Mirroristas/orbitdb
2
0
Fork 0
mirror of https://github.com/orbitdb/orbitdb.git synced 2025-10-07 22:57:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into fix/manifest-store

Browse Source
This commit is contained in:
Hayden Young 2024-10-31 22:45:40 +00:00
commit bef35b013c
23 changed files with 3720 additions and 5036 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.well-known/funding-manifest-urls Normal file
View File

@ -0,0 +1 @@
https://orbitdb.org/funding.json

8
CHANGELOG.md
View File

@ -2,4 +2,10 @@
For now, please refer to our Git commit history for a list of changes.
https://github.com/orbitdb/orbitdb/commits/v1.0.2
https://github.com/orbitdb/orbitdb/compare/v2.4.0...v2.4.1
You can also use the following git command to generate a log of changes:
```
git log v2.4.0..v2.4.1 --oneline
```

2
README.md
View File

@ -28,7 +28,7 @@ Install OrbitDB and its dependencies:
npm install @orbitdb/core helia
```
### Browser <script> tag
### Browser `<script>` tag
OrbitDB can be loaded in the browser using the distributed js file with the `<script/>` tag. OrbitDB is the global namespace and all external functions are available via this namespace:

8
docs/CONNECTING_PEERS.md
View File

@ -72,7 +72,7 @@ const options = {
filter: filters.all
})
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
services: {
identify: identify(),
@ -114,7 +114,7 @@ const options = {
discoverRelays: 1
})
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => {
@ -180,7 +180,7 @@ const options = {
discoverRelays: 1
})
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => {
@ -247,7 +247,7 @@ const options = {
discoverRelays: 1
})
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => {

2
docs/DATABASES.md
View File

@ -264,7 +264,7 @@ CustomDB.type = type
export default CustomDB
```
[Documents](../src/db/documents.js), [Events](../src/db/events.js) and [KeyValue](../src/db/keyvalue.js) provide good examples of how a database is implemented in OrbitDB and how to add the logic for returning records from the database (the state of the database).
[Documents](../src/databases/documents.js), [Events](../src/databases/events.js) and [KeyValue](../src/databases/keyvalue.js) provide good examples of how a database is implemented in OrbitDB and how to add the logic for returning records from the database (the state of the database).
To use a custom database, add it to the list of supported database types:

12
docs/GETTING_STARTED.md
View File

@ -50,7 +50,7 @@ A simple Node.js example might look something like:
transports: [
tcp()
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
services: {
identify: identify(),
@ -79,7 +79,7 @@ export const Libp2pOptions = {
transports: [
tcp()
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
services: {
identify: identify(),
@ -159,7 +159,7 @@ const db = await orbitdb.open('my-db')
to:
```js
const db = await orbitdb.open('my-documents-db', { type: 'documents '})
const db = await orbitdb.open('my-documents-db', { type: 'documents' })
```
Also replace:
@ -172,8 +172,8 @@ await db.add('hello world 2')
with:
```js
await db.put('doc1', { hello: "world 1", hits: 5 })
await db.put('doc2', { hello: "world 2", hits: 2 })
await db.put({ _id: "doc1", hello: "world 1", hits: 5 })
await db.put({ _id: "doc2", hello: "world 2", hits: 2 })
```
Run index.js again:
@ -195,7 +195,7 @@ To create an OrbitDB database peer, create a new project called `orbitdb-peer`:
mkdir orbitdb-peer
cd orbitdb-peer
npm init
npm i helia orbitdb/core blockstore-level @chainsafe/libp2p-gossipsub
npm i helia @orbitdb/core blockstore-level @chainsafe/libp2p-gossipsub
```
Create a new file called index.js and paste in the following code:

2
docs/IDENTITIES.md
View File

@ -109,7 +109,7 @@ const identity = await identities.createIdentity({ id: 'me' })
const cid = CID.parse(identity.hash, base58btc)
// Extract the hash from the full db path.
const bytes = await ipfs.block.get(cid)
const bytes = await ipfs.blockstore.get(cid)
// Defines how we serialize/hash the data.
const codec = dagCbor

2
docs/jsdoc/README.md
View File

@ -1,4 +1,4 @@
## OrbitDB API - v2.2
## OrbitDB API - v2.4
OrbitDB is a serverless, distributed, peer-to-peer database. OrbitDB uses IPFS
as its data storage and Libp2p Pubsub to automatically sync databases with peers. It's an eventually consistent database that uses Merkle-CRDTs for conflict-free database writes and merges making OrbitDB an excellent choice for p2p and decentralized apps, blockchain applications and local first web applications.

8562
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

16
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@orbitdb/core",
"version": "2.2.0",
"version": "2.4.1",
"description": "Distributed p2p database on IPFS",
"author": "Haad",
"license": "MIT",
@ -19,7 +19,7 @@
"main": "src/index.js",
"dependencies": {
"@ipld/dag-cbor": "^9.0.6",
"@libp2p/crypto": "^3.0.2",
"@libp2p/crypto": "^5.0.5",
"it-pipe": "^3.0.1",
"level": "^8.0.0",
"lru": "^3.1.0",
@ -29,20 +29,18 @@
"uint8arrays": "^5.0.0"
},
"devDependencies": {
"@chainsafe/libp2p-gossipsub": "^13.0.0",
"@chainsafe/libp2p-yamux": "^6.0.1",
"@helia/block-brokers": "^1.0.0",
"@libp2p/circuit-relay-v2": "^1.0.10",
"blockstore-level": "^1.1.7",
"@chainsafe/libp2p-gossipsub": "^14.1.0",
"@libp2p/circuit-relay-v2": "^3.1.0",
"blockstore-level": "^2.0.1",
"c8": "^8.0.1",
"cross-env": "^7.0.3",
"fs-extra": "^11.2.0",
"helia": "^4.0.1",
"helia": "^5.1.0",
"it-all": "^3.0.4",
"jsdoc": "^4.0.2",
"mocha": "^10.2.0",
"path-browserify": "^1.0.1",
"playwright-test": "^14.0.0",
"playwright-test": "^14.1.6",
"rimraf": "^5.0.5",
"standard": "^17.1.0",
"webpack": "^5.89.0",

2
src/access-controllers/ipfs.js
View File

@ -84,7 +84,7 @@ const IPFSAccessController = ({ write, storage } = {}) => async ({ orbitdb, iden
// Allow if the write access list contain the writer's id or is '*'
if (write.includes(id) || write.includes('*')) {
// Check that the identity is valid
return identities.verifyIdentity(writerIdentity)
return await identities.verifyIdentity(writerIdentity)
}
return false
}

2
src/access-controllers/orbitdb.js
View File

@ -58,7 +58,7 @@ const OrbitDBAccessController = ({ write } = {}) => async ({ orbitdb, identities
// If the ACL contains the writer's public key or it contains '*'
const hasWriteAccess = await hasCapability('write', id) || await hasCapability('admin', id)
if (hasWriteAccess) {
return identities.verifyIdentity(writerIdentity)
return await identities.verifyIdentity(writerIdentity)
}
return false

2
src/identities/providers/publickey.js
View File

@ -52,7 +52,7 @@ const PublicKeyIdentityProvider = ({ keystore }) => async () => {
}
const key = await keystore.getKey(id) || await keystore.createKey(id)
return uint8ArrayToString(key.public.marshal(), 'base16')
return uint8ArrayToString(key.publicKey.raw, 'base16')
}
/**

23
src/key-store.js
View File

@ -8,7 +8,7 @@
* const storage = await MemoryStorage()
* const keystore = await KeyStore({ storage })
*/
import * as crypto from '@libp2p/crypto'
import { privateKeyFromRaw, publicKeyFromRaw, generateKeyPair } from '@libp2p/crypto/keys'
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { compare as uint8ArrayCompare } from 'uint8arrays/compare'
@ -16,9 +16,6 @@ import ComposedStorage from './storage/composed.js'
import LevelStorage from './storage/level.js'
import LRUStorage from './storage/lru.js'
const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
const unmarshalPubKey = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PublicKey
const verifySignature = async (signature, publicKey, data) => {
if (!signature) {
throw new Error('No signature given')
@ -38,7 +35,7 @@ const verifySignature = async (signature, publicKey, data) => {
let res = false
try {
const pubKey = unmarshalPubKey(uint8ArrayFromString(publicKey, 'base16'))
const pubKey = publicKeyFromRaw(uint8ArrayFromString(publicKey, 'base16'))
res = await isValid(pubKey, data, uint8ArrayFromString(signature, 'base16'))
} catch (e) {
// Catch error: sig length wrong
@ -195,7 +192,7 @@ const KeyStore = async ({ storage, path } = {}) => {
const { privateKey } = key
await storage.put('private_' + id, privateKey)
// Unmarshal the key and add it to the cache
const unmarshaledPrivateKey = unmarshal(privateKey)
const unmarshaledPrivateKey = privateKeyFromRaw(privateKey)
await keyCache.put(id, unmarshaledPrivateKey)
}
@ -213,17 +210,16 @@ const KeyStore = async ({ storage, path } = {}) => {
}
// Generate a private key
const keyPair = await crypto.keys.generateKeyPair('secp256k1')
const keys = await crypto.keys.unmarshalPrivateKey(keyPair.bytes)
const keyPair = await generateKeyPair('secp256k1')
const key = {
publicKey: keys.public.marshal(),
privateKey: keys.marshal()
publicKey: keyPair.publicKey.raw,
privateKey: keyPair.raw
}
await addKey(id, key)
return keys
return keyPair
}
/**
@ -254,7 +250,8 @@ const KeyStore = async ({ storage, path } = {}) => {
return
}
key = unmarshal(storedKey)
key = privateKeyFromRaw(storedKey)
await keyCache.put(id, key)
}
@ -281,7 +278,7 @@ const KeyStore = async ({ storage, path } = {}) => {
throw new Error('Supported formats are `hex` and `buffer`')
}
const pubKey = keys.public.marshal()
const pubKey = keys.publicKey.raw
return format === 'buffer' ? pubKey : uint8ArrayToString(pubKey, 'base16')
}

3
src/oplog/log.js
View File

@ -312,6 +312,9 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
await _heads.remove(hash)
}
/* 6. Add new entry to entries (for pinning) */
await _entries.put(entry.hash, entry.bytes)
/* 6. Add the new entry to heads (=union with current heads) */
await _heads.add(entry)

13
test/browser/setup-fixtures.js
View File

@ -1,10 +1,7 @@
import * as crypto from '@libp2p/crypto'
import { privateKeyFromRaw } from '@libp2p/crypto/keys'
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
import { Identities, KeyStore } from '../../src/index.js'
const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
const unmarshalPubKey = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PublicKey
const keysPath = './testkeys'
const isBrowser = () => typeof window !== 'undefined'
@ -52,10 +49,10 @@ before(async () => {
]
for (let user of users) {
const privateKey1 = unmarshal(uint8ArrayFromString(user.privateKey, 'base16'))
const privateKey2 = unmarshal(uint8ArrayFromString(user.identity.privateKey, 'base16'))
await keystore.addKey(user.id, { privateKey: privateKey1.marshal() })
await keystore.addKey(user.identity.id, { privateKey: privateKey2.marshal() })
const privateKey1 = privateKeyFromRaw(uint8ArrayFromString(user.privateKey, 'base16'))
const privateKey2 = privateKeyFromRaw(uint8ArrayFromString(user.identity.privateKey, 'base16'))
await keystore.addKey(user.id, { privateKey: privateKey1.raw })
await keystore.addKey(user.identity.id, { privateKey: privateKey2.raw })
}
await keystore.close()

2
test/databases/replication/documents.test.js
View File

@ -11,7 +11,7 @@ import createHelia from '../../utils/create-helia.js'
const keysPath = './testkeys'
describe('Documents Database Replication', function () {
this.timeout(30000)
this.timeout(10000)
let ipfs1, ipfs2
let keystore

14
test/identities/identities.test.js
View File

@ -38,7 +38,7 @@ describe('Identities', function () {
identities = await Identities({ path: keysPath })
identity = await identities.createIdentity({ id })
const key = await identities.keystore.getKey(id)
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
assert.strictEqual(identity.id, externalId)
})
})
@ -106,7 +106,7 @@ describe('Identities', function () {
identity = await identities.createIdentity({ id })
keystore = identities.keystore
const key = await keystore.getKey(id)
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
assert.strictEqual(identity.id, externalId)
})
@ -117,7 +117,7 @@ describe('Identities', function () {
it('has the correct public key', async () => {
const key = await keystore.getKey(id)
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
const signingKey = await keystore.getKey(externalId)
assert.notStrictEqual(signingKey, undefined)
assert.strictEqual(identity.publicKey, keystore.getPublic(signingKey))
@ -125,10 +125,10 @@ describe('Identities', function () {
it('has a signature for the id', async () => {
const key = await keystore.getKey(id)
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
const signingKey = await keystore.getKey(externalId)
const idSignature = await signMessage(signingKey, externalId)
const publicKey = uint8ArrayToString(signingKey.public.marshal(), 'base16')
const publicKey = uint8ArrayToString(signingKey.publicKey.raw, 'base16')
const verifies = await verifyMessage(idSignature, publicKey, externalId)
assert.strictEqual(verifies, true)
assert.strictEqual(identity.signatures.id, idSignature)
@ -136,7 +136,7 @@ describe('Identities', function () {
it('has a signature for the publicKey', async () => {
const key = await keystore.getKey(id)
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
const signingKey = await keystore.getKey(externalId)
const idSignature = await signMessage(signingKey, externalId)
const externalKey = await keystore.getKey(id)
@ -171,7 +171,7 @@ describe('Identities', function () {
it('has the correct id', async () => {
const key = await savedKeysKeyStore.getKey(id)
assert.strictEqual(identity.id, uint8ArrayToString(key.public.marshal(), 'base16'))
assert.strictEqual(identity.id, uint8ArrayToString(key.publicKey.raw, 'base16'))
})
it('has the correct public key', async () => {

2
test/key-store.test.js
View File

@ -145,7 +145,7 @@ describe('KeyStore', () => {
})
describe('Options', () => {
const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
const unmarshal = crypto.keys.privateKeyFromRaw
const privateKey = '198594a8de39fd97017d11996d619b3746211605a9d290964badf58bc79bdb33'
const publicKey = '0260baeaffa1de1e4135e5b395e0380563a622b9599d1b8e012a0f7603f516bdaa'
let privateKeyBuffer, publicKeyBuffer, unmarshalledPrivateKey

37
test/orbitdb-replication.test.js
View File

@ -1,9 +1,11 @@
import { deepStrictEqual } from 'assert'
import { deepStrictEqual, strictEqual } from 'assert'
import { rimraf } from 'rimraf'
import { createOrbitDB } from '../src/index.js'
import connectPeers from './utils/connect-nodes.js'
import waitFor from './utils/wait-for.js'
import createHelia from './utils/create-helia.js'
import { CID } from 'multiformats/cid'
import { base58btc } from 'multiformats/bases/base58'
describe('Replicating databases', function () {
this.timeout(10000)
@ -23,8 +25,8 @@ describe('Replicating databases', function () {
after(async () => {
await orbitdb1.stop()
await orbitdb2.stop()
await ipfs1.blockstore.child.child.close()
await ipfs2.blockstore.child.child.close()
await ipfs1.blockstore.child.child.child.close()
await ipfs2.blockstore.child.child.child.close()
await ipfs1.stop()
await ipfs2.stop()
@ -136,8 +138,12 @@ describe('Replicating databases', function () {
await orbitdb1.stop()
await orbitdb2.stop()
await ipfs1.blockstore.child.child.close()
await ipfs2.blockstore.child.child.close()
// TODO: Strange issue with ClassicLevel. Causes subsequent Helia
// instantiations to error with db closed. Explicitly closing the
// nested ClassicLevel db seems to resolve the issue. Requires further
// investigation.
await ipfs1.blockstore.child.child.child.close()
await ipfs2.blockstore.child.child.child.close()
await ipfs1.stop()
await ipfs2.stop()
@ -163,5 +169,26 @@ describe('Replicating databases', function () {
console.log('events:', amount)
})
it('pins all entries in the replicated database', async () => {
const db1 = await orbitdb1.open('helloworld', { referencesCount: 0 })
const hash = await db1.add('hello world')
let replicated = false
const onJoin = async (peerId, heads) => {
replicated = true
}
const db2 = await orbitdb2.open(db1.address)
db2.events.on('join', onJoin)
await waitFor(() => replicated, () => true)
const cid = CID.parse(hash, base58btc)
strictEqual(await ipfs1.pins.isPinned(cid), true)
strictEqual(await ipfs2.pins.isPinned(cid), true)
})
})
})

16
test/orbitdb.test.js
View File

@ -72,8 +72,8 @@ describe('OrbitDB', function () {
const privateKey = await orbitdb1.keystore.getKey(orbitdb1.identity.id)
notStrictEqual(privateKey, undefined)
strictEqual(privateKey.constructor.name, 'Secp256k1PrivateKey')
notStrictEqual(privateKey._key, undefined)
notStrictEqual(privateKey._publicKey, undefined)
notStrictEqual(privateKey.raw, undefined)
notStrictEqual(privateKey.publicKey, undefined)
})
it('has a keystore that contains a public key that matches the identity\'s public key', async () => {
@ -102,8 +102,8 @@ describe('OrbitDB', function () {
notStrictEqual(orbitdb1.peerId, undefined)
})
it('has a peerId of type Ed25519PeerIdImpl', async () => {
strictEqual(orbitdb1.peerId.constructor.name, 'Ed25519PeerIdImpl')
it('has a peerId of type Ed25519', async () => {
strictEqual(orbitdb1.peerId.type, 'Ed25519')
})
it('has a peerId that matches the IPFS id', async () => {
@ -164,8 +164,8 @@ describe('OrbitDB', function () {
const privateKey = await orbitdb1.keystore.getKey(orbitdb1.identity.id)
notStrictEqual(privateKey, undefined)
strictEqual(privateKey.constructor.name, 'Secp256k1PrivateKey')
notStrictEqual(privateKey._key, undefined)
notStrictEqual(privateKey._publicKey, undefined)
notStrictEqual(privateKey.raw, undefined)
notStrictEqual(privateKey.publicKey, undefined)
})
it('has a keystore that contains a public key that matches the identity\'s public key', async () => {
@ -194,8 +194,8 @@ describe('OrbitDB', function () {
notStrictEqual(orbitdb1.peerId, undefined)
})
it('has a peerId of type Ed25519PeerIdImpl', async () => {
strictEqual(orbitdb1.peerId.constructor.name, 'Ed25519PeerIdImpl')
it('has a peerId of type Ed25519', async () => {
strictEqual(orbitdb1.peerId.type, 'Ed25519')
})
it('has a peerId that matches the IPFS id', async () => {

14
test/utils/create-helia.js
View File

@ -21,13 +21,9 @@ const Libp2pOptions = {
transports: [
webSockets({
filter: all
}),
webRTC(),
circuitRelayTransport({
discoverRelays: 1
})
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => false
@ -43,18 +39,16 @@ const Libp2pOptions = {
*/
const Libp2pBrowserOptions = {
addresses: {
listen: ['/webrtc']
listen: ['/webrtc', '/p2p-circuit']
},
transports: [
webSockets({
filter: all
}),
webRTC(),
circuitRelayTransport({
discoverRelays: 1
})
circuitRelayTransport()
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => false

11
test/utils/relay.js
View File

@ -5,21 +5,18 @@ import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
import { webSockets } from '@libp2p/websockets'
import * as filters from '@libp2p/websockets/filters'
import { identify } from '@libp2p/identify'
import { createFromPrivKey } from '@libp2p/peer-id-factory'
import { unmarshalPrivateKey } from '@libp2p/crypto/keys'
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
import { privateKeyFromProtobuf } from '@libp2p/crypto/keys'
// output of: console.log(server.peerId.privateKey.toString('hex'))
const relayPrivKey = '08011240821cb6bc3d4547fcccb513e82e4d718089f8a166b23ffcd4a436754b6b0774cf07447d1693cd10ce11ef950d7517bad6e9472b41a927cd17fc3fb23f8c70cd99'
// the peer id of the above key
// const relayId = '12D3KooWAJjbRkp8FPF5MKgMU53aUTxWkqvDrs4zc1VMbwRwfsbE'
const encoded = uint8ArrayFromString(relayPrivKey, 'hex')
const privateKey = await unmarshalPrivateKey(encoded)
const peerId = await createFromPrivKey(privateKey)
const privateKey = privateKeyFromProtobuf(uint8ArrayFromString(relayPrivKey, 'hex'))
const server = await createLibp2p({
peerId,
privateKey,
addresses: {
listen: ['/ip4/0.0.0.0/tcp/12345/ws']
},
@ -28,7 +25,7 @@ const server = await createLibp2p({
filter: filters.all
})
],
connectionEncryption: [noise()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
services: {
identify: identify(),