Merge pull request #1198 from orbitdb/helia-v5

Helia v5

CHANGELOG.md

@@ -2,4 +2,4 @@
 
 For now, please refer to our Git commit history for a list of changes.
 
-https://github.com/orbitdb/orbitdb/commits/v1.0.2
+https://github.com/orbitdb/orbitdb/commits/v2.3.0

docs/CONNECTING_PEERS.md

@@ -72,7 +72,7 @@ const options = {
       filter: filters.all
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   services: {
     identify: identify(),
@@ -114,7 +114,7 @@ const options = {
       discoverRelays: 1
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   connectionGater: {
     denyDialMultiaddr: () => {
@@ -180,7 +180,7 @@ const options = {
       discoverRelays: 1
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   connectionGater: {
     denyDialMultiaddr: () => {
@@ -247,7 +247,7 @@ const options = {
       discoverRelays: 1
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   connectionGater: {
     denyDialMultiaddr: () => {

docs/GETTING_STARTED.md

@@ -50,7 +50,7 @@ A simple Node.js example might look something like:
   transports: [
     tcp()
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   services: {
     identify: identify(),
@@ -79,7 +79,7 @@ export const Libp2pOptions = {
   transports: [
     tcp()
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   services: {
     identify: identify(),

docs/jsdoc/README.md

@@ -1,4 +1,4 @@
-## OrbitDB API - v2.2
+## OrbitDB API - v2.3
 
 OrbitDB is a serverless, distributed, peer-to-peer database. OrbitDB uses IPFS
 as its data storage and Libp2p Pubsub to automatically sync databases with peers. It's an eventually consistent database that uses Merkle-CRDTs for conflict-free database writes and merges making OrbitDB an excellent choice for p2p and decentralized apps, blockchain applications and local first web applications.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orbitdb/core",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Distributed p2p database on IPFS",
   "author": "Haad",
   "license": "MIT",
@@ -19,7 +19,7 @@
   "main": "src/index.js",
   "dependencies": {
     "@ipld/dag-cbor": "^9.0.6",
-    "@libp2p/crypto": "^3.0.2",
+    "@libp2p/crypto": "^5.0.5",
     "it-pipe": "^3.0.1",
     "level": "^8.0.0",
     "lru": "^3.1.0",
@@ -29,15 +29,15 @@
     "uint8arrays": "^5.0.0"
   },
   "devDependencies": {
-    "@chainsafe/libp2p-gossipsub": "^13.0.0",
+    "@chainsafe/libp2p-gossipsub": "^14.1.0",
-    "@chainsafe/libp2p-yamux": "^6.0.1",
+    "@chainsafe/libp2p-yamux": "^7.0.1",
-    "@helia/block-brokers": "^1.0.0",
+    "@helia/block-brokers": "^4.0.0",
-    "@libp2p/circuit-relay-v2": "^1.0.10",
+    "@libp2p/circuit-relay-v2": "^2.1.5",
-    "blockstore-level": "^1.1.7",
+    "blockstore-level": "^2.0.1",
     "c8": "^8.0.1",
     "cross-env": "^7.0.3",
     "fs-extra": "^11.2.0",
-    "helia": "^4.0.1",
+    "helia": "^5.0.0",
     "it-all": "^3.0.4",
     "jsdoc": "^4.0.2",
     "mocha": "^10.2.0",

src/identities/providers/publickey.js

@@ -52,7 +52,7 @@ const PublicKeyIdentityProvider = ({ keystore }) => async () => {
     }
 
     const key = await keystore.getKey(id) || await keystore.createKey(id)
-    return uint8ArrayToString(key.public.marshal(), 'base16')
+    return uint8ArrayToString(key.publicKey.raw, 'base16')
   }
 
   /**

src/key-store.js

@@ -8,7 +8,7 @@
 * const storage = await MemoryStorage()
 * const keystore = await KeyStore({ storage })
 */
-import * as crypto from '@libp2p/crypto'
+import { privateKeyFromRaw, publicKeyFromRaw, generateKeyPair } from '@libp2p/crypto/keys'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { compare as uint8ArrayCompare } from 'uint8arrays/compare'
@@ -16,9 +16,6 @@ import ComposedStorage from './storage/composed.js'
 import LevelStorage from './storage/level.js'
 import LRUStorage from './storage/lru.js'
 
-const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
-const unmarshalPubKey = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PublicKey
-
 const verifySignature = async (signature, publicKey, data) => {
   if (!signature) {
     throw new Error('No signature given')
@@ -38,7 +35,7 @@ const verifySignature = async (signature, publicKey, data) => {
 
   let res = false
   try {
-    const pubKey = unmarshalPubKey(uint8ArrayFromString(publicKey, 'base16'))
+    const pubKey = publicKeyFromRaw(uint8ArrayFromString(publicKey, 'base16'))
     res = await isValid(pubKey, data, uint8ArrayFromString(signature, 'base16'))
   } catch (e) {
     // Catch error: sig length wrong
@@ -195,7 +192,7 @@ const KeyStore = async ({ storage, path } = {}) => {
     const { privateKey } = key
     await storage.put('private_' + id, privateKey)
     // Unmarshal the key and add it to the cache
-    const unmarshaledPrivateKey = unmarshal(privateKey)
+    const unmarshaledPrivateKey = privateKeyFromRaw(privateKey)
     await keyCache.put(id, unmarshaledPrivateKey)
   }
 
@@ -213,17 +210,16 @@ const KeyStore = async ({ storage, path } = {}) => {
     }
 
     // Generate a private key
-    const keyPair = await crypto.keys.generateKeyPair('secp256k1')
+    const keyPair = await generateKeyPair('secp256k1')
-    const keys = await crypto.keys.unmarshalPrivateKey(keyPair.bytes)
 
     const key = {
-      publicKey: keys.public.marshal(),
+      publicKey: keyPair.publicKey.raw,
-      privateKey: keys.marshal()
+      privateKey: keyPair.raw
     }
 
     await addKey(id, key)
 
-    return keys
+    return keyPair
   }
 
   /**
@@ -254,7 +250,8 @@ const KeyStore = async ({ storage, path } = {}) => {
         return
       }
 
-      key = unmarshal(storedKey)
+      key = privateKeyFromRaw(storedKey)
+
       await keyCache.put(id, key)
     }
 
@@ -281,7 +278,7 @@ const KeyStore = async ({ storage, path } = {}) => {
       throw new Error('Supported formats are `hex` and `buffer`')
     }
 
-    const pubKey = keys.public.marshal()
+    const pubKey = keys.publicKey.raw
 
     return format === 'buffer' ? pubKey : uint8ArrayToString(pubKey, 'base16')
   }

src/oplog/log.js

@@ -312,6 +312,9 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
         await _heads.remove(hash)
       }
 
+      /* 6. Add new entry to entries (for pinning) */
+      await _entries.put(entry.hash, entry.bytes)
+
       /* 6. Add the new entry to heads (=union with current heads) */
       await _heads.add(entry)
 

test/browser/setup-fixtures.js

@@ -1,10 +1,7 @@
-import * as crypto from '@libp2p/crypto'
+import { privateKeyFromRaw } from '@libp2p/crypto/keys'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { Identities, KeyStore } from '../../src/index.js'
 
-const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
-const unmarshalPubKey = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PublicKey
-
 const keysPath = './testkeys'
 
 const isBrowser = () => typeof window !== 'undefined'
@@ -52,10 +49,10 @@ before(async () => {
     ]
 
     for (let user of users) {
-      const privateKey1 = unmarshal(uint8ArrayFromString(user.privateKey, 'base16'))
+      const privateKey1 = privateKeyFromRaw(uint8ArrayFromString(user.privateKey, 'base16'))
-      const privateKey2 = unmarshal(uint8ArrayFromString(user.identity.privateKey, 'base16'))
+      const privateKey2 = privateKeyFromRaw(uint8ArrayFromString(user.identity.privateKey, 'base16'))
-      await keystore.addKey(user.id, { privateKey: privateKey1.marshal() })
+      await keystore.addKey(user.id, { privateKey: privateKey1.raw })
-      await keystore.addKey(user.identity.id, { privateKey: privateKey2.marshal() })
+      await keystore.addKey(user.identity.id, { privateKey: privateKey2.raw })
     }
 
     await keystore.close()

test/databases/replication/documents.test.js

@@ -11,7 +11,7 @@ import createHelia from '../../utils/create-helia.js'
 const keysPath = './testkeys'
 
 describe('Documents Database Replication', function () {
-  this.timeout(30000)
+  this.timeout(10000)
 
   let ipfs1, ipfs2
   let keystore

test/identities/identities.test.js

@@ -38,7 +38,7 @@ describe('Identities', function () {
       identities = await Identities({ path: keysPath })
       identity = await identities.createIdentity({ id })
       const key = await identities.keystore.getKey(id)
-      const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
+      const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
       assert.strictEqual(identity.id, externalId)
     })
   })
@@ -106,7 +106,7 @@ describe('Identities', function () {
       identity = await identities.createIdentity({ id })
       keystore = identities.keystore
       const key = await keystore.getKey(id)
-      const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
+      const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
       assert.strictEqual(identity.id, externalId)
     })
 
@@ -117,7 +117,7 @@ describe('Identities', function () {
 
     it('has the correct public key', async () => {
       const key = await keystore.getKey(id)
-      const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
+      const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
       const signingKey = await keystore.getKey(externalId)
       assert.notStrictEqual(signingKey, undefined)
       assert.strictEqual(identity.publicKey, keystore.getPublic(signingKey))
@@ -125,10 +125,10 @@ describe('Identities', function () {
 
     it('has a signature for the id', async () => {
       const key = await keystore.getKey(id)
-      const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
+      const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
       const signingKey = await keystore.getKey(externalId)
       const idSignature = await signMessage(signingKey, externalId)
-      const publicKey = uint8ArrayToString(signingKey.public.marshal(), 'base16')
+      const publicKey = uint8ArrayToString(signingKey.publicKey.raw, 'base16')
       const verifies = await verifyMessage(idSignature, publicKey, externalId)
       assert.strictEqual(verifies, true)
       assert.strictEqual(identity.signatures.id, idSignature)
@@ -136,7 +136,7 @@ describe('Identities', function () {
 
     it('has a signature for the publicKey', async () => {
       const key = await keystore.getKey(id)
-      const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
+      const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
       const signingKey = await keystore.getKey(externalId)
       const idSignature = await signMessage(signingKey, externalId)
       const externalKey = await keystore.getKey(id)
@@ -171,7 +171,7 @@ describe('Identities', function () {
 
     it('has the correct id', async () => {
       const key = await savedKeysKeyStore.getKey(id)
-      assert.strictEqual(identity.id, uint8ArrayToString(key.public.marshal(), 'base16'))
+      assert.strictEqual(identity.id, uint8ArrayToString(key.publicKey.raw, 'base16'))
     })
 
     it('has the correct public key', async () => {

test/key-store.test.js

@@ -145,7 +145,7 @@ describe('KeyStore', () => {
   })
 
   describe('Options', () => {
-    const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
+    const unmarshal = crypto.keys.privateKeyFromRaw
     const privateKey = '198594a8de39fd97017d11996d619b3746211605a9d290964badf58bc79bdb33'
     const publicKey = '0260baeaffa1de1e4135e5b395e0380563a622b9599d1b8e012a0f7603f516bdaa'
     let privateKeyBuffer, publicKeyBuffer, unmarshalledPrivateKey

test/orbitdb-replication.test.js

@@ -1,9 +1,11 @@
-import { deepStrictEqual } from 'assert'
+import { deepStrictEqual, strictEqual } from 'assert'
 import { rimraf } from 'rimraf'
 import { createOrbitDB } from '../src/index.js'
 import connectPeers from './utils/connect-nodes.js'
 import waitFor from './utils/wait-for.js'
 import createHelia from './utils/create-helia.js'
+import { CID } from 'multiformats/cid'
+import { base58btc } from 'multiformats/bases/base58'
 
 describe('Replicating databases', function () {
   this.timeout(10000)
@@ -23,8 +25,8 @@ describe('Replicating databases', function () {
   after(async () => {
     await orbitdb1.stop()
     await orbitdb2.stop()
-    await ipfs1.blockstore.child.child.close()
+    await ipfs1.blockstore.child.child.child.close()
-    await ipfs2.blockstore.child.child.close()
+    await ipfs2.blockstore.child.child.child.close()
     await ipfs1.stop()
     await ipfs2.stop()
 
@@ -136,8 +138,12 @@ describe('Replicating databases', function () {
 
       await orbitdb1.stop()
       await orbitdb2.stop()
-      await ipfs1.blockstore.child.child.close()
+      // TODO: Strange issue with ClassicLevel. Causes subsequent Helia
-      await ipfs2.blockstore.child.child.close()
+      // instantiations to error with db closed. Explicitly closing the
+      // nested ClassicLevel db seems to resolve the issue. Requires further
+      // investigation.
+      await ipfs1.blockstore.child.child.child.close()
+      await ipfs2.blockstore.child.child.child.close()
       await ipfs1.stop()
       await ipfs2.stop()
 
@@ -163,5 +169,26 @@ describe('Replicating databases', function () {
 
       console.log('events:', amount)
     })
+
+    it('pins all entries in the replicated database', async () => {
+      const db1 = await orbitdb1.open('helloworld', { referencesCount: 0 })
+      const hash = await db1.add('hello world')
+
+      let replicated = false
+
+      const onJoin = async (peerId, heads) => {
+        replicated = true
+      }
+
+      const db2 = await orbitdb2.open(db1.address)
+
+      db2.events.on('join', onJoin)
+
+      await waitFor(() => replicated, () => true)
+
+      const cid = CID.parse(hash, base58btc)
+      strictEqual(await ipfs1.pins.isPinned(cid), true)
+      strictEqual(await ipfs2.pins.isPinned(cid), true)
+    })
   })
 })

test/orbitdb.test.js

@@ -72,8 +72,8 @@ describe('OrbitDB', function () {
       const privateKey = await orbitdb1.keystore.getKey(orbitdb1.identity.id)
       notStrictEqual(privateKey, undefined)
       strictEqual(privateKey.constructor.name, 'Secp256k1PrivateKey')
-      notStrictEqual(privateKey._key, undefined)
+      notStrictEqual(privateKey.raw, undefined)
-      notStrictEqual(privateKey._publicKey, undefined)
+      notStrictEqual(privateKey.publicKey, undefined)
     })
 
     it('has a keystore that contains a public key that matches the identity\'s public key', async () => {
@@ -102,8 +102,8 @@ describe('OrbitDB', function () {
       notStrictEqual(orbitdb1.peerId, undefined)
     })
 
-    it('has a peerId of type Ed25519PeerIdImpl', async () => {
+    it('has a peerId of type Ed25519', async () => {
-      strictEqual(orbitdb1.peerId.constructor.name, 'Ed25519PeerIdImpl')
+      strictEqual(orbitdb1.peerId.type, 'Ed25519')
     })
 
     it('has a peerId that matches the IPFS id', async () => {
@@ -164,8 +164,8 @@ describe('OrbitDB', function () {
       const privateKey = await orbitdb1.keystore.getKey(orbitdb1.identity.id)
       notStrictEqual(privateKey, undefined)
       strictEqual(privateKey.constructor.name, 'Secp256k1PrivateKey')
-      notStrictEqual(privateKey._key, undefined)
+      notStrictEqual(privateKey.raw, undefined)
-      notStrictEqual(privateKey._publicKey, undefined)
+      notStrictEqual(privateKey.publicKey, undefined)
     })
 
     it('has a keystore that contains a public key that matches the identity\'s public key', async () => {
@@ -194,8 +194,8 @@ describe('OrbitDB', function () {
       notStrictEqual(orbitdb1.peerId, undefined)
     })
 
-    it('has a peerId of type Ed25519PeerIdImpl', async () => {
+    it('has a peerId of type Ed25519', async () => {
-      strictEqual(orbitdb1.peerId.constructor.name, 'Ed25519PeerIdImpl')
+      strictEqual(orbitdb1.peerId.type, 'Ed25519')
     })
 
     it('has a peerId that matches the IPFS id', async () => {

test/utils/create-helia.js

@@ -27,7 +27,7 @@ const Libp2pOptions = {
       discoverRelays: 1
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   connectionGater: {
     denyDialMultiaddr: () => false
@@ -54,7 +54,7 @@ const Libp2pBrowserOptions = {
       discoverRelays: 1
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   connectionGater: {
     denyDialMultiaddr: () => false

test/utils/relay.js

@@ -5,21 +5,18 @@ import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
 import { webSockets } from '@libp2p/websockets'
 import * as filters from '@libp2p/websockets/filters'
 import { identify } from '@libp2p/identify'
-import { createFromPrivKey } from '@libp2p/peer-id-factory'
-import { unmarshalPrivateKey } from '@libp2p/crypto/keys'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { privateKeyFromProtobuf } from '@libp2p/crypto/keys'
 
 // output of: console.log(server.peerId.privateKey.toString('hex'))
 const relayPrivKey = '08011240821cb6bc3d4547fcccb513e82e4d718089f8a166b23ffcd4a436754b6b0774cf07447d1693cd10ce11ef950d7517bad6e9472b41a927cd17fc3fb23f8c70cd99'
 // the peer id of the above key
 // const relayId = '12D3KooWAJjbRkp8FPF5MKgMU53aUTxWkqvDrs4zc1VMbwRwfsbE'
 
-const encoded = uint8ArrayFromString(relayPrivKey, 'hex')
+const privateKey = privateKeyFromProtobuf(uint8ArrayFromString(relayPrivKey, 'hex'))
-const privateKey = await unmarshalPrivateKey(encoded)
-const peerId = await createFromPrivKey(privateKey)
 
 const server = await createLibp2p({
-  peerId,
+  privateKey,
   addresses: {
     listen: ['/ip4/0.0.0.0/tcp/12345/ws']
   },
@@ -28,7 +25,7 @@ const server = await createLibp2p({
       filter: filters.all
     })
   ],
-  connectionEncryption: [noise()],
+  connectionEncrypters: [noise()],
   streamMuxers: [yamux()],
   services: {
     identify: identify(),