From 4a1f5cba332470f16d31338ee7a3aaf05e546d4c Mon Sep 17 00:00:00 2001 From: Hayden Young Date: Sat, 13 Jan 2024 02:36:45 +0000 Subject: [PATCH] test: Replication when ipfs peers reconnect or a relaunched. --- package-lock.json | 20 ++++ package.json | 1 + src/storage/lru.js | 2 +- test/orbitdb-replication.test.js | 151 ++++++++++++++++++++++++++++--- test/utils/create-helia.js | 7 +- 5 files changed, 168 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 99a314a..17fc880 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "devDependencies": { "@chainsafe/libp2p-yamux": "^6.0.1", "@libp2p/circuit-relay-v2": "^1.0.10", + "blockstore-level": "^1.1.7", "c8": "^8.0.1", "cross-env": "^7.0.3", "fs-extra": "^11.2.0", @@ -7127,6 +7128,25 @@ "integrity": "sha512-xiIB0p7EKmETm3wyKedOg/xuyQ18PoWwXCzzgpZAiDxL9ktl3XTh8AqoDT5kAqRg+DU48XAGPsUJL2Rn6Bx3Lw==", "dev": true }, + "node_modules/blockstore-level": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/blockstore-level/-/blockstore-level-1.1.7.tgz", + "integrity": "sha512-ua1obA5WulSWkN1mCQVAZ3VSfYACUjD6pMbqNk5LOgqNAD87pLENrdCFnfY3lYzZJDP4UfK6dBM0tp5fK8fvwA==", + "dev": true, + "dependencies": { + "blockstore-core": "^4.0.0", + "interface-blockstore": "^5.0.0", + "interface-store": "^5.0.0", + "level": "^8.0.0", + "multiformats": "^13.0.0" + } + }, + "node_modules/blockstore-level/node_modules/multiformats": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.0.1.tgz", + "integrity": "sha512-bt3R5iXe2O8xpp3wkmQhC73b/lC4S2ihU8Dndwcsysqbydqb8N+bpP116qMcClZ17g58iSIwtXUTcg2zT4sniA==", + "dev": true + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", diff --git a/package.json b/package.json index af05390..ee037e8 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "devDependencies": { "@chainsafe/libp2p-yamux": "^6.0.1", "@libp2p/circuit-relay-v2": "^1.0.10", + "blockstore-level": "^1.1.7", "c8": "^8.0.1", "cross-env": "^7.0.3", "fs-extra": "^11.2.0", diff --git a/src/storage/lru.js b/src/storage/lru.js index bbe05f5..004f089 100644 --- a/src/storage/lru.js +++ b/src/storage/lru.js @@ -12,7 +12,7 @@ const defaultSize = 1000000 * Creates an instance of LRUStorage. * @function * @param {Object} [params={}] One or more parameters for configuring - * IPFSBlockStorage. + * LRUStorage. * @param {string} [params.size=defaultSize] The number of elements to store. * @return {module:Storage.Storage-LRU} An instance of LRUStorage. * @memberof module:Storage diff --git a/test/orbitdb-replication.test.js b/test/orbitdb-replication.test.js index 8b864c5..cc590e1 100644 --- a/test/orbitdb-replication.test.js +++ b/test/orbitdb-replication.test.js @@ -4,30 +4,38 @@ import { createOrbitDB } from '../src/index.js' import connectPeers from './utils/connect-nodes.js' import waitFor from './utils/wait-for.js' import createHelia from './utils/create-helia.js' +import { LevelBlockstore } from 'blockstore-level' -describe('Replicating databases', function () { - this.timeout(10000) +describe.only('Replicating databases', function () { + this.timeout(45000) + let blockstore1, blockstore2 let ipfs1, ipfs2 let orbitdb1, orbitdb2 - before(async () => { - [ipfs1, ipfs2] = await Promise.all([createHelia(), createHelia()]) + beforeEach(async () => { + blockstore1 = new LevelBlockstore('./ipfs1') + blockstore2 = new LevelBlockstore('./ipfs2') + ipfs1 = await createHelia({ blockstore: blockstore1 }) + ipfs2 = await createHelia({ blockstore: blockstore2 }) await connectPeers(ipfs1, ipfs2) orbitdb1 = await createOrbitDB({ ipfs: ipfs1, id: 'user1', directory: './orbitdb1' }) orbitdb2 = await createOrbitDB({ ipfs: ipfs2, id: 'user2', directory: './orbitdb2' }) }) - after(async () => { - await ipfs1.stop() - await ipfs2.stop() - await rimraf('./ipfs1') - await rimraf('./ipfs2') + afterEach(async () => { await orbitdb1.stop() await orbitdb2.stop() + await blockstore1.close() + await blockstore2.close() + await ipfs1.stop() + await ipfs2.stop() + await rimraf('./orbitdb1') await rimraf('./orbitdb2') + await rimraf('./ipfs1') + await rimraf('./ipfs2') }) describe('replicating a database', () => { @@ -40,7 +48,7 @@ describe('Replicating databases', function () { let db1, db2 - before(async () => { + beforeEach(async () => { db1 = await orbitdb1.open('helloworld', { referencesCount: 0 }) console.time('write') @@ -50,7 +58,7 @@ describe('Replicating databases', function () { console.timeEnd('write') }) - after(async () => { + afterEach(async () => { await db1.close() await db2.close() }) @@ -98,5 +106,126 @@ describe('Replicating databases', function () { console.log('events:', amount) }) + + it('returns all entries in the replicated database after reconnect', async () => { + console.time('replicate') + + let replicated = false + + const onJoin = async (peerId, heads) => { + replicated = true + } + + const onError = (err) => { + console.error(err) + } + + db2 = await orbitdb2.open(db1.address) + + db2.events.on('join', onJoin) + db2.events.on('error', onError) + db1.events.on('error', onError) + + await waitFor(() => replicated, () => true) + + console.time('query 1') + const eventsFromDb2 = [] + for await (const event of db2.iterator()) { + eventsFromDb2.unshift(event) + } + console.timeEnd('query 1') + + console.timeEnd('replicate') + + deepStrictEqual(eventsFromDb2.map(e => e.value), expected) + + await orbitdb1.stop() + await orbitdb2.stop() + await orbitdb1.ipfs.stop() + await orbitdb2.ipfs.stop() + + await orbitdb1.ipfs.start() + await orbitdb2.ipfs.start() + + db1 = await orbitdb1.open('helloworld', { referencesCount: 0 }) + db2 = await orbitdb2.open(db1.address) + + console.time('query 2') + const eventsFromDb1 = [] + for await (const event of db1.iterator()) { + eventsFromDb1.unshift(event) + } + console.timeEnd('query 2') + + deepStrictEqual(eventsFromDb1.map(e => e.value), expected) + + console.log('events:', amount) + }) + + it('returns all entries in the replicated database after recreating orbitdb/ipfs instances', async () => { + console.time('replicate') + + let replicated = false + + const onJoin = async (peerId, heads) => { + replicated = true + } + + const onError = (err) => { + console.error(err) + } + + db2 = await orbitdb2.open(db1.address) + + db2.events.on('join', onJoin) + db2.events.on('error', onError) + db1.events.on('error', onError) + + await waitFor(() => replicated, () => true) + + console.time('query 1') + const eventsFromDb2 = [] + for await (const event of db2.iterator()) { + eventsFromDb2.unshift(event) + } + console.timeEnd('query 1') + + console.timeEnd('replicate') + + deepStrictEqual(eventsFromDb2.map(e => e.value), expected) + + await orbitdb1.stop() + await orbitdb2.stop() + await blockstore1.close() + await blockstore2.close() + await ipfs1.stop() + await ipfs2.stop() + + blockstore1 = new LevelBlockstore('./ipfs1') + blockstore2 = new LevelBlockstore('./ipfs2') + ipfs1 = await createHelia({ blockstore: blockstore1 }) + ipfs2 = await createHelia({ blockstore: blockstore2 }) + + await orbitdb1.ipfs.start() + await orbitdb2.ipfs.start() + await connectPeers(ipfs1, ipfs2) + + orbitdb1 = await createOrbitDB({ ipfs: ipfs1, id: 'user1', directory: './orbitdb1' }) + orbitdb2 = await createOrbitDB({ ipfs: ipfs2, id: 'user2', directory: './orbitdb2' }) + + db1 = await orbitdb1.open('helloworld', { referencesCount: 0 }) + db2 = await orbitdb2.open(db1.address) + + console.time('query 2') + const eventsFromDb1 = [] + for await (const event of db1.iterator()) { + eventsFromDb1.unshift(event) + } + console.timeEnd('query 2') + + deepStrictEqual(eventsFromDb1.map(e => e.value), expected) + + console.log('events:', amount) + }) }) }) diff --git a/test/utils/create-helia.js b/test/utils/create-helia.js index 68ff04d..fc12969 100644 --- a/test/utils/create-helia.js +++ b/test/utils/create-helia.js @@ -1,16 +1,21 @@ import { createHelia } from 'helia' import { bitswap } from 'helia/block-brokers' import { createLibp2p } from 'libp2p' +import { MemoryBlockstore } from 'blockstore-core' +// import { LevelBlockstore } from 'blockstore-level' import { DefaultLibp2pOptions, DefaultLibp2pBrowserOptions } from '../../src/index.js' const isBrowser = () => typeof window !== 'undefined' -export default async () => { +export default async ({ blockstore } = {}) => { const options = isBrowser() ? DefaultLibp2pBrowserOptions : DefaultLibp2pOptions const libp2p = await createLibp2p({ ...options }) + blockstore = blockstore || new MemoryBlockstore() + const heliaOptions = { + blockstore, libp2p, blockBrokers: [bitswap()] }