Test/database (#38)

* test: Database.

* test: Remove caching test. Caching is superseded by the various storage options.

* test: db2 replicates data in existing db1.

* test: Move event tests to generic database tests.

* test: Single instance cleanup.

* fix: Linting.

* Fix Sync race condition (#39)

* test: Fix key path import.

---------

Co-authored-by: Haad <haadcode@users.noreply.github.com>

src/database.js

@@ -39,7 +39,9 @@ const Database = async ({ OpLog, ipfs, identity, address, name, accessController
       events.emit('update', entry)
       return entry.hash
     }
-    return queue.add(task)
+    const hash = await queue.add(task)
+    await queue.onIdle()
+    return hash
   }
 
   const applyOperation = async (bytes) => {
@@ -53,7 +55,6 @@ const Database = async ({ OpLog, ipfs, identity, address, name, accessController
       }
     }
     await queue.add(task)
-    await queue.onIdle()
   }
 
   const close = async () => {

src/sync.js

@@ -67,7 +67,6 @@ const Sync = async ({ ipfs, log, events, onSynced, start }) => {
         } catch (e) {
           if (e.code === 'ERR_UNSUPPORTED_PROTOCOL') {
             // Skip peer, they don't have this database currently
-            console.log(e.message)
           } else {
             console.error(e)
             peers.delete(peerId)
@@ -79,8 +78,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start }) => {
         events.emit('leave', peerId)
       }
     }
-    await queue.onIdle()
-    await queue.add(task)
+    queue.add(task)
   }
 
   const handleUpdateMessage = async (message) => {
@@ -97,8 +95,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start }) => {
         events.emit('error', e)
       }
     }
-    await queue.onIdle()
-    await queue.add(task)
+    queue.add(task)
   }
 
   const add = async (entry) => {

test/custom-cache.test.js

@@ -1,59 +0,0 @@
-// import assert from 'assert'
-// import rmrf from 'rimraf'
-// import path from 'path'
-// import OrbitDB from '../src/OrbitDB.js'
-// import CustomCache from 'orbit-db-cache'
-// import localdown from 'localstorage-down'
-// import storageAdapter from 'orbit-db-storage-adapter'
-
-// // Include test utilities
-// import {
-//   config,
-//   startIpfs,
-//   stopIpfs,
-//   testAPIs,
-// } from 'orbit-db-test-utils'
-
-// import { databases } from './utils/index.js'
-
-// const storage = storageAdapter(localdown)
-
-// const dbPath = './orbitdb/tests/customKeystore'
-
-// Object.keys(testAPIs).forEach(API => {
-//   describe(`orbit-db - Use a Custom Cache (${API})`, function() {
-//     this.timeout(20000)
-
-//     let ipfsd, ipfs, orbitdb1, store
-
-//     before(async () => {
-//       store = await storage.createStore("orbitdb/test/local")
-//       const cache = new CustomCache(store)
-
-//       rmrf.sync(dbPath)
-//       ipfsd = await startIpfs(API, config.daemon1)
-//       ipfs = ipfsd.api
-//       orbitdb1 = await OrbitDB.createInstance(ipfs, {
-//         directory: path.join(dbPath, '1'),
-//         cache: cache
-//       })
-//     })
-
-//     after(async () => {
-//       await orbitdb1.stop()
-//       await stopIpfs(ipfsd)
-//     })
-
-//     describe('allows orbit to use a custom cache with different store types', function() {
-//       for (let database of databases) {
-//         it(database.type + ' allows custom cache', async () => {
-//           const db1 = await database.create(orbitdb1, 'custom-keystore')
-//           await database.tryInsert(db1)
-
-//           assert.deepEqual(database.getTestValue(db1), database.expectedValue)
-//           await db1.close()
-//         })
-//       }
-//     })
-//   })
-// })

test/database-replication.test.js

@@ -0,0 +1,277 @@
+import { strictEqual, deepStrictEqual } from 'assert'
+import rmrf from 'rimraf'
+import { copy } from 'fs-extra'
+import * as IPFS from 'ipfs'
+import { Log, Entry, Database, KeyStore, Identities } from '../src/index.js'
+import config from './config.js'
+import testKeysPath from './fixtures/test-keys-path.js'
+import connectPeers from './utils/connect-nodes.js'
+import waitFor from './utils/wait-for.js'
+
+const OpLog = { Log, Entry }
+const keysPath = './testkeys'
+
+describe('Database - Replication', function () {
+  this.timeout(30000)
+
+  let ipfs1, ipfs2
+  let keystore
+  let identities
+  let testIdentity1, testIdentity2
+  let db1, db2
+
+  const databaseId = 'documentstore-AAA'
+
+  const accessController = {
+    canAppend: async (entry) => {
+      const identity1 = await identities.getIdentity(entry.identity)
+      const identity2 = await identities.getIdentity(entry.identity)
+      return identity1.id === testIdentity1.id || identity2.id === testIdentity2.id
+    }
+  }
+
+  beforeEach(async () => {
+    ipfs1 = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
+    ipfs2 = await IPFS.create({ ...config.daemon2, repo: './ipfs2' })
+    await connectPeers(ipfs1, ipfs2)
+
+    await copy(testKeysPath, keysPath)
+    keystore = await KeyStore({ path: keysPath })
+    identities = await Identities({ keystore })
+    testIdentity1 = await identities.createIdentity({ id: 'userA' })
+    testIdentity2 = await identities.createIdentity({ id: 'userB' })
+
+    db1 = await Database({ OpLog, ipfs: ipfs1, identity: testIdentity1, address: databaseId, accessController, directory: './orbitdb1' })
+    db2 = await Database({ OpLog, ipfs: ipfs2, identity: testIdentity2, address: databaseId, accessController, directory: './orbitdb2' })
+  })
+
+  afterEach(async () => {
+    if (db1) {
+      await db1.drop()
+      await db1.close()
+
+      await rmrf('./orbitdb1')
+    }
+    if (db2) {
+      await db2.drop()
+      await db2.close()
+
+      await rmrf('./orbitdb2')
+    }
+
+    if (ipfs1) {
+      await ipfs1.stop()
+    }
+
+    if (ipfs2) {
+      await ipfs2.stop()
+    }
+
+    if (keystore) {
+      await keystore.close()
+    }
+
+    await rmrf(keysPath)
+    await rmrf('./ipfs1')
+    await rmrf('./ipfs2')
+  })
+
+  it('replicates databases across two peers', async () => {
+    let connected1 = false
+    let connected2 = false
+
+    const onConnected1 = (peerId, heads) => {
+      connected1 = true
+    }
+
+    const onConnected2 = (peerId, heads) => {
+      connected2 = true
+    }
+
+    db1.events.on('join', onConnected1)
+    db2.events.on('join', onConnected2)
+
+    await db1.addOperation({ op: 'PUT', key: 1, value: 'record 1 on db 1' })
+    await db1.addOperation({ op: 'PUT', key: 2, value: 'record 2 on db 1' })
+    await db1.addOperation({ op: 'PUT', key: 3, value: 'record 3 on db 1' })
+    await db1.addOperation({ op: 'PUT', key: 4, value: 'record 4 on db 1' })
+
+    await waitFor(() => connected1, () => true)
+    await waitFor(() => connected2, () => true)
+
+    const all1 = []
+    for await (const item of db1.log.iterator()) {
+      all1.unshift(item)
+    }
+
+    const all2 = []
+    for await (const item of db2.log.iterator()) {
+      all2.unshift(item)
+    }
+
+    deepStrictEqual(all1, all2)
+  })
+
+  it('replicates databases across two peers with delays', async () => {
+    let connected1 = false
+    let connected2 = false
+
+    const onConnected1 = (peerId, heads) => {
+      connected1 = true
+    }
+
+    const onConnected2 = (peerId, heads) => {
+      connected2 = true
+    }
+
+    db1.events.on('join', onConnected1)
+    db2.events.on('join', onConnected2)
+
+    await db1.addOperation({ op: 'PUT', key: 1, value: 'record 1 on db 1' })
+
+    await new Promise(resolve => {
+      setTimeout(() => resolve(), 1000)
+    })
+
+    await db1.addOperation({ op: 'PUT', key: 2, value: 'record 2 on db 1' })
+    await db1.addOperation({ op: 'PUT', key: 3, value: 'record 3 on db 1' })
+
+    await new Promise(resolve => {
+      setTimeout(() => resolve(), 1000)
+    })
+
+    await db1.addOperation({ op: 'PUT', key: 4, value: 'record 4 on db 1' })
+
+    await waitFor(() => connected1, () => true)
+    await waitFor(() => connected2, () => true)
+
+    const all1 = []
+    for await (const item of db1.log.iterator()) {
+      all1.unshift(item)
+    }
+
+    const all2 = []
+    for await (const item of db2.log.iterator()) {
+      all2.unshift(item)
+    }
+
+    deepStrictEqual(all1, all2)
+  })
+
+  it('adds an operation before db2 is instantiated', async () => {
+    let connected = false
+    const onConnected = (peerId, heads) => {
+      connected = true
+    }
+
+    await db2.drop()
+    await db2.close()
+
+    await rmrf('./orbitdb2')
+
+    await db1.addOperation({ op: 'PUT', key: 1, value: 'record 1 on db 1' })
+
+    db2 = await Database({ OpLog, ipfs: ipfs2, identity: testIdentity2, address: databaseId, accessController, directory: './orbitdb2' })
+
+    db2.events.on('join', onConnected)
+
+    await waitFor(() => connected, () => true)
+
+    const all1 = []
+    for await (const item of db1.log.iterator()) {
+      all1.unshift(item)
+    }
+
+    const all2 = []
+    for await (const item of db2.log.iterator()) {
+      all2.unshift(item)
+    }
+
+    deepStrictEqual(all1, all2)
+  })
+
+  describe('Events', () => {
+    it('emits \'update\' once when one operation is added', async () => {
+      const expected = 1
+      let connected1 = false
+      let connected2 = false
+      let updateCount1 = 0
+      let updateCount2 = 0
+
+      const onConnected1 = (peerId, heads) => {
+        connected1 = true
+      }
+
+      const onConnected2 = (peerId, heads) => {
+        connected2 = true
+      }
+
+      const onUpdate1 = async (entry) => {
+        ++updateCount1
+      }
+
+      const onUpdate2 = async (entry) => {
+        ++updateCount2
+      }
+
+      db1.events.on('join', onConnected1)
+      db2.events.on('join', onConnected2)
+      db1.events.on('update', onUpdate1)
+      db2.events.on('update', onUpdate2)
+
+      await waitFor(() => connected1, () => true)
+      await waitFor(() => connected2, () => true)
+
+      await db1.addOperation({ op: 'PUT', key: 1, value: 'record 1 on db 1' })
+
+      await waitFor(() => updateCount1 >= expected, () => true)
+      await waitFor(() => updateCount2 >= expected, () => true)
+
+      strictEqual(updateCount1, expected)
+      strictEqual(updateCount2, expected)
+    })
+
+    it('emits \'update\' 4 times when 4 documents are added', async () => {
+      const expected = 4
+      let connected1 = false
+      let connected2 = false
+      let updateCount1 = 0
+      let updateCount2 = 0
+
+      const onConnected1 = async (peerId, heads) => {
+        connected1 = true
+      }
+
+      const onConnected2 = async (peerId, heads) => {
+        connected2 = true
+      }
+
+      const onUpdate1 = async (entry) => {
+        ++updateCount1
+      }
+
+      const onUpdate2 = async (entry) => {
+        ++updateCount2
+      }
+
+      db1.events.on('join', onConnected1)
+      db2.events.on('join', onConnected2)
+      db1.events.on('update', onUpdate1)
+      db2.events.on('update', onUpdate2)
+
+      await waitFor(() => connected1, () => true)
+      await waitFor(() => connected2, () => true)
+
+      await db1.addOperation({ op: 'PUT', key: 1, value: '11' })
+      await db1.addOperation({ op: 'PUT', key: 2, value: '22' })
+      await db1.addOperation({ op: 'PUT', key: 3, value: '33' })
+      await db1.addOperation({ op: 'PUT', key: 4, value: '44' })
+
+      await waitFor(() => updateCount1 >= expected, () => true)
+      await waitFor(() => updateCount2 >= expected, () => true)
+
+      strictEqual(updateCount1, expected)
+      strictEqual(updateCount2, expected)
+    })
+  })
+})

test/database.test.js

@@ -0,0 +1,99 @@
+import { strictEqual, deepStrictEqual } from 'assert'
+import rmrf from 'rimraf'
+import { copy } from 'fs-extra'
+import * as IPFS from 'ipfs'
+import { Log, Entry, Database, KeyStore, Identities } from '../src/index.js'
+import config from './config.js'
+import testKeysPath from './fixtures/test-keys-path.js'
+
+const OpLog = { Log, Entry }
+const keysPath = './testkeys'
+
+describe('Database', function () {
+  this.timeout(30000)
+
+  let ipfs
+  let keystore
+  let identities
+  let testIdentity
+  let db
+
+  const databaseId = 'documentstore-AAA'
+
+  const accessController = {
+    canAppend: async (entry) => {
+      const identity1 = await identities.getIdentity(entry.identity)
+      return identity1.id === testIdentity.id
+    }
+  }
+
+  before(async () => {
+    ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
+
+    await copy(testKeysPath, keysPath)
+    keystore = await KeyStore({ path: keysPath })
+    identities = await Identities({ keystore })
+    testIdentity = await identities.createIdentity({ id: 'userA' })
+  })
+
+  after(async () => {
+    if (ipfs) {
+      await ipfs.stop()
+    }
+
+    if (keystore) {
+      await keystore.close()
+    }
+
+    await rmrf(keysPath)
+    await rmrf('./ipfs1')
+  })
+
+  beforeEach(async () => {
+    db = await Database({ OpLog, ipfs, identity: testIdentity, address: databaseId, accessController, directory: './orbitdb1' })
+  })
+
+  afterEach(async () => {
+    await rmrf('./orbitdb1')
+  })
+
+  it('adds an operation', async () => {
+    const expected = 'zdpuAqQ9TJpMhPShuT315m2D9LUBkBPy8YX9zatjEynd2suZv'
+    const op = { op: 'PUT', key: 1, value: 'record 1 on db 1' }
+    const actual = await db.addOperation(op)
+
+    deepStrictEqual(actual, expected)
+
+    await db.close()
+  })
+
+  describe('Events', () => {
+    it('emits \'close\' when the database is closed', async () => {
+      let closed = false
+      const onClose = () => {
+        closed = true
+      }
+
+      db.events.on('close', onClose)
+
+      await db.close()
+
+      strictEqual(closed, true)
+    })
+
+    it('emits \'drop\' when the database is dropped', async () => {
+      let dropped = false
+      const onDrop = () => {
+        dropped = true
+      }
+
+      db.events.on('drop', onDrop)
+
+      await db.drop()
+
+      strictEqual(dropped, true)
+
+      await db.close()
+    })
+  })
+})

test/db/replication/document-store.test.js

@@ -1,4 +1,4 @@
-import { strictEqual, deepStrictEqual } from 'assert'
+import { deepStrictEqual } from 'assert'
 import rmrf from 'rimraf'
 import { copy } from 'fs-extra'
 import * as IPFS from 'ipfs'
@@ -83,11 +83,11 @@ describe('Documents Database Replication', function () {
     let connected1 = false
     let connected2 = false
 
-    const onConnected1 = (entry) => {
+    const onConnected1 = async (peerId, heads) => {
       connected1 = true
     }
 
-    const onConnected2 = (entry) => {
+    const onConnected2 = async (peerId, heads) => {
       connected2 = true
     }
 
@@ -114,27 +114,4 @@ describe('Documents Database Replication', function () {
 
     deepStrictEqual(all1, all2)
   })
-
-  it('emits \'update\' once when one document is added', async () => {
-    let connected = false
-    let updateCount = 0
-
-    const onConnected = async (peerId) => {
-      connected = true
-    }
-
-    const onUpdate = async (entry) => {
-      ++updateCount
-    }
-
-    db2.events.on('join', onConnected)
-    db2.events.on('update', onUpdate)
-
-    await db1.put({ _id: 1, msg: 'record 1 on db 1' })
-
-    await waitFor(() => connected, () => true)
-    await waitFor(() => updateCount > 0, () => true)
-
-    strictEqual(updateCount, 1)
-  })
 })

test/db/replication/event-store.test.js

@@ -88,7 +88,7 @@ describe('Events Database Replication', function () {
     let connected = false
     let updateCount = 0
 
-    const onConnected = async (peerId) => {
+    const onConnected = async (peerId, heads) => {
       connected = true
     }
 
@@ -138,7 +138,7 @@ describe('Events Database Replication', function () {
     let connected = false
     let updateCount = 0
 
-    const onConnected = async (peerId) => {
+    const onConnected = async (peerId, heads) => {
       connected = true
     }
 

test/db/replication/keyvalue-persisted.test.js

@@ -77,7 +77,7 @@ describe('KeyValue Database Replication', function () {
     let connected = false
     let updateCount = 0
 
-    const onConnected = async (peerId) => {
+    const onConnected = async (peerId, heads) => {
       connected = true
     }
 
@@ -145,7 +145,7 @@ describe('KeyValue Database Replication', function () {
     let updateCount = 0
     let connected = false
 
-    const onConnected = async (peerId) => {
+    const onConnected = async (peerId, heads) => {
       connected = true
     }
 

test/db/replication/keyvalue.test.js

@@ -77,7 +77,7 @@ describe('KeyValue Database Replication', function () {
     let connected = false
     let updateCount = 0
 
-    const onConnected = async (peerId) => {
+    const onConnected = async (peerId, heads) => {
       connected = true
     }
 
@@ -145,7 +145,7 @@ describe('KeyValue Database Replication', function () {
     let updateCount = 0
     let connected = false
 
-    const onConnected = async (peerId) => {
+    const onConnected = async (peerId, heads) => {
       connected = true
     }
 

test/orbitdb-open.test.js

@@ -309,7 +309,7 @@ describe('Open databases', function () {
         console.error(err)
       }
 
-      const onConnected = async (peerId) => {
+      const onConnected = async (peerId, heads) => {
         connected = true
       }