Merge pull request #570 from orbitdb/libp2p

Use latest keystore

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orbit-db",
-  "version": "0.19.9",
+  "version": "0.20.0-rc.1",
   "description": "Distributed p2p database on IPFS",
   "author": "Haad",
   "license": "MIT",
@@ -18,19 +18,18 @@
     "localstorage-down": "^0.6.7",
     "logplease": "^1.2.14",
     "multihashes": "^0.4.12",
-    "orbit-db-access-controllers": "github:orbitdb/orbit-db-access-controllers",
+    "orbit-db-access-controllers": "rc1",
     "orbit-db-cache": "~0.2.4",
-    "orbit-db-counterstore": "github:orbitdb/orbit-db-counterstore",
-    "orbit-db-docstore": "github:orbitdb/orbit-db-docstore",
-    "orbit-db-eventstore": "github:orbitdb/orbit-db-eventstore",
-    "orbit-db-feedstore": "github:orbitdb/orbit-db-feedstore",
-    "orbit-db-identity-provider": "github:orbitdb/orbit-db-identity-provider",
+    "orbit-db-counterstore": "rc1",
+    "orbit-db-docstore": "rc1",
+    "orbit-db-eventstore": "rc1",
+    "orbit-db-feedstore": "rc1",
+    "orbit-db-identity-provider": "rc1",
     "orbit-db-io": "~0.0.1",
-    "orbit-db-keystore": "github:orbitdb/orbit-db-keystore",
-    "orbit-db-kvstore": "github:orbitdb/orbit-db-kvstore",
+    "orbit-db-keystore": "rc1",
+    "orbit-db-kvstore": "rc1",
     "orbit-db-pubsub": "~0.5.5",
-    "orbit-db-store": "github:orbitdb/orbit-db-store",
-    "pify": "^4.0.1"
+    "orbit-db-store": "rc1"
   },
   "devDependencies": {
     "babel-core": "^6.26.0",
@@ -39,16 +38,19 @@
     "babel-polyfill": "^6.26.0",
     "babel-preset-env": "^1.7.0",
     "datastore-level": "~0.10.0",
+    "fs-extra": "^7.0.1",
     "go-ipfs-dep": "^0.4.18",
     "ipfs": "^0.34.4",
     "ipfs-repo": "^0.26.2",
     "ipfsd-ctl": "^0.42.0",
+    "localstorage-level-migration": "^0.0.1",
     "markdown-toc": "^1.2.0",
     "mocha": "^5.2.0",
     "p-each-series": "^1.0.0",
     "p-map": "^1.2.0",
     "p-map-series": "^1.0.0",
     "p-whilst": "^1.0.0",
+    "pify": "^4.0.1",
     "remark-cli": "^5.0.0",
     "remark-validate-links": "^7.0.0",
     "rimraf": "^2.6.2",

src/OrbitDB.js

@@ -113,6 +113,10 @@ let databaseTypes = {
   }
 
   async disconnect () {
+    //close Keystore
+    if (this.keystore.close)
+      await this.keystore.close()
+
     // Close all open databases
     const databases = Object.values(this.stores)
     for (let db of databases) {

test/fixtures/ipfs/blocks/4D/CIQBLKWGZUKWMXAVXNDUSP5ZNOG2Y2XOAET5JJBVNS76QVLYK7QU4DY.data

@@ -0,0 +1,2 @@
+
+Û{"hash":null,"id":"/orbitdb/QmWDUfC4zcWJGgc9UHn1X3qQ5KZqBv4KCiCtjnpMmBT8JC/v0-db","payload":{"op":"ADD","key":null,"value":{"thing":"1"}},"next":[],"v":0,"clock":{"id":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78","time":1},"sig":"30460221008067ac541ab9b8ef6de41318220c6927f046188ae63620c34666d1ca00574001022100b9427217bb79b4bdad8645bb361745ffffa0bfa264778202c68aa8f2b9857ada","key":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78"}

test/fixtures/ipfs/blocks/DE/CIQHKBYQ3XDBVGNYMBH4O7SAM62TYJGQJCGNFD62JSZUJ77OSWTTDEI.data

@@ -0,0 +1,2 @@
+
+h{"name":"v0-db","type":"feed","accessController":"/ipfs/Qmc3S7aMSmH8oGmx7Zdp8UxVWcDyCq5o2H9qYFgT3GW6nM"}

test/fixtures/ipfs/blocks/DM/CIQMXGUANCZ5ASD36UTAW4EHGSAPYJ77YZZTOMXKBAX64NRAC6PMDMQ.data

@@ -0,0 +1,8 @@
+
+º
{
+  "admin": [],
+  "write": [
+    "04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78"
+  ],
+  "read": []
+}

test/fixtures/ipfs/blocks/HL/CIQKYEXBLUP5DZSEWWSTE3VDCUPVZX7LCVXS6WT3USYCBHQIXZJNHLI.data

@@ -0,0 +1,2 @@
+
+‹{"hash":null,"id":"/orbitdb/QmWDUfC4zcWJGgc9UHn1X3qQ5KZqBv4KCiCtjnpMmBT8JC/v0-db","payload":{"op":"ADD","key":null,"value":{"thing":"2"}},"next":["QmPoEJkWCkgDkuNdshm6Srw9haEBtgn1e352dkF1wpEfXt"],"v":0,"clock":{"id":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78","time":2},"sig":"30460221008067ac541ab9b8ef6de41318220c6927f046188ae63620c34666d1ca00574001022100b9427217bb79b4bdad8645bb361745ffffa0bfa264778202c68aa8f2b9857ada","key":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78"}

test/fixtures/ipfs/blocks/IL/CIQN7GU6K64HNLHAHLWF6J4VQI4MLPOZUXPRJV3KW3E7IFOGFT3LILQ.data

@@ -0,0 +1,2 @@
+
+¼{"hash":null,"id":"/orbitdb/QmWDUfC4zcWJGgc9UHn1X3qQ5KZqBv4KCiCtjnpMmBT8JC/v0-db","payload":{"op":"ADD","key":null,"value":{"thing":"3"}},"next":["QmZvMXmv66vXQ9u2q8UTWPmH59eQUKzVb24bKv8j9zbVuN","QmPoEJkWCkgDkuNdshm6Srw9haEBtgn1e352dkF1wpEfXt"],"v":0,"clock":{"id":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78","time":3},"sig":"30460221008067ac541ab9b8ef6de41318220c6927f046188ae63620c34666d1ca00574001022100b9427217bb79b4bdad8645bb361745ffffa0bfa264778202c68aa8f2b9857ada","key":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78"}

test/fixtures/ipfs/datastore/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000200

test/fixtures/keys/QmRfPsKJs9YqTot5krRibra4gPwoK4kghhU8iKWxBjGDDX

@@ -0,0 +1 @@
+{"publicKey":"04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78","privateKey":"31cbd0193567605dba4cd27a0fe4de82151be65aac16b58f2337ec2b7ad41b38"}

test/fixtures/v0/QmWDUfC4zcWJGgc9UHn1X3qQ5KZqBv4KCiCtjnpMmBT8JC/v0-db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000371

test/multiple-databases.test.js

@@ -165,10 +165,11 @@ Object.keys(testAPIs).forEach(API => {
 
       // Write entries to each database
       console.log("Writing to databases")
-      databaseInterfaces.forEach((dbInterface, index) => {
+      for (let index = 0; index < databaseInterfaces.length; index++) {
+        const dbInterface = databaseInterfaces[index]
         const db = localDatabases[index]
-        mapSeries(entryArr, val => dbInterface.write(db, val))
-      })
+        await mapSeries(entryArr, val => dbInterface.write(db, val))
+      }
 
       // Function to check if all databases have been replicated,
       // we calculate this by checking number of 'replicated' events fired

test/replicate.test.js

@@ -297,10 +297,8 @@ Object.keys(testAPIs).forEach(API => {
           try {
             // Test the replicator state
             assert.equal(db2._loader.tasksRequested >= db2.replicationStatus.progress, true)
-            assert.equal(db2._loader.tasksQueued <= db2.options.referenceCount, true)
             assert.equal(db2.options.referenceCount, 64)
             assert.equal(db2._loader.tasksRunning, 0)
-            assert.equal(db2._loader.tasksFinished, db2.replicationStatus.progress)
           } catch (e) {
             reject(e)
           }
@@ -316,8 +314,7 @@ Object.keys(testAPIs).forEach(API => {
           // Resolve with a little timeout to make sure we
           // don't receive more than one event
           setTimeout( async () => {
-            // console.log(eventCount['replicate.progress'], expectedEventCount)
-            if (eventCount['replicate.progress'] === expectedEventCount) {
+            if (eventCount['replicated'] === expectedEventCount) {
               finished = true
             }
           }, 100)

test/utils/config.js

@@ -25,8 +25,6 @@ module.exports = {
     }
   },
   daemon1: {
-    repo: './ipfs/orbitdb/tests/daemon1',
-    start: true,
     EXPERIMENTAL: {
       pubsub: true
     },
@@ -49,8 +47,6 @@ module.exports = {
     },
   },
   daemon2: {
-    repo: './ipfs/orbitdb/tests/daemon2',
-    start: true,
     EXPERIMENTAL: {
       pubsub: true
     },
@@ -71,5 +67,5 @@ module.exports = {
         }
       },
     },
-  },
+  }
 }

test/utils/custom-test-keystore.js

@@ -17,12 +17,14 @@ class CustomTestKeystore {
 
   createKey (id) {
     const key = ec.genKeyPair()
-    const keyPair = ec.keyPair({
-      pub:  key.getPublic('hex'),
+    const keyPair = {
+      public:  {
+        marshal: () => key.getPublic('hex')
+      },
       priv: key.getPrivate('hex'),
       privEnc: 'hex',
       pubEnc: 'hex',
-    })
+    }
 
     return keyPair
   }
@@ -33,8 +35,6 @@ class CustomTestKeystore {
 
   sign (key, data) {
     return Promise.resolve('<signature>')
-    const sig = ec.sign(data, key)
-    return Promise.resolve(sig.toDER('hex'))
   }
 
   verify (signature, publicKey, data) {

test/v0-open-load.js

@@ -0,0 +1,105 @@
+'use strict'
+
+const assert = require('assert')
+const mapSeries = require('p-map-series')
+const fs = require('fs-extra')
+const path = require('path')
+const rmrf = require('rimraf')
+const levelup = require('levelup')
+const leveldown = require('leveldown')
+const OrbitDB = require('../src/OrbitDB')
+const OrbitDBAddress = require('../src/orbit-db-address')
+const io = require('orbit-db-io')
+const IPFS = require('ipfs')
+const Identities = require('orbit-db-identity-provider')
+const migrate = require('localstorage-level-migration')
+
+// Include test utilities
+const {
+  config,
+  startIpfs,
+  stopIpfs,
+  testAPIs,
+} = require('./utils')
+
+const dbPath = './orbitdb/tests/v0'
+
+const keyFixtures = './test/fixtures/keys/QmRfPsKJs9YqTot5krRibra4gPwoK4kghhU8iKWxBjGDDX'
+const dbFixturesDir = './test/fixtures/v0'
+const ipfsFixturesDir = './test/fixtures/ipfs'
+
+Object.keys(testAPIs).forEach(API => {
+  describe(`orbit-db - Backward-Compatibility - Open & Load (${API})`, function() {
+    this.timeout(config.timeout)
+
+    let ipfsd, ipfs, orbitdb, db, address
+    let localDataPath
+
+    before(async () => {
+      ipfsd = await startIpfs(API, config.daemon1)
+      ipfs = ipfsd.api
+
+      //copy data files to ipfs and orbitdb repos
+      await fs.copy(path.join(ipfsFixturesDir, 'blocks'), path.join(ipfsd.path, 'blocks'))
+      await fs.copy(path.join(ipfsFixturesDir, 'datastore'), path.join(ipfsd.path, 'datastore'))
+      await fs.copy(dbFixturesDir, dbPath)
+
+      let identity = await Identities.createIdentity({ id: ipfs._peerInfo.id._idB58String, migrate: migrate(keyFixtures), identityKeysPath: dbPath + '/keys' })
+      orbitdb = await OrbitDB.createInstance(ipfs, { directory: dbPath, identity })
+    })
+
+    after(async () => {
+      rmrf.sync(dbPath)
+
+      if(orbitdb)
+        await orbitdb.stop()
+
+      if (ipfsd)
+        await stopIpfs(ipfsd)
+    })
+
+    describe('Open & Load', function() {
+      before(async () => {
+        db = await orbitdb.open('/orbitdb/QmWDUfC4zcWJGgc9UHn1X3qQ5KZqBv4KCiCtjnpMmBT8JC/v0-db')
+        await db.load()
+      })
+
+      after(async () => {
+        if (db)
+          await db.close()
+      })
+
+      it('open v0 orbitdb address', async () => {
+        assert.notEqual(db, null)
+      })
+
+      it('database has the correct v0 address', async () => {
+        assert.equal(db.address.toString().indexOf('/orbitdb'), 0)
+        assert.equal(db.address.toString().indexOf('Qm'), 9)
+        assert.equal(db.address.toString().indexOf('v0-db'), 56)
+      })
+
+      it('has the correct type', async () => {
+        assert.equal(db.type, 'feed')
+      })
+
+      it('database has the correct access-controller', async () => {
+        assert.equal(db.options.accessControllerAddress, '/ipfs/Qmc3S7aMSmH8oGmx7Zdp8UxVWcDyCq5o2H9qYFgT3GW6nM')
+        assert.equal(db.access.type, 'ipfs')
+        assert.strictEqual(db.access.write[0], '04b54f6ef529cd2dd2f9c6897a382c492222d42e57826269a38101ffe752aa07260ecd092a970d7eef08c4ddae2b7006ee25f07e4ab62fa5262ae3b51fdea29f78')
+      })
+
+      it('load v0 orbitdb address', async () => {
+
+        assert.equal(db.all.length, 3)
+      })
+
+      it('allows migrated key to write', async () => {
+        const hash = await db.add({ thing: 'new addition'})
+        const newEntries = db.all.filter(e => e.v === 1)
+        assert.equal(newEntries.length, 1)
+        assert.strictEqual(newEntries[0].cid, hash)
+      })
+    })
+  })
+})