Mirroristas/orbitdb
2
0
Fork 0
mirror of https://github.com/orbitdb/orbitdb.git synced 2025-03-30 15:08:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix replication

Browse Source 
Add tests to make sure a database can be saved and loaded multiple times in a row
Add a test to make sure the correct replication progress is emitted
Add mem-store for faster development testing
Improve tests
Improve logging
Fix replication benchmark
Update packages and build
This commit is contained in:
haad 2017-12-11 19:48:20 +01:00
parent 86849c26bb
commit 6061b8cb53
14 changed files with 774 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
benchmarks/benchmark-replication.js
View File

@ -67,7 +67,7 @@ const queryLoop = async (db) => {
try {
await db.add(metrics1.totalQueries)
} catch (e) {
console.error("!!", e)
console.error(e)
}
metrics1.totalQueries ++
metrics1.lastTenSeconds ++
@ -89,7 +89,7 @@ const outputMetrics = (name, db, metrics) => {
}
const database = 'benchmark-replication'
const updateCount = 2000
const updateCount = 20000
// Start
console.log("Starting IPFS daemons...")
@ -139,8 +139,7 @@ pMapSeries([conf1, conf2], d => startIpfs(d))
let prevCount = 0
setInterval(() => {
try {
const result = db2.iterator({ limit: -1 }).collect()
metrics2.totalQueries = result.length
metrics2.totalQueries = db2._oplog.length
metrics2.queriesPerSecond = metrics2.totalQueries - prevCount
metrics2.lastTenSeconds += metrics2.queriesPerSecond
prevCount = metrics2.totalQueries
@ -152,7 +151,7 @@ pMapSeries([conf1, conf2], d => startIpfs(d))
process.exit(0)
}
} catch (e) {
console.error("!", e)
console.error(e)
}
}, 1000)
}

8
dist/es5/OrbitDB.js vendored
View File

@ -213,7 +213,9 @@ var OrbitDB = function () {
value: async function _onMessage(address, heads) {
var store = this.stores[address];
try {
logger.debug('Received heads for \'' + address + '\':\n', (0, _stringify2.default)(heads, null, 2));
logger.debug('Received ' + heads.length + ' heads for \'' + address + '\':\n', (0, _stringify2.default)(heads.map(function (e) {
return e.hash;
}), null, 2));
await store.sync(heads);
} catch (e) {
logger.error(e);
@ -231,7 +233,9 @@ var OrbitDB = function () {
// Send the newly connected peer our latest heads
var heads = store._oplog.heads;
if (heads.length > 0) {
logger.debug('Send latest heads of \'' + address + '\':\n', (0, _stringify2.default)(heads, null, 2));
logger.debug('Send latest heads of \'' + address + '\':\n', (0, _stringify2.default)(heads.map(function (e) {
return e.hash;
}), null, 2));
room.sendTo(peer, (0, _stringify2.default)(heads));
}
store.events.emit('peer', peer);

2
dist/orbitdb.min.js vendored
View File

File diff suppressed because one or more lines are too long

4
examples/browser/example.js
View File

@ -81,6 +81,8 @@ const main = (IPFS, ORBITDB) => {
// When we update the database, display result
db.events.on('write', () => queryAndRender(db))
db.events.on('replicate.progress', () => queryAndRender(db))
// Hook up to the load progress event and render the progress
let maxTotal = 0, loaded = 0
db.events.on('load.progress', (address, hash, entry, progress, total) => {
@ -242,7 +244,7 @@ const main = (IPFS, ORBITDB) => {
<br>
<div><b>Peer ID:</b> ${orbitdb.id}</div>
<div><b>Peers (database/network):</b> ${databasePeers.length} / ${networkPeers.length}</div>
<div><b>Oplog Size:</b> ${db._oplog.length} / ${db._replicationInfo.max}</div>
<div><b>Oplog Size:</b> ${db._replicationInfo.progress} / ${db._replicationInfo.max}</div>
<h2>Results</h2>
<div id="results">
<div>

298
package-lock.json generated
View File

@ -86,9 +86,9 @@
"dev": true
},
"ajv": {
"version": "5.5.1",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.1.tgz",
"integrity": "sha1-s4u4h22ehr7plJVqBOch6IskjrI=",
"version": "5.5.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
"integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
"dev": true,
"requires": {
"co": "4.6.0",
@ -791,7 +791,7 @@
"dev": true,
"requires": {
"babel-runtime": "6.26.0",
"core-js": "2.5.2",
"core-js": "2.5.3",
"regenerator-runtime": "0.10.5"
},
"dependencies": {
@ -843,7 +843,7 @@
"requires": {
"babel-core": "6.26.0",
"babel-runtime": "6.26.0",
"core-js": "2.5.2",
"core-js": "2.5.3",
"home-or-tmp": "2.0.0",
"lodash": "4.17.4",
"mkdirp": "0.5.1",
@ -856,7 +856,7 @@
"integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
"dev": true,
"requires": {
"core-js": "2.5.2",
"core-js": "2.5.3",
"regenerator-runtime": "0.11.1"
}
},
@ -1767,9 +1767,9 @@
}
},
"core-js": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.2.tgz",
"integrity": "sha1-vEZIZW59ydyA19PHu8Fy2W50TmM=",
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.3.tgz",
"integrity": "sha1-isw4NFgk8W2DZbfJtCWRaOjtYD4=",
"dev": true
},
"core-util-is": {
@ -2023,7 +2023,7 @@
"integrity": "sha512-/cLUpQduF6bNrWuAC4pwtUKA5t669pCsCi2XbmojG2tFeOr9j6ShtdDCtFFQO1DRt+EVZhx9gPzP9G2bUaG4ig==",
"dev": true,
"requires": {
"errno": "0.1.4"
"errno": "0.1.6"
}
},
"level-iterator-stream": {
@ -2072,12 +2072,6 @@
"integrity": "sha1-2Vv3IeyHfgjbJ27T/G63j5CDrUY=",
"dev": true
},
"prr": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
"integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
"dev": true
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
@ -2320,13 +2314,13 @@
"dev": true
},
"encoding-down": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-3.0.0.tgz",
"integrity": "sha1-IGjLZ7E3G14frJtfF44FpVUr+l4=",
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-3.0.1.tgz",
"integrity": "sha512-uvx+39YNqiPLqhXAvOSGBVy/oYBh4p2ShwG9YFCipwgfOhnVIOxuOPE3R9dEGM44bn0VHIrC3ojXq6lNf2ulwg==",
"requires": {
"abstract-leveldown": "3.0.0",
"level-codec": "8.0.0",
"level-errors": "1.1.1"
"level-errors": "1.1.2"
}
},
"end-of-stream": {
@ -2349,7 +2343,7 @@
"debug": "2.6.9",
"engine.io-parser": "2.1.1",
"uws": "0.14.5",
"ws": "3.3.2"
"ws": "3.3.3"
},
"dependencies": {
"debug": {
@ -2377,7 +2371,7 @@
"indexof": "0.0.1",
"parseqs": "0.0.5",
"parseuri": "0.0.5",
"ws": "3.3.2",
"ws": "3.3.3",
"xmlhttprequest-ssl": "1.5.4",
"yeast": "0.1.2"
},
@ -2428,11 +2422,11 @@
}
},
"errno": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/errno/-/errno-0.1.4.tgz",
"integrity": "sha1-uJbiOp5ei6M4cfyZar02NfyaHH0=",
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/errno/-/errno-0.1.6.tgz",
"integrity": "sha512-IsORQDpaaSwcDP4ZZnHxgE85werpo34VYn1Ud3mq+eUsF593faR8oCZNXrROVkpFu2TsbrNhHin0aUrTsQ9vNw==",
"requires": {
"prr": "0.0.0"
"prr": "1.0.1"
}
},
"error-ex": {
@ -4172,7 +4166,7 @@
"dev": true,
"optional": true,
"requires": {
"ajv": "5.5.1",
"ajv": "5.5.2",
"har-schema": "2.0.0"
}
},
@ -4564,9 +4558,9 @@
}
},
"ipfs": {
"version": "0.27.3",
"resolved": "https://registry.npmjs.org/ipfs/-/ipfs-0.27.3.tgz",
"integrity": "sha512-pUvmBE/Vo5vGPuChJchl/1AQBldjmtJU7iddtbR9iLz8f1qdtz2y7p9HyHsPM8N7t+VjB1Y+B2JchUap4wPdUg==",
"version": "0.27.5",
"resolved": "https://registry.npmjs.org/ipfs/-/ipfs-0.27.5.tgz",
"integrity": "sha512-msXTU+vgXArtC9PVEvOIv9auSfNRpGwFFgcoTB0ZLkmGDkG0LuUieRHMuibjYySqVE5rP0upUdqPeBXzOx40NA==",
"dev": true,
"requires": {
"async": "2.6.0",
@ -4586,23 +4580,23 @@
"hapi-set-header": "1.0.2",
"hoek": "5.0.2",
"ipfs-api": "17.2.4",
"ipfs-bitswap": "0.17.4",
"ipfs-bitswap": "0.18.0",
"ipfs-block": "0.6.1",
"ipfs-block-service": "0.13.0",
"ipfs-multipart": "0.1.0",
"ipfs-repo": "0.18.4",
"ipfs-repo": "0.18.5",
"ipfs-unixfs": "0.1.14",
"ipfs-unixfs-engine": "0.24.1",
"ipfs-unixfs-engine": "0.24.2",
"ipld-resolver": "0.14.1",
"is-ipfs": "0.3.2",
"is-stream": "1.1.0",
"joi": "13.0.2",
"libp2p": "0.13.3",
"libp2p": "0.14.3",
"libp2p-circuit": "0.1.4",
"libp2p-floodsub": "0.13.1",
"libp2p-kad-dht": "0.6.0",
"libp2p-mdns": "0.9.1",
"libp2p-multiplex": "0.5.0",
"libp2p-multiplex": "0.5.1",
"libp2p-railing": "0.7.1",
"libp2p-secio": "0.8.1",
"libp2p-tcp": "0.11.1",
@ -4622,12 +4616,13 @@
"path-exists": "3.0.0",
"peer-book": "0.5.2",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"progress": "2.0.0",
"prom-client": "10.2.2",
"prometheus-gc-stats": "0.5.0",
"promisify-es6": "1.0.3",
"pull-abortable": "4.1.1",
"pull-defer": "0.2.2",
"pull-file": "1.1.0",
"pull-ndjson": "0.1.1",
"pull-paramap": "1.2.2",
@ -4673,7 +4668,7 @@
"ndjson": "1.5.0",
"once": "1.4.0",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"promisify-es6": "1.0.3",
"pull-defer": "0.2.2",
"pull-pushable": "2.1.1",
@ -4687,12 +4682,13 @@
}
},
"ipfs-bitswap": {
"version": "0.17.4",
"resolved": "https://registry.npmjs.org/ipfs-bitswap/-/ipfs-bitswap-0.17.4.tgz",
"integrity": "sha512-IMDA0dr377fVaDdi6tVOTPlmbPQkWxe/KBsL85epjIhsKtFoZwz9WvY2INpNotT6OdTA/Wiwr2pWrFQsymDIeA==",
"version": "0.18.0",
"resolved": "https://registry.npmjs.org/ipfs-bitswap/-/ipfs-bitswap-0.18.0.tgz",
"integrity": "sha512-nPtMoIA68J/vwEiGHVTiWscG1QtyHCwyl+OhNlqArHw6EGeNLIE+6/Gm2F/jc1keboNstJo+mGMHF5L7YR0eYw==",
"dev": true,
"requires": {
"async": "2.6.0",
"big.js": "5.0.3",
"cids": "0.5.2",
"debug": "3.1.0",
"ipfs-block": "0.6.1",
@ -4705,6 +4701,7 @@
"lodash.sortby": "4.7.0",
"lodash.uniqwith": "4.5.0",
"lodash.values": "4.3.0",
"moving-average": "1.0.0",
"multicodec": "0.2.5",
"multihashing-async": "0.4.7",
"protons": "1.0.1",
@ -4714,6 +4711,14 @@
"pull-stream": "3.6.1",
"safe-buffer": "5.1.1",
"varint-decoder": "0.1.1"
},
"dependencies": {
"big.js": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/big.js/-/big.js-5.0.3.tgz",
"integrity": "sha512-av8LNZGBl4cg2r4ZhWqghJOxi2P8UCcWhdmrFgcHPMmUJ6jx1FbnyxjwL4URYzMK3QJg60qeMefQhv9G14oYKA==",
"dev": true
}
}
},
"ipfs-block": {
@ -4732,9 +4737,9 @@
"dev": true
},
"ipfs-log": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/ipfs-log/-/ipfs-log-4.0.5.tgz",
"integrity": "sha512-ZCf1dE8HsLs0G7hNrRs2ICZ5+e83pcddEH2ZY/VPsbrShZumsq4zHLL0QN+bmgm1AbdAZrzpS7wuqHPNeFRO7g==",
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ipfs-log/-/ipfs-log-4.0.6.tgz",
"integrity": "sha512-LcCFq8AF8CDCKFAM78QkbH5+6VHk0sX5qu6k7/b7wVetuaTx3Xq85axfFjj7D5YeQdssBGLqex1x6EAx9MX9+Q==",
"requires": {
"p-map": "1.2.0",
"p-whilst": "1.0.0"
@ -4751,9 +4756,7 @@
}
},
"ipfs-pubsub-room": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/ipfs-pubsub-room/-/ipfs-pubsub-room-1.0.1.tgz",
"integrity": "sha512-kuEjjrKIIWiK0JNW+NA684g3k/hMR9qYG93WXGMzsVGLcy2+nPxLSv680itAie3wJxtFtXcYzD4VjRO9rhygyQ==",
"version": "github:haadcode/ipfs-pubsub-room#68aecffc0c89b2430349d59c2556c0f70b2f96e5",
"requires": {
"hyperdiff": "2.0.4",
"lodash.clonedeep": "4.5.0",
@ -4763,9 +4766,9 @@
}
},
"ipfs-repo": {
"version": "0.18.4",
"resolved": "https://registry.npmjs.org/ipfs-repo/-/ipfs-repo-0.18.4.tgz",
"integrity": "sha512-sBrde6bIpzywv2hxqkTsOT4njJjme4cXuFNKbcaH/hGuz17MUpX+bGgeZsOhc4lVBSWh6VM/uVbS+vFfqEVdzw==",
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/ipfs-repo/-/ipfs-repo-0.18.5.tgz",
"integrity": "sha512-QELzb2fLqM2t9WQuiD60iRbC8KrxQ3GKODAOxTSnfrT+KKWls/2n/yw70mW80eemCRz6/9HSWXZdjmG2eISSaA==",
"dev": true,
"requires": {
"async": "2.6.0",
@ -4847,9 +4850,9 @@
}
},
"ipfs-unixfs-engine": {
"version": "0.24.1",
"resolved": "https://registry.npmjs.org/ipfs-unixfs-engine/-/ipfs-unixfs-engine-0.24.1.tgz",
"integrity": "sha512-Xm4jxdZFsBe7mwKhpqPxGaH2rMvoAbdovYKrA75ZZJwHT7rocepE+4Tx0Gpr+0mJnmGO9eZ2bzbvtqA71nNWmw==",
"version": "0.24.2",
"resolved": "https://registry.npmjs.org/ipfs-unixfs-engine/-/ipfs-unixfs-engine-0.24.2.tgz",
"integrity": "sha512-9HCnC+gkXqK35eARGYm77hnHZwIk9MoHiTG6eaXFueP341EIvuTWy6o1NJ0dw5Hp6AiK1fwN0AG6P8GqnZIWZA==",
"dev": true,
"requires": {
"async": "2.6.0",
@ -4868,7 +4871,7 @@
"pull-cat": "1.1.11",
"pull-pair": "1.1.0",
"pull-paramap": "1.2.2",
"pull-pause": "0.0.1",
"pull-pause": "0.0.2",
"pull-pushable": "2.1.1",
"pull-stream": "3.6.1",
"pull-traverse": "1.0.3",
@ -4973,7 +4976,7 @@
"interface-datastore": "0.4.2",
"ipfs-block": "0.6.1",
"ipfs-block-service": "0.13.0",
"ipfs-repo": "0.18.4",
"ipfs-repo": "0.18.5",
"ipld-dag-cbor": "0.11.2",
"ipld-dag-pb": "0.11.4",
"ipld-ethereum": "1.4.4",
@ -5454,11 +5457,11 @@
}
},
"level": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/level/-/level-2.1.0.tgz",
"integrity": "sha512-J1MuO0iJuG93xsJjugr2qrAcC95RVKMqrAiio1AcoM2FaAxrUJacAbtfLqYqr7xwQ/NiTXCMfBX71gNmGROXOA==",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/level/-/level-2.1.1.tgz",
"integrity": "sha512-jhsItEs/L5bf5ctKsvIPcMzeh4nwXxnL3Tsxm4E9My07jcFfqEYwKP57Y9AZu7tmtECdyK6bvCqoClwuOBuY9w==",
"requires": {
"level-packager": "2.0.2",
"level-packager": "2.1.0",
"leveldown": "2.1.1"
}
},
@ -5468,11 +5471,11 @@
"integrity": "sha512-gNZlo1HRHz0BWxzGCyNf7xntAs2HKOPvvRBWtXsoDvEX4vMYnSTBS6ZnxoaiX7nhxSBPpegRa8CQ/hnfGBKk3Q=="
},
"level-errors": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/level-errors/-/level-errors-1.1.1.tgz",
"integrity": "sha512-9MIIbizlJgWFQ6m45ehVuSrpzFxwJQmZYD6sfmiizhdmWMNUf41mBYpUJEeCslIa3sB4vdsIFCimPdDZkWznwA==",
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/level-errors/-/level-errors-1.1.2.tgz",
"integrity": "sha512-Sw/IJwWbPKF5Ai4Wz60B52yj0zYeqzObLh8k1Tk88jVmD51cJSKWSYpRyhVIvFzZdvsPqlH5wfhp/yxdsaQH4w==",
"requires": {
"errno": "0.1.4"
"errno": "0.1.6"
}
},
"level-iterator-stream": {
@ -5528,11 +5531,11 @@
}
},
"level-packager": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/level-packager/-/level-packager-2.0.2.tgz",
"integrity": "sha1-AztxFnhELPlWyVFFYnPcY+QXQEc=",
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/level-packager/-/level-packager-2.1.0.tgz",
"integrity": "sha512-I97zvGOZ6fJ9OFfYv+QmgBpDWbC+UaP5ERJ3oraTyk1v+ABAL4tazris5ym5qL2iLe+qNjXNM/iP8LQcoZMEWw==",
"requires": {
"encoding-down": "3.0.0",
"encoding-down": "3.0.1",
"levelup": "2.0.1"
}
},
@ -5599,25 +5602,25 @@
"integrity": "sha1-PckbPmMtN8nlRiOchkEYsATJ+GA=",
"requires": {
"deferred-leveldown": "2.0.3",
"level-errors": "1.1.1",
"level-errors": "1.1.2",
"level-iterator-stream": "2.0.0",
"xtend": "4.0.1"
}
},
"libp2p": {
"version": "0.13.3",
"resolved": "https://registry.npmjs.org/libp2p/-/libp2p-0.13.3.tgz",
"integrity": "sha512-u2STVP3FcC7kL16RykcS1ik5fBgNYgLVpPERpObRtwVpfnWgqWT9/faaJ649LGHBlH/4z/4piOrvgIeIdlnDVg==",
"version": "0.14.3",
"resolved": "https://registry.npmjs.org/libp2p/-/libp2p-0.14.3.tgz",
"integrity": "sha512-5eL/gA1XuYAo0XrqznvkJvagF/nHakNX+bC7Ksuqq574k1cNZYiqnFc0O+zdD8up0BJt0LAXgd9IPB4TX3rUVg==",
"dev": true,
"requires": {
"async": "2.6.0",
"libp2p-ping": "0.6.0",
"libp2p-swarm": "0.33.2",
"libp2p-swarm": "0.34.0",
"mafmt": "3.0.2",
"multiaddr": "3.0.1",
"peer-book": "0.5.2",
"peer-id": "0.10.3",
"peer-info": "0.11.3"
"peer-info": "0.11.4"
}
},
"libp2p-circuit": {
@ -5635,7 +5638,7 @@
"multiaddr": "3.0.1",
"multistream-select": "0.14.1",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"protons": "1.0.1",
"pull-abortable": "4.1.1",
"pull-handshake": "1.1.4",
@ -5701,7 +5704,7 @@
"requires": {
"multiaddr": "3.0.1",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"protons": "1.0.1",
"pull-length-prefixed": "1.3.0",
"pull-stream": "3.6.1"
@ -5725,7 +5728,7 @@
"libp2p-record": "0.5.1",
"multihashing-async": "0.4.7",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"priorityqueue": "0.2.0",
"protons": "1.0.1",
"pull-length-prefixed": "1.3.0",
@ -5745,13 +5748,13 @@
"multiaddr": "3.0.1",
"multicast-dns": "6.2.1",
"peer-id": "0.10.3",
"peer-info": "0.11.3"
"peer-info": "0.11.4"
}
},
"libp2p-multiplex": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/libp2p-multiplex/-/libp2p-multiplex-0.5.0.tgz",
"integrity": "sha512-cZjK66wr2zcaC5sW0QjBZ/lXouBlYQ9Hr3Y6j2DImTtOjPufccnaULkCxbnpnfQ/g52AEsIQh0DlwW4L0gZ8Ag==",
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/libp2p-multiplex/-/libp2p-multiplex-0.5.1.tgz",
"integrity": "sha512-XZagf1B31Vkbd+iAai8cOfRkcZWba+MtSRf/xQJ7uOz7O4heLMNC4orcDqT0n+Hkkn3zfqllH75Kp1xJ+Vog2w==",
"dev": true,
"requires": {
"async": "2.6.0",
@ -5759,7 +5762,20 @@
"pull-catch": "1.0.0",
"pull-stream": "3.6.1",
"pull-stream-to-stream": "1.3.4",
"pump": "2.0.0",
"stream-to-pull-stream": "1.7.2"
},
"dependencies": {
"pump": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-2.0.0.tgz",
"integrity": "sha512-6MYypjOvtiXhBSTOD0Zs5eNjCGfnqi5mPsCsW+dgKTxrZzQMZQNpBo3XRkLx7id753f3EeyHLBqzqqUymIolgw==",
"dev": true,
"requires": {
"end-of-stream": "1.4.0",
"once": "1.4.0"
}
}
}
},
"libp2p-ping": {
@ -5784,7 +5800,7 @@
"lodash": "4.17.4",
"multiaddr": "3.0.1",
"peer-id": "0.10.3",
"peer-info": "0.11.3"
"peer-info": "0.11.4"
}
},
"libp2p-record": {
@ -5822,9 +5838,9 @@
}
},
"libp2p-swarm": {
"version": "0.33.2",
"resolved": "https://registry.npmjs.org/libp2p-swarm/-/libp2p-swarm-0.33.2.tgz",
"integrity": "sha512-+detttstBXcQFiUqKv2kEYf08QntKqLIlRVDjFERMQsLp2ce4wRIymd75Hj5CvqAjdlcjh8WU/sjsNsPU/Dd+A==",
"version": "0.34.0",
"resolved": "https://registry.npmjs.org/libp2p-swarm/-/libp2p-swarm-0.34.0.tgz",
"integrity": "sha512-4I4T3+Zajb1uhdGMU5m1vel07AQY2m/RmIpC9WsheQsWh1AL86keMw6fL5iijrpMUWAU6mlfyuL6r73kdAaBfw==",
"dev": true,
"requires": {
"async": "2.6.0",
@ -5838,7 +5854,7 @@
"multistream-select": "0.14.1",
"once": "1.4.0",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"pull-stream": "3.6.1"
}
},
@ -5876,7 +5892,7 @@
"multiaddr": "3.0.1",
"once": "1.4.0",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"pull-stream": "3.6.1",
"simple-peer": "8.2.0",
"socket.io": "2.0.4",
@ -5901,7 +5917,7 @@
"multiaddr": "3.0.1",
"once": "1.4.0",
"peer-id": "0.10.3",
"peer-info": "0.11.3",
"peer-info": "0.11.4",
"pull-stream": "3.6.1",
"socket.io-client": "2.0.4",
"socket.io-pull-stream": "0.1.3",
@ -6266,7 +6282,7 @@
"integrity": "sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=",
"dev": true,
"requires": {
"errno": "0.1.4",
"errno": "0.1.6",
"readable-stream": "2.3.3"
}
},
@ -6350,7 +6366,7 @@
"integrity": "sha512-/cLUpQduF6bNrWuAC4pwtUKA5t669pCsCi2XbmojG2tFeOr9j6ShtdDCtFFQO1DRt+EVZhx9gPzP9G2bUaG4ig==",
"dev": true,
"requires": {
"errno": "0.1.4"
"errno": "0.1.6"
}
},
"level-iterator-stream": {
@ -6394,12 +6410,6 @@
"xtend": "4.0.1"
}
},
"prr": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
"integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
"dev": true
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
@ -6592,6 +6602,12 @@
"run-queue": "1.0.3"
}
},
"moving-average": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/moving-average/-/moving-average-1.0.0.tgz",
"integrity": "sha512-97cgMz0U2zciiDp4xRl/n+MYgrm9l7UiYbtsBLPr0rhw6KH3m4LyK2w4d96V6+UwKo+ph7KtQSoL2qgnqZVgvA==",
"dev": true
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -6955,11 +6971,11 @@
"dev": true
},
"orbit-db-cache": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/orbit-db-cache/-/orbit-db-cache-0.1.1.tgz",
"integrity": "sha512-iDzcLMSzuUTdypiaP0fXcIHtxdqJJrlwkYhTpAnpIr+3f+t+hRtyC99/ZwGdHOuWIIFiajDDoC8JPYCv4Rfkqg==",
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/orbit-db-cache/-/orbit-db-cache-0.1.2.tgz",
"integrity": "sha512-0I2o6VEPJ/kYMFTLWRqB7ek7G28GbewEFzGP1wOP1o3Uy+P2B/MKBT7sfa9yZ2pJRq4X0R5xZLtVWldpYK5nzQ==",
"requires": {
"level": "2.1.0",
"level": "2.1.1",
"mkdirp": "0.5.1"
}
},
@ -6969,7 +6985,7 @@
"integrity": "sha512-25CtAWuT5Ni4rDLziNj5qbjgAP0d+nyWWV9Tx30VqPXgCA4hYG3hAgIs3EYCVLWIRFZH7xdCfLyHfXY/v8pQvg==",
"requires": {
"crdts": "0.1.5",
"orbit-db-store": "2.1.0"
"orbit-db-store": "2.1.1"
}
},
"orbit-db-docstore": {
@ -6977,7 +6993,7 @@
"resolved": "https://registry.npmjs.org/orbit-db-docstore/-/orbit-db-docstore-1.1.0.tgz",
"integrity": "sha512-cELdzNVMZcjXP6QalX5xjd5a0ajy1gI+d837z8VEZJiJSqokJXgj7jhsEV6nZYtF3yTS5ky5eGD6f446cVrmzA==",
"requires": {
"orbit-db-store": "2.1.0",
"orbit-db-store": "2.1.1",
"p-map": "1.1.1"
},
"dependencies": {
@ -6989,11 +7005,11 @@
}
},
"orbit-db-eventstore": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/orbit-db-eventstore/-/orbit-db-eventstore-1.1.0.tgz",
"integrity": "sha512-+ND1O52dztkpeSrUWFT5IzkW0w13EUOEeUvil13ML0SIyMEEdpIHLDbF4/eNqdfO4wJYthtu1AryP1nTzK8hmQ==",
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/orbit-db-eventstore/-/orbit-db-eventstore-1.1.3.tgz",
"integrity": "sha512-5R01vEl98iztagCab7Wu+futTR2Ud6g8YaoHzNEZBtDdaJQ7UZhJCzzD0T42oIw9VchEeRUpZjChYSLrv7DgNw==",
"requires": {
"orbit-db-store": "2.1.0"
"orbit-db-store": "2.1.1"
}
},
"orbit-db-feedstore": {
@ -7001,7 +7017,7 @@
"resolved": "https://registry.npmjs.org/orbit-db-feedstore/-/orbit-db-feedstore-1.1.0.tgz",
"integrity": "sha512-dbE7eCVd9sGPx+cGtQjdnNoysavbe5HT7fst4CNsr5Ks4Nbrp+Y/e7kj4lkh/Ltl6TokSqoKtA5mEdmlsQA/Ew==",
"requires": {
"orbit-db-eventstore": "1.1.0"
"orbit-db-eventstore": "1.1.3"
}
},
"orbit-db-keystore": {
@ -7019,24 +7035,25 @@
"resolved": "https://registry.npmjs.org/orbit-db-kvstore/-/orbit-db-kvstore-1.1.0.tgz",
"integrity": "sha512-P9SKWklMcdyc1ZPQmsHJzKTfyLwNvTp/3tlVtPOllHlOwopDt3ItwQnIR4s163RtUQoRDyYEmft3F/FDKhNdbg==",
"requires": {
"orbit-db-store": "2.1.0"
"orbit-db-store": "2.1.1"
}
},
"orbit-db-pubsub": {
"version": "0.3.6",
"resolved": "https://registry.npmjs.org/orbit-db-pubsub/-/orbit-db-pubsub-0.3.6.tgz",
"integrity": "sha512-8hrnwJe6JjzIycUHoBUYYEY6LEtSYkUTX6VcJsneLooqRm5nbIXHwe9Bcoimyui9uH1GJ67+AZKUEePUvdBFFg==",
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/orbit-db-pubsub/-/orbit-db-pubsub-0.3.9.tgz",
"integrity": "sha512-0Pxq7Z3nGP20H9JWqfvFfyjAAnwr8sRNquU2aUIPsBb0jrcHw+fAtaj0m9/5ML2ift4IO17Mn9lydHxkS4M5Ug==",
"requires": {
"ipfs-pubsub-room": "1.0.1",
"ipfs-pubsub-room": "github:haadcode/ipfs-pubsub-room#68aecffc0c89b2430349d59c2556c0f70b2f96e5",
"logplease": "1.2.14"
}
},
"orbit-db-store": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/orbit-db-store/-/orbit-db-store-2.1.0.tgz",
"integrity": "sha512-GSmxqPPNL3JkR1e2Gwd11PejXWqircpy239hwOg4qThUBo05Y6YG6/G92cIgxypu5VgUe0QolCa+C3GOme4c5Q==",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/orbit-db-store/-/orbit-db-store-2.1.1.tgz",
"integrity": "sha512-rW/LiEr0W7i51BDprJNIB1BKqLZhFp74ksxIalQBckZvY7/QmYw3ZNsrO5zaZI2GMRgj+XVKbyikURn5VP7q7A==",
"requires": {
"ipfs-log": "4.0.5",
"ipfs-log": "4.0.6",
"logplease": "1.2.14",
"orbit-db-keystore": "0.0.2",
"p-each-series": "1.0.0",
"readable-stream": "2.3.3"
@ -7289,7 +7306,7 @@
"requires": {
"bs58": "4.0.1",
"peer-id": "0.10.3",
"peer-info": "0.11.3"
"peer-info": "0.11.4"
}
},
"peer-id": {
@ -7305,9 +7322,9 @@
}
},
"peer-info": {
"version": "0.11.3",
"resolved": "https://registry.npmjs.org/peer-info/-/peer-info-0.11.3.tgz",
"integrity": "sha512-FzNzL5PYGr7rpHFpoSMdEGwRVb7X9D88YDmTkamXd4CO05bNeHITlrwIV7ctshBhVRTvokpnI59HUSEQly5rnA==",
"version": "0.11.4",
"resolved": "https://registry.npmjs.org/peer-info/-/peer-info-0.11.4.tgz",
"integrity": "sha512-p+NpRgZpnlz0BGz6ZLFF8vVlqOBDxGwN7AA+QCc4nCICxVpbf4PlmtzwePVtkDqlNwUXYCDKK8pG0FGC5E8B2g==",
"dev": true,
"requires": {
"lodash.uniqby": "4.7.0",
@ -7549,9 +7566,9 @@
}
},
"prr": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/prr/-/prr-0.0.0.tgz",
"integrity": "sha1-GoS4WQgyVQFBGFPQCB7j+obikmo="
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
"integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY="
},
"pseudomap": {
"version": "1.0.2",
@ -7691,9 +7708,9 @@
}
},
"pull-pause": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/pull-pause/-/pull-pause-0.0.1.tgz",
"integrity": "sha1-xJm0Fhqt2+qE9S6JjlcPol1foiw=",
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/pull-pause/-/pull-pause-0.0.2.tgz",
"integrity": "sha1-GdRb6PqmFfpVbxSpb9czRiw3+6M=",
"dev": true
},
"pull-pushable": {
@ -8222,7 +8239,7 @@
"integrity": "sha1-9YdyIs4+kx7a4DnxfrNxbnE3+M8=",
"dev": true,
"requires": {
"ajv": "5.5.1"
"ajv": "5.5.2"
}
},
"secp256k1": {
@ -8261,6 +8278,12 @@
"semver": "5.4.1"
}
},
"serialize-javascript": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.4.0.tgz",
"integrity": "sha1-fJWFFNtqwkQ6irwGLcn3iGp/YAU=",
"dev": true
},
"set-blocking": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
@ -9103,9 +9126,9 @@
"dev": true
},
"uglify-es": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.2.1.tgz",
"integrity": "sha512-c+Fy4VuGvPmT7mj7vEPjRR/iNFuXuOAkufhCtCvTGX0Hr4gCM9YwCnLgHkxr1ngqSODQaDObU3g8SF8uE/tY1w==",
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.2.2.tgz",
"integrity": "sha512-l+s5VLzFwGJfS+fbqaGf/Dfwo1MF13jLOF2ekL0PytzqEqQ6cVppvHf4jquqFok+35USMpKjqkYxy6pQyUcuug==",
"dev": true,
"requires": {
"commander": "2.12.2",
@ -9128,16 +9151,17 @@
"optional": true
},
"uglifyjs-webpack-plugin": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-1.1.2.tgz",
"integrity": "sha512-k07cmJTj+8vZMSc3BaQ9uW7qVl2MqDts4ti4KaNACXEcXSw2vQM2S8olSk/CODxvcSFGvUHzNSqA8JQlhgUJPw==",
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-1.1.4.tgz",
"integrity": "sha512-fRrOJ5tv6YCsJIhP9mPRnfgyo4DVNSIfNOa7Gs9aT1NNpeJc85W7GcbVxQgc+9rU3No6tnkbMqZ4xsgRBU+HGQ==",
"dev": true,
"requires": {
"cacache": "10.0.1",
"find-cache-dir": "1.0.0",
"schema-utils": "0.3.0",
"serialize-javascript": "1.4.0",
"source-map": "0.6.1",
"uglify-es": "3.2.1",
"uglify-es": "3.2.2",
"webpack-sources": "1.1.0",
"worker-farm": "1.5.2"
},
@ -9390,7 +9414,7 @@
"requires": {
"acorn": "5.2.1",
"acorn-dynamic-import": "2.0.2",
"ajv": "5.5.1",
"ajv": "5.5.2",
"ajv-keywords": "2.1.1",
"async": "2.6.0",
"enhanced-resolve": "3.4.1",
@ -9747,7 +9771,7 @@
"integrity": "sha512-XxiQ9kZN5n6mmnW+mFJ+wXjNNI/Nx4DIdaAKLX1Bn6LYBWlN/zaBhu34DQYPZ1AJobQuu67S2OfDdNSVULvXkQ==",
"dev": true,
"requires": {
"errno": "0.1.4",
"errno": "0.1.6",
"xtend": "4.0.1"
}
},
@ -9804,9 +9828,9 @@
}
},
"ws": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.2.tgz",
"integrity": "sha512-t+WGpsNxhMR4v6EClXS8r8km5ZljKJzyGhJf7goJz9k5Ye3+b5Bvno5rjqPuIBn5mnn5GBb7o8IrIWHxX1qOLQ==",
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"dev": true,
"requires": {
"async-limiter": "1.0.0",

2
package.json
View File

@ -26,7 +26,7 @@
"orbit-db-feedstore": "~1.1.0",
"orbit-db-keystore": "~0.0.2",
"orbit-db-kvstore": "~1.1.0",
"orbit-db-pubsub": "~0.3.6"
"orbit-db-pubsub": "~0.3.8"
},
"devDependencies": {
"babel-core": "^6.26.0",

4
src/OrbitDB.js
View File

@ -132,7 +132,7 @@ class OrbitDB {
async _onMessage (address, heads) {
const store = this.stores[address]
try {
logger.debug(`Received heads for '${address}':\n`, JSON.stringify(heads, null, 2))
logger.debug(`Received ${heads.length} heads for '${address}':\n`, JSON.stringify(heads.map(e => e.hash), null, 2))
await store.sync(heads)
} catch (e) {
logger.error(e)
@ -147,7 +147,7 @@ class OrbitDB {
// Send the newly connected peer our latest heads
let heads = store._oplog.heads
if (heads.length > 0) {
logger.debug(`Send latest heads of '${address}':\n`, JSON.stringify(heads, null, 2))
logger.debug(`Send latest heads of '${address}':\n`, JSON.stringify(heads.map(e => e.hash), null, 2))
room.sendTo(peer, JSON.stringify(heads))
}
store.events.emit('peer', peer)

2
test/create-open.test.js
View File

@ -227,7 +227,7 @@ describe('orbit-db - Create & Open', function() {
it('doesn\'t open a database if we don\'t have it locally', async () => {
const address = new OrbitDBAddress(db.address.root.slice(0, -1) + 'A', 'non-existent')
return new Promise((resolve, reject) => {
setTimeout(resolve, 1000)
setTimeout(resolve, 900)
orbitdb.open(address)
.then(() => reject(new Error('Shouldn\'t open the database')))
})

3
test/mocha.opts
View File

@ -1,4 +1,5 @@
--reporter spec
--colors
--recursive
--exit
--exit
--slow 1000

43
test/persistency.js
View File

@ -68,9 +68,37 @@ describe('orbit-db - Persistency', function() {
const items = db.iterator({ limit: -1 }).collect()
assert.equal(items.length, amount)
assert.equal(items[0].payload.value, 'hello' + (entryCount - amount))
assert.equal(items[1].payload.value, 'hello' + (entryCount - amount + 1))
assert.equal(items[items.length - 1].payload.value, 'hello99')
})
it('load and close several times', async () => {
const amount = 16
for (let i = 0; i < amount; i ++) {
db = await orbitdb1.eventlog(address)
await db.load()
const items = db.iterator({ limit: -1 }).collect()
assert.equal(items.length, entryCount)
assert.equal(items[0].payload.value, 'hello0')
assert.equal(items[1].payload.value, 'hello1')
assert.equal(items[items.length - 1].payload.value, 'hello99')
await db.close()
}
})
it('load, add one, close - several times', async () => {
const amount = 8
for (let i = 0; i < amount; i ++) {
db = await orbitdb1.eventlog(address)
await db.load()
await db.add('hello' + (entryCount + i))
const items = db.iterator({ limit: -1 }).collect()
assert.equal(items.length, entryCount + i + 1)
assert.equal(items[items.length - 1].payload.value, 'hello' + (entryCount + i))
await db.close()
}
})
it('loading a database emits \'ready\' event', async () => {
db = await orbitdb1.eventlog(address)
return new Promise(async (resolve) => {
@ -155,6 +183,21 @@ describe('orbit-db - Persistency', function() {
assert.equal(items[entryCount - 1].payload.value, 'hello99')
})
it('load, add one and save snapshot several times', async () => {
const amount = 8
for (let i = 0; i < amount; i ++) {
db = await orbitdb1.eventlog(address)
await db.loadFromSnapshot()
await db.add('hello' + (entryCount + i))
const items = db.iterator({ limit: -1 }).collect()
assert.equal(items.length, entryCount + i + 1)
assert.equal(items[0].payload.value, 'hello0')
assert.equal(items[items.length - 1].payload.value, 'hello' + (entryCount + i))
await db.saveSnapshot()
await db.close()
}
})
it('throws an error when trying to load a missing snapshot', async () => {
db = await orbitdb1.eventlog(address)
await db.drop()

13
test/replicate-automatically.test.js
View File

@ -41,11 +41,16 @@ describe('orbit-db - Automatic Replication', function() {
if(orbitdb2)
await orbitdb2.stop()
if (ipfs1)
await ipfs1.stop()
return new Promise((resolve) => {
setTimeout(async () => {
if (ipfs1)
await ipfs1.stop()
if (ipfs2)
await ipfs2.stop()
if (ipfs2)
await ipfs2.stop()
resolve()
}, 2000)
})
})
beforeEach(async () => {

476
test/replicate.test.js
View File

@ -10,47 +10,44 @@ const waitForPeers = require('./utils/wait-for-peers')
const dbPath1 = './orbitdb/tests/replication/1'
const dbPath2 = './orbitdb/tests/replication/2'
const ipfsPath1 = './orbitdb/tests/replication/1/ipfs'
const ipfsPath2 = './orbitdb/tests/replication/2/ipfs'
const ipfsPath1 = './orbitdb/tests/replication/ipfs/1'
const ipfsPath2 = './orbitdb/tests/replication/ipfs/2'
const MemStore = require('./utils/mem-store')
describe('orbit-db - Replication', function() {
this.timeout(config.timeout)
this.timeout(config.timeout * 2)
let ipfs1, ipfs2, orbitdb1, orbitdb2, db1, db2
before(async () => {
config.daemon1.repo = ipfsPath1
config.daemon2.repo = ipfsPath2
rmrf.sync(config.daemon1.repo)
rmrf.sync(config.daemon2.repo)
rmrf.sync(dbPath1)
rmrf.sync(dbPath2)
ipfs1 = await startIpfs(config.daemon1)
ipfs2 = await startIpfs(config.daemon2)
// Connect the peers manually to speed up test times
await ipfs2.swarm.connect(ipfs1._peerInfo.multiaddrs._multiaddrs[0].toString())
await ipfs1.swarm.connect(ipfs2._peerInfo.multiaddrs._multiaddrs[0].toString())
orbitdb1 = new OrbitDB(ipfs1, dbPath1)
orbitdb2 = new OrbitDB(ipfs2, dbPath2)
})
after(async () => {
if(orbitdb1)
await orbitdb1.stop()
if(orbitdb2)
await orbitdb2.stop()
if (ipfs1)
await ipfs1.stop()
if (ipfs2)
await ipfs2.stop()
})
describe('two peers', function() {
let timer
let options
beforeEach(async () => {
let options = {
clearInterval(timer)
config.daemon1.repo = ipfsPath1
config.daemon2.repo = ipfsPath2
rmrf.sync(config.daemon1.repo)
rmrf.sync(config.daemon2.repo)
rmrf.sync(dbPath1)
rmrf.sync(dbPath2)
ipfs1 = await startIpfs(config.daemon1)
ipfs2 = await startIpfs(config.daemon2)
// Use memory store for quicker tests
const memstore = new MemStore()
ipfs1.object.put = memstore.put.bind(memstore)
ipfs1.object.get = memstore.get.bind(memstore)
ipfs2.object.put = memstore.put.bind(memstore)
ipfs2.object.get = memstore.get.bind(memstore)
// Connect the peers manually to speed up test times
await ipfs2.swarm.connect(ipfs1._peerInfo.multiaddrs._multiaddrs[0].toString())
await ipfs1.swarm.connect(ipfs2._peerInfo.multiaddrs._multiaddrs[0].toString())
orbitdb1 = new OrbitDB(ipfs1, dbPath1)
orbitdb2 = new OrbitDB(ipfs2, dbPath2)
options = {
// Set write access for both clients
write: [
orbitdb1.key.getPublic('hex'),
@ -59,24 +56,52 @@ describe('orbit-db - Replication', function() {
}
options = Object.assign({}, options, { path: dbPath1 })
db1 = await orbitdb1.eventlog('replication tests', options)
db1 = await orbitdb1.eventlog('replication-tests', options)
// Set 'sync' flag on. It'll prevent creating a new local database and rather
// fetch the database from the network
options = Object.assign({}, options, { path: dbPath2, sync: true })
db2 = await orbitdb2.eventlog(db1.address.toString(), options)
// options = Object.assign({}, options, { path: dbPath2, sync: true })
// db2 = await orbitdb2.eventlog(db1.address.toString(), options)
assert.equal(db1.address.toString(), db2.address.toString())
// assert.equal(db1.address.toString(), db2.address.toString())
await waitForPeers(ipfs1, [orbitdb2.id], db1.address.toString())
await waitForPeers(ipfs2, [orbitdb1.id], db1.address.toString())
// await waitForPeers(ipfs1, [orbitdb2.id], db1.address.toString())
// await waitForPeers(ipfs2, [orbitdb1.id], db1.address.toString())
})
afterEach(async () => {
await db1.drop()
await db2.drop()
clearInterval(timer)
options = {}
if (db1)
await db1.drop()
if (db2)
await db2.drop()
if(orbitdb1)
await orbitdb1.stop()
if(orbitdb2)
await orbitdb2.stop()
return new Promise((resolve) => {
setTimeout(async () => {
if (ipfs1)
await ipfs1.stop()
if (ipfs2)
await ipfs2.stop()
resolve()
}, 2000)
})
})
it('replicates database of 1 entry', async () => {
options = Object.assign({}, options, { path: dbPath2, sync: true })
db2 = await orbitdb2.eventlog(db1.address.toString(), options)
await waitForPeers(ipfs2, [orbitdb1.id], db1.address.toString())
await db1.add('hello')
return new Promise(resolve => {
setTimeout(() => {
@ -89,9 +114,12 @@ describe('orbit-db - Replication', function() {
})
it('replicates database of 100 entries', async () => {
options = Object.assign({}, options, { path: dbPath2, sync: true })
db2 = await orbitdb2.eventlog(db1.address.toString(), options)
await waitForPeers(ipfs2, [orbitdb1.id], db1.address.toString())
const entryCount = 100
const entryArr = []
let timer
for (let i = 0; i < entryCount; i ++)
entryArr.push(i)
@ -111,5 +139,367 @@ describe('orbit-db - Replication', function() {
}, 1000)
})
})
it('emits correct replication info', async () => {
options = Object.assign({}, options, { path: dbPath2, sync: true })
db2 = await orbitdb2.eventlog(db1.address.toString(), options)
await waitForPeers(ipfs2, [orbitdb1.id], db1.address.toString())
let finished = false
let eventCount = { 'replicate': 0, 'replicate.progress': 0, 'replicated': 0 }
let events = []
let expectedEventCount = 99
db2.events.on('replicate', (address, entry) => {
eventCount['replicate'] ++
events.push({
event: 'replicate',
count: eventCount['replicate'],
entry: entry,
})
})
db2.events.on('replicate.progress', (address, hash, entry, progress) => {
eventCount['replicate.progress'] ++
events.push({
event: 'replicate.progress',
count: eventCount['replicate.progress'],
entry: entry ,
replicationInfo: {
max: db2._replicationInfo.max,
progress: db2._replicationInfo.progress,
have: db2._replicationInfo.have,
},
})
})
db2.events.on('replicated', (address) => {
eventCount['replicated'] ++
events.push({
event: 'replicated',
count: eventCount['replicate'],
replicationInfo: {
max: db2._replicationInfo.max,
progress: db2._replicationInfo.progress,
have: db2._replicationInfo.have,
},
})
// Resolve with a little timeout to make sure we
// don't receive more than one event
setTimeout(() => {
finished = db2.iterator({ limit: -1 }).collect().length === expectedEventCount
}, 500)
})
return new Promise((resolve, reject) => {
timer = setInterval(() => {
if (finished) {
clearInterval(timer)
assert.equal(eventCount['replicate'], expectedEventCount)
assert.equal(eventCount['replicate.progress'], expectedEventCount)
const replicateEvents = events.filter(e => e.event === 'replicate')
assert.equal(replicateEvents.length, expectedEventCount)
assert.equal(replicateEvents[0].entry.payload.value.split(' ')[0], 'hello')
assert.equal(replicateEvents[0].entry.clock.time, 1)
const replicateProgressEvents = events.filter(e => e.event === 'replicate.progress')
assert.equal(replicateProgressEvents.length, expectedEventCount)
assert.equal(replicateProgressEvents[0].entry.payload.value.split(' ')[0], 'hello')
assert.equal(replicateProgressEvents[0].entry.clock.time, 1)
assert.equal(replicateProgressEvents[0].replicationInfo.max, 1)
assert.equal(replicateProgressEvents[0].replicationInfo.progress, 1)
const replicatedEvents = events.filter(e => e.event === 'replicated')
assert.equal(replicatedEvents[0].replicationInfo.max, 1)
assert.equal(replicatedEvents[0].replicationInfo.progress, 1)
resolve()
}
}, 100)
// Trigger replication
let adds = []
for (let i = 0; i < expectedEventCount; i ++) {
adds.push(i)
}
mapSeries(adds, i => db1.add('hello ' + i))
})
})
it('emits correct replication info on fresh replication', async () => {
return new Promise(async (resolve, reject) => {
let finished = false
let eventCount = { 'replicate': 0, 'replicate.progress': 0, 'replicated': 0 }
let events = []
let expectedEventCount = 512
// Close second instance
// await db2.close()
// await db2.drop()
// Trigger replication
let adds = []
for (let i = 0; i < expectedEventCount; i ++) {
adds.push(i)
}
const add = async (i) => {
process.stdout.write("\rWriting " + (i + 1) + " / " + expectedEventCount)
await db1.add('hello ' + i)
}
await mapSeries(adds, add)
console.log()
// Open second instance again
options = {
path: dbPath2,
overwrite: true,
sync: true,
// Set write access for both clients
write: [
orbitdb1.key.getPublic('hex'),
orbitdb2.key.getPublic('hex')
],
}
db2 = await orbitdb2.eventlog(db1.address.toString(), options)
let current = 0
let total = 0
db2.events.on('replicate', (address, entry) => {
eventCount['replicate'] ++
total = db2._replicationInfo.max
// console.log("[replicate] ", '#' + eventCount['replicate'] + ':', current, '/', total, '| Tasks (in/queued/running/out):', db2._loader.tasksRequested, '/', db2._loader.tasksQueued, '/', db2._loader.tasksRunning, '/', db2._loader.tasksFinished)
events.push({
event: 'replicate',
count: eventCount['replicate'],
entry: entry,
})
})
db2.events.on('replicate.progress', (address, hash, entry) => {
eventCount['replicate.progress'] ++
current = db2._replicationInfo.progress
// console.log("[progress] ", '#' + eventCount['replicate.progress'] + ':', current, '/', total, '| Tasks (in/queued/running/out):', db2._loader.tasksRequested, '/', db2._loader.tasksQueued, '/', db2._loader.tasksRunning, '/', db2._loader.tasksFinished)
// assert.equal(db2._replicationInfo.progress, eventCount['replicate.progress'])
events.push({
event: 'replicate.progress',
count: eventCount['replicate.progress'],
entry: entry ,
replicationInfo: {
max: db2._replicationInfo.max,
progress: db2._replicationInfo.progress,
have: db2._replicationInfo.have,
},
})
})
db2.events.on('replicated', (address, length) => {
eventCount['replicated'] += length
current = db2._replicationInfo.progress
// console.log("[replicated]", '#' + eventCount['replicated'] + ':', current, '/', total, '| Tasks (in/queued/running/out):', db2._loader.tasksRequested, '/', db2._loader.tasksQueued, '/', db2._loader.tasksRunning, '/', db2._loader.tasksFinished, "|", db2._loader._stats.a, db2._loader._stats.b, db2._loader._stats.c, db2._loader._stats.d)
assert.equal(current, eventCount['replicated'])
assert.equal(total, expectedEventCount)
// Test the replicator state
assert.equal(db2._loader.tasksRequested >= current, true)
assert.equal(db2._loader.tasksQueued <= db2.options.referenceCount, true)
assert.equal(db2.options.referenceCount, 64)
assert.equal(db2._loader.tasksRunning, 0)
assert.equal(db2._loader.tasksFinished, current)
events.push({
event: 'replicated',
count: eventCount['replicate'],
replicationInfo: {
max: db2._replicationInfo.max,
progress: db2._replicationInfo.progress,
have: db2._replicationInfo.have,
},
})
// Resolve with a little timeout to make sure we
// don't receive more than one event
setTimeout(() => {
//console.log(eventCount['replicate.progress'], expectedEventCount)
if (eventCount['replicate.progress'] === expectedEventCount)
finished = true
}, 500)
})
const st = new Date().getTime()
timer = setInterval(async () => {
if (finished) {
clearInterval(timer)
const et = new Date().getTime()
console.log("Duration:", et - st, "ms")
assert.equal(eventCount['replicate'], expectedEventCount)
assert.equal(eventCount['replicate.progress'], expectedEventCount)
const replicateEvents = events.filter(e => e.event === 'replicate')
assert.equal(replicateEvents.length, expectedEventCount)
assert.equal(replicateEvents[0].entry.payload.value.split(' ')[0], 'hello')
assert.equal(replicateEvents[0].entry.clock.time, expectedEventCount)
const replicateProgressEvents = events.filter(e => e.event === 'replicate.progress')
assert.equal(replicateProgressEvents.length, expectedEventCount)
assert.equal(replicateProgressEvents[0].entry.payload.value.split(' ')[0], 'hello')
assert.equal(replicateProgressEvents[0].entry.clock.time, expectedEventCount)
assert.equal(replicateProgressEvents[0].replicationInfo.max, expectedEventCount)
assert.equal(replicateProgressEvents[0].replicationInfo.progress, 1)
const replicatedEvents = events.filter(e => e.event === 'replicated')
assert.equal(replicatedEvents[0].replicationInfo.max, expectedEventCount)
assert.equal(replicatedEvents[replicatedEvents.length - 1].replicationInfo.progress, expectedEventCount)
resolve()
}
}, 100)
})
})
it('emits correct replication info in two-way replication', async () => {
return new Promise(async (resolve, reject) => {
let finished = false
let eventCount = { 'replicate': 0, 'replicate.progress': 0, 'replicated': 0 }
let events = []
let expectedEventCount = 100
// Trigger replication
let adds = []
for (let i = 0; i < expectedEventCount; i ++) {
adds.push(i)
}
const add = async (i) => {
// process.stdout.write("\rWriting " + (i + 1) + " / " + expectedEventCount)
await Promise.all([db1.add('hello-1-' + i), db2.add('hello-2-' + i)])
}
// Open second instance again
let options = {
path: dbPath2,
overwrite: true,
sync: true,
// Set write access for both clients
write: [
orbitdb1.key.getPublic('hex'),
orbitdb2.key.getPublic('hex')
],
}
db2 = await orbitdb2.eventlog(db1.address.toString(), options)
await waitForPeers(ipfs2, [orbitdb1.id], db1.address.toString())
let current = 0
let total = 0
db2.events.on('replicate', (address, entry) => {
eventCount['replicate'] ++
current = db2._replicationInfo.progress
total = db2._replicationInfo.max
// console.log("[replicate] ", '#' + eventCount['replicate'] + ':', current, '/', total, '| Tasks (in/queued/running/out):', db2._loader.tasksRequested, '/', db2._loader.tasksQueued, '/', db2._loader.tasksRunning, '/', db2._loader.tasksFinished)
events.push({
event: 'replicate',
count: eventCount['replicate'],
entry: entry,
})
})
let prevProgress = 0
db2.events.on('replicate.progress', (address, hash, entry) => {
eventCount['replicate.progress'] ++
current = db2._replicationInfo.progress
total = db2._replicationInfo.max
// console.log("[progress] ", '#' + eventCount['replicate.progress'] + ':', current, '/', total, '| Tasks (in/queued/running/out):', db2._loader.tasksRequested, '/', db2._loader.tasksQueued, '/', db2._loader.tasksRunning, '/', db2._loader.tasksFinished)
// assert.equal(current, total)
events.push({
event: 'replicate.progress',
count: eventCount['replicate.progress'],
entry: entry ,
replicationInfo: {
max: db2._replicationInfo.max,
progress: db2._replicationInfo.progress,
have: db2._replicationInfo.have,
},
})
})
db2.events.on('replicated', (address, length) => {
eventCount['replicated'] += length
current = db2._replicationInfo.progress
total = db2._replicationInfo.max
const values = db2.iterator({limit: -1}).collect()
// console.log(current, "/", total, "/", values.length)
//console.log("[replicated]", '#' + eventCount['replicated'] + ':', current, '/', total, '| Tasks (in/queued/running/out):', db2._loader.tasksRequested, '/', db2._loader.tasksQueued, '/', db2._loader.tasksRunning, '/', db2._loader.tasksFinished, "|", db2._loader._stats.a, db2._loader._stats.b, db2._loader._stats.c, db2._loader._stats.d)
assert.equal(current <= total, true)
events.push({
event: 'replicated',
count: eventCount['replicate'],
replicationInfo: {
max: db2._replicationInfo.max,
progress: db2._replicationInfo.progress,
have: db2._replicationInfo.have,
},
})
if (db2._replicationInfo.max >= expectedEventCount * 2
&& db2._replicationInfo.progress >= expectedEventCount * 2)
finished = true
})
const st = new Date().getTime()
await mapSeries(adds, add)
timer = setInterval(async () => {
if (finished) {
clearInterval(timer)
const et = new Date().getTime()
console.log("Duration:", et - st, "ms")
assert.equal(eventCount['replicate'], expectedEventCount)
assert.equal(eventCount['replicate.progress'], expectedEventCount)
assert.equal(eventCount['replicated'], expectedEventCount)
const replicateEvents = events.filter(e => e.event === 'replicate')
assert.equal(replicateEvents.length, expectedEventCount)
const replicateProgressEvents = events.filter(e => e.event === 'replicate.progress')
assert.equal(replicateProgressEvents.length, expectedEventCount)
assert.equal(replicateProgressEvents[replicateProgressEvents.length - 1].entry.clock.time, expectedEventCount)
assert.equal(replicateProgressEvents[replicateProgressEvents.length - 1].replicationInfo.max, expectedEventCount * 2)
assert.equal(replicateProgressEvents[replicateProgressEvents.length - 1].replicationInfo.progress, expectedEventCount * 2)
const replicatedEvents = events.filter(e => e.event === 'replicated')
assert.equal(replicatedEvents[replicatedEvents.length - 1].replicationInfo.progress, expectedEventCount * 2)
assert.equal(replicatedEvents[replicatedEvents.length - 1].replicationInfo.max, expectedEventCount * 2)
const values1 = db1.iterator({limit: -1}).collect()
const values2 = db2.iterator({limit: -1}).collect()
assert.deepEqual(values1, values2)
// Test the replicator state
assert.equal(db1._loader.tasksRequested, expectedEventCount)
assert.equal(db1._loader.tasksQueued, 0)
assert.equal(db1._loader.tasksRunning, 0)
assert.equal(db1._loader.tasksFinished, expectedEventCount)
assert.equal(db2._loader.tasksRequested, expectedEventCount)
assert.equal(db2._loader.tasksQueued, 0)
assert.equal(db2._loader.tasksRunning, 0)
assert.equal(db2._loader.tasksFinished, expectedEventCount)
resolve()
}
}, 100)
})
})
})
})

95
test/utils/mem-store.js Normal file
View File

@ -0,0 +1,95 @@
'use strict'
const multihashing = require('multihashing-async')
const mh = require('multihashes')
const defaultHashAlg = 'sha2-256'
// 'use strict'
// const ImmutableDB = require('./immutabledb-interface')
const defaultFormat = { format: 'dag-cbor', hashAlg: 'sha2-256' }
/* ImmutableDB using IPLD (through IPFS) */
class IPLDStore {
constructor (ipfs) {
// super()
this._ipfs = ipfs
}
async put (value) {
const cid = await this._ipfs.dag.put(value, defaultFormat)
return cid.toBaseEncodedString()
}
async get (key) {
const result = await this._ipfs.dag.get(key)
return result.value
}
}
const createMultihash = (data, hashAlg) => {
return new Promise((resolve, reject) => {
multihashing(data, hashAlg || defaultHashAlg, (err, multihash) => {
if (err)
return reject(err)
resolve(mh.toB58String(multihash))
})
})
}
// const LRU = require('lru')
// const ImmutableDB = require('./immutabledb-interface')
// const createMultihash = require('./create-multihash')
/* Memory store using an LRU cache */
class MemStore {
constructor () {
this._store = {}//new LRU(1000)
}
async put (value) {
const data = value//new Buffer(JSON.stringify(value))
const hash = await createMultihash(data)
// console.log(this._store)
// this._store.set(hash, data)
if (!this._store) this._store = {}
// console.log(this._store)
// console.log(hash, data)
this._store[hash] = data
// return hash
return {
toJSON: () => {
return {
data: value,
multihash: hash,
}
}
}
}
async get (key) {
// const data = this._store.get(key)
const data = this._store[key]
// if (data) {
// const value = JSON.parse(data)
// return value
// }
// return data
return {
toJSON: () => {
return {
data: this._store[key],
multihash: key,
}
}
}
}
}
module.exports = MemStore

17
test/write-permissions.test.js
View File

@ -97,6 +97,9 @@ describe('orbit-db - Write Permissions', function() {
assert.deepEqual(database.getTestValue(db1), database.expectedValue)
assert.deepEqual(database.getTestValue(db2), database.expectedValue)
await db1.close()
await db2.close()
})
})
})
@ -122,9 +125,11 @@ describe('orbit-db - Write Permissions', function() {
db1.sync(db2._oplog.heads)
return new Promise(resolve => {
setTimeout(() => {
setTimeout(async () => {
const value = database.getTestValue(db1)
assert.deepEqual(value, database.expectedValue)
await db1.close()
await db2.close()
resolve()
}, 300)
})
@ -150,9 +155,11 @@ describe('orbit-db - Write Permissions', function() {
db1.sync(db2._oplog.heads)
return new Promise(resolve => {
setTimeout(() => {
setTimeout(async () => {
const value = database.getTestValue(db1)
assert.deepEqual(value, database.expectedValue)
await db1.close()
await db2.close()
resolve()
}, 300)
})
@ -194,9 +201,11 @@ describe('orbit-db - Write Permissions', function() {
db1.sync(db2._oplog.heads)
return new Promise((resolve, reject) => {
setTimeout(() => {
setTimeout(async () => {
// Make sure nothing was added
assert.equal(database.query(db1).length, 0)
await db1.close()
await db2.close()
if (err) {
reject(err)
} else {
@ -219,7 +228,7 @@ describe('orbit-db - Write Permissions', function() {
let err
try {
const db1 = await database.create(orbitdb1, 'write error test 2', options)
options = Object.assign({}, options, { sync: true })
options = Object.assign({}, options, { sync: true })
const db2 = await database.create(orbitdb2, db1.address.toString(), options)
await database.tryInsert(db2)
} catch (e) {