From aaeb33efd900f74eff6d38d16af476ae6220a174 Mon Sep 17 00:00:00 2001 From: haad Date: Thu, 4 Feb 2016 15:49:13 +0700 Subject: [PATCH 01/30] Pubsub communication WIP --- src/HashCacheItem.js | 7 ++++++ src/OrbitClient.js | 17 +++++++++----- src/PubSub.js | 40 +++++++++++++++++++++++++++++++++ test/orbit-client-tests.js | 45 +++++++++++++++++++------------------- 4 files changed, 82 insertions(+), 27 deletions(-) create mode 100644 src/PubSub.js diff --git a/src/HashCacheItem.js b/src/HashCacheItem.js index cfd03f5..d4c63e1 100644 --- a/src/HashCacheItem.js +++ b/src/HashCacheItem.js @@ -15,7 +15,11 @@ class HashCacheItem { class EncryptedHashCacheItem extends HashCacheItem { constructor(operation, key, sequenceNumber, targetHash, metaInfo, next, publicKey, privateKey, salt) { + if(key) + key = Encryption.encrypt(key, privateKey, publicKey); + super(operation, key, sequenceNumber, targetHash, metaInfo, next); + try { this.pubkey = publicKey; this.target = Encryption.encrypt(targetHash, privateKey, publicKey); @@ -42,6 +46,9 @@ class EncryptedHashCacheItem extends HashCacheItem { data.target = targetDec; data.meta = JSON.parse(metaDec); + if(data.key) + data.key = Encryption.decrypt(data.key, privateKey, 'TODO: pubkey'); + const item = new HashCacheItem(data.op, data.key, data.seq, data.target, data.meta, next, publicKey, privateKey, salt); return item; } diff --git a/src/OrbitClient.js b/src/OrbitClient.js index a31f4f1..19e8da5 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -13,6 +13,7 @@ var ItemTypes = require('./ItemTypes'); var MetaInfo = require('./MetaInfo'); var Post = require('./Post'); var Aggregator = require('./Aggregator'); +var PubSub = require('./PubSub'); var pubkey = Keystore.getKeys().publicKey; var privkey = Keystore.getKeys().privateKey; @@ -82,7 +83,8 @@ class OrbitClient { if(lt || lte) { startFromHash = lte ? lte : lt; } else { - var channel = await (this.client.linkedList(channel, password).head()); + // var channel = await (this.client.linkedList(channel, password).head()); + var channel = PubSub.latest(channel); startFromHash = channel.head ? channel.head : null; } @@ -125,7 +127,8 @@ class OrbitClient { _createMessage(channel, password, operation, key, target) { // Get the current channel head and bump the sequence number let seq = 0; - const currentHead = await(this.client.linkedList(channel, password).head()) + // const currentHead = await(this.client.linkedList(channel, password).head()) + const currentHead = PubSub.latest(channel); if(currentHead.head) { const headItem = await (ipfsAPI.getObject(this.ipfs, currentHead.head)); seq = JSON.parse(headItem.Data)["seq"] + 1; @@ -170,12 +173,14 @@ class OrbitClient { _createOperation(channel, password, operation, key, value) { const message = this._createMessage(channel, password, operation, key, value); - await(this.client.linkedList(channel, password).add(message.Hash)); + // await(this.client.linkedList(channel, password).add(message.Hash)); + PubSub.publish(channel, message.Hash) return message.Hash; } _deleteChannel(channel, password) { - await(this.client.linkedList(channel, password).delete()); + // await(this.client.linkedList(channel, password).delete()); + PubSub.delete(channel); return true; } @@ -190,7 +195,9 @@ class OrbitClient { } _info(channel, password) { - return await(this.client.linkedList(channel, password).head()); + // return await(this.client.linkedList(channel, password).head()); + var l = PubSub.latest(channel); + return l; } _connect(host, username, password) { diff --git a/src/PubSub.js b/src/PubSub.js new file mode 100644 index 0000000..ae113ff --- /dev/null +++ b/src/PubSub.js @@ -0,0 +1,40 @@ +'use strict'; + +let messages = {}; + +class PubSub { + constructor() { + + } + + static latest(hash) { + return { head: messages[hash] && messages[hash].length > 0 ? messages[hash][messages[hash].length - 1] : null, modes: {} }; + } + + static publish(hash, message) { + if(!messages[hash]) messages[hash] = []; + messages[hash].push(message); + } + + static delete(hash) { + messages[hash] = []; + } + + onNewMessage(channel, message) { + /* + // From orbit-server: + var hash = req.params.hash; + var head = req.body.head; + if(!head) throw "Invalid request"; + var user = authorize(req, res); + var channel = await(Database.getChannel(hash)); + channel.authenticateRead(req.body.password); + var uid = await (ipfsAPI.putObject(ipfs, JSON.stringify(user.get()))); + channel.authenticateWrite(uid.Hash); + await(verifyMessage(head, channel)); + await(channel.updateHead(head)) + */ + } +} + +module.exports = PubSub; diff --git a/test/orbit-client-tests.js b/test/orbit-client-tests.js index b148890..8773e0c 100644 --- a/test/orbit-client-tests.js +++ b/test/orbit-client-tests.js @@ -99,27 +99,27 @@ describe('Orbit Client', () => { done(); })); - it('gets channel info when channel has modes set', async((done) => { - try { - orbit.channel(channel).delete(); - var mode = { - mode: "+r", - params: { - password: 'password' - } - }; - var res = orbit.channel(channel, '').setMode(mode) - var info = orbit.channel(channel, 'password').info(); - assert.notEqual(info, null); - assert.equal(info.head, null); - assert.equal(JSON.stringify(info.modes), JSON.stringify(res)); - orbit.channel(channel, 'password').delete(); - } catch(e) { - orbit.channel(channel, 'password').delete(); - assert.equal(e, null); - } - done(); - })); + // it('gets channel info when channel has modes set', async((done) => { + // try { + // orbit.channel(channel).delete(); + // var mode = { + // mode: "+r", + // params: { + // password: 'password' + // } + // }; + // var res = orbit.channel(channel, '').setMode(mode) + // var info = orbit.channel(channel, 'password').info(); + // assert.notEqual(info, null); + // assert.equal(info.head, null); + // assert.equal(JSON.stringify(info.modes), JSON.stringify(res)); + // orbit.channel(channel, 'password').delete(); + // } catch(e) { + // orbit.channel(channel, 'password').delete(); + // assert.equal(e, null); + // } + // done(); + // })); }); @@ -501,6 +501,7 @@ describe('Orbit Client', () => { }); +/* describe('Modes', function() { var password = 'hello'; @@ -574,5 +575,5 @@ describe('Orbit Client', () => { })); }); - +*/ }); From 8b905c2bdae20852e559f28581e05fcd4738ac2a Mon Sep 17 00:00:00 2001 From: haad Date: Tue, 9 Feb 2016 10:32:11 +0100 Subject: [PATCH 02/30] Add 'from' field to MetaInfo --- src/MetaInfo.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MetaInfo.js b/src/MetaInfo.js index f68f953..b460e7c 100644 --- a/src/MetaInfo.js +++ b/src/MetaInfo.js @@ -1,9 +1,10 @@ 'use strict'; class MetaInfo { - constructor(type, size, ts) { + constructor(type, size, from, ts) { this.type = type; this.size = size; + this.from = from; this.ts = ts; } } From bfb7dfc3113bb36b251c84db3b57a3ec0f901d3f Mon Sep 17 00:00:00 2001 From: haad Date: Tue, 9 Feb 2016 10:32:26 +0100 Subject: [PATCH 03/30] First version of pubsub communication --- examples/readMessages.js | 18 ++++++---- src/OrbitClient.js | 30 +++++++++++----- src/PubSub.js | 75 ++++++++++++++++++++++++++++------------ 3 files changed, 86 insertions(+), 37 deletions(-) diff --git a/examples/readMessages.js b/examples/readMessages.js index b427519..a789feb 100644 --- a/examples/readMessages.js +++ b/examples/readMessages.js @@ -41,8 +41,8 @@ let run = (async(() => { }); console.log(JSON.stringify(items, null, 2)); - console.log("--> remove", hash1); - orbit.channel(c1).remove({ key: hash1 }); + // console.log("--> remove", hash1); + // orbit.channel(c1).remove({ key: hash1 }); items = orbit.channel(c1).iterator({ limit: -1 }).collect(); items = items.map((e) => { @@ -50,15 +50,21 @@ let run = (async(() => { }); console.log(JSON.stringify(items, null, 2)); + setInterval(async(() => { + orbit.channel(c1).add("hello at " + new Date().getTime()); + }), 1234); +/* // You can also get the event based on its hash var value = orbit.channel(c1).get(hash2); console.log("key:", hash2, "value:", value); +*/ + // console.log("--> remove", hash2); + // orbit.channel(c1).remove({ key: hash2 }); - console.log("--> remove", hash2); - orbit.channel(c1).remove({ key: hash2 }); + // items = orbit.channel(c1).iterator({ limit: -1 }).collect(); + // console.log(JSON.stringify(items, null, 2)); - items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - console.log(JSON.stringify(items, null, 2)); + // process.exit(0); } catch(e) { console.error("error:", e); diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 19e8da5..d1c791e 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -27,6 +27,16 @@ class OrbitClient { channel(hash, password) { if(password === undefined) password = ''; + + this._pubsub.subscribe(hash, password, (channel, message) => { + const m = this._getMessages(hash, password, { gte: message }); + m.forEach((e) => { + const userData = await(ipfsAPI.getObject(this.ipfs, e.item.meta.from)) + const user = JSON.parse(userData.Data)["user"]; + console.log(`${user}>`, e.item.Payload, `(op: ${e.item.op}, ${e.item.key})`); + }); + }); + return { info: (options) => this._info(hash, password), delete: () => this._deleteChannel(hash, password), @@ -80,11 +90,13 @@ class OrbitClient { const key = options.key ? options.key : null; let startFromHash; - if(lt || lte) { + if(lte || lt) { startFromHash = lte ? lte : lt; + } else if (gte || gt) { + startFromHash = gte ? gte : gt; } else { // var channel = await (this.client.linkedList(channel, password).head()); - var channel = PubSub.latest(channel); + var channel = this._pubsub.latest(channel); startFromHash = channel.head ? channel.head : null; } @@ -128,7 +140,7 @@ class OrbitClient { // Get the current channel head and bump the sequence number let seq = 0; // const currentHead = await(this.client.linkedList(channel, password).head()) - const currentHead = PubSub.latest(channel); + const currentHead = this._pubsub.latest(channel); if(currentHead.head) { const headItem = await (ipfsAPI.getObject(this.ipfs, currentHead.head)); seq = JSON.parse(headItem.Data)["seq"] + 1; @@ -136,7 +148,7 @@ class OrbitClient { // Create meta info const size = -1; - const metaInfo = new MetaInfo(ItemTypes.Message, size, new Date().getTime()); + const metaInfo = new MetaInfo(ItemTypes.Message, size, this.user.id, new Date().getTime()); // Create the hash cache item const hcItem = new HashCacheItem(operation, key, seq, target, metaInfo, null, pubkey, privkey, password); @@ -174,13 +186,13 @@ class OrbitClient { _createOperation(channel, password, operation, key, value) { const message = this._createMessage(channel, password, operation, key, value); // await(this.client.linkedList(channel, password).add(message.Hash)); - PubSub.publish(channel, message.Hash) + this._pubsub.publish(channel, message.Hash) return message.Hash; } _deleteChannel(channel, password) { // await(this.client.linkedList(channel, password).delete()); - PubSub.delete(channel); + this._pubsub.delete(channel); return true; } @@ -196,12 +208,14 @@ class OrbitClient { _info(channel, password) { // return await(this.client.linkedList(channel, password).head()); - var l = PubSub.latest(channel); + var l = this._pubsub.latest(channel); return l; } _connect(host, username, password) { - this.client = await(HashCache.connect(host, username, password)); + this._pubsub = new PubSub(host, username, password); + // this.client = await(HashCache.connect(host, username, password)); + this.client = this._pubsub._client; this.user = this.client.info.user; this.network = { id: this.client.info.networkId, diff --git a/src/PubSub.js b/src/PubSub.js index ae113ff..37c97f3 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -1,39 +1,68 @@ 'use strict'; -let messages = {}; +var async = require('asyncawait/async'); +var await = require('asyncawait/await'); +var HashCache = require('./HashCacheClient'); class PubSub { - constructor() { + constructor(host, username, password) { + this._subscriptions = []; + this._messages = {}; + this._client = await(HashCache.connect(host, username, password)); + // Poll for the new head + setInterval(async(() => { + Object.keys(this._subscriptions).forEach(this._poll.bind(this)); + }), 500); } - static latest(hash) { - return { head: messages[hash] && messages[hash].length > 0 ? messages[hash][messages[hash].length - 1] : null, modes: {} }; + _poll(hash) { + const currentHead = this._subscriptions[hash].head; + const channel = await(this._client.linkedList(hash, this._subscriptions[hash].password).head()); + const newHead = channel.head; + if(currentHead !== newHead) { + // console.log("NEW HEAD!", newHead); + + this._subscriptions[hash].head = newHead; + + if(!this._messages[hash]) + this._messages[hash] = []; + + this._messages[hash].push(newHead); + + if(this._subscriptions[hash].callback) + this._subscriptions[hash].callback(hash, newHead); + } } - static publish(hash, message) { - if(!messages[hash]) messages[hash] = []; - messages[hash].push(message); + subscribe(channel, password, callback) { + if(!this._subscriptions[channel] || this._subscriptions[channel].password !== password) { + console.log("SUBSCRIBE:", channel); + this._subscriptions[channel] = { + channel: channel, + password: password, + head: null, + callback: callback + }; + } } - static delete(hash) { - messages[hash] = []; + unsubscribe(channel) { + delete this._subscriptions[channel]; + delete this._messages[channel]; } - onNewMessage(channel, message) { - /* - // From orbit-server: - var hash = req.params.hash; - var head = req.body.head; - if(!head) throw "Invalid request"; - var user = authorize(req, res); - var channel = await(Database.getChannel(hash)); - channel.authenticateRead(req.body.password); - var uid = await (ipfsAPI.putObject(ipfs, JSON.stringify(user.get()))); - channel.authenticateWrite(uid.Hash); - await(verifyMessage(head, channel)); - await(channel.updateHead(head)) - */ + publish(hash, message) { + if(!this._messages[hash]) this._messages[hash] = []; + await(this._client.linkedList(hash, this._subscriptions[hash].password).add(message)); + } + + latest(hash) { + return { head: this._messages[hash] && this._messages[hash].length > 0 ? this._messages[hash][this._messages[hash].length - 1] : null, modes: {} }; + } + + delete(hash) { + this._messages[hash] = []; } } From c39c8a9983af55b295815bd9c312bae42dca24df Mon Sep 17 00:00:00 2001 From: haad Date: Tue, 9 Feb 2016 17:01:39 +0100 Subject: [PATCH 04/30] Pubsub proto (WIP) --- examples/readMessages.js | 50 ++++++++++++++++-- examples/writeMessages.js | 45 +++++++++++++++- package.json | 1 + src/OrbitClient.js | 73 +++++++++++--------------- src/PubSub.js | 105 +++++++++++++++++++++---------------- test/orbit-client-tests.js | 13 ++--- 6 files changed, 188 insertions(+), 99 deletions(-) diff --git a/examples/readMessages.js b/examples/readMessages.js index a789feb..277f7ee 100644 --- a/examples/readMessages.js +++ b/examples/readMessages.js @@ -8,12 +8,22 @@ var host = 'localhost:3006'; var username = 'testrunner'; var password = ''; +var util = require('util'); +var exec = require('child_process').exec; + +function clear(cb){ + exec('clear', function(error, stdout, stderr){ + util.puts(stdout); + cb(); + }); +} + let run = (async(() => { try { // Connect var orbit = OrbitClient.connect(host, username, password); - var timer = new Timer(true); +/* var timer = new Timer(true); console.log("-------- KV store -------") var channel = 'keyspace1' @@ -49,10 +59,44 @@ let run = (async(() => { return { key: e.item.key, val: e.item.Payload }; }); console.log(JSON.stringify(items, null, 2)); +*/ + const id = 'a'; + const c1 = 'c1'; + const cc = orbit.channel(c1); + let i = 0; + let running = false; + let missCount = 0; setInterval(async(() => { - orbit.channel(c1).add("hello at " + new Date().getTime()); - }), 1234); + if(!running) { + let timer = new Timer(true); + running = true; + // orbit.channel(c1).add("hello at #" + i); + cc.add(id + i); + i ++; + console.log(`Insert took ${timer.stop(true)} ms`); + + let timer2 = new Timer(true); + var items = cc.iterator({ limit: 10 }).collect(); + console.log("Iterator took " + timer2.stop(true) + " ms"); + items = items.map((e) => { + return e.item; + }); + + var g = items.filter((e) => e.Payload.startsWith(id)) + var prev = -1; + g.reverse().forEach((e) => { + var a = parseInt(e.Payload.replace(id, '')); + if(prev > -1 && prev + 1 !== a) { + console.log("Missing message: " + id, prev + 1) + } + prev = a; + }) + console.log(JSON.stringify(items.map((e) => e.seq + " - " + e.Payload), null, 2)); + // console.log("\n\n"); + running = false; + } + }), 50); /* // You can also get the event based on its hash var value = orbit.channel(c1).get(hash2); diff --git a/examples/writeMessages.js b/examples/writeMessages.js index 60088be..69c4502 100644 --- a/examples/writeMessages.js +++ b/examples/writeMessages.js @@ -5,12 +5,12 @@ var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); var host = 'localhost:3006'; -var username = 'testrunner'; +var username = 'LambOfGod'; var password = ''; let run = (async(() => { try { - var channel = 'hello-world-test1' +/* var channel = 'hello-world-test1' // Connect var orbit = OrbitClient.connect(host, username, password); @@ -51,6 +51,47 @@ let run = (async(() => { // orbit.channel(channel, '').remove(items[items.length - 10].hash); // 11 // orbit.channel(channel, '').remove(items[items.length - 9].hash); // 10 // orbit.channel(channel, '').remove(items[items.length - 8].hash); // 9 +*/ + + var orbit = OrbitClient.connect(host, username, password); + const c1 = 'c1'; + const cc = orbit.channel(c1); + + let i = 0; + let running = false; + setInterval(async(() => { + try { + if(!running) { + let timer = new Timer(true); + running = true; + + cc.add("b" + i); + + let items = cc.iterator({ limit: 10 }).collect(); + + var g = items.filter((e) => e.item.Payload.startsWith('b')) + var prev = -1; + g.reverse().forEach((e) => { + var a = parseInt(e.item.Payload.replace('b', '')); + if(prev > -1 && prev + 1 !== a) { + console.log("MISSSS!!!", prev + 1) + } + prev = a; + }) + + items = items.map((e) => { + return e.item.seq + " - " + e.item.Payload; + }); + console.log(JSON.stringify(items, null, 2)); + console.log(`Query took ${timer.stop(true)} ms`); + running = false; + } + // console.log("\n\n"); + } catch(e) { + console.error(e); + } + i ++; + }), 100); } catch(e) { console.error("error:", e); diff --git a/package.json b/package.json index d3cae03..de9aad6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "bluebird": "^3.1.1", "bs58": "^3.0.0", "orbit-common": "^0.1.0", + "redis": "^2.4.2", "unirest": "^0.4.2" }, "devDependencies": { diff --git a/src/OrbitClient.js b/src/OrbitClient.js index d1c791e..c591dfc 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -18,6 +18,8 @@ var PubSub = require('./PubSub'); var pubkey = Keystore.getKeys().publicKey; var privkey = Keystore.getKeys().privateKey; +let vvv = {}; + class OrbitClient { constructor(ipfs) { this.ipfs = ipfs; @@ -28,14 +30,7 @@ class OrbitClient { channel(hash, password) { if(password === undefined) password = ''; - this._pubsub.subscribe(hash, password, (channel, message) => { - const m = this._getMessages(hash, password, { gte: message }); - m.forEach((e) => { - const userData = await(ipfsAPI.getObject(this.ipfs, e.item.meta.from)) - const user = JSON.parse(userData.Data)["user"]; - console.log(`${user}>`, e.item.Payload, `(op: ${e.item.op}, ${e.item.key})`); - }); - }); + this._pubsub.subscribe(hash, password); return { info: (options) => this._info(hash, password), @@ -92,11 +87,8 @@ class OrbitClient { let startFromHash; if(lte || lt) { startFromHash = lte ? lte : lt; - } else if (gte || gt) { - startFromHash = gte ? gte : gt; } else { - // var channel = await (this.client.linkedList(channel, password).head()); - var channel = this._pubsub.latest(channel); + var channel = this._info(channel, password); startFromHash = channel.head ? channel.head : null; } @@ -137,19 +129,13 @@ class OrbitClient { } _createMessage(channel, password, operation, key, target) { - // Get the current channel head and bump the sequence number - let seq = 0; - // const currentHead = await(this.client.linkedList(channel, password).head()) - const currentHead = this._pubsub.latest(channel); - if(currentHead.head) { - const headItem = await (ipfsAPI.getObject(this.ipfs, currentHead.head)); - seq = JSON.parse(headItem.Data)["seq"] + 1; - } - // Create meta info const size = -1; const metaInfo = new MetaInfo(ItemTypes.Message, size, this.user.id, new Date().getTime()); + // Get the current channel head and bump the sequence number + let seq = this._info(channel, password).seq + 1; + // Create the hash cache item const hcItem = new HashCacheItem(operation, key, seq, target, metaInfo, null, pubkey, privkey, password); @@ -159,7 +145,7 @@ class OrbitClient { // If this is not the first item in the channel, patch with the previous (ie. link as next) if(seq > 0) - newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, currentHead.head)); + newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, this._info(channel, password).head)); return newHead; } @@ -168,13 +154,13 @@ class OrbitClient { _add(channel, password, data) { const post = this._publish(data); const key = post.Hash; - this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash); + await(this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash)); return key; } _put(channel, password, key, data) { const post = this._publish(data); - return this._createOperation(channel, password, HashCacheOps.Put, key, post.Hash); + return await(this._createOperation(channel, password, HashCacheOps.Put, key, post.Hash)); } _remove(channel, password, options) { @@ -184,15 +170,18 @@ class OrbitClient { } _createOperation(channel, password, operation, key, value) { - const message = this._createMessage(channel, password, operation, key, value); - // await(this.client.linkedList(channel, password).add(message.Hash)); - this._pubsub.publish(channel, message.Hash) + let message, res = false; + while(!res) { + message = this._createMessage(channel, password, operation, key, value); + res = await(this._pubsub.publish(channel, message)); + if(!res) console.log("Retry <-->") + } + // console.log("Posted!") return message.Hash; } _deleteChannel(channel, password) { - // await(this.client.linkedList(channel, password).delete()); - this._pubsub.delete(channel); + this._pubsub.delete(channel, password); return true; } @@ -202,26 +191,26 @@ class OrbitClient { m.push(modes); else m = modes; - const res = await(this.client.linkedList(channel, password).setMode(m)); - return res.modes; + // const res = await(this.client.linkedList(channel, password).setMode(m)); + // return res.modes; + return { todo: 'TODO!' } } _info(channel, password) { - // return await(this.client.linkedList(channel, password).head()); - var l = this._pubsub.latest(channel); + var l = this._pubsub.latest(channel, password); return l; } _connect(host, username, password) { - this._pubsub = new PubSub(host, username, password); - // this.client = await(HashCache.connect(host, username, password)); - this.client = this._pubsub._client; - this.user = this.client.info.user; - this.network = { - id: this.client.info.networkId, - name: this.client.info.name, - config: this.client.info.config - }; + this._pubsub = new PubSub(this.ipfs, host, username, password); + // this.client = this._pubsub._client; + // this.user = this.client.info.user; + this.user = { id: 'hello' } + // this.network = { + // id: this.client.info.networkId, + // name: this.client.info.name, + // config: this.client.info.config + // }; } } diff --git a/src/PubSub.js b/src/PubSub.js index 37c97f3..6a8c450 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -1,69 +1,82 @@ 'use strict'; -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var HashCache = require('./HashCacheClient'); +var async = require('asyncawait/async'); +var await = require('asyncawait/await'); +var redis = require("redis"); +var Aggregator = require('./Aggregator'); class PubSub { - constructor(host, username, password) { - this._subscriptions = []; - this._messages = {}; - this._client = await(HashCache.connect(host, username, password)); + constructor(ipfs, host, username, password) { + this.ipfs = ipfs; + this._subscriptions = {}; + this.client1 = redis.createClient(); + this.client2 = redis.createClient(); + this.currentPost = null; - // Poll for the new head - setInterval(async(() => { - Object.keys(this._subscriptions).forEach(this._poll.bind(this)); - }), 500); + this.client1.on("message", async((hash, message) => { + const currentHead = this._subscriptions[hash] ? this._subscriptions[hash].head : null; + if(this._subscriptions[hash]) { + let item = Aggregator._fetchOne(this.ipfs, message, this._subscriptions[hash].password); + + if(item.seq > this._subscriptions[hash].seq) { + this._subscriptions[hash].seq = item.seq; + + if(currentHead !== message) + this._handleNewMessage(hash, message); + + if(this.currentPost) { + if(message === this.currentPost.hash) { + this.currentPost.callback(true); + this.currentPost = null; + } else { + this.currentPost.callback(false); + } + } + } + } + })); } - _poll(hash) { - const currentHead = this._subscriptions[hash].head; - const channel = await(this._client.linkedList(hash, this._subscriptions[hash].password).head()); - const newHead = channel.head; - if(currentHead !== newHead) { - // console.log("NEW HEAD!", newHead); - - this._subscriptions[hash].head = newHead; - - if(!this._messages[hash]) - this._messages[hash] = []; - - this._messages[hash].push(newHead); - - if(this._subscriptions[hash].callback) - this._subscriptions[hash].callback(hash, newHead); - } - } - - subscribe(channel, password, callback) { - if(!this._subscriptions[channel] || this._subscriptions[channel].password !== password) { - console.log("SUBSCRIBE:", channel); - this._subscriptions[channel] = { - channel: channel, + subscribe(hash, password, callback) { + if(!this._subscriptions[hash] || this._subscriptions[hash].password !== password) { + this._subscriptions[hash] = { + topic: hash, password: password, head: null, - callback: callback + callback: callback, + seq: -1 }; + this.client1.subscribe(hash); } } - unsubscribe(channel) { - delete this._subscriptions[channel]; - delete this._messages[channel]; + unsubscribe(hash) { + delete this._subscriptions[hash]; + this.client1.unsubscribe(); + this.client2.unsubscribe(); } - publish(hash, message) { - if(!this._messages[hash]) this._messages[hash] = []; - await(this._client.linkedList(hash, this._subscriptions[hash].password).add(message)); + publish(hash, message, callback) { + return new Promise((resolve, reject) => { + this.currentPost = { hash: message.Hash, callback: resolve }; + this.client2.publish(hash, message.Hash); + }); } - latest(hash) { - return { head: this._messages[hash] && this._messages[hash].length > 0 ? this._messages[hash][this._messages[hash].length - 1] : null, modes: {} }; + latest(hash, password) { + return { head: this._subscriptions[hash].head, modes: {}, seq: this._subscriptions[hash].seq }; } - delete(hash) { - this._messages[hash] = []; + delete(hash, password) { + delete this._subscriptions[hash]; } + + _handleNewMessage(hash, newHead) { + this._subscriptions[hash].head = newHead; + if(this._subscriptions[hash].callback) + this._subscriptions[hash].callback(hash, newHead); + } + } module.exports = PubSub; diff --git a/test/orbit-client-tests.js b/test/orbit-client-tests.js index 8773e0c..2b134cc 100644 --- a/test/orbit-client-tests.js +++ b/test/orbit-client-tests.js @@ -70,12 +70,12 @@ describe('Orbit Client', () => { describe('Connect', function() { it('connects to hash-cache-server', async((done) => { assert.notEqual(orbit, null); - assert.notEqual(orbit.client, null); - assert.equal(orbit.user.id, 'Qmf5A5RSTQmcfvigT3j29Fqh2fAHRANk5ooBYKdWsPtr8U'); - assert.equal(orbit.network.id, serverConfig.networkId); - assert.equal(orbit.network.name, serverConfig.networkName); - assert.notEqual(orbit.network.config.SupernodeRouting, null); - assert.notEqual(orbit.network.config.Bootstrap.length, 0); + // assert.notEqual(orbit.client, null); + // assert.equal(orbit.user.id, 'hello'); + // assert.equal(orbit.network.id, serverConfig.networkId); + // assert.equal(orbit.network.name, serverConfig.networkName); + // assert.notEqual(orbit.network.config.SupernodeRouting, null); + // assert.notEqual(orbit.network.config.Bootstrap.length, 0); done(); })); }); @@ -389,6 +389,7 @@ describe('Orbit Client', () => { var iter = orbit.channel(channel, '').iterator({ gte: gte, limit: -1 }); var messages = iter.collect().map((e) => e.hash); + // console.log(messages, all) assert.equal(messages.length, 2); assert.equal(messages[0], all[0].hash); assert.equal(messages[1], all[1].hash); From 78481cd96ad3bb3024a1c0c448d36a6e4da1f701 Mon Sep 17 00:00:00 2001 From: haad Date: Wed, 10 Feb 2016 13:43:07 +0100 Subject: [PATCH 05/30] WOrking version (WIP) --- examples/pubsubBenchmark.js | 56 ++++++++++++++++++++++ examples/pubsubReader.js | 67 +++++++++++++++++++++++++++ examples/readMessages.js | 80 ++++++++++++++++---------------- examples/writeMessages.js | 92 +++++++++++++++++++++++++------------ src/OrbitClient.js | 22 ++++----- src/PubSub.js | 63 ++++++++++++------------- test/orbit-client-tests.js | 4 +- 7 files changed, 268 insertions(+), 116 deletions(-) create mode 100644 examples/pubsubBenchmark.js create mode 100644 examples/pubsubReader.js diff --git a/examples/pubsubBenchmark.js b/examples/pubsubBenchmark.js new file mode 100644 index 0000000..5d5a83d --- /dev/null +++ b/examples/pubsubBenchmark.js @@ -0,0 +1,56 @@ +'use strict'; + +var async = require('asyncawait/async'); +var OrbitClient = require('../src/OrbitClient'); +var Timer = require('./Timer'); + +// Redis host +var host = '188.166.73.174'; +var port = 6379; + +var username = 'testrunner'; +var password = ''; + +let run = (async(() => { + try { + // Connect + var orbit = OrbitClient.connect(host, port, username, password); + + const id = process.argv[2] ? process.argv[2] : 'a'; + const channelName = 'c1'; + const channel = orbit.channel(channelName); + + // Metrics + let totalQueries = 0; + let seconds = 0; + let queriesPerSecond = 0; + let lastTenSeconds = 0; + + // Metrics output + setInterval(() => { + seconds ++; + + if(seconds % 10 === 0) { + console.log(`--> Average of ${lastTenSeconds/10} q/s in the last 10 seconds`) + lastTenSeconds = 0 + } + + console.log(`${queriesPerSecond} queries per second, ${totalQueries} queries in ${seconds} seconds`) + queriesPerSecond = 0; + }, 1000); + + while(true) { + channel.add(id + totalQueries); + totalQueries ++; + lastTenSeconds ++; + queriesPerSecond ++; + } + + } catch(e) { + console.error("error:", e); + console.error(e.stack); + process.exit(1); + } +}))(); + +module.exports = run; diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js new file mode 100644 index 0000000..2588212 --- /dev/null +++ b/examples/pubsubReader.js @@ -0,0 +1,67 @@ +'use strict'; + +var async = require('asyncawait/async'); +var await = require('asyncawait/await'); +var OrbitClient = require('../src/OrbitClient'); +var Timer = require('./Timer'); + +var host = '188.166.73.174'; +var port = 6379; + +var username = 'LambOfGod'; +var password = ''; + +let run = (async(() => { + try { + var orbit = OrbitClient.connect(host, port, username, password); + const c1 = 'c1'; + const channel = orbit.channel(c1); + + let count = 1; + let id = 'Log: Query ' + let running = false; + + setInterval(async(() => { + if(!running) { + let timer = new Timer(true); + running = true; + + channel.add(id + count); + + let items = channel.iterator({ limit: 20 }).collect(); + + var g = items.filter((e) => e.item.Payload.startsWith(id)) + var prev = -1; + g.reverse().forEach((e) => { + var a = parseInt(e.item.Payload.replace(id, '')); + if(prev > -1 && prev + 1 !== a) { + console.log("MISSING VALUE!!!", prev + 1, items) + process.exit(1); + } + prev = a; + }) + + items = items.map((e) => { + return e.item.seq + " | " + e.item.Payload; + }); + + console.log("---------------------------------------------------") + console.log("Seq | Payload") + console.log("---------------------------------------------------") + console.log(items.join("\n")); + console.log("---------------------------------------------------") + console.log(`Query #${count} took ${timer.stop(true)} ms\n`); + + running = false; + count ++; + } + }), 1000); + + } catch(e) { + console.error("error:", e); + console.error(e.stack); + process.exit(1); + } +}))(); + +module.exports = run; diff --git a/examples/readMessages.js b/examples/readMessages.js index 277f7ee..7083470 100644 --- a/examples/readMessages.js +++ b/examples/readMessages.js @@ -4,24 +4,20 @@ var async = require('asyncawait/async'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -var host = 'localhost:3006'; +// Redis host +var host = 'localhost'; +var port = '6379' + var username = 'testrunner'; var password = ''; var util = require('util'); var exec = require('child_process').exec; -function clear(cb){ - exec('clear', function(error, stdout, stderr){ - util.puts(stdout); - cb(); - }); -} - let run = (async(() => { try { // Connect - var orbit = OrbitClient.connect(host, username, password); + var orbit = OrbitClient.connect(host, port, username, password); /* var timer = new Timer(true); @@ -60,43 +56,49 @@ let run = (async(() => { }); console.log(JSON.stringify(items, null, 2)); */ - const id = 'a'; + const id = process.argv[2] ? process.argv[2] : 'a'; const c1 = 'c1'; const cc = orbit.channel(c1); let i = 0; - let running = false; - let missCount = 0; - setInterval(async(() => { - if(!running) { - let timer = new Timer(true); - running = true; - // orbit.channel(c1).add("hello at #" + i); - cc.add(id + i); - i ++; - console.log(`Insert took ${timer.stop(true)} ms`); + let seconds = 0; + let round = 0; + let lastTen = 0; - let timer2 = new Timer(true); - var items = cc.iterator({ limit: 10 }).collect(); - console.log("Iterator took " + timer2.stop(true) + " ms"); - items = items.map((e) => { - return e.item; - }); + // Metrics + setInterval(() => { + seconds ++; - var g = items.filter((e) => e.Payload.startsWith(id)) - var prev = -1; - g.reverse().forEach((e) => { - var a = parseInt(e.Payload.replace(id, '')); - if(prev > -1 && prev + 1 !== a) { - console.log("Missing message: " + id, prev + 1) - } - prev = a; - }) - console.log(JSON.stringify(items.map((e) => e.seq + " - " + e.Payload), null, 2)); - // console.log("\n\n"); - running = false; + if(seconds % 10 === 0) { + console.log(`--> Average of ${lastTen/10} q/s in the last 10 seconds`) + lastTen = 0 } - }), 50); + + console.log(`${round} queries per second, ${i} queries in ${seconds} seconds`) + round = 0; + }, 1000); + + while(true) { + cc.add(id + i); + + i ++; + lastTen ++; + round ++; + + // let items = cc.iterator({ limit: 10 }).collect(); + // items = items.map((e) => e.item); + // let g = items.filter((e) => e.Payload.startsWith(id)) + // let prev = -1; + // g.reverse().forEach((e) => { + // const a = parseInt(e.Payload.replace(id, '')); + // if(prev > -1 && prev + 1 !== a) { + // console.log("!! Missing message: " + id, prev + 1) + // process.exit(1); + // } + // prev = a; + // }) + } + /* // You can also get the event based on its hash var value = orbit.channel(c1).get(hash2); diff --git a/examples/writeMessages.js b/examples/writeMessages.js index 69c4502..3877df0 100644 --- a/examples/writeMessages.js +++ b/examples/writeMessages.js @@ -1,10 +1,13 @@ 'use strict'; var async = require('asyncawait/async'); +var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -var host = 'localhost:3006'; +var host = 'localhost'; +var port = 6379; + var username = 'LambOfGod'; var password = ''; @@ -53,45 +56,74 @@ let run = (async(() => { // orbit.channel(channel, '').remove(items[items.length - 8].hash); // 9 */ - var orbit = OrbitClient.connect(host, username, password); + var orbit = OrbitClient.connect(host, port, username, password); const c1 = 'c1'; const cc = orbit.channel(c1); - let i = 0; + let i = 1; + let id = 'b' let running = false; setInterval(async(() => { - try { - if(!running) { - let timer = new Timer(true); - running = true; + if(!running) { + let timer = new Timer(true); + running = true; - cc.add("b" + i); + await(cc.add(id + i)); - let items = cc.iterator({ limit: 10 }).collect(); + let items = cc.iterator({ limit: 10 }).collect(); - var g = items.filter((e) => e.item.Payload.startsWith('b')) - var prev = -1; - g.reverse().forEach((e) => { - var a = parseInt(e.item.Payload.replace('b', '')); - if(prev > -1 && prev + 1 !== a) { - console.log("MISSSS!!!", prev + 1) - } - prev = a; - }) + var g = items.filter((e) => e.item.Payload.startsWith(id)) + var prev = -1; + g.reverse().forEach((e) => { + var a = parseInt(e.item.Payload.replace(id, '')); + if(prev > -1 && prev + 1 !== a) { + console.log("MISSSS!!!", prev + 1, items) + process.exit(1); + } + prev = a; + }) - items = items.map((e) => { - return e.item.seq + " - " + e.item.Payload; - }); - console.log(JSON.stringify(items, null, 2)); - console.log(`Query took ${timer.stop(true)} ms`); - running = false; - } - // console.log("\n\n"); - } catch(e) { - console.error(e); + items = items.map((e) => { + return e.item.seq + " - " + e.item.Payload; + }); + console.log(JSON.stringify(items, null, 2)); + console.log(`Query ${i} took ${timer.stop(true)} ms`); + running = false; + i ++; } - i ++; - }), 100); + // while(true) { + // } + }), 1000); + + // setInterval(async(() => { + // if(!running) { + // let timer = new Timer(true); + // running = true; + + // await(cc.add(id + i)); + + // let items = cc.iterator({ limit: 10 }).collect(); + + // var g = items.filter((e) => e.item.Payload.startsWith(id)) + // var prev = -1; + // g.reverse().forEach((e) => { + // var a = parseInt(e.item.Payload.replace(id, '')); + // if(prev > -1 && prev + 1 !== a) { + // console.log("MISSSS!!!", prev + 1, items) + // process.exit(1); + // } + // prev = a; + // }) + + // items = items.map((e) => { + // return e.item.seq + " - " + e.item.Payload; + // }); + // console.log(JSON.stringify(items, null, 2)); + // console.log(`Query took ${timer.stop(true)} ms`); + // running = false; + // i ++; + // } + // }), 36); } catch(e) { console.error("error:", e); diff --git a/src/OrbitClient.js b/src/OrbitClient.js index c591dfc..792c9c9 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -1,5 +1,6 @@ 'use strict'; +var a = require('async'); var async = require('asyncawait/async'); var await = require('asyncawait/await'); var Keystore = require('orbit-common/lib/Keystore'); @@ -135,6 +136,7 @@ class OrbitClient { // Get the current channel head and bump the sequence number let seq = this._info(channel, password).seq + 1; + let head = this._info(channel, password).head; // Create the hash cache item const hcItem = new HashCacheItem(operation, key, seq, target, metaInfo, null, pubkey, privkey, password); @@ -145,16 +147,16 @@ class OrbitClient { // If this is not the first item in the channel, patch with the previous (ie. link as next) if(seq > 0) - newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, this._info(channel, password).head)); + newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); - return newHead; + return { hash: newHead, seq: seq }; } /* DB Operations */ _add(channel, password, data) { const post = this._publish(data); const key = post.Hash; - await(this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash)); + await(this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash, data)); return key; } @@ -169,14 +171,12 @@ class OrbitClient { return this._createOperation(channel, password, HashCacheOps.Delete, key, target); } - _createOperation(channel, password, operation, key, value) { + _createOperation(channel, password, operation, key, value, data) { let message, res = false; while(!res) { message = this._createMessage(channel, password, operation, key, value); - res = await(this._pubsub.publish(channel, message)); - if(!res) console.log("Retry <-->") + res = await(this._pubsub.publish(channel, message.hash, message.seq)); } - // console.log("Posted!") return message.Hash; } @@ -201,8 +201,8 @@ class OrbitClient { return l; } - _connect(host, username, password) { - this._pubsub = new PubSub(this.ipfs, host, username, password); + _connect(host, port, username, password) { + this._pubsub = new PubSub(this.ipfs, host, port, username, password); // this.client = this._pubsub._client; // this.user = this.client.info.user; this.user = { id: 'hello' } @@ -215,14 +215,14 @@ class OrbitClient { } class OrbitClientFactory { - static connect(host, username, password, ipfs) { + static connect(host, port, username, password, ipfs) { if(!ipfs) { let ipfsd = await(ipfsDaemon()); ipfs = ipfsd.daemon; } const client = new OrbitClient(ipfs); - await(client._connect(host, username, password)) + await(client._connect(host, port, username, password)) return client; } } diff --git a/src/PubSub.js b/src/PubSub.js index 6a8c450..2acd3ea 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -6,35 +6,13 @@ var redis = require("redis"); var Aggregator = require('./Aggregator'); class PubSub { - constructor(ipfs, host, username, password) { + constructor(ipfs, host, port, username, password) { this.ipfs = ipfs; this._subscriptions = {}; - this.client1 = redis.createClient(); - this.client2 = redis.createClient(); - this.currentPost = null; - - this.client1.on("message", async((hash, message) => { - const currentHead = this._subscriptions[hash] ? this._subscriptions[hash].head : null; - if(this._subscriptions[hash]) { - let item = Aggregator._fetchOne(this.ipfs, message, this._subscriptions[hash].password); - - if(item.seq > this._subscriptions[hash].seq) { - this._subscriptions[hash].seq = item.seq; - - if(currentHead !== message) - this._handleNewMessage(hash, message); - - if(this.currentPost) { - if(message === this.currentPost.hash) { - this.currentPost.callback(true); - this.currentPost = null; - } else { - this.currentPost.callback(false); - } - } - } - } - })); + this.client1 = redis.createClient({ host: host, port: port }); + this.client2 = redis.createClient({ host: host, port: port }); + this.publishQueue = []; + this.client1.on("message", this._handleMessage.bind(this)); } subscribe(hash, password, callback) { @@ -56,10 +34,10 @@ class PubSub { this.client2.unsubscribe(); } - publish(hash, message, callback) { + publish(hash, message, seq, callback) { return new Promise((resolve, reject) => { - this.currentPost = { hash: message.Hash, callback: resolve }; - this.client2.publish(hash, message.Hash); + this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); + this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); }); } @@ -71,10 +49,27 @@ class PubSub { delete this._subscriptions[hash]; } - _handleNewMessage(hash, newHead) { - this._subscriptions[hash].head = newHead; - if(this._subscriptions[hash].callback) - this._subscriptions[hash].callback(hash, newHead); + _handleMessage(hash, event) { + if(this._subscriptions[hash]) { + var e = JSON.parse(event) + var newHead = e.hash; + var seq = e.seq; + var isNewer = seq > this._subscriptions[hash].seq; + + var item = this.publishQueue[this.publishQueue.length - 1]; + if(item && item.hash === newHead) { + item.callback(isNewer); + this.publishQueue.pop(); + } + + if(isNewer) + this._updateSubscription(hash, newHead, seq); + } + } + + _updateSubscription(hash, message, seq) { + this._subscriptions[hash].seq = seq; + this._subscriptions[hash].head = message; } } diff --git a/test/orbit-client-tests.js b/test/orbit-client-tests.js index 2b134cc..3b879d9 100644 --- a/test/orbit-client-tests.js +++ b/test/orbit-client-tests.js @@ -20,7 +20,7 @@ var serverConfig = { // Orbit const host = 'localhost'; -const port = 3006; +const port = 6379; const username = 'testrunner'; const password = ''; @@ -47,7 +47,7 @@ describe('Orbit Client', () => { before(async((done) => { var initialize = () => new Promise(async((resolve, reject) => { - orbit = OrbitClient.connect(`${host}:${port}`, username, password); + orbit = OrbitClient.connect(host, port, username, password); orbit.channel(channel, '').delete(); resolve(); })); From 079ea9853b6076714d9716deb7c44c3aceea107b Mon Sep 17 00:00:00 2001 From: haad Date: Wed, 10 Feb 2016 17:04:23 +0100 Subject: [PATCH 06/30] Testing --- .gitignore | 1 + examples/pubsubReader.js | 6 ++++-- src/OrbitClient.js | 7 +++++-- src/PubSub.js | 18 +++++++++++------- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 1be490e..ea32c98 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules/ debug.log WIP/ +.vagrant/ diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index 2588212..acd532e 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -26,9 +26,11 @@ let run = (async(() => { let timer = new Timer(true); running = true; - channel.add(id + count); + // channel.add(id + count); - let items = channel.iterator({ limit: 20 }).collect(); + console.log("Query..."); + let items = channel.iterator({ limit: 1 }).collect(); + console.log(`Found items ${items.length} items`); var g = items.filter((e) => e.item.Payload.startsWith(id)) var prev = -1; diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 792c9c9..c8e1f4e 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -1,6 +1,5 @@ 'use strict'; -var a = require('async'); var async = require('asyncawait/async'); var await = require('asyncawait/await'); var Keystore = require('orbit-common/lib/Keystore'); @@ -31,7 +30,10 @@ class OrbitClient { channel(hash, password) { if(password === undefined) password = ''; - this._pubsub.subscribe(hash, password); + this._pubsub.subscribe(hash, password, async((hash, message, seq) => { + // let m = Aggregator._fetchOne(this.ipfs, message, password); + // console.log(">", message); + })); return { info: (options) => this._info(hash, password), @@ -206,6 +208,7 @@ class OrbitClient { // this.client = this._pubsub._client; // this.user = this.client.info.user; this.user = { id: 'hello' } + console.log("Connected to redis") // this.network = { // id: this.client.info.networkId, // name: this.client.info.name, diff --git a/src/PubSub.js b/src/PubSub.js index 2acd3ea..51ceacc 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -11,8 +11,8 @@ class PubSub { this._subscriptions = {}; this.client1 = redis.createClient({ host: host, port: port }); this.client2 = redis.createClient({ host: host, port: port }); - this.publishQueue = []; this.client1.on("message", this._handleMessage.bind(this)); + this.publishQueue = []; } subscribe(hash, password, callback) { @@ -51,17 +51,20 @@ class PubSub { _handleMessage(hash, event) { if(this._subscriptions[hash]) { - var e = JSON.parse(event) - var newHead = e.hash; - var seq = e.seq; + var message = JSON.parse(event) + var newHead = message.hash; + var seq = message.seq; var isNewer = seq > this._subscriptions[hash].seq; + var item = this.publishQueue[this.publishQueue.length - 1]; - var item = this.publishQueue[this.publishQueue.length - 1]; - if(item && item.hash === newHead) { - item.callback(isNewer); + // console.log(".", newHead, item ? item.hash : '') + + if(item) { + item.callback(isNewer && newHead === item.hash); this.publishQueue.pop(); } + // console.log(isNewer, seq, this._subscriptions[hash].seq) if(isNewer) this._updateSubscription(hash, newHead, seq); } @@ -70,6 +73,7 @@ class PubSub { _updateSubscription(hash, message, seq) { this._subscriptions[hash].seq = seq; this._subscriptions[hash].head = message; + this._subscriptions[hash].callback(hash, message, seq); } } From a419b0d52736f04071ee3585243a56f09cb47467 Mon Sep 17 00:00:00 2001 From: haad Date: Wed, 10 Feb 2016 17:27:26 +0100 Subject: [PATCH 07/30] Testing --- examples/pubsubReader.js | 4 ++-- src/OrbitClient.js | 3 +++ src/PubSub.js | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index acd532e..9258420 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -26,10 +26,10 @@ let run = (async(() => { let timer = new Timer(true); running = true; - // channel.add(id + count); + channel.add(id + count); console.log("Query..."); - let items = channel.iterator({ limit: 1 }).collect(); + let items = channel.iterator({ limit: 3 }).collect(); console.log(`Found items ${items.length} items`); var g = items.filter((e) => e.item.Payload.startsWith(id)) diff --git a/src/OrbitClient.js b/src/OrbitClient.js index c8e1f4e..9e3e8fb 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -176,9 +176,12 @@ class OrbitClient { _createOperation(channel, password, operation, key, value, data) { let message, res = false; while(!res) { + // console.log("posting...") message = this._createMessage(channel, password, operation, key, value); res = await(this._pubsub.publish(channel, message.hash, message.seq)); + // if(!res) console.log("retry") } + // console.log("posted") return message.Hash; } diff --git a/src/PubSub.js b/src/PubSub.js index 51ceacc..b82f50b 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -36,8 +36,10 @@ class PubSub { publish(hash, message, seq, callback) { return new Promise((resolve, reject) => { - this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); - this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); + if(this.publishQueue.length === 0) + this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); + this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); + setTimeout(() => resolve(false), 1000) }); } @@ -57,7 +59,7 @@ class PubSub { var isNewer = seq > this._subscriptions[hash].seq; var item = this.publishQueue[this.publishQueue.length - 1]; - // console.log(".", newHead, item ? item.hash : '') + // console.log(".", isNewer, newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq, message) if(item) { item.callback(isNewer && newHead === item.hash); From 1a99681a5bffc77b14b57a793353024969a0f77e Mon Sep 17 00:00:00 2001 From: haad Date: Wed, 10 Feb 2016 17:58:01 +0100 Subject: [PATCH 08/30] Testing --- src/PubSub.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/PubSub.js b/src/PubSub.js index b82f50b..201c1ce 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -36,10 +36,17 @@ class PubSub { publish(hash, message, seq, callback) { return new Promise((resolve, reject) => { - if(this.publishQueue.length === 0) + if(this.publishQueue.length === 0) { this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); - setTimeout(() => resolve(false), 1000) + } else { + console.log("too early") + // resolve(false); + } + setTimeout(() => { + this.publishQueue.pop(); + resolve(false); + }, 2000) }); } @@ -59,14 +66,14 @@ class PubSub { var isNewer = seq > this._subscriptions[hash].seq; var item = this.publishQueue[this.publishQueue.length - 1]; - // console.log(".", isNewer, newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq, message) - - if(item) { - item.callback(isNewer && newHead === item.hash); - this.publishQueue.pop(); - } + // console.log(".", isNewer, newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq) // console.log(isNewer, seq, this._subscriptions[hash].seq) + if(item) { + this.publishQueue.pop(); + item.callback(isNewer && newHead === item.hash); + } + if(isNewer) this._updateSubscription(hash, newHead, seq); } From 03ef4fe5ffbb62e2b89410778218e8f977f7f7d8 Mon Sep 17 00:00:00 2001 From: haad Date: Thu, 11 Feb 2016 09:19:53 +0100 Subject: [PATCH 09/30] Change redis server IP --- examples/pubsubBenchmark.js | 2 +- examples/pubsubReader.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pubsubBenchmark.js b/examples/pubsubBenchmark.js index 5d5a83d..94e2273 100644 --- a/examples/pubsubBenchmark.js +++ b/examples/pubsubBenchmark.js @@ -5,7 +5,7 @@ var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); // Redis host -var host = '188.166.73.174'; +var host = '178.62.229.175'; var port = 6379; var username = 'testrunner'; diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index 9258420..5c3f035 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -5,7 +5,7 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -var host = '188.166.73.174'; +var host = '178.62.229.175'; var port = 6379; var username = 'LambOfGod'; From d60f8d66a23b47080e38cd9ca9973f0d0824cfd2 Mon Sep 17 00:00:00 2001 From: haad Date: Thu, 11 Feb 2016 09:20:34 +0100 Subject: [PATCH 10/30] Testing --- src/OrbitClient.js | 5 ++--- src/PubSub.js | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 9e3e8fb..a446c39 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -176,10 +176,10 @@ class OrbitClient { _createOperation(channel, password, operation, key, value, data) { let message, res = false; while(!res) { - // console.log("posting...") + // console.log("posting...") message = this._createMessage(channel, password, operation, key, value); res = await(this._pubsub.publish(channel, message.hash, message.seq)); - // if(!res) console.log("retry") + // if(!res) console.log("retry", message) } // console.log("posted") return message.Hash; @@ -211,7 +211,6 @@ class OrbitClient { // this.client = this._pubsub._client; // this.user = this.client.info.user; this.user = { id: 'hello' } - console.log("Connected to redis") // this.network = { // id: this.client.info.networkId, // name: this.client.info.name, diff --git a/src/PubSub.js b/src/PubSub.js index 201c1ce..fc9c5ef 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -40,13 +40,13 @@ class PubSub { this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); } else { - console.log("too early") + // console.log("too early") // resolve(false); } - setTimeout(() => { - this.publishQueue.pop(); - resolve(false); - }, 2000) + // setTimeout(() => { + // this.publishQueue.pop(); + // resolve(false); + // }, 200) }); } From 61e063a7ed4ab36736981abfc7983d7e221ac2cf Mon Sep 17 00:00:00 2001 From: haad Date: Thu, 11 Feb 2016 11:57:10 +0100 Subject: [PATCH 11/30] Timeouts, save head in redis --- examples/pubsubReader.js | 4 ++-- src/OrbitClient.js | 30 ++++++++++++++++++------------ src/PubSub.js | 36 +++++++++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index 5c3f035..88b6780 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -26,10 +26,10 @@ let run = (async(() => { let timer = new Timer(true); running = true; - channel.add(id + count); + // channel.add(id + count); console.log("Query..."); - let items = channel.iterator({ limit: 3 }).collect(); + let items = channel.iterator({ limit: 1 }).collect(); console.log(`Found items ${items.length} items`); var g = items.filter((e) => e.item.Payload.startsWith(id)) diff --git a/src/OrbitClient.js b/src/OrbitClient.js index a446c39..5f96dcd 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -30,10 +30,11 @@ class OrbitClient { channel(hash, password) { if(password === undefined) password = ''; - this._pubsub.subscribe(hash, password, async((hash, message, seq) => { + await(this._pubsub.subscribe(hash, password)); + // this._pubsub.subscribe(hash, password, async((hash, message, seq) => { // let m = Aggregator._fetchOne(this.ipfs, message, password); // console.log(">", message); - })); + // })); return { info: (options) => this._info(hash, password), @@ -179,7 +180,7 @@ class OrbitClient { // console.log("posting...") message = this._createMessage(channel, password, operation, key, value); res = await(this._pubsub.publish(channel, message.hash, message.seq)); - // if(!res) console.log("retry", message) + if(!res) console.log("retry", message) } // console.log("posted") return message.Hash; @@ -207,15 +208,20 @@ class OrbitClient { } _connect(host, port, username, password) { - this._pubsub = new PubSub(this.ipfs, host, port, username, password); - // this.client = this._pubsub._client; - // this.user = this.client.info.user; - this.user = { id: 'hello' } - // this.network = { - // id: this.client.info.networkId, - // name: this.client.info.name, - // config: this.client.info.config - // }; + return new Promise((resolve, reject) => { + this._pubsub = new PubSub(this.ipfs, host, port, username, password, resolve); + // this.client = this._pubsub._client; + // this.user = this.client.info.user; + this.user = { id: 'hello' } + // this.network = { + // id: this.client.info.networkId, + // name: this.client.info.name, + // config: this.client.info.config + // }; + // setTimeout(() => { + // resolve(); + // }, 1000); + }); } } diff --git a/src/PubSub.js b/src/PubSub.js index fc9c5ef..70df347 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -6,13 +6,23 @@ var redis = require("redis"); var Aggregator = require('./Aggregator'); class PubSub { - constructor(ipfs, host, port, username, password) { + constructor(ipfs, host, port, username, password, resolve) { this.ipfs = ipfs; this._subscriptions = {}; this.client1 = redis.createClient({ host: host, port: port }); this.client2 = redis.createClient({ host: host, port: port }); + this.client3 = redis.createClient({ host: host, port: port }); this.client1.on("message", this._handleMessage.bind(this)); this.publishQueue = []; + + this.client1.on('connect', function() { + console.log('redis connected'); + resolve(); + }); + + this.client1.on("subscribe", function (channel, count) { + console.log("subscribed to pubsub topic '" + channel + "' (" + count + " peers)"); + }); } subscribe(hash, password, callback) { @@ -24,8 +34,24 @@ class PubSub { callback: callback, seq: -1 }; + this.client3.get("orbit." + hash, (err, reply) => { + if(reply) { + let d = JSON.parse(reply); + this._subscriptions[hash].seq = d.seq + 1; + this._subscriptions[hash].head = d.head; + if(err) console.log(err); + console.log(`head of '${hash}' is`, this._subscriptions[hash].head, "seq:", this._subscriptions[hash].seq); + } + }); this.client1.subscribe(hash); } + + return new Promise((resolve, reject) => { + setTimeout(() => { + console.log("pubsub initialized") + resolve(); + }, 1000); + }); } unsubscribe(hash) { @@ -59,6 +85,7 @@ class PubSub { } _handleMessage(hash, event) { + // console.log(".") if(this._subscriptions[hash]) { var message = JSON.parse(event) var newHead = message.hash; @@ -66,7 +93,7 @@ class PubSub { var isNewer = seq > this._subscriptions[hash].seq; var item = this.publishQueue[this.publishQueue.length - 1]; - // console.log(".", isNewer, newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq) + // console.log(".", newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq, isNewer) // console.log(isNewer, seq, this._subscriptions[hash].seq) if(item) { @@ -80,9 +107,12 @@ class PubSub { } _updateSubscription(hash, message, seq) { + this.client3.set("orbit." + hash, JSON.stringify({ head: message, seq: seq })); this._subscriptions[hash].seq = seq; this._subscriptions[hash].head = message; - this._subscriptions[hash].callback(hash, message, seq); + + if(this._subscriptions[hash].callback) + this._subscriptions[hash].callback(hash, message, seq); } } From aa619a81c8b340c1b704f406f0816495407fd834 Mon Sep 17 00:00:00 2001 From: haad Date: Thu, 11 Feb 2016 14:10:23 +0100 Subject: [PATCH 12/30] Tweaks --- examples/pubsubBenchmark.js | 6 ++++-- src/OrbitClient.js | 16 ++++++++++++---- src/PubSub.js | 20 +++++++++++--------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/examples/pubsubBenchmark.js b/examples/pubsubBenchmark.js index 94e2273..34a7eb8 100644 --- a/examples/pubsubBenchmark.js +++ b/examples/pubsubBenchmark.js @@ -39,12 +39,14 @@ let run = (async(() => { queriesPerSecond = 0; }, 1000); - while(true) { + setInterval(async(() => { + // while(true) { channel.add(id + totalQueries); totalQueries ++; lastTenSeconds ++; queriesPerSecond ++; - } + // } + }), 100); } catch(e) { console.error("error:", e); diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 5f96dcd..6770636 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -30,7 +30,11 @@ class OrbitClient { channel(hash, password) { if(password === undefined) password = ''; - await(this._pubsub.subscribe(hash, password)); + await(this._pubsub.subscribe(hash, password, async((hash, message, seq) => { + let m = await(Aggregator._fetchOne(this.ipfs, message, password)); + // console.log(">", m.key, m.seq, m.Payload); + }))); + // await(this._pubsub.subscribe(hash, password)); // this._pubsub.subscribe(hash, password, async((hash, message, seq) => { // let m = Aggregator._fetchOne(this.ipfs, message, password); // console.log(">", message); @@ -149,8 +153,12 @@ class OrbitClient { let newHead = { Hash: data.Hash }; // If this is not the first item in the channel, patch with the previous (ie. link as next) - if(seq > 0) - newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); + try { + if(seq > 0) + newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); + } catch(e) { + console.error("!!!!", e) + } return { hash: newHead, seq: seq }; } @@ -180,7 +188,7 @@ class OrbitClient { // console.log("posting...") message = this._createMessage(channel, password, operation, key, value); res = await(this._pubsub.publish(channel, message.hash, message.seq)); - if(!res) console.log("retry", message) + if(!res) console.log("retry", message.hash, message.seq) } // console.log("posted") return message.Hash; diff --git a/src/PubSub.js b/src/PubSub.js index 70df347..efba9d3 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -37,7 +37,7 @@ class PubSub { this.client3.get("orbit." + hash, (err, reply) => { if(reply) { let d = JSON.parse(reply); - this._subscriptions[hash].seq = d.seq + 1; + this._subscriptions[hash].seq = d.seq; this._subscriptions[hash].head = d.head; if(err) console.log(err); console.log(`head of '${hash}' is`, this._subscriptions[hash].head, "seq:", this._subscriptions[hash].seq); @@ -62,17 +62,19 @@ class PubSub { publish(hash, message, seq, callback) { return new Promise((resolve, reject) => { - if(this.publishQueue.length === 0) { - this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); - this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); - } else { - // console.log("too early") - // resolve(false); - } // setTimeout(() => { + // console.log("timeout") // this.publishQueue.pop(); // resolve(false); - // }, 200) + // }, 2000) + + // if(this.publishQueue.length === 0) { + this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); + this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); + // } else { + // console.log("too early") + // resolve(false); + // } }); } From 085cf4775f6dd3c9d8950f63eca9748fa2986e1e Mon Sep 17 00:00:00 2001 From: haad Date: Thu, 11 Feb 2016 15:26:41 +0100 Subject: [PATCH 13/30] Tweaks --- examples/pubsubBenchmark.js | 6 ++---- src/OrbitClient.js | 6 +----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/examples/pubsubBenchmark.js b/examples/pubsubBenchmark.js index 34a7eb8..94e2273 100644 --- a/examples/pubsubBenchmark.js +++ b/examples/pubsubBenchmark.js @@ -39,14 +39,12 @@ let run = (async(() => { queriesPerSecond = 0; }, 1000); - setInterval(async(() => { - // while(true) { + while(true) { channel.add(id + totalQueries); totalQueries ++; lastTenSeconds ++; queriesPerSecond ++; - // } - }), 100); + } } catch(e) { console.error("error:", e); diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 6770636..233c119 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -30,11 +30,7 @@ class OrbitClient { channel(hash, password) { if(password === undefined) password = ''; - await(this._pubsub.subscribe(hash, password, async((hash, message, seq) => { - let m = await(Aggregator._fetchOne(this.ipfs, message, password)); - // console.log(">", m.key, m.seq, m.Payload); - }))); - // await(this._pubsub.subscribe(hash, password)); + await(this._pubsub.subscribe(hash, password)); // this._pubsub.subscribe(hash, password, async((hash, message, seq) => { // let m = Aggregator._fetchOne(this.ipfs, message, password); // console.log(">", message); From cbb149ffa18299c53f16208763af3da9de1c0090 Mon Sep 17 00:00:00 2001 From: haad Date: Fri, 12 Feb 2016 09:27:57 +0100 Subject: [PATCH 14/30] Update usage in README --- README.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 76da568..4db1a98 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ _See Usage below_ .get(key) // Retrieve value - .remove({ key: }) // Remove entry + .del({ key: }) // Remove entry .setMode(modes) // Set channel modes, can be an object or an array of objects @@ -65,17 +65,20 @@ async(() => { const orbit = OrbitClient.connect(host, username, password); const channelName = 'hello-world'; + const db = orbit.channel(channelName); /* Event Log */ - const hash = orbit.channel(channelName).add('hello'); // - orbit.channel(channelName).remove({ key: hash }); + const hash = db.add('hello'); // + + // Remove event + db.remove(hash); // Iterator options - const options = { limit: -1 }; // fetch all messages + const options = { limit: -1 }; // fetch all messages // Get events - const iter = orbit.channel(channelName).iterator(options); // Symbol.iterator - const next = iter.next(); // { value: , done: false|true} + const iter = db.iterator(options); // Symbol.iterator + const next = iter.next(); // { value: , done: false|true} // OR: // var all = iter.collect(); // returns all elements as an array @@ -85,20 +88,21 @@ async(() => { // console.log(i.hash, i.item); /* KV Store */ - orbit.channel(channelName).put("key1", "hello world"); - orbit.channel(channelName).get("key1"); // returns "hello world" - orbit.channel(channelName).remove("key1"); + db.put('key1', 'hello world'); + db.get('key1'); // returns "hello world" + db.remove('key1'); /* Modes */ const password = 'hello'; let channelModes; - channelModes = orbit.channel(channel).setMode({ mode: "+r", params: { password: password } }); // { modes: { r: { password: 'hello' } } } - channelModes = orbit.channel(channel, password).setMode({ mode: "+w", params: { ops: [orbit.user.id] } }); // { modes: { ... } } - channelModes = orbit.channel(channel, password).setMode({ mode: "-r" }); // { modes: { ... } } - channelModes = orbit.channel(channel, '').setMode({ mode: "-w" }); // { modes: {} } + channelModes = db.setMode({ mode: '+r', params: { password: password } }); // { modes: { r: { password: 'hello' } } } + const privateChannel = orbit.channel(channel, password); + channelModes = privateChannel.setMode({ mode: '+w', params: { ops: [orbit.user.id] } }); // { modes: { ... } } + channelModes = privateChannel.setMode({ mode: '-r' }); // { modes: { ... } } + channelModes = privateChannel.setMode({ mode: '-w' }); // { modes: {} } /* Delete channel */ - const result = orbit.channel(channelName, channelPwd).delete(); // true | false + const result = db.delete(); // true | false })(); ``` From 8b35764952cadb6785811ade9816fd4f72d12fbc Mon Sep 17 00:00:00 2001 From: haad Date: Fri, 12 Feb 2016 09:28:08 +0100 Subject: [PATCH 15/30] Testing head tracking --- examples/pubsubBenchmark.js | 8 ++-- examples/pubsubReader.js | 4 +- examples/readMessages.js | 4 +- examples/writeMessages.js | 18 ++++----- src/OrbitClient.js | 21 ++++------ src/PubSub.js | 76 +++++++++++++++++++------------------ 6 files changed, 66 insertions(+), 65 deletions(-) diff --git a/examples/pubsubBenchmark.js b/examples/pubsubBenchmark.js index 94e2273..01100d7 100644 --- a/examples/pubsubBenchmark.js +++ b/examples/pubsubBenchmark.js @@ -39,12 +39,14 @@ let run = (async(() => { queriesPerSecond = 0; }, 1000); - while(true) { - channel.add(id + totalQueries); + setInterval(async(() => { + // while(true) { + let g = channel.add(id + totalQueries); totalQueries ++; lastTenSeconds ++; queriesPerSecond ++; - } + // } + }), 1000) } catch(e) { console.error("error:", e); diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index 88b6780..cde793f 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -29,7 +29,7 @@ let run = (async(() => { // channel.add(id + count); console.log("Query..."); - let items = channel.iterator({ limit: 1 }).collect(); + let items = channel.iterator({ limit: 3 }).collect(); console.log(`Found items ${items.length} items`); var g = items.filter((e) => e.item.Payload.startsWith(id)) @@ -57,7 +57,7 @@ let run = (async(() => { running = false; count ++; } - }), 1000); + }), 500); } catch(e) { console.error("error:", e); diff --git a/examples/readMessages.js b/examples/readMessages.js index 7083470..aebc6dd 100644 --- a/examples/readMessages.js +++ b/examples/readMessages.js @@ -31,7 +31,7 @@ let run = (async(() => { orbit.channel(channel).put("key4", "this will be deleted"); var val2 = orbit.channel(channel).get("key4"); console.log("key4:", val2); - orbit.channel(channel).remove({ key: "key4" }); + orbit.channel(channel).del({ key: "key4" }); val2 = orbit.channel(channel).get("key4"); console.log("key4:", val2); @@ -48,7 +48,7 @@ let run = (async(() => { console.log(JSON.stringify(items, null, 2)); // console.log("--> remove", hash1); - // orbit.channel(c1).remove({ key: hash1 }); + // orbit.channel(c1).del({ key: hash1 }); items = orbit.channel(c1).iterator({ limit: -1 }).collect(); items = items.map((e) => { diff --git a/examples/writeMessages.js b/examples/writeMessages.js index 3877df0..9dd24ab 100644 --- a/examples/writeMessages.js +++ b/examples/writeMessages.js @@ -24,7 +24,7 @@ let run = (async(() => { // Add the first message and delete it immediately // orbit.channel(channel, '').put("hello world!"); // var e = orbit.channel(channel, '').iterator({ limit: -1 }).collect()[0].hash; - // orbit.channel(channel, '').remove(e); + // orbit.channel(channel, '').del(e); orbit.channel(channel, '').put("key two", "hello world!!!"); var messages = 10; @@ -38,7 +38,7 @@ let run = (async(() => { if(i === 4) { console.log("remove", head); - // orbit.channel(channel, '').remove(head); + // orbit.channel(channel, '').del(head); } i ++; @@ -47,13 +47,13 @@ let run = (async(() => { var items = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); // console.log(items); var e = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); - orbit.channel(channel, '').remove({ key: "key one" }); - // orbit.channel(channel, '').remove(items[2].hash); // 97 - // orbit.channel(channel, '').remove(items[3].hash); // 96 - // orbit.channel(channel, '').remove(items[66].hash); // 34 - // orbit.channel(channel, '').remove(items[items.length - 10].hash); // 11 - // orbit.channel(channel, '').remove(items[items.length - 9].hash); // 10 - // orbit.channel(channel, '').remove(items[items.length - 8].hash); // 9 + orbit.channel(channel, '').del({ key: "key one" }); + // orbit.channel(channel, '').del(items[2].hash); // 97 + // orbit.channel(channel, '').del(items[3].hash); // 96 + // orbit.channel(channel, '').del(items[66].hash); // 34 + // orbit.channel(channel, '').del(items[items.length - 10].hash); // 11 + // orbit.channel(channel, '').del(items[items.length - 9].hash); // 10 + // orbit.channel(channel, '').del(items[items.length - 8].hash); // 9 */ var orbit = OrbitClient.connect(host, port, username, password); diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 233c119..f84fa21 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -43,7 +43,7 @@ class OrbitClient { setMode: (mode) => this._setMode(hash, password, mode), add: (data) => this._add(hash, password, data), //TODO: tests - remove: (options) => this._remove(hash, password, options), + del: (options) => this._remove(hash, password, options), put: (key, data) => this._put(hash, password, key, data), get: (key, options) => { const items = this._iterator(hash, password, { key: key }).collect(); @@ -149,12 +149,8 @@ class OrbitClient { let newHead = { Hash: data.Hash }; // If this is not the first item in the channel, patch with the previous (ie. link as next) - try { - if(seq > 0) - newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); - } catch(e) { - console.error("!!!!", e) - } + if(seq > 0) + newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); return { hash: newHead, seq: seq }; } @@ -163,8 +159,7 @@ class OrbitClient { _add(channel, password, data) { const post = this._publish(data); const key = post.Hash; - await(this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash, data)); - return key; + return await(this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash, data)); } _put(channel, password, key, data) { @@ -181,13 +176,13 @@ class OrbitClient { _createOperation(channel, password, operation, key, value, data) { let message, res = false; while(!res) { - // console.log("posting...") + console.log("posting...") message = this._createMessage(channel, password, operation, key, value); res = await(this._pubsub.publish(channel, message.hash, message.seq)); if(!res) console.log("retry", message.hash, message.seq) } - // console.log("posted") - return message.Hash; + console.log("posted") + return message.hash.Hash; } _deleteChannel(channel, password) { @@ -207,7 +202,7 @@ class OrbitClient { } _info(channel, password) { - var l = this._pubsub.latest(channel, password); + var l = this._pubsub.latest(channel); return l; } diff --git a/src/PubSub.js b/src/PubSub.js index efba9d3..37993ba 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -25,7 +25,7 @@ class PubSub { }); } - subscribe(hash, password, callback) { + subscribe(hash, password, head, callback) { if(!this._subscriptions[hash] || this._subscriptions[hash].password !== password) { this._subscriptions[hash] = { topic: hash, @@ -34,16 +34,17 @@ class PubSub { callback: callback, seq: -1 }; - this.client3.get("orbit." + hash, (err, reply) => { - if(reply) { - let d = JSON.parse(reply); - this._subscriptions[hash].seq = d.seq; - this._subscriptions[hash].head = d.head; - if(err) console.log(err); - console.log(`head of '${hash}' is`, this._subscriptions[hash].head, "seq:", this._subscriptions[hash].seq); - } - }); + // this.client3.get("orbit." + hash, (err, reply) => { + // if(reply) { + // let d = JSON.parse(reply); + // this._subscriptions[hash].seq = d.seq; + // this._subscriptions[hash].head = d.head; + // if(err) console.log(err); + // console.log(`head of '${hash}' is`, this._subscriptions[hash].head, "seq:", this._subscriptions[hash].seq); + // } + // }); this.client1.subscribe(hash); + this.client2.publish(hash, JSON.stringify({ r: "HEAD" })); } return new Promise((resolve, reject) => { @@ -62,23 +63,16 @@ class PubSub { publish(hash, message, seq, callback) { return new Promise((resolve, reject) => { - // setTimeout(() => { - // console.log("timeout") - // this.publishQueue.pop(); - // resolve(false); - // }, 2000) - - // if(this.publishQueue.length === 0) { + if(this.publishQueue.length === 0) { this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); - // } else { - // console.log("too early") - // resolve(false); - // } + } else { + resolve(false); + } }); } - latest(hash, password) { + latest(hash) { return { head: this._subscriptions[hash].head, modes: {}, seq: this._subscriptions[hash].seq }; } @@ -87,29 +81,39 @@ class PubSub { } _handleMessage(hash, event) { - // console.log(".") if(this._subscriptions[hash]) { var message = JSON.parse(event) - var newHead = message.hash; - var seq = message.seq; - var isNewer = seq > this._subscriptions[hash].seq; - var item = this.publishQueue[this.publishQueue.length - 1]; + if(message.hash) { + var newHead = message.hash; + var seq = message.seq; + var isNewer = seq > this._subscriptions[hash].seq; + var item = this.publishQueue[this.publishQueue.length - 1]; - // console.log(".", newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq, isNewer) + // console.log(".", newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq, isNewer) - // console.log(isNewer, seq, this._subscriptions[hash].seq) - if(item) { - this.publishQueue.pop(); - item.callback(isNewer && newHead === item.hash); + if(item) { + this.publishQueue.pop(); + item.callback(isNewer && newHead === item.hash); + } + + if(isNewer) + this._updateSubscription(hash, newHead, seq); + } else if(message.r === 'HEAD') { + console.log("SEND HEAD!") + this.client2.publish(hash, JSON.stringify(this.latest(hash))); + } else { + console.log("GOT HEAD!", message) + var isNewer = message.seq > this._subscriptions[hash].seq; + if(isNewer) { + console.log("NEW HEAD!") + this._updateSubscription(hash, message.head, message.seq); + } } - - if(isNewer) - this._updateSubscription(hash, newHead, seq); } } _updateSubscription(hash, message, seq) { - this.client3.set("orbit." + hash, JSON.stringify({ head: message, seq: seq })); + // this.client3.set("orbit." + hash, JSON.stringify({ head: message, seq: seq })); this._subscriptions[hash].seq = seq; this._subscriptions[hash].head = message; From 0aa0e0afd37db49223644d2aa3048207989e9fa1 Mon Sep 17 00:00:00 2001 From: haad Date: Tue, 16 Feb 2016 09:20:53 +0100 Subject: [PATCH 16/30] Testing lists --- .gitignore | 3 + MOCK.json | 198 ++++++ examples/pubsubReader.js | 2 +- package.json | 1 + src/OrbitClient.js | 7 +- src/PubSub.js | 4 + test/list-perf-tests.js | 34 ++ test/list-tests.js | 458 ++++++++++++++ test/orbit-client-tests.js | 1160 ++++++++++++++++++------------------ test1.js | 345 +++++++++++ 10 files changed, 1630 insertions(+), 582 deletions(-) create mode 100644 MOCK.json create mode 100644 test/list-perf-tests.js create mode 100644 test/list-tests.js create mode 100644 test1.js diff --git a/.gitignore b/.gitignore index ea32c98..7153fbd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ node_modules/ debug.log WIP/ .vagrant/ +.idea/ +isolate*.log +dump.rdb \ No newline at end of file diff --git a/MOCK.json b/MOCK.json new file mode 100644 index 0000000..54922e3 --- /dev/null +++ b/MOCK.json @@ -0,0 +1,198 @@ +// Data set grouped and sorted +"A" : [ + { "id": "A", "seq": 0, "ver": 0, "prev": null}, + { "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, + { "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, + { "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, + { "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, + { "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} +], +"B" : [ + { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, + { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} +], +"C" : [ + { "id": "C", "seq": 0, "ver": 0, "prev": null}, + { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, + { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, + { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} +] + + A B C + | +0.0 + | +0.1 + | +0.2 0.0 + | | +0.3 0.1 + | | +0.4 0.2 + | \ / | + | 1.0 | + | | | + | 1.1 | + | / | +2.0 | + \ | + \ | + \ | + 3.0 + +// expected order A +{ "id": "A", "seq": 0, "ver": 0, "prev": null}, +{ "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, +{ "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, +{ "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, +{ "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, + { "id": "C", "seq": 0, "ver": 0, "prev": null}, + { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, + { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, + { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, + { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} +{ "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} + { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} + +"VersionClock": { + "seq": 0, + "ver": 0 +} + +"Item": { + "id": "", + "VersionClock": "", + "prev": [] +} + +"List": { + "items": [""] +} + +/* + list.add(data) { + this.ver ++; + const heads = _findHeads(); + const i = new Item(id, this.seq, this.ver, heads) + outgoing.push(data) + } +*/ + +/* + list.join(other) { + // increase seq on join, reset version + if(other.first.seq >= this.seq) + this.seq = other.first.seq + 1 + this.ver = 0 + + items = items.concat(outgoing.concat(other)) + items = items.sortBy("seq", "id", "ver") + outgoing = [] + } +*/ + +/* + nextHeads() { + referenced = [] + heads = all.groupBy("id").map((items) => items[items.length - 1]) + cleaned = heads.reverse().filter((e) => !isReferencedInChain(referenced, e)) + return cleaned; + } +*/ + +/* + isReferencedInChain(list, other) { + const res = other.map((o) => { + const ref = list.map((e) => !(e.id == o.id && e.seq == o.seq && e.ver == o.ver)) + if(!ref) + list.push(ref) + //return false + + //list.concat(list.filter((e) => !(e.id == o.id && e.seq == o.seq && e.ver == o.ver))) + if(o.prev) + ref = isReferencedInChain(list, o.prev) + + return ref + }) + return res.anyEqual(true) + } +*/ + + + A B C + 0.0 + | +0.0 0.1 + | | +0.1 0.2 + \ / | + 1.0 | + | | + 1.1 | + / | +2.0 | + \ | + \ | + \ | + 3.0 + +// Sequence, --> syncs to +listA.add("mango") // { "id": "A", "seq": 0, "ver": 0, "prev": null} +listA.add("banana") // { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +--> B + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} + +listC.add("apple") // { "id": "C", "seq": 0, "ver": 0, "prev": null} +listC.add("strawberry") // { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +listC.add("orange") // { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +--> A,B + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} + +listB.add("pineapple") // { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +listB.add("papaya") // { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} +--> A + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} + +listA.add("kiwi") // { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} +--> C + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} +// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} + +listC.add("blueberry") // { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} +--> A,B + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} +// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} +// { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index cde793f..ba7293d 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -30,7 +30,7 @@ let run = (async(() => { console.log("Query..."); let items = channel.iterator({ limit: 3 }).collect(); - console.log(`Found items ${items.length} items`); + console.log(`Found ${items.length} items`); var g = items.filter((e) => e.item.Payload.startsWith(id)) var prev = -1; diff --git a/package.json b/package.json index de9aad6..c303053 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "asyncawait": "^1.0.1", "bluebird": "^3.1.1", "bs58": "^3.0.0", + "lodash": "^4.3.0", "orbit-common": "^0.1.0", "redis": "^2.4.2", "unirest": "^0.4.2" diff --git a/src/OrbitClient.js b/src/OrbitClient.js index f84fa21..30f753a 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -144,14 +144,17 @@ class OrbitClient { // Create the hash cache item const hcItem = new HashCacheItem(operation, key, seq, target, metaInfo, null, pubkey, privkey, password); + console.log("1") // Save the item to ipfs const data = await (ipfsAPI.putObject(this.ipfs, JSON.stringify(hcItem))); + console.log("2", data.Hash, head) let newHead = { Hash: data.Hash }; // If this is not the first item in the channel, patch with the previous (ie. link as next) if(seq > 0) newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); + console.log("3") return { hash: newHead, seq: seq }; } @@ -170,17 +173,19 @@ class OrbitClient { _remove(channel, password, options) { const key = null; const target = options.key ? options.key : (options.hash ? options.hash : null); - return this._createOperation(channel, password, HashCacheOps.Delete, key, target); + return await(this._createOperation(channel, password, HashCacheOps.Delete, key, target)); } _createOperation(channel, password, operation, key, value, data) { let message, res = false; while(!res) { + this.posting = true; console.log("posting...") message = this._createMessage(channel, password, operation, key, value); res = await(this._pubsub.publish(channel, message.hash, message.seq)); if(!res) console.log("retry", message.hash, message.seq) } + this.posting = false; console.log("posted") return message.hash.Hash; } diff --git a/src/PubSub.js b/src/PubSub.js index 37993ba..01fc892 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -65,8 +65,11 @@ class PubSub { return new Promise((resolve, reject) => { if(this.publishQueue.length === 0) { this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); + console.log("...") this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); + console.log("published") } else { + console.log("queue full!") resolve(false); } }); @@ -106,6 +109,7 @@ class PubSub { var isNewer = message.seq > this._subscriptions[hash].seq; if(isNewer) { console.log("NEW HEAD!") + this.publishQueue.pop(); this._updateSubscription(hash, message.head, message.seq); } } diff --git a/test/list-perf-tests.js b/test/list-perf-tests.js new file mode 100644 index 0000000..50431a8 --- /dev/null +++ b/test/list-perf-tests.js @@ -0,0 +1,34 @@ +// 'use strict'; + +// var assert = require('assert'); +// var async = require('asyncawait/async'); +// var await = require('asyncawait/await'); +// var List = require('../test1').List; +// var Node = require('../test1').Node; +// var Timer = require('../examples/Timer'); + +// describe('List - Performance Measurement', function() { +// this.timeout(60000); + +// it('add', (done) => { +// let ms = 0; + +// for(let t = 1000; t <= 10000; t += 1000) { +// const list = new List('A'); +// let timer = new Timer(true); + +// for(let i = 0; i < t; i ++) { +// list.add("hello" + i); +// } + +// ms = timer.stop(true); +// console.log(` > ${t} took ${ms} ms`) + +// // assert.equal(ms < t, true); +// } + +// assert.equal(true, true); +// done(); +// }); + +// }); diff --git a/test/list-tests.js b/test/list-tests.js new file mode 100644 index 0000000..bdbe3b3 --- /dev/null +++ b/test/list-tests.js @@ -0,0 +1,458 @@ +'use strict'; + +var assert = require('assert'); +var async = require('asyncawait/async'); +var await = require('asyncawait/await'); +var List = require('../test1').List; +var Node = require('../test1').Node; + +describe('Node', () => { + describe('Constructor', () => { + it('initializes member variables', (done) => { + const node = new Node('A', 0, 0, 'hello', []); + assert.equal(node.id, 'A'); + assert.equal(node.seq, 0); + assert.equal(node.ver, 0); + assert.equal(node.data, 'hello'); + assert.equal(node.next instanceof Array, true); + done(); + }); + + it('initializes member variables without data and next', (done) => { + const node = new Node('A', 0, 0); + assert.equal(node.id, 'A'); + assert.equal(node.seq, 0); + assert.equal(node.ver, 0); + assert.equal(node.data, null); + assert.equal(node.next instanceof Array, true); + done(); + }); + }); + + describe('compactId', () => { + it('presents the node as a string with id, sequence and version', (done) => { + const node1 = new Node('A', 0, 0); + const node2 = new Node('B', 123, 456); + assert.equal(node1.compactId, 'A.0.0'); + assert.equal(node2.compactId, 'B.123.456'); + done(); + }); + }); + + describe('compact', () => { + it('presents the node as a compacted object', (done) => { + const node1 = new Node('A', 0, 0, 'hello'); + const node2 = new Node('B', 0, 0, 'hello', [node1]); + const compacted1 = node1.compact(); + const compacted2 = node2.compact(); + + assert.notEqual(compacted1, null); + assert.equal(compacted1.id, 'A'); + assert.equal(compacted1.seq, 0); + assert.equal(compacted1.ver, 0); + assert.equal(compacted1.data, 'hello'); + assert.equal(compacted1.next instanceof Array, true); + assert.equal(compacted1.next.length, 0); + + assert.equal(compacted2.id, 'B'); + assert.equal(compacted2.next.length, 1); + assert.equal(compacted2.next[0], 'A.0.0'); + done(); + }); + }); +}); + +describe('List', () => { + describe('Constructor', () => { + it('initializes member variables', (done) => { + const list = new List('A'); + assert.equal(list.id, 'A'); + assert.equal(list.seq, 0); + assert.equal(list.ver, 0); + assert.equal(list._items instanceof Array, true); + assert.equal(list._currentBatch instanceof Array, true); + assert.equal(list._items.length, 0); + assert.equal(list._currentBatch.length, 0); + done(); + }); + }); + + describe('items', () => { + it('returns items', (done) => { + const list = new List('A'); + let items = list.items; + assert.equal(list.items instanceof Array, true); + assert.equal(list.items.length, 0); + list.add("hello1") + list.add("hello2") + assert.equal(list.items instanceof Array, true); + assert.equal(list.items.length, 2); + assert.equal(list.items[0].data, 'hello1'); + assert.equal(list.items[1].data, 'hello2'); + done(); + }); + }); + + describe('toJson', () => { + it('presents the list in a compacted json form', (done) => { + const list = new List('A'); + list.add("hello1") + list.add("hello2") + list.add("hello3") + // list.add("hello4") + // list.add("hello5") + let compacted = list.toJson(); + // console.log(compacted) + assert.equal(compacted.id, 'A'); + assert.equal(compacted.seq, 0); + assert.equal(compacted.ver, 3); + assert.equal(compacted.items.length, 3); + assert.equal(compacted.items[0].id, 'A'); + assert.equal(compacted.items[0].seq, 0); + assert.equal(compacted.items[0].ver, 0); + assert.equal(compacted.items[0].next.length, 0); + assert.equal(compacted.items[compacted.items.length - 1].id, 'A'); + assert.equal(compacted.items[compacted.items.length - 1].seq, 0); + assert.equal(compacted.items[compacted.items.length - 1].ver, 2); + assert.equal(compacted.items[compacted.items.length - 1].next.length, 1); + assert.equal(compacted.items[compacted.items.length - 1].next[0], 'A.0.1'); + assert.equal(compacted.items[compacted.items.length - 2].id, 'A'); + assert.equal(compacted.items[compacted.items.length - 2].seq, 0); + assert.equal(compacted.items[compacted.items.length - 2].ver, 1); + assert.equal(compacted.items[compacted.items.length - 2].next.length, 1); + assert.equal(compacted.items[compacted.items.length - 2].next[0], 'A.0.0'); + done(); + }); + }); + + describe('add', () => { + it('adds an item to an empty list', (done) => { + const list = new List('A'); + list.add("hello1") + const item = list.items[0]; + assert.equal(list.id, 'A'); + assert.equal(list.seq, 0); + assert.equal(list.ver, 1); + assert.equal(list.items.length, 1); + assert.equal(list._currentBatch.length, 1); + assert.equal(list._items.length, 0); + assert.equal(item, list._currentBatch[0]); + assert.equal(item.id, 'A'); + assert.equal(item.seq, 0); + assert.equal(item.ver, 0); + assert.equal(item.data, 'hello1'); + done(); + }); + + it('adds 100 items to a list', (done) => { + const list = new List('A'); + + for(let i = 1; i < 101; i ++) { + list.add("hello" + i); + } + + assert.equal(list.id, 'A'); + assert.equal(list.seq, 0); + assert.equal(list.ver, 100); + assert.equal(list.items.length, 100); + assert.equal(list._currentBatch.length, 100); + assert.equal(list._items.length, 0); + + const item = list.items[list.items.length - 1]; + assert.equal(item, list._currentBatch[list._currentBatch.length - 1]); + assert.equal(item.id, 'A'); + assert.equal(item.seq, 0); + assert.equal(item.ver, 99); + assert.equal(item.data, 'hello100'); + assert.equal(item.next, 'A.0.98'); + + done(); + }); + }); + + describe('join', () => { + it('increases the sequence and resets the version if other list has the same or higher sequence', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + + list2.seq = 7; + list1.add("helloA1") + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 0); + assert.equal(list1.ver, 1); + + list2.add("helloB1") + list1.join(list2); + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 8); + assert.equal(list1.ver, 0); + done(); + }); + + it('increases the sequence by one if other list has lower sequence', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + list1.seq = 4; + list2.seq = 1; + list2.add("helloB1") + list1.join(list2); + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 5); + assert.equal(list1.ver, 0); + done(); + }); + + it('finds the next head when adding a new element', (done) => { + const list1 = new List('A'); + list1.add("helloA1") + list1.add("helloA2") + list1.add("helloA3") + + // console.log(list1.toString()) + assert.equal(list1._currentBatch.length, 3); + assert.equal(list1._currentBatch[2].next.length, 1); + assert.equal(list1._currentBatch[2].next[0], 'A.0.1'); + done(); + }); + + it('finds the next heads after joining a list with another', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + list1.add("helloA1") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + list1.add("helloA2") + + assert.equal(list1._currentBatch.length, 1); + assert.equal(list1._currentBatch[0].next.length, 2); + assert.equal(list1._currentBatch[0].next[0], 'A.0.0'); + assert.equal(list1._currentBatch[0].next[1], 'B.0.1'); + done(); + }); + + it('finds the next head after a two-way join', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + list1.add("helloA1") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + list1.add("helloA2") + list1.add("helloA3") + + assert.equal(list1._currentBatch.length, 2); + assert.equal(list1._currentBatch[1].next.length, 1); + assert.equal(list1._currentBatch[1].next[0], 'A.1.0'); + done(); + }); + + it('find sthe next heads after join two-way join', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + list1.add("helloA1") + list1.add("helloA2") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + list1.add("helloA3") + + list1.join(list2); + + list1.add("helloA4") + list1.add("helloA5") + + const lastItem = list1.items[list1.items.length - 1]; + // console.log(list1.toString()) + assert.equal(list1.items.length, 7); + assert.equal(lastItem.next.length, 1); + assert.equal(lastItem.next[0], 'A.2.0'); + done(); + }); + + it('joins list of one item with list of two items', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + list1.add("helloA1") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + // console.log(list1) + // console.log(list2) + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 1); + assert.equal(list1.ver, 0); + assert.equal(list1._currentBatch.length, 0); + assert.equal(list1._items.length, 3); + assert.equal(list1._items[list1._items.length - 1].id, 'B'); + assert.equal(list1._items[list1._items.length - 1].seq, 0); + assert.equal(list1._items[list1._items.length - 1].ver, 1); + assert.equal(list1._items[list1._items.length - 1].data, 'helloB2'); + done(); + }); + + it('joins lists two ways', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + list1.add("helloA1") + list1.add("helloA2") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + list2.join(list1); + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 1); + assert.equal(list1.ver, 0); + assert.equal(list1._currentBatch.length, 0); + assert.equal(list1._items.length, 4); + assert.equal(list1._items[list1._items.length - 1].id, 'B'); + assert.equal(list1._items[list1._items.length - 1].seq, 0); + assert.equal(list1._items[list1._items.length - 1].ver, 1); + assert.equal(list1._items[list1._items.length - 1].data, 'helloB2'); + + assert.equal(list2.id, 'B'); + assert.equal(list2.seq, 2); + assert.equal(list2.ver, 0); + assert.equal(list2._currentBatch.length, 0); + assert.equal(list2._items.length, 4); + assert.equal(list2._items[list2._items.length - 1].id, 'A'); + assert.equal(list2._items[list2._items.length - 1].seq, 0); + assert.equal(list2._items[list2._items.length - 1].ver, 1); + assert.equal(list2._items[list2._items.length - 1].data, 'helloA2'); + done(); + }); + + it('joins lists twice', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + + list1.add("helloA1") + list2.add("helloB1") + list2.join(list1); + + list1.add("helloA2") + list2.add("helloB2") + list2.join(list1); + + assert.equal(list2.id, 'B'); + assert.equal(list2.seq, 2); + assert.equal(list2.ver, 0); + assert.equal(list2._currentBatch.length, 0); + assert.equal(list2._items.length, 4); + assert.equal(list2._items[1].id, 'A'); + assert.equal(list2._items[1].seq, 0); + assert.equal(list2._items[1].ver, 0); + assert.equal(list2._items[1].data, 'helloA1'); + assert.equal(list2._items[list2._items.length - 1].id, 'A'); + assert.equal(list2._items[list2._items.length - 1].seq, 0); + assert.equal(list2._items[list2._items.length - 1].ver, 1); + assert.equal(list2._items[list2._items.length - 1].data, 'helloA2'); + done(); + }); + + it('joins 4 lists', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + const list3 = new List('C'); + const list4 = new List('D'); + + list1.add("helloA1") + list2.add("helloB1") + // list2.join(list1); + + list1.add("helloA2") + list2.add("helloB2") + // list2.join(list1); + + list3.add("helloC1") + list4.add("helloD1") + // list2.join(list1); + + list3.add("helloC2") + list4.add("helloD2") + list1.join(list2); + list1.join(list3); + list1.join(list4); + + assert.equal(list1.id, 'A'); + // assert.equal(list1.seq, 1); + // assert.equal(list1.ver, 1); + assert.equal(list1._currentBatch.length, 0); + assert.equal(list1._items.length, 8); + assert.equal(list1._items[1].id, 'A'); + assert.equal(list1._items[1].seq, 0); + assert.equal(list1._items[1].ver, 1); + assert.equal(list1._items[1].data, 'helloA2'); + assert.equal(list1._items[list1._items.length - 1].id, 'D'); + assert.equal(list1._items[list1._items.length - 1].seq, 0); + assert.equal(list1._items[list1._items.length - 1].ver, 1); + assert.equal(list1._items[list1._items.length - 1].data, 'helloD2'); + done(); + }); + + it('joins 4 lists sequentally', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + const list3 = new List('C'); + const list4 = new List('D'); + + list1.add("helloA1") + list1.join(list2); + list2.add("helloB1") + list2.join(list1); + + list1.add("helloA2") + list2.add("helloB2") + list1.join(list3); + list3.join(list1); + + list3.add("helloC1") + list4.add("helloD1") + + list3.add("helloC2") + list4.add("helloD2") + + list1.join(list3); + list1.join(list2); + list4.join(list2); + list4.join(list1); + list4.join(list3); + + list4.add("helloD3") + list4.add("helloD4") + + // console.log(list4.toString()); + assert.equal(list4.id, 'D'); + assert.equal(list4.seq, 7); + assert.equal(list4.ver, 2); + assert.equal(list4._currentBatch.length, 2); + assert.equal(list4._items.length, 8); + assert.equal(list4._items[1].id, 'D'); + assert.equal(list4._items[1].seq, 0); + assert.equal(list4._items[1].ver, 1); + assert.equal(list4._items[1].data, 'helloD2'); + assert.equal(list4._items[list1._items.length - 1].id, 'A'); + assert.equal(list4._items[list1._items.length - 1].seq, 1); + assert.equal(list4._items[list1._items.length - 1].ver, 0); + assert.equal(list4._items[list1._items.length - 1].data, 'helloA2'); + assert.equal(list4.items[list4.items.length - 1].id, 'D'); + assert.equal(list4.items[list4.items.length - 1].seq, 7); + assert.equal(list4.items[list4.items.length - 1].ver, 1); + assert.equal(list4.items[list4.items.length - 1].data, 'helloD4'); + done(); + }); + + }); + + it('solves a graph', (done) => { + done(); + }); + +}); diff --git a/test/orbit-client-tests.js b/test/orbit-client-tests.js index 3b879d9..ae3d8df 100644 --- a/test/orbit-client-tests.js +++ b/test/orbit-client-tests.js @@ -1,580 +1,580 @@ -'use strict'; - -var fs = require('fs'); -var path = require('path'); -var assert = require('assert'); -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); -var logger = require('orbit-common/lib/logger'); -var Server = require('orbit-server/src/server'); -var OrbitClient = require('../src/OrbitClient'); - -var serverConfig = { - networkId: "orbitdb-test", - networkName: "OrbitDB Test Network", - salt: "hellothisisdog", - userDataPath: "/tmp/orbitdb-tests", - verifyMessages: true -} - -// Orbit -const host = 'localhost'; -const port = 6379; -const username = 'testrunner'; -const password = ''; - -const startServer = async (() => { - return new Promise(async((resolve, reject) => { - logger.setLevel('ERROR'); - const ipfsd = await(ipfsDaemon()); - const server = Server(ipfsd.daemon, ipfsd.nodeInfo, serverConfig); - server.app.listen(port, () => { - resolve(server); - }).on('error', (err) => { - resolve(server); - }); - })); -}); - - -describe('Orbit Client', () => { - let server, orbit; - - let head = ''; - let items = []; - let channel = 'abcdefgh'; - - before(async((done) => { - var initialize = () => new Promise(async((resolve, reject) => { - orbit = OrbitClient.connect(host, port, username, password); - orbit.channel(channel, '').delete(); - resolve(); - })); - server = await(startServer()); - await(initialize()); - done(); - })); - - after(function(done) { - var deleteChannel = () => new Promise(async((resolve, reject) => { - if(orbit) orbit.channel(channel, '').delete(); - resolve(); - })); - server.shutdown(); - server = null; - deleteChannel().then(done); - }); - - /* TESTS */ - describe('Connect', function() { - it('connects to hash-cache-server', async((done) => { - assert.notEqual(orbit, null); - // assert.notEqual(orbit.client, null); - // assert.equal(orbit.user.id, 'hello'); - // assert.equal(orbit.network.id, serverConfig.networkId); - // assert.equal(orbit.network.name, serverConfig.networkName); - // assert.notEqual(orbit.network.config.SupernodeRouting, null); - // assert.notEqual(orbit.network.config.Bootstrap.length, 0); - done(); - })); - }); - - describe('Info', function() { - it('gets channel info on empty channel', async((done) => { - var info = orbit.channel(channel, '').info(); - assert.notEqual(info, null); - assert.equal(info.head, null); - assert.notEqual(info.modes, null); - done(); - })); - - it('gets channel info on an existing channel', async((done) => { - var msg = orbit.channel(channel, '').add('hello'); - var info = orbit.channel(channel, '').info(); - assert.notEqual(info, null); - assert.notEqual(info.head, null); - assert.notEqual(info.modes, null); - assert.equal(info.modes.r, null); - done(); - })); - - // it('gets channel info when channel has modes set', async((done) => { - // try { - // orbit.channel(channel).delete(); - // var mode = { - // mode: "+r", - // params: { - // password: 'password' - // } - // }; - // var res = orbit.channel(channel, '').setMode(mode) - // var info = orbit.channel(channel, 'password').info(); - // assert.notEqual(info, null); - // assert.equal(info.head, null); - // assert.equal(JSON.stringify(info.modes), JSON.stringify(res)); - // orbit.channel(channel, 'password').delete(); - // } catch(e) { - // orbit.channel(channel, 'password').delete(); - // assert.equal(e, null); - // } - // done(); - // })); - - }); - - describe('Delete', function() { - it('deletes a channel from the database', async((done) => { - var result = orbit.channel(channel, '').delete(); - assert.equal(result, true); - var iter = orbit.channel(channel, '').iterator(); - assert.equal(iter.next().value, null); - done(); - })); - - it('deletes a channel with a password', async((done) => { - done(); - })); - - it('doesn\'t delete a channel when password is wrong', async((done) => { - done(); - })); - - it('doesn\'t delete a channel when user is not an op', async((done) => { - done(); - })); - }); - - describe('Add events', function() { - it('adds an item to an empty channel', async((done) => { - try { - orbit.channel(channel, '').delete(); - const head = orbit.channel(channel, '').add('hello'); - assert.notEqual(head, null); - assert.equal(head.startsWith('Qm'), true); - assert.equal(head.length, 46); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - - it('adds a new item to a channel with one item', async((done) => { - try { - const head = orbit.channel(channel, '').iterator().collect()[0]; - const second = orbit.channel(channel, '').add('hello'); - assert.notEqual(second, null); - assert.notEqual(second, head); - assert.equal(second.startsWith('Qm'), true); - assert.equal(second.length, 46); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - - it('adds five items', async((done) => { - for(var i = 0; i < 5; i ++) { - try { - var s = orbit.channel(channel, '').add('hello'); - assert.notEqual(s, null); - assert.equal(s.startsWith('Qm'), true); - assert.equal(s.length, 46); - } catch(e) { - assert.equal(e, null); - } - } - done(); - })); - - it('adds an item that is > 256 bytes', async((done) => { - try { - var msg = new Buffer(512); - msg.fill('a') - var s = orbit.channel(channel, '').add(msg.toString()); - assert.notEqual(s, null); - assert.equal(s.startsWith('Qm'), true); - assert.equal(s.length, 46); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - }); - - - describe('Iterator', function() { - var items = []; - var itemCount = 5; - - before(function(done) { - var addMessages = () => new Promise(async((resolve, reject) => { - var result = orbit.channel(channel, '').delete(); - var iter = orbit.channel(channel, '').iterator(); - for(var i = 0; i < itemCount; i ++) { - var s = orbit.channel(channel, '').add('hello' + i); - items.push(s); - } - resolve(); - })); - addMessages().then(done); - }); - - describe('Defaults', function() { - it('returns an iterator', async((done) => { - var iter = orbit.channel(channel, '').iterator(); - var next = iter.next().value; - assert.notEqual(iter, null); - assert.notEqual(next, null); - assert.notEqual(next.item, null); - assert.notEqual(next.item.op, null); - assert.equal(next.item.seq, 4); - assert.notEqual(next.item.target, null); - assert.notEqual(next.item.next, null); - assert.notEqual(next.item.Payload, null); - assert.equal(next.item.Payload, 'hello4'); - done(); - })); - - it('implements Iterator interface', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: -1 }); - var messages = []; - - for(let i of iter) - messages.push(i.hash); - - assert.equal(messages.length, items.length); - done(); - })); - - it('returns 1 item as default', async((done) => { - var iter = orbit.channel(channel, '').iterator(); - var first = iter.next().value; - var second = iter.next().value; - assert.equal(first.item.key, items[items.length - 1]); - assert.equal(second, null); - assert.equal(first.item.Payload, 'hello4'); - done(); - })); - }); - - describe('Collect', function() { - it('returns all items', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: -1 }); - var messages = iter.collect(); - assert.equal(messages.length, items.length); - assert.equal(messages[messages.length - 1].item.Payload, 'hello0'); - assert.equal(messages[0].item.Payload, 'hello4'); - done(); - })); - - it('returns 1 item', async((done) => { - var iter = orbit.channel(channel, '').iterator(); - var messages = iter.collect(); - assert.equal(messages.length, 1); - done(); - })); - - it('returns 3 items', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: 3 }); - var messages = iter.collect(); - assert.equal(messages.length, 3); - done(); - })); - }); - - describe('Options: limit', function() { - it('returns 1 item when limit is 0', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: 0 }); - var first = iter.next().value; - var second = iter.next().value; - assert.equal(first.item.key, items[items.length - 1]); - assert.equal(second, null); - done(); - })); - - it('returns 1 item when limit is 1', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: 1 }); - var first = iter.next().value; - var second = iter.next().value; - assert.equal(first.item.key, items[items.length - 1]); - assert.equal(second, null); - done(); - })); - - it('returns 3 items', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: 3 }); - var first = iter.next().value; - var second = iter.next().value; - var third = iter.next().value; - var fourth = iter.next().value; - assert.equal(first.item.key, items[items.length - 1]); - assert.equal(second.item.key, items[items.length - 2]); - assert.equal(third.item.key, items[items.length - 3]); - assert.equal(fourth, null); - done(); - })); - - it('returns all items', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: -1 }); - var messages = iter.collect().map((e) => e.item.key); - - messages.reverse(); - assert.equal(messages.length, items.length); - assert.equal(messages[0], items[0]); - done(); - })); - - it('returns all items when limit is bigger than -1', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: -300 }); - var messages = iter.collect().map((e) => e.item.key); - - assert.equal(messages.length, items.length); - assert.equal(messages[0], items[items.length - 1]); - done(); - })); - - it('returns all items when limit is bigger than number of items', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: 300 }); - var messages = iter.collect().map((e) => e.item.key); - - assert.equal(messages.length, items.length); - assert.equal(messages[0], items[items.length - 1]); - done(); - })); - }); - - describe('Options: reverse', function() { - it('returns all items reversed', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: -1, reverse: true }); - var messages = iter.collect().map((e) => e.item.key); - - assert.equal(messages.length, items.length); - assert.equal(messages[0], items[0]); - done(); - })); - }); - - describe('Options: ranges', function() { - var all = []; - var head; - - before((done) => { - var fetchAll = () => new Promise(async((resolve, reject) => { - all = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); - head = all[0]; - resolve(); - })); - fetchAll().then(done); - }); - - describe('gt & gte', function() { - it('returns 0 items when gt is the head', async((done) => { - var messages = orbit.channel(channel, '').iterator({ gt: head.hash }).collect(); - assert.equal(messages.length, 0); - done(); - })); - - it('returns 1 item when gte is the head', async((done) => { - var iter2 = orbit.channel(channel, '').iterator({ gte: head.hash, limit: -1 }); - var messages = iter2.collect().map((e) => e.item.key); - - assert.equal(messages.length, 1); - assert.equal(messages[0], items[items.length -1]); - done(); - })); - - it('returns 2 item when gte is defined', async((done) => { - var gte = all[1].hash; - var iter = orbit.channel(channel, '').iterator({ gte: gte, limit: -1 }); - var messages = iter.collect().map((e) => e.hash); - - // console.log(messages, all) - assert.equal(messages.length, 2); - assert.equal(messages[0], all[0].hash); - assert.equal(messages[1], all[1].hash); - done(); - })); - - it('returns all items when gte is the root item', async((done) => { - var iter = orbit.channel(channel, '').iterator({ gte: all[all.length -1], limit: -1 }); - var messages = iter.collect().map((e) => e.item.key); - - assert.equal(messages.length, itemCount); - assert.equal(messages[0], items[items.length - 1]); - assert.equal(messages[messages.length - 1], items[0]); - done(); - })); - - it('returns items when gt is the root item', async((done) => { - var iter = orbit.channel(channel, '').iterator({ gt: all[all.length - 1], limit: -1 }); - var messages = iter.collect().map((e) => e.item.key); - - assert.equal(messages.length, itemCount - 1); - assert.equal(messages[0], items[items.length - 1]); - assert.equal(messages[3], items[1]); - done(); - })); - - it('returns items when gt is defined', async((done) => { - var iter = orbit.channel(channel, '').iterator({ limit: -1}); - var messages = iter.collect().map((e) => e.hash); - - var gt = messages[2]; - var iter2 = orbit.channel(channel, '').iterator({ gt: gt, limit: 100 }); - var messages2 = iter2.collect().map((e) => e.hash); - - assert.equal(messages2.length, 2); - assert.equal(messages2[0], messages[0]); - assert.equal(messages2[1], messages[1]); - done(); - })); - }); - - describe('lt & lte', function() { - it('returns one item when lt is the head', async((done) => { - var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash }); - var messages = iter2.collect().map((e) => e.hash); - - assert.equal(messages.length, 1); - assert.equal(messages[0], head.hash); - done(); - })); - - it('returns all items when lt is head and limit is -1', async((done) => { - var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash, limit: -1 }); - var messages = iter2.collect().map((e) => e.hash); - - assert.equal(messages.length, itemCount); - assert.equal(messages[0], head.hash); - assert.equal(messages[4], all[all.length - 1].hash); - done(); - })); - - it('returns 3 items when lt is head and limit is 3', async((done) => { - var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash, limit: 3 }); - var messages = iter2.collect().map((e) => e.hash); - - assert.equal(messages.length, 3); - assert.equal(messages[0], head.hash); - assert.equal(messages[2], all[2].hash); - done(); - })); - - it('returns null when lt is the root item', async((done) => { - var messages = orbit.channel(channel, '').iterator({ lt: all[all.length - 1].hash }).collect(); - assert.equal(messages.length, 0); - done(); - })); - - it('returns one item when lte is the root item', async((done) => { - var iter = orbit.channel(channel, '').iterator({ lte: all[all.length - 1].hash }); - var messages = iter.collect().map((e) => e.hash); - - assert.equal(messages.length, 1); - assert.equal(messages[0], all[all.length - 1].hash); - done(); - })); - - it('returns all items when lte is the head', async((done) => { - var iter2 = orbit.channel(channel, '').iterator({ lte: head.hash, limit: -1 }); - var messages = iter2.collect().map((e) => e.hash); - - assert.equal(messages.length, itemCount); - assert.equal(messages[0], all[0].hash); - assert.equal(messages[4], all[all.length - 1].hash); - done(); - })); - - it('returns 3 items when lte is the head', async((done) => { - var iter2 = orbit.channel(channel, '').iterator({ lte: head.hash, limit: 3 }); - var messages = iter2.collect().map((e) => e.hash); - - assert.equal(messages.length, 3); - assert.equal(messages[0], all[0].hash); - assert.equal(messages[1], all[1].hash); - assert.equal(messages[2], all[2].hash); - done(); - })); - }); - }); - - }); - - -/* - describe('Modes', function() { - var password = 'hello'; - - it('sets read mode', async((done) => { - try { - var mode = { - mode: "+r", - params: { - password: password - } - }; - var modes = orbit.channel(channel, '').setMode(mode) - assert.notEqual(modes.r, null); - assert.equal(modes.r.password, password); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - - it('can\'t read with wrong password', async((done) => { - try { - var modes = orbit.channel(channel, 'invalidpassword').iterator(); - assert.equal(true, false); - } catch(e) { - assert.equal(e, 'Unauthorized'); - } - done(); - })); - - it('sets write mode', async((done) => { - try { - var mode = { - mode: "+w", - params: { - ops: [orbit.user.id] - } - }; - var modes = orbit.channel(channel, password).setMode(mode); - assert.notEqual(modes.w, null); - assert.equal(modes.w.ops[0], orbit.user.id); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - - it('can\'t write when user not an op', async((done) => { - // TODO - done(); - })); - - it('removes write mode', async((done) => { - try { - var modes = orbit.channel(channel, password).setMode({ mode: "-w" }); - assert.equal(modes.w, null); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - - it('removes read mode', async((done) => { - try { - var modes = orbit.channel(channel, password).setMode({ mode: "-r" }); - assert.equal(modes.r, null); - } catch(e) { - assert.equal(e, null); - } - done(); - })); - - }); -*/ -}); +// 'use strict'; + +// var fs = require('fs'); +// var path = require('path'); +// var assert = require('assert'); +// var async = require('asyncawait/async'); +// var await = require('asyncawait/await'); +// var ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +// var logger = require('orbit-common/lib/logger'); +// var Server = require('orbit-server/src/server'); +// var OrbitClient = require('../src/OrbitClient'); + +// var serverConfig = { +// networkId: "orbitdb-test", +// networkName: "OrbitDB Test Network", +// salt: "hellothisisdog", +// userDataPath: "/tmp/orbitdb-tests", +// verifyMessages: true +// } + +// // Orbit +// const host = 'localhost'; +// const port = 6379; +// const username = 'testrunner'; +// const password = ''; + +// const startServer = async (() => { +// return new Promise(async((resolve, reject) => { +// logger.setLevel('ERROR'); +// const ipfsd = await(ipfsDaemon()); +// const server = Server(ipfsd.daemon, ipfsd.nodeInfo, serverConfig); +// server.app.listen(port, () => { +// resolve(server); +// }).on('error', (err) => { +// resolve(server); +// }); +// })); +// }); + + +// describe('Orbit Client', () => { +// let server, orbit; + +// let head = ''; +// let items = []; +// let channel = 'abcdefgh'; + +// before(async((done) => { +// var initialize = () => new Promise(async((resolve, reject) => { +// orbit = OrbitClient.connect(host, port, username, password); +// orbit.channel(channel, '').delete(); +// resolve(); +// })); +// server = await(startServer()); +// await(initialize()); +// done(); +// })); + +// after(function(done) { +// var deleteChannel = () => new Promise(async((resolve, reject) => { +// if(orbit) orbit.channel(channel, '').delete(); +// resolve(); +// })); +// server.shutdown(); +// server = null; +// deleteChannel().then(done); +// }); + +// /* TESTS */ +// describe('Connect', function() { +// it('connects to hash-cache-server', async((done) => { +// assert.notEqual(orbit, null); +// // assert.notEqual(orbit.client, null); +// // assert.equal(orbit.user.id, 'hello'); +// // assert.equal(orbit.network.id, serverConfig.networkId); +// // assert.equal(orbit.network.name, serverConfig.networkName); +// // assert.notEqual(orbit.network.config.SupernodeRouting, null); +// // assert.notEqual(orbit.network.config.Bootstrap.length, 0); +// done(); +// })); +// }); + +// describe('Info', function() { +// it('gets channel info on empty channel', async((done) => { +// var info = orbit.channel(channel, '').info(); +// assert.notEqual(info, null); +// assert.equal(info.head, null); +// assert.notEqual(info.modes, null); +// done(); +// })); + +// it('gets channel info on an existing channel', async((done) => { +// var msg = orbit.channel(channel, '').add('hello'); +// var info = orbit.channel(channel, '').info(); +// assert.notEqual(info, null); +// assert.notEqual(info.head, null); +// assert.notEqual(info.modes, null); +// assert.equal(info.modes.r, null); +// done(); +// })); + +// // it('gets channel info when channel has modes set', async((done) => { +// // try { +// // orbit.channel(channel).delete(); +// // var mode = { +// // mode: "+r", +// // params: { +// // password: 'password' +// // } +// // }; +// // var res = orbit.channel(channel, '').setMode(mode) +// // var info = orbit.channel(channel, 'password').info(); +// // assert.notEqual(info, null); +// // assert.equal(info.head, null); +// // assert.equal(JSON.stringify(info.modes), JSON.stringify(res)); +// // orbit.channel(channel, 'password').delete(); +// // } catch(e) { +// // orbit.channel(channel, 'password').delete(); +// // assert.equal(e, null); +// // } +// // done(); +// // })); + +// }); + +// describe('Delete', function() { +// it('deletes a channel from the database', async((done) => { +// var result = orbit.channel(channel, '').delete(); +// assert.equal(result, true); +// var iter = orbit.channel(channel, '').iterator(); +// assert.equal(iter.next().value, null); +// done(); +// })); + +// it('deletes a channel with a password', async((done) => { +// done(); +// })); + +// it('doesn\'t delete a channel when password is wrong', async((done) => { +// done(); +// })); + +// it('doesn\'t delete a channel when user is not an op', async((done) => { +// done(); +// })); +// }); + +// describe('Add events', function() { +// it('adds an item to an empty channel', async((done) => { +// try { +// orbit.channel(channel, '').delete(); +// const head = orbit.channel(channel, '').add('hello'); +// assert.notEqual(head, null); +// assert.equal(head.startsWith('Qm'), true); +// assert.equal(head.length, 46); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); + +// it('adds a new item to a channel with one item', async((done) => { +// try { +// const head = orbit.channel(channel, '').iterator().collect()[0]; +// const second = orbit.channel(channel, '').add('hello'); +// assert.notEqual(second, null); +// assert.notEqual(second, head); +// assert.equal(second.startsWith('Qm'), true); +// assert.equal(second.length, 46); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); + +// it('adds five items', async((done) => { +// for(var i = 0; i < 5; i ++) { +// try { +// var s = orbit.channel(channel, '').add('hello'); +// assert.notEqual(s, null); +// assert.equal(s.startsWith('Qm'), true); +// assert.equal(s.length, 46); +// } catch(e) { +// assert.equal(e, null); +// } +// } +// done(); +// })); + +// it('adds an item that is > 256 bytes', async((done) => { +// try { +// var msg = new Buffer(512); +// msg.fill('a') +// var s = orbit.channel(channel, '').add(msg.toString()); +// assert.notEqual(s, null); +// assert.equal(s.startsWith('Qm'), true); +// assert.equal(s.length, 46); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); +// }); + + +// describe('Iterator', function() { +// var items = []; +// var itemCount = 5; + +// before(function(done) { +// var addMessages = () => new Promise(async((resolve, reject) => { +// var result = orbit.channel(channel, '').delete(); +// var iter = orbit.channel(channel, '').iterator(); +// for(var i = 0; i < itemCount; i ++) { +// var s = orbit.channel(channel, '').add('hello' + i); +// items.push(s); +// } +// resolve(); +// })); +// addMessages().then(done); +// }); + +// describe('Defaults', function() { +// it('returns an iterator', async((done) => { +// var iter = orbit.channel(channel, '').iterator(); +// var next = iter.next().value; +// assert.notEqual(iter, null); +// assert.notEqual(next, null); +// assert.notEqual(next.item, null); +// assert.notEqual(next.item.op, null); +// assert.equal(next.item.seq, 4); +// assert.notEqual(next.item.target, null); +// assert.notEqual(next.item.next, null); +// assert.notEqual(next.item.Payload, null); +// assert.equal(next.item.Payload, 'hello4'); +// done(); +// })); + +// it('implements Iterator interface', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: -1 }); +// var messages = []; + +// for(let i of iter) +// messages.push(i.hash); + +// assert.equal(messages.length, items.length); +// done(); +// })); + +// it('returns 1 item as default', async((done) => { +// var iter = orbit.channel(channel, '').iterator(); +// var first = iter.next().value; +// var second = iter.next().value; +// assert.equal(first.item.key, items[items.length - 1]); +// assert.equal(second, null); +// assert.equal(first.item.Payload, 'hello4'); +// done(); +// })); +// }); + +// describe('Collect', function() { +// it('returns all items', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: -1 }); +// var messages = iter.collect(); +// assert.equal(messages.length, items.length); +// assert.equal(messages[messages.length - 1].item.Payload, 'hello0'); +// assert.equal(messages[0].item.Payload, 'hello4'); +// done(); +// })); + +// it('returns 1 item', async((done) => { +// var iter = orbit.channel(channel, '').iterator(); +// var messages = iter.collect(); +// assert.equal(messages.length, 1); +// done(); +// })); + +// it('returns 3 items', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: 3 }); +// var messages = iter.collect(); +// assert.equal(messages.length, 3); +// done(); +// })); +// }); + +// describe('Options: limit', function() { +// it('returns 1 item when limit is 0', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: 0 }); +// var first = iter.next().value; +// var second = iter.next().value; +// assert.equal(first.item.key, items[items.length - 1]); +// assert.equal(second, null); +// done(); +// })); + +// it('returns 1 item when limit is 1', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: 1 }); +// var first = iter.next().value; +// var second = iter.next().value; +// assert.equal(first.item.key, items[items.length - 1]); +// assert.equal(second, null); +// done(); +// })); + +// it('returns 3 items', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: 3 }); +// var first = iter.next().value; +// var second = iter.next().value; +// var third = iter.next().value; +// var fourth = iter.next().value; +// assert.equal(first.item.key, items[items.length - 1]); +// assert.equal(second.item.key, items[items.length - 2]); +// assert.equal(third.item.key, items[items.length - 3]); +// assert.equal(fourth, null); +// done(); +// })); + +// it('returns all items', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: -1 }); +// var messages = iter.collect().map((e) => e.item.key); + +// messages.reverse(); +// assert.equal(messages.length, items.length); +// assert.equal(messages[0], items[0]); +// done(); +// })); + +// it('returns all items when limit is bigger than -1', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: -300 }); +// var messages = iter.collect().map((e) => e.item.key); + +// assert.equal(messages.length, items.length); +// assert.equal(messages[0], items[items.length - 1]); +// done(); +// })); + +// it('returns all items when limit is bigger than number of items', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: 300 }); +// var messages = iter.collect().map((e) => e.item.key); + +// assert.equal(messages.length, items.length); +// assert.equal(messages[0], items[items.length - 1]); +// done(); +// })); +// }); + +// describe('Options: reverse', function() { +// it('returns all items reversed', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: -1, reverse: true }); +// var messages = iter.collect().map((e) => e.item.key); + +// assert.equal(messages.length, items.length); +// assert.equal(messages[0], items[0]); +// done(); +// })); +// }); + +// describe('Options: ranges', function() { +// var all = []; +// var head; + +// before((done) => { +// var fetchAll = () => new Promise(async((resolve, reject) => { +// all = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); +// head = all[0]; +// resolve(); +// })); +// fetchAll().then(done); +// }); + +// describe('gt & gte', function() { +// it('returns 0 items when gt is the head', async((done) => { +// var messages = orbit.channel(channel, '').iterator({ gt: head.hash }).collect(); +// assert.equal(messages.length, 0); +// done(); +// })); + +// it('returns 1 item when gte is the head', async((done) => { +// var iter2 = orbit.channel(channel, '').iterator({ gte: head.hash, limit: -1 }); +// var messages = iter2.collect().map((e) => e.item.key); + +// assert.equal(messages.length, 1); +// assert.equal(messages[0], items[items.length -1]); +// done(); +// })); + +// it('returns 2 item when gte is defined', async((done) => { +// var gte = all[1].hash; +// var iter = orbit.channel(channel, '').iterator({ gte: gte, limit: -1 }); +// var messages = iter.collect().map((e) => e.hash); + +// // console.log(messages, all) +// assert.equal(messages.length, 2); +// assert.equal(messages[0], all[0].hash); +// assert.equal(messages[1], all[1].hash); +// done(); +// })); + +// it('returns all items when gte is the root item', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ gte: all[all.length -1], limit: -1 }); +// var messages = iter.collect().map((e) => e.item.key); + +// assert.equal(messages.length, itemCount); +// assert.equal(messages[0], items[items.length - 1]); +// assert.equal(messages[messages.length - 1], items[0]); +// done(); +// })); + +// it('returns items when gt is the root item', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ gt: all[all.length - 1], limit: -1 }); +// var messages = iter.collect().map((e) => e.item.key); + +// assert.equal(messages.length, itemCount - 1); +// assert.equal(messages[0], items[items.length - 1]); +// assert.equal(messages[3], items[1]); +// done(); +// })); + +// it('returns items when gt is defined', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ limit: -1}); +// var messages = iter.collect().map((e) => e.hash); + +// var gt = messages[2]; +// var iter2 = orbit.channel(channel, '').iterator({ gt: gt, limit: 100 }); +// var messages2 = iter2.collect().map((e) => e.hash); + +// assert.equal(messages2.length, 2); +// assert.equal(messages2[0], messages[0]); +// assert.equal(messages2[1], messages[1]); +// done(); +// })); +// }); + +// describe('lt & lte', function() { +// it('returns one item when lt is the head', async((done) => { +// var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash }); +// var messages = iter2.collect().map((e) => e.hash); + +// assert.equal(messages.length, 1); +// assert.equal(messages[0], head.hash); +// done(); +// })); + +// it('returns all items when lt is head and limit is -1', async((done) => { +// var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash, limit: -1 }); +// var messages = iter2.collect().map((e) => e.hash); + +// assert.equal(messages.length, itemCount); +// assert.equal(messages[0], head.hash); +// assert.equal(messages[4], all[all.length - 1].hash); +// done(); +// })); + +// it('returns 3 items when lt is head and limit is 3', async((done) => { +// var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash, limit: 3 }); +// var messages = iter2.collect().map((e) => e.hash); + +// assert.equal(messages.length, 3); +// assert.equal(messages[0], head.hash); +// assert.equal(messages[2], all[2].hash); +// done(); +// })); + +// it('returns null when lt is the root item', async((done) => { +// var messages = orbit.channel(channel, '').iterator({ lt: all[all.length - 1].hash }).collect(); +// assert.equal(messages.length, 0); +// done(); +// })); + +// it('returns one item when lte is the root item', async((done) => { +// var iter = orbit.channel(channel, '').iterator({ lte: all[all.length - 1].hash }); +// var messages = iter.collect().map((e) => e.hash); + +// assert.equal(messages.length, 1); +// assert.equal(messages[0], all[all.length - 1].hash); +// done(); +// })); + +// it('returns all items when lte is the head', async((done) => { +// var iter2 = orbit.channel(channel, '').iterator({ lte: head.hash, limit: -1 }); +// var messages = iter2.collect().map((e) => e.hash); + +// assert.equal(messages.length, itemCount); +// assert.equal(messages[0], all[0].hash); +// assert.equal(messages[4], all[all.length - 1].hash); +// done(); +// })); + +// it('returns 3 items when lte is the head', async((done) => { +// var iter2 = orbit.channel(channel, '').iterator({ lte: head.hash, limit: 3 }); +// var messages = iter2.collect().map((e) => e.hash); + +// assert.equal(messages.length, 3); +// assert.equal(messages[0], all[0].hash); +// assert.equal(messages[1], all[1].hash); +// assert.equal(messages[2], all[2].hash); +// done(); +// })); +// }); +// }); + +// }); + + +// /* +// describe('Modes', function() { +// var password = 'hello'; + +// it('sets read mode', async((done) => { +// try { +// var mode = { +// mode: "+r", +// params: { +// password: password +// } +// }; +// var modes = orbit.channel(channel, '').setMode(mode) +// assert.notEqual(modes.r, null); +// assert.equal(modes.r.password, password); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); + +// it('can\'t read with wrong password', async((done) => { +// try { +// var modes = orbit.channel(channel, 'invalidpassword').iterator(); +// assert.equal(true, false); +// } catch(e) { +// assert.equal(e, 'Unauthorized'); +// } +// done(); +// })); + +// it('sets write mode', async((done) => { +// try { +// var mode = { +// mode: "+w", +// params: { +// ops: [orbit.user.id] +// } +// }; +// var modes = orbit.channel(channel, password).setMode(mode); +// assert.notEqual(modes.w, null); +// assert.equal(modes.w.ops[0], orbit.user.id); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); + +// it('can\'t write when user not an op', async((done) => { +// // TODO +// done(); +// })); + +// it('removes write mode', async((done) => { +// try { +// var modes = orbit.channel(channel, password).setMode({ mode: "-w" }); +// assert.equal(modes.w, null); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); + +// it('removes read mode', async((done) => { +// try { +// var modes = orbit.channel(channel, password).setMode({ mode: "-r" }); +// assert.equal(modes.r, null); +// } catch(e) { +// assert.equal(e, null); +// } +// done(); +// })); + +// }); +// */ +// }); diff --git a/test1.js b/test1.js new file mode 100644 index 0000000..5c6d04a --- /dev/null +++ b/test1.js @@ -0,0 +1,345 @@ +'use strict'; + +const _ = require('lodash'); +const Timer = require('./examples/Timer'); + +Array.prototype.allEqual = function(val) { + for(var i = 1; i < this.length; i++) { + if(this[i] !== val || this[i] !== this[0]) + return false; + } + return true; +} + +if (!Array.prototype.last){ + Array.prototype.last = function(){ + return this[this.length - 1]; + }; +} + +class Node { + constructor(id, seq, ver, data, next) { + this.id = id; + this.seq = seq; + this.ver = ver; + this.data = data || null; + this.next = next ? next.map((f) => f.compactId ? f.compactId : f) : []; + } + + get compactId() { + return "" + this.id + "." + this.seq + "." + this.ver; + } + + compact() { + return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next } + // return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next.map((f) => f.compactId) } + } + + // static parseCompact(t) { + // let v = t.split('.'); + // return { id: v[0], seq: parseInt(v[1]), ver: parseInt(v[2]) }; + // } + +} + +class List { + constructor(id) { + this.id = id; + this.seq = 0; + this.ver = 0; + this._items = []; + this._currentBatch = []; + this._referenced = []; + } + + static fromJson(json) { + let list = new List(json.id); + list.seq = json.seq; + list.ver = json.ver; + // list._items = json.items.map((f) => new Node(f.id, f.seq, f.ver, f.data, f.next)); + list._items = _.uniqWith(json.items.map((f) => { + // console.log(JSON.stringify(f.next, null, 2)); + return new Node(f.id, f.seq, f.ver, f.data, f.next); + }), _.isEqual); + return list; + } + + get items() { + return this._items.concat(this._currentBatch); + } + + toJson() { + return { + id: this.id, + seq: this.seq, + ver: this.ver, + items: this._currentBatch.map((f) => f.compact()) + } + } + + toString() { + const items = this.items.map((f) => JSON.stringify(f.compact())).join("\n"); + return ` + id: ${this.id}, + seq: ${this.seq}, + ver: ${this.ver}, + items: \n${items} + ` + } + + add(data) { + // var ii = this.items.map((f) => f.compact()); + // console.log("--->", this.seq, this.ver) + const heads = this._findHeads(this.items);//.map((f) => f.compact()); + // const heads = this._findHeads(this.items) + // console.log("jjjj", this._referenced, "\n\n") + // console.log("jjjj", this.items, "\n\n") + const node = new Node(this.id, this.seq, this.ver, data, heads); + // console.log("aaaa", node) + this._currentBatch.push(node); + // this._referenced.push(node); + this.ver ++; + } + + join(other) { + let ms; + + if(other.seq && other.seq > this.seq) { + this.seq = other.seq + 1; + this.ver = 0; + } else { + this.seq = this.seq + 1; + this.ver = 0; + } + // if(other.items.last() && other.items.last().seq > this.seq) { + // this.seq = other.seq + 1; + // this.ver = 0; + // } + + // return items that are only in this._currentBatch + const current = _.differenceWith(this._currentBatch, this._items, this._equals); + // return items that are only in other.items + + let timer = new Timer(true); + const others = _.differenceWith(other.items, this._items, this._equals); + ms = timer.stop(true); + // return unique items from e and d + // const final = _.unionWith(others, this._currentBatch, this._equals); + const final = _.unionWith(current, others, this._equals); + // append new items to all items + this._items = this._items.concat(final); + + this._currentBatch = []; + if(ms > 20) { + console.log("OVER 20MS!!", other.items.length) + console.log(`join took ${timer.stop(true)} ms`); + } + } + + _findHeads(list) { + let timer = new Timer(true); + let ms; + const grouped = _.groupBy(list, 'id'); + // const heads = Object.keys(grouped).sort((a, b) => a === this.id ? -1 : (a < b ? 1 : 0)).map((g) => grouped[g].last()); + const heads = Object.keys(grouped).map((g) => grouped[g].last()); + // const heads = _.differenceWith(list, this._referenced); + // console.log("HEADS", JSON.stringify(heads)); + // const cleaned = heads.filter((e) => !this._isReferencedInChain(list, e, this._referenced, 0)); + const cleaned = heads.filter((e) => !this._isReferencedInChain2(list, e)); + // const cleaned = heads; + // console.log("CLEANED", JSON.stringify(cleaned), "\n"); + // this._referenced = []; + ms = timer.stop(true); + if(ms > 20) { + console.log("OVER 20MS!!") + console.log(`_findHeads took ${ms} ms`); + } + // console.log(`_findHeads took ${ms} ms`); + return cleaned; + } + + + _isReferencedInChain2(all, item) { + // console.log("item:", item); + let isReferenced = all.find((e) => this._references(e, item)) !== undefined; + if(!isReferenced) { + // console.log("check children", item.next) + let childHasRef = item.next.map((f) => { + const next = all.find((e) => this._equals(e, f)); + const res = next ? this._isReferencedInChain2(all, next) : false; + return _.find(res, (e) => e === true) !== undefined; + }); + isReferenced = _.find(childHasRef, (e) => e === true) !== undefined; + // console.log("children have ref", isReferenced) + } + + // console.log("red:", isReferenced); + return isReferenced; + } + + // _parse(t) { + // let v = t.split('.'); + // return { id: v[0], seq: parseInt(v[1]), ver: parseInt(v[2]) }; + // } + + _equals(a, b) { + return a.id == b.id && a.seq == b.seq && a.ver == b.ver; + } + + _references(a, b) { + // return _.includes(a.next, b.compactId); + // faster than _.includes() + for(let i = 0; i < a.next.length; i ++) { + if(b.compactId === a.next[i]) return true; + } + return false; + } + +/* + _isReferencedInChain(all, item, next2, refs, depth) { + console.log("...item:", item); + depth += 1; + let timer = new Timer(true); + let ms; + + if(!item) + return false; + + // let isReferenced = refs.find((e) => e.id == item.id && e.seq == item.seq && e.ver == item.ver) !== undefined; + let isReferenced = refs.find((e) => this._equals(e, item)) !== undefined; + + if(isReferenced) + console.log("ref", item) + + if(isReferenced) + return true; + + // if(item.id == 'C' && item.seq == 3 && item.ver == 1) { + // console.log("!!!!", refs,"\n", item) + // } + refs.splice(0, 0, item); + + var check = (o) => { + const next = all.find((e) => this._equals(e, this._parse(o))); + if(!next) + return false; + + return this._isReferencedInChain(all, item, next, refs, depth); + }; + + // refs.push(item); + + // console.log(item.next); + // let res = item.next.map(this._parse).map(check); + let res = item.next.map(check); + + if(depth > 5) { + console.log("WTF!!!", depth, item, refs, res); + } + + // refs.splice(0, 0, item); + + // if(item.id == 'A' && item.seq == 0 && item.ver == 1) { + // console.log("????", isReferenced, refs, "\n\n", item, "\n\n", res) + // } + + // if(item.id == 'A' && item.seq == 1 && item.ver == 2) { + // console.log("+++", res) + // } + + // const hasRef2 = _.find(res, (e) => e === true) !== undefined; + // const hasRef2 = res.allEqual(true) + + ms = timer.stop(true); + if(ms > 20) { + console.log("OVER 20MS!!", refs.length) + console.log(`_isReferencedInChain took ${ms} ms`); + } + let hasRef2 = _.find(res, (e) => e === true) !== undefined; + // console.log("...res:", hasRef2, res); + // if(hasRef2) + // refs.splice(0, 0, item); + + return hasRef2; + } +*/ +} + +var run = () => { + var redis = require("redis"); + this.client1 = redis.createClient({ host: "localhost", port: 6379 }); + this.client2 = redis.createClient({ host: "localhost", port: 6379 }); + var hash = "ccc" + this.client1.subscribe(hash); + this.client1.subscribe(hash); + + + let listA = new List("A"); + let listB = new List("B"); + let listC = new List("C"); + + const handleMessage = (hash, event) => { + const l = List.fromJson(JSON.parse(event)); + // console.log("LIST", l); + + // var l = List.fromJson(JSON.parse(event)); + if(l.id === 'A') { + listB.join(l); + listC.join(l); + // console.log(listB); + } else if(l.id === 'B') { + listA.join(l); + listC.join(l); + // console.log("ListA:", listA); + } else if(l.id === 'C') { + listA.join(l); + // console.log("LIST", event); + console.log("Items:", listA.items.length); + // console.log(JSON.stringify(listA, null, 1)); + } + + } + + this.client1.on("message", handleMessage); + this.client2.on("message", handleMessage); + + setInterval(() => { + // listC.join(listB); + // listC.join(listA); + }, 5000); + + let h = 0; + setInterval(() => { + listC.add("C--"+h); + this.client2.publish(hash, JSON.stringify(listC.toJson())); + h++; + }, 1000); + + let i = 0; + setInterval(() => { + let a = 0; + // for(let a = 0; a < 100; a ++) { + listB.add("B--"+(i+a)); + // } + this.client2.publish(hash, JSON.stringify(listB.toJson())); + i++; + }, 10); + + let k = 0; + setInterval(() => { + listA.add("A--"+k); + k++; + listA.add("A--"+k); + k++; + listA.add("A--"+k); + k++; + this.client2.publish(hash, JSON.stringify(listA.toJson())); + }, 100); +}; + +run(); + +module.exports = { + Node: Node, + List: List +}; From 523e67b05e89a2631e166c8f38c92d167a368b3b Mon Sep 17 00:00:00 2001 From: haad Date: Tue, 16 Feb 2016 11:10:02 +0100 Subject: [PATCH 17/30] Cleanup lists --- src/list/List.js | 88 ++++++++++++ src/list/Node.js | 21 +++ test/list-node-tests.js | 61 +++++++++ test/list-perf-tests.js | 5 +- test/list-tests.js | 289 +++++++++++++++++++++++----------------- test1.js | 284 +-------------------------------------- 6 files changed, 342 insertions(+), 406 deletions(-) create mode 100644 src/list/List.js create mode 100644 src/list/Node.js create mode 100644 test/list-node-tests.js diff --git a/src/list/List.js b/src/list/List.js new file mode 100644 index 0000000..fbf8736 --- /dev/null +++ b/src/list/List.js @@ -0,0 +1,88 @@ +'use strict'; + +const _ = require('lodash'); +const Node = require('./Node'); + +class List { + constructor(id) { + this.id = id; + this.seq = 0; + this.ver = 0; + this._items = []; + this._currentBatch = []; + } + + get items() { + return this._items.concat(this._currentBatch); + } + + add(data) { + const heads = this._findHeads(this.items); + const node = new Node(this.id, this.seq, this.ver, data, heads); + this._currentBatch.push(node); + this.ver ++; + } + + join(other) { + if(other.seq && other.seq > this.seq) { + this.seq = other.seq + 1; + this.ver = 0; + } else { + this.seq = this.seq + 1; + this.ver = 0; + } + const current = _.differenceWith(this._currentBatch, this._items, this._equals); + const others = _.differenceWith(other.items, this._items, this._equals); + const final = _.unionWith(current, others, this._equals); + this._items = this._items.concat(final); + this._currentBatch = []; + } + + _findHeads(list) { + const grouped = _.groupBy(list, 'id'); + const heads = Object.keys(grouped).map((g) => _.last(grouped[g])); + const cleaned = heads.filter((e) => !this._isReferencedInChain(list, e)); + return cleaned; + } + + _isReferencedInChain(all, item) { + let isReferenced = _.findLast(all, (e) => this._references(e, item)) !== undefined; + return isReferenced; + } + + _equals(a, b) { + return a.id == b.id && a.seq == b.seq && a.ver == b.ver; + } + + _references(a, b) { + for(let i = 0; i < a.next.length; i ++) { + if(b.compactId === a.next[i]) + return true; + } + return false; + } + + static fromJson(json) { + let list = new List(json.id); + list.seq = json.seq; + list.ver = json.ver; + list._items = _.uniqWith(json.items.map((f) => new Node(f.id, f.seq, f.ver, f.data, f.next)), _.isEqual); + return list; + } + + toJson() { + return { + id: this.id, + seq: this.seq, + ver: this.ver, + items: this._currentBatch.map((f) => f.compact()) + } + } + + toString() { + const items = this.items.map((f) => JSON.stringify(f.compact())).join("\n"); + return `id: ${this.id}, seq: ${this.seq}, ver: ${this.ver}, items:\n${items}`; + } +} + +module.exports = List; diff --git a/src/list/Node.js b/src/list/Node.js new file mode 100644 index 0000000..aa0544f --- /dev/null +++ b/src/list/Node.js @@ -0,0 +1,21 @@ +'use strict'; + +class Node { + constructor(id, seq, ver, data, next) { + this.id = id; + this.seq = seq; + this.ver = ver; + this.data = data || null; + this.next = next ? next.map((f) => f.compactId ? f.compactId : f) : []; + } + + get compactId() { + return "" + this.id + "." + this.seq + "." + this.ver; + } + + compact() { + return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next } + } +} + +module.exports = Node; diff --git a/test/list-node-tests.js b/test/list-node-tests.js new file mode 100644 index 0000000..f0ad641 --- /dev/null +++ b/test/list-node-tests.js @@ -0,0 +1,61 @@ +'use strict'; + +const assert = require('assert'); +const List = require('../src/list/List'); +const Node = require('../src/list/Node'); + +describe('Node', () => { + describe('Constructor', () => { + it('initializes member variables', (done) => { + const node = new Node('A', 0, 0, 'hello', []); + assert.equal(node.id, 'A'); + assert.equal(node.seq, 0); + assert.equal(node.ver, 0); + assert.equal(node.data, 'hello'); + assert.equal(node.next instanceof Array, true); + done(); + }); + + it('initializes member variables without specified data and next', (done) => { + const node = new Node('A', 0, 0); + assert.equal(node.id, 'A'); + assert.equal(node.seq, 0); + assert.equal(node.ver, 0); + assert.equal(node.data, null); + assert.equal(node.next instanceof Array, true); + done(); + }); + }); + + describe('compactId', () => { + it('presents the node as a string with id, sequence and version', (done) => { + const node1 = new Node('A', 0, 0); + const node2 = new Node('B', 123, 456); + assert.equal(node1.compactId, 'A.0.0'); + assert.equal(node2.compactId, 'B.123.456'); + done(); + }); + }); + + describe('compact', () => { + it('presents the node as a compacted object', (done) => { + const node1 = new Node('A', 0, 0, 'hello'); + const node2 = new Node('B', 0, 0, 'hello', [node1]); + const compacted1 = node1.compact(); + const compacted2 = node2.compact(); + + assert.notEqual(compacted1, null); + assert.equal(compacted1.id, 'A'); + assert.equal(compacted1.seq, 0); + assert.equal(compacted1.ver, 0); + assert.equal(compacted1.data, 'hello'); + assert.equal(compacted1.next instanceof Array, true); + assert.equal(compacted1.next.length, 0); + + assert.equal(compacted2.id, 'B'); + assert.equal(compacted2.next.length, 1); + assert.equal(compacted2.next[0], 'A.0.0'); + done(); + }); + }); +}); diff --git a/test/list-perf-tests.js b/test/list-perf-tests.js index 50431a8..43e97bc 100644 --- a/test/list-perf-tests.js +++ b/test/list-perf-tests.js @@ -3,8 +3,7 @@ // var assert = require('assert'); // var async = require('asyncawait/async'); // var await = require('asyncawait/await'); -// var List = require('../test1').List; -// var Node = require('../test1').Node; +// var List = require('../src/list/List'); // var Timer = require('../examples/Timer'); // describe('List - Performance Measurement', function() { @@ -23,8 +22,6 @@ // ms = timer.stop(true); // console.log(` > ${t} took ${ms} ms`) - -// // assert.equal(ms < t, true); // } // assert.equal(true, true); diff --git a/test/list-tests.js b/test/list-tests.js index bdbe3b3..46bfc46 100644 --- a/test/list-tests.js +++ b/test/list-tests.js @@ -1,66 +1,8 @@ 'use strict'; +const _ = require('lodash'); var assert = require('assert'); -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var List = require('../test1').List; -var Node = require('../test1').Node; - -describe('Node', () => { - describe('Constructor', () => { - it('initializes member variables', (done) => { - const node = new Node('A', 0, 0, 'hello', []); - assert.equal(node.id, 'A'); - assert.equal(node.seq, 0); - assert.equal(node.ver, 0); - assert.equal(node.data, 'hello'); - assert.equal(node.next instanceof Array, true); - done(); - }); - - it('initializes member variables without data and next', (done) => { - const node = new Node('A', 0, 0); - assert.equal(node.id, 'A'); - assert.equal(node.seq, 0); - assert.equal(node.ver, 0); - assert.equal(node.data, null); - assert.equal(node.next instanceof Array, true); - done(); - }); - }); - - describe('compactId', () => { - it('presents the node as a string with id, sequence and version', (done) => { - const node1 = new Node('A', 0, 0); - const node2 = new Node('B', 123, 456); - assert.equal(node1.compactId, 'A.0.0'); - assert.equal(node2.compactId, 'B.123.456'); - done(); - }); - }); - - describe('compact', () => { - it('presents the node as a compacted object', (done) => { - const node1 = new Node('A', 0, 0, 'hello'); - const node2 = new Node('B', 0, 0, 'hello', [node1]); - const compacted1 = node1.compact(); - const compacted2 = node2.compact(); - - assert.notEqual(compacted1, null); - assert.equal(compacted1.id, 'A'); - assert.equal(compacted1.seq, 0); - assert.equal(compacted1.ver, 0); - assert.equal(compacted1.data, 'hello'); - assert.equal(compacted1.next instanceof Array, true); - assert.equal(compacted1.next.length, 0); - - assert.equal(compacted2.id, 'B'); - assert.equal(compacted2.next.length, 1); - assert.equal(compacted2.next[0], 'A.0.0'); - done(); - }); - }); -}); +var List = require('../src/list/List'); describe('List', () => { describe('Constructor', () => { @@ -77,6 +19,60 @@ describe('List', () => { }); }); + describe('fromJson', () => { + it('creates a list from parsed json', (done) => { + const list = new List('A'); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const str = JSON.stringify(list.toJson()) + const res = List.fromJson(JSON.parse(str)); + assert.equal(res.id, 'A'); + assert.equal(res.seq, 0); + assert.equal(res.ver, 3); + assert.equal(res.items.length, 3); + assert.equal(res.items[0].compactId, 'A.0.0'); + assert.equal(res.items[1].compactId, 'A.0.1'); + assert.equal(res.items[2].compactId, 'A.0.2'); + done(); + }); + }); + + describe('toJson', () => { + it('presents the list as json', (done) => { + const list = new List('A'); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const json = list.toJson(); + const expected = { + id: 'A', + seq: 0, + ver: 3, + items: [ + { id: 'A', seq: 0, ver: 0, data: 'hello1', next: [] }, + { id: 'A', seq: 0, ver: 1, data: 'hello2', next: ['A.0.0'] }, + { id: 'A', seq: 0, ver: 2, data: 'hello3', next: ['A.0.1'] } + ] + }; + assert.equal(_.isEqual(json, expected), true); + done(); + }); + }); + + describe('toString', () => { + it('presents the list as a string', (done) => { + const list = new List('A'); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const str = list.toString(); + const expected = `id: A, seq: 0, ver: 3, items:\n{"id":"A","seq":0,"ver":0,"data":"hello1","next":[]}\n{"id":"A","seq":0,"ver":1,"data":"hello2","next":["A.0.0"]}\n{"id":"A","seq":0,"ver":2,"data":"hello3","next":["A.0.1"]}`; + assert.equal(str, expected); + done(); + }); + }); + describe('items', () => { it('returns items', (done) => { const list = new List('A'); @@ -99,10 +95,8 @@ describe('List', () => { list.add("hello1") list.add("hello2") list.add("hello3") - // list.add("hello4") - // list.add("hello5") + let compacted = list.toJson(); - // console.log(compacted) assert.equal(compacted.id, 'A'); assert.equal(compacted.seq, 0); assert.equal(compacted.ver, 3); @@ -210,14 +204,13 @@ describe('List', () => { list1.add("helloA2") list1.add("helloA3") - // console.log(list1.toString()) assert.equal(list1._currentBatch.length, 3); assert.equal(list1._currentBatch[2].next.length, 1); assert.equal(list1._currentBatch[2].next[0], 'A.0.1'); done(); }); - it('finds the next heads after joining a list with another', (done) => { + it('finds the next heads (two) after a join', (done) => { const list1 = new List('A'); const list2 = new List('B'); list1.add("helloA1") @@ -233,7 +226,7 @@ describe('List', () => { done(); }); - it('finds the next head after a two-way join', (done) => { + it('finds the next head (one) after a join', (done) => { const list1 = new List('A'); const list2 = new List('B'); list1.add("helloA1") @@ -250,7 +243,7 @@ describe('List', () => { done(); }); - it('find sthe next heads after join two-way join', (done) => { + it('finds the next heads after two joins', (done) => { const list1 = new List('A'); const list2 = new List('B'); list1.add("helloA1") @@ -267,13 +260,49 @@ describe('List', () => { list1.add("helloA5") const lastItem = list1.items[list1.items.length - 1]; - // console.log(list1.toString()) + assert.equal(list1.items.length, 7); assert.equal(lastItem.next.length, 1); assert.equal(lastItem.next[0], 'A.2.0'); done(); }); + it('finds the next heads after multiple joins', (done) => { + const list1 = new List('A'); + const list2 = new List('B'); + const list3 = new List('C'); + const list4 = new List('D'); + list1.add("helloA1") + list1.add("helloA2") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + list3.add("helloC1") + list4.add("helloD1") + list1.join(list3); + + list1.add("helloA3") + list2.join(list1); + list1.join(list2); + list2.join(list4); + + list4.add("helloD2") + list4.add("helloD3") + list1.add("helloA4") + list1.join(list4); + + list1.add("helloA5") + + const lastItem = list1.items[list1.items.length - 1]; + + assert.equal(list1.items.length, 11); + assert.equal(lastItem.next.length, 2); + assert.equal(lastItem.next[0], 'A.4.0'); + assert.equal(lastItem.next[1], 'D.0.2'); + done(); + }); + it('joins list of one item with list of two items', (done) => { const list1 = new List('A'); const list2 = new List('B'); @@ -282,18 +311,17 @@ describe('List', () => { list2.add("helloB2") list1.join(list2); - // console.log(list1) - // console.log(list2) + const lastItem = list1.items[list1.items.length - 1]; assert.equal(list1.id, 'A'); assert.equal(list1.seq, 1); assert.equal(list1.ver, 0); assert.equal(list1._currentBatch.length, 0); assert.equal(list1._items.length, 3); - assert.equal(list1._items[list1._items.length - 1].id, 'B'); - assert.equal(list1._items[list1._items.length - 1].seq, 0); - assert.equal(list1._items[list1._items.length - 1].ver, 1); - assert.equal(list1._items[list1._items.length - 1].data, 'helloB2'); + assert.equal(lastItem.id, 'B'); + assert.equal(lastItem.seq, 0); + assert.equal(lastItem.ver, 1); + assert.equal(lastItem.data, 'helloB2'); done(); }); @@ -307,25 +335,29 @@ describe('List', () => { list1.join(list2); list2.join(list1); + const lastItem1 = list1.items[list1.items.length - 1]; + assert.equal(list1.id, 'A'); assert.equal(list1.seq, 1); assert.equal(list1.ver, 0); assert.equal(list1._currentBatch.length, 0); assert.equal(list1._items.length, 4); - assert.equal(list1._items[list1._items.length - 1].id, 'B'); - assert.equal(list1._items[list1._items.length - 1].seq, 0); - assert.equal(list1._items[list1._items.length - 1].ver, 1); - assert.equal(list1._items[list1._items.length - 1].data, 'helloB2'); + assert.equal(lastItem1.id, 'B'); + assert.equal(lastItem1.seq, 0); + assert.equal(lastItem1.ver, 1); + assert.equal(lastItem1.data, 'helloB2'); + + const lastItem2 = list2.items[list2.items.length - 1]; assert.equal(list2.id, 'B'); assert.equal(list2.seq, 2); assert.equal(list2.ver, 0); assert.equal(list2._currentBatch.length, 0); assert.equal(list2._items.length, 4); - assert.equal(list2._items[list2._items.length - 1].id, 'A'); - assert.equal(list2._items[list2._items.length - 1].seq, 0); - assert.equal(list2._items[list2._items.length - 1].ver, 1); - assert.equal(list2._items[list2._items.length - 1].data, 'helloA2'); + assert.equal(lastItem2.id, 'A'); + assert.equal(lastItem2.seq, 0); + assert.equal(lastItem2.ver, 1); + assert.equal(lastItem2.data, 'helloA2'); done(); }); @@ -341,23 +373,26 @@ describe('List', () => { list2.add("helloB2") list2.join(list1); + const secondItem = list2.items[1]; + const lastItem = list2.items[list2.items.length - 1]; + assert.equal(list2.id, 'B'); assert.equal(list2.seq, 2); assert.equal(list2.ver, 0); assert.equal(list2._currentBatch.length, 0); assert.equal(list2._items.length, 4); - assert.equal(list2._items[1].id, 'A'); - assert.equal(list2._items[1].seq, 0); - assert.equal(list2._items[1].ver, 0); - assert.equal(list2._items[1].data, 'helloA1'); - assert.equal(list2._items[list2._items.length - 1].id, 'A'); - assert.equal(list2._items[list2._items.length - 1].seq, 0); - assert.equal(list2._items[list2._items.length - 1].ver, 1); - assert.equal(list2._items[list2._items.length - 1].data, 'helloA2'); + assert.equal(secondItem.id, 'A'); + assert.equal(secondItem.seq, 0); + assert.equal(secondItem.ver, 0); + assert.equal(secondItem.data, 'helloA1'); + assert.equal(lastItem.id, 'A'); + assert.equal(lastItem.seq, 0); + assert.equal(lastItem.ver, 1); + assert.equal(lastItem.data, 'helloA2'); done(); }); - it('joins 4 lists', (done) => { + it('joins 4 lists to one', (done) => { const list1 = new List('A'); const list2 = new List('B'); const list3 = new List('C'); @@ -365,39 +400,36 @@ describe('List', () => { list1.add("helloA1") list2.add("helloB1") - // list2.join(list1); - list1.add("helloA2") list2.add("helloB2") - // list2.join(list1); - list3.add("helloC1") list4.add("helloD1") - // list2.join(list1); - list3.add("helloC2") list4.add("helloD2") list1.join(list2); list1.join(list3); list1.join(list4); + const secondItem = list1.items[1]; + const lastItem = list1.items[list1.items.length - 1]; + assert.equal(list1.id, 'A'); - // assert.equal(list1.seq, 1); - // assert.equal(list1.ver, 1); + assert.equal(list1.seq, 3); + assert.equal(list1.ver, 0); assert.equal(list1._currentBatch.length, 0); assert.equal(list1._items.length, 8); - assert.equal(list1._items[1].id, 'A'); - assert.equal(list1._items[1].seq, 0); - assert.equal(list1._items[1].ver, 1); - assert.equal(list1._items[1].data, 'helloA2'); - assert.equal(list1._items[list1._items.length - 1].id, 'D'); - assert.equal(list1._items[list1._items.length - 1].seq, 0); - assert.equal(list1._items[list1._items.length - 1].ver, 1); - assert.equal(list1._items[list1._items.length - 1].data, 'helloD2'); + assert.equal(secondItem.id, 'A'); + assert.equal(secondItem.seq, 0); + assert.equal(secondItem.ver, 1); + assert.equal(secondItem.data, 'helloA2'); + assert.equal(lastItem.id, 'D'); + assert.equal(lastItem.seq, 0); + assert.equal(lastItem.ver, 1); + assert.equal(lastItem.data, 'helloD2'); done(); }); - it('joins 4 lists sequentally', (done) => { + it('joins lists from 4 lists', (done) => { const list1 = new List('A'); const list2 = new List('B'); const list3 = new List('C'); @@ -428,31 +460,42 @@ describe('List', () => { list4.add("helloD3") list4.add("helloD4") - // console.log(list4.toString()); + const secondItem = list4.items[1]; + const lastItem1 = list4._items[list4._items.length - 1]; + const lastItem2 = list4.items[list4.items.length - 1]; + assert.equal(list4.id, 'D'); assert.equal(list4.seq, 7); assert.equal(list4.ver, 2); assert.equal(list4._currentBatch.length, 2); assert.equal(list4._items.length, 8); - assert.equal(list4._items[1].id, 'D'); - assert.equal(list4._items[1].seq, 0); - assert.equal(list4._items[1].ver, 1); - assert.equal(list4._items[1].data, 'helloD2'); - assert.equal(list4._items[list1._items.length - 1].id, 'A'); - assert.equal(list4._items[list1._items.length - 1].seq, 1); - assert.equal(list4._items[list1._items.length - 1].ver, 0); - assert.equal(list4._items[list1._items.length - 1].data, 'helloA2'); - assert.equal(list4.items[list4.items.length - 1].id, 'D'); - assert.equal(list4.items[list4.items.length - 1].seq, 7); - assert.equal(list4.items[list4.items.length - 1].ver, 1); - assert.equal(list4.items[list4.items.length - 1].data, 'helloD4'); + assert.equal(secondItem.id, 'D'); + assert.equal(secondItem.seq, 0); + assert.equal(secondItem.ver, 1); + assert.equal(secondItem.data, 'helloD2'); + assert.equal(lastItem1.id, 'C'); + assert.equal(lastItem1.seq, 3); + assert.equal(lastItem1.ver, 1); + assert.equal(lastItem1.data, 'helloC2'); + assert.equal(lastItem2.id, 'D'); + assert.equal(lastItem2.seq, 7); + assert.equal(lastItem2.ver, 1); + assert.equal(lastItem2.data, 'helloD4'); done(); }); }); - it('solves a graph', (done) => { - done(); + describe('_findHeads', () => { + it('TODO', (done) => { + done(); + }); + }); + + describe('_isReferencedInChain', () => { + it('TODO', (done) => { + done(); + }); }); }); diff --git a/test1.js b/test1.js index 5c6d04a..1cff702 100644 --- a/test1.js +++ b/test1.js @@ -2,268 +2,8 @@ const _ = require('lodash'); const Timer = require('./examples/Timer'); - -Array.prototype.allEqual = function(val) { - for(var i = 1; i < this.length; i++) { - if(this[i] !== val || this[i] !== this[0]) - return false; - } - return true; -} - -if (!Array.prototype.last){ - Array.prototype.last = function(){ - return this[this.length - 1]; - }; -} - -class Node { - constructor(id, seq, ver, data, next) { - this.id = id; - this.seq = seq; - this.ver = ver; - this.data = data || null; - this.next = next ? next.map((f) => f.compactId ? f.compactId : f) : []; - } - - get compactId() { - return "" + this.id + "." + this.seq + "." + this.ver; - } - - compact() { - return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next } - // return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next.map((f) => f.compactId) } - } - - // static parseCompact(t) { - // let v = t.split('.'); - // return { id: v[0], seq: parseInt(v[1]), ver: parseInt(v[2]) }; - // } - -} - -class List { - constructor(id) { - this.id = id; - this.seq = 0; - this.ver = 0; - this._items = []; - this._currentBatch = []; - this._referenced = []; - } - - static fromJson(json) { - let list = new List(json.id); - list.seq = json.seq; - list.ver = json.ver; - // list._items = json.items.map((f) => new Node(f.id, f.seq, f.ver, f.data, f.next)); - list._items = _.uniqWith(json.items.map((f) => { - // console.log(JSON.stringify(f.next, null, 2)); - return new Node(f.id, f.seq, f.ver, f.data, f.next); - }), _.isEqual); - return list; - } - - get items() { - return this._items.concat(this._currentBatch); - } - - toJson() { - return { - id: this.id, - seq: this.seq, - ver: this.ver, - items: this._currentBatch.map((f) => f.compact()) - } - } - - toString() { - const items = this.items.map((f) => JSON.stringify(f.compact())).join("\n"); - return ` - id: ${this.id}, - seq: ${this.seq}, - ver: ${this.ver}, - items: \n${items} - ` - } - - add(data) { - // var ii = this.items.map((f) => f.compact()); - // console.log("--->", this.seq, this.ver) - const heads = this._findHeads(this.items);//.map((f) => f.compact()); - // const heads = this._findHeads(this.items) - // console.log("jjjj", this._referenced, "\n\n") - // console.log("jjjj", this.items, "\n\n") - const node = new Node(this.id, this.seq, this.ver, data, heads); - // console.log("aaaa", node) - this._currentBatch.push(node); - // this._referenced.push(node); - this.ver ++; - } - - join(other) { - let ms; - - if(other.seq && other.seq > this.seq) { - this.seq = other.seq + 1; - this.ver = 0; - } else { - this.seq = this.seq + 1; - this.ver = 0; - } - // if(other.items.last() && other.items.last().seq > this.seq) { - // this.seq = other.seq + 1; - // this.ver = 0; - // } - - // return items that are only in this._currentBatch - const current = _.differenceWith(this._currentBatch, this._items, this._equals); - // return items that are only in other.items - - let timer = new Timer(true); - const others = _.differenceWith(other.items, this._items, this._equals); - ms = timer.stop(true); - // return unique items from e and d - // const final = _.unionWith(others, this._currentBatch, this._equals); - const final = _.unionWith(current, others, this._equals); - // append new items to all items - this._items = this._items.concat(final); - - this._currentBatch = []; - if(ms > 20) { - console.log("OVER 20MS!!", other.items.length) - console.log(`join took ${timer.stop(true)} ms`); - } - } - - _findHeads(list) { - let timer = new Timer(true); - let ms; - const grouped = _.groupBy(list, 'id'); - // const heads = Object.keys(grouped).sort((a, b) => a === this.id ? -1 : (a < b ? 1 : 0)).map((g) => grouped[g].last()); - const heads = Object.keys(grouped).map((g) => grouped[g].last()); - // const heads = _.differenceWith(list, this._referenced); - // console.log("HEADS", JSON.stringify(heads)); - // const cleaned = heads.filter((e) => !this._isReferencedInChain(list, e, this._referenced, 0)); - const cleaned = heads.filter((e) => !this._isReferencedInChain2(list, e)); - // const cleaned = heads; - // console.log("CLEANED", JSON.stringify(cleaned), "\n"); - // this._referenced = []; - ms = timer.stop(true); - if(ms > 20) { - console.log("OVER 20MS!!") - console.log(`_findHeads took ${ms} ms`); - } - // console.log(`_findHeads took ${ms} ms`); - return cleaned; - } - - - _isReferencedInChain2(all, item) { - // console.log("item:", item); - let isReferenced = all.find((e) => this._references(e, item)) !== undefined; - if(!isReferenced) { - // console.log("check children", item.next) - let childHasRef = item.next.map((f) => { - const next = all.find((e) => this._equals(e, f)); - const res = next ? this._isReferencedInChain2(all, next) : false; - return _.find(res, (e) => e === true) !== undefined; - }); - isReferenced = _.find(childHasRef, (e) => e === true) !== undefined; - // console.log("children have ref", isReferenced) - } - - // console.log("red:", isReferenced); - return isReferenced; - } - - // _parse(t) { - // let v = t.split('.'); - // return { id: v[0], seq: parseInt(v[1]), ver: parseInt(v[2]) }; - // } - - _equals(a, b) { - return a.id == b.id && a.seq == b.seq && a.ver == b.ver; - } - - _references(a, b) { - // return _.includes(a.next, b.compactId); - // faster than _.includes() - for(let i = 0; i < a.next.length; i ++) { - if(b.compactId === a.next[i]) return true; - } - return false; - } - -/* - _isReferencedInChain(all, item, next2, refs, depth) { - console.log("...item:", item); - depth += 1; - let timer = new Timer(true); - let ms; - - if(!item) - return false; - - // let isReferenced = refs.find((e) => e.id == item.id && e.seq == item.seq && e.ver == item.ver) !== undefined; - let isReferenced = refs.find((e) => this._equals(e, item)) !== undefined; - - if(isReferenced) - console.log("ref", item) - - if(isReferenced) - return true; - - // if(item.id == 'C' && item.seq == 3 && item.ver == 1) { - // console.log("!!!!", refs,"\n", item) - // } - refs.splice(0, 0, item); - - var check = (o) => { - const next = all.find((e) => this._equals(e, this._parse(o))); - if(!next) - return false; - - return this._isReferencedInChain(all, item, next, refs, depth); - }; - - // refs.push(item); - - // console.log(item.next); - // let res = item.next.map(this._parse).map(check); - let res = item.next.map(check); - - if(depth > 5) { - console.log("WTF!!!", depth, item, refs, res); - } - - // refs.splice(0, 0, item); - - // if(item.id == 'A' && item.seq == 0 && item.ver == 1) { - // console.log("????", isReferenced, refs, "\n\n", item, "\n\n", res) - // } - - // if(item.id == 'A' && item.seq == 1 && item.ver == 2) { - // console.log("+++", res) - // } - - // const hasRef2 = _.find(res, (e) => e === true) !== undefined; - // const hasRef2 = res.allEqual(true) - - ms = timer.stop(true); - if(ms > 20) { - console.log("OVER 20MS!!", refs.length) - console.log(`_isReferencedInChain took ${ms} ms`); - } - let hasRef2 = _.find(res, (e) => e === true) !== undefined; - // console.log("...res:", hasRef2, res); - // if(hasRef2) - // refs.splice(0, 0, item); - - return hasRef2; - } -*/ -} +const List = require('./src/list/List'); +// const Node = require('./src/list/Node'); var run = () => { var redis = require("redis"); @@ -282,18 +22,14 @@ var run = () => { const l = List.fromJson(JSON.parse(event)); // console.log("LIST", l); - // var l = List.fromJson(JSON.parse(event)); if(l.id === 'A') { listB.join(l); listC.join(l); - // console.log(listB); } else if(l.id === 'B') { listA.join(l); listC.join(l); - // console.log("ListA:", listA); } else if(l.id === 'C') { listA.join(l); - // console.log("LIST", event); console.log("Items:", listA.items.length); // console.log(JSON.stringify(listA, null, 1)); } @@ -303,11 +39,6 @@ var run = () => { this.client1.on("message", handleMessage); this.client2.on("message", handleMessage); - setInterval(() => { - // listC.join(listB); - // listC.join(listA); - }, 5000); - let h = 0; setInterval(() => { listC.add("C--"+h); @@ -318,12 +49,12 @@ var run = () => { let i = 0; setInterval(() => { let a = 0; - // for(let a = 0; a < 100; a ++) { + for(let a = 0; a < 10; a ++) { listB.add("B--"+(i+a)); - // } + } this.client2.publish(hash, JSON.stringify(listB.toJson())); i++; - }, 10); + }, 20); let k = 0; setInterval(() => { @@ -338,8 +69,3 @@ var run = () => { }; run(); - -module.exports = { - Node: Node, - List: List -}; From 38631506249e319a99bf7077c3b86b4f89c41f26 Mon Sep 17 00:00:00 2001 From: haad Date: Tue, 16 Feb 2016 23:01:37 +0100 Subject: [PATCH 18/30] OrbitList --- examples/pubsubReader.js | 36 +-- examples/pubsubWriter.js | 53 ++++ src/OrbitClient.js | 45 +-- src/PubSub.js | 60 +++- src/list/OrbitList.js | 48 +++ src/list/OrbitNode.js | 28 ++ test/list-perf-tests.js | 54 +++- test/list-tests.js | 33 +- test/orbit-list-tests.js | 569 +++++++++++++++++++++++++++++++++++ test/orbitlist-node-tests.js | 63 ++++ test1.js | 1 - test2.js | 92 ++++++ 12 files changed, 994 insertions(+), 88 deletions(-) create mode 100644 examples/pubsubWriter.js create mode 100644 src/list/OrbitList.js create mode 100644 src/list/OrbitNode.js create mode 100644 test/orbit-list-tests.js create mode 100644 test/orbitlist-node-tests.js create mode 100644 test2.js diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index ba7293d..d83641b 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -5,7 +5,8 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -var host = '178.62.229.175'; +// var host = '178.62.229.175'; +var host = 'localhost'; var port = 6379; var username = 'LambOfGod'; @@ -23,36 +24,21 @@ let run = (async(() => { setInterval(async(() => { if(!running) { - let timer = new Timer(true); running = true; - // channel.add(id + count); - - console.log("Query..."); - let items = channel.iterator({ limit: 3 }).collect(); - console.log(`Found ${items.length} items`); - - var g = items.filter((e) => e.item.Payload.startsWith(id)) - var prev = -1; - g.reverse().forEach((e) => { - var a = parseInt(e.item.Payload.replace(id, '')); - if(prev > -1 && prev + 1 !== a) { - console.log("MISSING VALUE!!!", prev + 1, items) - process.exit(1); - } - prev = a; - }) - - items = items.map((e) => { - return e.item.seq + " | " + e.item.Payload; - }); + // let timer = new Timer(true); + channel.add(id + count); + // console.log(`Query #${count} took ${timer.stop(true)} ms\n`); + // console.log("Query..."); + const c = channel.iterator({ limit: -1 }).collect().length; + let items = channel.iterator({ limit: 5 }).collect(); console.log("---------------------------------------------------") - console.log("Seq | Payload") + console.log("Id | Seq | Ver | Data") console.log("---------------------------------------------------") - console.log(items.join("\n")); + console.log(items.map((e) => `${e.id} | ${e.seq} | ${e.ver} | ${e.data}`).join("\n")); console.log("---------------------------------------------------") - console.log(`Query #${count} took ${timer.stop(true)} ms\n`); + console.log(`Found ${items.length} items from ${c}\n`); running = false; count ++; diff --git a/examples/pubsubWriter.js b/examples/pubsubWriter.js new file mode 100644 index 0000000..067234a --- /dev/null +++ b/examples/pubsubWriter.js @@ -0,0 +1,53 @@ +'use strict'; + +var async = require('asyncawait/async'); +var await = require('asyncawait/await'); +var OrbitClient = require('../src/OrbitClient'); +var Timer = require('./Timer'); + +// var host = '178.62.229.175'; +var host = 'localhost'; +var port = 6379; + +var username = process.argv[2] ? process.argv[2] : 'DankoJones'; +var password = ''; + +let run = (async(() => { + try { + var orbit = OrbitClient.connect(host, port, username, password); + const c1 = 'c1'; + + let count = 1; + let id = 'Log: Query ' + let channel; + + setInterval(async(() => { + if(channel) { + channel.add(id + count); + count ++; + } + }), process.argv[3] ? process.argv[3] : 1000); + + setInterval(async(() => { + if(!channel) { + channel = orbit.channel(c1); + console.log("subscribed to pubsub topic '" + c1); + setTimeout(() => { + if(channel) { + console.log("leave"); + channel.leave(); + channel = null; + } + }, 2000); + } + }), 5000); + + + } catch(e) { + console.error("error:", e); + console.error(e.stack); + process.exit(1); + } +}))(); + +module.exports = run; diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 30f753a..689db56 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -14,6 +14,8 @@ var MetaInfo = require('./MetaInfo'); var Post = require('./Post'); var Aggregator = require('./Aggregator'); var PubSub = require('./PubSub'); +const List = require('./list/OrbitList'); +var Timer = require('../examples/Timer'); var pubkey = Keystore.getKeys().publicKey; var privkey = Keystore.getKeys().privateKey; @@ -25,16 +27,21 @@ class OrbitClient { this.ipfs = ipfs; this.network = {}; this.user = null; + this.list = null; // TODO move to DataStore } channel(hash, password) { if(password === undefined) password = ''; - await(this._pubsub.subscribe(hash, password)); - // this._pubsub.subscribe(hash, password, async((hash, message, seq) => { - // let m = Aggregator._fetchOne(this.ipfs, message, password); - // console.log(">", message); - // })); + this._pubsub.subscribe(hash, password, async((hash, message) => { + const other = await(List.fromIpfsHash(this.ipfs, message)); + // console.log(">", other.id, other.seq, other.ver); + if(other.id !== this.user.username) { + let timer = new Timer(true); + this.list.join(other); // TODO: move to DataStore + console.log(`Join took ${timer.stop(true)} ms`); + } + })); return { info: (options) => this._info(hash, password), @@ -49,6 +56,7 @@ class OrbitClient { const items = this._iterator(hash, password, { key: key }).collect(); return items[0] ? items[0].item.Payload : null; }, + leave: () => this._pubsub.unsubscribe(hash) } } @@ -68,7 +76,6 @@ class OrbitClient { return item; }, collect: () => messages - // TODO: add first() and last() ? } return iterator; @@ -106,7 +113,8 @@ class OrbitClient { }; // Get messages - messages = Aggregator.fetchRecursive(this.ipfs, startFromHash, password, opts); + // messages = Aggregator.fetchRecursive(this.ipfs, startFromHash, password, opts); + messages = this.list.items.map((f) => f.compact()); // TODO: move to DataStore // Slice the array let startIndex = 0; @@ -144,17 +152,14 @@ class OrbitClient { // Create the hash cache item const hcItem = new HashCacheItem(operation, key, seq, target, metaInfo, null, pubkey, privkey, password); - console.log("1") // Save the item to ipfs const data = await (ipfsAPI.putObject(this.ipfs, JSON.stringify(hcItem))); - console.log("2", data.Hash, head) let newHead = { Hash: data.Hash }; // If this is not the first item in the channel, patch with the previous (ie. link as next) if(seq > 0) newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); - console.log("3") return { hash: newHead, seq: seq }; } @@ -177,16 +182,10 @@ class OrbitClient { } _createOperation(channel, password, operation, key, value, data) { - let message, res = false; - while(!res) { - this.posting = true; - console.log("posting...") - message = this._createMessage(channel, password, operation, key, value); - res = await(this._pubsub.publish(channel, message.hash, message.seq)); - if(!res) console.log("retry", message.hash, message.seq) - } - this.posting = false; - console.log("posted") + let message = this._createMessage(channel, password, operation, key, value); + this.list.add(message.hash.Hash); // TODO: move to DataStore + const listHash = await(this.list.getIpfsHash()); + await(this._pubsub.publish(channel, listHash)); return message.hash.Hash; } @@ -213,10 +212,12 @@ class OrbitClient { _connect(host, port, username, password) { return new Promise((resolve, reject) => { - this._pubsub = new PubSub(this.ipfs, host, port, username, password, resolve); + this._pubsub = new PubSub(this.ipfs, host, port, username, password); // this.client = this._pubsub._client; // this.user = this.client.info.user; - this.user = { id: 'hello' } + this.user = { id: 'hello-todo', username: username } + this.list = new List(username, this.ipfs); // TODO: move to DataStore + resolve(); // this.network = { // id: this.client.info.networkId, // name: this.client.info.name, diff --git a/src/PubSub.js b/src/PubSub.js index 01fc892..2625f7f 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -4,7 +4,63 @@ var async = require('asyncawait/async'); var await = require('asyncawait/await'); var redis = require("redis"); var Aggregator = require('./Aggregator'); +const List = require('./list/OrbitList'); +class Pubsub2 { + constructor(ipfs, host, port, username, password) { + this.ipfs = ipfs; + this._subscriptions = {}; + this.client1 = redis.createClient({ host: host, port: port }); + this.client2 = redis.createClient({ host: host, port: port }); + this.client1.on("message", this._handleMessage.bind(this)); + this.client1.on('connect', () => { + console.log('redis connected'); + }); + // this.client1.on("subscribe", (channel, count) => { + // }); + } + + subscribe(hash, password, callback) { + if(!this._subscriptions[hash] || this._subscriptions[hash].password !== password) { + this._subscriptions[hash] = { + topic: hash, + password: password, + head: null, + callback: callback + }; + this.client1.subscribe(hash); + } + } + + unsubscribe(hash) { + delete this._subscriptions[hash]; + this.client1.unsubscribe(); + this.client2.unsubscribe(); + } + + publish(hash, message) { + this.client2.publish(hash, message); + } + + latest(hash) { + return { head: this._subscriptions[hash] ? this._subscriptions[hash].head : null }; + } + + delete(hash, password) { + delete this._subscriptions[hash]; + } + + _handleMessage(hash, message) { + if(this._subscriptions[hash]) { + this._subscriptions[hash].head = message; + + if(this._subscriptions[hash].callback) + this._subscriptions[hash].callback(hash, message); + } + } +} + +/* class PubSub { constructor(ipfs, host, port, username, password, resolve) { this.ipfs = ipfs; @@ -124,7 +180,7 @@ class PubSub { if(this._subscriptions[hash].callback) this._subscriptions[hash].callback(hash, message, seq); } - } +*/ -module.exports = PubSub; +module.exports = Pubsub2; diff --git a/src/list/OrbitList.js b/src/list/OrbitList.js new file mode 100644 index 0000000..40544ea --- /dev/null +++ b/src/list/OrbitList.js @@ -0,0 +1,48 @@ +'use strict'; + +const _ = require('lodash'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const List = require('./List'); +const Node = require('./OrbitNode'); + +class OrbitList extends List { + constructor(id, ipfs) { + super(id); + this._ipfs = ipfs; + this.hash = null; + } + + add(data) { + const heads = super._findHeads(this.items); + const node = new Node(this._ipfs, this.id, this.seq, this.ver, data, heads); + this._currentBatch.push(node); + this.ver ++; + } + + getIpfsHash() { + return new Promise(async((resolve, reject) => { + const list = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(this.toJson()))); + resolve(list.Hash); + })); + } + + static fromJson(ipfs, json) { + let list = new List(json.id); + list.seq = json.seq; + list.ver = json.ver; + list._items = _.uniqWith(json.items.map((f) => new Node(ipfs, f.id, f.seq, f.ver, f.data, f.next)), _.isEqual); + return list; + } + + static fromIpfsHash(ipfs, hash) { + return new Promise(async((resolve, reject) => { + const l = await(ipfsAPI.getObject(ipfs, hash)); + const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data)); + resolve(list); + })); + } +} + +module.exports = OrbitList; diff --git a/src/list/OrbitNode.js b/src/list/OrbitNode.js new file mode 100644 index 0000000..0350e7b --- /dev/null +++ b/src/list/OrbitNode.js @@ -0,0 +1,28 @@ +'use strict'; + +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const await = require('asyncawait/await'); +const Node = require('./Node'); + +class OrbitNode extends Node { + constructor(ipfs, id, seq, ver, data, next) { + super(id, seq, ver, data, next); + this.hash = null; + this._ipfs = ipfs; + } + + get compactId() { + if(!this.hash) { + const t = this.compact(); + const r = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(t))); + this.hash = r.Hash; + } + return "" + this.id + "." + this.seq + "." + this.ver + "." + this.hash; + } + + compact() { + return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next } + } +} + +module.exports = OrbitNode; diff --git a/test/list-perf-tests.js b/test/list-perf-tests.js index 43e97bc..409e6cf 100644 --- a/test/list-perf-tests.js +++ b/test/list-perf-tests.js @@ -1,10 +1,14 @@ // 'use strict'; -// var assert = require('assert'); -// var async = require('asyncawait/async'); -// var await = require('asyncawait/await'); -// var List = require('../src/list/List'); -// var Timer = require('../examples/Timer'); +// const assert = require('assert'); +// const async = require('asyncawait/async'); +// const await = require('asyncawait/await'); +// const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +// const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +// const List = require('../src/list/List'); +// const OrbitList = require('../src/list/OrbitList'); +// const Timer = require('../examples/Timer'); + // describe('List - Performance Measurement', function() { // this.timeout(60000); @@ -12,7 +16,7 @@ // it('add', (done) => { // let ms = 0; -// for(let t = 1000; t <= 10000; t += 1000) { +// for(let t = 1000; t <= 5000; t += 1000) { // const list = new List('A'); // let timer = new Timer(true); @@ -29,3 +33,41 @@ // }); // }); + +// describe('OrbitList - Performance Measurement', function() { +// const startIpfs = async (() => { +// return new Promise(async((resolve, reject) => { +// const ipfsd = await(ipfsDaemon()); +// resolve(ipfsd.daemon); +// })); +// }); + +// let ipfs; + +// this.timeout(60000); + +// before(async((done) => { +// ipfs = await(startIpfs()); +// done(); +// })); + +// it('add', async((done) => { +// let ms = 0; + +// for(let t = 100; t <= 1000; t += 300) { +// const list = new OrbitList('A', ipfs); +// let timer = new Timer(true); + +// for(let i = 0; i < t; i ++) { +// list.add("hello" + i); +// } + +// ms = timer.stop(true); +// console.log(` > ${t} took ${ms} ms`) +// } + +// assert.equal(true, true); +// done(); +// })); + +// }); diff --git a/test/list-tests.js b/test/list-tests.js index 46bfc46..9b08f28 100644 --- a/test/list-tests.js +++ b/test/list-tests.js @@ -25,7 +25,7 @@ describe('List', () => { list.add("hello1") list.add("hello2") list.add("hello3") - const str = JSON.stringify(list.toJson()) + const str = JSON.stringify(list.toJson(), null, 2) const res = List.fromJson(JSON.parse(str)); assert.equal(res.id, 'A'); assert.equal(res.seq, 0); @@ -89,36 +89,6 @@ describe('List', () => { }); }); - describe('toJson', () => { - it('presents the list in a compacted json form', (done) => { - const list = new List('A'); - list.add("hello1") - list.add("hello2") - list.add("hello3") - - let compacted = list.toJson(); - assert.equal(compacted.id, 'A'); - assert.equal(compacted.seq, 0); - assert.equal(compacted.ver, 3); - assert.equal(compacted.items.length, 3); - assert.equal(compacted.items[0].id, 'A'); - assert.equal(compacted.items[0].seq, 0); - assert.equal(compacted.items[0].ver, 0); - assert.equal(compacted.items[0].next.length, 0); - assert.equal(compacted.items[compacted.items.length - 1].id, 'A'); - assert.equal(compacted.items[compacted.items.length - 1].seq, 0); - assert.equal(compacted.items[compacted.items.length - 1].ver, 2); - assert.equal(compacted.items[compacted.items.length - 1].next.length, 1); - assert.equal(compacted.items[compacted.items.length - 1].next[0], 'A.0.1'); - assert.equal(compacted.items[compacted.items.length - 2].id, 'A'); - assert.equal(compacted.items[compacted.items.length - 2].seq, 0); - assert.equal(compacted.items[compacted.items.length - 2].ver, 1); - assert.equal(compacted.items[compacted.items.length - 2].next.length, 1); - assert.equal(compacted.items[compacted.items.length - 2].next[0], 'A.0.0'); - done(); - }); - }); - describe('add', () => { it('adds an item to an empty list', (done) => { const list = new List('A'); @@ -483,7 +453,6 @@ describe('List', () => { assert.equal(lastItem2.data, 'helloD4'); done(); }); - }); describe('_findHeads', () => { diff --git a/test/orbit-list-tests.js b/test/orbit-list-tests.js new file mode 100644 index 0000000..a42a923 --- /dev/null +++ b/test/orbit-list-tests.js @@ -0,0 +1,569 @@ +'use strict'; + +const _ = require('lodash'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const assert = require('assert'); +const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const List = require('../src/list/OrbitList'); + +const startIpfs = async (() => { + return new Promise(async((resolve, reject) => { + const ipfsd = await(ipfsDaemon()); + resolve(ipfsd.daemon); + })); +}); + +let ipfs; + +describe('OrbitList', async(function() { + this.timeout(5000); + + before(async((done) => { + ipfs = await(startIpfs()); + done(); + })); + + describe('Constructor', async(() => { + it('initializes member variables', async((done) => { + const list = new List('A', ipfs); + assert.equal(list.id, 'A'); + assert.equal(list.seq, 0); + assert.equal(list.ver, 0); + assert.equal(list._items instanceof Array, true); + assert.equal(list._currentBatch instanceof Array, true); + assert.equal(list._items.length, 0); + assert.equal(list._currentBatch.length, 0); + assert.equal(list._ipfs, ipfs); + assert.equal(list.hash, null); + done(); + })); + })); + + describe('add', async(() => { + it('saves the data to ipfs', async((done) => { + const list = new List('A', ipfs); + const text = 'testing 1 2 3 4'; + list.add(text) + const hash = await(list.getIpfsHash()); + assert.equal(hash, 'QmbV4JSx25tZ7P3HVpcUXuqju4rNcPsoLPpiG1pcE1AdVw'); + + const l = await(ipfsAPI.getObject(ipfs, hash)); + const list2 = List.fromJson(ipfs, JSON.parse(l.Data)); + assert.equal(list2.items[0].data, text); + + done(); + })); + + it('updates the data to ipfs', async((done) => { + const list = new List('A', ipfs); + const text1 = 'testing 1 2 3'; + const text2 = 'testing 456'; + let hash; + + list.add(text1) + + hash = await(list.getIpfsHash()); + assert.equal(hash, 'QmcBjB93PsJGz2LrVy5e1Z8mtwH99B8yynsa5f4q3GanEe'); + + list.add(text2) + hash = await(list.getIpfsHash()); + assert.equal(hash, 'Qmf358H1wjuX3Bbaag4SSEiujoruowVUNR5pLCNQs8vivP'); + + const l = await(ipfsAPI.getObject(ipfs, hash)); + const list2 = List.fromJson(ipfs, JSON.parse(l.Data)); + assert.equal(list2.items[0].data, text1); + assert.equal(list2.items[1].data, text2); + + done(); + })); + })); + + describe('getIpfsHash', async(() => { + it('returns the list as ipfs hash', async((done) => { + const list = new List('A', ipfs); + const hash = await(list.getIpfsHash()); + assert.equal(hash, 'QmVkddks6YBH88TqJf7nFHdyb9PjebPmJAxaRvWdu8ueoE'); + done(); + })); + + it('saves the list to ipfs', async((done) => { + const list = new List('A', ipfs); + const hash = await(list.getIpfsHash()); + const l = await(ipfsAPI.getObject(ipfs, hash)); + assert.equal(l.toString(), ({ Links: [], Data: '{"id":"A","seq":0,"ver":0,"items":[]}' }).toString()); + done(); + })); + })); + + describe('fromJson', () => { + it('creates a list from parsed json', async((done) => { + const list = new List('A', ipfs); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const str = JSON.stringify(list.toJson(), null, 2) + const res = List.fromJson(ipfs, JSON.parse(str)); + assert.equal(res.id, 'A'); + assert.equal(res.seq, 0); + assert.equal(res.ver, 3); + assert.equal(res.items.length, 3); + assert.equal(res.items[0].compactId, 'A.0.0.QmZfdeMV77si491NPX83Q8eRYE9WNzVorHrfWJPrJ51brt'); + assert.equal(res.items[1].compactId, 'A.0.1.QmbbtEWe4qHLSjtW2HkPuszFW3zfBTXBdPrkXMdbePxqfK'); + assert.equal(res.items[2].compactId, 'A.0.2.QmT6wQwBZsH6b3jQVxmM5L7kqV39nr3F99yd5tN6nviQPe'); + done(); + })); + }); + + describe('fromIpfsHash', () => { + it('creates a list from ipfs hash', async((done) => { + const list = new List('A', ipfs); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const hash = await(list.getIpfsHash()); + assert.equal(hash, 'QmThvyS6FUsHvT7oC2pGNMTAdhjUncNsVMbXAkUB72J8n1'); + const res = await(List.fromIpfsHash(ipfs, hash)); + assert.equal(res.id, 'A'); + assert.equal(res.seq, 0); + assert.equal(res.ver, 3); + assert.equal(res.items.length, 3); + assert.equal(res.items[0].compactId, 'A.0.0.QmZfdeMV77si491NPX83Q8eRYE9WNzVorHrfWJPrJ51brt'); + assert.equal(res.items[1].compactId, 'A.0.1.QmbbtEWe4qHLSjtW2HkPuszFW3zfBTXBdPrkXMdbePxqfK'); + assert.equal(res.items[2].compactId, 'A.0.2.QmT6wQwBZsH6b3jQVxmM5L7kqV39nr3F99yd5tN6nviQPe'); + done(); + })); + }); + + describe('toJson', async(() => { + it('presents the list as json', async((done) => { + const list = new List('A', ipfs); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const json = list.toJson(); + const expected = { + id: 'A', + seq: 0, + ver: 3, + items: [ + { id: 'A', seq: 0, ver: 0, data: 'hello1', next: [] }, + { id: 'A', seq: 0, ver: 1, data: 'hello2', next: ['A.0.0.QmZfdeMV77si491NPX83Q8eRYE9WNzVorHrfWJPrJ51brt'] }, + { id: 'A', seq: 0, ver: 2, data: 'hello3', next: ['A.0.1.QmbbtEWe4qHLSjtW2HkPuszFW3zfBTXBdPrkXMdbePxqfK'] } + ] + }; + // console.log(JSON.stringify(json, null, 1)) + assert.equal(_.isEqual(json, expected), true); + done(); + })); + })); + + describe('toString', () => { + it('presents the list as a string', async((done) => { + const list = new List('A', ipfs); + list.add("hello1") + list.add("hello2") + list.add("hello3") + const str = list.toString(); + const expected = `id: A, seq: 0, ver: 3, items:\n{"id":"A","seq":0,"ver":0,"data":"hello1","next":[]}\n{"id":"A","seq":0,"ver":1,"data":"hello2","next":["A.0.0.QmZfdeMV77si491NPX83Q8eRYE9WNzVorHrfWJPrJ51brt"]}\n{"id":"A","seq":0,"ver":2,"data":"hello3","next":["A.0.1.QmbbtEWe4qHLSjtW2HkPuszFW3zfBTXBdPrkXMdbePxqfK"]}`; + assert.equal(str, expected); + done(); + })); + }); + + describe('items', () => { + it('returns items', async((done) => { + const list = new List('A', ipfs); + let items = list.items; + assert.equal(list.items instanceof Array, true); + assert.equal(list.items.length, 0); + list.add("hello1") + list.add("hello2") + assert.equal(list.items instanceof Array, true); + assert.equal(list.items.length, 2); + assert.equal(list.items[0].data, 'hello1'); + assert.equal(list.items[1].data, 'hello2'); + done(); + })); + }); + + describe('add', () => { + it('adds an item to an empty list', async((done) => { + const list = new List('A', ipfs); + list.add("hello1") + const item = list.items[0]; + assert.equal(list.id, 'A'); + assert.equal(list.seq, 0); + assert.equal(list.ver, 1); + assert.equal(list.items.length, 1); + assert.equal(list._currentBatch.length, 1); + assert.equal(list._items.length, 0); + assert.equal(item, list._currentBatch[0]); + assert.equal(item.id, 'A'); + assert.equal(item.seq, 0); + assert.equal(item.ver, 0); + assert.equal(item.data, 'hello1'); + done(); + })); + + it('adds 100 items to a list', async((done) => { + const list = new List('A', ipfs); + + for(let i = 1; i < 101; i ++) { + list.add("hello" + i); + } + + assert.equal(list.id, 'A'); + assert.equal(list.seq, 0); + assert.equal(list.ver, 100); + assert.equal(list.items.length, 100); + assert.equal(list._currentBatch.length, 100); + assert.equal(list._items.length, 0); + + const item = list.items[list.items.length - 1]; + assert.equal(item, list._currentBatch[list._currentBatch.length - 1]); + assert.equal(item.id, 'A'); + assert.equal(item.seq, 0); + assert.equal(item.ver, 99); + assert.equal(item.data, 'hello100'); + assert.equal(item.next, 'A.0.98.QmPZ1Qmf52ko62xh9RDYcVGNMWx8ZCtfFNyrvqyE1UmhG1'); + + done(); + })); + }); + + describe('join', () => { + it('increases the sequence and resets the version if other list has the same or higher sequence', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + + list2.seq = 7; + list1.add("helloA1") + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 0); + assert.equal(list1.ver, 1); + + list2.add("helloB1") + list1.join(list2); + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 8); + assert.equal(list1.ver, 0); + done(); + })); + + it('increases the sequence by one if other list has lower sequence', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + list1.seq = 4; + list2.seq = 1; + list2.add("helloB1") + list1.join(list2); + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 5); + assert.equal(list1.ver, 0); + done(); + })); + + it('finds the next head when adding a new element', async((done) => { + const list1 = new List('A', ipfs); + list1.add("helloA1") + list1.add("helloA2") + list1.add("helloA3") + + assert.equal(list1._currentBatch.length, 3); + assert.equal(list1._currentBatch[2].next.length, 1); + assert.equal(list1._currentBatch[2].next[0], 'A.0.1.QmW3cnX41CNSAEkZE23w4qMRcsAY8MEUtsCT4wZmRZfQ76'); + done(); + })); + + it('finds the next heads (two) after a join', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + list1.add("helloA1") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + list1.add("helloA2") + + assert.equal(list1._currentBatch.length, 1); + assert.equal(list1._currentBatch[0].next.length, 2); + assert.equal(list1._currentBatch[0].next[0], 'A.0.0.QmaHqKY1GUJTKGF6KA3QLoDaD3TS7oa6wHGTAxY6sVLKD9'); + assert.equal(list1._currentBatch[0].next[1], 'B.0.1.QmbsBfrDfqtTbaPNzuF8KNR1jbK74LwMe4UM2G6DgN6zmQ'); + done(); + })); + + it('finds the next head (one) after a join', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + list1.add("helloA1") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + list1.add("helloA2") + list1.add("helloA3") + + assert.equal(list1._currentBatch.length, 2); + assert.equal(list1._currentBatch[1].next.length, 1); + assert.equal(list1._currentBatch[1].next[0], 'A.1.0.QmPxBabxGovTzTphiwoiEDCRnTGYwqZ7M7jahVVctbaJdF'); + done(); + })); + + it('finds the next heads after two joins', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + list1.add("helloA1") + list1.add("helloA2") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + list1.add("helloA3") + + list1.join(list2); + + list1.add("helloA4") + list1.add("helloA5") + + const lastItem = list1.items[list1.items.length - 1]; + + assert.equal(list1.items.length, 7); + assert.equal(lastItem.next.length, 1); + assert.equal(lastItem.next[0], 'A.2.0.QmTpRBszPFnxtuKccYJ4YShQoeYm2caeFhmMVBfiY1u7Jc'); + done(); + })); + + it('finds the next heads after multiple joins', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + const list3 = new List('C', ipfs); + const list4 = new List('D', ipfs); + list1.add("helloA1") + list1.add("helloA2") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + list3.add("helloC1") + list4.add("helloD1") + list1.join(list3); + + list1.add("helloA3") + list2.join(list1); + list1.join(list2); + list2.join(list4); + + list4.add("helloD2") + list4.add("helloD3") + list1.add("helloA4") + list1.join(list4); + + list1.add("helloA5") + + const lastItem = list1.items[list1.items.length - 1]; + + assert.equal(list1.items.length, 11); + assert.equal(lastItem.next.length, 2); + assert.equal(lastItem.next[0], 'A.4.0.Qmb7oeViDbsKTDNo7HAueFn47z3pon2fVptXNdXhcAigFz'); + assert.equal(lastItem.next[1], 'D.0.2.QmajSkuVj64RLy8YGVPqkDb4V52FjqDsvbGhJsLmkQLxsL'); + done(); + })); + + it('joins list of one item with list of two items', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + list1.add("helloA1") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + + const lastItem = list1.items[list1.items.length - 1]; + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 1); + assert.equal(list1.ver, 0); + assert.equal(list1._currentBatch.length, 0); + assert.equal(list1._items.length, 3); + assert.equal(lastItem.id, 'B'); + assert.equal(lastItem.seq, 0); + assert.equal(lastItem.ver, 1); + assert.equal(lastItem.data, 'helloB2'); + done(); + })); + + it('joins lists two ways', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + list1.add("helloA1") + list1.add("helloA2") + list2.add("helloB1") + list2.add("helloB2") + list1.join(list2); + list2.join(list1); + + const lastItem1 = list1.items[list1.items.length - 1]; + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 1); + assert.equal(list1.ver, 0); + assert.equal(list1._currentBatch.length, 0); + assert.equal(list1._items.length, 4); + assert.equal(lastItem1.id, 'B'); + assert.equal(lastItem1.seq, 0); + assert.equal(lastItem1.ver, 1); + assert.equal(lastItem1.data, 'helloB2'); + + const lastItem2 = list2.items[list2.items.length - 1]; + + assert.equal(list2.id, 'B'); + assert.equal(list2.seq, 2); + assert.equal(list2.ver, 0); + assert.equal(list2._currentBatch.length, 0); + assert.equal(list2._items.length, 4); + assert.equal(lastItem2.id, 'A'); + assert.equal(lastItem2.seq, 0); + assert.equal(lastItem2.ver, 1); + assert.equal(lastItem2.data, 'helloA2'); + done(); + })); + + it('joins lists twice', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + + list1.add("helloA1") + list2.add("helloB1") + list2.join(list1); + + list1.add("helloA2") + list2.add("helloB2") + list2.join(list1); + + const secondItem = list2.items[1]; + const lastItem = list2.items[list2.items.length - 1]; + + assert.equal(list2.id, 'B'); + assert.equal(list2.seq, 2); + assert.equal(list2.ver, 0); + assert.equal(list2._currentBatch.length, 0); + assert.equal(list2._items.length, 4); + assert.equal(secondItem.id, 'A'); + assert.equal(secondItem.seq, 0); + assert.equal(secondItem.ver, 0); + assert.equal(secondItem.data, 'helloA1'); + assert.equal(lastItem.id, 'A'); + assert.equal(lastItem.seq, 0); + assert.equal(lastItem.ver, 1); + assert.equal(lastItem.data, 'helloA2'); + done(); + })); + + it('joins 4 lists to one', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + const list3 = new List('C', ipfs); + const list4 = new List('D', ipfs); + + list1.add("helloA1") + list2.add("helloB1") + list1.add("helloA2") + list2.add("helloB2") + list3.add("helloC1") + list4.add("helloD1") + list3.add("helloC2") + list4.add("helloD2") + list1.join(list2); + list1.join(list3); + list1.join(list4); + + const secondItem = list1.items[1]; + const lastItem = list1.items[list1.items.length - 1]; + + assert.equal(list1.id, 'A'); + assert.equal(list1.seq, 3); + assert.equal(list1.ver, 0); + assert.equal(list1._currentBatch.length, 0); + assert.equal(list1._items.length, 8); + assert.equal(secondItem.id, 'A'); + assert.equal(secondItem.seq, 0); + assert.equal(secondItem.ver, 1); + assert.equal(secondItem.data, 'helloA2'); + assert.equal(lastItem.id, 'D'); + assert.equal(lastItem.seq, 0); + assert.equal(lastItem.ver, 1); + assert.equal(lastItem.data, 'helloD2'); + done(); + })); + + it('joins lists from 4 lists', async((done) => { + const list1 = new List('A', ipfs); + const list2 = new List('B', ipfs); + const list3 = new List('C', ipfs); + const list4 = new List('D', ipfs); + + list1.add("helloA1") + list1.join(list2); + list2.add("helloB1") + list2.join(list1); + + list1.add("helloA2") + list2.add("helloB2") + list1.join(list3); + list3.join(list1); + + list3.add("helloC1") + list4.add("helloD1") + + list3.add("helloC2") + list4.add("helloD2") + + list1.join(list3); + list1.join(list2); + list4.join(list2); + list4.join(list1); + list4.join(list3); + + list4.add("helloD3") + list4.add("helloD4") + + const secondItem = list4.items[1]; + const lastItem1 = list4._items[list4._items.length - 1]; + const lastItem2 = list4.items[list4.items.length - 1]; + + assert.equal(list4.id, 'D'); + assert.equal(list4.seq, 7); + assert.equal(list4.ver, 2); + assert.equal(list4._currentBatch.length, 2); + assert.equal(list4._items.length, 8); + assert.equal(secondItem.id, 'D'); + assert.equal(secondItem.seq, 0); + assert.equal(secondItem.ver, 1); + assert.equal(secondItem.data, 'helloD2'); + assert.equal(lastItem1.id, 'C'); + assert.equal(lastItem1.seq, 3); + assert.equal(lastItem1.ver, 1); + assert.equal(lastItem1.data, 'helloC2'); + assert.equal(lastItem2.id, 'D'); + assert.equal(lastItem2.seq, 7); + assert.equal(lastItem2.ver, 1); + assert.equal(lastItem2.data, 'helloD4'); + done(); + })); + }); + + describe('_findHeads', () => { + it('TODO', (done) => { + done(); + }); + }); + + describe('_isReferencedInChain', () => { + it('TODO', (done) => { + done(); + }); + }); + +})); diff --git a/test/orbitlist-node-tests.js b/test/orbitlist-node-tests.js new file mode 100644 index 0000000..10a491e --- /dev/null +++ b/test/orbitlist-node-tests.js @@ -0,0 +1,63 @@ +'use strict'; + +const _ = require('lodash'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const assert = require('assert'); +const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const Node = require('../src/list/OrbitNode'); + +const startIpfs = async (() => { + return new Promise(async((resolve, reject) => { + const ipfsd = await(ipfsDaemon()); + resolve(ipfsd.daemon); + })); +}); + +let ipfs; + +describe('OrbitNode', function() { + this.timeout(10000); + + before(async((done) => { + ipfs = await(startIpfs()); + done(); + })); + + describe('Constructor', () => { + it('initializes member variables', async((done) => { + const node = new Node(ipfs, 'A', 0, 0); + assert.equal(node.id, 'A'); + assert.equal(node.seq, 0); + assert.equal(node.ver, 0); + assert.equal(node.data, null); + assert.equal(node.next instanceof Array, true); + assert.equal(node.hash, null); + assert.equal(node._ipfs, ipfs); + done(); + })); + + it('initializes member variables with data', async((done) => { + const node = new Node(ipfs, 'A', 0, 0, 'QmTnaGEpw4totXN7rhv2jPMXKfL8s65PhhCKL5pwtJfRxn'); + assert.equal(node.id, 'A'); + assert.equal(node.seq, 0); + assert.equal(node.ver, 0); + assert.equal(node.data, 'QmTnaGEpw4totXN7rhv2jPMXKfL8s65PhhCKL5pwtJfRxn'); + assert.equal(node.next instanceof Array, true); + assert.equal(node.hash, null); + assert.equal(node._ipfs, ipfs); + done(); + })); + }); + + describe('compactId', () => { + it('presents the node as a string with id, sequence, version and hash', async((done) => { + const node1 = new Node(ipfs, 'A', 0, 0, "QmTnaGEpw4totXN7rhv2jPMXKfL8s65PhhCKL5pwtJfRxn"); + const node2 = new Node(ipfs, 'B', 123, 456, "QmdcCucbM2rnHHaVhAmjMxWDY5cCDwtTtjhYuS5nBHThQq"); + assert.equal(node1.compactId, 'A.0.0.QmcfXxBTpZGmWnYVUiPTpW4Uaf9e1x34Qh9vthvuAjmhTb'); + assert.equal(node2.compactId, 'B.123.456.QmWCVngHttRQQhrmgr94GZzY5F57m3g6fDdDwK9mgHFRn2'); + done(); + })); + }); +}); diff --git a/test1.js b/test1.js index 1cff702..da3caf9 100644 --- a/test1.js +++ b/test1.js @@ -3,7 +3,6 @@ const _ = require('lodash'); const Timer = require('./examples/Timer'); const List = require('./src/list/List'); -// const Node = require('./src/list/Node'); var run = () => { var redis = require("redis"); diff --git a/test2.js b/test2.js new file mode 100644 index 0000000..5bb9322 --- /dev/null +++ b/test2.js @@ -0,0 +1,92 @@ +'use strict'; + +const _ = require('lodash'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const List = require('./src/list/OrbitList'); +const Timer = require('./examples/Timer'); + +const startIpfs = async (() => { + return new Promise(async((resolve, reject) => { + const ipfsd = await(ipfsDaemon()); + resolve(ipfsd.daemon); + })); +}); + +let ipfs; + +var run = async(() => { + ipfs = await(startIpfs()); + + var redis = require("redis"); + this.client1 = redis.createClient({ host: "localhost", port: 6379 }); + this.client2 = redis.createClient({ host: "localhost", port: 6379 }); + var hash = "ccc" + this.client1.subscribe(hash); + this.client1.subscribe(hash); + + + let listA = new List("A", ipfs); + let listB = new List("B", ipfs); + let listC = new List("C", ipfs); + + const handleMessage = async((hash, event) => { + // const l = List.fromJson(JSON.parse(event)); + console.log(">", event); + const l = await(List.fromIpfsHash(ipfs, event)); + // console.log("ITEMS RECEIVED", l.items.length); + + if(l.id === 'A') { + listB.join(l); + listC.join(l); + } else if(l.id === 'B') { + // listA.join(l); + // listC.join(l); + } else if(l.id === 'C') { + listB.join(l); + var timer = new Timer('a'); + listC.join(listB); + console.log("join took " + timer.stop(true) + " ms"); + console.log("Items:", listC.items.length); + // console.log(listC.toString()); + } + }); + + this.client1.on("message", handleMessage); + this.client2.on("message", handleMessage); + + let h = 0; + setInterval(async(() => { + listC.add("C--"+h); + const list = await(listC.getIpfsHash()); + this.client2.publish(hash, list); + h++; + }), 1000); + + let i = 0; + setInterval(async(() => { + let a = 0; + // for(let a = 0; a < 10; a ++) { + listB.add("B--"+(i+a)); + // } + const list = await(listB.getIpfsHash()); + this.client2.publish(hash, list); + i++; + }), 50); + +// let k = 0; +// setInterval(async(() => { +// listA.add("A--"+k); +// k++; +// listA.add("A--"+k); +// k++; +// listA.add("A--"+k); +// k++; +// this.client2.publish(hash, JSON.stringify(listA.toJson())); +// }), 100); +// }); +}); + +run(); From 8caf8cbb9a615a7be9b1cdd7ea07c07d8b932baf Mon Sep 17 00:00:00 2001 From: haad Date: Sun, 21 Feb 2016 17:52:20 +0200 Subject: [PATCH 19/30] Refactor OrbitClient and Pubsub to use Lists --- examples/pubsubKeyValue.js | 52 ++ examples/pubsubReader.js | 10 +- examples/pubsubWriter.js | 4 +- src/DataStore.js | 102 +++ src/HashCacheItem.js | 10 + src/OrbitClient.js | 145 ++--- src/PubSub.js | 140 +---- src/list/OrbitList.js | 21 +- src/list/OrbitNode.js | 21 +- test/orbit-client-tests.js | 1192 ++++++++++++++++++------------------ test/orbit-list-tests.js | 6 +- 11 files changed, 879 insertions(+), 824 deletions(-) create mode 100644 examples/pubsubKeyValue.js create mode 100644 src/DataStore.js diff --git a/examples/pubsubKeyValue.js b/examples/pubsubKeyValue.js new file mode 100644 index 0000000..746daf9 --- /dev/null +++ b/examples/pubsubKeyValue.js @@ -0,0 +1,52 @@ +'use strict'; + +var async = require('asyncawait/async'); +var await = require('asyncawait/await'); +var OrbitClient = require('../src/OrbitClient'); +var Timer = require('./Timer'); + +// var host = '178.62.229.175'; +var host = 'localhost'; +var port = 6379; + +var username = 'LambOfGod'; +var password = ''; + +let run = (async(() => { + try { + var orbit = OrbitClient.connect(host, port, username, password); + const c1 = 'c1'; + const channel = orbit.channel(c1); + + let count = 1; + let id = 'Log: Query ' + let running = false; + + setInterval(async(() => { + if(!running) { + running = true; + + // let timer = new Timer(true); + channel.put("lamb", "of god" + count); + // console.log(`Query #${count} took ${timer.stop(true)} ms\n`); + let v = channel.get("lamb"); + + console.log("---------------------------------------------------") + console.log("Id | Seq | Ver | Data") + console.log("---------------------------------------------------") + console.log(v); + console.log("---------------------------------------------------") + + running = false; + count ++; + } + }), 500); + + } catch(e) { + console.error("error:", e); + console.error(e.stack); + process.exit(1); + } +}))(); + +module.exports = run; diff --git a/examples/pubsubReader.js b/examples/pubsubReader.js index d83641b..3653ba4 100644 --- a/examples/pubsubReader.js +++ b/examples/pubsubReader.js @@ -27,16 +27,18 @@ let run = (async(() => { running = true; // let timer = new Timer(true); - channel.add(id + count); + channel.add("Hello " + count); // console.log(`Query #${count} took ${timer.stop(true)} ms\n`); - // console.log("Query..."); const c = channel.iterator({ limit: -1 }).collect().length; let items = channel.iterator({ limit: 5 }).collect(); + // console.log(items); console.log("---------------------------------------------------") - console.log("Id | Seq | Ver | Data") + // console.log("Id | Seq | Ver | Data") + console.log("Key | Value") console.log("---------------------------------------------------") - console.log(items.map((e) => `${e.id} | ${e.seq} | ${e.ver} | ${e.data}`).join("\n")); + // console.log(items.map((e) => `${e.id} | ${e.seq} | ${e.ver} | ${e.data}`).join("\n")); + console.log(items.map((e) => `${e.payload.key} | ${e.payload.value}`).join("\n")); console.log("---------------------------------------------------") console.log(`Found ${items.length} items from ${c}\n`); diff --git a/examples/pubsubWriter.js b/examples/pubsubWriter.js index 067234a..6270e0c 100644 --- a/examples/pubsubWriter.js +++ b/examples/pubsubWriter.js @@ -16,14 +16,14 @@ let run = (async(() => { try { var orbit = OrbitClient.connect(host, port, username, password); const c1 = 'c1'; + let channel; let count = 1; let id = 'Log: Query ' - let channel; setInterval(async(() => { if(channel) { - channel.add(id + count); + channel.add(username + " " + count); count ++; } }), process.argv[3] ? process.argv[3] : 1000); diff --git a/src/DataStore.js b/src/DataStore.js new file mode 100644 index 0000000..28ccacf --- /dev/null +++ b/src/DataStore.js @@ -0,0 +1,102 @@ +'use strict'; + +const _ = require('lodash'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const OrbitList = require('./list/OrbitList'); +const HashCacheOps = require('./HashCacheOps'); + +const DefaultAmount = 1; + +class DataStore { + constructor(id, ipfs) { + this._ipfs = ipfs; + this.list = new OrbitList(id, this._ipfs); + } + + add(hash) { + this.list.add(hash); + } + + join(other) { + this.list.join(other); + } + + clear() { + this.list.clear(); + } + + get(options) { + return this._fetchRecursive(options); + } + + _fetchOne(index) { + const item = this.list.items[this.list.items.length - index - 1]; + if(item) { + await(item.getPayload()); + const f = item.compact(); + return { hash: f.data, payload: f.Payload }; + } + return null; + } + + _fetchRecursive(options, currentAmount, deleted, res) { + // console.log("-->") + // console.log("opts:", options, currentAmount) + const opts = { + amount: options && options.amount ? options.amount : DefaultAmount, + first: options && options.first ? options.first : null, + last: options && options.last ? options.last : null, + key: options && options.key ? options.key : null + }; + + let result = res ? res : []; + let handledItems = deleted ? deleted : []; + + if(!currentAmount) currentAmount = 0; + + const item = this._fetchOne(currentAmount); + // console.log("ITEM", item) + + if(item && item.payload) { + const wasHandled = _.includes(handledItems, item.payload.key); + if((item.payload.op === HashCacheOps.Put || item.payload.op === HashCacheOps.Add) && !wasHandled) { + if((!opts.key || (opts.key && opts.key === item.payload.key)) && + (!opts.first || (opts.first && (opts.first === item.payload.key && result.length === 0)) + || (opts.first && (opts.first !== item.payload.key && result.length > 0)))) + { + // console.log("PUSH!", item, currentAmount, result.length); + result.push(item); + handledItems.push(item.payload.key); + } + } else if(item.payload.op === HashCacheOps.Delete) { + // console.log("DELETE!", item); + handledItems.push(item.payload.key); + } + + currentAmount ++; + + if(opts.key && item.payload.key === opts.key) + return result; + + // console.log("ITEM", item.payload.key, opts.last) + if(opts.last && item.payload.key === opts.last) + return result; + + if(!opts.last && opts.amount > -1 && result.length >= opts.amount) + return result; + + if(currentAmount >= this.list.items.length) + return result; + + // console.log("RES!", result) + result = this._fetchRecursive(opts, currentAmount, handledItems, result); + } + + return result; + + } +} + +module.exports = DataStore; diff --git a/src/HashCacheItem.js b/src/HashCacheItem.js index d4c63e1..838091f 100644 --- a/src/HashCacheItem.js +++ b/src/HashCacheItem.js @@ -2,6 +2,15 @@ const Encryption = require('orbit-common/lib/Encryption'); +class OrbitDBItem { + constructor(operation, key, value, metaInfo) { + this.op = operation; + this.key = key; + this.value = value; + this.meta = metaInfo; + } +} + class HashCacheItem { constructor(operation, key, sequenceNumber, targetHash, metaInfo, next) { this.op = operation; @@ -55,6 +64,7 @@ class EncryptedHashCacheItem extends HashCacheItem { } module.exports = { + OrbitDBItem: OrbitDBItem, HashCacheItem: HashCacheItem, EncryptedHashCacheItem: EncryptedHashCacheItem }; diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 689db56..ee7470a 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -6,56 +6,54 @@ var Keystore = require('orbit-common/lib/Keystore'); var Encryption = require('orbit-common/lib/Encryption'); var ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); var ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); -var HashCache = require('./HashCacheClient'); -var HashCacheItem = require('./HashCacheItem').EncryptedHashCacheItem; +var HashCacheItem = require('./HashCacheItem').HashCacheItem; +var OrbitDBItem = require('./HashCacheItem').OrbitDBItem; var HashCacheOps = require('./HashCacheOps'); var ItemTypes = require('./ItemTypes'); var MetaInfo = require('./MetaInfo'); var Post = require('./Post'); var Aggregator = require('./Aggregator'); var PubSub = require('./PubSub'); -const List = require('./list/OrbitList'); var Timer = require('../examples/Timer'); +const List = require('./list/OrbitList'); +const DataStore = require('./DataStore'); var pubkey = Keystore.getKeys().publicKey; var privkey = Keystore.getKeys().privateKey; -let vvv = {}; - class OrbitClient { constructor(ipfs) { - this.ipfs = ipfs; + this._ipfs = ipfs; this.network = {}; this.user = null; - this.list = null; // TODO move to DataStore } channel(hash, password) { if(password === undefined) password = ''; this._pubsub.subscribe(hash, password, async((hash, message) => { - const other = await(List.fromIpfsHash(this.ipfs, message)); + const other = await(List.fromIpfsHash(this._ipfs, message)); // console.log(">", other.id, other.seq, other.ver); if(other.id !== this.user.username) { - let timer = new Timer(true); - this.list.join(other); // TODO: move to DataStore - console.log(`Join took ${timer.stop(true)} ms`); + // let timer = new Timer(true); + this._store.join(other); + // console.log(`Join took ${timer.stop(true)} ms`); } })); return { - info: (options) => this._info(hash, password), + // info: (options) => this._info(hash, password), delete: () => this._deleteChannel(hash, password), iterator: (options) => this._iterator(hash, password, options), setMode: (mode) => this._setMode(hash, password, mode), add: (data) => this._add(hash, password, data), - //TODO: tests - del: (options) => this._remove(hash, password, options), + del: (key) => this._remove(hash, password, key), put: (key, data) => this._put(hash, password, key, data), get: (key, options) => { const items = this._iterator(hash, password, { key: key }).collect(); - return items[0] ? items[0].item.Payload : null; + return items[0] ? items[0].payload.value : null; }, + //TODO: tests leave: () => this._pubsub.unsubscribe(hash) } } @@ -95,72 +93,45 @@ class OrbitClient { const reverse = options.reverse ? options.reverse : false; const key = options.key ? options.key : null; - let startFromHash; - if(lte || lt) { - startFromHash = lte ? lte : lt; - } else { - var channel = this._info(channel, password); - startFromHash = channel.head ? channel.head : null; - } - if((gt || lt) && limit > -1) limit += 1; - if(startFromHash) { - const opts = { - amount: limit, - last: gte ? gte : gt, - key: key - }; + const opts = { + amount: limit, + first: lte ? lte : lt, + last: gte ? gte : gt, + key: key + }; - // Get messages - // messages = Aggregator.fetchRecursive(this.ipfs, startFromHash, password, opts); - messages = this.list.items.map((f) => f.compact()); // TODO: move to DataStore + // Get messages + messages = await(this._store.get(opts)); + // console.log("M", messages) - // Slice the array - let startIndex = 0; - let endIndex = messages.length; - if(limit < 0) { - endIndex = messages.length - (gt ? 1 : 0); - } else { - startIndex = Math.max(0, messages.length - limit); - endIndex = messages.length - ((gt || lt) ? 1 : 0); - } + // Remove the first/last item if greater/lesser than is set + let startIndex = lt ? 1 : 0; + let endIndex = gt ? messages.length - 1 : messages.length; + messages = messages.slice(startIndex, endIndex) + // console.log("M2", messages) - messages = messages.slice(startIndex, endIndex) - } - - if(reverse) messages.reverse(); + if(!reverse) messages.reverse(); return messages; } _publish(data) { let post = new Post(data); - post.encrypt(privkey, pubkey); - return await (ipfsAPI.putObject(this.ipfs, JSON.stringify(post))); + // post.encrypt(privkey, pubkey); + return await (ipfsAPI.putObject(this._ipfs, JSON.stringify(post))); } - _createMessage(channel, password, operation, key, target) { + _createMessage(channel, password, operation, key, value) { // Create meta info const size = -1; const metaInfo = new MetaInfo(ItemTypes.Message, size, this.user.id, new Date().getTime()); - - // Get the current channel head and bump the sequence number - let seq = this._info(channel, password).seq + 1; - let head = this._info(channel, password).head; - // Create the hash cache item - const hcItem = new HashCacheItem(operation, key, seq, target, metaInfo, null, pubkey, privkey, password); - + const item = new OrbitDBItem(operation, key, value, metaInfo); // Save the item to ipfs - const data = await (ipfsAPI.putObject(this.ipfs, JSON.stringify(hcItem))); - let newHead = { Hash: data.Hash }; - - // If this is not the first item in the channel, patch with the previous (ie. link as next) - if(seq > 0) - newHead = await (ipfsAPI.patchObject(this.ipfs, data.Hash, head)); - - return { hash: newHead, seq: seq }; + const data = await (ipfsAPI.putObject(this._ipfs, JSON.stringify(item))); + return data.Hash; } /* DB Operations */ @@ -175,48 +146,46 @@ class OrbitClient { return await(this._createOperation(channel, password, HashCacheOps.Put, key, post.Hash)); } - _remove(channel, password, options) { - const key = null; - const target = options.key ? options.key : (options.hash ? options.hash : null); - return await(this._createOperation(channel, password, HashCacheOps.Delete, key, target)); + _remove(channel, password, hash) { + return await(this._createOperation(channel, password, HashCacheOps.Delete, hash, null)); } _createOperation(channel, password, operation, key, value, data) { - let message = this._createMessage(channel, password, operation, key, value); - this.list.add(message.hash.Hash); // TODO: move to DataStore - const listHash = await(this.list.getIpfsHash()); + let hash = this._createMessage(channel, password, operation, key, value); + this._store.add(hash); + const listHash = await(this._store.list.getIpfsHash()); await(this._pubsub.publish(channel, listHash)); - return message.hash.Hash; + return key; } _deleteChannel(channel, password) { - this._pubsub.delete(channel, password); + this._store.clear(); return true; } - _setMode(channel, password, modes) { - let m = []; - if(typeof modes !== 'Array') - m.push(modes); - else - m = modes; - // const res = await(this.client.linkedList(channel, password).setMode(m)); - // return res.modes; - return { todo: 'TODO!' } - } + // _setMode(channel, password, modes) { + // let m = []; + // if(typeof modes !== 'Array') + // m.push(modes); + // else + // m = modes; + // // const res = await(this.client.linkedList(channel, password).setMode(m)); + // // return res.modes; + // return { todo: 'TODO!' } + // } - _info(channel, password) { - var l = this._pubsub.latest(channel); - return l; - } + // _info(channel, password) { + // var l = this._pubsub.latest(channel); + // return l; + // } _connect(host, port, username, password) { return new Promise((resolve, reject) => { - this._pubsub = new PubSub(this.ipfs, host, port, username, password); + this._pubsub = new PubSub(this._ipfs, host, port, username, password); // this.client = this._pubsub._client; // this.user = this.client.info.user; this.user = { id: 'hello-todo', username: username } - this.list = new List(username, this.ipfs); // TODO: move to DataStore + this._store = new DataStore(username, this._ipfs); resolve(); // this.network = { // id: this.client.info.networkId, diff --git a/src/PubSub.js b/src/PubSub.js index 2625f7f..5cfe961 100644 --- a/src/PubSub.js +++ b/src/PubSub.js @@ -1,9 +1,6 @@ 'use strict'; -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var redis = require("redis"); -var Aggregator = require('./Aggregator'); +const redis = require("redis"); const List = require('./list/OrbitList'); class Pubsub2 { @@ -13,17 +10,13 @@ class Pubsub2 { this.client1 = redis.createClient({ host: host, port: port }); this.client2 = redis.createClient({ host: host, port: port }); this.client1.on("message", this._handleMessage.bind(this)); - this.client1.on('connect', () => { - console.log('redis connected'); - }); - // this.client1.on("subscribe", (channel, count) => { - // }); + // this.client1.on('connect', () => console.log('redis connected')); + // this.client1.on("subscribe", (channel, count) => console.log(`subscribed to ${channel}`)); } subscribe(hash, password, callback) { if(!this._subscriptions[hash] || this._subscriptions[hash].password !== password) { this._subscriptions[hash] = { - topic: hash, password: password, head: null, callback: callback @@ -46,10 +39,6 @@ class Pubsub2 { return { head: this._subscriptions[hash] ? this._subscriptions[hash].head : null }; } - delete(hash, password) { - delete this._subscriptions[hash]; - } - _handleMessage(hash, message) { if(this._subscriptions[hash]) { this._subscriptions[hash].head = message; @@ -60,127 +49,4 @@ class Pubsub2 { } } -/* -class PubSub { - constructor(ipfs, host, port, username, password, resolve) { - this.ipfs = ipfs; - this._subscriptions = {}; - this.client1 = redis.createClient({ host: host, port: port }); - this.client2 = redis.createClient({ host: host, port: port }); - this.client3 = redis.createClient({ host: host, port: port }); - this.client1.on("message", this._handleMessage.bind(this)); - this.publishQueue = []; - - this.client1.on('connect', function() { - console.log('redis connected'); - resolve(); - }); - - this.client1.on("subscribe", function (channel, count) { - console.log("subscribed to pubsub topic '" + channel + "' (" + count + " peers)"); - }); - } - - subscribe(hash, password, head, callback) { - if(!this._subscriptions[hash] || this._subscriptions[hash].password !== password) { - this._subscriptions[hash] = { - topic: hash, - password: password, - head: null, - callback: callback, - seq: -1 - }; - // this.client3.get("orbit." + hash, (err, reply) => { - // if(reply) { - // let d = JSON.parse(reply); - // this._subscriptions[hash].seq = d.seq; - // this._subscriptions[hash].head = d.head; - // if(err) console.log(err); - // console.log(`head of '${hash}' is`, this._subscriptions[hash].head, "seq:", this._subscriptions[hash].seq); - // } - // }); - this.client1.subscribe(hash); - this.client2.publish(hash, JSON.stringify({ r: "HEAD" })); - } - - return new Promise((resolve, reject) => { - setTimeout(() => { - console.log("pubsub initialized") - resolve(); - }, 1000); - }); - } - - unsubscribe(hash) { - delete this._subscriptions[hash]; - this.client1.unsubscribe(); - this.client2.unsubscribe(); - } - - publish(hash, message, seq, callback) { - return new Promise((resolve, reject) => { - if(this.publishQueue.length === 0) { - this.publishQueue.splice(0, 0, { hash: message.Hash, callback: resolve }); - console.log("...") - this.client2.publish(hash, JSON.stringify({ hash: message.Hash, seq: seq })); - console.log("published") - } else { - console.log("queue full!") - resolve(false); - } - }); - } - - latest(hash) { - return { head: this._subscriptions[hash].head, modes: {}, seq: this._subscriptions[hash].seq }; - } - - delete(hash, password) { - delete this._subscriptions[hash]; - } - - _handleMessage(hash, event) { - if(this._subscriptions[hash]) { - var message = JSON.parse(event) - if(message.hash) { - var newHead = message.hash; - var seq = message.seq; - var isNewer = seq > this._subscriptions[hash].seq; - var item = this.publishQueue[this.publishQueue.length - 1]; - - // console.log(".", newHead, item ? item.hash : '', seq, this._subscriptions[hash].seq, isNewer) - - if(item) { - this.publishQueue.pop(); - item.callback(isNewer && newHead === item.hash); - } - - if(isNewer) - this._updateSubscription(hash, newHead, seq); - } else if(message.r === 'HEAD') { - console.log("SEND HEAD!") - this.client2.publish(hash, JSON.stringify(this.latest(hash))); - } else { - console.log("GOT HEAD!", message) - var isNewer = message.seq > this._subscriptions[hash].seq; - if(isNewer) { - console.log("NEW HEAD!") - this.publishQueue.pop(); - this._updateSubscription(hash, message.head, message.seq); - } - } - } - } - - _updateSubscription(hash, message, seq) { - // this.client3.set("orbit." + hash, JSON.stringify({ head: message, seq: seq })); - this._subscriptions[hash].seq = seq; - this._subscriptions[hash].head = message; - - if(this._subscriptions[hash].callback) - this._subscriptions[hash].callback(hash, message, seq); - } -} -*/ - module.exports = Pubsub2; diff --git a/src/list/OrbitList.js b/src/list/OrbitList.js index 40544ea..627beb3 100644 --- a/src/list/OrbitList.js +++ b/src/list/OrbitList.js @@ -21,6 +21,11 @@ class OrbitList extends List { this.ver ++; } + clear() { + this._items = []; + this._currentBatch = []; + } + getIpfsHash() { return new Promise(async((resolve, reject) => { const list = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(this.toJson()))); @@ -28,6 +33,14 @@ class OrbitList extends List { })); } + static fromIpfsHash(ipfs, hash) { + return new Promise(async((resolve, reject) => { + const l = await(ipfsAPI.getObject(ipfs, hash)); + const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data)); + resolve(list); + })); + } + static fromJson(ipfs, json) { let list = new List(json.id); list.seq = json.seq; @@ -35,14 +48,6 @@ class OrbitList extends List { list._items = _.uniqWith(json.items.map((f) => new Node(ipfs, f.id, f.seq, f.ver, f.data, f.next)), _.isEqual); return list; } - - static fromIpfsHash(ipfs, hash) { - return new Promise(async((resolve, reject) => { - const l = await(ipfsAPI.getObject(ipfs, hash)); - const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data)); - resolve(list); - })); - } } module.exports = OrbitList; diff --git a/src/list/OrbitNode.js b/src/list/OrbitNode.js index 0350e7b..38cfc5d 100644 --- a/src/list/OrbitNode.js +++ b/src/list/OrbitNode.js @@ -1,7 +1,8 @@ 'use strict'; -const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const async = require('asyncawait/async'); const await = require('asyncawait/await'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); const Node = require('./Node'); class OrbitNode extends Node { @@ -20,8 +21,24 @@ class OrbitNode extends Node { return "" + this.id + "." + this.seq + "." + this.ver + "." + this.hash; } + getPayload() { + if(!this.Payload) { + return new Promise(async((resolve, reject) => { + const payload = await(ipfsAPI.getObject(this._ipfs, this.data)); + this.Payload = JSON.parse(payload.Data); + if(this.Payload.value) { + const value = await(ipfsAPI.getObject(this._ipfs, this.Payload.value)); + this.Payload.value = JSON.parse(value.Data)["content"]; + } + resolve(this); + })); + } else { + return this; + } + } + compact() { - return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next } + return { id: this.id, seq: this.seq, ver: this.ver, data: this.data, next: this.next, Payload: this.Payload } } } diff --git a/test/orbit-client-tests.js b/test/orbit-client-tests.js index ae3d8df..9437013 100644 --- a/test/orbit-client-tests.js +++ b/test/orbit-client-tests.js @@ -1,580 +1,612 @@ -// 'use strict'; - -// var fs = require('fs'); -// var path = require('path'); -// var assert = require('assert'); -// var async = require('asyncawait/async'); -// var await = require('asyncawait/await'); -// var ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); -// var logger = require('orbit-common/lib/logger'); -// var Server = require('orbit-server/src/server'); -// var OrbitClient = require('../src/OrbitClient'); - -// var serverConfig = { -// networkId: "orbitdb-test", -// networkName: "OrbitDB Test Network", -// salt: "hellothisisdog", -// userDataPath: "/tmp/orbitdb-tests", -// verifyMessages: true -// } - -// // Orbit -// const host = 'localhost'; -// const port = 6379; -// const username = 'testrunner'; -// const password = ''; - -// const startServer = async (() => { -// return new Promise(async((resolve, reject) => { -// logger.setLevel('ERROR'); -// const ipfsd = await(ipfsDaemon()); -// const server = Server(ipfsd.daemon, ipfsd.nodeInfo, serverConfig); -// server.app.listen(port, () => { -// resolve(server); -// }).on('error', (err) => { -// resolve(server); -// }); -// })); -// }); - - -// describe('Orbit Client', () => { -// let server, orbit; - -// let head = ''; -// let items = []; -// let channel = 'abcdefgh'; - -// before(async((done) => { -// var initialize = () => new Promise(async((resolve, reject) => { -// orbit = OrbitClient.connect(host, port, username, password); -// orbit.channel(channel, '').delete(); -// resolve(); -// })); -// server = await(startServer()); -// await(initialize()); -// done(); -// })); - -// after(function(done) { -// var deleteChannel = () => new Promise(async((resolve, reject) => { -// if(orbit) orbit.channel(channel, '').delete(); -// resolve(); -// })); -// server.shutdown(); -// server = null; -// deleteChannel().then(done); -// }); - -// /* TESTS */ -// describe('Connect', function() { -// it('connects to hash-cache-server', async((done) => { -// assert.notEqual(orbit, null); -// // assert.notEqual(orbit.client, null); -// // assert.equal(orbit.user.id, 'hello'); -// // assert.equal(orbit.network.id, serverConfig.networkId); -// // assert.equal(orbit.network.name, serverConfig.networkName); -// // assert.notEqual(orbit.network.config.SupernodeRouting, null); -// // assert.notEqual(orbit.network.config.Bootstrap.length, 0); -// done(); -// })); -// }); - -// describe('Info', function() { -// it('gets channel info on empty channel', async((done) => { -// var info = orbit.channel(channel, '').info(); -// assert.notEqual(info, null); -// assert.equal(info.head, null); -// assert.notEqual(info.modes, null); -// done(); -// })); - -// it('gets channel info on an existing channel', async((done) => { -// var msg = orbit.channel(channel, '').add('hello'); -// var info = orbit.channel(channel, '').info(); -// assert.notEqual(info, null); -// assert.notEqual(info.head, null); -// assert.notEqual(info.modes, null); -// assert.equal(info.modes.r, null); -// done(); -// })); - -// // it('gets channel info when channel has modes set', async((done) => { -// // try { -// // orbit.channel(channel).delete(); -// // var mode = { -// // mode: "+r", -// // params: { -// // password: 'password' -// // } -// // }; -// // var res = orbit.channel(channel, '').setMode(mode) -// // var info = orbit.channel(channel, 'password').info(); -// // assert.notEqual(info, null); -// // assert.equal(info.head, null); -// // assert.equal(JSON.stringify(info.modes), JSON.stringify(res)); -// // orbit.channel(channel, 'password').delete(); -// // } catch(e) { -// // orbit.channel(channel, 'password').delete(); -// // assert.equal(e, null); -// // } -// // done(); -// // })); - -// }); - -// describe('Delete', function() { -// it('deletes a channel from the database', async((done) => { -// var result = orbit.channel(channel, '').delete(); -// assert.equal(result, true); -// var iter = orbit.channel(channel, '').iterator(); -// assert.equal(iter.next().value, null); -// done(); -// })); - -// it('deletes a channel with a password', async((done) => { -// done(); -// })); - -// it('doesn\'t delete a channel when password is wrong', async((done) => { -// done(); -// })); - -// it('doesn\'t delete a channel when user is not an op', async((done) => { -// done(); -// })); -// }); - -// describe('Add events', function() { -// it('adds an item to an empty channel', async((done) => { -// try { -// orbit.channel(channel, '').delete(); -// const head = orbit.channel(channel, '').add('hello'); -// assert.notEqual(head, null); -// assert.equal(head.startsWith('Qm'), true); -// assert.equal(head.length, 46); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); - -// it('adds a new item to a channel with one item', async((done) => { -// try { -// const head = orbit.channel(channel, '').iterator().collect()[0]; -// const second = orbit.channel(channel, '').add('hello'); -// assert.notEqual(second, null); -// assert.notEqual(second, head); -// assert.equal(second.startsWith('Qm'), true); -// assert.equal(second.length, 46); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); - -// it('adds five items', async((done) => { -// for(var i = 0; i < 5; i ++) { -// try { -// var s = orbit.channel(channel, '').add('hello'); -// assert.notEqual(s, null); -// assert.equal(s.startsWith('Qm'), true); -// assert.equal(s.length, 46); -// } catch(e) { -// assert.equal(e, null); -// } -// } -// done(); -// })); - -// it('adds an item that is > 256 bytes', async((done) => { -// try { -// var msg = new Buffer(512); -// msg.fill('a') -// var s = orbit.channel(channel, '').add(msg.toString()); -// assert.notEqual(s, null); -// assert.equal(s.startsWith('Qm'), true); -// assert.equal(s.length, 46); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); -// }); - - -// describe('Iterator', function() { -// var items = []; -// var itemCount = 5; - -// before(function(done) { -// var addMessages = () => new Promise(async((resolve, reject) => { -// var result = orbit.channel(channel, '').delete(); -// var iter = orbit.channel(channel, '').iterator(); -// for(var i = 0; i < itemCount; i ++) { -// var s = orbit.channel(channel, '').add('hello' + i); -// items.push(s); -// } -// resolve(); -// })); -// addMessages().then(done); -// }); - -// describe('Defaults', function() { -// it('returns an iterator', async((done) => { -// var iter = orbit.channel(channel, '').iterator(); -// var next = iter.next().value; -// assert.notEqual(iter, null); -// assert.notEqual(next, null); -// assert.notEqual(next.item, null); -// assert.notEqual(next.item.op, null); -// assert.equal(next.item.seq, 4); -// assert.notEqual(next.item.target, null); -// assert.notEqual(next.item.next, null); -// assert.notEqual(next.item.Payload, null); -// assert.equal(next.item.Payload, 'hello4'); -// done(); -// })); - -// it('implements Iterator interface', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: -1 }); -// var messages = []; - -// for(let i of iter) -// messages.push(i.hash); - -// assert.equal(messages.length, items.length); -// done(); -// })); - -// it('returns 1 item as default', async((done) => { -// var iter = orbit.channel(channel, '').iterator(); -// var first = iter.next().value; -// var second = iter.next().value; -// assert.equal(first.item.key, items[items.length - 1]); -// assert.equal(second, null); -// assert.equal(first.item.Payload, 'hello4'); -// done(); -// })); -// }); - -// describe('Collect', function() { -// it('returns all items', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: -1 }); -// var messages = iter.collect(); -// assert.equal(messages.length, items.length); -// assert.equal(messages[messages.length - 1].item.Payload, 'hello0'); -// assert.equal(messages[0].item.Payload, 'hello4'); -// done(); -// })); - -// it('returns 1 item', async((done) => { -// var iter = orbit.channel(channel, '').iterator(); -// var messages = iter.collect(); -// assert.equal(messages.length, 1); -// done(); -// })); - -// it('returns 3 items', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: 3 }); -// var messages = iter.collect(); -// assert.equal(messages.length, 3); -// done(); -// })); -// }); - -// describe('Options: limit', function() { -// it('returns 1 item when limit is 0', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: 0 }); -// var first = iter.next().value; -// var second = iter.next().value; -// assert.equal(first.item.key, items[items.length - 1]); -// assert.equal(second, null); -// done(); -// })); - -// it('returns 1 item when limit is 1', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: 1 }); -// var first = iter.next().value; -// var second = iter.next().value; -// assert.equal(first.item.key, items[items.length - 1]); -// assert.equal(second, null); -// done(); -// })); - -// it('returns 3 items', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: 3 }); -// var first = iter.next().value; -// var second = iter.next().value; -// var third = iter.next().value; -// var fourth = iter.next().value; -// assert.equal(first.item.key, items[items.length - 1]); -// assert.equal(second.item.key, items[items.length - 2]); -// assert.equal(third.item.key, items[items.length - 3]); -// assert.equal(fourth, null); -// done(); -// })); - -// it('returns all items', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: -1 }); -// var messages = iter.collect().map((e) => e.item.key); - -// messages.reverse(); -// assert.equal(messages.length, items.length); -// assert.equal(messages[0], items[0]); -// done(); -// })); - -// it('returns all items when limit is bigger than -1', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: -300 }); -// var messages = iter.collect().map((e) => e.item.key); - -// assert.equal(messages.length, items.length); -// assert.equal(messages[0], items[items.length - 1]); -// done(); -// })); - -// it('returns all items when limit is bigger than number of items', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: 300 }); -// var messages = iter.collect().map((e) => e.item.key); - -// assert.equal(messages.length, items.length); -// assert.equal(messages[0], items[items.length - 1]); -// done(); -// })); -// }); - -// describe('Options: reverse', function() { -// it('returns all items reversed', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: -1, reverse: true }); -// var messages = iter.collect().map((e) => e.item.key); - -// assert.equal(messages.length, items.length); -// assert.equal(messages[0], items[0]); -// done(); -// })); -// }); - -// describe('Options: ranges', function() { -// var all = []; -// var head; - -// before((done) => { -// var fetchAll = () => new Promise(async((resolve, reject) => { -// all = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); -// head = all[0]; -// resolve(); -// })); -// fetchAll().then(done); -// }); - -// describe('gt & gte', function() { -// it('returns 0 items when gt is the head', async((done) => { -// var messages = orbit.channel(channel, '').iterator({ gt: head.hash }).collect(); -// assert.equal(messages.length, 0); -// done(); -// })); - -// it('returns 1 item when gte is the head', async((done) => { -// var iter2 = orbit.channel(channel, '').iterator({ gte: head.hash, limit: -1 }); -// var messages = iter2.collect().map((e) => e.item.key); - -// assert.equal(messages.length, 1); -// assert.equal(messages[0], items[items.length -1]); -// done(); -// })); - -// it('returns 2 item when gte is defined', async((done) => { -// var gte = all[1].hash; -// var iter = orbit.channel(channel, '').iterator({ gte: gte, limit: -1 }); -// var messages = iter.collect().map((e) => e.hash); - -// // console.log(messages, all) -// assert.equal(messages.length, 2); -// assert.equal(messages[0], all[0].hash); -// assert.equal(messages[1], all[1].hash); -// done(); -// })); - -// it('returns all items when gte is the root item', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ gte: all[all.length -1], limit: -1 }); -// var messages = iter.collect().map((e) => e.item.key); - -// assert.equal(messages.length, itemCount); -// assert.equal(messages[0], items[items.length - 1]); -// assert.equal(messages[messages.length - 1], items[0]); -// done(); -// })); - -// it('returns items when gt is the root item', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ gt: all[all.length - 1], limit: -1 }); -// var messages = iter.collect().map((e) => e.item.key); - -// assert.equal(messages.length, itemCount - 1); -// assert.equal(messages[0], items[items.length - 1]); -// assert.equal(messages[3], items[1]); -// done(); -// })); - -// it('returns items when gt is defined', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ limit: -1}); -// var messages = iter.collect().map((e) => e.hash); - -// var gt = messages[2]; -// var iter2 = orbit.channel(channel, '').iterator({ gt: gt, limit: 100 }); -// var messages2 = iter2.collect().map((e) => e.hash); - -// assert.equal(messages2.length, 2); -// assert.equal(messages2[0], messages[0]); -// assert.equal(messages2[1], messages[1]); -// done(); -// })); -// }); - -// describe('lt & lte', function() { -// it('returns one item when lt is the head', async((done) => { -// var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash }); -// var messages = iter2.collect().map((e) => e.hash); - -// assert.equal(messages.length, 1); -// assert.equal(messages[0], head.hash); -// done(); -// })); - -// it('returns all items when lt is head and limit is -1', async((done) => { -// var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash, limit: -1 }); -// var messages = iter2.collect().map((e) => e.hash); - -// assert.equal(messages.length, itemCount); -// assert.equal(messages[0], head.hash); -// assert.equal(messages[4], all[all.length - 1].hash); -// done(); -// })); - -// it('returns 3 items when lt is head and limit is 3', async((done) => { -// var iter2 = orbit.channel(channel, '').iterator({ lt: head.hash, limit: 3 }); -// var messages = iter2.collect().map((e) => e.hash); - -// assert.equal(messages.length, 3); -// assert.equal(messages[0], head.hash); -// assert.equal(messages[2], all[2].hash); -// done(); -// })); - -// it('returns null when lt is the root item', async((done) => { -// var messages = orbit.channel(channel, '').iterator({ lt: all[all.length - 1].hash }).collect(); -// assert.equal(messages.length, 0); -// done(); -// })); - -// it('returns one item when lte is the root item', async((done) => { -// var iter = orbit.channel(channel, '').iterator({ lte: all[all.length - 1].hash }); -// var messages = iter.collect().map((e) => e.hash); - -// assert.equal(messages.length, 1); -// assert.equal(messages[0], all[all.length - 1].hash); -// done(); -// })); - -// it('returns all items when lte is the head', async((done) => { -// var iter2 = orbit.channel(channel, '').iterator({ lte: head.hash, limit: -1 }); -// var messages = iter2.collect().map((e) => e.hash); - -// assert.equal(messages.length, itemCount); -// assert.equal(messages[0], all[0].hash); -// assert.equal(messages[4], all[all.length - 1].hash); -// done(); -// })); - -// it('returns 3 items when lte is the head', async((done) => { -// var iter2 = orbit.channel(channel, '').iterator({ lte: head.hash, limit: 3 }); -// var messages = iter2.collect().map((e) => e.hash); - -// assert.equal(messages.length, 3); -// assert.equal(messages[0], all[0].hash); -// assert.equal(messages[1], all[1].hash); -// assert.equal(messages[2], all[2].hash); -// done(); -// })); -// }); -// }); - -// }); - - -// /* -// describe('Modes', function() { -// var password = 'hello'; - -// it('sets read mode', async((done) => { -// try { -// var mode = { -// mode: "+r", -// params: { -// password: password -// } -// }; -// var modes = orbit.channel(channel, '').setMode(mode) -// assert.notEqual(modes.r, null); -// assert.equal(modes.r.password, password); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); - -// it('can\'t read with wrong password', async((done) => { -// try { -// var modes = orbit.channel(channel, 'invalidpassword').iterator(); -// assert.equal(true, false); -// } catch(e) { -// assert.equal(e, 'Unauthorized'); -// } -// done(); -// })); - -// it('sets write mode', async((done) => { -// try { -// var mode = { -// mode: "+w", -// params: { -// ops: [orbit.user.id] -// } -// }; -// var modes = orbit.channel(channel, password).setMode(mode); -// assert.notEqual(modes.w, null); -// assert.equal(modes.w.ops[0], orbit.user.id); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); - -// it('can\'t write when user not an op', async((done) => { -// // TODO -// done(); -// })); - -// it('removes write mode', async((done) => { -// try { -// var modes = orbit.channel(channel, password).setMode({ mode: "-w" }); -// assert.equal(modes.w, null); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); - -// it('removes read mode', async((done) => { -// try { -// var modes = orbit.channel(channel, password).setMode({ mode: "-r" }); -// assert.equal(modes.r, null); -// } catch(e) { -// assert.equal(e, null); -// } -// done(); -// })); - -// }); -// */ -// }); +'use strict'; + +const _ = require('lodash'); +const assert = require('assert'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const OrbitClient = require('../src/OrbitClient'); + +// Orbit +const host = 'localhost'; +const port = 6379; +const username = 'testrunner'; +const password = ''; + +describe('Orbit Client', () => { + let client, db; + let items = []; + + let channel = 'abcdefgh'; + + before(async((done) => { + client = OrbitClient.connect(host, port, username, password); + db = client.channel(channel); + db.delete(); + done(); + })); + + after(async((done) => { + if(db) db.delete(); + done(); + })); + +/* + describe('Info', function() { // } + // }; + // var res = db.setMode(mode) + // var info = orbit.channel(channel, 'password').info(); + // assert.notEqual(info, null); + // assert.equal(info.head, null); + // assert.equal(JSON.stringify(info.modes), JSON.stringify(res)); + // orbit.channel(channel, 'password').delete(); + // } catch(e) { + // orbit.channel(channel, 'password').delete(); + // assert.equal(e, null); + // } + // done(); + // })); + }); +*/ + + describe('Add events', function() { + it('adds an item to an empty channel', async((done) => { + const head = db.add('hello'); + assert.notEqual(head, null); + assert.equal(head.startsWith('Qm'), true); + assert.equal(head.length, 46); + done(); + })); + + it('adds a new item to a channel with one item', async((done) => { + const head = db.iterator().collect()[0]; + const second = db.add('hello'); + assert.notEqual(second, null); + assert.notEqual(second, head); + assert.equal(second.startsWith('Qm'), true); + assert.equal(second.length, 46); + done(); + })); + + it('adds five items', async((done) => { + for(let i = 0; i < 5; i ++) { + let hash = db.add('hello'); + assert.notEqual(hash, null); + assert.equal(hash.startsWith('Qm'), true); + assert.equal(hash.length, 46); + } + done(); + })); + + it('adds an item that is > 256 bytes', async((done) => { + let msg = new Buffer(512); + msg.fill('a') + const hash = db.add(msg.toString()); + assert.notEqual(hash, null); + assert.equal(hash.startsWith('Qm'), true); + assert.equal(hash.length, 46); + done(); + })); + }); + + describe('Delete events', function() { + it('deletes an item when only one item in the database', async((done) => { + db.delete(); + const head = db.add('hello1'); + let item = db.iterator().collect(); + const delop = db.del(head); + const items = db.iterator().collect(); + assert.equal(delop.startsWith('Qm'), true); + assert.equal(items.length, 0); + done(); + })); + + it('deletes an item when two items in the database', async((done) => { + db.add('hello1'); + const head = db.add('hello2'); + db.del(head); + const items = db.iterator().collect(); + assert.equal(items.length, 1); + assert.equal(items[0].hash.startsWith('Qm'), true); + assert.equal(items[0].payload.op, 'ADD'); + assert.equal(items[0].payload.value, 'hello1'); + assert.notEqual(items[0].payload.meta, null); + done(); + })); + + it('deletes an item between adds', async((done) => { + const head = db.add('hello1'); + db.add('hello2'); + db.del(head); + db.add('hello3'); + const items = db.iterator().collect(); + assert.equal(items.length, 1); + assert.equal(items[0].hash.startsWith('Qm'), true); + assert.equal(items[0].payload.op, 'ADD'); + assert.equal(items[0].payload.value, 'hello3'); + assert.notEqual(items[0].payload.meta, null); + done(); + })); + }); + + describe('Iterator', function() { + let items = []; + const itemCount = 5; + + before(async((done) => { + db.delete(); + for(let i = 0; i < itemCount; i ++) { + const hash = db.add('hello' + i); + items.push(hash); + } + done(); + })); + + describe('Defaults', function() { + it('returns an iterator', async((done) => { + const iter = db.iterator(); + const next = iter.next().value; + assert.notEqual(iter, null); + assert.notEqual(next, null); + done(); + })); + + it('returns an item with the correct structure', async((done) => { + const iter = db.iterator(); + const next = iter.next().value; + + assert.notEqual(next, null); + assert.notEqual(next.hash, null); + assert.equal(next.hash.startsWith('Qm'), true); + assert.notEqual(next.payload, null); + assert.equal(next.payload.op, 'ADD'); + assert.equal(next.payload.key.startsWith('Qm'), true); + assert.equal(next.payload.value, 'hello4'); + assert.notEqual(next.payload.meta, null); + done(); + })); + + it('implements Iterator interface', async((done) => { + const iter = db.iterator({ limit: -1 }); + let messages = []; + + for(let i of iter) + messages.push(i.hash); + + assert.equal(messages.length, items.length); + done(); + })); + + it('returns 1 item as default', async((done) => { + const iter = db.iterator(); + const first = iter.next().value; + const second = iter.next().value; + assert.equal(first.payload.key, items[items.length - 1]); + assert.equal(second, null); + assert.equal(first.payload.value, 'hello4'); + done(); + })); + }); + + describe('Collect', function() { + it('returns all items', async((done) => { + const messages = db.iterator({ limit: -1 }).collect(); + assert.equal(messages.length, items.length); + assert.equal(messages[0].payload.value, 'hello0'); + assert.equal(messages[messages.length - 1].payload.value, 'hello4'); + done(); + })); + + it('returns 1 item', async((done) => { + const messages = db.iterator().collect(); + assert.equal(messages.length, 1); + done(); + })); + + it('returns 3 items', async((done) => { + const messages = db.iterator({ limit: 3 }).collect(); + assert.equal(messages.length, 3); + done(); + })); + }); + + describe('Options: limit', function() { + it('returns 1 item when limit is 0', async((done) => { + const iter = db.iterator({ limit: 0 }); + const first = iter.next().value; + const second = iter.next().value; + assert.equal(first.payload.key, items[items.length - 1]); + assert.equal(second, null); + done(); + })); + + it('returns 1 item when limit is 1', async((done) => { + const iter = db.iterator({ limit: 1 }); + const first = iter.next().value; + const second = iter.next().value; + assert.equal(first.payload.key, items[items.length - 1]); + assert.equal(second, null); + done(); + })); + + it('returns 3 items', async((done) => { + const iter = db.iterator({ limit: 3 }); + const first = iter.next().value; + const second = iter.next().value; + const third = iter.next().value; + const fourth = iter.next().value; + assert.equal(first.payload.key, items[items.length - 3]); + assert.equal(second.payload.key, items[items.length - 2]); + assert.equal(third.payload.key, items[items.length - 1]); + assert.equal(fourth, null); + done(); + })); + + it('returns all items', async((done) => { + const messages = db.iterator({ limit: -1 }) + .collect() + .map((e) => e.payload.key); + + messages.reverse(); + assert.equal(messages.length, items.length); + assert.equal(messages[0], items[items.length - 1]); + done(); + })); + + it('returns all items when limit is bigger than -1', async((done) => { + const messages = db.iterator({ limit: -300 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, items.length); + assert.equal(messages[0], items[0]); + done(); + })); + + it('returns all items when limit is bigger than number of items', async((done) => { + const messages = db.iterator({ limit: 300 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, items.length); + assert.equal(messages[0], items[0]); + done(); + })); + }); + + describe('Options: reverse', function() { + it('returns all items reversed', async((done) => { + const messages = db.iterator({ limit: -1, reverse: true }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, items.length); + assert.equal(messages[0], items[items.length - 1]); + done(); + })); + }); + + describe('Options: ranges', function() { + + describe('gt & gte', function() { + it('returns 1 item when gte is the head', async((done) => { + const messages = db.iterator({ gte: _.last(items), limit: -1 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, 1); + assert.equal(messages[0], items[items.length -1]); + done(); + })); + + it('returns 0 items when gt is the head', async((done) => { + const messages = db.iterator({ gt: _.last(items) }).collect(); + assert.equal(messages.length, 0); + done(); + })); + + it('returns 2 item when gte is defined', async((done) => { + const gte = items[items.length - 2]; + const messages = db.iterator({ gte: gte, limit: -1 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, 2); + assert.equal(messages[0], items[items.length - 2]); + assert.equal(messages[1], items[items.length - 1]); + done(); + })); + + it('returns all items when gte is the root item', async((done) => { + const messages = db.iterator({ gte: items[0], limit: -1 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, items.length); + assert.equal(messages[0], items[0]); + assert.equal(messages[messages.length - 1], items[items.length - 1]); + done(); + })); + + it('returns items when gt is the root item', async((done) => { + const messages = db.iterator({ gt: items[0], limit: -1 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, itemCount - 1); + assert.equal(messages[0], items[1]); + assert.equal(messages[3], items[items.length - 1]); + done(); + })); + + it('returns items when gt is defined', async((done) => { + const messages = db.iterator({ limit: -1}) + .collect() + .map((e) => e.payload.key); + + const gt = messages[2]; + + const messages2 = db.iterator({ gt: gt, limit: 100 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages2.length, 2); + assert.equal(messages2[0], messages[messages.length - 2]); + assert.equal(messages2[1], messages[messages.length - 1]); + done(); + })); + }); + + describe('lt & lte', function() { + it('returns one item after head when lt is the head', async((done) => { + const messages = db.iterator({ lt: _.last(items) }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, 1); + assert.equal(messages[0], items[items.length - 2]); + done(); + })); + + it('returns all items when lt is head and limit is -1', async((done) => { + const messages = db.iterator({ lt: _.last(items), limit: -1 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, items.length - 1); + assert.equal(messages[0], items[0]); + assert.equal(messages[messages.length - 1], items[items.length - 2]); + done(); + })); + + it('returns 3 items when lt is head and limit is 3', async((done) => { + const messages = db.iterator({ lt: _.last(items), limit: 3 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, 3); + assert.equal(messages[0], items[items.length - 4]); + assert.equal(messages[2], items[items.length - 2]); + done(); + })); + + it('returns null when lt is the root item', async((done) => { + const messages = db.iterator({ lt: items[0] }).collect(); + assert.equal(messages.length, 0); + done(); + })); + + it('returns one item when lte is the root item', async((done) => { + const messages = db.iterator({ lte: items[0] }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, 1); + assert.equal(messages[0], items[0]); + done(); + })); + + it('returns all items when lte is the head', async((done) => { + const messages = db.iterator({ lte: _.last(items), limit: -1 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, itemCount); + assert.equal(messages[0], items[0]); + assert.equal(messages[4], _.last(items)); + done(); + })); + + it('returns 3 items when lte is the head', async((done) => { + const messages = db.iterator({ lte: _.last(items), limit: 3 }) + .collect() + .map((e) => e.payload.key); + + assert.equal(messages.length, 3); + assert.equal(messages[0], items[items.length - 3]); + assert.equal(messages[1], items[items.length - 2]); + assert.equal(messages[2], _.last(items)); + done(); + })); + }); + }); + }); + + +/* + describe('Modes', function() { + var password = 'hello'; + + it('sets read mode', async((done) => { + try { + var mode = { + mode: "+r", + params: { + password: password + } + }; + var modes = db.setMode(mode) + assert.notEqual(modes.r, null); + assert.equal(modes.r.password, password); + } catch(e) { + assert.equal(e, null); + } + done(); + })); + + it('can\'t read with wrong password', async((done) => { + try { + var modes = orbit.channel(channel, 'invalidpassword').iterator(); + assert.equal(true, false); + } catch(e) { + assert.equal(e, 'Unauthorized'); + } + done(); + })); + + it('sets write mode', async((done) => { + try { + var mode = { + mode: "+w", + params: { + ops: [orbit.user.id] + } + }; + var modes = orbit.channel(channel, password).setMode(mode); + assert.notEqual(modes.w, null); + assert.equal(modes.w.ops[0], orbit.user.id); + } catch(e) { + assert.equal(e, null); + } + done(); + })); + + it('can\'t write when user not an op', async((done) => { + // TODO + done(); + })); + + it('removes write mode', async((done) => { + try { + var modes = orbit.channel(channel, password).setMode({ mode: "-w" }); + assert.equal(modes.w, null); + } catch(e) { + assert.equal(e, null); + } + done(); + })); + + it('removes read mode', async((done) => { + try { + var modes = orbit.channel(channel, password).setMode({ mode: "-r" }); + assert.equal(modes.r, null); + } catch(e) { + assert.equal(e, null); + } + done(); + })); + + }); +*/ + + describe('Delete', function() { + it('deletes a channel from the database', async((done) => { + const result = db.delete(); + assert.equal(result, true); + const iter = db.iterator(); + assert.equal(iter.next().value, null); + done(); + })); + }); + + describe('Key-Value Store', function() { + it('put', async((done) => { + db.put('key1', 'hello!'); + let all = db.iterator().collect(); + assert.equal(all.length, 1); + assert.equal(all[0].hash.startsWith('Qm'), true); + assert.equal(all[0].payload.key, 'key1'); + assert.equal(all[0].payload.op, 'PUT'); + assert.notEqual(all[0].payload.meta, null); + done(); + })); + + it('get', async((done) => { + db.put('key1', 'hello!'); + const value = db.get('key1'); + assert.equal(value, 'hello!'); + done(); + })); + + it('put updates a value', async((done) => { + db.put('key1', 'hello!'); + db.put('key1', 'hello again'); + const value = db.get('key1'); + assert.equal(value, 'hello again'); + done(); + })); + + it('deletes a key', async((done) => { + db.put('key1', 'hello!'); + db.del('key1'); + const value = db.get('key1'); + assert.equal(value, null); + done(); + })); + + it('deletes a key after multiple updates', async((done) => { + db.put('key1', 'hello1'); + db.put('key1', 'hello2'); + db.put('key1', 'hello3'); + db.del('key1'); + const value = db.get('key1'); + assert.equal(value, null); + done(); + })); + + it('put - multiple keys', async((done) => { + db.put('key1', 'hello1'); + db.put('key2', 'hello2'); + db.put('key3', 'hello3'); + const all = db.iterator().collect(); + assert.equal(all.length, 1); + done(); + })); + + it('get - multiple keys', async((done) => { + db.put('key1', 'hello1'); + db.put('key2', 'hello2'); + db.put('key3', 'hello3'); + const v1 = db.get('key1'); + const v2 = db.get('key2'); + const v3 = db.get('key3'); + assert.equal(v1, 'hello1'); + assert.equal(v2, 'hello2'); + assert.equal(v3, 'hello3'); + done(); + })); + + it('get - integer value', async((done) => { + db.put('key1', 123); + const v1 = db.get('key1'); + assert.equal(v1, 123); + done(); + })); + + it('get - object value', async((done) => { + const val = { one: 'first', two: 2 }; + db.put('key1', val); + const v1 = db.get('key1'); + assert.equal(_.isEqual(v1, val), true); + done(); + })); + + it('get - array value', async((done) => { + const val = [1, 2, 3, 4, 5]; + db.put('key1', val); + const v1 = db.get('key1'); + assert.equal(_.isEqual(v1, val), true); + done(); + })); + }); + +}); diff --git a/test/orbit-list-tests.js b/test/orbit-list-tests.js index a42a923..af905f2 100644 --- a/test/orbit-list-tests.js +++ b/test/orbit-list-tests.js @@ -148,9 +148,9 @@ describe('OrbitList', async(function() { seq: 0, ver: 3, items: [ - { id: 'A', seq: 0, ver: 0, data: 'hello1', next: [] }, - { id: 'A', seq: 0, ver: 1, data: 'hello2', next: ['A.0.0.QmZfdeMV77si491NPX83Q8eRYE9WNzVorHrfWJPrJ51brt'] }, - { id: 'A', seq: 0, ver: 2, data: 'hello3', next: ['A.0.1.QmbbtEWe4qHLSjtW2HkPuszFW3zfBTXBdPrkXMdbePxqfK'] } + { id: 'A', seq: 0, ver: 0, data: 'hello1', next: [], Payload: undefined }, + { id: 'A', seq: 0, ver: 1, data: 'hello2', next: ['A.0.0.QmZfdeMV77si491NPX83Q8eRYE9WNzVorHrfWJPrJ51brt'], Payload: undefined }, + { id: 'A', seq: 0, ver: 2, data: 'hello3', next: ['A.0.1.QmbbtEWe4qHLSjtW2HkPuszFW3zfBTXBdPrkXMdbePxqfK'], Payload: undefined } ] }; // console.log(JSON.stringify(json, null, 1)) From 4c0e8f6a9ee30b7793105bdb36171c683dc3ca1c Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 11:33:38 +0200 Subject: [PATCH 20/30] Fix a perf bottleneck by commiting lists in batches. 100 tests! --- examples/pubsubKeyValue.js | 23 +++++++++++++---------- src/DataStore.js | 4 +++- src/OrbitClient.js | 12 +++++------- src/list/List.js | 9 ++------- src/list/OrbitList.js | 36 ++++++++++++++++++++++++++---------- src/list/OrbitNode.js | 28 ++++++++++++++++------------ test/orbit-client-tests.js | 3 ++- test/orbit-list-tests.js | 24 ++++++++++++++++++++++++ 8 files changed, 91 insertions(+), 48 deletions(-) diff --git a/examples/pubsubKeyValue.js b/examples/pubsubKeyValue.js index 746daf9..0fb697e 100644 --- a/examples/pubsubKeyValue.js +++ b/examples/pubsubKeyValue.js @@ -22,25 +22,28 @@ let run = (async(() => { let id = 'Log: Query ' let running = false; - setInterval(async(() => { - if(!running) { + // setInterval(async(() => { + // if(!running) { + while(true) { running = true; - // let timer = new Timer(true); - channel.put("lamb", "of god" + count); - // console.log(`Query #${count} took ${timer.stop(true)} ms\n`); - let v = channel.get("lamb"); + const key = "Lamb"; + let timer = new Timer(true); + channel.put(key, "Of God " + count); + let v = channel.get(key); + console.log(`Query #${count} took ${timer.stop(true)} ms\n`); console.log("---------------------------------------------------") - console.log("Id | Seq | Ver | Data") + console.log("Key | Value") console.log("---------------------------------------------------") - console.log(v); + console.log(`${key} | ${v}`); console.log("---------------------------------------------------") running = false; count ++; - } - }), 500); + } + // } + // }), 500); } catch(e) { console.error("error:", e); diff --git a/src/DataStore.js b/src/DataStore.js index 28ccacf..cc67a12 100644 --- a/src/DataStore.js +++ b/src/DataStore.js @@ -7,6 +7,8 @@ const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); const OrbitList = require('./list/OrbitList'); const HashCacheOps = require('./HashCacheOps'); +var Timer = require('../examples/Timer'); + const DefaultAmount = 1; class DataStore { @@ -16,7 +18,7 @@ class DataStore { } add(hash) { - this.list.add(hash); + return this.list.add(hash); } join(other) { diff --git a/src/OrbitClient.js b/src/OrbitClient.js index ee7470a..42b083f 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -124,12 +124,9 @@ class OrbitClient { } _createMessage(channel, password, operation, key, value) { - // Create meta info const size = -1; - const metaInfo = new MetaInfo(ItemTypes.Message, size, this.user.id, new Date().getTime()); - // Create the hash cache item - const item = new OrbitDBItem(operation, key, value, metaInfo); - // Save the item to ipfs + const meta = new MetaInfo(ItemTypes.Message, size, this.user.id, new Date().getTime()); + const item = new OrbitDBItem(operation, key, value, meta); const data = await (ipfsAPI.putObject(this._ipfs, JSON.stringify(item))); return data.Hash; } @@ -151,11 +148,12 @@ class OrbitClient { } _createOperation(channel, password, operation, key, value, data) { - let hash = this._createMessage(channel, password, operation, key, value); - this._store.add(hash); + const hash = this._createMessage(channel, password, operation, key, value); + const res = await(this._store.add(hash)); const listHash = await(this._store.list.getIpfsHash()); await(this._pubsub.publish(channel, listHash)); return key; + // return res; } _deleteChannel(channel, password) { diff --git a/src/list/List.js b/src/list/List.js index fbf8736..10ca246 100644 --- a/src/list/List.js +++ b/src/list/List.js @@ -24,13 +24,8 @@ class List { } join(other) { - if(other.seq && other.seq > this.seq) { - this.seq = other.seq + 1; - this.ver = 0; - } else { - this.seq = this.seq + 1; - this.ver = 0; - } + this.seq = (other.seq && other.seq > this.seq ? other.seq : this.seq) + 1; + this.ver = 0; const current = _.differenceWith(this._currentBatch, this._items, this._equals); const others = _.differenceWith(other.items, this._items, this._equals); const final = _.unionWith(current, others, this._equals); diff --git a/src/list/OrbitList.js b/src/list/OrbitList.js index 627beb3..db8c071 100644 --- a/src/list/OrbitList.js +++ b/src/list/OrbitList.js @@ -7,6 +7,8 @@ const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); const List = require('./List'); const Node = require('./OrbitNode'); +const MaxBatchSize = 200; + class OrbitList extends List { constructor(id, ipfs) { super(id); @@ -19,6 +21,11 @@ class OrbitList extends List { const node = new Node(this._ipfs, this.id, this.seq, this.ver, data, heads); this._currentBatch.push(node); this.ver ++; + + if(this.ver >= MaxBatchSize) + this._commit(); + + return node.ipfsHash; } clear() { @@ -27,27 +34,36 @@ class OrbitList extends List { } getIpfsHash() { - return new Promise(async((resolve, reject) => { - const list = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(this.toJson()))); - resolve(list.Hash); - })); + const list = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(this.toJson()))); + return list.Hash; } static fromIpfsHash(ipfs, hash) { - return new Promise(async((resolve, reject) => { - const l = await(ipfsAPI.getObject(ipfs, hash)); - const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data)); - resolve(list); - })); + const l = await(ipfsAPI.getObject(ipfs, hash)); + const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data)); + return list; } static fromJson(ipfs, json) { let list = new List(json.id); list.seq = json.seq; list.ver = json.ver; - list._items = _.uniqWith(json.items.map((f) => new Node(ipfs, f.id, f.seq, f.ver, f.data, f.next)), _.isEqual); + // list._items = _.uniqWith(json.items.map((f) => new Node(ipfs, f.id, f.seq, f.ver, f.data, f.next)), _.isEqual); + list._items = json.items.map((f) => new Node(ipfs, f.id, f.seq, f.ver, f.data, f.next)); return list; } + + static get batchSize() { + return MaxBatchSize; + } + + _commit() { + const current = _.differenceWith(this._currentBatch, this._items, this._equals); + this._items = this._items.concat(current); + this._currentBatch = []; + this.ver = 0; + this.seq ++; + } } module.exports = OrbitList; diff --git a/src/list/OrbitNode.js b/src/list/OrbitNode.js index 38cfc5d..14cc9e8 100644 --- a/src/list/OrbitNode.js +++ b/src/list/OrbitNode.js @@ -21,19 +21,23 @@ class OrbitNode extends Node { return "" + this.id + "." + this.seq + "." + this.ver + "." + this.hash; } + get ipfsHash() { + if(!this.hash) { + const t = this.compact(); + const r = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(t))); + this.hash = r.Hash; + } + return this.hash; + } + getPayload() { - if(!this.Payload) { - return new Promise(async((resolve, reject) => { - const payload = await(ipfsAPI.getObject(this._ipfs, this.data)); - this.Payload = JSON.parse(payload.Data); - if(this.Payload.value) { - const value = await(ipfsAPI.getObject(this._ipfs, this.Payload.value)); - this.Payload.value = JSON.parse(value.Data)["content"]; - } - resolve(this); - })); - } else { - return this; + if(!this.Payload) { + const payload = await(ipfsAPI.getObject(this._ipfs, this.data)); + this.Payload = JSON.parse(payload.Data); + if(this.Payload.value) { + const value = await(ipfsAPI.getObject(this._ipfs, this.Payload.value)); + this.Payload.value = JSON.parse(value.Data)["content"]; + } } } diff --git a/test/orbit-client-tests.js b/test/orbit-client-tests.js index 9437013..d700486 100644 --- a/test/orbit-client-tests.js +++ b/test/orbit-client-tests.js @@ -70,6 +70,7 @@ describe('Orbit Client', () => { it('adds five items', async((done) => { for(let i = 0; i < 5; i ++) { let hash = db.add('hello'); + // console.log(hash) assert.notEqual(hash, null); assert.equal(hash.startsWith('Qm'), true); assert.equal(hash.length, 46); @@ -78,7 +79,7 @@ describe('Orbit Client', () => { })); it('adds an item that is > 256 bytes', async((done) => { - let msg = new Buffer(512); + let msg = new Buffer(1024); msg.fill('a') const hash = db.add(msg.toString()); assert.notEqual(hash, null); diff --git a/test/orbit-list-tests.js b/test/orbit-list-tests.js index af905f2..036277a 100644 --- a/test/orbit-list-tests.js +++ b/test/orbit-list-tests.js @@ -231,6 +231,30 @@ describe('OrbitList', async(function() { done(); })); + + it('commits a list after batch size was reached', async((done) => { + const list = new List('A', ipfs); + + for(let i = 1; i <= List.batchSize; i ++) { + list.add("hello" + i); + } + + assert.equal(list.id, 'A'); + assert.equal(list.seq, 1); + assert.equal(list.ver, 0); + assert.equal(list.items.length, List.batchSize); + assert.equal(list._currentBatch.length, 0); + assert.equal(list._items.length, List.batchSize); + + const item = list.items[list.items.length - 1]; + assert.equal(item.id, 'A'); + assert.equal(item.seq, 0); + assert.equal(item.ver, List.batchSize - 1); + assert.equal(item.data, 'hello' + List.batchSize); + assert.equal(item.next, 'A.0.198.QmRKrcfkejCvxTxApZACjHpxzAKKGnCtFi2rD31CT7RkBS'); + + done(); + })); }); describe('join', () => { From bd8f8876ddf5077a89e396ace45c72ebaeaf2fa9 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:08:17 +0200 Subject: [PATCH 21/30] Add WIP/ to repo --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7153fbd..ee786e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ *sublime* node_modules/ debug.log -WIP/ .vagrant/ .idea/ isolate*.log From 04b3e262d04dc773d3f852d5e21f4482e63091a8 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:08:34 +0200 Subject: [PATCH 22/30] Remove obsolete files --- MOCK.json | 198 ------------------------------------------ examples/cacheTest.js | 58 ------------- test1.js | 70 --------------- test2.js | 92 -------------------- 4 files changed, 418 deletions(-) delete mode 100644 MOCK.json delete mode 100644 examples/cacheTest.js delete mode 100644 test1.js delete mode 100644 test2.js diff --git a/MOCK.json b/MOCK.json deleted file mode 100644 index 54922e3..0000000 --- a/MOCK.json +++ /dev/null @@ -1,198 +0,0 @@ -// Data set grouped and sorted -"A" : [ - { "id": "A", "seq": 0, "ver": 0, "prev": null}, - { "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, - { "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, - { "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, - { "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, - { "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} -], -"B" : [ - { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, - { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} -], -"C" : [ - { "id": "C", "seq": 0, "ver": 0, "prev": null}, - { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, - { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, - { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} -] - - A B C - | -0.0 - | -0.1 - | -0.2 0.0 - | | -0.3 0.1 - | | -0.4 0.2 - | \ / | - | 1.0 | - | | | - | 1.1 | - | / | -2.0 | - \ | - \ | - \ | - 3.0 - -// expected order A -{ "id": "A", "seq": 0, "ver": 0, "prev": null}, -{ "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, -{ "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, -{ "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, -{ "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, - { "id": "C", "seq": 0, "ver": 0, "prev": null}, - { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, - { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, - { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, - { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} -{ "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} - { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} - -"VersionClock": { - "seq": 0, - "ver": 0 -} - -"Item": { - "id": "", - "VersionClock": "", - "prev": [] -} - -"List": { - "items": [""] -} - -/* - list.add(data) { - this.ver ++; - const heads = _findHeads(); - const i = new Item(id, this.seq, this.ver, heads) - outgoing.push(data) - } -*/ - -/* - list.join(other) { - // increase seq on join, reset version - if(other.first.seq >= this.seq) - this.seq = other.first.seq + 1 - this.ver = 0 - - items = items.concat(outgoing.concat(other)) - items = items.sortBy("seq", "id", "ver") - outgoing = [] - } -*/ - -/* - nextHeads() { - referenced = [] - heads = all.groupBy("id").map((items) => items[items.length - 1]) - cleaned = heads.reverse().filter((e) => !isReferencedInChain(referenced, e)) - return cleaned; - } -*/ - -/* - isReferencedInChain(list, other) { - const res = other.map((o) => { - const ref = list.map((e) => !(e.id == o.id && e.seq == o.seq && e.ver == o.ver)) - if(!ref) - list.push(ref) - //return false - - //list.concat(list.filter((e) => !(e.id == o.id && e.seq == o.seq && e.ver == o.ver))) - if(o.prev) - ref = isReferencedInChain(list, o.prev) - - return ref - }) - return res.anyEqual(true) - } -*/ - - - A B C - 0.0 - | -0.0 0.1 - | | -0.1 0.2 - \ / | - 1.0 | - | | - 1.1 | - / | -2.0 | - \ | - \ | - \ | - 3.0 - -// Sequence, --> syncs to -listA.add("mango") // { "id": "A", "seq": 0, "ver": 0, "prev": null} -listA.add("banana") // { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} ---> B - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} - -listC.add("apple") // { "id": "C", "seq": 0, "ver": 0, "prev": null} -listC.add("strawberry") // { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -listC.add("orange") // { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} ---> A,B - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} - -listB.add("pineapple") // { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} -listB.add("papaya") // { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} ---> A - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} -// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} -// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} - -listA.add("kiwi") // { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} ---> C - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} -// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} -// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} -// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} - -listC.add("blueberry") // { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} ---> A,B - -// A -// { "id": "A", "seq": 0, "ver": 0, "prev": null} -// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} -// { "id": "C", "seq": 0, "ver": 0, "prev": null} -// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} -// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} -// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} -// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} -// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} -// { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} diff --git a/examples/cacheTest.js b/examples/cacheTest.js deleted file mode 100644 index 72f2a78..0000000 --- a/examples/cacheTest.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -var async = require('asyncawait/async'); -var OrbitClient = require('../src/OrbitClient'); -var Timer = require('./Timer'); - -var host = 'localhost:3006'; -var username = 'testrunner'; -var password = ''; - -let run = (async(() => { - try { - // Connect - var orbit = OrbitClient.connect(host, username, password); - - console.log("-------- EVENT log -------") - const c1 = 'cache-test'; - orbit.channel(c1).delete(); - - var timer1 = new Timer(true); - console.log("Writing..."); - for(let i = 0; i < 100; i ++) { - orbit.channel(c1).add("hello " + i); - } - console.log("Write took", timer1.stop() + "ms"); - - var timer2 = new Timer(true); - console.log("Reading 1st time..."); - var items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - items = items.map((e) => { - return { key: e.item.key, val: e.item.Payload }; - }); - console.log("Reading 1st time took", timer2.stop() + "ms"); - - var timer3 = new Timer(true); - console.log("Reading 2nd time..."); - var items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - items = items.map((e) => { - return { key: e.item.key, val: e.item.Payload }; - }); - console.log("Reading 2nd time took", timer3.stop() + "ms"); - - var timer4 = new Timer(true); - console.log("Reading 3rd time..."); - var items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - items = items.map((e) => { - return { key: e.item.key, val: e.item.Payload }; - }); - console.log("Reading 3rd time took", timer4.stop() + "ms"); - - } catch(e) { - console.error("error:", e); - console.error(e.stack); - process.exit(1); - } -}))(); - -module.exports = run; diff --git a/test1.js b/test1.js deleted file mode 100644 index da3caf9..0000000 --- a/test1.js +++ /dev/null @@ -1,70 +0,0 @@ -'use strict'; - -const _ = require('lodash'); -const Timer = require('./examples/Timer'); -const List = require('./src/list/List'); - -var run = () => { - var redis = require("redis"); - this.client1 = redis.createClient({ host: "localhost", port: 6379 }); - this.client2 = redis.createClient({ host: "localhost", port: 6379 }); - var hash = "ccc" - this.client1.subscribe(hash); - this.client1.subscribe(hash); - - - let listA = new List("A"); - let listB = new List("B"); - let listC = new List("C"); - - const handleMessage = (hash, event) => { - const l = List.fromJson(JSON.parse(event)); - // console.log("LIST", l); - - if(l.id === 'A') { - listB.join(l); - listC.join(l); - } else if(l.id === 'B') { - listA.join(l); - listC.join(l); - } else if(l.id === 'C') { - listA.join(l); - console.log("Items:", listA.items.length); - // console.log(JSON.stringify(listA, null, 1)); - } - - } - - this.client1.on("message", handleMessage); - this.client2.on("message", handleMessage); - - let h = 0; - setInterval(() => { - listC.add("C--"+h); - this.client2.publish(hash, JSON.stringify(listC.toJson())); - h++; - }, 1000); - - let i = 0; - setInterval(() => { - let a = 0; - for(let a = 0; a < 10; a ++) { - listB.add("B--"+(i+a)); - } - this.client2.publish(hash, JSON.stringify(listB.toJson())); - i++; - }, 20); - - let k = 0; - setInterval(() => { - listA.add("A--"+k); - k++; - listA.add("A--"+k); - k++; - listA.add("A--"+k); - k++; - this.client2.publish(hash, JSON.stringify(listA.toJson())); - }, 100); -}; - -run(); diff --git a/test2.js b/test2.js deleted file mode 100644 index 5bb9322..0000000 --- a/test2.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict'; - -const _ = require('lodash'); -const async = require('asyncawait/async'); -const await = require('asyncawait/await'); -const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); -const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); -const List = require('./src/list/OrbitList'); -const Timer = require('./examples/Timer'); - -const startIpfs = async (() => { - return new Promise(async((resolve, reject) => { - const ipfsd = await(ipfsDaemon()); - resolve(ipfsd.daemon); - })); -}); - -let ipfs; - -var run = async(() => { - ipfs = await(startIpfs()); - - var redis = require("redis"); - this.client1 = redis.createClient({ host: "localhost", port: 6379 }); - this.client2 = redis.createClient({ host: "localhost", port: 6379 }); - var hash = "ccc" - this.client1.subscribe(hash); - this.client1.subscribe(hash); - - - let listA = new List("A", ipfs); - let listB = new List("B", ipfs); - let listC = new List("C", ipfs); - - const handleMessage = async((hash, event) => { - // const l = List.fromJson(JSON.parse(event)); - console.log(">", event); - const l = await(List.fromIpfsHash(ipfs, event)); - // console.log("ITEMS RECEIVED", l.items.length); - - if(l.id === 'A') { - listB.join(l); - listC.join(l); - } else if(l.id === 'B') { - // listA.join(l); - // listC.join(l); - } else if(l.id === 'C') { - listB.join(l); - var timer = new Timer('a'); - listC.join(listB); - console.log("join took " + timer.stop(true) + " ms"); - console.log("Items:", listC.items.length); - // console.log(listC.toString()); - } - }); - - this.client1.on("message", handleMessage); - this.client2.on("message", handleMessage); - - let h = 0; - setInterval(async(() => { - listC.add("C--"+h); - const list = await(listC.getIpfsHash()); - this.client2.publish(hash, list); - h++; - }), 1000); - - let i = 0; - setInterval(async(() => { - let a = 0; - // for(let a = 0; a < 10; a ++) { - listB.add("B--"+(i+a)); - // } - const list = await(listB.getIpfsHash()); - this.client2.publish(hash, list); - i++; - }), 50); - -// let k = 0; -// setInterval(async(() => { -// listA.add("A--"+k); -// k++; -// listA.add("A--"+k); -// k++; -// listA.add("A--"+k); -// k++; -// this.client2.publish(hash, JSON.stringify(listA.toJson())); -// }), 100); -// }); -}); - -run(); From c10836b31f20e50c1851aa98811e97c3bd1843e6 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:08:50 +0200 Subject: [PATCH 23/30] Remove obsolete code --- src/Aggregator.js | 99 ------------------------------------------ src/BetterRequest.js | 65 --------------------------- src/HashCacheClient.js | 85 ------------------------------------ src/MemoryCache.js | 15 ------- src/OrbitClient.js | 35 +++++++-------- 5 files changed, 16 insertions(+), 283 deletions(-) delete mode 100644 src/Aggregator.js delete mode 100644 src/BetterRequest.js delete mode 100644 src/HashCacheClient.js delete mode 100644 src/MemoryCache.js diff --git a/src/Aggregator.js b/src/Aggregator.js deleted file mode 100644 index 7ed3356..0000000 --- a/src/Aggregator.js +++ /dev/null @@ -1,99 +0,0 @@ -'use strict'; - -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); -var Keystore = require('orbit-common/lib/Keystore'); -var Encryption = require('orbit-common/lib/Encryption'); -var HashCache = require('./HashCacheClient'); -var HashCacheItem = require('./HashCacheItem').EncryptedHashCacheItem; -var HashCacheOps = require('./HashCacheOps'); -var MemoryCache = require('./MemoryCache'); - -const pubkey = Keystore.getKeys().publicKey; -const privkey = Keystore.getKeys().privateKey; - -const DefaultAmount = 1; - -class Aggregator { - static fetchRecursive(ipfs, hash, password, options, currentAmount, deleted) { - const opts = { - amount: options.amount ? options.amount : DefaultAmount, - last: options.last ? options.last : null, - key: options.key ? options.key : null - }; - - let result = []; - let handledItems = deleted ? deleted : []; - - if(!currentAmount) currentAmount = 0; - - const item = await (this._fetchOne(ipfs, hash, password)); - - if(item) { - if((item.op === HashCacheOps.Put || item.op === HashCacheOps.Add) && !this._contains(handledItems, item.key)) { - if(!opts.key || (opts.key && opts.key === item.key)) { - result.push({ hash: hash, item: item }); - currentAmount ++; - handledItems.push(item.target); - } - } else if(item.op === HashCacheOps.Delete) { - handledItems.push(item.target); - } - - if(opts.key && item.key === opts.key) - return result; - - if(opts.last && hash === opts.last) - return result; - - if(!opts.last && opts.amount > -1 && currentAmount >= opts.amount) - return result; - - if(item.next) { - const items = this.fetchRecursive(ipfs, item.next, password, opts, currentAmount, handledItems); - result = result.concat(items); - } - } - - return result; - } - - static _fetchOne(ipfs, hash, password) { - // 1. Try fetching from memory - let data = MemoryCache.get(hash); - // TODO: 2. Try fetching from local cache - - // 3. Fetch from network - if(!data) - data = await (ipfsAPI.getObject(ipfs, hash)); - - // Cache the fetched item (encrypted) - MemoryCache.put(hash, data); - - // Decrypt the item - let item = HashCacheItem.fromEncrypted(data, pubkey, privkey, password); - - // TODO: add possibility to fetch content separately - // fetch and decrypt content - if(item.op === HashCacheOps.Add || item.op === HashCacheOps.Put) { - let payload = MemoryCache.get(item.target); - if(!payload) - payload = await (ipfsAPI.getObject(ipfs, item.target)); - - MemoryCache.put(item.target, payload); - - const contentEnc = JSON.parse(payload.Data)["content"]; - const contentDec = Encryption.decrypt(contentEnc, privkey, 'TODO: pubkey'); - item.Payload = contentDec; - } - - return item; - } - - static _contains(src, e) { - return src.filter((f) => f.toString() === e.toString()).length > 0; - } -} - -module.exports = Aggregator; diff --git a/src/BetterRequest.js b/src/BetterRequest.js deleted file mode 100644 index b080cab..0000000 --- a/src/BetterRequest.js +++ /dev/null @@ -1,65 +0,0 @@ -'use strict' - -var unirest = require('unirest') - -class Request { - constructor() { - this.url = ''; - this.method = 'GET'; - this.headers = {}; - this.body = {}; - } - - get(url) { - return this._init('GET', url); - } - - post(url) { - return this._init('POST', url); - } - - put(url) { - return this._init('PUT', url); - } - - delete(url) { - return this._init('DELETE', url); - } - - _init(method, url) { - this.url = url; - this.method = method; - this.body = {}; - this.headers = {}; - return this; - } - - set(key, value) { - this.headers[key] = value; - return this; - } - - send(body) { - this.body = body; - return this; - } - - end(callback) { - if(!this.url.startsWith("http")) - this.url = "http://" + this.url - - unirest(this.method, this.url) - .headers(this.headers) - .type('application/json') - .send(this.body) - .end((res) => { - if(res.error) - callback(res.body ? res.body.message : "Connection refused", null); - else - callback(null, res.body); - }); - } - -} - -module.exports = new Request(); diff --git a/src/HashCacheClient.js b/src/HashCacheClient.js deleted file mode 100644 index 05a9858..0000000 --- a/src/HashCacheClient.js +++ /dev/null @@ -1,85 +0,0 @@ -'use strict' - -var request = require('./BetterRequest'); - -class HashCacheClient { - constructor(host, credentials, info) { - this.host = host - this.credentials = credentials - this.info = info; - this.linkedList = this.linkedList.bind(this) - } - - linkedList(hash, password) { - return { - head: () => this._get(hash, password), - add: (head) => this._add(hash, password, head), - setMode: (mode) => this._setModes(hash, password, mode), - delete: () => this._delete(hash, password) - } - } - - _get(hash, password) { - return new Promise((resolve, reject) => { - request - .get(this.host + '/channel/' + hash) - .set('Authorization', this.credentials) - .send({ password: password }) - .end((err, res) => { this._resolveRequest(err, res, resolve, reject) }); - }) - } - - _add(hash, password, head) { - return new Promise((resolve, reject) => { - request - .put(this.host + '/channel/' + hash + '/add') - .set('Authorization', this.credentials) - .send({ head: head, password: password }) - .end((err, res) => { this._resolveRequest(err, res, resolve, reject) }); - }) - } - - _setModes(hash, password, modes) { - return new Promise((resolve, reject) => { - request - .post(this.host + '/channel/' + hash) - .set('Authorization', this.credentials) - .send({ modes: modes, password: password }) - .end((err, res) => { this._resolveRequest(err, res, resolve, reject) }); - }) - } - - _delete(hash, password) { - return new Promise((resolve, reject) => { - request - .delete(this.host + '/channel/' + hash) - .set('Authorization', this.credentials) - .send({ password: password }) - .end((err, res) => { this._resolveRequest(err, res, resolve, reject) }); - }) - } - - _resolveRequest(err, res, resolve, reject) { - if(err) - reject(res ? res : err.toString()); - else - resolve(res ? res : {}); - } -} - -module.exports = { - connect: (host, username, password) => { - var credentials = `Basic ${username}=${password}`; - return new Promise((resolve, reject) => { - request - .post(host + '/register') - .set('Authorization', credentials) - .end((err, res) => { - if(err) - reject(res ? res.body.message : err.toString()) - else - resolve(new HashCacheClient(host, credentials, res)); - }) - }) - } -} diff --git a/src/MemoryCache.js b/src/MemoryCache.js deleted file mode 100644 index d8cabf4..0000000 --- a/src/MemoryCache.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -let items = {}; - -class MemoryCache { - static put(hash, item) { - items[hash] = item; - } - - static get(hash) { - return items[hash]; - } -} - -module.exports = MemoryCache; diff --git a/src/OrbitClient.js b/src/OrbitClient.js index 42b083f..c3ac36a 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -1,25 +1,22 @@ 'use strict'; -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var Keystore = require('orbit-common/lib/Keystore'); -var Encryption = require('orbit-common/lib/Encryption'); -var ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); -var ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); -var HashCacheItem = require('./HashCacheItem').HashCacheItem; -var OrbitDBItem = require('./HashCacheItem').OrbitDBItem; -var HashCacheOps = require('./HashCacheOps'); -var ItemTypes = require('./ItemTypes'); -var MetaInfo = require('./MetaInfo'); -var Post = require('./Post'); -var Aggregator = require('./Aggregator'); -var PubSub = require('./PubSub'); -var Timer = require('../examples/Timer'); -const List = require('./list/OrbitList'); -const DataStore = require('./DataStore'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const Keystore = require('orbit-common/lib/Keystore'); +const Encryption = require('orbit-common/lib/Encryption'); +const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const OrbitDBItem = require('./HashCacheItem').OrbitDBItem; +const HashCacheOps = require('./HashCacheOps'); +const ItemTypes = require('./ItemTypes'); +const MetaInfo = require('./MetaInfo'); +const Post = require('./Post'); +const PubSub = require('./Pubsub'); +const List = require('./list/OrbitList'); +const DataStore = require('./DataStore'); -var pubkey = Keystore.getKeys().publicKey; -var privkey = Keystore.getKeys().privateKey; +const pubkey = Keystore.getKeys().publicKey; +const privkey = Keystore.getKeys().privateKey; class OrbitClient { constructor(ipfs) { From 61674d06990051451d6225c5c0458163a86ea089 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:09:22 +0200 Subject: [PATCH 24/30] Add WIP files --- WIP/MOCK.json | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++ WIP/test1.js | 70 ++++++++++++++++++ WIP/test2.js | 92 +++++++++++++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 WIP/MOCK.json create mode 100644 WIP/test1.js create mode 100644 WIP/test2.js diff --git a/WIP/MOCK.json b/WIP/MOCK.json new file mode 100644 index 0000000..54922e3 --- /dev/null +++ b/WIP/MOCK.json @@ -0,0 +1,198 @@ +// Data set grouped and sorted +"A" : [ + { "id": "A", "seq": 0, "ver": 0, "prev": null}, + { "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, + { "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, + { "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, + { "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, + { "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} +], +"B" : [ + { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, + { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} +], +"C" : [ + { "id": "C", "seq": 0, "ver": 0, "prev": null}, + { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, + { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, + { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} +] + + A B C + | +0.0 + | +0.1 + | +0.2 0.0 + | | +0.3 0.1 + | | +0.4 0.2 + | \ / | + | 1.0 | + | | | + | 1.1 | + | / | +2.0 | + \ | + \ | + \ | + 3.0 + +// expected order A +{ "id": "A", "seq": 0, "ver": 0, "prev": null}, +{ "id": "A", "seq": 0, "ver": 1, "prev": "A0.0"}, +{ "id": "A", "seq": 0, "ver": 2, "prev": "A0.1"}, +{ "id": "A", "seq": 0, "ver": 3, "prev": "A0.2"}, +{ "id": "A", "seq": 0, "ver": 4, "prev": "A0.3"}, + { "id": "C", "seq": 0, "ver": 0, "prev": null}, + { "id": "C", "seq": 0, "ver": 1, "prev": "C0.0"}, + { "id": "C", "seq": 0, "ver": 2, "prev": "C0.1"}, + { "id": "B", "seq": 1, "ver": 0, "prev": ["A0.4", "C0.2"]}, + { "id": "B", "seq": 1, "ver": 1, "prev": "B1.0"} +{ "id": "A", "seq": 2, "ver": 0, "prev": ["A0.4", "B1.1"]} + { "id": "C", "seq": 3, "ver": 0, "prev": ["C0.2", "A2.0]"]} + +"VersionClock": { + "seq": 0, + "ver": 0 +} + +"Item": { + "id": "", + "VersionClock": "", + "prev": [] +} + +"List": { + "items": [""] +} + +/* + list.add(data) { + this.ver ++; + const heads = _findHeads(); + const i = new Item(id, this.seq, this.ver, heads) + outgoing.push(data) + } +*/ + +/* + list.join(other) { + // increase seq on join, reset version + if(other.first.seq >= this.seq) + this.seq = other.first.seq + 1 + this.ver = 0 + + items = items.concat(outgoing.concat(other)) + items = items.sortBy("seq", "id", "ver") + outgoing = [] + } +*/ + +/* + nextHeads() { + referenced = [] + heads = all.groupBy("id").map((items) => items[items.length - 1]) + cleaned = heads.reverse().filter((e) => !isReferencedInChain(referenced, e)) + return cleaned; + } +*/ + +/* + isReferencedInChain(list, other) { + const res = other.map((o) => { + const ref = list.map((e) => !(e.id == o.id && e.seq == o.seq && e.ver == o.ver)) + if(!ref) + list.push(ref) + //return false + + //list.concat(list.filter((e) => !(e.id == o.id && e.seq == o.seq && e.ver == o.ver))) + if(o.prev) + ref = isReferencedInChain(list, o.prev) + + return ref + }) + return res.anyEqual(true) + } +*/ + + + A B C + 0.0 + | +0.0 0.1 + | | +0.1 0.2 + \ / | + 1.0 | + | | + 1.1 | + / | +2.0 | + \ | + \ | + \ | + 3.0 + +// Sequence, --> syncs to +listA.add("mango") // { "id": "A", "seq": 0, "ver": 0, "prev": null} +listA.add("banana") // { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +--> B + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} + +listC.add("apple") // { "id": "C", "seq": 0, "ver": 0, "prev": null} +listC.add("strawberry") // { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +listC.add("orange") // { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +--> A,B + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} + +listB.add("pineapple") // { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +listB.add("papaya") // { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} +--> A + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.1", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} + +listA.add("kiwi") // { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} +--> C + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} +// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} + +listC.add("blueberry") // { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} +--> A,B + +// A +// { "id": "A", "seq": 0, "ver": 0, "prev": null} +// { "id": "A", "seq": 0, "ver": 1, "prev": "A.0.0"} +// { "id": "C", "seq": 0, "ver": 0, "prev": null} +// { "id": "C", "seq": 0, "ver": 1, "prev": "C.0.0"} +// { "id": "C", "seq": 0, "ver": 2, "prev": "C.0.1"} +// { "id": "B", "seq": 1, "ver": 0, "prev": ["A.0.0", "C.0.2"]} +// { "id": "B", "seq": 1, "ver": 1, "prev": "B.1.0"} +// { "id": "A", "seq": 2, "ver": 0, "prev": ["A.0.1", "B1.1", "C0.2"]} +// { "id": "C", "seq": 3, "ver": 0, "prev": ["A.2.0", "C.0.2"]} diff --git a/WIP/test1.js b/WIP/test1.js new file mode 100644 index 0000000..da3caf9 --- /dev/null +++ b/WIP/test1.js @@ -0,0 +1,70 @@ +'use strict'; + +const _ = require('lodash'); +const Timer = require('./examples/Timer'); +const List = require('./src/list/List'); + +var run = () => { + var redis = require("redis"); + this.client1 = redis.createClient({ host: "localhost", port: 6379 }); + this.client2 = redis.createClient({ host: "localhost", port: 6379 }); + var hash = "ccc" + this.client1.subscribe(hash); + this.client1.subscribe(hash); + + + let listA = new List("A"); + let listB = new List("B"); + let listC = new List("C"); + + const handleMessage = (hash, event) => { + const l = List.fromJson(JSON.parse(event)); + // console.log("LIST", l); + + if(l.id === 'A') { + listB.join(l); + listC.join(l); + } else if(l.id === 'B') { + listA.join(l); + listC.join(l); + } else if(l.id === 'C') { + listA.join(l); + console.log("Items:", listA.items.length); + // console.log(JSON.stringify(listA, null, 1)); + } + + } + + this.client1.on("message", handleMessage); + this.client2.on("message", handleMessage); + + let h = 0; + setInterval(() => { + listC.add("C--"+h); + this.client2.publish(hash, JSON.stringify(listC.toJson())); + h++; + }, 1000); + + let i = 0; + setInterval(() => { + let a = 0; + for(let a = 0; a < 10; a ++) { + listB.add("B--"+(i+a)); + } + this.client2.publish(hash, JSON.stringify(listB.toJson())); + i++; + }, 20); + + let k = 0; + setInterval(() => { + listA.add("A--"+k); + k++; + listA.add("A--"+k); + k++; + listA.add("A--"+k); + k++; + this.client2.publish(hash, JSON.stringify(listA.toJson())); + }, 100); +}; + +run(); diff --git a/WIP/test2.js b/WIP/test2.js new file mode 100644 index 0000000..5bb9322 --- /dev/null +++ b/WIP/test2.js @@ -0,0 +1,92 @@ +'use strict'; + +const _ = require('lodash'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const ipfsDaemon = require('orbit-common/lib/ipfs-daemon'); +const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); +const List = require('./src/list/OrbitList'); +const Timer = require('./examples/Timer'); + +const startIpfs = async (() => { + return new Promise(async((resolve, reject) => { + const ipfsd = await(ipfsDaemon()); + resolve(ipfsd.daemon); + })); +}); + +let ipfs; + +var run = async(() => { + ipfs = await(startIpfs()); + + var redis = require("redis"); + this.client1 = redis.createClient({ host: "localhost", port: 6379 }); + this.client2 = redis.createClient({ host: "localhost", port: 6379 }); + var hash = "ccc" + this.client1.subscribe(hash); + this.client1.subscribe(hash); + + + let listA = new List("A", ipfs); + let listB = new List("B", ipfs); + let listC = new List("C", ipfs); + + const handleMessage = async((hash, event) => { + // const l = List.fromJson(JSON.parse(event)); + console.log(">", event); + const l = await(List.fromIpfsHash(ipfs, event)); + // console.log("ITEMS RECEIVED", l.items.length); + + if(l.id === 'A') { + listB.join(l); + listC.join(l); + } else if(l.id === 'B') { + // listA.join(l); + // listC.join(l); + } else if(l.id === 'C') { + listB.join(l); + var timer = new Timer('a'); + listC.join(listB); + console.log("join took " + timer.stop(true) + " ms"); + console.log("Items:", listC.items.length); + // console.log(listC.toString()); + } + }); + + this.client1.on("message", handleMessage); + this.client2.on("message", handleMessage); + + let h = 0; + setInterval(async(() => { + listC.add("C--"+h); + const list = await(listC.getIpfsHash()); + this.client2.publish(hash, list); + h++; + }), 1000); + + let i = 0; + setInterval(async(() => { + let a = 0; + // for(let a = 0; a < 10; a ++) { + listB.add("B--"+(i+a)); + // } + const list = await(listB.getIpfsHash()); + this.client2.publish(hash, list); + i++; + }), 50); + +// let k = 0; +// setInterval(async(() => { +// listA.add("A--"+k); +// k++; +// listA.add("A--"+k); +// k++; +// listA.add("A--"+k); +// k++; +// this.client2.publish(hash, JSON.stringify(listA.toJson())); +// }), 100); +// }); +}); + +run(); From 7aaadfff6f1e16de0678190142f7cd18ad661324 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:09:29 +0200 Subject: [PATCH 25/30] Rename examples --- examples/{pubsubBenchmark.js => benchmark.js} | 10 +- examples/{pubsubKeyValue.js => keyvalue.js} | 0 examples/readMessages.js | 122 ---------------- examples/{pubsubReader.js => reader.js} | 5 +- examples/writeMessages.js | 135 ------------------ examples/{pubsubWriter.js => writer.js} | 5 +- 6 files changed, 8 insertions(+), 269 deletions(-) rename examples/{pubsubBenchmark.js => benchmark.js} (87%) rename examples/{pubsubKeyValue.js => keyvalue.js} (100%) delete mode 100644 examples/readMessages.js rename examples/{pubsubReader.js => reader.js} (95%) delete mode 100644 examples/writeMessages.js rename examples/{pubsubWriter.js => writer.js} (92%) diff --git a/examples/pubsubBenchmark.js b/examples/benchmark.js similarity index 87% rename from examples/pubsubBenchmark.js rename to examples/benchmark.js index 01100d7..2f22fd1 100644 --- a/examples/pubsubBenchmark.js +++ b/examples/benchmark.js @@ -18,7 +18,7 @@ let run = (async(() => { const id = process.argv[2] ? process.argv[2] : 'a'; const channelName = 'c1'; - const channel = orbit.channel(channelName); + const db = orbit.channel(channelName); // Metrics let totalQueries = 0; @@ -39,14 +39,12 @@ let run = (async(() => { queriesPerSecond = 0; }, 1000); - setInterval(async(() => { - // while(true) { - let g = channel.add(id + totalQueries); + while(true) { + let g = db.add(id + totalQueries); totalQueries ++; lastTenSeconds ++; queriesPerSecond ++; - // } - }), 1000) + } } catch(e) { console.error("error:", e); diff --git a/examples/pubsubKeyValue.js b/examples/keyvalue.js similarity index 100% rename from examples/pubsubKeyValue.js rename to examples/keyvalue.js diff --git a/examples/readMessages.js b/examples/readMessages.js deleted file mode 100644 index aebc6dd..0000000 --- a/examples/readMessages.js +++ /dev/null @@ -1,122 +0,0 @@ -'use strict'; - -var async = require('asyncawait/async'); -var OrbitClient = require('../src/OrbitClient'); -var Timer = require('./Timer'); - -// Redis host -var host = 'localhost'; -var port = '6379' - -var username = 'testrunner'; -var password = ''; - -var util = require('util'); -var exec = require('child_process').exec; - -let run = (async(() => { - try { - // Connect - var orbit = OrbitClient.connect(host, port, username, password); - -/* var timer = new Timer(true); - - console.log("-------- KV store -------") - var channel = 'keyspace1' - // orbit.channel(channel, '').delete(); - orbit.channel(channel).put("key3", "this is the value you're looking for: " + new Date().getTime()); - var val = orbit.channel(channel).get("key3"); - console.log("key3:", val); - - orbit.channel(channel).put("key4", "this will be deleted"); - var val2 = orbit.channel(channel).get("key4"); - console.log("key4:", val2); - orbit.channel(channel).del({ key: "key4" }); - val2 = orbit.channel(channel).get("key4"); - console.log("key4:", val2); - - console.log("-------- EVENT log -------") - const c1 = 'c1'; - orbit.channel(c1).delete(); - var hash1 = orbit.channel(c1).add("hello1"); - var hash2 = orbit.channel(c1).add("hello2"); - - var items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - items = items.map((e) => { - return { key: e.item.key, val: e.item.Payload }; - }); - console.log(JSON.stringify(items, null, 2)); - - // console.log("--> remove", hash1); - // orbit.channel(c1).del({ key: hash1 }); - - items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - items = items.map((e) => { - return { key: e.item.key, val: e.item.Payload }; - }); - console.log(JSON.stringify(items, null, 2)); -*/ - const id = process.argv[2] ? process.argv[2] : 'a'; - const c1 = 'c1'; - const cc = orbit.channel(c1); - - let i = 0; - let seconds = 0; - let round = 0; - let lastTen = 0; - - // Metrics - setInterval(() => { - seconds ++; - - if(seconds % 10 === 0) { - console.log(`--> Average of ${lastTen/10} q/s in the last 10 seconds`) - lastTen = 0 - } - - console.log(`${round} queries per second, ${i} queries in ${seconds} seconds`) - round = 0; - }, 1000); - - while(true) { - cc.add(id + i); - - i ++; - lastTen ++; - round ++; - - // let items = cc.iterator({ limit: 10 }).collect(); - // items = items.map((e) => e.item); - // let g = items.filter((e) => e.Payload.startsWith(id)) - // let prev = -1; - // g.reverse().forEach((e) => { - // const a = parseInt(e.Payload.replace(id, '')); - // if(prev > -1 && prev + 1 !== a) { - // console.log("!! Missing message: " + id, prev + 1) - // process.exit(1); - // } - // prev = a; - // }) - } - -/* - // You can also get the event based on its hash - var value = orbit.channel(c1).get(hash2); - console.log("key:", hash2, "value:", value); -*/ - // console.log("--> remove", hash2); - // orbit.channel(c1).remove({ key: hash2 }); - - // items = orbit.channel(c1).iterator({ limit: -1 }).collect(); - // console.log(JSON.stringify(items, null, 2)); - - // process.exit(0); - - } catch(e) { - console.error("error:", e); - console.error(e.stack); - process.exit(1); - } -}))(); - -module.exports = run; diff --git a/examples/pubsubReader.js b/examples/reader.js similarity index 95% rename from examples/pubsubReader.js rename to examples/reader.js index 3653ba4..58a23e8 100644 --- a/examples/pubsubReader.js +++ b/examples/reader.js @@ -5,9 +5,8 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -// var host = '178.62.229.175'; -var host = 'localhost'; -var port = 6379; +var host = 'localhost'; +var port = 6379; var username = 'LambOfGod'; var password = ''; diff --git a/examples/writeMessages.js b/examples/writeMessages.js deleted file mode 100644 index 9dd24ab..0000000 --- a/examples/writeMessages.js +++ /dev/null @@ -1,135 +0,0 @@ -'use strict'; - -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var OrbitClient = require('../src/OrbitClient'); -var Timer = require('./Timer'); - -var host = 'localhost'; -var port = 6379; - -var username = 'LambOfGod'; -var password = ''; - -let run = (async(() => { - try { -/* var channel = 'hello-world-test1' - - // Connect - var orbit = OrbitClient.connect(host, username, password); - - // Delete channel and its data - var result = orbit.channel(channel, '').delete(); - - // Add the first message and delete it immediately - // orbit.channel(channel, '').put("hello world!"); - // var e = orbit.channel(channel, '').iterator({ limit: -1 }).collect()[0].hash; - // orbit.channel(channel, '').del(e); - orbit.channel(channel, '').put("key two", "hello world!!!"); - - var messages = 10; - var i = 1; - while(i <= messages) { - var timer = new Timer(true); - // Send a message - // var head = orbit.channel(channel, '').send(JSON.stringify({ omg: "hello" })); - var head = orbit.channel(channel, '').put("key one", "hello world " + i); - console.log(i, head, timer.stop() + "ms"); - - if(i === 4) { - console.log("remove", head); - // orbit.channel(channel, '').del(head); - } - - i ++; - } - - var items = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); - // console.log(items); - var e = orbit.channel(channel, '').iterator({ limit: -1 }).collect(); - orbit.channel(channel, '').del({ key: "key one" }); - // orbit.channel(channel, '').del(items[2].hash); // 97 - // orbit.channel(channel, '').del(items[3].hash); // 96 - // orbit.channel(channel, '').del(items[66].hash); // 34 - // orbit.channel(channel, '').del(items[items.length - 10].hash); // 11 - // orbit.channel(channel, '').del(items[items.length - 9].hash); // 10 - // orbit.channel(channel, '').del(items[items.length - 8].hash); // 9 -*/ - - var orbit = OrbitClient.connect(host, port, username, password); - const c1 = 'c1'; - const cc = orbit.channel(c1); - - let i = 1; - let id = 'b' - let running = false; - setInterval(async(() => { - if(!running) { - let timer = new Timer(true); - running = true; - - await(cc.add(id + i)); - - let items = cc.iterator({ limit: 10 }).collect(); - - var g = items.filter((e) => e.item.Payload.startsWith(id)) - var prev = -1; - g.reverse().forEach((e) => { - var a = parseInt(e.item.Payload.replace(id, '')); - if(prev > -1 && prev + 1 !== a) { - console.log("MISSSS!!!", prev + 1, items) - process.exit(1); - } - prev = a; - }) - - items = items.map((e) => { - return e.item.seq + " - " + e.item.Payload; - }); - console.log(JSON.stringify(items, null, 2)); - console.log(`Query ${i} took ${timer.stop(true)} ms`); - running = false; - i ++; - } - // while(true) { - // } - }), 1000); - - // setInterval(async(() => { - // if(!running) { - // let timer = new Timer(true); - // running = true; - - // await(cc.add(id + i)); - - // let items = cc.iterator({ limit: 10 }).collect(); - - // var g = items.filter((e) => e.item.Payload.startsWith(id)) - // var prev = -1; - // g.reverse().forEach((e) => { - // var a = parseInt(e.item.Payload.replace(id, '')); - // if(prev > -1 && prev + 1 !== a) { - // console.log("MISSSS!!!", prev + 1, items) - // process.exit(1); - // } - // prev = a; - // }) - - // items = items.map((e) => { - // return e.item.seq + " - " + e.item.Payload; - // }); - // console.log(JSON.stringify(items, null, 2)); - // console.log(`Query took ${timer.stop(true)} ms`); - // running = false; - // i ++; - // } - // }), 36); - - } catch(e) { - console.error("error:", e); - console.error(e.stack); - process.exit(1); - } -}))(); - -module.exports = run; diff --git a/examples/pubsubWriter.js b/examples/writer.js similarity index 92% rename from examples/pubsubWriter.js rename to examples/writer.js index 6270e0c..0a1b0f2 100644 --- a/examples/pubsubWriter.js +++ b/examples/writer.js @@ -5,9 +5,8 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -// var host = '178.62.229.175'; -var host = 'localhost'; -var port = 6379; +var host = 'localhost'; +var port = 6379; var username = process.argv[2] ? process.argv[2] : 'DankoJones'; var password = ''; From 0da4bf7aa6eec8e7237dc1b2832a785c1b815230 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:15:36 +0200 Subject: [PATCH 26/30] Cleanup examples --- examples/keyvalue.js | 11 +++++------ examples/reader.js | 4 +--- examples/writer.js | 1 + 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/examples/keyvalue.js b/examples/keyvalue.js index 0fb697e..eeda35a 100644 --- a/examples/keyvalue.js +++ b/examples/keyvalue.js @@ -5,9 +5,9 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); -// var host = '178.62.229.175'; -var host = 'localhost'; -var port = 6379; +// Redis +var host = 'localhost'; +var port = 6379; var username = 'LambOfGod'; var password = ''; @@ -19,7 +19,6 @@ let run = (async(() => { const channel = orbit.channel(c1); let count = 1; - let id = 'Log: Query ' let running = false; // setInterval(async(() => { @@ -27,9 +26,9 @@ let run = (async(() => { while(true) { running = true; - const key = "Lamb"; + const key = "username"; let timer = new Timer(true); - channel.put(key, "Of God " + count); + channel.put(key, "Lamb Of God " + count); let v = channel.get(key); console.log(`Query #${count} took ${timer.stop(true)} ms\n`); diff --git a/examples/reader.js b/examples/reader.js index 58a23e8..32f6ec9 100644 --- a/examples/reader.js +++ b/examples/reader.js @@ -5,6 +5,7 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); +// Redis var host = 'localhost'; var port = 6379; @@ -31,12 +32,9 @@ let run = (async(() => { const c = channel.iterator({ limit: -1 }).collect().length; let items = channel.iterator({ limit: 5 }).collect(); - // console.log(items); console.log("---------------------------------------------------") - // console.log("Id | Seq | Ver | Data") console.log("Key | Value") console.log("---------------------------------------------------") - // console.log(items.map((e) => `${e.id} | ${e.seq} | ${e.ver} | ${e.data}`).join("\n")); console.log(items.map((e) => `${e.payload.key} | ${e.payload.value}`).join("\n")); console.log("---------------------------------------------------") console.log(`Found ${items.length} items from ${c}\n`); diff --git a/examples/writer.js b/examples/writer.js index 0a1b0f2..5bbab7f 100644 --- a/examples/writer.js +++ b/examples/writer.js @@ -5,6 +5,7 @@ var await = require('asyncawait/await'); var OrbitClient = require('../src/OrbitClient'); var Timer = require('./Timer'); +// Redis var host = 'localhost'; var port = 6379; From a8dd379a233db9d5487af5e4eec11b96012456d9 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:35:54 +0200 Subject: [PATCH 27/30] Cleanup code --- examples/keyvalue.js | 52 +++++++++++++++++++------------------------- examples/reader.js | 16 +++++++------- src/DataStore.js | 10 --------- src/ItemTypes.js | 2 +- src/OrbitClient.js | 44 +++++-------------------------------- src/Post.js | 2 +- 6 files changed, 37 insertions(+), 89 deletions(-) diff --git a/examples/keyvalue.js b/examples/keyvalue.js index eeda35a..fd600bf 100644 --- a/examples/keyvalue.js +++ b/examples/keyvalue.js @@ -1,48 +1,40 @@ 'use strict'; -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var OrbitClient = require('../src/OrbitClient'); -var Timer = require('./Timer'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const OrbitClient = require('../src/OrbitClient'); +const Timer = require('./Timer'); // Redis -var host = 'localhost'; -var port = 6379; +const host = 'localhost'; +const port = 6379; -var username = 'LambOfGod'; -var password = ''; +const username = 'LambOfGod'; +const password = ''; let run = (async(() => { try { - var orbit = OrbitClient.connect(host, port, username, password); - const c1 = 'c1'; - const channel = orbit.channel(c1); + const orbit = OrbitClient.connect(host, port, username, password); + const channel = 'testing123'; + const db = orbit.channel(channel); let count = 1; - let running = false; - // setInterval(async(() => { - // if(!running) { while(true) { - running = true; + const key = "username"; + let timer = new Timer(true); + db.put(key, "Lamb Of God " + count); + let v = db.get(key); - const key = "username"; - let timer = new Timer(true); - channel.put(key, "Lamb Of God " + count); - let v = channel.get(key); - console.log(`Query #${count} took ${timer.stop(true)} ms\n`); + console.log("---------------------------------------------------") + console.log("Key | Value") + console.log("---------------------------------------------------") + console.log(`${key} | ${v}`); + console.log("---------------------------------------------------") + console.log(`Query #${count} took ${timer.stop(true)} ms\n`); - console.log("---------------------------------------------------") - console.log("Key | Value") - console.log("---------------------------------------------------") - console.log(`${key} | ${v}`); - console.log("---------------------------------------------------") - - running = false; - count ++; + count ++; } - // } - // }), 500); } catch(e) { console.error("error:", e); diff --git a/examples/reader.js b/examples/reader.js index 32f6ec9..65a13a6 100644 --- a/examples/reader.js +++ b/examples/reader.js @@ -1,16 +1,16 @@ 'use strict'; -var async = require('asyncawait/async'); -var await = require('asyncawait/await'); -var OrbitClient = require('../src/OrbitClient'); -var Timer = require('./Timer'); +const async = require('asyncawait/async'); +const await = require('asyncawait/await'); +const OrbitClient = require('../src/OrbitClient'); +const Timer = require('./Timer'); // Redis -var host = 'localhost'; -var port = 6379; +const host = 'localhost'; +const port = 6379; -var username = 'LambOfGod'; -var password = ''; +const username = 'LambOfGod'; +const password = ''; let run = (async(() => { try { diff --git a/src/DataStore.js b/src/DataStore.js index cc67a12..3a73721 100644 --- a/src/DataStore.js +++ b/src/DataStore.js @@ -3,12 +3,9 @@ const _ = require('lodash'); const async = require('asyncawait/async'); const await = require('asyncawait/await'); -const ipfsAPI = require('orbit-common/lib/ipfs-api-promised'); const OrbitList = require('./list/OrbitList'); const HashCacheOps = require('./HashCacheOps'); -var Timer = require('../examples/Timer'); - const DefaultAmount = 1; class DataStore { @@ -44,8 +41,6 @@ class DataStore { } _fetchRecursive(options, currentAmount, deleted, res) { - // console.log("-->") - // console.log("opts:", options, currentAmount) const opts = { amount: options && options.amount ? options.amount : DefaultAmount, first: options && options.first ? options.first : null, @@ -59,7 +54,6 @@ class DataStore { if(!currentAmount) currentAmount = 0; const item = this._fetchOne(currentAmount); - // console.log("ITEM", item) if(item && item.payload) { const wasHandled = _.includes(handledItems, item.payload.key); @@ -68,12 +62,10 @@ class DataStore { (!opts.first || (opts.first && (opts.first === item.payload.key && result.length === 0)) || (opts.first && (opts.first !== item.payload.key && result.length > 0)))) { - // console.log("PUSH!", item, currentAmount, result.length); result.push(item); handledItems.push(item.payload.key); } } else if(item.payload.op === HashCacheOps.Delete) { - // console.log("DELETE!", item); handledItems.push(item.payload.key); } @@ -82,7 +74,6 @@ class DataStore { if(opts.key && item.payload.key === opts.key) return result; - // console.log("ITEM", item.payload.key, opts.last) if(opts.last && item.payload.key === opts.last) return result; @@ -92,7 +83,6 @@ class DataStore { if(currentAmount >= this.list.items.length) return result; - // console.log("RES!", result) result = this._fetchRecursive(opts, currentAmount, handledItems, result); } diff --git a/src/ItemTypes.js b/src/ItemTypes.js index 0893a7f..bddb7a8 100644 --- a/src/ItemTypes.js +++ b/src/ItemTypes.js @@ -1,6 +1,6 @@ 'use strict'; -let ItemTypes = { +const ItemTypes = { Message: "text", Snippet: "snippet", File: "file", diff --git a/src/OrbitClient.js b/src/OrbitClient.js index c3ac36a..9517b7d 100644 --- a/src/OrbitClient.js +++ b/src/OrbitClient.js @@ -21,7 +21,6 @@ const privkey = Keystore.getKeys().privateKey; class OrbitClient { constructor(ipfs) { this._ipfs = ipfs; - this.network = {}; this.user = null; } @@ -30,16 +29,11 @@ class OrbitClient { this._pubsub.subscribe(hash, password, async((hash, message) => { const other = await(List.fromIpfsHash(this._ipfs, message)); - // console.log(">", other.id, other.seq, other.ver); - if(other.id !== this.user.username) { - // let timer = new Timer(true); + if(other.id !== this.user.username) this._store.join(other); - // console.log(`Join took ${timer.stop(true)} ms`); - } })); return { - // info: (options) => this._info(hash, password), delete: () => this._deleteChannel(hash, password), iterator: (options) => this._iterator(hash, password, options), setMode: (mode) => this._setMode(hash, password, mode), @@ -101,13 +95,11 @@ class OrbitClient { // Get messages messages = await(this._store.get(opts)); - // console.log("M", messages) // Remove the first/last item if greater/lesser than is set let startIndex = lt ? 1 : 0; let endIndex = gt ? messages.length - 1 : messages.length; messages = messages.slice(startIndex, endIndex) - // console.log("M2", messages) if(!reverse) messages.reverse(); @@ -122,7 +114,7 @@ class OrbitClient { _createMessage(channel, password, operation, key, value) { const size = -1; - const meta = new MetaInfo(ItemTypes.Message, size, this.user.id, new Date().getTime()); + const meta = new MetaInfo(ItemTypes.Message, size, this.user.username, new Date().getTime()); const item = new OrbitDBItem(operation, key, value, meta); const data = await (ipfsAPI.putObject(this._ipfs, JSON.stringify(item))); return data.Hash; @@ -149,8 +141,8 @@ class OrbitClient { const res = await(this._store.add(hash)); const listHash = await(this._store.list.getIpfsHash()); await(this._pubsub.publish(channel, listHash)); - return key; // return res; + return key; } _deleteChannel(channel, password) { @@ -158,38 +150,12 @@ class OrbitClient { return true; } - // _setMode(channel, password, modes) { - // let m = []; - // if(typeof modes !== 'Array') - // m.push(modes); - // else - // m = modes; - // // const res = await(this.client.linkedList(channel, password).setMode(m)); - // // return res.modes; - // return { todo: 'TODO!' } - // } - - // _info(channel, password) { - // var l = this._pubsub.latest(channel); - // return l; - // } - _connect(host, port, username, password) { return new Promise((resolve, reject) => { + this.user = { username: username, id: 'hello-todo' } this._pubsub = new PubSub(this._ipfs, host, port, username, password); - // this.client = this._pubsub._client; - // this.user = this.client.info.user; - this.user = { id: 'hello-todo', username: username } - this._store = new DataStore(username, this._ipfs); + this._store = new DataStore(username, this._ipfs); resolve(); - // this.network = { - // id: this.client.info.networkId, - // name: this.client.info.name, - // config: this.client.info.config - // }; - // setTimeout(() => { - // resolve(); - // }, 1000); }); } } diff --git a/src/Post.js b/src/Post.js index f9419ea..270cba7 100644 --- a/src/Post.js +++ b/src/Post.js @@ -1,6 +1,6 @@ 'use strict'; -var Encryption = require('orbit-common/lib/Encryption'); +const Encryption = require('orbit-common/lib/Encryption'); class Post { constructor(content) { From 11c573a380f24a1da2c35ac2007284abb86f30cc Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:36:22 +0200 Subject: [PATCH 28/30] Update circleci to run redis --- circle.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 0a53938..b568530 100644 --- a/circle.yml +++ b/circle.yml @@ -1,3 +1,5 @@ machine: node: - version: 4.2.2 \ No newline at end of file + version: 4.2.2 + services: + - redis \ No newline at end of file From 89a004dcd84e89943c2e787146588faf3c85aab8 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:36:32 +0200 Subject: [PATCH 29/30] Remove obsolete dependencies --- package.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index c303053..cb68bca 100644 --- a/package.json +++ b/package.json @@ -10,16 +10,12 @@ "main": "src/OrbitClient.js", "dependencies": { "asyncawait": "^1.0.1", - "bluebird": "^3.1.1", - "bs58": "^3.0.0", "lodash": "^4.3.0", "orbit-common": "^0.1.0", - "redis": "^2.4.2", - "unirest": "^0.4.2" + "redis": "^2.4.2" }, "devDependencies": { - "mocha": "^2.3.4", - "orbit-server": "^0.1.2" + "mocha": "^2.3.4" }, "scripts": { "test": "mocha" From 47e3999ecbda97f2c443177c6d1ae1696fa9c503 Mon Sep 17 00:00:00 2001 From: haad Date: Mon, 22 Feb 2016 12:36:38 +0200 Subject: [PATCH 30/30] Update README --- README.md | 64 ++++++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 4db1a98..69c8a6e 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,34 @@ Distributed, peer-to-peer* Key-Value Store and Event Log on IPFS. -Requires `orbit-server` to connect to. Get from https://github.com/haadcode/orbit-server. - -_* Currently requires a centralized server. This will change in the future as required p2p features land in IPFS_ +_* Currently requires a redis-server server for pubsub communication. This will change in the future as soon as IPFS provides pubsub_ ## Features - Distributed kv-store and event log database - Stores all data in IPFS -- Data encrypted on the wire and at rest -- Per channel access rights _Channel is similar to "table", "keyspace", "topic", "feed" or "collection" in other systems_ +## Example +``` +npm install +``` + +Key-Value store example: +``` +node examples/keyvalue.js +``` + +Event log example (run in separate shells): +``` +node examples/reader.js +node examples/writer.js +``` + ## API _See Usage below_ - connect(host, username, password) + connect(host, port, username, password) channel(name, password) @@ -42,27 +54,20 @@ _See Usage below_ .del({ key: }) // Remove entry - .setMode(modes) // Set channel modes, can be an object or an array of objects - - // { mode: "+r", params: { password: password } } // Set read mode - // { mode: "-r" } // Remove read-mode - // { mode: "+w", params: { ops: [orbit.user.id] } } // Set write-mode, only users in ops can write - // { mode: "-w" } // Remove write-mode - - .info() // Returns channel's current head and modes - - .delete() // Deletes the channel, all data will be "removed" (unassociated with the channel, actual data is not deleted) + .delete() // Deletes the channel, all data will be "removed" (unassociated with the channel, actual data is not deleted from ipfs) ## Usage ```javascript -var async = require('asyncawait/async'); -var OrbitClient = require('./OrbitClient'); +const async = require('asyncawait/async'); +const OrbitClient = require('./OrbitClient'); -var host = 'localhost:3006'; // orbit-server address +// Redis +const host = 'localhost'; +const port = 6379; async(() => { // Connect - const orbit = OrbitClient.connect(host, username, password); + const orbit = OrbitClient.connect(host, port, username, password); const channelName = 'hello-world'; const db = orbit.channel(channelName); @@ -90,16 +95,7 @@ async(() => { /* KV Store */ db.put('key1', 'hello world'); db.get('key1'); // returns "hello world" - db.remove('key1'); - - /* Modes */ - const password = 'hello'; - let channelModes; - channelModes = db.setMode({ mode: '+r', params: { password: password } }); // { modes: { r: { password: 'hello' } } } - const privateChannel = orbit.channel(channel, password); - channelModes = privateChannel.setMode({ mode: '+w', params: { ops: [orbit.user.id] } }); // { modes: { ... } } - channelModes = privateChannel.setMode({ mode: '-r' }); // { modes: { ... } } - channelModes = privateChannel.setMode({ mode: '-w' }); // { modes: {} } + db.del('key1'); /* Delete channel */ const result = db.delete(); // true | false @@ -108,7 +104,7 @@ async(() => { ### Development #### Run Tests -**Note!** Requires Aerospike, see http://www.aerospike.com/docs/operations/install/ +**Note!** Requires a redis-server at `localhost:6379` ``` npm test @@ -120,8 +116,4 @@ mocha -w ``` ### TODO -- Tests for remove(), put() and get() -- pubsub communication (use redis to mock ipfs pubsub) -- Local caching of messages -- Possibility to fetch content separately from data structure -- Use HTTPS instead of HTTP (channel password are sent in plaintext atm) +- Caching