Refactor data slicing/search. Remove DataStore. Rename and move files to their appropriate places.

This commit is contained in:
haad 2016-03-03 11:51:28 +01:00
parent 12df0ea5df
commit 03e3279766
18 changed files with 527 additions and 430 deletions

40
Vagrantfile vendored Normal file
View File

@ -0,0 +1,40 @@
Vagrant.configure(2) do |config|
config.vm.box = "digital_ocean"
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.define "benchmark" do |node|
node.vm.provision "shell", inline: <<-SHELL
sudo apt-get update
sudo apt-get install -y python gcc make g++ git
sudo apt-get install -y wget htop screen nano
wget --quiet https://nodejs.org/dist/v4.3.0/node-v4.3.0-linux-x64.tar.gz
tar -C /usr/local -zxf node-v4.3.0-linux-x64.tar.gz --strip 1
screen -X -S app quit
sudo rm -rf /orbit-db
sudo mkdir /orbit-db
cd /orbit-db
git clone https://github.com/haadcode/orbit-db.git
cd orbit-db
npm install --production
nohup screen -S app -d -m node examples/reader.js 178.62.241.75 benchmark b1 benchmark
SHELL
node.vm.provider :digital_ocean do |provider, override|
override.ssh.private_key_path = "~/.ssh/digital-ocean"
override.vm.box = 'digital_ocean'
override.vm.box_url = "https://github.com/smdahlen/vagrant-digitalocean/raw/master/box/digital_ocean.box"
provider.token = '20f523cae97809404ccb5c8877d84e580bb200cd6d7eb05d8d63b453b46c7cc8'
provider.image = 'ubuntu-14-04-x64'
provider.region = 'AMS3'
provider.size = '2GB'
end
end
end

View File

@ -10,6 +10,7 @@
"main": "src/OrbitClient.js",
"dependencies": {
"asyncawait": "^1.0.1",
"lazy.js": "^0.4.2",
"lodash": "^4.3.0",
"orbit-common": "^0.1.0",
"socket.io-client": "1.3.7"

View File

@ -1,93 +0,0 @@
'use strict';
const _ = require('lodash');
const async = require('asyncawait/async');
const await = require('asyncawait/await');
const OrbitList = require('./list/OrbitList');
const HashCacheOps = require('./HashCacheOps');
const DefaultAmount = 1;
class DataStore {
constructor(id, ipfs) {
this._ipfs = ipfs;
this.list = new OrbitList(id, this._ipfs);
}
add(hash) {
return this.list.add(hash);
}
join(other) {
this.list.join(other);
}
clear() {
this.list.clear();
}
get(options) {
return this._fetchRecursive(options);
}
_fetchRecursive(options, currentAmount, deleted, res) {
const opts = {
amount: options && options.amount ? options.amount : DefaultAmount,
first: options && options.first ? options.first : null,
last: options && options.last ? options.last : null,
key: options && options.key ? options.key : null
};
if(!currentAmount) currentAmount = 0;
if(!opts.first && !opts.last && !opts.key && opts.amount == -1)
return this.list.items.map((e) => e.fetchPayload()).reverse();
let result = res ? res : [];
let handledItems = deleted ? deleted : [];
let item;
// Fetch the item from ipfs
const node = this.list.items[this.list.items.length - currentAmount - 1];
if(node) item = await(node.fetchPayload());
const canAdd = (firstHash, key, foundItemsCount) => {
return (!opts.key || (opts.key && opts.key === item.payload.key)) &&
(!opts.first || (opts.first && (opts.first === item.payload.key && foundItemsCount === 0))
|| (opts.first && (opts.first !== item.payload.key && foundItemsCount > 0)))
};
if(item && item.payload) {
const wasHandled = _.includes(handledItems, item.payload.key); // Last Write Wins, if it was handled, ignore the rest
if((item.payload.op === HashCacheOps.Put || item.payload.op === HashCacheOps.Add) && !wasHandled) {
if(canAdd(opts.first, item.payload.key, result.length)) {
result.push(item);
handledItems.push(item.payload.key);
}
} else if(item.payload.op === HashCacheOps.Delete) {
handledItems.push(item.payload.key);
}
currentAmount ++;
if(opts.key && item.payload.key === opts.key)
return result;
if(opts.last && item.payload.key === opts.last)
return result;
if(!opts.last && opts.amount > -1 && result.length >= opts.amount)
return result;
if(currentAmount >= this.list.items.length)
return result;
result = this._fetchRecursive(opts, currentAmount, handledItems, result);
}
return result;
}
}
module.exports = DataStore;

View File

@ -1,9 +0,0 @@
'use strict';
const HashCacheOps = {
Add: "ADD",
Put: "PUT",
Delete: "DELETE"
};
module.exports = HashCacheOps;

View File

@ -1,20 +0,0 @@
'use strict';
var fs = require('fs');
var path = require('path');
var getAppPath = () => {
return process.type && process.env.ENV !== "dev" ? process.resourcesPath + "/app/" : process.cwd();
}
var privkey = fs.readFileSync(path.resolve(getAppPath(), 'keys/private.pem')).toString('ascii');
var pubkey = fs.readFileSync(path.resolve(getAppPath(), 'keys/public.pem')).toString('ascii');
module.exports = {
getKeys: (hash) => {
return {
publicKey: pubkey,
privateKey: privkey
};
}
}

View File

@ -1,48 +1,59 @@
'use strict';
const EventEmitter = require('events').EventEmitter;
const async = require('asyncawait/async');
const await = require('asyncawait/await');
const Keystore = require('orbit-common/lib/Keystore');
const Encryption = require('orbit-common/lib/Encryption');
const ipfsDaemon = require('orbit-common/lib/ipfs-daemon');
const ipfsAPI = require('orbit-common/lib/ipfs-api-promised');
const OrbitDBItem = require('./HashCacheItem').OrbitDBItem;
const HashCacheOps = require('./HashCacheOps');
const ItemTypes = require('./ItemTypes');
const MetaInfo = require('./MetaInfo');
const Post = require('./Post');
const PubSub = require('./PubSub');
const Operations = require('./list/Operations');
const List = require('./list/OrbitList');
const DataStore = require('./DataStore');
const OrbitDBItem = require('./db/OrbitDBItem');
const ItemTypes = require('./db/ItemTypes');
const MetaInfo = require('./db/MetaInfo');
const Post = require('./db/Post');
const PubSub = require('./PubSub');
const pubkey = Keystore.getKeys().publicKey;
const privkey = Keystore.getKeys().privateKey;
class OrbitClient {
constructor(ipfs) {
class OrbitDB {
constructor(ipfs, daemon) {
this._ipfs = ipfs;
this._store = {};
this._pubsub = null;
this.user = null;
this.network = null;
this.events = new EventEmitter();
}
channel(hash, password) {
channel(hash, password, subscribe) {
if(password === undefined) password = '';
if(subscribe === undefined) subscribe = true;
this._store[hash] = new List(this._ipfs, this.user.username);
// const onMessage = async((hash, message) => {
// // console.log("--> New head:", message)
// const other = List.fromIpfsHash(this._ipfs, message);
// // if(other.id !== this.user.username) {
// this._store[hash].join(other);
// // }
// this.events.emit('data', hash, message);
// });
const onMessage = async((hash, message) => {
const other = await(List.fromIpfsHash(this._ipfs, message));
if(other.id !== this.user.username) {
this._store.join(other);
// console.log("--> Head:", message)
if(message && this._store[hash]) {
const other = List.fromIpfsHash(this._ipfs, message);
this._store[hash].join(other);
}
this.events.emit('data', hash, message);
});
const onLatest = async((hash, message) => {
console.log("--> Received latest list:", message)
if(message) {
const other = await(List.fromIpfsHash(this._ipfs, message));
this._store.join(other);
}
});
this._pubsub.subscribe(hash, password, onMessage, onLatest);
if(subscribe)
this._pubsub.subscribe(hash, password, onMessage, onMessage);
return {
iterator: (options) => this._iterator(hash, password, options),
@ -59,6 +70,13 @@ class OrbitClient {
}
}
disconnect() {
this._pubsub.disconnect();
this._store = {};
this.user = null;
this.network = null;
}
_iterator(channel, password, options) {
const messages = this._getMessages(channel, password, options);
let currentIndex = 0;
@ -81,38 +99,10 @@ class OrbitClient {
}
_getMessages(channel, password, options) {
let messages = [];
if(!options) options = {};
// Options
let limit = options.limit ? options.limit : 1;
const gt = options.gt ? options.gt : null;
const gte = options.gte ? options.gte : null;
const lt = options.lt ? options.lt : null;
const lte = options.lte ? options.lte : null;
const reverse = options.reverse ? options.reverse : false;
const key = options.key ? options.key : null;
if((gt || lt) && limit > -1) limit += 1;
const opts = {
amount: limit,
first: lte ? lte : lt,
last: gte ? gte : gt,
key: key
};
// Get messages
messages = await(this._store.get(opts));
// Remove the first/last item if greater/lesser than is set
let startIndex = lt ? 1 : 0;
let endIndex = gt ? messages.length - 1 : messages.length;
messages = messages.slice(startIndex, endIndex)
if(!reverse) messages.reverse();
let opts = options || {};
Object.assign(opts, { amount: opts.limit || 1 });
let messages = await(this._store[channel].findAll(opts));
if(opts.reverse) messages.reverse();
return messages;
}
@ -137,45 +127,43 @@ class OrbitClient {
_add(channel, password, data) {
const post = await(this._publish(data));
const key = post.Hash;
return await(this._createOperation(channel, password, HashCacheOps.Add, key, post.Hash, data));
return await(this._createOperation(channel, password, Operations.Add, key, post.Hash, data));
}
_put(channel, password, key, data) {
const post = await(this._publish(data));
return await(this._createOperation(channel, password, HashCacheOps.Put, key, post.Hash));
return await(this._createOperation(channel, password, Operations.Put, key, post.Hash));
}
_remove(channel, password, hash) {
return await(this._createOperation(channel, password, HashCacheOps.Delete, hash, null));
return await(this._createOperation(channel, password, Operations.Delete, hash, null));
}
_createOperation(channel, password, operation, key, value, data) {
var create = async(() => {
var createOperation = async(() => {
return new Promise(async((resolve, reject) => {
const hash = this._createMessage(channel, password, operation, key, value);
const res = await(this._store.add(hash));
const listHash = await(this._store.list.getIpfsHash());
const res = await(this._store[channel].add(hash));
const listHash = await(this._store[channel].ipfsHash);
await(this._pubsub.publish(channel, listHash));
resolve();
}));
})
await(create());
await(createOperation());
return key;
// return res;
}
_deleteChannel(channel, password) {
this._store.clear();
this._store[channel].clear();
return true;
}
_connect(host, port, username, password) {
return new Promise((resolve, reject) => {
this.user = { username: username, id: 'hello-todo' }
this._pubsub = new PubSub(this._ipfs, host, port, username, password);
this._store = new DataStore(username, this._ipfs);
resolve();
});
this._pubsub = new PubSub(this._ipfs);
await(this._pubsub.connect(host, port, username, password));
this.user = { username: username, id: 'TODO: user id' }
this.network = { host: host, port: port, name: 'TODO: network name' }
}
}
@ -183,10 +171,10 @@ class OrbitClientFactory {
static connect(host, port, username, password, ipfs) {
if(!ipfs) {
let ipfsd = await(ipfsDaemon());
ipfs = ipfsd.daemon;
ipfs = ipfsd.ipfs;
}
const client = new OrbitClient(ipfs);
const client = new OrbitDB(ipfs);
await(client._connect(host, port, username, password))
return client;
}

View File

@ -4,25 +4,43 @@ const io = require('socket.io-client');
const List = require('./list/OrbitList');
class Pubsub {
constructor(ipfs, host, port, username, password) {
constructor(ipfs) {
this.ipfs = ipfs;
this._subscriptions = {};
this._socket = io(`http://${host}:${port}`);
this._socket.on('connect', (socket) => console.log(`Connected to http://${host}:${port}`));
this._socket.on('disconnect', (socket) => console.log(`Disconnected from http://${host}:${port}`));
this._socket.on('error', (e) => console.log('error:', e));
this._socket.on('message', this._handleMessage.bind(this));
this._socket.on('latest', (hash, message) => {
console.log(">", hash, message);
if(this._subscriptions[hash]) {
this._subscriptions[hash].head = message;
this.onConnected = null;
this.onConnectionError = null;
}
if(this._subscriptions[hash].onLatest)
this._subscriptions[hash].onLatest(hash, message);
}
connect(host, port, username, password) {
return new Promise((resolve, reject) => {
this._socket = io.connect(`http://${host}:${port}`, { 'forceNew': true });
this._socket.on('connect', resolve);
this._socket.on('connect_error', (err) => reject(new Error(`Connection refused to ${host}:${port}`)));
// TODO: cleanup
this._socket.on('disconnect', (socket) => {
// console.log(`Disconnected from http://${host}:${port}`)
});
this._socket.on('error', (e) => console.log('error:', e));
this._socket.on('message', this._handleMessage.bind(this));
this._socket.on('latest', (hash, message) => {
console.log(">", hash, message);
if(this._subscriptions[hash]) {
this._subscriptions[hash].head = message;
if(this._subscriptions[hash].onLatest)
this._subscriptions[hash].onLatest(hash, message);
}
});
});
}
disconnect() {
if(this._socket)
this._socket.disconnect();
}
subscribe(hash, password, callback, onLatest) {
if(!this._subscriptions[hash]) {
this._subscriptions[hash] = { head: null, callback: callback, onLatest: onLatest };

View File

@ -64,8 +64,4 @@ class EncryptedHashCacheItem extends HashCacheItem {
}
}
*/
module.exports = {
OrbitDBItem: OrbitDBItem,
// HashCacheItem: HashCacheItem,
// EncryptedHashCacheItem: EncryptedHashCacheItem
};
module.exports = OrbitDBItem;

View File

@ -12,6 +12,13 @@ class List {
this._currentBatch = [];
}
clear() {
this._items = [];
this._currentBatch = [];
this.seq = 0;
this.ver = 0;
}
get compactId() {
return "" + this.id + "." + this.seq + "." + this.ver;
}
@ -45,8 +52,7 @@ class List {
}
_isReferencedInChain(all, item) {
const isReferenced = _.findLast(all, (e) => Node.hasChild(e, item)) !== undefined;
return isReferenced;
return _.findLast(all, (e) => Node.hasChild(e, item)) !== undefined;
}
toJson() {

10
src/list/Operations.js Normal file
View File

@ -0,0 +1,10 @@
'use strict';
const DBOperations = {
Add: "ADD",
Put: "PUT",
Delete: "DELETE",
isUpdate: (op) => op === "ADD" || op === "PUT"
};
module.exports = DBOperations;

View File

@ -1,21 +1,22 @@
'use strict';
const _ = require('lodash');
const async = require('asyncawait/async');
const await = require('asyncawait/await');
const ipfsAPI = require('orbit-common/lib/ipfs-api-promised');
const List = require('./List');
const Node = require('./OrbitNode');
const _ = require('lodash');
const Lazy = require('lazy.js');
const async = require('asyncawait/async');
const await = require('asyncawait/await');
const ipfsAPI = require('orbit-common/lib/ipfs-api-promised');
const List = require('./List');
const Node = require('./OrbitNode');
const Operations = require('./Operations');
const MaxBatchSize = 10; // How many items per sequence. Saves a snapshot to ipfs in batches of this many items.
const MaxHistory = 32; // How many items to fetch in the chain per join
const MaxBatchSize = 10; // How many items per sequence. Saves a snapshot to ipfs in batches of this many items.
const MaxHistory = 1000; // How many items to fetch in the chain per join
// class IPFSLWWSet extends LWWSet {
class OrbitList extends List {
constructor(id, ipfs) {
super(id);
constructor(ipfs, id, seq, ver, items) {
super(id, seq, ver, items);
this._ipfs = ipfs;
this.hash = null;
this.next = null;
}
add(data) {
@ -45,7 +46,7 @@ class OrbitList extends List {
all.push(hash);
const item = Node.fromIpfsHash(this._ipfs, hash);
res.push(item);
item.heads.map((head) => fetchRecursive(head, amount - 1, all, res));
item.heads.map((head) => fetchRecursive(head, amount, all, res));
}
return res;
@ -63,29 +64,68 @@ class OrbitList extends List {
const idx = indices.length > 0 ? Math.max(_.max(indices) + 1, 0) : 0;
this._items.splice(idx, 0, item)
});
// console.log("--> Fetched", MaxHistory, "items from the history\n");
// console.log("--> Fetched", res.length, "items from the history\n");
}
clear() {
this._items = [];
this._currentBatch = [];
// The LWW-set query interface
findAll(opts) {
let list = Lazy(this.items);
const hash = (opts.gt ? opts.gt : (opts.gte ? opts.gte : (opts.lt ? opts.lt : opts.lte)));
const amount = opts.amount ? (opts.amount && opts.amount > -1 ? opts.amount : this.items.length) : 1;
// Last-Write-Wins set
let handled = [];
const _createLWWSet = (f) => {
const wasHandled = _.findIndex(handled, (b) => b === f.payload.key) > -1;
if(!wasHandled) handled.push(f.payload.key);
if(Operations.isUpdate(f.payload.op) && !wasHandled)
return f;
return null;
};
// Find an item from the lazy sequence (list)
const _findFrom = (sequence, key, amount, inclusive) => {
return sequence
.map((f) => await(f.fetchPayload())) // IO - fetch the actual OP from ipfs. consider merging with LL.
.skipWhile((f) => key && f.payload.key !== key) // Drop elements until we have the first one requested
.drop(!inclusive ? 1 : 0) // Drop the 'gt/lt' item, include 'gte/lte' item
.map(_createLWWSet) // Return items as LWW (ignore values after the first found)
.filter((f) => f !== null) // Make sure we don't have empty ones
.take(amount)
};
// Key-Value
if(opts.key)
return _findFrom(list.reverse(), opts.key, 1, true).toArray();
// Greater than case
if(opts.gt || opts.gte)
return _findFrom(list, hash, amount, opts.gte || opts.lte).toArray();
// Lower than and lastN case, search latest first by reversing the sequence
return _findFrom(list.reverse(), hash, amount, opts.lte || opts.gte || (!opts.lt && !opts.gt)).reverse().toArray();
}
getIpfsHash() {
return new Promise(async((resolve, reject) => {
var data = await(this.toJson())
const list = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(data)));
resolve(list.Hash);
}));
get ipfsHash() {
const toIpfs = async(() => {
return new Promise(async((resolve, reject) => {
var data = await(this.toJson())
const list = await(ipfsAPI.putObject(this._ipfs, JSON.stringify(data)));
resolve(list.Hash);
}));
});
return await(toIpfs());
}
static fromIpfsHash(ipfs, hash) {
return new Promise(async((resolve, reject) => {
const l = await(ipfsAPI.getObject(ipfs, hash));
const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data));
resolve(list);
}));
const fromIpfs = async(() => {
return new Promise(async((resolve, reject) => {
const l = await(ipfsAPI.getObject(ipfs, hash));
const list = OrbitList.fromJson(ipfs, JSON.parse(l.Data));
resolve(list);
}));
});
return await(fromIpfs());
}
toJson() {
@ -100,11 +140,8 @@ class OrbitList extends List {
}
static fromJson(ipfs, json) {
const items = Object.keys(json.items).map((f) => {
const hash = json.items[f];
return Node.fromIpfsHash(ipfs, hash);
});
return new List(json.id, json.seq, json.ver, items);
const items = Object.keys(json.items).map((f) => Node.fromIpfsHash(ipfs, json.items[f]));
return new OrbitList(ipfs, json.id, json.seq, json.ver, items);
}
static get batchSize() {
@ -112,8 +149,7 @@ class OrbitList extends List {
}
_isReferencedInChain(all, item) {
const isReferenced = _.findLast(all, (e) => Node.hasChild(e, item)) !== undefined;
return isReferenced;
return _.findLast(all, (e) => Node.hasChild(e, item)) !== undefined;
}
_commit() {

View File

@ -31,6 +31,7 @@ class OrbitNode extends Node {
return new Promise(async((resolve, reject) => {
await(this._getPayload());
resolve({ hash: this.data, payload: this.Payload });
// resolve(this);
}));
}

View File

@ -8,11 +8,13 @@ const OrbitClient = require('../src/OrbitClient');
// Orbit
const host = 'localhost';
const port = 6379;
const port = 3333;
const username = 'testrunner';
const password = '';
describe('Orbit Client', () => {
describe('Orbit Client', function() {
this.timeout(3000);
let client, db;
let items = [];
@ -20,17 +22,26 @@ describe('Orbit Client', () => {
before(async((done) => {
client = OrbitClient.connect(host, port, username, password);
db = client.channel(channel);
db = client.channel(channel, '', false);
db.delete();
done();
}));
after(async((done) => {
if(db) db.delete();
db.delete();
client.disconnect();
done();
}));
describe('Add events', function() {
let items2 = [];
const itemCount = 5;
after(async((done) => {
db.delete();
done();
}));
it('adds an item to an empty channel', async((done) => {
const head = db.add('hello');
assert.notEqual(head, null);
@ -62,7 +73,7 @@ describe('Orbit Client', () => {
it('adds an item that is > 256 bytes', async((done) => {
let msg = new Buffer(1024);
msg.fill('a')
const hash = db.add(msg.toString());
const hash = await(db.add(msg.toString()));
assert.notEqual(hash, null);
assert.equal(hash.startsWith('Qm'), true);
assert.equal(hash.length, 46);
@ -71,9 +82,20 @@ describe('Orbit Client', () => {
});
describe('Delete events', function() {
it('deletes an item when only one item in the database', async((done) => {
before(async((done) => {
db.delete();
const head = db.add('hello1');
let items = db.iterator().collect();
assert.equal(items.length, 0);
done();
}));
after(async((done) => {
db.delete();
done();
}));
it('deletes an item when only one item in the database', async((done) => {
const head = db.add('hello-');
let item = db.iterator().collect();
const delop = db.del(head);
const items = db.iterator().collect();
@ -83,6 +105,7 @@ describe('Orbit Client', () => {
}));
it('deletes an item when two items in the database', async((done) => {
db.delete();
db.add('hello1');
const head = db.add('hello2');
db.del(head);
@ -112,6 +135,7 @@ describe('Orbit Client', () => {
describe('Iterator', function() {
let items = [];
let items2 = [];
const itemCount = 5;
before(async((done) => {
@ -123,6 +147,12 @@ describe('Orbit Client', () => {
done();
}));
after(async((done) => {
db.delete();
done();
}));
describe('Defaults', function() {
it('returns an iterator', async((done) => {
const iter = db.iterator();
@ -148,13 +178,19 @@ describe('Orbit Client', () => {
}));
it('implements Iterator interface', async((done) => {
db.delete();
for(let i = 0; i < itemCount; i ++) {
const hash = db.add('hello' + i);
items2.push(hash);
}
const iter = db.iterator({ limit: -1 });
let messages = [];
for(let i of iter)
messages.push(i.hash);
assert.equal(messages.length, items.length);
assert.equal(messages.length, items2.length);
done();
}));
@ -162,7 +198,7 @@ describe('Orbit Client', () => {
const iter = db.iterator();
const first = iter.next().value;
const second = iter.next().value;
assert.equal(first.payload.key, items[items.length - 1]);
assert.equal(first.payload.key, items2[items.length - 1]);
assert.equal(second, null);
assert.equal(first.payload.value, 'hello4');
done();
@ -170,6 +206,23 @@ describe('Orbit Client', () => {
});
describe('Collect', function() {
let items2;
before(async((done) => {
db.delete();
items2 = [];
for(let i = 0; i < itemCount; i ++) {
const hash = db.add('hello' + i);
items2.push(hash);
}
done();
}));
after(async((done) => {
db.delete();
items2 = [];
done();
}));
it('returns all items', async((done) => {
const messages = db.iterator({ limit: -1 }).collect();
assert.equal(messages.length, items.length);
@ -192,11 +245,28 @@ describe('Orbit Client', () => {
});
describe('Options: limit', function() {
let items2;
before(async((done) => {
db.delete();
items2 = [];
for(let i = 0; i < itemCount; i ++) {
const hash = db.add('hello' + i);
items2.push(hash);
}
done();
}));
after(async((done) => {
db.delete();
items2 = [];
done();
}));
it('returns 1 item when limit is 0', async((done) => {
const iter = db.iterator({ limit: 0 });
const iter = db.iterator({ limit: 1 });
const first = iter.next().value;
const second = iter.next().value;
assert.equal(first.payload.key, items[items.length - 1]);
assert.equal(first.payload.key, items2[items.length - 1]);
assert.equal(second, null);
done();
}));
@ -205,7 +275,7 @@ describe('Orbit Client', () => {
const iter = db.iterator({ limit: 1 });
const first = iter.next().value;
const second = iter.next().value;
assert.equal(first.payload.key, items[items.length - 1]);
assert.equal(first.payload.key, items2[items.length - 1]);
assert.equal(second, null);
done();
}));
@ -216,9 +286,9 @@ describe('Orbit Client', () => {
const second = iter.next().value;
const third = iter.next().value;
const fourth = iter.next().value;
assert.equal(first.payload.key, items[items.length - 3]);
assert.equal(second.payload.key, items[items.length - 2]);
assert.equal(third.payload.key, items[items.length - 1]);
assert.equal(first.payload.key, items2[items.length - 3]);
assert.equal(second.payload.key, items2[items.length - 2]);
assert.equal(third.payload.key, items2[items.length - 1]);
assert.equal(fourth, null);
done();
}));
@ -229,8 +299,8 @@ describe('Orbit Client', () => {
.map((e) => e.payload.key);
messages.reverse();
assert.equal(messages.length, items.length);
assert.equal(messages[0], items[items.length - 1]);
assert.equal(messages.length, items2.length);
assert.equal(messages[0], items2[items2.length - 1]);
done();
}));
@ -239,8 +309,8 @@ describe('Orbit Client', () => {
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, items.length);
assert.equal(messages[0], items[0]);
assert.equal(messages.length, items2.length);
assert.equal(messages[0], items2[0]);
done();
}));
@ -249,74 +319,107 @@ describe('Orbit Client', () => {
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, items.length);
assert.equal(messages[0], items[0]);
assert.equal(messages.length, items2.length);
assert.equal(messages[0], items2[0]);
done();
}));
});
describe('Options: reverse', function() {
let items2;
before(async((done) => {
db.delete();
items2 = [];
for(let i = 0; i < itemCount; i ++) {
const hash = db.add('hello' + i);
items2.push(hash);
}
done();
}));
after(async((done) => {
db.delete();
items2 = [];
done();
}));
it('returns all items reversed', async((done) => {
const messages = db.iterator({ limit: -1, reverse: true })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, items.length);
assert.equal(messages[0], items[items.length - 1]);
assert.equal(messages.length, items2.length);
assert.equal(messages[0], items2[items.length - 1]);
done();
}));
});
describe('Options: ranges', function() {
let items2;
before(async((done) => {
db.delete();
items2 = [];
for(let i = 0; i < itemCount; i ++) {
const hash = db.add('hello' + i);
items2.push(hash);
}
done();
}));
after(async((done) => {
db.delete();
items2 = [];
done();
}));
describe('gt & gte', function() {
it('returns 1 item when gte is the head', async((done) => {
const messages = db.iterator({ gte: _.last(items), limit: -1 })
const messages = db.iterator({ gte: _.last(items2), limit: -1 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, 1);
assert.equal(messages[0], items[items.length -1]);
assert.equal(messages[0], items2[items.length -1]);
done();
}));
it('returns 0 items when gt is the head', async((done) => {
const messages = db.iterator({ gt: _.last(items) }).collect();
const messages = db.iterator({ gt: _.last(items2) }).collect();
assert.equal(messages.length, 0);
done();
}));
it('returns 2 item when gte is defined', async((done) => {
const gte = items[items.length - 2];
const gte = items2[items2.length - 2];
const messages = db.iterator({ gte: gte, limit: -1 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, 2);
assert.equal(messages[0], items[items.length - 2]);
assert.equal(messages[1], items[items.length - 1]);
assert.equal(messages[0], items2[items2.length - 2]);
assert.equal(messages[1], items2[items2.length - 1]);
done();
}));
it('returns all items when gte is the root item', async((done) => {
const messages = db.iterator({ gte: items[0], limit: -1 })
const messages = db.iterator({ gte: items2[0], limit: -1 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, items.length);
assert.equal(messages[0], items[0]);
assert.equal(messages[messages.length - 1], items[items.length - 1]);
assert.equal(messages.length, items2.length);
assert.equal(messages[0], items2[0]);
assert.equal(messages[messages.length - 1], items2[items2.length - 1]);
done();
}));
it('returns items when gt is the root item', async((done) => {
const messages = db.iterator({ gt: items[0], limit: -1 })
const messages = db.iterator({ gt: items2[0], limit: -1 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, itemCount - 1);
assert.equal(messages[0], items[1]);
assert.equal(messages[3], items[items.length - 1]);
assert.equal(messages[0], items2[1]);
assert.equal(messages[3], items2[items.length - 1]);
done();
}));
@ -340,73 +443,73 @@ describe('Orbit Client', () => {
describe('lt & lte', function() {
it('returns one item after head when lt is the head', async((done) => {
const messages = db.iterator({ lt: _.last(items) })
const messages = db.iterator({ lt: _.last(items2) })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, 1);
assert.equal(messages[0], items[items.length - 2]);
assert.equal(messages[0], items2[items2.length - 2]);
done();
}));
it('returns all items when lt is head and limit is -1', async((done) => {
const messages = db.iterator({ lt: _.last(items), limit: -1 })
const messages = db.iterator({ lt: _.last(items2), limit: -1 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, items.length - 1);
assert.equal(messages[0], items[0]);
assert.equal(messages[messages.length - 1], items[items.length - 2]);
assert.equal(messages.length, items2.length - 1);
assert.equal(messages[0], items2[0]);
assert.equal(messages[messages.length - 1], items2[items2.length - 2]);
done();
}));
it('returns 3 items when lt is head and limit is 3', async((done) => {
const messages = db.iterator({ lt: _.last(items), limit: 3 })
const messages = db.iterator({ lt: _.last(items2), limit: 3 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, 3);
assert.equal(messages[0], items[items.length - 4]);
assert.equal(messages[2], items[items.length - 2]);
assert.equal(messages[0], items2[items2.length - 4]);
assert.equal(messages[2], items2[items2.length - 2]);
done();
}));
it('returns null when lt is the root item', async((done) => {
const messages = db.iterator({ lt: items[0] }).collect();
const messages = db.iterator({ lt: items2[0] }).collect();
assert.equal(messages.length, 0);
done();
}));
it('returns one item when lte is the root item', async((done) => {
const messages = db.iterator({ lte: items[0] })
const messages = db.iterator({ lte: items2[0] })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, 1);
assert.equal(messages[0], items[0]);
assert.equal(messages[0], items2[0]);
done();
}));
it('returns all items when lte is the head', async((done) => {
const messages = db.iterator({ lte: _.last(items), limit: -1 })
const messages = db.iterator({ lte: _.last(items2), limit: -1 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, itemCount);
assert.equal(messages[0], items[0]);
assert.equal(messages[4], _.last(items));
assert.equal(messages[0], items2[0]);
assert.equal(messages[4], _.last(items2));
done();
}));
it('returns 3 items when lte is the head', async((done) => {
const messages = db.iterator({ lte: _.last(items), limit: 3 })
const messages = db.iterator({ lte: _.last(items2), limit: 3 })
.collect()
.map((e) => e.payload.key);
assert.equal(messages.length, 3);
assert.equal(messages[0], items[items.length - 3]);
assert.equal(messages[1], items[items.length - 2]);
assert.equal(messages[2], _.last(items));
assert.equal(messages[0], items2[items2.length - 3]);
assert.equal(messages[1], items2[items2.length - 2]);
assert.equal(messages[2], _.last(items2));
done();
}));
});
@ -414,82 +517,6 @@ describe('Orbit Client', () => {
});
/*
describe('Modes', function() {
var password = 'hello';
it('sets read mode', async((done) => {
try {
var mode = {
mode: "+r",
params: {
password: password
}
};
var modes = db.setMode(mode)
assert.notEqual(modes.r, null);
assert.equal(modes.r.password, password);
} catch(e) {
assert.equal(e, null);
}
done();
}));
it('can\'t read with wrong password', async((done) => {
try {
var modes = orbit.channel(channel, 'invalidpassword').iterator();
assert.equal(true, false);
} catch(e) {
assert.equal(e, 'Unauthorized');
}
done();
}));
it('sets write mode', async((done) => {
try {
var mode = {
mode: "+w",
params: {
ops: [orbit.user.id]
}
};
var modes = orbit.channel(channel, password).setMode(mode);
assert.notEqual(modes.w, null);
assert.equal(modes.w.ops[0], orbit.user.id);
} catch(e) {
assert.equal(e, null);
}
done();
}));
it('can\'t write when user not an op', async((done) => {
// TODO
done();
}));
it('removes write mode', async((done) => {
try {
var modes = orbit.channel(channel, password).setMode({ mode: "-w" });
assert.equal(modes.w, null);
} catch(e) {
assert.equal(e, null);
}
done();
}));
it('removes read mode', async((done) => {
try {
var modes = orbit.channel(channel, password).setMode({ mode: "-r" });
assert.equal(modes.r, null);
} catch(e) {
assert.equal(e, null);
}
done();
}));
});
*/
describe('Delete', function() {
it('deletes a channel from the database', async((done) => {
const result = db.delete();
@ -501,6 +528,11 @@ describe('Orbit Client', () => {
});
describe('Key-Value Store', function() {
before(async((done) => {
db.delete();
done();
}));
it('put', async((done) => {
db.put('key1', 'hello!');
let all = db.iterator().collect();
@ -591,4 +623,79 @@ describe('Orbit Client', () => {
}));
});
/*
describe('Modes', function() {
var password = 'hello';
it('sets read mode', async((done) => {
try {
var mode = {
mode: "+r",
params: {
password: password
}
};
var modes = db.setMode(mode)
assert.notEqual(modes.r, null);
assert.equal(modes.r.password, password);
} catch(e) {
assert.equal(e, null);
}
done();
}));
it('can\'t read with wrong password', async((done) => {
try {
var modes = orbit.channel(channel, 'invalidpassword').iterator();
assert.equal(true, false);
} catch(e) {
assert.equal(e, 'Unauthorized');
}
done();
}));
it('sets write mode', async((done) => {
try {
var mode = {
mode: "+w",
params: {
ops: [orbit.user.id]
}
};
var modes = orbit.channel(channel, password).setMode(mode);
assert.notEqual(modes.w, null);
assert.equal(modes.w.ops[0], orbit.user.id);
} catch(e) {
assert.equal(e, null);
}
done();
}));
it('can\'t write when user not an op', async((done) => {
// TODO
done();
}));
it('removes write mode', async((done) => {
try {
var modes = orbit.channel(channel, password).setMode({ mode: "-w" });
assert.equal(modes.w, null);
} catch(e) {
assert.equal(e, null);
}
done();
}));
it('removes read mode', async((done) => {
try {
var modes = orbit.channel(channel, password).setMode({ mode: "-r" });
assert.equal(modes.r, null);
} catch(e) {
assert.equal(e, null);
}
done();
}));
});
*/
});

View File

@ -12,14 +12,13 @@ const Node = require('../src/list/OrbitNode');
const startIpfs = async (() => {
return new Promise(async((resolve, reject) => {
const ipfsd = await(ipfsDaemon());
resolve(ipfsd.daemon);
resolve(ipfsd.ipfs);
}));
});
let ipfs;
describe('OrbitList', async(function() {
this.timeout(5000);
before(async((done) => {
ipfs = await(startIpfs());
@ -28,7 +27,7 @@ describe('OrbitList', async(function() {
describe('Constructor', async(() => {
it('initializes member variables', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
assert.equal(list.id, 'A');
assert.equal(list.seq, 0);
assert.equal(list.ver, 0);
@ -44,10 +43,10 @@ describe('OrbitList', async(function() {
describe('add', async(() => {
it('saves the data to ipfs', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
const text = 'testing 1 2 3 4';
list.add(text)
const hash = await(list.getIpfsHash());
const hash = list.ipfsHash;
const l = await(ipfsAPI.getObject(ipfs, hash));
const list2 = List.fromJson(ipfs, JSON.parse(l.Data));
@ -62,18 +61,18 @@ describe('OrbitList', async(function() {
}));
it('updates the data to ipfs', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
const text1 = 'testing 1 2 3';
const text2 = 'testing 456';
let hash;
list.add(text1)
hash = await(list.getIpfsHash());
hash = list.ipfsHash;
// assert.equal(hash, 'QmcBjB93PsJGz2LrVy5e1Z8mtwH99B8yynsa5f4q3GanEe');
list.add(text2)
hash = await(list.getIpfsHash());
hash = list.ipfsHash;
// assert.equal(hash, 'Qmf358H1wjuX3Bbaag4SSEiujoruowVUNR5pLCNQs8vivP');
const l = await(ipfsAPI.getObject(ipfs, hash));
@ -85,17 +84,17 @@ describe('OrbitList', async(function() {
}));
}));
describe('getIpfsHash', async(() => {
describe('ipfsHash', async(() => {
it('returns the list as ipfs hash', async((done) => {
const list = new List('A', ipfs);
const hash = await(list.getIpfsHash());
const list = new List(ipfs, 'A');
const hash = list.ipfsHash;
assert.equal(hash.startsWith('Qm'), true);
done();
}));
it('saves the list to ipfs', async((done) => {
const list = new List('A', ipfs);
const hash = await(list.getIpfsHash());
const list = new List(ipfs, 'A');
const hash = list.ipfsHash;
const l = await(ipfsAPI.getObject(ipfs, hash));
assert.equal(l.toString(), ({ Links: [], Data: '{"id":"A","seq":0,"ver":0,"items":[]}' }).toString());
done();
@ -104,12 +103,12 @@ describe('OrbitList', async(function() {
describe('fromIpfsHash', () => {
it('creates a list from ipfs hash', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
list.add("hello1")
list.add("hello2")
list.add("hello3")
const hash = await(list.getIpfsHash());
const res = await(List.fromIpfsHash(ipfs, hash));
const hash = list.ipfsHash;
const res = List.fromIpfsHash(ipfs, hash);
assert.equal(res.id, 'A');
assert.equal(res.seq, 0);
@ -142,7 +141,7 @@ describe('OrbitList', async(function() {
};
before(async((done) => {
list = new List('A', ipfs);
list = new List(ipfs, 'A');
list.add("hello1")
list.add("hello2")
list.add("hello3")
@ -177,7 +176,7 @@ describe('OrbitList', async(function() {
describe('items', () => {
it('returns items', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
let items = list.items;
assert.equal(list.items instanceof Array, true);
assert.equal(list.items.length, 0);
@ -193,7 +192,7 @@ describe('OrbitList', async(function() {
describe('add', () => {
it('adds an item to an empty list', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
list.add("hello1")
const item = list.items[0];
assert.equal(list.id, 'A');
@ -211,7 +210,7 @@ describe('OrbitList', async(function() {
}));
it('adds 100 items to a list', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
const amount = 100;
for(let i = 1; i <= amount; i ++) {
@ -230,7 +229,7 @@ describe('OrbitList', async(function() {
}));
it('commits a list after batch size was reached', async((done) => {
const list = new List('A', ipfs);
const list = new List(ipfs, 'A');
for(let i = 1; i <= List.batchSize; i ++) {
list.add("hello" + i);
@ -256,8 +255,8 @@ describe('OrbitList', async(function() {
describe('join', () => {
it('increases the sequence and resets the version if other list has the same or higher sequence', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list2.seq = 7;
list1.add("helloA1")
@ -276,8 +275,8 @@ describe('OrbitList', async(function() {
}));
it('increases the sequence by one if other list has lower sequence', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.seq = 4;
list2.seq = 1;
list2.add("helloB1")
@ -289,7 +288,7 @@ describe('OrbitList', async(function() {
}));
it('finds the next head when adding a new element', async((done) => {
const list1 = new List('A', ipfs);
const list1 = new List(ipfs, 'A');
list1.add("helloA1")
list1.add("helloA2")
list1.add("helloA3")
@ -306,8 +305,8 @@ describe('OrbitList', async(function() {
}));
it('finds the next heads (two) after a join', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.add("helloA1")
list2.add("helloB1")
list2.add("helloB2")
@ -324,8 +323,8 @@ describe('OrbitList', async(function() {
}));
it('finds the next head (one) after a join', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.add("helloA1")
list2.add("helloB1")
list2.add("helloB2")
@ -344,8 +343,8 @@ describe('OrbitList', async(function() {
}));
it('finds the next heads after two joins', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.add("helloA1")
list1.add("helloA2")
list2.add("helloB1")
@ -371,10 +370,10 @@ describe('OrbitList', async(function() {
}));
it('finds the next heads after multiple joins', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list3 = new List('C', ipfs);
const list4 = new List('D', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
const list3 = new List(ipfs, 'C');
const list4 = new List(ipfs, 'D');
list1.add("helloA1")
list1.add("helloA2")
list2.add("helloB1")
@ -413,8 +412,8 @@ describe('OrbitList', async(function() {
}));
it('joins list of one item with list of two items', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.add("helloA1")
list2.add("helloB1")
list2.add("helloB2")
@ -435,8 +434,8 @@ describe('OrbitList', async(function() {
}));
it('joins lists two ways', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.add("helloA1")
list1.add("helloA2")
list2.add("helloB1")
@ -471,8 +470,8 @@ describe('OrbitList', async(function() {
}));
it('joins lists twice', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
list1.add("helloA1")
list2.add("helloB1")
@ -502,10 +501,10 @@ describe('OrbitList', async(function() {
}));
it('joins 4 lists to one', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list3 = new List('C', ipfs);
const list4 = new List('D', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
const list3 = new List(ipfs, 'C');
const list4 = new List(ipfs, 'D');
list1.add("helloA1")
list2.add("helloB1")
@ -539,10 +538,10 @@ describe('OrbitList', async(function() {
}));
it('joins lists from 4 lists', async((done) => {
const list1 = new List('A', ipfs);
const list2 = new List('B', ipfs);
const list3 = new List('C', ipfs);
const list4 = new List('D', ipfs);
const list1 = new List(ipfs, 'A');
const list2 = new List(ipfs, 'B');
const list3 = new List(ipfs, 'C');
const list4 = new List(ipfs, 'D');
list1.add("helloA1")
list1.join(list2);
@ -592,6 +591,23 @@ describe('OrbitList', async(function() {
assert.equal(lastItem2.data, 'helloD4');
done();
}));
it('joins itself', async((done) => {
const list1 = new List(ipfs, 'A');
list1.add("helloA1")
list1.add("helloA2")
list1.add("helloA3")
list1.join(list1);
assert.equal(list1.id, 'A');
assert.equal(list1.seq, 1);
assert.equal(list1.ver, 0);
assert.equal(list1.items.length, 3);
assert.equal(list1.items[0].ver, 0);
assert.equal(list1.items[1].ver, 1);
assert.equal(list1.items[2].ver, 2);
done();
}));
});
describe('_findHeads', () => {

View File

@ -11,7 +11,7 @@ const Node = require('../src/list/OrbitNode');
const startIpfs = async (() => {
return new Promise(async((resolve, reject) => {
const ipfsd = await(ipfsDaemon());
resolve(ipfsd.daemon);
resolve(ipfsd.ipfs);
}));
});