From 0f592c65db684d1b378839c9d6c634f738145b2f Mon Sep 17 00:00:00 2001 From: haad Date: Fri, 4 Mar 2016 23:13:56 +0100 Subject: [PATCH] Refaactor history fetching --- src/list/OrbitList.js | 72 +++++++++++++++++++++------------------- test/orbit-list-tests.js | 72 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 37 deletions(-) diff --git a/src/list/OrbitList.js b/src/list/OrbitList.js index b428563..6b31148 100644 --- a/src/list/OrbitList.js +++ b/src/list/OrbitList.js @@ -32,39 +32,7 @@ class OrbitList extends List { join(other) { super.join(other); - - // WIP: fetch history - const isReferenced = (all, item) => _.findLast(all, (f) => f === item) !== undefined; - const fetchRecursive = (hash, amount, all, res) => { - let result = res ? res : []; - hash = hash instanceof Node === true ? hash.hash : hash; - - if(res.length >= amount) - return res; - - if(!isReferenced(all, hash)) { - all.push(hash); - const item = Node.fromIpfsHash(this._ipfs, hash); - res.push(item); - item.heads.map((head) => fetchRecursive(head, amount, all, res)); - } - - return res; - }; - - let allHashes = this._items.map((a) => a.hash); - - const res = _.flatten(other.items.map((e) => _.flatten(e.heads.map((f) => { - const remaining = (MaxHistory); - return _.flatten(fetchRecursive(f, MaxHistory, allHashes, [])); - })))); - - res.slice(0, MaxHistory).forEach((item) => { - const indices = item.heads.map((k) => _.findIndex(this._items, (b) => b.hash === k)); - const idx = indices.length > 0 ? Math.max(_.max(indices) + 1, 0) : 0; - this._items.splice(idx, 0, item) - }); - // console.log("--> Fetched", res.length, "items from the history\n"); + this._fetchHistory(other.items); } // The LWW-set query interface @@ -106,11 +74,45 @@ class OrbitList extends List { } /* Private methods */ - // Store to IPFS + _fetchHistory(items) { + let allHashes = this._items.map((a) => a.hash); + const res = Lazy(items) + .reverse() + .map((f) => f.heads).flatten() // Go through all heads + .filter((f) => !(f instanceof Node === true)) // OrbitNode vs. {}, filter out instances (we already have them in mem) + .map((f) => this._fetchRecursive(f, MaxHistory, allHashes)).flatten() // IO - get the data from IPFS + .map((f) => this._insert(f)) // Insert to the list + .take(MaxHistory) // How many items from the history we should fetch + .toArray(); + // console.log("--> Fetched", res.length, "items from the history\n"); + } + + _fetchRecursive(hash, amount, all) { + const isReferenced = (list, item) => Lazy(list).find((f) => f === item) !== undefined; + let result = []; + if(!isReferenced(all, hash)) { + all.push(hash); + const item = await(Node.fromIpfsHash(this._ipfs, hash)); + result.push(item); + result = result.concat(Lazy(item.heads) + .map((f) => this._fetchRecursive(f, amount, all)) + .flatten() + .toArray()); + } + return result; + } + + // Insert to the list right after the latest parent + _insert(item) { + const index = Lazy(item.heads) + .map((next) => Lazy(this._items).map((f) => f.hash).indexOf(next)) // Find the item's parent's indices + .reduce((max, a) => a > max ? a : max, 0); // find the largest index (latest parent) + + this._items.splice(index, 0, item); + } /* Properties */ get ipfsHash() { - // await(this._commit()); const toIpfs = async(() => { return new Promise(async((resolve, reject) => { var data = await(this.asJson) diff --git a/test/orbit-list-tests.js b/test/orbit-list-tests.js index 5d53ac7..fc9ba1a 100644 --- a/test/orbit-list-tests.js +++ b/test/orbit-list-tests.js @@ -76,8 +76,7 @@ describe('OrbitList', async(function() { hash = list.ipfsHash; assert.equal(hash, 'Qmecju6aNyQF8LHUNbUrujMmXPfUit7tDkqnmLKLF22aRk'); - const l = await(ipfsAPI.getObject(ipfs, hash)); - const list2 = List.fromJson(ipfs, JSON.parse(l.Data)); + const list2 = List.fromIpfsHash(ipfs, hash); assert.equal(list2.items[0].data, text1); assert.equal(list2.items[1].data, text2); @@ -608,6 +607,75 @@ describe('OrbitList', async(function() { assert.equal(list1.items[2].ver, 2); done(); })); + + it('fetches items from history', async((done) => { + const list1 = new List(ipfs, 'A'); + const list2 = new List(ipfs, 'AAA'); + + const count = 10; + for(let i = 1; i < count + 1; i ++) { + list1.add("first " + i); + list2.add("second " + i); + } + + const hash1 = list1.ipfsHash; + const hash2 = list2.ipfsHash; + assert.equal(hash1, 'QmaoGci9eiSYdANo63JAkvUpnyXe2uQH1BkwAiKsJHNUWp'); + assert.equal(hash2, 'QmTXu5g5BzZW3vMBKaXnerZTGXf5XPRFB6y3DXNEmcWWtU'); + + const final = new List(ipfs, 'B'); + const other1 = List.fromIpfsHash(ipfs, hash1); + const other2 = List.fromIpfsHash(ipfs, hash2); + final.join(other1); + + assert.equal(final.items.length, count); + assert.equal(final.items[0].data, "first 1"); + assert.equal(final.items[final.items.length - 1].data, "first 10"); + + final.join(other2); + assert.equal(final.items.length, count * 2); + assert.equal(final.items[0].data, "first 1"); + assert.equal(final.items[final.items.length - 1].data, "second 10"); + done(); + })); + + it('orders fetched items correctly', async((done) => { + const list1 = new List(ipfs, 'A'); + const list2 = new List(ipfs, 'AAA'); + + const count = List.batchSize * 3; + for(let i = 1; i < (count * 2) + 1; i ++) + list1.add("first " + i); + + const hash1 = list1.ipfsHash; + assert.equal(hash1, 'QmaJ2a1AxPBhKcis1HLRnc1UNixSmwd9XBNJzxdnqQSyYa'); + + const final = new List(ipfs, 'B'); + const other1 = List.fromIpfsHash(ipfs, hash1); + final.join(other1); + + assert.equal(final.items[0].data, "first 1"); + assert.equal(final.items[final.items.length - 1].data, "first " + count * 2); + assert.equal(final.items.length, count * 2); + + // Second batch + for(let i = 1; i < count + 1; i ++) + list2.add("second " + i); + + const hash2 = list2.ipfsHash; + assert.equal(hash2, 'QmVQ55crzwWY21D7LwMLrxT7aKvCoSVtpo23WRdajSHtBN'); + + const other2 = List.fromIpfsHash(ipfs, hash2); + final.join(other2); + + // console.log(final.items.map((e) => e.comptactId)) + assert.equal(final.items.length, count + count * 2); + assert.equal(final.items[0].data, "second 1"); + assert.equal(final.items[1].data, "second 2"); + assert.equal(final.items[final.items.length - 1].data, "second " + count); + done(); + })); + }); describe('_findHeads', () => {