Refaactor history fetching

src/list/OrbitList.js

@@ -32,39 +32,7 @@ class OrbitList extends List {
 
   join(other) {
     super.join(other);
-
+    this._fetchHistory(other.items);
-    // WIP: fetch history
-    const isReferenced = (all, item) => _.findLast(all, (f) => f === item) !== undefined;
-    const fetchRecursive = (hash, amount, all, res) => {
-      let result = res ? res : [];
-      hash = hash instanceof Node === true ? hash.hash : hash;
-
-      if(res.length >= amount)
-        return res;
-
-      if(!isReferenced(all, hash)) {
-        all.push(hash);
-        const item = Node.fromIpfsHash(this._ipfs, hash);
-        res.push(item);
-        item.heads.map((head) => fetchRecursive(head, amount, all, res));
-      }
-
-      return res;
-    };
-
-    let allHashes = this._items.map((a) => a.hash);
-
-    const res = _.flatten(other.items.map((e) => _.flatten(e.heads.map((f) => {
-      const remaining = (MaxHistory);
-      return _.flatten(fetchRecursive(f, MaxHistory, allHashes, []));
-    }))));
-
-    res.slice(0, MaxHistory).forEach((item) => {
-      const indices = item.heads.map((k) => _.findIndex(this._items, (b) => b.hash === k));
-      const idx = indices.length > 0 ? Math.max(_.max(indices) + 1, 0) : 0;
-      this._items.splice(idx, 0, item)
-    });
-    // console.log("--> Fetched", res.length, "items from the history\n");
   }
 
   // The LWW-set query interface
@@ -106,11 +74,45 @@ class OrbitList extends List {
   }
 
   /* Private methods */
-  // Store to IPFS
+  _fetchHistory(items) {
+    let allHashes = this._items.map((a) => a.hash);
+    const res = Lazy(items)
+      .reverse()
+      .map((f) => f.heads).flatten() // Go through all heads
+      .filter((f) => !(f instanceof Node === true)) // OrbitNode vs. {}, filter out instances (we already have them in mem)
+      .map((f) => this._fetchRecursive(f, MaxHistory, allHashes)).flatten() // IO - get the data from IPFS
+      .map((f) => this._insert(f)) // Insert to the list
+      .take(MaxHistory) // How many items from the history we should fetch
+      .toArray();
+    // console.log("--> Fetched", res.length, "items from the history\n");
+  }
+
+  _fetchRecursive(hash, amount, all) {
+    const isReferenced = (list, item) => Lazy(list).find((f) => f === item) !== undefined;
+    let result = [];
+    if(!isReferenced(all, hash)) {
+      all.push(hash);
+      const item = await(Node.fromIpfsHash(this._ipfs, hash));
+      result.push(item);
+      result = result.concat(Lazy(item.heads)
+        .map((f) => this._fetchRecursive(f, amount, all))
+        .flatten()
+        .toArray());
+    }
+    return result;
+  }
+
+  // Insert to the list right after the latest parent
+  _insert(item) {
+    const index = Lazy(item.heads)
+      .map((next) => Lazy(this._items).map((f) => f.hash).indexOf(next)) // Find the item's parent's indices
+      .reduce((max, a) => a > max ? a : max, 0); // find the largest index (latest parent)
+
+    this._items.splice(index, 0, item);
+  }
 
   /* Properties */
   get ipfsHash() {
-    // await(this._commit());
     const toIpfs = async(() => {
       return new Promise(async((resolve, reject) => {
         var data = await(this.asJson)

test/orbit-list-tests.js

@@ -76,8 +76,7 @@ describe('OrbitList', async(function() {
       hash = list.ipfsHash;
       assert.equal(hash, 'Qmecju6aNyQF8LHUNbUrujMmXPfUit7tDkqnmLKLF22aRk');
 
-      const l = await(ipfsAPI.getObject(ipfs, hash));
+      const list2 = List.fromIpfsHash(ipfs, hash);
-      const list2 = List.fromJson(ipfs, JSON.parse(l.Data));
       assert.equal(list2.items[0].data, text1);
       assert.equal(list2.items[1].data, text2);
 
@@ -608,6 +607,75 @@ describe('OrbitList', async(function() {
       assert.equal(list1.items[2].ver, 2);
       done();
     }));
+
+    it('fetches items from history', async((done) => {
+      const list1 = new List(ipfs, 'A');
+      const list2 = new List(ipfs, 'AAA');
+
+      const count = 10;
+      for(let i = 1; i < count + 1; i ++) {
+        list1.add("first " + i);
+        list2.add("second " + i);
+      }
+
+      const hash1 = list1.ipfsHash;
+      const hash2 = list2.ipfsHash;
+      assert.equal(hash1, 'QmaoGci9eiSYdANo63JAkvUpnyXe2uQH1BkwAiKsJHNUWp');
+      assert.equal(hash2, 'QmTXu5g5BzZW3vMBKaXnerZTGXf5XPRFB6y3DXNEmcWWtU');
+
+      const final = new List(ipfs, 'B');
+      const other1 = List.fromIpfsHash(ipfs, hash1);
+      const other2 = List.fromIpfsHash(ipfs, hash2);
+      final.join(other1);
+
+      assert.equal(final.items.length, count);
+      assert.equal(final.items[0].data, "first 1");
+      assert.equal(final.items[final.items.length - 1].data, "first 10");
+
+      final.join(other2);
+      assert.equal(final.items.length, count * 2);
+      assert.equal(final.items[0].data, "first 1");
+      assert.equal(final.items[final.items.length - 1].data, "second 10");
+      done();
+    }));
+
+    it('orders fetched items correctly', async((done) => {
+      const list1 = new List(ipfs, 'A');
+      const list2 = new List(ipfs, 'AAA');
+
+      const count = List.batchSize * 3;
+      for(let i = 1; i < (count * 2) + 1; i ++)
+        list1.add("first " + i);
+
+      const hash1 = list1.ipfsHash;
+      assert.equal(hash1, 'QmaJ2a1AxPBhKcis1HLRnc1UNixSmwd9XBNJzxdnqQSyYa');
+
+      const final = new List(ipfs, 'B');
+      const other1 = List.fromIpfsHash(ipfs, hash1);
+      final.join(other1);
+
+      assert.equal(final.items[0].data, "first 1");
+      assert.equal(final.items[final.items.length - 1].data, "first " + count * 2);
+      assert.equal(final.items.length, count * 2);
+
+      // Second batch
+      for(let i = 1; i < count + 1; i ++)
+        list2.add("second " + i);
+
+      const hash2 = list2.ipfsHash;
+      assert.equal(hash2, 'QmVQ55crzwWY21D7LwMLrxT7aKvCoSVtpo23WRdajSHtBN');
+
+      const other2 = List.fromIpfsHash(ipfs, hash2);
+      final.join(other2);
+
+      // console.log(final.items.map((e) => e.comptactId))
+      assert.equal(final.items.length, count + count * 2);
+      assert.equal(final.items[0].data, "second 1");
+      assert.equal(final.items[1].data, "second 2");
+      assert.equal(final.items[final.items.length - 1].data, "second " + count);
+      done();
+    }));
+
   });
 
   describe('_findHeads', () => {