From a5c14d53777fac2d207f63ac057e153f5f2c0b05 Mon Sep 17 00:00:00 2001 From: Haad Date: Wed, 29 Mar 2023 16:10:24 +0300 Subject: [PATCH] Fix log fetching parallelism (#60) * Fix log traverse parallelism * Rename pointerCount to referencesCount --- package-lock.json | 15 +-- package.json | 1 + src/database.js | 8 +- src/oplog/log.js | 80 ++++++++-------- test/db/keyvalue-persisted.js | 10 +- test/db/keyvalue.test.js | 10 +- test/oplog/append.test.js | 12 ++- test/oplog/references.test.js | 153 +++++++++++++------------------ test/orbitdb-replication.test.js | 4 +- 9 files changed, 132 insertions(+), 161 deletions(-) diff --git a/package-lock.json b/package-lock.json index aa64455..5387932 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "logplease": "^1.2.15", "lru": "^3.1.0", "multiformats": "^11.0.1", + "p-map": "^5.5.0", "p-queue": "^7.3.4", "timeout-abort-controller": "^3.0.0", "uint8arrays": "^4.0.3" @@ -3445,7 +3446,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-4.0.1.tgz", "integrity": "sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==", - "dev": true, "dependencies": { "clean-stack": "^4.0.0", "indent-string": "^5.0.0" @@ -4323,7 +4323,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-4.2.0.tgz", "integrity": "sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==", - "dev": true, "dependencies": { "escape-string-regexp": "5.0.0" }, @@ -4338,7 +4337,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "dev": true, "engines": { "node": ">=12" }, @@ -6881,7 +6879,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", - "dev": true, "engines": { "node": ">=12" }, @@ -10926,7 +10923,6 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-5.5.0.tgz", "integrity": "sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==", - "dev": true, "dependencies": { "aggregate-error": "^4.0.0" }, @@ -17330,7 +17326,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-4.0.1.tgz", "integrity": "sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==", - "dev": true, "requires": { "clean-stack": "^4.0.0", "indent-string": "^5.0.0" @@ -17946,7 +17941,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-4.2.0.tgz", "integrity": "sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==", - "dev": true, "requires": { "escape-string-regexp": "5.0.0" }, @@ -17954,8 +17948,7 @@ "escape-string-regexp": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "dev": true + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==" } } }, @@ -19801,8 +19794,7 @@ "indent-string": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", - "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", - "dev": true + "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==" }, "inflight": { "version": "1.0.6", @@ -22669,7 +22661,6 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-5.5.0.tgz", "integrity": "sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==", - "dev": true, "requires": { "aggregate-error": "^4.0.0" } diff --git a/package.json b/package.json index 156828c..cb108cc 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "logplease": "^1.2.15", "lru": "^3.1.0", "multiformats": "^11.0.1", + "p-map": "^5.5.0", "p-queue": "^7.3.4", "timeout-abort-controller": "^3.0.0", "uint8arrays": "^4.0.3" diff --git a/src/database.js b/src/database.js index 5b7af4b..229a56b 100644 --- a/src/database.js +++ b/src/database.js @@ -4,15 +4,15 @@ import Path from 'path' import Sync from './sync.js' import { ComposedStorage, LRUStorage, IPFSBlockStorage, LevelStorage } from './storage/index.js' -const defaultPointerCount = 0 +const defaultReferencesCount = 16 const defaultCacheSize = 1000 -const Database = async ({ OpLog, ipfs, identity, address, name, accessController, directory, meta, headsStorage, entryStorage, indexStorage, pointerCount, syncAutomatically }) => { +const Database = async ({ OpLog, ipfs, identity, address, name, accessController, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically }) => { const { Log, Entry } = OpLog directory = Path.join(directory || './orbitdb', `./${address}/`) meta = meta || {} - pointerCount = pointerCount || defaultPointerCount + referencesCount = referencesCount || defaultReferencesCount entryStorage = entryStorage || await ComposedStorage( await LRUStorage({ size: defaultCacheSize }), @@ -36,7 +36,7 @@ const Database = async ({ OpLog, ipfs, identity, address, name, accessController const addOperation = async (op) => { const task = async () => { - const entry = await log.append(op, { pointerCount }) + const entry = await log.append(op, { referencesCount }) await sync.add(entry) events.emit('update', entry) return entry.hash diff --git a/src/oplog/log.js b/src/oplog/log.js index dae5f3b..d485af8 100644 --- a/src/oplog/log.js +++ b/src/oplog/log.js @@ -4,6 +4,7 @@ import Clock from './lamport-clock.js' import Heads from './heads.js' import Sorting from './sorting.js' import MemoryStorage from '../storage/memory.js' +import pMap from 'p-map' const { LastWriteWins, NoZeroes } = Sorting @@ -124,16 +125,21 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora * @param {data} data Payload to add to the entry * @return {Promise} Entry that was appended */ - const append = async (data, options = { pointerCount: 0 }) => { + const append = async (data, options = { referencesCount: 0 }) => { // 1. Prepare entry // 2. Authorize entry // 3. Store entry // 4. return Entry - - // Get references (entry at every pow2 of distance) - const refs = await getReferences(options.pointerCount) + // Get current heads of the log + const heads_ = await heads() + // Get references (we skip the heads which are covered by the next field) + let refs = [] + for await (const { hash } of iterator({ amount: options.referencesCount + heads_.length })) { + refs.push(hash) + } + refs = refs.slice(heads_.length, options.referencesCount + heads_.length) // Create the next pointers from heads - const nexts = (await heads()).map(entry => entry.hash) + const nexts = heads_.map(entry => entry.hash) // Create the entry const entry = await Entry.create( identity, @@ -253,14 +259,20 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora let stack = rootEntries.sort(sortFn) // Keep a record of all the hashes of entries we've traversed and yielded const traversed = {} + // Keep a record of all the hashes we are fetching or have already fetched + let toFetch = [] + const fetched = {} + // A function to check if we've seen a hash + const notIndexed = (hash) => !(traversed[hash] || fetched[hash]) // Current entry during traversal let entry // Start traversal and process stack until it's empty (traversed the full log) while (stack.length > 0) { + stack = stack.sort(sortFn) // Get the next entry from the stack entry = stack.pop() if (entry) { - const hash = entry.hash + const { hash, next, refs } = entry // If we have an entry that we haven't traversed yet, process it if (!traversed[hash]) { // Yield the current entry @@ -270,21 +282,28 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora if (done === true) { break } - // Add to the hashes we've traversed + // Add to the hash indices traversed[hash] = true - // Add hashes of next entries to the stack from entry's - // causal connection (next) and references to history (refs) - for (const nextHash of [...entry.next, ...entry.refs]) { - // Check if we've already traversed this entry - if (!traversed[nextHash]) { - // Fetch the next entry - const next = await get(nextHash) - if (next) { - // Add the next entry in front of the stack and sort - stack = [next, ...stack].sort(sortFn) - } + fetched[hash] = true + // Add the next and refs hashes to the list of hashes to fetch next, + // filter out traversed and fetched hashes + toFetch = [...toFetch, ...next, ...refs].filter(notIndexed) + // Function to fetch an entry and making sure it's not a duplicate (check the hash indices) + const fetchEntries = async (hash) => { + if (!traversed[hash] && !fetched[hash]) { + fetched[hash] = true + return get(hash) } } + // Fetch the next/reference entries + const nexts = await pMap(toFetch, fetchEntries) + // Add the next and refs fields from the fetched entries to the next round + toFetch = nexts + .filter(e => e != null) + .reduce((res, acc) => [...res, ...acc.next, ...acc.refs], []) + .filter(notIndexed) + // Add the fetched entries to the stack to be processed + stack = [...nexts, ...stack] } } } @@ -405,31 +424,6 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora await _entries.close() } - /** - * TODO - * Get references at every pow2 distance - * If pointer count is 4, returns 2 - * If pointer count is 8, returns 3 references - * If pointer count is 512, returns 9 references - * If pointer count is 2048, returns 11 references - */ - const getReferences = async (pointerCount = 1) => { - let nextPointerDistance = 2 - let distance = 0 - const refs = [] - const shouldStopFn = () => distance >= pointerCount - for await (const entry of traverse(null, shouldStopFn)) { - distance++ - if (distance === nextPointerDistance) { - if (entry.hash) { - refs.push(entry.hash) - } - nextPointerDistance *= 2 - } - } - return refs - } - /** * Check if an object is a Log. * @param {Log} obj diff --git a/test/db/keyvalue-persisted.js b/test/db/keyvalue-persisted.js index c98df0e..1e849f2 100644 --- a/test/db/keyvalue-persisted.js +++ b/test/db/keyvalue-persisted.js @@ -175,11 +175,11 @@ describe('KeyValuePersisted Database', function () { const keyvalue = [ { hash: 'zdpuAm6QEA29wFnd6re7X2XWe7AmrzVbsvdHhSPXci2CqXryw', key: 'key1', value: 'init' }, { hash: 'zdpuAvfTQwogEAhEaAtb85ugEzxvfDVUnALoZeNbrz3s4jMYd', key: 'key2', value: true }, - { hash: 'zdpuB2CBCwvPBdHjZRKfFtL5JuDo9sc5QinKhbtYu1YkCLq22', key: 'key3', value: 'hello' }, - { hash: 'zdpuAyWWtvFfxKWMcV3NJ7XXbjiQC6MkA8h6TrhFA2ihLrt82', key: 'key4', value: 'friend' }, - { hash: 'zdpuB2Z5coKXGMAZtb7X8UQYgo6vWAP4VshBvE4xwBCrR5Laa', key: 'key5', value: '12345' }, - { hash: 'zdpuAnn2kuStzcTCJ5ULMxCvB7RtgAScJPmg3YAVYju4oPEtC', key: 'key6', value: 'empty' }, - { hash: 'zdpuAv1jSFz4GHRieAXGvRGnVWdEdxDp2HefREoTJJWYC8Zqw', key: 'key7', value: 'friend33' } + { hash: 'zdpuB2ZCXwfkbgXQDHaP13rGSLVzZdZXuFPAk988VCZyMV1Er', key: 'key3', value: 'hello' }, + { hash: 'zdpuAnDHm5qkyzkdtEiedF2VwyuUvrgsgM7mCVrjLya3G7nFS', key: 'key4', value: 'friend' }, + { hash: 'zdpuB2XjqLhSEEB6CxCwCLWoas77Db6T9TJDNKcyX35kwbNmb', key: 'key5', value: '12345' }, + { hash: 'zdpuB1GyECVHxwFBxa9QYeTYRFJRLDnujyekhXAmStG26stU9', key: 'key6', value: 'empty' }, + { hash: 'zdpuAsj9ZgSCWSuRYFkQ56Eiffpi6j6761ueHHNwNf3VaZfms', key: 'key7', value: 'friend33' } ] for (const { key, value } of Object.values(keyvalue)) { diff --git a/test/db/keyvalue.test.js b/test/db/keyvalue.test.js index 57136bf..3696ae8 100644 --- a/test/db/keyvalue.test.js +++ b/test/db/keyvalue.test.js @@ -167,11 +167,11 @@ describe('KeyValue Database', function () { const keyvalue = [ { hash: 'zdpuAm6QEA29wFnd6re7X2XWe7AmrzVbsvdHhSPXci2CqXryw', key: 'key1', value: 'init' }, { hash: 'zdpuAvfTQwogEAhEaAtb85ugEzxvfDVUnALoZeNbrz3s4jMYd', key: 'key2', value: true }, - { hash: 'zdpuB2CBCwvPBdHjZRKfFtL5JuDo9sc5QinKhbtYu1YkCLq22', key: 'key3', value: 'hello' }, - { hash: 'zdpuAyWWtvFfxKWMcV3NJ7XXbjiQC6MkA8h6TrhFA2ihLrt82', key: 'key4', value: 'friend' }, - { hash: 'zdpuB2Z5coKXGMAZtb7X8UQYgo6vWAP4VshBvE4xwBCrR5Laa', key: 'key5', value: '12345' }, - { hash: 'zdpuAnn2kuStzcTCJ5ULMxCvB7RtgAScJPmg3YAVYju4oPEtC', key: 'key6', value: 'empty' }, - { hash: 'zdpuAv1jSFz4GHRieAXGvRGnVWdEdxDp2HefREoTJJWYC8Zqw', key: 'key7', value: 'friend33' } + { hash: 'zdpuB2ZCXwfkbgXQDHaP13rGSLVzZdZXuFPAk988VCZyMV1Er', key: 'key3', value: 'hello' }, + { hash: 'zdpuAnDHm5qkyzkdtEiedF2VwyuUvrgsgM7mCVrjLya3G7nFS', key: 'key4', value: 'friend' }, + { hash: 'zdpuB2XjqLhSEEB6CxCwCLWoas77Db6T9TJDNKcyX35kwbNmb', key: 'key5', value: '12345' }, + { hash: 'zdpuB1GyECVHxwFBxa9QYeTYRFJRLDnujyekhXAmStG26stU9', key: 'key6', value: 'empty' }, + { hash: 'zdpuAsj9ZgSCWSuRYFkQ56Eiffpi6j6761ueHHNwNf3VaZfms', key: 'key7', value: 'friend33' } ] for (const { key, value } of Object.values(keyvalue)) { diff --git a/test/oplog/append.test.js b/test/oplog/append.test.js index 2b95695..7c71a90 100644 --- a/test/oplog/append.test.js +++ b/test/oplog/append.test.js @@ -72,7 +72,7 @@ describe('Log - Append', function () { describe('append 100 items to a log', async () => { const amount = 100 - const nextPointerAmount = 64 + const referencesCount = 64 let log let values = [] @@ -81,7 +81,7 @@ describe('Log - Append', function () { before(async () => { log = await Log(testIdentity, { logId: 'A' }) for (let i = 0; i < amount; i++) { - await log.append('hello' + i, { pointerCount: nextPointerAmount }) + await log.append('hello' + i, { referencesCount }) } values = await log.values() heads = await log.heads() @@ -110,8 +110,12 @@ describe('Log - Append', function () { }) it('added the correct amount of refs pointers', async () => { - values.forEach((entry, index) => { - strictEqual(entry.refs.length, index > 0 ? Math.floor(Math.log2(Math.min(nextPointerAmount, index))) : 0) + values.reverse().forEach((entry, index) => { + index = values.length - index - 1 + const expectedRefCount = index < referencesCount + ? Math.max(0, index - 1) + : Math.max(0, Math.min(referencesCount, index - 1)) + strictEqual(entry.refs.length, expectedRefCount) }) }) }) diff --git a/test/oplog/references.test.js b/test/oplog/references.test.js index 7498dbc..cc1b109 100644 --- a/test/oplog/references.test.js +++ b/test/oplog/references.test.js @@ -2,19 +2,12 @@ import { strictEqual } from 'assert' import rmrf from 'rimraf' import { copy } from 'fs-extra' import { Log } from '../../src/oplog/index.js' -import { Identities, KeyStore, MemoryStorage } from '../../src/index.js' +import { Identities, KeyStore } from '../../src/index.js' import testKeysPath from '../fixtures/test-keys-path.js' const keysPath = './testkeys' -const isBrowser = () => typeof window !== 'undefined' - describe('Log - References', function () { - if (isBrowser()) { - // Skip these tests when running in the browser since they take a long time - return - } - this.timeout(60000) let keystore @@ -36,132 +29,118 @@ describe('Log - References', function () { }) describe('References', async () => { - const amount = 64 - - it('creates entries with 2 references', async () => { - const maxReferenceDistance = 2 + it('creates entries with 1 references', async () => { + const amount = 32 + const referencesCount = 1 const log1 = await Log(testIdentity, { logId: 'A' }) for (let i = 0; i < amount; i++) { - await log1.append(i.toString(), { pointerCount: maxReferenceDistance }) + await log1.append(i.toString(), { referencesCount }) } const values1 = await log1.values() - strictEqual(values1[values1.length - 1].refs.length, 1) + strictEqual(values1[values1.length - 1].refs.length, referencesCount) + }) + + it('creates entries with 2 references', async () => { + const amount = 32 + const referencesCount = 2 + const log1 = await Log(testIdentity, { logId: 'A' }) + + for (let i = 0; i < amount; i++) { + await log1.append(i.toString(), { referencesCount }) + } + + const values1 = await log1.values() + + strictEqual(values1[values1.length - 1].refs.length, referencesCount) }) it('creates entries with 4 references', async () => { - const maxReferenceDistance = 2 + const amount = 32 + const referencesCount = 4 const log2 = await Log(testIdentity, { logId: 'B' }) - for (let i = 0; i < amount * 2; i++) { - await log2.append(i.toString(), { pointerCount: Math.pow(maxReferenceDistance, 2) }) + for (let i = 0; i < amount; i++) { + await log2.append(i.toString(), { referencesCount }) } const values2 = await log2.values() - strictEqual(values2[values2.length - 1].refs.length, 2) + strictEqual(values2[values2.length - 1].refs.length, referencesCount) }) it('creates entries with 8 references', async () => { - const maxReferenceDistance = 2 + const amount = 64 + const referencesCount = 8 const log3 = await Log(testIdentity, { logId: 'C' }) - for (let i = 0; i < amount * 3; i++) { - await log3.append(i.toString(), { pointerCount: Math.pow(maxReferenceDistance, 3) }) + for (let i = 0; i < amount; i++) { + await log3.append(i.toString(), { referencesCount }) } const values3 = await log3.values() - strictEqual(values3[values3.length - 1].refs.length, 3) + strictEqual(values3[values3.length - 1].refs.length, referencesCount) }) it('creates entries with 16 references', async () => { - const maxReferenceDistance = 2 + const amount = 64 + const referencesCount = 16 const log4 = await Log(testIdentity, { logId: 'D' }) - for (let i = 0; i < amount * 4; i++) { - await log4.append(i.toString(), { pointerCount: Math.pow(maxReferenceDistance, 4) }) + for (let i = 0; i < amount; i++) { + await log4.append(i.toString(), { referencesCount }) } const values4 = await log4.values() - strictEqual(values4[values4.length - 1].refs.length, 4) + strictEqual(values4[values4.length - 1].refs.length, referencesCount) }) - const inputs = [ - { amount: 1, referenceCount: 1, refLength: 0 }, - { amount: 1, referenceCount: 2, refLength: 0 }, - { amount: 2, referenceCount: 1, refLength: 1 }, - { amount: 2, referenceCount: 2, refLength: 1 }, - { amount: 3, referenceCount: 2, refLength: 1 }, - { amount: 3, referenceCount: 4, refLength: 1 }, - { amount: 4, referenceCount: 4, refLength: 2 }, - { amount: 4, referenceCount: 4, refLength: 2 }, - { amount: 32, referenceCount: 4, refLength: 2 }, - { amount: 32, referenceCount: 8, refLength: 3 }, - { amount: 32, referenceCount: 16, refLength: 4 }, - { amount: 18, referenceCount: 32, refLength: 5 }, - { amount: 128, referenceCount: 32, refLength: 5 }, - { amount: 63, referenceCount: 64, refLength: 5 }, - { amount: 64, referenceCount: 64, refLength: 6 }, - { amount: 65, referenceCount: 64, refLength: 6 }, - { amount: 91, referenceCount: 64, refLength: 6 }, - { amount: 128, referenceCount: 64, refLength: 6 }, - { amount: 128, referenceCount: 1, refLength: 0 }, - { amount: 128, referenceCount: 2, refLength: 1 }, - { amount: 256, referenceCount: 1, refLength: 0 }, - { amount: 256, referenceCount: 4, refLength: 2 }, - { amount: 256, referenceCount: 8, refLength: 3 }, - { amount: 256, referenceCount: 16, refLength: 4 }, - { amount: 256, referenceCount: 32, refLength: 5 }, - { amount: 1024, referenceCount: 2, refLength: 1 } - ] + it('creates entries with 32 references', async () => { + const amount = 64 + const referencesCount = 32 + const log4 = await Log(testIdentity, { logId: 'D' }) - inputs.forEach(input => { - it(`has ${input.refLength} references, max distance ${input.referenceCount}, total of ${input.amount} entries`, async () => { - const test = async (amount, referenceCount, refLength) => { - const storage = await MemoryStorage() - const log1 = await Log(testIdentity, { logId: 'A', storage }) - for (let i = 0; i < amount; i++) { - await log1.append((i + 1).toString(), { pointerCount: referenceCount }) - } + for (let i = 0; i < amount; i++) { + await log4.append(i.toString(), { referencesCount }) + } - const values = await log1.values() + const values4 = await log4.values() - strictEqual(values.length, input.amount) - strictEqual(values[values.length - 1].clock.time, input.amount) + strictEqual(values4[values4.length - 1].refs.length, referencesCount) + }) - for (let k = 0; k < input.amount; k++) { - const idx = values.length - k - 1 - strictEqual(values[idx].clock.time, idx + 1) + it('creates entries with 64 references', async () => { + const amount = 128 + const referencesCount = 64 + const log4 = await Log(testIdentity, { logId: 'D' }) - // Check the first ref (distance 2) - if (values[idx].refs.length > 0) { strictEqual(values[idx].refs[0], values[idx - 2].hash) } + for (let i = 0; i < amount; i++) { + await log4.append(i.toString(), { referencesCount }) + } - // Check the second ref (distance 4) - if (values[idx].refs.length > 1 && idx > referenceCount) { strictEqual(values[idx].refs[1], values[idx - 4].hash) } + const values4 = await log4.values() - // Check the third ref (distance 8) - if (values[idx].refs.length > 2 && idx > referenceCount) { strictEqual(values[idx].refs[2], values[idx - 8].hash) } + strictEqual(values4[values4.length - 1].refs.length, referencesCount) + }) - // Check the fourth ref (distance 16) - if (values[idx].refs.length > 3 && idx > referenceCount) { strictEqual(values[idx].refs[3], values[idx - 16].hash) } + it('creates entries with 128 references', async () => { + // +2 because first ref is always skipped (covered by next field) and + // we need 129 entries to have 128 back references + const amount = 128 + 2 + const referencesCount = 128 + const log4 = await Log(testIdentity, { logId: 'D' }) - // Check the fifth ref (distance 32) - if (values[idx].refs.length > 4 && idx > referenceCount) { strictEqual(values[idx].refs[4], values[idx - 32].hash) } + for (let i = 0; i < amount; i++) { + await log4.append(i.toString(), { referencesCount }) + } - // Check the fifth ref (distance 64) - if (values[idx].refs.length > 5 && idx > referenceCount) { strictEqual(values[idx].refs[5], values[idx - 64].hash) } + const values4 = await log4.values() - // Check the reference of each entry - if (idx > referenceCount) { strictEqual(values[idx].refs.length, refLength) } - } - } - - await test(input.amount, input.referenceCount, input.refLength) - }) + strictEqual(values4[values4.length - 1].refs.length, referencesCount) }) }) }) diff --git a/test/orbitdb-replication.test.js b/test/orbitdb-replication.test.js index 3034da0..d9629a0 100644 --- a/test/orbitdb-replication.test.js +++ b/test/orbitdb-replication.test.js @@ -44,9 +44,11 @@ describe('Replicating databases', function () { before(async () => { db1 = await orbitdb1.open('helloworld') + console.time('write') for (let i = 0; i < expected.length; i++) { await db1.add(expected[i]) } + console.timeEnd('write') }) after(async () => { @@ -59,7 +61,7 @@ describe('Replicating databases', function () { let replicated = false - const onJoin = async (peerId) => { + const onJoin = async (peerId, heads) => { const head = (await db2.log.heads())[0] if (head && head.clock.time === amount) { replicated = true