Mirroristas/orbitdb
2
0
Fork 0
mirror of https://github.com/orbitdb/orbitdb.git synced 2025-03-30 15:08:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'helia' into fix/helia-6

Browse Source
This commit is contained in:
Hayden Young 2023-12-16 00:27:42 +08:00 committed by GitHub
commit 7003912d2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 675 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
benchmarks/orbitdb-events.js
View File

@ -1,33 +1,10 @@
import { createOrbitDB } from '../src/index.js'
import { rimraf as rmrf } from 'rimraf'
import * as IPFS from 'ipfs-core'
import createHelia from '../test/utils/create-helia.js'
import { EventEmitter } from 'events'
EventEmitter.defaultMaxListeners = 10000
const ipfsConfig = {
preload: {
enabled: false
},
config: {
Addresses: {
API: '/ip4/127.0.0.1/tcp/0',
Swarm: ['/ip4/0.0.0.0/tcp/0'],
Gateway: '/ip4/0.0.0.0/tcp/0'
},
Bootstrap: [],
Discovery: {
MDNS: {
Enabled: false,
Interval: 0
},
webRTCStar: {
Enabled: false
}
}
}
}
;(async () => {
console.log('Starting benchmark...')
@ -36,7 +13,7 @@ const ipfsConfig = {
await rmrf('./ipfs')
await rmrf('./orbitdb')
const ipfs = await IPFS.create({ ...ipfsConfig, repo: './ipfs' })
const ipfs = await createHelia()
const orbitdb = await createOrbitDB({ ipfs })
console.log(`Create ${entryCount} events`)

27
benchmarks/orbitdb-keyvalue.js
View File

@ -1,33 +1,10 @@
import { createOrbitDB } from '../src/index.js'
import { rimraf as rmrf } from 'rimraf'
import * as IPFS from 'ipfs-core'
import createHelia from '../test/utils/create-helia.js'
import { EventEmitter } from 'events'
EventEmitter.defaultMaxListeners = 10000
const ipfsConfig = {
preload: {
enabled: false
},
config: {
Addresses: {
API: '/ip4/127.0.0.1/tcp/0',
Swarm: ['/ip4/0.0.0.0/tcp/0'],
Gateway: '/ip4/0.0.0.0/tcp/0'
},
Bootstrap: [],
Discovery: {
MDNS: {
Enabled: false,
Interval: 0
},
webRTCStar: {
Enabled: false
}
}
}
}
;(async () => {
console.log('Starting benchmark...')
@ -36,7 +13,7 @@ const ipfsConfig = {
await rmrf('./ipfs')
await rmrf('./orbitdb')
const ipfs = await IPFS.create({ ...ipfsConfig, repo: './ipfs' })
const ipfs = await createHelia()
const orbitdb = await createOrbitDB({ ipfs })
console.log(`Set ${entryCount} keys/values`)

42
benchmarks/orbitdb-replicate.js
View File

@ -1,47 +1,22 @@
import { createOrbitDB } from '../src/index.js'
import { rimraf as rmrf } from 'rimraf'
import * as IPFS from 'ipfs-core'
import createHelia from '../test/utils/create-helia.js'
import connectPeers from '../test/utils/connect-nodes.js'
import waitFor from '../test/utils/wait-for.js'
import { EventEmitter } from 'events'
EventEmitter.defaultMaxListeners = 10000
const ipfsConfig = {
preload: {
enabled: false
},
config: {
Addresses: {
API: '/ip4/127.0.0.1/tcp/0',
Swarm: ['/ip4/0.0.0.0/tcp/0'],
Gateway: '/ip4/0.0.0.0/tcp/0'
},
Bootstrap: [],
Discovery: {
MDNS: {
Enabled: true,
Interval: 0
},
webRTCStar: {
Enabled: false
}
}
}
}
;(async () => {
console.log('Starting benchmark...')
const entryCount = 1000
await rmrf('./ipfs1')
await rmrf('./ipfs2')
await rmrf('./orbitdb1')
await rmrf('./orbitdb2')
const ipfs1 = await IPFS.create({ ...ipfsConfig, repo: './ipfs1' })
const ipfs2 = await IPFS.create({ ...ipfsConfig, repo: './ipfs2' })
const [ipfs1, ipfs2] = await Promise.all([createHelia(), createHelia()])
const orbitdb1 = await createOrbitDB({ ipfs: ipfs1, directory: './orbitdb1' })
const orbitdb2 = await createOrbitDB({ ipfs: ipfs2, directory: './orbitdb2' })
@ -75,19 +50,12 @@ const ipfsConfig = {
await waitFor(() => connected, () => true)
console.log(`Iterate ${entryCount} events to replicate them`)
const all = []
for await (const { value } of db2.iterator()) {
all.unshift(value)
}
const endTime2 = new Date().getTime()
const duration2 = endTime2 - startTime2
const operationsPerSecond2 = Math.floor(entryCount / (duration2 / 1000))
const millisecondsPerOp2 = duration2 / entryCount
console.log(`Replicating ${all.length} events took ${duration2} ms, ${operationsPerSecond2} ops/s, ${millisecondsPerOp2} ms/op`)
console.log(`Replicating ${entryCount} events took ${duration2} ms, ${operationsPerSecond2} ops/s, ${millisecondsPerOp2} ms/op`)
await db1.drop()
await db1.close()
@ -99,8 +67,6 @@ const ipfsConfig = {
await ipfs1.stop()
await ipfs2.stop()
await rmrf('./ipfs1')
await rmrf('./ipfs2')
await rmrf('./orbitdb1')
await rmrf('./orbitdb2')

806
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

20
package.json
View File

@ -18,29 +18,29 @@
"type": "module",
"main": "src/index.js",
"dependencies": {
"@ipld/dag-cbor": "^9.0.5",
"@libp2p/crypto": "^2.0.4",
"@ipld/dag-cbor": "^9.0.6",
"@libp2p/crypto": "^3.0.2",
"it-pipe": "^3.0.1",
"level": "^8.0.0",
"lru": "^3.1.0",
"multiformats": "^12.1.1",
"p-queue": "^7.4.1",
"multiformats": "^12.1.3",
"p-queue": "^8.0.1",
"timeout-abort-controller": "^3.0.0",
"uint8arrays": "^4.0.6"
"uint8arrays": "^5.0.0"
},
"devDependencies": {
"c8": "^8.0.1",
"cross-env": "^7.0.3",
"fs-extra": "^11.1.1",
"fs-extra": "^11.2.0",
"helia": "^2.1.0",
"it-all": "^3.0.3",
"it-all": "^3.0.4",
"jsdoc": "^4.0.2",
"mocha": "^10.2.0",
"path-browserify": "^1.0.1",
"playwright-test": "^12.6.1",
"rimraf": "^5.0.1",
"playwright-test": "^14.0.0",
"rimraf": "^5.0.5",
"standard": "^17.1.0",
"webpack": "^5.88.2",
"webpack": "^5.89.0",
"webpack-cli": "^5.1.4"
},
"scripts": {

18
src/config/libp2p/index.js
View File

@ -8,7 +8,7 @@ import { gossipsub } from '@chainsafe/libp2p-gossipsub'
import { circuitRelayTransport } from 'libp2p/circuit-relay'
/**
* A basic Libp2p configuration for node servers.
* A basic Libp2p configuration for Node.js nodes.
*/
export const DefaultLibp2pOptions = {
addresses: {
@ -24,13 +24,9 @@ export const DefaultLibp2pOptions = {
})
],
connectionEncryption: [noise()],
streamMuxers: [
yamux()
],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => {
return false
}
denyDialMultiaddr: () => false
},
services: {
identify: identifyService(),
@ -55,13 +51,9 @@ export const DefaultLibp2pBrowserOptions = {
})
],
connectionEncryption: [noise()],
streamMuxers: [
yamux()
],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => {
return false
}
denyDialMultiaddr: () => false
},
services: {
identify: identifyService(),

32
src/sync.js
View File

@ -115,6 +115,9 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
if (!ipfs) throw new Error('An instance of ipfs is required.')
if (!log) throw new Error('An instance of log is required.')
const libp2p = ipfs.libp2p
const pubsub = ipfs.libp2p.services.pubsub
const address = log.id
const headsSyncAddress = pathJoin('/orbitdb/heads/', address)
@ -194,7 +197,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
const { signal } = timeoutController
try {
peers.add(peerId)
const stream = await ipfs.libp2p.dialProtocol(remotePeer, headsSyncAddress, { signal })
const stream = await libp2p.dialProtocol(remotePeer, headsSyncAddress, { signal })
await pipe(sendHeads, stream, receiveHeads(peerId))
} catch (e) {
console.error(e)
@ -218,18 +221,19 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
}
const handleUpdateMessage = async message => {
const { topic, data } = message.detail
const task = async () => {
const messageHasData = message => message.detail.data !== undefined
try {
if (messageHasData(message) && onSynced) {
await onSynced(message.detail.data)
if (data && onSynced) {
await onSynced(data)
}
} catch (e) {
events.emit('error', e)
}
}
if (message.detail.topic === address) {
if (topic === address) {
queue.add(task)
}
}
@ -243,7 +247,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
*/
const add = async (entry) => {
if (started) {
await ipfs.libp2p.services.pubsub.publish(address, entry.bytes)
await pubsub.publish(address, entry.bytes)
}
}
@ -257,10 +261,10 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
if (started) {
started = false
await queue.onIdle()
ipfs.libp2p.services.pubsub.removeEventListener('subscription-change', handlePeerSubscribed)
ipfs.libp2p.services.pubsub.removeEventListener('message', handleUpdateMessage)
await ipfs.libp2p.unhandle(headsSyncAddress)
await ipfs.libp2p.services.pubsub.unsubscribe(address)
pubsub.removeEventListener('subscription-change', handlePeerSubscribed)
pubsub.removeEventListener('message', handleUpdateMessage)
await libp2p.unhandle(headsSyncAddress)
await pubsub.unsubscribe(address)
peers.clear()
}
}
@ -274,11 +278,11 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
const startSync = async () => {
if (!started) {
// Exchange head entries with peers when connected
await ipfs.libp2p.handle(headsSyncAddress, handleReceiveHeads)
ipfs.libp2p.services.pubsub.addEventListener('subscription-change', handlePeerSubscribed)
ipfs.libp2p.services.pubsub.addEventListener('message', handleUpdateMessage)
await libp2p.handle(headsSyncAddress, handleReceiveHeads)
pubsub.addEventListener('subscription-change', handlePeerSubscribed)
pubsub.addEventListener('message', handleUpdateMessage)
// Subscribe to the pubsub channel for this database through which updates are sent
await ipfs.libp2p.services.pubsub.subscribe(address)
await pubsub.subscribe(address)
started = true
}
}

1
test/database.test.js
View File

@ -30,7 +30,6 @@ describe('Database', function () {
}
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
await copy(testKeysPath, keysPath)
keystore = await KeyStore({ path: keysPath })

1
test/databases/documents.test.js
View File

@ -19,7 +19,6 @@ describe('Documents Database', function () {
const databaseId = 'documents-AAA'
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
await copy(testKeysPath, keysPath)

1
test/databases/events.test.js
View File

@ -19,7 +19,6 @@ describe('Events Database', function () {
const databaseId = 'events-AAA'
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
await copy(testKeysPath, keysPath)

1
test/databases/keyvalue-indexed.js
View File

@ -21,7 +21,6 @@ describe('KeyValueIndexed Database', function () {
const databaseId = 'keyvalue-AAA'
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
await copy(testKeysPath, keysPath)

1
test/databases/keyvalue.test.js
View File

@ -19,7 +19,6 @@ describe('KeyValue Database', function () {
const databaseId = 'keyvalue-AAA'
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
await copy(testKeysPath, keysPath)

1
test/manifest-store.test.js
View File

@ -9,7 +9,6 @@ describe('Manifest', () => {
let manifestStore
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo })
ipfs = await createHelia()
manifestStore = await ManifestStore({ ipfs })
})

1
test/orbitdb-custom-database-types.test.js
View File

@ -26,7 +26,6 @@ describe('Add a custom database type', function () {
let orbitdb
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
orbitdb = await createOrbitDB({ ipfs })
})

1
test/orbitdb-custom-identity-providers.test.js
View File

@ -11,7 +11,6 @@ describe('Add a custom identity provider', function () {
let ipfs
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
})

1
test/orbitdb-drop.test.js
View File

@ -11,7 +11,6 @@ describe('Drop databases', function () {
let db
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs' })
ipfs = await createHelia()
})

2
test/orbitdb.test.js
View File

@ -50,7 +50,7 @@ describe('OrbitDB', function () {
it('has the IPFS instance given as a parameter', async () => {
const { id: expectedId } = ipfs1.libp2p.peerId
const { id: resultId } = ipfs1.libp2p.peerId
const { id: resultId } = orbitdb1.ipfs.libp2p.peerId
strictEqual(expectedId, resultId)
})

1
test/storage.test.js
View File

@ -16,7 +16,6 @@ describe('Storages', function () {
let testIdentity
before(async () => {
// ipfs = await IPFS.create({ ...config.daemon1, repo: './ipfs1' })
ipfs = await createHelia()
await copy(testKeysPath, keysPath)
keystore = await KeyStore({ path: keysPath })

2
test/sync.test.js
View File

@ -62,7 +62,7 @@ describe('Sync protocol', function () {
await sync.stop()
}
if (log) {
await log.close
await log.close()
}
})

19
test/utils/connect-nodes.js
View File

@ -14,29 +14,18 @@ const connectIpfsNodes = async (ipfs1, ipfs2, options = {
await ipfs1.libp2p.dial(multiaddr(`/ip4/127.0.0.1/tcp/12345/ws/p2p/${relayId}`))
let a1
let address1
await waitFor(() => {
a1 = ipfs1.libp2p.getMultiaddrs().filter(ma => WebRTC.matches(ma)).pop()
if (a1 != null) {
return true
} else {
return false
}
address1 = ipfs1.libp2p.getMultiaddrs().filter(ma => WebRTC.matches(ma)).pop()
return address1 != null
}, () => true)
await ipfs2.libp2p.dial(a1)
await ipfs2.libp2p.dial(address1)
} else {
await ipfs2.libp2p.peerStore.save(ipfs1.libp2p.peerId, { multiaddrs: ipfs1.libp2p.getMultiaddrs().filter(options.filter) })
await ipfs2.libp2p.dial(ipfs1.libp2p.peerId)
}
return new Promise((resolve) => {
setTimeout(() => {
resolve()
}, 1000)
})
}
export default connectIpfsNodes

8
test/utils/create-helia.js
View File

@ -1,4 +1,5 @@
import { createHelia } from 'helia'
import { bitswap } from 'helia/block-brokers'
import { createLibp2p } from 'libp2p'
import { DefaultLibp2pOptions, DefaultLibp2pBrowserOptions } from '../../src/index.js'
@ -9,5 +10,10 @@ export default async () => {
const libp2p = await createLibp2p({ ...options })
return createHelia({ libp2p })
const heliaOptions = {
libp2p,
blockBrokers: [bitswap()]
}
return createHelia({ ...heliaOptions })
}