mirror of
https://github.com/orbitdb/orbitdb.git
synced 2025-07-31 07:43:14 +00:00
Compare commits
No commits in common. "main" and "v2.2.0" have entirely different histories.
4
.github/workflows/api-publish.yml
vendored
4
.github/workflows/api-publish.yml
vendored
@ -25,7 +25,7 @@ jobs:
|
||||
registry-url: https://registry.npmjs.org/
|
||||
- run: npm ci
|
||||
- run: npm run build:docs
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
- uses: actions/upload-pages-artifact@v2
|
||||
with:
|
||||
path: ./docs/api/
|
||||
|
||||
@ -37,5 +37,5 @@ jobs:
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4 # or the latest "vX.X.X" version tag for this action
|
||||
uses: actions/deploy-pages@v2 # or the latest "vX.X.X" version tag for this action
|
||||
|
||||
20
.github/workflows/run-test-node-min-ver.yml
vendored
20
.github/workflows/run-test-node-min-ver.yml
vendored
@ -1,20 +0,0 @@
|
||||
---
|
||||
name: Run Tests (Node Minimum Version)
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 20
|
||||
registry-url: https://registry.npmjs.org/
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Run linter
|
||||
run: npm run lint
|
||||
- name: Run tests
|
||||
run: npm run test:ci
|
||||
@ -1 +0,0 @@
|
||||
https://orbitdb.org/funding.json
|
||||
@ -2,10 +2,4 @@
|
||||
|
||||
For now, please refer to our Git commit history for a list of changes.
|
||||
|
||||
https://github.com/orbitdb/orbitdb/compare/v2.5.0...v3.0.2
|
||||
|
||||
You can also use the following git command to generate a log of changes:
|
||||
|
||||
```
|
||||
git log v2.5.0..v3.0.2 --oneline
|
||||
```
|
||||
https://github.com/orbitdb/orbitdb/commits/v1.0.2
|
||||
|
||||
@ -28,7 +28,7 @@ Install OrbitDB and its dependencies:
|
||||
npm install @orbitdb/core helia
|
||||
```
|
||||
|
||||
### Browser `<script>` tag
|
||||
### Browser <script> tag
|
||||
|
||||
OrbitDB can be loaded in the browser using the distributed js file with the `<script/>` tag. OrbitDB is the global namespace and all external functions are available via this namespace:
|
||||
|
||||
@ -53,7 +53,7 @@ const Libp2pOptions = {
|
||||
services: {
|
||||
pubsub: gossipsub({
|
||||
// neccessary to run a single peer
|
||||
allowPublishToZeroTopicPeers: true
|
||||
allowPublishToZeroPeers: true
|
||||
}),
|
||||
identify: identify()
|
||||
}
|
||||
@ -109,7 +109,6 @@ You can find more advanced topics in our [docs](https://github.com/orbitdb/orbit
|
||||
- [Connecting Peers](https://github.com/orbitdb/orbitdb/blob/main/docs/CONNECTING_PEERS.md)
|
||||
- [Replication](https://github.com/orbitdb/orbitdb/blob/main/docs/REPLICATION.md)
|
||||
- [Oplog](https://github.com/orbitdb/orbitdb/blob/main/docs/OPLOG.md)
|
||||
- [Encryption](https://github.com/orbitdb/orbitdb/blob/main/docs/ENCRYPTION.md)
|
||||
|
||||
### API
|
||||
|
||||
|
||||
@ -1,68 +0,0 @@
|
||||
import { createOrbitDB } from '../src/index.js'
|
||||
// import { createOrbitDB, MemoryStorage } from '../src/index.js'
|
||||
import { rimraf as rmrf } from 'rimraf'
|
||||
import createHelia from '../test/utils/create-helia.js'
|
||||
|
||||
import { EventEmitter } from 'events'
|
||||
EventEmitter.defaultMaxListeners = 10000
|
||||
|
||||
;(async () => {
|
||||
console.log('Starting benchmark...')
|
||||
|
||||
const entryCount = 1000
|
||||
|
||||
await rmrf('./ipfs')
|
||||
await rmrf('./orbitdb')
|
||||
|
||||
const ipfs = await createHelia()
|
||||
const orbitdb = await createOrbitDB({ ipfs })
|
||||
|
||||
console.log(`Insert ${entryCount} documents`)
|
||||
|
||||
// const entryStorage = await MemoryStorage()
|
||||
// const headsStorage = await MemoryStorage()
|
||||
// const indexStorage = await MemoryStorage()
|
||||
|
||||
// const db1 = await orbitdb.open('benchmark-documents', { type: 'documents', referencesCount: 16, entryStorage, headsStorage, indexStorage })
|
||||
|
||||
const db1 = await orbitdb.open('benchmark-documents', { type: 'documents' })
|
||||
|
||||
const startTime1 = new Date().getTime()
|
||||
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
const doc = { _id: i.toString(), message: 'hello ' + i }
|
||||
await db1.put(doc)
|
||||
}
|
||||
|
||||
const endTime1 = new Date().getTime()
|
||||
const duration1 = endTime1 - startTime1
|
||||
const operationsPerSecond1 = Math.floor(entryCount / (duration1 / 1000))
|
||||
const millisecondsPerOp1 = duration1 / entryCount
|
||||
console.log(`Inserting ${entryCount} documents took ${duration1} ms, ${operationsPerSecond1} ops/s, ${millisecondsPerOp1} ms/op`)
|
||||
|
||||
console.log(`Query ${entryCount} documents`)
|
||||
const startTime2 = new Date().getTime()
|
||||
|
||||
const all = []
|
||||
for await (const { key, value } of db1.iterator()) {
|
||||
all.unshift({ key, value })
|
||||
}
|
||||
|
||||
const endTime2 = new Date().getTime()
|
||||
const duration2 = endTime2 - startTime2
|
||||
const operationsPerSecond2 = Math.floor(entryCount / (duration2 / 1000))
|
||||
const millisecondsPerOp2 = duration2 / entryCount
|
||||
|
||||
console.log(`Querying ${all.length} documents took ${duration2} ms, ${operationsPerSecond2} ops/s, ${millisecondsPerOp2} ms/op`)
|
||||
|
||||
await db1.drop()
|
||||
await db1.close()
|
||||
|
||||
await orbitdb.stop()
|
||||
await ipfs.stop()
|
||||
|
||||
await rmrf('./ipfs')
|
||||
await rmrf('./orbitdb')
|
||||
|
||||
process.exit(0)
|
||||
})()
|
||||
@ -45,10 +45,8 @@ EventEmitter.defaultMaxListeners = 10000
|
||||
let connected = false
|
||||
|
||||
const onJoin = async (peerId) => (connected = true)
|
||||
const onError = async (err) => console.error(err)
|
||||
|
||||
db2.events.on('join', onJoin)
|
||||
db2.events.on('error', onError)
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
|
||||
|
||||
@ -38,8 +38,6 @@ export default (env, argv) => {
|
||||
],
|
||||
fallback: {
|
||||
path: require.resolve('path-browserify'),
|
||||
crypto: false,
|
||||
stream: require.resolve('stream-browserify'),
|
||||
process: false
|
||||
}
|
||||
},
|
||||
|
||||
@ -72,7 +72,7 @@ const options = {
|
||||
filter: filters.all
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
services: {
|
||||
identify: identify(),
|
||||
@ -114,7 +114,7 @@ const options = {
|
||||
discoverRelays: 1
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
connectionGater: {
|
||||
denyDialMultiaddr: () => {
|
||||
@ -180,7 +180,7 @@ const options = {
|
||||
discoverRelays: 1
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
connectionGater: {
|
||||
denyDialMultiaddr: () => {
|
||||
@ -247,7 +247,7 @@ const options = {
|
||||
discoverRelays: 1
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
connectionGater: {
|
||||
denyDialMultiaddr: () => {
|
||||
|
||||
@ -208,7 +208,7 @@ for await (const record of db2.iterator()) {
|
||||
}
|
||||
```
|
||||
|
||||
To learn more, check out [OrbitDB's sychronization protocol](https://api.orbitdb.org/module-Sync.html) and the [OrbitDB replication documentation](./REPLICATION.md).
|
||||
To learn more, check out [OrbitDB's sychronization protocol](https://orbitdb.org/api/module-Sync.html) and the [OrbitDB replication documentation](./REPLICATION.md).
|
||||
|
||||
## Custom databases
|
||||
|
||||
@ -264,7 +264,7 @@ CustomDB.type = type
|
||||
export default CustomDB
|
||||
```
|
||||
|
||||
[Documents](../src/databases/documents.js), [Events](../src/databases/events.js) and [KeyValue](../src/databases/keyvalue.js) provide good examples of how a database is implemented in OrbitDB and how to add the logic for returning records from the database (the state of the database).
|
||||
[Documents](../src/db/documents.js), [Events](../src/db/events.js) and [KeyValue](../src/db/keyvalue.js) provide good examples of how a database is implemented in OrbitDB and how to add the logic for returning records from the database (the state of the database).
|
||||
|
||||
To use a custom database, add it to the list of supported database types:
|
||||
|
||||
|
||||
@ -1,94 +0,0 @@
|
||||
# Encryption
|
||||
|
||||
OrbitDB features a modular architecture for database encryption. By passing a module to an OrbitDB database, different encryption methods can be employed.
|
||||
|
||||
OrbitDB project currently maintains a [SimpleEncryption](https://github.com/orbitdb/simple-encryption) module that can be used to get started with encrypted databases.
|
||||
|
||||
**WARNING:** SimpleEncryption is an unaudited encryption module. Use at your own risk.
|
||||
|
||||
## How it works
|
||||
|
||||
OrbitDB encrypts databases in two layers: encrypting the payload and encrypting the log entry.
|
||||
|
||||
Log entry encryption encrypts each log entry fully. Payload encryption encrypts just the value.
|
||||
|
||||
This makes it possible to separate users of a database and replicators of a database, ie. an orbitdb peer can replicate a database without being able to decrypt the payloads (=data) of the database.
|
||||
|
||||
## Configuring encryption
|
||||
|
||||
You can configure OrbitDB to encrypt either the payload data being stored or the entire database.
|
||||
|
||||
To ***encrypt payload data only***, specify an encryption module and pass it to OtbiDB via the encryption object using the `data` property:
|
||||
|
||||
```
|
||||
const data = await EncryptionModule()
|
||||
const encryption = { data }
|
||||
const db = await orbitdb.open(dbNameOrAddress, { encryption })
|
||||
```
|
||||
|
||||
To ***encrypt the database log entries***, specify an encryption module and pass it to OrbitDB via the encryption object using the `replication` property:
|
||||
|
||||
```
|
||||
const replication = await EncryptionModule()
|
||||
const encryption = { replication }
|
||||
const db = await orbitdb.open(dbNameOrAddress, { encryption })
|
||||
```
|
||||
|
||||
To ***encrypt the whole database***, payload data and oplog entries separately, specify an encryption module and pass it to OrbitDB via the encryption object using both the `replication` and `data` properties:
|
||||
|
||||
```
|
||||
const replication = await EncryptionModule()
|
||||
const data = await EncryptionModule()
|
||||
const encryption = { replication, data }
|
||||
const db = await orbitdb.open(dbNameOrAddress, { encryption })
|
||||
```
|
||||
|
||||
## Encrypting Databases
|
||||
|
||||
OrbitDB provides simple password-based encryption via an external module called [SimpleEncryption](https://github.com/orbitdb/simple-encryption).
|
||||
|
||||
**WARNING:** This is an unaudited encryption module. Use at your own risk.
|
||||
|
||||
To install SimpleEncryption:
|
||||
|
||||
```
|
||||
npm i @orbitdb/simple-encryption
|
||||
```
|
||||
|
||||
To use encryption, initiate SimpleEncryption with a password and pass it to OrbitDB when opening your database:
|
||||
|
||||
```js
|
||||
import { SimpleEncryption } from '@orbitdb/simple-encryption'
|
||||
|
||||
const replication = await SimpleEncryption({ password: 'hello' })
|
||||
const data = await SimpleEncryption({ password: 'world' })
|
||||
|
||||
const encryption = { data, replication }
|
||||
|
||||
const db = await orbitdb.open(dbNameOrAddress, { encryption })
|
||||
```
|
||||
|
||||
If you wish to use another encryption type, simply replace SimpleEncryption with the module of your choice.
|
||||
|
||||
## Custom Encryption
|
||||
|
||||
To implement a custom encryption module for OrbitDB, expose encrypt and decrypt functions:
|
||||
|
||||
```
|
||||
const CustomEncryption = async () => {
|
||||
const encrypt = (value) => {
|
||||
// return encrypted value
|
||||
}
|
||||
|
||||
const decrypt = (value) => {
|
||||
// return decrypted value
|
||||
}
|
||||
|
||||
return {
|
||||
encrypt,
|
||||
decrypt
|
||||
}
|
||||
}
|
||||
|
||||
export default CustomEncryption
|
||||
```
|
||||
@ -50,11 +50,11 @@ A simple Node.js example might look something like:
|
||||
transports: [
|
||||
tcp()
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
services: {
|
||||
identify: identify(),
|
||||
pubsub: gossipsub({ allowPublishToZeroTopicPeers: true })
|
||||
pubsub: gossipsub({ allowPublishToZeroPeers: true })
|
||||
}
|
||||
}
|
||||
```
|
||||
@ -79,11 +79,11 @@ export const Libp2pOptions = {
|
||||
transports: [
|
||||
tcp()
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
services: {
|
||||
identify: identify(),
|
||||
pubsub: gossipsub({ allowPublishToZeroTopicPeers: true })
|
||||
pubsub: gossipsub({ allowPublishToZeroPeers: true })
|
||||
}
|
||||
}
|
||||
```
|
||||
@ -104,7 +104,7 @@ Assuming you have a Node.js development environment installed, create a new proj
|
||||
mkdir orbitdb-app
|
||||
cd orbitdb-app
|
||||
npm init
|
||||
npm i helia @orbitdb/core blockstore-level @chainsafe/libp2p-gossipsub
|
||||
npm i helia orbitdb/core blockstore-level @chainsafe/libp2p-gossipsub
|
||||
```
|
||||
|
||||
Create a file in your project called index.js and add the following code to it:
|
||||
@ -159,7 +159,7 @@ const db = await orbitdb.open('my-db')
|
||||
to:
|
||||
|
||||
```js
|
||||
const db = await orbitdb.open('my-documents-db', { type: 'documents' })
|
||||
const db = await orbitdb.open('my-documents-db', { type: 'documents '})
|
||||
```
|
||||
|
||||
Also replace:
|
||||
@ -172,8 +172,8 @@ await db.add('hello world 2')
|
||||
with:
|
||||
|
||||
```js
|
||||
await db.put({ _id: "doc1", hello: "world 1", hits: 5 })
|
||||
await db.put({ _id: "doc2", hello: "world 2", hits: 2 })
|
||||
await db.put('doc1', { hello: "world 1", hits: 5 })
|
||||
await db.put('doc2', { hello: "world 2", hits: 2 })
|
||||
```
|
||||
|
||||
Run index.js again:
|
||||
@ -195,7 +195,7 @@ To create an OrbitDB database peer, create a new project called `orbitdb-peer`:
|
||||
mkdir orbitdb-peer
|
||||
cd orbitdb-peer
|
||||
npm init
|
||||
npm i helia @orbitdb/core blockstore-level @chainsafe/libp2p-gossipsub
|
||||
npm i helia orbitdb/core blockstore-level @chainsafe/libp2p-gossipsub
|
||||
```
|
||||
|
||||
Create a new file called index.js and paste in the following code:
|
||||
@ -206,7 +206,6 @@ import { createHelia } from 'helia'
|
||||
import { createOrbitDB, IPFSAccessController } from '@orbitdb/core'
|
||||
import { LevelBlockstore } from 'blockstore-level'
|
||||
import { Libp2pOptions } from './config/libp2p.js'
|
||||
import { multiaddr } from '@multiformats/multiaddr'
|
||||
|
||||
const main = async () => {
|
||||
// create a random directory to avoid OrbitDB conflicts.
|
||||
@ -220,9 +219,7 @@ const main = async () => {
|
||||
|
||||
let db
|
||||
|
||||
if (process.argv[2] && process.argv[3]) {
|
||||
await orbitdb.ipfs.libp2p.dial(multiaddr(process.argv[3]))
|
||||
console.log('opening db', process.argv[2])
|
||||
if (process.argv[2]) {
|
||||
db = await orbitdb.open(process.argv[2])
|
||||
} else {
|
||||
// When we open a new database, write access is only available to the
|
||||
@ -233,8 +230,6 @@ const main = async () => {
|
||||
// access using grant and revoke.
|
||||
db = await orbitdb.open('my-db', { AccessController: IPFSAccessController({ write: ['*']}) })
|
||||
|
||||
console.log('libp2p address', '(copy one of these addresses then dial into this node from the second node)', orbitdb.ipfs.libp2p.getMultiaddrs())
|
||||
|
||||
// Copy this output if you want to connect a peer to another.
|
||||
console.log('my-db address', '(copy my db address and use when launching peer 2)', db.address)
|
||||
}
|
||||
@ -268,33 +263,27 @@ const main = async () => {
|
||||
main()
|
||||
```
|
||||
|
||||
Launch peer 1 from the terminal:
|
||||
Open two consoles in your command line terminal.
|
||||
|
||||
```bash
|
||||
node test.js
|
||||
In terminal 1, run the first peer:
|
||||
|
||||
```sh
|
||||
node index.js
|
||||
```
|
||||
|
||||
Once launched you will see some output which may look something like this:
|
||||
When running, you should see the address of the database, for example:
|
||||
|
||||
```
|
||||
libp2p address (copy one of these addresses then dial into this node from the second node) [
|
||||
Multiaddr(/ip4/127.0.0.1/tcp/36161/p2p/12D3KooWKFWB78Hka2uPVNYYoXfucWp6rDLsQzr5CFiP67NAo7YF),
|
||||
Multiaddr(/ip4/192.168.1.22/tcp/36161/p2p/12D3KooWKFWB78Hka2uPVNYYoXfucWp6rDLsQzr5CFiP67NAo7YF),
|
||||
Multiaddr(/ip4/100.64.100.6/tcp/36161/p2p/12D3KooWKFWB78Hka2uPVNYYoXfucWp6rDLsQzr5CFiP67NAo7YF)
|
||||
]
|
||||
```sh
|
||||
my-db address (copy my db address and use when launching peer 2) /orbitdb/zdpuB2aYUCnZ7YUBrDkCWpRLQ8ieUbqJEVRZEd5aDhJBDpBqj
|
||||
```
|
||||
|
||||
It contains the libp2p address and db address. You will need both of these when connecting from peer 2.
|
||||
Copy the database's address from terminal 1 and, in terminal 2, run:
|
||||
|
||||
Open another terminal and launch peer 2. The command takes the form `node test.js <orbitdb-address> <libp2p-address>`
|
||||
|
||||
```bash
|
||||
node test.js /orbitdb/zdpuB2aYUCnZ7YUBrDkCWpRLQ8ieUbqJEVRZEd5aDhJBDpBqj /ip4/127.0.0.1/tcp/36161/p2p/12D3KooWKFWB78Hka2uPVNYYoXfucWp6rDLsQzr5CFiP67NAo7YF
|
||||
```sh
|
||||
node index.js /orbitdb/zdpuB2aYUCnZ7YUBrDkCWpRLQ8ieUbqJEVRZEd5aDhJBDpBqj
|
||||
```
|
||||
|
||||
What is happening is the second peer is dialing the first peer on the /ip4/ address then opens the database.
|
||||
|
||||
Both peers will print new records to the terminal as the log is updated. When you stop each peer using ctrl+c, the final state of the database will be printed to the terminal. They should match.
|
||||
|
||||
**PLEASE NOTE:**
|
||||
|
||||
@ -306,6 +295,4 @@ These kinds of connectivity configurations are beyond the scope of OrbitDB. To f
|
||||
|
||||
[Databases](./DATABASES.md) covers database management and data entry in more detail.
|
||||
|
||||
[Replication](./REPLICATION.md) provides a comprehensive overview of how to perform data replication across multiple peers.
|
||||
|
||||
[Encryption](./ENCRYPTION.md) discusses database encryption using OrbitDB's modular architecture.
|
||||
[Replication](./REPLICATION.md) provides a comprehensive overview of how to perform data replication across multiple peers.
|
||||
@ -109,7 +109,7 @@ const identity = await identities.createIdentity({ id: 'me' })
|
||||
const cid = CID.parse(identity.hash, base58btc)
|
||||
|
||||
// Extract the hash from the full db path.
|
||||
const bytes = await ipfs.blockstore.get(cid)
|
||||
const bytes = await ipfs.block.get(cid)
|
||||
|
||||
// Defines how we serialize/hash the data.
|
||||
const codec = dagCbor
|
||||
|
||||
@ -80,4 +80,4 @@ await orbitdb2.stop()
|
||||
await ipfs2.stop()
|
||||
```
|
||||
|
||||
Refer to the API for more information about [OrbitDB's synchronization protocol](https://api.orbitdb.org/module-Sync.html).
|
||||
Refer to the API for more information about [OrbitDB's synchronization protocol](https://orbitdb.org/api/module-Sync.html).
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
## OrbitDB API - v3.0
|
||||
## OrbitDB API - v2.2
|
||||
|
||||
OrbitDB is a serverless, distributed, peer-to-peer database. OrbitDB uses IPFS
|
||||
as its data storage and Libp2p Pubsub to automatically sync databases with peers. It's an eventually consistent database that uses Merkle-CRDTs for conflict-free database writes and merges making OrbitDB an excellent choice for p2p and decentralized apps, blockchain applications and local first web applications.
|
||||
|
||||
11237
package-lock.json
generated
11237
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
17
package.json
17
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@orbitdb/core",
|
||||
"version": "3.0.2",
|
||||
"version": "2.2.0",
|
||||
"description": "Distributed p2p database on IPFS",
|
||||
"author": "Haad",
|
||||
"license": "MIT",
|
||||
@ -19,7 +19,7 @@
|
||||
"main": "src/index.js",
|
||||
"dependencies": {
|
||||
"@ipld/dag-cbor": "^9.0.6",
|
||||
"@libp2p/crypto": "^5.0.5",
|
||||
"@libp2p/crypto": "^3.0.2",
|
||||
"it-pipe": "^3.0.1",
|
||||
"level": "^8.0.0",
|
||||
"lru": "^3.1.0",
|
||||
@ -29,19 +29,20 @@
|
||||
"uint8arrays": "^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@chainsafe/libp2p-gossipsub": "^14.1.0",
|
||||
"@libp2p/circuit-relay-v2": "^3.1.0",
|
||||
"@orbitdb/simple-encryption": "^0.0.2",
|
||||
"blockstore-level": "^2.0.1",
|
||||
"@chainsafe/libp2p-gossipsub": "^12.0.0",
|
||||
"@chainsafe/libp2p-yamux": "^6.0.1",
|
||||
"@helia/block-brokers": "^1.0.0",
|
||||
"@libp2p/circuit-relay-v2": "^1.0.10",
|
||||
"blockstore-level": "^1.1.7",
|
||||
"c8": "^8.0.1",
|
||||
"cross-env": "^7.0.3",
|
||||
"fs-extra": "^11.2.0",
|
||||
"helia": "^5.3.0",
|
||||
"helia": "^4.0.1",
|
||||
"it-all": "^3.0.4",
|
||||
"jsdoc": "^4.0.2",
|
||||
"mocha": "^10.2.0",
|
||||
"path-browserify": "^1.0.1",
|
||||
"playwright-test": "^14.1.9",
|
||||
"playwright-test": "^14.0.0",
|
||||
"rimraf": "^5.0.5",
|
||||
"standard": "^17.1.0",
|
||||
"webpack": "^5.89.0",
|
||||
|
||||
@ -84,7 +84,7 @@ const IPFSAccessController = ({ write, storage } = {}) => async ({ orbitdb, iden
|
||||
// Allow if the write access list contain the writer's id or is '*'
|
||||
if (write.includes(id) || write.includes('*')) {
|
||||
// Check that the identity is valid
|
||||
return await identities.verifyIdentity(writerIdentity)
|
||||
return identities.verifyIdentity(writerIdentity)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@ -58,7 +58,7 @@ const OrbitDBAccessController = ({ write } = {}) => async ({ orbitdb, identities
|
||||
// If the ACL contains the writer's public key or it contains '*'
|
||||
const hasWriteAccess = await hasCapability('write', id) || await hasCapability('admin', id)
|
||||
if (hasWriteAccess) {
|
||||
return await identities.verifyIdentity(writerIdentity)
|
||||
return identities.verifyIdentity(writerIdentity)
|
||||
}
|
||||
|
||||
return false
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
import { EventEmitter } from 'events'
|
||||
import PQueue from 'p-queue'
|
||||
import Sync from './sync.js'
|
||||
import { Log } from './oplog/index.js'
|
||||
import { Log, Entry } from './oplog/index.js'
|
||||
import { ComposedStorage, LRUStorage, IPFSBlockStorage, LevelStorage } from './storage/index.js'
|
||||
import pathJoin from './utils/path-join.js'
|
||||
|
||||
@ -39,12 +39,10 @@ const defaultCacheSize = 1000
|
||||
* automatically. Otherwise, false.
|
||||
* @param {function} [params.onUpdate] A function callback. Fired when an
|
||||
* entry is added to the oplog.
|
||||
* @param {Function} options.encryptFn An encryption function.
|
||||
* @param {Function} options.decryptFn A decryption function.
|
||||
* @return {module:Databases~Database} An instance of Database.
|
||||
* @instance
|
||||
*/
|
||||
const Database = async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encryption }) => {
|
||||
const Database = async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate }) => {
|
||||
/**
|
||||
* @namespace module:Databases~Database
|
||||
* @description The instance returned by {@link module:Database~Database}.
|
||||
@ -110,9 +108,7 @@ const Database = async ({ ipfs, identity, address, name, access, directory, meta
|
||||
await LevelStorage({ path: pathJoin(directory, '/log/_index/') })
|
||||
)
|
||||
|
||||
encryption = encryption || {}
|
||||
|
||||
const log = await Log(identity, { logId: address, access, entryStorage, headsStorage, indexStorage, encryption })
|
||||
const log = await Log(identity, { logId: address, access, entryStorage, headsStorage, indexStorage })
|
||||
|
||||
const events = new EventEmitter()
|
||||
|
||||
@ -138,23 +134,21 @@ const Database = async ({ ipfs, identity, address, name, access, directory, meta
|
||||
return entry.hash
|
||||
}
|
||||
const hash = await queue.add(task)
|
||||
await queue.onIdle()
|
||||
return hash
|
||||
}
|
||||
|
||||
const applyOperation = async (entry) => {
|
||||
const applyOperation = async (bytes) => {
|
||||
const task = async () => {
|
||||
try {
|
||||
if (entry) {
|
||||
const updated = await log.joinEntry(entry)
|
||||
if (updated) {
|
||||
if (onUpdate) {
|
||||
await onUpdate(log, entry)
|
||||
}
|
||||
events.emit('update', entry)
|
||||
const entry = await Entry.decode(bytes)
|
||||
if (entry) {
|
||||
const updated = await log.joinEntry(entry)
|
||||
if (updated) {
|
||||
if (onUpdate) {
|
||||
await onUpdate(log, entry)
|
||||
}
|
||||
events.emit('update', entry)
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
}
|
||||
await queue.add(task)
|
||||
@ -183,7 +177,7 @@ const Database = async ({ ipfs, identity, address, name, access, directory, meta
|
||||
* @async
|
||||
*/
|
||||
const drop = async () => {
|
||||
await queue.clear()
|
||||
await queue.onIdle()
|
||||
await log.clear()
|
||||
if (access && access.drop) {
|
||||
await access.drop()
|
||||
|
||||
@ -25,8 +25,8 @@ const DefaultOptions = { indexBy: '_id' }
|
||||
* @return {module:Databases.Databases-Documents} A Documents function.
|
||||
* @memberof module:Databases
|
||||
*/
|
||||
const Documents = ({ indexBy } = DefaultOptions) => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encrypt }) => {
|
||||
const database = await Database({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, encrypt })
|
||||
const Documents = ({ indexBy } = DefaultOptions) => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate }) => {
|
||||
const database = await Database({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically })
|
||||
|
||||
const { addOperation, log } = database
|
||||
|
||||
|
||||
@ -15,8 +15,8 @@ const type = 'events'
|
||||
* @return {module:Databases.Databases-Events} A Events function.
|
||||
* @memberof module:Databases
|
||||
*/
|
||||
const Events = () => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encryption }) => {
|
||||
const database = await Database({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encryption })
|
||||
const Events = () => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate }) => {
|
||||
const database = await Database({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate })
|
||||
|
||||
const { addOperation, log } = database
|
||||
|
||||
|
||||
@ -109,7 +109,7 @@ const Index = ({ directory } = {}) => async () => {
|
||||
* function.
|
||||
* @memberof module:Databases
|
||||
*/
|
||||
const KeyValueIndexed = () => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encryption }) => {
|
||||
const KeyValueIndexed = () => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate }) => {
|
||||
// Set up the directory for an index
|
||||
directory = pathJoin(directory || './orbitdb', `./${address}/_index/`)
|
||||
|
||||
@ -117,7 +117,7 @@ const KeyValueIndexed = () => async ({ ipfs, identity, address, name, access, di
|
||||
const index = await Index({ directory })()
|
||||
|
||||
// Set up the underlying KeyValue database
|
||||
const keyValueStore = await KeyValue()({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate: index.update, encryption })
|
||||
const keyValueStore = await KeyValue()({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate: index.update })
|
||||
|
||||
/**
|
||||
* Gets a value from the store by key.
|
||||
|
||||
@ -15,8 +15,8 @@ const type = 'keyvalue'
|
||||
* @return {module:Databases.Databases-KeyValue} A KeyValue function.
|
||||
* @memberof module:Databases
|
||||
*/
|
||||
const KeyValue = () => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encryption }) => {
|
||||
const database = await Database({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate, encryption })
|
||||
const KeyValue = () => async ({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate }) => {
|
||||
const database = await Database({ ipfs, identity, address, name, access, directory, meta, headsStorage, entryStorage, indexStorage, referencesCount, syncAutomatically, onUpdate })
|
||||
|
||||
const { addOperation, log } = database
|
||||
|
||||
|
||||
@ -52,7 +52,7 @@ const PublicKeyIdentityProvider = ({ keystore }) => async () => {
|
||||
}
|
||||
|
||||
const key = await keystore.getKey(id) || await keystore.createKey(id)
|
||||
return uint8ArrayToString(key.publicKey.raw, 'base16')
|
||||
return uint8ArrayToString(key.public.marshal(), 'base16')
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* const storage = await MemoryStorage()
|
||||
* const keystore = await KeyStore({ storage })
|
||||
*/
|
||||
import { privateKeyFromRaw, publicKeyFromRaw, generateKeyPair } from '@libp2p/crypto/keys'
|
||||
import * as crypto from '@libp2p/crypto'
|
||||
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
|
||||
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
|
||||
import { compare as uint8ArrayCompare } from 'uint8arrays/compare'
|
||||
@ -16,6 +16,9 @@ import ComposedStorage from './storage/composed.js'
|
||||
import LevelStorage from './storage/level.js'
|
||||
import LRUStorage from './storage/lru.js'
|
||||
|
||||
const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
|
||||
const unmarshalPubKey = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PublicKey
|
||||
|
||||
const verifySignature = async (signature, publicKey, data) => {
|
||||
if (!signature) {
|
||||
throw new Error('No signature given')
|
||||
@ -35,7 +38,7 @@ const verifySignature = async (signature, publicKey, data) => {
|
||||
|
||||
let res = false
|
||||
try {
|
||||
const pubKey = publicKeyFromRaw(uint8ArrayFromString(publicKey, 'base16'))
|
||||
const pubKey = unmarshalPubKey(uint8ArrayFromString(publicKey, 'base16'))
|
||||
res = await isValid(pubKey, data, uint8ArrayFromString(signature, 'base16'))
|
||||
} catch (e) {
|
||||
// Catch error: sig length wrong
|
||||
@ -120,13 +123,8 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
* @namespace module:KeyStore~KeyStore
|
||||
* @description The instance returned by {@link module:KeyStore}.
|
||||
*/
|
||||
|
||||
// Persistent storage for keys
|
||||
storage = storage || await ComposedStorage(await LRUStorage({ size: 1000 }), await LevelStorage({ path: path || defaultPath }))
|
||||
|
||||
// Cache for deserialized/unmarshaled keys
|
||||
const keyCache = await LRUStorage({ size: 1000 })
|
||||
|
||||
/**
|
||||
* Closes the KeyStore's underlying storage.
|
||||
* @memberof module:KeyStore~KeyStore
|
||||
@ -135,7 +133,6 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
*/
|
||||
const close = async () => {
|
||||
await storage.close()
|
||||
await keyCache.close()
|
||||
}
|
||||
|
||||
/**
|
||||
@ -146,7 +143,6 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
*/
|
||||
const clear = async () => {
|
||||
await storage.clear()
|
||||
await keyCache.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
@ -164,17 +160,12 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
}
|
||||
|
||||
let hasKey = false
|
||||
let key = await keyCache.get(id)
|
||||
if (key) {
|
||||
hasKey = true
|
||||
} else {
|
||||
try {
|
||||
key = await storage.get('private_' + id)
|
||||
hasKey = key !== undefined && key !== null
|
||||
} catch (e) {
|
||||
// Catches 'Error: ENOENT: no such file or directory, open <path>'
|
||||
console.error('Error: ENOENT: no such file or directory')
|
||||
}
|
||||
try {
|
||||
const storedKey = await storage.get('private_' + id)
|
||||
hasKey = storedKey !== undefined && storedKey !== null
|
||||
} catch (e) {
|
||||
// Catches 'Error: ENOENT: no such file or directory, open <path>'
|
||||
console.error('Error: ENOENT: no such file or directory')
|
||||
}
|
||||
|
||||
return hasKey
|
||||
@ -189,11 +180,7 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
* @instance
|
||||
*/
|
||||
const addKey = async (id, key) => {
|
||||
const { privateKey } = key
|
||||
await storage.put('private_' + id, privateKey)
|
||||
// Unmarshal the key and add it to the cache
|
||||
const unmarshaledPrivateKey = privateKeyFromRaw(privateKey)
|
||||
await keyCache.put(id, unmarshaledPrivateKey)
|
||||
await storage.put('private_' + id, key.privateKey)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -210,16 +197,18 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
}
|
||||
|
||||
// Generate a private key
|
||||
const keyPair = await generateKeyPair('secp256k1')
|
||||
const pair = await crypto.keys.generateKeyPair('secp256k1')
|
||||
const keys = await crypto.keys.unmarshalPrivateKey(pair.bytes)
|
||||
const pubKey = keys.public.marshal()
|
||||
|
||||
const key = {
|
||||
publicKey: keyPair.publicKey.raw,
|
||||
privateKey: keyPair.raw
|
||||
publicKey: pubKey,
|
||||
privateKey: keys.marshal()
|
||||
}
|
||||
|
||||
await addKey(id, key)
|
||||
|
||||
return keyPair
|
||||
return keys
|
||||
}
|
||||
|
||||
/**
|
||||
@ -236,26 +225,18 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
throw new Error('id needed to get a key')
|
||||
}
|
||||
|
||||
let key = await keyCache.get(id)
|
||||
|
||||
if (!key) {
|
||||
let storedKey
|
||||
try {
|
||||
storedKey = await storage.get('private_' + id)
|
||||
} catch (e) {
|
||||
// ignore ENOENT error
|
||||
}
|
||||
|
||||
if (!storedKey) {
|
||||
return
|
||||
}
|
||||
|
||||
key = privateKeyFromRaw(storedKey)
|
||||
|
||||
await keyCache.put(id, key)
|
||||
let storedKey
|
||||
try {
|
||||
storedKey = await storage.get('private_' + id)
|
||||
} catch (e) {
|
||||
// ignore ENOENT error
|
||||
}
|
||||
|
||||
return key
|
||||
if (!storedKey) {
|
||||
return
|
||||
}
|
||||
|
||||
return unmarshal(storedKey)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -278,7 +259,7 @@ const KeyStore = async ({ storage, path } = {}) => {
|
||||
throw new Error('Supported formats are `hex` and `buffer`')
|
||||
}
|
||||
|
||||
const pubKey = keys.publicKey.raw
|
||||
const pubKey = keys.public.marshal()
|
||||
|
||||
return format === 'buffer' ? pubKey : uint8ArrayToString(pubKey, 'base16')
|
||||
}
|
||||
|
||||
@ -16,17 +16,13 @@ const ManifestStore = async ({ ipfs, storage } = {}) => {
|
||||
*/
|
||||
|
||||
storage = storage || await ComposedStorage(
|
||||
await LRUStorage({ size: 100000 }),
|
||||
await LRUStorage({ size: 1000 }),
|
||||
await IPFSBlockStorage({ ipfs, pin: true })
|
||||
)
|
||||
|
||||
const get = async (address) => {
|
||||
const bytes = await storage.get(address)
|
||||
const { value } = await Block.decode({ bytes, codec, hasher })
|
||||
if (value) {
|
||||
// Write to storage to make sure it gets pinned on IPFS
|
||||
await storage.put(address, bytes)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
|
||||
@ -55,7 +55,7 @@ const hashStringEncoding = base58btc
|
||||
* // { payload: "hello", next: [], ... }
|
||||
* @private
|
||||
*/
|
||||
const create = async (identity, id, payload, encryptPayloadFn, clock = null, next = [], refs = []) => {
|
||||
const create = async (identity, id, payload, clock = null, next = [], refs = []) => {
|
||||
if (identity == null) throw new Error('Identity is required, cannot create entry')
|
||||
if (id == null) throw new Error('Entry requires an id')
|
||||
if (payload == null) throw new Error('Entry requires a payload')
|
||||
@ -63,16 +63,9 @@ const create = async (identity, id, payload, encryptPayloadFn, clock = null, nex
|
||||
|
||||
clock = clock || Clock(identity.publicKey)
|
||||
|
||||
let encryptedPayload
|
||||
|
||||
if (encryptPayloadFn) {
|
||||
const { bytes: encodedPayloadBytes } = await Block.encode({ value: payload, codec, hasher })
|
||||
encryptedPayload = await encryptPayloadFn(encodedPayloadBytes)
|
||||
}
|
||||
|
||||
const entry = {
|
||||
id, // For determining a unique chain
|
||||
payload: encryptedPayload || payload, // Can be any dag-cbor encodeable data
|
||||
payload, // Can be any dag-cbor encodeable data
|
||||
next, // Array of strings of CIDs
|
||||
refs, // Array of strings of CIDs
|
||||
clock, // Clock
|
||||
@ -85,13 +78,8 @@ const create = async (identity, id, payload, encryptPayloadFn, clock = null, nex
|
||||
entry.key = identity.publicKey
|
||||
entry.identity = identity.hash
|
||||
entry.sig = signature
|
||||
entry.payload = payload
|
||||
|
||||
if (encryptPayloadFn) {
|
||||
entry._payload = encryptedPayload
|
||||
}
|
||||
|
||||
return entry
|
||||
return encode(entry)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -109,15 +97,13 @@ const verify = async (identities, entry) => {
|
||||
if (!entry.key) throw new Error("Entry doesn't have a key")
|
||||
if (!entry.sig) throw new Error("Entry doesn't have a signature")
|
||||
|
||||
const e = Object.assign({}, entry)
|
||||
|
||||
const value = {
|
||||
id: e.id,
|
||||
payload: e._payload || e.payload,
|
||||
next: e.next,
|
||||
refs: e.refs,
|
||||
clock: e.clock,
|
||||
v: e.v
|
||||
id: entry.id,
|
||||
payload: entry.payload,
|
||||
next: entry.next,
|
||||
refs: entry.refs,
|
||||
clock: entry.clock,
|
||||
v: entry.v
|
||||
}
|
||||
|
||||
const { bytes } = await Block.encode({ value, codec, hasher })
|
||||
@ -150,7 +136,7 @@ const isEntry = (obj) => {
|
||||
* @private
|
||||
*/
|
||||
const isEqual = (a, b) => {
|
||||
return a && b && a.hash && a.hash === b.hash
|
||||
return a && b && a.hash === b.hash
|
||||
}
|
||||
|
||||
/**
|
||||
@ -160,40 +146,9 @@ const isEqual = (a, b) => {
|
||||
* @memberof module:Log~Entry
|
||||
* @private
|
||||
*/
|
||||
const decode = async (bytes, decryptEntryFn, decryptPayloadFn) => {
|
||||
let cid
|
||||
|
||||
if (decryptEntryFn) {
|
||||
try {
|
||||
const encryptedEntry = await Block.decode({ bytes, codec, hasher })
|
||||
bytes = await decryptEntryFn(encryptedEntry.value)
|
||||
cid = encryptedEntry.cid
|
||||
} catch (e) {
|
||||
throw new Error('Could not decrypt entry')
|
||||
}
|
||||
}
|
||||
|
||||
const decodedEntry = await Block.decode({ bytes, codec, hasher })
|
||||
const entry = decodedEntry.value
|
||||
|
||||
if (decryptPayloadFn) {
|
||||
try {
|
||||
const decryptedPayloadBytes = await decryptPayloadFn(entry.payload)
|
||||
const { value: decryptedPayload } = await Block.decode({ bytes: decryptedPayloadBytes, codec, hasher })
|
||||
entry._payload = entry.payload
|
||||
entry.payload = decryptedPayload
|
||||
} catch (e) {
|
||||
throw new Error('Could not decrypt payload')
|
||||
}
|
||||
}
|
||||
|
||||
cid = cid || decodedEntry.cid
|
||||
const hash = cid.toString(hashStringEncoding)
|
||||
|
||||
return {
|
||||
...entry,
|
||||
hash
|
||||
}
|
||||
const decode = async (bytes) => {
|
||||
const { value } = await Block.decode({ bytes, codec, hasher })
|
||||
return encode(value)
|
||||
}
|
||||
|
||||
/**
|
||||
@ -203,28 +158,13 @@ const decode = async (bytes, decryptEntryFn, decryptPayloadFn) => {
|
||||
* @memberof module:Log~Entry
|
||||
* @private
|
||||
*/
|
||||
const encode = async (entry, encryptEntryFn, encryptPayloadFn) => {
|
||||
const e = Object.assign({}, entry)
|
||||
|
||||
if (encryptPayloadFn) {
|
||||
e.payload = e._payload
|
||||
}
|
||||
|
||||
delete e._payload
|
||||
delete e.hash
|
||||
|
||||
let { cid, bytes } = await Block.encode({ value: e, codec, hasher })
|
||||
|
||||
if (encryptEntryFn) {
|
||||
bytes = await encryptEntryFn(bytes)
|
||||
const encryptedEntry = await Block.encode({ value: bytes, codec, hasher })
|
||||
cid = encryptedEntry.cid
|
||||
bytes = encryptedEntry.bytes
|
||||
}
|
||||
|
||||
const encode = async (entry) => {
|
||||
const { cid, bytes } = await Block.encode({ value: entry, codec, hasher })
|
||||
const hash = cid.toString(hashStringEncoding)
|
||||
|
||||
const clock = Clock(entry.clock.id, entry.clock.time)
|
||||
return {
|
||||
...entry,
|
||||
clock,
|
||||
hash,
|
||||
bytes
|
||||
}
|
||||
|
||||
@ -9,17 +9,19 @@ import MemoryStorage from '../storage/memory.js'
|
||||
|
||||
const DefaultStorage = MemoryStorage
|
||||
|
||||
const Heads = async ({ storage, heads, decryptPayloadFn, decryptEntryFn }) => {
|
||||
const Heads = async ({ storage, heads }) => {
|
||||
storage = storage || await DefaultStorage()
|
||||
|
||||
const encoder = new TextEncoder()
|
||||
const decoder = new TextDecoder()
|
||||
|
||||
const put = async (heads) => {
|
||||
heads = findHeads(heads)
|
||||
const newHeads = heads.map(e => ({ hash: e.hash, next: e.next }))
|
||||
const bytes = encoder.encode(JSON.stringify(newHeads))
|
||||
await storage.put('heads', bytes)
|
||||
for (const head of heads) {
|
||||
await storage.put(head.hash, head.bytes)
|
||||
}
|
||||
}
|
||||
|
||||
const set = async (heads) => {
|
||||
await storage.clear()
|
||||
await put(heads)
|
||||
}
|
||||
|
||||
const add = async (head) => {
|
||||
@ -28,21 +30,22 @@ const Heads = async ({ storage, heads, decryptPayloadFn, decryptEntryFn }) => {
|
||||
return
|
||||
}
|
||||
const newHeads = findHeads([...currentHeads, head])
|
||||
await put(newHeads)
|
||||
await set(newHeads)
|
||||
|
||||
return newHeads
|
||||
}
|
||||
|
||||
const remove = async (hash) => {
|
||||
const currentHeads = await all()
|
||||
const newHeads = currentHeads.filter(e => e.hash !== hash)
|
||||
await put(newHeads)
|
||||
await set(newHeads)
|
||||
}
|
||||
|
||||
const iterator = async function * () {
|
||||
const bytes = await storage.get('heads')
|
||||
const headHashes = bytes ? JSON.parse(decoder.decode(bytes)) : []
|
||||
for (const hash of headHashes) {
|
||||
yield hash
|
||||
const it = storage.iterator()
|
||||
for await (const [, bytes] of it) {
|
||||
const head = await Entry.decode(bytes)
|
||||
yield head
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,13 +66,11 @@ const Heads = async ({ storage, heads, decryptPayloadFn, decryptEntryFn }) => {
|
||||
}
|
||||
|
||||
// Initialize the heads if given as parameter
|
||||
if (heads) {
|
||||
await put(heads)
|
||||
}
|
||||
await put(heads || [])
|
||||
|
||||
return {
|
||||
put,
|
||||
set: put,
|
||||
set,
|
||||
add,
|
||||
remove,
|
||||
iterator,
|
||||
|
||||
103
src/oplog/log.js
103
src/oplog/log.js
@ -10,14 +10,18 @@ import LRU from 'lru'
|
||||
import PQueue from 'p-queue'
|
||||
import Entry from './entry.js'
|
||||
import Clock, { tickClock } from './clock.js'
|
||||
import Heads from './heads.js'
|
||||
import ConflictResolution from './conflict-resolution.js'
|
||||
import OplogStore from './oplog-store.js'
|
||||
import MemoryStorage from '../storage/memory.js'
|
||||
|
||||
const { LastWriteWins, NoZeroes } = ConflictResolution
|
||||
|
||||
const randomId = () => new Date().getTime().toString()
|
||||
const maxClockTimeReducer = (res, acc) => Math.max(res, acc.clock.time)
|
||||
|
||||
// Default storage for storing the Log and its entries. Default: Memory. Options: Memory, LRU, IPFS.
|
||||
const DefaultStorage = MemoryStorage
|
||||
|
||||
// Default AccessController for the Log.
|
||||
// Default policy is that anyone can write to the Log.
|
||||
// Signature of an entry will always be verified regardless of AccessController policy.
|
||||
@ -52,7 +56,7 @@ const DefaultAccessController = async () => {
|
||||
* @memberof module:Log
|
||||
* @instance
|
||||
*/
|
||||
const Log = async (identity, { logId, logHeads, access, entryStorage, headsStorage, indexStorage, sortFn, encryption } = {}) => {
|
||||
const Log = async (identity, { logId, logHeads, access, entryStorage, headsStorage, indexStorage, sortFn } = {}) => {
|
||||
/**
|
||||
* @namespace Log
|
||||
* @description The instance returned by {@link module:Log}
|
||||
@ -64,23 +68,20 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
if (logHeads != null && !Array.isArray(logHeads)) {
|
||||
throw new Error('\'logHeads\' argument must be an array')
|
||||
}
|
||||
|
||||
// Set Log's id
|
||||
const id = logId || randomId()
|
||||
|
||||
// Encryption of entries and payloads
|
||||
encryption = encryption || {}
|
||||
const encryptPayloadFn = encryption.data?.encrypt
|
||||
|
||||
// Access Controller
|
||||
access = access || await DefaultAccessController()
|
||||
|
||||
// Index and storage of entries for this Log
|
||||
const oplogStore = await OplogStore({ logHeads, entryStorage, indexStorage, headsStorage, encryption })
|
||||
|
||||
// Oplog entry storage
|
||||
const _entries = entryStorage || await DefaultStorage()
|
||||
// Entry index for keeping track which entries are already in the log
|
||||
const _index = indexStorage || await DefaultStorage()
|
||||
// Heads storage
|
||||
headsStorage = headsStorage || await DefaultStorage()
|
||||
// Add heads to the state storage, ie. init the log state
|
||||
const _heads = await Heads({ storage: headsStorage, heads: logHeads })
|
||||
// Conflict-resolution sorting function
|
||||
sortFn = NoZeroes(sortFn || LastWriteWins)
|
||||
|
||||
// Internal queues for processing appends and joins in their call-order
|
||||
const appendQueue = new PQueue({ concurrency: 1 })
|
||||
const joinQueue = new PQueue({ concurrency: 1 })
|
||||
@ -105,8 +106,8 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
* @instance
|
||||
*/
|
||||
const heads = async () => {
|
||||
const heads_ = await oplogStore.heads()
|
||||
return heads_.sort(sortFn).reverse()
|
||||
const res = await _heads.all()
|
||||
return res.sort(sortFn).reverse()
|
||||
}
|
||||
|
||||
/**
|
||||
@ -133,14 +134,16 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
* @instance
|
||||
*/
|
||||
const get = async (hash) => {
|
||||
if (!hash) {
|
||||
throw new Error('hash is required')
|
||||
const bytes = await _entries.get(hash)
|
||||
if (bytes) {
|
||||
const entry = await Entry.decode(bytes)
|
||||
return entry
|
||||
}
|
||||
return oplogStore.get(hash)
|
||||
}
|
||||
|
||||
const has = async (hash) => {
|
||||
return oplogStore.has(hash)
|
||||
const entry = await _index.get(hash)
|
||||
return entry != null
|
||||
}
|
||||
|
||||
/**
|
||||
@ -159,7 +162,6 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
// 2. Authorize entry
|
||||
// 3. Store entry
|
||||
// 4. return Entry
|
||||
|
||||
// Get current heads of the log
|
||||
const heads_ = await heads()
|
||||
// Create the next pointers from heads
|
||||
@ -167,29 +169,29 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
// Get references (pointers) to multiple entries in the past
|
||||
// (skips the heads which are covered by the next field)
|
||||
const refs = await getReferences(heads_, options.referencesCount + heads_.length)
|
||||
|
||||
// Create the entry
|
||||
const entry = await Entry.create(
|
||||
identity,
|
||||
id,
|
||||
data,
|
||||
encryptPayloadFn,
|
||||
tickClock(await clock()),
|
||||
nexts,
|
||||
refs
|
||||
)
|
||||
|
||||
// Authorize the entry
|
||||
const canAppend = await access.canAppend(entry)
|
||||
if (!canAppend) {
|
||||
throw new Error(`Could not append entry:\nKey "${identity.hash}" is not allowed to write to the log`)
|
||||
}
|
||||
|
||||
// Add the entry to the oplog store (=store and index it)
|
||||
const hash = await oplogStore.setHead(entry)
|
||||
|
||||
// The appended entry is now the latest head
|
||||
await _heads.set([entry])
|
||||
// Add entry to the entry storage
|
||||
await _entries.put(entry.hash, entry.bytes)
|
||||
// Add entry to the entry index
|
||||
await _index.put(entry.hash, true)
|
||||
// Return the appended entry
|
||||
return { ...entry, hash }
|
||||
return entry
|
||||
}
|
||||
|
||||
return appendQueue.add(task)
|
||||
@ -216,7 +218,9 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
if (!isLog(log)) {
|
||||
throw new Error('Given argument is not an instance of Log')
|
||||
}
|
||||
await oplogStore.storage.merge(log.storage)
|
||||
if (_entries.merge) {
|
||||
await _entries.merge(log.storage)
|
||||
}
|
||||
const heads = await log.heads()
|
||||
for (const entry of heads) {
|
||||
await joinEntry(entry)
|
||||
@ -298,12 +302,18 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
|
||||
await traverseAndVerify()
|
||||
|
||||
/* 4. Add missing entries to the oplog store (=to the log) */
|
||||
await oplogStore.addVerified(hashesToAdd.values())
|
||||
/* 6. Remove heads which new entries are connect to */
|
||||
await oplogStore.removeHeads(connectedHeads.values())
|
||||
/* 7. Add the new entry to heads (=union with current heads) */
|
||||
await oplogStore.addHead(entry)
|
||||
/* 4. Add missing entries to the index (=to the log) */
|
||||
for (const hash of hashesToAdd.values()) {
|
||||
await _index.put(hash, true)
|
||||
}
|
||||
|
||||
/* 5. Remove heads which new entries are connect to */
|
||||
for (const hash of connectedHeads.values()) {
|
||||
await _heads.remove(hash)
|
||||
}
|
||||
|
||||
/* 6. Add the new entry to heads (=union with current heads) */
|
||||
await _heads.add(entry)
|
||||
|
||||
return true
|
||||
}
|
||||
@ -316,7 +326,7 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
* @memberof module:Log~Log
|
||||
* @instance
|
||||
*/
|
||||
const traverse = async function * (rootEntries, shouldStopFn) {
|
||||
const traverse = async function * (rootEntries, shouldStopFn, useRefs = true) {
|
||||
// By default, we don't stop traversal and traverse
|
||||
// until the end of the log
|
||||
const defaultStopFn = () => false
|
||||
@ -340,7 +350,7 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
// Get the next entry from the stack
|
||||
entry = stack.pop()
|
||||
if (entry) {
|
||||
const { hash, next } = entry
|
||||
const { hash, next, refs } = entry
|
||||
// If we have an entry that we haven't traversed yet, process it
|
||||
if (!traversed[hash]) {
|
||||
// Yield the current entry
|
||||
@ -355,7 +365,7 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
fetched[hash] = true
|
||||
// Add the next and refs hashes to the list of hashes to fetch next,
|
||||
// filter out traversed and fetched hashes
|
||||
toFetch = [...toFetch, ...next].filter(notIndexed)
|
||||
toFetch = [...toFetch, ...next, ...(useRefs ? refs : [])].filter(notIndexed)
|
||||
// Function to fetch an entry and making sure it's not a duplicate (check the hash indices)
|
||||
const fetchEntries = (hash) => {
|
||||
if (!traversed[hash] && !fetched[hash]) {
|
||||
@ -369,7 +379,7 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
// Add the next and refs fields from the fetched entries to the next round
|
||||
toFetch = nexts
|
||||
.filter(e => e !== null && e !== undefined)
|
||||
.reduce((res, acc) => Array.from(new Set([...res, ...acc.next])), [])
|
||||
.reduce((res, acc) => Array.from(new Set([...res, ...acc.next, ...(useRefs ? acc.refs : [])])), [])
|
||||
.filter(notIndexed)
|
||||
// Add the fetched entries to the stack to be processed
|
||||
stack = [...nexts, ...stack]
|
||||
@ -488,9 +498,9 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
* @instance
|
||||
*/
|
||||
const clear = async () => {
|
||||
await appendQueue.clear()
|
||||
await joinQueue.clear()
|
||||
await oplogStore.clear()
|
||||
await _index.clear()
|
||||
await _heads.clear()
|
||||
await _entries.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
@ -499,9 +509,9 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
* @instance
|
||||
*/
|
||||
const close = async () => {
|
||||
await appendQueue.onIdle()
|
||||
await joinQueue.onIdle()
|
||||
await oplogStore.close()
|
||||
await _index.close()
|
||||
await _heads.close()
|
||||
await _entries.close()
|
||||
}
|
||||
|
||||
/**
|
||||
@ -533,7 +543,7 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
const shouldStopTraversal = async (entry) => {
|
||||
return refs.length >= amount && amount !== -1
|
||||
}
|
||||
for await (const { hash } of traverse(heads, shouldStopTraversal)) {
|
||||
for await (const { hash } of traverse(heads, shouldStopTraversal, false)) {
|
||||
refs.push(hash)
|
||||
}
|
||||
refs = refs.slice(heads.length + 1, amount)
|
||||
@ -557,8 +567,7 @@ const Log = async (identity, { logId, logHeads, access, entryStorage, headsStora
|
||||
close,
|
||||
access,
|
||||
identity,
|
||||
storage: oplogStore.storage,
|
||||
encryption
|
||||
storage: _entries
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,112 +0,0 @@
|
||||
import Entry from './entry.js'
|
||||
import Heads from './heads.js'
|
||||
import MemoryStorage from '../storage/memory.js'
|
||||
|
||||
// Default storage for storing the Log and its entries. Default: Memory. Options: Memory, LRU, IPFS.
|
||||
const DefaultStorage = MemoryStorage
|
||||
|
||||
const OplogStore = async ({ logHeads, entryStorage, headsStorage, indexStorage, encryption }) => {
|
||||
// Setup encryption and decryption functions
|
||||
const encryptEntryFn = encryption?.replication?.encrypt
|
||||
const decryptEntryFn = encryption?.replication?.decrypt
|
||||
const encryptPayloadFn = encryption?.data?.encrypt
|
||||
const decryptPayloadFn = encryption?.data?.decrypt
|
||||
// Oplog entry storage
|
||||
const _entries = entryStorage || await DefaultStorage()
|
||||
// Entry index for keeping track which entries are already in the log
|
||||
const _index = indexStorage || await DefaultStorage()
|
||||
// Heads storage
|
||||
headsStorage = headsStorage || await DefaultStorage()
|
||||
// Add heads to the state storage, ie. init the log state
|
||||
const _heads = await Heads({ storage: headsStorage, heads: logHeads, decryptPayloadFn, decryptEntryFn })
|
||||
|
||||
const get = async (hash) => {
|
||||
const bytes = await _entries.get(hash)
|
||||
if (bytes) {
|
||||
const entry = await Entry.decode(bytes, decryptEntryFn, decryptPayloadFn)
|
||||
return entry
|
||||
}
|
||||
}
|
||||
|
||||
const getBytes = async (hash) => {
|
||||
return _entries.get(hash)
|
||||
}
|
||||
|
||||
const has = async (hash) => {
|
||||
const entry = await _index.get(hash)
|
||||
return entry != null
|
||||
}
|
||||
|
||||
const heads = async () => {
|
||||
const heads_ = []
|
||||
for (const { hash } of await _heads.all()) {
|
||||
const head = await get(hash)
|
||||
heads_.push(head)
|
||||
}
|
||||
return heads_
|
||||
}
|
||||
|
||||
const setHead = async (entry) => {
|
||||
const { hash, bytes } = await Entry.encode(entry, encryptEntryFn, encryptPayloadFn)
|
||||
// Add entry to the entry storage
|
||||
await _entries.put(hash, bytes)
|
||||
// Add entry to the entry index
|
||||
await _index.put(hash, true)
|
||||
// The appended entry is now the latest head
|
||||
await _heads.set([{ hash, next: entry.next }])
|
||||
|
||||
return hash
|
||||
}
|
||||
|
||||
const addHead = async (entry) => {
|
||||
/* 7. Add the new entry to heads (=union with current heads) */
|
||||
await _heads.add(entry)
|
||||
return entry.hash
|
||||
}
|
||||
|
||||
const removeHeads = async (hashes) => {
|
||||
/* 5. Remove heads which new entries are connect to */
|
||||
for (const hash of hashes) {
|
||||
await _heads.remove(hash)
|
||||
}
|
||||
}
|
||||
|
||||
const addVerified = async (hashes) => {
|
||||
/* 4. Add missing entries to the index (=to the log) */
|
||||
for (const hash of hashes) {
|
||||
await _index.put(hash, true)
|
||||
/* 5. Add new entry to entries (for pinning) */
|
||||
if (_entries.persist) {
|
||||
await _entries.persist(hash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const clear = async () => {
|
||||
await _index.clear()
|
||||
await _heads.clear()
|
||||
await _entries.clear()
|
||||
}
|
||||
|
||||
const close = async () => {
|
||||
await _index.close()
|
||||
await _heads.close()
|
||||
await _entries.close()
|
||||
}
|
||||
|
||||
return {
|
||||
get,
|
||||
getBytes,
|
||||
has,
|
||||
heads,
|
||||
setHead,
|
||||
addHead,
|
||||
removeHeads,
|
||||
addVerified,
|
||||
storage: _entries,
|
||||
clear,
|
||||
close
|
||||
}
|
||||
}
|
||||
|
||||
export default OplogStore
|
||||
@ -104,7 +104,6 @@ const OrbitDB = async ({ ipfs, id, identity, identities, directory } = {}) => {
|
||||
* @param {module:Storage} [params.indexStorage=[ComposedStorage]{@link module:Storage.Storage-Composed}] A compatible storage instance for storing an " index of log entries. Defaults to ComposedStorage(LRUStorage, LevelStorage).
|
||||
* @param {number} [params.referencesCount] The number of references to
|
||||
* use for [Log]{@link module:Log} entries.
|
||||
* @param {number} [params.encryption] Encryption module to encrypt/decrypt database payloads and entries. If provided, the encryption object must take the form { replication: { encrypt, decrypt }, data: { encrypt, decrypt } }.
|
||||
* @memberof module:OrbitDB
|
||||
* @return {module:Database} A database instance.
|
||||
* @throws "Unsupported database type" if the type specified is not in the list
|
||||
@ -113,7 +112,7 @@ const OrbitDB = async ({ ipfs, id, identity, identities, directory } = {}) => {
|
||||
* @instance
|
||||
* @async
|
||||
*/
|
||||
const open = async (address, { type, meta, sync, Database, AccessController, headsStorage, entryStorage, indexStorage, referencesCount, encryption } = {}) => {
|
||||
const open = async (address, { type, meta, sync, Database, AccessController, headsStorage, entryStorage, indexStorage, referencesCount } = {}) => {
|
||||
let name, manifest, accessController
|
||||
|
||||
if (databases[address]) {
|
||||
@ -154,7 +153,7 @@ const OrbitDB = async ({ ipfs, id, identity, identities, directory } = {}) => {
|
||||
|
||||
address = address.toString()
|
||||
|
||||
const db = await Database({ ipfs, identity, address, name, access: accessController, directory, meta, syncAutomatically: sync, headsStorage, entryStorage, indexStorage, referencesCount, encryption })
|
||||
const db = await Database({ ipfs, identity, address, name, access: accessController, directory, meta, syncAutomatically: sync, headsStorage, entryStorage, indexStorage, referencesCount })
|
||||
|
||||
db.events.on('close', onDatabaseClosed(address))
|
||||
|
||||
@ -194,7 +193,6 @@ const OrbitDB = async ({ ipfs, id, identity, identities, directory } = {}) => {
|
||||
ipfs,
|
||||
directory,
|
||||
keystore,
|
||||
identities,
|
||||
identity,
|
||||
peerId
|
||||
}
|
||||
|
||||
@ -114,16 +114,6 @@ const ComposedStorage = async (storage1, storage2) => {
|
||||
await storage2.clear()
|
||||
}
|
||||
|
||||
const persist = async (hash) => {
|
||||
if (storage1.persist) {
|
||||
await storage1.persist(hash)
|
||||
}
|
||||
|
||||
if (storage2.persist) {
|
||||
await storage2.persist(hash)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls close on each of the composed storages.
|
||||
* @function
|
||||
@ -139,7 +129,6 @@ const ComposedStorage = async (storage1, storage2) => {
|
||||
put,
|
||||
get,
|
||||
del,
|
||||
persist,
|
||||
iterator,
|
||||
merge,
|
||||
clear,
|
||||
|
||||
@ -28,8 +28,6 @@ const DefaultTimeout = 30000 // 30 seconds
|
||||
const IPFSBlockStorage = async ({ ipfs, pin, timeout } = {}) => {
|
||||
if (!ipfs) throw new Error('An instance of ipfs is required.')
|
||||
|
||||
const timeoutControllers = new Set()
|
||||
|
||||
/**
|
||||
* Puts data to an IPFS block.
|
||||
* @function
|
||||
@ -43,7 +41,9 @@ const IPFSBlockStorage = async ({ ipfs, pin, timeout } = {}) => {
|
||||
const { signal } = new TimeoutController(timeout || DefaultTimeout)
|
||||
await ipfs.blockstore.put(cid, data, { signal })
|
||||
|
||||
await persist(hash)
|
||||
if (pin && !(await ipfs.pins.isPinned(cid))) {
|
||||
await drain(ipfs.pins.add(cid))
|
||||
}
|
||||
}
|
||||
|
||||
const del = async (hash) => {}
|
||||
@ -58,40 +58,25 @@ const IPFSBlockStorage = async ({ ipfs, pin, timeout } = {}) => {
|
||||
*/
|
||||
const get = async (hash) => {
|
||||
const cid = CID.parse(hash, base58btc)
|
||||
const controller = new TimeoutController(timeout || DefaultTimeout)
|
||||
timeoutControllers.add(controller)
|
||||
const block = await ipfs.blockstore.get(cid, { signal: controller.signal })
|
||||
timeoutControllers.delete(controller)
|
||||
const { signal } = new TimeoutController(timeout || DefaultTimeout)
|
||||
const block = await ipfs.blockstore.get(cid, { signal })
|
||||
if (block) {
|
||||
return block
|
||||
}
|
||||
}
|
||||
|
||||
const persist = async (hash) => {
|
||||
const cid = CID.parse(hash, base58btc)
|
||||
if (pin && !(await ipfs.pins.isPinned(cid))) {
|
||||
await drain(ipfs.pins.add(cid))
|
||||
}
|
||||
}
|
||||
|
||||
const iterator = async function * () {}
|
||||
|
||||
const merge = async (other) => {}
|
||||
|
||||
const clear = async () => {}
|
||||
|
||||
const close = async () => {
|
||||
for (const controller in timeoutControllers) {
|
||||
controller.abort()
|
||||
}
|
||||
timeoutControllers.clear()
|
||||
}
|
||||
const close = async () => {}
|
||||
|
||||
return {
|
||||
put,
|
||||
del,
|
||||
get,
|
||||
persist,
|
||||
iterator,
|
||||
merge,
|
||||
clear,
|
||||
|
||||
37
src/sync.js
37
src/sync.js
@ -3,7 +3,6 @@ import PQueue from 'p-queue'
|
||||
import { EventEmitter } from 'events'
|
||||
import { TimeoutController } from 'timeout-abort-controller'
|
||||
import pathJoin from './utils/path-join.js'
|
||||
import { Entry } from './oplog/index.js'
|
||||
|
||||
const DefaultTimeout = 30000 // 30 seconds
|
||||
|
||||
@ -20,9 +19,7 @@ const DefaultTimeout = 30000 // 30 seconds
|
||||
* Upon subscribing to the topic, peers already connected to the topic receive
|
||||
* the subscription message and "dial" the subscribing peer using a libp2p
|
||||
* custom protocol. Once connected to the subscribing peer on a direct
|
||||
* peer-to-peer connection, the dialing peer and the subscribing peer exchange
|
||||
* the heads of the Log each peer currently has. Once completed, the peers have
|
||||
* the same "local state".
|
||||
* peer-to-peer connection, the dialing peer and the subscribing peer exchange * the heads of the Log each peer currently has. Once completed, the peers have * the same "local state".
|
||||
*
|
||||
* Once the initial sync has completed, peers notify one another of updates to
|
||||
* the log, ie. updates to the database, using the initially opened pubsub
|
||||
@ -135,7 +132,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
*/
|
||||
events = events || new EventEmitter()
|
||||
|
||||
timeout ??= DefaultTimeout
|
||||
timeout = timeout || DefaultTimeout
|
||||
|
||||
let started = false
|
||||
|
||||
@ -147,8 +144,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
const sendHeads = (source) => {
|
||||
return (async function * () {
|
||||
const heads = await log.heads()
|
||||
for await (const { hash } of heads) {
|
||||
const bytes = await log.storage.get(hash)
|
||||
for await (const { bytes } of heads) {
|
||||
yield bytes
|
||||
}
|
||||
})()
|
||||
@ -158,8 +154,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
for await (const value of source) {
|
||||
const headBytes = value.subarray()
|
||||
if (headBytes && onSynced) {
|
||||
const entry = await Entry.decode(headBytes, log.encryption.replication?.decrypt, log.encryption.data?.decrypt)
|
||||
await onSynced(entry)
|
||||
await onSynced(headBytes)
|
||||
}
|
||||
}
|
||||
if (started) {
|
||||
@ -197,8 +192,9 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
const stream = await libp2p.dialProtocol(remotePeer, headsSyncAddress, { signal })
|
||||
await pipe(sendHeads, stream, receiveHeads(peerId))
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
peers.delete(peerId)
|
||||
if (e.name === 'UnsupportedProtocolError') {
|
||||
if (e.code === 'ERR_UNSUPPORTED_PROTOCOL') {
|
||||
// Skip peer, they don't have this database currently
|
||||
} else {
|
||||
events.emit('error', e)
|
||||
@ -222,8 +218,7 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
const task = async () => {
|
||||
try {
|
||||
if (data && onSynced) {
|
||||
const entry = await Entry.decode(data, log.encryption.replication?.decrypt, log.encryption.data?.decrypt)
|
||||
await onSynced(entry)
|
||||
await onSynced(data)
|
||||
}
|
||||
} catch (e) {
|
||||
events.emit('error', e)
|
||||
@ -235,10 +230,6 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
}
|
||||
}
|
||||
|
||||
const handlePeerDisconnected = async event => {
|
||||
peers.delete(event.detail.toString())
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a log entry to the Sync Protocol to be sent to peers.
|
||||
* @function add
|
||||
@ -247,9 +238,8 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
* @instance
|
||||
*/
|
||||
const add = async (entry) => {
|
||||
if (started && entry && entry.hash) {
|
||||
const bytes = await log.storage.get(entry.hash)
|
||||
await pubsub.publish(address, bytes)
|
||||
if (started) {
|
||||
await pubsub.publish(address, entry.bytes)
|
||||
}
|
||||
}
|
||||
|
||||
@ -262,12 +252,11 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
const stopSync = async () => {
|
||||
if (started) {
|
||||
started = false
|
||||
await queue.clear()
|
||||
await queue.onIdle()
|
||||
pubsub.removeEventListener('subscription-change', handlePeerSubscribed)
|
||||
pubsub.removeEventListener('message', handleUpdateMessage)
|
||||
await libp2p.unhandle(headsSyncAddress)
|
||||
await pubsub.unsubscribe(address)
|
||||
libp2p.removeEventListener('peer:disconnect', handlePeerDisconnected)
|
||||
peers.clear()
|
||||
}
|
||||
}
|
||||
@ -280,14 +269,12 @@ const Sync = async ({ ipfs, log, events, onSynced, start, timeout }) => {
|
||||
*/
|
||||
const startSync = async () => {
|
||||
if (!started) {
|
||||
// Exchange head entries with peers when connected
|
||||
await libp2p.handle(headsSyncAddress, handleReceiveHeads)
|
||||
pubsub.addEventListener('subscription-change', handlePeerSubscribed)
|
||||
pubsub.addEventListener('message', handleUpdateMessage)
|
||||
// Subscribe to the pubsub channel for this database through which updates are sent
|
||||
await pubsub.subscribe(address)
|
||||
// Exchange head entries with peers when connected
|
||||
await libp2p.handle(headsSyncAddress, handleReceiveHeads)
|
||||
// Remove disconnected peers from `peers`, as otherwise they will not resync heads on reconnection
|
||||
libp2p.addEventListener('peer:disconnect', handlePeerDisconnected)
|
||||
started = true
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
import { privateKeyFromRaw } from '@libp2p/crypto/keys'
|
||||
import * as crypto from '@libp2p/crypto'
|
||||
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
|
||||
import { Identities, KeyStore } from '../../src/index.js'
|
||||
|
||||
const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
|
||||
const unmarshalPubKey = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PublicKey
|
||||
|
||||
const keysPath = './testkeys'
|
||||
|
||||
const isBrowser = () => typeof window !== 'undefined'
|
||||
@ -49,10 +52,10 @@ before(async () => {
|
||||
]
|
||||
|
||||
for (let user of users) {
|
||||
const privateKey1 = privateKeyFromRaw(uint8ArrayFromString(user.privateKey, 'base16'))
|
||||
const privateKey2 = privateKeyFromRaw(uint8ArrayFromString(user.identity.privateKey, 'base16'))
|
||||
await keystore.addKey(user.id, { privateKey: privateKey1.raw })
|
||||
await keystore.addKey(user.identity.id, { privateKey: privateKey2.raw })
|
||||
const privateKey1 = unmarshal(uint8ArrayFromString(user.privateKey, 'base16'))
|
||||
const privateKey2 = unmarshal(uint8ArrayFromString(user.identity.privateKey, 'base16'))
|
||||
await keystore.addKey(user.id, { privateKey: privateKey1.marshal() })
|
||||
await keystore.addKey(user.identity.id, { privateKey: privateKey2.marshal() })
|
||||
}
|
||||
|
||||
await keystore.close()
|
||||
|
||||
@ -196,20 +196,27 @@ describe('Database - Replication', function () {
|
||||
db1 = await Database({ ipfs: ipfs1, identity: testIdentity1, address: databaseId, accessController, directory: './orbitdb1', entryStorage: storage1 })
|
||||
db2 = await Database({ ipfs: ipfs2, identity: testIdentity2, address: databaseId, accessController, directory: './orbitdb2', entryStorage: storage2 })
|
||||
|
||||
let connected = false
|
||||
let connected1 = false
|
||||
let connected2 = false
|
||||
|
||||
const onConnected = (peerId, heads) => {
|
||||
connected = true
|
||||
const onConnected1 = (peerId, heads) => {
|
||||
connected1 = true
|
||||
}
|
||||
|
||||
db2.events.on('join', onConnected)
|
||||
const onConnected2 = (peerId, heads) => {
|
||||
connected2 = true
|
||||
}
|
||||
|
||||
db1.events.on('join', onConnected1)
|
||||
db2.events.on('join', onConnected2)
|
||||
|
||||
await db1.addOperation({ op: 'PUT', key: 1, value: 'record 1 on db 1' })
|
||||
await db1.addOperation({ op: 'PUT', key: 2, value: 'record 2 on db 1' })
|
||||
await db1.addOperation({ op: 'PUT', key: 3, value: 'record 3 on db 1' })
|
||||
await db1.addOperation({ op: 'PUT', key: 4, value: 'record 4 on db 1' })
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
await waitFor(() => connected1, () => true)
|
||||
await waitFor(() => connected2, () => true)
|
||||
|
||||
const all1 = []
|
||||
for await (const item of db1.log.iterator()) {
|
||||
|
||||
@ -3,7 +3,7 @@ import { rimraf } from 'rimraf'
|
||||
import { existsSync } from 'fs'
|
||||
import { copy } from 'fs-extra'
|
||||
import Path from 'path'
|
||||
import { Database, KeyStore, Identities } from '../src/index.js'
|
||||
import { Database, Entry, KeyStore, Identities } from '../src/index.js'
|
||||
import LevelStorage from '../src/storage/level.js'
|
||||
import MemoryStorage from '../src/storage/memory.js'
|
||||
import testKeysPath from './fixtures/test-keys-path.js'
|
||||
@ -68,13 +68,8 @@ describe('Database', function () {
|
||||
describe('Options', () => {
|
||||
it('uses default directory for headsStorage', async () => {
|
||||
db = await Database({ ipfs, identity: testIdentity, address: databaseId, accessController })
|
||||
|
||||
const op1 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 1' }
|
||||
const op2 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 2' }
|
||||
|
||||
await db.addOperation(op1)
|
||||
const hash = await db.addOperation(op2)
|
||||
const entry = await db.log.get(hash)
|
||||
const op = { op: 'PUT', key: 1, value: 'record 1 on db 1' }
|
||||
const hash = await db.addOperation(op)
|
||||
|
||||
const headsPath = Path.join('./orbitdb/', `${databaseId}/`, '/log/_heads/')
|
||||
|
||||
@ -83,13 +78,8 @@ describe('Database', function () {
|
||||
await db.close()
|
||||
|
||||
const headsStorage = await LevelStorage({ path: headsPath })
|
||||
const bytes = await headsStorage.get('heads')
|
||||
const heads = JSON.parse(new TextDecoder().decode(bytes))
|
||||
|
||||
strictEqual(heads.length, 1)
|
||||
strictEqual(heads.at(0).hash, hash)
|
||||
strictEqual(heads.at(0).next.length, 1)
|
||||
strictEqual(heads.at(0).next.at(0), entry.next.at(0))
|
||||
deepStrictEqual((await Entry.decode(await headsStorage.get(hash))).payload, op)
|
||||
|
||||
await headsStorage.close()
|
||||
|
||||
@ -98,12 +88,8 @@ describe('Database', function () {
|
||||
|
||||
it('uses given directory for headsStorage', async () => {
|
||||
db = await Database({ ipfs, identity: testIdentity, address: databaseId, accessController, directory: './custom-directory' })
|
||||
const op1 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 1' }
|
||||
const op2 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 2' }
|
||||
|
||||
await db.addOperation(op1)
|
||||
const hash = await db.addOperation(op2)
|
||||
const entry = await db.log.get(hash)
|
||||
const op = { op: 'PUT', key: 1, value: 'record 1 on db 1' }
|
||||
const hash = await db.addOperation(op)
|
||||
|
||||
const headsPath = Path.join('./custom-directory/', `${databaseId}/`, '/log/_heads/')
|
||||
|
||||
@ -113,13 +99,7 @@ describe('Database', function () {
|
||||
|
||||
const headsStorage = await LevelStorage({ path: headsPath })
|
||||
|
||||
const bytes = await headsStorage.get('heads')
|
||||
const heads = JSON.parse(new TextDecoder().decode(bytes))
|
||||
|
||||
strictEqual(heads.length, 1)
|
||||
strictEqual(heads.at(0).hash, hash)
|
||||
strictEqual(heads.at(0).next.length, 1)
|
||||
strictEqual(heads.at(0).next.at(0), entry.next.at(0))
|
||||
deepStrictEqual((await Entry.decode(await headsStorage.get(hash))).payload, op)
|
||||
|
||||
await headsStorage.close()
|
||||
|
||||
@ -130,51 +110,23 @@ describe('Database', function () {
|
||||
it('uses given MemoryStorage for headsStorage', async () => {
|
||||
const headsStorage = await MemoryStorage()
|
||||
db = await Database({ ipfs, identity: testIdentity, address: databaseId, accessController, directory: './orbitdb', headsStorage })
|
||||
const op1 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 1' }
|
||||
const op2 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 2' }
|
||||
const op = { op: 'PUT', key: 1, value: 'record 1 on db 1' }
|
||||
const hash = await db.addOperation(op)
|
||||
|
||||
await db.addOperation(op1)
|
||||
const hash = await db.addOperation(op2)
|
||||
const entry = await db.log.get(hash)
|
||||
|
||||
const bytes = await headsStorage.get('heads')
|
||||
const heads = JSON.parse(new TextDecoder().decode(bytes))
|
||||
|
||||
strictEqual(heads.length, 1)
|
||||
strictEqual(heads.at(0).hash, hash)
|
||||
strictEqual(heads.at(0).next.length, 1)
|
||||
strictEqual(heads.at(0).next.at(0), entry.next.at(0))
|
||||
deepStrictEqual((await Entry.decode(await headsStorage.get(hash))).payload, op)
|
||||
|
||||
await db.close()
|
||||
|
||||
await headsStorage.close()
|
||||
await rimraf('./orbitdb')
|
||||
})
|
||||
|
||||
it('uses given MemoryStorage for entryStorage', async () => {
|
||||
const entryStorage = await MemoryStorage()
|
||||
const headsStorage = await MemoryStorage()
|
||||
db = await Database({ ipfs, identity: testIdentity, address: databaseId, accessController, directory: './orbitdb', headsStorage, entryStorage })
|
||||
const op1 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 1' }
|
||||
const op2 = { op: 'PUT', key: 1, value: 'record 1 on db 1 version 2' }
|
||||
db = await Database({ ipfs, identity: testIdentity, address: databaseId, accessController, directory: './orbitdb', entryStorage })
|
||||
const op = { op: 'PUT', key: 1, value: 'record 1 on db 1' }
|
||||
const hash = await db.addOperation(op)
|
||||
|
||||
await db.addOperation(op1)
|
||||
const hash = await db.addOperation(op2)
|
||||
const entry = await db.log.get(hash)
|
||||
|
||||
const bytes = await headsStorage.get('heads')
|
||||
const heads = JSON.parse(new TextDecoder().decode(bytes))
|
||||
|
||||
strictEqual(heads.length, 1)
|
||||
strictEqual(heads.at(0).hash, hash)
|
||||
strictEqual(heads.at(0).next.length, 1)
|
||||
strictEqual(heads.at(0).next.at(0), entry.next.at(0))
|
||||
deepStrictEqual((await Entry.decode(await entryStorage.get(hash))).payload, op)
|
||||
|
||||
await db.close()
|
||||
|
||||
await entryStorage.close()
|
||||
await headsStorage.close()
|
||||
await rimraf('./orbitdb')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ import createHelia from '../../utils/create-helia.js'
|
||||
const keysPath = './testkeys'
|
||||
|
||||
describe('Documents Database Replication', function () {
|
||||
this.timeout(10000)
|
||||
this.timeout(30000)
|
||||
|
||||
let ipfs1, ipfs2
|
||||
let keystore
|
||||
|
||||
@ -320,12 +320,11 @@ describe('KeyValueIndexed Database Replication', function () {
|
||||
})
|
||||
|
||||
it('indexes deletes correctly', async () => {
|
||||
const databaseId = 'kv-CCC'
|
||||
let replicated = false
|
||||
let err
|
||||
|
||||
const onError = (error) => {
|
||||
err = error
|
||||
const onError = (err) => {
|
||||
console.error(err)
|
||||
deepStrictEqual(err, undefined)
|
||||
}
|
||||
|
||||
kv1 = await KeyValueIndexed()({ ipfs: ipfs1, identity: testIdentity1, address: databaseId, accessController, directory: './orbitdb11' })
|
||||
@ -340,11 +339,11 @@ describe('KeyValueIndexed Database Replication', function () {
|
||||
|
||||
kv2 = await KeyValueIndexed()({ ipfs: ipfs2, identity: testIdentity2, address: databaseId, accessController, directory: './orbitdb22' })
|
||||
|
||||
const onConnected = (entry) => {
|
||||
const onUpdate = (entry) => {
|
||||
replicated = true
|
||||
}
|
||||
|
||||
kv2.events.on('join', onConnected)
|
||||
kv2.events.on('update', onUpdate)
|
||||
kv2.events.on('error', onError)
|
||||
|
||||
await waitFor(() => replicated, () => true)
|
||||
@ -359,8 +358,6 @@ describe('KeyValueIndexed Database Replication', function () {
|
||||
all2.push(keyValue)
|
||||
}
|
||||
|
||||
deepStrictEqual(err, undefined)
|
||||
|
||||
deepStrictEqual(all2.map(e => { return { key: e.key, value: e.value } }), [
|
||||
{ key: 'init', value: true },
|
||||
{ key: 'hello', value: 'friend' }
|
||||
|
||||
@ -38,7 +38,7 @@ describe('Identities', function () {
|
||||
identities = await Identities({ path: keysPath })
|
||||
identity = await identities.createIdentity({ id })
|
||||
const key = await identities.keystore.getKey(id)
|
||||
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
|
||||
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
|
||||
assert.strictEqual(identity.id, externalId)
|
||||
})
|
||||
})
|
||||
@ -106,7 +106,7 @@ describe('Identities', function () {
|
||||
identity = await identities.createIdentity({ id })
|
||||
keystore = identities.keystore
|
||||
const key = await keystore.getKey(id)
|
||||
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
|
||||
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
|
||||
assert.strictEqual(identity.id, externalId)
|
||||
})
|
||||
|
||||
@ -117,7 +117,7 @@ describe('Identities', function () {
|
||||
|
||||
it('has the correct public key', async () => {
|
||||
const key = await keystore.getKey(id)
|
||||
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
|
||||
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
|
||||
const signingKey = await keystore.getKey(externalId)
|
||||
assert.notStrictEqual(signingKey, undefined)
|
||||
assert.strictEqual(identity.publicKey, keystore.getPublic(signingKey))
|
||||
@ -125,10 +125,10 @@ describe('Identities', function () {
|
||||
|
||||
it('has a signature for the id', async () => {
|
||||
const key = await keystore.getKey(id)
|
||||
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
|
||||
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
|
||||
const signingKey = await keystore.getKey(externalId)
|
||||
const idSignature = await signMessage(signingKey, externalId)
|
||||
const publicKey = uint8ArrayToString(signingKey.publicKey.raw, 'base16')
|
||||
const publicKey = uint8ArrayToString(signingKey.public.marshal(), 'base16')
|
||||
const verifies = await verifyMessage(idSignature, publicKey, externalId)
|
||||
assert.strictEqual(verifies, true)
|
||||
assert.strictEqual(identity.signatures.id, idSignature)
|
||||
@ -136,7 +136,7 @@ describe('Identities', function () {
|
||||
|
||||
it('has a signature for the publicKey', async () => {
|
||||
const key = await keystore.getKey(id)
|
||||
const externalId = uint8ArrayToString(key.publicKey.raw, 'base16')
|
||||
const externalId = uint8ArrayToString(key.public.marshal(), 'base16')
|
||||
const signingKey = await keystore.getKey(externalId)
|
||||
const idSignature = await signMessage(signingKey, externalId)
|
||||
const externalKey = await keystore.getKey(id)
|
||||
@ -171,7 +171,7 @@ describe('Identities', function () {
|
||||
|
||||
it('has the correct id', async () => {
|
||||
const key = await savedKeysKeyStore.getKey(id)
|
||||
assert.strictEqual(identity.id, uint8ArrayToString(key.publicKey.raw, 'base16'))
|
||||
assert.strictEqual(identity.id, uint8ArrayToString(key.public.marshal(), 'base16'))
|
||||
})
|
||||
|
||||
it('has the correct public key', async () => {
|
||||
|
||||
@ -145,7 +145,7 @@ describe('KeyStore', () => {
|
||||
})
|
||||
|
||||
describe('Options', () => {
|
||||
const unmarshal = crypto.keys.privateKeyFromRaw
|
||||
const unmarshal = crypto.keys.supportedKeys.secp256k1.unmarshalSecp256k1PrivateKey
|
||||
const privateKey = '198594a8de39fd97017d11996d619b3746211605a9d290964badf58bc79bdb33'
|
||||
const publicKey = '0260baeaffa1de1e4135e5b395e0380563a622b9599d1b8e012a0f7603f516bdaa'
|
||||
let privateKeyBuffer, publicKeyBuffer, unmarshalledPrivateKey
|
||||
|
||||
@ -33,8 +33,7 @@ describe('Entry', function () {
|
||||
it('creates a an empty entry', async () => {
|
||||
const expectedHash = 'zdpuAsKzwUEa8cz9pkJxxFMxLuP3cutA9PDGoLZytrg4RSVEa'
|
||||
const entry = await create(testIdentity, 'A', 'hello')
|
||||
const { hash } = await Entry.encode(entry)
|
||||
strictEqual(hash, expectedHash)
|
||||
strictEqual(entry.hash, expectedHash)
|
||||
strictEqual(entry.id, 'A')
|
||||
strictEqual(entry.clock.id, testIdentity.publicKey)
|
||||
strictEqual(entry.clock.time, 0)
|
||||
@ -48,8 +47,7 @@ describe('Entry', function () {
|
||||
const expectedHash = 'zdpuAmthfqpHRQjdSpKN5etr1GrreJb7QcU1Hshm6pERnzsxi'
|
||||
const payload = 'hello world'
|
||||
const entry = await create(testIdentity, 'A', payload)
|
||||
const { hash } = await Entry.encode(entry)
|
||||
strictEqual(hash, expectedHash)
|
||||
strictEqual(entry.hash, expectedHash)
|
||||
strictEqual(entry.payload, payload)
|
||||
strictEqual(entry.id, 'A')
|
||||
strictEqual(entry.clock.id, testIdentity.publicKey)
|
||||
@ -83,7 +81,7 @@ describe('Entry', function () {
|
||||
const payload2 = 'hello again'
|
||||
const entry1 = await create(testIdentity, 'A', payload1)
|
||||
entry1.clock = tickClock(entry1.clock)
|
||||
const entry2 = await create(testIdentity, 'A', payload2, null, entry1.clock, [entry1])
|
||||
const entry2 = await create(testIdentity, 'A', payload2, entry1.clock, [entry1])
|
||||
strictEqual(entry2.payload, payload2)
|
||||
strictEqual(entry2.next.length, 1)
|
||||
// strictEqual(entry2.hash, expectedHash)
|
||||
@ -93,8 +91,7 @@ describe('Entry', function () {
|
||||
|
||||
it('`next` parameter can be an array of strings', async () => {
|
||||
const entry1 = await create(testIdentity, 'A', 'hello1')
|
||||
const { hash } = await Entry.encode(entry1)
|
||||
const entry2 = await create(testIdentity, 'A', 'hello2', null, null, [hash])
|
||||
const entry2 = await create(testIdentity, 'A', 'hello2', null, [entry1.hash])
|
||||
strictEqual(typeof entry2.next[0] === 'string', true)
|
||||
})
|
||||
|
||||
@ -141,7 +138,7 @@ describe('Entry', function () {
|
||||
it('throws an error if next is not an array', async () => {
|
||||
let err
|
||||
try {
|
||||
await create(testIdentity, 'A', 'hello', null, null, {})
|
||||
await create(testIdentity, 'A', 'hello', null, {})
|
||||
} catch (e) {
|
||||
err = e
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { strictEqual, notStrictEqual, deepStrictEqual } from 'assert'
|
||||
import { rimraf } from 'rimraf'
|
||||
import { copy } from 'fs-extra'
|
||||
import { Log, Identities, KeyStore } from '../../src/index.js'
|
||||
import { Log, Entry, Identities, KeyStore } from '../../src/index.js'
|
||||
import { Clock } from '../../src/oplog/log.js'
|
||||
import { MemoryStorage } from '../../src/storage/index.js'
|
||||
import testKeysPath from '../fixtures/test-keys-path.js'
|
||||
@ -760,7 +760,7 @@ describe('Log - Join', async function () {
|
||||
|
||||
await log1.storage.merge(log0.storage)
|
||||
|
||||
await headsStorage1.put('heads', new TextEncoder().encode(JSON.stringify([{ hash: e0.hash, next: e0.next }])))
|
||||
await headsStorage1.put(e0.hash, e0.bytes)
|
||||
|
||||
await log1.append('hello1')
|
||||
await log1.append('hello2')
|
||||
@ -863,7 +863,7 @@ describe('Log - Join', async function () {
|
||||
})
|
||||
|
||||
describe('throws an error if verification of an entry in given entry\'s history fails', async () => {
|
||||
let e1
|
||||
let e1, e3
|
||||
let headsStorage1, headsStorage2
|
||||
|
||||
before(async () => {
|
||||
@ -875,19 +875,23 @@ describe('Log - Join', async function () {
|
||||
|
||||
e1 = await log1.append('hello1')
|
||||
await log1.append('hello2')
|
||||
await log1.append('hello3')
|
||||
e3 = await log1.append('hello3')
|
||||
})
|
||||
|
||||
it('throws an error if an entry doesn\'t have a payload field', async () => {
|
||||
const e = Object.assign({}, e1)
|
||||
delete e.payload
|
||||
|
||||
await headsStorage1.put('heads', JSON.stringify([{ hash: e1.hash, next: e1.next }]))
|
||||
delete e.bytes
|
||||
delete e.hash
|
||||
const ee = await Entry.encode(e)
|
||||
|
||||
await headsStorage1.put(e1.hash, ee.bytes)
|
||||
await log2.storage.merge(headsStorage1)
|
||||
|
||||
let err
|
||||
try {
|
||||
await log2.joinEntry(e)
|
||||
await log2.joinEntry(e3)
|
||||
} catch (e) {
|
||||
err = e
|
||||
}
|
||||
@ -902,12 +906,16 @@ describe('Log - Join', async function () {
|
||||
const e = Object.assign({}, e1)
|
||||
delete e.key
|
||||
|
||||
await headsStorage1.put('heads', JSON.stringify([{ hash: e1.hash, next: e1.next }]))
|
||||
delete e.bytes
|
||||
delete e.hash
|
||||
const ee = await Entry.encode(e)
|
||||
|
||||
await headsStorage1.put(e1.hash, ee.bytes)
|
||||
await log2.storage.merge(headsStorage1)
|
||||
|
||||
let err
|
||||
try {
|
||||
await log2.joinEntry(e)
|
||||
await log2.joinEntry(e3)
|
||||
} catch (e) {
|
||||
err = e
|
||||
}
|
||||
@ -922,12 +930,16 @@ describe('Log - Join', async function () {
|
||||
const e = Object.assign({}, e1)
|
||||
delete e.sig
|
||||
|
||||
await headsStorage1.put('heads', JSON.stringify([{ hash: e1.hash, next: e1.next }]))
|
||||
delete e.bytes
|
||||
delete e.hash
|
||||
const ee = await Entry.encode(e)
|
||||
|
||||
await headsStorage1.put(e1.hash, ee.bytes)
|
||||
await log2.storage.merge(headsStorage1)
|
||||
|
||||
let err
|
||||
try {
|
||||
await log2.joinEntry(e)
|
||||
await log2.joinEntry(e3)
|
||||
} catch (e) {
|
||||
err = e
|
||||
}
|
||||
@ -941,19 +953,22 @@ describe('Log - Join', async function () {
|
||||
it('throws an error if an entry signature doesn\'t verify', async () => {
|
||||
const e = Object.assign({}, e1)
|
||||
e.sig = '1234567890'
|
||||
delete e.bytes
|
||||
delete e.hash
|
||||
const ee = await Entry.encode(e)
|
||||
|
||||
await headsStorage1.put('heads', JSON.stringify([{ hash: e1.hash, next: e1.next }]))
|
||||
await headsStorage1.put(e1.hash, ee.bytes)
|
||||
await log2.storage.merge(headsStorage1)
|
||||
|
||||
let err
|
||||
try {
|
||||
await log2.joinEntry(e)
|
||||
await log2.joinEntry(e3)
|
||||
} catch (e) {
|
||||
err = e
|
||||
}
|
||||
|
||||
notStrictEqual(err, undefined)
|
||||
strictEqual(err.message, 'Could not validate signature for entry "zdpuAxyE4ScWLf4X6VvkhMrpDQvwdvQno1DhzY5p1U3GPHrBT"')
|
||||
strictEqual(err.message, 'Could not validate signature for entry "zdpuAvkAJ8C46cnGdtFpcBratA5MqK7CcjqCJjjmuKuFvZir3"')
|
||||
deepStrictEqual(await log2.all(), [])
|
||||
deepStrictEqual(await log2.heads(), [])
|
||||
})
|
||||
|
||||
@ -60,21 +60,15 @@ describe('Log', function () {
|
||||
})
|
||||
|
||||
it('sets one head if multiple are given as params', async () => {
|
||||
const one = await create(testIdentity, 'A', 'entryA', null, null, [])
|
||||
const { hash: hash1, bytes: bytes1 } = await Entry.encode(one)
|
||||
const two = await create(testIdentity, 'A', 'entryB', null, null, [hash1])
|
||||
const { hash: hash2, bytes: bytes2 } = await Entry.encode(two)
|
||||
const three = await create(testIdentity, 'A', 'entryC', null, null, [hash2])
|
||||
const { hash: hash3, bytes: bytes3 } = await Entry.encode(three)
|
||||
const four = await create(testIdentity, 'A', 'entryD', null, null, [hash3])
|
||||
const { hash: hash4, bytes: bytes4 } = await Entry.encode(four)
|
||||
const one = await create(testIdentity, 'A', 'entryA', null, [])
|
||||
const two = await create(testIdentity, 'A', 'entryB', null, [one.hash])
|
||||
const three = await create(testIdentity, 'A', 'entryC', null, [two.hash])
|
||||
const four = await create(testIdentity, 'A', 'entryD', null, [two.hash])
|
||||
const entryStorage = await MemoryStorage()
|
||||
await entryStorage.put(hash1, bytes1)
|
||||
await entryStorage.put(hash2, bytes2)
|
||||
await entryStorage.put(hash3, bytes3)
|
||||
await entryStorage.put(hash4, bytes4)
|
||||
three.hash = hash3
|
||||
two.hash = hash2
|
||||
await entryStorage.put(one.hash, one.bytes)
|
||||
await entryStorage.put(two.hash, two.bytes)
|
||||
await entryStorage.put(three.hash, three.bytes)
|
||||
await entryStorage.put(four.hash, four.bytes)
|
||||
const log = await Log(testIdentity, { logId: 'A', logHeads: [three, three, two, two], entryStorage })
|
||||
const values = await log.values()
|
||||
const heads = await log.heads()
|
||||
@ -84,22 +78,15 @@ describe('Log', function () {
|
||||
})
|
||||
|
||||
it('sets two heads if two given as params', async () => {
|
||||
const one = await create(testIdentity, 'A', 'entryA', null, null, [])
|
||||
const { hash: hash1, bytes: bytes1 } = await Entry.encode(one)
|
||||
const two = await create(testIdentity, 'A', 'entryB', null, null, [hash1])
|
||||
const { hash: hash2, bytes: bytes2 } = await Entry.encode(two)
|
||||
const three = await create(testIdentity, 'A', 'entryC', null, null, [hash2])
|
||||
const { hash: hash3, bytes: bytes3 } = await Entry.encode(three)
|
||||
const four = await create(testIdentity, 'A', 'entryD', null, null, [hash2])
|
||||
const { hash: hash4, bytes: bytes4 } = await Entry.encode(four)
|
||||
const one = await create(testIdentity, 'A', 'entryA', null, [])
|
||||
const two = await create(testIdentity, 'A', 'entryB', null, [one.hash])
|
||||
const three = await create(testIdentity, 'A', 'entryC', null, [two.hash])
|
||||
const four = await create(testIdentity, 'A', 'entryD', null, [two.hash])
|
||||
const entryStorage = await MemoryStorage()
|
||||
await entryStorage.put(hash1, bytes1)
|
||||
await entryStorage.put(hash2, bytes2)
|
||||
await entryStorage.put(hash3, bytes3)
|
||||
await entryStorage.put(hash4, bytes4)
|
||||
three.hash = hash3
|
||||
four.hash = hash4
|
||||
two.hash = hash2
|
||||
await entryStorage.put(one.hash, one.bytes)
|
||||
await entryStorage.put(two.hash, two.bytes)
|
||||
await entryStorage.put(three.hash, three.bytes)
|
||||
await entryStorage.put(four.hash, four.bytes)
|
||||
const log = await Log(testIdentity, { logId: 'A', logHeads: [three, four, two], entryStorage })
|
||||
const values = await log.values()
|
||||
const heads = await log.heads()
|
||||
|
||||
@ -69,7 +69,7 @@ describe('Log - Replication', function () {
|
||||
try {
|
||||
if (!messageIsFromMe(message)) {
|
||||
const entry = await Entry.decode(message.detail.data)
|
||||
await storage1.put(entry.hash, message.detail.data)
|
||||
await storage1.put(entry.hash, entry.bytes)
|
||||
await log1.joinEntry(entry)
|
||||
}
|
||||
} catch (e) {
|
||||
@ -83,7 +83,7 @@ describe('Log - Replication', function () {
|
||||
try {
|
||||
if (!messageIsFromMe(message)) {
|
||||
const entry = await Entry.decode(message.detail.data)
|
||||
await storage2.put(entry.hash, message.detail.data)
|
||||
await storage2.put(entry.hash, entry.bytes)
|
||||
await log2.joinEntry(entry)
|
||||
}
|
||||
} catch (e) {
|
||||
@ -114,10 +114,8 @@ describe('Log - Replication', function () {
|
||||
for (let i = 1; i <= amount; i++) {
|
||||
const entry1 = await input1.append('A' + i)
|
||||
const entry2 = await input2.append('B' + i)
|
||||
const bytes1 = await input1.storage.get(entry1.hash)
|
||||
const bytes2 = await input1.storage.get(entry2.hash)
|
||||
await ipfs1.libp2p.services.pubsub.publish(logId, bytes1)
|
||||
await ipfs2.libp2p.services.pubsub.publish(logId, bytes2)
|
||||
await ipfs1.libp2p.services.pubsub.publish(logId, entry1.bytes)
|
||||
await ipfs2.libp2p.services.pubsub.publish(logId, entry2.bytes)
|
||||
}
|
||||
|
||||
console.log('Messages sent')
|
||||
|
||||
@ -1,376 +0,0 @@
|
||||
import { strictEqual, notEqual } from 'assert'
|
||||
import { rimraf } from 'rimraf'
|
||||
import path from 'path'
|
||||
import { createOrbitDB } from '../src/index.js'
|
||||
import connectPeers from './utils/connect-nodes.js'
|
||||
import waitFor from './utils/wait-for.js'
|
||||
import createHelia from './utils/create-helia.js'
|
||||
|
||||
import * as Block from 'multiformats/block'
|
||||
import * as dagCbor from '@ipld/dag-cbor'
|
||||
import { sha256 } from 'multiformats/hashes/sha2'
|
||||
|
||||
import SimpleEncryption from '@orbitdb/simple-encryption'
|
||||
|
||||
const codec = dagCbor
|
||||
const hasher = sha256
|
||||
|
||||
const dbPath = './orbitdb/tests/write-permissions'
|
||||
|
||||
describe('Encryption', function () {
|
||||
this.timeout(5000)
|
||||
|
||||
let ipfs1, ipfs2
|
||||
let orbitdb1, orbitdb2
|
||||
let db1, db2
|
||||
|
||||
let replicationEncryption
|
||||
let dataEncryption
|
||||
|
||||
before(async () => {
|
||||
[ipfs1, ipfs2] = await Promise.all([createHelia(), createHelia()])
|
||||
await connectPeers(ipfs1, ipfs2)
|
||||
|
||||
await rimraf('./orbitdb')
|
||||
|
||||
orbitdb1 = await createOrbitDB({ ipfs: ipfs1, id: 'user1', directory: path.join(dbPath, '1') })
|
||||
orbitdb2 = await createOrbitDB({ ipfs: ipfs2, id: 'user2', directory: path.join(dbPath, '2') })
|
||||
|
||||
replicationEncryption = await SimpleEncryption({ password: 'hello' })
|
||||
dataEncryption = await SimpleEncryption({ password: 'world' })
|
||||
})
|
||||
|
||||
after(async () => {
|
||||
if (orbitdb1) {
|
||||
await orbitdb1.stop()
|
||||
}
|
||||
|
||||
if (orbitdb2) {
|
||||
await orbitdb2.stop()
|
||||
}
|
||||
|
||||
if (ipfs1) {
|
||||
await ipfs1.stop()
|
||||
}
|
||||
|
||||
if (ipfs2) {
|
||||
await ipfs2.stop()
|
||||
}
|
||||
|
||||
await rimraf('./orbitdb')
|
||||
await rimraf('./ipfs1')
|
||||
await rimraf('./ipfs2')
|
||||
})
|
||||
|
||||
describe('Data is encrypted when replicated to peers', async () => {
|
||||
afterEach(async () => {
|
||||
if (db1) {
|
||||
await db1.drop()
|
||||
await db1.close()
|
||||
}
|
||||
if (db2) {
|
||||
await db2.drop()
|
||||
await db2.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('encrypts/decrypts data', async () => {
|
||||
let connected = false
|
||||
let updated = false
|
||||
let error = false
|
||||
|
||||
const encryption = {
|
||||
data: dataEncryption
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
db2 = await orbitdb2.open(db1.address, { encryption })
|
||||
|
||||
const onJoin = async (peerId, heads) => {
|
||||
connected = true
|
||||
}
|
||||
db2.events.on('join', onJoin)
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
|
||||
const onUpdate = async (peerId, heads) => {
|
||||
updated = true
|
||||
}
|
||||
db2.events.on('update', onUpdate)
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
console.log(err)
|
||||
error = true
|
||||
}
|
||||
db2.events.on('error', onError)
|
||||
|
||||
const hash1 = await db1.add('record 1')
|
||||
const hash2 = await db1.add('record 2')
|
||||
|
||||
strictEqual(await db1.get(hash1), 'record 1')
|
||||
strictEqual(await db1.get(hash2), 'record 2')
|
||||
|
||||
await waitFor(() => updated || error, () => true)
|
||||
|
||||
const all = await db2.all()
|
||||
|
||||
strictEqual(all.length, 2)
|
||||
strictEqual(all[0].value, 'record 1')
|
||||
strictEqual(all[1].value, 'record 2')
|
||||
})
|
||||
|
||||
it('encrypts/decrypts log', async () => {
|
||||
let connected = false
|
||||
let updated = false
|
||||
let error = false
|
||||
|
||||
const encryption = {
|
||||
replication: replicationEncryption
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
db2 = await orbitdb2.open(db1.address, { encryption })
|
||||
|
||||
const onJoin = async (peerId, heads) => {
|
||||
connected = true
|
||||
}
|
||||
db2.events.on('join', onJoin)
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
|
||||
const onUpdate = async (peerId, heads) => {
|
||||
updated = true
|
||||
}
|
||||
db2.events.on('update', onUpdate)
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
console.log(err)
|
||||
error = true
|
||||
}
|
||||
db2.events.on('error', onError)
|
||||
|
||||
const hash1 = await db1.add('record 1')
|
||||
const hash2 = await db1.add('record 2')
|
||||
|
||||
strictEqual(await db1.get(hash1), 'record 1')
|
||||
strictEqual(await db1.get(hash2), 'record 2')
|
||||
|
||||
await waitFor(() => updated || error, () => true)
|
||||
|
||||
const all = await db2.all()
|
||||
|
||||
strictEqual(all.length, 2)
|
||||
strictEqual(all[0].value, 'record 1')
|
||||
strictEqual(all[1].value, 'record 2')
|
||||
})
|
||||
|
||||
it('encrypts/decrypts log and data', async () => {
|
||||
let connected = false
|
||||
let updated = false
|
||||
let error = false
|
||||
|
||||
const encryption = {
|
||||
replication: replicationEncryption,
|
||||
data: dataEncryption
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
db2 = await orbitdb2.open(db1.address, { encryption })
|
||||
|
||||
const onJoin = async (peerId, heads) => {
|
||||
connected = true
|
||||
}
|
||||
db2.events.on('join', onJoin)
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
|
||||
const onUpdate = async (peerId, heads) => {
|
||||
updated = true
|
||||
}
|
||||
db2.events.on('update', onUpdate)
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
console.log(err)
|
||||
error = true
|
||||
}
|
||||
db2.events.on('error', onError)
|
||||
|
||||
const hash1 = await db1.add('record 1')
|
||||
const hash2 = await db1.add('record 2')
|
||||
|
||||
strictEqual(await db1.get(hash1), 'record 1')
|
||||
strictEqual(await db1.get(hash2), 'record 2')
|
||||
|
||||
await waitFor(() => updated || error, () => true)
|
||||
|
||||
const all = await db2.all()
|
||||
|
||||
strictEqual(all.length, 2)
|
||||
strictEqual(all[0].value, 'record 1')
|
||||
strictEqual(all[1].value, 'record 2')
|
||||
})
|
||||
|
||||
it('throws an error if log can\'t be decrypted', async () => {
|
||||
let connected = false
|
||||
let hasError = false
|
||||
let error
|
||||
|
||||
const replicationEncryptionWithFailure = await SimpleEncryption({ password: 'goodbye' })
|
||||
|
||||
const encryption = {
|
||||
replication: replicationEncryption
|
||||
}
|
||||
|
||||
const encryptionWithFailure = {
|
||||
replication: replicationEncryptionWithFailure
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
db2 = await orbitdb2.open(db1.address, { encryption: encryptionWithFailure })
|
||||
|
||||
const onJoin = async (peerId, heads) => {
|
||||
connected = true
|
||||
}
|
||||
db2.events.on('join', onJoin)
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
error = err
|
||||
hasError = true
|
||||
}
|
||||
db2.events.on('error', onError)
|
||||
|
||||
await db1.add('record 1')
|
||||
|
||||
await waitFor(() => hasError, () => true)
|
||||
|
||||
strictEqual(error.message, 'Could not decrypt entry')
|
||||
|
||||
const all = await db2.all()
|
||||
|
||||
strictEqual(all.length, 0)
|
||||
})
|
||||
|
||||
it('throws an error if data can\'t be decrypted', async () => {
|
||||
let connected = false
|
||||
let hasError = false
|
||||
let error
|
||||
|
||||
const dataEncryptionWithFailure = await SimpleEncryption({ password: 'goodbye' })
|
||||
|
||||
const encryption = {
|
||||
data: dataEncryption
|
||||
}
|
||||
|
||||
const encryptionWithFailure = {
|
||||
data: dataEncryptionWithFailure
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
db2 = await orbitdb2.open(db1.address, { encryption: encryptionWithFailure })
|
||||
|
||||
const onJoin = async (peerId, heads) => {
|
||||
connected = true
|
||||
}
|
||||
db2.events.on('join', onJoin)
|
||||
|
||||
await waitFor(() => connected, () => true)
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
error = err
|
||||
hasError = true
|
||||
}
|
||||
db2.events.on('error', onError)
|
||||
|
||||
await db1.add('record 1')
|
||||
|
||||
await waitFor(() => hasError, () => true)
|
||||
|
||||
strictEqual(error.message, 'Could not decrypt payload')
|
||||
|
||||
const all = await db2.all()
|
||||
|
||||
strictEqual(all.length, 0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Data is encrypted in storage', async () => {
|
||||
afterEach(async () => {
|
||||
if (db1) {
|
||||
await db1.drop()
|
||||
await db1.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('payload bytes are encrypted in storage', async () => {
|
||||
let error
|
||||
|
||||
const encryption = {
|
||||
data: dataEncryption
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
console.log(err)
|
||||
error = true
|
||||
}
|
||||
db1.events.on('error', onError)
|
||||
|
||||
const hash1 = await db1.add('record 1')
|
||||
|
||||
const bytes = await db1.log.storage.get(hash1)
|
||||
const { value } = await Block.decode({ bytes, codec, hasher })
|
||||
const payload = value.payload
|
||||
|
||||
strictEqual(payload.constructor, Uint8Array)
|
||||
|
||||
try {
|
||||
await Block.decode({ bytes: payload, codec, hasher })
|
||||
} catch (e) {
|
||||
error = e
|
||||
}
|
||||
|
||||
strictEqual(error.message.startsWith('CBOR decode error'), true)
|
||||
})
|
||||
|
||||
it('entry bytes are encrypted in storage', async () => {
|
||||
let error
|
||||
|
||||
const encryption = {
|
||||
replication: replicationEncryption
|
||||
}
|
||||
|
||||
db1 = await orbitdb1.open('encryption-test-1', { encryption })
|
||||
|
||||
const onError = async (err) => {
|
||||
// Catch "Could not decrypt entry" errors
|
||||
console.log(err)
|
||||
error = true
|
||||
}
|
||||
db1.events.on('error', onError)
|
||||
|
||||
const hash1 = await db1.add('record 1')
|
||||
let decodedBytes
|
||||
|
||||
try {
|
||||
const bytes = await db1.log.storage.get(hash1)
|
||||
decodedBytes = await Block.decode({ bytes, codec, hasher })
|
||||
await Block.decode({ bytes: decodedBytes, codec, hasher })
|
||||
} catch (e) {
|
||||
error = e
|
||||
}
|
||||
|
||||
notEqual(error, undefined)
|
||||
strictEqual(error.message.startsWith('CBOR decode error'), true)
|
||||
strictEqual(decodedBytes.value.constructor, Uint8Array)
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,11 +1,9 @@
|
||||
import { deepStrictEqual, strictEqual } from 'assert'
|
||||
import { deepStrictEqual } from 'assert'
|
||||
import { rimraf } from 'rimraf'
|
||||
import { createOrbitDB } from '../src/index.js'
|
||||
import connectPeers from './utils/connect-nodes.js'
|
||||
import waitFor from './utils/wait-for.js'
|
||||
import createHelia from './utils/create-helia.js'
|
||||
import { CID } from 'multiformats/cid'
|
||||
import { base58btc } from 'multiformats/bases/base58'
|
||||
|
||||
describe('Replicating databases', function () {
|
||||
this.timeout(10000)
|
||||
@ -25,8 +23,8 @@ describe('Replicating databases', function () {
|
||||
after(async () => {
|
||||
await orbitdb1.stop()
|
||||
await orbitdb2.stop()
|
||||
await ipfs1.blockstore.child.child.child.close()
|
||||
await ipfs2.blockstore.child.child.child.close()
|
||||
await ipfs1.blockstore.child.child.close()
|
||||
await ipfs2.blockstore.child.child.close()
|
||||
await ipfs1.stop()
|
||||
await ipfs2.stop()
|
||||
|
||||
@ -138,12 +136,8 @@ describe('Replicating databases', function () {
|
||||
|
||||
await orbitdb1.stop()
|
||||
await orbitdb2.stop()
|
||||
// TODO: Strange issue with ClassicLevel. Causes subsequent Helia
|
||||
// instantiations to error with db closed. Explicitly closing the
|
||||
// nested ClassicLevel db seems to resolve the issue. Requires further
|
||||
// investigation.
|
||||
await ipfs1.blockstore.child.child.child.close()
|
||||
await ipfs2.blockstore.child.child.child.close()
|
||||
await ipfs1.blockstore.child.child.close()
|
||||
await ipfs2.blockstore.child.child.close()
|
||||
await ipfs1.stop()
|
||||
await ipfs2.stop()
|
||||
|
||||
@ -169,26 +163,5 @@ describe('Replicating databases', function () {
|
||||
|
||||
console.log('events:', amount)
|
||||
})
|
||||
|
||||
it('pins all entries in the replicated database', async () => {
|
||||
const db1 = await orbitdb1.open('helloworld', { referencesCount: 0 })
|
||||
const hash = await db1.add('hello world')
|
||||
|
||||
let replicated = false
|
||||
|
||||
const onJoin = async (peerId, heads) => {
|
||||
replicated = true
|
||||
}
|
||||
|
||||
const db2 = await orbitdb2.open(db1.address)
|
||||
|
||||
db2.events.on('join', onJoin)
|
||||
|
||||
await waitFor(() => replicated, () => true)
|
||||
|
||||
const cid = CID.parse(hash, base58btc)
|
||||
strictEqual(await ipfs1.pins.isPinned(cid), true)
|
||||
strictEqual(await ipfs2.pins.isPinned(cid), true)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -72,8 +72,8 @@ describe('OrbitDB', function () {
|
||||
const privateKey = await orbitdb1.keystore.getKey(orbitdb1.identity.id)
|
||||
notStrictEqual(privateKey, undefined)
|
||||
strictEqual(privateKey.constructor.name, 'Secp256k1PrivateKey')
|
||||
notStrictEqual(privateKey.raw, undefined)
|
||||
notStrictEqual(privateKey.publicKey, undefined)
|
||||
notStrictEqual(privateKey._key, undefined)
|
||||
notStrictEqual(privateKey._publicKey, undefined)
|
||||
})
|
||||
|
||||
it('has a keystore that contains a public key that matches the identity\'s public key', async () => {
|
||||
@ -102,8 +102,8 @@ describe('OrbitDB', function () {
|
||||
notStrictEqual(orbitdb1.peerId, undefined)
|
||||
})
|
||||
|
||||
it('has a peerId of type Ed25519', async () => {
|
||||
strictEqual(orbitdb1.peerId.type, 'Ed25519')
|
||||
it('has a peerId of type Ed25519PeerIdImpl', async () => {
|
||||
strictEqual(orbitdb1.peerId.constructor.name, 'Ed25519PeerIdImpl')
|
||||
})
|
||||
|
||||
it('has a peerId that matches the IPFS id', async () => {
|
||||
@ -164,8 +164,8 @@ describe('OrbitDB', function () {
|
||||
const privateKey = await orbitdb1.keystore.getKey(orbitdb1.identity.id)
|
||||
notStrictEqual(privateKey, undefined)
|
||||
strictEqual(privateKey.constructor.name, 'Secp256k1PrivateKey')
|
||||
notStrictEqual(privateKey.raw, undefined)
|
||||
notStrictEqual(privateKey.publicKey, undefined)
|
||||
notStrictEqual(privateKey._key, undefined)
|
||||
notStrictEqual(privateKey._publicKey, undefined)
|
||||
})
|
||||
|
||||
it('has a keystore that contains a public key that matches the identity\'s public key', async () => {
|
||||
@ -194,8 +194,8 @@ describe('OrbitDB', function () {
|
||||
notStrictEqual(orbitdb1.peerId, undefined)
|
||||
})
|
||||
|
||||
it('has a peerId of type Ed25519', async () => {
|
||||
strictEqual(orbitdb1.peerId.type, 'Ed25519')
|
||||
it('has a peerId of type Ed25519PeerIdImpl', async () => {
|
||||
strictEqual(orbitdb1.peerId.constructor.name, 'Ed25519PeerIdImpl')
|
||||
})
|
||||
|
||||
it('has a peerId that matches the IPFS id', async () => {
|
||||
|
||||
@ -139,7 +139,8 @@ describe('Sync protocol', function () {
|
||||
log1 = await Log(testIdentity1, { logId: 'synclog111', entryStorage: entryStorage1 })
|
||||
log2 = await Log(testIdentity2, { logId: 'synclog111', entryStorage: entryStorage2 })
|
||||
|
||||
const onSynced = async (entry) => {
|
||||
const onSynced = async (bytes) => {
|
||||
const entry = await Entry.decode(bytes)
|
||||
if (await log2.joinEntry(entry)) {
|
||||
syncedHead = entry
|
||||
syncedEventFired = true
|
||||
@ -206,7 +207,8 @@ describe('Sync protocol', function () {
|
||||
log1 = await Log(testIdentity1, { logId: 'synclog7', entryStorage: entryStorage1 })
|
||||
log2 = await Log(testIdentity2, { logId: 'synclog7', entryStorage: entryStorage2 })
|
||||
|
||||
const onSynced = async (entry) => {
|
||||
const onSynced = async (bytes) => {
|
||||
const entry = await Entry.decode(bytes)
|
||||
if (await log2.joinEntry(entry)) {
|
||||
syncedHead = entry
|
||||
}
|
||||
@ -289,8 +291,8 @@ describe('Sync protocol', function () {
|
||||
log1 = await Log(testIdentity1, { logId: 'synclog1' })
|
||||
log2 = await Log(testIdentity2, { logId: 'synclog1' })
|
||||
|
||||
const onSynced = async (entry) => {
|
||||
syncedHead = entry
|
||||
const onSynced = async (bytes) => {
|
||||
syncedHead = await Entry.decode(bytes)
|
||||
syncedEventFired = expectedEntry.hash === syncedHead.hash
|
||||
}
|
||||
|
||||
@ -346,8 +348,8 @@ describe('Sync protocol', function () {
|
||||
log1 = await Log(testIdentity1, { logId: 'synclog1' })
|
||||
log2 = await Log(testIdentity2, { logId: 'synclog1' })
|
||||
|
||||
const onSynced = async (entry) => {
|
||||
syncedHead = entry
|
||||
const onSynced = async (bytes) => {
|
||||
syncedHead = await Entry.decode(bytes)
|
||||
if (expectedEntry) {
|
||||
syncedEventFired = expectedEntry.hash === syncedHead.hash
|
||||
}
|
||||
@ -432,9 +434,9 @@ describe('Sync protocol', function () {
|
||||
log1 = await Log(testIdentity1, { logId: 'synclog1' })
|
||||
log2 = await Log(testIdentity2, { logId: 'synclog1' })
|
||||
|
||||
const onSynced = async (entry) => {
|
||||
const onSynced = async (bytes) => {
|
||||
if (expectedEntry && !syncedEventFired) {
|
||||
syncedHead = entry
|
||||
syncedHead = await Entry.decode(bytes)
|
||||
syncedEventFired = expectedEntry.hash === syncedHead.hash
|
||||
}
|
||||
}
|
||||
@ -516,8 +518,8 @@ describe('Sync protocol', function () {
|
||||
log1 = await Log(testIdentity1, { logId: 'synclog2' })
|
||||
log2 = await Log(testIdentity2, { logId: 'synclog2' })
|
||||
|
||||
const onSynced = async (entry) => {
|
||||
syncedHead = entry
|
||||
const onSynced = async (bytes) => {
|
||||
syncedHead = await Entry.decode(bytes)
|
||||
if (expectedEntry) {
|
||||
syncedEventFired = expectedEntry ? expectedEntry.hash === syncedHead.hash : false
|
||||
}
|
||||
@ -663,7 +665,7 @@ describe('Sync protocol', function () {
|
||||
let sync1, sync2
|
||||
let log1, log2
|
||||
|
||||
const timeoutTime = 0 // 0 milliseconds
|
||||
const timeoutTime = 1 // 1 millisecond
|
||||
|
||||
before(async () => {
|
||||
[ipfs1, ipfs2] = await Promise.all([createHelia(), createHelia()])
|
||||
@ -699,13 +701,13 @@ describe('Sync protocol', function () {
|
||||
let err = null
|
||||
|
||||
const onError = (error) => {
|
||||
err ??= error
|
||||
(!err) && (err = error)
|
||||
}
|
||||
|
||||
sync1 = await Sync({ ipfs: ipfs1, log: log1, timeout: timeoutTime })
|
||||
sync1.events.on('error', onError)
|
||||
|
||||
sync2 = await Sync({ ipfs: ipfs2, log: log2, start: false, timeout: timeoutTime })
|
||||
|
||||
sync1.events.on('error', onError)
|
||||
sync2.events.on('error', onError)
|
||||
|
||||
await log1.append('hello1')
|
||||
@ -716,7 +718,7 @@ describe('Sync protocol', function () {
|
||||
|
||||
notStrictEqual(err, null)
|
||||
strictEqual(err.type, 'aborted')
|
||||
strictEqual(err.message.includes('aborted'), true)
|
||||
strictEqual(err.message, 'Read aborted')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -21,16 +21,20 @@ const Libp2pOptions = {
|
||||
transports: [
|
||||
webSockets({
|
||||
filter: all
|
||||
}),
|
||||
webRTC(),
|
||||
circuitRelayTransport({
|
||||
discoverRelays: 1
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
connectionGater: {
|
||||
denyDialMultiaddr: () => false
|
||||
},
|
||||
services: {
|
||||
identify: identify(),
|
||||
pubsub: gossipsub({ allowPublishToZeroTopicPeers: true })
|
||||
pubsub: gossipsub({ allowPublishToZeroPeers: true })
|
||||
}
|
||||
}
|
||||
|
||||
@ -39,23 +43,25 @@ const Libp2pOptions = {
|
||||
*/
|
||||
const Libp2pBrowserOptions = {
|
||||
addresses: {
|
||||
listen: ['/webrtc', '/p2p-circuit']
|
||||
listen: ['/webrtc']
|
||||
},
|
||||
transports: [
|
||||
webSockets({
|
||||
filter: all
|
||||
}),
|
||||
webRTC(),
|
||||
circuitRelayTransport()
|
||||
circuitRelayTransport({
|
||||
discoverRelays: 1
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
connectionGater: {
|
||||
denyDialMultiaddr: () => false
|
||||
},
|
||||
services: {
|
||||
identify: identify(),
|
||||
pubsub: gossipsub({ allowPublishToZeroTopicPeers: true })
|
||||
pubsub: gossipsub({ allowPublishToZeroPeers: true })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -5,18 +5,21 @@ import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
|
||||
import { webSockets } from '@libp2p/websockets'
|
||||
import * as filters from '@libp2p/websockets/filters'
|
||||
import { identify } from '@libp2p/identify'
|
||||
import { createFromPrivKey } from '@libp2p/peer-id-factory'
|
||||
import { unmarshalPrivateKey } from '@libp2p/crypto/keys'
|
||||
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
|
||||
import { privateKeyFromProtobuf } from '@libp2p/crypto/keys'
|
||||
|
||||
// output of: console.log(server.peerId.privateKey.toString('hex'))
|
||||
const relayPrivKey = '08011240821cb6bc3d4547fcccb513e82e4d718089f8a166b23ffcd4a436754b6b0774cf07447d1693cd10ce11ef950d7517bad6e9472b41a927cd17fc3fb23f8c70cd99'
|
||||
// the peer id of the above key
|
||||
// const relayId = '12D3KooWAJjbRkp8FPF5MKgMU53aUTxWkqvDrs4zc1VMbwRwfsbE'
|
||||
|
||||
const privateKey = privateKeyFromProtobuf(uint8ArrayFromString(relayPrivKey, 'hex'))
|
||||
const encoded = uint8ArrayFromString(relayPrivKey, 'hex')
|
||||
const privateKey = await unmarshalPrivateKey(encoded)
|
||||
const peerId = await createFromPrivKey(privateKey)
|
||||
|
||||
const server = await createLibp2p({
|
||||
privateKey,
|
||||
peerId,
|
||||
addresses: {
|
||||
listen: ['/ip4/0.0.0.0/tcp/12345/ws']
|
||||
},
|
||||
@ -25,13 +28,14 @@ const server = await createLibp2p({
|
||||
filter: filters.all
|
||||
})
|
||||
],
|
||||
connectionEncrypters: [noise()],
|
||||
connectionEncryption: [noise()],
|
||||
streamMuxers: [yamux()],
|
||||
services: {
|
||||
identify: identify(),
|
||||
relay: circuitRelayServer({
|
||||
reservations: {
|
||||
maxReservations: 5000,
|
||||
reservationTtl: 1000,
|
||||
defaultDataLimit: BigInt(1024 * 1024 * 1024)
|
||||
}
|
||||
})
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user