Merge pull request #70 from orbitdb/dev/cleanups

Cleanups
This commit is contained in:
Haad 2023-04-13 08:05:47 +03:00 committed by GitHub
commit df7cd85a59
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 102 additions and 508 deletions

View File

@ -1,4 +1,4 @@
import { IPFSBlockStorage } from '../storage/index.js'
import { IPFSBlockStorage, LRUStorage, ComposedStorage } from '../storage/index.js'
import * as Block from 'multiformats/block'
import * as dagCbor from '@ipld/dag-cbor'
import { sha256 } from 'multiformats/hashes/sha2'
@ -23,7 +23,10 @@ const AccessControlList = async ({ storage, type, params }) => {
const type = 'ipfs'
const IPFSAccessController = ({ write, storage } = {}) => async ({ orbitdb, identities, address }) => {
storage = storage || await IPFSBlockStorage({ ipfs: orbitdb.ipfs, pin: true })
storage = storage || await ComposedStorage(
await LRUStorage({ size: 1000 }),
await IPFSBlockStorage({ ipfs: orbitdb.ipfs, pin: true })
)
write = write || [orbitdb.identity.id]
if (address) {

View File

@ -2,30 +2,59 @@ import * as Block from 'multiformats/block'
import * as dagCbor from '@ipld/dag-cbor'
import { sha256 } from 'multiformats/hashes/sha2'
import { base58btc } from 'multiformats/bases/base58'
import { ComposedStorage, IPFSBlockStorage, LRUStorage } from './storage/index.js'
const codec = dagCbor
const hasher = sha256
const hashStringEncoding = base58btc
// Creates a DB manifest file and saves it in IPFS
export default async ({ storage, name, type, accessController, meta }) => {
if (!storage) throw new Error('storage is required')
if (!name) throw new Error('name is required')
if (!type) throw new Error('type is required')
if (!accessController) throw new Error('accessController is required')
const manifest = Object.assign(
{
name,
type,
accessController
},
// meta field is only added to manifest if meta parameter is defined
meta !== undefined ? { meta } : {}
const Manifest = async ({ ipfs, storage } = {}) => {
storage = storage || await ComposedStorage(
await LRUStorage({ size: 1000 }),
await IPFSBlockStorage({ ipfs, pin: true })
)
const { cid, bytes } = await Block.encode({ value: manifest, codec, hasher })
const hash = cid.toString(hashStringEncoding)
await storage.put(hash, bytes)
return { hash, manifest }
const get = async (address) => {
const bytes = await storage.get(address)
const { value } = await Block.decode({ bytes, codec, hasher })
return value
}
const create = async ({ name, type, accessController, meta }) => {
if (!name) throw new Error('name is required')
if (!type) throw new Error('type is required')
if (!accessController) throw new Error('accessController is required')
const manifest = Object.assign(
{
name,
type,
accessController
},
// meta field is only added to manifest if meta parameter is defined
meta !== undefined ? { meta } : {}
)
const { cid, bytes } = await Block.encode({ value: manifest, codec, hasher })
const hash = cid.toString(hashStringEncoding)
await storage.put(hash, bytes)
return {
hash,
manifest
}
}
const close = async () => {
await storage.close()
}
return {
get,
create,
close
}
}
export default Manifest

View File

@ -1,34 +1,19 @@
import { Events, KeyValue, Documents } from './db/index.js'
import { ComposedStorage, IPFSBlockStorage, LRUStorage } from './storage/index.js'
import KeyStore from './key-store.js'
import { Identities } from './identities/index.js'
import OrbitDBAddress, { isValidAddress } from './address.js'
import DBManifest from './manifest.js'
import Manifests from './manifest.js'
import { createId } from './utils/index.js'
// import Logger from 'logplease'
import pathJoin from './utils/path-join.js'
import * as Block from 'multiformats/block'
import * as dagCbor from '@ipld/dag-cbor'
import { sha256 } from 'multiformats/hashes/sha2'
import * as AccessControllers from './access-controllers/index.js'
import IPFSAccessController from './access-controllers/ipfs.js'
const codec = dagCbor
const hasher = sha256
// const logger = Logger.create('orbit-db')
// Logger.setLogLevel('ERROR')
// Mapping for 'database type' -> Store
// Mapping for database types
const databaseTypes = {
events: Events,
documents: Documents,
keyvalue: KeyValue
}
//
// const accessControllers = {
// ipfs: IPFSAccessController,
// orbitdb: OrbitDBAccessController
// }
const addDatabaseType = (type, store) => {
if (databaseTypes[type]) {
@ -36,15 +21,8 @@ const addDatabaseType = (type, store) => {
}
databaseTypes[type] = store
}
//
// const addAccessController = (type, accessController) => {
// if (accessControllers[type]) {
// throw new Error(`Access Controller already exists: ${type}`)
// }
// accessControllers[type] = accessController
// }
// const defaultTimeout = 30000 // 30 seconds
const DefaultAccessController = IPFSAccessController
const OrbitDB = async ({ ipfs, id, identity, keystore, directory } = {}) => {
if (ipfs == null) {
@ -58,10 +36,7 @@ const OrbitDB = async ({ ipfs, id, identity, keystore, directory } = {}) => {
const identities = await Identities({ ipfs, keystore })
identity = identity || await identities.createIdentity({ id })
const manifestStorage = await ComposedStorage(
await LRUStorage({ size: 1000 }),
await IPFSBlockStorage({ ipfs, pin: true })
)
const manifests = await Manifests({ ipfs })
let databases = {}
@ -79,26 +54,22 @@ const OrbitDB = async ({ ipfs, id, identity, keystore, directory } = {}) => {
if (isValidAddress(address)) {
// If the address given was a valid OrbitDB address, eg. '/orbitdb/zdpuAuK3BHpS7NvMBivynypqciYCuy2UW77XYBPUYRnLjnw13'
const addr = OrbitDBAddress(address)
const bytes = await manifestStorage.get(addr.path)
const { value } = await Block.decode({ bytes, codec, hasher })
manifest = value
manifest = await manifests.get(addr.path)
const acType = manifest.accessController.split('/', 2).pop()
const acAddress = manifest.accessController.replaceAll(`/${acType}/`, '')
accessController = await AccessControllers.get(acType)()({ orbitdb: { open, identity, ipfs }, identities, address: acAddress })
AccessController = AccessControllers.get(acType)()
accessController = await AccessController({ orbitdb: { open, identity, ipfs }, identities, address: acAddress })
name = manifest.name
type = type || manifest.type
meta = manifest.meta
} else {
// If the address given was not valid, eg. just the name of the database
type = type || 'events'
AccessController = AccessController || AccessControllers.get('ipfs')({ storage: manifestStorage })
AccessController = AccessController || DefaultAccessController()
accessController = await AccessController({ orbitdb: { open, identity, ipfs }, identities })
const m = await DBManifest({ storage: manifestStorage, name: address, type, accessController: accessController.address, meta })
const m = await manifests.create({ name: address, type, accessController: accessController.address, meta })
manifest = m.manifest
address = OrbitDBAddress(m.hash)
// accessController = manifest.accessController
name = manifest.name
meta = manifest.meta
}
@ -108,11 +79,14 @@ const OrbitDB = async ({ ipfs, id, identity, keystore, directory } = {}) => {
if (!Database) {
throw new Error(`Unsupported database type: '${type}'`)
}
const db = await Database({ ipfs, identity, address: address.toString(), name, access: accessController, directory, meta, syncAutomatically: sync != null ? sync : true, headsStorage, entryStorage, indexStorage, referencesCount })
db.events.on('close', onDatabaseClosed(address.toString()))
address = address.toString()
databases[address.toString()] = db
const db = await Database({ ipfs, identity, address, name, access: accessController, directory, meta, syncAutomatically: sync, headsStorage, entryStorage, indexStorage, referencesCount })
db.events.on('close', onDatabaseClosed(address))
databases[address] = db
return db
}
@ -125,8 +99,8 @@ const OrbitDB = async ({ ipfs, id, identity, keystore, directory } = {}) => {
if (keystore) {
await keystore.close()
}
if (manifestStorage) {
await manifestStorage.close()
if (manifests) {
await manifests.close()
}
databases = {}
}
@ -143,418 +117,3 @@ const OrbitDB = async ({ ipfs, id, identity, keystore, directory } = {}) => {
}
export { OrbitDB as default, OrbitDBAddress, addDatabaseType, databaseTypes, AccessControllers }
// class OrbitDB2 {
// constructor (ipfs, identity, options = {}) {
// if (!isDefined(ipfs)) {
// throw new Error('IPFS is a required argument. See https://github.com/orbitdb/orbit-db/blob/master/API.md#createinstance')
// }
// if (!isDefined(identity)) {
// throw new Error('identity is a required argument. See https://github.com/orbitdb/orbit-db/blob/master/API.md#createinstance')
// }
// this._ipfs = ipfs
// this.identity = identity
// this.directory = options.directory || './orbitdb'
// this._directConnections = {}
// this.keystore = options.keystore
// this.stores = {}
// // AccessControllers module can be passed in to enable
// // testing with orbit-db-access-controller
// AccessControllers = options.AccessControllers || AccessControllers
// }
// static get KeyStore () { return KeyStore }
// static get Identities () { return Identities }
// static get AccessControllers () { return AccessControllers }
// static get OrbitDBAddress () { return OrbitDBAddress }
// static get Store () { return Store }
// static get EventStore () { return EventStore }
// static get FeedStore () { return FeedStore }
// static get KeyValueStore () { return KeyValueStore }
// static get CounterStore () { return CounterStore }
// static get DocumentStore () { return DocumentStore }
// static async createInstance (ipfs, options = {}) {
// if (!isDefined(ipfs)) { throw new Error('IPFS is a required argument. See https://github.com/orbitdb/orbit-db/blob/master/API.md#createinstance') }
// if (options.offline === undefined) {
// options.offline = false
// }
// if (options.offline && !options.id) {
// throw new Error('Offline mode requires passing an `id` in the options')
// }
// // TODO: don't use ipfs.id(), generate random id if one is not passed in with options
// const { id } = options.id || options.offline ? ({ id: options.id }) : await ipfs.id()
// if (!options.directory) { options.directory = './orbitdb' }
// if (options.identity) {
// // TODO: isIdentity(options.identity)
// options.keystore = options.identity.keystore
// }
// if (!options.keystore) {
// const keystorePath = path.join(options.directory, typeof id !== 'object' ? id : id.toString(), '/keystore')
// options.keystore = new KeyStore(keystorePath)
// }
// if (!options.identity) {
// options.identity = await Identities.createIdentity({
// id,
// keystore: options.keystore
// })
// }
// const finalOptions = Object.assign({}, options, { peerId: typeof id !== 'object' ? id : id.toString() })
// return new OrbitDB2(ipfs, options.identity, finalOptions)
// }
// /* Databases */
// async feed (address, options = {}) {
// options = Object.assign({ create: true, type: 'feed' }, options)
// return this.open(address, options)
// }
// async log (address, options = {}) {
// const accessController = {
// canAppend: async (entry) => {
// return true
// // const identity = await identities1.getIdentity(entry.identity)
// // return identity.id === testIdentity1.id
// }
// }
// const db = await EventStore({ OpLog, Database, ipfs: this._ipfs, identity: this.identity, databaseId: address, accessController })
// return db
// // options = Object.assign({ create: true, type: 'eventlog' }, options)
// // return this.open(address, options)
// }
// async eventlog (address, options = {}) {
// return this.log(address, options)
// }
// async keyvalue (address, options = {}) {
// options = Object.assign({ create: true, type: 'keyvalue' }, options)
// return this.open(address, options)
// }
// async kvstore (address, options = {}) {
// return this.keyvalue(address, options)
// }
// async counter (address, options = {}) {
// options = Object.assign({ create: true, type: 'counter' }, options)
// return this.open(address, options)
// }
// async docs (address, options = {}) {
// options = Object.assign({ create: true, type: 'docstore' }, options)
// return this.open(address, options)
// }
// async docstore (address, options = {}) {
// return this.docs(address, options)
// }
// /*
// options = {
// accessController: { write: [] } // array of keys that can write to this database
// overwrite: false, // whether we should overwrite the existing database if it exists
// }
// */
// async create (name, type, options = {}) {
// logger.debug(`Creating database '${name}' as ${type}`)
// // Create the database address
// const dbAddress = await this._determineAddress(name, type, options)
// // Check if we have the database locally
// const haveDB = await this._haveLocalData(options.cache, dbAddress)
// if (haveDB && !options.overwrite) { throw new Error(`Database '${dbAddress}' already exists!`) }
// // Save the database locally
// await this._addManifestToCache(options.cache, dbAddress)
// logger.debug(`Created database '${dbAddress}'`)
// // Open the database
// return this.open(dbAddress, options)
// }
// /*
// options = {
// localOnly: false // if set to true, throws an error if database can't be found locally
// create: false // whether to create the database
// type: TODO
// overwrite: TODO
// }
// */
// async open (address, options = {}) {
// logger.debug('open()')
// options = Object.assign({ localOnly: false, create: false }, options)
// logger.debug(`Open database '${address}'`)
// // If address is just the name of database, check the options to crate the database
// if (!OrbitDBAddress.isValid(address)) {
// if (!options.create) {
// throw new Error('\'options.create\' set to \'false\'. If you want to create a database, set \'options.create\' to \'true\'.')
// } else if (options.create && !options.type) {
// throw new Error(`Database type not provided! Provide a type with 'options.type' (${OrbitDB.databaseTypes.join('|')})`)
// } else {
// logger.warn(`Not a valid OrbitDB address '${address}', creating the database`)
// options.overwrite = options.overwrite ? options.overwrite : true
// return this.create(address, options.type, options)
// }
// }
// // Parse the database address
// const dbAddress = OrbitDBAddress.parse(address)
// // If database is already open, return early by returning the instance
// // if (this.stores[dbAddress]) {
// // return this.stores[dbAddress]
// // }
// // Check if we have the database
// const haveDB = await this._haveLocalData(options.cache, dbAddress)
// logger.debug((haveDB ? 'Found' : 'Didn\'t find') + ` database '${dbAddress}'`)
// // If we want to try and open the database local-only, throw an error
// // if we don't have the database locally
// if (options.localOnly && !haveDB) {
// logger.warn(`Database '${dbAddress}' doesn't exist!`)
// throw new Error(`Database '${dbAddress}' doesn't exist!`)
// }
// logger.debug(`Loading Manifest for '${dbAddress}'`)
// let manifest
// try {
// // Get the database manifest from IPFS
// manifest = await io.read(this._ipfs, dbAddress.root, { timeout: options.timeout || defaultTimeout })
// logger.debug(`Manifest for '${dbAddress}':\n${JSON.stringify(manifest, null, 2)}`)
// } catch (e) {
// if (e.name === 'TimeoutError' && e.code === 'ERR_TIMEOUT') {
// console.error(e)
// throw new Error('ipfs unable to find and fetch manifest for this address.')
// } else {
// throw e
// }
// }
// if (manifest.name !== dbAddress.path) {
// logger.warn(`Manifest name '${manifest.name}' and path name '${dbAddress.path}' do not match`)
// }
// // Make sure the type from the manifest matches the type that was given as an option
// if (options.type && manifest.type !== options.type) {
// throw new Error(`Database '${dbAddress}' is type '${manifest.type}' but was opened as '${options.type}'`)
// }
// // Save the database locally
// await this._addManifestToCache(options.cache, dbAddress)
// // Open the the database
// options = Object.assign({}, options, { accessControllerAddress: manifest.accessController, meta: manifest.meta })
// return this._createStore(options.type || manifest.type, dbAddress, options)
// }
// /* Private methods */
// async _createStore (type, address, options) {
// // Get the type -> class mapping
// const Store = databaseTypes[type]
// if (!Store) { throw new Error(`Invalid database type '${type}'`) }
// let accessController
// if (options.accessControllerAddress) {
// accessController = await AccessControllers.resolve(this, options.accessControllerAddress, options.accessController)
// }
// const opts = Object.assign(
// { replicate: true },
// options,
// { accessController }
// )
// const identity = options.identity || this.identity
// // TODO: await Database(...)
// const store = new Store(this._ipfs, identity, address, opts)
// // TODO: store.events.on('update', ...)
// store.events.on('update', this._onUpdate.bind(this))
// store.events.on('drop', this._onDrop.bind(this))
// store.events.on('close', this._onClose.bind(this))
// // ID of the store is the address as a string
// const addr = address.toString()
// this.stores[addr] = store
// return store
// }
// // Callback for local writes to the database. We the update to pubsub.
// _onUpdate (address, entry, heads) {
// // TODO
// }
// // Callback for receiving a message from the network
// async _onMessage (address, heads, peer) {
// const store = this.stores[address]
// try {
// logger.debug(`Received ${heads.length} heads for '${address}':\n`, JSON.stringify(heads.map(e => e.hash), null, 2))
// if (store && heads) {
// if (heads.length > 0) {
// await store.sync(heads)
// }
// store.events.emit('peer.exchanged', peer, address, heads)
// }
// } catch (e) {
// logger.error(e)
// }
// }
// // Callback for when a peer connected to a database
// async _onPeerConnected (address, peer) {
// logger.debug(`New peer '${peer}' connected to '${address}'`)
// const getStore = address => this.stores[address]
// const getDirectConnection = peer => this._directConnections[peer]
// const onChannelCreated = channel => { this._directConnections[channel._receiverID] = channel }
// const onMessage = (address, heads) => this._onMessage(address, heads, peer)
// await exchangeHeads(
// this._ipfs,
// address,
// peer,
// getStore,
// getDirectConnection,
// onMessage,
// onChannelCreated
// )
// if (getStore(address)) { getStore(address).events.emit('peer', peer) }
// }
// // Callback when database was closed
// async _onClose (db) {
// const address = db.address.toString()
// logger.debug(`Close ${address}`)
// delete this.stores[address]
// }
// async _onDrop (db) {
// }
// async _onLoad (db) {
// }
// async _determineAddress (name, type, options = {}) {
// if (!OrbitDB.isValidType(type)) { throw new Error(`Invalid database type '${type}'`) }
// if (OrbitDBAddress.isValid(name)) { throw new Error('Given database name is an address. Please give only the name of the database!') }
// // Create an AccessController, use IPFS AC as the default
// options.accessController = Object.assign({}, { name, type: 'ipfs' }, options.accessController)
// const accessControllerAddress = await AccessControllers.create(this, options.accessController.type, options.accessController || {})
// // Save the manifest to IPFS
// const manifestHash = await createDBManifest(this._ipfs, name, type, accessControllerAddress, options)
// // Create the database address
// return OrbitDBAddress.parse(OrbitDBAddress.join(manifestHash, name))
// }
// async determineAddress (name, type, options = {}) {
// const opts = Object.assign({}, { onlyHash: true }, options)
// return this._determineAddress(name, type, opts)
// }
// // Save the database locally
// async _addManifestToCache (cache, dbAddress) {
// await cache.set(path.join(dbAddress.toString(), '_manifest'), dbAddress.root)
// logger.debug(`Saved manifest to IPFS as '${dbAddress.root}'`)
// }
// /**
// * Check if we have the database, or part of it, saved locally
// * @param {[Cache]} cache [The OrbitDBCache instance containing the local data]
// * @param {[OrbitDBAddress]} dbAddress [Address of the database to check]
// * @return {[Boolean]} [Returns true if we have cached the db locally, false if not]
// */
// async _haveLocalData (cache, dbAddress) {
// if (!cache) {
// return false
// }
// const addr = dbAddress.toString()
// const data = await cache.get(path.join(addr, '_manifest'))
// return data !== undefined && data !== null
// }
// /**
// * Shutdown OrbitDB by closing network connections, the keystore
// * and databases opened by this OrbitDB instance.
// */
// async stop () {
// // Close a direct connection and remove it from internal state
// for (const connection of Object.values(this._directConnections)) {
// connection.close()
// }
// this._directConnections = {}
// // close keystore
// await this.keystore.close()
// // Close all open databases
// for (const db of Object.values(this.stores)) {
// await db.close()
// }
// this.stores = {}
// }
// /**
// * Returns supported database types as an Array of strings
// * Eg. [ 'counter', 'eventlog', 'feed', 'docstore', 'keyvalue']
// * @return {[Array]} [Supported database types]
// */
// static get databaseTypes () {
// return Object.keys(databaseTypes)
// }
// static isValidType (type) {
// return Object.keys(databaseTypes).includes(type)
// }
// static addDatabaseType (type, store) {
// if (databaseTypes[type]) throw new Error(`Type already exists: ${type}`)
// databaseTypes[type] = store
// }
// static getDatabaseTypes () {
// return databaseTypes
// }
// static isValidAddress (address) {
// return OrbitDBAddress.isValid(address)
// }
// static parseAddress (address) {
// return OrbitDBAddress.parse(address)
// }
// }
// // OrbitDB2.prototype.AccessControllers = AccessControllers
// // OrbitDB2.prototype.Identities = Identities
// // OrbitDB2.prototype.Keystore = Keystore

View File

@ -1,73 +1,76 @@
import { strictEqual, deepStrictEqual } from 'assert'
import rmrf from 'rimraf'
import * as IPFS from 'ipfs-core'
import Manifest from '../src/manifest.js'
import IPFSBlockStorage from '../src/storage/ipfs-block.js'
import Manifests from '../src/manifest.js'
import config from './config.js'
describe('Manifest', () => {
const repo = './ipfs'
let ipfs
let storage
let manifests
before(async () => {
ipfs = await IPFS.create({ ...config.daemon1, repo })
storage = await IPFSBlockStorage({ ipfs })
manifests = await Manifests({ ipfs })
})
after(async () => {
await storage.close()
await manifests.close()
await ipfs.stop()
await rmrf(repo)
})
it('creates a manifest', async () => {
const name = 'manifest'
const type = 'manifest'
const name = 'database'
const type = 'keyvalue'
const accessController = 'test/default-access-controller'
const expectedHash = 'zdpuAx3LaygjPHa2zsUmRoR4jQPm2WYrExsvz2gncfm62aRKv'
const expectedHash = 'zdpuAn26ookFToGNmpVHgEM71YMULiyS8mAs9UQtV1g6eEyRP'
const expectedManifest = {
name,
type,
accessController
}
const { hash, manifest } = await Manifest({ storage, name, type, accessController })
const { hash, manifest } = await manifests.create({ name, type, accessController })
strictEqual(hash, expectedHash)
deepStrictEqual(manifest, expectedManifest)
})
it('creates a manifest with metadata', async () => {
const name = 'manifest'
const type = 'manifest'
it('loads a manifest', async () => {
const name = 'database'
const type = 'keyvalue'
const accessController = 'test/default-access-controller'
const expectedHash = 'zdpuAmegd2PpDfTQRVhGiATCkWQDvp3JygT9WksWgJkG2u313'
const expectedHash = 'zdpuAn26ookFToGNmpVHgEM71YMULiyS8mAs9UQtV1g6eEyRP'
const expectedManifest = {
name,
type,
accessController
}
const manifest = await manifests.get(expectedHash)
deepStrictEqual(manifest, expectedManifest)
})
it('creates a manifest with metadata', async () => {
const name = 'database'
const type = 'keyvalue'
const accessController = 'test/default-access-controller'
const expectedHash = 'zdpuAyWPs4yAXS6W7CY4UM68pV2NCpzAJr98aMA4zS5XRq5ga'
const meta = { name, description: 'more information about the database' }
const { hash, manifest } = await Manifest({ storage, name, type, accessController, meta })
const { hash, manifest } = await manifests.create({ name, type, accessController, meta })
strictEqual(hash, expectedHash)
deepStrictEqual(manifest.meta, meta)
})
it('throws an error if storage is not specified', async () => {
let err
try {
await Manifest({})
} catch (e) {
err = e.toString()
}
strictEqual(err, 'Error: storage is required')
})
it('throws an error if name is not specified', async () => {
let err
try {
await Manifest({ storage })
await manifests.create({})
} catch (e) {
err = e.toString()
}
@ -79,7 +82,7 @@ describe('Manifest', () => {
let err
try {
await Manifest({ storage, name: 'manifest' })
await manifests.create({ name: 'database' })
} catch (e) {
err = e.toString()
}
@ -91,7 +94,7 @@ describe('Manifest', () => {
let err
try {
await Manifest({ storage, name: 'manifest', type: 'manifest' })
await manifests.create({ name: 'database', type: 'keyvalue' })
} catch (e) {
err = e.toString()
}