diff --git a/config/storage/key-value/memory.json b/config/storage/key-value/memory.json index 92409512f..571b73b3c 100644 --- a/config/storage/key-value/memory.json +++ b/config/storage/key-value/memory.json @@ -1,43 +1,18 @@ { "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^3.0.0/components/context.jsonld", + "import": [ + "files-scs:config/storage/key-value/storages/storages.json" + ], "@graph": [ { - "comment": "These storage solutions store their data in memory." - }, - { - "comment": "Used for internal storage by the locker.", - "@id": "urn:solid-server:default:LockStorage", + "comment": "Internal value storage. No different from urn:solid-server:default:KeyValueStorage, but required to be consistent with other storage solutions.", + "@id": "urn:solid-server:default:BackendKeyValueStorage", "@type": "MemoryMapStorage" }, { - "comment": "Storage used by the IDP adapter.", - "@id": "urn:solid-server:default:IdpAdapterStorage", + "comment": "Internal value storage.", + "@id": "urn:solid-server:default:KeyValueStorage", "@type": "MemoryMapStorage" - }, - { - "comment": "Storage used for the IDP keys.", - "@id": "urn:solid-server:default:IdpKeyStorage", - "@type": "MemoryMapStorage" - }, - { - "comment": "Storage used for IDP ownership tokens.", - "@id": "urn:solid-server:default:IdpTokenStorage", - "@type": "MemoryMapStorage" - }, - { - "comment": "Storage used for account management.", - "@id": "urn:solid-server:default:AccountStorage", - "@type": "MemoryMapStorage" - }, - { - "comment": "Storage used by setup components.", - "@id": "urn:solid-server:default:SetupStorage", - "@type": "MemoryMapStorage" - }, - { - "comment": "Storage used for ForgotPassword records", - "@id": "urn:solid-server:default:ForgotPasswordStorage", - "@type":"MemoryMapStorage" } ] } diff --git a/config/storage/key-value/resource-store.json b/config/storage/key-value/resource-store.json index 58f5c711c..064bb7a4a 100644 --- a/config/storage/key-value/resource-store.json +++ b/config/storage/key-value/resource-store.json @@ -1,67 +1,24 @@ { "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^3.0.0/components/context.jsonld", + "import": [ + "files-scs:config/storage/key-value/storages/storages.json" + ], "@graph": [ { - "comment": "These storage solutions use the specified container in the ResourceStore to store their data." - }, - { - "comment": [ - "This is the internal storage for the locker, which maintains what resources are in use.", - "It writes directly to a low-level store, because higher-level storage typically already uses the locker and would thus cause a loop." - ], - "@id": "urn:solid-server:default:LockStorage", + "comment": "A storage that writes directly to a low-level store. This is necessary to prevent infinite loops with stores that also use storage.", + "@id": "urn:solid-server:default:BackendKeyValueStorage", "@type": "JsonResourceStorage", "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/locks/" + "container": "/.internal/" }, { - "comment": "Storage used by the IDP adapter.", - "@id": "urn:solid-server:default:IdpAdapterStorage", + "comment": "Internal value storage.", + "@id": "urn:solid-server:default:KeyValueStorage", "@type": "JsonResourceStorage", "source": { "@id": "urn:solid-server:default:ResourceStore" }, "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/idp/adapter/" - }, - { - "comment": "Storage used for the IDP keys.", - "@id": "urn:solid-server:default:IdpKeyStorage", - "@type": "JsonResourceStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/idp/keys/" - }, - { - "comment": "Storage used for IDP ownership tokens.", - "@id": "urn:solid-server:default:IdpTokenStorage", - "@type": "JsonResourceStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/idp/tokens/" - }, - { - "comment": "Storage used for account management.", - "@id": "urn:solid-server:default:AccountStorage", - "@type": "JsonResourceStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/accounts/" - }, - { - "comment": "Storage used for ForgotPassword records", - "@id": "urn:solid-server:default:ForgotPasswordStorage", - "@type":"JsonResourceStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/forgot-password/" - }, - { - "comment": "Storage used by setup components.", - "@id": "urn:solid-server:default:SetupStorage", - "@type": "JsonResourceStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/setup/" + "container": "/.internal/" }, { "comment": "Block external access to the storage containers to avoid exposing internal data.", diff --git a/config/storage/key-value/storages/storages.json b/config/storage/key-value/storages/storages.json new file mode 100644 index 000000000..e96d52531 --- /dev/null +++ b/config/storage/key-value/storages/storages.json @@ -0,0 +1,54 @@ +{ + "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^3.0.0/components/context.jsonld", + "@graph": [ + { + "comment": "Used for internal storage by the locker.", + "@id": "urn:solid-server:default:LockStorage", + "@type": "EncodingPathStorage", + "relativePath": "/locks/", + "source": { "@id": "urn:solid-server:default:BackendKeyValueStorage" } + }, + { + "comment": "Storage used by the IDP adapter.", + "@id": "urn:solid-server:default:IdpAdapterStorage", + "@type": "EncodingPathStorage", + "relativePath": "/idp/adapter/", + "source": { "@id": "urn:solid-server:default:KeyValueStorage" } + }, + { + "comment": "Storage used for the IDP keys.", + "@id": "urn:solid-server:default:IdpKeyStorage", + "@type": "EncodingPathStorage", + "relativePath": "/idp/keys/", + "source": { "@id": "urn:solid-server:default:KeyValueStorage" } + }, + { + "comment": "Storage used for IDP ownership tokens.", + "@id": "urn:solid-server:default:IdpTokenStorage", + "@type": "EncodingPathStorage", + "relativePath": "/idp/tokens/", + "source": { "@id": "urn:solid-server:default:KeyValueStorage" } + }, + { + "comment": "Storage used for account management.", + "@id": "urn:solid-server:default:AccountStorage", + "@type": "EncodingPathStorage", + "relativePath": "/idp/accounts/", + "source": { "@id": "urn:solid-server:default:KeyValueStorage" } + }, + { + "comment": "Storage used for ForgotPassword records", + "@id": "urn:solid-server:default:ForgotPasswordStorage", + "@type": "EncodingPathStorage", + "relativePath": "/forgot-password/", + "source": { "@id": "urn:solid-server:default:KeyValueStorage" } + }, + { + "comment": "Storage used by setup components.", + "@id": "urn:solid-server:default:SetupStorage", + "@type": "EncodingPathStorage", + "relativePath": "/setup/", + "source": { "@id": "urn:solid-server:default:KeyValueStorage" } + } + ] +} diff --git a/src/index.ts b/src/index.ts index 542cf9a77..a7081bb40 100644 --- a/src/index.ts +++ b/src/index.ts @@ -304,6 +304,7 @@ export * from './storage/conversion/RepresentationConverter'; export * from './storage/conversion/TypedRepresentationConverter'; // Storage/KeyValue +export * from './storage/keyvalue/EncodingPathStorage'; export * from './storage/keyvalue/ExpiringStorage'; export * from './storage/keyvalue/JsonFileStorage'; export * from './storage/keyvalue/JsonResourceStorage'; diff --git a/src/storage/keyvalue/EncodingPathStorage.ts b/src/storage/keyvalue/EncodingPathStorage.ts new file mode 100644 index 000000000..bc29f4e8d --- /dev/null +++ b/src/storage/keyvalue/EncodingPathStorage.ts @@ -0,0 +1,64 @@ +import { ensureTrailingSlash, joinUrl } from '../../util/PathUtil'; +import type { KeyValueStorage } from './KeyValueStorage'; + +/** + * Transforms the keys into relative paths, to be used by the source storage. + * Encodes the input key with base64 encoding, + * to make sure there are no invalid or special path characters, + * and prepends it with the stored relative path. + * This can be useful to eventually generate URLs in specific containers + * without having to worry about cleaning the input keys. + */ +export class EncodingPathStorage implements KeyValueStorage { + private readonly basePath: string; + private readonly source: KeyValueStorage; + + public constructor(relativePath: string, source: KeyValueStorage) { + this.source = source; + this.basePath = ensureTrailingSlash(relativePath); + } + + public async get(key: string): Promise { + const path = this.keyToPath(key); + return this.source.get(path); + } + + public async has(key: string): Promise { + const path = this.keyToPath(key); + return this.source.has(path); + } + + public async set(key: string, value: T): Promise { + const path = this.keyToPath(key); + await this.source.set(path, value); + return this; + } + + public async delete(key: string): Promise { + const path = this.keyToPath(key); + return this.source.delete(path); + } + + public async* entries(): AsyncIterableIterator<[string, T]> { + for await (const [ path, value ] of this.source.entries()) { + const key = this.pathToKey(path); + yield [ key, value ]; + } + } + + /** + * Converts a key into a path for internal storage. + */ + private keyToPath(key: string): string { + const encodedKey = Buffer.from(key).toString('base64'); + return joinUrl(this.basePath, encodedKey); + } + + /** + * Converts an internal storage path string into the original path key. + */ + private pathToKey(path: string): string { + const buffer = Buffer.from(path.slice(this.basePath.length), 'base64'); + return buffer.toString('utf-8'); + } +} diff --git a/src/storage/keyvalue/JsonResourceStorage.ts b/src/storage/keyvalue/JsonResourceStorage.ts index 8e7ea7bf1..c8d747d35 100644 --- a/src/storage/keyvalue/JsonResourceStorage.ts +++ b/src/storage/keyvalue/JsonResourceStorage.ts @@ -1,34 +1,32 @@ -import { URL } from 'url'; import { BasicRepresentation } from '../../http/representation/BasicRepresentation'; -import type { Representation } from '../../http/representation/Representation'; import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier'; import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError'; -import { ensureTrailingSlash } from '../../util/PathUtil'; +import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError'; +import { ensureTrailingSlash, joinUrl } from '../../util/PathUtil'; import { readableToString } from '../../util/StreamUtil'; -import { LDP } from '../../util/Vocabularies'; import type { ResourceStore } from '../ResourceStore'; import type { KeyValueStorage } from './KeyValueStorage'; /** * A {@link KeyValueStorage} for JSON-like objects using a {@link ResourceStore} as backend. * - * The keys will be transformed so they can be safely used - * as a resource name in the given container. - * Values will be sent as data streams, - * so how these are stored depends on the underlying store. + * Creates a base URL by joining the input base URL with the container string. + * + * Assumes the input keys can be safely used to generate identifiers, + * which will be appended to the stored base URL. * * All non-404 errors will be re-thrown. */ -export class JsonResourceStorage implements KeyValueStorage { +export class JsonResourceStorage implements KeyValueStorage { private readonly source: ResourceStore; private readonly container: string; public constructor(source: ResourceStore, baseUrl: string, container: string) { this.source = source; - this.container = ensureTrailingSlash(new URL(container, baseUrl).href); + this.container = ensureTrailingSlash(joinUrl(baseUrl, container)); } - public async get(key: string): Promise { + public async get(key: string): Promise { try { const identifier = this.createIdentifier(key); const representation = await this.source.getRepresentation(identifier, { type: { 'application/json': 1 }}); @@ -65,42 +63,15 @@ export class JsonResourceStorage implements KeyValueStorage { } } - public async* entries(): AsyncIterableIterator<[string, unknown]> { - // Getting ldp:contains metadata from container to find entries - let container: Representation; - try { - container = await this.source.getRepresentation({ path: this.container }, {}); - } catch (error: unknown) { - // Container might not exist yet, will be created the first time `set` gets called - if (!NotFoundHttpError.isInstance(error)) { - throw error; - } - return; - } - - // Only need the metadata - container.data.destroy(); - const members = container.metadata.getAll(LDP.terms.contains).map((term): string => term.value); - for (const member of members) { - const representation = await this.source.getRepresentation({ path: member }, { type: { 'application/json': 1 }}); - const json = JSON.parse(await readableToString(representation.data)); - yield [ this.parseMember(member), json ]; - } + public entries(): never { + // There is no way of knowing which resources were added, or we should keep track in an index file + throw new NotImplementedHttpError(); } /** * Converts a key into an identifier for internal storage. */ private createIdentifier(key: string): ResourceIdentifier { - const buffer = Buffer.from(key); - return { path: `${this.container}${buffer.toString('base64')}` }; - } - - /** - * Converts an internal storage identifier string into the original identifier key. - */ - private parseMember(member: string): string { - const buffer = Buffer.from(member.slice(this.container.length), 'base64'); - return buffer.toString('utf-8'); + return { path: joinUrl(this.container, key) }; } } diff --git a/test/unit/storage/keyvalue/EncodingPathStorage.test.ts b/test/unit/storage/keyvalue/EncodingPathStorage.test.ts new file mode 100644 index 000000000..7ab472206 --- /dev/null +++ b/test/unit/storage/keyvalue/EncodingPathStorage.test.ts @@ -0,0 +1,33 @@ +import { EncodingPathStorage } from '../../../../src/storage/keyvalue/EncodingPathStorage'; +import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage'; + +describe('An EncodingPathStorage', (): void => { + const relativePath = '/container/'; + let map: Map; + let source: KeyValueStorage; + let storage: EncodingPathStorage; + + beforeEach(async(): Promise => { + map = new Map(); + source = map as any; + storage = new EncodingPathStorage(relativePath, source); + }); + + it('encodes the input key and joins it with the relativePath to create a new key.', async(): Promise => { + const key = 'key'; + // Base 64 encoding of 'key' + const encodedKey = 'a2V5'; + const generatedPath = `${relativePath}${encodedKey}`; + const data = 'data'; + + await expect(storage.set(key, data)).resolves.toBe(storage); + expect(map.get(generatedPath)).toBe(data); + + await expect(storage.has(key)).resolves.toBe(true); + await expect(storage.get(key)).resolves.toBe(data); + await expect(storage.entries().next()).resolves.toEqual({ done: false, value: [ key, data ]}); + + await expect(storage.delete(key)).resolves.toBe(true); + expect([ ...map.keys() ]).toHaveLength(0); + }); +}); diff --git a/test/unit/storage/keyvalue/JsonResourceStorage.test.ts b/test/unit/storage/keyvalue/JsonResourceStorage.test.ts index 6fd7cede7..d6687027d 100644 --- a/test/unit/storage/keyvalue/JsonResourceStorage.test.ts +++ b/test/unit/storage/keyvalue/JsonResourceStorage.test.ts @@ -5,6 +5,7 @@ import type { ResourceIdentifier } from '../../../../src/http/representation/Res import { JsonResourceStorage } from '../../../../src/storage/keyvalue/JsonResourceStorage'; import type { ResourceStore } from '../../../../src/storage/ResourceStore'; import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError'; +import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplementedHttpError'; import { readableToString } from '../../../../src/util/StreamUtil'; import { LDP } from '../../../../src/util/Vocabularies'; @@ -14,7 +15,7 @@ describe('A JsonResourceStorage', (): void => { const identifier1 = 'http://test.com/foo'; const identifier2 = 'http://test.com/bar'; let store: ResourceStore; - let storage: JsonResourceStorage; + let storage: JsonResourceStorage; beforeEach(async(): Promise => { const data: Record = { }; @@ -52,14 +53,13 @@ describe('A JsonResourceStorage', (): void => { await expect(storage.get(identifier1)).resolves.toBeUndefined(); }); - it('returns no entry data if the container does not exist yet.', async(): Promise => { - await expect(storage.entries().next()).resolves.toEqual({ done: true }); + it('errors when trying to request entries.', async(): Promise => { + expect((): never => storage.entries()).toThrow(NotImplementedHttpError); }); it('returns data if it was set beforehand.', async(): Promise => { await expect(storage.set(identifier1, 'apple')).resolves.toBe(storage); await expect(storage.get(identifier1)).resolves.toBe('apple'); - await expect(storage.entries().next()).resolves.toEqual({ done: false, value: [ identifier1, 'apple' ]}); }); it('can check if data is present.', async(): Promise => { @@ -89,7 +89,6 @@ describe('A JsonResourceStorage', (): void => { it('re-throws errors thrown by the store.', async(): Promise => { store.getRepresentation = jest.fn().mockRejectedValue(new Error('bad GET')); await expect(storage.get(identifier1)).rejects.toThrow('bad GET'); - await expect(storage.entries().next()).rejects.toThrow('bad GET'); store.deleteResource = jest.fn().mockRejectedValueOnce(new Error('bad DELETE')); await expect(storage.delete(identifier1)).rejects.toThrow('bad DELETE');