feat: Create separate storage to generate keys

config/storage/key-value/memory.json

@@ -1,43 +1,18 @@
 {
   "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^3.0.0/components/context.jsonld",
+  "import": [
+    "files-scs:config/storage/key-value/storages/storages.json"
+  ],
   "@graph": [
     {
-      "comment": "These storage solutions store their data in memory."
-    },
-    {
-      "comment": "Used for internal storage by the locker.",
-      "@id": "urn:solid-server:default:LockStorage",
+      "comment": "Internal value storage. No different from urn:solid-server:default:KeyValueStorage, but required to be consistent with other storage solutions.",
+      "@id": "urn:solid-server:default:BackendKeyValueStorage",
       "@type": "MemoryMapStorage"
     },
     {
-      "comment": "Storage used by the IDP adapter.",
-      "@id": "urn:solid-server:default:IdpAdapterStorage",
+      "comment": "Internal value storage.",
+      "@id": "urn:solid-server:default:KeyValueStorage",
       "@type": "MemoryMapStorage"
-    },
-    {
-      "comment": "Storage used for the IDP keys.",
-      "@id": "urn:solid-server:default:IdpKeyStorage",
-      "@type": "MemoryMapStorage"
-    },
-    {
-      "comment": "Storage used for IDP ownership tokens.",
-      "@id": "urn:solid-server:default:IdpTokenStorage",
-      "@type": "MemoryMapStorage"
-    },
-    {
-      "comment": "Storage used for account management.",
-      "@id": "urn:solid-server:default:AccountStorage",
-      "@type": "MemoryMapStorage"
-    },
-    {
-      "comment": "Storage used by setup components.",
-      "@id": "urn:solid-server:default:SetupStorage",
-      "@type": "MemoryMapStorage"
-    },
-    {
-      "comment": "Storage used for ForgotPassword records",
-      "@id": "urn:solid-server:default:ForgotPasswordStorage",
-      "@type":"MemoryMapStorage"
     }
   ]
 }

config/storage/key-value/resource-store.json

@@ -1,67 +1,24 @@
 {
   "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^3.0.0/components/context.jsonld",
+  "import": [
+    "files-scs:config/storage/key-value/storages/storages.json"
+  ],
   "@graph": [
     {
-      "comment": "These storage solutions use the specified container in the ResourceStore to store their data."
-    },
-    {
-      "comment": [
-        "This is the internal storage for the locker, which maintains what resources are in use.",
-        "It writes directly to a low-level store, because higher-level storage typically already uses the locker and would thus cause a loop."
-      ],
-      "@id": "urn:solid-server:default:LockStorage",
+      "comment": "A storage that writes directly to a low-level store. This is necessary to prevent infinite loops with stores that also use storage.",
+      "@id": "urn:solid-server:default:BackendKeyValueStorage",
       "@type": "JsonResourceStorage",
       "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
       "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/locks/"
+      "container": "/.internal/"
     },
     {
-      "comment": "Storage used by the IDP adapter.",
-      "@id": "urn:solid-server:default:IdpAdapterStorage",
+      "comment": "Internal value storage.",
+      "@id": "urn:solid-server:default:KeyValueStorage",
       "@type": "JsonResourceStorage",
       "source": { "@id": "urn:solid-server:default:ResourceStore" },
       "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/idp/adapter/"
-    },
-    {
-      "comment": "Storage used for the IDP keys.",
-      "@id": "urn:solid-server:default:IdpKeyStorage",
-      "@type": "JsonResourceStorage",
-      "source": { "@id": "urn:solid-server:default:ResourceStore" },
-      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/idp/keys/"
-    },
-    {
-      "comment": "Storage used for IDP ownership tokens.",
-      "@id": "urn:solid-server:default:IdpTokenStorage",
-      "@type": "JsonResourceStorage",
-      "source": { "@id": "urn:solid-server:default:ResourceStore" },
-      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/idp/tokens/"
-    },
-    {
-      "comment": "Storage used for account management.",
-      "@id": "urn:solid-server:default:AccountStorage",
-      "@type": "JsonResourceStorage",
-      "source": { "@id": "urn:solid-server:default:ResourceStore" },
-      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/accounts/"
-    },
-    {
-      "comment": "Storage used for ForgotPassword records",
-      "@id": "urn:solid-server:default:ForgotPasswordStorage",
-      "@type":"JsonResourceStorage",
-      "source": { "@id": "urn:solid-server:default:ResourceStore" },
-      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/forgot-password/"
-    },
-    {
-      "comment": "Storage used by setup components.",
-      "@id": "urn:solid-server:default:SetupStorage",
-      "@type": "JsonResourceStorage",
-      "source": { "@id": "urn:solid-server:default:ResourceStore" },
-      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-      "container": "/.internal/setup/"
+      "container": "/.internal/"
     },
     {
       "comment": "Block external access to the storage containers to avoid exposing internal data.",

config/storage/key-value/storages/storages.json

@@ -0,0 +1,54 @@
+{
+  "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^3.0.0/components/context.jsonld",
+  "@graph": [
+    {
+      "comment": "Used for internal storage by the locker.",
+      "@id": "urn:solid-server:default:LockStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/locks/",
+      "source": { "@id": "urn:solid-server:default:BackendKeyValueStorage" }
+    },
+    {
+      "comment": "Storage used by the IDP adapter.",
+      "@id": "urn:solid-server:default:IdpAdapterStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/idp/adapter/",
+      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+    },
+    {
+      "comment": "Storage used for the IDP keys.",
+      "@id": "urn:solid-server:default:IdpKeyStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/idp/keys/",
+      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+    },
+    {
+      "comment": "Storage used for IDP ownership tokens.",
+      "@id": "urn:solid-server:default:IdpTokenStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/idp/tokens/",
+      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+    },
+    {
+      "comment": "Storage used for account management.",
+      "@id": "urn:solid-server:default:AccountStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/idp/accounts/",
+      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+    },
+    {
+      "comment": "Storage used for ForgotPassword records",
+      "@id": "urn:solid-server:default:ForgotPasswordStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/forgot-password/",
+      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+    },
+    {
+      "comment": "Storage used by setup components.",
+      "@id": "urn:solid-server:default:SetupStorage",
+      "@type": "EncodingPathStorage",
+      "relativePath": "/setup/",
+      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+    }
+  ]
+}

src/index.ts

@@ -304,6 +304,7 @@ export * from './storage/conversion/RepresentationConverter';
 export * from './storage/conversion/TypedRepresentationConverter';
 
 // Storage/KeyValue
+export * from './storage/keyvalue/EncodingPathStorage';
 export * from './storage/keyvalue/ExpiringStorage';
 export * from './storage/keyvalue/JsonFileStorage';
 export * from './storage/keyvalue/JsonResourceStorage';

src/storage/keyvalue/EncodingPathStorage.ts

@@ -0,0 +1,64 @@
+import { ensureTrailingSlash, joinUrl } from '../../util/PathUtil';
+import type { KeyValueStorage } from './KeyValueStorage';
+
+/**
+ * Transforms the keys into relative paths, to be used by the source storage.
+ * Encodes the input key with base64 encoding,
+ * to make sure there are no invalid or special path characters,
+ * and prepends it with the stored relative path.
+ * This can be useful to eventually generate URLs in specific containers
+ * without having to worry about cleaning the input keys.
+ */
+export class EncodingPathStorage<T> implements KeyValueStorage<string, T> {
+  private readonly basePath: string;
+  private readonly source: KeyValueStorage<string, T>;
+
+  public constructor(relativePath: string, source: KeyValueStorage<string, T>) {
+    this.source = source;
+    this.basePath = ensureTrailingSlash(relativePath);
+  }
+
+  public async get(key: string): Promise<T | undefined> {
+    const path = this.keyToPath(key);
+    return this.source.get(path);
+  }
+
+  public async has(key: string): Promise<boolean> {
+    const path = this.keyToPath(key);
+    return this.source.has(path);
+  }
+
+  public async set(key: string, value: T): Promise<this> {
+    const path = this.keyToPath(key);
+    await this.source.set(path, value);
+    return this;
+  }
+
+  public async delete(key: string): Promise<boolean> {
+    const path = this.keyToPath(key);
+    return this.source.delete(path);
+  }
+
+  public async* entries(): AsyncIterableIterator<[string, T]> {
+    for await (const [ path, value ] of this.source.entries()) {
+      const key = this.pathToKey(path);
+      yield [ key, value ];
+    }
+  }
+
+  /**
+   * Converts a key into a path for internal storage.
+   */
+  private keyToPath(key: string): string {
+    const encodedKey = Buffer.from(key).toString('base64');
+    return joinUrl(this.basePath, encodedKey);
+  }
+
+  /**
+   * Converts an internal storage path string into the original path key.
+   */
+  private pathToKey(path: string): string {
+    const buffer = Buffer.from(path.slice(this.basePath.length), 'base64');
+    return buffer.toString('utf-8');
+  }
+}

src/storage/keyvalue/JsonResourceStorage.ts

@@ -1,34 +1,32 @@
-import { URL } from 'url';
 import { BasicRepresentation } from '../../http/representation/BasicRepresentation';
-import type { Representation } from '../../http/representation/Representation';
 import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier';
 import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError';
-import { ensureTrailingSlash } from '../../util/PathUtil';
+import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError';
+import { ensureTrailingSlash, joinUrl } from '../../util/PathUtil';
 import { readableToString } from '../../util/StreamUtil';
-import { LDP } from '../../util/Vocabularies';
 import type { ResourceStore } from '../ResourceStore';
 import type { KeyValueStorage } from './KeyValueStorage';
 
 /**
  * A {@link KeyValueStorage} for JSON-like objects using a {@link ResourceStore} as backend.
  *
- * The keys will be transformed so they can be safely used
- * as a resource name in the given container.
- * Values will be sent as data streams,
- * so how these are stored depends on the underlying store.
+ * Creates a base URL by joining the input base URL with the container string.
+ *
+ * Assumes the input keys can be safely used to generate identifiers,
+ * which will be appended to the stored base URL.
  *
  * All non-404 errors will be re-thrown.
  */
-export class JsonResourceStorage implements KeyValueStorage<string, unknown> {
+export class JsonResourceStorage<T> implements KeyValueStorage<string, T> {
   private readonly source: ResourceStore;
   private readonly container: string;
 
   public constructor(source: ResourceStore, baseUrl: string, container: string) {
     this.source = source;
-    this.container = ensureTrailingSlash(new URL(container, baseUrl).href);
+    this.container = ensureTrailingSlash(joinUrl(baseUrl, container));
   }
 
-  public async get(key: string): Promise<unknown | undefined> {
+  public async get(key: string): Promise<T | undefined> {
     try {
       const identifier = this.createIdentifier(key);
       const representation = await this.source.getRepresentation(identifier, { type: { 'application/json': 1 }});
@@ -65,42 +63,15 @@ export class JsonResourceStorage implements KeyValueStorage<string, unknown> {
     }
   }
 
-  public async* entries(): AsyncIterableIterator<[string, unknown]> {
-    // Getting ldp:contains metadata from container to find entries
-    let container: Representation;
-    try {
-      container = await this.source.getRepresentation({ path: this.container }, {});
-    } catch (error: unknown) {
-      // Container might not exist yet, will be created the first time `set` gets called
-      if (!NotFoundHttpError.isInstance(error)) {
-        throw error;
-      }
-      return;
-    }
-
-    // Only need the metadata
-    container.data.destroy();
-    const members = container.metadata.getAll(LDP.terms.contains).map((term): string => term.value);
-    for (const member of members) {
-      const representation = await this.source.getRepresentation({ path: member }, { type: { 'application/json': 1 }});
-      const json = JSON.parse(await readableToString(representation.data));
-      yield [ this.parseMember(member), json ];
-    }
+  public entries(): never {
+    // There is no way of knowing which resources were added, or we should keep track in an index file
+    throw new NotImplementedHttpError();
   }
 
   /**
    * Converts a key into an identifier for internal storage.
    */
   private createIdentifier(key: string): ResourceIdentifier {
-    const buffer = Buffer.from(key);
-    return { path: `${this.container}${buffer.toString('base64')}` };
-  }
-
-  /**
-   * Converts an internal storage identifier string into the original identifier key.
-   */
-  private parseMember(member: string): string {
-    const buffer = Buffer.from(member.slice(this.container.length), 'base64');
-    return buffer.toString('utf-8');
+    return { path: joinUrl(this.container, key) };
   }
 }

test/unit/storage/keyvalue/EncodingPathStorage.test.ts

@@ -0,0 +1,33 @@
+import { EncodingPathStorage } from '../../../../src/storage/keyvalue/EncodingPathStorage';
+import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage';
+
+describe('An EncodingPathStorage', (): void => {
+  const relativePath = '/container/';
+  let map: Map<string, string>;
+  let source: KeyValueStorage<string, unknown>;
+  let storage: EncodingPathStorage<unknown>;
+
+  beforeEach(async(): Promise<void> => {
+    map = new Map<string, string>();
+    source = map as any;
+    storage = new EncodingPathStorage(relativePath, source);
+  });
+
+  it('encodes the input key and joins it with the relativePath to create a new key.', async(): Promise<void> => {
+    const key = 'key';
+    // Base 64 encoding of 'key'
+    const encodedKey = 'a2V5';
+    const generatedPath = `${relativePath}${encodedKey}`;
+    const data = 'data';
+
+    await expect(storage.set(key, data)).resolves.toBe(storage);
+    expect(map.get(generatedPath)).toBe(data);
+
+    await expect(storage.has(key)).resolves.toBe(true);
+    await expect(storage.get(key)).resolves.toBe(data);
+    await expect(storage.entries().next()).resolves.toEqual({ done: false, value: [ key, data ]});
+
+    await expect(storage.delete(key)).resolves.toBe(true);
+    expect([ ...map.keys() ]).toHaveLength(0);
+  });
+});

test/unit/storage/keyvalue/JsonResourceStorage.test.ts

@@ -5,6 +5,7 @@ import type { ResourceIdentifier } from '../../../../src/http/representation/Res
 import { JsonResourceStorage } from '../../../../src/storage/keyvalue/JsonResourceStorage';
 import type { ResourceStore } from '../../../../src/storage/ResourceStore';
 import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError';
+import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplementedHttpError';
 import { readableToString } from '../../../../src/util/StreamUtil';
 import { LDP } from '../../../../src/util/Vocabularies';
 
@@ -14,7 +15,7 @@ describe('A JsonResourceStorage', (): void => {
   const identifier1 = 'http://test.com/foo';
   const identifier2 = 'http://test.com/bar';
   let store: ResourceStore;
-  let storage: JsonResourceStorage;
+  let storage: JsonResourceStorage<unknown>;
 
   beforeEach(async(): Promise<void> => {
     const data: Record<string, string> = { };
@@ -52,14 +53,13 @@ describe('A JsonResourceStorage', (): void => {
     await expect(storage.get(identifier1)).resolves.toBeUndefined();
   });
 
-  it('returns no entry data if the container does not exist yet.', async(): Promise<void> => {
-    await expect(storage.entries().next()).resolves.toEqual({ done: true });
+  it('errors when trying to request entries.', async(): Promise<void> => {
+    expect((): never => storage.entries()).toThrow(NotImplementedHttpError);
   });
 
   it('returns data if it was set beforehand.', async(): Promise<void> => {
     await expect(storage.set(identifier1, 'apple')).resolves.toBe(storage);
     await expect(storage.get(identifier1)).resolves.toBe('apple');
-    await expect(storage.entries().next()).resolves.toEqual({ done: false, value: [ identifier1, 'apple' ]});
   });
 
   it('can check if data is present.', async(): Promise<void> => {
@@ -89,7 +89,6 @@ describe('A JsonResourceStorage', (): void => {
   it('re-throws errors thrown by the store.', async(): Promise<void> => {
     store.getRepresentation = jest.fn().mockRejectedValue(new Error('bad GET'));
     await expect(storage.get(identifier1)).rejects.toThrow('bad GET');
-    await expect(storage.entries().next()).rejects.toThrow('bad GET');
 
     store.deleteResource = jest.fn().mockRejectedValueOnce(new Error('bad DELETE'));
     await expect(storage.delete(identifier1)).rejects.toThrow('bad DELETE');