feat: Split up EncodingPathStorage functionality into different classes

RELEASE_NOTES.md

@@ -23,15 +23,23 @@ The following changes pertain to the imports in the default configs:
 
 - There is a new `static-root.json` import option for `app/init`, setting a static page for the root container.
 
-The following changes are relevant for v5 custom configs that replaced certain features.
+The following changes are relevant for v6 custom configs that replaced certain features.
 
 - `/app/init/*` imports have changed. Functionality remained the same though.
+- All imports that define storages have been updated with new storage classes.
+    - `/http/notifications/base/storage.json`
+    - `/identity/*`
+    - `/storage/keyvalue/storages/storages.json`
 
 ### Interface changes
 
 These changes are relevant if you wrote custom modules for the server that depend on existing interfaces.
 
 - The `AppRunner` functions to create and start the server now take a singular arguments object as input.
+- Most of the key/value related storages had their constructors changed to allow more values.
+- `EncodingPathStorage` has been removed
+  and its functionality split up over `Base64EncodingStorage` and `ContainerPathStorage`.
+  `HashEncodingPathStorage` has similarly been replaced by introducing `HashEncodingStorage`.
 
 ## v6.0.0
 

config/http/notifications/base/storage.json

@@ -7,9 +7,12 @@
       "@type": "KeyValueChannelStorage",
       "locker": { "@id": "urn:solid-server:default:ResourceLocker" },
       "storage": {
-        "@type": "EncodingPathStorage",
-        "relativePath": "/notifications/",
-        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        "@type": "Base64EncodingStorage",
+        "source": {
+          "@type": "ContainerPathStorage",
+          "relativePath": "/notifications/",
+          "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        }
       }
     }
   ]

config/identity/handler/account-store/default.json

@@ -7,9 +7,12 @@
       "@type": "BaseAccountStore",
       "saltRounds": 10,
       "storage": {
-        "@type": "EncodingPathStorage",
-        "relativePath": "/accounts/",
-        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        "@type": "Base64EncodingStorage",
+        "source": {
+          "@type": "ContainerPathStorage",
+          "relativePath": "/accounts/",
+          "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        }
       },
       "forgotPasswordStorage": {
         "@id": "urn:solid-server:default:ExpiringForgotPasswordStorage"
@@ -20,9 +23,12 @@
       "@id": "urn:solid-server:default:ExpiringForgotPasswordStorage",
       "@type": "WrappedExpiringStorage",
       "source": {
-        "@type": "EncodingPathStorage",
-        "relativePath": "/forgot-password/",
-        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        "@type": "Base64EncodingStorage",
+        "source": {
+          "@type": "ContainerPathStorage",
+          "relativePath": "/forgot-password/",
+          "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        }
       }
     }
   ]

config/identity/handler/adapter-factory/webid.json

@@ -12,9 +12,12 @@
         "source": {
           "@type": "ExpiringAdapterFactory",
           "storage": {
-            "@type": "EncodingPathStorage",
-            "relativePath": "/idp/adapter/",
-            "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+            "@type": "Base64EncodingStorage",
+            "source": {
+              "@type": "ContainerPathStorage",
+              "relativePath": "/idp/adapter/",
+              "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+            }
           }
         }
       }

config/identity/handler/interaction/routes/credentials.json

@@ -4,9 +4,12 @@
     {
       "comment": "Stores all client credential tokens.",
       "@id": "urn:solid-server:auth:password:CredentialsStorage",
-      "@type": "EncodingPathStorage",
-      "relativePath": "/accounts/credentials/",
-      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+      "@type": "Base64EncodingStorage",
+      "source": {
+        "@type": "ContainerPathStorage",
+        "relativePath": "/accounts/credentials/",
+        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+      }
     },
     {
       "comment": "Handles credential tokens. These can be used to automate clients. See documentation for more info.",

config/identity/handler/jwks/default.json

@@ -11,9 +11,12 @@
     },
     {
       "@id": "urn:solid-server:default:KeyStorage",
-      "@type": "EncodingPathStorage",
-      "relativePath": "/idp/keys/",
-      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+      "@type": "Base64EncodingStorage",
+      "source": {
+        "@type": "ContainerPathStorage",
+        "relativePath": "/idp/keys/",
+        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+      }
     }
   ]
 }

config/identity/ownership/token.json

@@ -13,9 +13,12 @@
       "@id": "urn:solid-server:default:ExpiringTokenStorage",
       "@type": "WrappedExpiringStorage",
       "source": {
-        "@type": "EncodingPathStorage",
-        "relativePath": "/idp/tokens/",
-        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        "@type": "Base64EncodingStorage",
+        "source": {
+          "@type": "ContainerPathStorage",
+          "relativePath": "/idp/tokens/",
+          "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+        }
       }
     }
   ]

config/storage/key-value/storages/storages.json

@@ -4,16 +4,22 @@
     {
       "comment": "Used for internal storage by the locker.",
       "@id": "urn:solid-server:default:LockStorage",
-      "@type": "HashEncodingPathStorage",
-      "relativePath": "/locks/",
-      "source": { "@id": "urn:solid-server:default:BackendKeyValueStorage" }
+      "@type": "HashEncodingStorage",
+      "source": {
+        "@type": "ContainerPathStorage",
+        "relativePath": "/locks/",
+        "source": { "@id": "urn:solid-server:default:BackendKeyValueStorage" }
+      }
     },
     {
       "comment": "Storage used by setup components.",
       "@id": "urn:solid-server:default:SetupStorage",
-      "@type": "EncodingPathStorage",
-      "relativePath": "/setup/",
-      "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+      "@type": "Base64EncodingStorage",
+      "source": {
+        "@type": "ContainerPathStorage",
+        "relativePath": "/setup/",
+        "source": { "@id": "urn:solid-server:default:KeyValueStorage" }
+      }
     }
   ]
 }

src/index.ts

@@ -398,13 +398,15 @@ export * from './storage/conversion/RepresentationConverter';
 export * from './storage/conversion/TypedRepresentationConverter';
 
 // Storage/KeyValue
-export * from './storage/keyvalue/EncodingPathStorage';
+export * from './storage/keyvalue/Base64EncodingStorage';
+export * from './storage/keyvalue/ContainerPathStorage';
 export * from './storage/keyvalue/ExpiringStorage';
-export * from './storage/keyvalue/HashEncodingPathStorage';
+export * from './storage/keyvalue/HashEncodingStorage';
 export * from './storage/keyvalue/JsonFileStorage';
 export * from './storage/keyvalue/JsonResourceStorage';
 export * from './storage/keyvalue/KeyValueStorage';
 export * from './storage/keyvalue/MemoryMapStorage';
+export * from './storage/keyvalue/PassthroughKeyValueStorage';
 export * from './storage/keyvalue/WrappedExpiringStorage';
 
 // Storage/Mapping

src/storage/keyvalue/Base64EncodingStorage.ts

@@ -0,0 +1,20 @@
+import type { KeyValueStorage } from './KeyValueStorage';
+import { PassthroughKeyValueStorage } from './PassthroughKeyValueStorage';
+
+/**
+ * Encodes the input key with base64 encoding,
+ * to make sure there are no invalid or special path characters.
+ */
+export class Base64EncodingStorage<T> extends PassthroughKeyValueStorage<T> {
+  public constructor(source: KeyValueStorage<string, T>) {
+    super(source);
+  }
+
+  protected toNewKey(key: string): string {
+    return Buffer.from(key).toString('base64');
+  }
+
+  protected toOriginalKey(key: string): string {
+    return Buffer.from(key, 'base64').toString('utf-8');
+  }
+}

src/storage/keyvalue/ContainerPathStorage.ts

@@ -0,0 +1,33 @@
+import { ensureTrailingSlash, joinUrl } from '../../util/PathUtil';
+import type { KeyValueStorage } from './KeyValueStorage';
+import { PassthroughKeyValueStorage } from './PassthroughKeyValueStorage';
+
+/**
+ * A {@link KeyValueStorage} that prepends a relative path to the key.
+ */
+export class ContainerPathStorage<T> extends PassthroughKeyValueStorage<T> {
+  protected readonly basePath: string;
+
+  public constructor(source: KeyValueStorage<string, T>, relativePath: string) {
+    super(source);
+    this.basePath = ensureTrailingSlash(relativePath);
+  }
+
+  public async* entries(): AsyncIterableIterator<[string, T]> {
+    for await (const [ key, value ] of this.source.entries()) {
+      // The only relevant entries for this storage are those that start with the base path
+      if (!key.startsWith(this.basePath)) {
+        continue;
+      }
+      yield [ this.toOriginalKey(key), value ];
+    }
+  }
+
+  protected toNewKey(key: string): string {
+    return joinUrl(this.basePath, key);
+  }
+
+  protected toOriginalKey(path: string): string {
+    return path.slice(this.basePath.length);
+  }
+}

src/storage/keyvalue/EncodingPathStorage.ts

@@ -1,68 +0,0 @@
-import { ensureTrailingSlash, joinUrl } from '../../util/PathUtil';
-import type { KeyValueStorage } from './KeyValueStorage';
-
-/**
- * Transforms the keys into relative paths, to be used by the source storage.
- * Encodes the input key with base64 encoding,
- * to make sure there are no invalid or special path characters,
- * and prepends it with the stored relative path.
- * This can be useful to eventually generate URLs in specific containers
- * without having to worry about cleaning the input keys.
- */
-export class EncodingPathStorage<T> implements KeyValueStorage<string, T> {
-  protected readonly basePath: string;
-  protected readonly source: KeyValueStorage<string, T>;
-
-  public constructor(relativePath: string, source: KeyValueStorage<string, T>) {
-    this.source = source;
-    this.basePath = ensureTrailingSlash(relativePath);
-  }
-
-  public async get(key: string): Promise<T | undefined> {
-    const path = this.keyToPath(key);
-    return this.source.get(path);
-  }
-
-  public async has(key: string): Promise<boolean> {
-    const path = this.keyToPath(key);
-    return this.source.has(path);
-  }
-
-  public async set(key: string, value: T): Promise<this> {
-    const path = this.keyToPath(key);
-    await this.source.set(path, value);
-    return this;
-  }
-
-  public async delete(key: string): Promise<boolean> {
-    const path = this.keyToPath(key);
-    return this.source.delete(path);
-  }
-
-  public async* entries(): AsyncIterableIterator<[string, T]> {
-    for await (const [ path, value ] of this.source.entries()) {
-      // The only relevant entries for this storage are those that start with the base path
-      if (!path.startsWith(this.basePath)) {
-        continue;
-      }
-      const key = this.pathToKey(path);
-      yield [ key, value ];
-    }
-  }
-
-  /**
-   * Converts a key into a path for internal storage.
-   */
-  protected keyToPath(key: string): string {
-    const encodedKey = Buffer.from(key).toString('base64');
-    return joinUrl(this.basePath, encodedKey);
-  }
-
-  /**
-   * Converts an internal storage path string into the original path key.
-   */
-  protected pathToKey(path: string): string {
-    const buffer = Buffer.from(path.slice(this.basePath.length), 'base64');
-    return buffer.toString('utf-8');
-  }
-}

src/storage/keyvalue/HashEncodingStorage.ts

@@ -1,11 +1,12 @@
 import { createHash } from 'crypto';
 import { getLoggerFor } from '../../logging/LogUtil';
 import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError';
-import { joinUrl } from '../../util/PathUtil';
-import { EncodingPathStorage } from './EncodingPathStorage';
+import type { KeyValueStorage } from './KeyValueStorage';
+import { PassthroughKeyValueStorage } from './PassthroughKeyValueStorage';
 
 /**
- * A variant of the {@link EncodingPathStorage} that hashes the key instead of converting to base64 encoding.
+ * Encodes the input key with SHA-256 hashing,
+ * to make sure there are no invalid or special path characters.
  *
  * This class was created specifically to prevent the issue of identifiers being too long when storing data:
  * https://github.com/CommunitySolidServer/CommunitySolidServer/issues/1013
@@ -13,16 +14,20 @@ import { EncodingPathStorage } from './EncodingPathStorage';
  * This should eventually be replaced by a more structural approach once internal storage has been refactored
  * and data migration from older versions and formats is supported.
  */
-export class HashEncodingPathStorage<T> extends EncodingPathStorage<T> {
+export class HashEncodingStorage<T> extends PassthroughKeyValueStorage<T> {
   protected readonly logger = getLoggerFor(this);
 
-  protected keyToPath(key: string): string {
-    const hash = createHash('sha256').update(key).digest('hex');
-    this.logger.debug(`Hashing key ${key} to ${hash}`);
-    return joinUrl(this.basePath, hash);
+  public constructor(source: KeyValueStorage<string, T>) {
+    super(source);
   }
 
-  protected pathToKey(): string {
+  protected toNewKey(key: string): string {
+    const hash = createHash('sha256').update(key).digest('hex');
+    this.logger.debug(`Hashing key ${key} to ${hash}`);
+    return hash;
+  }
+
+  protected toOriginalKey(): string {
     throw new NotImplementedHttpError('Hash keys cannot be converted back.');
   }
 }

src/storage/keyvalue/PassthroughKeyValueStorage.ts

@@ -0,0 +1,60 @@
+import type { KeyValueStorage } from './KeyValueStorage';
+
+/**
+ * Abstract class to create a {@link KeyValueStorage} by wrapping around another one.
+ *
+ * Exposes abstract functions to modify the key before passing it to the the source storage.
+ */
+export abstract class PassthroughKeyValueStorage<TVal> implements KeyValueStorage<string, TVal> {
+  protected readonly source: KeyValueStorage<string, TVal>;
+
+  protected constructor(source: KeyValueStorage<string, TVal>) {
+    this.source = source;
+  }
+
+  public async get(key: string): Promise<TVal | undefined> {
+    const path = this.toNewKey(key);
+    return this.source.get(path);
+  }
+
+  public async has(key: string): Promise<boolean> {
+    const path = this.toNewKey(key);
+    return this.source.has(path);
+  }
+
+  public async set(key: string, value: TVal): Promise<this> {
+    const path = this.toNewKey(key);
+    await this.source.set(path, value);
+    return this;
+  }
+
+  public async delete(key: string): Promise<boolean> {
+    const path = this.toNewKey(key);
+    return this.source.delete(path);
+  }
+
+  public async* entries(): AsyncIterableIterator<[string, TVal]> {
+    for await (const [ path, value ] of this.source.entries()) {
+      const key = this.toOriginalKey(path);
+      yield [ key, value ];
+    }
+  }
+
+  /**
+   * This function will be called on the input key and used as a new key when calling the source.
+   *
+   * @param key - Original input key.
+   *
+   * @returns A new key to use with the source storage.
+   */
+  protected abstract toNewKey(key: string): string;
+
+  /**
+   * This function is used when calling `entries()` to revert the key generated by `toNewKey()`.
+   *
+   * @param key - A key generated by `toNewKey()`
+   *
+   * @returns The original key.
+   */
+  protected abstract toOriginalKey(key: string): string;
+}

test/unit/storage/keyvalue/Base64EncodingStorage.test.ts

@@ -0,0 +1,40 @@
+import { Base64EncodingStorage } from '../../../../src/storage/keyvalue/Base64EncodingStorage';
+import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage';
+
+describe('A Base64EncodingStorage', (): void => {
+  let map: Map<string, string>;
+  let source: KeyValueStorage<string, string>;
+  let storage: Base64EncodingStorage<string>;
+
+  beforeEach(async(): Promise<void> => {
+    map = new Map<string, string>();
+    source = map as any;
+    storage = new Base64EncodingStorage<string>(source);
+  });
+
+  it('encodes the keys.', async(): Promise<void> => {
+    const key = 'key';
+    // Base 64 encoding of 'key'
+    const encodedKey = 'a2V5';
+    const data = 'data';
+    await storage.set(key, data);
+    expect(map.size).toBe(1);
+    expect(map.get(encodedKey)).toBe(data);
+    await expect(storage.get(key)).resolves.toBe(data);
+  });
+
+  it('decodes the keys.', async(): Promise<void> => {
+    // Base 64 encoding of 'key'
+    const encodedKey = 'a2V5';
+    const data = 'data';
+
+    map.set(encodedKey, data);
+
+    const results = [];
+    for await (const entry of storage.entries()) {
+      results.push(entry);
+    }
+    expect(results).toHaveLength(1);
+    expect(results[0]).toEqual([ 'key', data ]);
+  });
+});

test/unit/storage/keyvalue/ContainerPathStorage.test.ts

@@ -1,23 +1,21 @@
-import { EncodingPathStorage } from '../../../../src/storage/keyvalue/EncodingPathStorage';
+import { ContainerPathStorage } from '../../../../src/storage/keyvalue/ContainerPathStorage';
 import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage';
 
-describe('An EncodingPathStorage', (): void => {
+describe('An ContainerPathStorage', (): void => {
   const relativePath = '/container/';
   let map: Map<string, string>;
   let source: KeyValueStorage<string, unknown>;
-  let storage: EncodingPathStorage<unknown>;
+  let storage: ContainerPathStorage<unknown>;
 
   beforeEach(async(): Promise<void> => {
     map = new Map<string, string>();
     source = map as any;
-    storage = new EncodingPathStorage(relativePath, source);
+    storage = new ContainerPathStorage(source, relativePath);
   });
 
-  it('encodes the input key and joins it with the relativePath to create a new key.', async(): Promise<void> => {
+  it('joins the input key with the relativePath to create a new key.', async(): Promise<void> => {
     const key = 'key';
-    // Base 64 encoding of 'key'
-    const encodedKey = 'a2V5';
-    const generatedPath = `${relativePath}${encodedKey}`;
+    const generatedPath = `${relativePath}${key}`;
     const data = 'data';
 
     await expect(storage.set(key, data)).resolves.toBe(storage);
@@ -32,10 +30,8 @@ describe('An EncodingPathStorage', (): void => {
   });
 
   it('only returns entries from the source storage matching the relative path.', async(): Promise<void> => {
-    // Base 64 encoding of 'key'
-    const encodedKey = 'a2V5';
-    const generatedPath = `${relativePath}${encodedKey}`;
-    const otherPath = `/otherContainer/${encodedKey}`;
+    const generatedPath = `${relativePath}key`;
+    const otherPath = `/otherContainer/key`;
     const data = 'data';
 
     map.set(generatedPath, data);

test/unit/storage/keyvalue/HashEncodingStorage.test.ts

@@ -1,17 +1,16 @@
-import { HashEncodingPathStorage } from '../../../../src/storage/keyvalue/HashEncodingPathStorage';
+import { HashEncodingStorage } from '../../../../src/storage/keyvalue/HashEncodingStorage';
 import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage';
 import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplementedHttpError';
 
-describe('A HashEncodingPathStorage', (): void => {
-  const relativePath = '/container/';
+describe('A HashEncodingStorage', (): void => {
   let map: Map<string, string>;
   let source: KeyValueStorage<string, string>;
-  let storage: HashEncodingPathStorage<string>;
+  let storage: HashEncodingStorage<string>;
 
   beforeEach(async(): Promise<void> => {
     map = new Map<string, string>();
     source = map as any;
-    storage = new HashEncodingPathStorage<string>(relativePath, source);
+    storage = new HashEncodingStorage<string>(source);
   });
 
   it('hashes the keys.', async(): Promise<void> => {
@@ -20,7 +19,7 @@ describe('A HashEncodingPathStorage', (): void => {
     const data = 'data';
     await storage.set(key, data);
     expect(map.size).toBe(1);
-    expect(map.get(`${relativePath}${hash}`)).toBe(data);
+    expect(map.get(hash)).toBe(data);
     await expect(storage.get(key)).resolves.toBe(data);
   });
 

test/unit/storage/keyvalue/PassthroughKeyValueStorage.test.ts

@@ -0,0 +1,62 @@
+import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage';
+import { PassthroughKeyValueStorage } from '../../../../src/storage/keyvalue/PassthroughKeyValueStorage';
+
+class DummyStorage extends PassthroughKeyValueStorage<string> {
+  public constructor(source: KeyValueStorage<string, string>) {
+    super(source);
+  }
+
+  protected toNewKey(key: string): string {
+    return `dummy-${key}`;
+  }
+
+  protected toOriginalKey(key: string): string {
+    return key.slice('dummy-'.length);
+  }
+}
+
+describe('A PassthroughKeyValueStorage', (): void => {
+  let source: jest.Mocked<KeyValueStorage<string, string>>;
+  let storage: DummyStorage;
+
+  beforeEach(async(): Promise<void> => {
+    source = {
+      has: jest.fn(),
+      get: jest.fn(),
+      set: jest.fn(),
+      delete: jest.fn(),
+      entries: jest.fn(),
+    };
+
+    storage = new DummyStorage(source);
+  });
+
+  it('calls the source storage with the updated key.', async(): Promise<void> => {
+    await storage.has('key');
+    expect(source.has).toHaveBeenCalledTimes(1);
+    expect(source.has).toHaveBeenLastCalledWith('dummy-key');
+
+    await storage.get('key');
+    expect(source.get).toHaveBeenCalledTimes(1);
+    expect(source.get).toHaveBeenLastCalledWith('dummy-key');
+
+    await storage.set('key', 'data');
+    expect(source.set).toHaveBeenCalledTimes(1);
+    expect(source.set).toHaveBeenLastCalledWith('dummy-key', 'data');
+
+    await storage.delete('key');
+    expect(source.delete).toHaveBeenCalledTimes(1);
+    expect(source.delete).toHaveBeenLastCalledWith('dummy-key');
+
+    // Set up data to test entries call
+    const map = new Map<string, string>([[ 'dummy-key', 'value' ], [ 'dummy-key2', 'value2' ]]);
+    source.entries.mockReturnValue(map.entries() as unknown as AsyncIterableIterator<[string, string]>);
+    const results = [];
+    for await (const entry of storage.entries()) {
+      results.push(entry);
+    }
+    expect(results).toHaveLength(2);
+    expect(results[0]).toEqual([ 'key', 'value' ]);
+    expect(results[1]).toEqual([ 'key2', 'value2' ]);
+  });
+});