feat: Update migration to clear all old non-account data

This is necessary to support the new internal format

RELEASE_NOTES.md

@@ -12,7 +12,7 @@
   for an overview of the new structure.
   Creating an account now requires multiple steps, but allows you to have multiple pods or WebIDs for 1 account.
   The architecture has been updated to be more easily extensible.
-- Pod seeding has been updated to account for the new account management, with an update CLI parameter `--seedConfig`,
+- Pod seeding has been updated to account for the new account management, with an updated CLI parameter `--seedConfig`,
   see the [updated documentation](https://communitysolidserver.github.io/CommunitySolidServer/7.x/usage/seeding-pods/)
   for more details.
 - Migration was added to update account data automatically from previous versions. See below for more details.
@@ -24,13 +24,17 @@
 
 ### Data migration
 
-Old account data will need to be migrated.
+Old internal data will need to be migrated.
 When starting the server for the first time after updating the version,
 this will happen automatically.
 A prompt will be shown to confirm.
-It is advised to first backup the account data in case something goes wrong.
+It is advised to first backup the internal data in case something goes wrong.
 When using the filesystem backend with default storage options,
-these can be found in the `.internal/accounts/` folder.
+these can be found in the `.internal` folder.
+
+Only account data will be migrated,
+other internal data such as OIDC sessions and notification subscriptions will be removed
+as how they are stored has changed as well.
 
 In case the prompt causes issues, it can be skipped automatically with the `--confirmMigration` CLI option.
 

config/app/init/migration/v6.json

@@ -13,28 +13,44 @@
         "@id": "urn:solid-server:default:V6MigrationInitializer",
         "@type": "V6MigrationInitializer",
         "versionKey": "current-server-version",
-        "versionStorage": { "@id": "urn:solid-server:default:SetupStorage" },
+        "versionStorage": { "@id": "urn:solid-server:default:V6MigrationSetupStorage" },
         "accountStorage": { "@id": "urn:solid-server:default:V6MigrationAccountStorage" },
         "clientCredentialsStorage": { "@id": "urn:solid-server:default:V6MigrationClientCredentialsStorage" },
-        "forgotPasswordStorage": { "@id": "urn:solid-server:default:V6MigrationForgotPasswordStorage" },
+        "cleanupStorages": [
+          { "@id": "urn:solid-server:default:V6MigrationAccountStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationClientCredentialsStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationForgotPasswordStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationKeyStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationAdapterStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationTokenStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationNotificationStorage" },
+          { "@id": "urn:solid-server:default:V6MigrationSetupStorage" }
+        ],
         "newStorage": { "@id": "urn:solid-server:default:AccountStorage" },
         "skipConfirmation": { "@id": "urn:solid-server:default:variable:confirmMigration" }
       }
     },
 
+    {
+      "comment": "All storages changed so we need the old setup storage to correctly read the version key",
+      "@id": "urn:solid-server:default:V6MigrationSetupStorage",
+      "@type": "Base64EncodingStorage",
+      "source": {
+        "@type": "SingleContainerJsonStorage",
+        "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+        "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+        "container": "/.internal/setup/"
+      }
+    },
+
     {
       "@id": "urn:solid-server:default:V6MigrationAccountStorage",
       "@type": "Base64EncodingStorage",
       "source": {
-        "comment": "Relative path of `/` is necessary to strip leading slash from keys.",
+        "@type": "SingleContainerJsonStorage",
-        "@type": "ContainerPathStorage",
+        "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
-        "relativePath": "/",
+        "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-        "source": {
+        "container": "/.internal/accounts/"
-          "@type": "SingleContainerJsonStorage",
-          "source": { "@id": "urn:solid-server:default:ResourceStore" },
-          "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-          "container": "/.internal/accounts/"
-        }
       }
     },
 
@@ -42,32 +58,47 @@
       "@id": "urn:solid-server:default:V6MigrationClientCredentialsStorage",
       "@type": "Base64EncodingStorage",
       "source": {
-        "comment": "Relative path of `/` is necessary to strip leading slash from keys.",
+        "@type": "SingleContainerJsonStorage",
-        "@type": "ContainerPathStorage",
+        "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
-        "relativePath": "/",
+        "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-        "source": {
+        "container": "/.internal/accounts/credentials/"
-          "@type": "SingleContainerJsonStorage",
-          "source": { "@id": "urn:solid-server:default:ResourceStore" },
-          "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-          "container": "/.internal/accounts/credentials/"
-        }
       }
     },
 
     {
       "@id": "urn:solid-server:default:V6MigrationForgotPasswordStorage",
-      "@type": "Base64EncodingStorage",
+      "@type": "SingleContainerJsonStorage",
-      "source": {
+      "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
-        "comment": "Relative path of `/` is necessary to strip leading slash from keys.",
+      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-        "@type": "ContainerPathStorage",
+      "container": "/.internal/forgot-password/"
-        "relativePath": "/",
+    },
-        "source": {
+    {
-          "@type": "SingleContainerJsonStorage",
+      "@id": "urn:solid-server:default:V6MigrationKeyStorage",
-          "source": { "@id": "urn:solid-server:default:ResourceStore" },
+      "@type": "SingleContainerJsonStorage",
-          "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+      "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
-          "container": "/.internal/forgot-password/"
+      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
-        }
+      "container": "/.internal/idp/keys/"
-      }
+    },
+    {
+      "@id": "urn:solid-server:default:V6MigrationAdapterStorage",
+      "@type": "SingleContainerJsonStorage",
+      "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+      "container": "/.internal/idp/adapter/"
+    },
+    {
+      "@id": "urn:solid-server:default:V6MigrationTokenStorage",
+      "@type": "SingleContainerJsonStorage",
+      "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+      "container": "/.internal/idp/tokens/"
+    },
+    {
+      "@id": "urn:solid-server:default:V6MigrationNotificationStorage",
+      "@type": "SingleContainerJsonStorage",
+      "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+      "container": "/.internal/notifications/"
     }
   ]
 }

src/init/migration/V6MigrationInitializer.ts

@@ -66,10 +66,9 @@ export interface V6MigrationInitializerArgs {
    */
   clientCredentialsStorage: KeyValueStorage<string, ClientCredentials>;
   /**
-   * The storage in which the forgot password entries of the previous version are stored.
+   * Storages for which all entries need to be removed.
-   * These will all just be removed, not migrated.
    */
-  forgotPasswordStorage: KeyValueStorage<string, unknown>;
+  cleanupStorages: KeyValueStorage<string, any>[];
   /**
    * The storage that will contain the account data in the new format.
    */
@@ -97,7 +96,7 @@ export class V6MigrationInitializer extends Initializer {
 
   private readonly accountStorage: KeyValueStorage<string, Account | Settings>;
   private readonly clientCredentialsStorage: KeyValueStorage<string, ClientCredentials>;
-  private readonly forgotPasswordStorage: KeyValueStorage<string, unknown>;
+  private readonly cleanupStorages: KeyValueStorage<string, any>[];
 
   private readonly newStorage: AccountLoginStorage<typeof STORAGE_DESCRIPTION>;
 
@@ -108,7 +107,7 @@ export class V6MigrationInitializer extends Initializer {
     this.versionStorage = args.versionStorage;
     this.accountStorage = args.accountStorage;
     this.clientCredentialsStorage = args.clientCredentialsStorage;
-    this.forgotPasswordStorage = args.forgotPasswordStorage;
+    this.cleanupStorages = args.cleanupStorages;
     this.newStorage = args.newStorage;
   }
 
@@ -125,24 +124,19 @@ export class V6MigrationInitializer extends Initializer {
       return;
     }
 
-    const accountIterator = this.accountStorage.entries();
-    const next = await accountIterator.next();
-    if (next.done) {
-      this.logger.debug('No account data was found so no migration is necessary.');
-      return;
-    }
-
     // Ask the user for confirmation
     if (!this.skipConfirmation) {
       const readline = createInterface({ input: process.stdin, output: process.stdout });
       const answer = await new Promise<string>((resolve): void => {
         readline.question([
-          'The server is now going to migrate v6 account data to the new storage format internally.',
+          'The server is now going to migrate v6 data to the new storage format internally.',
+          'Existing accounts will be migrated.',
+          'All other internal data, such as notification subscriptions will be removed.',
           'In case you have not yet done this,',
-          'it is recommended to cancel startup and first backup the existing account data,',
+          'it is recommended to cancel startup and first backup the existing data,',
           'in case something goes wrong.',
           'When using default configurations with a file backend,',
-          'this data can be found in the ".internal/accounts" folder.',
+          'this data can be found in the ".internal" folder.',
           '\n\nDo you want to migrate the data now? [y/N] ',
         ].join(' '), resolve);
       });
@@ -152,18 +146,11 @@ export class V6MigrationInitializer extends Initializer {
       }
     }
 
-    this.logger.info('Migrating v6 account data to the new format...');
+    this.logger.info('Migrating v6 data...');
 
     const webIdAccountMap: Record<string, string> = {};
 
-    // Need to migrate the first entry we already extracted from the iterator above
+    for await (const [ , account ] of this.accountStorage.entries()) {
-    const firstResult = await this.createAccount(next.value[1]);
-    if (firstResult) {
-      // Store link between WebID and account ID for client credentials
-      webIdAccountMap[firstResult.webId] = firstResult.accountId;
-    }
-
-    for await (const [ , account ] of accountIterator) {
       const result = await this.createAccount(account);
       if (result) {
         // Store link between WebID and account ID for client credentials
@@ -171,6 +158,7 @@ export class V6MigrationInitializer extends Initializer {
       }
     }
 
+    this.logger.debug('Converting client credentials tokens.');
     // Convert the existing client credentials tokens
     for await (const [ label, { webId, secret }] of this.clientCredentialsStorage.entries()) {
       const accountId = webIdAccountMap[webId];
@@ -181,18 +169,15 @@ export class V6MigrationInitializer extends Initializer {
       await this.newStorage.create(CLIENT_CREDENTIALS_STORAGE_TYPE, { webId, label, secret, accountId });
     }
 
-    // Delete all old entries
+    // Cleanup all old entries
-    for await (const [ key ] of this.accountStorage.entries()) {
+    this.logger.debug('Cleaning up older entries.');
-      await this.accountStorage.delete(key);
+    for (const storage of this.cleanupStorages) {
-    }
+      for await (const [ key ] of storage.entries()) {
-    for await (const [ key ] of this.clientCredentialsStorage.entries()) {
+        await storage.delete(key);
-      await this.clientCredentialsStorage.delete(key);
+      }
-    }
-    for await (const [ key ] of this.forgotPasswordStorage.entries()) {
-      await this.forgotPasswordStorage.delete(key);
     }
 
-    this.logger.info('Finished migrating v6 account data.');
+    this.logger.info('Finished migrating v6 data.');
   }
 
   protected isAccount(data: Account | Settings): data is Account {

test/unit/init/migration/SingleContainerJsonStorage.test.ts

@@ -40,8 +40,8 @@ describe('A SingleContainerJsonStorage', (): void => {
       entries.push(entry);
     }
     expect(entries).toEqual([
-      [ '/foo', { id: 'http://example.com/.internal/accounts/foo' }],
+      [ 'foo', { id: 'http://example.com/.internal/accounts/foo' }],
-      [ '/baz', { id: 'http://example.com/.internal/accounts/baz' }],
+      [ 'baz', { id: 'http://example.com/.internal/accounts/baz' }],
     ]);
     expect(store.getRepresentation).toHaveBeenCalledTimes(4);
     expect(store.getRepresentation).toHaveBeenNthCalledWith(1,

test/unit/init/migration/V6MigrationInitializer.test.ts

@@ -98,7 +98,7 @@ describe('A V6MigrationInitializer', (): void => {
       versionStorage,
       accountStorage,
       clientCredentialsStorage,
-      forgotPasswordStorage,
+      cleanupStorages: [ accountStorage, clientCredentialsStorage, forgotPasswordStorage ],
       newStorage,
       skipConfirmation: true,
     });
@@ -190,7 +190,7 @@ describe('A V6MigrationInitializer', (): void => {
         versionStorage,
         accountStorage,
         clientCredentialsStorage,
-        forgotPasswordStorage,
+        cleanupStorages: [ accountStorage, clientCredentialsStorage, forgotPasswordStorage ],
         newStorage,
         skipConfirmation: false,
       });
@@ -210,14 +210,5 @@ describe('A V6MigrationInitializer', (): void => {
       await expect(initializer.handle()).rejects.toThrow('Stopping server as migration was cancelled.');
       expect(newStorage.create).toHaveBeenCalledTimes(0);
     });
-
-    it('does not show the prompt if there are no accounts.', async(): Promise<void> => {
-      settings = {};
-      accounts = {};
-      await expect(initializer.handle()).resolves.toBeUndefined();
-      expect(questionMock).toHaveBeenCalledTimes(0);
-      expect(accountStorage.get).toHaveBeenCalledTimes(0);
-      expect(newStorage.create).toHaveBeenCalledTimes(0);
-    });
   });
 });