From 9daeaf89ace822d8283a7a75b0d300628ab573d3 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Tue, 10 Oct 2023 15:00:07 +0200 Subject: [PATCH] feat: Update migration to clear all old non-account data This is necessary to support the new internal format --- RELEASE_NOTES.md | 12 ++- config/app/init/migration/v6.json | 95 ++++++++++++------- src/init/migration/V6MigrationInitializer.ts | 53 ++++------- .../SingleContainerJsonStorage.test.ts | 4 +- .../migration/V6MigrationInitializer.test.ts | 13 +-- 5 files changed, 94 insertions(+), 83 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 473c22d48..fb04da80e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -12,7 +12,7 @@ for an overview of the new structure. Creating an account now requires multiple steps, but allows you to have multiple pods or WebIDs for 1 account. The architecture has been updated to be more easily extensible. -- Pod seeding has been updated to account for the new account management, with an update CLI parameter `--seedConfig`, +- Pod seeding has been updated to account for the new account management, with an updated CLI parameter `--seedConfig`, see the [updated documentation](https://communitysolidserver.github.io/CommunitySolidServer/7.x/usage/seeding-pods/) for more details. - Migration was added to update account data automatically from previous versions. See below for more details. @@ -24,13 +24,17 @@ ### Data migration -Old account data will need to be migrated. +Old internal data will need to be migrated. When starting the server for the first time after updating the version, this will happen automatically. A prompt will be shown to confirm. -It is advised to first backup the account data in case something goes wrong. +It is advised to first backup the internal data in case something goes wrong. When using the filesystem backend with default storage options, -these can be found in the `.internal/accounts/` folder. +these can be found in the `.internal` folder. + +Only account data will be migrated, +other internal data such as OIDC sessions and notification subscriptions will be removed +as how they are stored has changed as well. In case the prompt causes issues, it can be skipped automatically with the `--confirmMigration` CLI option. diff --git a/config/app/init/migration/v6.json b/config/app/init/migration/v6.json index a8ffadb7b..fa90076d9 100644 --- a/config/app/init/migration/v6.json +++ b/config/app/init/migration/v6.json @@ -13,28 +13,44 @@ "@id": "urn:solid-server:default:V6MigrationInitializer", "@type": "V6MigrationInitializer", "versionKey": "current-server-version", - "versionStorage": { "@id": "urn:solid-server:default:SetupStorage" }, + "versionStorage": { "@id": "urn:solid-server:default:V6MigrationSetupStorage" }, "accountStorage": { "@id": "urn:solid-server:default:V6MigrationAccountStorage" }, "clientCredentialsStorage": { "@id": "urn:solid-server:default:V6MigrationClientCredentialsStorage" }, - "forgotPasswordStorage": { "@id": "urn:solid-server:default:V6MigrationForgotPasswordStorage" }, + "cleanupStorages": [ + { "@id": "urn:solid-server:default:V6MigrationAccountStorage" }, + { "@id": "urn:solid-server:default:V6MigrationClientCredentialsStorage" }, + { "@id": "urn:solid-server:default:V6MigrationForgotPasswordStorage" }, + { "@id": "urn:solid-server:default:V6MigrationKeyStorage" }, + { "@id": "urn:solid-server:default:V6MigrationAdapterStorage" }, + { "@id": "urn:solid-server:default:V6MigrationTokenStorage" }, + { "@id": "urn:solid-server:default:V6MigrationNotificationStorage" }, + { "@id": "urn:solid-server:default:V6MigrationSetupStorage" } + ], "newStorage": { "@id": "urn:solid-server:default:AccountStorage" }, "skipConfirmation": { "@id": "urn:solid-server:default:variable:confirmMigration" } } }, + { + "comment": "All storages changed so we need the old setup storage to correctly read the version key", + "@id": "urn:solid-server:default:V6MigrationSetupStorage", + "@type": "Base64EncodingStorage", + "source": { + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/setup/" + } + }, + { "@id": "urn:solid-server:default:V6MigrationAccountStorage", "@type": "Base64EncodingStorage", "source": { - "comment": "Relative path of `/` is necessary to strip leading slash from keys.", - "@type": "ContainerPathStorage", - "relativePath": "/", - "source": { - "@type": "SingleContainerJsonStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/accounts/" - } + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/accounts/" } }, @@ -42,32 +58,47 @@ "@id": "urn:solid-server:default:V6MigrationClientCredentialsStorage", "@type": "Base64EncodingStorage", "source": { - "comment": "Relative path of `/` is necessary to strip leading slash from keys.", - "@type": "ContainerPathStorage", - "relativePath": "/", - "source": { - "@type": "SingleContainerJsonStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/accounts/credentials/" - } + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/accounts/credentials/" } }, { "@id": "urn:solid-server:default:V6MigrationForgotPasswordStorage", - "@type": "Base64EncodingStorage", - "source": { - "comment": "Relative path of `/` is necessary to strip leading slash from keys.", - "@type": "ContainerPathStorage", - "relativePath": "/", - "source": { - "@type": "SingleContainerJsonStorage", - "source": { "@id": "urn:solid-server:default:ResourceStore" }, - "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, - "container": "/.internal/forgot-password/" - } - } + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/forgot-password/" + }, + { + "@id": "urn:solid-server:default:V6MigrationKeyStorage", + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/idp/keys/" + }, + { + "@id": "urn:solid-server:default:V6MigrationAdapterStorage", + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/idp/adapter/" + }, + { + "@id": "urn:solid-server:default:V6MigrationTokenStorage", + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/idp/tokens/" + }, + { + "@id": "urn:solid-server:default:V6MigrationNotificationStorage", + "@type": "SingleContainerJsonStorage", + "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }, + "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" }, + "container": "/.internal/notifications/" } ] } diff --git a/src/init/migration/V6MigrationInitializer.ts b/src/init/migration/V6MigrationInitializer.ts index 1317d1b2f..e8c7f3baa 100644 --- a/src/init/migration/V6MigrationInitializer.ts +++ b/src/init/migration/V6MigrationInitializer.ts @@ -66,10 +66,9 @@ export interface V6MigrationInitializerArgs { */ clientCredentialsStorage: KeyValueStorage; /** - * The storage in which the forgot password entries of the previous version are stored. - * These will all just be removed, not migrated. + * Storages for which all entries need to be removed. */ - forgotPasswordStorage: KeyValueStorage; + cleanupStorages: KeyValueStorage[]; /** * The storage that will contain the account data in the new format. */ @@ -97,7 +96,7 @@ export class V6MigrationInitializer extends Initializer { private readonly accountStorage: KeyValueStorage; private readonly clientCredentialsStorage: KeyValueStorage; - private readonly forgotPasswordStorage: KeyValueStorage; + private readonly cleanupStorages: KeyValueStorage[]; private readonly newStorage: AccountLoginStorage; @@ -108,7 +107,7 @@ export class V6MigrationInitializer extends Initializer { this.versionStorage = args.versionStorage; this.accountStorage = args.accountStorage; this.clientCredentialsStorage = args.clientCredentialsStorage; - this.forgotPasswordStorage = args.forgotPasswordStorage; + this.cleanupStorages = args.cleanupStorages; this.newStorage = args.newStorage; } @@ -125,24 +124,19 @@ export class V6MigrationInitializer extends Initializer { return; } - const accountIterator = this.accountStorage.entries(); - const next = await accountIterator.next(); - if (next.done) { - this.logger.debug('No account data was found so no migration is necessary.'); - return; - } - // Ask the user for confirmation if (!this.skipConfirmation) { const readline = createInterface({ input: process.stdin, output: process.stdout }); const answer = await new Promise((resolve): void => { readline.question([ - 'The server is now going to migrate v6 account data to the new storage format internally.', + 'The server is now going to migrate v6 data to the new storage format internally.', + 'Existing accounts will be migrated.', + 'All other internal data, such as notification subscriptions will be removed.', 'In case you have not yet done this,', - 'it is recommended to cancel startup and first backup the existing account data,', + 'it is recommended to cancel startup and first backup the existing data,', 'in case something goes wrong.', 'When using default configurations with a file backend,', - 'this data can be found in the ".internal/accounts" folder.', + 'this data can be found in the ".internal" folder.', '\n\nDo you want to migrate the data now? [y/N] ', ].join(' '), resolve); }); @@ -152,18 +146,11 @@ export class V6MigrationInitializer extends Initializer { } } - this.logger.info('Migrating v6 account data to the new format...'); + this.logger.info('Migrating v6 data...'); const webIdAccountMap: Record = {}; - // Need to migrate the first entry we already extracted from the iterator above - const firstResult = await this.createAccount(next.value[1]); - if (firstResult) { - // Store link between WebID and account ID for client credentials - webIdAccountMap[firstResult.webId] = firstResult.accountId; - } - - for await (const [ , account ] of accountIterator) { + for await (const [ , account ] of this.accountStorage.entries()) { const result = await this.createAccount(account); if (result) { // Store link between WebID and account ID for client credentials @@ -171,6 +158,7 @@ export class V6MigrationInitializer extends Initializer { } } + this.logger.debug('Converting client credentials tokens.'); // Convert the existing client credentials tokens for await (const [ label, { webId, secret }] of this.clientCredentialsStorage.entries()) { const accountId = webIdAccountMap[webId]; @@ -181,18 +169,15 @@ export class V6MigrationInitializer extends Initializer { await this.newStorage.create(CLIENT_CREDENTIALS_STORAGE_TYPE, { webId, label, secret, accountId }); } - // Delete all old entries - for await (const [ key ] of this.accountStorage.entries()) { - await this.accountStorage.delete(key); - } - for await (const [ key ] of this.clientCredentialsStorage.entries()) { - await this.clientCredentialsStorage.delete(key); - } - for await (const [ key ] of this.forgotPasswordStorage.entries()) { - await this.forgotPasswordStorage.delete(key); + // Cleanup all old entries + this.logger.debug('Cleaning up older entries.'); + for (const storage of this.cleanupStorages) { + for await (const [ key ] of storage.entries()) { + await storage.delete(key); + } } - this.logger.info('Finished migrating v6 account data.'); + this.logger.info('Finished migrating v6 data.'); } protected isAccount(data: Account | Settings): data is Account { diff --git a/test/unit/init/migration/SingleContainerJsonStorage.test.ts b/test/unit/init/migration/SingleContainerJsonStorage.test.ts index daaabdea5..c7d999516 100644 --- a/test/unit/init/migration/SingleContainerJsonStorage.test.ts +++ b/test/unit/init/migration/SingleContainerJsonStorage.test.ts @@ -40,8 +40,8 @@ describe('A SingleContainerJsonStorage', (): void => { entries.push(entry); } expect(entries).toEqual([ - [ '/foo', { id: 'http://example.com/.internal/accounts/foo' }], - [ '/baz', { id: 'http://example.com/.internal/accounts/baz' }], + [ 'foo', { id: 'http://example.com/.internal/accounts/foo' }], + [ 'baz', { id: 'http://example.com/.internal/accounts/baz' }], ]); expect(store.getRepresentation).toHaveBeenCalledTimes(4); expect(store.getRepresentation).toHaveBeenNthCalledWith(1, diff --git a/test/unit/init/migration/V6MigrationInitializer.test.ts b/test/unit/init/migration/V6MigrationInitializer.test.ts index da8bf541c..3d893abe4 100644 --- a/test/unit/init/migration/V6MigrationInitializer.test.ts +++ b/test/unit/init/migration/V6MigrationInitializer.test.ts @@ -98,7 +98,7 @@ describe('A V6MigrationInitializer', (): void => { versionStorage, accountStorage, clientCredentialsStorage, - forgotPasswordStorage, + cleanupStorages: [ accountStorage, clientCredentialsStorage, forgotPasswordStorage ], newStorage, skipConfirmation: true, }); @@ -190,7 +190,7 @@ describe('A V6MigrationInitializer', (): void => { versionStorage, accountStorage, clientCredentialsStorage, - forgotPasswordStorage, + cleanupStorages: [ accountStorage, clientCredentialsStorage, forgotPasswordStorage ], newStorage, skipConfirmation: false, }); @@ -210,14 +210,5 @@ describe('A V6MigrationInitializer', (): void => { await expect(initializer.handle()).rejects.toThrow('Stopping server as migration was cancelled.'); expect(newStorage.create).toHaveBeenCalledTimes(0); }); - - it('does not show the prompt if there are no accounts.', async(): Promise => { - settings = {}; - accounts = {}; - await expect(initializer.handle()).resolves.toBeUndefined(); - expect(questionMock).toHaveBeenCalledTimes(0); - expect(accountStorage.get).toHaveBeenCalledTimes(0); - expect(newStorage.create).toHaveBeenCalledTimes(0); - }); }); });