From 327ce7409ac3f62f8213bd8e300b3726fa848efb Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Tue, 6 Sep 2022 08:40:32 +0200 Subject: [PATCH 1/7] fix: Prevent websockets from being used with worker threads --- src/http/UnsecureWebSocketsProtocol.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/http/UnsecureWebSocketsProtocol.ts b/src/http/UnsecureWebSocketsProtocol.ts index 26921b842..bc3d6ca65 100644 --- a/src/http/UnsecureWebSocketsProtocol.ts +++ b/src/http/UnsecureWebSocketsProtocol.ts @@ -1,6 +1,7 @@ import { EventEmitter } from 'events'; import type { TLSSocket } from 'tls'; import type { WebSocket } from 'ws'; +import type { SingleThreaded } from '../init/cluster/SingleThreaded'; import { getLoggerFor } from '../logging/LogUtil'; import type { HttpRequest } from '../server/HttpRequest'; import { WebSocketHandler } from '../server/WebSocketHandler'; @@ -119,7 +120,7 @@ class WebSocketListener extends EventEmitter { * Provides live update functionality following * the Solid WebSockets API Spec solid-0.1 */ -export class UnsecureWebSocketsProtocol extends WebSocketHandler { +export class UnsecureWebSocketsProtocol extends WebSocketHandler implements SingleThreaded { private readonly logger = getLoggerFor(this); private readonly listeners = new Set(); From 2a5a93fe846e844498e5290de88169d4c2b97dc6 Mon Sep 17 00:00:00 2001 From: Jasper Vaneessen Date: Wed, 7 Sep 2022 13:32:18 +0200 Subject: [PATCH 2/7] ci: Add Conformance Test Harness checks to PRs * ci: split up workflows * ci: tweaks and housekeeping * ci: additional comments and requested changes * ci: additional comments * chore: clean up .gitignore * ci: CTH testing on PRs + reusable * ci: allow failure on job instead of command * ci: remove npm-test dependency for CTH * ci: fix branch for scheduled run of CTH * ci: add concurrency group for PRs --- .github/workflows/cth-test.yml | 90 ++++++++++++++++++++++++++++++++++ .github/workflows/main.yml | 14 +++++- .github/workflows/schedule.yml | 63 ++---------------------- 3 files changed, 107 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/cth-test.yml diff --git a/.github/workflows/cth-test.yml b/.github/workflows/cth-test.yml new file mode 100644 index 000000000..48a12cf2a --- /dev/null +++ b/.github/workflows/cth-test.yml @@ -0,0 +1,90 @@ +name: Conformance test harness + +on: + workflow_call: + inputs: + version: + type: string + required: false + default: latest + description: Version of the Conformance Test Harness that has to be tested against + branch: + type: string + required: false + description: branch to run tests against (used for matrix strategy) + ignore_failures: + type: boolean + default: false + description: Return succes even if there are failures? + required: false + workflow_dispatch: + inputs: + version: + type: string + required: false + default: latest + description: Version of the Conformance Test Harness that has to be tested against. + ignore_failures: + type: boolean + default: false + description: Return succes even if there are failures? + required: false + +jobs: + # Run the Conformance Test Harness and upload output as artifacts to GitHub + conformance: + continue-on-error: ${{ inputs.ignore_failures }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Use Node.js 16.x + uses: actions/setup-node@v3 + with: + node-version: 16.x + - name: Check out the project + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch || github.ref }} + - name: Install dependencies and run build scripts + run: npm ci + - name: Start the server in the background + run: npm start > server-output.log & + - name: Create the necessary folders + run: mkdir -p reports/css + - name: Pull the conformance harness docker + run: "docker pull solidproject/conformance-test-harness:${{ inputs.version }}" + - name: Wait until the server has started + run: > + curl --output /dev/null --silent --head --fail --retry 30 + --retry-connrefused --retry-delay 0 -k http://localhost:3000/ + - name: Create users + run: npx ts-node test/deploy/createAccountCredentials.ts http://localhost:3000/ >> test/deploy/conformance.env + - name: Run the test harness + run: > + docker run -i --rm + -v "$(pwd)"/reports/css:/reports + --env-file=./test/deploy/conformance.env + --network="host" + solidproject/conformance-test-harness + --skip-teardown + --output=/reports + --target=https://github.com/solid/conformance-test-harness/css + # Steps below use `always()` to make sure logs get uploaded in case the CTH errors + - name: Sanitize branch name for artifact upload + id: sanitize + if: always() + uses: yeouchien/sanitize-branch-name-action@v1 + with: + branch-name: ${{ inputs.branch || github.head_ref }} + - name: Save the reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.sanitize.outputs.sanitized-branch-name }} reports + path: reports + - name: Save the server output + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.sanitize.outputs.sanitized-branch-name }} server output + path: server-output.log diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0bb5fcd6..04fc42644 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,14 +7,24 @@ on: tags: - 'v*' pull_request: - branches: - - '**' + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true jobs: npm-test: # Run the tests in every case uses: ./.github/workflows/npm-test.yml + cth-test: + # Run the Conformance Test Harness on PRs targeting main or versions/ + if: github.event_name == 'pull_request' && ( github.base_ref == 'main' || startsWith( github.base_ref, 'versions/' ) ) + uses: ./.github/workflows/cth-test.yml + with: + ignore_failures: true + version: 1.1.7 # The latest version that CSS is confirmed to pass + docker: # Build docker containers on version tag, push to main and push to versions/ needs: npm-test diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 407cbabfc..33244eec3 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -3,69 +3,16 @@ name: Conformance test harness on: schedule: - cron: '23 5 * * *' - push: - branches: - - test/conformance + workflow_dispatch: + jobs: conformance: - runs-on: ubuntu-latest strategy: fail-fast: false matrix: branch: - 'main' - 'versions/6.0.0' - timeout-minutes: 10 - steps: - - name: Use Node.js 16.x - uses: actions/setup-node@v3 - with: - node-version: 16.x - - name: Check out the project - uses: actions/checkout@v3 - with: - ref: ${{ matrix.branch }} - - name: Install dependencies and run build scripts - run: npm ci - - name: Start the server in the background - run: npm start > server-output.log & - - name: Create the necessary folders - run: mkdir -p reports/css - - name: Pull the conformance harness docker - run: docker pull solidproject/conformance-test-harness - - name: Wait until the server has started - run: | - until $(curl --output /dev/null --silent --head --fail -k http://localhost:3000/); do - sleep 1 - done - - name: Create users - run: npx ts-node test/deploy/createAccountCredentials.ts http://localhost:3000/ >> test/deploy/conformance.env - - name: Run the test harness - run: > - docker run -i --rm - -v "$(pwd)"/reports/css:/reports - --env-file=./test/deploy/conformance.env - --network="host" - solidproject/conformance-test-harness - --skip-teardown - --output=/reports - --target=https://github.com/solid/conformance-test-harness/css - # Steps below use `always()` to make sure logs get uploaded in case the CTH errors - - name: Sanitize branch name for artifact upload - id: sanitize - if: always() - uses: yeouchien/sanitize-branch-name-action@v1 - with: - branch-name: ${{ matrix.branch }} - - name: Save the reports - if: always() - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.sanitize.outputs.sanitized-branch-name }} reports - path: reports - - name: Save the server output - if: always() - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.sanitize.outputs.sanitized-branch-name }} server output - path: server-output.log + uses: ./.github/workflows/cth-test.yml + with: + branch: ${{ matrix.branch }} From 3beb049afc9f861309f1a572e283a0ecc4f647bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 18:12:27 +0000 Subject: [PATCH 3/7] chore(deps): bump jose from 4.8.3 to 4.9.2 Bumps [jose](https://github.com/panva/jose) from 4.8.3 to 4.9.2. - [Release notes](https://github.com/panva/jose/releases) - [Changelog](https://github.com/panva/jose/blob/main/CHANGELOG.md) - [Commits](https://github.com/panva/jose/compare/v4.8.3...v4.9.2) --- updated-dependencies: - dependency-name: jose dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index e29dbd5be..168b89661 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10904,9 +10904,9 @@ "dev": true }, "node_modules/jose": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.8.3.tgz", - "integrity": "sha512-7rySkpW78d8LBp4YU70Wb7+OTgE3OwAALNVZxhoIhp4Kscp+p/fBkdpxGAMKxvCAMV4QfXBU9m6l9nX/vGwd2g==", + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.9.2.tgz", + "integrity": "sha512-EqKvu2PqJCD3Jrg3PvcYZVS7D21qMVLSYMDAFcOdGUEOpJSLNtJO7NjLANvu3SYHVl6pdP2ff7ve6EZW2nX7Nw==", "funding": { "url": "https://github.com/sponsors/panva" } @@ -23934,9 +23934,9 @@ "dev": true }, "jose": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.8.3.tgz", - "integrity": "sha512-7rySkpW78d8LBp4YU70Wb7+OTgE3OwAALNVZxhoIhp4Kscp+p/fBkdpxGAMKxvCAMV4QfXBU9m6l9nX/vGwd2g==" + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.9.2.tgz", + "integrity": "sha512-EqKvu2PqJCD3Jrg3PvcYZVS7D21qMVLSYMDAFcOdGUEOpJSLNtJO7NjLANvu3SYHVl6pdP2ff7ve6EZW2nX7Nw==" }, "js-tokens": { "version": "4.0.0", From 79878240682b2f7e335db4edd4c6a4ed8044a170 Mon Sep 17 00:00:00 2001 From: Ruben Verborgh Date: Mon, 19 Sep 2022 20:15:30 +0200 Subject: [PATCH 4/7] fix: Clarify application consent screen. --- .../identity/email-password/consent.html.ejs | 92 ++++++++++--------- templates/scripts/util.js | 10 ++ templates/styles/main.css | 39 +++++++- 3 files changed, 96 insertions(+), 45 deletions(-) diff --git a/templates/identity/email-password/consent.html.ejs b/templates/identity/email-password/consent.html.ejs index b38a888f1..543203e8b 100644 --- a/templates/identity/email-password/consent.html.ejs +++ b/templates/identity/email-password/consent.html.ejs @@ -1,8 +1,12 @@ -

Authorize

-

Your WebID:

-

The following client wants to do authorized requests in your name:

-
    -
+

An application is requesting access

+

+ Your WebID is +

+

+ Do you trust this application + to read and write data on your behalf? +

+

@@ -15,52 +19,54 @@

- - + + +

- - diff --git a/templates/scripts/util.js b/templates/scripts/util.js index 2b07423b5..117315be7 100644 --- a/templates/scripts/util.js +++ b/templates/scripts/util.js @@ -1,3 +1,13 @@ +/** + * Returns an object that maps IDs to the corresponding element. + * + * @param ...ids - IDs of the element (empty to retrieve all elements) + */ +function getElements(...ids) { + ids = ids.length ? ids : [...document.querySelectorAll("[id]")].map(e => e.id); + return Object.fromEntries(ids.map(id => [id, document.getElementById(id)])); +} + /** * Acquires all data from the given form and POSTs it as JSON to the target URL. * In case of failure this function will throw an error. diff --git a/templates/styles/main.css b/templates/styles/main.css index 77d8a1bf2..7e5e21ed4 100644 --- a/templates/styles/main.css +++ b/templates/styles/main.css @@ -138,6 +138,20 @@ ul ul, ol ul { padding-left: 1em; } +dl { + padding: 0 0 0 2em; + margin: .75em 0 0; +} +dl dt { + font-weight: bold; + float: left; + clear: left; + min-width: 4em; +} +dl dt:after { + content: ": "; +} + pre { overflow-x: scroll; } @@ -207,12 +221,30 @@ button { border: 0px; background-color: var(--solid-purple); color: white; - - font-weight: 600; } button:hover { background-color: var(--solid-blue); } +button[type=submit] { + font-weight: 600; +} + +button.alternate { + display: block; + padding: 0; + margin: .5em 0; + + background: none; + color: var(--solid-purple); + font-weight: normal; +} +button.alternate:hover { + color: var(--solid-blue); +} + +input:focus, button:focus { + outline: var(--solid-blue) solid 1.5px; +} form p.actions { margin: .5em 0 1em 11em; @@ -222,6 +254,9 @@ form p.error { color: #ad0f0f; font-weight: 600; } +form p.error:empty { + display :none; +} form ul.actions { padding: 0; From 7884348c2f4e8f646f38cf987c58a8a47135facd Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Fri, 2 Sep 2022 13:37:46 +0200 Subject: [PATCH 5/7] fix: Always render OIDC errors correctly --- .../configuration/IdentityProviderFactory.ts | 55 +++++++++++++++++-- test/integration/Identity.test.ts | 10 +++- .../IdentityProviderFactory.test.ts | 41 +++++++++++++- 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/src/identity/configuration/IdentityProviderFactory.ts b/src/identity/configuration/IdentityProviderFactory.ts index a3fc51489..10bf782ad 100644 --- a/src/identity/configuration/IdentityProviderFactory.ts +++ b/src/identity/configuration/IdentityProviderFactory.ts @@ -17,6 +17,7 @@ import type { Operation } from '../../http/Operation'; import type { ErrorHandler } from '../../http/output/error/ErrorHandler'; import type { ResponseWriter } from '../../http/output/ResponseWriter'; import { BasicRepresentation } from '../../http/representation/BasicRepresentation'; +import { getLoggerFor } from '../../logging/LogUtil'; import type { KeyValueStorage } from '../../storage/keyvalue/KeyValueStorage'; import { InternalServerError } from '../../util/errors/InternalServerError'; import { RedirectHttpError } from '../../util/errors/RedirectHttpError'; @@ -77,6 +78,8 @@ const COOKIES_KEY = 'cookie-secret'; * Routes will be updated based on the `baseUrl` and `oidcPath`. */ export class IdentityProviderFactory implements ProviderFactory { + protected readonly logger = getLoggerFor(this); + private readonly config: Configuration; private readonly adapterFactory: AdapterFactory; private readonly baseUrl: string; @@ -136,9 +139,44 @@ export class IdentityProviderFactory implements ProviderFactory { const provider = new Provider(this.baseUrl, config); provider.proxy = true; + this.captureErrorResponses(provider); + return provider; } + /** + * In the `configureErrors` function below, we configure the `renderError` function of the provider configuration. + * This function is called by the OIDC provider library to render errors, + * but only does this if the accept header is HTML. + * Otherwise, it just returns the error object iself as a JSON object. + * See https://github.com/panva/node-oidc-provider/blob/0fcc112e0a95b3b2dae4eba6da812253277567c9/lib/shared/error_handler.js#L48-L52. + * + * In this function we override the `ctx.accepts` function + * to make the above code think HTML is always requested there. + * This way we have full control over error representation as configured in `configureErrors`. + * We still check the accept headers ourselves so there still is content negotiation on the output, + * the client will not simply always receive HTML. + * + * Should this part of the OIDC library code ever change, our function will break, + * at which point behaviour will simply revert to what it was before. + */ + private captureErrorResponses(provider: Provider): void { + provider.use(async(ctx, next): Promise => { + const accepts = ctx.accepts.bind(ctx); + + // Using `any` typings to make sure we support all different versions of `ctx.accepts` + ctx.accepts = (...types: any[]): any => { + // Make sure we only override our specific case + if (types.length === 2 && types[0] === 'json' && types[1] === 'html') { + return 'html'; + } + return accepts(...types); + }; + + return next(); + }); + } + /** * Creates a configuration by copying the internal configuration * and adding the adapter, default audience and jwks/cookie keys. @@ -340,14 +378,19 @@ export class IdentityProviderFactory implements ProviderFactory { // Doesn't really matter which type it is since all relevant fields are optional const oidcError = error as errors.OIDCProviderError; + let detailedError = error.message; + if (oidcError.error_description) { + detailedError += ` - ${oidcError.error_description}`; + } + if (oidcError.error_detail) { + detailedError += ` - ${oidcError.error_detail}`; + } + + this.logger.warn(`OIDC request failed: ${detailedError}`); + // OIDC library hides extra details in these fields if (this.showStackTrace) { - if (oidcError.error_description) { - error.message += ` - ${oidcError.error_description}`; - } - if (oidcError.error_detail) { - oidcError.message += ` - ${oidcError.error_detail}`; - } + error.message = detailedError; // Also change the error message in the stack trace if (error.stack) { diff --git a/test/integration/Identity.test.ts b/test/integration/Identity.test.ts index 5814ba264..d1c3ee3c8 100644 --- a/test/integration/Identity.test.ts +++ b/test/integration/Identity.test.ts @@ -621,9 +621,13 @@ describe('A Solid server with IDP', (): void => { }); it('should return correct error output.', async(): Promise => { - const res = await fetch(`${baseUrl}.oidc/auth`); - expect(res.status).toBe(400); - await expect(res.text()).resolves.toContain('InvalidRequest: invalid_request'); + const res = await fetch(`${baseUrl}.oidc/foo`, { headers: { accept: 'application/json' }}); + expect(res.status).toBe(404); + const json = await res.json(); + expect(json.name).toBe(`InvalidRequest`); + expect(json.message).toBe(`invalid_request - unrecognized route or not allowed method (GET on /.oidc/foo)`); + expect(json.statusCode).toBe(404); + expect(json.stack).toBeDefined(); }); }); }); diff --git a/test/unit/identity/configuration/IdentityProviderFactory.test.ts b/test/unit/identity/configuration/IdentityProviderFactory.test.ts index 45bb09b4b..05701709d 100644 --- a/test/unit/identity/configuration/IdentityProviderFactory.test.ts +++ b/test/unit/identity/configuration/IdentityProviderFactory.test.ts @@ -1,4 +1,5 @@ import { Readable } from 'stream'; +import type * as Koa from 'koa'; import type { errors, Configuration, KoaContextWithOIDC } from 'oidc-provider'; import type { ErrorHandler } from '../../../../src/http/output/error/ErrorHandler'; import type { ResponseWriter } from '../../../../src/http/output/ResponseWriter'; @@ -14,7 +15,8 @@ import { FoundHttpError } from '../../../../src/util/errors/FoundHttpError'; /* eslint-disable @typescript-eslint/naming-convention */ jest.mock('oidc-provider', (): any => ({ - Provider: jest.fn().mockImplementation((issuer: string, config: Configuration): any => ({ issuer, config })), + Provider: jest.fn().mockImplementation((issuer: string, config: Configuration): any => + ({ issuer, config, use: jest.fn() })), })); const routes = { @@ -58,6 +60,7 @@ describe('An IdentityProviderFactory', (): void => { request: { href: 'http://example.com/idp/', }, + accepts: jest.fn().mockReturnValue('type'), } as any; interactionHandler = { @@ -146,10 +149,11 @@ describe('An IdentityProviderFactory', (): void => { }); // Test the renderError function - await expect((config.renderError as any)(ctx, {}, 'error!')).resolves.toBeUndefined(); + const error = new Error('error!'); + await expect((config.renderError as any)(ctx, {}, error)).resolves.toBeUndefined(); expect(errorHandler.handleSafe).toHaveBeenCalledTimes(1); expect(errorHandler.handleSafe) - .toHaveBeenLastCalledWith({ error: 'error!', request: ctx.req }); + .toHaveBeenLastCalledWith({ error, request: ctx.req }); expect(responseWriter.handleSafe).toHaveBeenCalledTimes(1); expect(responseWriter.handleSafe).toHaveBeenLastCalledWith({ response: ctx.res, result: { statusCode: 500 }}); }); @@ -230,4 +234,35 @@ describe('An IdentityProviderFactory', (): void => { expect(error.message).toBe('bad data - more info - more details'); expect(error.stack).toContain('Error: bad data - more info - more details'); }); + + it('adds middleware to make the OIDC provider think the request wants HTML.', async(): Promise => { + const provider = await factory.getProvider() as any; + const use = provider.use as jest.Mock, Parameters>; + expect(use).toHaveBeenCalledTimes(1); + const middleware = use.mock.calls[0][0]; + + const oldAccept = ctx.accepts; + const next = jest.fn(); + await expect(middleware(ctx, next)).resolves.toBeUndefined(); + expect(next).toHaveBeenCalledTimes(1); + + expect(ctx.accepts('json', 'html')).toBe('html'); + expect(oldAccept).toHaveBeenCalledTimes(0); + }); + + it('does not modify the context accepts function in other cases.', async(): Promise => { + const provider = await factory.getProvider() as any; + const use = provider.use as jest.Mock, Parameters>; + expect(use).toHaveBeenCalledTimes(1); + const middleware = use.mock.calls[0][0]; + + const oldAccept = ctx.accepts; + const next = jest.fn(); + await expect(middleware(ctx, next)).resolves.toBeUndefined(); + expect(next).toHaveBeenCalledTimes(1); + + expect(ctx.accepts('something')).toBe('type'); + expect(oldAccept).toHaveBeenCalledTimes(1); + expect(oldAccept).toHaveBeenLastCalledWith('something'); + }); }); From e050f8be93b7ea9596fdaf250de9f0bf32fa4fd8 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Tue, 23 Aug 2022 16:37:07 +0200 Subject: [PATCH 6/7] feat: Allow multiple configurations to be used during startup --- README.md | 26 +++++++++++++------------- config/app/variables/cli/cli.json | 4 ++-- src/init/AppRunner.ts | 22 ++++++++++++---------- test/integration/Cli.test.ts | 2 +- test/unit/init/AppRunner.test.ts | 31 +++++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c9b2db73b..3027b684c 100644 --- a/README.md +++ b/README.md @@ -122,19 +122,19 @@ by passing parameters to the server command. These parameters give you direct access to some commonly used settings: -| parameter name | default value | description | -|------------------------|----------------------------|--------------------------------------------------------------------------------------------------------------------------------------| -| `--port, -p` | `3000` | The TCP port on which the server should listen. | -| `--baseUrl, -b` | `http://localhost:$PORT/` | The base URL used internally to generate URLs. Change this if your server does not run on `http://localhost:$PORT/`. | -| `--loggingLevel, -l` | `info` | The detail level of logging; useful for debugging problems. Use `debug` for full information. | -| `--config, -c` | `@css:config/default.json` | The configuration for the server. The default only stores data in memory; to persist to your filesystem, use `@css:config/file.json` | -| `--rootFilePath, -f` | `./` | Root folder where the server stores data, when using a file-based configuration. | -| `--sparqlEndpoint, -s` | | URL of the SPARQL endpoint, when using a quadstore-based configuration. | -| `--showStackTrace, -t` | false | Enables detailed logging on error output. | -| `--podConfigJson` | `./pod-config.json` | Path to the file that keeps track of dynamic Pod configurations. Only relevant when using `@css:config/dynamic.json`. | -| `--seededPodConfigJson`| | Path to the file that keeps track of seeded Pod configurations. | -| `--mainModulePath, -m` | | Path from where Components.js will start its lookup when initializing configurations. | -| `--workers, -w` | `1` | Run in multithreaded mode using workers. Special values are `-1` (scale to `num_cores-1`), `0` (scale to `num_cores`) and 1 (singlethreaded). | +| parameter name | default value | description | +|------------------------|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `--port, -p` | `3000` | The TCP port on which the server should listen. | +| `--baseUrl, -b` | `http://localhost:$PORT/` | The base URL used internally to generate URLs. Change this if your server does not run on `http://localhost:$PORT/`. | +| `--loggingLevel, -l` | `info` | The detail level of logging; useful for debugging problems. Use `debug` for full information. | +| `--config, -c` | `@css:config/default.json` | The configuration(s) for the server. The default only stores data in memory; to persist to your filesystem, use `@css:config/file.json` | +| `--rootFilePath, -f` | `./` | Root folder where the server stores data, when using a file-based configuration. | +| `--sparqlEndpoint, -s` | | URL of the SPARQL endpoint, when using a quadstore-based configuration. | +| `--showStackTrace, -t` | false | Enables detailed logging on error output. | +| `--podConfigJson` | `./pod-config.json` | Path to the file that keeps track of dynamic Pod configurations. Only relevant when using `@css:config/dynamic.json`. | +| `--seededPodConfigJson`| | Path to the file that keeps track of seeded Pod configurations. | +| `--mainModulePath, -m` | | Path from where Components.js will start its lookup when initializing configurations. | +| `--workers, -w` | `1` | Run in multithreaded mode using workers. Special values are `-1` (scale to `num_cores-1`), `0` (scale to `num_cores`) and 1 (singlethreaded). | ### 🔀 Multithreading diff --git a/config/app/variables/cli/cli.json b/config/app/variables/cli/cli.json index b9eeaf73c..bbeb70111 100644 --- a/config/app/variables/cli/cli.json +++ b/config/app/variables/cli/cli.json @@ -12,8 +12,8 @@ "options": { "alias": "c", "requiresArg": true, - "type": "string", - "describe": "The configuration for the server. The default only stores data in memory; to persist to your filesystem, use @css:config/file.json." + "type": "array", + "describe": "The configuration(s) for the server. The default only stores data in memory; to persist to your filesystem, use @css:config/file.json." } }, { diff --git a/src/init/AppRunner.ts b/src/init/AppRunner.ts index 09e8126ca..32cd1237d 100644 --- a/src/init/AppRunner.ts +++ b/src/init/AppRunner.ts @@ -21,7 +21,7 @@ const DEFAULT_CLI_RESOLVER = 'urn:solid-server-app-setup:default:CliResolver'; const DEFAULT_APP = 'urn:solid-server:default:App'; const CORE_CLI_PARAMETERS = { - config: { type: 'string', alias: 'c', default: DEFAULT_CONFIG, requiresArg: true }, + config: { type: 'array', alias: 'c', default: [ DEFAULT_CONFIG ], requiresArg: true }, loggingLevel: { type: 'string', alias: 'l', default: 'info', requiresArg: true, choices: LOG_LEVELS }, mainModulePath: { type: 'string', alias: 'm', requiresArg: true }, } as const; @@ -48,13 +48,13 @@ export class AppRunner { * The values in `variableBindings` take priority over those in `shorthand`. * * @param loaderProperties - Components.js loader properties. - * @param configFile - Path to the server config file. + * @param configFile - Path to the server config file(s). * @param variableBindings - Bindings of Components.js variables. * @param shorthand - Shorthand values that need to be resolved. */ public async run( loaderProperties: IComponentsManagerBuilderOptions, - configFile: string, + configFile: string | string[], variableBindings?: VariableBindings, shorthand?: Shorthand, ): Promise { @@ -75,13 +75,13 @@ export class AppRunner { * The values in `variableBindings` take priority over those in `shorthand`. * * @param loaderProperties - Components.js loader properties. - * @param configFile - Path to the server config file. + * @param configFile - Path to the server config file(s). * @param variableBindings - Bindings of Components.js variables. * @param shorthand - Shorthand values that need to be resolved. */ public async create( loaderProperties: IComponentsManagerBuilderOptions, - configFile: string, + configFile: string | string[], variableBindings?: VariableBindings, shorthand?: Shorthand, ): Promise { @@ -152,16 +152,16 @@ export class AppRunner { typeChecking: false, }; - const config = resolveAssetPath(params.config); + const configs = params.config.map(resolveAssetPath); // Create the Components.js manager used to build components from the provided config let componentsManager: ComponentsManager; try { - componentsManager = await this.createComponentsManager(loaderProperties, config); + componentsManager = await this.createComponentsManager(loaderProperties, configs); } catch (error: unknown) { // Print help of the expected core CLI parameters const help = await yargv.getHelp(); - this.resolveError(`${help}\n\nCould not build the config files from ${config}`, error); + this.resolveError(`${help}\n\nCould not build the config files from ${configs}`, error); } // Build the CLI components and use them to generate values for the Components.js variables @@ -176,10 +176,12 @@ export class AppRunner { */ public async createComponentsManager( loaderProperties: IComponentsManagerBuilderOptions, - configFile: string, + configFile: string | string[], ): Promise> { const componentsManager = await ComponentsManager.build(loaderProperties); - await componentsManager.configRegistry.register(configFile); + for (const config of Array.isArray(configFile) ? configFile : [ configFile ]) { + await componentsManager.configRegistry.register(config); + } return componentsManager; } diff --git a/test/integration/Cli.test.ts b/test/integration/Cli.test.ts index ff7fd6252..8c9f94c71 100644 --- a/test/integration/Cli.test.ts +++ b/test/integration/Cli.test.ts @@ -29,7 +29,7 @@ describe('An instantiated CliResolver', (): void => { '-s', 's', '-w', '2', ]); - expect(shorthand.config).toBe('c'); + expect(shorthand.config).toEqual([ 'c' ]); expect(shorthand.mainModulePath).toBe('m'); expect(shorthand.loggingLevel).toBe('l'); expect(shorthand.baseUrl).toBe('b'); diff --git a/test/unit/init/AppRunner.test.ts b/test/unit/init/AppRunner.test.ts index e975f9747..77e90f8c6 100644 --- a/test/unit/init/AppRunner.test.ts +++ b/test/unit/init/AppRunner.test.ts @@ -281,6 +281,37 @@ describe('AppRunner', (): void => { expect(app.start).toHaveBeenCalledTimes(0); }); + it('can apply multiple configurations.', async(): Promise => { + const params = [ + 'node', 'script', + '-c', 'config1.json', 'config2.json', + ]; + await expect(new AppRunner().createCli(params)).resolves.toBe(app); + + expect(ComponentsManager.build).toHaveBeenCalledTimes(1); + expect(ComponentsManager.build).toHaveBeenCalledWith({ + dumpErrorState: true, + logLevel: 'info', + mainModulePath: joinFilePath(__dirname, '../../../'), + typeChecking: false, + }); + expect(manager.configRegistry.register).toHaveBeenCalledTimes(2); + expect(manager.configRegistry.register).toHaveBeenNthCalledWith(1, '/var/cwd/config1.json'); + expect(manager.configRegistry.register).toHaveBeenNthCalledWith(2, '/var/cwd/config2.json'); + expect(manager.instantiate).toHaveBeenCalledTimes(2); + expect(manager.instantiate).toHaveBeenNthCalledWith(1, 'urn:solid-server-app-setup:default:CliResolver', {}); + expect(cliExtractor.handleSafe).toHaveBeenCalledTimes(1); + expect(cliExtractor.handleSafe).toHaveBeenCalledWith(params); + expect(shorthandResolver.handleSafe).toHaveBeenCalledTimes(1); + expect(shorthandResolver.handleSafe).toHaveBeenCalledWith(defaultParameters); + expect(manager.instantiate).toHaveBeenNthCalledWith(1, 'urn:solid-server-app-setup:default:CliResolver', {}); + expect(manager.instantiate).toHaveBeenNthCalledWith(2, + 'urn:solid-server:default:App', + { variables: defaultVariables }); + expect(app.clusterManager.isSingleThreaded()).toBeFalsy(); + expect(app.start).toHaveBeenCalledTimes(0); + }); + it('uses the default process.argv in case none are provided.', async(): Promise => { const { argv } = process; const argvParameters = [ From b0924bf168500070a287cef300047f874deebe0c Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Mon, 26 Sep 2022 16:06:49 +0200 Subject: [PATCH 7/7] feat: Allow JSON-LD contexts to be stored locally --- package-lock.json | 1 + package.json | 2 +- src/storage/conversion/RdfToQuadConverter.ts | 45 +++++++++++++- test/assets/contexts/test.jsonld | 7 +++ .../conversion/RdfToQuadConverter.test.ts | 61 ++++++++++++++++++- 5 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 test/assets/contexts/test.jsonld diff --git a/package-lock.json b/package-lock.json index 168b89661..a013e844c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "5.0.0", "license": "MIT", "dependencies": { + "@comunica/context-entries": "^2.2.0", "@comunica/query-sparql": "^2.2.1", "@rdfjs/types": "^1.1.0", "@solid/access-token-verifier": "^2.0.3", diff --git a/package.json b/package.json index f2a9778fc..4f1f804d9 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "templates" ], "dependencies": { + "@comunica/context-entries": "^2.2.0", "@comunica/query-sparql": "^2.2.1", "@rdfjs/types": "^1.1.0", "@solid/access-token-verifier": "^2.0.3", @@ -193,7 +194,6 @@ "nodemon": "^2.0.19", "set-cookie-parser": "^2.5.1", "simple-git": "^3.12.0", - "commit-and-tag-version": "^10.1.0", "supertest": "^6.2.4", "ts-jest": "^27.1.5", "ts-node": "^10.9.1", diff --git a/src/storage/conversion/RdfToQuadConverter.ts b/src/storage/conversion/RdfToQuadConverter.ts index e68b45162..1a8839c88 100644 --- a/src/storage/conversion/RdfToQuadConverter.ts +++ b/src/storage/conversion/RdfToQuadConverter.ts @@ -1,25 +1,61 @@ import { PassThrough } from 'stream'; +import { KeysRdfParseJsonLd } from '@comunica/context-entries'; import type { NamedNode } from '@rdfjs/types'; +import fetch from 'cross-fetch'; +import { readJsonSync } from 'fs-extra'; +import { FetchDocumentLoader } from 'jsonld-context-parser'; +import type { IJsonLdContext } from 'jsonld-context-parser'; import rdfParser from 'rdf-parse'; import { BasicRepresentation } from '../../http/representation/BasicRepresentation'; import type { Representation } from '../../http/representation/Representation'; import { RepresentationMetadata } from '../../http/representation/RepresentationMetadata'; import { INTERNAL_QUADS } from '../../util/ContentTypes'; import { BadRequestHttpError } from '../../util/errors/BadRequestHttpError'; +import { resolveAssetPath } from '../../util/PathUtil'; import { pipeSafely } from '../../util/StreamUtil'; import { PREFERRED_PREFIX_TERM, SOLID_META } from '../../util/Vocabularies'; import { BaseTypedRepresentationConverter } from './BaseTypedRepresentationConverter'; import type { RepresentationConverterArgs } from './RepresentationConverter'; +/** + * First checks if a context is stored locally before letting the super class do a fetch. + */ +class ContextDocumentLoader extends FetchDocumentLoader { + private readonly contexts: Record; + + public constructor(contexts: Record) { + super(fetch); + this.contexts = {}; + for (const [ key, path ] of Object.entries(contexts)) { + this.contexts[key] = readJsonSync(resolveAssetPath(path)); + } + } + + public async load(url: string): Promise { + if (url in this.contexts) { + return this.contexts[url]; + } + return super.load(url); + } +} + /** * Converts most major RDF serializations to `internal/quads`. + * + * Custom contexts can be defined to be used when parsing JSON-LD. + * The keys of the object should be the URL of the context, + * and the values the file path of the contexts to use when the JSON-LD parser would fetch the given context. + * We use filepaths because embedding them directly into the configurations breaks Components.js. */ export class RdfToQuadConverter extends BaseTypedRepresentationConverter { - public constructor() { + private readonly documentLoader: ContextDocumentLoader; + + public constructor(contexts: Record = {}) { const inputTypes = rdfParser.getContentTypes() // ContentType application/json MAY NOT be converted to Quad. .then((types): string[] => types.filter((type): boolean => type !== 'application/json')); super(inputTypes, INTERNAL_QUADS); + this.documentLoader = new ContextDocumentLoader(contexts); } public async handle({ representation, identifier }: RepresentationConverterArgs): Promise { @@ -27,7 +63,12 @@ export class RdfToQuadConverter extends BaseTypedRepresentationConverter { const rawQuads = rdfParser.parse(representation.data, { contentType: representation.metadata.contentType!, baseIRI: identifier.path, - }) + // All extra keys get passed in the Comunica ActionContext + // and this is the key that is used to define the document loader. + // See https://github.com/rubensworks/rdf-parse.js/blob/master/lib/RdfParser.ts + // and https://github.com/comunica/comunica/blob/master/packages/actor-rdf-parse-jsonld/lib/ActorRdfParseJsonLd.ts + [KeysRdfParseJsonLd.documentLoader.name]: this.documentLoader, + } as any) // This works only for those cases where the data stream has been completely read before accessing the metadata. // Eg. the PATCH operation, which is the main case why we store the prefixes in metadata here if there are any. // See also https://github.com/CommunitySolidServer/CommunitySolidServer/issues/126 diff --git a/test/assets/contexts/test.jsonld b/test/assets/contexts/test.jsonld new file mode 100644 index 000000000..1e2299865 --- /dev/null +++ b/test/assets/contexts/test.jsonld @@ -0,0 +1,7 @@ +{ + "@context": { + "@version": 1.1, + "test": "http://example.com/context#", + "testPredicate": { "@id": "test:predicate" } + } +} diff --git a/test/unit/storage/conversion/RdfToQuadConverter.test.ts b/test/unit/storage/conversion/RdfToQuadConverter.test.ts index a246740d0..0b1878aa1 100644 --- a/test/unit/storage/conversion/RdfToQuadConverter.test.ts +++ b/test/unit/storage/conversion/RdfToQuadConverter.test.ts @@ -1,6 +1,7 @@ import 'jest-rdf'; import { Readable } from 'stream'; import arrayifyStream from 'arrayify-stream'; +import fetch, { Headers } from 'cross-fetch'; import { DataFactory } from 'n3'; import rdfParser from 'rdf-parse'; import { PREFERRED_PREFIX_TERM, SOLID_META } from '../../../../src'; @@ -14,10 +15,25 @@ import { INTERNAL_QUADS } from '../../../../src/util/ContentTypes'; import { BadRequestHttpError } from '../../../../src/util/errors/BadRequestHttpError'; const { namedNode, triple, literal, quad } = DataFactory; -describe('A RdfToQuadConverter', (): void => { - const converter = new RdfToQuadConverter(); - const identifier: ResourceIdentifier = { path: 'path' }; +// All of this is necessary to not break the cross-fetch imports that happen in `rdf-parse` +jest.mock('cross-fetch', (): any => { + const mock = jest.fn(); + // Require the original module to not be mocked... + const originalFetch = jest.requireActual('cross-fetch'); + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + __esModule: true, + ...originalFetch, + fetch: mock, + default: mock, + }; +}); +// Not mocking `fs` since this breaks the `rdf-parser` library +describe('A RdfToQuadConverter', (): void => { + const fetchMock: jest.Mock = fetch as any; + const converter = new RdfToQuadConverter(); + const identifier: ResourceIdentifier = { path: 'http://example.com/resource' }; it('supports serializing as quads.', async(): Promise => { const types = rdfParser.getContentTypes() .then((inputTypes): string[] => inputTypes.filter((type): boolean => type !== 'application/json')); @@ -123,4 +139,43 @@ describe('A RdfToQuadConverter', (): void => { expect(result.metadata.contentType).toEqual(INTERNAL_QUADS); await expect(arrayifyStream(result.data)).rejects.toThrow(BadRequestHttpError); }); + + it('can use locally stored contexts.', async(): Promise => { + const fetchedContext = { + '@context': { + '@version': 1.1, + test: 'http://example.com/context2#', + testPredicate2: { '@id': 'test:predicate2' }, + }, + }; + // This depends on the fields needed by the `jsonld-context-parser` so could break if library changes + fetchMock.mockResolvedValueOnce({ + json: (): any => fetchedContext, + status: 200, + ok: true, + headers: new Headers({ 'content-type': 'application/ld+json' }), + }); + + const contextConverter = new RdfToQuadConverter( + { 'http://example.com/context.jsonld': '@css:test/assets/contexts/test.jsonld' }, + ); + const jsonld = { + '@context': [ 'http://example.com/context.jsonld', 'http://example.com/context2.jsonld' ], + '@id': 'http://example.com/resource', + testPredicate: 123, + testPredicate2: 456, + }; + const representation = new BasicRepresentation(JSON.stringify(jsonld), 'application/ld+json'); + const preferences: RepresentationPreferences = { type: { [INTERNAL_QUADS]: 1 }}; + const result = await contextConverter.handle({ identifier, representation, preferences }); + await expect(arrayifyStream(result.data)).resolves.toEqualRdfQuadArray([ triple( + namedNode('http://example.com/resource'), + namedNode('http://example.com/context#predicate'), + literal(123), + ), triple( + namedNode('http://example.com/resource'), + namedNode('http://example.com/context2#predicate2'), + literal(456), + ) ]); + }); });