feat: Add support for StreamingHTTPChannel2023 notifications

* feat: initial StremingHTTPChannel2023 notifications

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* test: unit for StremingHTTPChannel2023 notifications

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* test: integration for StremingHTTPChannel2023 notifications

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* emit initial notification on streaming http channel

* fix linting erros

* ensure canceling fetch body in integration tests

* extract defaultChannel for topic into util

* add documentation

* Apply suggestions from code review

Co-authored-by: Ted Thibodeau Jr <tthibodeau@openlinksw.com>

* only generate notifications when needed

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* test: set body timeout to pass on node >21

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* address review feedback

* remove node 21 workaround

* add architecture documentation

* Apply suggestions from code review

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>

---------

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>
Co-authored-by: Ted Thibodeau Jr <tthibodeau@openlinksw.com>
Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>
This commit is contained in:
elf Pavlik 2024-05-22 00:58:26 -06:00 committed by GitHub
parent 203f80020c
commit cb38613b4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 1121 additions and 1 deletions

View File

@ -6,6 +6,7 @@
"css:config/http/notifications/base/http.json",
"css:config/http/notifications/base/listener.json",
"css:config/http/notifications/base/storage.json",
"css:config/http/notifications/streaming-http/http.json",
"css:config/http/notifications/websockets/handler.json",
"css:config/http/notifications/websockets/http.json",
"css:config/http/notifications/websockets/subscription.json",

View File

@ -0,0 +1,14 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
"import": [
"css:config/http/notifications/base/handler.json",
"css:config/http/notifications/base/http.json",
"css:config/http/notifications/base/storage.json",
"css:config/http/notifications/streaming-http/http.json"
],
"@graph": [
{
"comment": "All the relevant components are made in the specific imports seen above."
}
]
}

View File

@ -0,0 +1,87 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
"@graph": [
{
"comment": "Path prefix used by streaming HTTP receiveFrom endpoints",
"@id": "urn:solid-server:default:variable:streamingHTTPReceiveFromPrefix",
"valueRaw": ".notifications/StreamingHTTPChannel2023/"
},
{
"comment": "Creates updatesViaStreamingHttp2023 Link relations",
"@id": "urn:solid-server:default:StreamingHttpMetadataWriter",
"@type": "StreamingHttpMetadataWriter",
"baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
"pathPrefix": { "@id": "urn:solid-server:default:variable:streamingHTTPReceiveFromPrefix" }
},
{
"comment": "Allows discovery of the corresponding streaming HTTP channel",
"@id": "urn:solid-server:default:MetadataWriter",
"@type": "ParallelHandler",
"handlers": [
{ "@id": "urn:solid-server:default:StreamingHttpMetadataWriter" }
]
},
{
"comment": "Handles the request targeting a StreamingHTTPChannel2023 receiveFrom endpoint.",
"@id": "urn:solid-server:default:StreamingHttp2023Router",
"@type": "OperationRouterHandler",
"baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
"allowedMethods": [ "GET" ],
"allowedPathNames": [ "/StreamingHTTPChannel2023/" ],
"handler": {
"@id": "urn:solid-server:default:StreamingHttp2023RequestHandler",
"@type": "StreamingHttpRequestHandler",
"streamMap": { "@id": "urn:solid-server:default:StreamingHttpMap" },
"pathPrefix": { "@id": "urn:solid-server:default:variable:streamingHTTPReceiveFromPrefix" },
"generator": { "@id": "urn:solid-server:default:BaseNotificationGenerator" },
"serializer": { "@id": "urn:solid-server:default:BaseNotificationSerializer" },
"credentialsExtractor": { "@id": "urn:solid-server:default:CredentialsExtractor" },
"permissionReader": { "@id": "urn:solid-server:default:PermissionReader" },
"authorizer": { "@id": "urn:solid-server:default:Authorizer" }
}
},
{
"comment": "Add the router to notification type handler",
"@id": "urn:solid-server:default:NotificationTypeHandler",
"@type": "WaterfallHandler",
"handlers": [
{ "@id": "urn:solid-server:default:StreamingHttp2023Router" }
]
},
{
"comment": "Opened response streams will be stored in this Map.",
"@id": "urn:solid-server:default:StreamingHttpMap",
"@type": "StreamingHttpMap"
},
{
"comment": "Emits serialized notifications through Streaming HTTP.",
"@id": "urn:solid-server:default:StreamingHttp2023Emitter",
"@type": "StreamingHttp2023Emitter",
"streamMap": { "@id": "urn:solid-server:default:StreamingHttpMap" }
},
{
"comment": "Listens to the activities emitted by the MonitoringStore.",
"@id": "urn:solid-server:default:StreamingHttpListeningActivityHandler",
"@type": "StreamingHttpListeningActivityHandler",
"emitter": { "@id": "urn:solid-server:default:ResourceStore" },
"streamMap": { "@id": "urn:solid-server:default:StreamingHttpMap" },
"source": {
"comment": "Handles the generation and serialization of notifications for StreamingHTTPChannel2023",
"@id": "urn:solid-server:default:StreamingHttpNotificationHandler",
"@type": "ComposedNotificationHandler",
"generator": { "@id": "urn:solid-server:default:BaseNotificationGenerator" },
"serializer": { "@id": "urn:solid-server:default:BaseNotificationSerializer" },
"emitter": { "@id": "urn:solid-server:default:StreamingHttp2023Emitter" },
"eTagHandler": { "@id": "urn:solid-server:default:ETagHandler" }
}
},
{
"comment": "Add the activity handler to the primary initializer",
"@id": "urn:solid-server:default:PrimaryParallelInitializer",
"@type": "ParallelHandler",
"handlers": [
{ "@id": "urn:solid-server:default:StreamingHttpListeningActivityHandler" }
]
}
]
}

View File

@ -184,3 +184,45 @@ are quite similar to those needed for WebSocketChannel2023:
* The `WebhookChannel2023Type` class contains all the necessary typing information.
* `WebhookEmitter` is the `NotificationEmitter` that sends the request.
* `WebhookUnsubscriber` and `WebhookWebId` are additional utility classes to support the spec requirements.
## StreamingHTTPChannel2023
Currently, support for [StreamingHTTPChannel2023](https://solid.github.io/notifications/streaming-http-channel-2023)
only covers default, pre-established channels made available for every resource. Those channels output `text/turtle`.
Support for custom, subscription-based channels can be added in the future.
* For discovery, there is a `StreamingHttpMetadataWriter`, which adds `Link` to every `HTTP` response header
using `rel="http://www.w3.org/ns/solid/terms#updatesViaStreamingHttp2023"`. It links directly to the `receiveFrom`
endpoint of the default, pre-established channel for that topic resource.
* Requests to `receiveFrom` endpoints are handled by a `StreamingHttpRequestHandler`.
* It performs an authorization check.
* It creates a new response stream and adds it to the `StreamingHttpMap`, indexed by the topic resource.
* It sends an initial notification, similar to notification channels using a `state` feature.
* `StreamingHttp2023Emitter` is the `NotificationEmitter` that writes notifications to matching response streams.
* `StreamingHttpListeningActivityHandler` is responsible for observing the `MonitoringStore`
and emitting notifications when needed.
It doesn't use a `NotificationChannelStorage` since the default, pre-established channels are not
subscription-based. Instead, it uses a `StreamingHttpMap` to check for active receivers.
```mermaid
flowchart TB
StreamingHttpListeningActivityHandler("<strong>StreamingHttpListeningActivityHandler</strong><br>StreamingHttpListeningActivityHandler")
StreamingHttpListeningActivityHandler --> StreamingHttpListeningActivityHandlerArgs
subgraph StreamingHttpListeningActivityHandlerArgs[" "]
StreamingHttpMap("<strong>StreamingHttpMap</strong><br><i>StreamingHttpMap</i>")
ResourceStore("<strong>ResourceStore</strong><br><i>ActivityEmitter</i>")
StreamingHttpNotificationHandler("<strong>StreamingHttpNotificationHandler</strong><br><i>ComposedNotificationHandler</i>")
end
StreamingHttpNotificationHandler --> StreamingHttpNotificationHandlerArgs
subgraph StreamingHttpNotificationHandlerArgs[" "]
direction TB
Generator("<strong>BaseNotificationGenerator</strong>")
Serializer("<strong>BaseNotificationSerializer</strong>")
Emitter("<strong>StreamingHttp2023Emitter</strong><br><i>StreamingHttp2023Emitter</i>")
ETagHandler("<strong>ETagHandler</strong>")
end
```

View File

@ -127,6 +127,31 @@ The response would then be something like this:
}
```
### Streaming HTTP
Currently, Streaming HTTP channels are only available as pre-established channels on each resource.
This means that subscribing and unsubscribing are not supported, and no subscription services are advertised.
Instead, each resource advertises the `receiveFrom` of its pre-established notification channel using HTTP Link header,
using `rel="http://www.w3.org/ns/solid/terms#updatesViaStreamingHttp2023"`.
For example, this —
```shell
curl --head 'http://localhost:3000/foo/'
```
```http
HTTP/1.1 200 OK
Link: <http://localhost:3000/.notifications/StreamingHTTPChannel2023/foo/>; rel="http://www.w3.org/ns/solid/terms#updatesViaStreamingHttp2023"
```
It is essential to remember that any HTTP request to that `receiveFrom` endpoint requires the same authorization
as a `GET` request on the resource which advertises it.
Currently, all pre-established Streaming HTTP channels have `Content-Type: text/turtle`.
Information on how to consume Streaming HTTP responses [is available on MDN](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream)
## Unsubscribing from a notification channel
!!! note

View File

@ -404,6 +404,14 @@ export * from './server/notifications/WebSocketChannel2023/WebSocket2023Util';
export * from './server/notifications/WebSocketChannel2023/WebSocketMap';
export * from './server/notifications/WebSocketChannel2023/WebSocketChannel2023Type';
// Server/Notifications/StreamingHTTPChannel2023
export * from './server/notifications/StreamingHttpChannel2023/StreamingHttp2023Emitter';
export * from './server/notifications/StreamingHttpChannel2023/StreamingHttp2023Util';
export * from './server/notifications/StreamingHttpChannel2023/StreamingHttpListeningActivityHandler';
export * from './server/notifications/StreamingHttpChannel2023/StreamingHttpMap';
export * from './server/notifications/StreamingHttpChannel2023/StreamingHttpMetadataWriter';
export * from './server/notifications/StreamingHttpChannel2023/StreamingHttpRequestHandler';
// Server/Notifications
export * from './server/notifications/ActivityEmitter';
export * from './server/notifications/BaseChannelType';

View File

@ -0,0 +1,37 @@
import { getLoggerFor } from '../../../logging/LogUtil';
import type { Representation } from '../../../http/representation/Representation';
import { AsyncHandler } from '../../../util/handlers/AsyncHandler';
import type { NotificationChannel } from '../NotificationChannel';
import type { StreamingHttpMap } from './StreamingHttpMap';
export interface StreamingHttpEmitterInput {
channel: NotificationChannel;
representation: Representation;
}
/**
* Emits notifications on StreamingHTTPChannel2023 streams.
* Uses the response streams found in the provided map.
* The key should be the identifier of the topic resource.
*/
export class StreamingHttp2023Emitter extends AsyncHandler<StreamingHttpEmitterInput> {
protected readonly logger = getLoggerFor(this);
public constructor(
private readonly streamMap: StreamingHttpMap,
) {
super();
}
public async handle({ channel, representation }: StreamingHttpEmitterInput): Promise<void> {
// Called as a NotificationEmitter: emit the notification
const streams = this.streamMap.get(channel.topic);
if (streams) {
for (const stream of streams) {
representation.data.pipe(stream, { end: false });
}
} else {
representation.data.destroy();
}
}
}

View File

@ -0,0 +1,17 @@
import type { ResourceIdentifier } from '../../../http/representation/ResourceIdentifier';
import { NOTIFY } from '../../../util/Vocabularies';
import type { NotificationChannel } from '../NotificationChannel';
/**
* Default StreamingHTTPChanel2023 for a topic.
* Currently channel description is only used internally and never sent to the client.
* The default channel uses Turtle.
*/
export function generateChannel(topic: ResourceIdentifier): NotificationChannel {
return {
id: `${topic.path}.channel`,
type: NOTIFY.StreamingHTTPChannel2023,
topic: topic.path,
accept: 'text/turtle',
};
}

View File

@ -0,0 +1,49 @@
import type { RepresentationMetadata } from '../../../http/representation/RepresentationMetadata';
import type { ResourceIdentifier } from '../../../http/representation/ResourceIdentifier';
import { getLoggerFor } from '../../../logging/LogUtil';
import { createErrorMessage } from '../../../util/errors/ErrorUtil';
import { StaticHandler } from '../../../util/handlers/StaticHandler';
import type { AS, VocabularyTerm } from '../../../util/Vocabularies';
import type { ActivityEmitter } from '../ActivityEmitter';
import type { NotificationHandler } from '../NotificationHandler';
import { generateChannel } from './StreamingHttp2023Util';
import type { StreamingHttpMap } from './StreamingHttpMap';
/**
* Listens to an {@link ActivityEmitter} and calls the stored {@link NotificationHandler}s in case of an event
* for every matching notification channel found.
*
* Extends {@link StaticHandler} so it can be more easily injected into a Components.js configuration.
* No class takes this one as input, so to make sure Components.js instantiates it,
* it needs to be added somewhere where its presence has no impact, such as the list of initializers.
*/
export class StreamingHttpListeningActivityHandler extends StaticHandler {
protected readonly logger = getLoggerFor(this);
public constructor(
emitter: ActivityEmitter,
private readonly streamMap: StreamingHttpMap,
private readonly source: NotificationHandler,
) {
super();
emitter.on('changed', (topic, activity, metadata): void => {
if (this.streamMap.has(topic.path)) {
this.emit(topic, activity, metadata).catch(
(error): void => {
this.logger.error(`Error trying to handle notification for ${topic.path}: ${createErrorMessage(error)}`);
},
);
}
});
}
private async emit(
topic: ResourceIdentifier,
activity: VocabularyTerm<typeof AS>,
metadata: RepresentationMetadata,
): Promise<void> {
const channel = generateChannel(topic);
return this.source.handleSafe({ channel, activity, topic, metadata });
}
}

View File

@ -0,0 +1,10 @@
import type { PassThrough } from 'node:stream';
import type { SingleThreaded } from '../../../init/cluster/SingleThreaded';
import { WrappedSetMultiMap } from '../../../util/map/WrappedSetMultiMap';
/**
* A {@link SetMultiMap} linking identifiers to a set of Streaming HTTP streams.
* An extension of {@link WrappedSetMultiMap} to make sure Components.js allows us to create this in the config,
* as {@link WrappedSetMultiMap} has a constructor not supported.
*/
export class StreamingHttpMap extends WrappedSetMultiMap<string, PassThrough> implements SingleThreaded {}

View File

@ -0,0 +1,28 @@
import { getLoggerFor } from '../../../logging/LogUtil';
import type { HttpResponse } from '../../HttpResponse';
import { addHeader } from '../../../util/HeaderUtil';
import type { RepresentationMetadata } from '../../../http/representation/RepresentationMetadata';
import { MetadataWriter } from '../../../http/output/metadata/MetadataWriter';
/**
* A {@link MetadataWriter} that adds a link to the receiveFrom endpoint
* of the corresponding Streaming HTTP notifications channel
*/
export class StreamingHttpMetadataWriter extends MetadataWriter {
protected readonly logger = getLoggerFor(this);
public constructor(
private readonly baseUrl: string,
private readonly pathPrefix: string,
) {
super();
}
public async handle(input: { response: HttpResponse; metadata: RepresentationMetadata }): Promise<void> {
const resourcePath = input.metadata.identifier.value.replace(this.baseUrl, '');
const receiveFrom = `${this.baseUrl}${this.pathPrefix}${resourcePath}`;
const link = `<${receiveFrom}>; rel="http://www.w3.org/ns/solid/terms#updatesViaStreamingHttp2023"`;
this.logger.debug('Adding updatesViaStreamingHttp2023 to the Link header');
addHeader(input.response, 'Link', link);
}
}

View File

@ -0,0 +1,79 @@
import { PassThrough } from 'node:stream';
import type { Credentials } from '../../../authentication/Credentials';
import type { CredentialsExtractor } from '../../../authentication/CredentialsExtractor';
import type { Authorizer } from '../../../authorization/Authorizer';
import type { PermissionReader } from '../../../authorization/PermissionReader';
import { AccessMode } from '../../../authorization/permissions/Permissions';
import { OkResponseDescription } from '../../../http/output/response/OkResponseDescription';
import type { ResponseDescription } from '../../../http/output/response/ResponseDescription';
import { BasicRepresentation } from '../../../http/representation/BasicRepresentation';
import { getLoggerFor } from '../../../logging/LogUtil';
import type { OperationHttpHandlerInput } from '../../OperationHttpHandler';
import { OperationHttpHandler } from '../../OperationHttpHandler';
import { guardStream } from '../../../util/GuardedStream';
import { IdentifierSetMultiMap } from '../../../util/map/IdentifierMap';
import { createErrorMessage } from '../../../util/errors/ErrorUtil';
import type { NotificationGenerator } from '../generate/NotificationGenerator';
import type { NotificationSerializer } from '../serialize/NotificationSerializer';
import type { StreamingHttpMap } from './StreamingHttpMap';
import { generateChannel } from './StreamingHttp2023Util';
/**
* Handles request to Streaming HTTP receiveFrom endopints.
* All allowed requests are stored in the {@link StreamingHttpMap}
*/
export class StreamingHttpRequestHandler extends OperationHttpHandler {
protected logger = getLoggerFor(this);
public constructor(
private readonly streamMap: StreamingHttpMap,
private readonly pathPrefix: string,
private readonly generator: NotificationGenerator,
private readonly serializer: NotificationSerializer,
private readonly credentialsExtractor: CredentialsExtractor,
private readonly permissionReader: PermissionReader,
private readonly authorizer: Authorizer,
) {
super();
}
public async handle({ operation, request }: OperationHttpHandlerInput): Promise<ResponseDescription> {
const topic = operation.target.path.replace(this.pathPrefix, '');
// Verify if the client is allowed to connect
const credentials = await this.credentialsExtractor.handleSafe(request);
await this.authorize(credentials, topic);
const stream = guardStream(new PassThrough());
this.streamMap.add(topic, stream);
stream.on('error', (): boolean => this.streamMap.deleteEntry(topic, stream));
stream.on('close', (): boolean => this.streamMap.deleteEntry(topic, stream));
const channel = generateChannel({ path: topic });
// Send initial notification
try {
const notification = await this.generator.handle({ channel, topic: { path: topic }});
const representation = await this.serializer.handleSafe({ channel, notification });
representation.data.pipe(stream, { end: false });
} catch (error: unknown) {
this.logger.error(`Problem emitting initial notification: ${createErrorMessage(error)}`);
}
// Pre-established channels use Turtle
const representation = new BasicRepresentation(topic, operation.target, channel.accept);
return new OkResponseDescription(
representation.metadata,
stream,
);
}
private async authorize(credentials: Credentials, topic: string): Promise<void> {
const requestedModes = new IdentifierSetMultiMap<AccessMode>([[{ path: topic }, AccessMode.read ]]);
this.logger.debug(`Retrieved required modes: ${[ ...requestedModes.entrySets() ].join(',')}`);
const availablePermissions = await this.permissionReader.handleSafe({ credentials, requestedModes });
this.logger.debug(`Available permissions are ${[ ...availablePermissions.entries() ].join(',')}`);
await this.authorizer.handleSafe({ credentials, requestedModes, availablePermissions });
this.logger.debug(`Authorization succeeded, creating notification channel`);
}
}

View File

@ -153,6 +153,7 @@ export const ACP = createVocabulary(
export const AS = createVocabulary(
'https://www.w3.org/ns/activitystreams#',
'object',
'target',
'Add',
'Create',
@ -231,6 +232,7 @@ export const NOTIFY = createVocabulary(
'WebhookChannel2023',
'WebSocketChannel2023',
'StreamingHTTPChannel2023',
);
export const OIDC = createVocabulary(

View File

@ -0,0 +1,329 @@
import { DataFactory, Parser, Store } from 'n3';
import { BasicRepresentation } from '../../src/http/representation/BasicRepresentation';
import type { App } from '../../src/init/App';
import type { ResourceStore } from '../../src/storage/ResourceStore';
import { joinUrl } from '../../src/util/PathUtil';
import { AS, RDF } from '../../src/util/Vocabularies';
import { getPort } from '../util/Util';
import {
getDefaultVariables,
getPresetConfigPath,
getTestConfigPath,
getTestFolder,
instantiateFromConfig,
removeFolder,
} from './Config';
import namedNode = DataFactory.namedNode;
const port = getPort('StreamingHTTPChannel2023');
const baseUrl = `http://localhost:${port}/`;
const rootFilePath = getTestFolder('StreamingHTTPChannel2023');
const stores: [string, any][] = [
[ 'in-memory storage', {
configs: [ 'storage/backend/memory.json', 'util/resource-locker/memory.json' ],
teardown: jest.fn(),
}],
[ 'on-disk storage', {
configs: [ 'storage/backend/file.json', 'util/resource-locker/file.json' ],
teardown: async(): Promise<void> => removeFolder(rootFilePath),
}],
];
async function readChunk(reader: ReadableStreamDefaultReader): Promise<Store> {
const decoder = new TextDecoder();
const parser = new Parser();
const { value } = await reader.read();
const notification = decoder.decode(value);
return new Store(parser.parse(notification));
}
describe.each(stores)('A server supporting StreamingHTTPChannel2023 using %s', (name, { configs, teardown }): void => {
let app: App;
let store: ResourceStore;
const webId = 'http://example.com/card/#me';
const topic = joinUrl(baseUrl, '/foo');
const pathPrefix = '.notifications/StreamingHTTPChannel2023';
const receiveFrom = joinUrl(baseUrl, pathPrefix, '/foo');
beforeAll(async(): Promise<void> => {
const variables = {
...getDefaultVariables(port, baseUrl),
'urn:solid-server:default:variable:rootFilePath': rootFilePath,
};
// Create and start the server
const instances = await instantiateFromConfig(
'urn:solid-server:test:Instances',
[
...configs.map(getPresetConfigPath),
getTestConfigPath('streaming-http-notifications.json'),
],
variables,
) as Record<string, any>;
({ app, store } = instances);
await app.start();
});
afterAll(async(): Promise<void> => {
await teardown();
await app.stop();
});
it('advertises streaming http endpoint in Link header.', async(): Promise<void> => {
await store.setRepresentation({ path: topic }, new BasicRepresentation('new', 'text/plain'));
const response = await fetch(topic);
expect(response.status).toBe(200);
const linkHeader = response.headers.get('link');
const match = /<([^>]+)>; rel="http:\/\/www\.w3\.org\/ns\/solid\/terms#updatesViaStreamingHttp2023"/u
.exec(linkHeader!);
expect(match![1]).toEqual(receiveFrom);
});
it('only allows GET on receiveFrom endpoint.', async(): Promise<void> => {
const methods = [ 'HEAD', 'PUT', 'POST' ];
for (const method of methods) {
const response = await fetch(receiveFrom, {
method,
});
expect(response.status).toBe(405);
}
// For some reason it differs
const del = await fetch(receiveFrom, {
method: 'DELETE',
});
expect(del.status).toBe(404);
});
it('emits initial Update if topic exists.', async(): Promise<void> => {
await store.setRepresentation({ path: topic }, new BasicRepresentation('new', 'text/plain'));
const streamingResponse = await fetch(receiveFrom);
const reader = streamingResponse.body!.getReader();
try {
const quads = await readChunk(reader);
expect(quads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Update ]);
expect(quads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
}
});
it('emits initial Delete if topic does not exist.', async(): Promise<void> => {
try {
await store.deleteResource({ path: topic });
} catch {}
const streamingResponse = await fetch(receiveFrom);
const reader = streamingResponse.body!.getReader();
try {
const quads = await readChunk(reader);
expect(quads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Delete ]);
expect(quads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
}
});
it('does not emit initial notification when other receivers connect.', async(): Promise<void> => {
await store.setRepresentation({ path: topic }, new BasicRepresentation('new', 'text/plain'));
const streamingResponse = await fetch(receiveFrom);
const reader = streamingResponse.body!.getReader();
const otherResponse = await fetch(receiveFrom);
const otherReader = otherResponse.body!.getReader();
try {
// Expected initial notification
const updateQuads = await readChunk(reader);
expect(updateQuads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Update ]);
expect(updateQuads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
// Expected initial notification on other receiver
const otherQuads = await readChunk(otherReader);
expect(otherQuads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Update ]);
expect(otherQuads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
// Delete resource
const response = await fetch(topic, {
method: 'DELETE',
});
expect(response.status).toBe(205);
// If it was caused by the other receiver connecting, it would have been Update as well
const deleteQuads = await readChunk(reader);
expect(deleteQuads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Delete ]);
expect(deleteQuads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
otherReader.releaseLock();
await otherResponse.body!.cancel();
}
});
it('emits Create events.', async(): Promise<void> => {
try {
await store.deleteResource({ path: topic });
} catch {}
const streamingResponse = await fetch(receiveFrom);
const reader = streamingResponse.body!.getReader();
try {
// Ignore initial notification
await readChunk(reader);
// Create resource
const response = await fetch(topic, {
method: 'PUT',
headers: { 'content-type': 'text/plain' },
body: 'abc',
});
expect(response.status).toBe(201);
const quads = await readChunk(reader);
expect(quads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Create ]);
expect(quads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
}
});
it('emits Update events.', async(): Promise<void> => {
await store.setRepresentation({ path: topic }, new BasicRepresentation('new', 'text/plain'));
const streamingResponse = await fetch(receiveFrom);
const reader = streamingResponse.body!.getReader();
try {
// Ignore initial notification
await readChunk(reader);
// Update resource
const response = await fetch(topic, {
method: 'PUT',
headers: { 'content-type': 'text/plain' },
body: 'abc',
});
expect(response.status).toBe(205);
const quads = await readChunk(reader);
expect(quads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Update ]);
expect(quads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
}
});
it('emits Delete events.', async(): Promise<void> => {
await store.setRepresentation({ path: topic }, new BasicRepresentation('new', 'text/plain'));
const streamingResponse = await fetch(receiveFrom);
const reader = streamingResponse.body!.getReader();
try {
// Ignore initial notification
await readChunk(reader);
// Delete resource
const response = await fetch(topic, {
method: 'DELETE',
});
expect(response.status).toBe(205);
const quads = await readChunk(reader);
expect(quads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Delete ]);
expect(quads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(topic) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
}
});
it('prevents connecting to channels of restricted topics.', async(): Promise<void> => {
const restricted = joinUrl(baseUrl, '/restricted');
const restrictedReceiveFrom = joinUrl(baseUrl, pathPrefix, '/restricted');
await store.setRepresentation({ path: restricted }, new BasicRepresentation('new', 'text/plain'));
// Only allow our WebID to read
const restrictedAcl = `
@prefix acl: <http://www.w3.org/ns/auth/acl#>.
@prefix foaf: <http://xmlns.com/foaf/0.1/>.
<#authorization>
a acl:Authorization;
acl:agent <${webId}>;
acl:mode acl:Read, acl:Write;
acl:accessTo <./restricted>.`;
await store.setRepresentation({ path: `${restricted}.acl` }, new BasicRepresentation(restrictedAcl, 'text/turtle'));
// Unauthenticated fetch fails
const unauthenticatedResponse = await fetch(restrictedReceiveFrom);
try {
expect(unauthenticatedResponse.status).toBe(401);
} finally {
await unauthenticatedResponse.body?.cancel();
}
// Authenticated fetch succeeds
const authenticatedResponse = await fetch(restrictedReceiveFrom, {
headers: {
authorization: `WebID ${webId}`,
},
});
try {
expect(authenticatedResponse.status).toBe(200);
} finally {
await authenticatedResponse.body!.cancel();
}
});
it('emits container notifications if contents get added or removed.', async(): Promise<void> => {
const resource = joinUrl(baseUrl, '/resource');
const baseReceiveFrom = joinUrl(baseUrl, pathPrefix, '/');
// Connecting to the base URL, which is the parent container
const streamingResponse = await fetch(baseReceiveFrom);
const reader = streamingResponse.body!.getReader();
try {
// Ignore initial notification
await readChunk(reader);
// Create contained resource
const createResponse = await fetch(resource, {
method: 'PUT',
headers: { 'content-type': 'text/plain' },
body: 'abc',
});
expect(createResponse.status).toBe(201);
// Will receive the Add notification
const addQuads = await readChunk(reader);
expect(addQuads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Add ]);
expect(addQuads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(resource) ]);
expect(addQuads.getObjects(null, AS.terms.target, null)).toEqual([ namedNode(baseUrl) ]);
// Remove contained resource
const removeResponse = await fetch(resource, {
method: 'DELETE',
});
expect(removeResponse.status).toBe(205);
// Will receive the Remove notification
const removeQuads = await readChunk(reader);
expect(removeQuads.getObjects(null, RDF.terms.type, null)).toEqual([ AS.terms.Remove ]);
expect(removeQuads.getObjects(null, AS.terms.object, null)).toEqual([ namedNode(resource) ]);
expect(removeQuads.getObjects(null, AS.terms.target, null)).toEqual([ namedNode(baseUrl) ]);
} finally {
reader.releaseLock();
await streamingResponse.body!.cancel();
}
});
});

View File

@ -0,0 +1,52 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
"import": [
"css:config/app/init/initialize-root.json",
"css:config/app/main/default.json",
"css:config/http/handler/default.json",
"css:config/http/middleware/default.json",
"css:config/http/notifications/streaming-http.json",
"css:config/http/server-factory/http.json",
"css:config/http/static/default.json",
"css:config/identity/access/public.json",
"css:config/identity/email/default.json",
"css:config/identity/handler/no-accounts.json",
"css:config/identity/oidc/default.json",
"css:config/identity/ownership/token.json",
"css:config/identity/pod/static.json",
"css:config/ldp/authentication/debug-auth-header.json",
"css:config/ldp/authorization/webacl.json",
"css:config/ldp/handler/default.json",
"css:config/ldp/metadata-parser/default.json",
"css:config/ldp/metadata-writer/default.json",
"css:config/ldp/modes/default.json",
"css:config/storage/key-value/resource-store.json",
"css:config/storage/location/root.json",
"css:config/storage/middleware/default.json",
"css:config/util/auxiliary/acl.json",
"css:config/util/identifiers/suffix.json",
"css:config/util/index/default.json",
"css:config/util/logging/winston.json",
"css:config/util/representation-conversion/default.json",
"css:config/util/variables/default.json"
],
"@graph": [
{
"comment": "Streaming HTTP notifications with debug authentication.",
"@id": "urn:solid-server:test:Instances",
"@type": "RecordObject",
"record": [
{
"RecordObject:_record_key": "app",
"RecordObject:_record_value": { "@id": "urn:solid-server:default:App" }
},
{
"RecordObject:_record_key": "store",
"RecordObject:_record_value": { "@id": "urn:solid-server:default:ResourceStore" }
}
]
}
]
}

View File

@ -0,0 +1,78 @@
import { PassThrough } from 'node:stream';
import { BasicRepresentation } from '../../../../../src/http/representation/BasicRepresentation';
import type { NotificationChannel } from '../../../../../src/server/notifications/NotificationChannel';
import {
StreamingHttp2023Emitter,
} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttp2023Emitter';
import { WrappedSetMultiMap } from '../../../../../src/util/map/WrappedSetMultiMap';
import type { StreamingHttpMap } from '../../../../../src';
describe('A StreamingHttp2023Emitter', (): void => {
const channel: NotificationChannel = {
id: 'id',
topic: 'http://example.com/foo',
type: 'type',
};
let stream: jest.Mocked<PassThrough>;
let streamMap: StreamingHttpMap;
let emitter: StreamingHttp2023Emitter;
beforeEach(async(): Promise<void> => {
stream = jest.mocked(new PassThrough());
streamMap = new WrappedSetMultiMap();
emitter = new StreamingHttp2023Emitter(streamMap);
});
it('emits notifications to the stored Streams.', async(): Promise<void> => {
streamMap.add(channel.topic, stream);
const representation = new BasicRepresentation('notification', 'text/plain');
const spy = jest.spyOn(representation.data, 'pipe');
await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
expect(spy).toHaveBeenCalledTimes(1);
expect(spy).toHaveBeenLastCalledWith(stream, { end: false });
});
it('destroys the representation if there is no matching Stream.', async(): Promise<void> => {
const representation = new BasicRepresentation('notification', 'text/plain');
const spy = jest.spyOn(representation.data, 'pipe');
await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
expect(spy).toHaveBeenCalledTimes(0);
expect(representation.data.destroyed).toBe(true);
});
it('can write to multiple matching Streams.', async(): Promise<void> => {
const stream2 = jest.mocked(new PassThrough());
streamMap.add(channel.topic, stream);
streamMap.add(channel.topic, stream2);
const representation = new BasicRepresentation('notification', 'text/plain');
const spy = jest.spyOn(representation.data, 'pipe');
await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
expect(spy).toHaveBeenCalledTimes(2);
expect(spy).toHaveBeenCalledWith(stream, { end: false });
expect(spy).toHaveBeenLastCalledWith(stream2, { end: false });
});
it('only writes to the matching topic Streams.', async(): Promise<void> => {
const stream2 = jest.mocked(new PassThrough());
const channel2: NotificationChannel = {
...channel,
id: 'other id',
topic: 'other topic',
};
streamMap.add(channel.topic, stream);
streamMap.add(channel2.topic, stream2);
const representation = new BasicRepresentation('notification', 'text/plain');
const spy = jest.spyOn(representation.data, 'pipe');
await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
expect(spy).toHaveBeenCalledTimes(1);
expect(spy).toHaveBeenLastCalledWith(stream, { end: false });
});
});

View File

@ -0,0 +1,19 @@
import {
generateChannel,
} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttp2023Util';
import { NOTIFY } from '../../../../../src/util/Vocabularies';
describe('StreamingHttp2023Util', (): void => {
describe('#generateChannel', (): void => {
it('returns description given topic.', (): void => {
const topic = { path: 'http://example.com/foo' };
const channel = generateChannel(topic);
expect(channel).toEqual({
id: `${topic.path}.channel`,
type: NOTIFY.StreamingHTTPChannel2023,
topic: topic.path,
accept: 'text/turtle',
});
});
});
});

View File

@ -0,0 +1,73 @@
import { EventEmitter } from 'node:events';
import { PassThrough } from 'node:stream';
import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata';
import type { ResourceIdentifier } from '../../../../../src/http/representation/ResourceIdentifier';
import type { Logger } from '../../../../../src/logging/Logger';
import { getLoggerFor } from '../../../../../src/logging/LogUtil';
import type { ActivityEmitter } from '../../../../../src/server/notifications/ActivityEmitter';
import type { NotificationHandler } from '../../../../../src/server/notifications/NotificationHandler';
import { AS } from '../../../../../src/util/Vocabularies';
import { flushPromises } from '../../../../util/Util';
import { StreamingHttpListeningActivityHandler, StreamingHttpMap } from '../../../../../src';
jest.mock('../../../../../src/logging/LogUtil', (): any => {
const logger: Logger = { error: jest.fn() } as any;
return { getLoggerFor: (): Logger => logger };
});
describe('A StreamingHttpListeningActivityHandler', (): void => {
const logger: jest.Mocked<Logger> = getLoggerFor('mock') as any;
const topic: ResourceIdentifier = { path: 'http://example.com/foo' };
const activity = AS.terms.Update;
const metadata = new RepresentationMetadata();
let emitter: ActivityEmitter;
let streamMap: StreamingHttpMap;
let notificationHandler: jest.Mocked<NotificationHandler>;
beforeEach(async(): Promise<void> => {
jest.clearAllMocks();
emitter = new EventEmitter() as any;
streamMap = new StreamingHttpMap();
notificationHandler = {
handleSafe: jest.fn().mockResolvedValue(undefined),
} as any;
// eslint-disable-next-line no-new
new StreamingHttpListeningActivityHandler(emitter, streamMap, notificationHandler);
});
it('calls the NotificationHandler if there is an event and a stream.', async(): Promise<void> => {
streamMap.add(topic.path, new PassThrough());
emitter.emit('changed', topic, activity, metadata);
await flushPromises();
expect(notificationHandler.handleSafe).toHaveBeenCalledTimes(1);
expect(notificationHandler.handleSafe).toHaveBeenLastCalledWith(
expect.objectContaining({ activity, topic, metadata }),
);
expect(logger.error).toHaveBeenCalledTimes(0);
});
it('does not call the NotificationHandler if there is an event but no stream.', async(): Promise<void> => {
emitter.emit('changed', topic, activity, metadata);
await flushPromises();
expect(notificationHandler.handleSafe).toHaveBeenCalledTimes(0);
expect(logger.error).toHaveBeenCalledTimes(0);
});
it('logs error from notification handler.', async(): Promise<void> => {
streamMap.add(topic.path, new PassThrough());
notificationHandler.handleSafe.mockRejectedValueOnce(new Error('bad input'));
emitter.emit('changed', topic, activity, metadata);
await flushPromises();
expect(logger.error).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenLastCalledWith(`Error trying to handle notification for ${topic.path}: bad input`);
});
});

View File

@ -0,0 +1,20 @@
import { createResponse } from 'node-mocks-http';
import {
StreamingHttpMetadataWriter,
} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttpMetadataWriter';
import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata';
import type { HttpResponse } from '../../../../../src/server/HttpResponse';
describe('A StreamingHttpMetadataWriter', (): void => {
const baseUrl = 'http://example.org/';
const pathPrefix = '.notifications/StreamingHTTPChannel2023/';
const writer = new StreamingHttpMetadataWriter(baseUrl, pathPrefix);
const rel = 'http://www.w3.org/ns/solid/terms#updatesViaStreamingHttp2023';
it('adds the correct link header.', async(): Promise<void> => {
const response = createResponse() as HttpResponse;
const metadata = new RepresentationMetadata({ path: 'http://example.org/foo/bar/baz' });
await expect(writer.handle({ response, metadata })).resolves.toBeUndefined();
expect(response.getHeaders()).toEqual({ link: `<http://example.org/.notifications/StreamingHTTPChannel2023/foo/bar/baz>; rel="${rel}"` });
});
});

View File

@ -0,0 +1,150 @@
import type { CredentialsExtractor } from '../../../../../src/authentication/CredentialsExtractor';
import type { Authorizer } from '../../../../../src/authorization/Authorizer';
import type { PermissionReader } from '../../../../../src/authorization/PermissionReader';
import { IdentifierMap } from '../../../../../src/util/map/IdentifierMap';
import { AccessMode } from '../../../../../src/authorization/permissions/Permissions';
import type { ResourceIdentifier } from '../../../../../src/http/representation/ResourceIdentifier';
import type { Operation } from '../../../../../src/http/Operation';
import type { NotificationChannel } from '../../../../../src/server/notifications/NotificationChannel';
import type { HttpRequest } from '../../../../../src/server/HttpRequest';
import type { HttpResponse } from '../../../../../src/server/HttpResponse';
import { BasicRepresentation } from '../../../../../src/http/representation/BasicRepresentation';
import type { Logger } from '../../../../../src/logging/Logger';
import { getLoggerFor } from '../../../../../src/logging/LogUtil';
import {
StreamingHttpRequestHandler,
} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttpRequestHandler';
import type { NotificationGenerator, NotificationSerializer } from '../../../../../src';
import { StreamingHttpMap } from '../../../../../src';
import type { Notification } from '../../../../../src/server/notifications/Notification';
import { flushPromises } from '../../../../util/Util';
jest.mock('../../../../../src/logging/LogUtil', (): any => {
const logger: Logger = { error: jest.fn(), debug: jest.fn() } as any;
return { getLoggerFor: (): Logger => logger };
});
describe('A StreamingHttpRequestHandler', (): void => {
const logger: jest.Mocked<Logger> = getLoggerFor('mock') as any;
const topic: ResourceIdentifier = { path: 'http://example.com/foo' };
const pathPrefix = '.notifications/StreamingHTTPChannel2023/';
const channel: NotificationChannel = {
id: 'id',
topic: topic.path,
type: 'type',
};
const notification: Notification = {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://www.w3.org/ns/solid/notification/v1',
],
id: `urn:123:http://example.com/foo`,
type: 'Update',
object: 'http://example.com/foo',
published: '123',
state: '"123456-text/turtle"',
};
const representation = new BasicRepresentation();
const request: HttpRequest = {} as any;
const response: HttpResponse = {} as any;
let streamMap: StreamingHttpMap;
let operation: Operation;
let generator: jest.Mocked<NotificationGenerator>;
let serializer: jest.Mocked<NotificationSerializer>;
let credentialsExtractor: jest.Mocked<CredentialsExtractor>;
let permissionReader: jest.Mocked<PermissionReader>;
let authorizer: jest.Mocked<Authorizer>;
let handler: StreamingHttpRequestHandler;
beforeEach(async(): Promise<void> => {
operation = {
method: 'GET',
target: { path: 'http://example.com/.notifications/StreamingHTTPChannel2023/foo' },
body: new BasicRepresentation(),
preferences: {},
};
streamMap = new StreamingHttpMap();
generator = {
canHandle: jest.fn(),
handle: jest.fn().mockResolvedValue(notification),
} as any;
serializer = {
handleSafe: jest.fn().mockResolvedValue(representation),
} as any;
credentialsExtractor = {
handleSafe: jest.fn().mockResolvedValue({ public: {}}),
} as any;
permissionReader = {
handleSafe: jest.fn().mockResolvedValue(new IdentifierMap([[ topic, AccessMode.read ]])),
} as any;
authorizer = {
handleSafe: jest.fn(),
} as any;
handler = new StreamingHttpRequestHandler(
streamMap,
pathPrefix,
generator,
serializer,
credentialsExtractor,
permissionReader,
authorizer,
);
});
it('stores streams.', async(): Promise<void> => {
await handler.handle({ operation, request, response });
expect([ ...streamMap.keys() ]).toHaveLength(1);
expect(streamMap.has(channel.topic)).toBe(true);
});
it('removes closed streams.', async(): Promise<void> => {
const description = await handler.handle({ operation, request, response });
expect(streamMap.has(channel.topic)).toBe(true);
description.data!.emit('close');
expect(streamMap.has(channel.topic)).toBe(false);
});
it('removes erroring streams.', async(): Promise<void> => {
const description = await handler.handle({ operation, request, response });
expect(streamMap.has(channel.topic)).toBe(true);
description.data!.emit('error');
expect(streamMap.has(channel.topic)).toBe(false);
});
it('sets content type to turtle.', async(): Promise<void> => {
const description = await handler.handle({ operation, request, response });
expect(description.metadata?.contentType).toBe('text/turtle');
});
it('responds with the stream.', async(): Promise<void> => {
const description = await handler.handle({ operation, request, response });
expect(description.data).toBeDefined();
});
it('sends initial notification.', async(): Promise<void> => {
const spy = jest.spyOn(representation.data, 'pipe');
await handler.handle({ operation, request, response });
expect(spy).toHaveBeenCalledTimes(1);
});
it('logs an error if sending initial notification fails.', async(): Promise<void> => {
serializer.handleSafe.mockRejectedValueOnce(new Error('failed'));
await handler.handle({ operation, request, response });
await flushPromises();
expect(logger.error).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenLastCalledWith(`Problem emitting initial notification: failed`);
});
it('errors on requests the Authorizer rejects.', async(): Promise<void> => {
authorizer.handleSafe.mockRejectedValue(new Error('not allowed'));
await expect(handler.handle({ operation, request, response })).rejects.toThrow('not allowed');
});
});

View File

@ -21,7 +21,6 @@ describe('A WebSocket2023Emitter', (): void => {
beforeEach(async(): Promise<void> => {
webSocket = {
send: jest.fn(),
close: jest.fn(),
} as any;
socketMap = new WrappedSetMultiMap();

View File

@ -29,6 +29,7 @@ const portNames = [
'ServerFetch',
'SetupMemory',
'SparqlStorage',
'StreamingHTTPChannel2023',
'Subdomains',
'WebhookChannel2023',
'WebhookChannel2023-client',