feat: Add support for StreamingHTTPChannel2023 notifications

* feat: initial StremingHTTPChannel2023 notifications

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* test: unit for StremingHTTPChannel2023 notifications

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* test: integration for StremingHTTPChannel2023 notifications

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* emit initial notification on streaming http channel

* fix linting erros

* ensure canceling fetch body in integration tests

* extract defaultChannel for topic into util

* add documentation

* Apply suggestions from code review

Co-authored-by: Ted Thibodeau Jr <tthibodeau@openlinksw.com>

* only generate notifications when needed

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* test: set body timeout to pass on node >21

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>

* address review feedback

* remove node 21 workaround

* add architecture documentation

* Apply suggestions from code review

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>

---------

Co-authored-by: Maciej Samoraj <maciej.samoraj@gmail.com>
Co-authored-by: Ted Thibodeau Jr <tthibodeau@openlinksw.com>
Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>

test/unit/server/notifications/StreamingHttpChannel2023/StreamingHttp2023Emitter.test.ts

@@ -0,0 +1,78 @@
+import { PassThrough } from 'node:stream';
+import { BasicRepresentation } from '../../../../../src/http/representation/BasicRepresentation';
+import type { NotificationChannel } from '../../../../../src/server/notifications/NotificationChannel';
+import {
+  StreamingHttp2023Emitter,
+} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttp2023Emitter';
+import { WrappedSetMultiMap } from '../../../../../src/util/map/WrappedSetMultiMap';
+import type { StreamingHttpMap } from '../../../../../src';
+
+describe('A StreamingHttp2023Emitter', (): void => {
+  const channel: NotificationChannel = {
+    id: 'id',
+    topic: 'http://example.com/foo',
+    type: 'type',
+  };
+
+  let stream: jest.Mocked<PassThrough>;
+  let streamMap: StreamingHttpMap;
+  let emitter: StreamingHttp2023Emitter;
+
+  beforeEach(async(): Promise<void> => {
+    stream = jest.mocked(new PassThrough());
+
+    streamMap = new WrappedSetMultiMap();
+
+    emitter = new StreamingHttp2023Emitter(streamMap);
+  });
+
+  it('emits notifications to the stored Streams.', async(): Promise<void> => {
+    streamMap.add(channel.topic, stream);
+
+    const representation = new BasicRepresentation('notification', 'text/plain');
+    const spy = jest.spyOn(representation.data, 'pipe');
+    await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
+    expect(spy).toHaveBeenCalledTimes(1);
+    expect(spy).toHaveBeenLastCalledWith(stream, { end: false });
+  });
+
+  it('destroys the representation if there is no matching Stream.', async(): Promise<void> => {
+    const representation = new BasicRepresentation('notification', 'text/plain');
+    const spy = jest.spyOn(representation.data, 'pipe');
+    await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
+    expect(spy).toHaveBeenCalledTimes(0);
+    expect(representation.data.destroyed).toBe(true);
+  });
+
+  it('can write to multiple matching Streams.', async(): Promise<void> => {
+    const stream2 = jest.mocked(new PassThrough());
+
+    streamMap.add(channel.topic, stream);
+    streamMap.add(channel.topic, stream2);
+
+    const representation = new BasicRepresentation('notification', 'text/plain');
+    const spy = jest.spyOn(representation.data, 'pipe');
+    await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
+    expect(spy).toHaveBeenCalledTimes(2);
+    expect(spy).toHaveBeenCalledWith(stream, { end: false });
+    expect(spy).toHaveBeenLastCalledWith(stream2, { end: false });
+  });
+
+  it('only writes to the matching topic Streams.', async(): Promise<void> => {
+    const stream2 = jest.mocked(new PassThrough());
+    const channel2: NotificationChannel = {
+      ...channel,
+      id: 'other id',
+      topic: 'other topic',
+    };
+
+    streamMap.add(channel.topic, stream);
+    streamMap.add(channel2.topic, stream2);
+
+    const representation = new BasicRepresentation('notification', 'text/plain');
+    const spy = jest.spyOn(representation.data, 'pipe');
+    await expect(emitter.handle({ channel, representation })).resolves.toBeUndefined();
+    expect(spy).toHaveBeenCalledTimes(1);
+    expect(spy).toHaveBeenLastCalledWith(stream, { end: false });
+  });
+});

test/unit/server/notifications/StreamingHttpChannel2023/StreamingHttp2023Util.test.ts

@@ -0,0 +1,19 @@
+import {
+  generateChannel,
+} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttp2023Util';
+import { NOTIFY } from '../../../../../src/util/Vocabularies';
+
+describe('StreamingHttp2023Util', (): void => {
+  describe('#generateChannel', (): void => {
+    it('returns description given topic.', (): void => {
+      const topic = { path: 'http://example.com/foo' };
+      const channel = generateChannel(topic);
+      expect(channel).toEqual({
+        id: `${topic.path}.channel`,
+        type: NOTIFY.StreamingHTTPChannel2023,
+        topic: topic.path,
+        accept: 'text/turtle',
+      });
+    });
+  });
+});

test/unit/server/notifications/StreamingHttpChannel2023/StreamingHttpListeningActivityHandler.test.ts

@@ -0,0 +1,73 @@
+import { EventEmitter } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata';
+import type { ResourceIdentifier } from '../../../../../src/http/representation/ResourceIdentifier';
+import type { Logger } from '../../../../../src/logging/Logger';
+import { getLoggerFor } from '../../../../../src/logging/LogUtil';
+import type { ActivityEmitter } from '../../../../../src/server/notifications/ActivityEmitter';
+import type { NotificationHandler } from '../../../../../src/server/notifications/NotificationHandler';
+import { AS } from '../../../../../src/util/Vocabularies';
+import { flushPromises } from '../../../../util/Util';
+import { StreamingHttpListeningActivityHandler, StreamingHttpMap } from '../../../../../src';
+
+jest.mock('../../../../../src/logging/LogUtil', (): any => {
+  const logger: Logger = { error: jest.fn() } as any;
+  return { getLoggerFor: (): Logger => logger };
+});
+
+describe('A StreamingHttpListeningActivityHandler', (): void => {
+  const logger: jest.Mocked<Logger> = getLoggerFor('mock') as any;
+  const topic: ResourceIdentifier = { path: 'http://example.com/foo' };
+  const activity = AS.terms.Update;
+  const metadata = new RepresentationMetadata();
+  let emitter: ActivityEmitter;
+  let streamMap: StreamingHttpMap;
+  let notificationHandler: jest.Mocked<NotificationHandler>;
+
+  beforeEach(async(): Promise<void> => {
+    jest.clearAllMocks();
+    emitter = new EventEmitter() as any;
+    streamMap = new StreamingHttpMap();
+
+    notificationHandler = {
+      handleSafe: jest.fn().mockResolvedValue(undefined),
+    } as any;
+
+    // eslint-disable-next-line no-new
+    new StreamingHttpListeningActivityHandler(emitter, streamMap, notificationHandler);
+  });
+
+  it('calls the NotificationHandler if there is an event and a stream.', async(): Promise<void> => {
+    streamMap.add(topic.path, new PassThrough());
+    emitter.emit('changed', topic, activity, metadata);
+
+    await flushPromises();
+
+    expect(notificationHandler.handleSafe).toHaveBeenCalledTimes(1);
+    expect(notificationHandler.handleSafe).toHaveBeenLastCalledWith(
+      expect.objectContaining({ activity, topic, metadata }),
+    );
+    expect(logger.error).toHaveBeenCalledTimes(0);
+  });
+
+  it('does not call the NotificationHandler if there is an event but no stream.', async(): Promise<void> => {
+    emitter.emit('changed', topic, activity, metadata);
+
+    await flushPromises();
+
+    expect(notificationHandler.handleSafe).toHaveBeenCalledTimes(0);
+    expect(logger.error).toHaveBeenCalledTimes(0);
+  });
+
+  it('logs error from notification handler.', async(): Promise<void> => {
+    streamMap.add(topic.path, new PassThrough());
+    notificationHandler.handleSafe.mockRejectedValueOnce(new Error('bad input'));
+
+    emitter.emit('changed', topic, activity, metadata);
+
+    await flushPromises();
+
+    expect(logger.error).toHaveBeenCalledTimes(1);
+    expect(logger.error).toHaveBeenLastCalledWith(`Error trying to handle notification for ${topic.path}: bad input`);
+  });
+});

test/unit/server/notifications/StreamingHttpChannel2023/StreamingHttpMetadataWriter.test.ts

@@ -0,0 +1,20 @@
+import { createResponse } from 'node-mocks-http';
+import {
+  StreamingHttpMetadataWriter,
+} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttpMetadataWriter';
+import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata';
+import type { HttpResponse } from '../../../../../src/server/HttpResponse';
+
+describe('A StreamingHttpMetadataWriter', (): void => {
+  const baseUrl = 'http://example.org/';
+  const pathPrefix = '.notifications/StreamingHTTPChannel2023/';
+  const writer = new StreamingHttpMetadataWriter(baseUrl, pathPrefix);
+  const rel = 'http://www.w3.org/ns/solid/terms#updatesViaStreamingHttp2023';
+
+  it('adds the correct link header.', async(): Promise<void> => {
+    const response = createResponse() as HttpResponse;
+    const metadata = new RepresentationMetadata({ path: 'http://example.org/foo/bar/baz' });
+    await expect(writer.handle({ response, metadata })).resolves.toBeUndefined();
+    expect(response.getHeaders()).toEqual({ link: `<http://example.org/.notifications/StreamingHTTPChannel2023/foo/bar/baz>; rel="${rel}"` });
+  });
+});

test/unit/server/notifications/StreamingHttpChannel2023/StreamingHttpRequestHandler.test.ts

@@ -0,0 +1,150 @@
+import type { CredentialsExtractor } from '../../../../../src/authentication/CredentialsExtractor';
+import type { Authorizer } from '../../../../../src/authorization/Authorizer';
+import type { PermissionReader } from '../../../../../src/authorization/PermissionReader';
+import { IdentifierMap } from '../../../../../src/util/map/IdentifierMap';
+import { AccessMode } from '../../../../../src/authorization/permissions/Permissions';
+import type { ResourceIdentifier } from '../../../../../src/http/representation/ResourceIdentifier';
+import type { Operation } from '../../../../../src/http/Operation';
+import type { NotificationChannel } from '../../../../../src/server/notifications/NotificationChannel';
+import type { HttpRequest } from '../../../../../src/server/HttpRequest';
+import type { HttpResponse } from '../../../../../src/server/HttpResponse';
+import { BasicRepresentation } from '../../../../../src/http/representation/BasicRepresentation';
+import type { Logger } from '../../../../../src/logging/Logger';
+import { getLoggerFor } from '../../../../../src/logging/LogUtil';
+
+import {
+  StreamingHttpRequestHandler,
+} from '../../../../../src/server/notifications/StreamingHttpChannel2023/StreamingHttpRequestHandler';
+import type { NotificationGenerator, NotificationSerializer } from '../../../../../src';
+import { StreamingHttpMap } from '../../../../../src';
+import type { Notification } from '../../../../../src/server/notifications/Notification';
+import { flushPromises } from '../../../../util/Util';
+
+jest.mock('../../../../../src/logging/LogUtil', (): any => {
+  const logger: Logger = { error: jest.fn(), debug: jest.fn() } as any;
+  return { getLoggerFor: (): Logger => logger };
+});
+
+describe('A StreamingHttpRequestHandler', (): void => {
+  const logger: jest.Mocked<Logger> = getLoggerFor('mock') as any;
+  const topic: ResourceIdentifier = { path: 'http://example.com/foo' };
+  const pathPrefix = '.notifications/StreamingHTTPChannel2023/';
+  const channel: NotificationChannel = {
+    id: 'id',
+    topic: topic.path,
+    type: 'type',
+  };
+  const notification: Notification = {
+    '@context': [
+      'https://www.w3.org/ns/activitystreams',
+      'https://www.w3.org/ns/solid/notification/v1',
+    ],
+    id: `urn:123:http://example.com/foo`,
+    type: 'Update',
+    object: 'http://example.com/foo',
+    published: '123',
+    state: '"123456-text/turtle"',
+  };
+  const representation = new BasicRepresentation();
+  const request: HttpRequest = {} as any;
+  const response: HttpResponse = {} as any;
+  let streamMap: StreamingHttpMap;
+  let operation: Operation;
+  let generator: jest.Mocked<NotificationGenerator>;
+  let serializer: jest.Mocked<NotificationSerializer>;
+  let credentialsExtractor: jest.Mocked<CredentialsExtractor>;
+  let permissionReader: jest.Mocked<PermissionReader>;
+  let authorizer: jest.Mocked<Authorizer>;
+  let handler: StreamingHttpRequestHandler;
+
+  beforeEach(async(): Promise<void> => {
+    operation = {
+      method: 'GET',
+      target: { path: 'http://example.com/.notifications/StreamingHTTPChannel2023/foo' },
+      body: new BasicRepresentation(),
+      preferences: {},
+    };
+
+    streamMap = new StreamingHttpMap();
+
+    generator = {
+      canHandle: jest.fn(),
+      handle: jest.fn().mockResolvedValue(notification),
+    } as any;
+
+    serializer = {
+      handleSafe: jest.fn().mockResolvedValue(representation),
+    } as any;
+
+    credentialsExtractor = {
+      handleSafe: jest.fn().mockResolvedValue({ public: {}}),
+    } as any;
+
+    permissionReader = {
+      handleSafe: jest.fn().mockResolvedValue(new IdentifierMap([[ topic, AccessMode.read ]])),
+    } as any;
+
+    authorizer = {
+      handleSafe: jest.fn(),
+    } as any;
+
+    handler = new StreamingHttpRequestHandler(
+      streamMap,
+      pathPrefix,
+      generator,
+      serializer,
+      credentialsExtractor,
+      permissionReader,
+      authorizer,
+    );
+  });
+
+  it('stores streams.', async(): Promise<void> => {
+    await handler.handle({ operation, request, response });
+    expect([ ...streamMap.keys() ]).toHaveLength(1);
+    expect(streamMap.has(channel.topic)).toBe(true);
+  });
+
+  it('removes closed streams.', async(): Promise<void> => {
+    const description = await handler.handle({ operation, request, response });
+    expect(streamMap.has(channel.topic)).toBe(true);
+    description.data!.emit('close');
+    expect(streamMap.has(channel.topic)).toBe(false);
+  });
+
+  it('removes erroring streams.', async(): Promise<void> => {
+    const description = await handler.handle({ operation, request, response });
+    expect(streamMap.has(channel.topic)).toBe(true);
+    description.data!.emit('error');
+    expect(streamMap.has(channel.topic)).toBe(false);
+  });
+
+  it('sets content type to turtle.', async(): Promise<void> => {
+    const description = await handler.handle({ operation, request, response });
+    expect(description.metadata?.contentType).toBe('text/turtle');
+  });
+
+  it('responds with the stream.', async(): Promise<void> => {
+    const description = await handler.handle({ operation, request, response });
+    expect(description.data).toBeDefined();
+  });
+
+  it('sends initial notification.', async(): Promise<void> => {
+    const spy = jest.spyOn(representation.data, 'pipe');
+    await handler.handle({ operation, request, response });
+    expect(spy).toHaveBeenCalledTimes(1);
+  });
+
+  it('logs an error if sending initial notification fails.', async(): Promise<void> => {
+    serializer.handleSafe.mockRejectedValueOnce(new Error('failed'));
+    await handler.handle({ operation, request, response });
+    await flushPromises();
+    expect(logger.error).toHaveBeenCalledTimes(1);
+    expect(logger.error).toHaveBeenLastCalledWith(`Problem emitting initial notification: failed`);
+  });
+
+  it('errors on requests the Authorizer rejects.', async(): Promise<void> => {
+    authorizer.handleSafe.mockRejectedValue(new Error('not allowed'));
+    await expect(handler.handle({ operation, request, response })).rejects.toThrow('not allowed');
+  });
+});

test/unit/server/notifications/WebSocketChannel2023/WebSocket2023Emitter.test.ts

@@ -21,7 +21,6 @@ describe('A WebSocket2023Emitter', (): void => {
   beforeEach(async(): Promise<void> => {
     webSocket = {
       send: jest.fn(),
-      close: jest.fn(),
     } as any;
 
     socketMap = new WrappedSetMultiMap();