fix: Add content-negotiation when fetching dataset from url

* Solution works but tests don't

* refactor(FetchUtil): use arrayifyStream

* refactor(FetchUtil): split fetchDataset into 2 separate functions

* style(FetchUtil): onelining instead of declaring new local var

* test: trying to mock rdfDereferencer

* refactor: promise can't have async function as arg

* test(FetchUtil): pass Quad array to mockDereference instead

* test: all tests should pass now and coverage is back to 100%

* style: comment typo

* chore: make package.json and package-lock.json compatible with main

* chore: fix package.json double entries

* chore: updated package.json to be alfabetical again

* refactor(AgentGroupAccessChecker): Remove converter from contructor and config

* refactor(TokenOwnerShipValidator): Remove converter from constructor and config

* refactor(FetchUtil): Return BadRequestHttpError instead of generic Error

* test(FetchUtil): return Response object instead of mocking fetch

* style: typos and newlines

test/unit/authorization/access/AgentGroupAccessChecker.test.ts

@@ -3,7 +3,6 @@ import type { AccessCheckerArgs } from '../../../../src/authorization/access/Acc
 import { AgentGroupAccessChecker } from '../../../../src/authorization/access/AgentGroupAccessChecker';
 import { BasicRepresentation } from '../../../../src/http/representation/BasicRepresentation';
 import type { Representation } from '../../../../src/http/representation/Representation';
-import type { RepresentationConverter } from '../../../../src/storage/conversion/RepresentationConverter';
 import type { ExpiringStorage } from '../../../../src/storage/keyvalue/ExpiringStorage';
 import { INTERNAL_QUADS } from '../../../../src/util/ContentTypes';
 import * as fetchUtil from '../../../../src/util/FetchUtil';
@@ -18,7 +17,6 @@ describe('An AgentGroupAccessChecker', (): void => {
   acl.addQuad(namedNode('noMatch'), ACL.terms.agentGroup, namedNode('badGroup'));
   let fetchMock: jest.SpyInstance;
   let representation: Representation;
-  const converter: RepresentationConverter = {} as any;
   let cache: ExpiringStorage<string, Promise<Store>>;
   let checker: AgentGroupAccessChecker;
 
@@ -31,7 +29,7 @@ describe('An AgentGroupAccessChecker', (): void => {
 
     cache = new Map() as any;
 
-    checker = new AgentGroupAccessChecker(converter, cache);
+    checker = new AgentGroupAccessChecker(cache);
   });
 
   it('can handle all requests.', async(): Promise<void> => {

test/unit/identity/ownership/TokenOwnershipValidator.test.ts

@@ -1,15 +1,17 @@
-import { fetch } from 'cross-fetch';
+import { Readable } from 'stream';
 import { DataFactory } from 'n3';
 import type { Quad } from 'n3';
+import rdfDereferencer from 'rdf-dereference';
 import { v4 } from 'uuid';
 import { TokenOwnershipValidator } from '../../../../src/identity/ownership/TokenOwnershipValidator';
-import { RdfToQuadConverter } from '../../../../src/storage/conversion/RdfToQuadConverter';
 import type { ExpiringStorage } from '../../../../src/storage/keyvalue/ExpiringStorage';
 import { SOLID } from '../../../../src/util/Vocabularies';
 const { literal, namedNode, quad } = DataFactory;
 
-jest.mock('cross-fetch');
 jest.mock('uuid');
+jest.mock('rdf-dereference', (): any => ({
+  dereference: jest.fn(),
+}));
 
 function quadToString(qq: Quad): string {
   const subPred = `<${qq.subject.value}> <${qq.predicate.value}>`;
@@ -20,21 +22,19 @@ function quadToString(qq: Quad): string {
 }
 
 describe('A TokenOwnershipValidator', (): void => {
-  const fetchMock: jest.Mock = fetch as any;
+  const rdfDereferenceMock: jest.Mocked<typeof rdfDereferencer> = rdfDereferencer as any;
   const webId = 'http://alice.test.com/#me';
   const token = 'randomlyGeneratedToken';
   const tokenTriple = quad(namedNode(webId), SOLID.terms.oidcIssuerRegistrationToken, literal(token));
   const tokenString = `${quadToString(tokenTriple)}.`;
-  const converter = new RdfToQuadConverter();
   let storage: ExpiringStorage<string, string>;
   let validator: TokenOwnershipValidator;
 
-  function mockFetch(body: string): void {
-    fetchMock.mockImplementation((url: string): any => ({
-      text: (): any => body,
-      url,
-      status: 200,
-      headers: { get: (): any => 'text/turtle' },
+  function mockDereference(qq?: Quad): any {
+    rdfDereferenceMock.dereference.mockImplementation((uri: string): any => ({
+      uri,
+      quads: Readable.from(qq ? [ qq ] : []),
+      exists: true,
     }));
   }
 
@@ -50,32 +50,32 @@ describe('A TokenOwnershipValidator', (): void => {
       delete: jest.fn().mockImplementation((key: string): any => map.delete(key)),
     } as any;
 
-    mockFetch('');
+    mockDereference();
 
-    validator = new TokenOwnershipValidator(converter, storage);
+    validator = new TokenOwnershipValidator(storage);
   });
 
   it('errors if no token is stored in the storage.', async(): Promise<void> => {
     // Even if the token is in the WebId, it will error since it's not in the storage
-    mockFetch(tokenString);
+    mockDereference(tokenTriple);
     await expect(validator.handle({ webId })).rejects.toThrow(expect.objectContaining({
       message: expect.stringContaining(tokenString),
       details: { quad: tokenString },
     }));
-    expect(fetch).toHaveBeenCalledTimes(0);
+    expect(rdfDereferenceMock.dereference).toHaveBeenCalledTimes(0);
   });
 
   it('errors if the expected triple is missing.', async(): Promise<void> => {
     // First call will add the token to the storage
     await expect(validator.handle({ webId })).rejects.toThrow(tokenString);
-    expect(fetch).toHaveBeenCalledTimes(0);
+    expect(rdfDereferenceMock.dereference).toHaveBeenCalledTimes(0);
     // Second call will fetch the WebId
     await expect(validator.handle({ webId })).rejects.toThrow(tokenString);
-    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(rdfDereferenceMock.dereference).toHaveBeenCalledTimes(1);
   });
 
   it('resolves if the WebId contains the verification triple.', async(): Promise<void> => {
-    mockFetch(tokenString);
+    mockDereference(tokenTriple);
     // First call will add the token to the storage
     await expect(validator.handle({ webId })).rejects.toThrow(tokenString);
     // Second call will succeed since it has the verification triple
@@ -84,7 +84,7 @@ describe('A TokenOwnershipValidator', (): void => {
 
   it('fails if the WebId contains the wrong verification triple.', async(): Promise<void> => {
     const wrongQuad = quad(namedNode(webId), SOLID.terms.oidcIssuerRegistrationToken, literal('wrongToken'));
-    mockFetch(`${quadToString(wrongQuad)} .`);
+    mockDereference(wrongQuad);
     // First call will add the token to the storage
     await expect(validator.handle({ webId })).rejects.toThrow(tokenString);
     // Second call will fail since it has the wrong verification triple

test/unit/util/FetchUtil.test.ts

@@ -1,55 +1,81 @@
+import { Readable } from 'stream';
+import type { Quad } from '@rdfjs/types';
 import arrayifyStream from 'arrayify-stream';
-import { fetch } from 'cross-fetch';
+import type { Response } from 'cross-fetch';
 import { DataFactory } from 'n3';
+import rdfDereferencer from 'rdf-dereference';
 import { RdfToQuadConverter } from '../../../src/storage/conversion/RdfToQuadConverter';
-import { fetchDataset } from '../../../src/util/FetchUtil';
+import { fetchDataset, responseToDataset } from '../../../src/util/FetchUtil';
 const { namedNode, quad } = DataFactory;
 
-jest.mock('cross-fetch');
+jest.mock('rdf-dereference', (): any => ({
+  dereference: jest.fn<string, any>(),
+}));
 
 describe('FetchUtil', (): void => {
-  describe('#fetchDataset', (): void => {
-    const fetchMock: jest.Mock = fetch as any;
-    const url = 'http://test.com/foo';
-    const converter = new RdfToQuadConverter();
+  const url = 'http://test.com/foo';
 
-    function mockFetch(body: string, status = 200): void {
-      fetchMock.mockImplementation((input: string): any => ({
-        text: (): any => body,
-        url: input,
-        status,
-        headers: { get: (): any => 'text/turtle' },
-      }));
+  function mockResponse(body: string, contentType: string | null, status = 200): Response {
+    return ({
+      text: (): any => body,
+      url,
+      status,
+      headers: { get: (): any => contentType },
+    }) as any;
+  }
+
+  describe('#fetchDataset', (): void => {
+    const rdfDereferenceMock: jest.Mocked<typeof rdfDereferencer> = rdfDereferencer as any;
+
+    function mockDereference(quads?: Quad[]): any {
+      rdfDereferenceMock.dereference.mockImplementation((uri: string): any => {
+        if (!quads) {
+          throw new Error('Throws error because url does not exist');
+        }
+        return {
+          uri,
+          quads: Readable.from(quads),
+          exists: true,
+        };
+      });
     }
 
-    it('errors if the status code is not 200.', async(): Promise<void> => {
-      mockFetch('Invalid URL!', 404);
-      await expect(fetchDataset(url, converter)).rejects.toThrow(`Unable to access data at ${url}`);
-      expect(fetchMock).toHaveBeenCalledWith(url);
-    });
-
-    it('errors if there is no content-type.', async(): Promise<void> => {
-      fetchMock.mockResolvedValueOnce({ url, text: (): any => '', status: 200, headers: { get: jest.fn() }});
-      await expect(fetchDataset(url, converter)).rejects.toThrow(`Unable to access data at ${url}`);
-      expect(fetchMock).toHaveBeenCalledWith(url);
+    it('errors if the URL does not exist.', async(): Promise<void> => {
+      mockDereference();
+      await expect(fetchDataset(url)).rejects.toThrow(`Could not parse resource at URL (${url})!`);
+      expect(rdfDereferenceMock.dereference).toHaveBeenCalledWith(url);
     });
 
     it('returns a Representation with quads.', async(): Promise<void> => {
-      mockFetch('<http://test.com/s> <http://test.com/p> <http://test.com/o>.');
-      const representation = await fetchDataset(url, converter);
-      await expect(arrayifyStream(representation.data)).resolves.toEqual([
-        quad(namedNode('http://test.com/s'), namedNode('http://test.com/p'), namedNode('http://test.com/o')),
-      ]);
-    });
-
-    it('accepts Response objects as input.', async(): Promise<void> => {
-      mockFetch('<http://test.com/s> <http://test.com/p> <http://test.com/o>.');
-      const response = await fetch(url);
-      const body = await response.text();
-      const representation = await fetchDataset(response, converter, body);
+      const quads = [ quad(namedNode('http://test.com/s'), namedNode('http://test.com/p'), namedNode('http://test.com/o')) ];
+      mockDereference(quads);
+      const representation = await fetchDataset(url);
       await expect(arrayifyStream(representation.data)).resolves.toEqual([
         quad(namedNode('http://test.com/s'), namedNode('http://test.com/p'), namedNode('http://test.com/o')),
       ]);
     });
   });
+
+  describe('#responseToDataset', (): void => {
+    const converter = new RdfToQuadConverter();
+
+    it('accepts Response objects as input.', async(): Promise<void> => {
+      const response = mockResponse('<http://test.com/s> <http://test.com/p> <http://test.com/o>.', 'text/turtle');
+      const body = await response.text();
+      const representation = await responseToDataset(response, converter, body);
+      await expect(arrayifyStream(representation.data)).resolves.toEqual([
+        quad(namedNode('http://test.com/s'), namedNode('http://test.com/p'), namedNode('http://test.com/o')),
+      ]);
+    });
+
+    it('errors if the status code is not 200.', async(): Promise<void> => {
+      const response = mockResponse('Incorrect status!', null, 400);
+      await expect(responseToDataset(response, converter)).rejects.toThrow(`Unable to access data at ${url}`);
+    });
+
+    it('errors if there is no content-type.', async(): Promise<void> => {
+      const response = mockResponse('No content-type!', null);
+      await expect(responseToDataset(response, converter)).rejects.toThrow(`Unable to access data at ${url}`);
+    });
+  });
 });