fix: Add content-negotiation when fetching dataset from url

* Solution works but tests don't

* refactor(FetchUtil): use arrayifyStream

* refactor(FetchUtil): split fetchDataset into 2 separate functions

* style(FetchUtil): onelining instead of declaring new local var

* test: trying to mock rdfDereferencer

* refactor: promise can't have async function as arg

* test(FetchUtil): pass Quad array to mockDereference instead

* test: all tests should pass now and coverage is back to 100%

* style: comment typo

* chore: make package.json and package-lock.json compatible with main

* chore: fix package.json double entries

* chore: updated package.json to be alfabetical again

* refactor(AgentGroupAccessChecker): Remove converter from contructor and config

* refactor(TokenOwnerShipValidator): Remove converter from constructor and config

* refactor(FetchUtil): Return BadRequestHttpError instead of generic Error

* test(FetchUtil): return Response object instead of mocking fetch

* style: typos and newlines

src/authorization/access/AgentGroupAccessChecker.ts

@@ -1,6 +1,5 @@
 import type { Store, Term } from 'n3';
 import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier';
-import type { RepresentationConverter } from '../../storage/conversion/RepresentationConverter';
 import type { ExpiringStorage } from '../../storage/keyvalue/ExpiringStorage';
 import { fetchDataset } from '../../util/FetchUtil';
 import { promiseSome } from '../../util/PromiseUtil';
@@ -19,14 +18,11 @@ import { AccessChecker } from './AccessChecker';
  * `expiration` parameter is how long entries in the cache should be stored in seconds, defaults to 3600.
  */
 export class AgentGroupAccessChecker extends AccessChecker {
-  private readonly converter: RepresentationConverter;
   private readonly cache: ExpiringStorage<string, Promise<Store>>;
   private readonly expiration: number;
 
-  public constructor(converter: RepresentationConverter, cache: ExpiringStorage<string, Promise<Store>>,
-    expiration = 3600) {
+  public constructor(cache: ExpiringStorage<string, Promise<Store>>, expiration = 3600) {
     super();
-    this.converter = converter;
     this.cache = cache;
     this.expiration = expiration * 1000;
   }
@@ -65,7 +61,7 @@ export class AgentGroupAccessChecker extends AccessChecker {
     let result = await this.cache.get(url);
     if (!result) {
       const prom = (async(): Promise<Store> => {
-        const representation = await fetchDataset(url, this.converter);
+        const representation = await fetchDataset(url);
         return readableToQuads(representation.data);
       })();
       await this.cache.set(url, prom, this.expiration);

src/identity/ownership/TokenOwnershipValidator.ts

@@ -2,7 +2,6 @@ import type { Quad } from 'n3';
 import { DataFactory } from 'n3';
 import { v4 } from 'uuid';
 import { getLoggerFor } from '../../logging/LogUtil';
-import type { RepresentationConverter } from '../../storage/conversion/RepresentationConverter';
 import type { ExpiringStorage } from '../../storage/keyvalue/ExpiringStorage';
 import { BadRequestHttpError } from '../../util/errors/BadRequestHttpError';
 import { fetchDataset } from '../../util/FetchUtil';
@@ -17,13 +16,11 @@ const { literal, namedNode, quad } = DataFactory;
 export class TokenOwnershipValidator extends OwnershipValidator {
   protected readonly logger = getLoggerFor(this);
 
-  private readonly converter: RepresentationConverter;
   private readonly storage: ExpiringStorage<string, string>;
   private readonly expiration: number;
 
-  public constructor(converter: RepresentationConverter, storage: ExpiringStorage<string, string>, expiration = 30) {
+  public constructor(storage: ExpiringStorage<string, string>, expiration = 30) {
     super();
-    this.converter = converter;
     this.storage = storage;
     // Convert minutes to milliseconds
     this.expiration = expiration * 60 * 1000;
@@ -66,7 +63,7 @@ export class TokenOwnershipValidator extends OwnershipValidator {
    * Fetches data from the WebID to determine if the token is present.
    */
   private async hasToken(webId: string, token: string): Promise<boolean> {
-    const representation = await fetchDataset(webId, this.converter);
+    const representation = await fetchDataset(webId);
     const expectedQuad = quad(namedNode(webId), SOLID.terms.oidcIssuerRegistrationToken, literal(token));
     for await (const data of representation.data) {
       const triple = data as Quad;

src/identity/storage/WebIdAdapterFactory.ts

@@ -5,7 +5,7 @@ import type { Adapter, AdapterPayload } from 'oidc-provider';
 import { getLoggerFor } from '../../logging/LogUtil';
 import type { RepresentationConverter } from '../../storage/conversion/RepresentationConverter';
 import { createErrorMessage } from '../../util/errors/ErrorUtil';
-import { fetchDataset } from '../../util/FetchUtil';
+import { responseToDataset } from '../../util/FetchUtil';
 import { OIDC } from '../../util/Vocabularies';
 import type { AdapterFactory } from './AdapterFactory';
 
@@ -91,7 +91,7 @@ export class WebIdAdapter implements Adapter {
    * @param response - Response object from the request.
    */
   private async parseRdfWebId(data: string, id: string, response: Response): Promise<AdapterPayload> {
-    const representation = await fetchDataset(response, this.converter, data);
+    const representation = await responseToDataset(response, this.converter, data);
 
     // Find the valid redirect URIs
     const redirectUris: string[] = [];

src/util/FetchUtil.ts

@@ -1,5 +1,8 @@
+import type { Readable } from 'stream';
+import type { Quad } from '@rdfjs/types';
+import arrayifyStream from 'arrayify-stream';
 import type { Response } from 'cross-fetch';
-import { fetch } from 'cross-fetch';
+import rdfDereferencer from 'rdf-dereference';
 import { BasicRepresentation } from '../http/representation/BasicRepresentation';
 import type { Representation } from '../http/representation/Representation';
 import { getLoggerFor } from '../logging/LogUtil';
@@ -12,24 +15,32 @@ const logger = getLoggerFor('FetchUtil');
 
 /**
  * Fetches an RDF dataset from the given URL.
- * Input can also be a Response if the request was already made.
+ *
+ * Response will be a Representation with content-type internal/quads.
+ */
+export async function fetchDataset(url: string): Promise<Representation> {
+  // Try content negotiation to parse quads from the URL
+  return (async(): Promise<Representation> => {
+    try {
+      const quadStream = (await rdfDereferencer.dereference(url)).quads as Readable;
+      const quadArray = await arrayifyStream(quadStream) as Quad[];
+      return new BasicRepresentation(quadArray, { path: url }, INTERNAL_QUADS, false);
+    } catch {
+      throw new BadRequestHttpError(`Could not parse resource at URL (${url})!`);
+    }
+  })();
+}
+
+/**
+ * Converts a given Response (from a request that was already made) to  an RDF dataset.
  * In case the given Response object was already parsed its body can be passed along as a string.
  *
  * The converter will be used to convert the response body to RDF.
  *
  * Response will be a Representation with content-type internal/quads.
  */
-export async function fetchDataset(url: string, converter: RepresentationConverter): Promise<Representation>;
-export async function fetchDataset(response: Response, converter: RepresentationConverter, body?: string):
-Promise<Representation>;
-export async function fetchDataset(input: string | Response, converter: RepresentationConverter, body?: string):
+export async function responseToDataset(response: Response, converter: RepresentationConverter, body?: string):
 Promise<Representation> {
-  let response: Response;
-  if (typeof input === 'string') {
-    response = await fetch(input);
-  } else {
-    response = input;
-  }
   if (!body) {
     body = await response.text();
   }