feat: add template based data generator

index.ts

@@ -130,7 +130,7 @@ export * from './src/storage/routing/RouterRule';
 export * from './src/storage/AtomicResourceStore';
 export * from './src/storage/Conditions';
 export * from './src/storage/DataAccessorBasedStore';
-export * from './src/storage/FileIdentifierMapper';
+export * from './src/storage/mapping/FileIdentifierMapper';
 export * from './src/storage/LockingResourceStore';
 export * from './src/storage/MonitoringStore';
 export * from './src/storage/PassthroughStore';

package-lock.json

@@ -4734,6 +4734,25 @@
       "dev": true,
       "optional": true
     },
+    "handlebars": {
+      "version": "4.7.6",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.6.tgz",
+      "integrity": "sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA==",
+      "requires": {
+        "minimist": "^1.2.5",
+        "neo-async": "^2.6.0",
+        "source-map": "^0.6.1",
+        "uglify-js": "^3.1.4",
+        "wordwrap": "^1.0.0"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
+        }
+      }
+    },
     "har-schema": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
@@ -6763,6 +6782,11 @@
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
       "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
     },
+    "neo-async": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
+      "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="
+    },
     "nested-error-stacks": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.0.1.tgz",
@@ -9613,6 +9637,12 @@
       "integrity": "sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ==",
       "dev": true
     },
+    "uglify-js": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.12.0.tgz",
+      "integrity": "sha512-8lBMSkFZuAK7gGF8LswsXmir8eX8d2AAMOnxSDWjKBx/fBR6MypQjs78m6ML9zQVp1/hD4TBdfeMZMC7nW1TAA==",
+      "optional": true
+    },
     "undefsafe": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz",
@@ -9991,6 +10021,11 @@
       "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
       "dev": true
     },
+    "wordwrap": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
+      "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus="
+    },
     "wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",

package.json

@@ -91,6 +91,7 @@
     "cors": "^2.8.5",
     "express": "^4.17.1",
     "fetch-sparql-endpoint": "^1.8.0",
+    "handlebars": "^4.7.6",
     "mime-types": "^2.1.27",
     "n3": "^1.6.4",
     "rdf-parse": "^1.5.0",

src/pods/generate/HandlebarsTemplateEngine.ts

@@ -0,0 +1,12 @@
+import { compile } from 'handlebars';
+import type { TemplateEngine } from './TemplateEngine';
+
+/**
+ * Fills in Handlebars templates.
+ */
+export class HandlebarsTemplateEngine implements TemplateEngine {
+  public apply(template: string, options: NodeJS.Dict<string>): string {
+    const compiled = compile(template);
+    return compiled(options);
+  }
+}

src/pods/generate/TemplateEngine.ts

@@ -0,0 +1,8 @@
+import Dict = NodeJS.Dict;
+
+/**
+ * A template engine takes as input a template and applies the given options to it.
+ */
+export interface TemplateEngine {
+  apply: (template: string, options: Dict<string>) => string;
+}

src/pods/generate/TemplatedResourcesGenerator.ts

@@ -0,0 +1,92 @@
+import { promises as fsPromises } from 'fs';
+import { posix } from 'path';
+import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata';
+import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
+import type { FileIdentifierMapper, FileIdentifierMapperFactory } from '../../storage/mapping/FileIdentifierMapper';
+import { guardedStreamFrom } from '../../util/StreamUtil';
+import type { Resource, ResourcesGenerator } from './ResourcesGenerator';
+import type { TemplateEngine } from './TemplateEngine';
+import Dict = NodeJS.Dict;
+
+const { join: joinPath } = posix;
+
+/**
+ * Generates resources by making use of a template engine.
+ * The template folder structure will be kept.
+ * Folders will be interpreted as containers and files as documents.
+ * A FileIdentifierMapper will be used to generate identifiers that correspond to the relative structure.
+ */
+export class TemplatedResourcesGenerator implements ResourcesGenerator {
+  private readonly templateFolder: string;
+  private readonly factory: FileIdentifierMapperFactory;
+  private readonly engine: TemplateEngine;
+
+  /**
+   * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier.
+   *
+   * @param templateFolder - Folder where the templates are located.
+   * @param factory - Factory used to generate mapper relative to the base identifier.
+   * @param engine - Template engine for generating the resources.
+   */
+  public constructor(templateFolder: string, factory: FileIdentifierMapperFactory, engine: TemplateEngine) {
+    this.templateFolder = templateFolder;
+    this.factory = factory;
+    this.engine = engine;
+  }
+
+  public async* generate(location: ResourceIdentifier, options: Dict<string>): AsyncIterable<Resource> {
+    const mapper = await this.factory.create(location.path, this.templateFolder);
+    yield* this.parseFolder(this.templateFolder, mapper, options);
+  }
+
+  /**
+   * Generates results for all entries in the given folder, including the folder itself.
+   */
+  private async* parseFolder(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
+  AsyncIterable<Resource> {
+    // Generate representation for the container
+    const link = await mapper.mapFilePathToUrl(filePath, true);
+    yield {
+      identifier: link.identifier,
+      representation: {
+        binary: true,
+        data: guardedStreamFrom([]),
+        metadata: new RepresentationMetadata(link.identifier.path),
+      },
+    };
+
+    // Generate representations for all resources in this container
+    const files = await fsPromises.readdir(filePath);
+    for (const childName of files) {
+      const childPath = joinPath(filePath, childName);
+      const childStats = await fsPromises.lstat(childPath);
+      if (childStats.isDirectory()) {
+        yield* this.parseFolder(childPath, mapper, options);
+      } else if (childStats.isFile()) {
+        yield this.generateDocument(childPath, mapper, options);
+      }
+    }
+  }
+
+  /**
+   * Generates a new Representation corresponding to the template file at the given location.
+   */
+  private async generateDocument(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
+  Promise<Resource> {
+    const link = await mapper.mapFilePathToUrl(filePath, false);
+    const metadata = new RepresentationMetadata(link.identifier.path);
+    metadata.contentType = link.contentType;
+
+    const raw = await fsPromises.readFile(filePath, 'utf8');
+    const compiled = this.engine.apply(raw, options);
+
+    return {
+      identifier: link.identifier,
+      representation: {
+        binary: true,
+        data: guardedStreamFrom([ compiled ]),
+        metadata,
+      },
+    };
+  }
+}

src/storage/accessors/FileDataAccessor.ts

@@ -19,7 +19,7 @@ import { parseQuads, pushQuad, serializeQuads } from '../../util/QuadUtil';
 import { generateContainmentQuads, generateResourceQuads } from '../../util/ResourceUtil';
 import { CONTENT_TYPE, DCTERMS, POSIX, RDF, XSD } from '../../util/UriConstants';
 import { toNamedNode, toTypedLiteral } from '../../util/UriUtil';
-import type { FileIdentifierMapper, ResourceLink } from '../FileIdentifierMapper';
+import type { FileIdentifierMapper, ResourceLink } from '../mapping/FileIdentifierMapper';
 import type { DataAccessor } from './DataAccessor';
 
 const { join: joinPath } = posix;

src/storage/mapping/ExtensionBasedMapper.ts

@@ -12,7 +12,7 @@ import {
   isContainerIdentifier,
   trimTrailingSlashes,
 } from '../../util/PathUtil';
-import type { FileIdentifierMapper, ResourceLink } from '../FileIdentifierMapper';
+import type { FileIdentifierMapper, FileIdentifierMapperFactory, ResourceLink } from './FileIdentifierMapper';
 import { getAbsolutePath, getRelativePath, validateRelativePath } from './MapperUtil';
 
 const { join: joinPath, normalize: normalizePath } = posix;
@@ -197,3 +197,10 @@ export class ExtensionBasedMapper implements FileIdentifierMapper {
     return extension && extension[1];
   }
 }
+
+export class ExtensionBasedMapperFactory implements FileIdentifierMapperFactory<ExtensionBasedMapper> {
+  public async create(base: string, rootFilePath: string): Promise<ExtensionBasedMapper> {
+    return new ExtensionBasedMapper(base, rootFilePath);
+  }
+}
+

src/storage/mapping/FileIdentifierMapper.ts

@@ -1,4 +1,4 @@
-import type { ResourceIdentifier } from '../ldp/representation/ResourceIdentifier';
+import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
 
 export interface ResourceLink {
   /**
@@ -38,3 +38,11 @@ export interface FileIdentifierMapper {
    */
   mapUrlToFilePath: (identifier: ResourceIdentifier, contentType?: string) => Promise<ResourceLink>;
 }
+
+/**
+ * Factory that can create FileIdentifierMappers so the base and rootFilePath can be set dynamically.
+ * Specifically used when identifiers need to be generated for a new pod (since pod identifiers are generated).
+ */
+export interface FileIdentifierMapperFactory<T extends FileIdentifierMapper = FileIdentifierMapper> {
+  create: (base: string, rootFilePath: string) => Promise<T>;
+}

src/storage/mapping/FixedContentTypeMapper.ts

@@ -7,7 +7,7 @@ import {
   ensureTrailingSlash, isContainerIdentifier,
   trimTrailingSlashes,
 } from '../../util/PathUtil';
-import type { FileIdentifierMapper, ResourceLink } from '../FileIdentifierMapper';
+import type { FileIdentifierMapper, ResourceLink } from './FileIdentifierMapper';
 import { getAbsolutePath, getRelativePath, validateRelativePath } from './MapperUtil';
 
 const { normalize: normalizePath } = posix;

test/unit/pods/generate/HandlebarsTemplateEngine.test.ts

@@ -0,0 +1,11 @@
+import { HandlebarsTemplateEngine } from '../../../../src/pods/generate/HandlebarsTemplateEngine';
+
+describe('A HandlebarsTemplateEngine', (): void => {
+  const engine = new HandlebarsTemplateEngine();
+
+  it('fills in Handlebars templates.', async(): Promise<void> => {
+    const template = '<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.';
+    const options = { webId: 'http://alice/#profile' };
+    expect(engine.apply(template, options)).toBe('<http://alice/#profile> a <http://xmlns.com/foaf/0.1/Person>.');
+  });
+});

test/unit/pods/generate/TemplatedResourcesGenerator.test.ts

@@ -0,0 +1,90 @@
+import type { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier';
+import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator';
+import type { TemplateEngine } from '../../../../src/pods/generate/TemplateEngine';
+import type {
+  FileIdentifierMapper,
+  FileIdentifierMapperFactory,
+  ResourceLink,
+} from '../../../../src/storage/mapping/FileIdentifierMapper';
+import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil';
+import { readableToString } from '../../../../src/util/StreamUtil';
+import { mockFs } from '../../../util/Util';
+import Dict = NodeJS.Dict;
+
+jest.mock('fs');
+
+class DummyFactory implements FileIdentifierMapperFactory {
+  public async create(base: string, rootFilePath: string): Promise<FileIdentifierMapper> {
+    const trimBase = trimTrailingSlashes(base);
+    const trimRoot = trimTrailingSlashes(rootFilePath);
+    return {
+      async mapFilePathToUrl(filePath: string, isContainer: boolean): Promise<ResourceLink> {
+        const path = `${trimBase}${filePath.slice(trimRoot.length)}`;
+        return {
+          identifier: { path: isContainer ? ensureTrailingSlash(path) : path },
+          filePath,
+          contentType: isContainer ? undefined : 'text/turtle',
+        };
+      },
+    } as any;
+  }
+}
+
+class DummyEngine implements TemplateEngine {
+  public apply(template: string, options: Dict<string>): string {
+    const keys = Object.keys(options);
+    return `${template}${keys.map((key): string => `{${key}:${options[key]}}`).join('')}`;
+  }
+}
+
+const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
+  const arr: T[] = [];
+  for await (const result of iterable) {
+    arr.push(result);
+  }
+  return arr;
+};
+
+describe('A TemplatedResourcesGenerator', (): void => {
+  const rootFilePath = 'templates';
+  const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new DummyEngine());
+  let cache: { data: any };
+  const template = '<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.';
+  const location = { path: 'http://test.com/alice/' };
+  const webId = 'http://alice/#profile';
+
+  beforeEach(async(): Promise<void> => {
+    cache = mockFs(rootFilePath);
+  });
+
+  it('fills in a template with the given options.', async(): Promise<void> => {
+    cache.data = { template };
+    const result = await genToArray(generator.generate(location, { webId }));
+    const identifiers = result.map((res): ResourceIdentifier => res.identifier);
+    const id = { path: `${location.path}template` };
+    expect(identifiers).toEqual([ location, id ]);
+
+    const { representation } = result[1];
+    expect(representation.binary).toBe(true);
+    expect(representation.metadata.contentType).toBe('text/turtle');
+    await expect(readableToString(representation.data)).resolves
+      .toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
+  });
+
+  it('creates the necessary containers and ignores non-files.', async(): Promise<void> => {
+    cache.data = { container: { container: { template }}, 2: 5 };
+    const result = await genToArray(generator.generate(location, { webId }));
+    const identifiers = result.map((res): ResourceIdentifier => res.identifier);
+    const id = { path: `${location.path}container/container/template` };
+    expect(identifiers).toEqual([
+      location,
+      { path: `${location.path}container/` },
+      { path: `${location.path}container/container/` },
+      id,
+    ]);
+
+    const { representation } = result[3];
+    await expect(readableToString(representation.data)).resolves
+      .toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
+  });
+});

test/unit/storage/mapping/ExtensionBasedMapper.test.ts

@@ -1,5 +1,8 @@
 import fs from 'fs';
-import { ExtensionBasedMapper } from '../../../../src/storage/mapping/ExtensionBasedMapper';
+import {
+  ExtensionBasedMapper,
+  ExtensionBasedMapperFactory,
+} from '../../../../src/storage/mapping/ExtensionBasedMapper';
 import { BadRequestHttpError } from '../../../../src/util/errors/BadRequestHttpError';
 import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError';
 import { trimTrailingSlashes } from '../../../../src/util/PathUtil';
@@ -135,4 +138,12 @@ describe('An ExtensionBasedMapper', (): void => {
       });
     });
   });
+
+  describe('An ExtensionBasedMapperFactory', (): void => {
+    const factory = new ExtensionBasedMapperFactory();
+
+    it('creates an ExtensionBasedMapper.', async(): Promise<void> => {
+      await expect(factory.create('base', 'filePath')).resolves.toBeInstanceOf(ExtensionBasedMapper);
+    });
+  });
 });

test/util/Util.ts

@@ -160,6 +160,10 @@ export const mockFs = (rootFilepath?: string, time?: Date): { data: any } => {
         }
         folder[name] = {};
       },
+      readFile(path: string): string {
+        const { folder, name } = getFolder(path);
+        return folder[name];
+      },
     },
   };