feat: Support .meta files for pod provisioning

src/pods/generate/TemplatedResourcesGenerator.ts

@@ -1,8 +1,14 @@
 import { promises as fsPromises } from 'fs';
 import { posix } from 'path';
+import { Parser } from 'n3';
 import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata';
 import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
-import type { FileIdentifierMapper, FileIdentifierMapperFactory } from '../../storage/mapping/FileIdentifierMapper';
+import type {
+  FileIdentifierMapper,
+  FileIdentifierMapperFactory,
+  ResourceLink,
+} from '../../storage/mapping/FileIdentifierMapper';
+import { isContainerIdentifier } from '../../util/PathUtil';
 import { guardedStreamFrom } from '../../util/StreamUtil';
 import type { Resource, ResourcesGenerator } from './ResourcesGenerator';
 import type { TemplateEngine } from './TemplateEngine';
@@ -20,6 +26,7 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
   private readonly templateFolder: string;
   private readonly factory: FileIdentifierMapperFactory;
   private readonly engine: TemplateEngine;
+  private readonly metaExtension = '.meta';
 
   /**
    * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier.
@@ -36,57 +43,132 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
 
   public async* generate(location: ResourceIdentifier, options: Dict<string>): AsyncIterable<Resource> {
     const mapper = await this.factory.create(location.path, this.templateFolder);
-    yield* this.parseFolder(this.templateFolder, mapper, options);
+    const folderLink = await mapper.mapFilePathToUrl(this.templateFolder, true);
+    yield* this.parseFolder(folderLink, mapper, options);
   }
 
   /**
    * Generates results for all entries in the given folder, including the folder itself.
    */
-  private async* parseFolder(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
+  private async* parseFolder(folderLink: ResourceLink, mapper: FileIdentifierMapper, options: Dict<string>):
   AsyncIterable<Resource> {
-    // Generate representation for the container
+    // Group resource links with their corresponding metadata links
-    const link = await mapper.mapFilePathToUrl(filePath, true);
+    const links = await this.groupLinks(this.generateLinks(folderLink.filePath, mapper));
-    yield {
-      identifier: link.identifier,
-      representation: {
-        binary: true,
-        data: guardedStreamFrom([]),
-        metadata: new RepresentationMetadata(link.identifier),
-      },
-    };
 
-    // Generate representations for all resources in this container
+    // Remove root metadata if it exists
-    const files = await fsPromises.readdir(filePath);
+    const metaLink = links[folderLink.identifier.path]?.meta;
-    for (const childName of files) {
+    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-      const childPath = joinPath(filePath, childName);
+    delete links[folderLink.identifier.path];
-      const childStats = await fsPromises.lstat(childPath);
+
-      if (childStats.isDirectory()) {
+    yield this.generateResource(folderLink, options, metaLink);
-        yield* this.parseFolder(childPath, mapper, options);
+
-      } else if (childStats.isFile()) {
+    for (const { link, meta } of Object.values(links)) {
-        yield this.generateDocument(childPath, mapper, options);
+      if (isContainerIdentifier(link.identifier)) {
+        yield* this.parseFolder(link, mapper, options);
+      } else {
+        yield this.generateResource(link, options, meta);
       }
     }
   }
 
   /**
-   * Generates a new Representation corresponding to the template file at the given location.
+   * Generates ResourceLinks for each entry in the given folder.
    */
-  private async generateDocument(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
+  private async* generateLinks(folderPath: string, mapper: FileIdentifierMapper): AsyncIterable<ResourceLink> {
-  Promise<Resource> {
+    const files = await fsPromises.readdir(folderPath);
-    const link = await mapper.mapFilePathToUrl(filePath, false);
+    for (const name of files) {
-    const metadata = new RepresentationMetadata(link.identifier);
+      const filePath = joinPath(folderPath, name);
-    metadata.contentType = link.contentType;
+      const stats = await fsPromises.lstat(filePath);
+      yield mapper.mapFilePathToUrl(filePath, stats.isDirectory());
+    }
+  }
 
-    const raw = await fsPromises.readFile(filePath, 'utf8');
+  /**
-    const compiled = this.engine.apply(raw, options);
+   * Parses a group of ResourceLinks so resources and their metadata are grouped together.
+   */
+  private async groupLinks(linkGen: AsyncIterable<ResourceLink>):
+  Promise<Record<string, { link: ResourceLink; meta?: ResourceLink }>> {
+    const links: Record<string, { link: ResourceLink; meta?: ResourceLink }> = { };
+    for await (const link of linkGen) {
+      const { path } = link.identifier;
+      if (this.isMeta(path)) {
+        const resourcePath = this.metaToResource(link.identifier).path;
+        links[resourcePath] = Object.assign(links[resourcePath] || {}, { meta: link });
+      } else {
+        links[path] = Object.assign(links[path] || {}, { link });
+      }
+    }
+    return links;
+  }
+
+  /**
+   * Generates a Resource object for the given ResourceLink.
+   * In the case of documents the corresponding template will be used.
+   * If a ResourceLink of metadata is provided the corresponding data will be added as metadata.
+   */
+  private async generateResource(link: ResourceLink, options: Dict<string>, metaLink?: ResourceLink):
+  Promise<Resource> {
+    const data: string[] = [];
+    const metadata = new RepresentationMetadata(link.identifier);
+
+    // Read file if it is not a container
+    if (!isContainerIdentifier(link.identifier)) {
+      const compiled = await this.parseTemplate(link.filePath, options);
+      data.push(compiled);
+      metadata.contentType = link.contentType;
+    }
+
+    // Add metadata from meta file if there is one
+    if (metaLink) {
+      const rawMetadata = await this.generateMetadata(metaLink, options);
+      metadata.addQuads(rawMetadata.quads());
+    }
 
     return {
       identifier: link.identifier,
       representation: {
         binary: true,
-        data: guardedStreamFrom([ compiled ]),
+        data: guardedStreamFrom(data),
         metadata,
       },
     };
   }
+
+  /**
+   * Generates a RepresentationMetadata using the given template.
+   */
+  private async generateMetadata(metaLink: ResourceLink, options: Dict<string>):
+  Promise<RepresentationMetadata> {
+    const identifier = this.metaToResource(metaLink.identifier);
+    const metadata = new RepresentationMetadata(identifier);
+
+    const data = await this.parseTemplate(metaLink.filePath, options);
+    const parser = new Parser({ format: metaLink.contentType, baseIRI: identifier.path });
+    const quads = parser.parse(data);
+    metadata.addQuads(quads);
+
+    return metadata;
+  }
+
+  /**
+   * Applies the given options to the template found at the given path.
+   */
+  private async parseTemplate(filePath: string, options: Dict<string>): Promise<string> {
+    const raw = await fsPromises.readFile(filePath, 'utf8');
+    return this.engine.apply(raw, options);
+  }
+
+  /**
+   * Verifies if the given path corresponds to a metadata file.
+   */
+  private isMeta(path: string): boolean {
+    return path.endsWith(this.metaExtension);
+  }
+
+  /**
+   * Converts a generated metadata identifier to the identifier of its corresponding resource.
+   */
+  private metaToResource(metaIdentifier: ResourceIdentifier): ResourceIdentifier {
+    return { path: metaIdentifier.path.slice(0, -this.metaExtension.length) };
+  }
 }

test/unit/pods/generate/TemplatedResourcesGenerator.test.ts

@@ -1,6 +1,6 @@
 import type { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier';
+import { HandlebarsTemplateEngine } from '../../../../src/pods/generate/HandlebarsTemplateEngine';
 import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator';
-import type { TemplateEngine } from '../../../../src/pods/generate/TemplateEngine';
 import type {
   FileIdentifierMapper,
   FileIdentifierMapperFactory,
@@ -9,7 +9,6 @@ import type {
 import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil';
 import { readableToString } from '../../../../src/util/StreamUtil';
 import { mockFs } from '../../../util/Util';
-import Dict = NodeJS.Dict;
 
 jest.mock('fs');
 
@@ -30,13 +29,6 @@ class DummyFactory implements FileIdentifierMapperFactory {
   }
 }
 
-class DummyEngine implements TemplateEngine {
-  public apply(template: string, options: Dict<string>): string {
-    const keys = Object.keys(options);
-    return `${template}${keys.map((key): string => `{${key}:${options[key]}}`).join('')}`;
-  }
-}
-
 const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
   const arr: T[] = [];
   for await (const result of iterable) {
@@ -47,7 +39,8 @@ const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
 
 describe('A TemplatedResourcesGenerator', (): void => {
   const rootFilePath = 'templates';
-  const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new DummyEngine());
+  // Using handlebars engine since it's smaller than any possible dummy
+  const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new HandlebarsTemplateEngine());
   let cache: { data: any };
   const template = '<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.';
   const location = { path: 'http://test.com/alice/' };
@@ -68,11 +61,11 @@ describe('A TemplatedResourcesGenerator', (): void => {
     expect(representation.binary).toBe(true);
     expect(representation.metadata.contentType).toBe('text/turtle');
     await expect(readableToString(representation.data)).resolves
-      .toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
+      .toEqual(`<${webId}> a <http://xmlns.com/foaf/0.1/Person>.`);
   });
 
-  it('creates the necessary containers and ignores non-files.', async(): Promise<void> => {
+  it('creates the necessary containers.', async(): Promise<void> => {
-    cache.data = { container: { container: { template }}, 2: 5 };
+    cache.data = { container: { container: { template }}};
     const result = await genToArray(generator.generate(location, { webId }));
     const identifiers = result.map((res): ResourceIdentifier => res.identifier);
     const id = { path: `${location.path}container/container/template` };
@@ -85,6 +78,37 @@ describe('A TemplatedResourcesGenerator', (): void => {
 
     const { representation } = result[3];
     await expect(readableToString(representation.data)).resolves
-      .toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
+      .toEqual(`<${webId}> a <http://xmlns.com/foaf/0.1/Person>.`);
+  });
+
+  it('adds metadata from .meta files.', async(): Promise<void> => {
+    const meta = '<> <pre:has> "metadata".';
+    cache.data = { '.meta': meta, container: { 'template.meta': meta, template }};
+
+    // Not using options since our dummy template generator generates invalid turtle
+    const result = await genToArray(generator.generate(location, { webId }));
+    const identifiers = result.map((res): ResourceIdentifier => res.identifier);
+    expect(identifiers).toEqual([
+      location,
+      { path: `${location.path}container/` },
+      { path: `${location.path}container/template` },
+    ]);
+    // Root has the 1 raw metadata triple (with <> changed to its identifier)
+    const rootMetadata = result[0].representation.metadata;
+    expect(rootMetadata.identifier.value).toBe(location.path);
+    expect(rootMetadata.quads()).toHaveLength(1);
+    expect(rootMetadata.get('pre:has')?.value).toBe('metadata');
+
+    // Container has no metadata triples
+    const contMetadata = result[1].representation.metadata;
+    expect(contMetadata.identifier.value).toBe(`${location.path}container/`);
+    expect(contMetadata.quads()).toHaveLength(0);
+
+    // Document has the 1 raw metadata triple (with <> changed to its identifier) and content-type
+    const docMetadata = result[2].representation.metadata;
+    expect(docMetadata.identifier.value).toBe(`${location.path}container/template`);
+    expect(docMetadata.quads()).toHaveLength(2);
+    expect(docMetadata.get('pre:has')?.value).toBe('metadata');
+    expect(docMetadata.contentType).toBe('text/turtle');
   });
 });