From e722cc67affbb189b48bfb4d133e5bc28bec5339 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Wed, 16 Dec 2020 14:19:23 +0100 Subject: [PATCH] feat: Support .meta files for pod provisioning --- .../generate/TemplatedResourcesGenerator.ts | 144 ++++++++++++++---- .../TemplatedResourcesGenerator.test.ts | 52 +++++-- 2 files changed, 151 insertions(+), 45 deletions(-) diff --git a/src/pods/generate/TemplatedResourcesGenerator.ts b/src/pods/generate/TemplatedResourcesGenerator.ts index ae16fd218..a35cfefc6 100644 --- a/src/pods/generate/TemplatedResourcesGenerator.ts +++ b/src/pods/generate/TemplatedResourcesGenerator.ts @@ -1,8 +1,14 @@ import { promises as fsPromises } from 'fs'; import { posix } from 'path'; +import { Parser } from 'n3'; import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata'; import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier'; -import type { FileIdentifierMapper, FileIdentifierMapperFactory } from '../../storage/mapping/FileIdentifierMapper'; +import type { + FileIdentifierMapper, + FileIdentifierMapperFactory, + ResourceLink, +} from '../../storage/mapping/FileIdentifierMapper'; +import { isContainerIdentifier } from '../../util/PathUtil'; import { guardedStreamFrom } from '../../util/StreamUtil'; import type { Resource, ResourcesGenerator } from './ResourcesGenerator'; import type { TemplateEngine } from './TemplateEngine'; @@ -20,6 +26,7 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator { private readonly templateFolder: string; private readonly factory: FileIdentifierMapperFactory; private readonly engine: TemplateEngine; + private readonly metaExtension = '.meta'; /** * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier. @@ -36,57 +43,132 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator { public async* generate(location: ResourceIdentifier, options: Dict): AsyncIterable { const mapper = await this.factory.create(location.path, this.templateFolder); - yield* this.parseFolder(this.templateFolder, mapper, options); + const folderLink = await mapper.mapFilePathToUrl(this.templateFolder, true); + yield* this.parseFolder(folderLink, mapper, options); } /** * Generates results for all entries in the given folder, including the folder itself. */ - private async* parseFolder(filePath: string, mapper: FileIdentifierMapper, options: Dict): + private async* parseFolder(folderLink: ResourceLink, mapper: FileIdentifierMapper, options: Dict): AsyncIterable { - // Generate representation for the container - const link = await mapper.mapFilePathToUrl(filePath, true); - yield { - identifier: link.identifier, - representation: { - binary: true, - data: guardedStreamFrom([]), - metadata: new RepresentationMetadata(link.identifier), - }, - }; + // Group resource links with their corresponding metadata links + const links = await this.groupLinks(this.generateLinks(folderLink.filePath, mapper)); - // Generate representations for all resources in this container - const files = await fsPromises.readdir(filePath); - for (const childName of files) { - const childPath = joinPath(filePath, childName); - const childStats = await fsPromises.lstat(childPath); - if (childStats.isDirectory()) { - yield* this.parseFolder(childPath, mapper, options); - } else if (childStats.isFile()) { - yield this.generateDocument(childPath, mapper, options); + // Remove root metadata if it exists + const metaLink = links[folderLink.identifier.path]?.meta; + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete links[folderLink.identifier.path]; + + yield this.generateResource(folderLink, options, metaLink); + + for (const { link, meta } of Object.values(links)) { + if (isContainerIdentifier(link.identifier)) { + yield* this.parseFolder(link, mapper, options); + } else { + yield this.generateResource(link, options, meta); } } } /** - * Generates a new Representation corresponding to the template file at the given location. + * Generates ResourceLinks for each entry in the given folder. */ - private async generateDocument(filePath: string, mapper: FileIdentifierMapper, options: Dict): - Promise { - const link = await mapper.mapFilePathToUrl(filePath, false); - const metadata = new RepresentationMetadata(link.identifier); - metadata.contentType = link.contentType; + private async* generateLinks(folderPath: string, mapper: FileIdentifierMapper): AsyncIterable { + const files = await fsPromises.readdir(folderPath); + for (const name of files) { + const filePath = joinPath(folderPath, name); + const stats = await fsPromises.lstat(filePath); + yield mapper.mapFilePathToUrl(filePath, stats.isDirectory()); + } + } - const raw = await fsPromises.readFile(filePath, 'utf8'); - const compiled = this.engine.apply(raw, options); + /** + * Parses a group of ResourceLinks so resources and their metadata are grouped together. + */ + private async groupLinks(linkGen: AsyncIterable): + Promise> { + const links: Record = { }; + for await (const link of linkGen) { + const { path } = link.identifier; + if (this.isMeta(path)) { + const resourcePath = this.metaToResource(link.identifier).path; + links[resourcePath] = Object.assign(links[resourcePath] || {}, { meta: link }); + } else { + links[path] = Object.assign(links[path] || {}, { link }); + } + } + return links; + } + + /** + * Generates a Resource object for the given ResourceLink. + * In the case of documents the corresponding template will be used. + * If a ResourceLink of metadata is provided the corresponding data will be added as metadata. + */ + private async generateResource(link: ResourceLink, options: Dict, metaLink?: ResourceLink): + Promise { + const data: string[] = []; + const metadata = new RepresentationMetadata(link.identifier); + + // Read file if it is not a container + if (!isContainerIdentifier(link.identifier)) { + const compiled = await this.parseTemplate(link.filePath, options); + data.push(compiled); + metadata.contentType = link.contentType; + } + + // Add metadata from meta file if there is one + if (metaLink) { + const rawMetadata = await this.generateMetadata(metaLink, options); + metadata.addQuads(rawMetadata.quads()); + } return { identifier: link.identifier, representation: { binary: true, - data: guardedStreamFrom([ compiled ]), + data: guardedStreamFrom(data), metadata, }, }; } + + /** + * Generates a RepresentationMetadata using the given template. + */ + private async generateMetadata(metaLink: ResourceLink, options: Dict): + Promise { + const identifier = this.metaToResource(metaLink.identifier); + const metadata = new RepresentationMetadata(identifier); + + const data = await this.parseTemplate(metaLink.filePath, options); + const parser = new Parser({ format: metaLink.contentType, baseIRI: identifier.path }); + const quads = parser.parse(data); + metadata.addQuads(quads); + + return metadata; + } + + /** + * Applies the given options to the template found at the given path. + */ + private async parseTemplate(filePath: string, options: Dict): Promise { + const raw = await fsPromises.readFile(filePath, 'utf8'); + return this.engine.apply(raw, options); + } + + /** + * Verifies if the given path corresponds to a metadata file. + */ + private isMeta(path: string): boolean { + return path.endsWith(this.metaExtension); + } + + /** + * Converts a generated metadata identifier to the identifier of its corresponding resource. + */ + private metaToResource(metaIdentifier: ResourceIdentifier): ResourceIdentifier { + return { path: metaIdentifier.path.slice(0, -this.metaExtension.length) }; + } } diff --git a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts b/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts index 3f48d1645..531bd5279 100644 --- a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts +++ b/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts @@ -1,6 +1,6 @@ import type { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier'; +import { HandlebarsTemplateEngine } from '../../../../src/pods/generate/HandlebarsTemplateEngine'; import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator'; -import type { TemplateEngine } from '../../../../src/pods/generate/TemplateEngine'; import type { FileIdentifierMapper, FileIdentifierMapperFactory, @@ -9,7 +9,6 @@ import type { import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil'; import { readableToString } from '../../../../src/util/StreamUtil'; import { mockFs } from '../../../util/Util'; -import Dict = NodeJS.Dict; jest.mock('fs'); @@ -30,13 +29,6 @@ class DummyFactory implements FileIdentifierMapperFactory { } } -class DummyEngine implements TemplateEngine { - public apply(template: string, options: Dict): string { - const keys = Object.keys(options); - return `${template}${keys.map((key): string => `{${key}:${options[key]}}`).join('')}`; - } -} - const genToArray = async(iterable: AsyncIterable): Promise => { const arr: T[] = []; for await (const result of iterable) { @@ -47,7 +39,8 @@ const genToArray = async(iterable: AsyncIterable): Promise => { describe('A TemplatedResourcesGenerator', (): void => { const rootFilePath = 'templates'; - const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new DummyEngine()); + // Using handlebars engine since it's smaller than any possible dummy + const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new HandlebarsTemplateEngine()); let cache: { data: any }; const template = '<{{webId}}> a .'; const location = { path: 'http://test.com/alice/' }; @@ -68,11 +61,11 @@ describe('A TemplatedResourcesGenerator', (): void => { expect(representation.binary).toBe(true); expect(representation.metadata.contentType).toBe('text/turtle'); await expect(readableToString(representation.data)).resolves - .toEqual(`<{{webId}}> a .{webId:${webId}}`); + .toEqual(`<${webId}> a .`); }); - it('creates the necessary containers and ignores non-files.', async(): Promise => { - cache.data = { container: { container: { template }}, 2: 5 }; + it('creates the necessary containers.', async(): Promise => { + cache.data = { container: { container: { template }}}; const result = await genToArray(generator.generate(location, { webId })); const identifiers = result.map((res): ResourceIdentifier => res.identifier); const id = { path: `${location.path}container/container/template` }; @@ -85,6 +78,37 @@ describe('A TemplatedResourcesGenerator', (): void => { const { representation } = result[3]; await expect(readableToString(representation.data)).resolves - .toEqual(`<{{webId}}> a .{webId:${webId}}`); + .toEqual(`<${webId}> a .`); + }); + + it('adds metadata from .meta files.', async(): Promise => { + const meta = '<> "metadata".'; + cache.data = { '.meta': meta, container: { 'template.meta': meta, template }}; + + // Not using options since our dummy template generator generates invalid turtle + const result = await genToArray(generator.generate(location, { webId })); + const identifiers = result.map((res): ResourceIdentifier => res.identifier); + expect(identifiers).toEqual([ + location, + { path: `${location.path}container/` }, + { path: `${location.path}container/template` }, + ]); + // Root has the 1 raw metadata triple (with <> changed to its identifier) + const rootMetadata = result[0].representation.metadata; + expect(rootMetadata.identifier.value).toBe(location.path); + expect(rootMetadata.quads()).toHaveLength(1); + expect(rootMetadata.get('pre:has')?.value).toBe('metadata'); + + // Container has no metadata triples + const contMetadata = result[1].representation.metadata; + expect(contMetadata.identifier.value).toBe(`${location.path}container/`); + expect(contMetadata.quads()).toHaveLength(0); + + // Document has the 1 raw metadata triple (with <> changed to its identifier) and content-type + const docMetadata = result[2].representation.metadata; + expect(docMetadata.identifier.value).toBe(`${location.path}container/template`); + expect(docMetadata.quads()).toHaveLength(2); + expect(docMetadata.get('pre:has')?.value).toBe('metadata'); + expect(docMetadata.contentType).toBe('text/turtle'); }); });