diff --git a/index.ts b/index.ts index 05f856a64..fd960e730 100644 --- a/index.ts +++ b/index.ts @@ -130,7 +130,7 @@ export * from './src/storage/routing/RouterRule'; export * from './src/storage/AtomicResourceStore'; export * from './src/storage/Conditions'; export * from './src/storage/DataAccessorBasedStore'; -export * from './src/storage/FileIdentifierMapper'; +export * from './src/storage/mapping/FileIdentifierMapper'; export * from './src/storage/LockingResourceStore'; export * from './src/storage/MonitoringStore'; export * from './src/storage/PassthroughStore'; diff --git a/package-lock.json b/package-lock.json index bbed4f5c8..566191a77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4734,6 +4734,25 @@ "dev": true, "optional": true }, + "handlebars": { + "version": "4.7.6", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.6.tgz", + "integrity": "sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA==", + "requires": { + "minimist": "^1.2.5", + "neo-async": "^2.6.0", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4", + "wordwrap": "^1.0.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + } + } + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -6763,6 +6782,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + }, "nested-error-stacks": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.0.1.tgz", @@ -9613,6 +9637,12 @@ "integrity": "sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ==", "dev": true }, + "uglify-js": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.12.0.tgz", + "integrity": "sha512-8lBMSkFZuAK7gGF8LswsXmir8eX8d2AAMOnxSDWjKBx/fBR6MypQjs78m6ML9zQVp1/hD4TBdfeMZMC7nW1TAA==", + "optional": true + }, "undefsafe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", @@ -9991,6 +10021,11 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" + }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index d8dee9821..a939ac161 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "cors": "^2.8.5", "express": "^4.17.1", "fetch-sparql-endpoint": "^1.8.0", + "handlebars": "^4.7.6", "mime-types": "^2.1.27", "n3": "^1.6.4", "rdf-parse": "^1.5.0", diff --git a/src/pods/generate/HandlebarsTemplateEngine.ts b/src/pods/generate/HandlebarsTemplateEngine.ts new file mode 100644 index 000000000..fcf474b96 --- /dev/null +++ b/src/pods/generate/HandlebarsTemplateEngine.ts @@ -0,0 +1,12 @@ +import { compile } from 'handlebars'; +import type { TemplateEngine } from './TemplateEngine'; + +/** + * Fills in Handlebars templates. + */ +export class HandlebarsTemplateEngine implements TemplateEngine { + public apply(template: string, options: NodeJS.Dict): string { + const compiled = compile(template); + return compiled(options); + } +} diff --git a/src/pods/generate/TemplateEngine.ts b/src/pods/generate/TemplateEngine.ts new file mode 100644 index 000000000..0f4224aa7 --- /dev/null +++ b/src/pods/generate/TemplateEngine.ts @@ -0,0 +1,8 @@ +import Dict = NodeJS.Dict; + +/** + * A template engine takes as input a template and applies the given options to it. + */ +export interface TemplateEngine { + apply: (template: string, options: Dict) => string; +} diff --git a/src/pods/generate/TemplatedResourcesGenerator.ts b/src/pods/generate/TemplatedResourcesGenerator.ts new file mode 100644 index 000000000..c3c0ae8d0 --- /dev/null +++ b/src/pods/generate/TemplatedResourcesGenerator.ts @@ -0,0 +1,92 @@ +import { promises as fsPromises } from 'fs'; +import { posix } from 'path'; +import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata'; +import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier'; +import type { FileIdentifierMapper, FileIdentifierMapperFactory } from '../../storage/mapping/FileIdentifierMapper'; +import { guardedStreamFrom } from '../../util/StreamUtil'; +import type { Resource, ResourcesGenerator } from './ResourcesGenerator'; +import type { TemplateEngine } from './TemplateEngine'; +import Dict = NodeJS.Dict; + +const { join: joinPath } = posix; + +/** + * Generates resources by making use of a template engine. + * The template folder structure will be kept. + * Folders will be interpreted as containers and files as documents. + * A FileIdentifierMapper will be used to generate identifiers that correspond to the relative structure. + */ +export class TemplatedResourcesGenerator implements ResourcesGenerator { + private readonly templateFolder: string; + private readonly factory: FileIdentifierMapperFactory; + private readonly engine: TemplateEngine; + + /** + * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier. + * + * @param templateFolder - Folder where the templates are located. + * @param factory - Factory used to generate mapper relative to the base identifier. + * @param engine - Template engine for generating the resources. + */ + public constructor(templateFolder: string, factory: FileIdentifierMapperFactory, engine: TemplateEngine) { + this.templateFolder = templateFolder; + this.factory = factory; + this.engine = engine; + } + + public async* generate(location: ResourceIdentifier, options: Dict): AsyncIterable { + const mapper = await this.factory.create(location.path, this.templateFolder); + yield* this.parseFolder(this.templateFolder, mapper, options); + } + + /** + * Generates results for all entries in the given folder, including the folder itself. + */ + private async* parseFolder(filePath: string, mapper: FileIdentifierMapper, options: Dict): + AsyncIterable { + // Generate representation for the container + const link = await mapper.mapFilePathToUrl(filePath, true); + yield { + identifier: link.identifier, + representation: { + binary: true, + data: guardedStreamFrom([]), + metadata: new RepresentationMetadata(link.identifier.path), + }, + }; + + // Generate representations for all resources in this container + const files = await fsPromises.readdir(filePath); + for (const childName of files) { + const childPath = joinPath(filePath, childName); + const childStats = await fsPromises.lstat(childPath); + if (childStats.isDirectory()) { + yield* this.parseFolder(childPath, mapper, options); + } else if (childStats.isFile()) { + yield this.generateDocument(childPath, mapper, options); + } + } + } + + /** + * Generates a new Representation corresponding to the template file at the given location. + */ + private async generateDocument(filePath: string, mapper: FileIdentifierMapper, options: Dict): + Promise { + const link = await mapper.mapFilePathToUrl(filePath, false); + const metadata = new RepresentationMetadata(link.identifier.path); + metadata.contentType = link.contentType; + + const raw = await fsPromises.readFile(filePath, 'utf8'); + const compiled = this.engine.apply(raw, options); + + return { + identifier: link.identifier, + representation: { + binary: true, + data: guardedStreamFrom([ compiled ]), + metadata, + }, + }; + } +} diff --git a/src/storage/accessors/FileDataAccessor.ts b/src/storage/accessors/FileDataAccessor.ts index 8a4d4a8ad..ea994347d 100644 --- a/src/storage/accessors/FileDataAccessor.ts +++ b/src/storage/accessors/FileDataAccessor.ts @@ -19,7 +19,7 @@ import { parseQuads, pushQuad, serializeQuads } from '../../util/QuadUtil'; import { generateContainmentQuads, generateResourceQuads } from '../../util/ResourceUtil'; import { CONTENT_TYPE, DCTERMS, POSIX, RDF, XSD } from '../../util/UriConstants'; import { toNamedNode, toTypedLiteral } from '../../util/UriUtil'; -import type { FileIdentifierMapper, ResourceLink } from '../FileIdentifierMapper'; +import type { FileIdentifierMapper, ResourceLink } from '../mapping/FileIdentifierMapper'; import type { DataAccessor } from './DataAccessor'; const { join: joinPath } = posix; diff --git a/src/storage/mapping/ExtensionBasedMapper.ts b/src/storage/mapping/ExtensionBasedMapper.ts index 012b48434..9e3a10185 100644 --- a/src/storage/mapping/ExtensionBasedMapper.ts +++ b/src/storage/mapping/ExtensionBasedMapper.ts @@ -12,7 +12,7 @@ import { isContainerIdentifier, trimTrailingSlashes, } from '../../util/PathUtil'; -import type { FileIdentifierMapper, ResourceLink } from '../FileIdentifierMapper'; +import type { FileIdentifierMapper, FileIdentifierMapperFactory, ResourceLink } from './FileIdentifierMapper'; import { getAbsolutePath, getRelativePath, validateRelativePath } from './MapperUtil'; const { join: joinPath, normalize: normalizePath } = posix; @@ -197,3 +197,10 @@ export class ExtensionBasedMapper implements FileIdentifierMapper { return extension && extension[1]; } } + +export class ExtensionBasedMapperFactory implements FileIdentifierMapperFactory { + public async create(base: string, rootFilePath: string): Promise { + return new ExtensionBasedMapper(base, rootFilePath); + } +} + diff --git a/src/storage/FileIdentifierMapper.ts b/src/storage/mapping/FileIdentifierMapper.ts similarity index 71% rename from src/storage/FileIdentifierMapper.ts rename to src/storage/mapping/FileIdentifierMapper.ts index 76148bf0a..e55aa07a7 100644 --- a/src/storage/FileIdentifierMapper.ts +++ b/src/storage/mapping/FileIdentifierMapper.ts @@ -1,4 +1,4 @@ -import type { ResourceIdentifier } from '../ldp/representation/ResourceIdentifier'; +import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier'; export interface ResourceLink { /** @@ -38,3 +38,11 @@ export interface FileIdentifierMapper { */ mapUrlToFilePath: (identifier: ResourceIdentifier, contentType?: string) => Promise; } + +/** + * Factory that can create FileIdentifierMappers so the base and rootFilePath can be set dynamically. + * Specifically used when identifiers need to be generated for a new pod (since pod identifiers are generated). + */ +export interface FileIdentifierMapperFactory { + create: (base: string, rootFilePath: string) => Promise; +} diff --git a/src/storage/mapping/FixedContentTypeMapper.ts b/src/storage/mapping/FixedContentTypeMapper.ts index 8a62967cc..1a01efd3e 100644 --- a/src/storage/mapping/FixedContentTypeMapper.ts +++ b/src/storage/mapping/FixedContentTypeMapper.ts @@ -7,7 +7,7 @@ import { ensureTrailingSlash, isContainerIdentifier, trimTrailingSlashes, } from '../../util/PathUtil'; -import type { FileIdentifierMapper, ResourceLink } from '../FileIdentifierMapper'; +import type { FileIdentifierMapper, ResourceLink } from './FileIdentifierMapper'; import { getAbsolutePath, getRelativePath, validateRelativePath } from './MapperUtil'; const { normalize: normalizePath } = posix; diff --git a/test/unit/pods/generate/HandlebarsTemplateEngine.test.ts b/test/unit/pods/generate/HandlebarsTemplateEngine.test.ts new file mode 100644 index 000000000..ba5d220bb --- /dev/null +++ b/test/unit/pods/generate/HandlebarsTemplateEngine.test.ts @@ -0,0 +1,11 @@ +import { HandlebarsTemplateEngine } from '../../../../src/pods/generate/HandlebarsTemplateEngine'; + +describe('A HandlebarsTemplateEngine', (): void => { + const engine = new HandlebarsTemplateEngine(); + + it('fills in Handlebars templates.', async(): Promise => { + const template = '<{{webId}}> a .'; + const options = { webId: 'http://alice/#profile' }; + expect(engine.apply(template, options)).toBe(' a .'); + }); +}); diff --git a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts b/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts new file mode 100644 index 000000000..3f48d1645 --- /dev/null +++ b/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts @@ -0,0 +1,90 @@ +import type { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier'; +import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator'; +import type { TemplateEngine } from '../../../../src/pods/generate/TemplateEngine'; +import type { + FileIdentifierMapper, + FileIdentifierMapperFactory, + ResourceLink, +} from '../../../../src/storage/mapping/FileIdentifierMapper'; +import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil'; +import { readableToString } from '../../../../src/util/StreamUtil'; +import { mockFs } from '../../../util/Util'; +import Dict = NodeJS.Dict; + +jest.mock('fs'); + +class DummyFactory implements FileIdentifierMapperFactory { + public async create(base: string, rootFilePath: string): Promise { + const trimBase = trimTrailingSlashes(base); + const trimRoot = trimTrailingSlashes(rootFilePath); + return { + async mapFilePathToUrl(filePath: string, isContainer: boolean): Promise { + const path = `${trimBase}${filePath.slice(trimRoot.length)}`; + return { + identifier: { path: isContainer ? ensureTrailingSlash(path) : path }, + filePath, + contentType: isContainer ? undefined : 'text/turtle', + }; + }, + } as any; + } +} + +class DummyEngine implements TemplateEngine { + public apply(template: string, options: Dict): string { + const keys = Object.keys(options); + return `${template}${keys.map((key): string => `{${key}:${options[key]}}`).join('')}`; + } +} + +const genToArray = async(iterable: AsyncIterable): Promise => { + const arr: T[] = []; + for await (const result of iterable) { + arr.push(result); + } + return arr; +}; + +describe('A TemplatedResourcesGenerator', (): void => { + const rootFilePath = 'templates'; + const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new DummyEngine()); + let cache: { data: any }; + const template = '<{{webId}}> a .'; + const location = { path: 'http://test.com/alice/' }; + const webId = 'http://alice/#profile'; + + beforeEach(async(): Promise => { + cache = mockFs(rootFilePath); + }); + + it('fills in a template with the given options.', async(): Promise => { + cache.data = { template }; + const result = await genToArray(generator.generate(location, { webId })); + const identifiers = result.map((res): ResourceIdentifier => res.identifier); + const id = { path: `${location.path}template` }; + expect(identifiers).toEqual([ location, id ]); + + const { representation } = result[1]; + expect(representation.binary).toBe(true); + expect(representation.metadata.contentType).toBe('text/turtle'); + await expect(readableToString(representation.data)).resolves + .toEqual(`<{{webId}}> a .{webId:${webId}}`); + }); + + it('creates the necessary containers and ignores non-files.', async(): Promise => { + cache.data = { container: { container: { template }}, 2: 5 }; + const result = await genToArray(generator.generate(location, { webId })); + const identifiers = result.map((res): ResourceIdentifier => res.identifier); + const id = { path: `${location.path}container/container/template` }; + expect(identifiers).toEqual([ + location, + { path: `${location.path}container/` }, + { path: `${location.path}container/container/` }, + id, + ]); + + const { representation } = result[3]; + await expect(readableToString(representation.data)).resolves + .toEqual(`<{{webId}}> a .{webId:${webId}}`); + }); +}); diff --git a/test/unit/storage/mapping/ExtensionBasedMapper.test.ts b/test/unit/storage/mapping/ExtensionBasedMapper.test.ts index 9e3ddaceb..2a0296285 100644 --- a/test/unit/storage/mapping/ExtensionBasedMapper.test.ts +++ b/test/unit/storage/mapping/ExtensionBasedMapper.test.ts @@ -1,5 +1,8 @@ import fs from 'fs'; -import { ExtensionBasedMapper } from '../../../../src/storage/mapping/ExtensionBasedMapper'; +import { + ExtensionBasedMapper, + ExtensionBasedMapperFactory, +} from '../../../../src/storage/mapping/ExtensionBasedMapper'; import { BadRequestHttpError } from '../../../../src/util/errors/BadRequestHttpError'; import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError'; import { trimTrailingSlashes } from '../../../../src/util/PathUtil'; @@ -135,4 +138,12 @@ describe('An ExtensionBasedMapper', (): void => { }); }); }); + + describe('An ExtensionBasedMapperFactory', (): void => { + const factory = new ExtensionBasedMapperFactory(); + + it('creates an ExtensionBasedMapper.', async(): Promise => { + await expect(factory.create('base', 'filePath')).resolves.toBeInstanceOf(ExtensionBasedMapper); + }); + }); }); diff --git a/test/util/Util.ts b/test/util/Util.ts index 16ecde136..0ca1aea27 100644 --- a/test/util/Util.ts +++ b/test/util/Util.ts @@ -160,6 +160,10 @@ export const mockFs = (rootFilepath?: string, time?: Date): { data: any } => { } folder[name] = {}; }, + readFile(path: string): string { + const { folder, name } = getFolder(path); + return folder[name]; + }, }, };