From 40f2c8ea42221fff706df66f01ddf6dccf58d462 Mon Sep 17 00:00:00 2001 From: Joachim Van Herwegen Date: Fri, 19 Aug 2022 11:12:02 +0200 Subject: [PATCH] feat: Update templates and generators to support ACP --- .../app/init/initializers/prefilled-root.json | 7 +- config/app/init/initializers/root.json | 7 +- config/app/setup/handlers/setup.json | 7 +- config/identity/access/initializers/idp.json | 7 +- .../access/initializers/well-known.json | 7 +- config/identity/pod/dynamic.json | 2 +- .../pod/resource-generators/templated.json | 33 ++- config/identity/pod/static.json | 2 +- config/ldp/authorization/acp.json | 6 + config/ldp/authorization/webacl.json | 6 + src/index.ts | 3 + src/pods/generate/BaseResourcesGenerator.ts | 237 ++++++++++++++++++ src/pods/generate/ResourcesGenerator.ts | 2 +- src/pods/generate/StaticFolderGenerator.ts | 21 ++ .../generate/SubfolderResourcesGenerator.ts | 56 +++++ .../generate/TemplatedResourcesGenerator.ts | 219 +--------------- src/util/ContentTypes.ts | 1 + src/util/IterableUtil.ts | 90 +++++++ templates/pod/acp/.acr.hbs | 39 +++ templates/pod/acp/README.acr | 18 ++ templates/pod/acp/profile/card.acr | 22 ++ templates/pod/{ => base}/.meta | 0 templates/pod/{ => base}/README$.md.hbs | 0 .../pod/{ => base}/profile/card$.ttl.hbs | 0 templates/pod/{ => wac}/.acl.hbs | 0 templates/pod/{ => wac}/README.acl.hbs | 0 templates/pod/{ => wac}/profile/card.acl.hbs | 0 templates/root/empty/acp/.acr | 32 +++ templates/root/empty/{ => base}/.meta | 0 templates/root/empty/{ => wac}/.acl | 0 templates/root/prefilled/acp/.acr | 32 +++ templates/root/prefilled/{ => base}/.meta | 0 .../root/prefilled/{ => base}/index.html | 2 +- templates/root/prefilled/{ => wac}/.acl | 0 test/assets/templates/profile/card$.ttl | 2 +- test/unit/authorization/AcpReader.test.ts | 10 +- ...test.ts => BaseResourcesGenerator.test.ts} | 47 ++-- .../generate/StaticFolderGenerator.test.ts | 26 ++ .../SubfolderResourcesGenerator.test.ts | 90 +++++++ test/unit/util/IterableUtil.test.ts | 30 ++- test/util/Util.ts | 8 + 41 files changed, 800 insertions(+), 271 deletions(-) create mode 100644 src/pods/generate/BaseResourcesGenerator.ts create mode 100644 src/pods/generate/StaticFolderGenerator.ts create mode 100644 src/pods/generate/SubfolderResourcesGenerator.ts create mode 100644 templates/pod/acp/.acr.hbs create mode 100644 templates/pod/acp/README.acr create mode 100644 templates/pod/acp/profile/card.acr rename templates/pod/{ => base}/.meta (100%) rename templates/pod/{ => base}/README$.md.hbs (100%) rename templates/pod/{ => base}/profile/card$.ttl.hbs (100%) rename templates/pod/{ => wac}/.acl.hbs (100%) rename templates/pod/{ => wac}/README.acl.hbs (100%) rename templates/pod/{ => wac}/profile/card.acl.hbs (100%) create mode 100644 templates/root/empty/acp/.acr rename templates/root/empty/{ => base}/.meta (100%) rename templates/root/empty/{ => wac}/.acl (100%) create mode 100644 templates/root/prefilled/acp/.acr rename templates/root/prefilled/{ => base}/.meta (100%) rename templates/root/prefilled/{ => base}/index.html (97%) rename templates/root/prefilled/{ => wac}/.acl (100%) rename test/unit/pods/generate/{TemplatedResourcesGenerator.test.ts => BaseResourcesGenerator.test.ts} (80%) create mode 100644 test/unit/pods/generate/StaticFolderGenerator.test.ts create mode 100644 test/unit/pods/generate/SubfolderResourcesGenerator.test.ts diff --git a/config/app/init/initializers/prefilled-root.json b/config/app/init/initializers/prefilled-root.json index f5c784ab2..9614c3116 100644 --- a/config/app/init/initializers/prefilled-root.json +++ b/config/app/init/initializers/prefilled-root.json @@ -14,12 +14,9 @@ "args_path": "/", "args_store": { "@id": "urn:solid-server:default:ResourceStore" }, "args_generator": { - "@type": "TemplatedResourcesGenerator", + "@type": "StaticFolderGenerator", "templateFolder": "@css:templates/root/prefilled", - "factory": { "@type": "ExtensionBasedMapperFactory" }, - "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" }, - "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, - "store": { "@id": "urn:solid-server:default:ResourceStore"} + "resourcesGenerator": { "@id": "urn:solid-server:default:TemplatedResourcesGenerator" } }, "args_storageKey": "rootInitialized", "args_storage": { "@id": "urn:solid-server:default:SetupStorage" } diff --git a/config/app/init/initializers/root.json b/config/app/init/initializers/root.json index 64a8abaae..277bb8149 100644 --- a/config/app/init/initializers/root.json +++ b/config/app/init/initializers/root.json @@ -14,12 +14,9 @@ "args_path": "/", "args_store": { "@id": "urn:solid-server:default:ResourceStore" }, "args_generator": { - "@type": "TemplatedResourcesGenerator", + "@type": "StaticFolderGenerator", "templateFolder": "@css:templates/root/empty", - "factory": { "@type": "ExtensionBasedMapperFactory" }, - "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" }, - "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, - "store": { "@id": "urn:solid-server:default:ResourceStore"} + "resourcesGenerator": { "@id": "urn:solid-server:default:TemplatedResourcesGenerator" } }, "args_storageKey": "rootInitialized", "args_storage": { "@id": "urn:solid-server:default:SetupStorage" } diff --git a/config/app/setup/handlers/setup.json b/config/app/setup/handlers/setup.json index 847444ef1..7764354ea 100644 --- a/config/app/setup/handlers/setup.json +++ b/config/app/setup/handlers/setup.json @@ -60,12 +60,9 @@ "args_path": "/", "args_store": { "@id": "urn:solid-server:default:ResourceStore" }, "args_generator": { - "@type": "TemplatedResourcesGenerator", + "@type": "StaticFolderGenerator", "templateFolder": "@css:templates/root/empty", - "factory": { "@type": "ExtensionBasedMapperFactory" }, - "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" }, - "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, - "store": { "@id": "urn:solid-server:default:ResourceStore"} + "resourcesGenerator": { "@id": "urn:solid-server:default:TemplatedResourcesGenerator" } }, "args_storageKey": "rootInitialized", "args_storage": { "@id": "urn:solid-server:default:SetupStorage" } diff --git a/config/identity/access/initializers/idp.json b/config/identity/access/initializers/idp.json index b4414ef1a..465966cfc 100644 --- a/config/identity/access/initializers/idp.json +++ b/config/identity/access/initializers/idp.json @@ -14,12 +14,9 @@ "args_path": "/idp/", "args_store": { "@id": "urn:solid-server:default:ResourceStore" }, "args_generator": { - "@type": "TemplatedResourcesGenerator", + "@type": "StaticFolderGenerator", "templateFolder": "@css:templates/root/empty", - "factory": { "@type": "ExtensionBasedMapperFactory" }, - "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" }, - "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, - "store": { "@id": "urn:solid-server:default:ResourceStore"} + "resourcesGenerator": { "@id": "urn:solid-server:default:TemplatedResourcesGenerator" } }, "args_storageKey": "idpContainerInitialized", "args_storage": { "@id": "urn:solid-server:default:SetupStorage" } diff --git a/config/identity/access/initializers/well-known.json b/config/identity/access/initializers/well-known.json index a16ad5d6a..d6f33342d 100644 --- a/config/identity/access/initializers/well-known.json +++ b/config/identity/access/initializers/well-known.json @@ -14,12 +14,9 @@ "args_path": "/.well-known/", "args_store": { "@id": "urn:solid-server:default:ResourceStore" }, "args_generator": { - "@type": "TemplatedResourcesGenerator", + "@type": "StaticFolderGenerator", "templateFolder": "@css:templates/root/empty", - "factory": { "@type": "ExtensionBasedMapperFactory" }, - "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" }, - "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, - "store": { "@id": "urn:solid-server:default:ResourceStore"} + "resourcesGenerator": { "@id": "urn:solid-server:default:TemplatedResourcesGenerator" } }, "args_storageKey": "wellKnownContainerInitialized", "args_storage": { "@id": "urn:solid-server:default:SetupStorage" } diff --git a/config/identity/pod/dynamic.json b/config/identity/pod/dynamic.json index 4ab11eec1..4523621fe 100644 --- a/config/identity/pod/dynamic.json +++ b/config/identity/pod/dynamic.json @@ -12,7 +12,7 @@ "@type": "ConfigPodManager", "podGenerator": { "@id": "urn:solid-server:default:PodGenerator" }, "routingStorage": { "@id": "urn:solid-server:default:PodRoutingStorage" }, - "resourcesGenerator": { "@id": "urn:solid-server:default:ResourcesGenerator" }, + "resourcesGenerator": { "@id": "urn:solid-server:default:PodResourcesGenerator" }, "store": { "@id": "urn:solid-server:default:ResourceStore" } }, diff --git a/config/identity/pod/resource-generators/templated.json b/config/identity/pod/resource-generators/templated.json index 6ac957731..c518fa6a7 100644 --- a/config/identity/pod/resource-generators/templated.json +++ b/config/identity/pod/resource-generators/templated.json @@ -2,16 +2,31 @@ "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^5.0.0/components/context.jsonld", "@graph": [ { - "comment": "Generates resources based on the templates stored in the template folder.", - "@id": "urn:solid-server:default:ResourcesGenerator", - "@type": "TemplatedResourcesGenerator", + "comment": "Generates pods based on the templates in the corresponding folder.", + "@id": "urn:solid-server:default:PodResourcesGenerator", + "@type": "StaticFolderGenerator", "templateFolder": "@css:templates/pod", - "factory": { - "@type": "ExtensionBasedMapperFactory" - }, - "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" }, - "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, - "store": { "@id": "urn:solid-server:default:ResourceStore"} + "resourcesGenerator": { "@id": "urn:solid-server:default:TemplatedResourcesGenerator" } + }, + { + "comment": [ + "Generates resources in the base subfolder of the provided folder.", + "Authorization configurations should add their corresponding subfolder containing the relevant authorization files." + ], + "@id": "urn:solid-server:default:TemplatedResourcesGenerator", + "@type": "SubfolderResourcesGenerator", + "subfolders": [ "base" ], + "resourcesGenerator": { + "@type": "BaseResourcesGenerator", + "factory": { + "@type": "ExtensionBasedMapperFactory" + }, + "templateEngine": { + "@id": "urn:solid-server:default:TemplateEngine" + }, + "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" }, + "store": { "@id": "urn:solid-server:default:ResourceStore"} + } } ] } diff --git a/config/identity/pod/static.json b/config/identity/pod/static.json index d69b5b40e..82302419e 100644 --- a/config/identity/pod/static.json +++ b/config/identity/pod/static.json @@ -9,7 +9,7 @@ "@id": "urn:solid-server:default:PodManager", "@type": "GeneratedPodManager", "store": { "@id": "urn:solid-server:default:ResourceStore" }, - "resourcesGenerator": { "@id": "urn:solid-server:default:ResourcesGenerator" } + "resourcesGenerator": { "@id": "urn:solid-server:default:PodResourcesGenerator" } } ] } diff --git a/config/ldp/authorization/acp.json b/config/ldp/authorization/acp.json index 2b3ed750c..90dc1fc23 100644 --- a/config/ldp/authorization/acp.json +++ b/config/ldp/authorization/acp.json @@ -31,6 +31,12 @@ ] } }, + { + "comment": "The templates for ACP authorization documents are in the acp subfolder.", + "@id": "urn:solid-server:default:TemplatedResourcesGenerator", + "@type": "SubfolderResourcesGenerator", + "subfolders": [ "acp" ] + }, { "comment": "In case of ACP authorization the ACR resources determine authorization.", "@id": "urn:solid-server:default:AuthResourceHttpHandler", diff --git a/config/ldp/authorization/webacl.json b/config/ldp/authorization/webacl.json index 4a49ee4bd..82283a9e1 100644 --- a/config/ldp/authorization/webacl.json +++ b/config/ldp/authorization/webacl.json @@ -31,6 +31,12 @@ ] } }, + { + "comment": "The templates for WAC authorization documents are in the wac subfolder.", + "@id": "urn:solid-server:default:TemplatedResourcesGenerator", + "@type": "SubfolderResourcesGenerator", + "subfolders": [ "wac" ] + }, { "comment": "In case of WebACL authorization the ACL resources determine authorization.", "@id": "urn:solid-server:default:AuthResourceHttpHandler", diff --git a/src/index.ts b/src/index.ts index 9e99e1402..323a990d0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -258,12 +258,15 @@ export * from './pods/generate/variables/VariableSetter'; // Pods/Generate export * from './pods/generate/BaseComponentsJsFactory'; +export * from './pods/generate/BaseResourcesGenerator'; export * from './pods/generate/ComponentsJsFactory'; export * from './pods/generate/GenerateUtil'; export * from './pods/generate/IdentifierGenerator'; export * from './pods/generate/PodGenerator'; export * from './pods/generate/ResourcesGenerator'; +export * from './pods/generate/StaticFolderGenerator'; export * from './pods/generate/SubdomainIdentifierGenerator'; +export * from './pods/generate/SubfolderResourcesGenerator'; export * from './pods/generate/SuffixIdentifierGenerator'; export * from './pods/generate/TemplatedPodGenerator'; export * from './pods/generate/TemplatedResourcesGenerator'; diff --git a/src/pods/generate/BaseResourcesGenerator.ts b/src/pods/generate/BaseResourcesGenerator.ts new file mode 100644 index 000000000..998c72f4e --- /dev/null +++ b/src/pods/generate/BaseResourcesGenerator.ts @@ -0,0 +1,237 @@ +import { createReadStream, promises as fsPromises } from 'fs'; +import type { Readable } from 'stream'; +import { pathExists } from 'fs-extra'; +import { Parser } from 'n3'; +import type { AuxiliaryStrategy } from '../../http/auxiliary/AuxiliaryStrategy'; +import { BasicRepresentation } from '../../http/representation/BasicRepresentation'; +import { RepresentationMetadata } from '../../http/representation/RepresentationMetadata'; +import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier'; +import { getLoggerFor } from '../../logging/LogUtil'; +import type { + FileIdentifierMapper, + FileIdentifierMapperFactory, + ResourceLink, +} from '../../storage/mapping/FileIdentifierMapper'; +import type { ResourceSet } from '../../storage/ResourceSet'; +import { INTERNAL_QUADS } from '../../util/ContentTypes'; +import { guardStream } from '../../util/GuardedStream'; +import type { Guarded } from '../../util/GuardedStream'; +import { joinFilePath, isContainerIdentifier, resolveAssetPath } from '../../util/PathUtil'; +import { addResourceMetadata } from '../../util/ResourceUtil'; +import { guardedStreamFrom, readableToString } from '../../util/StreamUtil'; +import type { TemplateEngine } from '../../util/templates/TemplateEngine'; +import type { Resource } from './ResourcesGenerator'; +import type { TemplatedResourcesGenerator } from './TemplatedResourcesGenerator'; +import Dict = NodeJS.Dict; + +interface TemplateResourceLink extends ResourceLink { + isTemplate: boolean; +} + +/** + * Input arguments required for {@link BaseResourcesGenerator} + */ +export interface SubfolderResourcesGeneratorArgs { + /** + * Factory used to generate mapper relative to the base identifier. + */ + factory: FileIdentifierMapperFactory; + /** + * Template engine for generating the resources. + */ + templateEngine: TemplateEngine; + /** + * The extension of files that need to be interpreted as templates. + * Will be removed to generate the identifier. + */ + templateExtension?: string; + /** + * The metadataStrategy + */ + metadataStrategy: AuxiliaryStrategy; + /** + * The default ResourceStore + */ + store: ResourceSet; +} + +// Comparator for the results of the `groupLinks` call +function comparator(left: { link: TemplateResourceLink }, right: { link: TemplateResourceLink }): number { + return left.link.identifier.path.localeCompare(right.link.identifier.path); +} + +/** + * Generates resources by making use of a template engine. + * The template folder structure will be kept. + * Folders will be interpreted as containers and files as documents. + * A FileIdentifierMapper will be used to generate identifiers that correspond to the relative structure. + * + * Metadata resources will be yielded separately from their subject resource. + * + * A relative `templateFolder` is resolved relative to cwd, + * unless it's preceded by `@css:`, e.g. `@css:foo/bar`. + */ +export class BaseResourcesGenerator implements TemplatedResourcesGenerator { + protected readonly logger = getLoggerFor(this); + + private readonly factory: FileIdentifierMapperFactory; + private readonly templateEngine: TemplateEngine; + private readonly templateExtension: string; + private readonly metadataStrategy: AuxiliaryStrategy; + private readonly store: ResourceSet; + + /** + * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier. + * + * @param args - TemplatedResourcesGeneratorArgs + */ + public constructor(args: SubfolderResourcesGeneratorArgs) { + this.factory = args.factory; + this.templateEngine = args.templateEngine; + this.templateExtension = args.templateExtension ?? '.hbs'; + this.metadataStrategy = args.metadataStrategy; + this.store = args.store; + } + + public async* generate(templateFolder: string, location: ResourceIdentifier, options: Dict): + AsyncIterable { + templateFolder = resolveAssetPath(templateFolder); + + // Ignore folders that don't exist + if (!await pathExists(templateFolder)) { + this.logger.warn(`Ignoring non-existing template folder ${templateFolder}`); + return; + } + + const mapper = await this.factory.create(location.path, templateFolder); + const folderLink = await this.toTemplateLink(templateFolder, mapper); + yield* this.processFolder(folderLink, mapper, options); + } + + /** + * Generates results for all entries in the given folder, including the folder itself. + */ + private async* processFolder(folderLink: TemplateResourceLink, mapper: FileIdentifierMapper, options: Dict): + AsyncIterable { + // Group resource links with their corresponding metadata links + const links = await this.groupLinks(folderLink.filePath, mapper); + + // Remove root metadata if it exists + const metaLink = links[folderLink.identifier.path]?.meta; + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete links[folderLink.identifier.path]; + + yield* this.generateResource(folderLink, options, metaLink); + + // Make sure the results are sorted + for (const { link, meta } of Object.values(links).sort(comparator)) { + if (isContainerIdentifier(link.identifier)) { + yield* this.processFolder(link, mapper, options); + } else { + yield* this.generateResource(link, options, meta); + } + } + } + + /** + * Creates a TemplateResourceLink for the given filePath, + * which connects a resource URL to its template file. + * The identifier will be based on the file path stripped from the template extension, + * but the filePath parameter will still point to the original file. + */ + private async toTemplateLink(filePath: string, mapper: FileIdentifierMapper): Promise { + const stats = await fsPromises.lstat(filePath); + + // Slice the template extension from the filepath for correct identifier generation + const isTemplate = filePath.endsWith(this.templateExtension); + const slicedPath = isTemplate ? filePath.slice(0, -this.templateExtension.length) : filePath; + const link = await mapper.mapFilePathToUrl(slicedPath, stats.isDirectory()); + // We still need the original file path for disk reading though + return { + ...link, + filePath, + isTemplate, + }; + } + + /** + * Generates TemplateResourceLinks for each entry in the given folder + * and combines the results so resources and their metadata are grouped together. + */ + private async groupLinks(folderPath: string, mapper: FileIdentifierMapper): + Promise> { + const files = await fsPromises.readdir(folderPath); + const links: Record = { }; + for (const name of files) { + const link = await this.toTemplateLink(joinFilePath(folderPath, name), mapper); + const { path } = link.identifier; + links[path] = Object.assign(links[path] || {}, link.isMetadata ? { meta: link } : { link }); + } + return links; + } + + /** + * Generates a Resource object for the given ResourceLink. + * In the case of documents the corresponding template will be used. + * If a ResourceLink of metadata is provided the corresponding metadata resource + * will be yielded as a separate resource. + */ + private async* generateResource(link: TemplateResourceLink, options: Dict, metaLink?: TemplateResourceLink): + AsyncIterable { + let data: Guarded | undefined; + const metadata = new RepresentationMetadata(link.identifier); + + // Read file if it is not a container + if (!isContainerIdentifier(link.identifier)) { + data = await this.processFile(link, options); + metadata.contentType = link.contentType; + } + // Do not yield a container resource if it already exists + if (!isContainerIdentifier(link.identifier) || !await this.store.hasResource(link.identifier)) { + this.logger.debug(`Generating resource ${link.identifier.path}`); + yield { + identifier: link.identifier, + representation: new BasicRepresentation(data ?? [], metadata), + }; + } + + // Add metadata from .meta file if there is one + if (metaLink) { + const rawMetadata = await this.generateMetadata(metaLink, options); + const metaIdentifier = this.metadataStrategy.getAuxiliaryIdentifier(link.identifier); + const descriptionMeta = new RepresentationMetadata(metaIdentifier); + addResourceMetadata(rawMetadata, isContainerIdentifier(link.identifier)); + this.logger.debug(`Generating resource ${metaIdentifier.path}`); + yield { + identifier: metaIdentifier, + representation: new BasicRepresentation(rawMetadata.quads(), descriptionMeta, INTERNAL_QUADS), + }; + } + } + + /** + * Generates a RepresentationMetadata using the given template. + */ + private async generateMetadata(metaLink: TemplateResourceLink, options: Dict): + Promise { + const metadata = new RepresentationMetadata(metaLink.identifier); + + const data = await this.processFile(metaLink, options); + const parser = new Parser({ format: metaLink.contentType, baseIRI: metaLink.identifier.path }); + const quads = parser.parse(await readableToString(data)); + metadata.addQuads(quads); + + return metadata; + } + + /** + * Creates a read stream from the file and applies the template if necessary. + */ + private async processFile(link: TemplateResourceLink, contents: Dict): Promise> { + if (link.isTemplate) { + const rendered = await this.templateEngine.handleSafe({ contents, template: { templateFile: link.filePath }}); + return guardedStreamFrom(rendered); + } + return guardStream(createReadStream(link.filePath)); + } +} diff --git a/src/pods/generate/ResourcesGenerator.ts b/src/pods/generate/ResourcesGenerator.ts index fcd44e40f..a6184cab0 100644 --- a/src/pods/generate/ResourcesGenerator.ts +++ b/src/pods/generate/ResourcesGenerator.ts @@ -14,7 +14,7 @@ export interface Resource { export interface ResourcesGenerator { /** * Generates resources with the given options. - * The output Map should be sorted so that containers always appear before their contents. + * The output Iterable should be sorted so that containers always appear before their contents. * @param location - Base identifier. * @param options - Options that can be used when generating resources. * diff --git a/src/pods/generate/StaticFolderGenerator.ts b/src/pods/generate/StaticFolderGenerator.ts new file mode 100644 index 000000000..23c4ce112 --- /dev/null +++ b/src/pods/generate/StaticFolderGenerator.ts @@ -0,0 +1,21 @@ +import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier'; +import type { Resource, ResourcesGenerator } from './ResourcesGenerator'; +import type { TemplatedResourcesGenerator } from './TemplatedResourcesGenerator'; +import Dict = NodeJS.Dict; + +/** + * Stores a static template folder that will be used to call the wrapped {@link TemplatedResourcesGenerator}. + */ +export class StaticFolderGenerator implements ResourcesGenerator { + private readonly resourcesGenerator: TemplatedResourcesGenerator; + private readonly templateFolder: string; + + public constructor(resourcesGenerator: TemplatedResourcesGenerator, templateFolder: string) { + this.resourcesGenerator = resourcesGenerator; + this.templateFolder = templateFolder; + } + + public generate(location: ResourceIdentifier, options: Dict): AsyncIterable { + return this.resourcesGenerator.generate(this.templateFolder, location, options); + } +} diff --git a/src/pods/generate/SubfolderResourcesGenerator.ts b/src/pods/generate/SubfolderResourcesGenerator.ts new file mode 100644 index 000000000..f1c26f542 --- /dev/null +++ b/src/pods/generate/SubfolderResourcesGenerator.ts @@ -0,0 +1,56 @@ +import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier'; +import { sortedAsyncMerge } from '../../util/IterableUtil'; +import { joinFilePath, resolveAssetPath } from '../../util/PathUtil'; +import type { Resource } from './ResourcesGenerator'; +import type { TemplatedResourcesGenerator } from './TemplatedResourcesGenerator'; +import Dict = NodeJS.Dict; + +// Sorts Resources based on their identifiers +function comparator(left: Resource, right: Resource): number { + return left.identifier.path.localeCompare(right.identifier.path); +} + +/** + * Generates all resources found in specific subfolders of the given template folder. + * In case the same resource is defined in several subfolders, + * the data of the last subfolder in the list will be used. + * + * The results of all the subfolders will be merged so the end result is still a sorted stream of identifiers. + * + * One of the main use cases for this class is so template resources can be in a separate folder + * than their corresponding authorization resources, + * allowing for authorization-independent templates. + */ +export class SubfolderResourcesGenerator implements TemplatedResourcesGenerator { + private readonly resourcesGenerator: TemplatedResourcesGenerator; + private readonly subfolders: string[]; + + public constructor(resourcesGenerator: TemplatedResourcesGenerator, subfolders: string[]) { + this.resourcesGenerator = resourcesGenerator; + this.subfolders = subfolders; + } + + public async* generate(templateFolder: string, location: ResourceIdentifier, options: Dict): + AsyncIterable { + const root = resolveAssetPath(templateFolder); + const templateSubfolders = this.subfolders.map((subfolder): string => joinFilePath(root, subfolder)); + + // Build all generators + const generators: AsyncIterator[] = []; + for (const templateSubfolder of templateSubfolders) { + generators.push(this.resourcesGenerator.generate(templateSubfolder, location, options)[Symbol.asyncIterator]()); + } + + let previous: ResourceIdentifier = { path: '' }; + for await (const result of sortedAsyncMerge(generators, comparator)) { + // Skip duplicate results. + // In practice these are just going to be the same empty containers. + if (result.identifier.path === previous.path) { + result.representation.data.destroy(); + } else { + previous = result.identifier; + yield result; + } + } + } +} diff --git a/src/pods/generate/TemplatedResourcesGenerator.ts b/src/pods/generate/TemplatedResourcesGenerator.ts index 9a2eb6529..1d1b8fda1 100644 --- a/src/pods/generate/TemplatedResourcesGenerator.ts +++ b/src/pods/generate/TemplatedResourcesGenerator.ts @@ -1,216 +1,21 @@ -import { createReadStream, promises as fsPromises } from 'fs'; -import type { Readable } from 'stream'; -import { Parser } from 'n3'; -import type { AuxiliaryStrategy } from '../../http/auxiliary/AuxiliaryStrategy'; -import { BasicRepresentation } from '../../http/representation/BasicRepresentation'; -import { RepresentationMetadata } from '../../http/representation/RepresentationMetadata'; import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier'; -import type { - FileIdentifierMapper, - FileIdentifierMapperFactory, - ResourceLink, -} from '../../storage/mapping/FileIdentifierMapper'; -import type { ResourceSet } from '../../storage/ResourceSet'; -import { INTERNAL_QUADS } from '../../util/ContentTypes'; -import { guardStream } from '../../util/GuardedStream'; -import type { Guarded } from '../../util/GuardedStream'; -import { joinFilePath, isContainerIdentifier, resolveAssetPath } from '../../util/PathUtil'; -import { addResourceMetadata } from '../../util/ResourceUtil'; -import { guardedStreamFrom, readableToString } from '../../util/StreamUtil'; -import type { TemplateEngine } from '../../util/templates/TemplateEngine'; -import type { Resource, ResourcesGenerator } from './ResourcesGenerator'; +import type { Resource } from './ResourcesGenerator'; import Dict = NodeJS.Dict; -interface TemplateResourceLink extends ResourceLink { - isTemplate: boolean; -} - /** - * Input arguments required for {@link TemplatedResourcesGenerator} + * Generator used to create resources relative to a given base identifier. + * Similar to {@link ResourcesGenerator}, but takes as input a string + * indicating where the templates are stored that need to be used for resource generation. */ -export interface TemplatedResourcesGeneratorArgs { +export interface TemplatedResourcesGenerator { /** - * Folder where the templates are located. - */ - templateFolder: string; - /** - * Factory used to generate mapper relative to the base identifier. - */ - factory: FileIdentifierMapperFactory; - /** - * Template engine for generating the resources. - */ - templateEngine: TemplateEngine; - /** - * The extension of files that need to be interpreted as templates. - * Will be removed to generate the identifier. - */ - templateExtension?: string; - /** - * The metadataStrategy - */ - metadataStrategy: AuxiliaryStrategy; - /** - * The default ResourceStore - */ - store: ResourceSet; -} -/** - * Generates resources by making use of a template engine. - * The template folder structure will be kept. - * Folders will be interpreted as containers and files as documents. - * A FileIdentifierMapper will be used to generate identifiers that correspond to the relative structure. - * - * A relative `templateFolder` is resolved relative to cwd, - * unless it's preceded by `@css:`, e.g. `@css:foo/bar`. - */ -export class TemplatedResourcesGenerator implements ResourcesGenerator { - private readonly templateFolder: string; - private readonly factory: FileIdentifierMapperFactory; - private readonly templateEngine: TemplateEngine; - private readonly templateExtension: string; - private readonly metadataStrategy: AuxiliaryStrategy; - private readonly store: ResourceSet; - - /** - * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier. + * Generates resources with the given options, based on the given template folder. + * The output Iterable should be sorted so that containers always appear before their contents. + * @param templateFolder - Folder where the templates are located. + * @param location - Base identifier. + * @param options - Options that can be used when generating resources. * - * @param args - TemplatedResourcesGeneratorArgs + * @returns A map where the keys are the identifiers and the values the corresponding representations to store. */ - public constructor(args: TemplatedResourcesGeneratorArgs) { - this.templateFolder = resolveAssetPath(args.templateFolder); - this.factory = args.factory; - this.templateEngine = args.templateEngine; - this.templateExtension = args.templateExtension ?? '.hbs'; - this.metadataStrategy = args.metadataStrategy; - this.store = args.store; - } - - public async* generate(location: ResourceIdentifier, options: Dict): AsyncIterable { - const mapper = await this.factory.create(location.path, this.templateFolder); - const folderLink = await this.toTemplateLink(this.templateFolder, mapper); - yield* this.processFolder(folderLink, mapper, options); - } - - /** - * Generates results for all entries in the given folder, including the folder itself. - */ - private async* processFolder(folderLink: TemplateResourceLink, mapper: FileIdentifierMapper, options: Dict): - AsyncIterable { - // Group resource links with their corresponding metadata links - const links = await this.groupLinks(folderLink.filePath, mapper); - - // Remove root metadata if it exists - const metaLink = links[folderLink.identifier.path]?.meta; - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete links[folderLink.identifier.path]; - - yield* this.generateResource(folderLink, options, metaLink); - - for (const { link, meta } of Object.values(links)) { - if (isContainerIdentifier(link.identifier)) { - yield* this.processFolder(link, mapper, options); - } else { - yield* this.generateResource(link, options, meta); - } - } - } - - /** - * Creates a TemplateResourceLink for the given filePath. - * The identifier will be based on the file path stripped from the template extension, - * but the filePath parameter will still point to the original file. - */ - private async toTemplateLink(filePath: string, mapper: FileIdentifierMapper): Promise { - const stats = await fsPromises.lstat(filePath); - - // Slice the template extension from the filepath for correct identifier generation - const isTemplate = filePath.endsWith(this.templateExtension); - const slicedPath = isTemplate ? filePath.slice(0, -this.templateExtension.length) : filePath; - const link = await mapper.mapFilePathToUrl(slicedPath, stats.isDirectory()); - // We still need the original file path for disk reading though - return { - ...link, - filePath, - isTemplate, - }; - } - - /** - * Generates TemplateResourceLinks for each entry in the given folder - * and combines the results so resources and their metadata are grouped together. - */ - private async groupLinks(folderPath: string, mapper: FileIdentifierMapper): - Promise> { - const files = await fsPromises.readdir(folderPath); - const links: Record = { }; - for (const name of files) { - const link = await this.toTemplateLink(joinFilePath(folderPath, name), mapper); - const { path } = link.identifier; - links[path] = Object.assign(links[path] || {}, link.isMetadata ? { meta: link } : { link }); - } - return links; - } - - /** - * Generates a Resource object for the given ResourceLink. - * In the case of documents the corresponding template will be used. - * If a ResourceLink of metadata is provided the corresponding data will be added as metadata. - */ - private async* generateResource(link: TemplateResourceLink, options: Dict, metaLink?: TemplateResourceLink): - AsyncIterable { - let data: Guarded | undefined; - const metadata = new RepresentationMetadata(link.identifier); - - // Read file if it is not a container - if (!isContainerIdentifier(link.identifier)) { - data = await this.processFile(link, options); - metadata.contentType = link.contentType; - } - // Do not yield a container resource if it already exists - if (!isContainerIdentifier(link.identifier) || !await this.store.hasResource(link.identifier)) { - yield { - identifier: link.identifier, - representation: new BasicRepresentation(data ?? [], metadata), - }; - } - - // Add metadata from .meta file if there is one - if (metaLink) { - const rawMetadata = await this.generateMetadata(metaLink, options); - const metaIdentifier = this.metadataStrategy.getAuxiliaryIdentifier(link.identifier); - const descriptionMeta = new RepresentationMetadata(metaIdentifier); - addResourceMetadata(rawMetadata, isContainerIdentifier(link.identifier)); - yield { - identifier: metaIdentifier, - representation: new BasicRepresentation(rawMetadata.quads(), descriptionMeta, INTERNAL_QUADS), - }; - } - } - - /** - * Generates a RepresentationMetadata using the given template. - */ - private async generateMetadata(metaLink: TemplateResourceLink, options: Dict): - Promise { - const metadata = new RepresentationMetadata(metaLink.identifier); - - const data = await this.processFile(metaLink, options); - const parser = new Parser({ format: metaLink.contentType, baseIRI: metaLink.identifier.path }); - const quads = parser.parse(await readableToString(data)); - metadata.addQuads(quads); - - return metadata; - } - - /** - * Creates a read stream from the file and applies the template if necessary. - */ - private async processFile(link: TemplateResourceLink, contents: Dict): Promise> { - if (link.isTemplate) { - const rendered = await this.templateEngine.handleSafe({ contents, template: { templateFile: link.filePath }}); - return guardedStreamFrom(rendered); - } - return guardStream(createReadStream(link.filePath)); - } + generate: (templateFolder: string, location: ResourceIdentifier, options: Dict) => AsyncIterable; } diff --git a/src/util/ContentTypes.ts b/src/util/ContentTypes.ts index 47481171d..2a11b2890 100644 --- a/src/util/ContentTypes.ts +++ b/src/util/ContentTypes.ts @@ -15,5 +15,6 @@ export const INTERNAL_ERROR = 'internal/error'; export const DEFAULT_CUSTOM_TYPES = { acl: TEXT_TURTLE, + acr: TEXT_TURTLE, meta: TEXT_TURTLE, }; diff --git a/src/util/IterableUtil.ts b/src/util/IterableUtil.ts index c207900a4..3190b5171 100644 --- a/src/util/IterableUtil.ts +++ b/src/util/IterableUtil.ts @@ -113,3 +113,93 @@ export function reduce(iterable: Iterable, } return previousValue; } + +/** + * Helper function for {@link sortedAsyncMerge}. + * + * Returns the next result of an AsyncIterator, or undefined if the iterator is finished. + */ +async function nextAsyncEntry(iterator: AsyncIterator): Promise { + const result = await iterator.next(); + if (result.done) { + return; + } + return result.value; +} + +/** + * Helper function for {@link sortedAsyncMerge}. + * + * Compares the next results of all `iterators` and returns the first one, + * determined by the provided `comparator`. + * + * `results` should contain the first result of all these iterators. + * This array will also be updated, replacing the result of the iterator whose result was chosen by the next one. + */ +async function findNextSorted(iterators: AsyncIterator[], results: (T | undefined)[], + comparator: (left: T, right: T) => number): Promise { + let best: { idx: number; value: T } | undefined; + // For every iterator: see if their next result is the best one so far + for (let i = 0; i < iterators.length; ++i) { + const value = results[i]; + if (typeof value !== 'undefined') { + let compare = 1; + if (best) { + compare = comparator(best.value, value); + } + + if (compare > 0) { + best = { idx: i, value }; + } + } + } + + if (best) { + // Advance the iterator that returned the new result + results[best.idx] = await nextAsyncEntry(iterators[best.idx]); + } + + // Will return undefined if `best` was never initialized above + return best?.value; +} + +/** + * Merges the results of several sorted iterators. + * In case the results of the individual iterators are not sorted the outcome results will also not be sorted. + * + * @param iterators - The iterators whose results need to be merged. + * @param comparator - The comparator to use to compare the results. + */ +export async function* sortedAsyncMerge(iterators: AsyncIterator[], comparator?: (left: T, right: T) => number): +AsyncIterable { + if (!comparator) { + // eslint-disable-next-line @typescript-eslint/no-extra-parens + comparator = (left, right): number => left < right ? -1 : (left > right ? 1 : 0); + } + + // Initialize the array to the first result of every iterator + const results: (T | undefined)[] = []; + for (const iterator of iterators) { + results.push(await nextAsyncEntry(iterator)); + } + + // Keep returning results as long as we find them + while (true) { + const next = await findNextSorted(iterators, results, comparator); + if (typeof next === 'undefined') { + return; + } + yield next; + } +} + +/** + * Converts an `AsyncIterator` to an array. + */ +export async function asyncToArray(iterable: AsyncIterable): Promise { + const arr: T[] = []; + for await (const result of iterable) { + arr.push(result); + } + return arr; +} diff --git a/templates/pod/acp/.acr.hbs b/templates/pod/acp/.acr.hbs new file mode 100644 index 000000000..d69972655 --- /dev/null +++ b/templates/pod/acp/.acr.hbs @@ -0,0 +1,39 @@ +# Root ACR for the agent account +@prefix acl: . +@prefix acp: . + +# The owner has full access to every resource in their pod. +# Other agents have no access rights, +# unless specifically authorized in other ACRs. +<#root> + a acp:AccessControlResource; + # Set the access to the root storage folder itself + acp:resource <./>; + # The homepage is readable by the public + acp:accessControl <#fullOwnerAccess>, <#publicReadAccess>; + # All resources will inherit this authorization + acp:memberAccessControl <#fullOwnerAccess>. + +# The public only has read access +<#publicReadAccess> + a acp:AccessControl; + acp:apply [ + a acp:Policy; + acp:allow acl:Read; + acp:anyOf [ + a acp:Matcher; + acp:agent acp:PublicAgent + ] + ]. + +# The owner has all of the access modes allowed +<#fullOwnerAccess> + a acp:AccessControl; + acp:apply [ + a acp:Policy; + acp:allow acl:Read, acl:Write, acl:Control; + acp:anyOf [ + a acp:Matcher; + acp:agent <{{webId}}> + ] + ]. diff --git a/templates/pod/acp/README.acr b/templates/pod/acp/README.acr new file mode 100644 index 000000000..be3fdd8f1 --- /dev/null +++ b/templates/pod/acp/README.acr @@ -0,0 +1,18 @@ +@prefix acl: . +@prefix acp: . + +<#card> + a acp:AccessControlResource; + acp:resource <./README>; + acp:accessControl <#publicReadAccess>. + +<#publicReadAccess> + a acp:AccessControl; + acp:apply [ + a acp:Policy; + acp:allow acl:Read; + acp:anyOf [ + a acp:Matcher; + acp:agent acp:PublicAgent + ] + ]. diff --git a/templates/pod/acp/profile/card.acr b/templates/pod/acp/profile/card.acr new file mode 100644 index 000000000..5febf6421 --- /dev/null +++ b/templates/pod/acp/profile/card.acr @@ -0,0 +1,22 @@ +# ACR for the WebID profile document +@prefix acl: . +@prefix acp: . + +# The WebID profile is readable by the public. +# This is required for discovery and verification, +# e.g. when checking identity providers. +<#card> + a acp:AccessControlResource; + acp:resource <./card>; + acp:accessControl <#publicReadAccess>. + +<#publicReadAccess> + a acp:AccessControl; + acp:apply [ + a acp:Policy; + acp:allow acl:Read; + acp:anyOf [ + a acp:Matcher; + acp:agent acp:PublicAgent + ] + ]. diff --git a/templates/pod/.meta b/templates/pod/base/.meta similarity index 100% rename from templates/pod/.meta rename to templates/pod/base/.meta diff --git a/templates/pod/README$.md.hbs b/templates/pod/base/README$.md.hbs similarity index 100% rename from templates/pod/README$.md.hbs rename to templates/pod/base/README$.md.hbs diff --git a/templates/pod/profile/card$.ttl.hbs b/templates/pod/base/profile/card$.ttl.hbs similarity index 100% rename from templates/pod/profile/card$.ttl.hbs rename to templates/pod/base/profile/card$.ttl.hbs diff --git a/templates/pod/.acl.hbs b/templates/pod/wac/.acl.hbs similarity index 100% rename from templates/pod/.acl.hbs rename to templates/pod/wac/.acl.hbs diff --git a/templates/pod/README.acl.hbs b/templates/pod/wac/README.acl.hbs similarity index 100% rename from templates/pod/README.acl.hbs rename to templates/pod/wac/README.acl.hbs diff --git a/templates/pod/profile/card.acl.hbs b/templates/pod/wac/profile/card.acl.hbs similarity index 100% rename from templates/pod/profile/card.acl.hbs rename to templates/pod/wac/profile/card.acl.hbs diff --git a/templates/root/empty/acp/.acr b/templates/root/empty/acp/.acr new file mode 100644 index 000000000..588cbbce0 --- /dev/null +++ b/templates/root/empty/acp/.acr @@ -0,0 +1,32 @@ +# WARNING: DO NOT USE UNMODIFIED UNLESS FOR TESTING PURPOSES. +# WHEN IN DOUBT, DELETE THIS DOCUMENT. +# +# This root ACR allows unrestricted public access to all documents and subcontainers. +# +# This document was automatically generated by the Community Solid Server +# because the "Expose a public root Pod" option was selected during setup, +# or because setup has been bypassed. +# +# We strongly suggest to edit this document such that it restricts permissions. + +@prefix acl: . +@prefix acp: . + +# Give all agents Read, Write, and Control permissions on everything +<#card> + a acp:AccessControlResource; + acp:resource <./>; + acp:accessControl <#publicReadAccess>; + acp:memberAccessControl <#publicReadAccess> . + +<#publicReadAccess> + a acp:AccessControl; + acp:apply [ + a acp:Policy; + acp:allow acl:Read, acl:Write, acl:Control; + acp:anyOf [ + a acp:Matcher; + acp:agent acp:PublicAgent + ] + ]. + diff --git a/templates/root/empty/.meta b/templates/root/empty/base/.meta similarity index 100% rename from templates/root/empty/.meta rename to templates/root/empty/base/.meta diff --git a/templates/root/empty/.acl b/templates/root/empty/wac/.acl similarity index 100% rename from templates/root/empty/.acl rename to templates/root/empty/wac/.acl diff --git a/templates/root/prefilled/acp/.acr b/templates/root/prefilled/acp/.acr new file mode 100644 index 000000000..588cbbce0 --- /dev/null +++ b/templates/root/prefilled/acp/.acr @@ -0,0 +1,32 @@ +# WARNING: DO NOT USE UNMODIFIED UNLESS FOR TESTING PURPOSES. +# WHEN IN DOUBT, DELETE THIS DOCUMENT. +# +# This root ACR allows unrestricted public access to all documents and subcontainers. +# +# This document was automatically generated by the Community Solid Server +# because the "Expose a public root Pod" option was selected during setup, +# or because setup has been bypassed. +# +# We strongly suggest to edit this document such that it restricts permissions. + +@prefix acl: . +@prefix acp: . + +# Give all agents Read, Write, and Control permissions on everything +<#card> + a acp:AccessControlResource; + acp:resource <./>; + acp:accessControl <#publicReadAccess>; + acp:memberAccessControl <#publicReadAccess> . + +<#publicReadAccess> + a acp:AccessControl; + acp:apply [ + a acp:Policy; + acp:allow acl:Read, acl:Write, acl:Control; + acp:anyOf [ + a acp:Matcher; + acp:agent acp:PublicAgent + ] + ]. + diff --git a/templates/root/prefilled/.meta b/templates/root/prefilled/base/.meta similarity index 100% rename from templates/root/prefilled/.meta rename to templates/root/prefilled/base/.meta diff --git a/templates/root/prefilled/index.html b/templates/root/prefilled/base/index.html similarity index 97% rename from templates/root/prefilled/index.html rename to templates/root/prefilled/base/index.html index 90b43b336..e8170600f 100644 --- a/templates/root/prefilled/index.html +++ b/templates/root/prefilled/base/index.html @@ -8,7 +8,7 @@
- [Solid logo] + [Solid logo]

Community Solid Server

diff --git a/templates/root/prefilled/.acl b/templates/root/prefilled/wac/.acl similarity index 100% rename from templates/root/prefilled/.acl rename to templates/root/prefilled/wac/.acl diff --git a/test/assets/templates/profile/card$.ttl b/test/assets/templates/profile/card$.ttl index 3461b7e23..5beaec769 100644 --- a/test/assets/templates/profile/card$.ttl +++ b/test/assets/templates/profile/card$.ttl @@ -1,5 +1,5 @@ @prefix foaf: . <{{webId}}> - a foaf:Person ; + a foaf:Person; foaf:name "{{name}}". diff --git a/test/unit/authorization/AcpReader.test.ts b/test/unit/authorization/AcpReader.test.ts index 087755236..e86a171b2 100644 --- a/test/unit/authorization/AcpReader.test.ts +++ b/test/unit/authorization/AcpReader.test.ts @@ -63,7 +63,7 @@ describe('An AcpReader', (): void => { const target = { path: joinUrl(baseUrl, 'foo') }; dataMap[baseUrl] = toQuads(` [] - acp:resource <./> ; + acp:resource <./>; acp:accessControl [ acp:apply _:policy ]. _:policy acp:allow acl:Read; @@ -89,7 +89,7 @@ describe('An AcpReader', (): void => { const target = { path: joinUrl(baseUrl, 'foo') }; dataMap[baseUrl] = toQuads(` [] - acp:resource <./> ; + acp:resource <./>; acp:memberAccessControl [ acp:apply _:policy ]. _:policy acp:allow acl:Read; @@ -109,7 +109,7 @@ describe('An AcpReader', (): void => { const target = { path: joinUrl(baseUrl, 'foo') }; dataMap[baseUrl] = toQuads(` [] - acp:resource <./> ; + acp:resource <./>; acp:accessControl [ acp:apply _:controlPolicy ]; acp:memberAccessControl [ acp:apply _:readPolicy ]. _:readPolicy @@ -122,7 +122,7 @@ describe('An AcpReader', (): void => { `, baseUrl); dataMap[target.path] = toQuads(` [] - acp:resource <./foo> ; + acp:resource <./foo>; acp:accessControl [ acp:apply _:appendPolicy ]. _:appendPolicy acp:allow acl:Append; @@ -143,7 +143,7 @@ describe('An AcpReader', (): void => { const target2 = { path: joinUrl(baseUrl, 'foo/bar') }; dataMap[baseUrl] = toQuads(` [] - acp:resource <./> ; + acp:resource <./>; acp:memberAccessControl [ acp:apply _:policy ]. _:policy acp:allow acl:Read; diff --git a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts b/test/unit/pods/generate/BaseResourcesGenerator.test.ts similarity index 80% rename from test/unit/pods/generate/TemplatedResourcesGenerator.test.ts rename to test/unit/pods/generate/BaseResourcesGenerator.test.ts index 1c9fde7c8..e4c0a1b1e 100644 --- a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts +++ b/test/unit/pods/generate/BaseResourcesGenerator.test.ts @@ -1,18 +1,20 @@ import type { ResourceIdentifier } from '../../../../src/http/representation/ResourceIdentifier'; -import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator'; +import { BaseResourcesGenerator } from '../../../../src/pods/generate/BaseResourcesGenerator'; import type { FileIdentifierMapper, FileIdentifierMapperFactory, ResourceLink, } from '../../../../src/storage/mapping/FileIdentifierMapper'; import type { ResourceStore } from '../../../../src/storage/ResourceStore'; -import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil'; +import { asyncToArray } from '../../../../src/util/IterableUtil'; +import { ensureTrailingSlash, joinFilePath, trimTrailingSlashes } from '../../../../src/util/PathUtil'; import { readableToQuads, readableToString } from '../../../../src/util/StreamUtil'; import { HandlebarsTemplateEngine } from '../../../../src/util/templates/HandlebarsTemplateEngine'; import { SimpleSuffixStrategy } from '../../../util/SimpleSuffixStrategy'; import { mockFileSystem } from '../../../util/Util'; jest.mock('fs'); +jest.mock('fs-extra'); class DummyFactory implements FileIdentifierMapperFactory { public async create(base: string, rootFilePath: string): Promise { @@ -36,20 +38,12 @@ class DummyFactory implements FileIdentifierMapperFactory { } } -async function genToArray(iterable: AsyncIterable): Promise { - const arr: T[] = []; - for await (const result of iterable) { - arr.push(result); - } - return arr; -} - -describe('A TemplatedResourcesGenerator', (): void => { +describe('A BaseResourcesGenerator', (): void => { const rootFilePath = '/templates/pod'; // Using handlebars engine since it's smaller than any possible dummy const metadataStrategy = new SimpleSuffixStrategy('.meta'); let store: jest.Mocked; - let generator: TemplatedResourcesGenerator; + let generator: BaseResourcesGenerator; let cache: { data: any }; const template = '<{{webId}}> a .'; const location = { path: 'http://test.com/alice/' }; @@ -61,8 +55,7 @@ describe('A TemplatedResourcesGenerator', (): void => { hasResource: jest.fn(), } as any; - generator = new TemplatedResourcesGenerator({ - templateFolder: rootFilePath, + generator = new BaseResourcesGenerator({ factory: new DummyFactory(), templateEngine: new HandlebarsTemplateEngine('http://test.com/'), metadataStrategy, @@ -72,7 +65,7 @@ describe('A TemplatedResourcesGenerator', (): void => { it('fills in a template with the given options.', async(): Promise => { cache.data = { 'template.hbs': template }; - const result = await genToArray(generator.generate(location, { webId })); + const result = await asyncToArray(generator.generate(rootFilePath, location, { webId })); const identifiers = result.map((res): ResourceIdentifier => res.identifier); const id = { path: `${location.path}template` }; expect(identifiers).toEqual([ location, id ]); @@ -86,7 +79,7 @@ describe('A TemplatedResourcesGenerator', (): void => { it('creates the necessary containers.', async(): Promise => { cache.data = { container: { container: { 'template.hbs': template }}}; - const result = await genToArray(generator.generate(location, { webId })); + const result = await asyncToArray(generator.generate(rootFilePath, location, { webId })); const identifiers = result.map((res): ResourceIdentifier => res.identifier); const id = { path: `${location.path}container/container/template` }; expect(identifiers).toEqual([ @@ -103,7 +96,7 @@ describe('A TemplatedResourcesGenerator', (): void => { it('copies the file stream directly if no template extension is found.', async(): Promise => { cache.data = { noTemplate: template }; - const result = await genToArray(generator.generate(location, { webId })); + const result = await asyncToArray(generator.generate(rootFilePath, location, { webId })); const identifiers = result.map((res): ResourceIdentifier => res.identifier); const id = { path: `${location.path}noTemplate` }; expect(identifiers).toEqual([ location, id ]); @@ -119,7 +112,7 @@ describe('A TemplatedResourcesGenerator', (): void => { cache.data = { '.meta': meta, container: { 'template.meta': meta, template }}; // Not using options since our dummy template generator generates invalid turtle - const result = await genToArray(generator.generate(location, { webId })); + const result = await asyncToArray(generator.generate(rootFilePath, location, { webId })); const identifiers = result.map((res): ResourceIdentifier => res.identifier); expect(identifiers).toEqual([ location, @@ -158,7 +151,7 @@ describe('A TemplatedResourcesGenerator', (): void => { cache.data = { '.meta': meta }; store.hasResource = jest.fn().mockResolvedValue(true); - const result = await genToArray(generator.generate(location, { webId })); + const result = await asyncToArray(generator.generate(rootFilePath, location, { webId })); const identifiers = result.map((res): ResourceIdentifier => res.identifier); expect(identifiers).toEqual([ { path: `${location.path}.meta` }, @@ -169,4 +162,20 @@ describe('A TemplatedResourcesGenerator', (): void => { expect(expQuads).toHaveLength(1); expect(expQuads[0].object.value).toBe('metadata'); }); + + it('returns no results if the target folder does not exist.', async(): Promise => { + const result = await asyncToArray(generator.generate(joinFilePath(rootFilePath, 'nope'), location, { webId })); + expect(result).toHaveLength(0); + }); + + it('makes sure the results are sorted.', async(): Promise => { + cache.data = { 'template2.hbs': template, 'template1.hbs': template }; + const result = await asyncToArray(generator.generate(rootFilePath, location, { webId })); + const identifiers = result.map((res): ResourceIdentifier => res.identifier); + expect(identifiers).toEqual([ + location, + { path: `${location.path}template1` }, + { path: `${location.path}template2` }, + ]); + }); }); diff --git a/test/unit/pods/generate/StaticFolderGenerator.test.ts b/test/unit/pods/generate/StaticFolderGenerator.test.ts new file mode 100644 index 000000000..a857cb10c --- /dev/null +++ b/test/unit/pods/generate/StaticFolderGenerator.test.ts @@ -0,0 +1,26 @@ +import type { ResourceIdentifier } from '../../../../src/http/representation/ResourceIdentifier'; +import type { Resource } from '../../../../src/pods/generate/ResourcesGenerator'; +import { StaticFolderGenerator } from '../../../../src/pods/generate/StaticFolderGenerator'; +import type { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator'; + +describe('A StaticFolderGenerator', (): void => { + const location: ResourceIdentifier = { path: 'http://example.com/foo' }; + const options = { foo: 'bar' }; + const folder = '/data/templates/'; + let source: jest.Mocked; + const response: AsyncIterable = {} as any; + let generator: StaticFolderGenerator; + + beforeEach(async(): Promise => { + source = { + generate: jest.fn().mockReturnValue(response), + }; + generator = new StaticFolderGenerator(source, folder); + }); + + it('calls the source generator with the stored template folder.', async(): Promise => { + expect(generator.generate(location, options)).toBe(response); + expect(source.generate).toHaveBeenCalledTimes(1); + expect(source.generate).toHaveBeenLastCalledWith(folder, location, options); + }); +}); diff --git a/test/unit/pods/generate/SubfolderResourcesGenerator.test.ts b/test/unit/pods/generate/SubfolderResourcesGenerator.test.ts new file mode 100644 index 000000000..3513f5a95 --- /dev/null +++ b/test/unit/pods/generate/SubfolderResourcesGenerator.test.ts @@ -0,0 +1,90 @@ +import { BasicRepresentation } from '../../../../src/http/representation/BasicRepresentation'; +import type { Resource } from '../../../../src/pods/generate/ResourcesGenerator'; +import { SubfolderResourcesGenerator } from '../../../../src/pods/generate/SubfolderResourcesGenerator'; +import type { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator'; +import { asyncToArray } from '../../../../src/util/IterableUtil'; + +async function* yieldResources(resources: Resource[]): AsyncIterable { + yield* resources; +} + +function createResource(path: string): Resource { + const identifier = { path }; + const representation = new BasicRepresentation('data', 'text/plain'); + return { identifier, representation }; +} + +describe('A SubfolderResourcesGenerator', (): void => { + const templateFolder = '/data/templates/'; + const identifier = { path: 'http://example.com/foo' }; + const options = { foo: 'bar' }; + const subfolders = [ 'base', 'empty', 'extra' ]; + let baseResources: Resource[]; + let extraResources: Resource[]; + let source: jest.Mocked; + let generator: SubfolderResourcesGenerator; + + beforeEach(async(): Promise => { + baseResources = []; + extraResources = []; + + source = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + generate: jest.fn((folder, loc, opt): AsyncIterable => { + if (folder.endsWith('base')) { + return yieldResources(baseResources); + } + if (folder.endsWith('extra')) { + return yieldResources(extraResources); + } + return yieldResources([]); + }), + }; + + generator = new SubfolderResourcesGenerator(source, subfolders); + }); + + it('merges the results of the subfolders into one sorted result.', async(): Promise => { + baseResources = [ createResource('a'), createResource('c'), createResource('d'), createResource('f') ]; + extraResources = [ createResource('b'), createResource('e'), createResource('g') ]; + + const resources = await asyncToArray(generator.generate(templateFolder, identifier, options)); + expect(resources.map((resource): string => resource.identifier.path)).toEqual([ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', + ]); + expect(source.generate).toHaveBeenCalledTimes(3); + expect(source.generate).toHaveBeenNthCalledWith(1, '/data/templates/base', identifier, options); + expect(source.generate).toHaveBeenNthCalledWith(2, '/data/templates/empty', identifier, options); + expect(source.generate).toHaveBeenNthCalledWith(3, '/data/templates/extra', identifier, options); + }); + + it('keeps the first result in case of duplicate identifiers.', async(): Promise => { + const resource1 = createResource('foo'); + const resource2 = createResource('foo'); + baseResources = [ createResource('b'), resource1, createResource('g') ]; + extraResources = [ createResource('a'), resource2, createResource('h') ]; + const resources = await asyncToArray(generator.generate(templateFolder, identifier, options)); + expect(resources.map((resource): string => resource.identifier.path)).toEqual([ + 'a', 'b', 'foo', 'g', 'h', + ]); + expect(resources[2]).toBe(resource1); + expect(resource2.representation.data.destroyed).toBe(true); + }); + + it('correctly sorts containers.', async(): Promise => { + baseResources = [ createResource('/'), createResource('/container/'), + createResource('/container/foo.acl'), createResource('README.acl') ]; + extraResources = [ createResource('/'), createResource('/container/'), + createResource('/container/foo'), createResource('README') ]; + + const resources = await asyncToArray(generator.generate(templateFolder, identifier, options)); + expect(resources.map((resource): string => resource.identifier.path)).toEqual([ + '/', + '/container/', + '/container/foo', + '/container/foo.acl', + 'README', + 'README.acl', + ]); + }); +}); diff --git a/test/unit/util/IterableUtil.test.ts b/test/unit/util/IterableUtil.test.ts index 136360218..bbebaf0aa 100644 --- a/test/unit/util/IterableUtil.test.ts +++ b/test/unit/util/IterableUtil.test.ts @@ -1,4 +1,4 @@ -import { concat, filter, find, map, reduce } from '../../../src/util/IterableUtil'; +import { asyncToArray, concat, filter, find, map, reduce, sortedAsyncMerge } from '../../../src/util/IterableUtil'; describe('IterableUtil', (): void => { describe('#map', (): void => { @@ -50,4 +50,32 @@ describe('IterableUtil', (): void => { expect((): number => reduce(input, (acc, cur): number => acc + cur)).toThrow(TypeError); }); }); + + describe('#sortedAsyncMerge', (): void => { + it('sorts the iterables.', async(): Promise => { + // eslint-disable-next-line unicorn/consistent-function-scoping + async function* left(): AsyncIterator { + yield* [ 1, 3, 5, 7, 9 ]; + } + // eslint-disable-next-line unicorn/consistent-function-scoping + async function* right(): AsyncIterator { + yield* [ 0, 2, 3, 4 ]; + } + await expect(asyncToArray(sortedAsyncMerge([ left(), right() ]))).resolves + .toEqual([ 0, 1, 2, 3, 3, 4, 5, 7, 9 ]); + }); + + it('accepts a custom comparator.', async(): Promise => { + // eslint-disable-next-line unicorn/consistent-function-scoping + async function* left(): AsyncIterator { + yield* [ 'apple', 'citrus', 'date' ]; + } + // eslint-disable-next-line unicorn/consistent-function-scoping + async function* right(): AsyncIterator { + yield* [ 'banana', 'donut' ]; + } + await expect(asyncToArray(sortedAsyncMerge([ left(), right() ]))).resolves + .toEqual([ 'apple', 'banana', 'citrus', 'date', 'donut' ]); + }); + }); }); diff --git a/test/util/Util.ts b/test/util/Util.ts index 3ced94099..efeecb8d4 100644 --- a/test/util/Util.ts +++ b/test/util/Util.ts @@ -272,6 +272,14 @@ export function mockFileSystem(rootFilepath?: string, time?: Date): { data: any // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete folder[name]; }, + async pathExists(path: string): Promise { + try { + const { folder, name } = getFolder(path); + return Boolean(folder[name]); + } catch { + return false; + } + }, createReadStream(path: string): any { return mockFs.createReadStream(path); },