Mirroristas/CommunitySolidServer
2
0
Fork 0
mirror of https://github.com/CommunitySolidServer/CommunitySolidServer.git synced 2024-10-03 14:55:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Support .meta files for pod provisioning

Browse Source
This commit is contained in:
Joachim Van Herwegen 2020-12-16 14:19:23 +01:00
parent a114d00827
commit e722cc67af
2 changed files with 151 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
src/pods/generate/TemplatedResourcesGenerator.ts
View File

@ -1,8 +1,14 @@
import { promises as fsPromises } from 'fs';
import { posix } from 'path';
import { Parser } from 'n3';
import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata';
import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
import type { FileIdentifierMapper, FileIdentifierMapperFactory } from '../../storage/mapping/FileIdentifierMapper';
import type {
FileIdentifierMapper,
FileIdentifierMapperFactory,
ResourceLink,
} from '../../storage/mapping/FileIdentifierMapper';
import { isContainerIdentifier } from '../../util/PathUtil';
import { guardedStreamFrom } from '../../util/StreamUtil';
import type { Resource, ResourcesGenerator } from './ResourcesGenerator';
import type { TemplateEngine } from './TemplateEngine';
@ -20,6 +26,7 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
private readonly templateFolder: string;
private readonly factory: FileIdentifierMapperFactory;
private readonly engine: TemplateEngine;
private readonly metaExtension = '.meta';
/**
* A mapper is needed to convert the template file paths to identifiers relative to the given base identifier.
@ -36,57 +43,132 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
public async* generate(location: ResourceIdentifier, options: Dict<string>): AsyncIterable<Resource> {
const mapper = await this.factory.create(location.path, this.templateFolder);
yield* this.parseFolder(this.templateFolder, mapper, options);
const folderLink = await mapper.mapFilePathToUrl(this.templateFolder, true);
yield* this.parseFolder(folderLink, mapper, options);
}
/**
* Generates results for all entries in the given folder, including the folder itself.
*/
private async* parseFolder(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
private async* parseFolder(folderLink: ResourceLink, mapper: FileIdentifierMapper, options: Dict<string>):
AsyncIterable<Resource> {
// Generate representation for the container
const link = await mapper.mapFilePathToUrl(filePath, true);
yield {
identifier: link.identifier,
representation: {
binary: true,
data: guardedStreamFrom([]),
metadata: new RepresentationMetadata(link.identifier),
},
};
// Group resource links with their corresponding metadata links
const links = await this.groupLinks(this.generateLinks(folderLink.filePath, mapper));
// Generate representations for all resources in this container
const files = await fsPromises.readdir(filePath);
for (const childName of files) {
const childPath = joinPath(filePath, childName);
const childStats = await fsPromises.lstat(childPath);
if (childStats.isDirectory()) {
yield* this.parseFolder(childPath, mapper, options);
} else if (childStats.isFile()) {
yield this.generateDocument(childPath, mapper, options);
// Remove root metadata if it exists
const metaLink = links[folderLink.identifier.path]?.meta;
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete links[folderLink.identifier.path];
yield this.generateResource(folderLink, options, metaLink);
for (const { link, meta } of Object.values(links)) {
if (isContainerIdentifier(link.identifier)) {
yield* this.parseFolder(link, mapper, options);
} else {
yield this.generateResource(link, options, meta);
}
}
}
/**
* Generates a new Representation corresponding to the template file at the given location.
* Generates ResourceLinks for each entry in the given folder.
*/
private async generateDocument(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
Promise<Resource> {
const link = await mapper.mapFilePathToUrl(filePath, false);
const metadata = new RepresentationMetadata(link.identifier);
metadata.contentType = link.contentType;
private async* generateLinks(folderPath: string, mapper: FileIdentifierMapper): AsyncIterable<ResourceLink> {
const files = await fsPromises.readdir(folderPath);
for (const name of files) {
const filePath = joinPath(folderPath, name);
const stats = await fsPromises.lstat(filePath);
yield mapper.mapFilePathToUrl(filePath, stats.isDirectory());
}
}
const raw = await fsPromises.readFile(filePath, 'utf8');
const compiled = this.engine.apply(raw, options);
/**
* Parses a group of ResourceLinks so resources and their metadata are grouped together.
*/
private async groupLinks(linkGen: AsyncIterable<ResourceLink>):
Promise<Record<string, { link: ResourceLink; meta?: ResourceLink }>> {
const links: Record<string, { link: ResourceLink; meta?: ResourceLink }> = { };
for await (const link of linkGen) {
const { path } = link.identifier;
if (this.isMeta(path)) {
const resourcePath = this.metaToResource(link.identifier).path;
links[resourcePath] = Object.assign(links[resourcePath] || {}, { meta: link });
} else {
links[path] = Object.assign(links[path] || {}, { link });
}
}
return links;
}
/**
* Generates a Resource object for the given ResourceLink.
* In the case of documents the corresponding template will be used.
* If a ResourceLink of metadata is provided the corresponding data will be added as metadata.
*/
private async generateResource(link: ResourceLink, options: Dict<string>, metaLink?: ResourceLink):
Promise<Resource> {
const data: string[] = [];
const metadata = new RepresentationMetadata(link.identifier);
// Read file if it is not a container
if (!isContainerIdentifier(link.identifier)) {
const compiled = await this.parseTemplate(link.filePath, options);
data.push(compiled);
metadata.contentType = link.contentType;
}
// Add metadata from meta file if there is one
if (metaLink) {
const rawMetadata = await this.generateMetadata(metaLink, options);
metadata.addQuads(rawMetadata.quads());
}
return {
identifier: link.identifier,
representation: {
binary: true,
data: guardedStreamFrom([ compiled ]),
data: guardedStreamFrom(data),
metadata,
},
};
}
/**
* Generates a RepresentationMetadata using the given template.
*/
private async generateMetadata(metaLink: ResourceLink, options: Dict<string>):
Promise<RepresentationMetadata> {
const identifier = this.metaToResource(metaLink.identifier);
const metadata = new RepresentationMetadata(identifier);
const data = await this.parseTemplate(metaLink.filePath, options);
const parser = new Parser({ format: metaLink.contentType, baseIRI: identifier.path });
const quads = parser.parse(data);
metadata.addQuads(quads);
return metadata;
}
/**
* Applies the given options to the template found at the given path.
*/
private async parseTemplate(filePath: string, options: Dict<string>): Promise<string> {
const raw = await fsPromises.readFile(filePath, 'utf8');
return this.engine.apply(raw, options);
}
/**
* Verifies if the given path corresponds to a metadata file.
*/
private isMeta(path: string): boolean {
return path.endsWith(this.metaExtension);
}
/**
* Converts a generated metadata identifier to the identifier of its corresponding resource.
*/
private metaToResource(metaIdentifier: ResourceIdentifier): ResourceIdentifier {
return { path: metaIdentifier.path.slice(0, -this.metaExtension.length) };
}
}

52
test/unit/pods/generate/TemplatedResourcesGenerator.test.ts
View File

@ -1,6 +1,6 @@
import type { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier';
import { HandlebarsTemplateEngine } from '../../../../src/pods/generate/HandlebarsTemplateEngine';
import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator';
import type { TemplateEngine } from '../../../../src/pods/generate/TemplateEngine';
import type {
FileIdentifierMapper,
FileIdentifierMapperFactory,
@ -9,7 +9,6 @@ import type {
import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil';
import { readableToString } from '../../../../src/util/StreamUtil';
import { mockFs } from '../../../util/Util';
import Dict = NodeJS.Dict;
jest.mock('fs');
@ -30,13 +29,6 @@ class DummyFactory implements FileIdentifierMapperFactory {
}
}
class DummyEngine implements TemplateEngine {
public apply(template: string, options: Dict<string>): string {
const keys = Object.keys(options);
return `${template}${keys.map((key): string => `{${key}:${options[key]}}`).join('')}`;
}
}
const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
const arr: T[] = [];
for await (const result of iterable) {
@ -47,7 +39,8 @@ const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
describe('A TemplatedResourcesGenerator', (): void => {
const rootFilePath = 'templates';
const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new DummyEngine());
// Using handlebars engine since it's smaller than any possible dummy
const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new HandlebarsTemplateEngine());
let cache: { data: any };
const template = '<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.';
const location = { path: 'http://test.com/alice/' };
@ -68,11 +61,11 @@ describe('A TemplatedResourcesGenerator', (): void => {
expect(representation.binary).toBe(true);
expect(representation.metadata.contentType).toBe('text/turtle');
await expect(readableToString(representation.data)).resolves
.toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
.toEqual(`<${webId}> a <http://xmlns.com/foaf/0.1/Person>.`);
});
it('creates the necessary containers and ignores non-files.', async(): Promise<void> => {
cache.data = { container: { container: { template }}, 2: 5 };
it('creates the necessary containers.', async(): Promise<void> => {
cache.data = { container: { container: { template }}};
const result = await genToArray(generator.generate(location, { webId }));
const identifiers = result.map((res): ResourceIdentifier => res.identifier);
const id = { path: `${location.path}container/container/template` };
@ -85,6 +78,37 @@ describe('A TemplatedResourcesGenerator', (): void => {
const { representation } = result[3];
await expect(readableToString(representation.data)).resolves
.toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
.toEqual(`<${webId}> a <http://xmlns.com/foaf/0.1/Person>.`);
});
it('adds metadata from .meta files.', async(): Promise<void> => {
const meta = '<> <pre:has> "metadata".';
cache.data = { '.meta': meta, container: { 'template.meta': meta, template }};
// Not using options since our dummy template generator generates invalid turtle
const result = await genToArray(generator.generate(location, { webId }));
const identifiers = result.map((res): ResourceIdentifier => res.identifier);
expect(identifiers).toEqual([
location,
{ path: `${location.path}container/` },
{ path: `${location.path}container/template` },
]);
// Root has the 1 raw metadata triple (with <> changed to its identifier)
const rootMetadata = result[0].representation.metadata;
expect(rootMetadata.identifier.value).toBe(location.path);
expect(rootMetadata.quads()).toHaveLength(1);
expect(rootMetadata.get('pre:has')?.value).toBe('metadata');
// Container has no metadata triples
const contMetadata = result[1].representation.metadata;
expect(contMetadata.identifier.value).toBe(`${location.path}container/`);
expect(contMetadata.quads()).toHaveLength(0);
// Document has the 1 raw metadata triple (with <> changed to its identifier) and content-type
const docMetadata = result[2].representation.metadata;
expect(docMetadata.identifier.value).toBe(`${location.path}container/template`);
expect(docMetadata.quads()).toHaveLength(2);
expect(docMetadata.get('pre:has')?.value).toBe('metadata');
expect(docMetadata.contentType).toBe('text/turtle');
});
});