Mirroristas/CommunitySolidServer
2
0
Fork 0
mirror of https://github.com/CommunitySolidServer/CommunitySolidServer.git synced 2024-10-03 14:55:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Move all metadata file knowledge to file mapper

Browse Source
This commit is contained in:
Joachim Van Herwegen
2021-07-20 15:58:45 +02:00
parent 6495d650c2
commit 57da67f9ee
11 changed files with 215 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/pods/generate/TemplatedResourcesGenerator.ts
View File

@@ -26,7 +26,6 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
private readonly templateFolder: string;
private readonly factory: FileIdentifierMapperFactory;
private readonly engine: TemplateEngine;
private readonly metaExtension = '.meta';
/**
* A mapper is needed to convert the template file paths to identifiers relative to the given base identifier.
@@ -91,9 +90,8 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
const links: Record<string, { link: ResourceLink; meta?: ResourceLink }> = { };
for await (const link of linkGen) {
const { path } = link.identifier;
if (this.isMeta(path)) {
const resourcePath = this.metaToResource(link.identifier).path;
links[resourcePath] = Object.assign(links[resourcePath] || {}, { meta: link });
if (link.isMetadata) {
links[path] = Object.assign(links[path] || {}, { meta: link });
} else {
links[path] = Object.assign(links[path] || {}, { link });
}
@@ -135,11 +133,10 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
*/
private async generateMetadata(metaLink: ResourceLink, options: Dict<string>):
Promise<RepresentationMetadata> {
const identifier = this.metaToResource(metaLink.identifier);
const metadata = new RepresentationMetadata(identifier);
const metadata = new RepresentationMetadata(metaLink.identifier);
const data = await this.parseTemplate(metaLink.filePath, options);
const parser = new Parser({ format: metaLink.contentType, baseIRI: identifier.path });
const parser = new Parser({ format: metaLink.contentType, baseIRI: metaLink.identifier.path });
const quads = parser.parse(data);
metadata.addQuads(quads);
@@ -153,18 +150,4 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
const raw = await fsPromises.readFile(filePath, 'utf8');
return this.engine.apply(raw, options);
}
/**
* Verifies if the given path corresponds to a metadata file.
*/
private isMeta(path: string): boolean {
return path.endsWith(this.metaExtension);
}
/**
* Converts a generated metadata identifier to the identifier of its corresponding resource.
*/
private metaToResource(metaIdentifier: ResourceIdentifier): ResourceIdentifier {
return { path: metaIdentifier.path.slice(0, -this.metaExtension.length) };
}
}

2
src/pods/generate/variables/RootFilePathHandler.ts
View File

@@ -21,7 +21,7 @@ export class RootFilePathHandler extends VariableHandler {
public async handle({ identifier, settings }: { identifier: ResourceIdentifier; settings: PodSettings }):
Promise<void> {
const path = (await this.fileMapper.mapUrlToFilePath(identifier)).filePath;
const path = (await this.fileMapper.mapUrlToFilePath(identifier, false)).filePath;
try {
// Even though we check if it already exists, there is still a potential race condition
// in between this check and the store being created.

52
src/storage/accessors/FileDataAccessor.ts
View File

@@ -5,7 +5,6 @@ import type { Quad } from 'rdf-js';
import type { Representation } from '../../ldp/representation/Representation';
import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata';
import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
import { ConflictHttpError } from '../../util/errors/ConflictHttpError';
import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError';
import { isSystemError } from '../../util/errors/SystemError';
import { UnsupportedMediaTypeHttpError } from '../../util/errors/UnsupportedMediaTypeHttpError';
@@ -43,7 +42,7 @@ export class FileDataAccessor implements DataAccessor {
* Will throw NotFoundHttpError if the input is a container.
*/
public async getData(identifier: ResourceIdentifier): Promise<Guarded<Readable>> {
const link = await this.resourceMapper.mapUrlToFilePath(identifier);
const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
const stats = await this.getStats(link.filePath);
if (stats.isFile()) {
@@ -58,7 +57,7 @@ export class FileDataAccessor implements DataAccessor {
* and adding file system specific metadata elements.
*/
public async getMetadata(identifier: ResourceIdentifier): Promise<RepresentationMetadata> {
const link = await this.resourceMapper.mapUrlToFilePath(identifier);
const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
const stats = await this.getStats(link.filePath);
if (!isContainerIdentifier(identifier) && stats.isFile()) {
return this.getFileMetadata(link, stats);
@@ -70,7 +69,7 @@ export class FileDataAccessor implements DataAccessor {
}
public async* getChildren(identifier: ResourceIdentifier): AsyncIterableIterator<RepresentationMetadata> {
const link = await this.resourceMapper.mapUrlToFilePath(identifier);
const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
yield* this.getChildMetadata(link);
}
@@ -80,10 +79,7 @@ export class FileDataAccessor implements DataAccessor {
*/
public async writeDocument(identifier: ResourceIdentifier, data: Guarded<Readable>, metadata: RepresentationMetadata):
Promise<void> {
if (this.isMetadataPath(identifier.path)) {
throw new ConflictHttpError('Not allowed to create files with the metadata extension.');
}
const link = await this.resourceMapper.mapUrlToFilePath(identifier, metadata.contentType);
const link = await this.resourceMapper.mapUrlToFilePath(identifier, false, metadata.contentType);
// Check if we already have a corresponding file with a different extension
await this.verifyExistingExtension(link);
@@ -95,7 +91,8 @@ export class FileDataAccessor implements DataAccessor {
} catch (error: unknown) {
// Delete the metadata if there was an error writing the file
if (wroteMetadata) {
await fsPromises.unlink((await this.getMetadataLink(link.identifier)).filePath);
const metaLink = await this.resourceMapper.mapUrlToFilePath(identifier, true);
await fsPromises.unlink(metaLink.filePath);
}
throw error;
}
@@ -105,7 +102,7 @@ export class FileDataAccessor implements DataAccessor {
* Creates corresponding folder if necessary and writes metadata to metadata file if necessary.
*/
public async writeContainer(identifier: ResourceIdentifier, metadata: RepresentationMetadata): Promise<void> {
const link = await this.resourceMapper.mapUrlToFilePath(identifier);
const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
try {
await fsPromises.mkdir(link.filePath, { recursive: true });
} catch (error: unknown) {
@@ -122,11 +119,12 @@ export class FileDataAccessor implements DataAccessor {
* Removes the corresponding file/folder (and metadata file).
*/
public async deleteResource(identifier: ResourceIdentifier): Promise<void> {
const link = await this.resourceMapper.mapUrlToFilePath(identifier);
const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
const stats = await this.getStats(link.filePath);
try {
await fsPromises.unlink((await this.getMetadataLink(link.identifier)).filePath);
const metaLink = await this.resourceMapper.mapUrlToFilePath(identifier, true);
await fsPromises.unlink(metaLink.filePath);
} catch (error: unknown) {
// Ignore if it doesn't exist
if (!isSystemError(error) || error.code !== 'ENOENT') {
@@ -161,21 +159,6 @@ export class FileDataAccessor implements DataAccessor {
}
}
/**
* Generates ResourceLink that corresponds to the metadata resource of the given identifier.
*/
private async getMetadataLink(identifier: ResourceIdentifier): Promise<ResourceLink> {
const metaIdentifier = { path: `${identifier.path}.meta` };
return this.resourceMapper.mapUrlToFilePath(metaIdentifier);
}
/**
* Checks if the given path is a metadata path.
*/
private isMetadataPath(path: string): boolean {
return path.endsWith('.meta');
}
/**
* Reads and generates all metadata relevant for the given file,
* ingesting it into a RepresentationMetadata object.
@@ -215,7 +198,7 @@ export class FileDataAccessor implements DataAccessor {
metadata.remove(RDF.type, LDP.terms.BasicContainer);
metadata.removeAll(CONTENT_TYPE);
const quads = metadata.quads();
const metadataLink = await this.getMetadataLink(link.identifier);
const metadataLink = await this.resourceMapper.mapUrlToFilePath(link.identifier, true);
let wroteMetadata: boolean;
// Write metadata to file if there are quads remaining
@@ -263,7 +246,7 @@ export class FileDataAccessor implements DataAccessor {
*/
private async getRawMetadata(identifier: ResourceIdentifier): Promise<Quad[]> {
try {
const metadataLink = await this.getMetadataLink(identifier);
const metadataLink = await this.resourceMapper.mapUrlToFilePath(identifier, true);
// Check if the metadata file exists first
await fsPromises.lstat(metadataLink.filePath);
@@ -290,10 +273,6 @@ export class FileDataAccessor implements DataAccessor {
// For every child in the container we want to generate specific metadata
for await (const entry of dir) {
const childName = entry.name;
// Hide metadata files
if (this.isMetadataPath(childName)) {
continue;
}
// Ignore non-file/directory entries in the folder
if (!entry.isFile() && !entry.isDirectory()) {
@@ -304,6 +283,11 @@ export class FileDataAccessor implements DataAccessor {
const childLink = await this.resourceMapper
.mapFilePathToUrl(joinFilePath(link.filePath, childName), entry.isDirectory());
// Hide metadata files
if (childLink.isMetadata) {
continue;
}
// Generate metadata of this specific child
const childStats = await fsPromises.lstat(joinFilePath(link.filePath, childName));
const metadata = new RepresentationMetadata(childLink.identifier);
@@ -340,7 +324,7 @@ export class FileDataAccessor implements DataAccessor {
private async verifyExistingExtension(link: ResourceLink): Promise<void> {
try {
// Delete the old file with the (now) wrong extension
const oldLink = await this.resourceMapper.mapUrlToFilePath(link.identifier);
const oldLink = await this.resourceMapper.mapUrlToFilePath(link.identifier, false);
if (oldLink.filePath !== link.filePath) {
await fsPromises.unlink(oldLink.filePath);
}

33
src/storage/mapping/BaseFileIdentifierMapper.ts
View File

@@ -2,6 +2,7 @@ import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdenti
import { getLoggerFor } from '../../logging/LogUtil';
import { APPLICATION_OCTET_STREAM } from '../../util/ContentTypes';
import { BadRequestHttpError } from '../../util/errors/BadRequestHttpError';
import { ConflictHttpError } from '../../util/errors/ConflictHttpError';
import { InternalServerError } from '../../util/errors/InternalServerError';
import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError';
import {
@@ -33,12 +34,23 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
* Determines the content type if none was provided.
* For containers the content-type input is ignored.
* @param identifier - The input identifier.
* @param isMetadata - If we need the data or metadata file path.
* @param contentType - The content-type provided with the request.
*
* @returns A ResourceLink with all the necessary metadata.
*/
public async mapUrlToFilePath(identifier: ResourceIdentifier, contentType?: string): Promise<ResourceLink> {
const path = this.getRelativePath(identifier);
public async mapUrlToFilePath(identifier: ResourceIdentifier, isMetadata: boolean, contentType?: string):
Promise<ResourceLink> {
// Technically we could allow paths ending on .meta as long as we make sure there is never a mixup.
// But this can lead to potential issues.
// This also immediately stops users that expect they can update metadata like this.
if (this.isMetadataPath(identifier.path)) {
throw new ConflictHttpError('Not allowed to create files with the metadata extension.');
}
let path = this.getRelativePath(identifier);
if (isMetadata) {
path += '.meta';
}
this.validateRelativePath(path, identifier);
const filePath = this.getAbsolutePath(path);
@@ -57,7 +69,7 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
*/
protected async mapUrlToContainerPath(identifier: ResourceIdentifier, filePath: string): Promise<ResourceLink> {
this.logger.debug(`URL ${identifier.path} points to the container ${filePath}`);
return { identifier, filePath };
return { identifier, filePath, isMetadata: this.isMetadataPath(filePath) };
}
/**
@@ -75,7 +87,7 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
Promise<ResourceLink> {
contentType = await this.getContentTypeFromUrl(identifier, contentType);
this.logger.debug(`The path for ${identifier.path} is ${filePath}`);
return { identifier, filePath, contentType };
return { identifier, filePath, contentType, isMetadata: this.isMetadataPath(filePath) };
}
/**
@@ -113,7 +125,11 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
this.logger.debug(`Document ${filePath} maps to URL ${url}`);
contentType = await this.getContentTypeFromPath(filePath);
}
return { identifier: { path: url }, filePath, contentType };
const isMetadata = this.isMetadataPath(filePath);
if (isMetadata) {
url = url.slice(0, -'.meta'.length);
}
return { identifier: { path: url }, filePath, contentType, isMetadata };
}
/**
@@ -194,4 +210,11 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
throw new BadRequestHttpError('Disallowed /.. segment in URL');
}
}
/**
* Checks if the given path is a metadata path.
*/
protected isMetadataPath(path: string): boolean {
return path.endsWith('.meta');
}
}

8
src/storage/mapping/FileIdentifierMapper.ts
View File

@@ -13,6 +13,10 @@ export interface ResourceLink {
* Content-type for a document (not defined for containers).
*/
contentType?: string;
/**
* If the resource is a metadata file.
*/
isMetadata: boolean;
}
/**
@@ -33,11 +37,13 @@ export interface FileIdentifierMapper {
* If there is no corresponding file a file path will be generated.
* For containers the content-type input gets ignored.
* @param identifier - The input identifier.
* @param isMetadata - If we are mapping the metadata of the resource instead of its data.
* @param contentType - The (optional) content-type of the resource.
*
* @returns A ResourceLink with all the necessary metadata.
*/
mapUrlToFilePath: (identifier: ResourceIdentifier, contentType?: string) => Promise<ResourceLink>;
mapUrlToFilePath: (identifier: ResourceIdentifier, isMetadata: boolean, contentType?: string) =>
Promise<ResourceLink>;
}
/**