feat: Implement resource mapper for the file resource store (#142)

* move file mapping logic to resourcemapper

* make filestore dependent of resource mapper

* set default contenttype

* refactor fileresourcemapper

* fix map function

* refactor

* add normalized parser

* refactor unit test

* fix metadata problem

* refactor names

* reverse change

* add getters

* add comments

* add comments, move code

* change text/turtle to constant

* add changes

* add requested changes

* add more requested changes

* add more requested changes

* more changes
This commit is contained in:
Freya 2020-09-11 17:08:53 +02:00 committed by GitHub
parent e9983d5837
commit 383da24601
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 219 additions and 113 deletions

View File

@ -86,7 +86,7 @@ export * from './src/storage/PassthroughStore';
export * from './src/storage/PatchingStore';
export * from './src/storage/RepresentationConvertingStore';
export * from './src/storage/ResourceLocker';
export * from './src/storage/ResourceMapper';
export * from './src/storage/FileIdentifierMapper';
export * from './src/storage/ResourceStore';
export * from './src/storage/SingleThreadedResourceLocker';
export * from './src/storage/UrlContainerManager';

View File

@ -0,0 +1,135 @@
import { posix } from 'path';
import { types } from 'mime-types';
import { RuntimeConfig } from '../init/RuntimeConfig';
import { ResourceIdentifier } from '../ldp/representation/ResourceIdentifier';
import { APPLICATION_OCTET_STREAM, TEXT_TURTLE } from '../util/ContentTypes';
import { ConflictHttpError } from '../util/errors/ConflictHttpError';
import { NotFoundHttpError } from '../util/errors/NotFoundHttpError';
import { trimTrailingSlashes } from '../util/Util';
import { FileIdentifierMapper } from './FileIdentifierMapper';
const { join: joinPath, normalize: normalizePath } = posix;
export interface ResourcePath {
/**
* The path of the container.
*/
containerPath: string;
/**
* The document name.
*/
documentName?: string;
}
export class ExtensionBasedMapper implements FileIdentifierMapper {
private readonly runtimeConfig: RuntimeConfig;
private readonly types: Record<string, any>;
public constructor(runtimeConfig: RuntimeConfig, overrideTypes = { acl: TEXT_TURTLE, metadata: TEXT_TURTLE }) {
this.runtimeConfig = runtimeConfig;
this.types = { ...types, ...overrideTypes };
}
// Using getters because the values of runtimeConfig get filled in at runtime (so they are still empty at
// construction time until issue #106 gets resolved.)
public get baseRequestURI(): string {
return trimTrailingSlashes(this.runtimeConfig.base);
}
public get rootFilepath(): string {
return trimTrailingSlashes(this.runtimeConfig.rootFilepath);
}
/**
* Strips the baseRequestURI from the identifier and checks if the stripped base URI matches the store's one.
* @param identifier - Incoming identifier.
*
* @throws {@link NotFoundHttpError}
* If the identifier does not match the baseRequestURI path of the store.
*
* @returns Absolute path of the file.
*/
public mapUrlToFilePath(identifier: ResourceIdentifier, id = ''): string {
return this.getAbsolutePath(this.getRelativePath(identifier), id);
}
/**
* Strips the rootFilepath path from the filepath and adds the baseRequestURI in front of it.
* @param path - The file path.
*
* @throws {@Link Error}
* If the file path does not match the rootFilepath path of the store.
*
* @returns Url of the file.
*/
public mapFilePathToUrl(path: string): string {
if (!path.startsWith(this.rootFilepath)) {
throw new Error(`File ${path} is not part of the file storage at ${this.rootFilepath}.`);
}
return this.baseRequestURI + path.slice(this.rootFilepath.length);
}
/**
* Get the content type from a file path, using its extension.
* @param path - The file path.
*
* @returns Content type of the file.
*/
public getContentTypeFromExtension(path: string): string {
const extension = /\.([^./]+)$/u.exec(path);
return (extension && this.types[extension[1].toLowerCase()]) || APPLICATION_OCTET_STREAM;
}
/**
* Get the absolute file path based on the rootFilepath of the store.
* @param path - The relative file path.
* @param identifier - Optional identifier to add to the path.
*
* @returns Absolute path of the file.
*/
public getAbsolutePath(path: string, identifier = ''): string {
return joinPath(this.rootFilepath, path, identifier);
}
/**
* Strips the baseRequestURI from the identifier and checks if the stripped base URI matches the store's one.
* @param identifier - Incoming identifier.
*
* @throws {@link NotFoundHttpError}
* If the identifier does not match the baseRequestURI path of the store.
*
* @returns A string representing the relative path.
*/
public getRelativePath(identifier: ResourceIdentifier): string {
if (!identifier.path.startsWith(this.baseRequestURI)) {
throw new NotFoundHttpError();
}
return identifier.path.slice(this.baseRequestURI.length);
}
/**
* Splits the identifier into the parent directory and slug.
* If the identifier specifies a directory, slug will be undefined.
* @param identifier - Incoming identifier.
*
* @throws {@link ConflictHttpError}
* If the root identifier is passed.
*
* @returns A ResourcePath object containing path and (optional) slug fields.
*/
public exctractDocumentName(identifier: ResourceIdentifier): ResourcePath {
const [ , containerPath, documentName ] = /^(.*\/)([^/]+\/?)?$/u.exec(this.getRelativePath(identifier)) ?? [];
if (
(typeof containerPath !== 'string' || normalizePath(containerPath) === '/') && typeof documentName !== 'string') {
throw new ConflictHttpError('Container with that identifier already exists (root).');
}
return {
containerPath: normalizePath(containerPath),
// If documentName is not undefined, return normalized documentName
documentName: typeof documentName === 'string' ? normalizePath(documentName) : undefined,
};
}
}

View File

@ -0,0 +1,21 @@
import { ResourceIdentifier } from '../ldp/representation/ResourceIdentifier';
/**
* Supports mapping a file to an URL and back.
*/
export interface FileIdentifierMapper {
/**
* Maps the given file path to an URL.
* @param file - The input file path.
*
* @returns The URL as a string.
*/
mapFilePathToUrl: (filePath: string) => string;
/**
* Maps the given resource identifier / URL to a file path.
* @param url - The input URL.
*
* @returns The file path as a string.
*/
mapUrlToFilePath: (identifier: ResourceIdentifier) => string;
}

View File

@ -1,10 +1,8 @@
import { createReadStream, createWriteStream, promises as fsPromises, Stats } from 'fs';
import { posix } from 'path';
import { Readable } from 'stream';
import { contentType as getContentTypeFromExtension } from 'mime-types';
import type { Quad } from 'rdf-js';
import streamifyArray from 'streamify-array';
import { RuntimeConfig } from '../init/RuntimeConfig';
import { Representation } from '../ldp/representation/Representation';
import { RepresentationMetadata } from '../ldp/representation/RepresentationMetadata';
import { ResourceIdentifier } from '../ldp/representation/ResourceIdentifier';
@ -15,38 +13,31 @@ import { NotFoundHttpError } from '../util/errors/NotFoundHttpError';
import { UnsupportedMediaTypeHttpError } from '../util/errors/UnsupportedMediaTypeHttpError';
import { InteractionController } from '../util/InteractionController';
import { MetadataController } from '../util/MetadataController';
import { ensureTrailingSlash, trimTrailingSlashes } from '../util/Util';
import { ensureTrailingSlash } from '../util/Util';
import { ExtensionBasedMapper } from './ExtensionBasedMapper';
import { ResourceStore } from './ResourceStore';
const { extname, join: joinPath, normalize: normalizePath } = posix;
const { join: joinPath } = posix;
/**
* Resource store storing its data in the file system backend.
* All requests will throw an {@link NotFoundHttpError} if unknown identifiers get passed.
*/
export class FileResourceStore implements ResourceStore {
private readonly runtimeConfig: RuntimeConfig;
private readonly interactionController: InteractionController;
private readonly metadataController: MetadataController;
private readonly resourceMapper: ExtensionBasedMapper;
/**
* @param runtimeConfig - The runtime config.
* @param resourceMapper - The file resource mapper.
* @param interactionController - Instance of InteractionController to use.
* @param metadataController - Instance of MetadataController to use.
*/
public constructor(runtimeConfig: RuntimeConfig, interactionController: InteractionController,
public constructor(resourceMapper: ExtensionBasedMapper, interactionController: InteractionController,
metadataController: MetadataController) {
this.runtimeConfig = runtimeConfig;
this.interactionController = interactionController;
this.metadataController = metadataController;
}
public get baseRequestURI(): string {
return trimTrailingSlashes(this.runtimeConfig.base);
}
public get rootFilepath(): string {
return trimTrailingSlashes(this.runtimeConfig.rootFilepath);
this.resourceMapper = resourceMapper;
}
/**
@ -63,7 +54,7 @@ export class FileResourceStore implements ResourceStore {
}
// Get the path from the request URI, all metadata triples if any, and the Slug and Link header values.
const path = this.parseIdentifier(container);
const path = this.resourceMapper.getRelativePath(container);
const { slug, raw } = representation.metadata;
const linkTypes = representation.metadata.linkRel?.type;
let metadata;
@ -84,13 +75,13 @@ export class FileResourceStore implements ResourceStore {
* @param identifier - Identifier of resource to delete.
*/
public async deleteResource(identifier: ResourceIdentifier): Promise<void> {
let path = this.parseIdentifier(identifier);
let path = this.resourceMapper.getRelativePath(identifier);
if (path === '' || ensureTrailingSlash(path) === '/') {
throw new MethodNotAllowedHttpError('Cannot delete root container.');
}
// Get the file status of the path defined by the request URI mapped to the corresponding filepath.
path = joinPath(this.rootFilepath, path);
path = this.resourceMapper.getAbsolutePath(path);
let stats;
try {
stats = await fsPromises.lstat(path);
@ -117,7 +108,7 @@ export class FileResourceStore implements ResourceStore {
*/
public async getRepresentation(identifier: ResourceIdentifier): Promise<Representation> {
// Get the file status of the path defined by the request URI mapped to the corresponding filepath.
const path = joinPath(this.rootFilepath, this.parseIdentifier(identifier));
const path = this.resourceMapper.mapUrlToFilePath(identifier);
let stats;
try {
stats = await fsPromises.lstat(path);
@ -152,12 +143,9 @@ export class FileResourceStore implements ResourceStore {
throw new UnsupportedMediaTypeHttpError('FileResourceStore only supports binary representations.');
}
// Break up the request URI in the different parts `path` and `slug` as we know their semantics from addResource
// to call the InteractionController in the same way.
const [ , path, slug ] = /^(.*\/)([^/]+\/?)?$/u.exec(this.parseIdentifier(identifier)) ?? [];
if ((typeof path !== 'string' || normalizePath(path) === '/') && typeof slug !== 'string') {
throw new ConflictHttpError('Container with that identifier already exists (root).');
}
// Break up the request URI in the different parts `containerPath` and `documentName` as we know their semantics
// from addResource to call the InteractionController in the same way.
const { containerPath, documentName } = this.resourceMapper.exctractDocumentName(identifier);
const { raw } = representation.metadata;
const linkTypes = representation.metadata.linkRel?.type;
let metadata: Readable | undefined;
@ -166,39 +154,11 @@ export class FileResourceStore implements ResourceStore {
}
// Create a new container or resource in the parent container with a specific name based on the incoming headers.
const isContainer = this.interactionController.isContainer(slug, linkTypes);
const newIdentifier = this.interactionController.generateIdentifier(isContainer, slug);
const isContainer = this.interactionController.isContainer(documentName, linkTypes);
const newIdentifier = this.interactionController.generateIdentifier(isContainer, documentName);
return isContainer ?
await this.setDirectoryRepresentation(path, newIdentifier, metadata) :
await this.setFileRepresentation(path, newIdentifier, representation.data, metadata);
}
/**
* Strips the baseRequestURI from the identifier and checks if the stripped base URI matches the store's one.
* @param identifier - Incoming identifier.
*
* @throws {@link NotFoundHttpError}
* If the identifier does not match the baseRequestURI path of the store.
*/
private parseIdentifier(identifier: ResourceIdentifier): string {
if (!identifier.path.startsWith(this.baseRequestURI)) {
throw new NotFoundHttpError();
}
return identifier.path.slice(this.baseRequestURI.length);
}
/**
* Strips the rootFilepath path from the filepath and adds the baseRequestURI in front of it.
* @param path - The filepath.
*
* @throws {@Link Error}
* If the filepath does not match the rootFilepath path of the store.
*/
private mapFilepathToUrl(path: string): string {
if (!path.startsWith(this.rootFilepath)) {
throw new Error(`File ${path} is not part of the file storage at ${this.rootFilepath}.`);
}
return this.baseRequestURI + path.slice(this.rootFilepath.length);
await this.setDirectoryRepresentation(containerPath, newIdentifier, metadata) :
await this.setFileRepresentation(containerPath, newIdentifier, representation.data, metadata);
}
/**
@ -247,7 +207,7 @@ export class FileResourceStore implements ResourceStore {
*/
private async getFileRepresentation(path: string, stats: Stats): Promise<Representation> {
const readStream = createReadStream(path);
const contentType = getContentTypeFromExtension(extname(path));
const contentType = this.resourceMapper.getContentTypeFromExtension(path);
let rawMetadata: Quad[] = [];
try {
const readMetadataStream = createReadStream(`${path}.metadata`);
@ -259,10 +219,9 @@ export class FileResourceStore implements ResourceStore {
raw: rawMetadata,
dateTime: stats.mtime,
byteSize: stats.size,
contentType,
};
if (contentType) {
metadata.contentType = contentType;
}
return { metadata, data: readStream, binary: true };
}
@ -280,7 +239,7 @@ export class FileResourceStore implements ResourceStore {
const files = await fsPromises.readdir(path);
const quads: Quad[] = [];
const containerURI = this.mapFilepathToUrl(path);
const containerURI = this.resourceMapper.mapFilePathToUrl(path);
quads.push(...this.metadataController.generateResourceQuads(containerURI, stats));
quads.push(...await this.getDirChildrenQuadRepresentation(files, path, containerURI));
@ -316,7 +275,7 @@ export class FileResourceStore implements ResourceStore {
const quads: Quad[] = [];
for (const childName of files) {
try {
const childURI = this.mapFilepathToUrl(joinPath(path, childName));
const childURI = this.resourceMapper.mapFilePathToUrl(joinPath(path, childName));
const childStats = await fsPromises.lstat(joinPath(path, childName));
if (!childStats.isFile() && !childStats.isDirectory()) {
continue;
@ -344,7 +303,7 @@ export class FileResourceStore implements ResourceStore {
let stats;
try {
stats = await fsPromises.lstat(
joinPath(this.rootFilepath, path, newIdentifier),
this.resourceMapper.getAbsolutePath(path, newIdentifier),
);
} catch (error) {
await this.createFile(path, newIdentifier, data, true, metadata);
@ -367,7 +326,7 @@ export class FileResourceStore implements ResourceStore {
// Create a container if the identifier doesn't exist yet.
try {
await fsPromises.access(
joinPath(this.rootFilepath, path, newIdentifier),
this.resourceMapper.getAbsolutePath(path, newIdentifier),
);
throw new ConflictHttpError('Resource with that identifier already exists.');
} catch (error) {
@ -400,7 +359,7 @@ export class FileResourceStore implements ResourceStore {
// Get the file status of the filepath of the directory where the file is to be created.
let stats;
try {
stats = await fsPromises.lstat(joinPath(this.rootFilepath, path));
stats = await fsPromises.lstat(this.resourceMapper.getAbsolutePath(path));
} catch (error) {
throw new MethodNotAllowedHttpError();
}
@ -411,16 +370,17 @@ export class FileResourceStore implements ResourceStore {
} else {
// If metadata is specified, save it in a corresponding metadata file.
if (metadata) {
await this.createDataFile(joinPath(this.rootFilepath, path, `${resourceName}.metadata`), metadata);
await this.createDataFile(this.resourceMapper.getAbsolutePath(path, `${resourceName}.metadata`), metadata);
}
// If no error thrown from above, indicating failed metadata file creation, create the actual resource file.
try {
await this.createDataFile(joinPath(this.rootFilepath, path, resourceName), data);
return { path: this.mapFilepathToUrl(joinPath(this.rootFilepath, path, resourceName)) };
const fullPath = this.resourceMapper.getAbsolutePath(path, resourceName);
await this.createDataFile(fullPath, data);
return { path: this.resourceMapper.mapFilePathToUrl(fullPath) };
} catch (error) {
// Normal file has not been created so we don't want the metadata file to remain.
await fsPromises.unlink(joinPath(this.rootFilepath, path, `${resourceName}.metadata`));
await fsPromises.unlink(this.resourceMapper.getAbsolutePath(path, `${resourceName}.metadata`));
throw error;
}
}
@ -437,12 +397,12 @@ export class FileResourceStore implements ResourceStore {
*/
private async createContainer(path: string, containerName: string,
allowRecursiveCreation: boolean, metadata?: Readable): Promise<ResourceIdentifier> {
const fullPath = ensureTrailingSlash(joinPath(this.rootFilepath, path, containerName));
const fullPath = ensureTrailingSlash(this.resourceMapper.getAbsolutePath(path, containerName));
// If recursive creation is not allowed, check if the parent container exists and then create the child directory.
try {
if (!allowRecursiveCreation) {
const stats = await fsPromises.lstat(joinPath(this.rootFilepath, path));
const stats = await fsPromises.lstat(this.resourceMapper.getAbsolutePath(path));
if (!stats.isDirectory()) {
throw new MethodNotAllowedHttpError('The given path is not a valid container.');
}
@ -466,7 +426,7 @@ export class FileResourceStore implements ResourceStore {
throw error;
}
}
return { path: this.mapFilepathToUrl(fullPath) };
return { path: this.resourceMapper.mapFilePathToUrl(fullPath) };
}
/**

View File

@ -1,22 +0,0 @@
import { RepresentationMetadata } from '../ldp/representation/RepresentationMetadata';
/**
* Supports mapping a file to an URL and back.
*/
export interface ResourceMapper {
/**
* Maps the given file to an URL.
* @param file - The input file.
*
* @returns A promise resolving to the corresponding URL and metadata of the representation.
*/
mapFilePathToUrl: (file: File) => Promise<{ url: URL; metadata: RepresentationMetadata }>;
/**
* Maps the given URL and metadata to a file.
* @param url - The input URL.
* @param metadata - The representation metadata.
*
* @returns A promise resolving to the corresponding file.
*/
mapUrlToFilePath: (url: URL, metadata: RepresentationMetadata) => Promise<File>;
}

View File

@ -1,5 +1,6 @@
// Well-known content types
export const TEXT_TURTLE = 'text/turtle';
export const APPLICATION_OCTET_STREAM = 'application/octet-stream';
// Internal (non-exposed) content types
export const INTERNAL_QUADS = 'internal/quads';

View File

@ -0,0 +1,21 @@
import { RuntimeConfig } from '../../../src/init/RuntimeConfig';
import { ExtensionBasedMapper } from '../../../src/storage/ExtensionBasedMapper';
describe('An ExtensionBasedMapper', (): void => {
const base = 'http://test.com/';
const rootFilepath = 'uploads/';
const resourceMapper = new ExtensionBasedMapper(new RuntimeConfig({ base, rootFilepath }));
it('returns the correct url of a file.', async(): Promise<void> => {
let result = resourceMapper.mapFilePathToUrl(`${rootFilepath}test.txt`);
expect(result).toEqual(`${base}test.txt`);
result = resourceMapper.mapFilePathToUrl(`${rootFilepath}image.jpg`);
expect(result).toEqual(`${base}image.jpg`);
});
it('errors when filepath does not contain rootFilepath.', async(): Promise<void> => {
expect((): string => resourceMapper.mapFilePathToUrl('random/test.txt')).toThrow(Error);
expect((): string => resourceMapper.mapFilePathToUrl('test.txt')).toThrow(Error);
});
});

View File

@ -8,6 +8,7 @@ import streamifyArray from 'streamify-array';
import { RuntimeConfig } from '../../../src/init/RuntimeConfig';
import { Representation } from '../../../src/ldp/representation/Representation';
import { RepresentationMetadata } from '../../../src/ldp/representation/RepresentationMetadata';
import { ExtensionBasedMapper } from '../../../src/storage/ExtensionBasedMapper';
import { FileResourceStore } from '../../../src/storage/FileResourceStore';
import { INTERNAL_QUADS } from '../../../src/util/ContentTypes';
import { ConflictHttpError } from '../../../src/util/errors/ConflictHttpError';
@ -50,7 +51,7 @@ describe('A FileResourceStore', (): void => {
jest.clearAllMocks();
store = new FileResourceStore(
new RuntimeConfig({ base, rootFilepath }),
new ExtensionBasedMapper(new RuntimeConfig({ base, rootFilepath })),
new InteractionController(),
new MetadataController(),
);
@ -218,7 +219,7 @@ describe('A FileResourceStore', (): void => {
raw: [],
dateTime: stats.mtime,
byteSize: stats.size,
contentType: 'text/plain; charset=utf-8',
contentType: 'text/plain',
},
});
await expect(arrayifyStream(result.data)).resolves.toEqual([ rawData ]);
@ -426,17 +427,6 @@ describe('A FileResourceStore', (): void => {
expect(fsPromises.access as jest.Mock).toBeCalledTimes(1);
});
it('errors when mapping a filepath that does not match the rootFilepath of the store.', async(): Promise<void> => {
expect((): any => {
// eslint-disable-next-line dot-notation
store['mapFilepathToUrl']('http://wrong.com/wrong');
}).toThrowError();
expect((): any => {
// eslint-disable-next-line dot-notation
store['mapFilepathToUrl'](`${base}file.txt`);
}).toThrowError();
});
it('undoes metadata file creation when resource creation fails.', async(): Promise<void> => {
// Mock the fs functions.
(fsPromises.mkdir as jest.Mock).mockReturnValueOnce(true);
@ -483,13 +473,12 @@ describe('A FileResourceStore', (): void => {
expect(fsPromises.mkdir as jest.Mock).toBeCalledWith(joinPath(rootFilepath, 'myContainer/'), { recursive: true });
});
it('returns no contentType when unknown for representation.', async(): Promise<void> => {
it('returns default contentType when unknown for representation.', async(): Promise<void> => {
// Mock the fs functions.
stats.isFile = jest.fn((): any => true);
(fsPromises.lstat as jest.Mock).mockReturnValueOnce(stats);
(fs.createReadStream as jest.Mock).mockReturnValueOnce(streamifyArray([ rawData ]));
(fs.createReadStream as jest.Mock).mockReturnValueOnce(new Readable()
.destroy(new Error('Metadata file does not exist.')));
(fs.createReadStream as jest.Mock).mockImplementationOnce((): any => new Error('Metadata file does not exist.'));
const result = await store.getRepresentation({ path: `${base}.htaccess` });
expect(result).toEqual({
@ -497,6 +486,7 @@ describe('A FileResourceStore', (): void => {
data: expect.any(Readable),
metadata: {
raw: [],
contentType: 'application/octet-stream',
dateTime: stats.mtime,
byteSize: stats.size,
},