Merge branch 'main' into versions/next-major

This commit is contained in:
Joachim Van Herwegen
2023-10-05 14:28:06 +02:00
58 changed files with 1061 additions and 215 deletions

View File

@@ -11,7 +11,7 @@ import type { RepresentationPreferences } from '../../representation/Representat
import { PreferenceParser } from './PreferenceParser';
const parsers: {
name: keyof RepresentationPreferences;
name: Exclude<keyof RepresentationPreferences, 'range'>;
header: string;
parse: (value: string) => AcceptHeader[];
}[] = [

View File

@@ -0,0 +1,48 @@
import type { HttpRequest } from '../../../server/HttpRequest';
import { BadRequestHttpError } from '../../../util/errors/BadRequestHttpError';
import type { RepresentationPreferences } from '../../representation/RepresentationPreferences';
import { PreferenceParser } from './PreferenceParser';
/**
* Parses the range header into range preferences.
* If the range corresponds to a suffix-length range, it will be stored in `start` as a negative value.
*/
export class RangePreferenceParser extends PreferenceParser {
public async handle({ request: { headers: { range }}}: { request: HttpRequest }): Promise<RepresentationPreferences> {
if (!range) {
return {};
}
const [ unit, rangeTail ] = range.split('=').map((entry): string => entry.trim());
if (unit.length === 0) {
throw new BadRequestHttpError(`Missing unit value from range header ${range}`);
}
if (!rangeTail) {
throw new BadRequestHttpError(`Invalid range header format ${range}`);
}
const ranges = rangeTail.split(',').map((entry): string => entry.trim());
const parts: { start: number; end?: number }[] = [];
for (const rangeEntry of ranges) {
const [ start, end ] = rangeEntry.split('-').map((entry): string => entry.trim());
// This can actually be undefined if the split results in less than 2 elements
if (typeof end !== 'string') {
throw new BadRequestHttpError(`Invalid range header format ${range}`);
}
if (start.length === 0) {
if (end.length === 0) {
throw new BadRequestHttpError(`Invalid range header format ${range}`);
}
parts.push({ start: -Number.parseInt(end, 10) });
} else {
const part: typeof parts[number] = { start: Number.parseInt(start, 10) };
if (end.length > 0) {
part.end = Number.parseInt(end, 10);
}
parts.push(part);
}
}
return { range: { unit, parts }};
}
}

View File

@@ -0,0 +1,32 @@
import { InternalServerError } from '../../../util/errors/InternalServerError';
import { UnionHandler } from '../../../util/handlers/UnionHandler';
import type { RepresentationPreferences } from '../../representation/RepresentationPreferences';
import type { PreferenceParser } from './PreferenceParser';
/**
* Combines the results of multiple {@link PreferenceParser}s.
* Will throw an error if multiple parsers return a range as these can't logically be combined.
*/
export class UnionPreferenceParser extends UnionHandler<PreferenceParser> {
public constructor(parsers: PreferenceParser[]) {
super(parsers, false, false);
}
protected async combine(results: RepresentationPreferences[]): Promise<RepresentationPreferences> {
const rangeCount = results.filter((result): boolean => Boolean(result.range)).length;
if (rangeCount > 1) {
throw new InternalServerError('Found multiple range values. This implies a misconfiguration.');
}
return results.reduce<RepresentationPreferences>((acc, val): RepresentationPreferences => {
for (const key of Object.keys(val) as (keyof RepresentationPreferences)[]) {
if (key === 'range') {
acc[key] = val[key];
} else {
acc[key] = { ...acc[key], ...val[key] };
}
}
return acc;
}, {});
}
}

View File

@@ -0,0 +1,50 @@
import { getLoggerFor } from '../../../logging/LogUtil';
import type { HttpResponse } from '../../../server/HttpResponse';
import { addHeader } from '../../../util/HeaderUtil';
import { termToInt } from '../../../util/QuadUtil';
import { POSIX, SOLID_HTTP } from '../../../util/Vocabularies';
import type { RepresentationMetadata } from '../../representation/RepresentationMetadata';
import { MetadataWriter } from './MetadataWriter';
/**
* Generates the necessary `content-range` header if there is range metadata.
* If the start or end is unknown, a `*` will be used instead.
* According to the RFC, this is incorrect,
* but is all we can do as long as we don't know the full length of the representation in advance.
* For the same reason, the total length of the representation will always be `*`.
*
* This class also adds the content-length header.
* This will contain either the full size for standard requests,
* or the size of the slice for range requests.
*/
export class RangeMetadataWriter extends MetadataWriter {
protected readonly logger = getLoggerFor(this);
public async handle(input: { response: HttpResponse; metadata: RepresentationMetadata }): Promise<void> {
const size = termToInt(input.metadata.get(POSIX.terms.size));
const unit = input.metadata.get(SOLID_HTTP.terms.unit)?.value;
if (!unit) {
if (typeof size === 'number') {
addHeader(input.response, 'Content-Length', `${size}`);
}
return;
}
let start = termToInt(input.metadata.get(SOLID_HTTP.terms.start));
if (typeof start === 'number' && start < 0 && typeof size === 'number') {
start = size + start;
}
let end = termToInt(input.metadata.get(SOLID_HTTP.terms.end));
if (typeof end !== 'number' && typeof size === 'number') {
end = size - 1;
}
const rangeHeader = `${unit} ${start ?? '*'}-${end ?? '*'}/${size ?? '*'}`;
addHeader(input.response, 'Content-Range', rangeHeader);
if (typeof start === 'number' && typeof end === 'number') {
addHeader(input.response, 'Content-Length', `${end - start + 1}`);
} else {
this.logger.warn(`Generating invalid content-range header due to missing size information: ${rangeHeader}`);
}
}
}

View File

@@ -1,10 +1,12 @@
import type { Readable } from 'stream';
import type { Guarded } from '../../../util/GuardedStream';
import { SOLID_HTTP } from '../../../util/Vocabularies';
import type { RepresentationMetadata } from '../../representation/RepresentationMetadata';
import { ResponseDescription } from './ResponseDescription';
/**
* Corresponds to a 200 response, containing relevant metadata and potentially data.
* Corresponds to a 200 or 206 response, containing relevant metadata and potentially data.
* A 206 will be returned if range metadata is found in the metadata object.
*/
export class OkResponseDescription extends ResponseDescription {
/**
@@ -12,6 +14,6 @@ export class OkResponseDescription extends ResponseDescription {
* @param data - Potential data. @ignored
*/
public constructor(metadata: RepresentationMetadata, data?: Guarded<Readable>) {
super(200, metadata, data);
super(metadata.has(SOLID_HTTP.terms.unit) ? 206 : 200, metadata, data);
}
}

View File

@@ -31,4 +31,6 @@ export interface RepresentationPreferences {
datetime?: ValuePreferences;
encoding?: ValuePreferences;
language?: ValuePreferences;
// `start` can be negative and implies the last X of a stream
range?: { unit: string; parts: { start: number; end?: number }[] };
}

View File

@@ -77,6 +77,8 @@ export * from './http/input/metadata/SlugParser';
// HTTP/Input/Preferences
export * from './http/input/preferences/AcceptPreferenceParser';
export * from './http/input/preferences/PreferenceParser';
export * from './http/input/preferences/RangePreferenceParser';
export * from './http/input/preferences/UnionPreferenceParser';
// HTTP/Input
export * from './http/input/BasicRequestParser';
@@ -106,6 +108,7 @@ export * from './http/output/metadata/LinkRelMetadataWriter';
export * from './http/output/metadata/MappedMetadataWriter';
export * from './http/output/metadata/MetadataWriter';
export * from './http/output/metadata/ModifiedMetadataWriter';
export * from './http/output/metadata/RangeMetadataWriter';
export * from './http/output/metadata/StorageDescriptionAdvertiser';
export * from './http/output/metadata/WacAllowMetadataWriter';
export * from './http/output/metadata/WwwAuthMetadataWriter';
@@ -452,6 +455,7 @@ export * from './storage/validators/QuotaValidator';
// Storage
export * from './storage/AtomicResourceStore';
export * from './storage/BaseResourceStore';
export * from './storage/BinarySliceResourceStore';
export * from './storage/CachedResourceSet';
export * from './storage/DataAccessorBasedStore';
export * from './storage/IndexRepresentationStore';
@@ -480,6 +484,7 @@ export * from './util/errors/NotFoundHttpError';
export * from './util/errors/NotImplementedHttpError';
export * from './util/errors/OAuthHttpError';
export * from './util/errors/PreconditionFailedHttpError';
export * from './util/errors/RangeNotSatisfiedHttpError';
export * from './util/errors/RedirectHttpError';
export * from './util/errors/SystemError';
export * from './util/errors/UnauthorizedHttpError';
@@ -550,6 +555,7 @@ export * from './util/PromiseUtil';
export * from './util/QuadUtil';
export * from './util/RecordObject';
export * from './util/ResourceUtil';
export * from './util/SliceStream';
export * from './util/StreamUtil';
export * from './util/StringUtil';
export * from './util/TermUtil';

View File

@@ -0,0 +1,68 @@
import type { Representation } from '../http/representation/Representation';
import type { RepresentationPreferences } from '../http/representation/RepresentationPreferences';
import type { ResourceIdentifier } from '../http/representation/ResourceIdentifier';
import { getLoggerFor } from '../logging/LogUtil';
import { InternalServerError } from '../util/errors/InternalServerError';
import { RangeNotSatisfiedHttpError } from '../util/errors/RangeNotSatisfiedHttpError';
import { guardStream } from '../util/GuardedStream';
import { termToInt } from '../util/QuadUtil';
import { SliceStream } from '../util/SliceStream';
import { toLiteral } from '../util/TermUtil';
import { POSIX, SOLID_HTTP, XSD } from '../util/Vocabularies';
import type { Conditions } from './conditions/Conditions';
import { PassthroughStore } from './PassthroughStore';
import type { ResourceStore } from './ResourceStore';
/**
* Resource store that slices the data stream if there are range preferences.
* Only works for `bytes` range preferences on binary data streams.
* Does not support multipart range requests.
*
* If the slice happens, unit/start/end values will be written to the metadata to indicate such.
* The values are dependent on the preferences we got as an input,
* as we don't know the actual size of the data stream.
*/
export class BinarySliceResourceStore<T extends ResourceStore = ResourceStore> extends PassthroughStore<T> {
protected readonly logger = getLoggerFor(this);
public async getRepresentation(identifier: ResourceIdentifier, preferences: RepresentationPreferences,
conditions?: Conditions): Promise<Representation> {
const result = await this.source.getRepresentation(identifier, preferences, conditions);
if (!preferences.range || preferences.range.unit !== 'bytes' || preferences.range.parts.length === 0) {
return result;
}
if (result.metadata.has(SOLID_HTTP.unit)) {
this.logger.debug('Not slicing stream that has already been sliced.');
return result;
}
if (!result.binary) {
throw new InternalServerError('Trying to slice a non-binary stream.');
}
if (preferences.range.parts.length > 1) {
throw new RangeNotSatisfiedHttpError('Multipart range requests are not supported.');
}
const [{ start, end }] = preferences.range.parts;
result.metadata.set(SOLID_HTTP.terms.unit, preferences.range.unit);
result.metadata.set(SOLID_HTTP.terms.start, toLiteral(start, XSD.terms.integer));
if (typeof end === 'number') {
result.metadata.set(SOLID_HTTP.terms.end, toLiteral(end, XSD.terms.integer));
}
try {
const size = termToInt(result.metadata.get(POSIX.terms.size));
// The reason we don't determine the object mode based on the object mode of the parent stream
// is that `guardedStreamFrom` does not create object streams when inputting streams/buffers.
// Something to potentially update in the future.
result.data = guardStream(new SliceStream(result.data, { start, end, size, objectMode: false }));
} catch (error: unknown) {
// Creating the slice stream can throw an error if some of the parameters are unacceptable.
// Need to make sure the stream is closed in that case.
result.data.destroy();
throw error;
}
return result;
}
}

View File

@@ -32,6 +32,9 @@ export interface DataAccessor {
/**
* Returns the metadata corresponding to the identifier.
* If possible, it is suggested to add a `posix:size` triple to the metadata indicating the binary size.
* This is necessary for range requests.
*
* @param identifier - Identifier for which the metadata is requested.
*/
getMetadata: (identifier: ResourceIdentifier) => Promise<RepresentationMetadata>;

View File

@@ -8,6 +8,8 @@ import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError';
import type { Guarded } from '../../util/GuardedStream';
import type { IdentifierStrategy } from '../../util/identifiers/IdentifierStrategy';
import { guardedStreamFrom } from '../../util/StreamUtil';
import { POSIX } from '../../util/Vocabularies';
import { isInternalContentType } from '../conversion/ConversionUtil';
import type { DataAccessor } from './DataAccessor';
interface DataEntry {
@@ -59,9 +61,17 @@ export class InMemoryDataAccessor implements DataAccessor, SingleThreaded {
public async writeDocument(identifier: ResourceIdentifier, data: Guarded<Readable>, metadata: RepresentationMetadata):
Promise<void> {
const parent = this.getParentEntry(identifier);
// Drain original stream and create copy
const dataArray = await arrayifyStream(data);
// Only add the size for binary streams, which are all streams that do not have an internal type.
if (metadata.contentType && !isInternalContentType(metadata.contentType)) {
const size = dataArray.reduce<number>((total, chunk: Buffer): number => total + chunk.length, 0);
metadata.set(POSIX.terms.size, `${size}`);
}
parent.entries[identifier.path] = {
// Drain original stream and create copy
data: await arrayifyStream(data),
data: dataArray,
metadata,
};
}

View File

@@ -5,6 +5,7 @@ import type { ValuePreferences } from '../../http/representation/RepresentationP
import { getLoggerFor } from '../../logging/LogUtil';
import { BadRequestHttpError } from '../../util/errors/BadRequestHttpError';
import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError';
import { POSIX } from '../../util/Vocabularies';
import { cleanPreferences, getBestPreference, getTypeWeight, preferencesToString } from './ConversionUtil';
import type { RepresentationConverterArgs } from './RepresentationConverter';
import { RepresentationConverter } from './RepresentationConverter';
@@ -100,6 +101,13 @@ export class ChainedConverter extends RepresentationConverter {
args.preferences = { type: { [outTypes[i]]: 1 }};
args.representation = await match.converters[i].handle(args);
}
// For now, we assume any kind of conversion invalidates the stored byte length.
// In the future, we could let converters handle this individually, as some might know the size of the result.
if (match.converters.length > 0) {
args.representation.metadata.removeAll(POSIX.terms.size);
}
return args.representation;
}

View File

@@ -15,6 +15,14 @@ export class Base64EncodingStorage<T> extends PassthroughKeyValueStorage<T> {
}
protected toOriginalKey(key: string): string {
return Buffer.from(key, 'base64').toString('utf-8');
// While the main part of a base64 encoded string is same from any changes from encoding or decoding URL parts,
// the `=` symbol that is used for padding is not.
// This can cause incorrect results when calling these function,
// where the original path contains `YXBwbGU%3D` instead of `YXBwbGU=`.
// This does not create any issues when the source store does not encode the string, so is safe to always call.
// For consistency, we might want to also always encode when creating the path in `keyToPath()`,
// but that would potentially break existing implementations that do not do encoding,
// and is not really necessary to solve any issues.
return Buffer.from(decodeURIComponent(key), 'base64').toString('utf-8');
}
}

View File

@@ -24,6 +24,8 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
protected readonly rootFilepath: string;
// Extension to use as a fallback when the media type is not supported (could be made configurable).
protected readonly unknownMediaTypeExtension = 'unknown';
// Path suffix for metadata
private readonly metadataSuffix = '.meta';
public constructor(base: string, rootFilepath: string) {
this.baseRequestURI = trimTrailingSlashes(base);
@@ -44,7 +46,7 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
Promise<ResourceLink> {
let path = this.getRelativePath(identifier);
if (isMetadata) {
path += '.meta';
path += this.metadataSuffix;
}
this.validateRelativePath(path, identifier);
@@ -125,7 +127,7 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
}
const isMetadata = this.isMetadataPath(filePath);
if (isMetadata) {
url = url.slice(0, -'.meta'.length);
url = url.slice(0, -this.metadataSuffix.length);
}
return { identifier: { path: url }, filePath, contentType, isMetadata };
}
@@ -213,6 +215,6 @@ export class BaseFileIdentifierMapper implements FileIdentifierMapper {
* Checks if the given path is a metadata path.
*/
protected isMetadataPath(path: string): boolean {
return path.endsWith('.meta');
return path.endsWith(this.metadataSuffix);
}
}

View File

@@ -64,8 +64,8 @@ export class FixedContentTypeMapper extends BaseFileIdentifierMapper {
}
protected async getDocumentUrl(relative: string): Promise<string> {
// Handle path suffix
if (this.pathSuffix) {
// Handle path suffix, but ignore metadata files
if (this.pathSuffix && !this.isMetadataPath(relative)) {
if (relative.endsWith(this.pathSuffix)) {
relative = relative.slice(0, -this.pathSuffix.length);
} else {

View File

@@ -3,7 +3,7 @@ import type { NamedNode } from '@rdfjs/types';
import arrayifyStream from 'arrayify-stream';
import type { ParserOptions } from 'n3';
import { StreamParser, StreamWriter } from 'n3';
import type { Quad } from 'rdf-js';
import type { Quad, Term } from 'rdf-js';
import type { Guarded } from './GuardedStream';
import { guardedStreamFrom, pipeSafely } from './StreamUtil';
import { toNamedTerm } from './TermUtil';
@@ -45,6 +45,18 @@ export function uniqueQuads(quads: Quad[]): Quad[] {
}, []);
}
/**
* Converts a term to a number. Returns undefined if the term was undefined.
*
* @param term - Term to parse.
* @param radix - Radix to use when parsing. Default is 10.
*/
export function termToInt(term?: Term, radix = 10): number | undefined {
if (term) {
return Number.parseInt(term.value, radix);
}
}
/**
* Represents a triple pattern to be used as a filter.
*/

107
src/util/SliceStream.ts Normal file
View File

@@ -0,0 +1,107 @@
import type { Readable, TransformCallback, TransformOptions } from 'stream';
import { Transform } from 'stream';
import { RangeNotSatisfiedHttpError } from './errors/RangeNotSatisfiedHttpError';
import { pipeSafely } from './StreamUtil';
export interface SliceStreamOptions extends TransformOptions {
start: number;
end?: number;
size?: number;
}
/**
* A stream that slices a part out of another stream.
* `start` and `end` are inclusive.
* If `end` is not defined it is until the end of the stream.
*
* Negative `start` values can be used to instead slice that many streams off the end of the stream.
* This requires the `size` field to be defined.
*
* Both object and non-object streams are supported.
* This needs to be explicitly specified,
* as the class makes no assumptions based on the object mode of the source stream.
*/
export class SliceStream extends Transform {
protected readonly source: Readable;
protected remainingSkip: number;
protected remainingRead: number;
public constructor(source: Readable, options: SliceStreamOptions) {
super(options);
let start = options.start;
const end = options.end ?? Number.POSITIVE_INFINITY;
if (options.start < 0) {
if (typeof options.size !== 'number') {
throw new RangeNotSatisfiedHttpError('Slicing data at the end of a stream requires a known size.');
} else {
// `start` is a negative number here so need to add
start = options.size + start;
}
}
if (start >= end) {
throw new RangeNotSatisfiedHttpError('Range start should be less than end.');
}
// Not using `end` variable as that could be infinity
if (typeof options.end === 'number' && typeof options.size === 'number' && options.end >= options.size) {
throw new RangeNotSatisfiedHttpError('Range end should be less than the total size.');
}
this.remainingSkip = start;
// End value is inclusive
this.remainingRead = end - options.start + 1;
this.source = source;
pipeSafely(source, this);
}
// eslint-disable-next-line @typescript-eslint/naming-convention
public _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void {
this.source.pause();
if (this.writableObjectMode) {
this.objectSlice(chunk);
} else {
this.binarySlice(chunk);
}
// eslint-disable-next-line callback-return
callback();
this.source.resume();
}
protected binarySlice(chunk: Buffer): void {
let length = chunk.length;
if (this.remainingSkip > 0) {
chunk = chunk.slice(this.remainingSkip);
this.remainingSkip -= length - chunk.length;
length = chunk.length;
}
if (length > 0 && this.remainingSkip <= 0) {
chunk = chunk.slice(0, this.remainingRead);
this.push(chunk);
this.remainingRead -= length;
this.checkEnd();
}
}
protected objectSlice(chunk: unknown): void {
if (this.remainingSkip > 0) {
this.remainingSkip -= 1;
} else {
this.remainingRead -= 1;
this.push(chunk);
this.checkEnd();
}
}
/**
* Stop piping the source stream and close everything once the slice is finished.
*/
protected checkEnd(): void {
if (this.remainingRead <= 0) {
this.source.unpipe();
this.end();
this.source.destroy();
}
}
}

View File

@@ -266,8 +266,12 @@ export const SOLID_ERROR_TERM = createVocabulary('urn:npm:solid:community-server
);
export const SOLID_HTTP = createVocabulary('urn:npm:solid:community-server:http:',
// Unit, start, and end are used for range headers
'end',
'location',
'start',
'slug',
'unit',
);
export const SOLID_META = createVocabulary('urn:npm:solid:community-server:meta:',

View File

@@ -0,0 +1,19 @@
import type { HttpErrorOptions } from './HttpError';
import { generateHttpErrorClass } from './HttpError';
// eslint-disable-next-line @typescript-eslint/naming-convention
const BaseHttpError = generateHttpErrorClass(416, 'RangeNotSatisfiedHttpError');
/**
* An error thrown when the requested range is not supported.
*/
export class RangeNotSatisfiedHttpError extends BaseHttpError {
/**
* Default message is 'The requested range is not supported.'.
* @param message - Optional, more specific, message.
* @param options - Optional error options.
*/
public constructor(message?: string, options?: HttpErrorOptions) {
super(message ?? 'The requested range is not supported.', options);
}
}