diff --git a/src/storage/keyvalue/JsonResourceStorage.ts b/src/storage/keyvalue/JsonResourceStorage.ts index 21c1cf809..f092fca91 100644 --- a/src/storage/keyvalue/JsonResourceStorage.ts +++ b/src/storage/keyvalue/JsonResourceStorage.ts @@ -1,13 +1,19 @@ +import { createHash } from 'crypto'; +import { parse } from 'path'; import { BasicRepresentation } from '../../http/representation/BasicRepresentation'; import type { Representation } from '../../http/representation/Representation'; import type { ResourceIdentifier } from '../../http/representation/ResourceIdentifier'; import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError'; -import { ensureLeadingSlash, ensureTrailingSlash, isContainerIdentifier, joinUrl } from '../../util/PathUtil'; +import { ensureLeadingSlash, ensureTrailingSlash, isContainerIdentifier, joinUrl, + joinFilePath } from '../../util/PathUtil'; import { readableToString } from '../../util/StreamUtil'; import { LDP } from '../../util/Vocabularies'; import type { ResourceStore } from '../ResourceStore'; import type { KeyValueStorage } from './KeyValueStorage'; +// Maximum allowed length for the keys, longer keys will be hashed. +const KEY_LENGTH_LIMIT = 255; + /** * A {@link KeyValueStorage} for JSON-like objects using a {@link ResourceStore} as backend. * @@ -114,6 +120,15 @@ export class JsonResourceStorage implements KeyValueStorage { * Converts a key into an identifier for internal storage. */ private keyToIdentifier(key: string): ResourceIdentifier { + // Parse the key as a file path + const parsedPath = parse(key); + // Hash long filenames to prevent issues with the underlying storage. + // E.g. a UNIX a file name cannot exceed 255 bytes. + // This is a temporary fix for https://github.com/CommunitySolidServer/CommunitySolidServer/issues/1013, + // until we have a solution for data migration. + if (parsedPath.base.length > KEY_LENGTH_LIMIT) { + key = joinFilePath(parsedPath.dir, this.applyHash(parsedPath.base)); + } return { path: joinUrl(this.container, key) }; } @@ -127,4 +142,8 @@ export class JsonResourceStorage implements KeyValueStorage { // on the `entries` results matching a key that was sent before. return ensureLeadingSlash(identifier.path.slice(this.container.length)); } + + private applyHash(key: string): string { + return createHash('sha256').update(key).digest('hex'); + } } diff --git a/test/unit/storage/keyvalue/JsonResourceStorage.test.ts b/test/unit/storage/keyvalue/JsonResourceStorage.test.ts index 9757933b3..41100a606 100644 --- a/test/unit/storage/keyvalue/JsonResourceStorage.test.ts +++ b/test/unit/storage/keyvalue/JsonResourceStorage.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'crypto'; import { BasicRepresentation } from '../../../../src/http/representation/BasicRepresentation'; import type { Representation } from '../../../../src/http/representation/Representation'; import { RepresentationMetadata } from '../../../../src/http/representation/RepresentationMetadata'; @@ -5,7 +6,7 @@ import type { ResourceIdentifier } from '../../../../src/http/representation/Res import { JsonResourceStorage } from '../../../../src/storage/keyvalue/JsonResourceStorage'; import type { ResourceStore } from '../../../../src/storage/ResourceStore'; import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError'; -import { isContainerIdentifier } from '../../../../src/util/PathUtil'; +import { isContainerIdentifier, joinUrl } from '../../../../src/util/PathUtil'; import { readableToString } from '../../../../src/util/StreamUtil'; import { LDP } from '../../../../src/util/Vocabularies'; @@ -123,7 +124,23 @@ describe('A JsonResourceStorage', (): void => { ]); }); - it('can handle resources being deleted while iterating in the entries call.', async(): Promise => { - // - }); + it('converts keys that would result in too large filenames into an identifier that uses a hash.', + async(): Promise => { + const longFileName = `${'sometext'.repeat(32)}.json`; + const b64LongFileName = Buffer.from(longFileName).toString('base64'); + const longKey = `/container/${b64LongFileName}`; + const longKeyId = joinUrl(subContainerIdentifier, createHash('sha256').update(b64LongFileName).digest('hex')); + + await storage.set(longKey, 'data'); + // Check if a hash of the key has been used for the filename part of the key. + expect(data.has(longKeyId)).toBeTruthy(); + + data.clear(); + + // Check that normal keys stay unaffected + const normalKey = '/container/test'; + const normalKeyId = joinUrl(containerIdentifier, normalKey); + await storage.set(normalKey, 'data'); + expect(data.has(normalKeyId)).toBeTruthy(); + }); });