diff --git a/.componentsignore b/.componentsignore index 74a204187..5bd08ec1f 100644 --- a/.componentsignore +++ b/.componentsignore @@ -18,6 +18,7 @@ "HashMap", "HttpErrorOptions", "HttpResponse", + "IndexTypeCollection", "IdentifierMap", "IdentifierSetMultiMap", "NodeJS.Dict", @@ -34,6 +35,7 @@ "ValuePreferencesArg", "VariableBindings", "UnionHandler", + "VirtualObject", "WinstonLogger", "WrappedSetMultiMap", "YargsOptions" diff --git a/src/index.ts b/src/index.ts index 248fb328d..3e9fde433 100644 --- a/src/index.ts +++ b/src/index.ts @@ -402,12 +402,14 @@ export * from './storage/keyvalue/Base64EncodingStorage'; export * from './storage/keyvalue/ContainerPathStorage'; export * from './storage/keyvalue/ExpiringStorage'; export * from './storage/keyvalue/HashEncodingStorage'; +export * from './storage/keyvalue/IndexedStorage'; export * from './storage/keyvalue/JsonFileStorage'; export * from './storage/keyvalue/JsonResourceStorage'; export * from './storage/keyvalue/KeyValueStorage'; export * from './storage/keyvalue/MemoryMapStorage'; export * from './storage/keyvalue/PassthroughKeyValueStorage'; export * from './storage/keyvalue/WrappedExpiringStorage'; +export * from './storage/keyvalue/WrappedIndexedStorage'; // Storage/Mapping export * from './storage/mapping/BaseFileIdentifierMapper'; diff --git a/src/storage/keyvalue/IndexedStorage.ts b/src/storage/keyvalue/IndexedStorage.ts new file mode 100644 index 000000000..e4edf1062 --- /dev/null +++ b/src/storage/keyvalue/IndexedStorage.ts @@ -0,0 +1,201 @@ +/** + * The key that needs to be present in all output results of {@link IndexedStorage}. + */ +export const INDEX_ID_KEY = 'id'; + +/** + * Used to define the value of a key in a type entry of a {@link IndexedStorage}. + * Valid values are `"string"`, `"boolean"`, `"number"` and `"id:TYPE"`, + * with TYPE being one of the types in the definition. + * In the latter case this means that key points to an identifier of the specified type. + * A `?` can be appended to the type to indicate this key is optional. + */ +export type ValueTypeDescription = + `${('string' | 'boolean' | 'number' | (TType extends string ? `${typeof INDEX_ID_KEY}:${TType}` : never))}${ + '?' | ''}`; + +/** + * Converts a {@link ValueTypeDescription} to the type it should be interpreted as. + */ +export type ValueType = + (T extends 'boolean' | 'boolean?' ? boolean : T extends 'number' | 'number?' ? number : string) | + (T extends `${string}?` ? undefined : never); + +/** + * Used to filter on optional keys in a {@link IndexedStorage} definition. + */ +export type OptionalKey = {[K in keyof T ]: T[K] extends `${string}?` ? K : never }[keyof T]; + +/** + * Converts a {@link IndexedStorage} definition of a specific type + * to the typing an object would have that is returned as an output on function calls. + */ +export type TypeObject> = { + -readonly [K in Exclude>]: ValueType; +} & { + -readonly [K in keyof TDesc]?: ValueType; +} & { [INDEX_ID_KEY]: string }; + +/** + * Input expected for `create()` call in {@link IndexedStorage}. + * This is the same as {@link TypeObject} but without the index key. + */ +export type CreateTypeObject> = Omit, typeof INDEX_ID_KEY>; + +/** + * Key of an object that is also a string. + */ +export type StringKey = keyof T & string; + +/** + * The description of a single type in an {@link IndexedStorage}. + */ +export type IndexTypeDescription = Record>; + +/** + * The full description of all the types of an {@link IndexedStorage}. + */ +export type IndexTypeCollection = Record>; + +// This is necessary to prevent infinite recursion in types +type Prev = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...0[]]; + +/** + * Object that represents a valid query starting from a specific type on an {@link IndexedStorage}. + * The keys of the object need to be one or more keys from the starting type, + * with the values being corresponding valid values of an object of that type. + * If the value definition of a key is one that contains the identifier of a different type, + * the value in the query can also be a nested object that has the same IndexedQuery requirements for that type. + * This can be done recursively. + * + * E.g., if the storage has the following definition: + *```ts + * { + * account: {}, + * pod: { baseUrl: 'string', account: 'id:account' }, + * pod: { owner: 'string', pod: 'id:pod' }, + * } + *``` + * A valid query on the `pod` type could be `{ pod: '123456' }`, + * but also `{ pod: { baseUrl: 'http://example.com/pod/', account: { id: '789' }}}`. + */ +export type IndexedQuery, TType extends keyof T, TDepth extends number = 10> = + [TDepth] extends [never] ? never : + {[K in keyof T[TType] | typeof INDEX_ID_KEY]?: + ValueType | + (T[TType][K] extends `${typeof INDEX_ID_KEY}:${infer U}` ? IndexedQuery : never) + }; + +/* eslint-disable @typescript-eslint/method-signature-style */ +/** + * A storage solution that allows for more complex queries than a key/value storage + * and allows setting indexes on specific keys. + */ +export interface IndexedStorage> { + /** + * Informs the storage of the definition of a specific type. + * A definition is a key/value object with the values being a valid {@link ValueTypeDescription}. + * Generally, this call needs to happen for every type of this storage, + * and before any calls are made to interact with the data. + * + * @param type - The type to define. + * @param description - A description of the values stored in objects of that type. + */ + defineType>(type: TType, description: T[TType]): Promise; + + /** + * Creates an index on a key of the given type, to allow for better queries involving those keys. + * Similar to {@link IndexedStorage.defineType} these calls need to happen first. + * + * @param type - The type to create an index on. + * @param key - The key of that type to create an index on. + */ + createIndex>(type: TType, key: StringKey): Promise; + + /** + * Creates an object of the given type. + * The storage will generate an identifier for the newly created object. + * + * @param type - The type to create. + * @param value - The value to set for the created object. + * + * @returns A representation of the newly created object, including its new identifier. + */ + create>(type: TType, value: CreateTypeObject): Promise>; + + /** + * Returns `true` if the object of the given type with the given identifier exists. + * + * @param type - The type of object to get. + * @param id - The identifier of that object. + * + * @returns Whether this object exists. + */ + has>(type: TType, id: string): Promise; + + /** + * Returns the object of the given type with the given identifier. + * + * @param type - The type of object to get. + * @param id - The identifier of that object. + * + * @returns A representation of the object, or `undefined` if there is no object of that type with that identifier. + */ + get>(type: TType, id: string): Promise | undefined>; + + /** + * Finds all objects matching a specific {@link IndexedQuery}. + * + * @param type - The type of objects to find. + * @param query - The query to execute. + * + * @returns A list of objects matching the query. + */ + find>(type: TType, query: IndexedQuery): Promise<(TypeObject)[]>; + + /** + * Similar to {@link IndexedStorage.find}, but only returns the identifiers of the found objects. + * + * @param type - The type of objects to find. + * @param query - The query to execute. + * + * @returns A list of identifiers of the matching objects. + */ + findIds>(type: TType, query: IndexedQuery): Promise; + + /** + * Sets the value of a specific object. + * The identifier in the object is used to identify the object. + * + * @param type - The type of the object to set. + * @param value - The new value for the object. + */ + set>(type: TType, value: TypeObject): Promise; + + /** + * Sets the value of one specific field in an object. + * + * @param type - The type of the object to update. + * @param id - The identifier of the object to update. + * @param key - The key to update. + * @param value - The new value for the given key. + */ + setField, TKey extends StringKey>( + type: TType, id: string, key: TKey, value: ValueType): Promise; + + /** + * Deletes the given object. + * This will also delete all objects that reference that object if the corresponding key is not optional. + * + * @param type - The type of the object to delete. + * @param id - The identifier of the object. + */ + delete>(type: TType, id: string): Promise; + + /** + * Returns an iterator over all objects of the given type. + * + * @param type - The type to iterate over. + */ + entries>(type: TType): AsyncIterableIterator>; +} diff --git a/src/storage/keyvalue/WrappedIndexedStorage.ts b/src/storage/keyvalue/WrappedIndexedStorage.ts new file mode 100644 index 000000000..e24667a71 --- /dev/null +++ b/src/storage/keyvalue/WrappedIndexedStorage.ts @@ -0,0 +1,645 @@ +import { v4 } from 'uuid'; +import { getLoggerFor } from '../../logging/LogUtil'; +import { InternalServerError } from '../../util/errors/InternalServerError'; +import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError'; +import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError'; +import { INDEX_ID_KEY } from './IndexedStorage'; +import type { StringKey, + IndexedStorage, + TypeObject, + IndexedQuery, + IndexTypeCollection, + CreateTypeObject, ValueType } from './IndexedStorage'; +import type { KeyValueStorage } from './KeyValueStorage'; + +/** + * Key used to link to child objects in a {@link WrappedIndexedStorage}. + */ +export type VirtualKey = TChild extends string ? `**${TChild}**` : never; + +/** + * Object stored in the wrapped {@link KeyValueStorage} in a {@link WrappedIndexedStorage}. + */ +export type VirtualObject = { + [key: VirtualKey]: Record; + [key: string]: unknown; + [INDEX_ID_KEY]: string; +}; + +/** + * A parent/child relation description in a {@link WrappedIndexedStorage}. + */ +export type IndexRelation = { + parent: { key: VirtualKey; type: StringKey }; + child: { key: string; type: StringKey }; +}; + +/** + * An {@link IndexedStorage} that makes use of 2 {@link KeyValueStorage}s to implement the interface. + * Due to being limited by key/value storages, there are some restrictions on the allowed type definitions: + * + * * There needs to be exactly 1 type with no references to other types. + * * All other types need to have exactly 1 reference to another type. + * * Types can't reference each other to create a cycle of references. + * + * All of the above to create a tree-like structure of references. + * Such a tree is then stored in one of the storages. + * The other storage is used to store all indexes that are used to find the correct tree object when solving queries. + */ +export class WrappedIndexedStorage> implements IndexedStorage { + protected readonly logger = getLoggerFor(this); + + private readonly valueStorage: KeyValueStorage; + private readonly indexStorage: KeyValueStorage; + + /** + * For every type, the keys on which an index tracks the values and which root object they are contained in. + * All types for which a `defineType` call was made will have a key in this object. + * For all types that are not the root, there will always be an index on their ID value. + */ + private readonly indexes: {[K in StringKey]?: Set> }; + + /** + * Keeps track of type validation. + * If true the defined types create a valid structure that can be used. + */ + private validDefinition = false; + + /** + * The variable in which the root type is stored. + * A separate getter is used to always return the value + * so the potential `undefined` does not have to be taken into account. + */ + private rootTypeVar: StringKey | undefined; + + /** + * All parent/child relations between all types in the storage, + * including the keys in both types that are used to reference each other. + */ + private readonly relations: IndexRelation[]; + + public constructor(valueStorage: KeyValueStorage, + indexStorage: KeyValueStorage) { + this.valueStorage = valueStorage; + this.indexStorage = indexStorage; + this.indexes = {}; + this.relations = []; + } + + public async defineType>(type: TType, description: T[TType]): Promise { + if (this.rootTypeVar) { + this.logger.error(`Trying to define new type "${type}" after types were already validated.`); + throw new InternalServerError(`Trying to define new type "${type}" after types were already validated.`); + } + + this.validDefinition = false; + let hasParentKey = false; + for (const [ key, desc ] of Object.entries(description)) { + if (desc.startsWith(`${INDEX_ID_KEY}:`)) { + if (hasParentKey) { + this.logger.error(`Type definition of ${type} has multiple references, only 1 is allowed.`); + throw new InternalServerError(`Type definition of ${type} has multiple references, only 1 is allowed.`); + } + if (desc.endsWith('?')) { + this.logger.error(`Type definition of ${type} has optional references, which is not allowed.`); + throw new InternalServerError(`Type definition of ${type} has optional references, which is not allowed.`); + } + hasParentKey = true; + this.relations.push({ parent: { type: desc.slice(3), key: `**${type}**` }, child: { type, key }}); + } + } + this.indexes[type] = new Set([ INDEX_ID_KEY ]); + } + + public async createIndex>(type: TType, key: StringKey): Promise { + // An index on the key targeting the parent is the same as having an index on the identifier of that parent. + // Such an index gets created automatically when the type is defined so this can now be ignored. + if (key === this.getParentRelation(type)?.child.key) { + return; + } + const typeIndexes = this.indexes[type]; + if (!typeIndexes) { + this.logger.error(`Trying to create index on key ${key} of undefined type ${type}`); + throw new InternalServerError(`Trying to create index on key ${key} of undefined type ${type}`); + } + typeIndexes.add(key); + } + + public async has>(type: TType, id: string): Promise { + this.validateDefinition(type); + if (type === this.rootType) { + return this.valueStorage.has(id); + } + const result = await this.find(type, { [INDEX_ID_KEY]: id } as IndexedQuery); + return result.length > 0; + } + + public async get>(type: TType, id: string): Promise | undefined> { + this.validateDefinition(type); + const result = await this.find(type, { [INDEX_ID_KEY]: id } as IndexedQuery); + if (result.length === 0) { + return; + } + return result[0]; + } + + public async create>(type: TType, value: CreateTypeObject): + Promise> { + this.validateDefinition(type); + const id = v4(); + const newObj = { ...value, [INDEX_ID_KEY]: id } as unknown as VirtualObject; + // Add the virtual keys + for (const relation of this.getChildRelations(type)) { + newObj[relation.parent.key] = {}; + } + + const relation = this.getParentRelation(type); + + // No parent relation implies that this is the root type + if (!relation) { + await this.valueStorage.set(id, newObj); + await this.updateTypeIndex(type, id, undefined, newObj); + return this.toTypeObject(type, newObj); + } + + // We know this will be a string due to the typing requirements and how the relations object is built + const parentId = newObj[relation.child.key] as string; + const root = await this.getRoot(relation.parent.type, parentId); + if (!root) { + throw new NotFoundHttpError(`Unknown object of type ${relation.parent.type} with ID ${parentId}`); + } + const parentObj = relation.parent.type === this.rootType ? + root : + this.getContainingRecord(root, relation.parent.type, parentId)[parentId]; + parentObj[relation.parent.key][id] = newObj; + + await this.valueStorage.set(root[INDEX_ID_KEY], root); + await this.updateTypeIndex(type, root[INDEX_ID_KEY], undefined, newObj); + + return this.toTypeObject(type, newObj); + } + + public async set>(type: TType, value: TypeObject): Promise { + this.validateDefinition(type); + return this.updateValue(type, value, true); + } + + public async setField, TKey extends StringKey>( + type: TType, id: string, key: TKey, value: ValueType, + ): Promise { + this.validateDefinition(type); + return this.updateValue(type, + { [INDEX_ID_KEY]: id, [key]: value } as Partial> & { [INDEX_ID_KEY]: string }, + false); + } + + public async delete>(type: TType, id: string): Promise { + this.validateDefinition(type); + const root = await this.getRoot(type, id); + if (!root) { + return; + } + + let oldObj: VirtualObject; + if (type === this.rootType) { + oldObj = root; + await this.valueStorage.delete(id); + } else { + const objs = this.getContainingRecord(root, type, id); + oldObj = objs[id]; + delete objs[id]; + await this.valueStorage.set(root[INDEX_ID_KEY], root); + } + + // Updating index of removed type and all children as those are also gone + await this.updateDeepTypeIndex(type, root[INDEX_ID_KEY], oldObj); + } + + public async findIds>(type: TType, query: IndexedQuery): Promise { + this.validateDefinition(type); + if (type === this.rootType) { + // Root IDs are the only ones we can get more efficiently. + // For all other types we have to find the full objects anyway. + const indexedRoots = await this.findIndexedRoots(type, query); + if (!Array.isArray(indexedRoots)) { + this.logger.error(`Attempting to execute query without index: ${JSON.stringify(query)}`); + throw new InternalServerError(`Attempting to execute query without index: ${JSON.stringify(query)}`); + } + return indexedRoots; + } + return (await this.solveQuery(type, query)).map((result): string => result.id); + } + + public async find>(type: TType, query: IndexedQuery): + Promise<(TypeObject)[]> { + this.validateDefinition(type); + return (await this.solveQuery(type, query)).map((result): TypeObject => this.toTypeObject(type, result)); + } + + public async* entries>(type: TType): AsyncIterableIterator> { + this.validateDefinition(type); + const path = this.getPathToType(type); + for await (const [ , root ] of this.valueStorage.entries()) { + const children = this.getChildObjects(root, path); + yield* children.map((child): TypeObject => this.toTypeObject(type, child)); + } + } + + // --------------------------------- OUTPUT HELPERS --------------------------------- + + /** + * Converts a {@link VirtualObject} into a {@link TypeObject}. + * To be used when outputting results. + */ + protected toTypeObject>(type: TType, obj: VirtualObject): TypeObject { + const result = { ...obj }; + for (const relation of this.getChildRelations(type)) { + delete result[relation.parent.key]; + } + return result as unknown as TypeObject; + } + + // --------------------------------- ROOT HELPERS --------------------------------- + + /** + * The root type for this storage. + * Use this instead of rootTypeVar to prevent having to check for `undefined`. + * This value will always be defined if the type definitions have been validated. + */ + protected get rootType(): string { + return this.rootTypeVar!; + } + + /** + * Finds the root object that contains the requested type/id combination. + */ + protected async getRoot>(type: TType, id: string): Promise { + let rootId: string; + if (type === this.rootType) { + rootId = id; + } else { + // We know there always is an index on the identifier key + const indexKey = this.getIndexKey(type, INDEX_ID_KEY, id); + const indexResult = await this.indexStorage.get(indexKey); + if (!indexResult || indexResult.length !== 1) { + return; + } + rootId = indexResult[0]; + } + return this.valueStorage.get(rootId); + } + + // --------------------------------- PATH HELPERS --------------------------------- + + /** + * Returns the sequence of virtual keys that need to be accessed to reach the given type, starting from the root. + */ + protected getPathToType(type: string): VirtualKey[] { + const result: VirtualKey[] = []; + let relation = this.getParentRelation(type); + while (relation) { + result.unshift(relation.parent.key); + relation = this.getParentRelation(relation.parent.type); + } + return result; + } + + /** + * Finds all records that can be found in the given object by following the given path of virtual keys. + */ + protected getPathRecords(obj: VirtualObject, path: VirtualKey[]): Record[] { + const record = obj[path[0]]; + if (path.length === 1) { + return [ record ]; + } + const subPath = path.slice(1); + return Object.values(record) + .flatMap((child): Record[] => this.getPathRecords(child, subPath)); + } + + /** + * Finds all objects in the provided object that can be found by following the provided path of virtual keys. + */ + protected getChildObjects(obj: VirtualObject, path: VirtualKey[]): VirtualObject[] { + if (path.length === 0) { + return [ obj ]; + } + return this.getPathRecords(obj, path).flatMap((record): VirtualObject[] => Object.values(record)); + } + + /** + * Finds the record in the given object that contains the given type/id combination. + * This function assumes it was already verified through an index that this object contains the given combination. + */ + protected getContainingRecord>(rootObj: VirtualObject, type: TType, id: string): + Record { + const path = this.getPathToType(type); + const records = this.getPathRecords(rootObj, path); + const match = records.find((record): boolean => Boolean(record[id])); + if (!match) { + this.logger.error(`Could not find ${type} ${id} in ${this.rootType} ${rootObj.id}`); + throw new InternalServerError(`Could not find ${type} ${id} in ${this.rootType} ${rootObj.id}`); + } + return match; + } + + // --------------------------------- UPDATE VALUE HELPERS --------------------------------- + + /** + * Replaces an object of the given type. + * The identifier in the value is used to determine which object. + */ + protected updateValue>(type: TType, value: TypeObject, replace: true): + Promise; + + /** + * Replaces part of an object of the given type with the given partial value. + * The identifier in the value is used to determine which object. + */ + protected updateValue>(type: TType, + partial: Partial> & { [INDEX_ID_KEY]: string }, replace: false): Promise; + + protected async updateValue>(type: TType, + partial: Partial> & { [INDEX_ID_KEY]: string }, replace: boolean): Promise { + const id = partial[INDEX_ID_KEY]; + let root = await this.getRoot(type, id); + if (!root) { + throw new NotFoundHttpError(`Unknown object of type ${type} with ID ${id}`); + } + + let oldObj: VirtualObject; + let newObj: VirtualObject; + const relation = this.getParentRelation(type); + if (!relation) { + oldObj = root; + newObj = (replace ? { ...partial } : { ...oldObj, ...partial }) as VirtualObject; + root = newObj; + } else { + const objs = this.getContainingRecord(root, type, id); + if (partial[relation.child.key] && objs[id][relation.child.key] !== partial[relation.child.key]) { + this.logger.error(`Trying to modify reference key ${objs[id][relation.child.key]} on "${type}" ${id}`); + throw new NotImplementedHttpError('Changing reference keys of existing objects is not supported.'); + } + oldObj = objs[id]; + newObj = (replace ? { ...partial } : { ...oldObj, ...partial }) as VirtualObject; + objs[id] = newObj; + } + + // Copy over the child relations + for (const childRelation of this.getChildRelations(type)) { + newObj[childRelation.parent.key] = oldObj[childRelation.parent.key]; + } + await this.valueStorage.set(root[INDEX_ID_KEY], root); + await this.updateTypeIndex(type, root[INDEX_ID_KEY], oldObj, newObj); + } + + // --------------------------------- TYPE HELPERS --------------------------------- + + /** + * Returns all relations where the given type is the parent. + */ + protected getChildRelations>(type: TType): IndexRelation[] { + return this.relations.filter((relation): boolean => relation.parent.type === type); + } + + /** + * Returns the relation where the given type is the child. + * Will return `undefined` for the root type as that one doesn't have a parent. + */ + protected getParentRelation>(type: TType): IndexRelation | undefined { + return this.relations.find((relation): boolean => relation.child.type === type); + } + + /** + * Makes sure the defined types fulfill all the requirements necessary for types on this storage. + * Will throw an error if this is not the case. + * This should be called before doing any data interactions. + * Stores success in a variable so future calls are instantaneous. + */ + protected validateDefinition(type: string): void { + // We can't know if all types are already defined. + // This prevents issues even if the other types together are valid. + if (!this.indexes[type]) { + const msg = `Type "${type}" was not defined. The defineType functions needs to be called before accessing data.`; + this.logger.error(msg); + throw new InternalServerError(msg); + } + + if (this.validDefinition) { + return; + } + + const rootTypes = new Set>(); + // `this.indexes` will contain a key for each type as we always have an index on the identifier of a type + for (let indexType of Object.keys(this.indexes)) { + const foundTypes = new Set([ indexType ]); + // Find path to root from this type, thereby ensuring that there is no cycle. + let relation = this.getParentRelation(indexType); + while (relation) { + indexType = relation.parent.type; + if (foundTypes.has(indexType)) { + const msg = `The following types cyclically reference each other: ${[ ...foundTypes ].join(', ')}`; + this.logger.error(msg); + throw new InternalServerError(msg); + } + foundTypes.add(indexType); + relation = this.getParentRelation(indexType); + } + rootTypes.add(indexType); + } + + if (rootTypes.size > 1) { + const msg = `Only one type definition with no references is allowed. Found ${[ ...rootTypes ].join(', ')}`; + this.logger.error(msg); + throw new InternalServerError(msg); + } + + this.rootTypeVar = [ ...rootTypes.values() ][0]; + + // Remove the root index as we don't need it, and it can cause confusion when resolving queries + this.indexes[this.rootTypeVar]?.delete(INDEX_ID_KEY); + + this.validDefinition = true; + } + + // --------------------------------- QUERY HELPERS --------------------------------- + + /** + * Finds the IDs of all root objects that contain objects of the given type matching the given query + * by making use of the indexes applicable to the keys in the query. + * This function only looks at the keys in the query with primitive values, + * object values in the query referencing parent objects are not considered. + * Similarly, only indexes are used, keys without index are also ignored. + * + * If an array of root IDs is provided as input, + * the result will be an intersection of this array and the found identifiers. + * + * If the result is an empty array, it means that there is no valid identifier matching the query, + * while an `undefined` result means there is no index matching any of the query keys, + * so a result can't be determined. + */ + protected async findIndexedRoots>(type: TType, match: IndexedQuery, + rootIds?: string[]): Promise { + if (type === this.rootType && match[INDEX_ID_KEY]) { + // If the input is the root type with a known ID in the query, + // and we have already established that it is not this ID, + // there is no result. + if (rootIds && !rootIds.includes(match[INDEX_ID_KEY] as string)) { + return []; + } + rootIds = [ match[INDEX_ID_KEY] as string ]; + } + + const indexIds: string[] = []; + for (const [ key, value ] of Object.entries(match)) { + if (this.indexes[type]?.has(key) && typeof value !== 'undefined') { + // We know value is a string (or boolean/number) since we can't have indexes on fields referencing other objects + indexIds.push(this.getIndexKey(type, key, value)); + } + } + + if (indexIds.length === 0) { + return rootIds; + } + + // Use all indexes found to find matching IDs + const indexResults = await Promise.all(indexIds.map(async(id): Promise => + await this.indexStorage.get(id) ?? [])); + if (Array.isArray(rootIds)) { + indexResults.push(rootIds); + } + + return indexResults.reduce((acc, ids): string[] => acc.filter((id): boolean => ids.includes(id))); + } + + /** + * Finds all objects of the given type matching the query. + * The `rootIds` array can be used to restrict the IDs of root objects to look at, + * which is relevant for the recursive calls the function does. + * + * Will throw an error if there is no index that can be used to solve the query. + */ + protected async solveQuery>(type: TType, query: IndexedQuery, + rootIds?: string[]): Promise { + this.logger.debug(`Executing "${type}" query ${JSON.stringify(query)}. Already found roots ${rootIds}.`); + + const indexedRoots = await this.findIndexedRoots(type, query, rootIds); + + // All objects of this type that we find through recursive calls + let objs: VirtualObject[]; + + // Either find all objects of the type from the found rootIds if the query is a leaf, + // or recursively query the parent object if it is not. + const relation = this.getParentRelation(type); + if (!relation || !query[relation.child.key]) { + // This is a leaf node of the query + if (!Array.isArray(indexedRoots)) { + this.logger.error(`Attempting to execute query without index: ${JSON.stringify(query)}`); + throw new InternalServerError(`Attempting to execute query without index: ${JSON.stringify(query)}`); + } + const pathFromRoot = this.getPathToType(type); + // All objects of this type for all root objects we have + const roots = (await Promise.all(indexedRoots.map(async(id): Promise => { + const root = await this.valueStorage.get(id); + if (!root) { + // Not throwing an error to sort of make server still work if an index is wrong. + this.logger.error( + `Data inconsistency: index contains ${this.rootType} with ID ${id}, but this object does not exist.`, + ); + } + return root; + }))).filter((root): boolean => typeof root !== 'undefined') as VirtualObject[]; + objs = roots.flatMap((root): VirtualObject[] => this.getChildObjects(root, pathFromRoot)); + } else { + const subQuery = (typeof query[relation.child.key] === 'string' ? + { [INDEX_ID_KEY]: query[relation.child.key] } : + query[relation.child.key]) as IndexedQuery; + // All objects by recursively calling this function on the parent object and extracting all children of this type + objs = (await this.solveQuery(relation.parent.type, subQuery, indexedRoots)) + .flatMap((parentObj): VirtualObject[] => Object.values(parentObj[relation.parent.key])); + } + + // For all keys that were not handled recursively: make sure that it matches the found objects + const remainingKeys = Object.keys(query).filter((key): boolean => + key !== relation?.child.key || typeof query[key] === 'string'); + return remainingKeys.reduce((acc, key): any[] => acc.filter((obj): boolean => obj[key] === query[key]), objs); + } + + // --------------------------------- INDEX HELPERS --------------------------------- + + /** + * Generate the key used to store the index in the index storage. + */ + protected getIndexKey(type: string, key: string, value: string | number): string { + return `${encodeURIComponent(type)}/${key === INDEX_ID_KEY ? '' : `${key}/`}${encodeURIComponent(`${value}`)}`; + } + + /** + * Update all indexes for an object of the given type, and all its children. + */ + protected async updateDeepTypeIndex>(type: TType, rootId: string, + oldObj: VirtualObject, newObj?: VirtualObject): Promise { + const promises: Promise[] = []; + promises.push(this.updateTypeIndex(type, rootId, oldObj, newObj)); + + for (const { parent, child } of this.getChildRelations(type)) { + const oldRecord: Record = oldObj[parent.key]; + const newRecord: Record = newObj?.[parent.key] ?? {}; + const uniqueKeys = new Set([ ...Object.keys(oldRecord), ...Object.keys(newRecord) ]); + for (const key of uniqueKeys) { + promises.push(this.updateDeepTypeIndex(child.type, rootId, oldRecord[key], newRecord[key])); + } + } + + await Promise.all(promises); + } + + /** + * Updates all indexes for an object of the given type. + */ + protected async updateTypeIndex>(type: TType, rootId: string, + oldObj?: VirtualObject, newObj?: VirtualObject): Promise { + const added: { key: string; value: string }[] = []; + const removed: { key: string; value: string }[] = []; + + for (const key of this.indexes[type]!) { + const oldValue = oldObj?.[key]; + const newValue = newObj?.[key]; + if (oldValue !== newValue) { + if (typeof oldValue !== 'undefined') { + removed.push({ key, value: oldValue as string }); + } + if (typeof newValue !== 'undefined') { + added.push({ key, value: newValue as string }); + } + } + } + + await Promise.all([ + ...added.map(({ key, value }): Promise => this.updateKeyIndex(type, key, value, rootId, true)), + ...removed.map(({ key, value }): Promise => this.updateKeyIndex(type, key, value, rootId, false)), + ]); + } + + /** + * Updates the index for a specific key of an object of the given type. + */ + protected async updateKeyIndex(type: string, key: string, value: string, rootId: string, add: boolean): + Promise { + const indexKey = this.getIndexKey(type, key, value); + const indexValues = await this.indexStorage.get(indexKey) ?? []; + this.logger.debug(`Updating index ${indexKey} by ${add ? 'adding' : 'removing'} ${rootId} from ${indexValues}`); + + if (add) { + if (!indexValues.includes(rootId)) { + indexValues.push(rootId); + } + await this.indexStorage.set(indexKey, indexValues); + } else { + const updatedValues = indexValues.filter((val): boolean => val !== rootId); + await (updatedValues.length === 0 ? + this.indexStorage.delete(indexKey) : + this.indexStorage.set(indexKey, updatedValues)); + } + } +} diff --git a/test/unit/storage/keyvalue/WrappedIndexedStorage.test.ts b/test/unit/storage/keyvalue/WrappedIndexedStorage.test.ts new file mode 100644 index 000000000..dc8e68bc4 --- /dev/null +++ b/test/unit/storage/keyvalue/WrappedIndexedStorage.test.ts @@ -0,0 +1,361 @@ +import { INDEX_ID_KEY } from '../../../../src/storage/keyvalue/IndexedStorage'; +import type { TypeObject } from '../../../../src/storage/keyvalue/IndexedStorage'; +import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage'; +import { WrappedIndexedStorage } from '../../../../src/storage/keyvalue/WrappedIndexedStorage'; +import { InternalServerError } from '../../../../src/util/errors/InternalServerError'; +import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError'; +import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplementedHttpError'; + +const dummyDescription = { + root: { required: 'number', optional: 'string?', notIndexed: 'number' }, + child: { parent: 'id:root', name: 'string', notIndexed: 'number' }, + grandchild: { parent: 'id:child', bool: 'boolean', notIndexed: 'number' }, + otherChild: { parent: 'id:root', name: 'string', notIndexed: 'number' }, +} as const; + +describe('A WrappedIndexedStorage', (): void => { + let valueMap: Map; + let valueStorage: jest.Mocked>; + let indexMap: Map; + let indexStorage: jest.Mocked>; + let storage: WrappedIndexedStorage; + + beforeEach(async(): Promise => { + valueMap = new Map(); + valueStorage = { + has: jest.fn(async(key): Promise => valueMap.has(key)), + get: jest.fn(async(key): Promise => valueMap.get(key)), + set: jest.fn(async(key, value): Promise => valueMap.set(key, value)), + delete: jest.fn(async(key): Promise => valueMap.delete(key)), + entries: jest.fn(async function* (): AsyncIterableIterator<[string, unknown]> { + yield* valueMap.entries(); + }), + }; + + indexMap = new Map(); + indexStorage = { + has: jest.fn(async(key): Promise => indexMap.has(key)), + get: jest.fn(async(key): Promise => indexMap.get(key)), + set: jest.fn(async(key, value): Promise => indexMap.set(key, value)), + delete: jest.fn(async(key): Promise => indexMap.delete(key)), + entries: jest.fn(async function* (): AsyncIterableIterator<[string, string[]]> { + yield* indexMap.entries(); + }), + }; + + storage = new WrappedIndexedStorage(valueStorage, indexStorage); + }); + + describe('that is empty', (): void => { + it('can define and initialize data.', async(): Promise => { + await expect(storage.defineType('root', dummyDescription.root)).resolves.toBeUndefined(); + await expect(storage.defineType('child', dummyDescription.child)).resolves.toBeUndefined(); + await expect(storage.defineType('grandchild', dummyDescription.grandchild)).resolves.toBeUndefined(); + await expect(storage.defineType('otherChild', dummyDescription.otherChild)).resolves.toBeUndefined(); + }); + + it('errors when defining types with multiple references.', async(): Promise => { + await expect(storage.defineType('root', { ref1: 'id:Type1', ref2: 'id:Type2' } as any)) + .rejects.toThrow(InternalServerError); + }); + + it('errors when defining types with optional references.', async(): Promise => { + await expect(storage.defineType('root', { ref: 'id:Type1?' } as any)).rejects.toThrow(InternalServerError); + }); + + it('errors trying to create an index on an undefined type.', async(): Promise => { + await expect(storage.createIndex('root', 'required')).rejects.toThrow(InternalServerError); + }); + + it('errors trying to access data before its type was defined.', async(): Promise => { + await expect(storage.has('root', '???')).rejects.toThrow(InternalServerError); + }); + + it('errors if type definitions are added after validation.', async(): Promise => { + await expect(storage.defineType('root', dummyDescription.root)).resolves.toBeUndefined(); + // Trigger data validation + await storage.has('root', '???'); + await expect(storage.defineType('root', dummyDescription.root)).rejects.toThrow(InternalServerError); + }); + + it('errors if the type definitions are cyclical.', async(): Promise => { + await expect(storage.defineType('root', { ...dummyDescription.root, invalidReference: 'id:grandchild' } as any)) + .resolves.toBeUndefined(); + await expect(storage.defineType('child', dummyDescription.child)).resolves.toBeUndefined(); + await expect(storage.defineType('grandchild', dummyDescription.grandchild)).resolves.toBeUndefined(); + + // Trigger data validation + await expect(storage.has('root', '???')).rejects.toThrow(InternalServerError); + }); + + it('errors if there are multiple root types.', async(): Promise => { + await expect(storage.defineType('root', dummyDescription.root)).resolves.toBeUndefined(); + await expect(storage.defineType('child', dummyDescription.root as any)).resolves.toBeUndefined(); + + // Trigger data validation + await expect(storage.has('root', '???')).rejects.toThrow(InternalServerError); + }); + }); + + describe('with data definitions', (): void => { + beforeEach(async(): Promise => { + await storage.defineType('root', dummyDescription.root); + await storage.defineType('child', dummyDescription.child); + await storage.defineType('grandchild', dummyDescription.grandchild); + await storage.defineType('otherChild', dummyDescription.otherChild); + }); + + it('can create new entries.', async(): Promise => { + const parent = await storage.create('root', { required: 5, notIndexed: 0 }); + expect(parent).toEqual({ id: expect.any(String), required: 5, notIndexed: 0 }); + const child = await storage.create('child', { name: 'child', parent: parent.id, notIndexed: 1 }); + expect(child).toEqual({ id: expect.any(String), name: 'child', parent: parent.id, notIndexed: 1 }); + const grandchild = await storage.create('grandchild', { bool: true, parent: child.id, notIndexed: 2 }); + expect(grandchild).toEqual({ id: expect.any(String), bool: true, parent: child.id, notIndexed: 2 }); + const otherChild = await storage.create('otherChild', { name: 'otherChild', parent: parent.id, notIndexed: 3 }); + expect(otherChild).toEqual({ id: expect.any(String), name: 'otherChild', parent: parent.id, notIndexed: 3 }); + }); + + it('errors when creating new entries with unknown references.', async(): Promise => { + await expect(storage.create('child', { name: 'child', parent: '???', notIndexed: 1 })) + .rejects.toThrow(NotFoundHttpError); + }); + + it('can create indexes.', async(): Promise => { + await expect(storage.createIndex('root', 'required')).resolves.toBeUndefined(); + await expect(storage.createIndex('root', 'optional')).resolves.toBeUndefined(); + await expect(storage.createIndex('child', 'name')).resolves.toBeUndefined(); + await expect(storage.createIndex('grandchild', 'bool')).resolves.toBeUndefined(); + await expect(storage.createIndex('otherChild', 'name')).resolves.toBeUndefined(); + + // This one does nothing + await expect(storage.createIndex('grandchild', 'parent')).resolves.toBeUndefined(); + }); + }); + + describe('with initialized data', (): void => { + let root: TypeObject; + let root2: TypeObject; + let child: TypeObject; + let child2: TypeObject; + let child3: TypeObject; + let grandchild: TypeObject; + let grandchild2: TypeObject; + let grandchild3: TypeObject; + let otherChild: TypeObject; + + beforeEach(async(): Promise => { + await storage.defineType('root', dummyDescription.root); + await storage.defineType('child', dummyDescription.child); + await storage.defineType('grandchild', dummyDescription.grandchild); + await storage.defineType('otherChild', dummyDescription.otherChild); + + await storage.createIndex('root', 'required'); + await storage.createIndex('root', 'optional'); + await storage.createIndex('child', 'name'); + await storage.createIndex('grandchild', 'bool'); + await storage.createIndex('otherChild', 'name'); + + root = await storage.create('root', { required: 5, notIndexed: 0 }); + child = await storage.create('child', { name: 'child', parent: root.id, notIndexed: 1 }); + grandchild = await storage.create('grandchild', { bool: true, parent: child.id, notIndexed: 2 }); + otherChild = await storage.create('otherChild', { name: 'otherChild', parent: root.id, notIndexed: 3 }); + + // Extra resources for query tests + root2 = await storage.create('root', { required: 5, optional: 'defined', notIndexed: 1 }); + child2 = await storage.create('child', { name: 'child2', parent: root.id, notIndexed: 1 }); + child3 = await storage.create('child', { name: 'child', parent: root2.id, notIndexed: 1 }); + grandchild2 = await storage.create('grandchild', { bool: false, parent: child.id, notIndexed: 2 }); + grandchild3 = await storage.create('grandchild', { bool: true, parent: child2.id, notIndexed: 2 }); + }); + + it('can verify existence.', async(): Promise => { + await expect(storage.has('root', root.id)).resolves.toBe(true); + await expect(storage.has('child', child.id)).resolves.toBe(true); + await expect(storage.has('grandchild', grandchild.id)).resolves.toBe(true); + await expect(storage.has('otherChild', otherChild.id)).resolves.toBe(true); + + await expect(storage.has('root', '???')).resolves.toBe(false); + await expect(storage.has('child', '???')).resolves.toBe(false); + }); + + it('can return data.', async(): Promise => { + await expect(storage.get('root', root.id)).resolves.toEqual(root); + await expect(storage.get('child', child.id)).resolves.toEqual(child); + await expect(storage.get('grandchild', grandchild.id)).resolves.toEqual(grandchild); + await expect(storage.get('otherChild', otherChild.id)).resolves.toEqual(otherChild); + }); + + it('returns undefined if there is no match.', async(): Promise => { + await expect(storage.get('root', child.id)).resolves.toBeUndefined(); + await expect(storage.get('child', root.id)).resolves.toBeUndefined(); + await expect(storage.get('grandchild', otherChild.id)).resolves.toBeUndefined(); + await expect(storage.get('otherChild', grandchild.id)).resolves.toBeUndefined(); + }); + + it('can update entries.', async(): Promise => { + await expect(storage.set('root', { [INDEX_ID_KEY]: root.id, required: -10, notIndexed: -1 })) + .resolves.toBeUndefined(); + await expect(storage.get('root', root.id)) + .resolves.toEqual({ [INDEX_ID_KEY]: root.id, required: -10, notIndexed: -1 }); + + await expect(storage.set('child', { ...child, name: 'newChild', notIndexed: -2 })).resolves.toBeUndefined(); + await expect(storage.get('child', child.id)).resolves.toEqual({ ...child, name: 'newChild', notIndexed: -2 }); + + await expect(storage.set('grandchild', { ...grandchild, bool: false, notIndexed: -3 })).resolves.toBeUndefined(); + await expect(storage.get('grandchild', grandchild.id)) + .resolves.toEqual({ ...grandchild, bool: false, notIndexed: -3 }); + + await expect(storage.set('otherChild', { ...otherChild, name: 'newOtherChild', notIndexed: -4 })) + .resolves.toBeUndefined(); + await expect(storage.get('otherChild', otherChild.id)) + .resolves.toEqual({ ...otherChild, name: 'newOtherChild', notIndexed: -4 }); + }); + + it('errors when trying to update unknown entries.', async(): Promise => { + await expect(storage.set('root', { [INDEX_ID_KEY]: '???', required: -10, notIndexed: -1 })) + .rejects.toThrow(NotFoundHttpError); + await expect(storage.set('child', { ...child, [INDEX_ID_KEY]: '???' })) + .rejects.toThrow(NotFoundHttpError); + }); + + it('errors when trying to update references.', async(): Promise => { + await expect(storage.set('child', { ...child, parent: 'somewhereElse' })) + .rejects.toThrow(NotImplementedHttpError); + }); + + it('can update specific fields.', async(): Promise => { + await expect(storage.setField('root', root.id, 'notIndexed', -1)) + .resolves.toBeUndefined(); + await expect(storage.get('root', root.id)) + .resolves.toEqual({ ...root, notIndexed: -1 }); + + await expect(storage.setField('child', child.id, 'notIndexed', -2)) + .resolves.toBeUndefined(); + await expect(storage.get('child', child.id)) + .resolves.toEqual({ ...child, notIndexed: -2 }); + + await expect(storage.setField('grandchild', grandchild.id, 'notIndexed', -2)) + .resolves.toBeUndefined(); + await expect(storage.get('grandchild', grandchild.id)) + .resolves.toEqual({ ...grandchild, notIndexed: -2 }); + + await expect(storage.setField('otherChild', otherChild.id, 'notIndexed', -3)) + .resolves.toBeUndefined(); + await expect(storage.get('otherChild', otherChild.id)) + .resolves.toEqual({ ...otherChild, notIndexed: -3 }); + }); + + it('errors when trying to update a field in unknown entries.', async(): Promise => { + await expect(storage.setField('root', '???', 'notIndexed', -1)) + .rejects.toThrow(NotFoundHttpError); + await expect(storage.setField('child', '???', 'notIndexed', -1)) + .rejects.toThrow(NotFoundHttpError); + }); + + it('errors when trying to update a reference field.', async(): Promise => { + await expect(storage.setField('child', child.id, 'parent', 'somewhereElse')) + .rejects.toThrow(NotImplementedHttpError); + }); + + it('can remove resource.', async(): Promise => { + await expect(storage.delete('otherChild', otherChild.id)).resolves.toBeUndefined(); + await expect(storage.get('otherChild', otherChild.id)).resolves.toBeUndefined(); + await expect(storage.delete('grandchild', grandchild.id)).resolves.toBeUndefined(); + await expect(storage.get('grandchild', grandchild.id)).resolves.toBeUndefined(); + await expect(storage.delete('child', child.id)).resolves.toBeUndefined(); + await expect(storage.get('child', child.id)).resolves.toBeUndefined(); + await expect(storage.delete('root', root.id)).resolves.toBeUndefined(); + await expect(storage.get('root', root.id)).resolves.toBeUndefined(); + }); + + it('does nothing when removing a resource that does not exist.', async(): Promise => { + await expect(storage.delete('otherChild', otherChild.id)).resolves.toBeUndefined(); + await expect(storage.delete('otherChild', otherChild.id)).resolves.toBeUndefined(); + + await expect(storage.delete('root', root.id)).resolves.toBeUndefined(); + await expect(storage.delete('root', root.id)).resolves.toBeUndefined(); + }); + + it('removes all dependent resources when deleting.', async(): Promise => { + await expect(storage.delete('child', child.id)).resolves.toBeUndefined(); + await expect(storage.get('grandchild', grandchild.id)).resolves.toBeUndefined(); + await expect(storage.get('otherChild', otherChild.id)).resolves.toEqual(otherChild); + + await expect(storage.delete('root', root.id)).resolves.toBeUndefined(); + await expect(storage.get('otherChild', otherChild.id)).resolves.toBeUndefined(); + }); + + it('can find objects using queries.', async(): Promise => { + await expect(storage.find('root', { required: 5 })).resolves.toEqual([ root, root2 ]); + await expect(storage.find('root', { required: 5, notIndexed: 0 })).resolves.toEqual([ root ]); + await expect(storage.find('root', { optional: 'defined' })).resolves.toEqual([ root2 ]); + await expect(storage.find('root', { required: 5, optional: undefined })).resolves.toEqual([ root ]); + + await expect(storage.find('child', { parent: root[INDEX_ID_KEY] })).resolves.toEqual([ child, child2 ]); + await expect(storage.find('child', { parent: root[INDEX_ID_KEY], name: 'child' })).resolves.toEqual([ child ]); + await expect(storage.find('child', { parent: root[INDEX_ID_KEY], name: 'child2' })).resolves.toEqual([ child2 ]); + await expect(storage.find('child', { name: 'child' })).resolves.toEqual([ child, child3 ]); + await expect(storage.find('child', { parent: { [INDEX_ID_KEY]: root[INDEX_ID_KEY] }, name: 'child0' })) + .resolves.toEqual([]); + + await expect(storage.find('grandchild', { parent: child[INDEX_ID_KEY] })) + .resolves.toEqual([ grandchild, grandchild2 ]); + await expect(storage.find('grandchild', { parent: child2[INDEX_ID_KEY] })) + .resolves.toEqual([ grandchild3 ]); + await expect(storage.find('grandchild', { bool: true })) + .resolves.toEqual([ grandchild, grandchild3 ]); + }); + + it('can perform nested queries.', async(): Promise => { + await expect(storage.find('grandchild', { parent: { name: 'child' }})) + .resolves.toEqual([ grandchild, grandchild2 ]); + await expect(storage.find('grandchild', { bool: true, parent: { notIndexed: 1 }})) + .resolves.toEqual([ grandchild, grandchild3 ]); + await expect(storage.find('grandchild', { bool: true, parent: { parent: { required: 5 }}})) + .resolves.toEqual([ grandchild, grandchild3 ]); + }); + + it('can also find just the IDs of the results.', async(): Promise => { + await expect(storage.findIds('root', { required: 5 })) + .resolves.toEqual([ root[INDEX_ID_KEY], root2[INDEX_ID_KEY] ]); + await expect(storage.findIds('child', { name: 'child' })) + .resolves.toEqual([ child[INDEX_ID_KEY], child3[INDEX_ID_KEY] ]); + }); + + it('requires at least one index when finding results.', async(): Promise => { + await expect(storage.findIds('root', { notIndexed: 0 })).rejects.toThrow(InternalServerError); + await expect(storage.findIds('root', { optional: undefined })).rejects.toThrow(InternalServerError); + await expect(storage.findIds('child', { notIndexed: 0 })).rejects.toThrow(InternalServerError); + await expect(storage.findIds('grandchild', { notIndexed: 0 })).rejects.toThrow(InternalServerError); + await expect(storage.findIds('otherChild', { notIndexed: 0 })).rejects.toThrow(InternalServerError); + }); + + it('can iterate over all entries of a type.', async(): Promise => { + const roots: unknown[] = []; + for await (const entry of storage.entries('root')) { + roots.push(entry); + } + expect(roots).toEqual([ root, root2 ]); + + const children: unknown[] = []; + for await (const entry of storage.entries('child')) { + children.push(entry); + } + expect(children).toEqual([ child, child2, child3 ]); + }); + + it('errors if there is index corruption.', async(): Promise => { + // Corrupt the index. Will break if we change how index keys get generated. + indexMap.set(`child/${child.id}`, [ root2.id ]); + await expect(storage.create('grandchild', { bool: false, notIndexed: 5, parent: child.id })) + .rejects.toThrow(InternalServerError); + }); + + it('ignores index corruption when deleting keys.', async(): Promise => { + // Corrupt the index. Will break if we change how index keys get generated. + indexMap.delete(`child/name/${child.name}`); + await expect(storage.delete('child', child.id)).resolves.toBeUndefined(); + }); + }); +});