Mirroristas/CommunitySolidServer
2
0
Fork 0
mirror of https://github.com/CommunitySolidServer/CommunitySolidServer.git synced 2024-10-03 14:55:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Introduce IndexedStorage for a more extensive storage solution

Browse Source
This commit is contained in:
Joachim Van Herwegen 2023-09-20 10:51:09 +02:00
parent 661357c985
commit 3ade2ad795
5 changed files with 1211 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.componentsignore
View File

@ -18,6 +18,7 @@
"HashMap",
"HttpErrorOptions",
"HttpResponse",
"IndexTypeCollection",
"IdentifierMap",
"IdentifierSetMultiMap",
"NodeJS.Dict",
@ -34,6 +35,7 @@
"ValuePreferencesArg",
"VariableBindings",
"UnionHandler",
"VirtualObject",
"WinstonLogger",
"WrappedSetMultiMap",
"YargsOptions"

2
src/index.ts
View File

@ -402,12 +402,14 @@ export * from './storage/keyvalue/Base64EncodingStorage';
export * from './storage/keyvalue/ContainerPathStorage';
export * from './storage/keyvalue/ExpiringStorage';
export * from './storage/keyvalue/HashEncodingStorage';
export * from './storage/keyvalue/IndexedStorage';
export * from './storage/keyvalue/JsonFileStorage';
export * from './storage/keyvalue/JsonResourceStorage';
export * from './storage/keyvalue/KeyValueStorage';
export * from './storage/keyvalue/MemoryMapStorage';
export * from './storage/keyvalue/PassthroughKeyValueStorage';
export * from './storage/keyvalue/WrappedExpiringStorage';
export * from './storage/keyvalue/WrappedIndexedStorage';
// Storage/Mapping
export * from './storage/mapping/BaseFileIdentifierMapper';

201
src/storage/keyvalue/IndexedStorage.ts Normal file
View File

@ -0,0 +1,201 @@
/**
* The key that needs to be present in all output results of {@link IndexedStorage}.
*/
export const INDEX_ID_KEY = 'id';
/**
* Used to define the value of a key in a type entry of a {@link IndexedStorage}.
* Valid values are `"string"`, `"boolean"`, `"number"` and `"id:TYPE"`,
* with TYPE being one of the types in the definition.
* In the latter case this means that key points to an identifier of the specified type.
* A `?` can be appended to the type to indicate this key is optional.
*/
export type ValueTypeDescription<TType = string> =
`${('string' | 'boolean' | 'number' | (TType extends string ? `${typeof INDEX_ID_KEY}:${TType}` : never))}${
'?' | ''}`;
/**
* Converts a {@link ValueTypeDescription} to the type it should be interpreted as.
*/
export type ValueType<T extends ValueTypeDescription> =
(T extends 'boolean' | 'boolean?' ? boolean : T extends 'number' | 'number?' ? number : string) |
(T extends `${string}?` ? undefined : never);
/**
* Used to filter on optional keys in a {@link IndexedStorage} definition.
*/
export type OptionalKey<T> = {[K in keyof T ]: T[K] extends `${string}?` ? K : never }[keyof T];
/**
* Converts a {@link IndexedStorage} definition of a specific type
* to the typing an object would have that is returned as an output on function calls.
*/
export type TypeObject<TDesc extends Record<string, ValueTypeDescription>> = {
-readonly [K in Exclude<keyof TDesc, OptionalKey<TDesc>>]: ValueType<TDesc[K]>;
} & {
-readonly [K in keyof TDesc]?: ValueType<TDesc[K]>;
} & { [INDEX_ID_KEY]: string };
/**
* Input expected for `create()` call in {@link IndexedStorage}.
* This is the same as {@link TypeObject} but without the index key.
*/
export type CreateTypeObject<T extends Record<string, ValueTypeDescription>> = Omit<TypeObject<T>, typeof INDEX_ID_KEY>;
/**
* Key of an object that is also a string.
*/
export type StringKey<T> = keyof T & string;
/**
* The description of a single type in an {@link IndexedStorage}.
*/
export type IndexTypeDescription<TType = never> = Record<string, ValueTypeDescription<TType>>;
/**
* The full description of all the types of an {@link IndexedStorage}.
*/
export type IndexTypeCollection<T> = Record<string, IndexTypeDescription<keyof T>>;
// This is necessary to prevent infinite recursion in types
type Prev = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...0[]];
/**
* Object that represents a valid query starting from a specific type on an {@link IndexedStorage}.
* The keys of the object need to be one or more keys from the starting type,
* with the values being corresponding valid values of an object of that type.
* If the value definition of a key is one that contains the identifier of a different type,
* the value in the query can also be a nested object that has the same IndexedQuery requirements for that type.
* This can be done recursively.
*
* E.g., if the storage has the following definition:
*```ts
* {
* account: {},
* pod: { baseUrl: 'string', account: 'id:account' },
* pod: { owner: 'string', pod: 'id:pod' },
* }
*```
* A valid query on the `pod` type could be `{ pod: '123456' }`,
* but also `{ pod: { baseUrl: 'http://example.com/pod/', account: { id: '789' }}}`.
*/
export type IndexedQuery<T extends IndexTypeCollection<T>, TType extends keyof T, TDepth extends number = 10> =
[TDepth] extends [never] ? never :
{[K in keyof T[TType] | typeof INDEX_ID_KEY]?:
ValueType<T[TType][K]> |
(T[TType][K] extends `${typeof INDEX_ID_KEY}:${infer U}` ? IndexedQuery<T, U, Prev[TDepth]> : never)
};
/* eslint-disable @typescript-eslint/method-signature-style */
/**
* A storage solution that allows for more complex queries than a key/value storage
* and allows setting indexes on specific keys.
*/
export interface IndexedStorage<T extends IndexTypeCollection<T>> {
/**
* Informs the storage of the definition of a specific type.
* A definition is a key/value object with the values being a valid {@link ValueTypeDescription}.
* Generally, this call needs to happen for every type of this storage,
* and before any calls are made to interact with the data.
*
* @param type - The type to define.
* @param description - A description of the values stored in objects of that type.
*/
defineType<TType extends StringKey<T>>(type: TType, description: T[TType]): Promise<void>;
/**
* Creates an index on a key of the given type, to allow for better queries involving those keys.
* Similar to {@link IndexedStorage.defineType} these calls need to happen first.
*
* @param type - The type to create an index on.
* @param key - The key of that type to create an index on.
*/
createIndex<TType extends StringKey<T>>(type: TType, key: StringKey<T[TType]>): Promise<void>;
/**
* Creates an object of the given type.
* The storage will generate an identifier for the newly created object.
*
* @param type - The type to create.
* @param value - The value to set for the created object.
*
* @returns A representation of the newly created object, including its new identifier.
*/
create<TType extends StringKey<T>>(type: TType, value: CreateTypeObject<T[TType]>): Promise<TypeObject<T[TType]>>;
/**
* Returns `true` if the object of the given type with the given identifier exists.
*
* @param type - The type of object to get.
* @param id - The identifier of that object.
*
* @returns Whether this object exists.
*/
has<TType extends StringKey<T>>(type: TType, id: string): Promise<boolean>;
/**
* Returns the object of the given type with the given identifier.
*
* @param type - The type of object to get.
* @param id - The identifier of that object.
*
* @returns A representation of the object, or `undefined` if there is no object of that type with that identifier.
*/
get<TType extends StringKey<T>>(type: TType, id: string): Promise<TypeObject<T[TType]> | undefined>;
/**
* Finds all objects matching a specific {@link IndexedQuery}.
*
* @param type - The type of objects to find.
* @param query - The query to execute.
*
* @returns A list of objects matching the query.
*/
find<TType extends StringKey<T>>(type: TType, query: IndexedQuery<T, TType>): Promise<(TypeObject<T[TType]>)[]>;
/**
* Similar to {@link IndexedStorage.find}, but only returns the identifiers of the found objects.
*
* @param type - The type of objects to find.
* @param query - The query to execute.
*
* @returns A list of identifiers of the matching objects.
*/
findIds<TType extends StringKey<T>>(type: TType, query: IndexedQuery<T, TType>): Promise<string[]>;
/**
* Sets the value of a specific object.
* The identifier in the object is used to identify the object.
*
* @param type - The type of the object to set.
* @param value - The new value for the object.
*/
set<TType extends StringKey<T>>(type: TType, value: TypeObject<T[TType]>): Promise<void>;
/**
* Sets the value of one specific field in an object.
*
* @param type - The type of the object to update.
* @param id - The identifier of the object to update.
* @param key - The key to update.
* @param value - The new value for the given key.
*/
setField<TType extends StringKey<T>, TKey extends StringKey<T[TType]>>(
type: TType, id: string, key: TKey, value: ValueType<T[TType][TKey]>): Promise<void>;
/**
* Deletes the given object.
* This will also delete all objects that reference that object if the corresponding key is not optional.
*
* @param type - The type of the object to delete.
* @param id - The identifier of the object.
*/
delete<TType extends StringKey<T>>(type: TType, id: string): Promise<void>;
/**
* Returns an iterator over all objects of the given type.
*
* @param type - The type to iterate over.
*/
entries<TType extends StringKey<T>>(type: TType): AsyncIterableIterator<TypeObject<T[TType]>>;
}

645
src/storage/keyvalue/WrappedIndexedStorage.ts Normal file
View File

@ -0,0 +1,645 @@
import { v4 } from 'uuid';
import { getLoggerFor } from '../../logging/LogUtil';
import { InternalServerError } from '../../util/errors/InternalServerError';
import { NotFoundHttpError } from '../../util/errors/NotFoundHttpError';
import { NotImplementedHttpError } from '../../util/errors/NotImplementedHttpError';
import { INDEX_ID_KEY } from './IndexedStorage';
import type { StringKey,
IndexedStorage,
TypeObject,
IndexedQuery,
IndexTypeCollection,
CreateTypeObject, ValueType } from './IndexedStorage';
import type { KeyValueStorage } from './KeyValueStorage';
/**
* Key used to link to child objects in a {@link WrappedIndexedStorage}.
*/
export type VirtualKey<TChild> = TChild extends string ? `**${TChild}**` : never;
/**
* Object stored in the wrapped {@link KeyValueStorage} in a {@link WrappedIndexedStorage}.
*/
export type VirtualObject = {
[key: VirtualKey<string>]: Record<string, VirtualObject>;
[key: string]: unknown;
[INDEX_ID_KEY]: string;
};
/**
* A parent/child relation description in a {@link WrappedIndexedStorage}.
*/
export type IndexRelation<TTypes> = {
parent: { key: VirtualKey<string>; type: StringKey<TTypes> };
child: { key: string; type: StringKey<TTypes> };
};
/**
* An {@link IndexedStorage} that makes use of 2 {@link KeyValueStorage}s to implement the interface.
* Due to being limited by key/value storages, there are some restrictions on the allowed type definitions:
*
* * There needs to be exactly 1 type with no references to other types.
* * All other types need to have exactly 1 reference to another type.
* * Types can't reference each other to create a cycle of references.
*
* All of the above to create a tree-like structure of references.
* Such a tree is then stored in one of the storages.
* The other storage is used to store all indexes that are used to find the correct tree object when solving queries.
*/
export class WrappedIndexedStorage<T extends IndexTypeCollection<T>> implements IndexedStorage<T> {
protected readonly logger = getLoggerFor(this);
private readonly valueStorage: KeyValueStorage<string, VirtualObject>;
private readonly indexStorage: KeyValueStorage<string, string[]>;
/**
* For every type, the keys on which an index tracks the values and which root object they are contained in.
* All types for which a `defineType` call was made will have a key in this object.
* For all types that are not the root, there will always be an index on their ID value.
*/
private readonly indexes: {[K in StringKey<T>]?: Set<StringKey<T[K]>> };
/**
* Keeps track of type validation.
* If true the defined types create a valid structure that can be used.
*/
private validDefinition = false;
/**
* The variable in which the root type is stored.
* A separate getter is used to always return the value
* so the potential `undefined` does not have to be taken into account.
*/
private rootTypeVar: StringKey<T> | undefined;
/**
* All parent/child relations between all types in the storage,
* including the keys in both types that are used to reference each other.
*/
private readonly relations: IndexRelation<T>[];
public constructor(valueStorage: KeyValueStorage<string, VirtualObject>,
indexStorage: KeyValueStorage<string, string[]>) {
this.valueStorage = valueStorage;
this.indexStorage = indexStorage;
this.indexes = {};
this.relations = [];
}
public async defineType<TType extends StringKey<T>>(type: TType, description: T[TType]): Promise<void> {
if (this.rootTypeVar) {
this.logger.error(`Trying to define new type "${type}" after types were already validated.`);
throw new InternalServerError(`Trying to define new type "${type}" after types were already validated.`);
}
this.validDefinition = false;
let hasParentKey = false;
for (const [ key, desc ] of Object.entries(description)) {
if (desc.startsWith(`${INDEX_ID_KEY}:`)) {
if (hasParentKey) {
this.logger.error(`Type definition of ${type} has multiple references, only 1 is allowed.`);
throw new InternalServerError(`Type definition of ${type} has multiple references, only 1 is allowed.`);
}
if (desc.endsWith('?')) {
this.logger.error(`Type definition of ${type} has optional references, which is not allowed.`);
throw new InternalServerError(`Type definition of ${type} has optional references, which is not allowed.`);
}
hasParentKey = true;
this.relations.push({ parent: { type: desc.slice(3), key: `**${type}**` }, child: { type, key }});
}
}
this.indexes[type] = new Set([ INDEX_ID_KEY ]);
}
public async createIndex<TType extends StringKey<T>>(type: TType, key: StringKey<T[TType]>): Promise<void> {
// An index on the key targeting the parent is the same as having an index on the identifier of that parent.
// Such an index gets created automatically when the type is defined so this can now be ignored.
if (key === this.getParentRelation(type)?.child.key) {
return;
}
const typeIndexes = this.indexes[type];
if (!typeIndexes) {
this.logger.error(`Trying to create index on key ${key} of undefined type ${type}`);
throw new InternalServerError(`Trying to create index on key ${key} of undefined type ${type}`);
}
typeIndexes.add(key);
}
public async has<TType extends StringKey<T>>(type: TType, id: string): Promise<boolean> {
this.validateDefinition(type);
if (type === this.rootType) {
return this.valueStorage.has(id);
}
const result = await this.find(type, { [INDEX_ID_KEY]: id } as IndexedQuery<T, TType>);
return result.length > 0;
}
public async get<TType extends StringKey<T>>(type: TType, id: string): Promise<TypeObject<T[TType]> | undefined> {
this.validateDefinition(type);
const result = await this.find(type, { [INDEX_ID_KEY]: id } as IndexedQuery<T, TType>);
if (result.length === 0) {
return;
}
return result[0];
}
public async create<TType extends StringKey<T>>(type: TType, value: CreateTypeObject<T[TType]>):
Promise<TypeObject<T[TType]>> {
this.validateDefinition(type);
const id = v4();
const newObj = { ...value, [INDEX_ID_KEY]: id } as unknown as VirtualObject;
// Add the virtual keys
for (const relation of this.getChildRelations(type)) {
newObj[relation.parent.key] = {};
}
const relation = this.getParentRelation(type);
// No parent relation implies that this is the root type
if (!relation) {
await this.valueStorage.set(id, newObj);
await this.updateTypeIndex(type, id, undefined, newObj);
return this.toTypeObject(type, newObj);
}
// We know this will be a string due to the typing requirements and how the relations object is built
const parentId = newObj[relation.child.key] as string;
const root = await this.getRoot(relation.parent.type, parentId);
if (!root) {
throw new NotFoundHttpError(`Unknown object of type ${relation.parent.type} with ID ${parentId}`);
}
const parentObj = relation.parent.type === this.rootType ?
root :
this.getContainingRecord(root, relation.parent.type, parentId)[parentId];
parentObj[relation.parent.key][id] = newObj;
await this.valueStorage.set(root[INDEX_ID_KEY], root);
await this.updateTypeIndex(type, root[INDEX_ID_KEY], undefined, newObj);
return this.toTypeObject(type, newObj);
}
public async set<TType extends StringKey<T>>(type: TType, value: TypeObject<T[TType]>): Promise<void> {
this.validateDefinition(type);
return this.updateValue(type, value, true);
}
public async setField<TType extends StringKey<T>, TKey extends StringKey<T[TType]>>(
type: TType, id: string, key: TKey, value: ValueType<T[TType][TKey]>,
): Promise<void> {
this.validateDefinition(type);
return this.updateValue(type,
{ [INDEX_ID_KEY]: id, [key]: value } as Partial<TypeObject<T[TType]>> & { [INDEX_ID_KEY]: string },
false);
}
public async delete<TType extends StringKey<T>>(type: TType, id: string): Promise<void> {
this.validateDefinition(type);
const root = await this.getRoot(type, id);
if (!root) {
return;
}
let oldObj: VirtualObject;
if (type === this.rootType) {
oldObj = root;
await this.valueStorage.delete(id);
} else {
const objs = this.getContainingRecord(root, type, id);
oldObj = objs[id];
delete objs[id];
await this.valueStorage.set(root[INDEX_ID_KEY], root);
}
// Updating index of removed type and all children as those are also gone
await this.updateDeepTypeIndex(type, root[INDEX_ID_KEY], oldObj);
}
public async findIds<TType extends StringKey<T>>(type: TType, query: IndexedQuery<T, TType>): Promise<string[]> {
this.validateDefinition(type);
if (type === this.rootType) {
// Root IDs are the only ones we can get more efficiently.
// For all other types we have to find the full objects anyway.
const indexedRoots = await this.findIndexedRoots(type, query);
if (!Array.isArray(indexedRoots)) {
this.logger.error(`Attempting to execute query without index: ${JSON.stringify(query)}`);
throw new InternalServerError(`Attempting to execute query without index: ${JSON.stringify(query)}`);
}
return indexedRoots;
}
return (await this.solveQuery(type, query)).map((result): string => result.id);
}
public async find<TType extends StringKey<T>>(type: TType, query: IndexedQuery<T, TType>):
Promise<(TypeObject<T[TType]>)[]> {
this.validateDefinition(type);
return (await this.solveQuery(type, query)).map((result): TypeObject<T[TType]> => this.toTypeObject(type, result));
}
public async* entries<TType extends StringKey<T>>(type: TType): AsyncIterableIterator<TypeObject<T[TType]>> {
this.validateDefinition(type);
const path = this.getPathToType(type);
for await (const [ , root ] of this.valueStorage.entries()) {
const children = this.getChildObjects(root, path);
yield* children.map((child): TypeObject<T[TType]> => this.toTypeObject(type, child));
}
}
// --------------------------------- OUTPUT HELPERS ---------------------------------
/**
* Converts a {@link VirtualObject} into a {@link TypeObject}.
* To be used when outputting results.
*/
protected toTypeObject<TType extends StringKey<T>>(type: TType, obj: VirtualObject): TypeObject<T[TType]> {
const result = { ...obj };
for (const relation of this.getChildRelations(type)) {
delete result[relation.parent.key];
}
return result as unknown as TypeObject<T[TType]>;
}
// --------------------------------- ROOT HELPERS ---------------------------------
/**
* The root type for this storage.
* Use this instead of rootTypeVar to prevent having to check for `undefined`.
* This value will always be defined if the type definitions have been validated.
*/
protected get rootType(): string {
return this.rootTypeVar!;
}
/**
* Finds the root object that contains the requested type/id combination.
*/
protected async getRoot<TType extends StringKey<T>>(type: TType, id: string): Promise<VirtualObject | undefined> {
let rootId: string;
if (type === this.rootType) {
rootId = id;
} else {
// We know there always is an index on the identifier key
const indexKey = this.getIndexKey(type, INDEX_ID_KEY, id);
const indexResult = await this.indexStorage.get(indexKey);
if (!indexResult || indexResult.length !== 1) {
return;
}
rootId = indexResult[0];
}
return this.valueStorage.get(rootId);
}
// --------------------------------- PATH HELPERS ---------------------------------
/**
* Returns the sequence of virtual keys that need to be accessed to reach the given type, starting from the root.
*/
protected getPathToType(type: string): VirtualKey<string>[] {
const result: VirtualKey<string>[] = [];
let relation = this.getParentRelation(type);
while (relation) {
result.unshift(relation.parent.key);
relation = this.getParentRelation(relation.parent.type);
}
return result;
}
/**
* Finds all records that can be found in the given object by following the given path of virtual keys.
*/
protected getPathRecords(obj: VirtualObject, path: VirtualKey<string>[]): Record<string, VirtualObject>[] {
const record = obj[path[0]];
if (path.length === 1) {
return [ record ];
}
const subPath = path.slice(1);
return Object.values(record)
.flatMap((child): Record<string, VirtualObject>[] => this.getPathRecords(child, subPath));
}
/**
* Finds all objects in the provided object that can be found by following the provided path of virtual keys.
*/
protected getChildObjects(obj: VirtualObject, path: VirtualKey<string>[]): VirtualObject[] {
if (path.length === 0) {
return [ obj ];
}
return this.getPathRecords(obj, path).flatMap((record): VirtualObject[] => Object.values(record));
}
/**
* Finds the record in the given object that contains the given type/id combination.
* This function assumes it was already verified through an index that this object contains the given combination.
*/
protected getContainingRecord<TType extends StringKey<T>>(rootObj: VirtualObject, type: TType, id: string):
Record<string, VirtualObject> {
const path = this.getPathToType(type);
const records = this.getPathRecords(rootObj, path);
const match = records.find((record): boolean => Boolean(record[id]));
if (!match) {
this.logger.error(`Could not find ${type} ${id} in ${this.rootType} ${rootObj.id}`);
throw new InternalServerError(`Could not find ${type} ${id} in ${this.rootType} ${rootObj.id}`);
}
return match;
}
// --------------------------------- UPDATE VALUE HELPERS ---------------------------------
/**
* Replaces an object of the given type.
* The identifier in the value is used to determine which object.
*/
protected updateValue<TType extends StringKey<T>>(type: TType, value: TypeObject<T[TType]>, replace: true):
Promise<void>;
/**
* Replaces part of an object of the given type with the given partial value.
* The identifier in the value is used to determine which object.
*/
protected updateValue<TType extends StringKey<T>>(type: TType,
partial: Partial<TypeObject<T[TType]>> & { [INDEX_ID_KEY]: string }, replace: false): Promise<void>;
protected async updateValue<TType extends StringKey<T>>(type: TType,
partial: Partial<TypeObject<T[TType]>> & { [INDEX_ID_KEY]: string }, replace: boolean): Promise<void> {
const id = partial[INDEX_ID_KEY];
let root = await this.getRoot(type, id);
if (!root) {
throw new NotFoundHttpError(`Unknown object of type ${type} with ID ${id}`);
}
let oldObj: VirtualObject;
let newObj: VirtualObject;
const relation = this.getParentRelation(type);
if (!relation) {
oldObj = root;
newObj = (replace ? { ...partial } : { ...oldObj, ...partial }) as VirtualObject;
root = newObj;
} else {
const objs = this.getContainingRecord(root, type, id);
if (partial[relation.child.key] && objs[id][relation.child.key] !== partial[relation.child.key]) {
this.logger.error(`Trying to modify reference key ${objs[id][relation.child.key]} on "${type}" ${id}`);
throw new NotImplementedHttpError('Changing reference keys of existing objects is not supported.');
}
oldObj = objs[id];
newObj = (replace ? { ...partial } : { ...oldObj, ...partial }) as VirtualObject;
objs[id] = newObj;
}
// Copy over the child relations
for (const childRelation of this.getChildRelations(type)) {
newObj[childRelation.parent.key] = oldObj[childRelation.parent.key];
}
await this.valueStorage.set(root[INDEX_ID_KEY], root);
await this.updateTypeIndex(type, root[INDEX_ID_KEY], oldObj, newObj);
}
// --------------------------------- TYPE HELPERS ---------------------------------
/**
* Returns all relations where the given type is the parent.
*/
protected getChildRelations<TType extends StringKey<T>>(type: TType): IndexRelation<T>[] {
return this.relations.filter((relation): boolean => relation.parent.type === type);
}
/**
* Returns the relation where the given type is the child.
* Will return `undefined` for the root type as that one doesn't have a parent.
*/
protected getParentRelation<TType extends StringKey<T>>(type: TType): IndexRelation<T> | undefined {
return this.relations.find((relation): boolean => relation.child.type === type);
}
/**
* Makes sure the defined types fulfill all the requirements necessary for types on this storage.
* Will throw an error if this is not the case.
* This should be called before doing any data interactions.
* Stores success in a variable so future calls are instantaneous.
*/
protected validateDefinition(type: string): void {
// We can't know if all types are already defined.
// This prevents issues even if the other types together are valid.
if (!this.indexes[type]) {
const msg = `Type "${type}" was not defined. The defineType functions needs to be called before accessing data.`;
this.logger.error(msg);
throw new InternalServerError(msg);
}
if (this.validDefinition) {
return;
}
const rootTypes = new Set<StringKey<T>>();
// `this.indexes` will contain a key for each type as we always have an index on the identifier of a type
for (let indexType of Object.keys(this.indexes)) {
const foundTypes = new Set<string>([ indexType ]);
// Find path to root from this type, thereby ensuring that there is no cycle.
let relation = this.getParentRelation(indexType);
while (relation) {
indexType = relation.parent.type;
if (foundTypes.has(indexType)) {
const msg = `The following types cyclically reference each other: ${[ ...foundTypes ].join(', ')}`;
this.logger.error(msg);
throw new InternalServerError(msg);
}
foundTypes.add(indexType);
relation = this.getParentRelation(indexType);
}
rootTypes.add(indexType);
}
if (rootTypes.size > 1) {
const msg = `Only one type definition with no references is allowed. Found ${[ ...rootTypes ].join(', ')}`;
this.logger.error(msg);
throw new InternalServerError(msg);
}
this.rootTypeVar = [ ...rootTypes.values() ][0];
// Remove the root index as we don't need it, and it can cause confusion when resolving queries
this.indexes[this.rootTypeVar]?.delete(INDEX_ID_KEY);
this.validDefinition = true;
}
// --------------------------------- QUERY HELPERS ---------------------------------
/**
* Finds the IDs of all root objects that contain objects of the given type matching the given query
* by making use of the indexes applicable to the keys in the query.
* This function only looks at the keys in the query with primitive values,
* object values in the query referencing parent objects are not considered.
* Similarly, only indexes are used, keys without index are also ignored.
*
* If an array of root IDs is provided as input,
* the result will be an intersection of this array and the found identifiers.
*
* If the result is an empty array, it means that there is no valid identifier matching the query,
* while an `undefined` result means there is no index matching any of the query keys,
* so a result can't be determined.
*/
protected async findIndexedRoots<TType extends StringKey<T>>(type: TType, match: IndexedQuery<T, TType>,
rootIds?: string[]): Promise<string[] | undefined> {
if (type === this.rootType && match[INDEX_ID_KEY]) {
// If the input is the root type with a known ID in the query,
// and we have already established that it is not this ID,
// there is no result.
if (rootIds && !rootIds.includes(match[INDEX_ID_KEY] as string)) {
return [];
}
rootIds = [ match[INDEX_ID_KEY] as string ];
}
const indexIds: string[] = [];
for (const [ key, value ] of Object.entries(match)) {
if (this.indexes[type]?.has(key) && typeof value !== 'undefined') {
// We know value is a string (or boolean/number) since we can't have indexes on fields referencing other objects
indexIds.push(this.getIndexKey(type, key, value));
}
}
if (indexIds.length === 0) {
return rootIds;
}
// Use all indexes found to find matching IDs
const indexResults = await Promise.all(indexIds.map(async(id): Promise<string[]> =>
await this.indexStorage.get(id) ?? []));
if (Array.isArray(rootIds)) {
indexResults.push(rootIds);
}
return indexResults.reduce((acc, ids): string[] => acc.filter((id): boolean => ids.includes(id)));
}
/**
* Finds all objects of the given type matching the query.
* The `rootIds` array can be used to restrict the IDs of root objects to look at,
* which is relevant for the recursive calls the function does.
*
* Will throw an error if there is no index that can be used to solve the query.
*/
protected async solveQuery<TType extends StringKey<T>>(type: TType, query: IndexedQuery<T, TType>,
rootIds?: string[]): Promise<VirtualObject[]> {
this.logger.debug(`Executing "${type}" query ${JSON.stringify(query)}. Already found roots ${rootIds}.`);
const indexedRoots = await this.findIndexedRoots(type, query, rootIds);
// All objects of this type that we find through recursive calls
let objs: VirtualObject[];
// Either find all objects of the type from the found rootIds if the query is a leaf,
// or recursively query the parent object if it is not.
const relation = this.getParentRelation(type);
if (!relation || !query[relation.child.key]) {
// This is a leaf node of the query
if (!Array.isArray(indexedRoots)) {
this.logger.error(`Attempting to execute query without index: ${JSON.stringify(query)}`);
throw new InternalServerError(`Attempting to execute query without index: ${JSON.stringify(query)}`);
}
const pathFromRoot = this.getPathToType(type);
// All objects of this type for all root objects we have
const roots = (await Promise.all(indexedRoots.map(async(id): Promise<VirtualObject | undefined> => {
const root = await this.valueStorage.get(id);
if (!root) {
// Not throwing an error to sort of make server still work if an index is wrong.
this.logger.error(
`Data inconsistency: index contains ${this.rootType} with ID ${id}, but this object does not exist.`,
);
}
return root;
}))).filter((root): boolean => typeof root !== 'undefined') as VirtualObject[];
objs = roots.flatMap((root): VirtualObject[] => this.getChildObjects(root, pathFromRoot));
} else {
const subQuery = (typeof query[relation.child.key] === 'string' ?
{ [INDEX_ID_KEY]: query[relation.child.key] } :
query[relation.child.key]) as IndexedQuery<T, typeof relation.parent.type>;
// All objects by recursively calling this function on the parent object and extracting all children of this type
objs = (await this.solveQuery(relation.parent.type, subQuery, indexedRoots))
.flatMap((parentObj): VirtualObject[] => Object.values(parentObj[relation.parent.key]));
}
// For all keys that were not handled recursively: make sure that it matches the found objects
const remainingKeys = Object.keys(query).filter((key): boolean =>
key !== relation?.child.key || typeof query[key] === 'string');
return remainingKeys.reduce((acc, key): any[] => acc.filter((obj): boolean => obj[key] === query[key]), objs);
}
// --------------------------------- INDEX HELPERS ---------------------------------
/**
* Generate the key used to store the index in the index storage.
*/
protected getIndexKey(type: string, key: string, value: string | number): string {
return `${encodeURIComponent(type)}/${key === INDEX_ID_KEY ? '' : `${key}/`}${encodeURIComponent(`${value}`)}`;
}
/**
* Update all indexes for an object of the given type, and all its children.
*/
protected async updateDeepTypeIndex<TType extends StringKey<T>>(type: TType, rootId: string,
oldObj: VirtualObject, newObj?: VirtualObject): Promise<void> {
const promises: Promise<void>[] = [];
promises.push(this.updateTypeIndex(type, rootId, oldObj, newObj));
for (const { parent, child } of this.getChildRelations(type)) {
const oldRecord: Record<string, VirtualObject> = oldObj[parent.key];
const newRecord: Record<string, VirtualObject> = newObj?.[parent.key] ?? {};
const uniqueKeys = new Set([ ...Object.keys(oldRecord), ...Object.keys(newRecord) ]);
for (const key of uniqueKeys) {
promises.push(this.updateDeepTypeIndex(child.type, rootId, oldRecord[key], newRecord[key]));
}
}
await Promise.all(promises);
}
/**
* Updates all indexes for an object of the given type.
*/
protected async updateTypeIndex<TType extends StringKey<T>>(type: TType, rootId: string,
oldObj?: VirtualObject, newObj?: VirtualObject): Promise<void> {
const added: { key: string; value: string }[] = [];
const removed: { key: string; value: string }[] = [];
for (const key of this.indexes[type]!) {
const oldValue = oldObj?.[key];
const newValue = newObj?.[key];
if (oldValue !== newValue) {
if (typeof oldValue !== 'undefined') {
removed.push({ key, value: oldValue as string });
}
if (typeof newValue !== 'undefined') {
added.push({ key, value: newValue as string });
}
}
}
await Promise.all([
...added.map(({ key, value }): Promise<unknown> => this.updateKeyIndex(type, key, value, rootId, true)),
...removed.map(({ key, value }): Promise<unknown> => this.updateKeyIndex(type, key, value, rootId, false)),
]);
}
/**
* Updates the index for a specific key of an object of the given type.
*/
protected async updateKeyIndex(type: string, key: string, value: string, rootId: string, add: boolean):
Promise<void> {
const indexKey = this.getIndexKey(type, key, value);
const indexValues = await this.indexStorage.get(indexKey) ?? [];
this.logger.debug(`Updating index ${indexKey} by ${add ? 'adding' : 'removing'} ${rootId} from ${indexValues}`);
if (add) {
if (!indexValues.includes(rootId)) {
indexValues.push(rootId);
}
await this.indexStorage.set(indexKey, indexValues);
} else {
const updatedValues = indexValues.filter((val): boolean => val !== rootId);
await (updatedValues.length === 0 ?
this.indexStorage.delete(indexKey) :
this.indexStorage.set(indexKey, updatedValues));
}
}
}

361
test/unit/storage/keyvalue/WrappedIndexedStorage.test.ts Normal file
View File

@ -0,0 +1,361 @@
import { INDEX_ID_KEY } from '../../../../src/storage/keyvalue/IndexedStorage';
import type { TypeObject } from '../../../../src/storage/keyvalue/IndexedStorage';
import type { KeyValueStorage } from '../../../../src/storage/keyvalue/KeyValueStorage';
import { WrappedIndexedStorage } from '../../../../src/storage/keyvalue/WrappedIndexedStorage';
import { InternalServerError } from '../../../../src/util/errors/InternalServerError';
import { NotFoundHttpError } from '../../../../src/util/errors/NotFoundHttpError';
import { NotImplementedHttpError } from '../../../../src/util/errors/NotImplementedHttpError';
const dummyDescription = {
root: { required: 'number', optional: 'string?', notIndexed: 'number' },
child: { parent: 'id:root', name: 'string', notIndexed: 'number' },
grandchild: { parent: 'id:child', bool: 'boolean', notIndexed: 'number' },
otherChild: { parent: 'id:root', name: 'string', notIndexed: 'number' },
} as const;
describe('A WrappedIndexedStorage', (): void => {
let valueMap: Map<string, any>;
let valueStorage: jest.Mocked<KeyValueStorage<string, any>>;
let indexMap: Map<string, string[]>;
let indexStorage: jest.Mocked<KeyValueStorage<string, string[]>>;
let storage: WrappedIndexedStorage<typeof dummyDescription>;
beforeEach(async(): Promise<void> => {
valueMap = new Map();
valueStorage = {
has: jest.fn(async(key): Promise<boolean> => valueMap.has(key)),
get: jest.fn(async(key): Promise<unknown> => valueMap.get(key)),
set: jest.fn(async(key, value): Promise<any> => valueMap.set(key, value)),
delete: jest.fn(async(key): Promise<boolean> => valueMap.delete(key)),
entries: jest.fn(async function* (): AsyncIterableIterator<[string, unknown]> {
yield* valueMap.entries();
}),
};
indexMap = new Map();
indexStorage = {
has: jest.fn(async(key): Promise<boolean> => indexMap.has(key)),
get: jest.fn(async(key): Promise<string[] | undefined> => indexMap.get(key)),
set: jest.fn(async(key, value): Promise<any> => indexMap.set(key, value)),
delete: jest.fn(async(key): Promise<boolean> => indexMap.delete(key)),
entries: jest.fn(async function* (): AsyncIterableIterator<[string, string[]]> {
yield* indexMap.entries();
}),
};
storage = new WrappedIndexedStorage<typeof dummyDescription>(valueStorage, indexStorage);
});
describe('that is empty', (): void => {
it('can define and initialize data.', async(): Promise<void> => {
await expect(storage.defineType('root', dummyDescription.root)).resolves.toBeUndefined();
await expect(storage.defineType('child', dummyDescription.child)).resolves.toBeUndefined();
await expect(storage.defineType('grandchild', dummyDescription.grandchild)).resolves.toBeUndefined();
await expect(storage.defineType('otherChild', dummyDescription.otherChild)).resolves.toBeUndefined();
});
it('errors when defining types with multiple references.', async(): Promise<void> => {
await expect(storage.defineType('root', { ref1: 'id:Type1', ref2: 'id:Type2' } as any))
.rejects.toThrow(InternalServerError);
});
it('errors when defining types with optional references.', async(): Promise<void> => {
await expect(storage.defineType('root', { ref: 'id:Type1?' } as any)).rejects.toThrow(InternalServerError);
});
it('errors trying to create an index on an undefined type.', async(): Promise<void> => {
await expect(storage.createIndex('root', 'required')).rejects.toThrow(InternalServerError);
});
it('errors trying to access data before its type was defined.', async(): Promise<void> => {
await expect(storage.has('root', '???')).rejects.toThrow(InternalServerError);
});
it('errors if type definitions are added after validation.', async(): Promise<void> => {
await expect(storage.defineType('root', dummyDescription.root)).resolves.toBeUndefined();
// Trigger data validation
await storage.has('root', '???');
await expect(storage.defineType('root', dummyDescription.root)).rejects.toThrow(InternalServerError);
});
it('errors if the type definitions are cyclical.', async(): Promise<void> => {
await expect(storage.defineType('root', { ...dummyDescription.root, invalidReference: 'id:grandchild' } as any))
.resolves.toBeUndefined();
await expect(storage.defineType('child', dummyDescription.child)).resolves.toBeUndefined();
await expect(storage.defineType('grandchild', dummyDescription.grandchild)).resolves.toBeUndefined();
// Trigger data validation
await expect(storage.has('root', '???')).rejects.toThrow(InternalServerError);
});
it('errors if there are multiple root types.', async(): Promise<void> => {
await expect(storage.defineType('root', dummyDescription.root)).resolves.toBeUndefined();
await expect(storage.defineType('child', dummyDescription.root as any)).resolves.toBeUndefined();
// Trigger data validation
await expect(storage.has('root', '???')).rejects.toThrow(InternalServerError);
});
});
describe('with data definitions', (): void => {
beforeEach(async(): Promise<void> => {
await storage.defineType('root', dummyDescription.root);
await storage.defineType('child', dummyDescription.child);
await storage.defineType('grandchild', dummyDescription.grandchild);
await storage.defineType('otherChild', dummyDescription.otherChild);
});
it('can create new entries.', async(): Promise<void> => {
const parent = await storage.create('root', { required: 5, notIndexed: 0 });
expect(parent).toEqual({ id: expect.any(String), required: 5, notIndexed: 0 });
const child = await storage.create('child', { name: 'child', parent: parent.id, notIndexed: 1 });
expect(child).toEqual({ id: expect.any(String), name: 'child', parent: parent.id, notIndexed: 1 });
const grandchild = await storage.create('grandchild', { bool: true, parent: child.id, notIndexed: 2 });
expect(grandchild).toEqual({ id: expect.any(String), bool: true, parent: child.id, notIndexed: 2 });
const otherChild = await storage.create('otherChild', { name: 'otherChild', parent: parent.id, notIndexed: 3 });
expect(otherChild).toEqual({ id: expect.any(String), name: 'otherChild', parent: parent.id, notIndexed: 3 });
});
it('errors when creating new entries with unknown references.', async(): Promise<void> => {
await expect(storage.create('child', { name: 'child', parent: '???', notIndexed: 1 }))
.rejects.toThrow(NotFoundHttpError);
});
it('can create indexes.', async(): Promise<void> => {
await expect(storage.createIndex('root', 'required')).resolves.toBeUndefined();
await expect(storage.createIndex('root', 'optional')).resolves.toBeUndefined();
await expect(storage.createIndex('child', 'name')).resolves.toBeUndefined();
await expect(storage.createIndex('grandchild', 'bool')).resolves.toBeUndefined();
await expect(storage.createIndex('otherChild', 'name')).resolves.toBeUndefined();
// This one does nothing
await expect(storage.createIndex('grandchild', 'parent')).resolves.toBeUndefined();
});
});
describe('with initialized data', (): void => {
let root: TypeObject<typeof dummyDescription.root>;
let root2: TypeObject<typeof dummyDescription.root>;
let child: TypeObject<typeof dummyDescription.child>;
let child2: TypeObject<typeof dummyDescription.child>;
let child3: TypeObject<typeof dummyDescription.child>;
let grandchild: TypeObject<typeof dummyDescription.grandchild>;
let grandchild2: TypeObject<typeof dummyDescription.grandchild>;
let grandchild3: TypeObject<typeof dummyDescription.grandchild>;
let otherChild: TypeObject<typeof dummyDescription.otherChild>;
beforeEach(async(): Promise<void> => {
await storage.defineType('root', dummyDescription.root);
await storage.defineType('child', dummyDescription.child);
await storage.defineType('grandchild', dummyDescription.grandchild);
await storage.defineType('otherChild', dummyDescription.otherChild);
await storage.createIndex('root', 'required');
await storage.createIndex('root', 'optional');
await storage.createIndex('child', 'name');
await storage.createIndex('grandchild', 'bool');
await storage.createIndex('otherChild', 'name');
root = await storage.create('root', { required: 5, notIndexed: 0 });
child = await storage.create('child', { name: 'child', parent: root.id, notIndexed: 1 });
grandchild = await storage.create('grandchild', { bool: true, parent: child.id, notIndexed: 2 });
otherChild = await storage.create('otherChild', { name: 'otherChild', parent: root.id, notIndexed: 3 });
// Extra resources for query tests
root2 = await storage.create('root', { required: 5, optional: 'defined', notIndexed: 1 });
child2 = await storage.create('child', { name: 'child2', parent: root.id, notIndexed: 1 });
child3 = await storage.create('child', { name: 'child', parent: root2.id, notIndexed: 1 });
grandchild2 = await storage.create('grandchild', { bool: false, parent: child.id, notIndexed: 2 });
grandchild3 = await storage.create('grandchild', { bool: true, parent: child2.id, notIndexed: 2 });
});
it('can verify existence.', async(): Promise<void> => {
await expect(storage.has('root', root.id)).resolves.toBe(true);
await expect(storage.has('child', child.id)).resolves.toBe(true);
await expect(storage.has('grandchild', grandchild.id)).resolves.toBe(true);
await expect(storage.has('otherChild', otherChild.id)).resolves.toBe(true);
await expect(storage.has('root', '???')).resolves.toBe(false);
await expect(storage.has('child', '???')).resolves.toBe(false);
});
it('can return data.', async(): Promise<void> => {
await expect(storage.get('root', root.id)).resolves.toEqual(root);
await expect(storage.get('child', child.id)).resolves.toEqual(child);
await expect(storage.get('grandchild', grandchild.id)).resolves.toEqual(grandchild);
await expect(storage.get('otherChild', otherChild.id)).resolves.toEqual(otherChild);
});
it('returns undefined if there is no match.', async(): Promise<void> => {
await expect(storage.get('root', child.id)).resolves.toBeUndefined();
await expect(storage.get('child', root.id)).resolves.toBeUndefined();
await expect(storage.get('grandchild', otherChild.id)).resolves.toBeUndefined();
await expect(storage.get('otherChild', grandchild.id)).resolves.toBeUndefined();
});
it('can update entries.', async(): Promise<void> => {
await expect(storage.set('root', { [INDEX_ID_KEY]: root.id, required: -10, notIndexed: -1 }))
.resolves.toBeUndefined();
await expect(storage.get('root', root.id))
.resolves.toEqual({ [INDEX_ID_KEY]: root.id, required: -10, notIndexed: -1 });
await expect(storage.set('child', { ...child, name: 'newChild', notIndexed: -2 })).resolves.toBeUndefined();
await expect(storage.get('child', child.id)).resolves.toEqual({ ...child, name: 'newChild', notIndexed: -2 });
await expect(storage.set('grandchild', { ...grandchild, bool: false, notIndexed: -3 })).resolves.toBeUndefined();
await expect(storage.get('grandchild', grandchild.id))
.resolves.toEqual({ ...grandchild, bool: false, notIndexed: -3 });
await expect(storage.set('otherChild', { ...otherChild, name: 'newOtherChild', notIndexed: -4 }))
.resolves.toBeUndefined();
await expect(storage.get('otherChild', otherChild.id))
.resolves.toEqual({ ...otherChild, name: 'newOtherChild', notIndexed: -4 });
});
it('errors when trying to update unknown entries.', async(): Promise<void> => {
await expect(storage.set('root', { [INDEX_ID_KEY]: '???', required: -10, notIndexed: -1 }))
.rejects.toThrow(NotFoundHttpError);
await expect(storage.set('child', { ...child, [INDEX_ID_KEY]: '???' }))
.rejects.toThrow(NotFoundHttpError);
});
it('errors when trying to update references.', async(): Promise<void> => {
await expect(storage.set('child', { ...child, parent: 'somewhereElse' }))
.rejects.toThrow(NotImplementedHttpError);
});
it('can update specific fields.', async(): Promise<void> => {
await expect(storage.setField('root', root.id, 'notIndexed', -1))
.resolves.toBeUndefined();
await expect(storage.get('root', root.id))
.resolves.toEqual({ ...root, notIndexed: -1 });
await expect(storage.setField('child', child.id, 'notIndexed', -2))
.resolves.toBeUndefined();
await expect(storage.get('child', child.id))
.resolves.toEqual({ ...child, notIndexed: -2 });
await expect(storage.setField('grandchild', grandchild.id, 'notIndexed', -2))
.resolves.toBeUndefined();
await expect(storage.get('grandchild', grandchild.id))
.resolves.toEqual({ ...grandchild, notIndexed: -2 });
await expect(storage.setField('otherChild', otherChild.id, 'notIndexed', -3))
.resolves.toBeUndefined();
await expect(storage.get('otherChild', otherChild.id))
.resolves.toEqual({ ...otherChild, notIndexed: -3 });
});
it('errors when trying to update a field in unknown entries.', async(): Promise<void> => {
await expect(storage.setField('root', '???', 'notIndexed', -1))
.rejects.toThrow(NotFoundHttpError);
await expect(storage.setField('child', '???', 'notIndexed', -1))
.rejects.toThrow(NotFoundHttpError);
});
it('errors when trying to update a reference field.', async(): Promise<void> => {
await expect(storage.setField('child', child.id, 'parent', 'somewhereElse'))
.rejects.toThrow(NotImplementedHttpError);
});
it('can remove resource.', async(): Promise<void> => {
await expect(storage.delete('otherChild', otherChild.id)).resolves.toBeUndefined();
await expect(storage.get('otherChild', otherChild.id)).resolves.toBeUndefined();
await expect(storage.delete('grandchild', grandchild.id)).resolves.toBeUndefined();
await expect(storage.get('grandchild', grandchild.id)).resolves.toBeUndefined();
await expect(storage.delete('child', child.id)).resolves.toBeUndefined();
await expect(storage.get('child', child.id)).resolves.toBeUndefined();
await expect(storage.delete('root', root.id)).resolves.toBeUndefined();
await expect(storage.get('root', root.id)).resolves.toBeUndefined();
});
it('does nothing when removing a resource that does not exist.', async(): Promise<void> => {
await expect(storage.delete('otherChild', otherChild.id)).resolves.toBeUndefined();
await expect(storage.delete('otherChild', otherChild.id)).resolves.toBeUndefined();
await expect(storage.delete('root', root.id)).resolves.toBeUndefined();
await expect(storage.delete('root', root.id)).resolves.toBeUndefined();
});
it('removes all dependent resources when deleting.', async(): Promise<void> => {
await expect(storage.delete('child', child.id)).resolves.toBeUndefined();
await expect(storage.get('grandchild', grandchild.id)).resolves.toBeUndefined();
await expect(storage.get('otherChild', otherChild.id)).resolves.toEqual(otherChild);
await expect(storage.delete('root', root.id)).resolves.toBeUndefined();
await expect(storage.get('otherChild', otherChild.id)).resolves.toBeUndefined();
});
it('can find objects using queries.', async(): Promise<void> => {
await expect(storage.find('root', { required: 5 })).resolves.toEqual([ root, root2 ]);
await expect(storage.find('root', { required: 5, notIndexed: 0 })).resolves.toEqual([ root ]);
await expect(storage.find('root', { optional: 'defined' })).resolves.toEqual([ root2 ]);
await expect(storage.find('root', { required: 5, optional: undefined })).resolves.toEqual([ root ]);
await expect(storage.find('child', { parent: root[INDEX_ID_KEY] })).resolves.toEqual([ child, child2 ]);
await expect(storage.find('child', { parent: root[INDEX_ID_KEY], name: 'child' })).resolves.toEqual([ child ]);
await expect(storage.find('child', { parent: root[INDEX_ID_KEY], name: 'child2' })).resolves.toEqual([ child2 ]);
await expect(storage.find('child', { name: 'child' })).resolves.toEqual([ child, child3 ]);
await expect(storage.find('child', { parent: { [INDEX_ID_KEY]: root[INDEX_ID_KEY] }, name: 'child0' }))
.resolves.toEqual([]);
await expect(storage.find('grandchild', { parent: child[INDEX_ID_KEY] }))
.resolves.toEqual([ grandchild, grandchild2 ]);
await expect(storage.find('grandchild', { parent: child2[INDEX_ID_KEY] }))
.resolves.toEqual([ grandchild3 ]);
await expect(storage.find('grandchild', { bool: true }))
.resolves.toEqual([ grandchild, grandchild3 ]);
});
it('can perform nested queries.', async(): Promise<void> => {
await expect(storage.find('grandchild', { parent: { name: 'child' }}))
.resolves.toEqual([ grandchild, grandchild2 ]);
await expect(storage.find('grandchild', { bool: true, parent: { notIndexed: 1 }}))
.resolves.toEqual([ grandchild, grandchild3 ]);
await expect(storage.find('grandchild', { bool: true, parent: { parent: { required: 5 }}}))
.resolves.toEqual([ grandchild, grandchild3 ]);
});
it('can also find just the IDs of the results.', async(): Promise<void> => {
await expect(storage.findIds('root', { required: 5 }))
.resolves.toEqual([ root[INDEX_ID_KEY], root2[INDEX_ID_KEY] ]);
await expect(storage.findIds('child', { name: 'child' }))
.resolves.toEqual([ child[INDEX_ID_KEY], child3[INDEX_ID_KEY] ]);
});
it('requires at least one index when finding results.', async(): Promise<void> => {
await expect(storage.findIds('root', { notIndexed: 0 })).rejects.toThrow(InternalServerError);
await expect(storage.findIds('root', { optional: undefined })).rejects.toThrow(InternalServerError);
await expect(storage.findIds('child', { notIndexed: 0 })).rejects.toThrow(InternalServerError);
await expect(storage.findIds('grandchild', { notIndexed: 0 })).rejects.toThrow(InternalServerError);
await expect(storage.findIds('otherChild', { notIndexed: 0 })).rejects.toThrow(InternalServerError);
});
it('can iterate over all entries of a type.', async(): Promise<void> => {
const roots: unknown[] = [];
for await (const entry of storage.entries('root')) {
roots.push(entry);
}
expect(roots).toEqual([ root, root2 ]);
const children: unknown[] = [];
for await (const entry of storage.entries('child')) {
children.push(entry);
}
expect(children).toEqual([ child, child2, child3 ]);
});
it('errors if there is index corruption.', async(): Promise<void> => {
// Corrupt the index. Will break if we change how index keys get generated.
indexMap.set(`child/${child.id}`, [ root2.id ]);
await expect(storage.create('grandchild', { bool: false, notIndexed: 5, parent: child.id }))
.rejects.toThrow(InternalServerError);
});
it('ignores index corruption when deleting keys.', async(): Promise<void> => {
// Corrupt the index. Will break if we change how index keys get generated.
indexMap.delete(`child/name/${child.name}`);
await expect(storage.delete('child', child.id)).resolves.toBeUndefined();
});
});
});