Files
CommunitySolidServer/test/util/Util.ts
Arthur Joppart 0cb4d7b161 feat: Add support for quota limits
* feat: implemented SizeReporter and FileSizeReporter

* test: FileSizeReporter tests

* feat: added QuotedDataAccessor

* test: added extra test to check recursiveness of filesizereporter

* feat: added QuotaStrategy interface

* feat: further progress in different files

* feat: wrote doc, tests and improved code

* feat: fixed bugs and code is now runnable and buildable

* feat: finished implementation

* fix: revert accidental chanegs

* fix: fileSizeReported did not count container size

* fix: bug calculating container sizes fixed

* test: FileSizeReporter tests

* test: QuotaDataValidator tests

* test: QuotaError tests

* fix: removed console.log

* doc: added doc to several files

* doc: changed doc for QuotaStrategy to new implementation

* fix: improved content length regex

* feat: improved GlobalQuotaStrategy code

* fix: made FileSizeReported readonly

* feat: added comments to quota-file.json

* fix: changed default tempFilePath variable

* test: included new tempFilePath variable in testing

* chore: created seperate command for start:file:quota to pass tests

* feat: removed all sync fs calls from FileSizeReporter

* feat: minor changes in multple files

* fix: changed function signatures to be in line with others

* feat: optimized quota data validation

* feat: improved FileSizeReporter code

* fix: corrected calculation of containersizes and fixed erroring edgecase

* feat: save content-length as number in metadata

* feat: added comments and changed GlobalQuotaStrategy constructor

* feat: changed file names and added small comment

* test: AtomicFileDataAccessor tests

* test: completed FileSizeReporter tests

* fix: content-length is now saved correctly in RepresentationMetadata

* feat: adapted content length metadata + tests

* fix: removed tempFilePath variable

* fix: reverted .gitignore

* fix: forgot to remove tempFilePath variable from componentsjs config

* test: GlobalQuotaStrategy tests

* feat: replaced DataValidator with Validator

* feat: reworked DataValidator

* feat: added calcultateChunkSize() to SizeReporter

* test: updated FileSizeReporter tests

* fix: tempFile location now relative to rootFilePath

* test: QuotaDataValidator tests

* fix: corrected FileSizeReporter tests

* fix: adapted FileSizeReporter tests

* fix: FileSizeReporter bug on Windows

* fix: regex linting error

* feat: changed Validator class

* feat: added PodQuotaStrategy to enable suota on a per pod basis

* chore: bump context versions

* fix: Capitalized comments in json file

* chore: renamed ValidatorArgs to ValidatorInput

* chore: order all exports

* fix: made TODO comment clearer

* chore: added seperated config files for global and pod based quota + fixed comments

* chore: made minor changes to comments

* feat: added PassthroughDataAccessor

* feat: added PasstroughtDataAccessor + tests

* fix: added invalid header check to ContentLengthParser

* chore: improved mocks

* chore: move quota limit higher up in config

* fix: atomicity issue in AtomicFileDataAccessor

* chore: moved .internal folder to config from FileSizeReporter

* fix: improved algorithm to ignore folders while calculating file size in FileSizeReporter

* fix: changes to support containers in the future

* fix: added error handling to prevent reading of unexistent files

* feat: added generic type to SizeReporter to calculate chunk sizes

* test: use mocked DataAccessor

* chore: added some comments to test and made minor improvement

* fix: fs mock rename

* chore: QuotaStrategy.estimateSize refactor

* chore: move trackAvailableSpace to abstract class QuotaStrategy

* fix: improved test case

* test: quota integration tests

* chore: edited some comments

* chore: change lstat to stat

* feat: moved estimateSize to SizeReporter to be consistent with calcultateChunkSize

* test: finish up tests to reach coverage

* fix: basic config

* fix: minor changes to test CI run

* fix: small fix for windows

* fix: improved writing to file

* chore: linting errors

* chore: rename trackAvailableSpace

* test: improved integration tests

* test: logging info for test debugging

* test: extra logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: improved Quota integration test setup

* test: improve quota tests for CI run

* test: debugging Quota test

* test: uncommented global quota test

* test: changed global quota parameters

* test: logging for debugging

* test: logging cleanup

* chore: minor changes, mostly typo fixes

* chore: remove console.log

* fix: getting inconsistent results

* chore: try fix index.ts CI error

* chore: try fix CI error

* chore: try fix CI error

* chore: revert last commits

* chore: fix inconsistent files with origin

* test: minor test improvements

* chore: minor refactors and improvements

* fix: added extra try catch for breaking bug

* chore: improve config

* chore: minor code improvements

* test: use mockFs

* feat: add extra check in podQuotaStrategy

* chore: replace handle by handleSafe in ValidatingDataAccessor

* chore: typo

* test: improved Quota integration tests

* test: made comment in test more correct

* fix: rm -> rmdir for backwards compatibility

* fix: fsPromises issue

* chore: leave out irrelevant config

* chore: removed start script from package.json

* fix: Small fixes

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>
2022-01-21 10:49:05 +01:00

227 lines
7.5 KiB
TypeScript

import type { Dirent, Stats } from 'fs';
import { PassThrough, Readable } from 'stream';
import type { SystemError } from '../../src/util/errors/SystemError';
const portNames = [
// Integration
'Conditions',
'ContentNegotiation',
'DynamicPods',
'Identity',
'LpdHandlerWithAuth',
'LpdHandlerWithoutAuth',
'Middleware',
'PodCreation',
'RedisResourceLocker',
'RestrictedIdentity',
'ServerFetch',
'SetupMemory',
'SparqlStorage',
'Subdomains',
'WebSocketsProtocol',
'PodQuota',
'GlobalQuota',
// Unit
'BaseHttpServerFactory',
] as const;
export function getPort(name: typeof portNames[number]): number {
const idx = portNames.indexOf(name);
// Just in case something doesn't listen to the typings
if (idx < 0) {
throw new Error(`Unknown port name ${name}`);
}
return 6000 + idx;
}
export function describeIf(envFlag: string, name: string, fn: () => void): void {
const flag = `TEST_${envFlag.toUpperCase()}`;
const enabled = !/^(|0|false)$/iu.test(process.env[flag] ?? '');
// eslint-disable-next-line jest/valid-describe, jest/valid-title, jest/no-disabled-tests
return enabled ? describe(name, fn) : describe.skip(name, fn);
}
/**
* Mocks (some) functions of the fs system library.
* It is important that you call `jest.mock('fs');` in your test file before calling this!!!
*
* This function will return an object of which the `data` field corresponds to the contents of the root folder.
* The file system can be "reset" by assigning an empty object (`{}`) to the data field.
*
* Only files and directories are supported.
* Files are stored as strings, directories as objects with the keys corresponding to its contents.
* File path `/folder/folder2/file` will correspond to `data['folder']['folder2']['file']`.
* This can both be used to check if a file/directory was created,
* or to specify in advance certain files on the "file system".
*
* Data streams will be converted to strings for files by concatenating the contents.
*
* @param rootFilepath - The name of the root folder in which fs will start.
* @param time - The date object to use for time functions (currently only mtime from lstats)
*/
export function mockFs(rootFilepath?: string, time?: Date): { data: any } {
const cache: { data: any } = { data: {}};
rootFilepath = rootFilepath ?? 'folder';
time = time ?? new Date();
// eslint-disable-next-line unicorn/consistent-function-scoping
function throwSystemError(code: string): void {
const error = new Error('error') as SystemError;
error.code = code;
error.syscall = 'this exists for isSystemError';
throw error;
}
function getFolder(path: string): { folder: any; name: string } {
let parts = path.slice(rootFilepath!.length).split('/').filter((part): boolean => part.length > 0);
if (parts.length === 0) {
return { folder: cache, name: 'data' };
}
const name = parts.slice(-1)[0];
parts = parts.slice(0, -1);
let folder = cache.data;
parts.forEach((part): any => {
if (typeof folder === 'string') {
throwSystemError('ENOTDIR');
}
folder = folder[part];
if (!folder) {
throwSystemError('ENOENT');
}
});
return { folder, name };
}
const mock = {
createReadStream(path: string): any {
const { folder, name } = getFolder(path);
return Readable.from([ folder[name] ]);
},
createWriteStream(path: string): any {
const { folder, name } = getFolder(path);
folder[name] = '';
const stream = new PassThrough();
stream.on('data', (data): any => {
folder[name] += data;
});
stream.on('end', (): any => stream.emit('finish'));
return stream;
},
promises: {
async stat(path: string): Promise<Stats> {
return this.lstat(await this.realpath(path));
},
async lstat(path: string): Promise<Stats> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
return {
isFile: (): boolean => typeof folder[name] === 'string',
isDirectory: (): boolean => typeof folder[name] === 'object',
isSymbolicLink: (): boolean => typeof folder[name] === 'symbol',
size: typeof folder[name] === 'string' ? folder[name].length : 4,
mtime: time,
} as Stats;
},
async unlink(path: string): Promise<void> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
if (!(await this.lstat(path)).isFile()) {
throwSystemError('EISDIR');
}
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete folder[name];
},
async symlink(target: string, path: string): Promise<void> {
const { folder, name } = getFolder(path);
folder[name] = Symbol(target);
},
async realpath(path: string): Promise<string> {
const { folder, name } = getFolder(path);
const entry = folder[name];
return typeof entry === 'symbol' ? entry.description ?? 'invalid' : path;
},
async rmdir(path: string): Promise<void> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
if (Object.keys(folder[name]).length > 0) {
throwSystemError('ENOTEMPTY');
}
if (!(await this.lstat(path)).isDirectory()) {
throwSystemError('ENOTDIR');
}
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete folder[name];
},
async readdir(path: string): Promise<string[]> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
return Object.keys(folder[name]);
},
async* opendir(path: string): AsyncIterableIterator<Dirent> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
for (const [ child, entry ] of Object.entries(folder[name])) {
yield {
name: child,
isFile: (): boolean => typeof entry === 'string',
isDirectory: (): boolean => typeof entry === 'object',
isSymbolicLink: (): boolean => typeof entry === 'symbol',
} as Dirent;
}
},
async mkdir(path: string): Promise<void> {
const { folder, name } = getFolder(path);
if (folder[name]) {
throwSystemError('EEXIST');
}
folder[name] = {};
},
async readFile(path: string): Promise<string> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
return folder[name];
},
async writeFile(path: string, data: string): Promise<void> {
const { folder, name } = getFolder(path);
folder[name] = data;
},
async rename(path: string, destination: string): Promise<void> {
const { folder, name } = getFolder(path);
if (!folder[name]) {
throwSystemError('ENOENT');
}
if (!(await this.lstat(path)).isFile()) {
throwSystemError('EISDIR');
}
const { folder: folderDest, name: nameDest } = getFolder(destination);
folderDest[nameDest] = folder[name];
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete folder[name];
},
},
};
const fs = jest.requireMock('fs');
Object.assign(fs, mock);
return cache;
}