mirror of
https://github.com/CommunitySolidServer/CommunitySolidServer.git
synced 2024-10-03 14:55:10 +00:00
* feat: implemented SizeReporter and FileSizeReporter * test: FileSizeReporter tests * feat: added QuotedDataAccessor * test: added extra test to check recursiveness of filesizereporter * feat: added QuotaStrategy interface * feat: further progress in different files * feat: wrote doc, tests and improved code * feat: fixed bugs and code is now runnable and buildable * feat: finished implementation * fix: revert accidental chanegs * fix: fileSizeReported did not count container size * fix: bug calculating container sizes fixed * test: FileSizeReporter tests * test: QuotaDataValidator tests * test: QuotaError tests * fix: removed console.log * doc: added doc to several files * doc: changed doc for QuotaStrategy to new implementation * fix: improved content length regex * feat: improved GlobalQuotaStrategy code * fix: made FileSizeReported readonly * feat: added comments to quota-file.json * fix: changed default tempFilePath variable * test: included new tempFilePath variable in testing * chore: created seperate command for start:file:quota to pass tests * feat: removed all sync fs calls from FileSizeReporter * feat: minor changes in multple files * fix: changed function signatures to be in line with others * feat: optimized quota data validation * feat: improved FileSizeReporter code * fix: corrected calculation of containersizes and fixed erroring edgecase * feat: save content-length as number in metadata * feat: added comments and changed GlobalQuotaStrategy constructor * feat: changed file names and added small comment * test: AtomicFileDataAccessor tests * test: completed FileSizeReporter tests * fix: content-length is now saved correctly in RepresentationMetadata * feat: adapted content length metadata + tests * fix: removed tempFilePath variable * fix: reverted .gitignore * fix: forgot to remove tempFilePath variable from componentsjs config * test: GlobalQuotaStrategy tests * feat: replaced DataValidator with Validator * feat: reworked DataValidator * feat: added calcultateChunkSize() to SizeReporter * test: updated FileSizeReporter tests * fix: tempFile location now relative to rootFilePath * test: QuotaDataValidator tests * fix: corrected FileSizeReporter tests * fix: adapted FileSizeReporter tests * fix: FileSizeReporter bug on Windows * fix: regex linting error * feat: changed Validator class * feat: added PodQuotaStrategy to enable suota on a per pod basis * chore: bump context versions * fix: Capitalized comments in json file * chore: renamed ValidatorArgs to ValidatorInput * chore: order all exports * fix: made TODO comment clearer * chore: added seperated config files for global and pod based quota + fixed comments * chore: made minor changes to comments * feat: added PassthroughDataAccessor * feat: added PasstroughtDataAccessor + tests * fix: added invalid header check to ContentLengthParser * chore: improved mocks * chore: move quota limit higher up in config * fix: atomicity issue in AtomicFileDataAccessor * chore: moved .internal folder to config from FileSizeReporter * fix: improved algorithm to ignore folders while calculating file size in FileSizeReporter * fix: changes to support containers in the future * fix: added error handling to prevent reading of unexistent files * feat: added generic type to SizeReporter to calculate chunk sizes * test: use mocked DataAccessor * chore: added some comments to test and made minor improvement * fix: fs mock rename * chore: QuotaStrategy.estimateSize refactor * chore: move trackAvailableSpace to abstract class QuotaStrategy * fix: improved test case * test: quota integration tests * chore: edited some comments * chore: change lstat to stat * feat: moved estimateSize to SizeReporter to be consistent with calcultateChunkSize * test: finish up tests to reach coverage * fix: basic config * fix: minor changes to test CI run * fix: small fix for windows * fix: improved writing to file * chore: linting errors * chore: rename trackAvailableSpace * test: improved integration tests * test: logging info for test debugging * test: extra logging for debugging * test: logging for debugging * test: logging for debugging * test: logging for debugging * test: improved Quota integration test setup * test: improve quota tests for CI run * test: debugging Quota test * test: uncommented global quota test * test: changed global quota parameters * test: logging for debugging * test: logging cleanup * chore: minor changes, mostly typo fixes * chore: remove console.log * fix: getting inconsistent results * chore: try fix index.ts CI error * chore: try fix CI error * chore: try fix CI error * chore: revert last commits * chore: fix inconsistent files with origin * test: minor test improvements * chore: minor refactors and improvements * fix: added extra try catch for breaking bug * chore: improve config * chore: minor code improvements * test: use mockFs * feat: add extra check in podQuotaStrategy * chore: replace handle by handleSafe in ValidatingDataAccessor * chore: typo * test: improved Quota integration tests * test: made comment in test more correct * fix: rm -> rmdir for backwards compatibility * fix: fsPromises issue * chore: leave out irrelevant config * chore: removed start script from package.json * fix: Small fixes Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>
227 lines
7.5 KiB
TypeScript
227 lines
7.5 KiB
TypeScript
import type { Dirent, Stats } from 'fs';
|
|
import { PassThrough, Readable } from 'stream';
|
|
import type { SystemError } from '../../src/util/errors/SystemError';
|
|
|
|
const portNames = [
|
|
// Integration
|
|
'Conditions',
|
|
'ContentNegotiation',
|
|
'DynamicPods',
|
|
'Identity',
|
|
'LpdHandlerWithAuth',
|
|
'LpdHandlerWithoutAuth',
|
|
'Middleware',
|
|
'PodCreation',
|
|
'RedisResourceLocker',
|
|
'RestrictedIdentity',
|
|
'ServerFetch',
|
|
'SetupMemory',
|
|
'SparqlStorage',
|
|
'Subdomains',
|
|
'WebSocketsProtocol',
|
|
'PodQuota',
|
|
'GlobalQuota',
|
|
// Unit
|
|
'BaseHttpServerFactory',
|
|
] as const;
|
|
|
|
export function getPort(name: typeof portNames[number]): number {
|
|
const idx = portNames.indexOf(name);
|
|
// Just in case something doesn't listen to the typings
|
|
if (idx < 0) {
|
|
throw new Error(`Unknown port name ${name}`);
|
|
}
|
|
return 6000 + idx;
|
|
}
|
|
|
|
export function describeIf(envFlag: string, name: string, fn: () => void): void {
|
|
const flag = `TEST_${envFlag.toUpperCase()}`;
|
|
const enabled = !/^(|0|false)$/iu.test(process.env[flag] ?? '');
|
|
// eslint-disable-next-line jest/valid-describe, jest/valid-title, jest/no-disabled-tests
|
|
return enabled ? describe(name, fn) : describe.skip(name, fn);
|
|
}
|
|
|
|
/**
|
|
* Mocks (some) functions of the fs system library.
|
|
* It is important that you call `jest.mock('fs');` in your test file before calling this!!!
|
|
*
|
|
* This function will return an object of which the `data` field corresponds to the contents of the root folder.
|
|
* The file system can be "reset" by assigning an empty object (`{}`) to the data field.
|
|
*
|
|
* Only files and directories are supported.
|
|
* Files are stored as strings, directories as objects with the keys corresponding to its contents.
|
|
* File path `/folder/folder2/file` will correspond to `data['folder']['folder2']['file']`.
|
|
* This can both be used to check if a file/directory was created,
|
|
* or to specify in advance certain files on the "file system".
|
|
*
|
|
* Data streams will be converted to strings for files by concatenating the contents.
|
|
*
|
|
* @param rootFilepath - The name of the root folder in which fs will start.
|
|
* @param time - The date object to use for time functions (currently only mtime from lstats)
|
|
*/
|
|
export function mockFs(rootFilepath?: string, time?: Date): { data: any } {
|
|
const cache: { data: any } = { data: {}};
|
|
|
|
rootFilepath = rootFilepath ?? 'folder';
|
|
time = time ?? new Date();
|
|
|
|
// eslint-disable-next-line unicorn/consistent-function-scoping
|
|
function throwSystemError(code: string): void {
|
|
const error = new Error('error') as SystemError;
|
|
error.code = code;
|
|
error.syscall = 'this exists for isSystemError';
|
|
throw error;
|
|
}
|
|
|
|
function getFolder(path: string): { folder: any; name: string } {
|
|
let parts = path.slice(rootFilepath!.length).split('/').filter((part): boolean => part.length > 0);
|
|
|
|
if (parts.length === 0) {
|
|
return { folder: cache, name: 'data' };
|
|
}
|
|
|
|
const name = parts.slice(-1)[0];
|
|
parts = parts.slice(0, -1);
|
|
let folder = cache.data;
|
|
parts.forEach((part): any => {
|
|
if (typeof folder === 'string') {
|
|
throwSystemError('ENOTDIR');
|
|
}
|
|
folder = folder[part];
|
|
if (!folder) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
});
|
|
|
|
return { folder, name };
|
|
}
|
|
|
|
const mock = {
|
|
createReadStream(path: string): any {
|
|
const { folder, name } = getFolder(path);
|
|
return Readable.from([ folder[name] ]);
|
|
},
|
|
createWriteStream(path: string): any {
|
|
const { folder, name } = getFolder(path);
|
|
folder[name] = '';
|
|
const stream = new PassThrough();
|
|
stream.on('data', (data): any => {
|
|
folder[name] += data;
|
|
});
|
|
stream.on('end', (): any => stream.emit('finish'));
|
|
return stream;
|
|
},
|
|
promises: {
|
|
async stat(path: string): Promise<Stats> {
|
|
return this.lstat(await this.realpath(path));
|
|
},
|
|
async lstat(path: string): Promise<Stats> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
return {
|
|
isFile: (): boolean => typeof folder[name] === 'string',
|
|
isDirectory: (): boolean => typeof folder[name] === 'object',
|
|
isSymbolicLink: (): boolean => typeof folder[name] === 'symbol',
|
|
size: typeof folder[name] === 'string' ? folder[name].length : 4,
|
|
mtime: time,
|
|
} as Stats;
|
|
},
|
|
async unlink(path: string): Promise<void> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
if (!(await this.lstat(path)).isFile()) {
|
|
throwSystemError('EISDIR');
|
|
}
|
|
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
|
|
delete folder[name];
|
|
},
|
|
async symlink(target: string, path: string): Promise<void> {
|
|
const { folder, name } = getFolder(path);
|
|
folder[name] = Symbol(target);
|
|
},
|
|
async realpath(path: string): Promise<string> {
|
|
const { folder, name } = getFolder(path);
|
|
const entry = folder[name];
|
|
return typeof entry === 'symbol' ? entry.description ?? 'invalid' : path;
|
|
},
|
|
async rmdir(path: string): Promise<void> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
if (Object.keys(folder[name]).length > 0) {
|
|
throwSystemError('ENOTEMPTY');
|
|
}
|
|
if (!(await this.lstat(path)).isDirectory()) {
|
|
throwSystemError('ENOTDIR');
|
|
}
|
|
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
|
|
delete folder[name];
|
|
},
|
|
async readdir(path: string): Promise<string[]> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
return Object.keys(folder[name]);
|
|
},
|
|
async* opendir(path: string): AsyncIterableIterator<Dirent> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
for (const [ child, entry ] of Object.entries(folder[name])) {
|
|
yield {
|
|
name: child,
|
|
isFile: (): boolean => typeof entry === 'string',
|
|
isDirectory: (): boolean => typeof entry === 'object',
|
|
isSymbolicLink: (): boolean => typeof entry === 'symbol',
|
|
} as Dirent;
|
|
}
|
|
},
|
|
async mkdir(path: string): Promise<void> {
|
|
const { folder, name } = getFolder(path);
|
|
if (folder[name]) {
|
|
throwSystemError('EEXIST');
|
|
}
|
|
folder[name] = {};
|
|
},
|
|
async readFile(path: string): Promise<string> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
return folder[name];
|
|
},
|
|
async writeFile(path: string, data: string): Promise<void> {
|
|
const { folder, name } = getFolder(path);
|
|
folder[name] = data;
|
|
},
|
|
async rename(path: string, destination: string): Promise<void> {
|
|
const { folder, name } = getFolder(path);
|
|
if (!folder[name]) {
|
|
throwSystemError('ENOENT');
|
|
}
|
|
if (!(await this.lstat(path)).isFile()) {
|
|
throwSystemError('EISDIR');
|
|
}
|
|
|
|
const { folder: folderDest, name: nameDest } = getFolder(destination);
|
|
folderDest[nameDest] = folder[name];
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
|
|
delete folder[name];
|
|
},
|
|
},
|
|
};
|
|
|
|
const fs = jest.requireMock('fs');
|
|
Object.assign(fs, mock);
|
|
|
|
return cache;
|
|
}
|