Arthur Joppart 0cb4d7b161
feat: Add support for quota limits
* feat: implemented SizeReporter and FileSizeReporter

* test: FileSizeReporter tests

* feat: added QuotedDataAccessor

* test: added extra test to check recursiveness of filesizereporter

* feat: added QuotaStrategy interface

* feat: further progress in different files

* feat: wrote doc, tests and improved code

* feat: fixed bugs and code is now runnable and buildable

* feat: finished implementation

* fix: revert accidental chanegs

* fix: fileSizeReported did not count container size

* fix: bug calculating container sizes fixed

* test: FileSizeReporter tests

* test: QuotaDataValidator tests

* test: QuotaError tests

* fix: removed console.log

* doc: added doc to several files

* doc: changed doc for QuotaStrategy to new implementation

* fix: improved content length regex

* feat: improved GlobalQuotaStrategy code

* fix: made FileSizeReported readonly

* feat: added comments to quota-file.json

* fix: changed default tempFilePath variable

* test: included new tempFilePath variable in testing

* chore: created seperate command for start:file:quota to pass tests

* feat: removed all sync fs calls from FileSizeReporter

* feat: minor changes in multple files

* fix: changed function signatures to be in line with others

* feat: optimized quota data validation

* feat: improved FileSizeReporter code

* fix: corrected calculation of containersizes and fixed erroring edgecase

* feat: save content-length as number in metadata

* feat: added comments and changed GlobalQuotaStrategy constructor

* feat: changed file names and added small comment

* test: AtomicFileDataAccessor tests

* test: completed FileSizeReporter tests

* fix: content-length is now saved correctly in RepresentationMetadata

* feat: adapted content length metadata + tests

* fix: removed tempFilePath variable

* fix: reverted .gitignore

* fix: forgot to remove tempFilePath variable from componentsjs config

* test: GlobalQuotaStrategy tests

* feat: replaced DataValidator with Validator

* feat: reworked DataValidator

* feat: added calcultateChunkSize() to SizeReporter

* test: updated FileSizeReporter tests

* fix: tempFile location now relative to rootFilePath

* test: QuotaDataValidator tests

* fix: corrected FileSizeReporter tests

* fix: adapted FileSizeReporter tests

* fix: FileSizeReporter bug on Windows

* fix: regex linting error

* feat: changed Validator class

* feat: added PodQuotaStrategy to enable suota on a per pod basis

* chore: bump context versions

* fix: Capitalized comments in json file

* chore: renamed ValidatorArgs to ValidatorInput

* chore: order all exports

* fix: made TODO comment clearer

* chore: added seperated config files for global and pod based quota + fixed comments

* chore: made minor changes to comments

* feat: added PassthroughDataAccessor

* feat: added PasstroughtDataAccessor + tests

* fix: added invalid header check to ContentLengthParser

* chore: improved mocks

* chore: move quota limit higher up in config

* fix: atomicity issue in AtomicFileDataAccessor

* chore: moved .internal folder to config from FileSizeReporter

* fix: improved algorithm to ignore folders while calculating file size in FileSizeReporter

* fix: changes to support containers in the future

* fix: added error handling to prevent reading of unexistent files

* feat: added generic type to SizeReporter to calculate chunk sizes

* test: use mocked DataAccessor

* chore: added some comments to test and made minor improvement

* fix: fs mock rename

* chore: QuotaStrategy.estimateSize refactor

* chore: move trackAvailableSpace to abstract class QuotaStrategy

* fix: improved test case

* test: quota integration tests

* chore: edited some comments

* chore: change lstat to stat

* feat: moved estimateSize to SizeReporter to be consistent with calcultateChunkSize

* test: finish up tests to reach coverage

* fix: basic config

* fix: minor changes to test CI run

* fix: small fix for windows

* fix: improved writing to file

* chore: linting errors

* chore: rename trackAvailableSpace

* test: improved integration tests

* test: logging info for test debugging

* test: extra logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: improved Quota integration test setup

* test: improve quota tests for CI run

* test: debugging Quota test

* test: uncommented global quota test

* test: changed global quota parameters

* test: logging for debugging

* test: logging cleanup

* chore: minor changes, mostly typo fixes

* chore: remove console.log

* fix: getting inconsistent results

* chore: try fix index.ts CI error

* chore: try fix CI error

* chore: try fix CI error

* chore: revert last commits

* chore: fix inconsistent files with origin

* test: minor test improvements

* chore: minor refactors and improvements

* fix: added extra try catch for breaking bug

* chore: improve config

* chore: minor code improvements

* test: use mockFs

* feat: add extra check in podQuotaStrategy

* chore: replace handle by handleSafe in ValidatingDataAccessor

* chore: typo

* test: improved Quota integration tests

* test: made comment in test more correct

* fix: rm -> rmdir for backwards compatibility

* fix: fsPromises issue

* chore: leave out irrelevant config

* chore: removed start script from package.json

* fix: Small fixes

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>
2022-01-21 10:49:05 +01:00

121 lines
5.2 KiB
TypeScript

import type { Readable } from 'stream';
import { PassThrough } from 'stream';
import type { ValidatorInput } from '../../../../src/http/auxiliary/Validator';
import { BasicRepresentation } from '../../../../src/http/representation/BasicRepresentation';
import { RepresentationMetadata } from '../../../../src/http/representation/RepresentationMetadata';
import type { ResourceIdentifier } from '../../../../src/http/representation/ResourceIdentifier';
import type { QuotaStrategy } from '../../../../src/storage/quota/QuotaStrategy';
import { UNIT_BYTES } from '../../../../src/storage/size-reporter/Size';
import type { SizeReporter } from '../../../../src/storage/size-reporter/SizeReporter';
import { QuotaValidator } from '../../../../src/storage/validators/QuotaValidator';
import { guardStream } from '../../../../src/util/GuardedStream';
import type { Guarded } from '../../../../src/util/GuardedStream';
import { guardedStreamFrom, readableToString } from '../../../../src/util/StreamUtil';
describe('QuotaValidator', (): void => {
let mockedStrategy: jest.Mocked<QuotaStrategy>;
let validator: QuotaValidator;
let identifier: ResourceIdentifier;
let mockMetadata: RepresentationMetadata;
let mockData: Guarded<Readable>;
let mockInput: ValidatorInput;
let mockReporter: jest.Mocked<SizeReporter<any>>;
beforeEach((): void => {
jest.clearAllMocks();
identifier = { path: 'http://localhost/' };
mockMetadata = new RepresentationMetadata();
mockData = guardedStreamFrom([ 'test string' ]);
mockInput = {
representation: new BasicRepresentation(mockData, mockMetadata),
identifier,
};
mockReporter = {
getSize: jest.fn(),
getUnit: jest.fn(),
calculateChunkSize: jest.fn(),
estimateSize: jest.fn().mockResolvedValue(8),
};
mockedStrategy = {
reporter: mockReporter,
limit: { unit: UNIT_BYTES, amount: 8 },
getAvailableSpace: jest.fn().mockResolvedValue({ unit: UNIT_BYTES, amount: 10 }),
estimateSize: jest.fn().mockResolvedValue({ unit: UNIT_BYTES, amount: 8 }),
createQuotaGuard: jest.fn().mockResolvedValue(guardStream(new PassThrough())),
} as any;
validator = new QuotaValidator(mockedStrategy);
});
describe('handle()', (): void => {
// Step 2
it('should destroy the stream when estimated size is larger than the available size.', async(): Promise<void> => {
mockedStrategy.estimateSize.mockResolvedValueOnce({ unit: UNIT_BYTES, amount: 11 });
const result = validator.handle(mockInput);
await expect(result).resolves.toBeDefined();
const awaitedResult = await result;
const prom = new Promise<void>((resolve, reject): void => {
awaitedResult.data.on('error', (): void => resolve());
awaitedResult.data.on('end', (): void => reject(new Error('reject')));
});
// Consume the stream
await expect(readableToString(awaitedResult.data))
.rejects.toThrow('Quota exceeded: Advertised Content-Length is');
await expect(prom).resolves.toBeUndefined();
});
// Step 3
it('should destroy the stream when quota is exceeded during write.', async(): Promise<void> => {
mockedStrategy.createQuotaGuard.mockResolvedValueOnce(guardStream(new PassThrough({
async transform(this): Promise<void> {
this.destroy(new Error('error'));
},
})));
const result = validator.handle(mockInput);
await expect(result).resolves.toBeDefined();
const awaitedResult = await result;
const prom = new Promise<void>((resolve, reject): void => {
awaitedResult.data.on('error', (): void => resolve());
awaitedResult.data.on('end', (): void => reject(new Error('reject')));
});
// Consume the stream
await expect(readableToString(awaitedResult.data)).rejects.toThrow('error');
expect(mockedStrategy.createQuotaGuard).toHaveBeenCalledTimes(1);
await expect(prom).resolves.toBeUndefined();
});
// Step 4
it('should throw when quota were exceeded after stream was finished.', async(): Promise<void> => {
const result = validator.handle(mockInput);
// Putting this after the handle / before consuming the stream will only effect
// this function in the flush part of the code.
mockedStrategy.getAvailableSpace.mockResolvedValueOnce({ unit: UNIT_BYTES, amount: -100 });
await expect(result).resolves.toBeDefined();
const awaitedResult = await result;
const prom = new Promise<void>((resolve, reject): void => {
awaitedResult.data.on('error', (): void => resolve());
awaitedResult.data.on('end', (): void => reject(new Error('reject')));
});
// Consume the stream
await expect(readableToString(awaitedResult.data)).rejects.toThrow('Quota exceeded after write completed');
await expect(prom).resolves.toBeUndefined();
});
it('should return a stream that is consumable without error if quota isn\'t exceeded.', async(): Promise<void> => {
const result = validator.handle(mockInput);
await expect(result).resolves.toBeDefined();
const awaitedResult = await result;
await expect(readableToString(awaitedResult.data)).resolves.toBe('test string');
});
});
});