feat: Add support for quota limits

* feat: implemented SizeReporter and FileSizeReporter

* test: FileSizeReporter tests

* feat: added QuotedDataAccessor

* test: added extra test to check recursiveness of filesizereporter

* feat: added QuotaStrategy interface

* feat: further progress in different files

* feat: wrote doc, tests and improved code

* feat: fixed bugs and code is now runnable and buildable

* feat: finished implementation

* fix: revert accidental chanegs

* fix: fileSizeReported did not count container size

* fix: bug calculating container sizes fixed

* test: FileSizeReporter tests

* test: QuotaDataValidator tests

* test: QuotaError tests

* fix: removed console.log

* doc: added doc to several files

* doc: changed doc for QuotaStrategy to new implementation

* fix: improved content length regex

* feat: improved GlobalQuotaStrategy code

* fix: made FileSizeReported readonly

* feat: added comments to quota-file.json

* fix: changed default tempFilePath variable

* test: included new tempFilePath variable in testing

* chore: created seperate command for start:file:quota to pass tests

* feat: removed all sync fs calls from FileSizeReporter

* feat: minor changes in multple files

* fix: changed function signatures to be in line with others

* feat: optimized quota data validation

* feat: improved FileSizeReporter code

* fix: corrected calculation of containersizes and fixed erroring edgecase

* feat: save content-length as number in metadata

* feat: added comments and changed GlobalQuotaStrategy constructor

* feat: changed file names and added small comment

* test: AtomicFileDataAccessor tests

* test: completed FileSizeReporter tests

* fix: content-length is now saved correctly in RepresentationMetadata

* feat: adapted content length metadata + tests

* fix: removed tempFilePath variable

* fix: reverted .gitignore

* fix: forgot to remove tempFilePath variable from componentsjs config

* test: GlobalQuotaStrategy tests

* feat: replaced DataValidator with Validator

* feat: reworked DataValidator

* feat: added calcultateChunkSize() to SizeReporter

* test: updated FileSizeReporter tests

* fix: tempFile location now relative to rootFilePath

* test: QuotaDataValidator tests

* fix: corrected FileSizeReporter tests

* fix: adapted FileSizeReporter tests

* fix: FileSizeReporter bug on Windows

* fix: regex linting error

* feat: changed Validator class

* feat: added PodQuotaStrategy to enable suota on a per pod basis

* chore: bump context versions

* fix: Capitalized comments in json file

* chore: renamed ValidatorArgs to ValidatorInput

* chore: order all exports

* fix: made TODO comment clearer

* chore: added seperated config files for global and pod based quota + fixed comments

* chore: made minor changes to comments

* feat: added PassthroughDataAccessor

* feat: added PasstroughtDataAccessor + tests

* fix: added invalid header check to ContentLengthParser

* chore: improved mocks

* chore: move quota limit higher up in config

* fix: atomicity issue in AtomicFileDataAccessor

* chore: moved .internal folder to config from FileSizeReporter

* fix: improved algorithm to ignore folders while calculating file size in FileSizeReporter

* fix: changes to support containers in the future

* fix: added error handling to prevent reading of unexistent files

* feat: added generic type to SizeReporter to calculate chunk sizes

* test: use mocked DataAccessor

* chore: added some comments to test and made minor improvement

* fix: fs mock rename

* chore: QuotaStrategy.estimateSize refactor

* chore: move trackAvailableSpace to abstract class QuotaStrategy

* fix: improved test case

* test: quota integration tests

* chore: edited some comments

* chore: change lstat to stat

* feat: moved estimateSize to SizeReporter to be consistent with calcultateChunkSize

* test: finish up tests to reach coverage

* fix: basic config

* fix: minor changes to test CI run

* fix: small fix for windows

* fix: improved writing to file

* chore: linting errors

* chore: rename trackAvailableSpace

* test: improved integration tests

* test: logging info for test debugging

* test: extra logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: improved Quota integration test setup

* test: improve quota tests for CI run

* test: debugging Quota test

* test: uncommented global quota test

* test: changed global quota parameters

* test: logging for debugging

* test: logging cleanup

* chore: minor changes, mostly typo fixes

* chore: remove console.log

* fix: getting inconsistent results

* chore: try fix index.ts CI error

* chore: try fix CI error

* chore: try fix CI error

* chore: revert last commits

* chore: fix inconsistent files with origin

* test: minor test improvements

* chore: minor refactors and improvements

* fix: added extra try catch for breaking bug

* chore: improve config

* chore: minor code improvements

* test: use mockFs

* feat: add extra check in podQuotaStrategy

* chore: replace handle by handleSafe in ValidatingDataAccessor

* chore: typo

* test: improved Quota integration tests

* test: made comment in test more correct

* fix: rm -> rmdir for backwards compatibility

* fix: fsPromises issue

* chore: leave out irrelevant config

* chore: removed start script from package.json

* fix: Small fixes

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>

test/unit/http/auxiliary/ComposedAuxiliaryStrategy.test.ts

@@ -61,10 +61,10 @@ describe('A ComposedAuxiliaryStrategy', (): void => {
   });
 
   it('validates data through the Validator.', async(): Promise<void> => {
-    const representation = { data: 'data!' } as any;
+    const representation = { data: 'data!', metadata: { identifier: { value: 'any' }}} as any;
     await expect(strategy.validate(representation)).resolves.toBeUndefined();
     expect(validator.handleSafe).toHaveBeenCalledTimes(1);
-    expect(validator.handleSafe).toHaveBeenLastCalledWith(representation);
+    expect(validator.handleSafe).toHaveBeenLastCalledWith({ representation, identifier: { path: 'any' }});
   });
 
   it('defaults isRequiredInRoot to false.', async(): Promise<void> => {

test/unit/http/auxiliary/RdfValidator.test.ts

@@ -1,5 +1,6 @@
 import { RdfValidator } from '../../../../src/http/auxiliary/RdfValidator';
 import { BasicRepresentation } from '../../../../src/http/representation/BasicRepresentation';
+import type { ResourceIdentifier } from '../../../../src/http/representation/ResourceIdentifier';
 import type { RepresentationConverter } from '../../../../src/storage/conversion/RepresentationConverter';
 import { readableToString } from '../../../../src/util/StreamUtil';
 import { StaticAsyncHandler } from '../../../util/StaticAsyncHandler';
@@ -8,6 +9,7 @@ import 'jest-rdf';
 describe('An RdfValidator', (): void => {
   let converter: RepresentationConverter;
   let validator: RdfValidator;
+  const identifier: ResourceIdentifier = { path: 'any/path' };
 
   beforeEach(async(): Promise<void> => {
     converter = new StaticAsyncHandler<any>(true, null);
@@ -20,14 +22,15 @@ describe('An RdfValidator', (): void => {
 
   it('always accepts content-type internal/quads.', async(): Promise<void> => {
     const representation = new BasicRepresentation('data', 'internal/quads');
-    await expect(validator.handle(representation)).resolves.toBeUndefined();
+    await expect(validator.handle({ representation, identifier })).resolves.toEqual(representation);
   });
 
   it('validates data by running it through a converter.', async(): Promise<void> => {
     converter.handleSafe = jest.fn().mockResolvedValue(new BasicRepresentation('transformedData', 'wrongType'));
     const representation = new BasicRepresentation('data', 'content-type');
     const quads = representation.metadata.quads();
-    await expect(validator.handle(representation)).resolves.toBeUndefined();
+    // Output is not important for this Validator
+    await expect(validator.handle({ representation, identifier })).resolves.toBeDefined();
     // Make sure the data can still be streamed
     await expect(readableToString(representation.data)).resolves.toBe('data');
     // Make sure the metadata was not changed
@@ -37,7 +40,7 @@ describe('An RdfValidator', (): void => {
   it('throws an error when validating invalid data.', async(): Promise<void> => {
     converter.handleSafe = jest.fn().mockRejectedValue(new Error('bad data!'));
     const representation = new BasicRepresentation('data', 'content-type');
-    await expect(validator.handle(representation)).rejects.toThrow('bad data!');
+    await expect(validator.handle({ representation, identifier })).rejects.toThrow('bad data!');
     // Make sure the data on the readable has not been reset
     expect(representation.data.destroyed).toBe(true);
   });

test/unit/http/input/metadata/ContentLengthParser.test.ts

@@ -0,0 +1,32 @@
+import { ContentLengthParser } from '../../../../../src/http/input/metadata/ContentLengthParser';
+import { RepresentationMetadata } from '../../../../../src/http/representation/RepresentationMetadata';
+import type { HttpRequest } from '../../../../../src/server/HttpRequest';
+
+describe('A ContentLengthParser', (): void => {
+  const parser = new ContentLengthParser();
+  let request: HttpRequest;
+  let metadata: RepresentationMetadata;
+
+  beforeEach(async(): Promise<void> => {
+    request = { headers: {}} as HttpRequest;
+    metadata = new RepresentationMetadata();
+  });
+
+  it('does nothing if there is no content-length header.', async(): Promise<void> => {
+    await expect(parser.handle({ request, metadata })).resolves.toBeUndefined();
+    expect(metadata.quads()).toHaveLength(0);
+  });
+
+  it('sets the given content-length as metadata.', async(): Promise<void> => {
+    request.headers['content-length'] = '50';
+    await expect(parser.handle({ request, metadata })).resolves.toBeUndefined();
+    expect(metadata.quads()).toHaveLength(1);
+    expect(metadata.contentLength).toBe(50);
+  });
+
+  it('does not set a content-length when the header is invalid.', async(): Promise<void> => {
+    request.headers['content-length'] = 'aabbcc50ccbbaa';
+    await expect(parser.handle({ request, metadata })).resolves.toBeUndefined();
+    expect(metadata.quads()).toHaveLength(0);
+  });
+});

test/unit/http/representation/RepresentationMetadata.test.ts

@@ -60,6 +60,16 @@ describe('A RepresentationMetadata', (): void => {
       expect(metadata.contentType).toEqual('text/turtle');
     });
 
+    it('stores the content-length correctly.', async(): Promise<void> => {
+      metadata = new RepresentationMetadata();
+      metadata.contentLength = 50;
+      expect(metadata.contentLength).toEqual(50);
+
+      metadata = new RepresentationMetadata();
+      metadata.contentLength = undefined;
+      expect(metadata.contentLength).toBeUndefined();
+    });
+
     it('copies an other metadata object.', async(): Promise<void> => {
       const other = new RepresentationMetadata({ path: 'otherId' }, { 'test:pred': 'objVal' });
       metadata = new RepresentationMetadata(other);