feat: Add support for quota limits

* feat: implemented SizeReporter and FileSizeReporter

* test: FileSizeReporter tests

* feat: added QuotedDataAccessor

* test: added extra test to check recursiveness of filesizereporter

* feat: added QuotaStrategy interface

* feat: further progress in different files

* feat: wrote doc, tests and improved code

* feat: fixed bugs and code is now runnable and buildable

* feat: finished implementation

* fix: revert accidental chanegs

* fix: fileSizeReported did not count container size

* fix: bug calculating container sizes fixed

* test: FileSizeReporter tests

* test: QuotaDataValidator tests

* test: QuotaError tests

* fix: removed console.log

* doc: added doc to several files

* doc: changed doc for QuotaStrategy to new implementation

* fix: improved content length regex

* feat: improved GlobalQuotaStrategy code

* fix: made FileSizeReported readonly

* feat: added comments to quota-file.json

* fix: changed default tempFilePath variable

* test: included new tempFilePath variable in testing

* chore: created seperate command for start:file:quota to pass tests

* feat: removed all sync fs calls from FileSizeReporter

* feat: minor changes in multple files

* fix: changed function signatures to be in line with others

* feat: optimized quota data validation

* feat: improved FileSizeReporter code

* fix: corrected calculation of containersizes and fixed erroring edgecase

* feat: save content-length as number in metadata

* feat: added comments and changed GlobalQuotaStrategy constructor

* feat: changed file names and added small comment

* test: AtomicFileDataAccessor tests

* test: completed FileSizeReporter tests

* fix: content-length is now saved correctly in RepresentationMetadata

* feat: adapted content length metadata + tests

* fix: removed tempFilePath variable

* fix: reverted .gitignore

* fix: forgot to remove tempFilePath variable from componentsjs config

* test: GlobalQuotaStrategy tests

* feat: replaced DataValidator with Validator

* feat: reworked DataValidator

* feat: added calcultateChunkSize() to SizeReporter

* test: updated FileSizeReporter tests

* fix: tempFile location now relative to rootFilePath

* test: QuotaDataValidator tests

* fix: corrected FileSizeReporter tests

* fix: adapted FileSizeReporter tests

* fix: FileSizeReporter bug on Windows

* fix: regex linting error

* feat: changed Validator class

* feat: added PodQuotaStrategy to enable suota on a per pod basis

* chore: bump context versions

* fix: Capitalized comments in json file

* chore: renamed ValidatorArgs to ValidatorInput

* chore: order all exports

* fix: made TODO comment clearer

* chore: added seperated config files for global and pod based quota + fixed comments

* chore: made minor changes to comments

* feat: added PassthroughDataAccessor

* feat: added PasstroughtDataAccessor + tests

* fix: added invalid header check to ContentLengthParser

* chore: improved mocks

* chore: move quota limit higher up in config

* fix: atomicity issue in AtomicFileDataAccessor

* chore: moved .internal folder to config from FileSizeReporter

* fix: improved algorithm to ignore folders while calculating file size in FileSizeReporter

* fix: changes to support containers in the future

* fix: added error handling to prevent reading of unexistent files

* feat: added generic type to SizeReporter to calculate chunk sizes

* test: use mocked DataAccessor

* chore: added some comments to test and made minor improvement

* fix: fs mock rename

* chore: QuotaStrategy.estimateSize refactor

* chore: move trackAvailableSpace to abstract class QuotaStrategy

* fix: improved test case

* test: quota integration tests

* chore: edited some comments

* chore: change lstat to stat

* feat: moved estimateSize to SizeReporter to be consistent with calcultateChunkSize

* test: finish up tests to reach coverage

* fix: basic config

* fix: minor changes to test CI run

* fix: small fix for windows

* fix: improved writing to file

* chore: linting errors

* chore: rename trackAvailableSpace

* test: improved integration tests

* test: logging info for test debugging

* test: extra logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: logging for debugging

* test: improved Quota integration test setup

* test: improve quota tests for CI run

* test: debugging Quota test

* test: uncommented global quota test

* test: changed global quota parameters

* test: logging for debugging

* test: logging cleanup

* chore: minor changes, mostly typo fixes

* chore: remove console.log

* fix: getting inconsistent results

* chore: try fix index.ts CI error

* chore: try fix CI error

* chore: try fix CI error

* chore: revert last commits

* chore: fix inconsistent files with origin

* test: minor test improvements

* chore: minor refactors and improvements

* fix: added extra try catch for breaking bug

* chore: improve config

* chore: minor code improvements

* test: use mockFs

* feat: add extra check in podQuotaStrategy

* chore: replace handle by handleSafe in ValidatingDataAccessor

* chore: typo

* test: improved Quota integration tests

* test: made comment in test more correct

* fix: rm -> rmdir for backwards compatibility

* fix: fsPromises issue

* chore: leave out irrelevant config

* chore: removed start script from package.json

* fix: Small fixes

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>

src/http/auxiliary/ComposedAuxiliaryStrategy.ts

@@ -58,7 +58,10 @@ export class ComposedAuxiliaryStrategy implements AuxiliaryStrategy {
 
   public async validate(representation: Representation): Promise<void> {
     if (this.validator) {
-      return this.validator.handleSafe(representation);
+      await this.validator.handleSafe({
+        representation,
+        identifier: { path: representation.metadata.identifier.value },
+      });
     }
   }
 }

src/http/auxiliary/RdfValidator.ts

@@ -3,6 +3,7 @@ import type { RepresentationConverter } from '../../storage/conversion/Represent
 import { INTERNAL_QUADS } from '../../util/ContentTypes';
 import { cloneRepresentation } from '../../util/ResourceUtil';
 import type { Representation } from '../representation/Representation';
+import type { ValidatorInput } from './Validator';
 import { Validator } from './Validator';
 
 /**
@@ -17,12 +18,11 @@ export class RdfValidator extends Validator {
     this.converter = converter;
   }
 
-  public async handle(representation: Representation): Promise<void> {
+  public async handle({ representation, identifier }: ValidatorInput): Promise<Representation> {
     // If the data already is quads format we know it's RDF
     if (representation.metadata.contentType === INTERNAL_QUADS) {
-      return;
+      return representation;
     }
-    const identifier = { path: representation.metadata.identifier.value };
     const preferences = { type: { [INTERNAL_QUADS]: 1 }};
     let result;
     try {
@@ -39,5 +39,7 @@ export class RdfValidator extends Validator {
     }
     // Drain stream to make sure data was parsed correctly
     await arrayifyStream(result.data);
+
+    return representation;
   }
 }

src/http/auxiliary/Validator.ts

@@ -1,7 +1,13 @@
 import { AsyncHandler } from '../../util/handlers/AsyncHandler';
 import type { Representation } from '../representation/Representation';
+import type { ResourceIdentifier } from '../representation/ResourceIdentifier';
+
+export type ValidatorInput = {
+  representation: Representation;
+  identifier: ResourceIdentifier;
+};
 
 /**
  * Generic interface for classes that validate Representations in some way.
  */
-export abstract class Validator extends AsyncHandler<Representation> { }
+export abstract class Validator extends AsyncHandler<ValidatorInput, Representation> { }

src/http/input/metadata/ContentLengthParser.ts

@@ -0,0 +1,23 @@
+import { getLoggerFor } from '../../../logging/LogUtil';
+import type { HttpRequest } from '../../../server/HttpRequest';
+import type { RepresentationMetadata } from '../../representation/RepresentationMetadata';
+import { MetadataParser } from './MetadataParser';
+
+/**
+ * Parser for the `content-length` header.
+ */
+export class ContentLengthParser extends MetadataParser {
+  protected readonly logger = getLoggerFor(this);
+
+  public async handle(input: { request: HttpRequest; metadata: RepresentationMetadata }): Promise<void> {
+    const contentLength = input.request.headers['content-length'];
+    if (contentLength) {
+      const length = /^\s*(\d+)\s*(?:;.*)?$/u.exec(contentLength)?.[1];
+      if (length) {
+        input.metadata.contentLength = Number(length);
+      } else {
+        this.logger.warn(`Invalid content-length header found: ${contentLength}.`);
+      }
+    }
+  }
+}

src/http/representation/RepresentationMetadata.ts

@@ -2,8 +2,8 @@ import { DataFactory, Store } from 'n3';
 import type { BlankNode, DefaultGraph, Literal, NamedNode, Quad, Term } from 'rdf-js';
 import { getLoggerFor } from '../../logging/LogUtil';
 import { InternalServerError } from '../../util/errors/InternalServerError';
-import { toNamedTerm, toObjectTerm, toCachedNamedNode, isTerm } from '../../util/TermUtil';
-import { CONTENT_TYPE, CONTENT_TYPE_TERM } from '../../util/Vocabularies';
+import { toNamedTerm, toObjectTerm, toCachedNamedNode, isTerm, toLiteral } from '../../util/TermUtil';
+import { CONTENT_TYPE, CONTENT_TYPE_TERM, CONTENT_LENGTH_TERM, XSD } from '../../util/Vocabularies';
 import type { ResourceIdentifier } from './ResourceIdentifier';
 import { isResourceIdentifier } from './ResourceIdentifier';
 
@@ -316,4 +316,18 @@ export class RepresentationMetadata {
   public set contentType(input) {
     this.set(CONTENT_TYPE_TERM, input);
   }
+
+  /**
+  * Shorthand for the CONTENT_LENGTH predicate.
+  */
+  public get contentLength(): number | undefined {
+    const length = this.get(CONTENT_LENGTH_TERM);
+    return length?.value ? Number(length.value) : undefined;
+  }
+
+  public set contentLength(input) {
+    if (input) {
+      this.set(CONTENT_LENGTH_TERM, toLiteral(input, XSD.terms.integer));
+    }
+  }
 }