refactor: Use fs-extra instead of fs to simplify file access

* refactor: use fs-extra instead of fs

* tests: manual mocks for fs-extra base + ensureDir

* refactor: mockFileSystem + mockFs and mockFsExtra

* add remove mock and some further test tweaks

* test: FileDataAccessor tests passing

* refactor: remove try-catch due to fs-extra handlin

* refactor: fs-extra in atomicFileDataAccessor

* refactor: AtomicFileDataAccessor fs-extra

* test: fix coverage

* refactor: use read/writeJson from fs-extra

* refactor: less duplicate mocking code

* refactor: re-use opendir mocking code

test/unit/pods/generate/TemplatedResourcesGenerator.test.ts

@@ -8,7 +8,7 @@ import type {
 import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil';
 import { readableToString } from '../../../../src/util/StreamUtil';
 import { HandlebarsTemplateEngine } from '../../../../src/util/templates/HandlebarsTemplateEngine';
-import { mockFs } from '../../../util/Util';
+import { mockFileSystem } from '../../../util/Util';
 
 jest.mock('fs');
 
@@ -52,7 +52,7 @@ describe('A TemplatedResourcesGenerator', (): void => {
   const webId = 'http://alice/#profile';
 
   beforeEach(async(): Promise<void> => {
-    cache = mockFs(rootFilePath);
+    cache = mockFileSystem(rootFilePath);
   });
 
   it('fills in a template with the given options.', async(): Promise<void> => {

test/unit/quota/PodQuotaStrategy.test.ts

@@ -9,7 +9,7 @@ import { NotFoundHttpError } from '../../../src/util/errors/NotFoundHttpError';
 import type { IdentifierStrategy } from '../../../src/util/identifiers/IdentifierStrategy';
 import { SingleRootIdentifierStrategy } from '../../../src/util/identifiers/SingleRootIdentifierStrategy';
 import { PIM, RDF } from '../../../src/util/Vocabularies';
-import { mockFs } from '../../util/Util';
+import { mockFileSystem } from '../../util/Util';
 
 jest.mock('fs');
 
@@ -24,7 +24,7 @@ describe('PodQuotaStrategy', (): void => {
 
   beforeEach((): void => {
     jest.restoreAllMocks();
-    mockFs(rootFilePath, new Date());
+    mockFileSystem(rootFilePath, new Date());
     mockSize = { amount: 2000, unit: UNIT_BYTES };
     identifierStrategy = new SingleRootIdentifierStrategy(base);
     mockReporter = {

test/unit/quota/QuotaStrategy.test.ts

@@ -4,7 +4,7 @@ import { UNIT_BYTES } from '../../../src/storage/size-reporter/Size';
 import type { Size } from '../../../src/storage/size-reporter/Size';
 import type { SizeReporter } from '../../../src/storage/size-reporter/SizeReporter';
 import { guardedStreamFrom, pipeSafely } from '../../../src/util/StreamUtil';
-import { mockFs } from '../../util/Util';
+import { mockFileSystem } from '../../util/Util';
 
 jest.mock('fs');
 
@@ -26,7 +26,7 @@ describe('A QuotaStrategy', (): void => {
 
   beforeEach((): void => {
     jest.restoreAllMocks();
-    mockFs(rootFilePath, new Date());
+    mockFileSystem(rootFilePath, new Date());
     mockSize = { amount: 2000, unit: UNIT_BYTES };
     mockReporter = {
       getSize: jest.fn().mockResolvedValue({ unit: mockSize.unit, amount: 50 }),

test/unit/storage/accessors/AtomicFileDataAccessor.test.ts

@@ -7,9 +7,10 @@ import { APPLICATION_OCTET_STREAM } from '../../../../src/util/ContentTypes';
 import type { Guarded } from '../../../../src/util/GuardedStream';
 import { guardedStreamFrom } from '../../../../src/util/StreamUtil';
 import { CONTENT_TYPE } from '../../../../src/util/Vocabularies';
-import { mockFs } from '../../../util/Util';
+import { mockFileSystem } from '../../../util/Util';
 
 jest.mock('fs');
+jest.mock('fs-extra');
 
 describe('AtomicFileDataAccessor', (): void => {
   const rootFilePath = 'uploads';
@@ -20,7 +21,7 @@ describe('AtomicFileDataAccessor', (): void => {
   let data: Guarded<Readable>;
 
   beforeEach(async(): Promise<void> => {
-    cache = mockFs(rootFilePath, new Date());
+    cache = mockFileSystem(rootFilePath, new Date());
     accessor = new AtomicFileDataAccessor(
       new ExtensionBasedMapper(base, rootFilePath),
       rootFilePath,
@@ -59,27 +60,27 @@ describe('AtomicFileDataAccessor', (): void => {
         data.emit('error', new Error('error'));
         return null;
       });
-      jest.requireMock('fs').promises.stat = jest.fn((): any => ({
+      jest.requireMock('fs-extra').stat = jest.fn((): any => ({
         isFile: (): boolean => false,
       }));
       await expect(accessor.writeDocument({ path: `${base}res.ttl` }, data, metadata)).rejects.toThrow('error');
     });
 
     it('should throw when renaming / moving the file goes wrong.', async(): Promise<void> => {
-      jest.requireMock('fs').promises.rename = jest.fn((): any => {
+      jest.requireMock('fs-extra').rename = jest.fn((): any => {
         throw new Error('error');
       });
-      jest.requireMock('fs').promises.stat = jest.fn((): any => ({
+      jest.requireMock('fs-extra').stat = jest.fn((): any => ({
         isFile: (): boolean => true,
       }));
       await expect(accessor.writeDocument({ path: `${base}res.ttl` }, data, metadata)).rejects.toThrow('error');
     });
 
     it('should (on error) not unlink the temp file if it does not exist.', async(): Promise<void> => {
-      jest.requireMock('fs').promises.rename = jest.fn((): any => {
+      jest.requireMock('fs-extra').rename = jest.fn((): any => {
         throw new Error('error');
       });
-      jest.requireMock('fs').promises.stat = jest.fn((): any => ({
+      jest.requireMock('fs-extra').stat = jest.fn((): any => ({
         isFile: (): boolean => false,
       }));
       await expect(accessor.writeDocument({ path: `${base}res.ttl` }, data, metadata)).rejects.toThrow('error');
@@ -87,10 +88,10 @@ describe('AtomicFileDataAccessor', (): void => {
 
     it('should throw when renaming / moving the file goes wrong and the temp file does not exist.',
       async(): Promise<void> => {
-        jest.requireMock('fs').promises.rename = jest.fn((): any => {
+        jest.requireMock('fs-extra').rename = jest.fn((): any => {
           throw new Error('error');
         });
-        jest.requireMock('fs').promises.stat = jest.fn();
+        jest.requireMock('fs-extra').stat = jest.fn();
         await expect(accessor.writeDocument({ path: `${base}res.ttl` }, data, metadata)).rejects.toThrow('error');
       });
   });

test/unit/storage/accessors/FileDataAccessor.test.ts

@@ -15,9 +15,10 @@ import { isContainerPath } from '../../../../src/util/PathUtil';
 import { guardedStreamFrom, readableToString } from '../../../../src/util/StreamUtil';
 import { toLiteral } from '../../../../src/util/TermUtil';
 import { CONTENT_TYPE, DC, LDP, POSIX, RDF, SOLID_META, XSD } from '../../../../src/util/Vocabularies';
-import { mockFs } from '../../../util/Util';
+import { mockFileSystem } from '../../../util/Util';
 
 jest.mock('fs');
+jest.mock('fs-extra');
 
 const rootFilePath = 'uploads';
 const now = new Date();
@@ -32,7 +33,7 @@ describe('A FileDataAccessor', (): void => {
   let data: Guarded<Readable>;
 
   beforeEach(async(): Promise<void> => {
-    cache = mockFs(rootFilePath, now);
+    cache = mockFileSystem(rootFilePath, now);
     accessor = new FileDataAccessor(new ExtensionBasedMapper(base, rootFilePath));
 
     metadata = new RepresentationMetadata(APPLICATION_OCTET_STREAM);
@@ -66,6 +67,13 @@ describe('A FileDataAccessor', (): void => {
       const stream = await accessor.getData({ path: `${base}resource` });
       await expect(readableToString(stream)).resolves.toBe('data');
     });
+
+    it('throws an error if something else went wrong.', async(): Promise<void> => {
+      jest.requireMock('fs-extra').stat = (): any => {
+        throw new Error('error');
+      };
+      await expect(accessor.getData({ path: `${base}resource` })).rejects.toThrow('error');
+    });
   });
 
   describe('getting metadata', (): void => {
@@ -83,7 +91,7 @@ describe('A FileDataAccessor', (): void => {
     });
 
     it('throws an error if something else went wrong.', async(): Promise<void> => {
-      jest.requireMock('fs').promises.lstat = (): any => {
+      jest.requireMock('fs-extra').lstat = (): any => {
         throw new Error('error');
       };
       await expect(accessor.getMetadata({ path: base })).rejects.toThrow('error');
@@ -248,7 +256,7 @@ describe('A FileDataAccessor', (): void => {
 
     it('errors if there is a problem deleting the old metadata file.', async(): Promise<void> => {
       cache.data = { resource: 'data', 'resource.meta': 'metadata!' };
-      jest.requireMock('fs').promises.unlink = (): any => {
+      jest.requireMock('fs-extra').remove = (): any => {
         throw new Error('error');
       };
       await expect(accessor.writeDocument({ path: `${base}resource` }, data, metadata))
@@ -302,7 +310,7 @@ describe('A FileDataAccessor', (): void => {
 
     it('throws an error if there is an issue deleting the original file.', async(): Promise<void> => {
       cache.data = { 'resource$.ttl': '<this> <is> <data>.' };
-      jest.requireMock('fs').promises.unlink = (): any => {
+      jest.requireMock('fs-extra').remove = (): any => {
         const error = new Error('error') as SystemError;
         error.code = 'EISDIR';
         error.syscall = 'unlink';
@@ -332,7 +340,7 @@ describe('A FileDataAccessor', (): void => {
     });
 
     it('throws other errors when making a directory.', async(): Promise<void> => {
-      jest.requireMock('fs').promises.mkdir = (): any => {
+      jest.requireMock('fs-extra').ensureDir = (): any => {
         throw new Error('error');
       };
       await expect(accessor.writeContainer({ path: base }, metadata)).rejects.toThrow('error');
@@ -395,6 +403,9 @@ describe('A FileDataAccessor', (): void => {
 
     it('throws error if there is a problem with deleting existing metadata.', async(): Promise<void> => {
       cache.data = { resource: 'apple', 'resource.meta': {}};
+      jest.requireMock('fs-extra').remove = (): any => {
+        throw new Error('error');
+      };
       await expect(accessor.deleteResource({ path: `${base}resource` })).rejects.toThrow();
     });
 

test/unit/storage/keyvalue/JsonFileStorage.test.ts

@@ -1,9 +1,10 @@
 import type { ResourceIdentifier } from '../../../../src/http/representation/ResourceIdentifier';
 import { JsonFileStorage } from '../../../../src/storage/keyvalue/JsonFileStorage';
 import type { ReadWriteLocker } from '../../../../src/util/locking/ReadWriteLocker';
-import { mockFs } from '../../../util/Util';
+import { mockFileSystem } from '../../../util/Util';
 
 jest.mock('fs');
+jest.mock('fs-extra');
 
 describe('A JsonFileStorage', (): void => {
   const rootFilePath = 'files/';
@@ -13,7 +14,7 @@ describe('A JsonFileStorage', (): void => {
   let storage: JsonFileStorage;
 
   beforeEach(async(): Promise<void> => {
-    cache = mockFs(rootFilePath);
+    cache = mockFileSystem(rootFilePath);
     locker = {
       withReadLock:
         jest.fn(async(identifier: ResourceIdentifier, whileLocked: () => any): Promise<any> => await whileLocked()),

test/unit/storage/size-reporter/FileSizeReporter.test.ts

@@ -5,7 +5,7 @@ import type { FileIdentifierMapper, ResourceLink } from '../../../../src/storage
 import { FileSizeReporter } from '../../../../src/storage/size-reporter/FileSizeReporter';
 import { UNIT_BYTES } from '../../../../src/storage/size-reporter/Size';
 import { joinFilePath } from '../../../../src/util/PathUtil';
-import { mockFs } from '../../../util/Util';
+import { mockFileSystem } from '../../../util/Util';
 
 jest.mock('fs');
 
@@ -28,7 +28,7 @@ describe('A FileSizeReporter', (): void => {
   );
 
   beforeEach(async(): Promise<void> => {
-    mockFs(fileRoot);
+    mockFileSystem(fileRoot);
   });
 
   it('should work without the ignoreFolders constructor parameter.', async(): Promise<void> => {

test/unit/util/templates/TemplateEngine.test.ts

@@ -1,6 +1,6 @@
 import { resolveAssetPath } from '../../../../src/util/PathUtil';
 import { getTemplateFilePath, readTemplate } from '../../../../src/util/templates/TemplateEngine';
-import { mockFs } from '../../../util/Util';
+import { mockFileSystem } from '../../../util/Util';
 
 jest.mock('fs');
 
@@ -10,7 +10,7 @@ describe('TemplateEngine', (): void => {
     const templatePath = 'other';
 
     beforeEach(async(): Promise<void> => {
-      const { data } = mockFs(resolveAssetPath(''));
+      const { data } = mockFileSystem(resolveAssetPath(''));
       Object.assign(data, {
         'template.xyz': '{{template}}',
         other: {
@@ -45,7 +45,7 @@ describe('TemplateEngine', (): void => {
     const templatePath = 'other';
 
     beforeEach(async(): Promise<void> => {
-      const { data } = mockFs(resolveAssetPath(''));
+      const { data } = mockFileSystem(resolveAssetPath(''));
       Object.assign(data, {
         'template.xyz': '{{template}}',
         other: {

test/util/Util.ts

@@ -60,7 +60,7 @@ export function describeIf(envFlag: string, name: string, fn: () => void): void
  * @param rootFilepath - The name of the root folder in which fs will start.
  * @param time - The date object to use for time functions (currently only mtime from lstats)
  */
-export function mockFs(rootFilepath?: string, time?: Date): { data: any } {
+export function mockFileSystem(rootFilepath?: string, time?: Date): { data: any } {
   const cache: { data: any } = { data: {}};
 
   rootFilepath = rootFilepath ?? 'folder';
@@ -97,7 +97,7 @@ export function mockFs(rootFilepath?: string, time?: Date): { data: any } {
     return { folder, name };
   }
 
-  const mock = {
+  const mockFs = {
     createReadStream(path: string): any {
       const { folder, name } = getFolder(path);
       return Readable.from([ folder[name] ]);
@@ -220,8 +220,79 @@ export function mockFs(rootFilepath?: string, time?: Date): { data: any } {
     },
   };
 
+  const mockFsExtra = {
+    async readJson(path: string): Promise<NodeJS.Dict<unknown>> {
+      const { folder, name } = getFolder(path);
+      if (!folder[name]) {
+        throwSystemError('ENOENT');
+      }
+      return JSON.parse(folder[name]);
+    },
+    async writeJson(path: string, json: NodeJS.Dict<unknown>): Promise<void> {
+      const { folder, name } = getFolder(path);
+      const data = JSON.stringify(json, null, 2);
+      folder[name] = data;
+    },
+    async ensureDir(path: string): Promise<void> {
+      const { folder, name } = getFolder(path);
+      folder[name] = {};
+    },
+    async remove(path: string): Promise<void> {
+      const { folder, name } = getFolder(path);
+      // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+      delete folder[name];
+    },
+    createReadStream(path: string): any {
+      return mockFs.createReadStream(path);
+    },
+    createWriteStream(path: string): any {
+      return mockFs.createWriteStream(path);
+    },
+    async realpath(path: string): Promise<string> {
+      return await mockFs.promises.realpath(path);
+    },
+    async stat(path: string): Promise<Stats> {
+      return mockFs.promises.lstat(await mockFs.promises.realpath(path));
+    },
+    async lstat(path: string): Promise<Stats> {
+      return mockFs.promises.lstat(path);
+    },
+    async unlink(path: string): Promise<void> {
+      await mockFs.promises.unlink(path);
+    },
+    async symlink(target: string, path: string): Promise<void> {
+      await mockFs.promises.symlink(target, path);
+    },
+    async rmdir(path: string): Promise<void> {
+      await mockFs.promises.rmdir(path);
+    },
+    async readdir(path: string): Promise<string[]> {
+      return await mockFs.promises.readdir(path);
+    },
+    async* opendir(path: string): AsyncIterableIterator<Dirent> {
+      for await (const entry of mockFs.promises.opendir(path)) {
+        yield entry;
+      }
+    },
+    async mkdir(path: string): Promise<void> {
+      await mockFs.promises.mkdir(path);
+    },
+    async readFile(path: string): Promise<string> {
+      return await mockFs.promises.readFile(path);
+    },
+    async writeFile(path: string, data: string): Promise<void> {
+      await mockFs.promises.writeFile(path, data);
+    },
+    async rename(path: string, destination: string): Promise<void> {
+      await mockFs.promises.rename(path, destination);
+    },
+  };
+
   const fs = jest.requireMock('fs');
-  Object.assign(fs, mock);
+  Object.assign(fs, mockFs);
+
+  const fsExtra = jest.requireMock('fs-extra');
+  Object.assign(fsExtra, mockFsExtra);
 
   return cache;
 }