Mirroristas/CommunitySolidServer
2
0
Fork 0
mirror of https://github.com/CommunitySolidServer/CommunitySolidServer.git synced 2024-10-03 14:55:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add redis based locking mechanism

Browse Source 
* feat: redis based locking mechanism

* fix: adapted to review and wrote unit tests

* fix: adapted to review(Woutermont)

* fix: adapted to review and expanded tests

* test: redlock integration tests

* test: corrected file name

* test: tests should run on CI now

* test: improved tests and minor changes according to review

* fix: forgot describeIf docker

* test: adapted to review

* test: Mock all redis dependencies

Co-authored-by: Joachim Van Herwegen <joachimvh@gmail.com>
This commit is contained in:
Arthur Joppart 2021-04-19 09:45:25 +02:00 committed by GitHub
parent 953458231b
commit 99d0173213
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 800 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@ -41,6 +41,10 @@ jobs:
SPARQL_UPDATE: true
ports:
- 4000:8890
redis:
image: redis
ports:
- 6379:6379
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v2

68
package-lock.json generated
View File

@ -1152,6 +1152,11 @@
"@babel/types": "^7.3.0"
}
},
"@types/bluebird": {
"version": "3.5.33",
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.33.tgz",
"integrity": "sha512-ndEo1xvnYeHxm7I/5sF6tBvnsA4Tdi3zj1keRKRs12SP+2ye2A27NDJ1B6PqkfMbGAcT+mqQVqbZRIrhfOp5PQ=="
},
"@types/cookiejar": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.2.tgz",
@ -1324,6 +1329,22 @@
"@types/node": "*"
}
},
"@types/redis": {
"version": "2.8.28",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.28.tgz",
"integrity": "sha512-8l2gr2OQ969ypa7hFOeKqtFoY70XkHxISV0pAwmQ2nm6CSPb1brmTmqJCGGrekCo+pAZyWlNXr+Kvo6L/1wijA==",
"requires": {
"@types/node": "*"
}
},
"@types/redlock": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/redlock/-/redlock-4.0.1.tgz",
"integrity": "sha512-YrPEPHOgLlWtsg/Ocv/gthGDQpx3HXFPhBvCNuBKBKUoMBzvCjCTC95dmJ0WLZgO+fgTxGQFyk6ZO/+/zG5mRg==",
"requires": {
"@types/bluebird": "*"
}
},
"@types/rimraf": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.0.tgz",
@ -2084,6 +2105,11 @@
"integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
"dev": true
},
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"boxen": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz",
@ -3029,6 +3055,11 @@
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true
},
"denque": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ=="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@ -8082,6 +8113,43 @@
"strip-indent": "^3.0.0"
}
},
"redis": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
"requires": {
"denque": "^1.4.1",
"redis-commands": "^1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0"
}
},
"redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"requires": {
"redis-errors": "^1.0.0"
}
},
"redlock": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/redlock/-/redlock-4.2.0.tgz",
"integrity": "sha512-j+oQlG+dOwcetUt2WJWttu4CZVeRzUrcVcISFmEmfyuwCVSJ93rDT7YSgg7H7rnxwoRyk/jU46kycVka5tW7jA==",
"requires": {
"bluebird": "^3.7.2"
}
},
"regex-not": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",

4
package.json
View File

@ -86,6 +86,8 @@
"@types/pump": "^1.1.0",
"@types/punycode": "^2.1.0",
"@types/rdf-js": "^4.0.0",
"@types/redis": "^2.8.28",
"@types/redlock": "^4.0.1",
"@types/sparqljs": "^3.1.0",
"@types/streamify-array": "^1.0.0",
"@types/uuid": "^8.3.0",
@ -106,6 +108,8 @@
"rdf-parse": "^1.7.0",
"rdf-serialize": "^1.1.0",
"rdf-terms": "^1.5.1",
"redis": "^3.0.2",
"redlock": "^4.2.0",
"sparqlalgebrajs": "^2.5.1",
"sparqljs": "^3.1.2",
"streamify-array": "^1.0.1",

1
src/index.ts
View File

@ -243,6 +243,7 @@ export * from './util/locking/ExpiringReadWriteLocker';
export * from './util/locking/EqualReadWriteLocker';
export * from './util/locking/GreedyReadWriteLocker';
export * from './util/locking/ReadWriteLocker';
export * from './util/locking/RedisResourceLocker';
export * from './util/locking/ResourceLocker';
export * from './util/locking/SingleThreadedResourceLocker';
export * from './util/locking/WrappedExpiringReadWriteLocker';

174
src/util/locking/RedisResourceLocker.ts Normal file
View File

@ -0,0 +1,174 @@
import { assert } from 'console';
import type { RedisClient } from 'redis';
import { createClient } from 'redis';
import type { Lock } from 'redlock';
import Redlock from 'redlock';
import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
import { getLoggerFor } from '../../logging/LogUtil';
import { InternalServerError } from '../errors/InternalServerError';
import type { ResourceLocker } from './ResourceLocker';
// The ttl set on a lock, not really important cause redlock wil not handle expiration
const ttl = 10000;
// The default redlock config
const defaultRedlockConfig = {
// The expected clock drift; for more details
// see http://redis.io/topics/distlock
// Multiplied by lock ttl to determine drift time
driftFactor: 0.01,
// The max number of times Redlock will attempt
// to lock a resource before erroring
retryCount: 1000000,
// The time in ms between attempts
retryDelay: 200,
// The max time in ms randomly added to retries
// to improve performance under high contention
// see https://www.awsarchitectureblog.com/2015/03/backoff.html
retryJitter: 200,
};
/**
* A locking system that uses a Redis server or any number of
* Redis nodes / clusters
* This solution has issues though:
* - Redlock wants to handle expiration itself, this is against the design of a ResourceLocker.
* The workaround for this is to extend an active lock indefinitely.
* - This solution is not multithreaded! If threadA locks a resource, only threadA can unlock this resource.
* If threadB wont be able to lock a resource if threadA already acquired that lock,
* in that sense it is kind of multithreaded.
* - Redlock does not provide the ability to see which locks have expired
*/
export class RedisResourceLocker implements ResourceLocker {
protected readonly logger = getLoggerFor(this);
private readonly redlock: Redlock;
private readonly lockMap: Map<string, { lock: Lock; interval: NodeJS.Timeout }>;
public constructor(redisClients: string[], redlockOptions?: Record<string, number>) {
this.lockMap = new Map();
const clients = this.createRedisClients(redisClients);
if (clients.length === 0) {
throw new Error('At least 1 client should be provided');
}
this.redlock = this.createRedlock(clients, redlockOptions);
this.redlock.on('clientError', (err): void => {
this.logger.error(`Redis/Redlock error: ${err}`);
});
}
/**
* Generate and return a list of RedisClients based on the provided strings
* @param redisClientsStrings - a list of strings that contain either a host address and a
* port number like '127.0.0.1:6379' or just a port number like '6379'
*/
private createRedisClients(redisClientsStrings: string[]): RedisClient[] {
const result: RedisClient[] = [];
if (redisClientsStrings && redisClientsStrings.length > 0) {
for (const client of redisClientsStrings) {
// Check if port number or ip with port number
// Definitely not perfect, but configuring this is only for experienced users
const match = new RegExp(/^(?:([^:]+):)?(\d{4,5})$/u, 'u').exec(client);
if (!match || !match[2]) {
// At least a port number should be provided
throw new Error(`Invalid data provided to create a Redis client: ${client}\n
Please provide a port number like '6379' or a host address and a port number like '127.0.0.1:6379'`);
}
const port = Number(match[2]);
const host = match[1];
const redisclient = createClient(port, host);
result.push(redisclient);
}
}
return result;
}
/**
* Generate and return a Redlock instance
* @param clients - a list of RedisClients you want to use for the redlock instance
* @param redlockOptions - extra redlock options to overwrite the default config
*/
private createRedlock(clients: RedisClient[], redlockOptions: Record<string, number> = {}): Redlock {
try {
return new Redlock(
clients,
{ ...defaultRedlockConfig, ...redlockOptions },
);
} catch (error: unknown) {
throw new InternalServerError(`Error initializing Redlock: ${error}`);
}
}
public async quit(): Promise<void> {
// This for loop is an extra failsafe,
// this extra code won't slow down anything, this function will only be called to shut down in peace
for (const [ , { lock }] of this.lockMap.entries()) {
await this.release({ path: lock.resource });
}
await this.redlock.quit();
}
public async acquire(identifier: ResourceIdentifier): Promise<void> {
const resource = identifier.path;
let lock: Lock | undefined;
try {
lock = await this.redlock.lock(resource, ttl);
assert(lock);
} catch (error: unknown) {
this.logger.debug(`Unable to acquire lock for ${resource}`);
throw new InternalServerError(`Unable to acquire lock for ${resource} (${error})`);
}
if (this.lockMap.get(resource)) {
throw new InternalServerError(`Acquired duplicate lock on ${resource}`);
}
// Lock acquired
this.logger.debug(`Acquired lock for resource: ${resource}!`);
const interval = this.extendLockIndefinitely(resource);
this.lockMap.set(resource, { lock, interval });
}
public async release(identifier: ResourceIdentifier): Promise<void> {
const resource = identifier.path;
const entry = this.lockMap.get(resource);
if (!entry) {
// Lock is invalid
this.logger.warn(`Unexpected release request for non-existent lock on ${resource}`);
throw new InternalServerError(`Trying to unlock resource that is not locked: ${resource}`);
}
try {
await this.redlock.unlock(entry.lock);
clearInterval(entry.interval);
this.lockMap.delete(resource);
// Successfully released lock
this.logger.debug(`Released lock for ${resource}, ${this.getLockCount()} active locks remaining!`);
} catch (error: unknown) {
this.logger.error(`Error releasing lock for ${resource} (${error})`);
throw new InternalServerError(`Unable to release lock for: ${resource}, ${error}`);
}
}
/**
* Counts the number of active locks.
*/
private getLockCount(): number {
return this.lockMap.size;
}
/**
* This function is internally used to keep an acquired lock active, a wrapper class will handle expiration
* @param identifier - Identifier of the lock to be extended
*/
private extendLockIndefinitely(identifier: string): NodeJS.Timeout {
return setInterval(async(): Promise<void> => {
const entry = this.lockMap.get(identifier)!;
try {
const newLock = await this.redlock.extend(entry.lock, ttl);
this.lockMap.set(identifier, { lock: newLock, interval: entry.interval });
this.logger.debug(`Extended (Redis)lock for resource: ${identifier}`);
} catch (error: unknown) {
// No error should be re-thrown because this means the lock has simply been released
this.logger.error(`Failed to extend this (Redis)lock for resource: ${identifier}, ${error}`);
clearInterval(entry.interval);
this.lockMap.delete(identifier);
}
}, ttl / 2);
}
}

173
test/integration/RedisResourceLockerIntegration.test.ts Normal file
View File

@ -0,0 +1,173 @@
import type { Server } from 'http';
import fetch from 'cross-fetch';
import type { RedisResourceLocker } from '../../src';
import { joinFilePath } from '../../src';
import type { HttpServerFactory } from '../../src/server/HttpServerFactory';
import { describeIf } from '../util/TestHelpers';
import { instantiateFromConfig } from './Config';
/**
* Test the general functionality of the server using a RedisResourceLocker
*/
describeIf('docker', 'A server with a RedisResourceLocker as ResourceLocker', (): void => {
const port = 6008;
const baseUrl = `http://localhost:${port}/`;
let server: Server;
let locker: RedisResourceLocker;
let factory: HttpServerFactory;
beforeAll(async(): Promise<void> => {
const instances = await instantiateFromConfig(
'urn:solid-server:test:Instances',
'run-with-redlock.json',
{
'urn:solid-server:default:variable:baseUrl': baseUrl,
'urn:solid-server:default:variable:podTemplateFolder': joinFilePath(__dirname, '../assets/templates'),
},
) as Record<string, any>;
({ factory, locker } = instances);
server = factory.startServer(port);
});
afterAll(async(): Promise<void> => {
await locker.quit();
await new Promise<void>((resolve, reject): void => {
server.close((error): void => error ? reject(error) : resolve());
});
});
it('can add a file to the store, read it and delete it.', async(): Promise<void> => {
// Create file
const fileUrl = `${baseUrl}testfile2.txt`;
const fileData = 'TESTFILE2';
let response = await fetch(fileUrl, {
method: 'PUT',
headers: {
'content-type': 'text/plain',
},
body: fileData,
});
expect(response.status).toBe(205);
// Get file
response = await fetch(fileUrl);
expect(response.status).toBe(200);
const body = await response.text();
expect(body).toContain('TESTFILE2');
// DELETE file
response = await fetch(fileUrl, { method: 'DELETE' });
expect(response.status).toBe(205);
response = await fetch(fileUrl);
expect(response.status).toBe(404);
});
it('can create a folder and delete it.', async(): Promise<void> => {
const containerPath = 'secondfolder/';
const containerUrl = `${baseUrl}${containerPath}`;
// PUT
let response = await fetch(containerUrl, {
method: 'PUT',
headers: {
'content-type': 'text/plain',
},
});
expect(response.status).toBe(205);
// GET
response = await fetch(containerUrl);
expect(response.status).toBe(200);
// DELETE
response = await fetch(containerUrl, { method: 'DELETE' });
expect(response.status).toBe(205);
response = await fetch(containerUrl);
expect(response.status).toBe(404);
});
it('can get a resource multiple times.', async(): Promise<void> => {
const fileUrl = `${baseUrl}image.png`;
const fileData = 'testtesttest';
let response = await fetch(fileUrl, {
method: 'PUT',
headers: {
'content-type': 'text/plain',
},
body: fileData,
});
expect(response.status).toBe(205);
// GET 4 times
for (let i = 0; i < 4; i++) {
const res = await fetch(fileUrl);
expect(res.status).toBe(200);
expect(res.headers.get('content-type')).toBe('text/plain');
const body = await res.text();
expect(body).toContain('testtesttest');
}
// DELETE
response = await fetch(fileUrl, { method: 'DELETE' });
expect(response.status).toBe(205);
response = await fetch(fileUrl);
expect(response.status).toBe(404);
});
describe('Test the ResourceLocker itself', (): void => {
const identifier = { path: 'http://test.com/foo' };
it('can lock and unlock a resource.', async(): Promise<void> => {
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).resolves.toBeUndefined();
});
it('can lock a resource again after it was unlocked.', async(): Promise<void> => {
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).resolves.toBeUndefined();
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).resolves.toBeUndefined();
});
it('can acquire different keys simultaneously.', async(): Promise<void> => {
const lock1 = locker.acquire({ path: 'path1' });
const lock2 = locker.acquire({ path: 'path2' });
const lock3 = locker.acquire({ path: 'path3' });
await expect(Promise.all([ lock1, lock2, lock3 ])).resolves.toBeDefined();
await locker.release({ path: 'path1' });
await locker.release({ path: 'path2' });
await locker.release({ path: 'path3' });
});
it('Cannot acquire the same lock simultaneously.', async(): Promise<void> => {
let res = '';
const lock1 = locker.acquire(identifier);
const lock2 = locker.acquire(identifier);
const lock3 = locker.acquire(identifier);
await new Promise((resolve): any => setImmediate(resolve));
const l2 = lock2.then(async(): Promise<void> => {
res += 'l2';
await locker.release(identifier);
res += 'r2';
});
const l1 = lock1.then(async(): Promise<void> => {
res += 'l1';
await locker.release(identifier);
res += 'r1';
});
const l3 = lock3.then(async(): Promise<void> => {
res += 'l3';
await locker.release(identifier);
res += 'r3';
});
await Promise.all([ l1, l2, l3 ]);
expect(res).toContain('l1r1');
expect(res).toContain('l2r2');
expect(res).toContain('l3r3');
});
});
});

168
test/integration/config/run-with-redlock.json Normal file
View File

@ -0,0 +1,168 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^0.0.0/components/context.jsonld",
"import": [
"files-scs:config/presets/acl.json",
"files-scs:config/presets/http.json",
"files-scs:config/presets/identifiers/suffix-identifiers.json",
"files-scs:config/presets/ldp/credentials-extractor.json",
"files-scs:config/presets/ldp/metadata-handler.json",
"files-scs:config/presets/ldp/operation-handler.json",
"files-scs:config/presets/ldp/permissions-extractor.json",
"files-scs:config/presets/ldp/response-writer.json",
"files-scs:config/presets/ldp/request-parser.json",
"files-scs:config/presets/ldp/websockets.json",
"files-scs:config/presets/middleware.json",
"files-scs:config/presets/pod-management.json",
"files-scs:config/presets/representation-conversion.json",
"files-scs:config/presets/static.json",
"files-scs:config/presets/storage/backend/storage-memory.json",
"files-scs:config/presets/cli-params.json"
],
"@graph": [
{
"@id": "urn:solid-server:test:Instances",
"@type": "RecordObject",
"RecordObject:_record": [
{
"RecordObject:_record_key": "factory",
"RecordObject:_record_value": {
"@id": "urn:solid-server:default:ServerFactory"
}
},
{
"RecordObject:_record_key": "locker",
"RecordObject:_record_value": {
"@id": "urn:solid-server:default:RedisResourceLocker"
}
}
]
},
{
"@id": "urn:solid-server:default:LdpHandler",
"@type": "AuthenticatedLdpHandler",
"AuthenticatedLdpHandler:_args_requestParser": {
"@id": "urn:solid-server:default:RequestParser"
},
"AuthenticatedLdpHandler:_args_credentialsExtractor": {
"@id": "urn:solid-server:default:CredentialsExtractor"
},
"AuthenticatedLdpHandler:_args_permissionsExtractor": {
"@id": "urn:solid-server:default:PermissionsExtractor"
},
"AuthenticatedLdpHandler:_args_authorizer": {
"@type": "AllowEverythingAuthorizer"
},
"AuthenticatedLdpHandler:_args_operationHandler": {
"@id": "urn:solid-server:default:OperationHandler"
},
"AuthenticatedLdpHandler:_args_responseWriter": {
"@id": "urn:solid-server:default:ResponseWriter"
}
},
{
"@id": "urn:solid-server:default:RoutingResourceStore",
"@type": "PassthroughStore",
"PassthroughStore:_source": {
"@id": "urn:solid-server:default:MemoryResourceStore"
}
},
{
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Locking"
}
},
{
"@id": "urn:solid-server:default:AuxiliaryStrategy",
"@type": "RoutingAuxiliaryStrategy",
"RoutingAuxiliaryStrategy:_sources": [
{
"@id": "urn:solid-server:default:AclStrategy"
}
]
},
{
"@id": "urn:solid-server:default:ResourceStore_Locking",
"@type": "LockingResourceStore",
"LockingResourceStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Patching"
},
"LockingResourceStore:_locks": {
"@id": "urn:solid-server:default:ResourceLocker"
},
"LockingResourceStore:_strategy": {
"@id": "urn:solid-server:default:AuxiliaryStrategy"
}
},
{
"@id": "urn:solid-server:default:ResourceStore_Patching",
"@type": "PatchingStore",
"PatchingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Converting"
},
"PatchingStore:_patcher": {
"@id": "urn:solid-server:default:PatchHandler",
"@type": "SparqlUpdatePatchHandler",
"SparqlUpdatePatchHandler:_source": {
"@id": "urn:solid-server:default:ResourceStore_ToTurtle"
}
}
},
{
"@id": "urn:solid-server:default:ResourceLocker",
"@type": "WrappedExpiringReadWriteLocker",
"WrappedExpiringReadWriteLocker:_locker": {
"@type": "GreedyReadWriteLocker",
"GreedyReadWriteLocker:_locker": {
"@id": "urn:solid-server:default:RedisResourceLocker",
"@type": "RedisResourceLocker",
"RedisResourceLocker:_redisClients": [ "6379" ]
},
"GreedyReadWriteLocker:_storage": {
"@type": "ResourceIdentifierStorage",
"ResourceIdentifierStorage:_source": {
"@type": "MemoryMapStorage"
}
},
"GreedyReadWriteLocker:_suffixes_count": "count",
"GreedyReadWriteLocker:_suffixes_read": "read",
"GreedyReadWriteLocker:_suffixes_write": "write"
},
"WrappedExpiringReadWriteLocker:_expiration": 3000
},
{
"@id": "urn:solid-server:default:ResourceStore_ToTurtle",
"@type": "RepresentationConvertingStore",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:RoutingResourceStore"
},
"RepresentationConvertingStore:_options_outConverter": {
"@id": "urn:solid-server:default:RepresentationConverter"
},
"RepresentationConvertingStore:_options_inConverter": {
"@id": "urn:solid-server:default:RepresentationConverter"
},
"RepresentationConvertingStore:_options_inType": "text/turtle"
},
{
"@id": "urn:solid-server:default:ResourceStore_Converting",
"@type": "RepresentationConvertingStore",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:RoutingResourceStore"
},
"RepresentationConvertingStore:_options_outConverter": {
"@id": "urn:solid-server:default:RepresentationConverter"
}
}
]
}

208
test/unit/util/locking/RedisResourceLocker.test.ts Normal file
View File

@ -0,0 +1,208 @@
import { EventEmitter } from 'events';
// eslint-disable-next-line import/default
import redis from 'redis';
import Redlock from 'redlock';
import type { Lock } from 'redlock';
import * as LogUtil from '../../../../src/logging/LogUtil';
import { InternalServerError } from '../../../../src/util/errors/InternalServerError';
import { RedisResourceLocker } from '../../../../src/util/locking/RedisResourceLocker';
const redlock: jest.Mocked<Redlock> = Object.assign(new EventEmitter(), {
lock: jest.fn().mockImplementation(async(resource: string, ttl: number): Promise<Lock> =>
({ resource, expiration: Date.now() + ttl } as Lock)),
unlock: jest.fn(),
extend: jest.fn().mockImplementation(
async(lock: Lock, ttl: number): Promise<Lock> => {
lock.expiration += ttl;
return lock;
},
),
quit: jest.fn(),
}) as any;
jest.mock('redlock', (): any => jest.fn().mockImplementation((): Redlock => redlock));
jest.useFakeTimers();
describe('A RedisResourceLocker', (): void => {
let locker: RedisResourceLocker;
const identifier = { path: 'http://test.com/foo' };
let createClient: jest.SpyInstance;
beforeEach(async(): Promise<void> => {
jest.clearAllMocks();
redlock.removeAllListeners();
createClient = jest.spyOn(redis, 'createClient').mockImplementation(jest.fn());
locker = new RedisResourceLocker([ '6379' ]);
});
afterEach(async(): Promise<void> => {
// In case some locks are not released by a test the timers will still be running
jest.clearAllTimers();
});
afterAll(async(): Promise<void> => {
jest.restoreAllMocks();
});
it('can lock and unlock a resource.', async(): Promise<void> => {
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).resolves.toBeUndefined();
expect(redlock.lock).toHaveBeenCalledTimes(1);
expect(redlock.unlock).toHaveBeenCalledTimes(1);
});
it('can lock a resource again after it was unlocked.', async(): Promise<void> => {
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).resolves.toBeUndefined();
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
expect(redlock.lock).toHaveBeenCalledTimes(2);
expect(redlock.unlock).toHaveBeenCalledTimes(1);
await expect(locker.release(identifier)).resolves.toBeUndefined();
});
it('errors when unlocking a resource that was not locked.', async(): Promise<void> => {
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).resolves.toBeUndefined();
await expect(locker.release(identifier)).rejects.toThrow(InternalServerError);
expect(redlock.lock).toHaveBeenCalledTimes(1);
expect(redlock.unlock).toHaveBeenCalledTimes(1);
});
it('errors when redlock.lock throws an error.', async(): Promise<void> => {
redlock.lock.mockRejectedValueOnce(new Error('random Error'));
const prom = locker.acquire(identifier);
await expect(prom).rejects.toThrow(InternalServerError);
await expect(prom).rejects.toThrow('Unable to acquire lock for ');
await expect(prom).rejects.toThrow('Error: random Error');
expect(redlock.lock).toHaveBeenCalledTimes(1);
});
it('errors if redlock.lock resolves but a lock is already stored.', async(): Promise<void> => {
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
// Works since redlock.lock is mocked to always succeed
const prom = locker.acquire(identifier);
await expect(prom).rejects.toThrow(InternalServerError);
await expect(prom).rejects.toThrow(`Acquired duplicate lock on ${identifier.path}`);
});
it('errors when redlock.unlock throws an error.', async(): Promise<void> => {
await locker.acquire(identifier);
redlock.unlock.mockRejectedValueOnce(new Error('random Error'));
const prom = locker.release(identifier);
await expect(prom).rejects.toThrow(InternalServerError);
await expect(prom).rejects.toThrow('Unable to release lock for: ');
await expect(prom).rejects.toThrow('Error: random Error');
expect(redlock.unlock).toHaveBeenCalledTimes(1);
await expect(locker.release(identifier)).resolves.toBeUndefined();
});
it('does not extend when there are no locks to extend.', async(): Promise<void> => {
await locker.acquire(identifier);
await locker.release(identifier);
jest.advanceTimersByTime(20000);
expect(redlock.extend).toHaveBeenCalledTimes(0);
});
it('cleans up if lock extension failed.', async(): Promise<void> => {
// This should never happen though
redlock.extend.mockImplementationOnce((): any => {
throw new Error('random error');
});
await locker.acquire(identifier);
jest.advanceTimersByTime(20000);
expect(redlock.extend).toHaveBeenCalledTimes(1);
// Will throw since we removed the lock entry
await expect(locker.release(identifier)).rejects.toThrow(InternalServerError);
});
it('can acquire different keys simultaneously.', async(): Promise<void> => {
const lock1 = locker.acquire({ path: 'path1' });
const lock2 = locker.acquire({ path: 'path2' });
const lock3 = locker.acquire({ path: 'path3' });
await expect(Promise.all([ lock1, lock2, lock3 ])).resolves.toBeDefined();
await locker.release({ path: 'path1' });
await locker.release({ path: 'path2' });
await locker.release({ path: 'path3' });
});
it('extends a lock indefinitely.', async(): Promise<void> => {
await locker.acquire(identifier);
jest.advanceTimersByTime(20000);
await expect(locker.release(identifier)).resolves.toBeUndefined();
});
it('uses users redlockOptions if passed to constructor.', async(): Promise<void> => {
// Reset calls done in `beforeEach`
jest.clearAllMocks();
const clients = [ '6379' ];
const options = {
driftFactor: 0.2,
retryCount: 20,
retryDelay: 2000,
retryJitter: 2000,
};
locker = new RedisResourceLocker(clients, options);
expect(Redlock).toHaveBeenCalledTimes(1);
expect(Redlock).toHaveBeenLastCalledWith(expect.any(Array), options);
});
it('errors on creation when no redis servers are passed to the constructor.', async(): Promise<void> => {
expect((): any => new RedisResourceLocker([])).toThrow('At least 1 client should be provided');
});
it('errors if there is an issue creating the Redlock.', async(): Promise<void> => {
(Redlock as unknown as jest.Mock).mockImplementationOnce((): never => {
throw new Error('redlock error!');
});
expect((): any => new RedisResourceLocker([ '1234' ]))
.toThrow('Error initializing Redlock: Error: redlock error!');
});
it('logs redis client errors.', async(): Promise<void> => {
const logger = { error: jest.fn() };
const mock = jest.spyOn(LogUtil, 'getLoggerFor');
mock.mockReturnValueOnce(logger as any);
locker = new RedisResourceLocker([ '6379' ]);
expect(logger.error).toHaveBeenCalledTimes(0);
redlock.emit('clientError', 'problem!');
expect(logger.error).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenLastCalledWith('Redis/Redlock error: problem!');
});
describe('createRedisClients', (): void => {
it('should create and return the right amount of redisClients.', async(): Promise<void> => {
// Reset calls done in `beforeEach`
jest.clearAllMocks();
const clientStrings = [ '6379', '127.0.0.1:6378' ];
locker = new RedisResourceLocker(clientStrings);
expect(createClient).toHaveBeenCalledTimes(2);
expect(createClient).toHaveBeenCalledWith(6379, undefined);
expect(createClient).toHaveBeenCalledWith(6378, '127.0.0.1');
});
it('errors when invalid string is passed.', async(): Promise<void> => {
// Reset calls done in `beforeEach`
jest.clearAllMocks();
const clientStrings = [ 'noHostOrPort' ];
expect((): any => new RedisResourceLocker(clientStrings))
.toThrow('Invalid data provided to create a Redis client: noHostOrPort');
expect(createClient).toHaveBeenCalledTimes(0);
});
});
describe('quit()', (): void => {
it('should clear all locks and intervals when quit() is called.', async(): Promise<void> => {
await locker.acquire(identifier);
await locker.quit();
expect(redlock.quit).toHaveBeenCalledTimes(1);
// This works since the Redlock is simply a mock and quit should have cleared the lockMap
await expect(locker.acquire(identifier)).resolves.toBeUndefined();
});
});
});