feat: Split ResourceStore config into 2 parts

This commit is contained in:
Joachim Van Herwegen 2021-07-15 14:41:58 +02:00
parent fdd6f853b6
commit ad7f4ed134
36 changed files with 153 additions and 220 deletions

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/memory.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/dynamic.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/dynamic.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/file.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/file.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/file.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/file.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/memory.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/subdomain.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/regex.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/regex.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/sparql.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/sparql.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -1,19 +1,22 @@
# Storage
Options related to how data and resources are stored.
## Backend
The final part of the ResourceStore chain that handles data access.
* *dynamic*: The routing store used here is needed when using dynamic pod creation.
* *file*: Default setup with a file backend.
* *memory*: Default setup with a memory backend.
* *regex*: Uses a different backend based on the container that is being used.
* *sparql*: Default setup with a SPARQL endpoint backend.
Also updates the converting store so all incoming data is transformed into quads.
## Key-Value
Used by certain classes for internal storage.
* *memory*: Store everything in memory.
* *resource-store*: Store everything in a specific container in the resource store.
## Resource-Store
The chain of ResourceStores that needs to be passed through before reaching the actual data.
There is much variety possible here so chances are higher that a custom solution is needed here.
Most configs here have the same default setup (Monitoring -> Locking -> Patching -> Converting)
and only differ at the tail.
* *dynamic*: The routing store used at the tail here is needed when using dynamic pod creation.
* *file*: Default setup with a file backend.
* *memory*: Default setup with a memory backend.
* *regex*: Uses a different backend based on the container that is being used.
* *sparql*: Default setup with a SPARQL endpoint backend.
Also updates the converting store so all incoming data is transformed into quads.
## Middleware
The chain of utility ResourceStores that needs to be passed through before reaching the backend stores.
The final store in this chain takes the store from the stores/backend config as source.
* *default*: Chains all the utility stores:
Monitoring -> IndexRepresentation -> Locking -> Patching -> Converting

View File

@ -0,0 +1,25 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/backend/data-accessors/file.json"
],
"@graph": [
{
"comment": "Setup when using dynamic pods. A router rule will be defined when activating dynamic pods.",
"@id": "urn:solid-server:default:ResourceStore_Backend",
"@type": "RoutingResourceStore",
"rule": { "@id": "urn:solid-server:default:RouterRule" }
},
{
"comment": "Which store to use for requests that do not match any pod, e.g. for storage.",
"@id": "urn:solid-server:default:RouterRule",
"BaseUrlRouterRule:_baseStore": {
"@id": "urn:solid-server:default:ResourceStore_DataAccessor",
"@type": "DataAccessorBasedStore",
"identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },
"auxiliaryStrategy": { "@id": "urn:solid-server:default:AuxiliaryStrategy" },
"accessor": { "@id": "urn:solid-server:default:FileDataAccessor" }
}
}
]
}

View File

@ -0,0 +1,16 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/backend/data-accessors/file.json"
],
"@graph": [
{
"comment": "A default store setup with a file system backend.",
"@id": "urn:solid-server:default:ResourceStore_Backend",
"@type": "DataAccessorBasedStore",
"identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },
"auxiliaryStrategy": { "@id": "urn:solid-server:default:AuxiliaryStrategy" },
"accessor": { "@id": "urn:solid-server:default:FileDataAccessor" }
}
]
}

View File

@ -1,12 +1,16 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/backend/data-accessors/memory.json"
],
"@graph": [
{
"comment": "Supports default Solid behaviour on any kind of data accessor.",
"@id": "urn:solid-server:default:ResourceStore_DataAccessor",
"comment": "A default store setup with a memory backend.",
"@id": "urn:solid-server:default:ResourceStore_Backend",
"@type": "DataAccessorBasedStore",
"identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },
"auxiliaryStrategy": { "@id": "urn:solid-server:default:AuxiliaryStrategy" }
"auxiliaryStrategy": { "@id": "urn:solid-server:default:AuxiliaryStrategy" },
"accessor": { "@id": "urn:solid-server:default:MemoryDataAccessor" }
}
]
}

View File

@ -1,34 +1,16 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/resource-store/data-accessors/file.json",
"files-scs:config/storage/resource-store/data-accessors/memory.json",
"files-scs:config/storage/resource-store/data-accessors/sparql-endpoint.json",
"files-scs:config/storage/resource-store/stores/converting.json",
"files-scs:config/storage/resource-store/stores/locking.json",
"files-scs:config/storage/resource-store/stores/patching.json",
"files-scs:config/storage/resource-store/stores/routing.json"
"files-scs:config/storage/backend/data-accessors/file.json",
"files-scs:config/storage/backend/data-accessors/memory.json",
"files-scs:config/storage/backend/data-accessors/sparql-endpoint.json"
],
"@graph": [
{
"comment": "A more complex example with 3 different stores being routed to.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Index",
"IndexRepresentationStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Locking",
"LockingResourceStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Patching",
"PatchingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Converting",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Routing"
}
}
}
}
}
"@id": "urn:solid-server:default:ResourceStore_Backend",
"@type": "RoutingResourceStore",
"rule": { "@id": "urn:solid-server:default:RouterRule" }
},
{

View File

@ -0,0 +1,22 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/backend/data-accessors/sparql-endpoint.json"
],
"@graph": [
{
"comment": "A default store setup with a memory backend.",
"@id": "urn:solid-server:default:ResourceStore_Backend",
"@type": "DataAccessorBasedStore",
"identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },
"auxiliaryStrategy": { "@id": "urn:solid-server:default:AuxiliaryStrategy" },
"accessor": { "@id": "urn:solid-server:default:SparqlDataAccessor" }
},
{
"comment": "This makes it so all incoming data is converted to quad objects.",
"@id": "urn:solid-server:default:ResourceStore_Converting",
"RepresentationConvertingStore:_options_inConverter": { "@id": "urn:solid-server:default:RepresentationConverter" },
"RepresentationConvertingStore:_options_inType": "internal/quads"
}
]
}

View File

@ -0,0 +1,32 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/middleware/stores/converting.json",
"files-scs:config/storage/middleware/stores/locking.json",
"files-scs:config/storage/middleware/stores/patching.json"
],
"@graph": [
{
"comment": "Sets up a stack of utility stores used by most instances.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": { "@id": "urn:solid-server:default:ResourceStore_Index" }
},
{
"@id": "urn:solid-server:default:ResourceStore_Index",
"IndexRepresentationStore:_source": { "@id": "urn:solid-server:default:ResourceStore_Locking" }
},
{
"@id": "urn:solid-server:default:ResourceStore_Locking",
"LockingResourceStore:_source": { "@id": "urn:solid-server:default:ResourceStore_Patching" }
},
{
"@id": "urn:solid-server:default:ResourceStore_Patching",
"PatchingStore:_source": { "@id": "urn:solid-server:default:ResourceStore_Converting" }
},
{
"@id": "urn:solid-server:default:ResourceStore_Converting",
"RepresentationConvertingStore:_source": { "@id": "urn:solid-server:default:ResourceStore_Backend" }
}
]
}

View File

@ -1,42 +0,0 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/resource-store/data-accessors/file.json",
"files-scs:config/storage/resource-store/stores/converting.json",
"files-scs:config/storage/resource-store/stores/data-accessor.json",
"files-scs:config/storage/resource-store/stores/locking.json",
"files-scs:config/storage/resource-store/stores/patching.json",
"files-scs:config/storage/resource-store/stores/routing.json"
],
"@graph": [
{
"comment": "Setup when using dynamic pods.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Index",
"IndexRepresentationStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Locking",
"LockingResourceStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Patching",
"PatchingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Converting",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Routing",
"comment": "A router rule will be defined when activating dynamic pods."
}
}
}
}
}
},
{
"comment": "Which store to use for requests that do not match any pod, e.g. for storage.",
"@id": "urn:solid-server:default:RouterRule",
"BaseUrlRouterRule:_baseStore": {
"@id": "urn:solid-server:default:ResourceStore_DataAccessor",
"DataAccessorBasedStore:_accessor": { "@id": "urn:solid-server:default:FileDataAccessor" }
}
}
]
}

View File

@ -1,35 +0,0 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/resource-store/data-accessors/file.json",
"files-scs:config/storage/resource-store/stores/converting.json",
"files-scs:config/storage/resource-store/stores/data-accessor.json",
"files-scs:config/storage/resource-store/stores/locking.json",
"files-scs:config/storage/resource-store/stores/patching.json"
],
"@graph": [
{
"comment": "A default store setup with a file system backend.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Index",
"IndexRepresentationStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Locking",
"LockingResourceStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Patching",
"PatchingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Converting",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_DataAccessor",
"DataAccessorBasedStore:_accessor": {
"@id": "urn:solid-server:default:FileDataAccessor"
}
}
}
}
}
}
}
]
}

View File

@ -1,36 +0,0 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/resource-store/data-accessors/memory.json",
"files-scs:config/storage/resource-store/stores/converting.json",
"files-scs:config/storage/resource-store/stores/data-accessor.json",
"files-scs:config/storage/resource-store/stores/locking.json",
"files-scs:config/storage/resource-store/stores/patching.json"
],
"@graph": [
{
"comment": "A default store setup with a memory backend.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Index",
"IndexRepresentationStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Locking",
"LockingResourceStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Patching",
"PatchingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Converting",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_DataAccessor",
"DataAccessorBasedStore:_accessor": {
"@id": "urn:solid-server:default:MemoryDataAccessor"
}
}
}
}
}
}
}
]
}

View File

@ -1,40 +0,0 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"import": [
"files-scs:config/storage/resource-store/data-accessors/sparql-endpoint.json",
"files-scs:config/storage/resource-store/stores/converting.json",
"files-scs:config/storage/resource-store/stores/data-accessor.json",
"files-scs:config/storage/resource-store/stores/locking.json",
"files-scs:config/storage/resource-store/stores/patching.json"
],
"@graph": [
{
"comment": "A default store setup with a memory backend.",
"@id": "urn:solid-server:default:ResourceStore",
"@type": "MonitoringStore",
"MonitoringStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Index",
"IndexRepresentationStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Locking",
"LockingResourceStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Patching",
"PatchingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_Converting",
"comment": "This makes it so all incoming data is converted to quad objects.",
"RepresentationConvertingStore:_options_inConverter": {
"@id": "urn:solid-server:default:RepresentationConverter"
},
"RepresentationConvertingStore:_options_inType": "internal/quads",
"RepresentationConvertingStore:_source": {
"@id": "urn:solid-server:default:ResourceStore_DataAccessor",
"DataAccessorBasedStore:_accessor": {
"@id": "urn:solid-server:default:SparqlDataAccessor"
}
}
}
}
}
}
}
]
}

View File

@ -1,11 +0,0 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^1.0.0/components/context.jsonld",
"@graph": [
{
"comment": "Routes data to different stores based on the chosen rule.",
"@id": "urn:solid-server:default:ResourceStore_Routing",
"@type": "RoutingResourceStore",
"rule": { "@id": "urn:solid-server:default:RouterRule" }
}
]
}

View File

@ -20,11 +20,11 @@ const baseUrl = `http://localhost:${port}/`;
const rootFilePath = getTestFolder('full-config-acl');
const stores: [string, any][] = [
[ 'in-memory storage', {
storeConfig: 'storage/resource-store/memory.json',
storeConfig: 'storage/backend/memory.json',
teardown: jest.fn(),
}],
[ 'on-disk storage', {
storeConfig: 'storage/resource-store/file.json',
storeConfig: 'storage/backend/file.json',
teardown: (): void => removeFolder(rootFilePath),
}],
];

View File

@ -21,11 +21,11 @@ const baseUrl = `http://localhost:${port}/`;
const rootFilePath = getTestFolder('full-config-no-auth');
const stores: [string, any][] = [
[ 'in-memory storage', {
storeConfig: 'storage/resource-store/memory.json',
storeConfig: 'storage/backend/memory.json',
teardown: jest.fn(),
}],
[ 'on-disk storage', {
storeConfig: 'storage/resource-store/file.json',
storeConfig: 'storage/backend/file.json',
teardown: (): void => removeFolder(rootFilePath),
}],
];

View File

@ -21,7 +21,7 @@ describeIf('docker', 'A server with a SPARQL endpoint as storage', (): void => {
const instances = await instantiateFromConfig(
'urn:solid-server:test:Instances',
[
getPresetConfigPath('storage/resource-store/sparql.json'),
getPresetConfigPath('storage/backend/sparql.json'),
getTestConfigPath('ldp-with-auth.json'),
],
variables,

View File

@ -17,11 +17,11 @@ const baseUrl = `http://localhost:${port}/`;
const rootFilePath = getTestFolder('subdomains');
const stores: [string, any][] = [
[ 'in-memory storage', {
storeConfig: 'storage/resource-store/memory.json',
storeConfig: 'storage/backend/memory.json',
teardown: jest.fn(),
}],
[ 'on-disk storage', {
storeConfig: 'storage/resource-store/file.json',
storeConfig: 'storage/backend/file.json',
teardown: (): void => removeFolder(rootFilePath),
}],
];

View File

@ -14,6 +14,7 @@
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -13,8 +13,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/memory.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -17,8 +17,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/dynamic.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/dynamic.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -16,8 +16,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/memory.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",

View File

@ -18,6 +18,7 @@
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/subdomain.json",
"files-scs:config/util/index/default.json",

View File

@ -13,8 +13,9 @@
"files-scs:config/ldp/metadata-parser/default.json",
"files-scs:config/ldp/metadata-writer/default.json",
"files-scs:config/ldp/permissions/acl.json",
"files-scs:config/storage/backend/memory.json",
"files-scs:config/storage/key-value/memory.json",
"files-scs:config/storage/resource-store/memory.json",
"files-scs:config/storage/middleware/default.json",
"files-scs:config/util/auxiliary/acl.json",
"files-scs:config/util/identifiers/suffix.json",
"files-scs:config/util/index/default.json",