Merge branch 'main' into versions/5.0.0

This commit is contained in:
Joachim Van Herwegen 2022-04-25 15:46:04 +02:00
commit 7e5483a36d
9 changed files with 435 additions and 44 deletions

View File

@ -58,8 +58,6 @@ jobs:
github-token: ${{ secrets.github_token }}
flag-name: test-unit-${{ matrix.node-version }}-${{ matrix.operating-system }}
parallel: true
- name: Run deployment tests
run: npm run test:deploy
test-integration:
runs-on: ubuntu-latest
@ -118,6 +116,27 @@ jobs:
- name: Run integration tests
run: npm run test:integration
test-configs:
runs-on: ubuntu-latest
services:
sparql-endpoint:
image: tenforce/virtuoso
env:
SPARQL_UPDATE: true
ports:
- 4000:8890
steps:
- name: Use Node.js
uses: actions/setup-node@v3
with:
node-version: '16.x'
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run deploy tests
run: npm run test:deploy
coveralls:
needs: test-unit
runs-on: ubuntu-latest
@ -134,6 +153,7 @@ jobs:
- test-unit
- test-integration
- test-integration-windows
- test-configs
# Only run on tag push events starting with v prefix for now OR main branch push events
if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main')
runs-on: ubuntu-latest
@ -175,6 +195,7 @@ jobs:
- test-unit
- test-integration
- test-integration-windows
- test-configs
# Only run on push events on a versions/* branch (ASSUMPTION: THERE SHOULD ONLY BE ONE THERE!)
if: startsWith(github.ref, 'refs/heads/versions/')
runs-on: ubuntu-latest
@ -231,6 +252,8 @@ jobs:
- lint
- test-unit
- test-integration
- test-integration-windows
- test-configs
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/v')
steps:

View File

@ -54,7 +54,7 @@
"start": "node ./bin/server.js",
"start:file": "node ./bin/server.js -c config/file.json -f ./data",
"test": "npm run test:ts && npm run jest",
"test:deploy": "test/deploy/validate-package.sh",
"test:deploy": "test/deploy/validate-configs.sh",
"test:ts": "tsc -p test --noEmit",
"test:integration": "jest --coverageReporters text-summary -- test/integration",
"test:unit": "jest --config=./jest.coverage.config.js test/unit",

View File

@ -107,17 +107,36 @@ export function getExtension(path: string): string {
}
/**
* Performs a transformation on the path components of a URI.
* Performs a transformation on the path components of a URI,
* preserving but normalizing path delimiters and their escaped forms.
*/
function transformPathComponents(path: string, transform: (part: string) => string): string {
const [ , base, queryString ] = /^([^?]*)(.*)$/u.exec(path)!;
const transformed = base.split('/').map((element): string => transform(element)).join('/');
const transformed = base
// We split on actual URI path component delimiters (slash and backslash),
// but also on things that could be wrongly interpreted as component delimiters,
// such that they cannot be transformed incorrectly.
// We thus ensure that encoded slashes (%2F) and backslashes (%5C) are preserved,
// since they would become _actual_ delimiters if accidentally decoded.
// Additionally, we need to preserve any encoded percent signs (%25)
// that precede them, because these might change their interpretation as well.
.split(/(\/|\\|%(?:25)*(?:2f|5c))/ui)
// Even parts map to components that need to be transformed,
// odd parts to (possibly escaped) delimiters that need to be normalized.
.map((part, index): string =>
index % 2 === 0 ? transform(part) : part.toUpperCase())
.join('');
return !queryString ? transformed : `${transformed}${queryString}`;
}
/**
* Converts a URI path to the canonical version by splitting on slashes,
* decoding any percent-based encodings, and then encoding any special characters.
* This function is used to clean unwanted characters in the components of
* the provided path.
*
* @param path - The path to convert to its canonical URI path form.
* @returns The canonical URI path form of the provided path.
*/
export function toCanonicalUriPath(path: string): string {
return transformPathComponents(path, (part): string =>
@ -125,14 +144,24 @@ export function toCanonicalUriPath(path: string): string {
}
/**
* Decodes all components of a URI path.
* This function is used when converting a URI to a file path. Decodes all components of a URI path,
* with the exception of encoded slash characters, as this would lead to unexpected file locations
* being targeted (resulting in erroneous behaviour of the file based backend).
*
* @param path - The path to decode the URI path components of.
* @returns A decoded copy of the provided URI path (ignoring encoded slash characters).
*/
export function decodeUriPathComponents(path: string): string {
return transformPathComponents(path, decodeURIComponent);
}
/**
* Encodes all (non-slash) special characters in a URI path.
* This function is used in the process of converting a file path to a URI. Encodes all (non-slash)
* special characters in a URI path, with the exception of encoded slash characters, as this would
* lead to unnecessary double encoding, resulting in a URI that differs from the expected result.
*
* @param path - The path to encode the URI path components of.
* @returns An encoded copy of the provided URI path (ignoring encoded slash characters).
*/
export function encodeUriPathComponents(path: string): string {
return transformPathComponents(path, encodeURIComponent);

120
test/deploy/validate-configs.sh Executable file
View File

@ -0,0 +1,120 @@
#!/usr/bin/env bash
# Script to validate the packaged module and configs
# Ensure our workdir is that of the project root
cd "${0%/*}/../.." || { echo "Error setting workdir to project directory."; exit 1; }
# This script takes config paths (from project directory) as optional input
# No arguments: all default configs are tested
# One ore more arguments: provided configs are tested
# Example: validate-configs.sh config/default.json config/file.json
TEST_NAME="Deployment testing"
declare -a CONFIG_ARRAY
if [[ $# -gt 0 ]]; then
for CONFIG in "$@"; do
if [ ! -f "$CONFIG" ]; then
echo "Config file $CONFIG does not exist, check the path (example: config/default.json)"
exit 1
fi
CONFIG_ARRAY+=("$CONFIG")
done
echo "Deployment testing ${#CONFIG_ARRAY[@]} configs:"
else
mapfile -t CONFIG_ARRAY < <(ls config/*.json)
echo "Deployment testing all configs:"
fi
printf " - %s\n" "${CONFIG_ARRAY[@]}"
mkdir -p test/tmp/data
echo "$TEST_NAME - Building and installing package"
npm pack --loglevel warn
npm install -g solid-community-server-*.tgz --loglevel warn
rm solid-community-server-*.tgz
run_server_with_config () {
if [[ $# -ne 2 ]]; then
echo "Config arguments not set"
return 1
elif [ ! -f "$1" ]; then
echo "Config file does not exist, check the path."
return 1
fi
CONFIG_PATH=$1
CONFIG_NAME=$2
mkdir -p test/tmp/data
CSS_ARGS=("-p" "8888" "-l" "warn" "-f" "test/tmp/data/" "-s" "http://localhost:4000/sparql")
CSS_BASE_URL="http://localhost:8888"
# HTTPS config specifics: self-signed key/cert + CSS base URL override
if [[ $CONFIG_NAME =~ "https" ]]; then
openssl req -x509 -nodes -days 1 -newkey rsa:2048 -keyout test/tmp/server.key -out test/tmp/server.cert -subj '/CN=localhost' &>/dev/null
CSS_BASE_URL="https://localhost:8888"
if [[ $CONFIG_NAME =~ "https-file-cli" ]]; then
CSS_ARGS+=("--httpsKey" "test/tmp/server.key" "--httpsCert" "test/tmp/server.cert")
elif [[ $CONFIG_NAME =~ "example-https" ]]; then
CONFIG_PATH=test/tmp/example-https-file.json
cp config/example-https-file.json $CONFIG_PATH
sed -i -E "s/(\W+\"options_key\".*\").+(\".*)/\1test\/tmp\/server.key\2/" $CONFIG_PATH
sed -i -E "s/(\W+\"options_cert\".*\").+(\".*)/\1test\/tmp\/server.cert\2/" $CONFIG_PATH
fi
fi
echo -e "----------------------------------"
echo "$TEST_NAME($CONFIG_NAME) - Starting the server"
community-solid-server "${CSS_ARGS[@]}" -b $CSS_BASE_URL -c $CONFIG_PATH &>test/tmp/"$CONFIG_NAME" &
PID=$!
FAILURE=1
if [ -z $PID ]; then
echo "$TEST_NAME($CONFIG_NAME) - FAILURE: Server did not start"
cat test/tmp/"$CONFIG_NAME"
else
echo "$TEST_NAME($CONFIG_NAME) - Attempting HTTP access to the server"
if curl -sfkI -X GET --retry 15 --retry-connrefused --retry-delay 1 $CSS_BASE_URL > test/tmp/"$CONFIG_NAME"-curl; then
echo "$TEST_NAME($CONFIG_NAME) - SUCCESS: server reached"
FAILURE=0
else
echo "$TEST_NAME($CONFIG_NAME) - FAILURE: Could not reach server"
fi
kill -9 $PID &> /dev/null
timeout 30s tail --pid=$PID -f /dev/null
fi
rm -rf test/tmp/data/*
return $FAILURE
}
VALIDATION_FAILURE=0
SUCCESSES=''
FAILURES=''
for CONFIG_PATH in "${CONFIG_ARRAY[@]}"; do
CONFIG_NAME=$(echo "$CONFIG_PATH" | sed -E 's/.+\/(.+)\.json/\1/')
run_server_with_config "$CONFIG_PATH" "$CONFIG_NAME"
SERVER_FAILURE=$?
if [ $SERVER_FAILURE -eq 0 ]; then
SUCCESSES="${SUCCESSES}[SUCCESS]\t$CONFIG_NAME\t($CONFIG_PATH)\n"
else
echo "$TEST_NAME($CONFIG_NAME) - CSS logs: ";
cat test/tmp/"$CONFIG_NAME"
echo "$TEST_NAME($CONFIG_NAME) - curl logs: ";
cat test/tmp/"$CONFIG_NAME"-curl
VALIDATION_FAILURE=1
FAILURES="${FAILURES}[FAILURE]\t$CONFIG_NAME\t($CONFIG_PATH)\n"
fi
echo ""
done;
echo "$TEST_NAME - Cleanup"
npm uninstall -g @solid/community-server
rm -rf test/tmp/*
echo -e "\n\n----------------------------------------"
echo "Config validation overview"
echo "----------------------------------------"
echo -e "$SUCCESSES"
echo -e "$FAILURES"
exit $VALIDATION_FAILURE

View File

@ -1,37 +0,0 @@
#!/usr/bin/env bash
# Script to validate the packaged module
TEST_NAME="Deployment test: packaged module"
echo "$TEST_NAME - Building and installing package"
npm pack --loglevel warn
npm install -g solid-community-server-*.tgz --loglevel warn
rm solid-community-server-*.tgz
echo "$TEST_NAME - Starting the server"
community-solid-server -p 8888 -l warn &
PID=$!
FAILURE=1
if [ -z $PID ]; then
echo "$TEST_NAME - Failure: Server did not start"
else
echo "$TEST_NAME - Attempting HTTP access to the server"
for i in {1..10}; do
sleep 1
if curl -s -f localhost:8888 > /dev/null; then
echo "$TEST_NAME - Success: server reached"
FAILURE=0
break
fi
done
if [ $FAILURE -eq 1 ]; then
echo "$TEST_NAME - Failure: Could not reach server"
fi
kill -9 $PID
fi
echo "$TEST_NAME - Cleanup"
npm uninstall -g @solid/community-server
exit $FAILURE

View File

@ -0,0 +1,149 @@
import fetch from 'cross-fetch';
import { pathExists } from 'fs-extra';
import type { App } from '../../src/init/App';
import { getPort } from '../util/Util';
import {
getDefaultVariables,
getTestConfigPath,
getTestFolder,
instantiateFromConfig,
removeFolder,
} from './Config';
const port = getPort('FileBackendEncodedSlashHandling');
const baseUrl = `http://localhost:${port}/`;
const rootFilePath = getTestFolder('file-backend-encoded-slash-handling');
describe('A server with a file backend storage', (): void => {
let app: App;
beforeAll(async(): Promise<void> => {
await removeFolder(rootFilePath);
const variables = {
...getDefaultVariables(port, baseUrl),
'urn:solid-server:default:variable:rootFilePath': rootFilePath,
};
// Create and start the server
const instances = await instantiateFromConfig(
'urn:solid-server:test:Instances',
[
getTestConfigPath('server-file.json'),
],
variables,
) as Record<string, any>;
({ app } = instances);
await app.start();
});
afterAll(async(): Promise<void> => {
await removeFolder(rootFilePath);
await app.stop();
});
it('can put a document for which the URI path contains URL-encoded separator characters.', async(): Promise<void> => {
const url = `${baseUrl}c1/c2/t1%2f`;
const res = await fetch(url, {
method: 'PUT',
headers: {
'content-type': 'text/plain',
},
body: 'abc',
});
expect(res.status).toBe(201);
expect(res.headers.get('location')).toBe(`${baseUrl}c1/c2/t1%2F`);
// The resource should not be accessible through ${baseUrl}c1/c2/t1/.
const check1 = await fetch(`${baseUrl}c1/c2/t1/}`, {
method: 'GET',
headers: {
accept: 'text/plain',
},
});
expect(check1.status).toBe(404);
// Check that the created resource is not a container
const check2 = await fetch(url, {
method: 'GET',
headers: {
accept: 'text/plain',
},
});
const linkHeaderValues = check2.headers.get('link')!.split(',').map((item): string => item.trim());
expect(linkHeaderValues).not.toContain('<http://www.w3.org/ns/ldp#Container>; rel="type"');
// Check that the appropriate file path exists
const check3 = await pathExists(`${rootFilePath}/c1/c2/t1%2F$.txt`);
expect(check3).toBe(true);
});
it('can post a document using a slug that contains URL-encoded separator characters.', async(): Promise<void> => {
const slug = 't1%2Faa';
const res = await fetch(baseUrl, {
method: 'POST',
headers: {
'content-type': 'text/plain',
slug,
},
body: 'abc',
});
expect(res.status).toBe(201);
expect(res.headers.get('location')).toBe(`${baseUrl}${slug}`);
// Check that the appropriate file path exists
const check = await pathExists(`${rootFilePath}/${slug}$.txt`);
expect(check).toBe(true);
});
it('prevents accessing a document via a different identifier that results in the same path after URL decoding.',
async(): Promise<void> => {
// First put a resource using a path without encoded separator characters: foo/bar
const url = `${baseUrl}foo/bar`;
await fetch(url, {
method: 'PUT',
headers: {
'content-type': 'text/plain',
},
body: 'abc',
});
// The resource at foo/bar should not be accessible using the url encoded variant of this path: foo%2Fbar
const check1 = await fetch(`${baseUrl}foo%2Fbar`, {
method: 'GET',
headers: {
accept: 'text/plain',
},
});
// Expect foo%2Fbar to correctly refer to a different document, which does not exist.
expect(check1.status).toBe(404);
// Check that the the appropriate file path for foo/bar exists
const check2 = await pathExists(`${rootFilePath}/foo/bar$.txt`);
expect(check2).toBe(true);
// Next, put a resource using a path with an encoded separator character: bar%2Ffoo
await fetch(`${baseUrl}bar%2Ffoo`, {
method: 'PUT',
headers: {
'content-type': 'text/plain',
},
body: 'abc',
});
// The resource at bar%2Ffoo should not be accessible through bar/foo
const check3 = await fetch(`${baseUrl}bar/foo`, {
method: 'GET',
headers: {
accept: 'text/plain',
},
});
// Expect bar/foo to correctly refer to a different document, which does not exist.
expect(check3.status).toBe(404);
// Check that the the appropriate file path for bar%foo exists
const check4 = await pathExists(`${rootFilePath}/bar%2Ffoo$.txt`);
expect(check4).toBe(true);
});
});

View File

@ -0,0 +1,54 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^4.0.0/components/context.jsonld",
"import": [
"css:config/app/main/default.json",
"css:config/app/init/initialize-root.json",
"css:config/app/setup/disabled.json",
"css:config/http/handler/default.json",
"css:config/http/middleware/websockets.json",
"css:config/http/server-factory/websockets.json",
"css:config/http/static/default.json",
"css:config/identity/access/public.json",
"css:config/identity/handler/default.json",
"css:config/identity/ownership/token.json",
"css:config/identity/pod/static.json",
"css:config/identity/registration/enabled.json",
"css:config/ldp/authentication/dpop-bearer.json",
"css:config/ldp/authorization/webacl.json",
"css:config/ldp/handler/default.json",
"css:config/ldp/metadata-parser/default.json",
"css:config/ldp/metadata-writer/default.json",
"css:config/ldp/modes/default.json",
"css:config/storage/backend/file.json",
"css:config/storage/key-value/resource-store.json",
"css:config/storage/middleware/default.json",
"css:config/util/auxiliary/acl.json",
"css:config/util/identifiers/suffix.json",
"css:config/util/index/default.json",
"css:config/util/logging/winston.json",
"css:config/util/representation-conversion/default.json",
"css:config/util/resource-locker/memory.json",
"css:config/util/variables/default.json"
],
"@graph": [
{
"@id": "urn:solid-server:test:Instances",
"@type": "RecordObject",
"RecordObject:_record": [
{
"RecordObject:_record_key": "app",
"RecordObject:_record_value": { "@id": "urn:solid-server:default:App" }
}
]
},
{
"@id": "urn:solid-server:default:EmailSender",
"@type": "BaseEmailSender",
"args_senderName": "Solid Server",
"args_emailConfig_host": "smtp.example.email",
"args_emailConfig_port": 587,
"args_emailConfig_auth_user": "alice@example.email",
"args_emailConfig_auth_pass": "NYEaCsqV7aVStRCbmC"
}
]
}

View File

@ -112,6 +112,32 @@ describe('PathUtil', (): void => {
it('leaves the query string untouched.', (): void => {
expect(decodeUriPathComponents('/a%20path&/name?abc=def&xyz')).toBe('/a path&/name?abc=def&xyz');
});
it('ignores URL-encoded path separator characters.', (): void => {
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%2F')).toBe('/a path&/c1/c2/t1%2F');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%5C')).toBe('/a path&/c1/c2/t1%5C');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%252F')).toBe('/a path&/c1/c2/t1%252F');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%255C')).toBe('/a path&/c1/c2/t1%255C');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%25%252F')).toBe('/a path&/c1/c2/t1%%252F');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%25%255C')).toBe('/a path&/c1/c2/t1%%255C');
});
it('normalizes to uppercase encoding.', (): void => {
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%2f')).toBe('/a path&/c1/c2/t1%2F');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%5c')).toBe('/a path&/c1/c2/t1%5C');
});
it('accepts paths with mixed lowercase and uppercase encoding.', (): void => {
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%2F%2f')).toBe('/a path&/c1/c2/t1%2F%2F');
expect(decodeUriPathComponents('/a%20path&/c1/c2/t1%5C%5c')).toBe('/a path&/c1/c2/t1%5C%5C');
});
it('takes sequences of encoded percent signs into account.', (): void => {
expect(decodeUriPathComponents('/a%2Fb')).toBe('/a%2Fb');
expect(decodeUriPathComponents('/a%252Fb')).toBe('/a%252Fb');
expect(decodeUriPathComponents('/a%25252Fb')).toBe('/a%25252Fb');
expect(decodeUriPathComponents('/a%2525252Fb')).toBe('/a%2525252Fb');
});
});
describe('#encodeUriPathComponents', (): void => {
@ -122,6 +148,32 @@ describe('PathUtil', (): void => {
it('leaves the query string untouched.', (): void => {
expect(encodeUriPathComponents('/a%20path&/name?abc=def&xyz')).toBe('/a%2520path%26/name?abc=def&xyz');
});
it('does not double-encode URL-encoded path separator characters.', (): void => {
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%2F')).toBe('/a%2520path%26/c1/c2/t1%2F');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%5C')).toBe('/a%2520path%26/c1/c2/t1%5C');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%252F')).toBe('/a%2520path%26/c1/c2/t1%252F');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%255C')).toBe('/a%2520path%26/c1/c2/t1%255C');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%%252F')).toBe('/a%2520path%26/c1/c2/t1%25%252F');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%%255C')).toBe('/a%2520path%26/c1/c2/t1%25%255C');
});
it('normalizes to uppercase encoding.', (): void => {
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%2f')).toBe('/a%2520path%26/c1/c2/t1%2F');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%5c')).toBe('/a%2520path%26/c1/c2/t1%5C');
});
it('accepts paths with mixed lowercase and uppercase encoding.', (): void => {
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%2F%2f')).toBe('/a%2520path%26/c1/c2/t1%2F%2F');
expect(encodeUriPathComponents('/a%20path&/c1/c2/t1%5C%5c')).toBe('/a%2520path%26/c1/c2/t1%5C%5C');
});
it('takes sequences of encoded percent signs into account.', (): void => {
expect(encodeUriPathComponents('/a%2Fb')).toBe('/a%2Fb');
expect(encodeUriPathComponents('/a%252Fb')).toBe('/a%252Fb');
expect(encodeUriPathComponents('/a%25252Fb')).toBe('/a%25252Fb');
expect(encodeUriPathComponents('/a%2525252Fb')).toBe('/a%2525252Fb');
});
});
describe('#isContainerPath', (): void => {

View File

@ -8,6 +8,7 @@ const portNames = [
'ContentNegotiation',
'DynamicPods',
'ExpiringDataCleanup',
'FileBackendEncodedSlashHandling',
'GlobalQuota',
'Identity',
'LpdHandlerWithAuth',