ci: Add Conformance Test Harness checks to PRs

* ci: split up workflows

* ci: tweaks and housekeeping

* ci: additional comments and requested changes

* ci: additional comments

* chore: clean up .gitignore

* ci: CTH testing on PRs + reusable

* ci: allow failure on job instead of command

* ci: remove npm-test dependency for CTH

* ci: fix branch for scheduled run of CTH

* ci: add concurrency group for PRs

.github/workflows/cth-test.yml

@@ -0,0 +1,90 @@
+name: Conformance test harness
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        type: string
+        required: false
+        default: latest
+        description: Version of the Conformance Test Harness that has to be tested against
+      branch:
+        type: string
+        required: false
+        description: branch to run tests against (used for matrix strategy)
+      ignore_failures:
+        type: boolean
+        default: false
+        description: Return succes even if there are failures?
+        required: false
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        required: false
+        default: latest
+        description: Version of the Conformance Test Harness that has to be tested against.
+      ignore_failures:
+        type: boolean
+        default: false
+        description: Return succes even if there are failures?
+        required: false
+
+jobs:
+  # Run the Conformance Test Harness and upload output as artifacts to GitHub
+  conformance:
+    continue-on-error: ${{ inputs.ignore_failures }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+      - name: Check out the project
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.branch || github.ref }}
+      - name: Install dependencies and run build scripts
+        run: npm ci
+      - name: Start the server in the background
+        run: npm start > server-output.log  &
+      - name: Create the necessary folders
+        run: mkdir -p reports/css
+      - name: Pull the conformance harness docker
+        run: "docker pull solidproject/conformance-test-harness:${{ inputs.version }}"
+      - name: Wait until the server has started
+        run: >
+          curl --output /dev/null --silent --head --fail --retry 30
+          --retry-connrefused --retry-delay 0 -k http://localhost:3000/
+      - name: Create users
+        run: npx ts-node test/deploy/createAccountCredentials.ts http://localhost:3000/ >> test/deploy/conformance.env
+      - name: Run the test harness
+        run: >
+          docker run -i --rm
+          -v "$(pwd)"/reports/css:/reports
+          --env-file=./test/deploy/conformance.env
+          --network="host"
+          solidproject/conformance-test-harness
+          --skip-teardown
+          --output=/reports
+          --target=https://github.com/solid/conformance-test-harness/css
+      # Steps below use `always()` to make sure logs get uploaded in case the CTH errors
+      - name: Sanitize branch name for artifact upload
+        id: sanitize
+        if: always()
+        uses: yeouchien/sanitize-branch-name-action@v1
+        with:
+          branch-name: ${{ inputs.branch || github.head_ref }}
+      - name: Save the reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.sanitize.outputs.sanitized-branch-name }} reports
+          path: reports
+      - name: Save the server output
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.sanitize.outputs.sanitized-branch-name }} server output
+          path: server-output.log

.github/workflows/main.yml

@@ -7,14 +7,24 @@ on:
     tags:
       - 'v*'
   pull_request:
-    branches:
+
-      - '**'
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 
 jobs:
   npm-test:
   # Run the tests in every case
     uses: ./.github/workflows/npm-test.yml
 
+  cth-test:
+  # Run the Conformance Test Harness on PRs targeting main or versions/
+    if: github.event_name == 'pull_request' && ( github.base_ref == 'main' || startsWith( github.base_ref, 'versions/' ) )
+    uses: ./.github/workflows/cth-test.yml
+    with:
+      ignore_failures: true
+      version: 1.1.7 # The latest version that CSS is confirmed to pass
+
   docker:
   # Build docker containers on version tag, push to main and push to versions/
     needs: npm-test

.github/workflows/schedule.yml

@@ -3,69 +3,16 @@ name: Conformance test harness
 on:
   schedule:
     - cron: '23 5 * * *'
-  push:
+  workflow_dispatch:
-    branches:
+
-     - test/conformance
 jobs:
   conformance:
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         branch:
           - 'main'
           - 'versions/6.0.0'
-    timeout-minutes: 10
+    uses: ./.github/workflows/cth-test.yml
-    steps:
+    with:
-      - name: Use Node.js 16.x
+      branch: ${{ matrix.branch }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16.x
-      - name: Check out the project
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ matrix.branch }}
-      - name: Install dependencies and run build scripts
-        run: npm ci
-      - name: Start the server in the background
-        run: npm start > server-output.log  &
-      - name: Create the necessary folders
-        run: mkdir -p reports/css
-      - name: Pull the conformance harness docker
-        run: docker pull solidproject/conformance-test-harness
-      - name: Wait until the server has started
-        run: |
-          until $(curl --output /dev/null --silent --head --fail -k http://localhost:3000/); do
-            sleep 1
-          done
-      - name: Create users
-        run: npx ts-node test/deploy/createAccountCredentials.ts http://localhost:3000/ >> test/deploy/conformance.env
-      - name: Run the test harness
-        run: >
-          docker run -i --rm
-          -v "$(pwd)"/reports/css:/reports
-          --env-file=./test/deploy/conformance.env
-          --network="host"
-          solidproject/conformance-test-harness
-          --skip-teardown
-          --output=/reports
-          --target=https://github.com/solid/conformance-test-harness/css
-      # Steps below use `always()` to make sure logs get uploaded in case the CTH errors
-      - name: Sanitize branch name for artifact upload
-        id: sanitize
-        if: always()
-        uses: yeouchien/sanitize-branch-name-action@v1
-        with:
-          branch-name: ${{ matrix.branch }}
-      - name: Save the reports
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ steps.sanitize.outputs.sanitized-branch-name }} reports
-          path: reports
-      - name: Save the server output
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ steps.sanitize.outputs.sanitized-branch-name }} server output
-          path: server-output.log