diff --git a/.github/workflows/cth-test.yml b/.github/workflows/cth-test.yml new file mode 100644 index 000000000..48a12cf2a --- /dev/null +++ b/.github/workflows/cth-test.yml @@ -0,0 +1,90 @@ +name: Conformance test harness + +on: + workflow_call: + inputs: + version: + type: string + required: false + default: latest + description: Version of the Conformance Test Harness that has to be tested against + branch: + type: string + required: false + description: branch to run tests against (used for matrix strategy) + ignore_failures: + type: boolean + default: false + description: Return succes even if there are failures? + required: false + workflow_dispatch: + inputs: + version: + type: string + required: false + default: latest + description: Version of the Conformance Test Harness that has to be tested against. + ignore_failures: + type: boolean + default: false + description: Return succes even if there are failures? + required: false + +jobs: + # Run the Conformance Test Harness and upload output as artifacts to GitHub + conformance: + continue-on-error: ${{ inputs.ignore_failures }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Use Node.js 16.x + uses: actions/setup-node@v3 + with: + node-version: 16.x + - name: Check out the project + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch || github.ref }} + - name: Install dependencies and run build scripts + run: npm ci + - name: Start the server in the background + run: npm start > server-output.log & + - name: Create the necessary folders + run: mkdir -p reports/css + - name: Pull the conformance harness docker + run: "docker pull solidproject/conformance-test-harness:${{ inputs.version }}" + - name: Wait until the server has started + run: > + curl --output /dev/null --silent --head --fail --retry 30 + --retry-connrefused --retry-delay 0 -k http://localhost:3000/ + - name: Create users + run: npx ts-node test/deploy/createAccountCredentials.ts http://localhost:3000/ >> test/deploy/conformance.env + - name: Run the test harness + run: > + docker run -i --rm + -v "$(pwd)"/reports/css:/reports + --env-file=./test/deploy/conformance.env + --network="host" + solidproject/conformance-test-harness + --skip-teardown + --output=/reports + --target=https://github.com/solid/conformance-test-harness/css + # Steps below use `always()` to make sure logs get uploaded in case the CTH errors + - name: Sanitize branch name for artifact upload + id: sanitize + if: always() + uses: yeouchien/sanitize-branch-name-action@v1 + with: + branch-name: ${{ inputs.branch || github.head_ref }} + - name: Save the reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.sanitize.outputs.sanitized-branch-name }} reports + path: reports + - name: Save the server output + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.sanitize.outputs.sanitized-branch-name }} server output + path: server-output.log diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0bb5fcd6..04fc42644 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,14 +7,24 @@ on: tags: - 'v*' pull_request: - branches: - - '**' + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true jobs: npm-test: # Run the tests in every case uses: ./.github/workflows/npm-test.yml + cth-test: + # Run the Conformance Test Harness on PRs targeting main or versions/ + if: github.event_name == 'pull_request' && ( github.base_ref == 'main' || startsWith( github.base_ref, 'versions/' ) ) + uses: ./.github/workflows/cth-test.yml + with: + ignore_failures: true + version: 1.1.7 # The latest version that CSS is confirmed to pass + docker: # Build docker containers on version tag, push to main and push to versions/ needs: npm-test diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 407cbabfc..33244eec3 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -3,69 +3,16 @@ name: Conformance test harness on: schedule: - cron: '23 5 * * *' - push: - branches: - - test/conformance + workflow_dispatch: + jobs: conformance: - runs-on: ubuntu-latest strategy: fail-fast: false matrix: branch: - 'main' - 'versions/6.0.0' - timeout-minutes: 10 - steps: - - name: Use Node.js 16.x - uses: actions/setup-node@v3 - with: - node-version: 16.x - - name: Check out the project - uses: actions/checkout@v3 - with: - ref: ${{ matrix.branch }} - - name: Install dependencies and run build scripts - run: npm ci - - name: Start the server in the background - run: npm start > server-output.log & - - name: Create the necessary folders - run: mkdir -p reports/css - - name: Pull the conformance harness docker - run: docker pull solidproject/conformance-test-harness - - name: Wait until the server has started - run: | - until $(curl --output /dev/null --silent --head --fail -k http://localhost:3000/); do - sleep 1 - done - - name: Create users - run: npx ts-node test/deploy/createAccountCredentials.ts http://localhost:3000/ >> test/deploy/conformance.env - - name: Run the test harness - run: > - docker run -i --rm - -v "$(pwd)"/reports/css:/reports - --env-file=./test/deploy/conformance.env - --network="host" - solidproject/conformance-test-harness - --skip-teardown - --output=/reports - --target=https://github.com/solid/conformance-test-harness/css - # Steps below use `always()` to make sure logs get uploaded in case the CTH errors - - name: Sanitize branch name for artifact upload - id: sanitize - if: always() - uses: yeouchien/sanitize-branch-name-action@v1 - with: - branch-name: ${{ matrix.branch }} - - name: Save the reports - if: always() - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.sanitize.outputs.sanitized-branch-name }} reports - path: reports - - name: Save the server output - if: always() - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.sanitize.outputs.sanitized-branch-name }} server output - path: server-output.log + uses: ./.github/workflows/cth-test.yml + with: + branch: ${{ matrix.branch }}