ci: split up into multiple workflows

* ci: split up workflows

* ci: tweaks and housekeeping

* ci: additional comments and requested changes

* ci: additional comments

* chore: clean up .gitignore
This commit is contained in:
Jasper Vaneessen 2022-09-05 13:11:20 +02:00 committed by GitHub
parent 9a5fc674f3
commit 2d884c7252
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 356 additions and 373 deletions

View File

@ -1,372 +0,0 @@
name: CI
on:
push:
branches:
- 'main'
- 'versions/*'
tags:
- 'v*'
pull_request:
branches:
- '**'
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16.x'
- run: npm ci --ignore-scripts
- run: npm run lint
test-unit:
runs-on: ${{ matrix.operating-system }}
strategy:
matrix:
operating-system:
- ubuntu-latest
- windows-latest
node-version:
- '14.2'
- '14.x'
- '16.0'
- '16.x'
- '18.0'
- '18.x'
timeout-minutes: 15
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Ensure line endings are consistent
run: git config --global core.autocrlf input
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Type-check tests
run: npm run test:ts
- name: Run unit tests
run: npm run test:unit
- name: Submit unit test coverage
uses: coverallsapp/github-action@master
with:
github-token: ${{ secrets.github_token }}
flag-name: test-unit-${{ matrix.node-version }}-${{ matrix.operating-system }}
parallel: true
test-integration:
runs-on: ubuntu-latest
strategy:
matrix:
node-version:
- '14.x'
- '16.x'
- '18.x'
env:
TEST_DOCKER: true
services:
sparql-endpoint:
image: tenforce/virtuoso
env:
SPARQL_UPDATE: true
ports:
- 4000:8890
redis:
image: redis
ports:
- 6379:6379
timeout-minutes: 15
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run integration tests
run: npm run test:integration
test-integration-windows:
runs-on: windows-latest
strategy:
matrix:
node-version:
- '14.x'
- '16.x'
- '18.x'
timeout-minutes: 15
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Ensure line endings are consistent
run: git config --global core.autocrlf input
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run integration tests
run: npm run test:integration
test-configs:
runs-on: ubuntu-latest
services:
sparql-endpoint:
image: tenforce/virtuoso
env:
SPARQL_UPDATE: true
ports:
- 4000:8890
steps:
- name: Use Node.js
uses: actions/setup-node@v3
with:
node-version: '16.x'
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run deploy tests
run: npm run test:deploy
coveralls:
needs: test-unit
runs-on: ubuntu-latest
steps:
- name: Consolidate test coverage from different jobs
uses: coverallsapp/github-action@master
with:
github-token: ${{ secrets.github_token }}
parallel-finished: true
docker-main-branch:
needs:
- lint
- test-unit
- test-integration
- test-integration-windows
- test-configs
# Only run on tag push events starting with v prefix for now OR main branch push events
if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main')
runs-on: ubuntu-latest
steps:
-
name: Install jq
run: sudo apt-get update && sudo apt-get install -y jq
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
solidproject/community-server
# edge will always be executed (without latest tag), semver only on tag push events (with latest tag)
tags: |
type=edge
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
github-token: ${{ secrets.github_token }}
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and export to docker
uses: docker/build-push-action@v3
with:
context: .
load: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
-
name: Store built tags in env variable
shell: bash
run: |
echo 'generated_tags<<EOM' >> $GITHUB_ENV
echo '${{ toJSON(fromJSON(steps.meta.outputs.json).tags) }}' >> $GITHUB_ENV
echo 'EOM' >> $GITHUB_ENV
-
name: "Test all docker-built image tags"
shell: bash
# Loop over all generated image:tag names and docker run them.
# If they aren't built previously, the command will error.
# If they don't work, the job will fail.
# If they work the --version argument should return a version. and the job succeeds.
run: |
jq -rc '.[]' <<< '${{env.generated_tags}}' | while read i; do
docker run --rm --pull never $i --version
done;
-
name: Build and push
uses: docker/build-push-action@v3
with:
context: .
push: true
platforms: linux/amd64,linux/arm/v7
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
docker-versions-branch:
needs:
- lint
- test-unit
- test-integration
- test-integration-windows
- test-configs
# Only run on push events on a versions/* branch (ASSUMPTION: THERE SHOULD ONLY BE ONE THERE!)
if: startsWith(github.ref, 'refs/heads/versions/')
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.ref }}
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
solidproject/community-server
# Just one label: next (no latest here) for the last pushed commit on this branch
tags: |
type=raw,value=next
github-token: ${{ secrets.github_token }}
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and export to docker
uses: docker/build-push-action@v3
with:
context: .
load: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
-
name: "Test docker image 'next'"
run: |
docker run --rm --pull never solidproject/community-server:next --version
-
name: Build and push
uses: docker/build-push-action@v3
with:
context: .
push: true
platforms: linux/amd64,linux/arm/v7
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
typedocs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16.x'
- run: npm ci --ignore-scripts
- run: npm run typedocs
- name: Save docs
uses: actions/upload-artifact@v3
with:
name: docs
path: docs
retention-days: 1
mkdocs-tag:
runs-on: ubuntu-latest
# Only trigger on tagged release that doesn't have a prerelease qualifier (like -alpha, -beta, -rc)
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
needs:
- docker-main-branch
- typedocs
steps:
- uses: actions/checkout@v2
- id: get_version
uses: battila7/get-version-action@v2
- uses: actions/setup-python@v4
with:
python-version: 3.x
- run: pip install mkdocs-material
- run: pip install mike
- run: git config user.name ci-bot
- run: git config user.email ci-bot@example.com
- run: git fetch origin gh-pages --depth=1
- run: cd documentation && mike deploy --push --update-aliases ${{steps.get_version.outputs.major}}.x latest
mkdocs-latest:
runs-on: ubuntu-latest
# Runs on main branch commits: thus rebuild documentation of the version in the package.json
if: github.ref == 'refs/heads/main'
needs:
- docker-main-branch
- typedocs
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: 3.x
- run: pip install mkdocs-material
- run: pip install mike
- run: sudo apt-get update && sudo apt-get install -y jq
- run: git config user.name ci-bot
- run: git config user.email ci-bot@example.com
- name: Extract major version from package.json
run: |
echo "VERSION=$(git show origin/main:package.json | jq -r .version | grep -Po '^(\d+)').x" >> $GITHUB_ENV
- run: git fetch origin gh-pages --depth=1
- run: cd documentation && mike deploy --push -u ${{ env.VERSION }} latest
publish-docs:
# When a new version is tagged, upload the generated typedocs to the appropriate mkdocs <major.x> folder.
needs:
- mkdocs-tag
- lint
- test-unit
- test-integration
- test-integration-windows
- test-configs
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/v')
steps:
- uses: actions/download-artifact@v3
with:
name: docs
path: docs
- id: get_version
uses: battila7/get-version-action@v2
- uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs
destination_dir: ${{steps.get_version.outputs.major}}.x/docs

94
.github/workflows/docker.yml vendored Normal file
View File

@ -0,0 +1,94 @@
name: Docker
on:
workflow_call:
secrets:
DOCKERHUB_TOKEN:
required: true
DOCKERHUB_USERNAME:
required: true
jobs:
docker-meta:
# Generates the metadata (labels and tags) for the docker containers
# - push to main results in the edge tag
# - version tag results in edge, latest and semver, major, major.minor tags
# - push to versions/ results in the next tag
runs-on: ubuntu-latest
outputs:
labels: ${{ steps.meta-main.outputs.labels || steps.meta-version.outputs.labels }}
tags: ${{ steps.meta-main.outputs.tags || steps.meta-version.outputs.tags }}
steps:
- name: Checkout
uses: actions/checkout@v3
- if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main')
name: Docker meta edge and version tag
id: meta-main
uses: docker/metadata-action@v4
with:
images: |
solidproject/community-server
# edge will always be executed (without latest tag), semver only on tag push events (with latest tag)
tags: |
type=edge
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
github-token: ${{ secrets.github_token }}
- if: startsWith(github.ref, 'refs/heads/versions/')
name: Docker meta next
id: meta-version
uses: docker/metadata-action@v4
with:
images: |
solidproject/community-server
# Just one label: next (no latest here) for the last pushed commit on this branch
tags: |
type=raw,value=next
github-token: ${{ secrets.github_token }}
docker:
# Builds, tests and pushes docker containers
# Containers are built for both linux/amd64 and linux/arm/v7 platforms
needs: docker-meta
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and export to docker
uses: docker/build-push-action@v3
with:
context: .
load: true
tags: ${{ needs.docker-meta.outputs.tags }}
labels: ${{ needs.docker-meta.outputs.labels }}
- name: "Test all docker-built image tags"
shell: bash
# Loop over all generated image:tag names and docker run them.
# If they aren't built previously, the command will error.
# If they don't work, the job will fail.
# If they work the --version argument should return a version and the job succeeds.
run: |
while read i; do
docker run --rm --pull never $i --version
done <<< "${{ needs.docker-meta.outputs.tags }}";
- name: Build and push
uses: docker/build-push-action@v3
with:
context: .
push: true
platforms: linux/amd64,linux/arm/v7
tags: ${{ needs.docker-meta.outputs.tags }}
labels: ${{ needs.docker-meta.outputs.labels }}

35
.github/workflows/main.yml vendored Normal file
View File

@ -0,0 +1,35 @@
name: CI
on:
push:
branches:
- 'main'
- 'versions/*'
tags:
- 'v*'
pull_request:
branches:
- '**'
jobs:
npm-test:
# Run the tests in every case
uses: ./.github/workflows/npm-test.yml
docker:
# Build docker containers on version tag, push to main and push to versions/
needs: npm-test
if: startsWith(github.ref, 'refs/tags/v') || (github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/heads/versions/')
uses: ./.github/workflows/docker.yml
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
typedocs-release:
# Release typedocs on version tag, but ignore pre-releases
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
uses: ./.github/workflows/typedocs.yml
mkdocs-release:
# Release mkdocs on version tag, but ignore pre-releases
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
uses: ./.github/workflows/mkdocs.yml

57
.github/workflows/mkdocs.yml vendored Normal file
View File

@ -0,0 +1,57 @@
name: Documentation
on:
workflow_call:
# Additional trigger to deploy changes to the documentation/ folder
# on push to main, ignoring tags so we don't trigger twice upon release
push:
branches:
- main
paths:
- documentation/**
tags-ignore:
- "*"
jobs:
mkdocs-prep:
# Runs the markdownlinter to ensure we don't release faulty markdown.
# Also gets the correct major version, wether the job is triggered by a version tag
# or a push to main to update the latest documentation.
runs-on: ubuntu-latest
outputs:
major: ${{ steps.tagged_version.outputs.major || steps.current_version.ouputs.major }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v3
with:
node-version: '16.x'
- run: npm ci --ignore-scripts
- name: Lint documentation markdown
run: npm run lint:markdown -- documentation/**/*.md
- if: startsWith(github.ref, 'refs/tags/v')
name: Get tagged version
id: tagged_version
uses: battila7/get-version-action@v2
- if: github.ref == 'refs/heads/main'
name: Get current version
id: current_version
run: |
VERSION=$(git show origin/main:package.json | jq -r .version | grep -Po '^(\d+)')
echo "::set-output name=major::$VERSION"
mkdocs:
runs-on: ubuntu-latest
needs: mkdocs-prep
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: 3.x
- run: pip install mkdocs-material
- run: pip install mike
- run: git config user.name ci-bot
- run: git config user.email ci-bot@example.com
- run: git fetch origin gh-pages --depth=1
- run: |
cd documentation && mike deploy --push --update-aliases \
${{ needs.mkdocs-prep.outputs.major}}.x latest

145
.github/workflows/npm-test.yml vendored Normal file
View File

@ -0,0 +1,145 @@
name: NPM test
on:
workflow_call:
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16.x'
- run: npm ci --ignore-scripts
- run: npm run lint
test-unit:
# Run unit tests on windows and linux
runs-on: ${{ matrix.operating-system }}
strategy:
matrix:
operating-system:
- ubuntu-latest
- windows-latest
node-version:
- '14.2'
- '14.x'
- '16.0'
- '16.x'
- '18.0'
- '18.x'
timeout-minutes: 15
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Ensure line endings are consistent
run: git config --global core.autocrlf input
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Type-check tests
run: npm run test:ts
- name: Run unit tests
run: npm run test:unit
- name: Submit unit test coverage
uses: coverallsapp/github-action@master
with:
github-token: ${{ secrets.github_token }}
flag-name: test-unit-${{ matrix.node-version }}-${{ matrix.operating-system }}
parallel: true
test-integration:
# Run integration tests on linux (full suite, with external dependencies)
runs-on: ubuntu-latest
strategy:
matrix:
node-version:
- '14.x'
- '16.x'
- '18.x'
env:
TEST_DOCKER: true
services:
sparql-endpoint:
image: tenforce/virtuoso
env:
SPARQL_UPDATE: true
ports:
- 4000:8890
redis:
image: redis
ports:
- 6379:6379
timeout-minutes: 15
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run integration tests
run: npm run test:integration
test-integration-windows:
# Run integration tests on windows (no external dependencies)
runs-on: windows-latest
strategy:
matrix:
node-version:
- '14.x'
- '16.x'
- '18.x'
timeout-minutes: 15
steps:
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Ensure line endings are consistent
run: git config --global core.autocrlf input
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run integration tests
run: npm run test:integration
test-configs:
# Test startup of CSS with all configs inside the config/ folder
runs-on: ubuntu-latest
services:
sparql-endpoint:
image: tenforce/virtuoso
env:
SPARQL_UPDATE: true
ports:
- 4000:8890
steps:
- name: Use Node.js
uses: actions/setup-node@v3
with:
node-version: '16.x'
- name: Check out repository
uses: actions/checkout@v3
- name: Install dependencies and run build scripts
run: npm ci
- name: Run deploy tests
run: npm run test:deploy
coveralls:
# Consolidate test coverage reports from all unit tests
needs: test-unit
runs-on: ubuntu-latest
steps:
- name: Consolidate test coverage from different jobs
uses: coverallsapp/github-action@master
with:
github-token: ${{ secrets.github_token }}
parallel-finished: true

25
.github/workflows/typedocs.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: Typedocs
on:
workflow_call:
jobs:
typedocs:
# Build typedocs and publish them to the GH page
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16.x'
- run: npm ci --ignore-scripts
- name: Generate typedocs
run: npm run typedocs
- name: Get tagged version
id: version
uses: battila7/get-version-action@v2
- name: Deploy typedocs
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs
destination_dir: ${{ steps.version.outputs.major }}.x/docs

1
.gitignore vendored
View File

@ -7,4 +7,3 @@
/docs
/node_modules
/test/tmp
tsconfig.tsbuildinfo