From a6b0375b7b0d8ebe6d3ad20d04cf3976581fb417 Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Wed, 30 Sep 2020 04:47:05 +0200 Subject: [PATCH] Travis: Reduce footprint of unit tests - so hopefully flakiness (#12350) * ./tests: Remove legacy coverage collection code The legacy tests/cover.test.bash script was not ./test script compatible for a long time. The following method of coverage collection works (also across packages) and does not make all the test execution slower. ``` COVERDIR=coverage PASSES="build build_cov cov" ./test go tool cover -html ./coverage/cover.out ``` * CI: Reduce duplicated coverage between different variants on Travis We used to execute unit tests in 3 different jobs, every time with --race detection and every time in 3 variants:1,2,4 CPUS. The proposed change makes each of the jobs use different variant of CPUS, and only 4-cpu variant is running with --race detection (as the more-parallel variant is more likely to experience races), --- .travis.yml | 28 ++++++++++++++-------------- test | 12 +++++------- tests/cover.test.bash | 36 ------------------------------------ 3 files changed, 19 insertions(+), 57 deletions(-) delete mode 100755 tests/cover.test.bash diff --git a/.travis.yml b/.travis.yml index 610e143d0..a47c98451 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,12 @@ env: - TARGET=linux-amd64-integration-2-cpu - TARGET=linux-amd64-integration-4-cpu - TARGET=linux-amd64-functional - - TARGET=linux-amd64-unit + - TARGET=linux-amd64-unit-4-cpu-race - TARGET=all-build - TARGET=linux-amd64-grpcproxy - TARGET=linux-amd64-coverage - - TARGET=linux-amd64-fmt-unit-go-tip - - TARGET=linux-386-unit + - TARGET=linux-amd64-fmt-unit-go-tip-2-cpu + - TARGET=linux-386-unit-1-cpu matrix: fast_finish: true @@ -35,9 +35,9 @@ matrix: - go: 1.14.3 env: TARGET=linux-amd64-coverage - go: tip - env: TARGET=linux-amd64-fmt-unit-go-tip + env: TARGET=linux-amd64-fmt-unit-go-tip-2-cpu - go: 1.14.3 - env: TARGET=linux-386-unit + env: TARGET=linux-386-unit-1-cpu exclude: - go: tip env: TARGET=linux-amd64-fmt @@ -50,7 +50,7 @@ matrix: - go: tip env: TARGET=linux-amd64-functional - go: tip - env: TARGET=linux-amd64-unit + env: TARGET=linux-amd64-unit-4-cpu-race - go: tip env: TARGET=all-build - go: tip @@ -58,9 +58,9 @@ matrix: - go: tip env: TARGET=linux-amd64-coverage - go: 1.14.3 - env: TARGET=linux-amd64-fmt-unit-go-tip + env: TARGET=linux-amd64-fmt-unit-go-tip-2-cpu - go: tip - env: TARGET=linux-386-unit + env: TARGET=linux-386-unit-1-cpu before_install: - if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi @@ -100,10 +100,10 @@ script: --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \ /bin/bash -c "GO_BUILD_FLAGS='-v -mod=readonly' ./build && GOARCH=amd64 PASSES='functional' ./test" ;; - linux-amd64-unit) + linux-amd64-unit-4-cpu-race) docker run --rm \ --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \ - /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test -p=2" + /bin/bash -c "GOARCH=amd64 PASSES='unit' RACE='true' CPU='4' ./test -p=2" ;; all-build) docker run --rm \ @@ -124,12 +124,12 @@ script: linux-amd64-coverage) sudo HOST_TMP_DIR=/tmp TEST_OPTS="VERBOSE='1'" make docker-test-coverage ;; - linux-amd64-fmt-unit-go-tip) - GOARCH=amd64 PASSES='fmt unit' ./test -p=2 + linux-amd64-fmt-unit-go-tip-2-cpu) + GOARCH=amd64 PASSES='fmt unit' 'CPU=2' ./test -p=2 ;; - linux-386-unit) + linux-386-unit-1-cpu) docker run --rm \ --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \ - /bin/bash -c "GOARCH=386 PASSES='unit' ./test -p=2" + /bin/bash -c "GOARCH=386 PASSES='unit' RACE='false' CPU='1' ./test -p=4" ;; esac diff --git a/test b/test index b5e1675b2..6fbe2bf76 100755 --- a/test +++ b/test @@ -29,7 +29,8 @@ # # Run code coverage # COVERDIR must either be a absolute path or a relative path to the etcd root -# $ COVERDIR=coverage PASSES="build_cov cov" ./test +# $ COVERDIR=coverage PASSES="build build_cov cov" ./test +# $ go tool cover -html ./coverage/cover.out set -e # The test script is not supposed to make any changes to the files @@ -43,16 +44,12 @@ source ./scripts/test_lib.sh PASSES=${PASSES:-"fmt bom dep build unit"} PKG=${PKG:-} -# Invoke ./tests/cover.test.bash for HTML output -COVER="--cover=${COVER:-true}" - if [ -z "$GOARCH" ]; then GOARCH=$(go env GOARCH); fi # determine the number of CPUs to use for Go tests -CPU=${CPU:-"1,2,4"} -echo "Running with CPU:" "${CPU}" +CPU=${CPU:-"4"} # determine whether target supports race detection if [ -z "${RACE}" ] ; then @@ -66,7 +63,8 @@ else fi # This options make sense for cases where SUT (System Under Test) is compiled by test. -COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}" "${COVER}") +COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}") +log_callout "Running with ${COMMON_TEST_FLAGS[*]}" RUN_ARG=() if [ -n "${TESTCASE}" ]; then diff --git a/tests/cover.test.bash b/tests/cover.test.bash deleted file mode 100755 index eb089a4d2..000000000 --- a/tests/cover.test.bash +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Generate coverage HTML for a package -# e.g. PKG=./unit ./tests/cover.test.bash -# -set -e - -if ! [[ "$0" =~ "tests/cover.test.bash" ]]; then - echo "must be run from repository root" - exit 255 -fi - -if [ -z "$PKG" ]; then - echo "cover only works with a single package, sorry" - exit 255 -fi - -COVEROUT="coverage" - -if ! [ -d "$COVEROUT" ]; then - mkdir "$COVEROUT" -fi - -# strip leading dot/slash and trailing slash and sanitize other slashes -# e.g. ./etcdserver/etcdhttp/ ==> etcdserver_etcdhttp -COVERPKG=${PKG/#./} -COVERPKG=${COVERPKG/#\//} -COVERPKG=${COVERPKG/%\//} -COVERPKG=${COVERPKG//\//_} - -# generate arg for "go test" -export COVER="-coverprofile ${COVEROUT}/${COVERPKG}.out" - -source ./test - -go tool cover -html=${COVEROUT}/${COVERPKG}.out