mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
482a7839d9
Was iterating over every file, reloading everything. Instead, analyze the package directories. On my machine, the time for vet checking goes from 34s to 3s. Scans more code too.
390 lines
12 KiB
Bash
Executable File
390 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Run all etcd tests
|
|
# ./test
|
|
# ./test -v
|
|
#
|
|
# Run tests for one package
|
|
#
|
|
# PKG=./wal ./test
|
|
# PKG=snap ./test
|
|
#
|
|
# Run code coverage
|
|
# COVERDIR must either be a absolute path or a relative path to the etcd root
|
|
# COVERDIR=coverage PASSES="build_cov cov" ./test
|
|
set -e
|
|
|
|
source ./build
|
|
|
|
# build tests with vendored dependencies
|
|
etcd_setup_gopath
|
|
|
|
if [ -z "$PASSES" ]; then
|
|
PASSES="fmt bom dep compile build unit"
|
|
fi
|
|
|
|
# Invoke ./cover for HTML output
|
|
COVER=${COVER:-"-cover"}
|
|
|
|
# Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
|
|
IGNORE_PKGS="(cmd/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
|
|
INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)"
|
|
|
|
# all github.com/coreos/etcd/whatever pkgs that are not auto-generated / tools
|
|
PKGS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | egrep -v "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g"`
|
|
# pkg1,pkg2,pkg3
|
|
PKGS_COMMA=`echo ${PKGS} | sed 's/ /,/g'`
|
|
|
|
TEST_PKGS=`find . -name \*_test.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
|
|
FORMATTABLE=`find . -name \*.go | while read a; do echo $(dirname $a)/"*.go"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
|
|
TESTABLE_AND_FORMATTABLE=`echo "$TEST_PKGS" | egrep -v "$INTEGRATION_PKGS"`
|
|
|
|
# TODO: 'client' pkg fails with gosimple from generated files
|
|
# TODO: 'rafttest' is failing with unused
|
|
STATIC_ANALYSIS_PATHS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | grep -v 'client'`
|
|
|
|
if [ -z "$GOARCH" ]; then
|
|
GOARCH=$(go env GOARCH);
|
|
fi
|
|
|
|
# user has not provided PKG override
|
|
if [ -z "$PKG" ]; then
|
|
TEST=$TESTABLE_AND_FORMATTABLE
|
|
FMT=$FORMATTABLE
|
|
|
|
# user has provided PKG override
|
|
else
|
|
# strip out leading dotslashes and trailing slashes from PKG=./foo/
|
|
TEST=${PKG/#./}
|
|
TEST=${TEST/#\//}
|
|
TEST=${TEST/%\//}
|
|
|
|
# only run gofmt on packages provided by user
|
|
FMT="$TEST"
|
|
fi
|
|
|
|
# split TEST into an array and prepend REPO_PATH to each local package
|
|
split=(${TEST// / })
|
|
TEST=${split[@]/#/${REPO_PATH}/}
|
|
|
|
# determine whether target supports race detection
|
|
if [ "$GOARCH" == "amd64" ]; then
|
|
RACE="--race"
|
|
fi
|
|
|
|
function unit_pass {
|
|
echo "Running unit tests..."
|
|
# only -run=Test so examples can run in integration tests
|
|
go test -timeout 3m ${COVER} ${RACE} -cpu 1,2,4 -run=Test $@ ${TEST}
|
|
}
|
|
|
|
function integration_pass {
|
|
echo "Running integration tests..."
|
|
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration
|
|
go test -timeout 1m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/client/integration
|
|
go test -timeout 10m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
|
|
go test -timeout 1m -v -cpu 1,2,4 $@ ${REPO_PATH}/contrib/raftexample
|
|
go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
|
|
}
|
|
|
|
function cov_pass {
|
|
echo "Running code coverage..."
|
|
# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
|
|
# gocovmerge merges coverage files
|
|
if ! which gocovmerge >/dev/null; then
|
|
echo "gocovmerge not installed"
|
|
exit 255
|
|
fi
|
|
|
|
if [ -z "$COVERDIR" ]; then
|
|
echo "COVERDIR undeclared"
|
|
exit 255
|
|
fi
|
|
|
|
if [ ! -f "bin/etcd_test" ]; then
|
|
echo "etcd_test binary not found"
|
|
exit 255
|
|
fi
|
|
|
|
mkdir -p "$COVERDIR"
|
|
|
|
# run code coverage for unit and integration tests
|
|
GOCOVFLAGS="-covermode=set -coverpkg $PKGS_COMMA -v -timeout 15m"
|
|
failed=""
|
|
for t in `echo "${TEST_PKGS}" | egrep -v "(e2e|functional-tester)"`; do
|
|
tf=`echo $t | tr / _`
|
|
# cache package compilation data for faster repeated builds
|
|
go test $GOCOVFLAGS -i ${REPO_PATH}/$t || true
|
|
# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
|
|
go test $GOCOVFLAGS -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile" ${REPO_PATH}/$t || failed="$failed $t"
|
|
done
|
|
|
|
# proxy tests
|
|
go test -tags cluster_proxy $GOCOVFLAGS -coverprofile "$COVERDIR/proxy_integration.coverprofile" ${REPO_PATH}/integration || failed="$failed proxy-integration"
|
|
go test -tags cluster_proxy $GOCOVFLAGS -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" ${REPO_PATH}/clientv3/integration || failed="$failed proxy-clientv3/integration"
|
|
|
|
# run code coverage for e2e tests
|
|
# use 30m timeout because e2e coverage takes longer
|
|
# due to many tests cause etcd process to wait
|
|
# on leadership transfer timeout during gracefully shutdown
|
|
go test -tags cov -timeout 30m -v ${REPO_PATH}"/e2e" || failed="$failed e2e"
|
|
|
|
gocovmerge "$COVERDIR"/*.coverprofile >"$COVERDIR"/cover.out
|
|
# strip out generated files (using GNU-style sed)
|
|
sed --in-place '/generated.go/d' "$COVERDIR"/cover.out || true
|
|
|
|
# held failures to generate the full coverage file, now fail
|
|
if [ -n "$failed" ]; then
|
|
for f in $failed; do
|
|
echo FAIL $f
|
|
done
|
|
exit 255
|
|
fi
|
|
}
|
|
|
|
function e2e_pass {
|
|
echo "Running e2e tests..."
|
|
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e
|
|
}
|
|
|
|
function integration_e2e_pass {
|
|
echo "Running integration and e2e tests..."
|
|
|
|
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e &
|
|
e2epid="$!"
|
|
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration &
|
|
intpid="$!"
|
|
wait $e2epid
|
|
wait $intpid
|
|
go test -timeout 1m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/client/integration
|
|
go test -timeout 10m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
|
|
go test -timeout 1m -v -cpu 1,2,4 $@ ${REPO_PATH}/contrib/raftexample
|
|
go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
|
|
}
|
|
|
|
function grpcproxy_pass {
|
|
go test -timeout 15m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/integration
|
|
go test -timeout 15m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
|
|
}
|
|
|
|
function release_pass {
|
|
rm -f ./bin/etcd-last-release
|
|
# to grab latest patch release; bump this up for every minor release
|
|
UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.2.*" | head -1)
|
|
if [ -n "$MANUAL_VER" ]; then
|
|
# in case, we need to test against different version
|
|
UPGRADE_VER=$MANUAL_VER
|
|
fi
|
|
|
|
local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
|
|
echo "Downloading $file"
|
|
|
|
set +e
|
|
curl --fail -L https://github.com/coreos/etcd/releases/download/$UPGRADE_VER/$file -o /tmp/$file
|
|
local result=$?
|
|
set -e
|
|
case $result in
|
|
0) ;;
|
|
22) return 0
|
|
;;
|
|
*) exit $result
|
|
;;
|
|
esac
|
|
|
|
tar xzvf /tmp/$file -C /tmp/ --strip-components=1
|
|
mkdir -p ./bin
|
|
mv /tmp/etcd ./bin/etcd-last-release
|
|
}
|
|
|
|
function fmt_pass {
|
|
toggle_failpoints disable
|
|
|
|
echo "Checking gofmt..."
|
|
fmtRes=$(gofmt -l -s -d $FMT)
|
|
if [ -n "${fmtRes}" ]; then
|
|
echo -e "gofmt checking failed:\n${fmtRes}"
|
|
exit 255
|
|
fi
|
|
|
|
echo "Checking govet..."
|
|
vetRes=$(go vet $TEST)
|
|
if [ -n "${vetRes}" ]; then
|
|
echo -e "govet checking failed:\n${vetRes}"
|
|
exit 255
|
|
fi
|
|
|
|
echo "Checking 'go tool vet -all -shadow'..."
|
|
fmtpkgs=$(echo $FMT | xargs dirname | sort | uniq | sed '/\./d')
|
|
vetRes=$(go tool vet -all -shadow ${fmtpkgs} 2>&1 | grep -v '/gw/' || true)
|
|
if [ -n "${vetRes}" ]; then
|
|
echo -e "govet -all -shadow checking failed:\n${vetRes}"
|
|
exit 255
|
|
fi
|
|
|
|
echo "Checking documentation style..."
|
|
# eschew you
|
|
yous=`find . -name \*.md | xargs egrep --color "[Yy]ou[r]?[ '.,;]" | grep -v /v2/ || true`
|
|
if [ ! -z "$yous" ]; then
|
|
echo -e "found 'you' in documentation:\n${yous}"
|
|
exit 255
|
|
fi
|
|
|
|
# TODO: check other markdown files when marker handles headers with '[]'
|
|
if which marker >/dev/null; then
|
|
echo "Checking marker to find broken links..."
|
|
markerResult=`marker --skip-http --root ./Documentation 2>&1 || true`
|
|
if [ -n "${markerResult}" ]; then
|
|
echo -e "marker checking failed:\n${markerResult}"
|
|
exit 255
|
|
fi
|
|
else
|
|
echo "Skipping marker..."
|
|
fi
|
|
|
|
if which goword >/dev/null; then
|
|
echo "Checking goword..."
|
|
# get all go files to process
|
|
gofiles=`find $FMT -iname '*.go' 2>/dev/null`
|
|
# ignore tests and protobuf files
|
|
gofiles=`echo ${gofiles} | sort | uniq | sed "s/ /\n/g" | egrep -v "(\\_test.go|\\.pb\\.go)"`
|
|
# only check for broken exported godocs
|
|
gowordRes=`goword -use-spell=false ${gofiles} | grep godoc-export | sort`
|
|
if [ ! -z "$gowordRes" ]; then
|
|
echo -e "goword checking failed:\n${gowordRes}"
|
|
exit 255
|
|
fi
|
|
else
|
|
echo "Skipping goword..."
|
|
fi
|
|
|
|
if which gosimple >/dev/null; then
|
|
echo "Checking gosimple..."
|
|
gosimpleResult=`gosimple ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
|
|
if [ -n "${gosimpleResult}" ]; then
|
|
# TODO: resolve these after go1.8 migration
|
|
SIMPLE_CHECK_MASK="S(1024)"
|
|
if echo "${gosimpleResult}" | egrep -v "$SIMPLE_CHECK_MASK"; then
|
|
echo -e "gosimple checking ${path} failed:\n${gosimpleResult}"
|
|
exit 255
|
|
else
|
|
echo -e "gosimple warning:\n${gosimpleResult}"
|
|
fi
|
|
fi
|
|
else
|
|
echo "Skipping gosimple..."
|
|
fi
|
|
|
|
if which unused >/dev/null; then
|
|
echo "Checking unused..."
|
|
unusedResult=`unused ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
|
|
if [ -n "${unusedResult}" ]; then
|
|
echo -e "unused checking failed:\n${unusedResult}"
|
|
exit 255
|
|
fi
|
|
else
|
|
echo "Skipping unused..."
|
|
fi
|
|
|
|
if which staticcheck >/dev/null; then
|
|
echo "Checking staticcheck..."
|
|
staticcheckResult=`staticcheck ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
|
|
if [ -n "${staticcheckResult}" ]; then
|
|
# TODO: resolve these after go1.8 migration
|
|
# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
|
|
STATIC_CHECK_MASK="SA(1019|2002)"
|
|
if echo "${staticcheckResult}" | egrep -v "$STATIC_CHECK_MASK"; then
|
|
echo -e "staticcheck checking ${path} failed:\n${staticcheckResult}"
|
|
exit 255
|
|
else
|
|
suppressed=`echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c`
|
|
echo -e "staticcheck suppressed warnings:\n${suppressed}"
|
|
fi
|
|
fi
|
|
else
|
|
echo "Skipping staticcheck..."
|
|
fi
|
|
|
|
echo "Checking for license header..."
|
|
licRes=$(for file in $(find . -type f -iname '*.go' ! -path './cmd/*' ! -path './gopath.proto/*'); do
|
|
head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" || echo -e " ${file}"
|
|
done;)
|
|
if [ -n "${licRes}" ]; then
|
|
echo -e "license header checking failed:\n${licRes}"
|
|
exit 255
|
|
fi
|
|
|
|
echo "Checking commit titles..."
|
|
git log --oneline `git merge-base HEAD master`...HEAD | while read l; do
|
|
commitMsg=`echo "$l" | cut -f2- -d' '`
|
|
if [[ "$commitMsg" == Merge* ]]; then
|
|
# ignore "Merge pull" commits
|
|
continue
|
|
fi
|
|
if [[ "$commitMsg" == Revert* ]]; then
|
|
# ignore revert commits
|
|
continue
|
|
fi
|
|
|
|
pkgPrefix=`echo "$commitMsg" | cut -f1 -d':'`
|
|
spaceCommas=`echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0`
|
|
commaSpaces=`echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0`
|
|
if [[ `echo $commitMsg | grep -c ":..*"` == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
|
|
echo "$l"...
|
|
echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
|
|
echo "Got: $l"
|
|
exit 255
|
|
fi
|
|
done
|
|
}
|
|
|
|
function bom_pass {
|
|
if ! which license-bill-of-materials >/dev/null; then
|
|
return
|
|
fi
|
|
echo "Checking bill of materials..."
|
|
license-bill-of-materials \
|
|
--override-file bill-of-materials.override.json \
|
|
github.com/coreos/etcd github.com/coreos/etcd/etcdctl >bom-now.json || true
|
|
if ! diff bill-of-materials.json bom-now.json; then
|
|
echo vendored licenses do not match given bill of materials
|
|
exit 255
|
|
fi
|
|
rm bom-now.json
|
|
}
|
|
|
|
function dep_pass {
|
|
echo "Checking package dependencies..."
|
|
# don't pull in etcdserver package
|
|
pushd clientv3 >/dev/null
|
|
badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
|
|
deps=`go list -f '{{ .Deps }}' | sed 's/ /\n/g' | egrep "${badpkg}" || echo ""`
|
|
popd >/dev/null
|
|
if [ ! -z "$deps" ]; then
|
|
echo -e "clientv3 has masked dependencies:\n${deps}"
|
|
exit 255
|
|
fi
|
|
}
|
|
|
|
function build_cov_pass {
|
|
out="bin"
|
|
if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
|
|
go test -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcd_test
|
|
go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcdctl_test ${REPO_PATH}/etcdctl
|
|
}
|
|
|
|
function compile_pass {
|
|
echo "Checking build..."
|
|
go build -v ./tools/...
|
|
}
|
|
|
|
# fail fast on static tests
|
|
function build_pass {
|
|
GO_BUILD_FLAGS="-a -v" etcd_build
|
|
}
|
|
|
|
for pass in $PASSES; do
|
|
${pass}_pass $@
|
|
done
|
|
|
|
echo "Success"
|