*: fix shellcheck warnings

Fixes scripts and removes shellcheck warning suppressions.

* regexp warnings
* use ./*glob* so names don't become options
* use $(..) instead of legacy `..`
* read with -r to avoid mangling backslashes
* double quote to prevent globbing and word splitting
This commit is contained in:
Anthony Romano
2017-09-06 10:38:08 -07:00
parent 3c1845604b
commit 9abe9da9db
10 changed files with 149 additions and 147 deletions

138
test
View File

@@ -38,13 +38,13 @@ IGNORE_PKGS="(cmd/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)"
# all github.com/coreos/etcd/whatever pkgs that are not auto-generated / tools
PKGS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | egrep -v "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo`
PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | egrep -v "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo)
# pkg1,pkg2,pkg3
PKGS_COMMA=${PKGS// /,}
TEST_PKGS=`find . -name \*_test.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
FORMATTABLE=`find . -name \*.go | while read a; do echo "$(dirname $a)/*.go"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
TESTABLE_AND_FORMATTABLE=`echo "$TEST_PKGS" | egrep -v "$INTEGRATION_PKGS"`
TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g")
FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g")
TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | egrep -v "$INTEGRATION_PKGS")
# check if user provided PKG override
if [ -z "${USERPKG}" ]; then
@@ -58,21 +58,23 @@ else
# only run gofmt on packages provided by user
FMT="$TEST"
fi
FMT=($FMT)
# prepend REPO_PATH to each local package
split=$TEST
TEST=""
for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done
TEST=($TEST)
# TODO: 'client' pkg fails with gosimple from generated files
# TODO: 'rafttest' is failing with unused
STATIC_ANALYSIS_PATHS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | grep -v 'client'`
STATIC_ANALYSIS_PATHS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | grep -v 'client')
STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS)
if [ -z "$GOARCH" ]; then
GOARCH=$(go env GOARCH);
fi
# determine whether target supports race detection
if [ "$GOARCH" == "amd64" ]; then
RACE="--race"
@@ -81,21 +83,21 @@ fi
function unit_pass {
echo "Running unit tests..."
# only -run=Test so examples can run in integration tests
go test -timeout 3m ${COVER} ${RACE} -cpu 1,2,4 -run=Test $@ ${TEST}
go test -timeout 3m "${COVER}" ${RACE} -cpu 1,2,4 -run=Test "$@" "${TEST[@]}"
}
function integration_pass {
echo "Running integration tests..."
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration
integration_extra $@
go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
integration_extra "$@"
}
function integration_extra {
go test -timeout 1m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/client/integration
go test -timeout 15m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
go test -timeout 1m -v -cpu 1,2,4 $@ ${REPO_PATH}/contrib/raftexample
go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
go test -timeout 5m -v ${RACE} -tags v2v3 $@ ${REPO_PATH}/store
go test -timeout 1m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/client/integration"
go test -timeout 15m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
go test -timeout 1m -v -cpu 1,2,4 "$@" "${REPO_PATH}/contrib/raftexample"
go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/store"
go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example "$@" "${TEST[@]}"
}
function functional_pass {
@@ -125,8 +127,9 @@ function functional_pass {
echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
echo "Waiting for processes to exit"
kill -s TERM ${agent_pids}
for a in ${agent_pids}; do wait $a || true; done
agent_pids=($agent_pids)
kill -s TERM "${agent_pids[@]}"
for a in "${agent_pids[@]}"; do wait "$a" || true; done
rm -rf ./agent-*
if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
@@ -157,38 +160,39 @@ function cov_pass {
mkdir -p "$COVERDIR"
# run code coverage for unit and integration tests
GOCOVFLAGS="-covermode=set -coverpkg $PKGS_COMMA -v -timeout 15m"
GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 15m"
GOCOVFLAGS=($GOCOVFLGS)
failed=""
for t in `echo "${TEST_PKGS}" | egrep -v "(e2e|functional-tester)"`; do
tf=`echo $t | tr / _`
for t in $(echo "${TEST_PKGS}" | egrep -v "(e2e|functional-tester)"); do
tf=$(echo "$t" | tr / _)
# cache package compilation data for faster repeated builds
go test $GOCOVFLAGS -i ${REPO_PATH}/$t || true
go test "${GOCOVFLAGS[@]}" -i "${REPO_PATH}/$t" || true
# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
go test $GOCOVFLAGS -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile" ${REPO_PATH}/$t || failed="$failed $t"
go test "${GOCOVFLAGS[@]}" -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile" "${REPO_PATH}/$t" || failed="$failed $t"
done
# v2v3 tests
go test -tags v2v3 $GOCOVFLAGS -coverprofile "$COVERDIR/store-v2v3.coverprofile" ${REPO_PATH}/clientv3/integration || failed="$failed store-v2v3"
go test -tags v2v3 "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/store-v2v3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed store-v2v3"
# proxy tests
go test -tags cluster_proxy $GOCOVFLAGS -coverprofile "$COVERDIR/proxy_integration.coverprofile" ${REPO_PATH}/integration || failed="$failed proxy-integration"
go test -tags cluster_proxy $GOCOVFLAGS -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" ${REPO_PATH}/clientv3/integration || failed="$failed proxy-clientv3/integration"
go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_integration.coverprofile" "${REPO_PATH}/integration" || failed="$failed proxy-integration"
go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed proxy-clientv3/integration"
# run code coverage for e2e tests
# use 30m timeout because e2e coverage takes longer
# due to many tests cause etcd process to wait
# on leadership transfer timeout during gracefully shutdown
echo Testing e2e without proxy...
go test -tags cov -timeout 30m -v ${REPO_PATH}"/e2e" || failed="$failed e2e"
go test -tags cov -timeout 30m -v "${REPO_PATH}/e2e" || failed="$failed e2e"
echo Testing e2e with proxy...
go test -tags "cov cluster_proxy" -timeout 30m -v ${REPO_PATH}"/e2e" || failed="$failed e2e-proxy"
go test -tags "cov cluster_proxy" -timeout 30m -v "${REPO_PATH}/e2e" || failed="$failed e2e-proxy"
# incrementally merge to get coverage data even if some coverage files are corrupted
# optimistically assume etcdserver package's coverage file is OK since gocovmerge
# expects to start with a non-empty file
cp "$COVERDIR"/etcdserver.coverprofile "$COVERDIR"/cover.out
for f in "$COVERDIR"/*.coverprofile; do
gocovmerge $f "$COVERDIR"/cover.out >"$COVERDIR"/cover.tmp || failed="$failed $f"
gocovmerge "$f" "$COVERDIR"/cover.out >"$COVERDIR"/cover.tmp || failed="$failed $f"
if [ -s "$COVERDIR"/cover.tmp ]; then
mv "$COVERDIR"/cover.tmp "$COVERDIR"/cover.out
fi
@@ -199,7 +203,7 @@ function cov_pass {
# held failures to generate the full coverage file, now fail
if [ -n "$failed" ]; then
for f in $failed; do
echo FAIL $f
echo FAIL "$f"
done
exit 255
fi
@@ -207,25 +211,25 @@ function cov_pass {
function e2e_pass {
echo "Running e2e tests..."
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e
go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e"
}
function integration_e2e_pass {
echo "Running integration and e2e tests..."
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e &
go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
e2epid="$!"
go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration &
go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
intpid="$!"
wait $e2epid
wait $intpid
integration_extra $@
integration_extra "$@"
}
function grpcproxy_pass {
go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/integration
go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
go test -timeout 15m -v -tags cluster_proxy $@ ${REPO_PATH}/e2e
go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
go test -timeout 15m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e"
}
function release_pass {
@@ -245,7 +249,7 @@ function release_pass {
echo "Downloading $file"
set +e
curl --fail -L https://github.com/coreos/etcd/releases/download/$UPGRADE_VER/$file -o /tmp/$file
curl --fail -L "https://github.com/coreos/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
local result=$?
set -e
case $result in
@@ -255,7 +259,7 @@ function release_pass {
;;
esac
tar xzvf /tmp/$file -C /tmp/ --strip-components=1
tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
mkdir -p ./bin
mv /tmp/etcd ./bin/etcd-last-release
}
@@ -264,22 +268,23 @@ function fmt_pass {
toggle_failpoints disable
echo "Checking gofmt..."
fmtRes=$(gofmt -l -s -d $FMT)
fmtRes=$(gofmt -l -s -d "${FMT[@]}")
if [ -n "${fmtRes}" ]; then
echo -e "gofmt checking failed:\n${fmtRes}"
exit 255
fi
echo "Checking govet..."
vetRes=$(go vet $TEST)
vetRes=$(go vet "${TEST[@]}")
if [ -n "${vetRes}" ]; then
echo -e "govet checking failed:\n${vetRes}"
exit 255
fi
echo "Checking 'go tool vet -all -shadow'..."
fmtpkgs=$(for a in $FMT; do dirname "$a"; done | sort | uniq | grep -v "\\.")
vetRes=$(go tool vet -all -shadow ${fmtpkgs} 2>&1 | grep -v '/gw/' || true)
fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
fmtpkgs=($fmtpkgs)
vetRes=$(go tool vet -all -shadow "${fmtpkgs[@]}" 2>&1 | grep -v '/gw/' || true)
if [ -n "${vetRes}" ]; then
echo -e "govet -all -shadow checking failed:\n${vetRes}"
exit 255
@@ -289,21 +294,14 @@ function fmt_pass {
echo "Checking shellcheck..."
shellcheckResult=$(shellcheck -fgcc build test scripts/* 2>&1 || true)
if [ -n "${shellcheckResult}" ]; then
# mask the most common ones; fix later
SHELLCHECK_MASK="SC(2086|2006|2068|2196|2035|2162|2076)"
errs=$(echo "${shellcheckResult}" | egrep -v "${SHELLCHECK_MASK}" || true)
if [ -n "${errs}" ]; then
echo -e "shellcheck checking failed:\n${shellcheckResult}\n===\nFailed:\n${errs}"
exit 255
fi
suppressed=$(echo "${shellcheckResult}" | cut -f4- -d':' | sort | uniq -c | sort -n)
echo -e "shellcheck suppressed warnings:\n${suppressed}"
echo -e "shellcheck checking failed:\n${shellcheckResult}"
exit 255
fi
fi
echo "Checking documentation style..."
# eschew you
yous=`find . -name \*.md -exec egrep --color "[Yy]ou[r]?[ '.,;]" {} + | grep -v /v2/ || true`
yous=$(find . -name \*.md -exec egrep --color "[Yy]ou[r]?[ '.,;]" {} + | grep -v /v2/ || true)
if [ ! -z "$yous" ]; then
echo -e "found 'you' in documentation:\n${yous}"
exit 255
@@ -312,7 +310,7 @@ function fmt_pass {
# TODO: check other markdown files when marker handles headers with '[]'
if which marker >/dev/null; then
echo "Checking marker to find broken links..."
markerResult=`marker --skip-http --root ./Documentation 2>&1 || true`
markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true)
if [ -n "${markerResult}" ]; then
echo -e "marker checking failed:\n${markerResult}"
exit 255
@@ -324,11 +322,13 @@ function fmt_pass {
if which goword >/dev/null; then
echo "Checking goword..."
# get all go files to process
gofiles=`find $FMT -iname '*.go' 2>/dev/null`
gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null)
gofiles_all=($gofiles)
# ignore tests and protobuf files
gofiles=`echo ${gofiles} | sort | uniq | sed "s/ /\n/g" | egrep -v "(\\_test.go|\\.pb\\.go)"`
gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | egrep -v "(\\_test.go|\\.pb\\.go)")
gofiles=($gofiles)
# only check for broken exported godocs
gowordRes=`goword -use-spell=false ${gofiles} | grep godoc-export | sort`
gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
if [ ! -z "$gowordRes" ]; then
echo -e "goword checking failed:\n${gowordRes}"
exit 255
@@ -339,7 +339,7 @@ function fmt_pass {
if which gosimple >/dev/null; then
echo "Checking gosimple..."
gosimpleResult=`gosimple ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
gosimpleResult=$(gosimple "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
if [ -n "${gosimpleResult}" ]; then
# TODO: resolve these after go1.8 migration
SIMPLE_CHECK_MASK="S(1024)"
@@ -356,7 +356,7 @@ function fmt_pass {
if which unused >/dev/null; then
echo "Checking unused..."
unusedResult=`unused ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
unusedResult=$(unused "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
if [ -n "${unusedResult}" ]; then
echo -e "unused checking failed:\n${unusedResult}"
exit 255
@@ -367,7 +367,7 @@ function fmt_pass {
if which staticcheck >/dev/null; then
echo "Checking staticcheck..."
staticcheckResult=`staticcheck ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
staticcheckResult=$(staticcheck "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
if [ -n "${staticcheckResult}" ]; then
# TODO: resolve these after go1.8 migration
# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
@@ -376,7 +376,7 @@ function fmt_pass {
echo -e "staticcheck checking failed:\n${staticcheckResult}"
exit 255
else
suppressed=`echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c`
suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c)
echo -e "staticcheck suppressed warnings:\n${suppressed}"
fi
fi
@@ -398,8 +398,8 @@ function fmt_pass {
fi
echo "Checking commit titles..."
git log --oneline "$(git merge-base HEAD master)"...HEAD | while read l; do
commitMsg=`echo "$l" | cut -f2- -d' '`
git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
commitMsg=$(echo "$l" | cut -f2- -d' ')
if [[ "$commitMsg" == Merge* ]]; then
# ignore "Merge pull" commits
continue
@@ -409,10 +409,10 @@ function fmt_pass {
continue
fi
pkgPrefix=`echo "$commitMsg" | cut -f1 -d':'`
spaceCommas=`echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0`
commaSpaces=`echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0`
if [[ `echo $commitMsg | grep -c ":..*"` == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':')
spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
echo "$l"...
echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
echo "Got: $l"
@@ -441,7 +441,7 @@ function dep_pass {
# don't pull in etcdserver package
pushd clientv3 >/dev/null
badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
deps=`go list -f '{{ .Deps }}' | sed 's/ /\n/g' | egrep "${badpkg}" || echo ""`
deps=$(go list -f '{{ .Deps }}' | sed 's/ /\n/g' | egrep "${badpkg}" || echo "")
popd >/dev/null
if [ ! -z "$deps" ]; then
echo -e "clientv3 has masked dependencies:\n${deps}"
@@ -452,8 +452,8 @@ function dep_pass {
function build_cov_pass {
out="bin"
if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcd_test
go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcdctl_test ${REPO_PATH}/etcdctl
go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcd_test"
go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcdctl_test" "${REPO_PATH}/etcdctl"
}
function compile_pass {
@@ -468,7 +468,7 @@ function build_pass {
for pass in $PASSES; do
echo "Starting '$pass' pass at $(date)"
${pass}_pass $@
"${pass}"_pass "$@"
echo "Finished '$pass' pass at $(date)"
done