From 84b5b87fb21ed5dfdc9822c0313a79e07704f941 Mon Sep 17 00:00:00 2001
From: Piotr Tabor <ptab@google.com>
Date: Thu, 7 Jan 2021 10:07:51 +0100
Subject: [PATCH] Make test.sh scripts OSX/BSD compatible:

- build & test scripts deprecated. Call *.sh variants.
  This will avoid delete the symlinks and get rid of
  subtle dependency on 'sed --follow-symlinks' on OsX/BSD sed.

- Fix parameters to mktemp
---
 Makefile                                      |  12 +-
 build                                         | 125 +---
 build.sh                                      | 126 +++-
 .../recipes/grpc_gateway/user_add.sh          |   3 +-
 scripts/fix.sh                                |   7 +-
 test                                          | 612 +----------------
 test.sh                                       | 624 +++++++++++++++++-
 7 files changed, 767 insertions(+), 742 deletions(-)
 mode change 120000 => 100755 build.sh
 mode change 120000 => 100755 test.sh

diff --git a/Makefile b/Makefile
index 2f6a41c6d..e8d34ef06 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,7 @@ XARGS += rm -r
 
 .PHONY: build
 build:
-	GO_BUILD_FLAGS="-v" ./build
+	GO_BUILD_FLAGS="-v" ./build.sh
 	./bin/etcd --version
 	./bin/etcdctl version
 
@@ -107,7 +107,7 @@ compile-with-docker-test:
 	  --rm \
 	  --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \
 	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
-	  /bin/bash -c "GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build && ./bin/etcd --version"
+	  /bin/bash -c "GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build.sh && ./bin/etcd --version"
 
 compile-setup-gopath-with-docker-test:
 	$(info GO_VERSION: $(GO_VERSION))
@@ -115,7 +115,7 @@ compile-setup-gopath-with-docker-test:
 	  --rm \
 	  --mount type=bind,source=`pwd`,destination=/etcd \
 	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
-	  /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build && ./bin/etcd --version && rm -rf ./gopath"
+	  /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build.sh && ./bin/etcd --version && rm -rf ./gopath"
 
 
 
@@ -148,7 +148,7 @@ compile-setup-gopath-with-docker-test:
 test:
 	$(info TEST_OPTS: $(TEST_OPTS))
 	$(info log-file: test-$(TEST_SUFFIX).log)
-	$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log
+	$(TEST_OPTS) ./test.sh 2>&1 | tee test-$(TEST_SUFFIX).log
 	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
 
 docker-test:
@@ -163,7 +163,7 @@ docker-test:
 	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \
 	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
-	  /bin/bash -c "$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log"
+	  /bin/bash -c "$(TEST_OPTS) ./test.sh 2>&1 | tee test-$(TEST_SUFFIX).log"
 	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
 
 docker-test-coverage:
@@ -177,7 +177,7 @@ docker-test-coverage:
 	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \
 	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
-	  /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1"
+	  /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test.sh 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1"
 	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 docker-test-coverage-$(TEST_SUFFIX).log
 
 
diff --git a/build b/build
index 0521fe6a4..b9e031074 100755
--- a/build
+++ b/build
@@ -1,125 +1,6 @@
 #!/usr/bin/env bash
 
-source ./scripts/test_lib.sh
+echo -e "\e[91mDEPRECATED!!! Use build.sh script instead.\e[0m\n"
+sleep 1
 
-GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
-if [[ -n "$FAILPOINTS" ]]; then
-  GIT_SHA="$GIT_SHA"-FAILPOINTS
-fi
-
-VERSION_SYMBOL="go.etcd.io/etcd/api/v3/version.GitSHA"
-
-# Set GO_LDFLAGS="-s" for building without symbols for debugging.
-# shellcheck disable=SC2206
-GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
-GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")
-
-# enable/disable failpoints
-toggle_failpoints() {
-  mode="$1"
-  if command -v gofail >/dev/null 2>&1; then
-    run gofail "$mode" server/etcdserver/ server/mvcc/backend/
-  elif [[ "$mode" != "disable" ]]; then
-    log_error "FAILPOINTS set but gofail not found"
-    exit 1
-  fi
-}
-
-toggle_failpoints_default() {
-  mode="disable"
-  if [[ -n "$FAILPOINTS" ]]; then mode="enable"; fi
-  toggle_failpoints "$mode"
-}
-
-etcd_build() {
-  out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
-  toggle_failpoints_default
-
-  run rm -f "${out}/etcd"
-  (
-    cd ./server
-    # Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
-    # shellcheck disable=SC2086
-    run env "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \
-      -installsuffix=cgo \
-      "-ldflags=${GO_LDFLAGS[*]}" \
-      -o="../${out}/etcd" . || return 2
-  ) || return 2
-
-  run rm -f "${out}/etcdctl"
-  # shellcheck disable=SC2086
-  (
-    cd ./etcdctl
-    run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build $GO_BUILD_FLAGS \
-      -installsuffix=cgo \
-      "-ldflags=${GO_LDFLAGS[*]}" \
-      -o="../${out}/etcdctl" . || return 2
-  ) || return 2
-  # Verify whether symbol we overriden exists
-  # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}"
-
-  # We need symbols to do this check:
-  if [[ "${GO_LDFLAGS[*]}" != *"-s"* ]]; then
-    go tool nm "${out}/etcd" | grep "${VERSION_SYMBOL}" > /dev/null
-    if [[ "${PIPESTATUS[*]}" != "0 0" ]]; then
-      log_error "FAIL: Symbol ${VERSION_SYMBOL} not found in binary: ${out}/etcd"
-      return 2
-    fi
-  fi
-}
-
-tools_build() {
-  out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
-  tools_path="tools/benchmark
-    tools/etcd-dump-db
-    tools/etcd-dump-logs
-    tools/local-tester/bridge"
-  for tool in ${tools_path}
-  do
-    echo "Building" "'${tool}'"...
-    run rm -f "${out}/${tool}"
-    # shellcheck disable=SC2086
-    run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} \
-      -installsuffix=cgo \
-      "-ldflags='${GO_LDFLAGS[*]}'" \
-      -o="${out}/${tool}" "./${tool}" || return 2
-  done
-  tests_build "${@}"
-}
-
-tests_build() {
-  out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
-  tools_path="
-    functional/cmd/etcd-agent
-    functional/cmd/etcd-proxy
-    functional/cmd/etcd-runner
-    functional/cmd/etcd-tester"
-  (
-    cd tests || exit 2
-    for tool in ${tools_path}; do
-      echo "Building" "'${tool}'"...
-      run rm -f "../${out}/${tool}"
-
-      # shellcheck disable=SC2086
-      run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build ${GO_BUILD_FLAGS} \
-        -installsuffix=cgo \
-        "-ldflags='${GO_LDFLAGS[*]}'" \
-        -o="../${out}/${tool}" "./${tool}" || return 2
-    done
-  ) || return 2
-}
-
-toggle_failpoints_default
-
-# only build when called directly, not sourced
-if echo "$0" | grep "build$" >/dev/null; then
-  if etcd_build; then
-    log_success "SUCCESS: etcd_build"
-  else
-    log_error "FAIL: etcd_build"
-    exit 2
-  fi
-fi
+source ./build.sh
diff --git a/build.sh b/build.sh
deleted file mode 120000
index c795b054e..000000000
--- a/build.sh
+++ /dev/null
@@ -1 +0,0 @@
-build
\ No newline at end of file
diff --git a/build.sh b/build.sh
new file mode 100755
index 000000000..ef06018d6
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+source ./scripts/test_lib.sh
+
+GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
+if [[ -n "$FAILPOINTS" ]]; then
+  GIT_SHA="$GIT_SHA"-FAILPOINTS
+fi
+
+VERSION_SYMBOL="go.etcd.io/etcd/api/v3/version.GitSHA"
+
+# Set GO_LDFLAGS="-s" for building without symbols for debugging.
+# shellcheck disable=SC2206
+GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
+GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")
+
+# enable/disable failpoints
+toggle_failpoints() {
+  mode="$1"
+  if command -v gofail >/dev/null 2>&1; then
+    run gofail "$mode" server/etcdserver/ server/mvcc/backend/
+  elif [[ "$mode" != "disable" ]]; then
+    log_error "FAILPOINTS set but gofail not found"
+    exit 1
+  fi
+}
+
+toggle_failpoints_default() {
+  mode="disable"
+  if [[ -n "$FAILPOINTS" ]]; then mode="enable"; fi
+  toggle_failpoints "$mode"
+}
+
+etcd_build() {
+  out="bin"
+  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  toggle_failpoints_default
+
+  run rm -f "${out}/etcd"
+  (
+    cd ./server
+    # Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
+    # shellcheck disable=SC2086
+    run env "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \
+      -installsuffix=cgo \
+      "-ldflags=${GO_LDFLAGS[*]}" \
+      -o="../${out}/etcd" . || return 2
+  ) || return 2
+
+  run rm -f "${out}/etcdctl"
+  # shellcheck disable=SC2086
+  (
+    cd ./etcdctl
+    run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build $GO_BUILD_FLAGS \
+      -installsuffix=cgo \
+      "-ldflags=${GO_LDFLAGS[*]}" \
+      -o="../${out}/etcdctl" . || return 2
+  ) || return 2
+  # Verify whether symbol we overriden exists
+  # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}"
+
+  # We need symbols to do this check:
+  if [[ "${GO_LDFLAGS[*]}" != *"-s"* ]]; then
+    go tool nm "${out}/etcd" | grep "${VERSION_SYMBOL}" > /dev/null
+    if [[ "${PIPESTATUS[*]}" != "0 0" ]]; then
+      log_error "FAIL: Symbol ${VERSION_SYMBOL} not found in binary: ${out}/etcd"
+      return 2
+    fi
+  fi
+}
+
+tools_build() {
+  out="bin"
+  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  tools_path="tools/benchmark
+    tools/etcd-dump-db
+    tools/etcd-dump-logs
+    tools/local-tester/bridge"
+  for tool in ${tools_path}
+  do
+    echo "Building" "'${tool}'"...
+    run rm -f "${out}/${tool}"
+    # shellcheck disable=SC2086
+    run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} \
+      -installsuffix=cgo \
+      "-ldflags='${GO_LDFLAGS[*]}'" \
+      -o="${out}/${tool}" "./${tool}" || return 2
+  done
+  tests_build "${@}"
+}
+
+tests_build() {
+  out="bin"
+  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  tools_path="
+    functional/cmd/etcd-agent
+    functional/cmd/etcd-proxy
+    functional/cmd/etcd-runner
+    functional/cmd/etcd-tester"
+  (
+    cd tests || exit 2
+    for tool in ${tools_path}; do
+      echo "Building" "'${tool}'"...
+      run rm -f "../${out}/${tool}"
+
+      # shellcheck disable=SC2086
+      run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build ${GO_BUILD_FLAGS} \
+        -installsuffix=cgo \
+        "-ldflags='${GO_LDFLAGS[*]}'" \
+        -o="../${out}/${tool}" "./${tool}" || return 2
+    done
+  ) || return 2
+}
+
+toggle_failpoints_default
+
+# only build when called directly, not sourced
+if echo "$0" | grep -E "build(.sh)?$" >/dev/null; then
+  if etcd_build; then
+    log_success "SUCCESS: etcd_build"
+  else
+    log_error "FAIL: etcd_build"
+    exit 2
+  fi
+fi
diff --git a/client/v3/experimental/recipes/grpc_gateway/user_add.sh b/client/v3/experimental/recipes/grpc_gateway/user_add.sh
index 2ad3a54e3..ec5386743 100644
--- a/client/v3/experimental/recipes/grpc_gateway/user_add.sh
+++ b/client/v3/experimental/recipes/grpc_gateway/user_add.sh
@@ -71,4 +71,5 @@ escape() {
 token=$(tokengen $user $pass)
 response=$(add_user $newuser $newpass $token)
 
-echo -e "\\n$response"
\ No newline at end of file
+echo -e "\\n$response"
+
diff --git a/scripts/fix.sh b/scripts/fix.sh
index 264854b9b..c512444b4 100755
--- a/scripts/fix.sh
+++ b/scripts/fix.sh
@@ -11,10 +11,13 @@ function mod_tidy_fix {
 }
 
 function bash_ws_fix {
+  TAB=$'\t'
+
   log_callout "Fixing whitespaces in the bash scripts"
   # Makes sure all bash scripts do use '  ' (double space) for indention. 
-  log_cmd "find ./ -name '*.sh' -print0 | xargs -0 sed --follow-symlinks -i 's|\t|  |g'"
-  find ./ -name '*.sh' -print0 | xargs -0 sed --follow-symlinks -i 's|\t|  |g'
+  log_cmd "find ./ -name '*.sh' -print0 | xargs -0 sed -i.bak 's|${TAB}|  |g'"
+  find ./ -name '*.sh' -print0 | xargs -0 sed -i.bak "s|${TAB}|  |g"
+  find ./ -name '*.sh.bak' -print0 | xargs -0 rm
 }
 
 log_callout -e "\nFixing etcd code for you...\n"
diff --git a/test b/test
index ee99a99f2..e47fcb6ef 100755
--- a/test
+++ b/test
@@ -1,612 +1,6 @@
 #!/usr/bin/env bash
-#
-# Run all etcd tests
-# ./test
-# ./test -v
-#
-#
-# Run specified test pass
-#
-# $ PASSES=unit ./test
-# $ PASSES=integration ./test
-#
-#
-# Run tests for one package
-# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
-# flag for different expectation
-#
-# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
-# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
-#
-# Run specified unit tests in one package
-# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
-# to run only "TestNew", set "TESTCASE="\bTestNew\b""
-#
-# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
-# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
-# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
-#
-#
-# Run code coverage
-# COVERDIR must either be a absolute path or a relative path to the etcd root
-# $ COVERDIR=coverage PASSES="build build_cov cov" ./test
-# $ go tool cover -html ./coverage/cover.out
-set -e
-set -o pipefail
 
+echo -e "\e[91mDEPRECATED!!! Use test.sh script instead.\e[0m\n"
+sleep 1
 
-# Consider command as failed when any component of the pipe fails:
-# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
-set -o pipefail
-
-# The test script is not supposed to make any changes to the files
-# e.g. add/update missing dependencies. Such divergences should be 
-# detected and trigger a failure that needs explicit developer's action.
-export GOFLAGS=-mod=readonly
-
-source ./scripts/test_lib.sh
-source ./build
-
-PASSES=${PASSES:-"fmt bom dep build unit"}
-PKG=${PKG:-}
-
-if [ -z "$GOARCH" ]; then
-  GOARCH=$(go env GOARCH);
-fi
-
-# determine the number of CPUs to use for Go tests
-CPU=${CPU:-"4"}
-
-# determine whether target supports race detection
-if [ -z "${RACE}" ] ; then
-  if [ "$GOARCH" == "amd64" ]; then
-    RACE="--race"
-  else
-    RACE="--race=false"
-  fi
-else
-  RACE="--race=${RACE:-true}"
-fi
-
-# This options make sense for cases where SUT (System Under Test) is compiled by test.
-COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}")
-log_callout "Running with ${COMMON_TEST_FLAGS[*]}"
-
-RUN_ARG=()
-if [ -n "${TESTCASE}" ]; then
-  RUN_ARG=("-run=${TESTCASE}")
-fi
-
-function build_pass {
-  log_callout "Building etcd"
-  run_for_modules run go build "${@}" || return 2
-  GO_BUILD_FLAGS="-v" etcd_build "${@}"
-  GO_BUILD_FLAGS="-v" tools_build "${@}"
-}
-
-################# REGULAR TESTS ################################################
-
-# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
-function run_unit_tests {
-  local pkgs="${1:-./...}"
-  shift 1
-  # shellcheck disable=SC2086
-  go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
-}
-
-function unit_pass {
-  run_for_modules run_unit_tests "$@"
-}
-
-function integration_extra {
-  if [ -z "${PKG}" ] ; then
-    run_for_module "."  go_test "./contrib/raftexample" "keep_going" :  -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
-    run_for_module "tests"  go_test "./integration/v2store/..." "keep_going" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
-  else
-    log_warning "integration_extra ignored when PKG is specified"
-  fi
-}
-
-function integration_pass {
-  local pkgs=${USERPKG:-"./integration/..."}
-  run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
-  integration_extra "$@"
-}
-
-function e2e_pass {
-  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
-  run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
-}
-
-function integration_e2e_pass {
-  run_pass "integration" "${@}"
-  run_pass "e2e" "${@}"
-}
-
-# generic_checker [cmd...]
-# executes given command in the current module, and clearly fails if it
-# failed or returned output.
-function generic_checker {
-  local cmd=("$@")
-  if ! output=$("${cmd[@]}"); then
-    echo "${output}"
-    log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
-    return 255
-  fi
-  if [ -n "${output}" ]; then
-    echo "${output}"
-    log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
-    return 255
-  fi
-}
-
-function functional_pass {
-  run ./tests/functional/build
-
-  # Clean up any data and logs from previous runs
-  rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
-
-  # TODO: These ports should be dynamically allocated instead of hard-coded.
-  for a in 1 2 3; do
-    ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 < /dev/null &
-    pid="$!"
-    agent_pids="${agent_pids} $pid"
-  done
-
-  for a in 1 2 3; do
-    log_callout "Waiting for 'etcd-agent' on ${a}9027..."
-    while ! nc -z localhost ${a}9027; do
-      sleep 1
-    done
-  done
-
-  log_callout "functional test START!"
-  run ./bin/etcd-tester --config ./tests/functional/functional.yaml && log_success "'etcd-tester' succeeded"
-  local etcd_tester_exit_code=$?
-
-  if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
-    log_error "ETCD_TESTER_EXIT_CODE:" ${etcd_tester_exit_code}
-  fi
-
-  # shellcheck disable=SC2206
-  agent_pids=($agent_pids)
-  kill -s TERM "${agent_pids[@]}" || true
-
-  if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
-    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
-    tail -1000 /tmp/etcd-functional-1/etcd.log
-
-    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
-    tail -1000 /tmp/etcd-functional-2/etcd.log
-
-    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
-    tail -1000 /tmp/etcd-functional-3/etcd.log
-
-    log_error "--- FAIL: exit code" ${etcd_tester_exit_code}
-    return ${etcd_tester_exit_code}
-  fi
-  log_success "functional test PASS!"
-}
-
-function grpcproxy_pass {
-  run_for_module "tests" go_test "./integration/... ./e2e" "fail_fast" : \
-      -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
-}
-
-################# COVERAGE #####################################################
-
-# Builds artifacts used by tests/e2e in coverage mode.
-function build_cov_pass {
-  local out="${BINDIR:-./bin}"
-  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test"
-  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl"
-}
-
-# pkg_to_coverflag [prefix] [pkgs]
-# produces name of .coverprofile file to be used for tests of this package
-function pkg_to_coverprofileflag {
-  local prefix="${1}"
-  local pkgs="${2}"
-  local pkgs_normalized
-  pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
-  echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
-}
-
-function cov_pass {
-  # shellcheck disable=SC2153
-  if [ -z "$COVERDIR" ]; then
-    log_error "COVERDIR undeclared"
-    return 255
-  fi
-
-  if [ ! -f "bin/etcd_test" ]; then
-    log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
-    return 255
-  fi
-
-  local coverdir
-  coverdir=$(readlink -f "${COVERDIR}")
-  mkdir -p "${coverdir}"
-  rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"
-
-  local covpkgs
-  covpkgs=$(pkgs_in_module "./...")
-  local coverpkg_comma
-  coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
-  local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")
-
-  local failed=""
-
-  log_callout "Collecting coverage from unit tests ..."
-  go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
-      "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
-
-  log_callout "Collecting coverage from integration tests ..."
-  run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration" \
-      -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
-  # integration-store-v2
-  run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
-      -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3"
-  # integration_cluster_proxy
-  run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
-      -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
-
-  log_callout "Collecting coverage from e2e tests ..."
-  # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
-  # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
-  run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
-
-  log_callout "Collecting coverage from e2e tests with proxy ..."
-  run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
-
-  log_callout "Merging coverage results ..."
-  local cover_out_file="${coverdir}/cover.out"
-  # gocovmerge requires not-empty test to start with:
-  echo "mode: set" > "${cover_out_file}"
-
-  # incrementally merge to get coverage data even if some coverage files are corrupted
-  for f in "${coverdir}"/*.coverprofile; do
-    echo "merging test coverage file ${f}"
-    run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
-    if [ -s "${coverdir}"/cover.tmp ]; then
-      mv "${coverdir}/cover.tmp" "${cover_out_file}"
-    fi
-  done
-  # strip out generated files (using GNU-style sed)
-  sed --in-place '/generated.go/d' "${cover_out_file}" || true
-
-  # held failures to generate the full coverage file, now fail
-  if [ -n "$failed" ]; then
-    for f in $failed; do
-      log_error "--- FAIL:" "$f"
-    done
-    log_warning "Despite failures, you can see partial report:"
-    log_warning "  go tool cover -html ${cover_out_file}"
-    return 255
-  fi
-
-  log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
-}
-
-######### Code formatting checkers #############################################
-
-function fmt_pass {
-  toggle_failpoints disable
-
-  # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
-  # after resolving ore-existing errors.
-  for p in shellcheck \
-      markdown_you \
-      goword \
-      gofmt \
-      govet \
-      revive \
-      license_header \
-      receiver_name \
-      mod_tidy \
-      dep \
-      shellcheck \
-      shellws \
-      ; do
-    run_pass "${p}" "${@}"
-  done
-}
-
-function shellcheck_pass {
-  if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
-    generic_checker run shellcheck -fgcc build test scripts/*.sh
-  fi
-}
-
-function shellws_pass {
-  log_callout "Ensuring no tab-based indention in shell scripts"
-  local files
-  files=$(find ./ -name '*.sh' -print0 | xargs -0 )
-  log_cmd "grep -E -n $'^ *\t' ${files}"
-  # shellcheck disable=SC2086
-  if grep -E -n $'^ *\t' ${files} | sed -s $'s|\t|[\\\\tab]|g'; then
-    log_error "FAIL: found tab-based indention in bash scripts. Use '  ' (double space)."
-    return 1
-  else
-    log_success "SUCCESS: no tabulators found."
-    return 0
-  fi
-}
-
-function markdown_you_find_eschew_you {
-  local find_you_cmd="find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' ! -path './release/*' -exec grep -E --color '[Yy]ou[r]?[ '\''.,;]' {} + || true"
-  run eval "${find_you_cmd}"
-}
-
-function markdown_you_pass {
-  generic_checker markdown_you_find_eschew_you
-}
-
-function markdown_marker_pass {
-  # TODO: check other markdown files when marker handles headers with '[]'
-  if tool_exists "marker" "https://crates.io/crates/marker"; then
-    generic_checker run marker --skip-http --root ./Documentation 2>&1
-  fi
-}
-
-function govet_pass {
-  run_for_modules generic_checker run go vet
-}
-
-function govet_shadow_pass {
-  local shadow
-  shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow")
-  run_for_modules generic_checker run go vet -all -vettool="${shadow}"
-}
-
-function unparam_pass {
-  run_for_modules generic_checker run_go_tool "mvdan.cc/unparam"
-}
-
-function staticcheck_pass {
-  run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck"
-}
-
-function revive_pass {
-  run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
-}
-
-function unconvert_pass {
-  run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v
-}
-
-function ineffassign_per_package {
-  mapfile -t gofiles < <(go_srcs_in_module "$1")
-  run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}"
-}
-
-function ineffassign_pass {
-  run_for_modules generic_checker ineffassign_per_package
-}
-
-function nakedret_pass {
-  run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret"
-}
-
-function license_header_pass {
-  mapfile -t gofiles < <(go_srcs_in_module "$1")
-  for file in "${gofiles[@]}"; do
-    if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
-      licRes="${licRes}"$(echo -e "  ${file}")
-    fi
-  done
-  if [ -n "${licRes}" ]; then
-    log_error -e "license header checking failed:\\n${licRes}"
-    return 255
-  fi
-}
-
-function receiver_name_for_package {
-  mapfile -t gofiles < <(go_srcs_in_module "$1")
-  recvs=$(grep 'func ([^*]' "${gofiles[@]}"  | tr  ':' ' ' |  \
-    awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
-    grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
-  if [ -n "${recvs}" ]; then
-    # shellcheck disable=SC2206
-    recvs=($recvs)
-    for recv in "${recvs[@]}"; do
-      log_error "Mismatched receiver for $recv..."
-      grep "$recv" "${gofiles[@]}" | grep 'func ('
-    done
-    return 255
-  fi
-}
-
-function receiver_name_pass {
-  run_for_modules receiver_name_for_package
-}
-
-# goword_for_package package
-# checks spelling and comments in the 'package' in the current module
-#
-function goword_for_package {
-  mapfile -t gofiles < <(go_srcs_in_module "$1")
-  local gowordRes
-
-  # spellchecking can be enabled with GOBINARGS="--tags=spell"
-  # but it requires heavy dependencies installation, like:
-  # apt-get install libaspell-dev libhunspell-dev hunspell-en-us aspell-en
-
-  # only check for broke exported godocs
-  if gowordRes=$(run_go_tool "github.com/chzchzchz/goword" -use-spell=false "${gofiles[@]}" | grep godoc-export | sort); then
-    log_error -e "goword checking failed:\\n${gowordRes}"
-    return 255
-  fi
-  if [ -n "$gowordRes" ]; then
-    log_error -e "goword checking returned output:\\n${gowordRes}"
-    return 255
-  fi
-}
-
-
-function goword_pass {
-  run_for_modules goword_for_package || return 255
-}
-
-function go_fmt_for_package {
-  # We utilize 'go fmt' to find all files suitable for formatting,
-  # but reuse full power gofmt to perform just RO check.
-  go fmt -n "$1" | sed 's| -w | -d |g' | sh
-}
-
-function gofmt_pass {
-  run_for_modules generic_checker go_fmt_for_package
-}
-
-function bom_pass {
-  log_callout "Checking bill of materials..."
-  # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
-  # shellcheck disable=SC2207
-  modules=($(modules_exp))
-
-  # Internally license-bill-of-materials tends to modify go.sum
-  run cp go.sum go.sum.tmp || return 2
-  run cp go.mod go.mod.tmp || return 2
-
-  output=$(GOFLAGS=-mod=mod run_go_tool github.com/coreos/license-bill-of-materials \
-    --override-file ./bill-of-materials.override.json \
-    "${modules[@]}")
-  code="$?"
-
-  run cp go.sum.tmp go.sum || return 2
-  run cp go.mod.tmp go.mod || return 2
-
-  if [ "${code}" -ne 0 ] ; then
-    log_error -e "license-bill-of-materials (code: ${code}) failed with:\n${output}"
-    return 255
-  else
-    echo "${output}" > "bom-now.json.tmp"
-  fi
-  if ! diff ./bill-of-materials.json bom-now.json.tmp; then
-    log_error "modularized licenses do not match given bill of materials"
-    return 255
-  fi
-  rm bom-now.json.tmp
-}
-
-######## VARIOUS CHECKERS ######################################################
-
-function dump_deps_of_module() {
-  local module
-  if ! module=$(run go list -m); then
-    return 255
-  fi
-  run go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all
-}
-
-# Checks whether dependencies are consistent across modules
-function dep_pass {
-  local all_dependencies
-  all_dependencies=$(run_for_modules dump_deps_of_module | sort) || return 2
-
-  local duplicates
-  duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2
-
-  for dup in ${duplicates}; do
-    log_error "FAIL: inconsistent versions for depencency: ${dup}"
-    echo "${all_dependencies}" | grep "${dup}" | sed "s|\([^,]*\),\([^,]*\),\([^,]*\)|  - \1@\2 from: \3|g"
-  done
-  if [[ -n "${duplicates}" ]]; then
-    log_error "FAIL: inconsistent dependencies"
-    return 2
-  else
-    log_success "SUCCESS: dependencies are consistent across modules"
-  fi
-}
-
-function release_pass {
-  rm -f ./bin/etcd-last-release
-  # to grab latest patch release; bump this up for every minor release
-  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
-  if [ -n "$MANUAL_VER" ]; then
-    # in case, we need to test against different version
-    UPGRADE_VER=$MANUAL_VER
-  fi
-  if [[ -z ${UPGRADE_VER} ]]; then
-    UPGRADE_VER="v3.3.0"
-    log_warning "fallback to" ${UPGRADE_VER}
-  fi
-
-  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
-  log_callout "Downloading $file"
-
-  set +e
-  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
-  local result=$?
-  set -e
-  case $result in
-    0)  ;;
-    *)  log_error "--- FAIL:" ${result}
-      return $result
-      ;;
-  esac
-
-  tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
-  mkdir -p ./bin
-  mv /tmp/etcd ./bin/etcd-last-release
-}
-
-function mod_tidy_for_module {
-  # Watch for upstream solution: https://github.com/golang/go/issues/27005
-  local tmpModDir
-  tmpModDir=$(mktemp -d --suffix "etcd-mod")
-  run cp "./go.mod" "./go.sum" "${tmpModDir}" || return 2
-
-  # Guarantees keeping go.sum minimal
-  # If this is causing too much problems, we should
-  # stop controlling go.sum at all.
-  rm go.sum
-  run go mod tidy || return 2
-
-  set +e
-  local tmpFileGoModInSync
-  diff -C 5 "${tmpModDir}/go.mod" "./go.mod"
-  tmpFileGoModInSync="$?"
-
-  local tmpFileGoSumInSync
-  diff -C 5 "${tmpModDir}/go.sum" "./go.sum"
-  tmpFileGoSumInSync="$?"
-  set -e
-
-  # Bring back initial state
-  mv "${tmpModDir}/go.mod" "./go.mod"
-  mv "${tmpModDir}/go.sum" "./go.sum"
-
-  if [ "${tmpFileGoModInSync}" -ne 0 ]; then
-    log_error "${PWD}/go.mod is not in sync with 'go mod tidy'"
-    return 255
-  fi
-  if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
-    log_error "${PWD}/go.sum is not in sync with 'rm go.sum; go mod tidy'"
-    return 255
-  fi
-}
-
-function mod_tidy_pass {
-  run_for_modules mod_tidy_for_module
-}
-
-########### MAIN ###############################################################
-
-function run_pass {
-  local pass="${1}"
-  shift 1
-  log_callout -e "\n'${pass}' started at $(date)"
-  if "${pass}_pass" "$@" ; then
-    log_success "'${pass}' completed at $(date)"
-  else
-    log_error "FAIL: '${pass}' failed at $(date)"
-    exit 255
-  fi
-}
-
-for pass in $PASSES; do
-  run_pass "${pass}" "${@}"
-done
-
-log_success "SUCCESS"
+source ./test.sh
diff --git a/test.sh b/test.sh
deleted file mode 120000
index 30d74d258..000000000
--- a/test.sh
+++ /dev/null
@@ -1 +0,0 @@
-test
\ No newline at end of file
diff --git a/test.sh b/test.sh
new file mode 100755
index 000000000..be5112062
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,623 @@
+#!/usr/bin/env bash
+#
+# Run all etcd tests
+# ./test
+# ./test -v
+#
+#
+# Run specified test pass
+#
+# $ PASSES=unit ./test
+# $ PASSES=integration ./test
+#
+#
+# Run tests for one package
+# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
+# flag for different expectation
+#
+# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
+# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
+#
+# Run specified unit tests in one package
+# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
+# to run only "TestNew", set "TESTCASE="\bTestNew\b""
+#
+# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
+# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
+# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
+#
+#
+# Run code coverage
+# COVERDIR must either be a absolute path or a relative path to the etcd root
+# $ COVERDIR=coverage PASSES="build build_cov cov" ./test
+# $ go tool cover -html ./coverage/cover.out
+set -e
+set -o pipefail
+
+
+# Consider command as failed when any component of the pipe fails:
+# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
+set -o pipefail
+
+# The test script is not supposed to make any changes to the files
+# e.g. add/update missing dependencies. Such divergences should be 
+# detected and trigger a failure that needs explicit developer's action.
+export GOFLAGS=-mod=readonly
+
+source ./scripts/test_lib.sh
+source ./build.sh
+
+PASSES=${PASSES:-"fmt bom dep build unit"}
+PKG=${PKG:-}
+
+if [ -z "$GOARCH" ]; then
+  GOARCH=$(go env GOARCH);
+fi
+
+# determine the number of CPUs to use for Go tests
+CPU=${CPU:-"4"}
+
+# determine whether target supports race detection
+if [ -z "${RACE}" ] ; then
+  if [ "$GOARCH" == "amd64" ]; then
+    RACE="--race"
+  else
+    RACE="--race=false"
+  fi
+else
+  RACE="--race=${RACE:-true}"
+fi
+
+# This options make sense for cases where SUT (System Under Test) is compiled by test.
+COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}")
+log_callout "Running with ${COMMON_TEST_FLAGS[*]}"
+
+RUN_ARG=()
+if [ -n "${TESTCASE}" ]; then
+  RUN_ARG=("-run=${TESTCASE}")
+fi
+
+function build_pass {
+  log_callout "Building etcd"
+  run_for_modules run go build "${@}" || return 2
+  GO_BUILD_FLAGS="-v" etcd_build "${@}"
+  GO_BUILD_FLAGS="-v" tools_build "${@}"
+}
+
+################# REGULAR TESTS ################################################
+
+# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
+function run_unit_tests {
+  local pkgs="${1:-./...}"
+  shift 1
+  # shellcheck disable=SC2086
+  go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
+}
+
+function unit_pass {
+  run_for_modules run_unit_tests "$@"
+}
+
+function integration_extra {
+  if [ -z "${PKG}" ] ; then
+    run_for_module "."  go_test "./contrib/raftexample" "keep_going" :  -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
+    run_for_module "tests"  go_test "./integration/v2store/..." "keep_going" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
+  else
+    log_warning "integration_extra ignored when PKG is specified"
+  fi
+}
+
+function integration_pass {
+  local pkgs=${USERPKG:-"./integration/..."}
+  run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
+  integration_extra "$@"
+}
+
+function e2e_pass {
+  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
+  run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
+}
+
+function integration_e2e_pass {
+  run_pass "integration" "${@}"
+  run_pass "e2e" "${@}"
+}
+
+# generic_checker [cmd...]
+# executes given command in the current module, and clearly fails if it
+# failed or returned output.
+function generic_checker {
+  local cmd=("$@")
+  if ! output=$("${cmd[@]}"); then
+    echo "${output}"
+    log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
+    return 255
+  fi
+  if [ -n "${output}" ]; then
+    echo "${output}"
+    log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
+    return 255
+  fi
+}
+
+function functional_pass {
+  run ./tests/functional/build
+
+  # Clean up any data and logs from previous runs
+  rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
+
+  # TODO: These ports should be dynamically allocated instead of hard-coded.
+  for a in 1 2 3; do
+    ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 < /dev/null &
+    pid="$!"
+    agent_pids="${agent_pids} $pid"
+  done
+
+  for a in 1 2 3; do
+    log_callout "Waiting for 'etcd-agent' on ${a}9027..."
+    while ! nc -z localhost ${a}9027; do
+      sleep 1
+    done
+  done
+
+  log_callout "functional test START!"
+  run ./bin/etcd-tester --config ./tests/functional/functional.yaml && log_success "'etcd-tester' succeeded"
+  local etcd_tester_exit_code=$?
+
+  if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
+    log_error "ETCD_TESTER_EXIT_CODE:" ${etcd_tester_exit_code}
+  fi
+
+  # shellcheck disable=SC2206
+  agent_pids=($agent_pids)
+  kill -s TERM "${agent_pids[@]}" || true
+
+  if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
+    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
+    tail -1000 /tmp/etcd-functional-1/etcd.log
+
+    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
+    tail -1000 /tmp/etcd-functional-2/etcd.log
+
+    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
+    tail -1000 /tmp/etcd-functional-3/etcd.log
+
+    log_error "--- FAIL: exit code" ${etcd_tester_exit_code}
+    return ${etcd_tester_exit_code}
+  fi
+  log_success "functional test PASS!"
+}
+
+function grpcproxy_pass {
+  run_for_module "tests" go_test "./integration/... ./e2e" "fail_fast" : \
+      -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
+}
+
+################# COVERAGE #####################################################
+
+# Builds artifacts used by tests/e2e in coverage mode.
+function build_cov_pass {
+  local out="${BINDIR:-./bin}"
+  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test"
+  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl"
+}
+
+# pkg_to_coverflag [prefix] [pkgs]
+# produces name of .coverprofile file to be used for tests of this package
+function pkg_to_coverprofileflag {
+  local prefix="${1}"
+  local pkgs="${2}"
+  local pkgs_normalized
+  pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
+  echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
+}
+
+function cov_pass {
+  # shellcheck disable=SC2153
+  if [ -z "$COVERDIR" ]; then
+    log_error "COVERDIR undeclared"
+    return 255
+  fi
+
+  if [ ! -f "bin/etcd_test" ]; then
+    log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
+    return 255
+  fi
+
+  local coverdir
+  coverdir=$(readlink -f "${COVERDIR}")
+  mkdir -p "${coverdir}"
+  rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"
+
+  local covpkgs
+  covpkgs=$(pkgs_in_module "./...")
+  local coverpkg_comma
+  coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
+  local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")
+
+  local failed=""
+
+  log_callout "Collecting coverage from unit tests ..."
+  go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
+      "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
+
+  log_callout "Collecting coverage from integration tests ..."
+  run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration" \
+      -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
+  # integration-store-v2
+  run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
+      -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3"
+  # integration_cluster_proxy
+  run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
+      -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
+
+  log_callout "Collecting coverage from e2e tests ..."
+  # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
+  # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
+  run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
+
+  log_callout "Collecting coverage from e2e tests with proxy ..."
+  run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
+
+  log_callout "Merging coverage results ..."
+  local cover_out_file="${coverdir}/cover.out"
+  # gocovmerge requires not-empty test to start with:
+  echo "mode: set" > "${cover_out_file}"
+
+  # incrementally merge to get coverage data even if some coverage files are corrupted
+  for f in "${coverdir}"/*.coverprofile; do
+    echo "merging test coverage file ${f}"
+    run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
+    if [ -s "${coverdir}"/cover.tmp ]; then
+      mv "${coverdir}/cover.tmp" "${cover_out_file}"
+    fi
+  done
+  # strip out generated files (using GNU-style sed)
+  sed --in-place '/generated.go/d' "${cover_out_file}" || true
+
+  # held failures to generate the full coverage file, now fail
+  if [ -n "$failed" ]; then
+    for f in $failed; do
+      log_error "--- FAIL:" "$f"
+    done
+    log_warning "Despite failures, you can see partial report:"
+    log_warning "  go tool cover -html ${cover_out_file}"
+    return 255
+  fi
+
+  log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
+}
+
+######### Code formatting checkers #############################################
+
+function fmt_pass {
+  toggle_failpoints disable
+
+  # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
+  # after resolving ore-existing errors.
+  for p in shellcheck \
+      markdown_you \
+      goword \
+      gofmt \
+      govet \
+      revive \
+      license_header \
+      receiver_name \
+      mod_tidy \
+      dep \
+      shellcheck \
+      shellws \
+      ; do
+    run_pass "${p}" "${@}"
+  done
+}
+
+function shellcheck_pass {
+  if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
+    generic_checker run shellcheck -fgcc build test scripts/*.sh ./*.sh
+  fi
+}
+
+function shellws_pass {
+  log_callout "Ensuring no tab-based indention in shell scripts"
+  local files
+  files=$(find ./ -name '*.sh' -print0 | xargs -0 )
+  log_cmd "grep -E -n $'^ *\t' ${files}"
+  # shellcheck disable=SC2086
+  if grep -E -n $'^ *\t' ${files} | sed $'s|\t|[\\\\tab]|g'; then
+    log_error "FAIL: found tab-based indention in bash scripts. Use '  ' (double space)."
+    return 1
+  else
+    log_success "SUCCESS: no tabulators found."
+    return 0
+  fi
+}
+
+function markdown_you_find_eschew_you {
+  local find_you_cmd="find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' ! -path './release/*' -exec grep -E --color '[Yy]ou[r]?[ '\''.,;]' {} + || true"
+  run eval "${find_you_cmd}"
+}
+
+function markdown_you_pass {
+  generic_checker markdown_you_find_eschew_you
+}
+
+function markdown_marker_pass {
+  # TODO: check other markdown files when marker handles headers with '[]'
+  if tool_exists "marker" "https://crates.io/crates/marker"; then
+    generic_checker run marker --skip-http --root ./Documentation 2>&1
+  fi
+}
+
+function govet_pass {
+  run_for_modules generic_checker run go vet
+}
+
+function govet_shadow_pass {
+  local shadow
+  shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow")
+  run_for_modules generic_checker run go vet -all -vettool="${shadow}"
+}
+
+function unparam_pass {
+  run_for_modules generic_checker run_go_tool "mvdan.cc/unparam"
+}
+
+function staticcheck_pass {
+  run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck"
+}
+
+function revive_pass {
+  run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
+}
+
+function unconvert_pass {
+  run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v
+}
+
+function ineffassign_per_package {
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  local gofiles=()
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}"
+}
+
+function ineffassign_pass {
+  run_for_modules generic_checker ineffassign_per_package
+}
+
+function nakedret_pass {
+  run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret"
+}
+
+function license_header_pass {
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  local gofiles=()
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  
+  for file in "${gofiles[@]}"; do
+    if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
+      licRes="${licRes}"$(echo -e "  ${file}")
+    fi
+  done
+  if [ -n "${licRes}" ]; then
+    log_error -e "license header checking failed:\\n${licRes}"
+    return 255
+  fi
+}
+
+function receiver_name_for_package {
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  local gofiles=()
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+
+  recvs=$(grep 'func ([^*]' "${gofiles[@]}"  | tr  ':' ' ' |  \
+    awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
+    grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
+  if [ -n "${recvs}" ]; then
+    # shellcheck disable=SC2206
+    recvs=($recvs)
+    for recv in "${recvs[@]}"; do
+      log_error "Mismatched receiver for $recv..."
+      grep "$recv" "${gofiles[@]}" | grep 'func ('
+    done
+    return 255
+  fi
+}
+
+function receiver_name_pass {
+  run_for_modules receiver_name_for_package
+}
+
+# goword_for_package package
+# checks spelling and comments in the 'package' in the current module
+#
+function goword_for_package {
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  local gofiles=()
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  
+  local gowordRes
+
+  # spellchecking can be enabled with GOBINARGS="--tags=spell"
+  # but it requires heavy dependencies installation, like:
+  # apt-get install libaspell-dev libhunspell-dev hunspell-en-us aspell-en
+
+  # only check for broke exported godocs
+  if gowordRes=$(run_go_tool "github.com/chzchzchz/goword" -use-spell=false "${gofiles[@]}" | grep godoc-export | sort); then
+    log_error -e "goword checking failed:\\n${gowordRes}"
+    return 255
+  fi
+  if [ -n "$gowordRes" ]; then
+    log_error -e "goword checking returned output:\\n${gowordRes}"
+    return 255
+  fi
+}
+
+
+function goword_pass {
+  run_for_modules goword_for_package || return 255
+}
+
+function go_fmt_for_package {
+  # We utilize 'go fmt' to find all files suitable for formatting,
+  # but reuse full power gofmt to perform just RO check.
+  go fmt -n "$1" | sed 's| -w | -d |g' | sh
+}
+
+function gofmt_pass {
+  run_for_modules generic_checker go_fmt_for_package
+}
+
+function bom_pass {
+  log_callout "Checking bill of materials..."
+  # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
+  # shellcheck disable=SC2207
+  modules=($(modules_exp))
+
+  # Internally license-bill-of-materials tends to modify go.sum
+  run cp go.sum go.sum.tmp || return 2
+  run cp go.mod go.mod.tmp || return 2
+
+  output=$(GOFLAGS=-mod=mod run_go_tool github.com/coreos/license-bill-of-materials \
+    --override-file ./bill-of-materials.override.json \
+    "${modules[@]}")
+  code="$?"
+
+  run cp go.sum.tmp go.sum || return 2
+  run cp go.mod.tmp go.mod || return 2
+
+  if [ "${code}" -ne 0 ] ; then
+    log_error -e "license-bill-of-materials (code: ${code}) failed with:\n${output}"
+    return 255
+  else
+    echo "${output}" > "bom-now.json.tmp"
+  fi
+  if ! diff ./bill-of-materials.json bom-now.json.tmp; then
+    log_error "modularized licenses do not match given bill of materials"
+    return 255
+  fi
+  rm bom-now.json.tmp
+}
+
+######## VARIOUS CHECKERS ######################################################
+
+function dump_deps_of_module() {
+  local module
+  if ! module=$(run go list -m); then
+    return 255
+  fi
+  run go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all
+}
+
+# Checks whether dependencies are consistent across modules
+function dep_pass {
+  local all_dependencies
+  all_dependencies=$(run_for_modules dump_deps_of_module | sort) || return 2
+
+  local duplicates
+  duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2
+
+  for dup in ${duplicates}; do
+    log_error "FAIL: inconsistent versions for depencency: ${dup}"
+    echo "${all_dependencies}" | grep "${dup}" | sed "s|\([^,]*\),\([^,]*\),\([^,]*\)|  - \1@\2 from: \3|g"
+  done
+  if [[ -n "${duplicates}" ]]; then
+    log_error "FAIL: inconsistent dependencies"
+    return 2
+  else
+    log_success "SUCCESS: dependencies are consistent across modules"
+  fi
+}
+
+function release_pass {
+  rm -f ./bin/etcd-last-release
+  # to grab latest patch release; bump this up for every minor release
+  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
+  if [ -n "$MANUAL_VER" ]; then
+    # in case, we need to test against different version
+    UPGRADE_VER=$MANUAL_VER
+  fi
+  if [[ -z ${UPGRADE_VER} ]]; then
+    UPGRADE_VER="v3.3.0"
+    log_warning "fallback to" ${UPGRADE_VER}
+  fi
+
+  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
+  log_callout "Downloading $file"
+
+  set +e
+  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
+  local result=$?
+  set -e
+  case $result in
+    0)  ;;
+    *)  log_error "--- FAIL:" ${result}
+      return $result
+      ;;
+  esac
+
+  tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
+  mkdir -p ./bin
+  mv /tmp/etcd ./bin/etcd-last-release
+}
+
+function mod_tidy_for_module {
+  # Watch for upstream solution: https://github.com/golang/go/issues/27005
+  local tmpModDir
+  tmpModDir=$(mktemp -d -t 'tmpModDir.XXXXXX')
+  run cp "./go.mod" "./go.sum" "${tmpModDir}" || return 2
+
+  # Guarantees keeping go.sum minimal
+  # If this is causing too much problems, we should
+  # stop controlling go.sum at all.
+  rm go.sum
+  run go mod tidy || return 2
+
+  set +e
+  local tmpFileGoModInSync
+  diff -C 5 "${tmpModDir}/go.mod" "./go.mod"
+  tmpFileGoModInSync="$?"
+
+  local tmpFileGoSumInSync
+  diff -C 5 "${tmpModDir}/go.sum" "./go.sum"
+  tmpFileGoSumInSync="$?"
+  set -e
+
+  # Bring back initial state
+  mv "${tmpModDir}/go.mod" "./go.mod"
+  mv "${tmpModDir}/go.sum" "./go.sum"
+
+  if [ "${tmpFileGoModInSync}" -ne 0 ]; then
+    log_error "${PWD}/go.mod is not in sync with 'go mod tidy'"
+    return 255
+  fi
+  if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
+    log_error "${PWD}/go.sum is not in sync with 'rm go.sum; go mod tidy'"
+    return 255
+  fi
+}
+
+function mod_tidy_pass {
+  run_for_modules mod_tidy_for_module
+}
+
+########### MAIN ###############################################################
+
+function run_pass {
+  local pass="${1}"
+  shift 1
+  log_callout -e "\n'${pass}' started at $(date)"
+  if "${pass}_pass" "$@" ; then
+    log_success "'${pass}' completed at $(date)"
+  else
+    log_error "FAIL: '${pass}' failed at $(date)"
+    exit 255
+  fi
+}
+
+for pass in $PASSES; do
+  run_pass "${pass}" "${@}"
+done
+
+log_success "SUCCESS"