diff --git a/.gitignore b/.gitignore
index 2d2ade487..a83fc002c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,5 +17,6 @@ hack/tls-setup/certs
 /contrib/raftexample/raftexample
 /contrib/raftexample/raftexample-*
 /vendor
+/tests/e2e/default.proxy
 
 *.bak
diff --git a/.travis.yml b/.travis.yml
index db63faf3f..610e143d0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -78,19 +78,22 @@ script:
           /bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
         ;;
       linux-amd64-integration-1-cpu)
+        # TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' RACE='false' ./test"
         ;;
       linux-amd64-integration-2-cpu)
+        # TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' RACE='false' ./test"
         ;;
       linux-amd64-integration-4-cpu)
+        # TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test"
         ;;
       linux-amd64-functional)
         docker run --rm \
@@ -100,7 +103,7 @@ script:
       linux-amd64-unit)
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test"
+          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test -p=2"
         ;;
       all-build)
         docker run --rm \
@@ -115,17 +118,18 @@ script:
             && GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=s390x ./build"
         ;;
       linux-amd64-grpcproxy)
-        sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test
+        # TODO: Reenable race when https://github.com/etcd-io/etcd/issues/12336 fixed.
+        sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy' VERBOSE='1' CPU='4' COVER='false' RACE='false'" make docker-test
         ;;
       linux-amd64-coverage)
-        sudo HOST_TMP_DIR=/tmp make docker-test-coverage
+        sudo HOST_TMP_DIR=/tmp TEST_OPTS="VERBOSE='1'" make docker-test-coverage
         ;;
       linux-amd64-fmt-unit-go-tip)
-        GOARCH=amd64 PASSES='fmt unit' ./test
+        GOARCH=amd64 PASSES='fmt unit' ./test -p=2
         ;;
       linux-386-unit)
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=386 PASSES='unit' ./test"
+          /bin/bash -c "GOARCH=386 PASSES='unit' ./test -p=2"
         ;;
     esac
diff --git a/Makefile b/Makefile
index 0d2211817..7169994f5 100644
--- a/Makefile
+++ b/Makefile
@@ -18,18 +18,15 @@ build:
 
 clean:
 	rm -f ./codecov
-	rm -rf ./agent-*
 	rm -rf ./covdir
-	rm -f ./*.coverprofile
-	rm -f ./*.log
 	rm -f ./bin/Dockerfile-release
 	rm -rf ./bin/etcd*
 	rm -rf ./default.etcd
 	rm -rf ./tests/e2e/default.etcd
 	rm -rf ./release
-	rm -f ./snapshot/localhost:*
-	rm -f ./tools/etcd-dump-metrics/localhost:*
-	find ./ -name "127.0.0.1:*" -o -name "localhost:*" -delete
+	rm -rf ./coverage/*.err ./coverage/*.out
+	rm -rf ./tests/e2e/default.proxy
+	find ./ -name "127.0.0.1:*" -o -name "localhost:*" -o -name "*.log" -o -name "agent-*" -o -name "*.coverprofile" -o -name "testname-proxy-*" | xargs -r rm -r
 
 docker-clean:
 	docker images
diff --git a/clientv3/integration/ordering_kv_test.go b/clientv3/integration/ordering_kv_test.go
new file mode 100644
index 000000000..f91c9382c
--- /dev/null
+++ b/clientv3/integration/ordering_kv_test.go
@@ -0,0 +1,166 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/v3/clientv3/ordering"
+	"go.etcd.io/etcd/v3/integration"
+	"go.etcd.io/etcd/v3/pkg/testutil"
+)
+
+func TestDetectKvOrderViolation(t *testing.T) {
+	var errOrderViolation = errors.New("Detected Order Violation")
+
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	cfg := clientv3.Config{
+		Endpoints: []string{
+			clus.Members[0].GRPCAddr(),
+			clus.Members[1].GRPCAddr(),
+			clus.Members[2].GRPCAddr(),
+		},
+	}
+	cli, err := clientv3.New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+	ctx := context.TODO()
+
+	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
+		t.Fatal(err)
+	}
+	// ensure that the second member has the current revision for the key foo
+	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
+		t.Fatal(err)
+	}
+
+	// stop third member in order to force the member to have an outdated revision
+	clus.Members[2].Stop(t)
+	time.Sleep(1 * time.Second) // give enough time for operation
+	_, err = cli.Put(ctx, "foo", "buzz")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// perform get request against the first member, in order to
+	// set up kvOrdering to expect "foo" revisions greater than that of
+	// the third member.
+	orderingKv := ordering.NewKV(cli.KV,
+		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
+			return errOrderViolation
+		})
+	_, err = orderingKv.Get(ctx, "foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that only the third member is queried during requests
+	clus.Members[0].Stop(t)
+	clus.Members[1].Stop(t)
+	clus.Members[2].Restart(t)
+	// force OrderingKv to query the third member
+	cli.SetEndpoints(clus.Members[2].GRPCAddr())
+	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
+
+	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
+	if err != errOrderViolation {
+		t.Fatalf("expected %v, got %v", errOrderViolation, err)
+	}
+}
+
+func TestDetectTxnOrderViolation(t *testing.T) {
+	var errOrderViolation = errors.New("Detected Order Violation")
+
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	cfg := clientv3.Config{
+		Endpoints: []string{
+			clus.Members[0].GRPCAddr(),
+			clus.Members[1].GRPCAddr(),
+			clus.Members[2].GRPCAddr(),
+		},
+	}
+	cli, err := clientv3.New(cfg)
+	defer cli.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+	ctx := context.TODO()
+
+	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
+		t.Fatal(err)
+	}
+	// ensure that the second member has the current revision for the key foo
+	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
+		t.Fatal(err)
+	}
+
+	// stop third member in order to force the member to have an outdated revision
+	clus.Members[2].Stop(t)
+	time.Sleep(1 * time.Second) // give enough time for operation
+	if _, err = clus.Client(1).Put(ctx, "foo", "buzz"); err != nil {
+		t.Fatal(err)
+	}
+
+	// perform get request against the first member, in order to
+	// set up kvOrdering to expect "foo" revisions greater than that of
+	// the third member.
+	orderingKv := ordering.NewKV(cli.KV,
+		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
+			return errOrderViolation
+		})
+	orderingTxn := orderingKv.Txn(ctx)
+	_, err = orderingTxn.If(
+		clientv3.Compare(clientv3.Value("b"), ">", "a"),
+	).Then(
+		clientv3.OpGet("foo"),
+	).Commit()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that only the third member is queried during requests
+	clus.Members[0].Stop(t)
+	clus.Members[1].Stop(t)
+	clus.Members[2].Restart(t)
+	// force OrderingKv to query the third member
+	cli.SetEndpoints(clus.Members[2].GRPCAddr())
+	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
+	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
+	if err != errOrderViolation {
+		t.Fatalf("expected %v, got %v", errOrderViolation, err)
+	}
+	orderingTxn = orderingKv.Txn(ctx)
+	_, err = orderingTxn.If(
+		clientv3.Compare(clientv3.Value("b"), ">", "a"),
+	).Then(
+		clientv3.OpGet("foo", clientv3.WithSerializable()),
+	).Commit()
+	if err != errOrderViolation {
+		t.Fatalf("expected %v, got %v", errOrderViolation, err)
+	}
+}
diff --git a/clientv3/ordering/kv_test.go b/clientv3/ordering/kv_test.go
index e10db9693..6dc1989e4 100644
--- a/clientv3/ordering/kv_test.go
+++ b/clientv3/ordering/kv_test.go
@@ -17,153 +17,13 @@ package ordering
 import (
 	"context"
 	gContext "context"
-	"errors"
 	"sync"
 	"testing"
-	"time"
 
 	"go.etcd.io/etcd/v3/clientv3"
 	pb "go.etcd.io/etcd/v3/etcdserver/etcdserverpb"
-	"go.etcd.io/etcd/v3/integration"
-	"go.etcd.io/etcd/v3/pkg/testutil"
 )
 
-func TestDetectKvOrderViolation(t *testing.T) {
-	var errOrderViolation = errors.New("Detected Order Violation")
-
-	defer testutil.AfterTest(t)
-	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
-	defer clus.Terminate(t)
-
-	cfg := clientv3.Config{
-		Endpoints: []string{
-			clus.Members[0].GRPCAddr(),
-			clus.Members[1].GRPCAddr(),
-			clus.Members[2].GRPCAddr(),
-		},
-	}
-	cli, err := clientv3.New(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx := context.TODO()
-
-	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
-		t.Fatal(err)
-	}
-	// ensure that the second member has the current revision for the key foo
-	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
-		t.Fatal(err)
-	}
-
-	// stop third member in order to force the member to have an outdated revision
-	clus.Members[2].Stop(t)
-	time.Sleep(1 * time.Second) // give enough time for operation
-	_, err = cli.Put(ctx, "foo", "buzz")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// perform get request against the first member, in order to
-	// set up kvOrdering to expect "foo" revisions greater than that of
-	// the third member.
-	orderingKv := NewKV(cli.KV,
-		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
-			return errOrderViolation
-		})
-	_, err = orderingKv.Get(ctx, "foo")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// ensure that only the third member is queried during requests
-	clus.Members[0].Stop(t)
-	clus.Members[1].Stop(t)
-	clus.Members[2].Restart(t)
-	// force OrderingKv to query the third member
-	cli.SetEndpoints(clus.Members[2].GRPCAddr())
-	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
-
-	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
-	if err != errOrderViolation {
-		t.Fatalf("expected %v, got %v", errOrderViolation, err)
-	}
-}
-
-func TestDetectTxnOrderViolation(t *testing.T) {
-	var errOrderViolation = errors.New("Detected Order Violation")
-
-	defer testutil.AfterTest(t)
-	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
-	defer clus.Terminate(t)
-
-	cfg := clientv3.Config{
-		Endpoints: []string{
-			clus.Members[0].GRPCAddr(),
-			clus.Members[1].GRPCAddr(),
-			clus.Members[2].GRPCAddr(),
-		},
-	}
-	cli, err := clientv3.New(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx := context.TODO()
-
-	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
-		t.Fatal(err)
-	}
-	// ensure that the second member has the current revision for the key foo
-	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
-		t.Fatal(err)
-	}
-
-	// stop third member in order to force the member to have an outdated revision
-	clus.Members[2].Stop(t)
-	time.Sleep(1 * time.Second) // give enough time for operation
-	if _, err = clus.Client(1).Put(ctx, "foo", "buzz"); err != nil {
-		t.Fatal(err)
-	}
-
-	// perform get request against the first member, in order to
-	// set up kvOrdering to expect "foo" revisions greater than that of
-	// the third member.
-	orderingKv := NewKV(cli.KV,
-		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
-			return errOrderViolation
-		})
-	orderingTxn := orderingKv.Txn(ctx)
-	_, err = orderingTxn.If(
-		clientv3.Compare(clientv3.Value("b"), ">", "a"),
-	).Then(
-		clientv3.OpGet("foo"),
-	).Commit()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// ensure that only the third member is queried during requests
-	clus.Members[0].Stop(t)
-	clus.Members[1].Stop(t)
-	clus.Members[2].Restart(t)
-	// force OrderingKv to query the third member
-	cli.SetEndpoints(clus.Members[2].GRPCAddr())
-	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
-	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
-	if err != errOrderViolation {
-		t.Fatalf("expected %v, got %v", errOrderViolation, err)
-	}
-	orderingTxn = orderingKv.Txn(ctx)
-	_, err = orderingTxn.If(
-		clientv3.Compare(clientv3.Value("b"), ">", "a"),
-	).Then(
-		clientv3.OpGet("foo", clientv3.WithSerializable()),
-	).Commit()
-	if err != errOrderViolation {
-		t.Fatalf("expected %v, got %v", errOrderViolation, err)
-	}
-}
-
 type mockKV struct {
 	clientv3.KV
 	response clientv3.OpResponse
diff --git a/clientv3/snapshot/v3_snapshot_test.go b/clientv3/snapshot/v3_snapshot_test.go
index 8ffc10ea1..3b6c3b096 100644
--- a/clientv3/snapshot/v3_snapshot_test.go
+++ b/clientv3/snapshot/v3_snapshot_test.go
@@ -33,8 +33,6 @@ import (
 	"go.uber.org/zap"
 )
 
-// TODO(ptabor): This is integration test. Skip it in --short and move to integration tests directory.
-
 // TestSnapshotV3RestoreSingle tests single node cluster restoring
 // from a snapshot file.
 func TestSnapshotV3RestoreSingle(t *testing.T) {
@@ -194,6 +192,8 @@ type kv struct {
 
 // creates a snapshot file and returns the file path.
 func createSnapshotFile(t *testing.T, kvs []kv) string {
+	testutil.SkipTestIfShortMode(t,
+		"Snapshot creation tests are depending on embedded etcServer so are integration-level tests.")
 	clusterN := 1
 	urls := newEmbedURLs(clusterN * 2)
 	cURLs, pURLs := urls[:clusterN], urls[clusterN:]
diff --git a/go.sum b/go.sum
index fbbfa0ede..7028b6bd9 100644
--- a/go.sum
+++ b/go.sum
@@ -183,7 +183,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8=
 golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/integration/cluster.go b/integration/cluster.go
index 44bbe8ba8..45d356144 100644
--- a/integration/cluster.go
+++ b/integration/cluster.go
@@ -1250,6 +1250,7 @@ type ClusterV3 struct {
 func NewClusterV3(t testing.TB, cfg *ClusterConfig) *ClusterV3 {
 	if t != nil {
 		t.Helper()
+		testutil.SkipTestIfShortMode(t, "Cannot create clusters in --short tests")
 	}
 
 	cfg.UseGRPC = true
diff --git a/integration/embed/embed_proxy_test.go b/integration/embed/embed_proxy_test.go
new file mode 100644
index 000000000..8bff9662e
--- /dev/null
+++ b/integration/embed/embed_proxy_test.go
@@ -0,0 +1,19 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build cluster_proxy
+
+// The purpose of this (empty) package is too keep following test working:
+// #  go test -tags=cluster_proxy ./integration/embed
+package embed_test
diff --git a/pkg/testutil/leak.go b/pkg/testutil/leak.go
index 77fce27e7..546ab05da 100644
--- a/pkg/testutil/leak.go
+++ b/pkg/testutil/leak.go
@@ -131,6 +131,7 @@ func interestingGoroutines() (gs []string) {
 			strings.Contains(stack, "go.etcd.io/etcd/v3/pkg/logutil.(*MergeLogger).outputLoop") ||
 			strings.Contains(stack, "github.com/golang/glog.(*loggingT).flushDaemon") ||
 			strings.Contains(stack, "created by runtime.gc") ||
+			strings.Contains(stack, "created by text/template/parse.lex") ||
 			strings.Contains(stack, "runtime.MHeap_Scavenger") {
 			continue
 		}
diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh
new file mode 100644
index 000000000..6207c9c8e
--- /dev/null
+++ b/scripts/test_lib.sh
@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+
+ETCD_ROOT_DIR="$(pwd)"
+
+####   Convenient IO methods #####
+
+COLOR_RED='\033[0;31m'
+COLOR_ORANGE='\033[0;33m'
+COLOR_GREEN='\033[0;32m'
+COLOR_LIGHTCYAN='\033[0;36m'
+
+COLOR_NONE='\033[0m' # No Color
+
+function log_error {
+  >&2 echo -n -e "${COLOR_RED}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+function log_warning {
+  >&2 echo -n -e "${COLOR_ORANGE}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+function log_callout {
+  >&2 echo -n -e "${COLOR_LIGHTCYAN}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+function log_success {
+  >&2 echo -n -e "${COLOR_GREEN}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+
+####   Discovery of files/packages within a go module #####
+
+# go_srcs_in_module [package]
+# returns list of all not-generated go sources in the current (dir) module.
+function go_srcs_in_module {
+  go fmt -n "$1"  | grep -Eo "([^ ]*)$" | grep -vE "(\\_test.go|\\.pb\\.go|\\.pb\\.gw.go)"
+}
+
+# pkgs_in_module [optional:package_pattern]
+# returns list of all packages in the current (dir) module.
+# if the package_pattern is given, its being resolved.
+function pkgs_in_module {
+  go list -mod=mod "${1:-./...}";
+}
+
+function filter_out_integration_style_tests {
+  grep -Ev '/(tests/e2e|integration|functional)(/|$)'
+}
+
+####    Running actions against multiple modules ####
+
+# run [command...] - runs given command, printing it first and
+# again if it failed (in RED). Use to wrap important test commands
+# that user might want to re-execute to shorten the feedback loop when fixing
+# the test.
+function run {
+  local rpath
+  rpath=$(realpath "--relative-to=${ETCD_ROOT_DIR}" "${PWD}")
+  local repro="$*"
+  if [ "${rpath}" != "." ]; then
+    repro="(cd ${rpath} && ${repro})"
+  fi
+
+  log_callout "% ${repro}"
+  "${@}"
+  local error_code=$?
+  if [ ${error_code} != 0 ]; then
+    log_error -e "FAIL: (code:${error_code}):\n  % ${repro}"
+    return ${error_code}
+  fi
+}
+
+# run_for_module [module] [cmd]
+# executes given command in the given module for given pkgs.
+#   module_name - "." (in future: tests, client, server)
+#   cmd         - cmd to be executed - that takes package as last argument
+function run_for_module {
+  local module=${1:-"."}
+  shift 1
+  (
+    cd "${module}" && "$@"
+  )
+}
+
+#  run_for_modules [cmd]
+#  run given command across all modules and packages
+#  (unless the set is limited using ${PKG} or / ${USERMOD})
+function run_for_modules {
+  local pkg="${PKG:-./...}"
+  if [ -z "${USERMOD}" ]; then
+    run_for_module "." "$@" "${pkg}" || return "$?"
+  else
+    run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?"
+  fi
+}
+
+
+####    Running go test  ########
+
+# go_test [packages] [mode] [flags_for_package_func] [$@]
+# [mode] supports 3 states:
+#   - "parallel": fastest as concurrently processes multiple packages, but silent
+#                 till the last package. See: https://github.com/golang/go/issues/2731
+#   - "keep_going" : executes tests package by package, but postpones reporting error to the last
+#   - "fail_fast"  : executes tests packages 1 by 1, exits on the first failure.
+#
+# [flags_for_package_func] is a name of function that takes list of packages as parameter
+#   and computes additional flags to the go_test commands.
+#   Use 'true' or ':' if you dont need additional arguments.
+#
+#  depends on the VERBOSE top-level variable.
+#
+#  Example:
+#    go_test "./..." "keep_going" ":" --short
+#
+#  The function returns != 0 code in case of test failure.
+function go_test {
+  local packages="${1}"
+  local mode="${2}"
+  local flags_for_package_func="${3}"
+
+  shift 3
+
+  local goTestFlags=""
+  local goTestEnv=""
+  if [ "${VERBOSE}" == "1" ]; then
+    goTestFlags="-v"
+  fi
+  if [ "${VERBOSE}" == "2" ]; then
+    goTestFlags="-v"
+    goTestEnv="CLIENT_DEBUG=1"
+  fi
+
+  # Expanding patterns (like ./...) into list of packages
+
+  local unpacked_packages=("${packages}")
+  if [ "${mode}" != "parallel" ]; then
+    # shellcheck disable=SC2207
+    # shellcheck disable=SC2086
+    if ! unpacked_packages=($(go list ${packages})); then
+      log_error "Cannot resolve packages: ${packages}"
+      return 255
+    fi
+  fi
+
+  local failures=""
+
+  # execution of tests against packages:
+  for pkg in "${unpacked_packages[@]}"; do
+    local additional_flags
+    # shellcheck disable=SC2086
+    additional_flags=$(${flags_for_package_func} ${pkg})
+
+    # shellcheck disable=SC2206
+    local cmd=( go test ${goTestFlags} ${additional_flags} "$@" ${pkg} )
+
+    if ! run env ${goTestEnv} "${cmd[@]}" ; then
+      if [ "${mode}" != "keep_going" ]; then
+        return 2
+      else
+        failures=("${failures[@]}" "${pkg}")
+      fi
+    fi
+    echo
+  done
+
+  if [ -n "${failures[*]}" ] ; then
+    log_error -e "ERROR: Tests for following packages failed:\n  ${failures[*]}"
+    return 2
+  fi
+}
+
+#### Other ####
+
+# tool_exists [tool] [instruction]
+# Checks whether given [tool] is installed. In case of failure,
+# prints a warning with installation [instruction] and returns !=0 code.
+function tool_exists {
+  local tool="${1}"
+  local instruction="${2}"
+  if ! command -v "${tool}" >/dev/null; then
+    log_warning "Tool: '${tool}' not found on PATH. ${instruction}"
+    return 255
+  fi
+}
+
diff --git a/test b/test
index ae3ca76eb..b5e1675b2 100755
--- a/test
+++ b/test
@@ -16,8 +16,7 @@
 # flag for different expectation
 #
 # $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
-# $ PASSES=integration PKG=client/integration TIMEOUT=1m ./test
-#
+# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
 #
 # Run specified unit tests in one package
 # To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
@@ -25,7 +24,7 @@
 #
 # $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
 # $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
-# $ PASSES=integration PKG=client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
+# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
 #
 #
 # Run code coverage
@@ -39,660 +38,519 @@ set -e
 export GOFLAGS=-mod=readonly
 
 source ./build
+source ./scripts/test_lib.sh
 
-# build before setting up test GOPATH
-if [[ "${PASSES}" == *"functional"* ]]; then
-	./functional/build
-fi
-
-if [ -z "$PASSES" ]; then
-	PASSES="fmt bom dep build unit"
-fi
-
-USERPKG=${PKG:-}
+PASSES=${PASSES:-"fmt bom dep build unit"}
+PKG=${PKG:-}
 
 # Invoke ./tests/cover.test.bash for HTML output
-COVER=${COVER:-"-cover"}
-
-# Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
-IGNORE_PKGS="(vendor/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
-INTEGRATION_PKGS="(integration|tests/e2e|contrib|functional)"
-
-# all github.com/etcd-io/etcd/whatever pkgs that are not auto-generated / tools
-# shellcheck disable=SC1117
-PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -vE "(tools/|contrib/|tests/e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo)
-# pkg1,pkg2,pkg3
-PKGS_COMMA=${PKGS// /,}
-
-# shellcheck disable=SC1117
-TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
-
-# shellcheck disable=SC1117
-FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
-
-TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | grep -vE "$INTEGRATION_PKGS")
-
-# check if user provided PKG override
-if [ -z "${USERPKG}" ]; then
-	TEST=$TESTABLE_AND_FORMATTABLE
-	FMT=$FORMATTABLE
-else
-	# strip out leading dotslashes and trailing slashes from PKG=./foo/
-	TEST=${USERPKG/#./}
-	TEST=${TEST/#\//}
-	TEST=${TEST/%\//}
-	# only run gofmt on packages provided by user
-	FMT="$TEST"
-fi
-
-# shellcheck disable=SC2206
-FMT=($FMT)
-if [ "${VERBOSE}" == "1" ]; then
-	# shellcheck disable=SC2128
-	echo "Running with FMT:" "${FMT[@]}"
-fi
-
-# prepend REPO_PATH to each local package
-split=$TEST
-TEST=""
-for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done
-
-# shellcheck disable=SC2206
-TEST=($TEST)
-if [ "${VERBOSE}" == "1" ]; then
-	# shellcheck disable=SC2128
-	echo "Running with TEST:" "${TEST[@]}"
-fi
-
-# TODO: 'rafttest' is failing with unused
-STATIC_ANALYSIS_PATHS=$(find . -name \*.go ! -path './vendor/*' ! -path './gopath.proto/*' ! -path '*pb/*' | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS")
-# shellcheck disable=SC2206
-STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS)
-if [ "${VERBOSE}" == "1" ]; then
-	# shellcheck disable=SC2128
-	echo "Running with STATIC_ANALYSIS_PATHS:" "${STATIC_ANALYSIS_PATHS[@]}"
-fi
+COVER="--cover=${COVER:-true}"
 
 if [ -z "$GOARCH" ]; then
-	GOARCH=$(go env GOARCH);
+  GOARCH=$(go env GOARCH);
 fi
 
 # determine the number of CPUs to use for Go tests
-TEST_CPUS="1,2,4"
-if [ -n "${CPU}" ]; then
-	TEST_CPUS="${CPU}"
-fi
-echo "Running with TEST_CPUS:" "${TEST_CPUS}"
+CPU=${CPU:-"1,2,4"}
+echo "Running with CPU:" "${CPU}"
 
 # determine whether target supports race detection
-if [ "$GOARCH" == "amd64" ]; then
-	RACE="--race"
+if [ -z "${RACE}" ] ; then
+  if [ "$GOARCH" == "amd64" ]; then
+    RACE="--race"
+  else
+    RACE="--race=false"
+  fi
+else
+  RACE="--race=${RACE:-true}"
 fi
 
-RUN_ARG=""
+# This options make sense for cases where SUT (System Under Test) is compiled by test.
+COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}" "${COVER}")
+
+RUN_ARG=()
 if [ -n "${TESTCASE}" ]; then
-	RUN_ARG="-run=${TESTCASE}"
+  RUN_ARG=("-run=${TESTCASE}")
 fi
 
-function unit_pass {
-	echo "Running unit tests..."
-	GO_TEST_FLAG=""
-	if [ "${VERBOSE}" == "1" ]; then
-		GO_TEST_FLAG="-v"
-	fi
-	if [ "${VERBOSE}" == "2" ]; then
-		GO_TEST_FLAG="-v"
-		export CLIENT_DEBUG=1
-	fi
-
-	if [ "${RUN_ARG}" == "" ]; then
-	    RUN_ARG="-run=Test"
-	fi
-
-	# check if user provided time out, especially useful when just run one test case
-	# expectation could be different
-	USERTIMEOUT=""
-	if [ -z "${TIMEOUT}" ]; then
-		USERTIMEOUT="3m"
-	else
-		USERTIMEOUT="${TIMEOUT}"
-	fi
-	go test ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}"  "${COVER}" ${RACE} -cpu "${TEST_CPUS}" ${RUN_ARG} "$@" "${TEST[@]}"
+function build_pass {
+  log_callout "Building etcd"
+  GO_BUILD_FLAGS="-v" etcd_build "${@}"
+  GO_BUILD_FLAGS="-v" tools_build "${@}"
 }
 
-function integration_pass {
-	echo "Running integration tests..."
+################# REGULAR TESTS ################################################
 
-	# check if user provided time out, especially useful when just run one test case
-	# expectation could be different
-	USERTIMEOUT=""
-	if [ -z "${TIMEOUT}" ]; then
-		USERTIMEOUT="30m"
-	else
-		USERTIMEOUT="${TIMEOUT}"
-	fi
+# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
+function run_unit_tests {
+  local pkgs="${1:-./...}"
+  shift 1
+  # shellcheck disable=SC2086
+  go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
+}
 
-	# if TESTCASE and PKG set, run specified test case in specified PKG
-	# if TESTCASE set, PKG not set, run specified test case in all integration and integration_extra packages
-	# if TESTCASE not set, PKG set, run all test cases in specified package
-	# if TESTCASE not set, PKG not set, run all tests in all integration and integration_extra packages
-	if [ -z "${TESTCASE}" ] && [ -z "${USERPKG}" ]; then
-		go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
-		integration_extra "$@"
-	else
-		if [ -z "${USERPKG}" ]; then
-			INTEGTESTPKG=("${REPO_PATH}/integration"
-						  "${REPO_PATH}/client/integration"
-						  "${REPO_PATH}/clientv3/integration"
-						  "${REPO_PATH}/contrib/raftexample")
-		else
-			INTEGTESTPKG=("${TEST[@]}")
-		fi
-		go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}"  "$@" "${INTEGTESTPKG[@]}"
-	fi
+function unit_pass {
+  run_for_modules run_unit_tests "$@"
 }
 
 function integration_extra {
-	go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/client/integration"
-	go test -timeout 25m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
-	go test -timeout 1m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/contrib/raftexample"
-	go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/etcdserver/api/v2store"
-	go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" -run=Example "$@" "${TEST[@]}"
+  if [ -z "${PKG}" ] ; then
+    if [[ -z "${RUN_ARG[*]}" ]]; then
+      go_test "./client/... ./clientv3/..." "parallel" : -timeout="${TIMEOUT:-5m}" "${COMMON_TEST_FLAGS[@]}" --run=Example "$@" || return $?
+    fi
+
+    go_test "./etcdserver/api/v2store/..." "parallel" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
+  else
+    log_warning "integration_extra ignored when PKG is specified"
+  fi
 }
 
-function functional_pass {
-  	# Clean up any data and logs from previous runs
-  	rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
-
-	for a in 1 2 3; do
-		./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 &
-		pid="$!"
-		agent_pids="${agent_pids} $pid"
-	done
-
-	for a in 1 2 3; do
-		echo "Waiting for 'etcd-agent' on ${a}9027..."
-		while ! nc -z localhost ${a}9027; do
-			sleep 1
-		done
-	done
-
-	echo "functional test START!"
-	./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded"
-	ETCD_TESTER_EXIT_CODE=$?
-	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
-
-	# shellcheck disable=SC2206
-	agent_pids=($agent_pids)
-	kill -s TERM "${agent_pids[@]}" || true
-
-	if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
-		printf "\n"
-		echo "FAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
-		tail -1000 /tmp/etcd-functional-1/etcd.log
-
-		printf "\n"
-		echo "FAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
-		tail -1000 /tmp/etcd-functional-2/etcd.log
-
-		printf "\n"
-		echo "FAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
-		tail -1000 /tmp/etcd-functional-3/etcd.log
-
-		echo "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
-		exit ${ETCD_TESTER_EXIT_CODE}
-	fi
-	echo "functional test PASS!"
-}
-
-function cov_pass {
-	echo "Running code coverage..."
-	# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
-	# gocovmerge merges coverage files
-	if ! command -v gocovmerge >/dev/null; then
-		echo "gocovmerge not installed"
-		exit 255
-	fi
-
-	if [ -z "$COVERDIR" ]; then
-		echo "COVERDIR undeclared"
-		exit 255
-	fi
-
-	if [ ! -f "bin/etcd_test" ]; then
-		echo "etcd_test binary not found"
-		exit 255
-	fi
-
-	mkdir -p "$COVERDIR"
-
-	# run code coverage for unit and integration tests
-	GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 30m"
-	# shellcheck disable=SC2206
-	GOCOVFLAGS=($GOCOVFLAGS)
-	failed=""
-	for t in $(echo "${TEST_PKGS}" | grep -vE "(tests/e2e|functional)"); do
-		tf=$(echo "$t" | tr / _)
-		# cache package compilation data for faster repeated builds
-		go test "${GOCOVFLAGS[@]}" -i "${REPO_PATH}/$t" || true
-		# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
-		go test "${GOCOVFLAGS[@]}" -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile"  "${REPO_PATH}/$t" || failed="$failed $t"
-	done
-
-	# v2v3 tests
-	go test -tags v2v3 "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/store-v2v3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed store-v2v3"
-
-	# proxy tests
-	go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_integration.coverprofile" "${REPO_PATH}/integration" || failed="$failed proxy-integration"
-	go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed proxy-clientv3/integration"
-
-	# run code coverage for e2e tests
-	# use 30m timeout because e2e coverage takes longer
-	# due to many tests cause etcd process to wait
-	# on leadership transfer timeout during gracefully shutdown
-	echo Testing tests/e2e without proxy...
-	go test -tags cov -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e"
-	echo Testing tests/e2e with proxy...
-	go test -tags "cov cluster_proxy" -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e-proxy"
-
-	# incrementally merge to get coverage data even if some coverage files are corrupted
-	# optimistically assume etcdserver package's coverage file is OK since gocovmerge
-	# expects to start with a non-empty file
-	cp "$COVERDIR"/etcdserver.coverprofile "$COVERDIR"/cover.out
-	for f in "$COVERDIR"/*.coverprofile; do
-		echo "merging test coverage file ${f}"
-		gocovmerge "$f" "$COVERDIR"/cover.out  >"$COVERDIR"/cover.tmp || failed="$failed $f"
-		if [ -s "$COVERDIR"/cover.tmp ]; then
-			mv "$COVERDIR"/cover.tmp "$COVERDIR"/cover.out
-		fi
-	done
-	# strip out generated files (using GNU-style sed)
-	sed --in-place '/generated.go/d' "$COVERDIR"/cover.out || true
-
-	echo -e "\nTo generate coverage report use:"
-	echo -e "  go tool cover --html=$COVERDIR/cover.out\n"
-
-	# held failures to generate the full coverage file, now fail
-	if [ -n "$failed" ]; then
-		for f in $failed; do
-			echo "--- FAIL:" "$f"
-		done
-		exit 255
-	fi
+function integration_pass {
+  local pkgs=${USERPKG:-"./integration/... ./clientv3/snapshot/... ./client/integration/... ./clientv3/integration/... ./contrib/raftexample"}
+  go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
+  integration_extra "$@"
 }
 
 function e2e_pass {
-	echo "Running e2e tests..."
-
-	# check if user provided time out, especially useful when just run one test case
-	# expectation could be different
-	USERTIMEOUT=""
-	if [ -z "${TIMEOUT}" ]; then
-		USERTIMEOUT="30m"
-	else
-		USERTIMEOUT="${TIMEOUT}"
-	fi
-
-	go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}"  "$@" "${REPO_PATH}/tests/e2e"
+  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
+  run_for_module "." go_test "./tests/e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
 }
 
 function integration_e2e_pass {
-	echo "Running integration and e2e tests..."
+  run_pass "integration" "${@}"
+  run_pass "e2e" "${@}"
+}
 
-	go test -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/tests/e2e" &
-	e2epid="$!"
-	go test -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" &
-	intpid="$!"
-	wait $e2epid
-	wait $intpid
-	integration_extra "$@"
+# generic_checker [cmd...]
+# executes given command in the current module, and clearly fails if it
+# failed or returned output.
+function generic_checker {
+  local cmd=("$@")
+  if ! output=$("${cmd[@]}"); then
+    echo "${output}"
+    log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
+    return 255
+  fi
+  if [ -n "${output}" ]; then
+    echo "${output}"
+    log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
+    return 255
+  fi
+}
+
+function functional_pass {
+  ./functional/build
+
+  # Clean up any data and logs from previous runs
+  rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
+
+  for a in 1 2 3; do
+    ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 &
+    pid="$!"
+    agent_pids="${agent_pids} $pid"
+  done
+
+  for a in 1 2 3; do
+    echo "Waiting for 'etcd-agent' on ${a}9027..."
+    while ! nc -z localhost ${a}9027; do
+      sleep 1
+    done
+  done
+
+  echo "functional test START!"
+  run ./bin/etcd-tester --config ./functional.yaml && log_success "'etcd-tester' succeeded"
+  ETCD_TESTER_EXIT_CODE=$?
+  echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
+
+  # shellcheck disable=SC2206
+  agent_pids=($agent_pids)
+  kill -s TERM "${agent_pids[@]}" || true
+
+  if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
+    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
+    tail -1000 /tmp/etcd-functional-1/etcd.log
+
+    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
+    tail -1000 /tmp/etcd-functional-2/etcd.log
+
+    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
+    tail -1000 /tmp/etcd-functional-3/etcd.log
+
+    log_error "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
+    return ${ETCD_TESTER_EXIT_CODE}
+  fi
+  log_success "functional test PASS!"
 }
 
 function grpcproxy_pass {
-	go test -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
-	go test -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
-	go test -timeout 30m -v -tags cluster_proxy "$@" "${REPO_PATH}/tests/e2e"
+  go_test "./integration ./clientv3/integration ./tests/e2e" "fail_fast" : \
+      -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
 }
 
-function release_pass {
-	rm -f ./bin/etcd-last-release
-	# to grab latest patch release; bump this up for every minor release
-	UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
-	if [ -n "$MANUAL_VER" ]; then
-		# in case, we need to test against different version
-		UPGRADE_VER=$MANUAL_VER
-	fi
-	if [[ -z ${UPGRADE_VER} ]]; then
-		UPGRADE_VER="v3.3.0"
-		echo "fallback to" ${UPGRADE_VER}
-	fi
+################# COVERAGE #####################################################
 
-	local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
-	echo "Downloading $file"
+# Builds artifacts used by tests/e2e in coverage mode.
+function build_cov_pass {
+  local out="${BINDIR:-./bin}"
+  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test"
+  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl"
+}
 
-	set +e
-	curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
-	local result=$?
-	set -e
-	case $result in
-		0)	;;
-		*)	echo "--- FAIL:" ${result}
-			exit $result
-			;;
-	esac
+# pkg_to_coverflag [prefix] [pkgs]
+# produces name of .coverprofile file to be used for tests of this package
+function pkg_to_coverprofileflag {
+  local prefix="${1}"
+  local pkgs="${2}"
+  local pkgs_normalized
+  pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
+  echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
+}
 
-	tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
-	mkdir -p ./bin
-	mv /tmp/etcd ./bin/etcd-last-release
+function cov_pass {
+  # gocovmerge merges coverage files
+  if ! command -v gocovmerge >/dev/null; then
+    log_error "gocovmerge not installed (need: ./scripts/install_tool.sh github.com/gyuho/gocovmerge)"
+    return 255
+  fi
+
+  # shellcheck disable=SC2153
+  if [ -z "$COVERDIR" ]; then
+    log_error "COVERDIR undeclared"
+    return 255
+  fi
+
+  if [ ! -f "bin/etcd_test" ]; then
+    log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
+    return 255
+  fi
+
+  local coverdir
+  coverdir=$(readlink -f "${COVERDIR}")
+  mkdir -p "${coverdir}"
+  rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"
+
+  local covpkgs
+  covpkgs=$(pkgs_in_module "./..." | filter_out_integration_style_tests)
+  local coverpkg_comma
+  coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
+  local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")
+
+  local failed=""
+
+  log_callout "Collecting coverage from unit tests ..."
+  go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
+      "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
+
+  log_callout "Collecting coverage from integration tests ..."
+  go_test "./integration ./clientv3/integration ./client/integration" "keep_going" "pkg_to_coverprofileflag integration" \
+      -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
+  # integration-extra
+  go_test "./client/... ./clientv3/..." "keep_going" "pkg_to_coverprofileflag integration_example" \
+      -timeout=5m --run=Example "${gocov_build_flags[@]}" "$@" || failed="$failed integration-examples"
+  # integration-store-v2
+  go_test "./etcdserver/api/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
+      -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed etcdserver_api_v2store"
+  # integration_cluster_proxy
+  go_test "./clientv3/... ./client/integration/... ./clientv3/snapshot/... ./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
+      -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
+
+  log_callout "Collecting coverage from e2e tests ..."
+  # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
+  # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
+  go_test "./tests/e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
+
+  log_callout "Collecting coverage from e2e tests with proxy ..."
+  go_test "./tests/e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
+
+  log_callout "Merging coverage results ..."
+  local cover_out_file="${coverdir}/cover.out"
+  # gocovmerge requires not-empty test to start with:
+  echo "mode: set" > "${cover_out_file}"
+
+  # incrementally merge to get coverage data even if some coverage files are corrupted
+  for f in "${coverdir}"/*.coverprofile; do
+    echo "merging test coverage file ${f}"
+    gocovmerge "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
+    if [ -s "${coverdir}"/cover.tmp ]; then
+      mv "${coverdir}/cover.tmp" "${cover_out_file}"
+    fi
+  done
+  # strip out generated files (using GNU-style sed)
+  sed --in-place '/generated.go/d' "${cover_out_file}" || true
+
+  # held failures to generate the full coverage file, now fail
+  if [ -n "$failed" ]; then
+    for f in $failed; do
+      log_error "--- FAIL:" "$f"
+    done
+    log_warning "Despite failures, you can see partial report:"
+    log_warning "  go tool cover -html ${cover_out_file}"
+    return 255
+  fi
+
+  log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
+}
+
+######### Code formatting checkers #############################################
+
+function fmt_pass {
+  toggle_failpoints disable
+
+  # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
+  # after resolving ore-existing errors.
+  for p in shellcheck \
+      markdown_you \
+      goword \
+      gofmt \
+      govet \
+      revive \
+      license_header \
+      receiver_name \
+      commit_title \
+      mod_tidy \
+      ; do
+    run_pass "${p}" "${@}"
+  done
 }
 
 function shellcheck_pass {
-	if command -v shellcheck >/dev/null; then
-		shellcheckResult=$(shellcheck -fgcc build test scripts/*.sh 2>&1 || true)
-		if [ -n "${shellcheckResult}" ]; then
-			echo -e "shellcheck checking failed:\\n${shellcheckResult}"
-			exit 255
-		fi
-  else
-    echo "Shellcheck not found !!! Check: https://github.com/koalaman/shellcheck#installing"
-	fi
+  if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
+    generic_checker run shellcheck -fgcc build test scripts/*.sh
+  fi
+}
+
+function markdown_you_find_eschew_you {
+  find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true
 }
 
 function markdown_you_pass {
-	# eschew you
-	yous=$(find . -name \*.md ! -path './vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true)
-	if [ -n "$yous" ]; then
-		echo -e "found 'you' in documentation:\\n${yous}"
-		exit 255
-	fi
+  generic_checker markdown_you_find_eschew_you
 }
 
 function markdown_marker_pass {
-	# TODO: check other markdown files when marker handles headers with '[]'
-	if command -v marker >/dev/null; then
-		markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true)
-		if [ -n "${markerResult}" ]; then
-			echo -e "marker checking failed:\\n${markerResult}"
-			exit 255
-		fi
-	else
-		echo "Skipping marker..."
-	fi
-}
-
-function goword_pass {
-	if command -v goword >/dev/null; then
-		# get all go files to process
-		gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null)
-		# shellcheck disable=SC2206
-		gofiles_all=($gofiles)
-		# ignore tests and protobuf files
-		# shellcheck disable=SC1117
-		gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | grep -vE "(\\_test.go|\\.pb\\.go)")
-		# shellcheck disable=SC2206
-		gofiles=($gofiles)
-		# only check for broken exported godocs
-		gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
-		if [ -n "$gowordRes" ]; then
-			echo -e "goword checking failed:\\n${gowordRes}"
-			exit 255
-		fi
-		# check some spelling
-		gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
-		if [ -n "$gowordRes" ]; then
-			echo -e "goword checking failed:\\n${gowordRes}"
-			exit 255
-		fi
-	else
-		echo "Skipping goword..."
-	fi
-}
-
-function gofmt_pass {
-	fmtRes=$(gofmt -l -s -d "${FMT[@]}")
-	if [ -n "${fmtRes}" ]; then
-		echo -e "gofmt checking failed:\\n${fmtRes}"
-		exit 255
-	fi
+  # TODO: check other markdown files when marker handles headers with '[]'
+  if tool_exists "marker" "https://crates.io/crates/marker"; then
+    generic_checker run marker --skip-http --root ./Documentation 2>&1
+  fi
 }
 
 function govet_pass {
-	vetRes=$(go vet "${TEST[@]}")
-	if [ -n "${vetRes}" ]; then
-		echo -e "govet checking failed:\\n${vetRes}"
-		exit 255
-	fi
+  run_for_modules generic_checker run go vet
 }
 
 function govet_shadow_pass {
-	fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
-	# shellcheck disable=SC2206
-	fmtpkgs=($fmtpkgs)
-	# Golang 1.12 onwards the experimental -shadow option is no longer available with go vet
-	./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
-	export PATH=${GOPATH}/bin:${PATH}
-	shadow_tool=$(which shadow)
-	vetRes=$(go vet -all -vettool="${shadow_tool}" "${TEST[@]}")
-	if [ -n "${vetRes}" ]; then
-		echo -e "govet -shadow checking failed:\\n${vetRes}"
-		exit 255
-	fi
+  if tool_exists "shadow" "./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow"; then
+    run_for_modules generic_checker run go vet -all -vettool="$(command -v shadow)"
+  fi
 }
 
 function unparam_pass {
-	if command -v unparam >/dev/null; then
-		unparamResult=$(unparam "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
-		if [ -n "${unparamResult}" ]; then
-			echo -e "unparam checking failed:\\n${unparamResult}"
-			exit 255
-		fi
-	else
-		echo "Skipping unparam..."
-	fi
+  if tool_exists "unparam" "./scripts/install_tool.sh mvdan.cc/unparam"; then
+    run_for_modules generic_checker run unparam
+  fi
 }
 
 function staticcheck_pass {
-	if command -v staticcheck >/dev/null; then
-		staticcheckResult=$(staticcheck "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
-		if [ -n "${staticcheckResult}" ]; then
-			# TODO: resolve these after go1.8 migration
-			# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
-			STATIC_CHECK_MASK="S(A|T)(1002|1005|1006|1008|1012|1019|1032|2002|4003|4006)"
-			if echo "${staticcheckResult}" | grep -vE "$STATIC_CHECK_MASK"; then
-				echo -e "staticcheck checking failed:\\n${staticcheckResult}"
-				exit 255
-			else
-				suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c)
-				echo -e "staticcheck suppressed warnings:\\n${suppressed}"
-			fi
-		fi
-	else
-		echo "Skipping staticcheck..."
-	fi
+  if tool_exists "staticcheck" "./scripts/install_tool.sh honnef.co/go/tools/cmd/staticcheck"; then
+    run_for_modules generic_checker run staticcheck
+  fi
 }
 
 function revive_pass {
-	if command -v revive >/dev/null; then
-		reviveResult=$(revive -config ./tests/revive.toml -exclude "vendor/..." ./... 2>&1 || true)
-		if [ -n "${reviveResult}" ]; then
-			echo -e "revive checking failed:\\n${reviveResult}"
-			exit 255
-		fi
-	else
-		echo "Skipping revive..."
-	fi
+  if tool_exists "revive" "./scripts/install_tool.sh github.com/mgechev/revive"; then
+    run_for_modules generic_checker run revive -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
+  fi
 }
 
 function unconvert_pass {
-	if command -v unconvert >/dev/null; then
-		unconvertResult=$(unconvert -v "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
-		if [ -n "${unconvertResult}" ]; then
-			echo -e "unconvert checking failed:\\n${unconvertResult}"
-			exit 255
-		fi
-	else
-		echo "Skipping unconvert..."
-	fi
+  if tool_exists "unconvert" "./scripts/install_tool.sh github.com/mdempsky/unconvert"; then
+    run_for_modules generic_checker run unconvert -v
+  fi
+}
+
+function ineffassign_per_package {
+  mapfile -t gofiles < <(go_srcs_in_module "$1")
+  run ineffassign "${gofiles[@]}"
 }
 
 function ineffassign_pass {
-	if command -v ineffassign >/dev/null; then
-		ineffassignResult=$(ineffassign "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
-		if [ -n "${ineffassignResult}" ]; then
-			echo -e "ineffassign checking failed:\\n${ineffassignResult}"
-			exit 255
-		fi
-	else
-		echo "Skipping ineffassign..."
-	fi
+  if tool_exists "ineffassign" "./scripts/install_tool.sh github.com/gordonklaus/ineffassign"; then
+    run_for_modules generic_checker ineffassign_per_package
+  fi
 }
 
 function nakedret_pass {
-	if command -v nakedret >/dev/null; then
-		nakedretResult=$(nakedret "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
-		if [ -n "${nakedretResult}" ]; then
-			echo -e "nakedret checking failed:\\n${nakedretResult}"
-			exit 255
-		fi
-	else
-		echo "Skipping nakedret..."
-	fi
+  if tool_exists "nakedret" "./scripts/install_tool.sh github.com/alexkohler/nakedret"; then
+    run_for_modules generic_checker run nakedret
+  fi
 }
 
 function license_header_pass {
-	licRes=""
-	files=$(find . -type f -iname '*.go' ! -path './vendor/*' ! -path './gopath.proto/*')
-	for file in $files; do
-		if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
-			licRes="${licRes}"$(echo -e "  ${file}")
-		fi
-	done
-	if [ -n "${licRes}" ]; then
-		echo -e "license header checking failed:\\n${licRes}"
-		exit 255
-	fi
+  mapfile -t gofiles < <(go_srcs_in_module "$1")
+  for file in "${gofiles[@]}"; do
+    if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
+      licRes="${licRes}"$(echo -e "  ${file}")
+    fi
+  done
+  if [ -n "${licRes}" ]; then
+    log_error -e "license header checking failed:\\n${licRes}"
+    return 255
+  fi
+}
+
+function receiver_name_for_package {
+  mapfile -t gofiles < <(go_srcs_in_module "$1")
+  recvs=$(grep 'func ([^*]' "${gofiles[@]}"  | tr  ':' ' ' |  \
+    awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
+    grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
+  if [ -n "${recvs}" ]; then
+    # shellcheck disable=SC2206
+    recvs=($recvs)
+    for recv in "${recvs[@]}"; do
+      log_error "Mismatched receiver for $recv..."
+      grep "$recv" "${gofiles[@]}" | grep 'func ('
+    done
+    return 255
+  fi
 }
 
 function receiver_name_pass {
-	# shellcheck disable=SC1117
-	recvs=$(grep 'func ([^*]' {*,*/*,*/*/*}.go  | grep -Ev "(generated|pb/)" | tr  ':' ' ' |  \
-		awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
-		grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
-	if [ -n "${recvs}" ]; then
-		# shellcheck disable=SC2206
-		recvs=($recvs)
-		for recv in "${recvs[@]}"; do
-			echo "Mismatched receiver for $recv..."
-			grep "$recv" "${FMT[@]}" | grep 'func ('
-		done
-		exit 255
-	fi
+  run_for_modules receiver_name_for_package
 }
 
 function commit_title_pass {
-	git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
-		commitMsg=$(echo "$l" | cut -f2- -d' ')
-		if [[ "$commitMsg" == Merge* ]]; then
-			# ignore "Merge pull" commits
-			continue
-		fi
-		if [[ "$commitMsg" == Revert* ]]; then
-			# ignore revert commits
-			continue
-		fi
+  git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
+    commitMsg=$(echo "$l" | cut -f2- -d' ')
+    if [[ "$commitMsg" == Merge* ]]; then
+      # ignore "Merge pull" commits
+      continue
+    fi
+    if [[ "$commitMsg" == Revert* ]]; then
+      # ignore revert commits
+      continue
+    fi
 
-		pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':')
-		spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
-		commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
-		if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
-			echo "$l"...
-			echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
-			echo "Got: $l"
-			exit 255
-		fi
-	done
+    pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':')
+    spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
+    commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
+    if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
+      log_error "$l"...
+      log_error "Expected commit title format '<package>{\", \"<package>}: <description>'"
+      log_error "Got: $l"
+      return 255
+    fi
+  done
 }
 
-# tools gosimple,unused,staticheck,unconvert,ineffasign,nakedret
-# are not module-aware. See https://github.com/golang/go/issues/24661
-# The module-aware versions need to be used when they become available
-function fmt_pass {
-	toggle_failpoints disable
+# goword_for_package package
+# checks spelling and comments in the 'package' in the current module
+function goword_for_package {
+  mapfile -t gofiles < <(go_srcs_in_module "$1")
+  # only check for broke exported godocs
+  gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
+  if [ -n "$gowordRes" ]; then
+    log_error -e "goword checking failed:\\n${gowordRes}"
+    return 255
+  fi
+  # check some spelling
+  gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
+  if [ -n "$gowordRes" ]; then
+    log_error -e "goword checking failed:\\n${gowordRes}"
+    return 255
+  fi
+}
 
-    # TODO: add "unparam"
-	for p in shellcheck \
-			markdown_you \
-			goword \
-			gofmt \
-			govet \
-			revive \
-			license_header \
-			receiver_name \
-			commit_title \
-			mod_tidy \
-			; do
-		echo "'$p' started at $(date)"
-		"${p}"_pass "$@"
-		echo "'$p' completed at $(date)"
-	done
+function goword_pass {
+  if tool_exists "goword" "./scripts_install_tool.sh github.com/chzchzchz/goword"; then
+    run_for_modules goword_for_package
+  fi
+}
+
+function go_fmt_for_package {
+  # We utilize 'go fmt' to find all files suitable for formatting,
+  # but reuse full power gofmt to perform just RO check.
+  go fmt -n "$1" | sed 's| -w | -d |g' | sh
+}
+
+function gofmt_pass {
+  run_for_modules generic_checker go_fmt_for_package
 }
 
 function bom_pass {
-	if ! command -v license-bill-of-materials >/dev/null; then
-		return
-	fi
-	if [ "${GO111MODULE}" == "on" ]; then
-		# license-bill-off-materials calls "go list std cmd" which cannot handle modules
-		# Please see https://github.com/golang/go/issues/26924
-		echo "Skipping license-bill-of-materials with go modules..."
-		return
-	fi
-	echo "Checking bill of materials..."
-	# https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
-	license-bill-of-materials \
-		--override-file bill-of-materials.override.json \
-		go.etcd.io/etcd/v3 go.etcd.io/etcd/v3/etcdctl >bom-now.json || true
-	if ! diff bill-of-materials.json bom-now.json; then
-		echo "vendored licenses do not match given bill of materials"
-		exit 255
-	fi
-	rm bom-now.json
+  if ! command -v license-bill-of-materials >/dev/null; then
+    log_warning "./license-bill-of-materials not FOUND"
+    log_warning "USE: ./scripts/install_tool.sh github.com/coreos/license-bill-of-materials"
+    return
+  fi
+  log_callout "Checking bill of materials..."
+  # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
+  run license-bill-of-materials \
+    --override-file bill-of-materials.override.json \
+    go.etcd.io/etcd/v3 go.etcd.io/etcd/v3/etcdctl >bom-now.json || true
+  if ! diff bill-of-materials.json bom-now.json; then
+    log_error "modularized licenses do not match given bill of materials"
+    return 255
+  fi
+  rm bom-now.json
 }
 
+######## VARIOUS CHECKERS ######################################################
+
 function dep_pass {
-	echo "Checking package dependencies..."
-	# don't pull in etcdserver package
-	pushd clientv3 >/dev/null
-	badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
-	deps=$(go list -f '{{ .Deps }}'  | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "")
-	popd >/dev/null
-	if [ -n "$deps" ]; then
-		echo -e "clientv3 has masked dependencies:\\n${deps}"
-		exit 255
-	fi
+  log_callout "Checking package dependencies..."
+  # don't pull in etcdserver package
+  pushd clientv3 >/dev/null
+  badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
+  deps=$(go list -f '{{ .Deps }}'  | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "")
+  popd >/dev/null
+  if [ -n "$deps" ]; then
+    log_error -e "clientv3 has masked dependencies:\\n${deps}"
+    return 255
+  fi
 }
 
-function build_cov_pass {
-	out="bin"
-	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
-	go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcd_test"
+function release_pass {
+  rm -f ./bin/etcd-last-release
+  # to grab latest patch release; bump this up for every minor release
+  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
+  if [ -n "$MANUAL_VER" ]; then
+    # in case, we need to test against different version
+    UPGRADE_VER=$MANUAL_VER
+  fi
+  if [[ -z ${UPGRADE_VER} ]]; then
+    UPGRADE_VER="v3.3.0"
+    log_warning "fallback to" ${UPGRADE_VER}
+  fi
 
-	go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcdctl_test" "${REPO_PATH}/etcdctl"
-}
+  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
+  log_callout "Downloading $file"
 
-# fail fast on static tests
-function build_pass {
-	echo "Checking build..."
-	GO_BUILD_FLAGS="-v" etcd_build
-	GO_BUILD_FLAGS="-v" tools_build
+  set +e
+  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
+  local result=$?
+  set -e
+  case $result in
+    0)  ;;
+    *)  log_error "--- FAIL:" ${result}
+      return $result
+      ;;
+  esac
+
+  tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
+  mkdir -p ./bin
+  mv /tmp/etcd ./bin/etcd-last-release
 }
 
 function mod_tidy_pass {
-	# Watch for upstream solution: https://github.com/golang/go/issues/27005
-	local tmpModDir
-	tmpModDir=$(mktemp -d --suffix "etcd-mod")
+  # Watch for upstream solution: https://github.com/golang/go/issues/27005
+  local tmpModDir
+  tmpModDir=$(mktemp -d --suffix "etcd-mod")
   cp "./go.mod" "./go.sum" "${tmpModDir}"
 
+  # Guarantees keeping go.sum minimal
+  # If this is causing too much problems, we should
+  # stop controlling go.sum at all.
+  rm go.sum
   go mod tidy
 
   set +e
@@ -710,20 +568,31 @@ function mod_tidy_pass {
   mv "${tmpModDir}/go.sum" "./go.sum"
 
   if [ "${tmpFileGoModInSync}" -ne 0 ]; then
-		echo "./go.mod is not in sync with 'go mod tidy'"
-		exit 255
-	fi
-	if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
-		echo "./go.sum is not in sync with 'go mod tidy'"
-		exit 255
-	fi
+    log_error "./go.mod is not in sync with 'go mod tidy'"
+    return 255
+  fi
+  if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
+    log_error "./go.sum is not in sync with 'rm go.sum; go mod tidy'"
+    return 255
+  fi
+}
+
+########### MAIN ###############################################################
+
+function run_pass {
+  local pass="${1}"
+  shift 1
+  log_callout -e "\n'${pass}' started at $(date)"
+  if "${pass}_pass" "$@" ; then
+    log_success "'${pass}' completed at $(date)"
+  else
+    log_error "FAIL: '${pass}' failed at $(date)"
+    exit 255
+  fi
 }
 
 for pass in $PASSES; do
-	echo "Starting '$pass' pass at $(date)"
-	"${pass}"_pass "$@"
-	echo "Finished '$pass' pass at $(date)"
+  run_pass "${pass}" "${@}"
 done
 
-echo "Success"
-
+log_success "SUCCESS"