Merge pull request #12322 from ptabor/20200920-test-script

./test: Refactoring of test script for modularization

.gitignore

@@ -17,5 +17,6 @@ hack/tls-setup/certs
 /contrib/raftexample/raftexample
 /contrib/raftexample/raftexample-*
 /vendor
+/tests/e2e/default.proxy
 
 *.bak

.travis.yml

@@ -78,19 +78,22 @@ script:
           /bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
         ;;
       linux-amd64-integration-1-cpu)
+        # TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' RACE='false' ./test"
         ;;
       linux-amd64-integration-2-cpu)
+        # TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' RACE='false' ./test"
         ;;
       linux-amd64-integration-4-cpu)
+        # TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test"
         ;;
       linux-amd64-functional)
         docker run --rm \
@@ -100,7 +103,7 @@ script:
       linux-amd64-unit)
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test"
+          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test -p=2"
         ;;
       all-build)
         docker run --rm \
@@ -115,17 +118,18 @@ script:
             && GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=s390x ./build"
         ;;
       linux-amd64-grpcproxy)
-        sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test
+        # TODO: Reenable race when https://github.com/etcd-io/etcd/issues/12336 fixed.
+        sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy' VERBOSE='1' CPU='4' COVER='false' RACE='false'" make docker-test
         ;;
       linux-amd64-coverage)
-        sudo HOST_TMP_DIR=/tmp make docker-test-coverage
+        sudo HOST_TMP_DIR=/tmp TEST_OPTS="VERBOSE='1'" make docker-test-coverage
         ;;
       linux-amd64-fmt-unit-go-tip)
-        GOARCH=amd64 PASSES='fmt unit' ./test
+        GOARCH=amd64 PASSES='fmt unit' ./test -p=2
         ;;
       linux-386-unit)
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=386 PASSES='unit' ./test"
+          /bin/bash -c "GOARCH=386 PASSES='unit' ./test -p=2"
         ;;
     esac

Makefile

@@ -18,18 +18,15 @@ build:
 
 clean:
 	rm -f ./codecov
-	rm -rf ./agent-*
 	rm -rf ./covdir
-	rm -f ./*.coverprofile
-	rm -f ./*.log
 	rm -f ./bin/Dockerfile-release
 	rm -rf ./bin/etcd*
 	rm -rf ./default.etcd
 	rm -rf ./tests/e2e/default.etcd
 	rm -rf ./release
-	rm -f ./snapshot/localhost:*
-	rm -f ./tools/etcd-dump-metrics/localhost:*
-	find ./ -name "127.0.0.1:*" -o -name "localhost:*" -delete
+	rm -rf ./coverage/*.err ./coverage/*.out
+	rm -rf ./tests/e2e/default.proxy
+	find ./ -name "127.0.0.1:*" -o -name "localhost:*" -o -name "*.log" -o -name "agent-*" -o -name "*.coverprofile" -o -name "testname-proxy-*" | xargs -r rm -r
 
 docker-clean:
 	docker images

clientv3/integration/ordering_kv_test.go

@@ -0,0 +1,166 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/v3/clientv3/ordering"
+	"go.etcd.io/etcd/v3/integration"
+	"go.etcd.io/etcd/v3/pkg/testutil"
+)
+
+func TestDetectKvOrderViolation(t *testing.T) {
+	var errOrderViolation = errors.New("Detected Order Violation")
+
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	cfg := clientv3.Config{
+		Endpoints: []string{
+			clus.Members[0].GRPCAddr(),
+			clus.Members[1].GRPCAddr(),
+			clus.Members[2].GRPCAddr(),
+		},
+	}
+	cli, err := clientv3.New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+	ctx := context.TODO()
+
+	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
+		t.Fatal(err)
+	}
+	// ensure that the second member has the current revision for the key foo
+	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
+		t.Fatal(err)
+	}
+
+	// stop third member in order to force the member to have an outdated revision
+	clus.Members[2].Stop(t)
+	time.Sleep(1 * time.Second) // give enough time for operation
+	_, err = cli.Put(ctx, "foo", "buzz")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// perform get request against the first member, in order to
+	// set up kvOrdering to expect "foo" revisions greater than that of
+	// the third member.
+	orderingKv := ordering.NewKV(cli.KV,
+		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
+			return errOrderViolation
+		})
+	_, err = orderingKv.Get(ctx, "foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that only the third member is queried during requests
+	clus.Members[0].Stop(t)
+	clus.Members[1].Stop(t)
+	clus.Members[2].Restart(t)
+	// force OrderingKv to query the third member
+	cli.SetEndpoints(clus.Members[2].GRPCAddr())
+	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
+
+	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
+	if err != errOrderViolation {
+		t.Fatalf("expected %v, got %v", errOrderViolation, err)
+	}
+}
+
+func TestDetectTxnOrderViolation(t *testing.T) {
+	var errOrderViolation = errors.New("Detected Order Violation")
+
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	cfg := clientv3.Config{
+		Endpoints: []string{
+			clus.Members[0].GRPCAddr(),
+			clus.Members[1].GRPCAddr(),
+			clus.Members[2].GRPCAddr(),
+		},
+	}
+	cli, err := clientv3.New(cfg)
+	defer cli.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+	ctx := context.TODO()
+
+	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
+		t.Fatal(err)
+	}
+	// ensure that the second member has the current revision for the key foo
+	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
+		t.Fatal(err)
+	}
+
+	// stop third member in order to force the member to have an outdated revision
+	clus.Members[2].Stop(t)
+	time.Sleep(1 * time.Second) // give enough time for operation
+	if _, err = clus.Client(1).Put(ctx, "foo", "buzz"); err != nil {
+		t.Fatal(err)
+	}
+
+	// perform get request against the first member, in order to
+	// set up kvOrdering to expect "foo" revisions greater than that of
+	// the third member.
+	orderingKv := ordering.NewKV(cli.KV,
+		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
+			return errOrderViolation
+		})
+	orderingTxn := orderingKv.Txn(ctx)
+	_, err = orderingTxn.If(
+		clientv3.Compare(clientv3.Value("b"), ">", "a"),
+	).Then(
+		clientv3.OpGet("foo"),
+	).Commit()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that only the third member is queried during requests
+	clus.Members[0].Stop(t)
+	clus.Members[1].Stop(t)
+	clus.Members[2].Restart(t)
+	// force OrderingKv to query the third member
+	cli.SetEndpoints(clus.Members[2].GRPCAddr())
+	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
+	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
+	if err != errOrderViolation {
+		t.Fatalf("expected %v, got %v", errOrderViolation, err)
+	}
+	orderingTxn = orderingKv.Txn(ctx)
+	_, err = orderingTxn.If(
+		clientv3.Compare(clientv3.Value("b"), ">", "a"),
+	).Then(
+		clientv3.OpGet("foo", clientv3.WithSerializable()),
+	).Commit()
+	if err != errOrderViolation {
+		t.Fatalf("expected %v, got %v", errOrderViolation, err)
+	}
+}

clientv3/ordering/kv_test.go

@@ -17,153 +17,13 @@ package ordering
 import (
 	"context"
 	gContext "context"
-	"errors"
 	"sync"
 	"testing"
-	"time"
 
 	"go.etcd.io/etcd/v3/clientv3"
 	pb "go.etcd.io/etcd/v3/etcdserver/etcdserverpb"
-	"go.etcd.io/etcd/v3/integration"
-	"go.etcd.io/etcd/v3/pkg/testutil"
 )
 
-func TestDetectKvOrderViolation(t *testing.T) {
-	var errOrderViolation = errors.New("Detected Order Violation")
-
-	defer testutil.AfterTest(t)
-	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
-	defer clus.Terminate(t)
-
-	cfg := clientv3.Config{
-		Endpoints: []string{
-			clus.Members[0].GRPCAddr(),
-			clus.Members[1].GRPCAddr(),
-			clus.Members[2].GRPCAddr(),
-		},
-	}
-	cli, err := clientv3.New(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx := context.TODO()
-
-	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
-		t.Fatal(err)
-	}
-	// ensure that the second member has the current revision for the key foo
-	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
-		t.Fatal(err)
-	}
-
-	// stop third member in order to force the member to have an outdated revision
-	clus.Members[2].Stop(t)
-	time.Sleep(1 * time.Second) // give enough time for operation
-	_, err = cli.Put(ctx, "foo", "buzz")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// perform get request against the first member, in order to
-	// set up kvOrdering to expect "foo" revisions greater than that of
-	// the third member.
-	orderingKv := NewKV(cli.KV,
-		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
-			return errOrderViolation
-		})
-	_, err = orderingKv.Get(ctx, "foo")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// ensure that only the third member is queried during requests
-	clus.Members[0].Stop(t)
-	clus.Members[1].Stop(t)
-	clus.Members[2].Restart(t)
-	// force OrderingKv to query the third member
-	cli.SetEndpoints(clus.Members[2].GRPCAddr())
-	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
-
-	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
-	if err != errOrderViolation {
-		t.Fatalf("expected %v, got %v", errOrderViolation, err)
-	}
-}
-
-func TestDetectTxnOrderViolation(t *testing.T) {
-	var errOrderViolation = errors.New("Detected Order Violation")
-
-	defer testutil.AfterTest(t)
-	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
-	defer clus.Terminate(t)
-
-	cfg := clientv3.Config{
-		Endpoints: []string{
-			clus.Members[0].GRPCAddr(),
-			clus.Members[1].GRPCAddr(),
-			clus.Members[2].GRPCAddr(),
-		},
-	}
-	cli, err := clientv3.New(cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx := context.TODO()
-
-	if _, err = clus.Client(0).Put(ctx, "foo", "bar"); err != nil {
-		t.Fatal(err)
-	}
-	// ensure that the second member has the current revision for the key foo
-	if _, err = clus.Client(1).Get(ctx, "foo"); err != nil {
-		t.Fatal(err)
-	}
-
-	// stop third member in order to force the member to have an outdated revision
-	clus.Members[2].Stop(t)
-	time.Sleep(1 * time.Second) // give enough time for operation
-	if _, err = clus.Client(1).Put(ctx, "foo", "buzz"); err != nil {
-		t.Fatal(err)
-	}
-
-	// perform get request against the first member, in order to
-	// set up kvOrdering to expect "foo" revisions greater than that of
-	// the third member.
-	orderingKv := NewKV(cli.KV,
-		func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
-			return errOrderViolation
-		})
-	orderingTxn := orderingKv.Txn(ctx)
-	_, err = orderingTxn.If(
-		clientv3.Compare(clientv3.Value("b"), ">", "a"),
-	).Then(
-		clientv3.OpGet("foo"),
-	).Commit()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// ensure that only the third member is queried during requests
-	clus.Members[0].Stop(t)
-	clus.Members[1].Stop(t)
-	clus.Members[2].Restart(t)
-	// force OrderingKv to query the third member
-	cli.SetEndpoints(clus.Members[2].GRPCAddr())
-	time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed
-	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
-	if err != errOrderViolation {
-		t.Fatalf("expected %v, got %v", errOrderViolation, err)
-	}
-	orderingTxn = orderingKv.Txn(ctx)
-	_, err = orderingTxn.If(
-		clientv3.Compare(clientv3.Value("b"), ">", "a"),
-	).Then(
-		clientv3.OpGet("foo", clientv3.WithSerializable()),
-	).Commit()
-	if err != errOrderViolation {
-		t.Fatalf("expected %v, got %v", errOrderViolation, err)
-	}
-}
-
 type mockKV struct {
 	clientv3.KV
 	response clientv3.OpResponse

clientv3/snapshot/v3_snapshot_test.go

@@ -33,8 +33,6 @@ import (
 	"go.uber.org/zap"
 )
 
-// TODO(ptabor): This is integration test. Skip it in --short and move to integration tests directory.
-
 // TestSnapshotV3RestoreSingle tests single node cluster restoring
 // from a snapshot file.
 func TestSnapshotV3RestoreSingle(t *testing.T) {
@@ -194,6 +192,8 @@ type kv struct {
 
 // creates a snapshot file and returns the file path.
 func createSnapshotFile(t *testing.T, kvs []kv) string {
+	testutil.SkipTestIfShortMode(t,
+		"Snapshot creation tests are depending on embedded etcServer so are integration-level tests.")
 	clusterN := 1
 	urls := newEmbedURLs(clusterN * 2)
 	cURLs, pURLs := urls[:clusterN], urls[clusterN:]

go.sum

@@ -183,7 +183,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8=
 golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

integration/cluster.go

@@ -1250,6 +1250,7 @@ type ClusterV3 struct {
 func NewClusterV3(t testing.TB, cfg *ClusterConfig) *ClusterV3 {
 	if t != nil {
 		t.Helper()
+		testutil.SkipTestIfShortMode(t, "Cannot create clusters in --short tests")
 	}
 
 	cfg.UseGRPC = true

integration/embed/embed_proxy_test.go

@@ -0,0 +1,19 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build cluster_proxy
+
+// The purpose of this (empty) package is too keep following test working:
+// #  go test -tags=cluster_proxy ./integration/embed
+package embed_test

pkg/testutil/leak.go

@@ -131,6 +131,7 @@ func interestingGoroutines() (gs []string) {
 			strings.Contains(stack, "go.etcd.io/etcd/v3/pkg/logutil.(*MergeLogger).outputLoop") ||
 			strings.Contains(stack, "github.com/golang/glog.(*loggingT).flushDaemon") ||
 			strings.Contains(stack, "created by runtime.gc") ||
+			strings.Contains(stack, "created by text/template/parse.lex") ||
 			strings.Contains(stack, "runtime.MHeap_Scavenger") {
 			continue
 		}

scripts/test_lib.sh

@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+
+ETCD_ROOT_DIR="$(pwd)"
+
+####   Convenient IO methods #####
+
+COLOR_RED='\033[0;31m'
+COLOR_ORANGE='\033[0;33m'
+COLOR_GREEN='\033[0;32m'
+COLOR_LIGHTCYAN='\033[0;36m'
+
+COLOR_NONE='\033[0m' # No Color
+
+function log_error {
+  >&2 echo -n -e "${COLOR_RED}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+function log_warning {
+  >&2 echo -n -e "${COLOR_ORANGE}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+function log_callout {
+  >&2 echo -n -e "${COLOR_LIGHTCYAN}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+function log_success {
+  >&2 echo -n -e "${COLOR_GREEN}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
+
+####   Discovery of files/packages within a go module #####
+
+# go_srcs_in_module [package]
+# returns list of all not-generated go sources in the current (dir) module.
+function go_srcs_in_module {
+  go fmt -n "$1"  | grep -Eo "([^ ]*)$" | grep -vE "(\\_test.go|\\.pb\\.go|\\.pb\\.gw.go)"
+}
+
+# pkgs_in_module [optional:package_pattern]
+# returns list of all packages in the current (dir) module.
+# if the package_pattern is given, its being resolved.
+function pkgs_in_module {
+  go list -mod=mod "${1:-./...}";
+}
+
+function filter_out_integration_style_tests {
+  grep -Ev '/(tests/e2e|integration|functional)(/|$)'
+}
+
+####    Running actions against multiple modules ####
+
+# run [command...] - runs given command, printing it first and
+# again if it failed (in RED). Use to wrap important test commands
+# that user might want to re-execute to shorten the feedback loop when fixing
+# the test.
+function run {
+  local rpath
+  rpath=$(realpath "--relative-to=${ETCD_ROOT_DIR}" "${PWD}")
+  local repro="$*"
+  if [ "${rpath}" != "." ]; then
+    repro="(cd ${rpath} && ${repro})"
+  fi
+
+  log_callout "% ${repro}"
+  "${@}"
+  local error_code=$?
+  if [ ${error_code} != 0 ]; then
+    log_error -e "FAIL: (code:${error_code}):\n  % ${repro}"
+    return ${error_code}
+  fi
+}
+
+# run_for_module [module] [cmd]
+# executes given command in the given module for given pkgs.
+#   module_name - "." (in future: tests, client, server)
+#   cmd         - cmd to be executed - that takes package as last argument
+function run_for_module {
+  local module=${1:-"."}
+  shift 1
+  (
+    cd "${module}" && "$@"
+  )
+}
+
+#  run_for_modules [cmd]
+#  run given command across all modules and packages
+#  (unless the set is limited using ${PKG} or / ${USERMOD})
+function run_for_modules {
+  local pkg="${PKG:-./...}"
+  if [ -z "${USERMOD}" ]; then
+    run_for_module "." "$@" "${pkg}" || return "$?"
+  else
+    run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?"
+  fi
+}
+
+
+####    Running go test  ########
+
+# go_test [packages] [mode] [flags_for_package_func] [$@]
+# [mode] supports 3 states:
+#   - "parallel": fastest as concurrently processes multiple packages, but silent
+#                 till the last package. See: https://github.com/golang/go/issues/2731
+#   - "keep_going" : executes tests package by package, but postpones reporting error to the last
+#   - "fail_fast"  : executes tests packages 1 by 1, exits on the first failure.
+#
+# [flags_for_package_func] is a name of function that takes list of packages as parameter
+#   and computes additional flags to the go_test commands.
+#   Use 'true' or ':' if you dont need additional arguments.
+#
+#  depends on the VERBOSE top-level variable.
+#
+#  Example:
+#    go_test "./..." "keep_going" ":" --short
+#
+#  The function returns != 0 code in case of test failure.
+function go_test {
+  local packages="${1}"
+  local mode="${2}"
+  local flags_for_package_func="${3}"
+
+  shift 3
+
+  local goTestFlags=""
+  local goTestEnv=""
+  if [ "${VERBOSE}" == "1" ]; then
+    goTestFlags="-v"
+  fi
+  if [ "${VERBOSE}" == "2" ]; then
+    goTestFlags="-v"
+    goTestEnv="CLIENT_DEBUG=1"
+  fi
+
+  # Expanding patterns (like ./...) into list of packages
+
+  local unpacked_packages=("${packages}")
+  if [ "${mode}" != "parallel" ]; then
+    # shellcheck disable=SC2207
+    # shellcheck disable=SC2086
+    if ! unpacked_packages=($(go list ${packages})); then
+      log_error "Cannot resolve packages: ${packages}"
+      return 255
+    fi
+  fi
+
+  local failures=""
+
+  # execution of tests against packages:
+  for pkg in "${unpacked_packages[@]}"; do
+    local additional_flags
+    # shellcheck disable=SC2086
+    additional_flags=$(${flags_for_package_func} ${pkg})
+
+    # shellcheck disable=SC2206
+    local cmd=( go test ${goTestFlags} ${additional_flags} "$@" ${pkg} )
+
+    if ! run env ${goTestEnv} "${cmd[@]}" ; then
+      if [ "${mode}" != "keep_going" ]; then
+        return 2
+      else
+        failures=("${failures[@]}" "${pkg}")
+      fi
+    fi
+    echo
+  done
+
+  if [ -n "${failures[*]}" ] ; then
+    log_error -e "ERROR: Tests for following packages failed:\n  ${failures[*]}"
+    return 2
+  fi
+}
+
+#### Other ####
+
+# tool_exists [tool] [instruction]
+# Checks whether given [tool] is installed. In case of failure,
+# prints a warning with installation [instruction] and returns !=0 code.
+function tool_exists {
+  local tool="${1}"
+  local instruction="${2}"
+  if ! command -v "${tool}" >/dev/null; then
+    log_warning "Tool: '${tool}' not found on PATH. ${instruction}"
+    return 255
+  fi
+}
+