From 351bdb33c520f3462eed58993ab5a168ef076841 Mon Sep 17 00:00:00 2001
From: Piotr Tabor <ptab@google.com>
Date: Thu, 21 Jan 2021 20:21:19 +0100
Subject: [PATCH] Split intengration/clientv3 tests into multiple packages

They used to take >10min with coverage, so were causing interrupted
Travis runs.

Know thay fit in 100-150s (together), thanks also to parallel
execution.
---
 codecov.yml                                   | 10 +--
 pkg/testutil/testutil.go                      | 12 +++
 test.sh                                       |  6 +-
 tests/functional/tester/cluster_run.go        |  4 +-
 .../clientv3/concurrency/main_test.go         |  2 +
 .../{ => connectivity}/black_hole_test.go     | 15 ++--
 .../clientv3/{ => connectivity}/dial_test.go  | 19 ++---
 .../integration/clientv3/connectivity/doc.go  | 15 ++++
 .../clientv3/connectivity/main_test.go        | 15 ++++
 .../network_partition_test.go                 | 19 ++---
 .../server_shutdown_test.go                   | 79 ++-----------------
 .../clientv3/examples/main_test.go            |  1 +
 tests/integration/clientv3/kv_test.go         |  7 +-
 tests/integration/clientv3/lease/doc.go       | 15 ++++
 .../clientv3/{ => lease}/lease_test.go        |  2 +-
 .../clientv3/{ => lease}/leasing_test.go      |  2 +-
 tests/integration/clientv3/lease/main_test.go | 15 ++++
 .../integration/clientv3/maintenance_test.go  |  4 +-
 tests/integration/clientv3/util.go            | 71 ++++++++++++++++-
 tests/integration/clientv3/watch_test.go      |  3 +-
 20 files changed, 197 insertions(+), 119 deletions(-)
 rename tests/integration/clientv3/{ => connectivity}/black_hole_test.go (90%)
 rename tests/integration/clientv3/{ => connectivity}/dial_test.go (92%)
 create mode 100644 tests/integration/clientv3/connectivity/doc.go
 create mode 100644 tests/integration/clientv3/connectivity/main_test.go
 rename tests/integration/clientv3/{ => connectivity}/network_partition_test.go (92%)
 rename tests/integration/clientv3/{ => connectivity}/server_shutdown_test.go (86%)
 create mode 100644 tests/integration/clientv3/lease/doc.go
 rename tests/integration/clientv3/{ => lease}/lease_test.go (99%)
 rename tests/integration/clientv3/{ => lease}/leasing_test.go (99%)
 create mode 100644 tests/integration/clientv3/lease/main_test.go

diff --git a/codecov.yml b/codecov.yml
index aed42c014..a4b3b7f27 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -6,13 +6,13 @@ fixes:
   - "go.etcd.io/etcd/api/v3/::api/"
   - "go.etcd.io/etcd/client/v3/::client/v3/"
   - "go.etcd.io/etcd/client/v2/::client/v2/"
-  - "go.etcd.io/etcd/etcdctl/v3::etcdctl/"
-  - "go.etcd.io/etcd/pkg/v3::pkg/"
-  - "go.etcd.io/etcd/raft/v3::raft/"
-  - "go.etcd.io/etcd/server/v3::server/"
+  - "go.etcd.io/etcd/etcdctl/v3/::etcdctl/"
+  - "go.etcd.io/etcd/pkg/v3/::pkg/"
+  - "go.etcd.io/etcd/raft/v3/::raft/"
+  - "go.etcd.io/etcd/server/v3/::server/"
 
 ignore:
   - "**/*.pb.go"
   - "**/*.pb.gw.go"
   - "tests/**/*"
-  - "go.etcd.io/etcd/tests/**/*"
\ No newline at end of file
+  - "go.etcd.io/etcd/tests/**/*"
diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go
index db51304e7..f12566ab2 100644
--- a/pkg/testutil/testutil.go
+++ b/pkg/testutil/testutil.go
@@ -17,6 +17,7 @@ package testutil
 
 import (
 	"net/url"
+	"os"
 	"runtime"
 	"testing"
 	"time"
@@ -91,3 +92,14 @@ func SkipTestIfShortMode(t testing.TB, reason string) {
 		}
 	}
 }
+
+// ExitInShortMode closes the current process (with 0) if the short test mode detected.
+//
+// To be used in Test-main, where test context (testing.TB) is not available.
+//
+// Requires custom env-variable (GOLANG_TEST_SHORT) apart of `go test --short flag`.
+func ExitInShortMode(reason string) {
+	if os.Getenv("GOLANG_TEST_SHORT") == "true" {
+		os.Exit(0)
+	}
+}
diff --git a/test.sh b/test.sh
index fdde245cf..29a6cf181 100755
--- a/test.sh
+++ b/test.sh
@@ -91,7 +91,7 @@ function run_unit_tests {
   local pkgs="${1:-./...}"
   shift 1
   # shellcheck disable=SC2086
-  go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
+  GOLANG_TEST_SHORT=true go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
 }
 
 function unit_pass {
@@ -109,7 +109,7 @@ function integration_extra {
 
 function integration_pass {
   local pkgs=${USERPKG:-"./integration/..."}
-  run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
+  run_for_module "tests" go_test "${pkgs}" "parallel" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
   integration_extra "$@"
 }
 
@@ -307,7 +307,7 @@ function cov_pass {
 
   log_callout "[$(date)] Collecting coverage from unit tests ..."
   for m in $(module_dirs); do
-    run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \
+    GOLANG_TEST_SHORT=true run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \
        "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
   done
 
diff --git a/tests/functional/tester/cluster_run.go b/tests/functional/tester/cluster_run.go
index d73f60d84..256540d55 100644
--- a/tests/functional/tester/cluster_run.go
+++ b/tests/functional/tester/cluster_run.go
@@ -47,7 +47,7 @@ func (clus *Cluster) Run() {
 		clus.rd = round
 
 		if err := clus.doRound(); err != nil {
-			clus.lg.Warn(
+			clus.lg.Error(
 				"round FAIL",
 				zap.Int("round", clus.rd),
 				zap.Int("case", clus.cs),
@@ -316,7 +316,7 @@ func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) {
 }
 
 func (clus *Cluster) failed() {
-	clus.lg.Info(
+	clus.lg.Error(
 		"functional-tester FAIL",
 		zap.Int("round", clus.rd),
 		zap.Int("case", clus.cs),
diff --git a/tests/integration/clientv3/concurrency/main_test.go b/tests/integration/clientv3/concurrency/main_test.go
index d149d872f..9462b3a9b 100644
--- a/tests/integration/clientv3/concurrency/main_test.go
+++ b/tests/integration/clientv3/concurrency/main_test.go
@@ -33,6 +33,8 @@ func forUnitTestsRunInMockedContext(mocking func(), example func()) {
 
 // TestMain sets up an etcd cluster if running the examples.
 func TestMain(m *testing.M) {
+	testutil.ExitInShortMode("Skipping: the tests require real cluster")
+
 	v := m.Run()
 	lazyCluster.Terminate()
 	if v == 0 {
diff --git a/tests/integration/clientv3/black_hole_test.go b/tests/integration/clientv3/connectivity/black_hole_test.go
similarity index 90%
rename from tests/integration/clientv3/black_hole_test.go
rename to tests/integration/clientv3/connectivity/black_hole_test.go
index 0e5d71c60..eebdc2e67 100644
--- a/tests/integration/clientv3/black_hole_test.go
+++ b/tests/integration/clientv3/connectivity/black_hole_test.go
@@ -14,7 +14,7 @@
 
 // +build !cluster_proxy
 
-package clientv3test
+package connectivity_test
 
 import (
 	"context"
@@ -25,6 +25,7 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
+	"go.etcd.io/etcd/tests/v3/integration/clientv3"
 	"google.golang.org/grpc"
 )
 
@@ -111,7 +112,7 @@ func TestBalancerUnderBlackholeKeepAliveWatch(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Put(ctx, "foo", "bar")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -121,7 +122,7 @@ func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAliveDelete(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Delete(ctx, "foo")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -134,7 +135,7 @@ func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) {
 			If(clientv3.Compare(clientv3.Version("foo"), "=", 0)).
 			Then(clientv3.OpPut("foo", "bar")).
 			Else(clientv3.OpPut("foo", "baz")).Commit()
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -144,7 +145,7 @@ func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -154,7 +155,7 @@ func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAliveSerializableGet(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a", clientv3.WithSerializable())
-		if isClientTimeout(err) || isServerCtxTimeout(err) {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) {
 			return errExpected
 		}
 		return err
@@ -186,7 +187,7 @@ func testBalancerUnderBlackholeNoKeepAlive(t *testing.T, op func(*clientv3.Clien
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
diff --git a/tests/integration/clientv3/dial_test.go b/tests/integration/clientv3/connectivity/dial_test.go
similarity index 92%
rename from tests/integration/clientv3/dial_test.go
rename to tests/integration/clientv3/connectivity/dial_test.go
index dcb8c662c..5e4955969 100644
--- a/tests/integration/clientv3/dial_test.go
+++ b/tests/integration/clientv3/connectivity/dial_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package connectivity_test
 
 import (
 	"context"
@@ -26,21 +26,22 @@ import (
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/pkg/v3/transport"
 	"go.etcd.io/etcd/tests/v3/integration"
+	clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3"
 	"google.golang.org/grpc"
 )
 
 var (
 	testTLSInfo = transport.TLSInfo{
-		KeyFile:        "../../fixtures/server.key.insecure",
-		CertFile:       "../../fixtures/server.crt",
-		TrustedCAFile:  "../../fixtures/ca.crt",
+		KeyFile:        "../../../fixtures/server.key.insecure",
+		CertFile:       "../../../fixtures/server.crt",
+		TrustedCAFile:  "../../../fixtures/ca.crt",
 		ClientCertAuth: true,
 	}
 
 	testTLSInfoExpired = transport.TLSInfo{
-		KeyFile:        "../fixtures-expired/server.key.insecure",
-		CertFile:       "../fixtures-expired/server.crt",
-		TrustedCAFile:  "../fixtures-expired/ca.crt",
+		KeyFile:        "../../fixtures-expired/server.key.insecure",
+		CertFile:       "../../fixtures-expired/server.crt",
+		TrustedCAFile:  "../../fixtures-expired/ca.crt",
 		ClientCertAuth: true,
 	}
 )
@@ -62,7 +63,7 @@ func TestDialTLSExpired(t *testing.T) {
 		DialOptions: []grpc.DialOption{grpc.WithBlock()},
 		TLS:         tls,
 	})
-	if !isClientTimeout(err) {
+	if !clientv3test.IsClientTimeout(err) {
 		t.Fatalf("expected dial timeout error, got %v", err)
 	}
 }
@@ -84,7 +85,7 @@ func TestDialTLSNoConfig(t *testing.T) {
 			c.Close()
 		}
 	}()
-	if !isClientTimeout(err) {
+	if !clientv3test.IsClientTimeout(err) {
 		t.Fatalf("expected dial timeout error, got %v", err)
 	}
 }
diff --git a/tests/integration/clientv3/connectivity/doc.go b/tests/integration/clientv3/connectivity/doc.go
new file mode 100644
index 000000000..c90413b44
--- /dev/null
+++ b/tests/integration/clientv3/connectivity/doc.go
@@ -0,0 +1,15 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectivity
diff --git a/tests/integration/clientv3/connectivity/main_test.go b/tests/integration/clientv3/connectivity/main_test.go
new file mode 100644
index 000000000..f857c6883
--- /dev/null
+++ b/tests/integration/clientv3/connectivity/main_test.go
@@ -0,0 +1,15 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package connectivity
+
+import (
+	"testing"
+
+	"go.etcd.io/etcd/pkg/v3/testutil"
+)
+
+func TestMain(m *testing.M) {
+	testutil.MustTestMainWithLeakDetection(m)
+}
diff --git a/tests/integration/clientv3/network_partition_test.go b/tests/integration/clientv3/connectivity/network_partition_test.go
similarity index 92%
rename from tests/integration/clientv3/network_partition_test.go
rename to tests/integration/clientv3/connectivity/network_partition_test.go
index bc2849d5a..1860fe490 100644
--- a/tests/integration/clientv3/network_partition_test.go
+++ b/tests/integration/clientv3/connectivity/network_partition_test.go
@@ -14,7 +14,7 @@
 
 // +build !cluster_proxy
 
-package clientv3test
+package connectivity_test
 
 import (
 	"context"
@@ -27,6 +27,7 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
+	"go.etcd.io/etcd/tests/v3/integration/clientv3"
 	"google.golang.org/grpc"
 )
 
@@ -38,7 +39,7 @@ var errExpected = errors.New("expected error")
 func TestBalancerUnderNetworkPartitionPut(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Put(ctx, "a", "b")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -48,7 +49,7 @@ func TestBalancerUnderNetworkPartitionPut(t *testing.T) {
 func TestBalancerUnderNetworkPartitionDelete(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Delete(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -61,7 +62,7 @@ func TestBalancerUnderNetworkPartitionTxn(t *testing.T) {
 			If(clientv3.Compare(clientv3.Version("foo"), "=", 0)).
 			Then(clientv3.OpPut("foo", "bar")).
 			Else(clientv3.OpPut("foo", "baz")).Commit()
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -74,7 +75,7 @@ func TestBalancerUnderNetworkPartitionTxn(t *testing.T) {
 func TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -87,7 +88,7 @@ func TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout(t *testing.
 func TestBalancerUnderNetworkPartitionLinearizableGetWithShortTimeout(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) {
 			return errExpected
 		}
 		return err
@@ -125,7 +126,7 @@ func testBalancerUnderNetworkPartition(t *testing.T, op func(*clientv3.Client, c
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add other endpoints for later endpoint switch
 	cli.SetEndpoints(eps...)
@@ -235,7 +236,7 @@ func testBalancerUnderNetworkPartitionWatch(t *testing.T, isolateLeader bool) {
 	defer watchCli.Close()
 
 	// wait for eps[target] to be pinned
-	mustWaitPinReady(t, watchCli)
+	clientv3test.MustWaitPinReady(t, watchCli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -290,7 +291,7 @@ func TestDropReadUnderNetworkPartition(t *testing.T) {
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add other endpoints for later endpoint switch
 	cli.SetEndpoints(eps...)
diff --git a/tests/integration/clientv3/server_shutdown_test.go b/tests/integration/clientv3/connectivity/server_shutdown_test.go
similarity index 86%
rename from tests/integration/clientv3/server_shutdown_test.go
rename to tests/integration/clientv3/connectivity/server_shutdown_test.go
index 32d6bb877..85aa08750 100644
--- a/tests/integration/clientv3/server_shutdown_test.go
+++ b/tests/integration/clientv3/connectivity/server_shutdown_test.go
@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package connectivity_test
 
 import (
 	"bytes"
 	"context"
-	"strings"
 	"testing"
 	"time"
 
@@ -25,9 +24,7 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
-
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
+	"go.etcd.io/etcd/tests/v3/integration/clientv3"
 )
 
 // TestBalancerUnderServerShutdownWatch expects that watch client
@@ -53,7 +50,7 @@ func TestBalancerUnderServerShutdownWatch(t *testing.T) {
 	defer watchCli.Close()
 
 	// wait for eps[lead] to be pinned
-	mustWaitPinReady(t, watchCli)
+	clientv3test.MustWaitPinReady(t, watchCli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -104,7 +101,7 @@ func TestBalancerUnderServerShutdownWatch(t *testing.T) {
 		if err == nil {
 			break
 		}
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail {
 			continue
 		}
 		t.Fatal(err)
@@ -163,7 +160,7 @@ func testBalancerUnderServerShutdownMutable(t *testing.T, op func(*clientv3.Clie
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -221,7 +218,7 @@ func testBalancerUnderServerShutdownImmutable(t *testing.T, op func(*clientv3.Cl
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -304,7 +301,7 @@ func testBalancerUnderServerStopInflightRangeOnRestart(t *testing.T, linearizabl
 	defer cli.Close()
 
 	// wait for eps[target] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -363,65 +360,3 @@ func testBalancerUnderServerStopInflightRangeOnRestart(t *testing.T, linearizabl
 	case <-donec:
 	}
 }
-
-// e.g. due to clock drifts in server-side,
-// client context times out first in server-side
-// while original client-side context is not timed out yet
-func isServerCtxTimeout(err error) bool {
-	if err == nil {
-		return false
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.DeadlineExceeded && strings.Contains(err.Error(), "context deadline exceeded")
-}
-
-// In grpc v1.11.3+ dial timeouts can error out with transport.ErrConnClosing. Previously dial timeouts
-// would always error out with context.DeadlineExceeded.
-func isClientTimeout(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err == context.DeadlineExceeded {
-		return true
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.DeadlineExceeded
-}
-
-func isCanceled(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err == context.Canceled {
-		return true
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.Canceled
-}
-
-func isUnavailable(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err == context.Canceled {
-		return true
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.Unavailable
-}
diff --git a/tests/integration/clientv3/examples/main_test.go b/tests/integration/clientv3/examples/main_test.go
index 31101209c..ff4f128b3 100644
--- a/tests/integration/clientv3/examples/main_test.go
+++ b/tests/integration/clientv3/examples/main_test.go
@@ -42,6 +42,7 @@ func forUnitTestsRunInMockedContext(mocking func(), example func()) {
 
 // TestMain sets up an etcd cluster if running the examples.
 func TestMain(m *testing.M) {
+	testutil.ExitInShortMode("Skipping: the tests require real cluster")
 	v := m.Run()
 	lazyCluster.Terminate()
 
diff --git a/tests/integration/clientv3/kv_test.go b/tests/integration/clientv3/kv_test.go
index 1b87515ea..8012a1d80 100644
--- a/tests/integration/clientv3/kv_test.go
+++ b/tests/integration/clientv3/kv_test.go
@@ -31,7 +31,6 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 )
@@ -838,7 +837,7 @@ func TestKVGetStoppedServerAndClose(t *testing.T) {
 	// this Get fails and triggers an asynchronous connection retry
 	_, err := cli.Get(ctx, "abc")
 	cancel()
-	if err != nil && !(isCanceled(err) || isClientTimeout(err)) {
+	if err != nil && !(IsCanceled(err) || IsClientTimeout(err)) {
 		t.Fatal(err)
 	}
 }
@@ -860,7 +859,7 @@ func TestKVPutStoppedServerAndClose(t *testing.T) {
 	// grpc finds out the original connection is down due to the member shutdown.
 	_, err := cli.Get(ctx, "abc")
 	cancel()
-	if err != nil && !(isCanceled(err) || isClientTimeout(err)) {
+	if err != nil && !(IsCanceled(err) || IsClientTimeout(err)) {
 		t.Fatal(err)
 	}
 
@@ -868,7 +867,7 @@ func TestKVPutStoppedServerAndClose(t *testing.T) {
 	// this Put fails and triggers an asynchronous connection retry
 	_, err = cli.Put(ctx, "abc", "123")
 	cancel()
-	if err != nil && !(isCanceled(err) || isClientTimeout(err) || isUnavailable(err)) {
+	if err != nil && !(IsCanceled(err) || IsClientTimeout(err) || IsUnavailable(err)) {
 		t.Fatal(err)
 	}
 }
diff --git a/tests/integration/clientv3/lease/doc.go b/tests/integration/clientv3/lease/doc.go
new file mode 100644
index 000000000..0061520d3
--- /dev/null
+++ b/tests/integration/clientv3/lease/doc.go
@@ -0,0 +1,15 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lease
diff --git a/tests/integration/clientv3/lease_test.go b/tests/integration/clientv3/lease/lease_test.go
similarity index 99%
rename from tests/integration/clientv3/lease_test.go
rename to tests/integration/clientv3/lease/lease_test.go
index c86d63fa0..e29c3e97e 100644
--- a/tests/integration/clientv3/lease_test.go
+++ b/tests/integration/clientv3/lease/lease_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package lease_test
 
 import (
 	"context"
diff --git a/tests/integration/clientv3/leasing_test.go b/tests/integration/clientv3/lease/leasing_test.go
similarity index 99%
rename from tests/integration/clientv3/leasing_test.go
rename to tests/integration/clientv3/lease/leasing_test.go
index b0edc3be9..0091505e2 100644
--- a/tests/integration/clientv3/leasing_test.go
+++ b/tests/integration/clientv3/lease/leasing_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package lease_test
 
 import (
 	"context"
diff --git a/tests/integration/clientv3/lease/main_test.go b/tests/integration/clientv3/lease/main_test.go
new file mode 100644
index 000000000..3328dda03
--- /dev/null
+++ b/tests/integration/clientv3/lease/main_test.go
@@ -0,0 +1,15 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lease_test
+
+import (
+	"testing"
+
+	"go.etcd.io/etcd/pkg/v3/testutil"
+)
+
+func TestMain(m *testing.M) {
+	testutil.MustTestMainWithLeakDetection(m)
+}
diff --git a/tests/integration/clientv3/maintenance_test.go b/tests/integration/clientv3/maintenance_test.go
index 93eb4eb52..f2b35150a 100644
--- a/tests/integration/clientv3/maintenance_test.go
+++ b/tests/integration/clientv3/maintenance_test.go
@@ -133,7 +133,7 @@ func TestMaintenanceSnapshotError(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	_, err = io.Copy(ioutil.Discard, rc2)
-	if err != nil && !isClientTimeout(err) {
+	if err != nil && !IsClientTimeout(err) {
 		t.Errorf("expected client timeout, got %v", err)
 	}
 }
@@ -192,7 +192,7 @@ func TestMaintenanceSnapshotErrorInflight(t *testing.T) {
 	// 300ms left and expect timeout while snapshot reading is in progress
 	time.Sleep(700 * time.Millisecond)
 	_, err = io.Copy(ioutil.Discard, rc2)
-	if err != nil && !isClientTimeout(err) {
+	if err != nil && !IsClientTimeout(err) {
 		t.Errorf("expected client timeout, got %v", err)
 	}
 }
diff --git a/tests/integration/clientv3/util.go b/tests/integration/clientv3/util.go
index 10df3b872..674a5d1c8 100644
--- a/tests/integration/clientv3/util.go
+++ b/tests/integration/clientv3/util.go
@@ -16,15 +16,18 @@ package clientv3test
 
 import (
 	"context"
+	"strings"
 	"testing"
 	"time"
 
 	"go.etcd.io/etcd/client/v3"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
-// mustWaitPinReady waits up to 3-second until connection is up (pin endpoint).
+// MustWaitPinReady waits up to 3-second until connection is up (pin endpoint).
 // Fatal on time-out.
-func mustWaitPinReady(t *testing.T, cli *clientv3.Client) {
+func MustWaitPinReady(t *testing.T, cli *clientv3.Client) {
 	// TODO: decrease timeout after balancer rewrite!!!
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	_, err := cli.Get(ctx, "foo")
@@ -33,3 +36,67 @@ func mustWaitPinReady(t *testing.T, cli *clientv3.Client) {
 		t.Fatal(err)
 	}
 }
+
+// IsServerCtxTimeout checks reason of the error.
+// e.g. due to clock drifts in server-side,
+// client context times out first in server-side
+// while original client-side context is not timed out yet
+func IsServerCtxTimeout(err error) bool {
+	if err == nil {
+		return false
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.DeadlineExceeded && strings.Contains(err.Error(), "context deadline exceeded")
+}
+
+// IsClientTimeout checks reason of the error.
+// In grpc v1.11.3+ dial timeouts can error out with transport.ErrConnClosing. Previously dial timeouts
+// would always error out with context.DeadlineExceeded.
+func IsClientTimeout(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err == context.DeadlineExceeded {
+		return true
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.DeadlineExceeded
+}
+
+func IsCanceled(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err == context.Canceled {
+		return true
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.Canceled
+}
+
+func IsUnavailable(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err == context.Canceled {
+		return true
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.Unavailable
+}
diff --git a/tests/integration/clientv3/watch_test.go b/tests/integration/clientv3/watch_test.go
index e68a31a02..87b5fe1d4 100644
--- a/tests/integration/clientv3/watch_test.go
+++ b/tests/integration/clientv3/watch_test.go
@@ -31,7 +31,6 @@ import (
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc"
 	"go.etcd.io/etcd/tests/v3/integration"
-
 	"google.golang.org/grpc/metadata"
 )
 
@@ -768,7 +767,7 @@ func TestWatchErrConnClosed(t *testing.T) {
 		defer close(donec)
 		ch := cli.Watch(context.TODO(), "foo")
 
-		if wr := <-ch; !isCanceled(wr.Err()) {
+		if wr := <-ch; !IsCanceled(wr.Err()) {
 			t.Errorf("expected context canceled, got %v", wr.Err())
 		}
 	}()