From 88ee005a0fbf76d10703d2d34a0fae4b2c3e7336 Mon Sep 17 00:00:00 2001
From: Piotr Tabor <ptab@google.com>
Date: Tue, 19 Jan 2021 20:00:40 +0100
Subject: [PATCH 1/2] Reduce verbosity of coverage collection.

---
 .travis.yml               |   5 +-
 Makefile                  |   2 +-
 codecov.yml               |  18 ++++++
 pkg/testutil/leak.go      |   3 +-
 scripts/codecov_upload.sh |  18 ++++++
 scripts/test_lib.sh       |   2 +-
 test.sh                   | 122 +++++++++++++++++++++++++++++---------
 7 files changed, 137 insertions(+), 33 deletions(-)
 create mode 100644 codecov.yml
 create mode 100755 scripts/codecov_upload.sh

diff --git a/.travis.yml b/.travis.yml
index 9378a58f4..ecfffe14e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -66,11 +66,10 @@ before_install:
   - if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi
 
 install:
-  - go get -t -v -d ./...
-  - (cd tests && go get -t -v -d ./...)
-  - (cd pkg && go get -t -v -d ./...)
+  - date
 
 script:
+  - date
   - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
   - >
     case "${TARGET}" in
diff --git a/Makefile b/Makefile
index b3bde92e0..cc12af5bf 100644
--- a/Makefile
+++ b/Makefile
@@ -177,7 +177,7 @@ docker-test-coverage:
 	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \
 	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
-	  /bin/bash -c "set -o pipefail; (COVERDIR=covdir PASSES='build build_cov cov' ./test.sh 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log) && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1"
+	  /bin/bash ./scripts/codecov_upload.sh docker-test-coverage-$(TEST_SUFFIX).log \
 	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 docker-test-coverage-$(TEST_SUFFIX).log
 
 
diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 000000000..aed42c014
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,18 @@
+codecov:
+  token: "6040de41-c073-4d6f-bbf8-d89256ef31e1"
+  disable_default_path_fixes: true
+
+fixes:
+  - "go.etcd.io/etcd/api/v3/::api/"
+  - "go.etcd.io/etcd/client/v3/::client/v3/"
+  - "go.etcd.io/etcd/client/v2/::client/v2/"
+  - "go.etcd.io/etcd/etcdctl/v3::etcdctl/"
+  - "go.etcd.io/etcd/pkg/v3::pkg/"
+  - "go.etcd.io/etcd/raft/v3::raft/"
+  - "go.etcd.io/etcd/server/v3::server/"
+
+ignore:
+  - "**/*.pb.go"
+  - "**/*.pb.gw.go"
+  - "tests/**/*"
+  - "go.etcd.io/etcd/tests/**/*"
\ No newline at end of file
diff --git a/pkg/testutil/leak.go b/pkg/testutil/leak.go
index 4b7a4a9b5..c24adcce8 100644
--- a/pkg/testutil/leak.go
+++ b/pkg/testutil/leak.go
@@ -124,7 +124,8 @@ func interestingGoroutines() (gs []string) {
 			strings.Contains(stack, "github.com/golang/glog.(*loggingT).flushDaemon") ||
 			strings.Contains(stack, "created by runtime.gc") ||
 			strings.Contains(stack, "created by text/template/parse.lex") ||
-			strings.Contains(stack, "runtime.MHeap_Scavenger") {
+			strings.Contains(stack, "runtime.MHeap_Scavenger") ||
+			strings.Contains(stack, "rcrypto/internal/boring.(*PublicKeyRSA).finalize") {
 			continue
 		}
 		gs = append(gs, stack)
diff --git a/scripts/codecov_upload.sh b/scripts/codecov_upload.sh
new file mode 100755
index 000000000..f75ad60e0
--- /dev/null
+++ b/scripts/codecov_upload.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Script used to collect and upload test coverage (mostly by travis).
+# Usage ./test_coverage_upload.sh [log_file]
+
+set -o pipefail
+
+LOG_FILE=${1:-test-coverage.log}
+
+# We collect the coverage
+COVERDIR=covdir PASSES='build build_cov cov' ./test.sh 2>&1 | tee "${LOG_FILE}"
+test_success="$?"
+
+# We try to upload whatever we have:
+bash <(curl -s https://codecov.io/bash) -f ./covdir/all.coverprofile -cF all || exit 2
+
+# Expose the original status of the test coverage execution.
+exit ${test_success}
diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh
index d5e422725..984689d62 100644
--- a/scripts/test_lib.sh
+++ b/scripts/test_lib.sh
@@ -143,7 +143,7 @@ function run {
   fi
 
   log_cmd "% ${repro}"
-  "${@}" 2> >(while read -r line; do echo -e "stderr: ${COLOR_MAGENTA}${line}${COLOR_NONE}" >&2; done)
+  "${@}" 2> >(while read -r line; do echo -e "${COLOR_NONE}stderr: ${COLOR_MAGENTA}${line}${COLOR_NONE}">&2; done)
   local error_code=$?
   if [ ${error_code} -ne 0 ]; then
     log_error -e "FAIL: (code:${error_code}):\n  % ${repro}"
diff --git a/test.sh b/test.sh
index be27e6beb..fdde245cf 100755
--- a/test.sh
+++ b/test.sh
@@ -207,12 +207,77 @@ function pkg_to_coverprofileflag {
   local prefix="${1}"
   local pkgs="${2}"
   local pkgs_normalized
-  pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
-  echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
+  prefix_normalized=$(echo "${prefix}" | tr "./ " "__+")
+  if [ "${pkgs}" == "./..." ]; then
+    pkgs_normalized="all"
+  else
+    pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
+  fi
+  mkdir -p "${coverdir}/${prefix_normalized}"
+  echo -n "-coverprofile=${coverdir}/${prefix_normalized}/${pkgs_normalized}.coverprofile"
 }
 
 function not_test_packages {
-  run_for_modules pkgs_in_module "./..." | grep -v /etcd/tests/v3/
+  for m in $(modules); do
+    if [[ $m =~ .*/etcd/tests/v3 ]]; then continue; fi
+    if [[ $m =~ .*/etcd/v3 ]]; then continue; fi
+    echo "${m}/..."
+  done
+}
+
+# split_dir [dir] [num]
+function split_dir {
+  local d="${1}"
+  local num="${2}"
+  local i=0
+  for f in "${d}/"*; do
+    local g=$(( "${i}" % "${num}" ))
+    mkdir -p "${d}_${g}"
+    mv "${f}" "${d}_${g}/"
+    (( i++ ))
+  done
+}
+
+function split_dir_pass {
+  split_dir ./covdir/integration 4
+}
+
+
+# merge_cov_files [coverdir] [outfile]
+# merges all coverprofile files into a single file in the given directory.
+function merge_cov_files {
+  local coverdir="${1}"
+  local cover_out_file="${2}"
+  log_callout "Merging coverage results in: ${coverdir}"
+  # gocovmerge requires not-empty test to start with:
+  echo "mode: set" > "${cover_out_file}"
+
+  local i=0
+  local count
+  count=$(find "${coverdir}"/*.coverprofile | wc -l)
+  for f in "${coverdir}"/*.coverprofile; do
+    # print once per 20 files
+    if ! (( "${i}" % 20 )); then
+      log_callout "${i} of ${count}: Merging file: ${f}"
+    fi
+    run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" 2>/dev/null
+    if [ -s "${coverdir}"/cover.tmp ]; then
+      mv "${coverdir}/cover.tmp" "${cover_out_file}"
+    fi
+    (( i++ ))
+  done
+}
+
+# merge_cov [coverdir]
+function merge_cov {
+  log_callout "[$(date)] Merging coverage files ..."
+  coverdir="${1}"
+  for d in "${coverdir}"/*/; do
+    d=${d%*/}  # remove the trailing "/"
+    merge_cov_files "${d}" "${d}.coverprofile" &
+  done
+  wait
+  merge_cov_files "${coverdir}" "${coverdir}/all.coverprofile"
 }
 
 function cov_pass {
@@ -230,7 +295,7 @@ function cov_pass {
   local coverdir
   coverdir=$(readlink -f "${COVERDIR}")
   mkdir -p "${coverdir}"
-  rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"
+  find "${coverdir}" -print0 -name '*.coverprofile' | xargs -0 rm
 
   local covpkgs
   covpkgs=$(not_test_packages)
@@ -240,45 +305,47 @@ function cov_pass {
 
   local failed=""
 
-  log_callout "Collecting coverage from unit tests ..."
+  log_callout "[$(date)] Collecting coverage from unit tests ..."
   for m in $(module_dirs); do
-    run_for_module "${m}" go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
+    run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \
        "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
   done
 
-  log_callout "Collecting coverage from integration tests ..."
-  run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration" \
+  log_callout "[$(date)] Collecting coverage from integration tests ..."
+  run_for_module "tests" go_test "./integration/..." "parallel" "pkg_to_coverprofileflag integration" \
       -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
   # integration-store-v2
   run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
       -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3"
   # integration_cluster_proxy
-  run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
+  run_for_module "tests" go_test "./integration/..." "parallel" "pkg_to_coverprofileflag integration_cluster_proxy" \
       -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
 
-  log_callout "Collecting coverage from e2e tests ..."
+  log_callout "[$(date)] Collecting coverage from e2e tests ..."
   # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
-  # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
-  run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
+  # as the coverage is collected from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
+  mkdir -p "${COVERDIR}/e2e"
+  COVERDIR="${COVERDIR}/e2e" run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
+  split_dir "${COVERDIR}/e2e" 10
 
-  log_callout "Collecting coverage from e2e tests with proxy ..."
-  run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
+  log_callout "[$(date)] Collecting coverage from e2e tests with proxy ..."
+  mkdir -p "${COVERDIR}/e2e_proxy"
+  COVERDIR="${COVERDIR}/e2e_proxy" run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
+  split_dir "${COVERDIR}/e2e_proxy" 10
 
-  log_callout "Merging coverage results ..."
-  local cover_out_file="${coverdir}/cover.out"
-  # gocovmerge requires not-empty test to start with:
-  echo "mode: set" > "${cover_out_file}"
+  local cover_out_file="${coverdir}/all.coverprofile"
+  merge_cov "${coverdir}"
 
-  # incrementally merge to get coverage data even if some coverage files are corrupted
-  for f in "${coverdir}"/*.coverprofile; do
-    echo "merging test coverage file ${f}"
-    run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
-    if [ -s "${coverdir}"/cover.tmp ]; then
-      mv "${coverdir}/cover.tmp" "${cover_out_file}"
-    fi
-  done
   # strip out generated files (using GNU-style sed)
-  sed --in-place '/generated.go/d' "${cover_out_file}" || true
+  sed --in-place -E "/[.]pb[.](gw[.])?go/d" "${cover_out_file}" || true
+
+  sed --in-place -E "s|go.etcd.io/etcd/api/v3/|api/|g" "${cover_out_file}" || true
+  sed --in-place -E "s|go.etcd.io/etcd/client/v3/|client/v3/|g" "${cover_out_file}" || true
+  sed --in-place -E "s|go.etcd.io/etcd/client/v2/|client/v2/|g" "${cover_out_file}" || true
+  sed --in-place -E "s|go.etcd.io/etcd/etcdctl/v3/|etcdctl/|g" "${cover_out_file}" || true
+  sed --in-place -E "s|go.etcd.io/etcd/pkg/v3/|pkg/|g" "${cover_out_file}" || true
+  sed --in-place -E "s|go.etcd.io/etcd/raft/v3/|raft/|g" "${cover_out_file}" || true
+  sed --in-place -E "s|go.etcd.io/etcd/server/v3/|server/|g" "${cover_out_file}" || true
 
   # held failures to generate the full coverage file, now fail
   if [ -n "$failed" ]; then
@@ -626,6 +693,7 @@ function run_pass {
   fi
 }
 
+log_callout "Starting at: $(date)"
 for pass in $PASSES; do
   run_pass "${pass}" "${@}"
 done

From 351bdb33c520f3462eed58993ab5a168ef076841 Mon Sep 17 00:00:00 2001
From: Piotr Tabor <ptab@google.com>
Date: Thu, 21 Jan 2021 20:21:19 +0100
Subject: [PATCH 2/2] Split intengration/clientv3 tests into multiple packages

They used to take >10min with coverage, so were causing interrupted
Travis runs.

Know thay fit in 100-150s (together), thanks also to parallel
execution.
---
 codecov.yml                                   | 10 +--
 pkg/testutil/testutil.go                      | 12 +++
 test.sh                                       |  6 +-
 tests/functional/tester/cluster_run.go        |  4 +-
 .../clientv3/concurrency/main_test.go         |  2 +
 .../{ => connectivity}/black_hole_test.go     | 15 ++--
 .../clientv3/{ => connectivity}/dial_test.go  | 19 ++---
 .../integration/clientv3/connectivity/doc.go  | 15 ++++
 .../clientv3/connectivity/main_test.go        | 15 ++++
 .../network_partition_test.go                 | 19 ++---
 .../server_shutdown_test.go                   | 79 ++-----------------
 .../clientv3/examples/main_test.go            |  1 +
 tests/integration/clientv3/kv_test.go         |  7 +-
 tests/integration/clientv3/lease/doc.go       | 15 ++++
 .../clientv3/{ => lease}/lease_test.go        |  2 +-
 .../clientv3/{ => lease}/leasing_test.go      |  2 +-
 tests/integration/clientv3/lease/main_test.go | 15 ++++
 .../integration/clientv3/maintenance_test.go  |  4 +-
 tests/integration/clientv3/util.go            | 71 ++++++++++++++++-
 tests/integration/clientv3/watch_test.go      |  3 +-
 20 files changed, 197 insertions(+), 119 deletions(-)
 rename tests/integration/clientv3/{ => connectivity}/black_hole_test.go (90%)
 rename tests/integration/clientv3/{ => connectivity}/dial_test.go (92%)
 create mode 100644 tests/integration/clientv3/connectivity/doc.go
 create mode 100644 tests/integration/clientv3/connectivity/main_test.go
 rename tests/integration/clientv3/{ => connectivity}/network_partition_test.go (92%)
 rename tests/integration/clientv3/{ => connectivity}/server_shutdown_test.go (86%)
 create mode 100644 tests/integration/clientv3/lease/doc.go
 rename tests/integration/clientv3/{ => lease}/lease_test.go (99%)
 rename tests/integration/clientv3/{ => lease}/leasing_test.go (99%)
 create mode 100644 tests/integration/clientv3/lease/main_test.go

diff --git a/codecov.yml b/codecov.yml
index aed42c014..a4b3b7f27 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -6,13 +6,13 @@ fixes:
   - "go.etcd.io/etcd/api/v3/::api/"
   - "go.etcd.io/etcd/client/v3/::client/v3/"
   - "go.etcd.io/etcd/client/v2/::client/v2/"
-  - "go.etcd.io/etcd/etcdctl/v3::etcdctl/"
-  - "go.etcd.io/etcd/pkg/v3::pkg/"
-  - "go.etcd.io/etcd/raft/v3::raft/"
-  - "go.etcd.io/etcd/server/v3::server/"
+  - "go.etcd.io/etcd/etcdctl/v3/::etcdctl/"
+  - "go.etcd.io/etcd/pkg/v3/::pkg/"
+  - "go.etcd.io/etcd/raft/v3/::raft/"
+  - "go.etcd.io/etcd/server/v3/::server/"
 
 ignore:
   - "**/*.pb.go"
   - "**/*.pb.gw.go"
   - "tests/**/*"
-  - "go.etcd.io/etcd/tests/**/*"
\ No newline at end of file
+  - "go.etcd.io/etcd/tests/**/*"
diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go
index db51304e7..f12566ab2 100644
--- a/pkg/testutil/testutil.go
+++ b/pkg/testutil/testutil.go
@@ -17,6 +17,7 @@ package testutil
 
 import (
 	"net/url"
+	"os"
 	"runtime"
 	"testing"
 	"time"
@@ -91,3 +92,14 @@ func SkipTestIfShortMode(t testing.TB, reason string) {
 		}
 	}
 }
+
+// ExitInShortMode closes the current process (with 0) if the short test mode detected.
+//
+// To be used in Test-main, where test context (testing.TB) is not available.
+//
+// Requires custom env-variable (GOLANG_TEST_SHORT) apart of `go test --short flag`.
+func ExitInShortMode(reason string) {
+	if os.Getenv("GOLANG_TEST_SHORT") == "true" {
+		os.Exit(0)
+	}
+}
diff --git a/test.sh b/test.sh
index fdde245cf..29a6cf181 100755
--- a/test.sh
+++ b/test.sh
@@ -91,7 +91,7 @@ function run_unit_tests {
   local pkgs="${1:-./...}"
   shift 1
   # shellcheck disable=SC2086
-  go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
+  GOLANG_TEST_SHORT=true go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
 }
 
 function unit_pass {
@@ -109,7 +109,7 @@ function integration_extra {
 
 function integration_pass {
   local pkgs=${USERPKG:-"./integration/..."}
-  run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
+  run_for_module "tests" go_test "${pkgs}" "parallel" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
   integration_extra "$@"
 }
 
@@ -307,7 +307,7 @@ function cov_pass {
 
   log_callout "[$(date)] Collecting coverage from unit tests ..."
   for m in $(module_dirs); do
-    run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \
+    GOLANG_TEST_SHORT=true run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \
        "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
   done
 
diff --git a/tests/functional/tester/cluster_run.go b/tests/functional/tester/cluster_run.go
index d73f60d84..256540d55 100644
--- a/tests/functional/tester/cluster_run.go
+++ b/tests/functional/tester/cluster_run.go
@@ -47,7 +47,7 @@ func (clus *Cluster) Run() {
 		clus.rd = round
 
 		if err := clus.doRound(); err != nil {
-			clus.lg.Warn(
+			clus.lg.Error(
 				"round FAIL",
 				zap.Int("round", clus.rd),
 				zap.Int("case", clus.cs),
@@ -316,7 +316,7 @@ func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) {
 }
 
 func (clus *Cluster) failed() {
-	clus.lg.Info(
+	clus.lg.Error(
 		"functional-tester FAIL",
 		zap.Int("round", clus.rd),
 		zap.Int("case", clus.cs),
diff --git a/tests/integration/clientv3/concurrency/main_test.go b/tests/integration/clientv3/concurrency/main_test.go
index d149d872f..9462b3a9b 100644
--- a/tests/integration/clientv3/concurrency/main_test.go
+++ b/tests/integration/clientv3/concurrency/main_test.go
@@ -33,6 +33,8 @@ func forUnitTestsRunInMockedContext(mocking func(), example func()) {
 
 // TestMain sets up an etcd cluster if running the examples.
 func TestMain(m *testing.M) {
+	testutil.ExitInShortMode("Skipping: the tests require real cluster")
+
 	v := m.Run()
 	lazyCluster.Terminate()
 	if v == 0 {
diff --git a/tests/integration/clientv3/black_hole_test.go b/tests/integration/clientv3/connectivity/black_hole_test.go
similarity index 90%
rename from tests/integration/clientv3/black_hole_test.go
rename to tests/integration/clientv3/connectivity/black_hole_test.go
index 0e5d71c60..eebdc2e67 100644
--- a/tests/integration/clientv3/black_hole_test.go
+++ b/tests/integration/clientv3/connectivity/black_hole_test.go
@@ -14,7 +14,7 @@
 
 // +build !cluster_proxy
 
-package clientv3test
+package connectivity_test
 
 import (
 	"context"
@@ -25,6 +25,7 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
+	"go.etcd.io/etcd/tests/v3/integration/clientv3"
 	"google.golang.org/grpc"
 )
 
@@ -111,7 +112,7 @@ func TestBalancerUnderBlackholeKeepAliveWatch(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Put(ctx, "foo", "bar")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -121,7 +122,7 @@ func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAliveDelete(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Delete(ctx, "foo")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -134,7 +135,7 @@ func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) {
 			If(clientv3.Compare(clientv3.Version("foo"), "=", 0)).
 			Then(clientv3.OpPut("foo", "bar")).
 			Else(clientv3.OpPut("foo", "baz")).Commit()
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -144,7 +145,7 @@ func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -154,7 +155,7 @@ func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) {
 func TestBalancerUnderBlackholeNoKeepAliveSerializableGet(t *testing.T) {
 	testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a", clientv3.WithSerializable())
-		if isClientTimeout(err) || isServerCtxTimeout(err) {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) {
 			return errExpected
 		}
 		return err
@@ -186,7 +187,7 @@ func testBalancerUnderBlackholeNoKeepAlive(t *testing.T, op func(*clientv3.Clien
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
diff --git a/tests/integration/clientv3/dial_test.go b/tests/integration/clientv3/connectivity/dial_test.go
similarity index 92%
rename from tests/integration/clientv3/dial_test.go
rename to tests/integration/clientv3/connectivity/dial_test.go
index dcb8c662c..5e4955969 100644
--- a/tests/integration/clientv3/dial_test.go
+++ b/tests/integration/clientv3/connectivity/dial_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package connectivity_test
 
 import (
 	"context"
@@ -26,21 +26,22 @@ import (
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/pkg/v3/transport"
 	"go.etcd.io/etcd/tests/v3/integration"
+	clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3"
 	"google.golang.org/grpc"
 )
 
 var (
 	testTLSInfo = transport.TLSInfo{
-		KeyFile:        "../../fixtures/server.key.insecure",
-		CertFile:       "../../fixtures/server.crt",
-		TrustedCAFile:  "../../fixtures/ca.crt",
+		KeyFile:        "../../../fixtures/server.key.insecure",
+		CertFile:       "../../../fixtures/server.crt",
+		TrustedCAFile:  "../../../fixtures/ca.crt",
 		ClientCertAuth: true,
 	}
 
 	testTLSInfoExpired = transport.TLSInfo{
-		KeyFile:        "../fixtures-expired/server.key.insecure",
-		CertFile:       "../fixtures-expired/server.crt",
-		TrustedCAFile:  "../fixtures-expired/ca.crt",
+		KeyFile:        "../../fixtures-expired/server.key.insecure",
+		CertFile:       "../../fixtures-expired/server.crt",
+		TrustedCAFile:  "../../fixtures-expired/ca.crt",
 		ClientCertAuth: true,
 	}
 )
@@ -62,7 +63,7 @@ func TestDialTLSExpired(t *testing.T) {
 		DialOptions: []grpc.DialOption{grpc.WithBlock()},
 		TLS:         tls,
 	})
-	if !isClientTimeout(err) {
+	if !clientv3test.IsClientTimeout(err) {
 		t.Fatalf("expected dial timeout error, got %v", err)
 	}
 }
@@ -84,7 +85,7 @@ func TestDialTLSNoConfig(t *testing.T) {
 			c.Close()
 		}
 	}()
-	if !isClientTimeout(err) {
+	if !clientv3test.IsClientTimeout(err) {
 		t.Fatalf("expected dial timeout error, got %v", err)
 	}
 }
diff --git a/tests/integration/clientv3/connectivity/doc.go b/tests/integration/clientv3/connectivity/doc.go
new file mode 100644
index 000000000..c90413b44
--- /dev/null
+++ b/tests/integration/clientv3/connectivity/doc.go
@@ -0,0 +1,15 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectivity
diff --git a/tests/integration/clientv3/connectivity/main_test.go b/tests/integration/clientv3/connectivity/main_test.go
new file mode 100644
index 000000000..f857c6883
--- /dev/null
+++ b/tests/integration/clientv3/connectivity/main_test.go
@@ -0,0 +1,15 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package connectivity
+
+import (
+	"testing"
+
+	"go.etcd.io/etcd/pkg/v3/testutil"
+)
+
+func TestMain(m *testing.M) {
+	testutil.MustTestMainWithLeakDetection(m)
+}
diff --git a/tests/integration/clientv3/network_partition_test.go b/tests/integration/clientv3/connectivity/network_partition_test.go
similarity index 92%
rename from tests/integration/clientv3/network_partition_test.go
rename to tests/integration/clientv3/connectivity/network_partition_test.go
index bc2849d5a..1860fe490 100644
--- a/tests/integration/clientv3/network_partition_test.go
+++ b/tests/integration/clientv3/connectivity/network_partition_test.go
@@ -14,7 +14,7 @@
 
 // +build !cluster_proxy
 
-package clientv3test
+package connectivity_test
 
 import (
 	"context"
@@ -27,6 +27,7 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
+	"go.etcd.io/etcd/tests/v3/integration/clientv3"
 	"google.golang.org/grpc"
 )
 
@@ -38,7 +39,7 @@ var errExpected = errors.New("expected error")
 func TestBalancerUnderNetworkPartitionPut(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Put(ctx, "a", "b")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -48,7 +49,7 @@ func TestBalancerUnderNetworkPartitionPut(t *testing.T) {
 func TestBalancerUnderNetworkPartitionDelete(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Delete(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -61,7 +62,7 @@ func TestBalancerUnderNetworkPartitionTxn(t *testing.T) {
 			If(clientv3.Compare(clientv3.Version("foo"), "=", 0)).
 			Then(clientv3.OpPut("foo", "bar")).
 			Else(clientv3.OpPut("foo", "baz")).Commit()
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -74,7 +75,7 @@ func TestBalancerUnderNetworkPartitionTxn(t *testing.T) {
 func TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
 			return errExpected
 		}
 		return err
@@ -87,7 +88,7 @@ func TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout(t *testing.
 func TestBalancerUnderNetworkPartitionLinearizableGetWithShortTimeout(t *testing.T) {
 	testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
 		_, err := cli.Get(ctx, "a")
-		if isClientTimeout(err) || isServerCtxTimeout(err) {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) {
 			return errExpected
 		}
 		return err
@@ -125,7 +126,7 @@ func testBalancerUnderNetworkPartition(t *testing.T, op func(*clientv3.Client, c
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add other endpoints for later endpoint switch
 	cli.SetEndpoints(eps...)
@@ -235,7 +236,7 @@ func testBalancerUnderNetworkPartitionWatch(t *testing.T, isolateLeader bool) {
 	defer watchCli.Close()
 
 	// wait for eps[target] to be pinned
-	mustWaitPinReady(t, watchCli)
+	clientv3test.MustWaitPinReady(t, watchCli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -290,7 +291,7 @@ func TestDropReadUnderNetworkPartition(t *testing.T) {
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add other endpoints for later endpoint switch
 	cli.SetEndpoints(eps...)
diff --git a/tests/integration/clientv3/server_shutdown_test.go b/tests/integration/clientv3/connectivity/server_shutdown_test.go
similarity index 86%
rename from tests/integration/clientv3/server_shutdown_test.go
rename to tests/integration/clientv3/connectivity/server_shutdown_test.go
index 32d6bb877..85aa08750 100644
--- a/tests/integration/clientv3/server_shutdown_test.go
+++ b/tests/integration/clientv3/connectivity/server_shutdown_test.go
@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package connectivity_test
 
 import (
 	"bytes"
 	"context"
-	"strings"
 	"testing"
 	"time"
 
@@ -25,9 +24,7 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
-
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
+	"go.etcd.io/etcd/tests/v3/integration/clientv3"
 )
 
 // TestBalancerUnderServerShutdownWatch expects that watch client
@@ -53,7 +50,7 @@ func TestBalancerUnderServerShutdownWatch(t *testing.T) {
 	defer watchCli.Close()
 
 	// wait for eps[lead] to be pinned
-	mustWaitPinReady(t, watchCli)
+	clientv3test.MustWaitPinReady(t, watchCli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -104,7 +101,7 @@ func TestBalancerUnderServerShutdownWatch(t *testing.T) {
 		if err == nil {
 			break
 		}
-		if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail {
+		if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail {
 			continue
 		}
 		t.Fatal(err)
@@ -163,7 +160,7 @@ func testBalancerUnderServerShutdownMutable(t *testing.T, op func(*clientv3.Clie
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -221,7 +218,7 @@ func testBalancerUnderServerShutdownImmutable(t *testing.T, op func(*clientv3.Cl
 	defer cli.Close()
 
 	// wait for eps[0] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -304,7 +301,7 @@ func testBalancerUnderServerStopInflightRangeOnRestart(t *testing.T, linearizabl
 	defer cli.Close()
 
 	// wait for eps[target] to be pinned
-	mustWaitPinReady(t, cli)
+	clientv3test.MustWaitPinReady(t, cli)
 
 	// add all eps to list, so that when the original pined one fails
 	// the client can switch to other available eps
@@ -363,65 +360,3 @@ func testBalancerUnderServerStopInflightRangeOnRestart(t *testing.T, linearizabl
 	case <-donec:
 	}
 }
-
-// e.g. due to clock drifts in server-side,
-// client context times out first in server-side
-// while original client-side context is not timed out yet
-func isServerCtxTimeout(err error) bool {
-	if err == nil {
-		return false
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.DeadlineExceeded && strings.Contains(err.Error(), "context deadline exceeded")
-}
-
-// In grpc v1.11.3+ dial timeouts can error out with transport.ErrConnClosing. Previously dial timeouts
-// would always error out with context.DeadlineExceeded.
-func isClientTimeout(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err == context.DeadlineExceeded {
-		return true
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.DeadlineExceeded
-}
-
-func isCanceled(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err == context.Canceled {
-		return true
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.Canceled
-}
-
-func isUnavailable(err error) bool {
-	if err == nil {
-		return false
-	}
-	if err == context.Canceled {
-		return true
-	}
-	ev, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	code := ev.Code()
-	return code == codes.Unavailable
-}
diff --git a/tests/integration/clientv3/examples/main_test.go b/tests/integration/clientv3/examples/main_test.go
index 31101209c..ff4f128b3 100644
--- a/tests/integration/clientv3/examples/main_test.go
+++ b/tests/integration/clientv3/examples/main_test.go
@@ -42,6 +42,7 @@ func forUnitTestsRunInMockedContext(mocking func(), example func()) {
 
 // TestMain sets up an etcd cluster if running the examples.
 func TestMain(m *testing.M) {
+	testutil.ExitInShortMode("Skipping: the tests require real cluster")
 	v := m.Run()
 	lazyCluster.Terminate()
 
diff --git a/tests/integration/clientv3/kv_test.go b/tests/integration/clientv3/kv_test.go
index 1b87515ea..8012a1d80 100644
--- a/tests/integration/clientv3/kv_test.go
+++ b/tests/integration/clientv3/kv_test.go
@@ -31,7 +31,6 @@ import (
 	"go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/tests/v3/integration"
-
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 )
@@ -838,7 +837,7 @@ func TestKVGetStoppedServerAndClose(t *testing.T) {
 	// this Get fails and triggers an asynchronous connection retry
 	_, err := cli.Get(ctx, "abc")
 	cancel()
-	if err != nil && !(isCanceled(err) || isClientTimeout(err)) {
+	if err != nil && !(IsCanceled(err) || IsClientTimeout(err)) {
 		t.Fatal(err)
 	}
 }
@@ -860,7 +859,7 @@ func TestKVPutStoppedServerAndClose(t *testing.T) {
 	// grpc finds out the original connection is down due to the member shutdown.
 	_, err := cli.Get(ctx, "abc")
 	cancel()
-	if err != nil && !(isCanceled(err) || isClientTimeout(err)) {
+	if err != nil && !(IsCanceled(err) || IsClientTimeout(err)) {
 		t.Fatal(err)
 	}
 
@@ -868,7 +867,7 @@ func TestKVPutStoppedServerAndClose(t *testing.T) {
 	// this Put fails and triggers an asynchronous connection retry
 	_, err = cli.Put(ctx, "abc", "123")
 	cancel()
-	if err != nil && !(isCanceled(err) || isClientTimeout(err) || isUnavailable(err)) {
+	if err != nil && !(IsCanceled(err) || IsClientTimeout(err) || IsUnavailable(err)) {
 		t.Fatal(err)
 	}
 }
diff --git a/tests/integration/clientv3/lease/doc.go b/tests/integration/clientv3/lease/doc.go
new file mode 100644
index 000000000..0061520d3
--- /dev/null
+++ b/tests/integration/clientv3/lease/doc.go
@@ -0,0 +1,15 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lease
diff --git a/tests/integration/clientv3/lease_test.go b/tests/integration/clientv3/lease/lease_test.go
similarity index 99%
rename from tests/integration/clientv3/lease_test.go
rename to tests/integration/clientv3/lease/lease_test.go
index c86d63fa0..e29c3e97e 100644
--- a/tests/integration/clientv3/lease_test.go
+++ b/tests/integration/clientv3/lease/lease_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package lease_test
 
 import (
 	"context"
diff --git a/tests/integration/clientv3/leasing_test.go b/tests/integration/clientv3/lease/leasing_test.go
similarity index 99%
rename from tests/integration/clientv3/leasing_test.go
rename to tests/integration/clientv3/lease/leasing_test.go
index b0edc3be9..0091505e2 100644
--- a/tests/integration/clientv3/leasing_test.go
+++ b/tests/integration/clientv3/lease/leasing_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package clientv3test
+package lease_test
 
 import (
 	"context"
diff --git a/tests/integration/clientv3/lease/main_test.go b/tests/integration/clientv3/lease/main_test.go
new file mode 100644
index 000000000..3328dda03
--- /dev/null
+++ b/tests/integration/clientv3/lease/main_test.go
@@ -0,0 +1,15 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lease_test
+
+import (
+	"testing"
+
+	"go.etcd.io/etcd/pkg/v3/testutil"
+)
+
+func TestMain(m *testing.M) {
+	testutil.MustTestMainWithLeakDetection(m)
+}
diff --git a/tests/integration/clientv3/maintenance_test.go b/tests/integration/clientv3/maintenance_test.go
index 93eb4eb52..f2b35150a 100644
--- a/tests/integration/clientv3/maintenance_test.go
+++ b/tests/integration/clientv3/maintenance_test.go
@@ -133,7 +133,7 @@ func TestMaintenanceSnapshotError(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	_, err = io.Copy(ioutil.Discard, rc2)
-	if err != nil && !isClientTimeout(err) {
+	if err != nil && !IsClientTimeout(err) {
 		t.Errorf("expected client timeout, got %v", err)
 	}
 }
@@ -192,7 +192,7 @@ func TestMaintenanceSnapshotErrorInflight(t *testing.T) {
 	// 300ms left and expect timeout while snapshot reading is in progress
 	time.Sleep(700 * time.Millisecond)
 	_, err = io.Copy(ioutil.Discard, rc2)
-	if err != nil && !isClientTimeout(err) {
+	if err != nil && !IsClientTimeout(err) {
 		t.Errorf("expected client timeout, got %v", err)
 	}
 }
diff --git a/tests/integration/clientv3/util.go b/tests/integration/clientv3/util.go
index 10df3b872..674a5d1c8 100644
--- a/tests/integration/clientv3/util.go
+++ b/tests/integration/clientv3/util.go
@@ -16,15 +16,18 @@ package clientv3test
 
 import (
 	"context"
+	"strings"
 	"testing"
 	"time"
 
 	"go.etcd.io/etcd/client/v3"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
-// mustWaitPinReady waits up to 3-second until connection is up (pin endpoint).
+// MustWaitPinReady waits up to 3-second until connection is up (pin endpoint).
 // Fatal on time-out.
-func mustWaitPinReady(t *testing.T, cli *clientv3.Client) {
+func MustWaitPinReady(t *testing.T, cli *clientv3.Client) {
 	// TODO: decrease timeout after balancer rewrite!!!
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	_, err := cli.Get(ctx, "foo")
@@ -33,3 +36,67 @@ func mustWaitPinReady(t *testing.T, cli *clientv3.Client) {
 		t.Fatal(err)
 	}
 }
+
+// IsServerCtxTimeout checks reason of the error.
+// e.g. due to clock drifts in server-side,
+// client context times out first in server-side
+// while original client-side context is not timed out yet
+func IsServerCtxTimeout(err error) bool {
+	if err == nil {
+		return false
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.DeadlineExceeded && strings.Contains(err.Error(), "context deadline exceeded")
+}
+
+// IsClientTimeout checks reason of the error.
+// In grpc v1.11.3+ dial timeouts can error out with transport.ErrConnClosing. Previously dial timeouts
+// would always error out with context.DeadlineExceeded.
+func IsClientTimeout(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err == context.DeadlineExceeded {
+		return true
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.DeadlineExceeded
+}
+
+func IsCanceled(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err == context.Canceled {
+		return true
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.Canceled
+}
+
+func IsUnavailable(err error) bool {
+	if err == nil {
+		return false
+	}
+	if err == context.Canceled {
+		return true
+	}
+	ev, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	code := ev.Code()
+	return code == codes.Unavailable
+}
diff --git a/tests/integration/clientv3/watch_test.go b/tests/integration/clientv3/watch_test.go
index e68a31a02..87b5fe1d4 100644
--- a/tests/integration/clientv3/watch_test.go
+++ b/tests/integration/clientv3/watch_test.go
@@ -31,7 +31,6 @@ import (
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc"
 	"go.etcd.io/etcd/tests/v3/integration"
-
 	"google.golang.org/grpc/metadata"
 )
 
@@ -768,7 +767,7 @@ func TestWatchErrConnClosed(t *testing.T) {
 		defer close(donec)
 		ch := cli.Watch(context.TODO(), "foo")
 
-		if wr := <-ch; !isCanceled(wr.Err()) {
+		if wr := <-ch; !IsCanceled(wr.Err()) {
 			t.Errorf("expected context canceled, got %v", wr.Err())
 		}
 	}()