etcd/test

#!/usr/bin/env bash
#
# Run all etcd tests
# ./test
# ./test -v
#
#
# Run specified test pass
#
# $ PASSES=unit ./test
# $ PASSES=integration ./test
#
#
# Run tests for one package
# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
# flag for different expectation
#
# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
#
# Run specified unit tests in one package
# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
# to run only "TestNew", set "TESTCASE="\bTestNew\b""
#
# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
#
#
# Run code coverage
# COVERDIR must either be a absolute path or a relative path to the etcd root
# $ COVERDIR=coverage PASSES="build_cov cov" ./test
set -e

# The test script is not supposed to make any changes to the files
# e.g. add/update missing dependencies. Such divergences should be 
# detected and trigger a failure that needs explicit developer's action.
export GOFLAGS=-mod=readonly

source ./build
source ./scripts/test_lib.sh

PASSES=${PASSES:-"fmt bom dep build unit"}
PKG=${PKG:-}

# Invoke ./tests/cover.test.bash for HTML output
COVER="--cover=${COVER:-true}"

if [ -z "$GOARCH" ]; then
  GOARCH=$(go env GOARCH);
fi

# determine the number of CPUs to use for Go tests
CPU=${CPU:-"1,2,4"}
echo "Running with CPU:" "${CPU}"

# determine whether target supports race detection
if [ -z "${RACE}" ] ; then
  if [ "$GOARCH" == "amd64" ]; then
    RACE="--race"
  else
    RACE="--race=false"
  fi
else
  RACE="--race=${RACE:-true}"
fi

# This options make sense for cases where SUT (System Under Test) is compiled by test.
COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}" "${COVER}")

RUN_ARG=()
if [ -n "${TESTCASE}" ]; then
  RUN_ARG=("-run=${TESTCASE}")
fi

function build_pass {
  log_callout "Building etcd"
  GO_BUILD_FLAGS="-v" etcd_build "${@}"
  GO_BUILD_FLAGS="-v" tools_build "${@}"
}

################# REGULAR TESTS ################################################

# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
function run_unit_tests {
  local pkgs="${1:-./...}"
  shift 1
  # shellcheck disable=SC2086
  go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
}

function unit_pass {
  run_for_modules run_unit_tests "$@"
}

function integration_extra {
  if [ -z "${PKG}" ] ; then
    if [[ -z "${RUN_ARG[*]}" ]]; then
      go_test "./client/... ./clientv3/..." "parallel" : -timeout="${TIMEOUT:-5m}" "${COMMON_TEST_FLAGS[@]}" --run=Example "$@" || return $?
    fi

    go_test "./etcdserver/api/v2store/..." "parallel" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
  else
    log_warning "integration_extra ignored when PKG is specified"
  fi
}

function integration_pass {
  local pkgs=${USERPKG:-"./integration/... ./clientv3/snapshot/... ./client/integration/... ./clientv3/integration/... ./contrib/raftexample"}
  go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
  integration_extra "$@"
}

function e2e_pass {
  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
  run_for_module "." go_test "./tests/e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
}

function integration_e2e_pass {
  run_pass "integration" "${@}"
  run_pass "e2e" "${@}"
}

# generic_checker [cmd...]
# executes given command in the current module, and clearly fails if it
# failed or returned output.
function generic_checker {
  local cmd=("$@")
  if ! output=$("${cmd[@]}"); then
    echo "${output}"
    log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
    return 255
  fi
  if [ -n "${output}" ]; then
    echo "${output}"
    log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
    return 255
  fi
}

function functional_pass {
  ./functional/build

  # Clean up any data and logs from previous runs
  rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup

  for a in 1 2 3; do
    ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 &
    pid="$!"
    agent_pids="${agent_pids} $pid"
  done

  for a in 1 2 3; do
    echo "Waiting for 'etcd-agent' on ${a}9027..."
    while ! nc -z localhost ${a}9027; do
      sleep 1
    done
  done

  echo "functional test START!"
  run ./bin/etcd-tester --config ./functional.yaml && log_success "'etcd-tester' succeeded"
  ETCD_TESTER_EXIT_CODE=$?
  echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}

  # shellcheck disable=SC2206
  agent_pids=($agent_pids)
  kill -s TERM "${agent_pids[@]}" || true

  if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
    tail -1000 /tmp/etcd-functional-1/etcd.log

    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
    tail -1000 /tmp/etcd-functional-2/etcd.log

    log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
    tail -1000 /tmp/etcd-functional-3/etcd.log

    log_error "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
    return ${ETCD_TESTER_EXIT_CODE}
  fi
  log_success "functional test PASS!"
}

function grpcproxy_pass {
  go_test "./integration ./clientv3/integration ./tests/e2e" "fail_fast" : \
      -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
}

################# COVERAGE #####################################################

# Builds artifacts used by tests/e2e in coverage mode.
function build_cov_pass {
  local out="${BINDIR:-./bin}"
  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test"
  run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl"
}

# pkg_to_coverflag [prefix] [pkgs]
# produces name of .coverprofile file to be used for tests of this package
function pkg_to_coverprofileflag {
  local prefix="${1}"
  local pkgs="${2}"
  local pkgs_normalized
  pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
  echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
}

function cov_pass {
  # gocovmerge merges coverage files
  if ! command -v gocovmerge >/dev/null; then
    log_error "gocovmerge not installed (need: ./scripts/install_tool.sh github.com/gyuho/gocovmerge)"
    return 255
  fi

  # shellcheck disable=SC2153
  if [ -z "$COVERDIR" ]; then
    log_error "COVERDIR undeclared"
    return 255
  fi

  if [ ! -f "bin/etcd_test" ]; then
    log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
    return 255
  fi

  local coverdir
  coverdir=$(readlink -f "${COVERDIR}")
  mkdir -p "${coverdir}"
  rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"

  local covpkgs
  covpkgs=$(pkgs_in_module "./..." | filter_out_integration_style_tests)
  local coverpkg_comma
  coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
  local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")

  local failed=""

  log_callout "Collecting coverage from unit tests ..."
  go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
      "${gocov_build_flags[@]}" "$@" || failed="$failed unit"

  log_callout "Collecting coverage from integration tests ..."
  go_test "./integration ./clientv3/integration ./client/integration" "keep_going" "pkg_to_coverprofileflag integration" \
      -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
  # integration-extra
  go_test "./client/... ./clientv3/..." "keep_going" "pkg_to_coverprofileflag integration_example" \
      -timeout=5m --run=Example "${gocov_build_flags[@]}" "$@" || failed="$failed integration-examples"
  # integration-store-v2
  go_test "./etcdserver/api/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
      -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed etcdserver_api_v2store"
  # integration_cluster_proxy
  go_test "./clientv3/... ./client/integration/... ./clientv3/snapshot/... ./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
      -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"

  log_callout "Collecting coverage from e2e tests ..."
  # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
  # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
  go_test "./tests/e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"

  log_callout "Collecting coverage from e2e tests with proxy ..."
  go_test "./tests/e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"

  log_callout "Merging coverage results ..."
  local cover_out_file="${coverdir}/cover.out"
  # gocovmerge requires not-empty test to start with:
  echo "mode: set" > "${cover_out_file}"

  # incrementally merge to get coverage data even if some coverage files are corrupted
  for f in "${coverdir}"/*.coverprofile; do
    echo "merging test coverage file ${f}"
    gocovmerge "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
    if [ -s "${coverdir}"/cover.tmp ]; then
      mv "${coverdir}/cover.tmp" "${cover_out_file}"
    fi
  done
  # strip out generated files (using GNU-style sed)
  sed --in-place '/generated.go/d' "${cover_out_file}" || true

  # held failures to generate the full coverage file, now fail
  if [ -n "$failed" ]; then
    for f in $failed; do
      log_error "--- FAIL:" "$f"
    done
    log_warning "Despite failures, you can see partial report:"
    log_warning "  go tool cover -html ${cover_out_file}"
    return 255
  fi

  log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
}

######### Code formatting checkers #############################################

function fmt_pass {
  toggle_failpoints disable

  # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
  # after resolving ore-existing errors.
  for p in shellcheck \
      markdown_you \
      goword \
      gofmt \
      govet \
      revive \
      license_header \
      receiver_name \
      commit_title \
      mod_tidy \
      ; do
    run_pass "${p}" "${@}"
  done
}

function shellcheck_pass {
  if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
    generic_checker run shellcheck -fgcc build test scripts/*.sh
  fi
}

function markdown_you_find_eschew_you {
  find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true
}

function markdown_you_pass {
  generic_checker markdown_you_find_eschew_you
}

function markdown_marker_pass {
  # TODO: check other markdown files when marker handles headers with '[]'
  if tool_exists "marker" "https://crates.io/crates/marker"; then
    generic_checker run marker --skip-http --root ./Documentation 2>&1
  fi
}

function govet_pass {
  run_for_modules generic_checker run go vet
}

function govet_shadow_pass {
  if tool_exists "shadow" "./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow"; then
    run_for_modules generic_checker run go vet -all -vettool="$(command -v shadow)"
  fi
}

function unparam_pass {
  if tool_exists "unparam" "./scripts/install_tool.sh mvdan.cc/unparam"; then
    run_for_modules generic_checker run unparam
  fi
}

function staticcheck_pass {
  if tool_exists "staticcheck" "./scripts/install_tool.sh honnef.co/go/tools/cmd/staticcheck"; then
    run_for_modules generic_checker run staticcheck
  fi
}

function revive_pass {
  if tool_exists "revive" "./scripts/install_tool.sh github.com/mgechev/revive"; then
    run_for_modules generic_checker run revive -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
  fi
}

function unconvert_pass {
  if tool_exists "unconvert" "./scripts/install_tool.sh github.com/mdempsky/unconvert"; then
    run_for_modules generic_checker run unconvert -v
  fi
}

function ineffassign_per_package {
  mapfile -t gofiles < <(go_srcs_in_module "$1")
  run ineffassign "${gofiles[@]}"
}

function ineffassign_pass {
  if tool_exists "ineffassign" "./scripts/install_tool.sh github.com/gordonklaus/ineffassign"; then
    run_for_modules generic_checker ineffassign_per_package
  fi
}

function nakedret_pass {
  if tool_exists "nakedret" "./scripts/install_tool.sh github.com/alexkohler/nakedret"; then
    run_for_modules generic_checker run nakedret
  fi
}

function license_header_pass {
  mapfile -t gofiles < <(go_srcs_in_module "$1")
  for file in "${gofiles[@]}"; do
    if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
      licRes="${licRes}"$(echo -e "  ${file}")
    fi
  done
  if [ -n "${licRes}" ]; then
    log_error -e "license header checking failed:\\n${licRes}"
    return 255
  fi
}

function receiver_name_for_package {
  mapfile -t gofiles < <(go_srcs_in_module "$1")
  recvs=$(grep 'func ([^*]' "${gofiles[@]}"  | tr  ':' ' ' |  \
    awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
    grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
  if [ -n "${recvs}" ]; then
    # shellcheck disable=SC2206
    recvs=($recvs)
    for recv in "${recvs[@]}"; do
      log_error "Mismatched receiver for $recv..."
      grep "$recv" "${gofiles[@]}" | grep 'func ('
    done
    return 255
  fi
}

function receiver_name_pass {
  run_for_modules receiver_name_for_package
}

function commit_title_pass {
  git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
    commitMsg=$(echo "$l" | cut -f2- -d' ')
    if [[ "$commitMsg" == Merge* ]]; then
      # ignore "Merge pull" commits
      continue
    fi
    if [[ "$commitMsg" == Revert* ]]; then
      # ignore revert commits
      continue
    fi

    pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':')
    spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
    commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
    if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
      log_error "$l"...
      log_error "Expected commit title format '<package>{\", \"<package>}: <description>'"
      log_error "Got: $l"
      return 255
    fi
  done
}

# goword_for_package package
# checks spelling and comments in the 'package' in the current module
function goword_for_package {
  mapfile -t gofiles < <(go_srcs_in_module "$1")
  # only check for broke exported godocs
  gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
  if [ -n "$gowordRes" ]; then
    log_error -e "goword checking failed:\\n${gowordRes}"
    return 255
  fi
  # check some spelling
  gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
  if [ -n "$gowordRes" ]; then
    log_error -e "goword checking failed:\\n${gowordRes}"
    return 255
  fi
}

function goword_pass {
  if tool_exists "goword" "./scripts_install_tool.sh github.com/chzchzchz/goword"; then
    run_for_modules goword_for_package
  fi
}

function go_fmt_for_package {
  # We utilize 'go fmt' to find all files suitable for formatting,
  # but reuse full power gofmt to perform just RO check.
  go fmt -n "$1" | sed 's| -w | -d |g' | sh
}

function gofmt_pass {
  run_for_modules generic_checker go_fmt_for_package
}

function bom_pass {
  if ! command -v license-bill-of-materials >/dev/null; then
    log_warning "./license-bill-of-materials not FOUND"
    log_warning "USE: ./scripts/install_tool.sh github.com/coreos/license-bill-of-materials"
    return
  fi
  log_callout "Checking bill of materials..."
  # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
  run license-bill-of-materials \
    --override-file bill-of-materials.override.json \
    go.etcd.io/etcd/v3 go.etcd.io/etcd/v3/etcdctl >bom-now.json || true
  if ! diff bill-of-materials.json bom-now.json; then
    log_error "modularized licenses do not match given bill of materials"
    return 255
  fi
  rm bom-now.json
}

######## VARIOUS CHECKERS ######################################################

function dep_pass {
  log_callout "Checking package dependencies..."
  # don't pull in etcdserver package
  pushd clientv3 >/dev/null
  badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
  deps=$(go list -f '{{ .Deps }}'  | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "")
  popd >/dev/null
  if [ -n "$deps" ]; then
    log_error -e "clientv3 has masked dependencies:\\n${deps}"
    return 255
  fi
}

function release_pass {
  rm -f ./bin/etcd-last-release
  # to grab latest patch release; bump this up for every minor release
  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
  if [ -n "$MANUAL_VER" ]; then
    # in case, we need to test against different version
    UPGRADE_VER=$MANUAL_VER
  fi
  if [[ -z ${UPGRADE_VER} ]]; then
    UPGRADE_VER="v3.3.0"
    log_warning "fallback to" ${UPGRADE_VER}
  fi

  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
  log_callout "Downloading $file"

  set +e
  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
  local result=$?
  set -e
  case $result in
    0)  ;;
    *)  log_error "--- FAIL:" ${result}
      return $result
      ;;
  esac

  tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
  mkdir -p ./bin
  mv /tmp/etcd ./bin/etcd-last-release
}

function mod_tidy_pass {
  # Watch for upstream solution: https://github.com/golang/go/issues/27005
  local tmpModDir
  tmpModDir=$(mktemp -d --suffix "etcd-mod")
  cp "./go.mod" "./go.sum" "${tmpModDir}"

  # Guarantees keeping go.sum minimal
  # If this is causing too much problems, we should
  # stop controlling go.sum at all.
  rm go.sum
  go mod tidy

  set +e
  local tmpFileGoModInSync
  diff -C 5 "${tmpModDir}/go.mod" "./go.mod"
  tmpFileGoModInSync="$?"

  local tmpFileGoSumInSync
  diff -C 5 "${tmpModDir}/go.sum" "./go.sum"
  tmpFileGoSumInSync="$?"
  set -e

  # Bring back initial state
  mv "${tmpModDir}/go.mod" "./go.mod"
  mv "${tmpModDir}/go.sum" "./go.sum"

  if [ "${tmpFileGoModInSync}" -ne 0 ]; then
    log_error "./go.mod is not in sync with 'go mod tidy'"
    return 255
  fi
  if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
    log_error "./go.sum is not in sync with 'rm go.sum; go mod tidy'"
    return 255
  fi
}

########### MAIN ###############################################################

function run_pass {
  local pass="${1}"
  shift 1
  log_callout -e "\n'${pass}' started at $(date)"
  if "${pass}_pass" "$@" ; then
    log_success "'${pass}' completed at $(date)"
  else
    log_error "FAIL: '${pass}' failed at $(date)"
    exit 255
  fi
}

for pass in $PASSES; do
  run_pass "${pass}" "${@}"
done

log_success "SUCCESS"