#!/usr/bin/env bash ROOT_MODULE="go.etcd.io/etcd" if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then echo "must be run from '${ROOT_MODULE}/v3' module directory" exit 255 fi function set_root_dir { ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3") } set_root_dir #### Convenient IO methods ##### COLOR_RED='\033[0;31m' COLOR_ORANGE='\033[0;33m' COLOR_GREEN='\033[0;32m' COLOR_LIGHTCYAN='\033[0;36m' COLOR_BLUE='\033[0;94m' COLOR_MAGENTA='\033[95m' COLOR_BOLD='\033[1m' COLOR_NONE='\033[0m' # No Color function log_error { >&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_warning { >&2 echo -n -e "${COLOR_ORANGE}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_callout { >&2 echo -n -e "${COLOR_LIGHTCYAN}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_cmd { >&2 echo -n -e "${COLOR_BLUE}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_success { >&2 echo -n -e "${COLOR_GREEN}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_info { >&2 echo -n -e "${COLOR_NONE}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } # From http://stackoverflow.com/a/12498485 function relativePath { # both $1 and $2 are absolute paths beginning with / # returns relative path to $2 from $1 local source=$1 local target=$2 local commonPart=$source local result="" while [[ "${target#"$commonPart"}" == "${target}" ]]; do # no match, means that candidate common part is not correct # go up one level (reduce common part) commonPart="$(dirname "$commonPart")" # and record that we went back, with correct / handling if [[ -z $result ]]; then result=".." else result="../$result" fi done if [[ $commonPart == "/" ]]; then # special case for root (no common path) result="$result/" fi # since we now have identified the common part, # compute the non-common part local forwardPart="${target#"$commonPart"}" # and now stick all parts together if [[ -n $result ]] && [[ -n $forwardPart ]]; then result="$result$forwardPart" elif [[ -n $forwardPart ]]; then # extra slash removal result="${forwardPart:1}" fi echo "$result" } #### Discovery of files/packages within a go module ##### # go_srcs_in_module [package] # returns list of all not-generated go sources in the current (dir) module. function go_srcs_in_module { go list -f "{{with \$c:=.}}{{range \$f:=\$c.GoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{end}}" ./... | grep -vE "(\\.pb\\.go|\\.pb\\.gw.go)" } # pkgs_in_module [optional:package_pattern] # returns list of all packages in the current (dir) module. # if the package_pattern is given, its being resolved. function pkgs_in_module { go list -mod=mod "${1:-./...}"; } # Prints subdirectory (from the repo root) for the current module. function module_subdir { relativePath "${ETCD_ROOT_DIR}" "${PWD}" } #### Running actions against multiple modules #### # run [command...] - runs given command, printing it first and # again if it failed (in RED). Use to wrap important test commands # that user might want to re-execute to shorten the feedback loop when fixing # the test. function run { local rpath local command rpath=$(module_subdir) # Quoting all components as the commands are fully copy-parsable: command=("${@}") command=("${command[@]@Q}") if [[ "${rpath}" != "." && "${rpath}" != "" ]]; then repro="(cd ${rpath} && ${command[*]})" else repro="${command[*]}" fi log_cmd "% ${repro}" "${@}" 2> >(while read -r line; do echo -e "${COLOR_NONE}stderr: ${COLOR_MAGENTA}${line}${COLOR_NONE}">&2; done) local error_code=$? if [ ${error_code} -ne 0 ]; then log_error -e "FAIL: (code:${error_code}):\\n % ${repro}" return ${error_code} fi } # run_for_module [module] [cmd] # executes given command in the given module for given pkgs. # module_name - "." (in future: tests, client, server) # cmd - cmd to be executed - that takes package as last argument function run_for_module { local module=${1:-"."} shift 1 ( cd "${ETCD_ROOT_DIR}/${module}" && "$@" ) } function module_dirs() { echo "api pkg raft client/pkg client/v2 client/v3 server etcdutl etcdctl tests ." } # maybe_run [cmd...] runs given command depending on the DRY_RUN flag. function maybe_run() { if ${DRY_RUN}; then log_warning -e "# DRY_RUN:\\n % ${*}" else run "${@}" fi } function modules() { modules=( "${ROOT_MODULE}/api/v3" "${ROOT_MODULE}/pkg/v3" "${ROOT_MODULE}/raft/v3" "${ROOT_MODULE}/client/pkg/v3" "${ROOT_MODULE}/client/v2" "${ROOT_MODULE}/client/v3" "${ROOT_MODULE}/server/v3" "${ROOT_MODULE}/etcdutl/v3" "${ROOT_MODULE}/etcdctl/v3" "${ROOT_MODULE}/tests/v3" "${ROOT_MODULE}/v3") echo "${modules[@]}" } function modules_exp() { for m in $(modules); do echo -n "${m}/... " done } # run_for_modules [cmd] # run given command across all modules and packages # (unless the set is limited using ${PKG} or / ${USERMOD}) function run_for_modules { local pkg="${PKG:-./...}" if [ -z "${USERMOD:-}" ]; then for m in $(module_dirs); do run_for_module "${m}" "$@" "${pkg}" || return "$?" done else run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?" fi } junitFilenamePrefix() { if [[ -z "${JUNIT_REPORT_DIR}" ]]; then echo "" return fi mkdir -p "${JUNIT_REPORT_DIR}" DATE=$( date +%s | base64 | head -c 15 ) echo "${JUNIT_REPORT_DIR}/junit_$DATE" } function produce_junit_xmlreport { local -r junit_filename_prefix=$1 if [[ -z "${junit_filename_prefix}" ]]; then return fi local junit_xml_filename junit_xml_filename="${junit_filename_prefix}.xml" # Ensure that gotestsum is run without cross-compiling run_go_tool gotest.tools/gotestsum --junitfile "${junit_xml_filename}" --raw-command cat "${junit_filename_prefix}"*.stdout || exit 1 if [ "${VERBOSE}" != "1" ]; then rm "${junit_filename_prefix}"*.stdout fi log_callout "Saved JUnit XML test report to ${junit_xml_filename}" } #### Running go test ######## # go_test [packages] [mode] [flags_for_package_func] [$@] # [mode] supports 3 states: # - "parallel": fastest as concurrently processes multiple packages, but silent # till the last package. See: https://github.com/golang/go/issues/2731 # - "keep_going" : executes tests package by package, but postpones reporting error to the last # - "fail_fast" : executes tests packages 1 by 1, exits on the first failure. # # [flags_for_package_func] is a name of function that takes list of packages as parameter # and computes additional flags to the go_test commands. # Use 'true' or ':' if you dont need additional arguments. # # depends on the VERBOSE top-level variable. # # Example: # go_test "./..." "keep_going" ":" --short # # The function returns != 0 code in case of test failure. function go_test { local packages="${1}" local mode="${2}" local flags_for_package_func="${3}" local junit_filename_prefix shift 3 local goTestFlags="" local goTestEnv="" ##### Create a junit-style XML test report in this directory if set. ##### JUNIT_REPORT_DIR=${JUNIT_REPORT_DIR:-} # If JUNIT_REPORT_DIR is unset, and ARTIFACTS is set, then have them match. if [[ -z "${JUNIT_REPORT_DIR:-}" && -n "${ARTIFACTS:-}" ]]; then export JUNIT_REPORT_DIR="${ARTIFACTS}" fi # Used to filter verbose test output. go_test_grep_pattern=".*" if [[ -n "${JUNIT_REPORT_DIR}" ]] ; then goTestFlags+="-v " goTestFlags+="-json " # Show only summary lines by matching lines like "status package/test" go_test_grep_pattern="^[^[:space:]]\+[[:space:]]\+[^[:space:]]\+/[^[[:space:]]\+" fi junit_filename_prefix=$(junitFilenamePrefix) if [ "${VERBOSE}" == "1" ]; then goTestFlags="-v" fi # Expanding patterns (like ./...) into list of packages local unpacked_packages=("${packages}") if [ "${mode}" != "parallel" ]; then # shellcheck disable=SC2207 # shellcheck disable=SC2086 if ! unpacked_packages=($(go list ${packages})); then log_error "Cannot resolve packages: ${packages}" return 255 fi fi local failures="" # execution of tests against packages: for pkg in "${unpacked_packages[@]}"; do local additional_flags # shellcheck disable=SC2086 additional_flags=$(${flags_for_package_func} ${pkg}) # shellcheck disable=SC2206 local cmd=( go test ${goTestFlags} ${additional_flags} "$@" ${pkg} ) # shellcheck disable=SC2086 if ! run env ${goTestEnv} ETCD_VERIFY="${ETCD_VERIFY}" "${cmd[@]}" | tee ${junit_filename_prefix:+"${junit_filename_prefix}.stdout"} | grep --binary-files=text "${go_test_grep_pattern}" ; then if [ "${mode}" != "keep_going" ]; then produce_junit_xmlreport "${junit_filename_prefix}" return 2 else failures=("${failures[@]}" "${pkg}") fi fi produce_junit_xmlreport "${junit_filename_prefix}" done if [ -n "${failures[*]}" ] ; then log_error -e "ERROR: Tests for following packages failed:\\n ${failures[*]}" return 2 fi } #### Other #### # tool_exists [tool] [instruction] # Checks whether given [tool] is installed. In case of failure, # prints a warning with installation [instruction] and returns !=0 code. # # WARNING: This depend on "any" version of the 'binary' that might be tricky # from hermetic build perspective. For go binaries prefer 'tool_go_run' function tool_exists { local tool="${1}" local instruction="${2}" if ! command -v "${tool}" >/dev/null; then log_warning "Tool: '${tool}' not found on PATH. ${instruction}" return 255 fi } # tool_get_bin [tool] - returns absolute path to a tool binary (or returns error) function tool_get_bin { local tool="$1" local pkg_part="$1" if [[ "$tool" == *"@"* ]]; then pkg_part=$(echo "${tool}" | cut -d'@' -f1) # shellcheck disable=SC2086 run go install ${GOBINARGS:-} "${tool}" || return 2 else # shellcheck disable=SC2086 run_for_module ./tools/mod run go install ${GOBINARGS:-} "${tool}" || return 2 fi # remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3". local cmd_base_name cmd_base_name=$(basename "${pkg_part}") if [[ ${cmd_base_name} =~ ^v[0-9]*$ ]]; then pkg_part=$(dirname "${pkg_part}") fi run_for_module ./tools/mod go list -f '{{.Target}}' "${pkg_part}" } # tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg. # The pkg versions must be defined in ./tools/mod directory. function tool_pkg_dir { run_for_module ./tools/mod run go list -f '{{.Dir}}' "${1}" } # tool_get_bin [tool] function run_go_tool { local cmdbin if ! cmdbin=$(GOARCH="" tool_get_bin "${1}"); then log_warning "Failed to install tool '${1}'" return 2 fi shift 1 GOARCH="" run "${cmdbin}" "$@" || return 2 } # assert_no_git_modifications fails if there are any uncommited changes. function assert_no_git_modifications { log_callout "Making sure everything is committed." if ! git diff --cached --exit-code; then log_error "Found staged by uncommited changes. Do commit/stash your changes first." return 2 fi if ! git diff --exit-code; then log_error "Found unstaged and uncommited changes. Do commit/stash your changes first." return 2 fi } # makes sure that the current branch is in sync with the origin branch: # - no uncommitted nor unstaged changes # - no differencing commits in relation to the origin/$branch function git_assert_branch_in_sync { local branch branch=$(run git rev-parse --abbrev-ref HEAD) # TODO: When git 2.22 popular, change to: # branch=$(git branch --show-current) if [[ $(run git status --porcelain --untracked-files=no) ]]; then log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes" log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard)" return 2 fi if [ -n "${branch}" ]; then ref_local=$(run git rev-parse "${branch}") ref_origin=$(run git rev-parse "origin/${branch}") if [ "x${ref_local}" != "x${ref_origin}" ]; then log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin." log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard origin/${branch})" return 2 fi else log_warning "Cannot verify consistency with the origin, as git is on detached branch." fi }