Modernize release script:

- making sure the DRY_RUN mode can finish e2e, so e.g. commits to
local copy of repository are OK in dry-run (while git pushes are NOT).
  - better interaction with ./test_lib.sh script.
  - more consistent logging
  - bringing back s390x architecture that on go 1.14.3 seems to work as
expected.

scripts/build-binary

@@ -24,7 +24,6 @@ function setup_env {
   fi
 
   pushd "${proj}" >/dev/null
-    run git checkout master
     run git fetch --all
     git_assert_branch_in_sync || exit 2
     run git checkout "${ver}"
@@ -76,8 +75,7 @@ function main {
     if [ ${GOOS} == "linux" ]; then
       TARGET_ARCHS+=("arm64")
       TARGET_ARCHS+=("ppc64le")
-#  TODO: Reenable when https://github.com/etcd-io/etcd/issues/12496 is fixed.
+      TARGET_ARCHS+=("s390x")
-#      TARGET_ARCHS+=("s390x")
     fi
 
     for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do

scripts/build-release.sh

@@ -24,8 +24,7 @@ pushd "${ETCD_ROOT}" >/dev/null
   log_callout "Building etcd binary..."
   ./scripts/build-binary "${VERSION}"
 
-#  TODO: Add "s390x" when https://github.com/etcd-io/etcd/issues/12496 is fixed.
+  for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do
-  for TARGET_ARCH in "amd64" "arm64" "ppc64le"; do
     log_callout "Building ${TARGET_ARCH} docker image..."
     GOOS=linux GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker "${VERSION}"
   done

scripts/release

@@ -5,6 +5,9 @@ set -o nounset
 set -o pipefail
 
 source ./scripts/test_lib.sh
+source ./scripts/release_mod.sh
+
+DRY_RUN=${DRY_RUN:-true}
 
 help() {
   echo "$(basename "$0") [version]"
@@ -27,7 +30,7 @@ help() {
 main() {
   VERSION=$1
   if [[ ! "${VERSION}" =~ [0-9]+.[0-9]+.[0-9]+ ]]; then
-    echo "Expected 'version' param of the form '<major-version>.<minor-version>.<patch-version>' but got '${VERSION}'"
+    log_error "Expected 'version' param of the form '<major-version>.<minor-version>.<patch-version>' but got '${VERSION}'"
     exit 1
   fi
   RELEASE_VERSION="v${VERSION}"
@@ -35,11 +38,18 @@ main() {
   BRANCH=${BRANCH:-"release-${MINOR_VERSION}"}
   REPOSITORY=${REPOSITORY:-"https://github.com/etcd-io/etcd"}
 
+  log_warning "DRY_RUN=${DRY_RUN}"
+  log_callout "RELEASE_VERSION=${RELEASE_VERSION}"
+  log_callout "MINOR_VERSION=${MINOR_VERSION}"
+  log_callout "BRANCH=${BRANCH}"
+  log_callout "REPOSITORY=${REPOSITORY}"
+  log_callout ""
+  
   # Required to enable 'docker manifest ...'
   export DOCKER_CLI_EXPERIMENTAL=enabled
 
   if ! command -v docker >/dev/null; then
-    echo "cannot find docker"
+    log_error "cannot find docker"
     exit 1
   fi
 
@@ -55,6 +65,9 @@ main() {
     run git clone "${REPOSITORY}" --branch "${BRANCH}"
   fi
   run cd "${reldir}/etcd" || exit 2
+  # mark local directory as root for test_lib scripts executions
+  set_root_dir
+
   run git checkout "${BRANCH}" || exit 2
   run git pull origin
   git_assert_branch_in_sync || exit 2
@@ -89,7 +102,7 @@ main() {
         exit 1
       fi
       log_callout "Updating modules definitions"
-      TARGET_VERSION="v${VERSION}" DRY_RUN=false ./scripts/release_mod.sh update_versions
+      TARGET_VERSION="v${VERSION}" update_versions_cmd
 
       log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go"
       sed -i "s/${source_version}/${VERSION}/g" api/version/version.go
@@ -97,11 +110,11 @@ main() {
 
 
     log_callout "Building etcd and checking --version output"
-    run ./build
+    run ./build.sh
     local etcd_version
     etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
     if [[ "${etcd_version}" != "${VERSION}" ]]; then
-      echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
+      log_error "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
       exit 1
     fi
 
@@ -118,7 +131,7 @@ main() {
     if [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then
       read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm
       [[ "${confirm,,}" == "y" ]] || exit 1
-      git push
+      maybe_run git push
     fi
 
     # Tag release.
@@ -126,20 +139,20 @@ main() {
       log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
     else
       log_callout "Tagging release..."
-      REMOTE_REPO="origin" DRY_RUN=false ./scripts/release_mod.sh push_mod_tags
+      REMOTE_REPO="origin" push_mod_tags_cmd
     fi
 
     # Verify the latest commit has the version tag
     local tag="$(git describe --exact-match HEAD)"
     if [ "${tag}" != "${RELEASE_VERSION}" ]; then
-      echo "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}"
+      log_error "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}"
       exit 1
     fi
 
     # Verify the version tag is on the right branch
     local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3)
     if [ "${branch}" != "${BRANCH}" ]; then
-      echo "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'"
+      log_error "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'"
       exit 1
     fi
   fi
@@ -150,7 +163,12 @@ main() {
     log_warning "Skipping release build step. /release directory already exists."
   else
     log_callout "Building release..."
-    REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}"
+    if ${DRY_RUN}; then
+      log_warning "In DRY_RUN mode we clone the current release directory (as there was no push)"
+      REPOSITORY=$(pwd) ./scripts/build-release.sh "${RELEASE_VERSION}"
+    else
+      REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}"
+    fi
   fi
 
   # Sanity checks.
@@ -158,67 +176,67 @@ main() {
   "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdctl" version | grep -q "etcdctl version: ${VERSION}" || true
 
   # Generate SHA256SUMS
-  echo -e "Generating sha256sums of release artifacts.\n"
+  log_callout "Generating sha256sums of release artifacts."
   pushd ./release
   ls . | grep -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS
   popd
   if [ -s ./release/SHA256SUMS ]; then
     cat ./release/SHA256SUMS
   else
-    echo "sha256sums is not valid. Aborting."
+    log_error "sha256sums is not valid. Aborting."
     exit 1
   fi
 
   # Upload artifacts.
   if [ "${NO_UPLOAD}" == 1 ]; then
-    echo "Skipping artifact upload to gs://etcd. --no-upload flat is set."
+    log_callout "Skipping artifact upload to gs://etcd. --no-upload flat is set."
   else
     read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " -r confirm
     [[ "${confirm,,}" == "y" ]] || exit 1
-    gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/"
+    maybe_run gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/"
-    gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/"
+    maybe_run gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/"
-    gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/"
+    maybe_run gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/"
-    gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/"
+    maybe_run gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/"
   fi
 
   # Push images.
   if [ "${NO_DOCKER_PUSH}" == 1 ]; then
-    echo "Skipping docker push. --no-docker-push flat is set."
+    log_callout "Skipping docker push. --no-docker-push flat is set."
   else
     read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " -r confirm
     [[ "${confirm,,}" == "y" ]] || exit 1
     # shellcheck disable=SC2034
     for i in {1..5}; do
       docker login quay.io && break
-      echo "login failed, retrying"
+      log_warning "login failed, retrying"
     done
-    gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+    maybe_run gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
 
     # NB: `docker manifest create` insists on at least one manifest.
     # We overwrite it later with --amend anyway.
-    docker manifest create "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-amd64"
+    maybe_run docker manifest create "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-amd64"
-    docker manifest create "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-amd64"
+    maybe_run docker manifest create "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-amd64"
 
     for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do
-      echo "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}"
+      log_callout "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}"
       docker push "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
       docker manifest create --amend "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
       docker manifest annotate "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}"
 
-      echo "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}"
+      log_callout "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}"
-      gcloud docker -- push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
+      maybe_run gcloud docker -- push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
-      docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
+      maybe_run docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
-      docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}"
+      maybe_run docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}"
     done
 
-    echo "Pushing container manifest list to quay.io ${RELEASE_VERSION}"
+    log_callout "Pushing container manifest list to quay.io ${RELEASE_VERSION}"
-    docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}"
+    maybe_run docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}"
 
-    echo "Pushing container manifest list to gcr.io ${RELEASE_VERSION}"
+    log_callout "Pushing container manifest list to gcr.io ${RELEASE_VERSION}"
-    gcloud docker -- manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"
+    maybe_run gcloud docker -- manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"
 
-    echo "Setting permissions using gsutil..."
+    log_callout "Setting permissions using gsutil..."
-    gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+    maybe_run gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
   fi
 
   ### Release validation
@@ -228,7 +246,7 @@ main() {
   for IMAGE in "quay.io/coreos/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"; do
     local image_version=$(docker run --rm "${IMAGE}" etcd --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]')
     if [ "${image_version}" != "${VERSION}" ]; then
-      echo "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}"
+      log_error "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}"
       exit 1
     fi
   done
@@ -239,15 +257,15 @@ main() {
   tar -zx -C downloads -f "downloads/${BINARY_TGZ}"
   local binary_version=$("./downloads/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]')
   if [ "${binary_version}" != "${VERSION}" ]; then
-    echo "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}"
+    log_error "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}"
     exit 1
   fi
 
   # TODO: signing process
-  echo ""
+  log_warning ""
-  echo "WARNING: The release has not been signed and published to github. This must be done manually."
+  log_warning "WARNING: The release has not been signed and published to github. This must be done manually."
-  echo ""
+  log_warning ""
-  echo "Success."
+  log_success "Success."
   exit 0
 }
 

scripts/release_mod.sh

@@ -12,15 +12,10 @@
 
 set -e
 
-DRY_RUN=${DRY_RUN:-true}
-
-if ! [[ "$0" =~ scripts/release_mod.sh ]]; then
-  echo "must be run from repository root"
-  exit 255
-fi
-
 source ./scripts/test_lib.sh
 
+DRY_RUN=${DRY_RUN:-true}
+
 # _cmd prints help message
 function _cmd() {
   log_error "Command required: ${0} [cmd]"
@@ -29,15 +24,6 @@ function _cmd() {
   log_info "  - push_mod_tags    - Tags HEAD with all modules versions tags and pushes it to \${REMOTE_REPO}."
 }
 
-# maybe_run [cmd...] runs given command depending on the DRY_RUN flag.
-function maybe_run() {
-  if ${DRY_RUN}; then
-    log_warning -e "# DRY_RUN:\n  % ${*}"
-  else
-    run "${@}"
-  fi
-}
-
 # update_module_version [v2version] [v3version]
 #   Updates versions of cross-references in all internal references in current module.
 function update_module_version() {
@@ -48,12 +34,12 @@ function update_module_version() {
 
   v3deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v3")
   for dep in ${v3deps}; do
-    maybe_run go mod edit -require "${dep}@${v3version}"
+    run go mod edit -require "${dep}@${v3version}"
   done
 
   v2deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v2")
   for dep in ${v2deps}; do
-    maybe_run go mod edit -require "${dep}@${v2version}"
+    run go mod edit -require "${dep}@${v2version}"
   done
 }
 
@@ -124,15 +110,18 @@ function push_mod_tags_cmd {
     # The sleep is ugly hack that guarantees that 'git describe' will
     # consider main-module's tag as the latest.
     run sleep 2
-    maybe_run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}"
+    run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}"
     tags=("${tags[@]}" "${tag}")
   done
   maybe_run git push -f "${REMOTE_REPO}" "${tags[@]}"
 }
 
-"${1}_cmd"
+# only release_mod when called directly, not sourced
+if echo "$0" | grep -E "release_mod.sh$" >/dev/null; then
+  "${1}_cmd"
 
-if "${DRY_RUN}"; then
+  if "${DRY_RUN}"; then
-  log_info
+    log_info
-  log_warning "WARNING: It was a DRY_RUN. No files were modified."
+    log_warning "WARNING: It was a DRY_RUN. No files were modified."
-fi
+  fi
+fi

scripts/test_lib.sh

@@ -7,7 +7,11 @@ if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then
   exit 255
 fi
 
-ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3")
+function set_root_dir {
+  ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3")
+}
+
+set_root_dir
 
 ####   Convenient IO methods #####
 
@@ -20,6 +24,7 @@ COLOR_MAGENTA='\033[95m'
 COLOR_BOLD='\033[1m'
 COLOR_NONE='\033[0m' # No Color
 
+
 function log_error {
   >&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}"
   >&2 echo "$@"
@@ -162,6 +167,15 @@ function module_dirs() {
   echo "api pkg raft client/v2 client/v3 server etcdctl tests ."
 }
 
+# maybe_run [cmd...] runs given command depending on the DRY_RUN flag.
+function maybe_run() {
+  if ${DRY_RUN}; then
+    log_warning -e "# DRY_RUN:\n  % ${*}"
+  else
+    run "${@}"
+  fi
+}
+
 function modules() {
   modules=(
     "${ROOT_MODULE}/api/v3"
@@ -344,14 +358,18 @@ function git_assert_branch_in_sync {
   branch=$(git branch --show-current)
   if [[ $(run git status --porcelain --untracked-files=no) ]]; then
     log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes"
-    log_error "Consider cleaning up / renaming this directory."
+    log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard)"
     return 2
   fi
-  ref_local=$(run git rev-parse "${branch}")
+  if [ -n "${branch}" ]; then
-  ref_origin=$(run git rev-parse "origin/${branch}")
+    ref_local=$(run git rev-parse "${branch}")
-  if [ "x${ref_local}" != "x${ref_origin}" ]; then
+    ref_origin=$(run git rev-parse "origin/${branch}")
-    log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin."
+    if [ "x${ref_local}" != "x${ref_origin}" ]; then
-    log_error "Consider cleaning up / renaming this directory."
+      log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin."
-    return 2
+      log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard origin/${branch})"
+      return 2
+    fi
+  else
+    log_warning "Cannot verify consistency with the origin, as git is on detached branch."
   fi
 }

tests/functional/Dockerfile

@@ -26,11 +26,11 @@ ADD ./tests/functional/functional.yaml /functional.yaml
 
 RUN go get -v go.etcd.io/gofail \
   && pushd ${GOPATH}/src/go.etcd.io/etcd \
-  && GO_BUILD_FLAGS="-v" ./build \
+  && GO_BUILD_FLAGS="-v" ./build.sh \
   && mkdir -p /bin \
   && cp ./bin/etcd /bin/etcd \
   && cp ./bin/etcdctl /bin/etcdctl \
-  && GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \
+  && GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build.sh \
   && cp ./bin/etcd /bin/etcd-failpoints \
   && ./tests/functional/build \
   && cp ./bin/etcd-agent /bin/etcd-agent \

tests/functional/README.md

@@ -39,7 +39,7 @@ Proxy layer that simulates various network conditions.
 Test locally
 
 ```bash
-$ ./build
+$ ./build.sh
 $ ./bin/etcd
 
 $ make build-functional
@@ -201,7 +201,7 @@ $ curl -L http://localhost:2378/blackhole-tx -X DELETE
 Trigger leader election
 
 ```bash
-$ ./build
+$ ./build.sh
 $ make build-functional
 
 $ rm -rf /tmp/etcd-proxy-data.s*

tests/functional/cmd/etcd-proxy/main.go

@@ -54,7 +54,7 @@ See README.md for more examples.
 Example:
 
 # build etcd
-$ ./build
+$ ./build.sh
 $ ./bin/etcd
 
 # build etcd-proxy

tools/local-tester/README.md

@@ -11,7 +11,7 @@ local-tester depends on `goreman` to manage its processes and `bash` to run faul
 local-tester needs `etcd`, `benchmark`, and `bridge` binaries. To build these binaries, run the following from the etcd repository root:
 
 ```sh
-./build
+./build.sh
 pushd tools/benchmark/ && go build && popd
 pushd tools/local-tester/bridge && go build && popd
 ```