scripts: Integrate ./scripts/release with new code for tagging modules.

Changes: - signing tags. - allows to override BRANCH and REPOSITORY using env variables. Tested by a release in my private fork: BRANCH="20201126-ptabor-release" REPOSITORY="git@github.com:ptabor/etcd.git" ./scripts/release 3.5.0-alpha.20
2024-09-27 06:25:44 +00:00 · 2020-11-27 11:11:09 +01:00
parent c95d9434f1
commit 577c898fee
12 changed files with 308 additions and 168 deletions
--- a/scripts/build-binary
+++ b/scripts/build-binary
@@ -2,97 +2,104 @@

 set -e

+source ./scripts/test_lib.sh
+
 VER=$1
-PROJ="etcd"
+REPOSITORY="${REPOSITORY:-https://github.com/etcd-io/etcd}"
+

 if [ -z "$1" ]; then
-	echo "Usage: ${0} VERSION" >> /dev/stderr
-	exit 255
+  echo "Usage: ${0} VERSION" >> /dev/stderr
+  exit 255
 fi

 set -u

 function setup_env {
-	local proj=${1}
-	local ver=${2}
+  local ver=${1}
+  local proj=${2}

-	if [ ! -d "${proj}" ]; then
-		git clone https://github.com/etcd-io/"${proj}"
-	fi
+  if [ ! -d "${proj}" ]; then
+    run git clone "${REPOSITORY}"
+  fi

-	pushd "${proj}" >/dev/null
-		git checkout master
-		git fetch --all
-		git reset --hard origin/master
-		git checkout "${ver}"
-	popd >/dev/null
+  pushd "${proj}" >/dev/null
+    run git checkout master
+    run git fetch --all
+    git_assert_branch_in_sync || exit 2
+    run git checkout "${ver}"
+    git_assert_branch_in_sync || exit 2
+  popd >/dev/null
 }


 function package {
-	local target=${1}
-	local srcdir="${2}/bin"
+  local target=${1}
+  local srcdir="${2}/bin"

-	local ccdir="${srcdir}/${GOOS}_${GOARCH}"
-	if [ -d "${ccdir}" ]; then
-		srcdir="${ccdir}"
-	fi
-	local ext=""
-	if [ "${GOOS}" == "windows" ]; then
-		ext=".exe"
-	fi
-	for bin in etcd etcdctl; do
-		cp "${srcdir}/${bin}" "${target}/${bin}${ext}"
-	done
+  local ccdir="${srcdir}/${GOOS}_${GOARCH}"
+  if [ -d "${ccdir}" ]; then
+    srcdir="${ccdir}"
+  fi
+  local ext=""
+  if [ "${GOOS}" == "windows" ]; then
+    ext=".exe"
+  fi
+  for bin in etcd etcdctl; do
+    cp "${srcdir}/${bin}" "${target}/${bin}${ext}"
+  done

-	cp etcd/README.md "${target}"/README.md
-	cp etcd/etcdctl/README.md "${target}"/README-etcdctl.md
-	cp etcd/etcdctl/READMEv2.md "${target}"/READMEv2-etcdctl.md
+  cp etcd/README.md "${target}"/README.md
+  cp etcd/etcdctl/README.md "${target}"/README-etcdctl.md
+  cp etcd/etcdctl/READMEv2.md "${target}"/READMEv2-etcdctl.md

-	cp -R etcd/Documentation "${target}"/Documentation
+  cp -R etcd/Documentation "${target}"/Documentation
 }

 function main {
-	mkdir release
-	cd release
-	setup_env "${PROJ}" "${VER}"
+  local proj=$(echo "${REPOSITORY}" | sed 's|^.*/\([^/]*\)$|\1|g')

-	tarcmd=tar
-	if [[ $(go env GOOS) == "darwin" ]]; then
-	  	echo "Please use linux machine for release builds."
-		exit 1
-	fi
+  mkdir -p release
+  cd release
+  setup_env "${VER}" "${proj}"

-	for os in darwin windows linux; do
-		export GOOS=${os}
-		TARGET_ARCHS=("amd64")
+  tarcmd=tar
+  if [[ $(go env GOOS) == "darwin" ]]; then
+      echo "Please use linux machine for release builds."
+    exit 1
+  fi

-		if [ ${GOOS} == "linux" ]; then
-			TARGET_ARCHS+=("arm64")
-			TARGET_ARCHS+=("ppc64le")
-			TARGET_ARCHS+=("s390x")
-		fi
+  for os in darwin windows linux; do
+    export GOOS=${os}
+    TARGET_ARCHS=("amd64")

-		for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do
-			export GOARCH=${TARGET_ARCH}
+    if [ ${GOOS} == "linux" ]; then
+      TARGET_ARCHS+=("arm64")
+      TARGET_ARCHS+=("ppc64le")
+#  TODO: Reenable when https://github.com/etcd-io/etcd/issues/12496 is fixed.
+#      TARGET_ARCHS+=("s390x")
+    fi

-			pushd etcd >/dev/null
-			GO_LDFLAGS="-s" ./build
-			popd >/dev/null
+    for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do
+      export GOARCH=${TARGET_ARCH}

-			TARGET="etcd-${VER}-${GOOS}-${GOARCH}"
-			mkdir "${TARGET}"
-			package "${TARGET}" "${PROJ}"
+      pushd etcd >/dev/null
+      GO_LDFLAGS="-s" ./build
+      popd >/dev/null

-			if [ ${GOOS} == "linux" ]; then
-				${tarcmd} cfz "${TARGET}.tar.gz" "${TARGET}"
-				echo "Wrote release/${TARGET}.tar.gz"
-			else
-				zip -qr "${TARGET}.zip" "${TARGET}"
-				echo "Wrote release/${TARGET}.zip"
-			fi
-		done
-	done
+      TARGET="etcd-${VER}-${GOOS}-${GOARCH}"
+      mkdir "${TARGET}"
+      package "${TARGET}" "${proj}"
+
+      if [ ${GOOS} == "linux" ]; then
+        ${tarcmd} cfz "${TARGET}.tar.gz" "${TARGET}"
+        echo "Wrote release/${TARGET}.tar.gz"
+      else
+        zip -qr "${TARGET}.zip" "${TARGET}"
+        echo "Wrote release/${TARGET}.zip"
+      fi
+    done
+  done
 }

 main
--- a/scripts/build-docker
+++ b/scripts/build-docker
@@ -3,8 +3,8 @@
 set -e

 if [ "$#" -ne 1 ]; then
-	echo "Usage: $0 VERSION" >&2
-	exit 1
+  echo "Usage: $0 VERSION" >&2
+  exit 1
 fi

 VERSION=${1}
@@ -12,20 +12,20 @@ ARCH=$(go env GOARCH)
 DOCKERFILE="Dockerfile-release"

 if [ -z "${BINARYDIR}" ]; then
-	RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH)
-	BINARYDIR="${RELEASE}"
-	TARFILE="${RELEASE}.tar.gz"
-	TARURL="https://github.com/etcd-io/etcd/releases/download/${1}/${TARFILE}"
-	if ! curl -f -L -o "${TARFILE}" "${TARURL}" ; then
-		echo "Failed to download ${TARURL}."
-		exit 1
-	fi
-	tar -zvxf "${TARFILE}"
+  RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH)
+  BINARYDIR="${RELEASE}"
+  TARFILE="${RELEASE}.tar.gz"
+  TARURL="https://github.com/etcd-io/etcd/releases/download/${1}/${TARFILE}"
+  if ! curl -f -L -o "${TARFILE}" "${TARURL}" ; then
+    echo "Failed to download ${TARURL}."
+    exit 1
+  fi
+  tar -zvxf "${TARFILE}"
 fi

 if [ "${ARCH}" != "amd64" ]; then
-	DOCKERFILE+=".${ARCH}"
-	VERSION+="-${ARCH}"
+  DOCKERFILE+=".${ARCH}"
+  VERSION+="-${ARCH}"
 fi

 BINARYDIR=${BINARYDIR:-.}
--- a/scripts/build-release.sh
+++ b/scripts/build-release.sh
@@ -5,6 +5,8 @@
 #
 set -e

+source ./scripts/test_lib.sh
+
 VERSION=$1
 if [ -z "${VERSION}" ]; then
  echo "Usage: ${0} VERSION" >> /dev/stderr
@@ -19,11 +21,12 @@ fi
 ETCD_ROOT=$(dirname "${BASH_SOURCE[0]}")/..

 pushd "${ETCD_ROOT}" >/dev/null
-  echo Building etcd binary...
+  log_callout "Building etcd binary..."
  ./scripts/build-binary "${VERSION}"

-  for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do
-    echo Building ${TARGET_ARCH} docker image...
+#  TODO: Add "s390x" when https://github.com/etcd-io/etcd/issues/12496 is fixed.
+  for TARGET_ARCH in "amd64" "arm64" "ppc64le"; do
+    log_callout "Building ${TARGET_ARCH} docker image..."
    GOOS=linux GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker "${VERSION}"
  done
 popd >/dev/null
--- a/scripts/release
+++ b/scripts/release
@@ -4,6 +4,8 @@ set -o errexit
 set -o nounset
 set -o pipefail

+source ./scripts/test_lib.sh
+
 help() {
  echo "$(basename "$0") [version]"
  echo "Release etcd using the same approach as the etcd-release-runbook (https://goo.gl/Gxwysq)"
@@ -30,7 +32,8 @@ main() {
  fi
  RELEASE_VERSION="v${VERSION}"
  MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2)
-  BRANCH="release-${MINOR_VERSION}"
+  BRANCH=${BRANCH:-"release-${MINOR_VERSION}"}
+  REPOSITORY=${REPOSITORY:-"https://github.com/etcd-io/etcd"}

  if ! command -v docker >/dev/null; then
    echo "cannot find docker"
@@ -42,28 +45,32 @@ main() {

  # Set up release directory.
  local reldir="/tmp/etcd-release-${VERSION}"
+  log_callout "Preparing temporary directory: ${reldir}"
  if [ ! -d "${reldir}/etcd" ]; then
    mkdir -p "${reldir}"
    cd "${reldir}"
-    git clone https://github.com/etcd-io/etcd.git --branch "${BRANCH}"
+    run git clone "${REPOSITORY}" --branch "${BRANCH}"
  fi
-  cd "${reldir}/etcd"
+  run cd "${reldir}/etcd" || exit 2
+  run git checkout "${BRANCH}" || exit 2
+  run git pull origin
+  git_assert_branch_in_sync || exit 2

  # If a release version tag already exists, use it.
  local remote_tag_exists
-  remote_tag_exists=$(git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}")
+  remote_tag_exists=$(run git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}" || true)
+
  if [ "${remote_tag_exists}" -gt 0 ]; then
-    echo "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}"
+    log_callout "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}"
    git checkout -q "tags/${RELEASE_VERSION}"
  fi

  # Check go version.
-  # download "yq" from https://github.com/mikefarah/yq
  local go_version current_go_version
-  go_version="go$(yq read .travis.yml "go[0]")"
+  go_version="go$(run_go_tool "github.com/mikefarah/yq/v3" read .travis.yml "go[0]")"
  current_go_version=$(go version | awk '{ print $3 }')
  if [[ "${current_go_version}" != "${go_version}" ]]; then
-    echo "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .travis.yml)."
+    log_error "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .travis.yml)."
    exit 1
  fi

@@ -71,19 +78,23 @@ main() {
  if [ "${remote_tag_exists}" -eq 0 ]; then
    # Bump version/version.go to release version.
    local source_version
-    source_version=$(grep -E "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
+    source_version=$(grep -E "\s+Version\s*=" api/version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
    if [[ "${source_version}" != "${VERSION}" ]]; then
      source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2)
      if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then
-        echo "Wrong etcd minor version in version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting."
+        log_error "Wrong etcd minor version in api/version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting."
        exit 1
      fi
-      echo "Updating version from ${source_version} to ${VERSION} in version/version.go"
-      sed -i "s/${source_version}/${VERSION}/g" version/version.go
+      log_callout "Updating modules definitions"
+      TARGET_VERSION="v${VERSION}" DRY_RUN=false ./scripts/release_mod.sh update_versions
+
+      log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go"
+      sed -i "s/${source_version}/${VERSION}/g" api/version/version.go
    fi

-    echo "Building etcd and checking --version output"
-    ./build
+
+    log_callout "Building etcd and checking --version output"
+    run ./build
    local etcd_version
    etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
    if [[ "${etcd_version}" != "${VERSION}" ]]; then
@@ -92,30 +103,27 @@ main() {
    fi

    if [[ -n $(git status -s) ]]; then
-      echo "Committing version/version.go update."
-      git add version/version.go
-      git commit -m "version: bump up to ${VERSION}"
-      git diff --staged
+      log_callout "Committing mods & api/version/version.go update."
+      run git add api/version/version.go
+      run git add $(find -name go.mod | xargs)
+      run git diff --staged | cat
+      run git commit -m "version: bump up to ${VERSION}"
+      run git diff --staged | cat
    fi

    # Push the version change if it's not already been pushed.
    if [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then
-      read -p "Push version bump up to ${VERSION} to github.com/etcd-io/etcd [y/N]? " -r confirm
+      read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm
      [[ "${confirm,,}" == "y" ]] || exit 1
      git push
    fi

    # Tag release.
    if [ "$(git tag --list | grep -c "${RELEASE_VERSION}")" -gt 0 ]; then
-      echo "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
+      log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
    else
-      echo "Tagging release..."
-      KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
-      if [[ -z "${KEYID}" ]]; then
-        echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
-        exit 1
-      fi
-      git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}"
+      log_callout "Tagging release..."
+      REMOTE_REPO="origin" DRY_RUN=false ./scripts/release_mod.sh push_mod_tags
    fi

    # Verify the latest commit has the version tag
@@ -126,31 +134,20 @@ main() {
    fi

    # Verify the version tag is on the right branch
-    local branch=$(git branch --contains "${RELEASE_VERSION}")
-    if [ "${branch}" != "release-${MINOR_VERSION}" ]; then
-      echo "Error: Git tag ${RELEASE_VERSION} should be on branch release-${MINOR_VERSION} but is on ${branch}"
+    local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3)
+    if [ "${branch}" != "${BRANCH}" ]; then
+      echo "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'"
      exit 1
    fi
-
-    # Push the tag change if it's not already been pushed.
-    read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " -r confirm
-    [[ "${confirm,,}" == "y" ]] || exit 1
-    git push origin "tags/${RELEASE_VERSION}"
  fi

  # Build release.
  # TODO: check the release directory for all required build artifacts.
  if [ -d release ]; then
-    echo "Skpping release build step. /release directory already exists."
+    log_warning "Skipping release build step. /release directory already exists."
  else
-    echo "Building release..."
-    # Check for old and new names of the release build script.
-    # TODO: Move the release script into this on as a function?
-    if [ -f ./scripts/release.sh ]; then
-      ./scripts/release.sh "${RELEASE_VERSION}"
-    else
-      ./scripts/build-release.sh "${RELEASE_VERSION}"
-    fi
+    log_callout "Building release..."
+    REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}"
  fi

  # Sanity checks.
--- a/scripts/release_mod.sh
+++ b/scripts/release_mod.sh
@@ -46,12 +46,12 @@ function update_module_version() {
  local modules
  modules=$(run go list -f '{{if not .Main}}{{if not .Indirect}}{{.Path}}{{end}}{{end}}' -m all)

-  v3deps=$(echo "${modules}" | grep -E "${REPO}/.*/v3")
+  v3deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v3")
  for dep in ${v3deps}; do
    maybe_run go mod edit -require "${dep}@${v3version}"
  done

-  v2deps=$(echo "${modules}" | grep -E "${REPO}/.*/v2")
+  v2deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v2")
  for dep in ${v2deps}; do
    maybe_run go mod edit -require "${dep}@${v2version}"
  done
@@ -81,6 +81,15 @@ function update_versions_cmd() {
  run_for_modules update_module_version "${v3version}" "${v2version}"
 }

+function get_gpg_key {
+  keyid=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
+  if [[ -z "${keyid}" ]]; then
+    log_error "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
+    return 2
+  fi
+  echo "$keyid"
+}
+
 function push_mod_tags_cmd {
  assert_no_git_modifications || return 2

@@ -92,15 +101,17 @@ function push_mod_tags_cmd {

  # Any module ccan be used for this
  local master_version
-  master_version=$(go list -f '{{.Version}}' -m "${REPO}/api/v3")
+  master_version=$(go list -f '{{.Version}}' -m "${ROOT_MODULE}/api/v3")
  local tags=()

+  keyid=$(get_gpg_key) || return 2
+
  for module in $(modules); do
    local version
    version=$(go list -f '{{.Version}}' -m "${module}")
    local path
    path=$(go list -f '{{.Path}}' -m "${module}")
-    local subdir="${path//${REPO}\//}"
+    local subdir="${path//${ROOT_MODULE}\//}"
    local tag
    if [ -z "${version}" ]; then
      tag="${master_version}"
@@ -110,7 +121,10 @@ function push_mod_tags_cmd {
    fi

    log_info "Tags for: ${module} version:${version} tag:${tag}"
-    maybe_run git tag -f "${tag}"
+    # The sleep is ugly hack that guarantees that 'git describe' will
+    # consider main-module's tag as the latest.
+    run sleep 2
+    maybe_run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}"
    tags=("${tags[@]}" "${tag}")
  done
  maybe_run git push -f "${REMOTE_REPO}" "${tags[@]}"
--- a/scripts/test_lib.sh
+++ b/scripts/test_lib.sh
@@ -1,13 +1,13 @@
 #!/usr/bin/env bash

-REPO="go.etcd.io/etcd"
+ROOT_MODULE="go.etcd.io/etcd"

-if [[ "$(go list)" != "${REPO}/v3" ]]; then
-  echo "must be run from '${REPO}/v3' module directory"
+if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then
+  echo "must be run from '${ROOT_MODULE}/v3' module directory"
  exit 255
 fi

-ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${REPO}/v3")
+ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3")

 ####   Convenient IO methods #####

@@ -160,15 +160,15 @@ function run_for_module {

 function modules() {
  modules=(
-    "${REPO}/api/v3"
-    "${REPO}/pkg/v3"
-    "${REPO}/raft/v3"
-    "${REPO}/client/v2"
-    "${REPO}/client/v3"
-    "${REPO}/server/v3"
-    "${REPO}/etcdctl/v3"
-    "${REPO}/tests/v3"
-    "${REPO}/v3")
+    "${ROOT_MODULE}/api/v3"
+    "${ROOT_MODULE}/pkg/v3"
+    "${ROOT_MODULE}/raft/v3"
+    "${ROOT_MODULE}/client/v2"
+    "${ROOT_MODULE}/client/v3"
+    "${ROOT_MODULE}/server/v3"
+    "${ROOT_MODULE}/etcdctl/v3"
+    "${ROOT_MODULE}/tests/v3"
+    "${ROOT_MODULE}/v3")
  echo "${modules[@]}"
 }

@@ -303,10 +303,10 @@ function tool_get_bin {
  local tool="$1"
  if [[ "$tool" == *"@"* ]]; then
    # shellcheck disable=SC2086
-    run gobin ${GOBINARGS} -p "${tool}" || return 2
+    run gobin ${GOBINARGS:-} -p "${tool}" || return 2
  else
    # shellcheck disable=SC2086
-    run_for_module ./tools/mod run gobin ${GOBINARGS} -p -m --mod=readonly "${tool}" || return 2
+    run_for_module ./tools/mod run gobin ${GOBINARGS:-} -p -m --mod=readonly "${tool}" || return 2
  fi
 }

@@ -339,3 +339,22 @@ function assert_no_git_modifications {
  fi
 }

+# makes sure that the current branch is in sync with the origin branch:
+#  - no uncommitted nor unstaged changes
+#  - no differencing commits in relation to the origin/$branch
+function git_assert_branch_in_sync {
+  local branch
+  branch=$(git branch --show-current)
+  if [[ $(run git status --porcelain --untracked-files=no) ]]; then
+    log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes"
+    log_error "Consider cleaning up / renaming this directory."
+    return 2
+  fi
+  ref_local=$(run git rev-parse "${branch}")
+  ref_origin=$(run git rev-parse "origin/${branch}")
+  if [ "x${ref_local}" != "x${ref_origin}" ]; then
+    log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin."
+    log_error "Consider cleaning up / renaming this directory."
+    return 2
+  fi
+}