Merge pull request #12629 from ptabor/20210118-release-attempt

Modernize release script
This commit is contained in:
Piotr Tabor 2021-01-21 20:33:30 +01:00 committed by GitHub
commit 7d444ee327
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 104 additions and 82 deletions

View File

@ -24,7 +24,6 @@ function setup_env {
fi
pushd "${proj}" >/dev/null
run git checkout master
run git fetch --all
git_assert_branch_in_sync || exit 2
run git checkout "${ver}"
@ -76,8 +75,7 @@ function main {
if [ ${GOOS} == "linux" ]; then
TARGET_ARCHS+=("arm64")
TARGET_ARCHS+=("ppc64le")
# TODO: Reenable when https://github.com/etcd-io/etcd/issues/12496 is fixed.
# TARGET_ARCHS+=("s390x")
TARGET_ARCHS+=("s390x")
fi
for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do

View File

@ -24,8 +24,7 @@ pushd "${ETCD_ROOT}" >/dev/null
log_callout "Building etcd binary..."
./scripts/build-binary "${VERSION}"
# TODO: Add "s390x" when https://github.com/etcd-io/etcd/issues/12496 is fixed.
for TARGET_ARCH in "amd64" "arm64" "ppc64le"; do
for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do
log_callout "Building ${TARGET_ARCH} docker image..."
GOOS=linux GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker "${VERSION}"
done

View File

@ -5,6 +5,9 @@ set -o nounset
set -o pipefail
source ./scripts/test_lib.sh
source ./scripts/release_mod.sh
DRY_RUN=${DRY_RUN:-true}
help() {
echo "$(basename "$0") [version]"
@ -27,7 +30,7 @@ help() {
main() {
VERSION=$1
if [[ ! "${VERSION}" =~ [0-9]+.[0-9]+.[0-9]+ ]]; then
echo "Expected 'version' param of the form '<major-version>.<minor-version>.<patch-version>' but got '${VERSION}'"
log_error "Expected 'version' param of the form '<major-version>.<minor-version>.<patch-version>' but got '${VERSION}'"
exit 1
fi
RELEASE_VERSION="v${VERSION}"
@ -35,11 +38,18 @@ main() {
BRANCH=${BRANCH:-"release-${MINOR_VERSION}"}
REPOSITORY=${REPOSITORY:-"https://github.com/etcd-io/etcd"}
log_warning "DRY_RUN=${DRY_RUN}"
log_callout "RELEASE_VERSION=${RELEASE_VERSION}"
log_callout "MINOR_VERSION=${MINOR_VERSION}"
log_callout "BRANCH=${BRANCH}"
log_callout "REPOSITORY=${REPOSITORY}"
log_callout ""
# Required to enable 'docker manifest ...'
export DOCKER_CLI_EXPERIMENTAL=enabled
if ! command -v docker >/dev/null; then
echo "cannot find docker"
log_error "cannot find docker"
exit 1
fi
@ -55,6 +65,9 @@ main() {
run git clone "${REPOSITORY}" --branch "${BRANCH}"
fi
run cd "${reldir}/etcd" || exit 2
# mark local directory as root for test_lib scripts executions
set_root_dir
run git checkout "${BRANCH}" || exit 2
run git pull origin
git_assert_branch_in_sync || exit 2
@ -89,7 +102,7 @@ main() {
exit 1
fi
log_callout "Updating modules definitions"
TARGET_VERSION="v${VERSION}" DRY_RUN=false ./scripts/release_mod.sh update_versions
TARGET_VERSION="v${VERSION}" update_versions_cmd
log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go"
sed -i "s/${source_version}/${VERSION}/g" api/version/version.go
@ -97,11 +110,11 @@ main() {
log_callout "Building etcd and checking --version output"
run ./build
run ./build.sh
local etcd_version
etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
if [[ "${etcd_version}" != "${VERSION}" ]]; then
echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
log_error "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
exit 1
fi
@ -118,7 +131,7 @@ main() {
if [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then
read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm
[[ "${confirm,,}" == "y" ]] || exit 1
git push
maybe_run git push
fi
# Tag release.
@ -126,20 +139,20 @@ main() {
log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
else
log_callout "Tagging release..."
REMOTE_REPO="origin" DRY_RUN=false ./scripts/release_mod.sh push_mod_tags
REMOTE_REPO="origin" push_mod_tags_cmd
fi
# Verify the latest commit has the version tag
local tag="$(git describe --exact-match HEAD)"
if [ "${tag}" != "${RELEASE_VERSION}" ]; then
echo "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}"
log_error "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}"
exit 1
fi
# Verify the version tag is on the right branch
local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3)
if [ "${branch}" != "${BRANCH}" ]; then
echo "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'"
log_error "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'"
exit 1
fi
fi
@ -150,7 +163,12 @@ main() {
log_warning "Skipping release build step. /release directory already exists."
else
log_callout "Building release..."
REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}"
if ${DRY_RUN}; then
log_warning "In DRY_RUN mode we clone the current release directory (as there was no push)"
REPOSITORY=$(pwd) ./scripts/build-release.sh "${RELEASE_VERSION}"
else
REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}"
fi
fi
# Sanity checks.
@ -158,67 +176,67 @@ main() {
"./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdctl" version | grep -q "etcdctl version: ${VERSION}" || true
# Generate SHA256SUMS
echo -e "Generating sha256sums of release artifacts.\n"
log_callout "Generating sha256sums of release artifacts."
pushd ./release
ls . | grep -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS
popd
if [ -s ./release/SHA256SUMS ]; then
cat ./release/SHA256SUMS
else
echo "sha256sums is not valid. Aborting."
log_error "sha256sums is not valid. Aborting."
exit 1
fi
# Upload artifacts.
if [ "${NO_UPLOAD}" == 1 ]; then
echo "Skipping artifact upload to gs://etcd. --no-upload flat is set."
log_callout "Skipping artifact upload to gs://etcd. --no-upload flat is set."
else
read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " -r confirm
[[ "${confirm,,}" == "y" ]] || exit 1
gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/"
gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/"
gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/"
gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/"
maybe_run gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/"
maybe_run gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/"
maybe_run gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/"
maybe_run gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/"
fi
# Push images.
if [ "${NO_DOCKER_PUSH}" == 1 ]; then
echo "Skipping docker push. --no-docker-push flat is set."
log_callout "Skipping docker push. --no-docker-push flat is set."
else
read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " -r confirm
[[ "${confirm,,}" == "y" ]] || exit 1
# shellcheck disable=SC2034
for i in {1..5}; do
docker login quay.io && break
echo "login failed, retrying"
log_warning "login failed, retrying"
done
gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
maybe_run gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
# NB: `docker manifest create` insists on at least one manifest.
# We overwrite it later with --amend anyway.
docker manifest create "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-amd64"
docker manifest create "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-amd64"
maybe_run docker manifest create "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-amd64"
maybe_run docker manifest create "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-amd64"
for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do
echo "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}"
log_callout "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}"
docker push "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
docker manifest create --amend "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
docker manifest annotate "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}"
echo "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}"
gcloud docker -- push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}"
log_callout "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}"
maybe_run gcloud docker -- push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
maybe_run docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}"
maybe_run docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}"
done
echo "Pushing container manifest list to quay.io ${RELEASE_VERSION}"
docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}"
log_callout "Pushing container manifest list to quay.io ${RELEASE_VERSION}"
maybe_run docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}"
echo "Pushing container manifest list to gcr.io ${RELEASE_VERSION}"
gcloud docker -- manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"
log_callout "Pushing container manifest list to gcr.io ${RELEASE_VERSION}"
maybe_run gcloud docker -- manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"
echo "Setting permissions using gsutil..."
gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
log_callout "Setting permissions using gsutil..."
maybe_run gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
fi
### Release validation
@ -228,7 +246,7 @@ main() {
for IMAGE in "quay.io/coreos/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"; do
local image_version=$(docker run --rm "${IMAGE}" etcd --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]')
if [ "${image_version}" != "${VERSION}" ]; then
echo "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}"
log_error "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}"
exit 1
fi
done
@ -239,15 +257,15 @@ main() {
tar -zx -C downloads -f "downloads/${BINARY_TGZ}"
local binary_version=$("./downloads/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]')
if [ "${binary_version}" != "${VERSION}" ]; then
echo "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}"
log_error "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}"
exit 1
fi
# TODO: signing process
echo ""
echo "WARNING: The release has not been signed and published to github. This must be done manually."
echo ""
echo "Success."
log_warning ""
log_warning "WARNING: The release has not been signed and published to github. This must be done manually."
log_warning ""
log_success "Success."
exit 0
}

View File

@ -12,15 +12,10 @@
set -e
DRY_RUN=${DRY_RUN:-true}
if ! [[ "$0" =~ scripts/release_mod.sh ]]; then
echo "must be run from repository root"
exit 255
fi
source ./scripts/test_lib.sh
DRY_RUN=${DRY_RUN:-true}
# _cmd prints help message
function _cmd() {
log_error "Command required: ${0} [cmd]"
@ -29,15 +24,6 @@ function _cmd() {
log_info " - push_mod_tags - Tags HEAD with all modules versions tags and pushes it to \${REMOTE_REPO}."
}
# maybe_run [cmd...] runs given command depending on the DRY_RUN flag.
function maybe_run() {
if ${DRY_RUN}; then
log_warning -e "# DRY_RUN:\n % ${*}"
else
run "${@}"
fi
}
# update_module_version [v2version] [v3version]
# Updates versions of cross-references in all internal references in current module.
function update_module_version() {
@ -48,12 +34,12 @@ function update_module_version() {
v3deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v3")
for dep in ${v3deps}; do
maybe_run go mod edit -require "${dep}@${v3version}"
run go mod edit -require "${dep}@${v3version}"
done
v2deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v2")
for dep in ${v2deps}; do
maybe_run go mod edit -require "${dep}@${v2version}"
run go mod edit -require "${dep}@${v2version}"
done
}
@ -124,15 +110,18 @@ function push_mod_tags_cmd {
# The sleep is ugly hack that guarantees that 'git describe' will
# consider main-module's tag as the latest.
run sleep 2
maybe_run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}"
run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}"
tags=("${tags[@]}" "${tag}")
done
maybe_run git push -f "${REMOTE_REPO}" "${tags[@]}"
}
"${1}_cmd"
# only release_mod when called directly, not sourced
if echo "$0" | grep -E "release_mod.sh$" >/dev/null; then
"${1}_cmd"
if "${DRY_RUN}"; then
log_info
log_warning "WARNING: It was a DRY_RUN. No files were modified."
fi
if "${DRY_RUN}"; then
log_info
log_warning "WARNING: It was a DRY_RUN. No files were modified."
fi
fi

View File

@ -7,7 +7,11 @@ if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then
exit 255
fi
ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3")
function set_root_dir {
ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3")
}
set_root_dir
#### Convenient IO methods #####
@ -20,6 +24,7 @@ COLOR_MAGENTA='\033[95m'
COLOR_BOLD='\033[1m'
COLOR_NONE='\033[0m' # No Color
function log_error {
>&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}"
>&2 echo "$@"
@ -162,6 +167,15 @@ function module_dirs() {
echo "api pkg raft client/v2 client/v3 server etcdctl tests ."
}
# maybe_run [cmd...] runs given command depending on the DRY_RUN flag.
function maybe_run() {
if ${DRY_RUN}; then
log_warning -e "# DRY_RUN:\n % ${*}"
else
run "${@}"
fi
}
function modules() {
modules=(
"${ROOT_MODULE}/api/v3"
@ -344,14 +358,18 @@ function git_assert_branch_in_sync {
branch=$(git branch --show-current)
if [[ $(run git status --porcelain --untracked-files=no) ]]; then
log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes"
log_error "Consider cleaning up / renaming this directory."
log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard)"
return 2
fi
ref_local=$(run git rev-parse "${branch}")
ref_origin=$(run git rev-parse "origin/${branch}")
if [ "x${ref_local}" != "x${ref_origin}" ]; then
log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin."
log_error "Consider cleaning up / renaming this directory."
return 2
if [ -n "${branch}" ]; then
ref_local=$(run git rev-parse "${branch}")
ref_origin=$(run git rev-parse "origin/${branch}")
if [ "x${ref_local}" != "x${ref_origin}" ]; then
log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin."
log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard origin/${branch})"
return 2
fi
else
log_warning "Cannot verify consistency with the origin, as git is on detached branch."
fi
}

View File

@ -26,11 +26,11 @@ ADD ./tests/functional/functional.yaml /functional.yaml
RUN go get -v go.etcd.io/gofail \
&& pushd ${GOPATH}/src/go.etcd.io/etcd \
&& GO_BUILD_FLAGS="-v" ./build \
&& GO_BUILD_FLAGS="-v" ./build.sh \
&& mkdir -p /bin \
&& cp ./bin/etcd /bin/etcd \
&& cp ./bin/etcdctl /bin/etcdctl \
&& GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \
&& GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build.sh \
&& cp ./bin/etcd /bin/etcd-failpoints \
&& ./tests/functional/build \
&& cp ./bin/etcd-agent /bin/etcd-agent \

View File

@ -39,7 +39,7 @@ Proxy layer that simulates various network conditions.
Test locally
```bash
$ ./build
$ ./build.sh
$ ./bin/etcd
$ make build-functional
@ -201,7 +201,7 @@ $ curl -L http://localhost:2378/blackhole-tx -X DELETE
Trigger leader election
```bash
$ ./build
$ ./build.sh
$ make build-functional
$ rm -rf /tmp/etcd-proxy-data.s*

View File

@ -54,7 +54,7 @@ See README.md for more examples.
Example:
# build etcd
$ ./build
$ ./build.sh
$ ./bin/etcd
# build etcd-proxy

View File

@ -11,7 +11,7 @@ local-tester depends on `goreman` to manage its processes and `bash` to run faul
local-tester needs `etcd`, `benchmark`, and `bridge` binaries. To build these binaries, run the following from the etcd repository root:
```sh
./build
./build.sh
pushd tools/benchmark/ && go build && popd
pushd tools/local-tester/bridge && go build && popd
```