scripts: update release, genproto, dep

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
This commit is contained in:
Gyuho Lee 2019-08-14 01:14:34 -07:00
parent cc1591aa4e
commit d7fc66bcbb
5 changed files with 80 additions and 301 deletions

View File

@ -1,84 +0,0 @@
#!/usr/bin/env bash
set -e
BINARYDIR=${BINARYDIR:-bin}
BUILDDIR=${BUILDDIR:-bin}
# A non-installed acbuild can be used, for example:
# ACBUILD=../../appc/acbuild/bin/acbuild
ACBUILD=${ACBUILD:-acbuild}
VERSION=$1
go2aci() {
case "${1}" in
"arm64") echo "aarch64";;
*) echo "${1}";;
esac
}
if ! command -v "${ACBUILD}" >/dev/null; then
echo "acbuild ($ACBUILD) is not executable"
exit 1
fi
if [ ! -x "${BINARYDIR}"/etcd ] ; then
echo "$BINARYDIR/etcd not found. Is it compiled?"
exit 1
fi
if [ -z "$VERSION" ] ; then
echo "Usage: scripts/build-aci VERSION"
exit 1
fi
acbuild --debug begin
TMPHOSTS="$(mktemp)"
ACI_ARCH=$(go2aci "${GOARCH}")
acbuildEnd() {
rm "$TMPHOSTS"
export EXIT=$?
acbuild --debug end && exit $EXIT
}
trap acbuildEnd EXIT
cat <<DF > "$TMPHOSTS"
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
DF
acbuild --debug set-name coreos.com/etcd
acbuild --debug annotation add appc.io/executor/supports-systemd-notify true
acbuild --debug copy "${BINARYDIR}"/etcd /usr/local/bin/etcd
acbuild --debug copy "${BINARYDIR}"/etcdctl /usr/local/bin/etcdctl
acbuild --debug copy README.md README.md
acbuild --debug copy etcdctl/README.md README-etcdctl.md
acbuild --debug copy etcdctl/READMEv2.md READMEv2-etcdctl.md
acbuild --debug copy-to-dir Documentation .
acbuild --debug environment add ETCD_DATA_DIR /var/lib/etcd
acbuild --debug mount add data-dir /var/lib/etcd
acbuild --debug label add version "$VERSION"
acbuild --debug set-exec -- /usr/local/bin/etcd
acbuild --debug port add client tcp 2379
acbuild --debug port add peer tcp 2380
acbuild --debug copy "$TMPHOSTS" /etc/hosts
acbuild --debug label add arch "${ACI_ARCH}"
# mkdir default data-dir
mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd
# symlinks for backward-compatibility
ln -s ./usr/local/bin/etcd .acbuild/currentaci/rootfs/etcd
ln -s ./usr/local/bin/etcdctl .acbuild/currentaci/rootfs/etcdctl
acbuild --debug write --overwrite "${BUILDDIR}/etcd-${1}-linux-${ACI_ARCH}.aci"

View File

@ -11,11 +11,6 @@ if [ -z "${VERSION}" ]; then
exit 255
fi
if ! command -v acbuild >/dev/null; then
echo "cannot find acbuild"
exit 1
fi
if ! command -v docker >/dev/null; then
echo "cannot find docker"
exit 1
@ -27,14 +22,8 @@ pushd "${ETCD_ROOT}" >/dev/null
echo Building etcd binary...
./scripts/build-binary "${VERSION}"
# ppc64le not yet supported by acbuild.
for TARGET_ARCH in "amd64" "arm64"; do
echo Building ${TARGET_ARCH} aci image...
GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-aci "${VERSION}"
done
for TARGET_ARCH in "amd64" "arm64" "ppc64le"; do
echo Building ${TARGET_ARCH} docker image...
GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker "${VERSION}"
GOOS=linux GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker "${VERSION}"
done
popd >/dev/null

View File

@ -10,17 +10,20 @@ if ! [[ "$0" =~ scripts/genproto.sh ]]; then
exit 255
fi
if [[ $(protoc --version | cut -f2 -d' ') != "3.5.0" ]]; then
echo "could not find protoc 3.5.0, is it installed + in PATH?"
if [[ $(protoc --version | cut -f2 -d' ') != "3.7.1" ]]; then
echo "could not find protoc 3.7.1, is it installed + in PATH?"
exit 255
fi
# directories containing protos to be built
DIRS="./wal/walpb ./etcdserver/etcdserverpb ./snap/snappb ./raft/raftpb ./mvcc/mvccpb ./lease/leasepb ./auth/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb"
# disable go mod
export GO111MODULE=off
# exact version of packages to build
GOGO_PROTO_SHA="41168f6614b7bb144818ec8967b8c702705df564"
GRPC_GATEWAY_SHA="a92d37fb6339375fa4bb7d9c364f92373fe199c3"
GOGO_PROTO_SHA="1adfc126b41513cc696b209667c8656ea7aac67c"
GRPC_GATEWAY_SHA="92583770e3f01b09a0d3e9bdf64321d8bebd48f2"
SCHWAG_SHA="b7d0fc9aadaaae3d61aaadfc12e4a2f945514912"
# set up self-contained GOPATH for building
@ -28,39 +31,57 @@ export GOPATH=${PWD}/gopath.proto
export GOBIN=${PWD}/bin
export PATH="${GOBIN}:${PATH}"
COREOS_ROOT="${GOPATH}/src/github.com/coreos"
ETCD_ROOT="${COREOS_ROOT}/etcd"
ETCD_IO_ROOT="${GOPATH}/src/github.com/coreos/etcd"
ETCD_ROOT="${ETCD_IO_ROOT}/etcd"
GOGOPROTO_ROOT="${GOPATH}/src/github.com/gogo/protobuf"
SCHWAG_ROOT="${GOPATH}/src/github.com/hexfusion/schwag"
GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf"
GRPC_GATEWAY_ROOT="${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway"
rm -f "${ETCD_ROOT}"
mkdir -p "${COREOS_ROOT}"
function cleanup {
# Remove the whole fake GOPATH which can really confuse go mod.
rm -rf "${PWD}/gopath.proto"
}
cleanup
trap cleanup EXIT
mkdir -p "${ETCD_IO_ROOT}"
ln -s "${PWD}" "${ETCD_ROOT}"
# Ensure we have the right version of protoc-gen-gogo by building it every time.
# TODO(jonboulle): vendor this instead of `go get`ting it.
go get -u github.com/gogo/protobuf/{proto,protoc-gen-gogo,gogoproto}
go get -u golang.org/x/tools/cmd/goimports
echo "Installing gogo/protobuf..."
GOGOPROTO_ROOT="$GOPATH/src/github.com/gogo/protobuf"
# rm -rf $GOGOPROTO_ROOT
mkdir -p $GOPATH/src/github.com/gogo
pushd $GOPATH/src/github.com/gogo
git clone https://github.com/gogo/protobuf.git
popd
pushd "${GOGOPROTO_ROOT}"
git reset --hard "${GOGO_PROTO_SHA}"
make install
git reset --hard HEAD
make install
popd
# generate gateway code
go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
echo "Installing grpc-ecosystem/grpc-gateway..."
GRPC_GATEWAY_ROOT="$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway"
# rm -rf $GRPC_GATEWAY_ROOT
mkdir -p $GOPATH/src/github.com/grpc-ecosystem
go get -v -d github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
go get -v -d github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
pushd $GOPATH/src/github.com/grpc-ecosystem
rm -rf ./grpc-gateway
git clone https://github.com/grpc-ecosystem/grpc-gateway.git
popd
pushd "${GRPC_GATEWAY_ROOT}"
git reset --hard "${GRPC_GATEWAY_SHA}"
go install ./protoc-gen-grpc-gateway
go install -v ./protoc-gen-grpc-gateway
go install -v ./protoc-gen-swagger
popd
for dir in ${DIRS}; do
pushd "${dir}"
protoc --gofast_out=plugins=grpc,import_prefix=github.com/coreos/:. -I=".:${GOGOPROTO_PATH}:${COREOS_ROOT}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" ./*.proto
protoc --gofast_out=plugins=grpc,import_prefix=github.com/coreos/:. -I=".:${GOGOPROTO_PATH}:${ETCD_IO_ROOT}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" ./*.proto
sed -i.bak -E "s/github\.com\/coreos\/(gogoproto|github\.com|golang\.org|google\.golang\.org)/\1/g" ./*.pb.go
sed -i.bak -E 's/github\.com\/coreos\/(errors|fmt|io)/\1/g' ./*.pb.go
sed -i.bak -E 's/github\.com\/coreos\/(errors|fmt|io|context|math\/bits)/\1/g' ./*.pb.go
sed -i.bak -E 's/import _ \"gogoproto\"//g' ./*.pb.go
sed -i.bak -E 's/import fmt \"fmt\"//g' ./*.pb.go
sed -i.bak -E 's/import _ \"github\.com\/coreos\/google\/api\"//g' ./*.pb.go
@ -69,77 +90,3 @@ for dir in ${DIRS}; do
goimports -w ./*.pb.go
popd
done
# remove old swagger files so it's obvious whether the files fail to generate
rm -rf Documentation/dev-guide/apispec/swagger/*json
for pb in etcdserverpb/rpc api/v3lock/v3lockpb/v3lock api/v3election/v3electionpb/v3election; do
protobase="etcdserver/${pb}"
protoc -I. \
-I"${GRPC_GATEWAY_ROOT}"/third_party/googleapis \
-I"${GOGOPROTO_PATH}" \
-I"${COREOS_ROOT}" \
--grpc-gateway_out=logtostderr=true:. \
--swagger_out=logtostderr=true:./Documentation/dev-guide/apispec/swagger/. \
${protobase}.proto
# hack to move gw files around so client won't include them
pkgpath=$(dirname "${protobase}")
pkg=$(basename "${pkgpath}")
gwfile="${protobase}.pb.gw.go"
sed -i.bak -E "s/package $pkg/package gw/g" ${gwfile}
sed -i.bak -E "s/protoReq /&$pkg\./g" ${gwfile}
sed -i.bak -E "s/, client /, client $pkg./g" ${gwfile}
sed -i.bak -E "s/Client /, client $pkg./g" ${gwfile}
sed -i.bak -E "s/[^(]*Client, runtime/${pkg}.&/" ${gwfile}
sed -i.bak -E "s/New[A-Za-z]*Client/${pkg}.&/" ${gwfile}
# darwin doesn't like newlines in sed...
sed -i.bak -E "s|import \(|& \"github.com/coreos/etcd/${pkgpath}\"|" ${gwfile}
mkdir -p "${pkgpath}"/gw/
go fmt ${gwfile}
mv ${gwfile} "${pkgpath}/gw/"
rm -f ./etcdserver/${pb}*.bak
swaggerName=$(basename ${pb})
mv Documentation/dev-guide/apispec/swagger/etcdserver/${pb}.swagger.json \
Documentation/dev-guide/apispec/swagger/"${swaggerName}".swagger.json
done
rm -rf Documentation/dev-guide/apispec/swagger/etcdserver/
# append security to swagger spec
go get -u "github.com/hexfusion/schwag"
pushd "${SCHWAG_ROOT}"
git reset --hard "${SCHWAG_SHA}"
go install .
popd
schwag -input=Documentation/dev-guide/apispec/swagger/rpc.swagger.json
# install protodoc
# go get -v -u github.com/coreos/protodoc
#
# by default, do not run this option.
# only run when './scripts/genproto.sh -g'
#
if [ "$1" = "-g" ]; then
echo "protodoc is auto-generating grpc API reference documentation..."
go get -v -u github.com/coreos/protodoc
SHA_PROTODOC="4372ee725035a208404e2d5465ba921469decc32"
PROTODOC_PATH="${GOPATH}/src/github.com/coreos/protodoc"
pushd "${PROTODOC_PATH}"
git reset --hard "${SHA_PROTODOC}"
go install
echo "protodoc is updated"
popd
protodoc --directories="etcdserver/etcdserverpb=service_message,mvcc/mvccpb=service_message,lease/leasepb=service_message,auth/authpb=service_message" \
--title="etcd API Reference" \
--output="Documentation/dev-guide/api_reference_v3.md" \
--message-only-from-this-file="etcdserver/etcdserverpb/rpc.proto" \
--disclaimer="This is a generated documentation. Please read the proto files for more."
protodoc --directories="etcdserver/api/v3lock/v3lockpb=service_message,etcdserver/api/v3election/v3electionpb=service_message,mvcc/mvccpb=service_message" \
--title="etcd concurrency API Reference" \
--output="Documentation/dev-guide/api_concurrency_reference_v3.md" \
--disclaimer="This is a generated documentation. Please read the proto files for more."
echo "protodoc is finished..."
else
echo "skipping grpc API reference document auto-generation..."
fi

View File

@ -32,11 +32,6 @@ main() {
MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2)
BRANCH="release-${MINOR_VERSION}"
if ! command -v acbuild >/dev/null; then
echo "cannot find acbuild"
exit 1
fi
if ! command -v docker >/dev/null; then
echo "cannot find docker"
exit 1
@ -56,7 +51,7 @@ main() {
if [ ! -d "${reldir}/etcd" ]; then
mkdir -p "${reldir}"
cd "${reldir}"
git clone git@github.com:coreos/etcd.git --branch "${BRANCH}"
git clone git@github.com:etcd-io/etcd.git --branch "${BRANCH}"
fi
cd "${reldir}/etcd"
@ -68,7 +63,8 @@ main() {
fi
# Check go version.
local go_version="go$(yq -r ".go[0]" .travis.yml)"
# download "yq" from https://github.com/mikefarah/yq
local go_version="go$(yq read .travis.yml "go[0]")"
local current_go_version=$(go version | awk '{ print $3 }')
if [[ "${current_go_version}" != "${go_version}" ]]; then
echo "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .travis.yml)."
@ -87,10 +83,10 @@ main() {
fi
echo "Updating version from ${source_version} to ${VERSION} in version/version.go"
sed -i "s/${source_version}/${VERSION}/g" version/version.go
echo "Building etcd with updated version"
./build
fi
echo "Building etcd and checking --version output"
./build
local etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
if [[ "${etcd_version}" != "${VERSION}" ]]; then
echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
@ -106,7 +102,7 @@ main() {
# Push the version change if it's not already been pushed.
if [ $(git rev-list --count "origin/${BRANCH}..${BRANCH}") -gt 0 ]; then
read -p "Push version bump up to ${VERSION} to github.com/coreos/etcd [y/N]? " confirm
read -p "Push version bump up to ${VERSION} to github.com/etcd-io/etcd [y/N]? " confirm
[[ "${confirm,,}" == "y" ]] || exit 1
git push
fi
@ -141,8 +137,8 @@ main() {
fi
# Sanity checks.
./release/etcd-${RELEASE_VERSION}-linux-amd64/etcd --version | grep -q "etcd Version: ${VERSION}"
ETCDCTL_API=3 ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcdctl version | grep -q "etcdctl version: ${VERSION}"
./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd --version | grep -q "etcd Version: ${VERSION}" || true
./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdctl version | grep -q "etcdctl version: ${VERSION}" || true
# Upload artifacts.
if [ "${NO_UPLOAD}" == 1 ]; then
@ -152,14 +148,13 @@ main() {
[[ "${confirm,,}" == "y" ]] || exit 1
gsutil -m cp ./release/*.zip gs://etcd/${RELEASE_VERSION}/
gsutil -m cp ./release/*.tar.gz gs://etcd/${RELEASE_VERSION}/
gsutil -m cp ./release/*.aci gs://etcd/${RELEASE_VERSION}/
gsutil -m acl ch -u allUsers:R -r gs://etcd/${RELEASE_VERSION}/
fi
# Push images.
if [ "${NO_DOCKER_PUSH}" == 1 ]; then
echo "Skipping docker push. --no-docker-push flat is set."
else
else
read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " confirm
[[ "${confirm,,}" == "y" ]] || exit 1
for i in {1..5}; do
@ -168,45 +163,24 @@ main() {
done
gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
echo "Pushing container images to quay.io" ${RELEASE_VERSION}
docker push quay.io/coreos/etcd:${RELEASE_VERSION}
echo "Pushing container images to gcr.io" ${RELEASE_VERSION}
gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}
if [ "${MINOR_VERSION}" != "3.1" ]; then
for TARGET_ARCH in "-arm64" "-ppc64le"; do
docker push quay.io/coreos/etcd:${RELEASE_VERSION}${TARGET_ARCH}
gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}${TARGET_ARCH}
done
fi
for TARGET_ARCH in "-arm64" "-ppc64le"; do
echo "Pushing container images to quay.io" ${RELEASE_VERSION}${TARGET_ARCH}
docker push quay.io/coreos/etcd:${RELEASE_VERSION}${TARGET_ARCH}
echo "Pushing container images to gcr.io" ${RELEASE_VERSION}${TARGET_ARCH}
gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}${TARGET_ARCH}
done
echo "Setting permissions using gsutil..."
gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
docker tag quay.io/coreos/etcd:${RELEASE_VERSION} quay.io/coreos/etcd:v${MINOR_VERSION}
docker push quay.io/coreos/etcd:v${MINOR_VERSION}
gcloud docker -- tag gcr.io/etcd-development/etcd:${RELEASE_VERSION} gcr.io/etcd-development/etcd:v${MINOR_VERSION}
gcloud docker -- push gcr.io/etcd-development/etcd:v${MINOR_VERSION}
fi
# TODO: test
# docker run --rm --name etcd-gcr-${RELEASE_VERSION} gcr.io/etcd-development/etcd:${RELEASE_VERSION};
# docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "/usr/local/bin/etcd --version"
# docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl version"
# docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl put foo bar"
# docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl get foo"
# Bump version to next development version.
git checkout -q "${BRANCH}" # Since we might be on a checkout of the remote version tag.
local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
if [[ "${source_version}" != "${VERSION}+git" ]]; then
echo "Updating version from ${source_version} to ${VERSION}+git in version/version.go"
sed -i "s/${source_version}/${VERSION}+git/g" version/version.go
echo "Building etcd with ${VERSION}+git in version/version.go"
git add version/version.go
git commit -m "version: bump up to ${VERSION}+git"
git diff --staged
read -p "Push version bump up to ${VERSION}+git to github.com/coreos/etcd [y/N]? " confirm
[[ "${confirm,,}" == "y" ]] || exit 1
git push
fi
# TODO: signing process
echo ""
echo "WARNING: The release has not been signed and published to github. This must be done manually."

View File

@ -1,69 +1,22 @@
#!/usr/bin/env bash
# A script for updating godep dependencies for the vendored directory /cmd/
# without pulling in etcd itself as a dependency.
#
# update depedency
# 1. edit glide.yaml with version, git SHA
# 2. run ./scripts/updatedep.sh
# 3. it automatically detects new git SHA, and vendors updates to cmd/vendor directory
#
# add depedency
# 1. run ./scripts/updatedep.sh github.com/USER/PROJECT#^1.0.0
# OR
# ./scripts/updatedep.sh github.com/USER/PROJECT#9b772b54b3bf0be1eec083c9669766a56332559a
# 2. make sure glide.yaml and glide.lock are updated
set -e
if ! [[ "$0" =~ scripts/updatedep.sh ]]; then
echo "must be run from repository root"
exit 255
echo "must be run from repository root"
exit 255
fi
rm -rf vendor
mv cmd/vendor vendor
if [ -d "gopath.proto" ]; then
# gopath.proto is created by genproto.sh and it thoroughly messes
# with go mod.
echo "Remove gopath.proto before running this script"
exit 255
fi
# TODO: glide doesn't play well with symlink
echo "manually deleting etcd-repo symlink in vendor"
rm -f vendor/github.com/coreos/etcd
GLIDE_ROOT="$GOPATH/src/github.com/Masterminds/glide"
GLIDE_SHA=21ff6d397ccca910873d8eaabab6a941c364cc70
go get -d -u github.com/Masterminds/glide
pushd "${GLIDE_ROOT}"
git reset --hard ${GLIDE_SHA}
go install
popd
GLIDE_VC_ROOT="$GOPATH/src/github.com/sgotti/glide-vc"
GLIDE_VC_SHA=d96375d23c85287e80296cdf48f9d21c227fa40a
go get -d -u github.com/sgotti/glide-vc
pushd "${GLIDE_VC_ROOT}"
git reset --hard ${GLIDE_VC_SHA}
go install
popd
if [ -n "$1" ]; then
echo "glide get on $1"
matches=$(grep "name: $1" glide.lock)
if [ ! -z "$matches" ]; then
echo "glide update on $1"
glide update --strip-vendor "$1"
else
echo "glide get on $1"
glide get --strip-vendor "$1"
fi
else
echo "glide update on *"
glide update --strip-vendor
fi;
echo "removing test files"
glide vc --only-code --no-tests
mv vendor cmd/
echo "recreating symlink to etcd"
ln -s ../../../../ cmd/vendor/github.com/coreos/etcd
echo "done"
if [[ $(go version) != "go version go1.12"* ]]; then
echo "expect Go 1.12+, got:" "$(go version)"
exit 255
fi
GO111MODULE=on go mod tidy -v
GO111MODULE=on go mod vendor -v