diff --git a/Documentation/dev-guide/api_concurrency_reference_v3.md b/Documentation/dev-guide/api_concurrency_reference_v3.md index 3ea9d4a60..3d3edc1e2 100644 --- a/Documentation/dev-guide/api_concurrency_reference_v3.md +++ b/Documentation/dev-guide/api_concurrency_reference_v3.md @@ -4,7 +4,7 @@ This is a generated documentation. Please read the proto files for more. -##### service `Lock` (etcdserver/api/v3lock/v3lockpb/v3lock.proto) +##### service `Lock` (server/etcdserver/api/v3lock/v3lockpb/v3lock.proto) The lock service exposes client-side locking facilities as a gRPC interface. @@ -15,7 +15,7 @@ The lock service exposes client-side locking facilities as a gRPC interface. -##### message `LockRequest` (etcdserver/api/v3lock/v3lockpb/v3lock.proto) +##### message `LockRequest` (server/etcdserver/api/v3lock/v3lockpb/v3lock.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -24,7 +24,7 @@ The lock service exposes client-side locking facilities as a gRPC interface. -##### message `LockResponse` (etcdserver/api/v3lock/v3lockpb/v3lock.proto) +##### message `LockResponse` (server/etcdserver/api/v3lock/v3lockpb/v3lock.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -33,7 +33,7 @@ The lock service exposes client-side locking facilities as a gRPC interface. -##### message `UnlockRequest` (etcdserver/api/v3lock/v3lockpb/v3lock.proto) +##### message `UnlockRequest` (server/etcdserver/api/v3lock/v3lockpb/v3lock.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -41,7 +41,7 @@ The lock service exposes client-side locking facilities as a gRPC interface. -##### message `UnlockResponse` (etcdserver/api/v3lock/v3lockpb/v3lock.proto) +##### message `UnlockResponse` (server/etcdserver/api/v3lock/v3lockpb/v3lock.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -49,7 +49,7 @@ The lock service exposes client-side locking facilities as a gRPC interface. -##### service `Election` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### service `Election` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) The election service exposes client-side election facilities as a gRPC interface. @@ -63,7 +63,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `CampaignRequest` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `CampaignRequest` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -73,7 +73,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `CampaignResponse` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `CampaignResponse` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -82,7 +82,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `LeaderKey` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `LeaderKey` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -93,7 +93,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `LeaderRequest` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `LeaderRequest` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -101,7 +101,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `LeaderResponse` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `LeaderResponse` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -110,7 +110,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `ProclaimRequest` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `ProclaimRequest` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -119,7 +119,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `ProclaimResponse` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `ProclaimResponse` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -127,7 +127,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `ResignRequest` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `ResignRequest` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -135,7 +135,7 @@ The election service exposes client-side election facilities as a gRPC interface -##### message `ResignResponse` (etcdserver/api/v3election/v3electionpb/v3election.proto) +##### message `ResignResponse` (server/etcdserver/api/v3election/v3electionpb/v3election.proto) | Field | Description | Type | | ----- | ----------- | ---- | diff --git a/Documentation/dev-guide/api_reference_v3.md b/Documentation/dev-guide/api_reference_v3.md index e27258506..ac8fcab23 100644 --- a/Documentation/dev-guide/api_reference_v3.md +++ b/Documentation/dev-guide/api_reference_v3.md @@ -984,7 +984,7 @@ Empty field. -##### message `Lease` (lease/leasepb/lease.proto) +##### message `Lease` (server/lease/leasepb/lease.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -994,7 +994,7 @@ Empty field. -##### message `LeaseInternalRequest` (lease/leasepb/lease.proto) +##### message `LeaseInternalRequest` (server/lease/leasepb/lease.proto) | Field | Description | Type | | ----- | ----------- | ---- | @@ -1002,7 +1002,7 @@ Empty field. -##### message `LeaseInternalResponse` (lease/leasepb/lease.proto) +##### message `LeaseInternalResponse` (server/lease/leasepb/lease.proto) | Field | Description | Type | | ----- | ----------- | ---- | diff --git a/Documentation/dev-guide/apispec/swagger/v3election.swagger.json b/Documentation/dev-guide/apispec/swagger/v3election.swagger.json index efd647ae6..312177460 100644 --- a/Documentation/dev-guide/apispec/swagger/v3election.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/v3election.swagger.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "title": "etcdserver/api/v3election/v3electionpb/v3election.proto", + "title": "server/etcdserver/api/v3election/v3electionpb/v3election.proto", "version": "version not set" }, "consumes": [ diff --git a/Documentation/dev-guide/apispec/swagger/v3lock.swagger.json b/Documentation/dev-guide/apispec/swagger/v3lock.swagger.json index b3692fd68..20fe37715 100644 --- a/Documentation/dev-guide/apispec/swagger/v3lock.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/v3lock.swagger.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "title": "etcdserver/api/v3lock/v3lockpb/v3lock.proto", + "title": "server/etcdserver/api/v3lock/v3lockpb/v3lock.proto", "version": "version not set" }, "consumes": [ diff --git a/MAINTAINERS b/MAINTAINERS index 6505be34e..ecf320ad3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14,12 +14,11 @@ Gyuho Lee (@gyuho) pkg:* Hitoshi Mitake (@mitake) pkg:* Jingyi Hu (@jingyih) pkg:* Joe Betz (@jpbetz) pkg:* +Piotr Tabor (@ptabor) pkg:* Sahdev Zala (@spzala) pkg:* Sam Batschelet (@hexfusion) pkg:* +Wenjia Zhang (@wenjiaswe) pkg:* Xiang Li (@xiang90) pkg:* Ben Darnell (@bdarnell) pkg:go.etcd.io/etcd/raft Tobias Grieger (@tbg) pkg:go.etcd.io/etcd/raft - -# REVIEWERS -Wenjia Zhang (@wenjiaswe) pkg:* diff --git a/Makefile b/Makefile index 2f6a41c6d..e8d34ef06 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ XARGS += rm -r .PHONY: build build: - GO_BUILD_FLAGS="-v" ./build + GO_BUILD_FLAGS="-v" ./build.sh ./bin/etcd --version ./bin/etcdctl version @@ -107,7 +107,7 @@ compile-with-docker-test: --rm \ --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \ gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \ - /bin/bash -c "GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build && ./bin/etcd --version" + /bin/bash -c "GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build.sh && ./bin/etcd --version" compile-setup-gopath-with-docker-test: $(info GO_VERSION: $(GO_VERSION)) @@ -115,7 +115,7 @@ compile-setup-gopath-with-docker-test: --rm \ --mount type=bind,source=`pwd`,destination=/etcd \ gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \ - /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build && ./bin/etcd --version && rm -rf ./gopath" + /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v GOOS=linux GOARCH=amd64 ./build.sh && ./bin/etcd --version && rm -rf ./gopath" @@ -148,7 +148,7 @@ compile-setup-gopath-with-docker-test: test: $(info TEST_OPTS: $(TEST_OPTS)) $(info log-file: test-$(TEST_SUFFIX).log) - $(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log + $(TEST_OPTS) ./test.sh 2>&1 | tee test-$(TEST_SUFFIX).log ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log docker-test: @@ -163,7 +163,7 @@ docker-test: $(TMP_DIR_MOUNT_FLAG) \ --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \ gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \ - /bin/bash -c "$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log" + /bin/bash -c "$(TEST_OPTS) ./test.sh 2>&1 | tee test-$(TEST_SUFFIX).log" ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log docker-test-coverage: @@ -177,7 +177,7 @@ docker-test-coverage: $(TMP_DIR_MOUNT_FLAG) \ --mount type=bind,source=`pwd`,destination=/go/src/go.etcd.io/etcd \ gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \ - /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1" + /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test.sh 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1" ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 docker-test-coverage-$(TEST_SUFFIX).log diff --git a/api/etcdserverpb/rpc.proto b/api/etcdserverpb/rpc.proto index e63d4057b..14391378a 100644 --- a/api/etcdserverpb/rpc.proto +++ b/api/etcdserverpb/rpc.proto @@ -238,7 +238,9 @@ service Maintenance { }; } - // Downgrade requests downgrade, cancel downgrade on the cluster version. + // Downgrade requests downgrades, verifies feasibility or cancels downgrade + // on the cluster version. + // Supported since etcd 3.5. rpc Downgrade(DowngradeRequest) returns (DowngradeResponse) { option (google.api.http) = { post: "/v3/maintenance/downgrade" diff --git a/build b/build index 0521fe6a4..b9e031074 100755 --- a/build +++ b/build @@ -1,125 +1,6 @@ #!/usr/bin/env bash -source ./scripts/test_lib.sh +echo -e "\e[91mDEPRECATED!!! Use build.sh script instead.\e[0m\n" +sleep 1 -GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") -if [[ -n "$FAILPOINTS" ]]; then - GIT_SHA="$GIT_SHA"-FAILPOINTS -fi - -VERSION_SYMBOL="go.etcd.io/etcd/api/v3/version.GitSHA" - -# Set GO_LDFLAGS="-s" for building without symbols for debugging. -# shellcheck disable=SC2206 -GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}") -GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}") - -# enable/disable failpoints -toggle_failpoints() { - mode="$1" - if command -v gofail >/dev/null 2>&1; then - run gofail "$mode" server/etcdserver/ server/mvcc/backend/ - elif [[ "$mode" != "disable" ]]; then - log_error "FAILPOINTS set but gofail not found" - exit 1 - fi -} - -toggle_failpoints_default() { - mode="disable" - if [[ -n "$FAILPOINTS" ]]; then mode="enable"; fi - toggle_failpoints "$mode" -} - -etcd_build() { - out="bin" - if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi - toggle_failpoints_default - - run rm -f "${out}/etcd" - ( - cd ./server - # Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK - # shellcheck disable=SC2086 - run env "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ - -installsuffix=cgo \ - "-ldflags=${GO_LDFLAGS[*]}" \ - -o="../${out}/etcd" . || return 2 - ) || return 2 - - run rm -f "${out}/etcdctl" - # shellcheck disable=SC2086 - ( - cd ./etcdctl - run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build $GO_BUILD_FLAGS \ - -installsuffix=cgo \ - "-ldflags=${GO_LDFLAGS[*]}" \ - -o="../${out}/etcdctl" . || return 2 - ) || return 2 - # Verify whether symbol we overriden exists - # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}" - - # We need symbols to do this check: - if [[ "${GO_LDFLAGS[*]}" != *"-s"* ]]; then - go tool nm "${out}/etcd" | grep "${VERSION_SYMBOL}" > /dev/null - if [[ "${PIPESTATUS[*]}" != "0 0" ]]; then - log_error "FAIL: Symbol ${VERSION_SYMBOL} not found in binary: ${out}/etcd" - return 2 - fi - fi -} - -tools_build() { - out="bin" - if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi - tools_path="tools/benchmark - tools/etcd-dump-db - tools/etcd-dump-logs - tools/local-tester/bridge" - for tool in ${tools_path} - do - echo "Building" "'${tool}'"... - run rm -f "${out}/${tool}" - # shellcheck disable=SC2086 - run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} \ - -installsuffix=cgo \ - "-ldflags='${GO_LDFLAGS[*]}'" \ - -o="${out}/${tool}" "./${tool}" || return 2 - done - tests_build "${@}" -} - -tests_build() { - out="bin" - if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi - tools_path=" - functional/cmd/etcd-agent - functional/cmd/etcd-proxy - functional/cmd/etcd-runner - functional/cmd/etcd-tester" - ( - cd tests || exit 2 - for tool in ${tools_path}; do - echo "Building" "'${tool}'"... - run rm -f "../${out}/${tool}" - - # shellcheck disable=SC2086 - run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build ${GO_BUILD_FLAGS} \ - -installsuffix=cgo \ - "-ldflags='${GO_LDFLAGS[*]}'" \ - -o="../${out}/${tool}" "./${tool}" || return 2 - done - ) || return 2 -} - -toggle_failpoints_default - -# only build when called directly, not sourced -if echo "$0" | grep "build$" >/dev/null; then - if etcd_build; then - log_success "SUCCESS: etcd_build" - else - log_error "FAIL: etcd_build" - exit 2 - fi -fi +source ./build.sh diff --git a/build.sh b/build.sh deleted file mode 120000 index c795b054e..000000000 --- a/build.sh +++ /dev/null @@ -1 +0,0 @@ -build \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..66d78973a --- /dev/null +++ b/build.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +source ./scripts/test_lib.sh + +GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") +if [[ -n "$FAILPOINTS" ]]; then + GIT_SHA="$GIT_SHA"-FAILPOINTS +fi + +VERSION_SYMBOL="${ROOT_MODULE}/api/v3/version.GitSHA" + +# Set GO_LDFLAGS="-s" for building without symbols for debugging. +# shellcheck disable=SC2206 +GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}") +GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}") + +# enable/disable failpoints +toggle_failpoints() { + mode="$1" + if command -v gofail >/dev/null 2>&1; then + run gofail "$mode" server/etcdserver/ server/mvcc/backend/ + elif [[ "$mode" != "disable" ]]; then + log_error "FAILPOINTS set but gofail not found" + exit 1 + fi +} + +toggle_failpoints_default() { + mode="disable" + if [[ -n "$FAILPOINTS" ]]; then mode="enable"; fi + toggle_failpoints "$mode" +} + +etcd_build() { + out="bin" + if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi + toggle_failpoints_default + + run rm -f "${out}/etcd" + ( + cd ./server + # Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK + # shellcheck disable=SC2086 + run env "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ + -installsuffix=cgo \ + "-ldflags=${GO_LDFLAGS[*]}" \ + -o="../${out}/etcd" . || return 2 + ) || return 2 + + run rm -f "${out}/etcdctl" + # shellcheck disable=SC2086 + ( + cd ./etcdctl + run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build $GO_BUILD_FLAGS \ + -installsuffix=cgo \ + "-ldflags=${GO_LDFLAGS[*]}" \ + -o="../${out}/etcdctl" . || return 2 + ) || return 2 + # Verify whether symbol we overriden exists + # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}" + + # We need symbols to do this check: + if [[ "${GO_LDFLAGS[*]}" != *"-s"* ]]; then + go tool nm "${out}/etcd" | grep "${VERSION_SYMBOL}" > /dev/null + if [[ "${PIPESTATUS[*]}" != "0 0" ]]; then + log_error "FAIL: Symbol ${VERSION_SYMBOL} not found in binary: ${out}/etcd" + return 2 + fi + fi +} + +tools_build() { + out="bin" + if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi + tools_path="tools/benchmark + tools/etcd-dump-db + tools/etcd-dump-logs + tools/local-tester/bridge" + for tool in ${tools_path} + do + echo "Building" "'${tool}'"... + run rm -f "${out}/${tool}" + # shellcheck disable=SC2086 + run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} \ + -installsuffix=cgo \ + "-ldflags='${GO_LDFLAGS[*]}'" \ + -o="${out}/${tool}" "./${tool}" || return 2 + done + tests_build "${@}" +} + +tests_build() { + out="bin" + if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi + tools_path=" + functional/cmd/etcd-agent + functional/cmd/etcd-proxy + functional/cmd/etcd-runner + functional/cmd/etcd-tester" + ( + cd tests || exit 2 + for tool in ${tools_path}; do + echo "Building" "'${tool}'"... + run rm -f "../${out}/${tool}" + + # shellcheck disable=SC2086 + run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build ${GO_BUILD_FLAGS} \ + -installsuffix=cgo \ + "-ldflags='${GO_LDFLAGS[*]}'" \ + -o="../${out}/${tool}" "./${tool}" || return 2 + done + ) || return 2 +} + +toggle_failpoints_default + +# only build when called directly, not sourced +if echo "$0" | grep -E "build(.sh)?$" >/dev/null; then + if etcd_build; then + log_success "SUCCESS: etcd_build (GOARCH=${GOARCH})" + else + log_error "FAIL: etcd_build (GOARCH=${GOARCH})" + exit 2 + fi +fi diff --git a/client/v3/experimental/recipes/grpc_gateway/user_add.sh b/client/v3/experimental/recipes/grpc_gateway/user_add.sh index 2ad3a54e3..ec5386743 100644 --- a/client/v3/experimental/recipes/grpc_gateway/user_add.sh +++ b/client/v3/experimental/recipes/grpc_gateway/user_add.sh @@ -71,4 +71,5 @@ escape() { token=$(tokengen $user $pass) response=$(add_user $newuser $newpass $token) -echo -e "\\n$response" \ No newline at end of file +echo -e "\\n$response" + diff --git a/client/v3/yaml/config_test.go b/client/v3/yaml/config_test.go index e74a6a2d0..dd6893237 100644 --- a/client/v3/yaml/config_test.go +++ b/client/v3/yaml/config_test.go @@ -98,6 +98,7 @@ func TestConfigFromFile(t *testing.T) { continue } if cerr != nil { + os.Remove(tmpfile.Name()) continue } diff --git a/etcdctl/READMEv2.md b/etcdctl/READMEv2.md index 5ee2329e3..63318dfd4 100644 --- a/etcdctl/READMEv2.md +++ b/etcdctl/READMEv2.md @@ -328,7 +328,7 @@ Releases will follow lockstep with the etcd release cycle. etcdctl is under the Apache 2.0 license. See the [LICENSE][license] file for details. -[authentication]: ../Documentation/v2/authentication.md +[authentication]: https://github.com/etcd-io/website/blob/master/content/docs/v2/authentication.md [etcd]: https://github.com/coreos/etcd [github-release]: https://github.com/coreos/etcd/releases/ [license]: ../LICENSE diff --git a/pkg/fileutil/fileutil_test.go b/pkg/fileutil/fileutil_test.go index 753115720..33b39a23a 100644 --- a/pkg/fileutil/fileutil_test.go +++ b/pkg/fileutil/fileutil_test.go @@ -135,6 +135,7 @@ func TestZeroToEnd(t *testing.T) { if err != nil { t.Fatal(err) } + defer os.Remove(f.Name()) defer f.Close() // Ensure 0 size is a nop so zero-to-end on an empty file won't give EINVAL. diff --git a/pkg/testutil/assert.go b/pkg/testutil/assert.go index 9cf03457d..e8e042021 100644 --- a/pkg/testutil/assert.go +++ b/pkg/testutil/assert.go @@ -21,6 +21,7 @@ import ( ) func AssertEqual(t *testing.T, e, a interface{}, msg ...string) { + t.Helper() if (e == nil || a == nil) && (isNil(e) && isNil(a)) { return } @@ -36,20 +37,24 @@ func AssertEqual(t *testing.T, e, a interface{}, msg ...string) { } func AssertNil(t *testing.T, v interface{}) { + t.Helper() AssertEqual(t, nil, v) } func AssertNotNil(t *testing.T, v interface{}) { + t.Helper() if v == nil { t.Fatalf("expected non-nil, got %+v", v) } } func AssertTrue(t *testing.T, v bool, msg ...string) { + t.Helper() AssertEqual(t, true, v, msg...) } func AssertFalse(t *testing.T, v bool, msg ...string) { + t.Helper() AssertEqual(t, false, v, msg...) } diff --git a/pkg/testutil/leak.go b/pkg/testutil/leak.go index 0d85c0c7c..4b7a4a9b5 100644 --- a/pkg/testutil/leak.go +++ b/pkg/testutil/leak.go @@ -111,10 +111,12 @@ func interestingGoroutines() (gs []string) { if stack == "" || strings.Contains(stack, "sync.(*WaitGroup).Done") || strings.Contains(stack, "os.(*file).close") || + strings.Contains(stack, "os.(*Process).Release") || strings.Contains(stack, "created by os/signal.init") || strings.Contains(stack, "runtime/panic.go") || strings.Contains(stack, "created by testing.RunTests") || strings.Contains(stack, "created by testing.runTests") || + strings.Contains(stack, "created by testing.(*T).Run") || strings.Contains(stack, "testing.Main(") || strings.Contains(stack, "runtime.goexit") || strings.Contains(stack, "go.etcd.io/etcd/pkg/v3/testutil.interestingGoroutines") || diff --git a/pkg/transport/listener.go b/pkg/transport/listener.go index 7ce11a3fb..df9a895bb 100644 --- a/pkg/transport/listener.go +++ b/pkg/transport/listener.go @@ -438,7 +438,7 @@ func (info TLSInfo) ClientConfig() (*tls.Config, error) { if info.EmptyCN { hasNonEmptyCN := false cn := "" - tlsutil.NewCert(info.CertFile, info.KeyFile, func(certPEMBlock []byte, keyPEMBlock []byte) (tls.Certificate, error) { + _, err := tlsutil.NewCert(info.CertFile, info.KeyFile, func(certPEMBlock []byte, keyPEMBlock []byte) (tls.Certificate, error) { var block *pem.Block block, _ = pem.Decode(certPEMBlock) cert, err := x509.ParseCertificate(block.Bytes) @@ -451,6 +451,9 @@ func (info TLSInfo) ClientConfig() (*tls.Config, error) { } return tls.X509KeyPair(certPEMBlock, keyPEMBlock) }) + if err != nil { + return nil, err + } if hasNonEmptyCN { return nil, fmt.Errorf("cert has non empty Common Name (%s): %s", cn, info.CertFile) } diff --git a/pkg/transport/listener_test.go b/pkg/transport/listener_test.go index dbded1946..a34d97055 100644 --- a/pkg/transport/listener_test.go +++ b/pkg/transport/listener_test.go @@ -292,14 +292,28 @@ func TestTLSInfoParseFuncError(t *testing.T) { } defer del() - tlsinfo.parseFunc = fakeCertificateParserFunc(tls.Certificate{}, errors.New("fake")) + tests := []struct { + info TLSInfo + }{ + { + info: *tlsinfo, + }, - if _, err = tlsinfo.ServerConfig(); err == nil { - t.Errorf("expected non-nil error from ServerConfig()") + { + info: TLSInfo{CertFile: "", KeyFile: "", TrustedCAFile: tlsinfo.CertFile, EmptyCN: true}, + }, } - if _, err = tlsinfo.ClientConfig(); err == nil { - t.Errorf("expected non-nil error from ClientConfig()") + for i, tt := range tests { + tt.info.parseFunc = fakeCertificateParserFunc(tls.Certificate{}, errors.New("fake")) + + if _, err = tt.info.ServerConfig(); err == nil { + t.Errorf("#%d: expected non-nil error from ServerConfig()", i) + } + + if _, err = tt.info.ClientConfig(); err == nil { + t.Errorf("#%d: expected non-nil error from ClientConfig()", i) + } } } diff --git a/raft/README.md b/raft/README.md index 5f67517fc..c25a176ea 100644 --- a/raft/README.md +++ b/raft/README.md @@ -195,3 +195,7 @@ This implementation is up to date with the final Raft thesis (https://github.com To ensure there is no attempt to commit two membership changes at once by matching log positions (which would be unsafe since they should have different quorum requirements), any proposed membership change is simply disallowed while any uncommitted change appears in the leader's log. This approach introduces a problem when removing a member from a two-member cluster: If one of the members dies before the other one receives the commit of the confchange entry, then the member cannot be removed any more since the cluster cannot make progress. For this reason it is highly recommended to use three or more nodes in every cluster. + +## Go docs + +More detailed development documentation can be found in go docs: https://pkg.go.dev/go.etcd.io/etcd/raft/v3. \ No newline at end of file diff --git a/raft/raft.go b/raft/raft.go index d9d9419d4..1dd9bdfa0 100644 --- a/raft/raft.go +++ b/raft/raft.go @@ -397,7 +397,8 @@ func (r *raft) hardState() pb.HardState { } } -// send persists state to stable storage and then sends to its mailbox. +// send schedules persisting state to a stable storage and AFTER that +// sending the message (as part of next Ready message processing). func (r *raft) send(m pb.Message) { if m.From == None { m.From = r.id diff --git a/raft/raftpb/raft.proto b/raft/raftpb/raft.proto index 23d62ec2f..ed4e45151 100644 --- a/raft/raftpb/raft.proto +++ b/raft/raftpb/raft.proto @@ -33,6 +33,8 @@ message Snapshot { optional SnapshotMetadata metadata = 2 [(gogoproto.nullable) = false]; } +// For description of different message types, see: +// https://pkg.go.dev/go.etcd.io/etcd/raft/v3#hdr-MessageType enum MessageType { MsgHup = 0; MsgBeat = 1; diff --git a/scripts/build-binary b/scripts/build-binary index 4875c4cd3..5bc31cc49 100755 --- a/scripts/build-binary +++ b/scripts/build-binary @@ -2,97 +2,104 @@ set -e +source ./scripts/test_lib.sh + VER=$1 -PROJ="etcd" +REPOSITORY="${REPOSITORY:-https://github.com/etcd-io/etcd}" + if [ -z "$1" ]; then - echo "Usage: ${0} VERSION" >> /dev/stderr - exit 255 + echo "Usage: ${0} VERSION" >> /dev/stderr + exit 255 fi set -u function setup_env { - local proj=${1} - local ver=${2} + local ver=${1} + local proj=${2} - if [ ! -d "${proj}" ]; then - git clone https://github.com/etcd-io/"${proj}" - fi + if [ ! -d "${proj}" ]; then + run git clone "${REPOSITORY}" + fi - pushd "${proj}" >/dev/null - git checkout master - git fetch --all - git reset --hard origin/master - git checkout "${ver}" - popd >/dev/null + pushd "${proj}" >/dev/null + run git checkout master + run git fetch --all + git_assert_branch_in_sync || exit 2 + run git checkout "${ver}" + git_assert_branch_in_sync || exit 2 + popd >/dev/null } function package { - local target=${1} - local srcdir="${2}/bin" + local target=${1} + local srcdir="${2}/bin" - local ccdir="${srcdir}/${GOOS}_${GOARCH}" - if [ -d "${ccdir}" ]; then - srcdir="${ccdir}" - fi - local ext="" - if [ "${GOOS}" == "windows" ]; then - ext=".exe" - fi - for bin in etcd etcdctl; do - cp "${srcdir}/${bin}" "${target}/${bin}${ext}" - done + local ccdir="${srcdir}/${GOOS}_${GOARCH}" + if [ -d "${ccdir}" ]; then + srcdir="${ccdir}" + fi + local ext="" + if [ "${GOOS}" == "windows" ]; then + ext=".exe" + fi + for bin in etcd etcdctl; do + cp "${srcdir}/${bin}" "${target}/${bin}${ext}" + done - cp etcd/README.md "${target}"/README.md - cp etcd/etcdctl/README.md "${target}"/README-etcdctl.md - cp etcd/etcdctl/READMEv2.md "${target}"/READMEv2-etcdctl.md + cp etcd/README.md "${target}"/README.md + cp etcd/etcdctl/README.md "${target}"/README-etcdctl.md + cp etcd/etcdctl/READMEv2.md "${target}"/READMEv2-etcdctl.md - cp -R etcd/Documentation "${target}"/Documentation + cp -R etcd/Documentation "${target}"/Documentation } function main { - mkdir release - cd release - setup_env "${PROJ}" "${VER}" + local proj=$(echo "${REPOSITORY}" | sed 's|^.*/\([^/]*\)$|\1|g') - tarcmd=tar - if [[ $(go env GOOS) == "darwin" ]]; then - echo "Please use linux machine for release builds." - exit 1 - fi + mkdir -p release + cd release + setup_env "${VER}" "${proj}" - for os in darwin windows linux; do - export GOOS=${os} - TARGET_ARCHS=("amd64") + tarcmd=tar + if [[ $(go env GOOS) == "darwin" ]]; then + echo "Please use linux machine for release builds." + exit 1 + fi - if [ ${GOOS} == "linux" ]; then - TARGET_ARCHS+=("arm64") - TARGET_ARCHS+=("ppc64le") - TARGET_ARCHS+=("s390x") - fi + for os in darwin windows linux; do + export GOOS=${os} + TARGET_ARCHS=("amd64") - for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do - export GOARCH=${TARGET_ARCH} + if [ ${GOOS} == "linux" ]; then + TARGET_ARCHS+=("arm64") + TARGET_ARCHS+=("ppc64le") +# TODO: Reenable when https://github.com/etcd-io/etcd/issues/12496 is fixed. +# TARGET_ARCHS+=("s390x") + fi - pushd etcd >/dev/null - GO_LDFLAGS="-s" ./build - popd >/dev/null + for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do + export GOARCH=${TARGET_ARCH} - TARGET="etcd-${VER}-${GOOS}-${GOARCH}" - mkdir "${TARGET}" - package "${TARGET}" "${PROJ}" + pushd etcd >/dev/null + GO_LDFLAGS="-s" ./build + popd >/dev/null - if [ ${GOOS} == "linux" ]; then - ${tarcmd} cfz "${TARGET}.tar.gz" "${TARGET}" - echo "Wrote release/${TARGET}.tar.gz" - else - zip -qr "${TARGET}.zip" "${TARGET}" - echo "Wrote release/${TARGET}.zip" - fi - done - done + TARGET="etcd-${VER}-${GOOS}-${GOARCH}" + mkdir "${TARGET}" + package "${TARGET}" "${proj}" + + if [ ${GOOS} == "linux" ]; then + ${tarcmd} cfz "${TARGET}.tar.gz" "${TARGET}" + echo "Wrote release/${TARGET}.tar.gz" + else + zip -qr "${TARGET}.zip" "${TARGET}" + echo "Wrote release/${TARGET}.zip" + fi + done + done } main diff --git a/scripts/build-docker b/scripts/build-docker index 1e9db2301..e32aedd9c 100755 --- a/scripts/build-docker +++ b/scripts/build-docker @@ -3,8 +3,8 @@ set -e if [ "$#" -ne 1 ]; then - echo "Usage: $0 VERSION" >&2 - exit 1 + echo "Usage: $0 VERSION" >&2 + exit 1 fi VERSION=${1} @@ -12,20 +12,20 @@ ARCH=$(go env GOARCH) DOCKERFILE="Dockerfile-release" if [ -z "${BINARYDIR}" ]; then - RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH) - BINARYDIR="${RELEASE}" - TARFILE="${RELEASE}.tar.gz" - TARURL="https://github.com/etcd-io/etcd/releases/download/${1}/${TARFILE}" - if ! curl -f -L -o "${TARFILE}" "${TARURL}" ; then - echo "Failed to download ${TARURL}." - exit 1 - fi - tar -zvxf "${TARFILE}" + RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH) + BINARYDIR="${RELEASE}" + TARFILE="${RELEASE}.tar.gz" + TARURL="https://github.com/etcd-io/etcd/releases/download/${1}/${TARFILE}" + if ! curl -f -L -o "${TARFILE}" "${TARURL}" ; then + echo "Failed to download ${TARURL}." + exit 1 + fi + tar -zvxf "${TARFILE}" fi if [ "${ARCH}" != "amd64" ]; then - DOCKERFILE+=".${ARCH}" - VERSION+="-${ARCH}" + DOCKERFILE+=".${ARCH}" + VERSION+="-${ARCH}" fi BINARYDIR=${BINARYDIR:-.} diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 85a7eb3ab..75df72966 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -5,6 +5,8 @@ # set -e +source ./scripts/test_lib.sh + VERSION=$1 if [ -z "${VERSION}" ]; then echo "Usage: ${0} VERSION" >> /dev/stderr @@ -19,11 +21,12 @@ fi ETCD_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. pushd "${ETCD_ROOT}" >/dev/null - echo Building etcd binary... + log_callout "Building etcd binary..." ./scripts/build-binary "${VERSION}" - for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do - echo Building ${TARGET_ARCH} docker image... +# TODO: Add "s390x" when https://github.com/etcd-io/etcd/issues/12496 is fixed. + for TARGET_ARCH in "amd64" "arm64" "ppc64le"; do + log_callout "Building ${TARGET_ARCH} docker image..." GOOS=linux GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker "${VERSION}" done popd >/dev/null diff --git a/scripts/fix.sh b/scripts/fix.sh index 264854b9b..c512444b4 100755 --- a/scripts/fix.sh +++ b/scripts/fix.sh @@ -11,10 +11,13 @@ function mod_tidy_fix { } function bash_ws_fix { + TAB=$'\t' + log_callout "Fixing whitespaces in the bash scripts" # Makes sure all bash scripts do use ' ' (double space) for indention. - log_cmd "find ./ -name '*.sh' -print0 | xargs -0 sed --follow-symlinks -i 's|\t| |g'" - find ./ -name '*.sh' -print0 | xargs -0 sed --follow-symlinks -i 's|\t| |g' + log_cmd "find ./ -name '*.sh' -print0 | xargs -0 sed -i.bak 's|${TAB}| |g'" + find ./ -name '*.sh' -print0 | xargs -0 sed -i.bak "s|${TAB}| |g" + find ./ -name '*.sh.bak' -print0 | xargs -0 rm } log_callout -e "\nFixing etcd code for you...\n" diff --git a/scripts/genproto.sh b/scripts/genproto.sh index c7656ac91..d8c5b6052 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -30,13 +30,12 @@ echo " - protoc-gen-grpc-gateway: ${GRPC_GATEWAY_BIN}" echo " - swagger: ${SWAGGER_BIN}" echo " - gogoproto-root: ${GOGOPROTO_ROOT}" echo " - grpc-gateway-root: ${GRPC_GATEWAY_ROOT}" - -# directories containing protos to be built -DIRS="./wal/walpb ./api/etcdserverpb ./etcdserver/api/snap/snappb ./raft/raftpb ./api/mvccpb ./lease/leasepb ./api/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb ./api/membershippb" - GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf" -log_callout -e "\nRunning gofast proto generation..." +# directories containing protos to be built +DIRS="./server/wal/walpb ./api/etcdserverpb ./server/etcdserver/api/snap/snappb ./raft/raftpb ./api/mvccpb ./server/lease/leasepb ./api/authpb ./server/etcdserver/api/v3lock/v3lockpb ./server/etcdserver/api/v3election/v3electionpb ./api/membershippb" + +log_callout -e "\nRunning gofast (gogo) proto generation..." for dir in ${DIRS}; do run pushd "${dir}" @@ -51,12 +50,11 @@ for dir in ${DIRS}; do run popd done -#return log_callout -e "\nRunning swagger & grpc_gateway proto generation..." # remove old swagger files so it's obvious whether the files fail to generate rm -rf Documentation/dev-guide/apispec/swagger/*json -for pb in api/etcdserverpb/rpc etcdserver/api/v3lock/v3lockpb/v3lock etcdserver/api/v3election/v3electionpb/v3election; do +for pb in api/etcdserverpb/rpc server/etcdserver/api/v3lock/v3lockpb/v3lock server/etcdserver/api/v3election/v3electionpb/v3election; do log_callout "grpc & swagger for: ${pb}.proto" run protoc -I. \ -I"${GRPC_GATEWAY_ROOT}"/third_party/googleapis \ @@ -77,6 +75,7 @@ for pb in api/etcdserverpb/rpc etcdserver/api/v3lock/v3lockpb/v3lock etcdserver/ sed -i -E "s# (New[a-zA-Z0-9_]*Client\()# ${pkg}.\1#g" "${gwfile}" sed -i -E "s|go.etcd.io/etcd|go.etcd.io/etcd/v3|g" "${gwfile}" sed -i -E "s|go.etcd.io/etcd/v3/api|go.etcd.io/etcd/api/v3|g" "${gwfile}" + sed -i -E "s|go.etcd.io/etcd/v3/server|go.etcd.io/etcd/server/v3|g" "${gwfile}" run go fmt "${gwfile}" @@ -96,14 +95,14 @@ if [ "$1" != "--skip-protodoc" ]; then log_callout "protodoc is auto-generating grpc API reference documentation..." run rm -rf Documentation/dev-guide/api_reference_v3.md - run_go_tool go.etcd.io/protodoc --directories="api/etcdserverpb=service_message,api/mvccpb=service_message,lease/leasepb=service_message,api/authpb=service_message" \ + run_go_tool go.etcd.io/protodoc --directories="api/etcdserverpb=service_message,api/mvccpb=service_message,server/lease/leasepb=service_message,api/authpb=service_message" \ --title="etcd API Reference" \ --output="Documentation/dev-guide/api_reference_v3.md" \ --message-only-from-this-file="api/etcdserverpb/rpc.proto" \ --disclaimer="This is a generated documentation. Please read the proto files for more." || exit 2 run rm -rf Documentation/dev-guide/api_concurrency_reference_v3.md - run_go_tool go.etcd.io/protodoc --directories="etcdserver/api/v3lock/v3lockpb=service_message,etcdserver/api/v3election/v3electionpb=service_message,api/mvccpb=service_message" \ + run_go_tool go.etcd.io/protodoc --directories="server/etcdserver/api/v3lock/v3lockpb=service_message,server/etcdserver/api/v3election/v3electionpb=service_message,api/mvccpb=service_message" \ --title="etcd concurrency API Reference" \ --output="Documentation/dev-guide/api_concurrency_reference_v3.md" \ --disclaimer="This is a generated documentation. Please read the proto files for more." || exit 2 diff --git a/scripts/release b/scripts/release index 2b71b9ffa..35084c810 100755 --- a/scripts/release +++ b/scripts/release @@ -4,6 +4,8 @@ set -o errexit set -o nounset set -o pipefail +source ./scripts/test_lib.sh + help() { echo "$(basename "$0") [version]" echo "Release etcd using the same approach as the etcd-release-runbook (https://goo.gl/Gxwysq)" @@ -30,7 +32,8 @@ main() { fi RELEASE_VERSION="v${VERSION}" MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2) - BRANCH="release-${MINOR_VERSION}" + BRANCH=${BRANCH:-"release-${MINOR_VERSION}"} + REPOSITORY=${REPOSITORY:-"https://github.com/etcd-io/etcd"} if ! command -v docker >/dev/null; then echo "cannot find docker" @@ -42,28 +45,32 @@ main() { # Set up release directory. local reldir="/tmp/etcd-release-${VERSION}" + log_callout "Preparing temporary directory: ${reldir}" if [ ! -d "${reldir}/etcd" ]; then mkdir -p "${reldir}" cd "${reldir}" - git clone https://github.com/etcd-io/etcd.git --branch "${BRANCH}" + run git clone "${REPOSITORY}" --branch "${BRANCH}" fi - cd "${reldir}/etcd" + run cd "${reldir}/etcd" || exit 2 + run git checkout "${BRANCH}" || exit 2 + run git pull origin + git_assert_branch_in_sync || exit 2 # If a release version tag already exists, use it. local remote_tag_exists - remote_tag_exists=$(git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}") + remote_tag_exists=$(run git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}" || true) + if [ "${remote_tag_exists}" -gt 0 ]; then - echo "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}" + log_callout "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}" git checkout -q "tags/${RELEASE_VERSION}" fi # Check go version. - # download "yq" from https://github.com/mikefarah/yq local go_version current_go_version - go_version="go$(yq read .travis.yml "go[0]")" + go_version="go$(run_go_tool "github.com/mikefarah/yq/v3" read .travis.yml "go[0]")" current_go_version=$(go version | awk '{ print $3 }') if [[ "${current_go_version}" != "${go_version}" ]]; then - echo "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .travis.yml)." + log_error "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .travis.yml)." exit 1 fi @@ -71,19 +78,23 @@ main() { if [ "${remote_tag_exists}" -eq 0 ]; then # Bump version/version.go to release version. local source_version - source_version=$(grep -E "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g") + source_version=$(grep -E "\s+Version\s*=" api/version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g") if [[ "${source_version}" != "${VERSION}" ]]; then source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2) if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then - echo "Wrong etcd minor version in version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting." + log_error "Wrong etcd minor version in api/version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting." exit 1 fi - echo "Updating version from ${source_version} to ${VERSION} in version/version.go" - sed -i "s/${source_version}/${VERSION}/g" version/version.go + log_callout "Updating modules definitions" + TARGET_VERSION="v${VERSION}" DRY_RUN=false ./scripts/release_mod.sh update_versions + + log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go" + sed -i "s/${source_version}/${VERSION}/g" api/version/version.go fi - echo "Building etcd and checking --version output" - ./build + + log_callout "Building etcd and checking --version output" + run ./build local etcd_version etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }') if [[ "${etcd_version}" != "${VERSION}" ]]; then @@ -92,30 +103,27 @@ main() { fi if [[ -n $(git status -s) ]]; then - echo "Committing version/version.go update." - git add version/version.go - git commit -m "version: bump up to ${VERSION}" - git diff --staged + log_callout "Committing mods & api/version/version.go update." + run git add api/version/version.go + run git add $(find -name go.mod | xargs) + run git diff --staged | cat + run git commit -m "version: bump up to ${VERSION}" + run git diff --staged | cat fi # Push the version change if it's not already been pushed. if [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then - read -p "Push version bump up to ${VERSION} to github.com/etcd-io/etcd [y/N]? " -r confirm + read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm [[ "${confirm,,}" == "y" ]] || exit 1 git push fi # Tag release. if [ "$(git tag --list | grep -c "${RELEASE_VERSION}")" -gt 0 ]; then - echo "Skipping tag step. git tag ${RELEASE_VERSION} already exists." + log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists." else - echo "Tagging release..." - KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }') - if [[ -z "${KEYID}" ]]; then - echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?" - exit 1 - fi - git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}" + log_callout "Tagging release..." + REMOTE_REPO="origin" DRY_RUN=false ./scripts/release_mod.sh push_mod_tags fi # Verify the latest commit has the version tag @@ -126,31 +134,20 @@ main() { fi # Verify the version tag is on the right branch - local branch=$(git branch --contains "${RELEASE_VERSION}") - if [ "${branch}" != "release-${MINOR_VERSION}" ]; then - echo "Error: Git tag ${RELEASE_VERSION} should be on branch release-${MINOR_VERSION} but is on ${branch}" + local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3) + if [ "${branch}" != "${BRANCH}" ]; then + echo "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'" exit 1 fi - - # Push the tag change if it's not already been pushed. - read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " -r confirm - [[ "${confirm,,}" == "y" ]] || exit 1 - git push origin "tags/${RELEASE_VERSION}" fi # Build release. # TODO: check the release directory for all required build artifacts. if [ -d release ]; then - echo "Skpping release build step. /release directory already exists." + log_warning "Skipping release build step. /release directory already exists." else - echo "Building release..." - # Check for old and new names of the release build script. - # TODO: Move the release script into this on as a function? - if [ -f ./scripts/release.sh ]; then - ./scripts/release.sh "${RELEASE_VERSION}" - else - ./scripts/build-release.sh "${RELEASE_VERSION}" - fi + log_callout "Building release..." + REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}" fi # Sanity checks. diff --git a/scripts/release_mod.sh b/scripts/release_mod.sh index 21afdb0df..f69824f61 100755 --- a/scripts/release_mod.sh +++ b/scripts/release_mod.sh @@ -46,12 +46,12 @@ function update_module_version() { local modules modules=$(run go list -f '{{if not .Main}}{{if not .Indirect}}{{.Path}}{{end}}{{end}}' -m all) - v3deps=$(echo "${modules}" | grep -E "${REPO}/.*/v3") + v3deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v3") for dep in ${v3deps}; do maybe_run go mod edit -require "${dep}@${v3version}" done - v2deps=$(echo "${modules}" | grep -E "${REPO}/.*/v2") + v2deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v2") for dep in ${v2deps}; do maybe_run go mod edit -require "${dep}@${v2version}" done @@ -81,6 +81,15 @@ function update_versions_cmd() { run_for_modules update_module_version "${v3version}" "${v2version}" } +function get_gpg_key { + keyid=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }') + if [[ -z "${keyid}" ]]; then + log_error "Failed to load gpg key. Is gpg set up correctly for etcd releases?" + return 2 + fi + echo "$keyid" +} + function push_mod_tags_cmd { assert_no_git_modifications || return 2 @@ -92,15 +101,17 @@ function push_mod_tags_cmd { # Any module ccan be used for this local master_version - master_version=$(go list -f '{{.Version}}' -m "${REPO}/api/v3") + master_version=$(go list -f '{{.Version}}' -m "${ROOT_MODULE}/api/v3") local tags=() + keyid=$(get_gpg_key) || return 2 + for module in $(modules); do local version version=$(go list -f '{{.Version}}' -m "${module}") local path path=$(go list -f '{{.Path}}' -m "${module}") - local subdir="${path//${REPO}\//}" + local subdir="${path//${ROOT_MODULE}\//}" local tag if [ -z "${version}" ]; then tag="${master_version}" @@ -110,7 +121,10 @@ function push_mod_tags_cmd { fi log_info "Tags for: ${module} version:${version} tag:${tag}" - maybe_run git tag -f "${tag}" + # The sleep is ugly hack that guarantees that 'git describe' will + # consider main-module's tag as the latest. + run sleep 2 + maybe_run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}" tags=("${tags[@]}" "${tag}") done maybe_run git push -f "${REMOTE_REPO}" "${tags[@]}" diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh index ec2045d7d..53e146a54 100644 --- a/scripts/test_lib.sh +++ b/scripts/test_lib.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -REPO="go.etcd.io/etcd" +ROOT_MODULE="go.etcd.io/etcd" -if [[ "$(go list)" != "${REPO}/v3" ]]; then - echo "must be run from '${REPO}/v3' module directory" +if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then + echo "must be run from '${ROOT_MODULE}/v3' module directory" exit 255 fi -ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${REPO}/v3") +ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3") #### Convenient IO methods ##### @@ -160,15 +160,15 @@ function run_for_module { function modules() { modules=( - "${REPO}/api/v3" - "${REPO}/pkg/v3" - "${REPO}/raft/v3" - "${REPO}/client/v2" - "${REPO}/client/v3" - "${REPO}/server/v3" - "${REPO}/etcdctl/v3" - "${REPO}/tests/v3" - "${REPO}/v3") + "${ROOT_MODULE}/api/v3" + "${ROOT_MODULE}/pkg/v3" + "${ROOT_MODULE}/raft/v3" + "${ROOT_MODULE}/client/v2" + "${ROOT_MODULE}/client/v3" + "${ROOT_MODULE}/server/v3" + "${ROOT_MODULE}/etcdctl/v3" + "${ROOT_MODULE}/tests/v3" + "${ROOT_MODULE}/v3") echo "${modules[@]}" } @@ -303,10 +303,10 @@ function tool_get_bin { local tool="$1" if [[ "$tool" == *"@"* ]]; then # shellcheck disable=SC2086 - run gobin ${GOBINARGS} -p "${tool}" || return 2 + run gobin ${GOBINARGS:-} -p "${tool}" || return 2 else # shellcheck disable=SC2086 - run_for_module ./tools/mod run gobin ${GOBINARGS} -p -m --mod=readonly "${tool}" || return 2 + run_for_module ./tools/mod run gobin ${GOBINARGS:-} -p -m --mod=readonly "${tool}" || return 2 fi } @@ -339,3 +339,22 @@ function assert_no_git_modifications { fi } +# makes sure that the current branch is in sync with the origin branch: +# - no uncommitted nor unstaged changes +# - no differencing commits in relation to the origin/$branch +function git_assert_branch_in_sync { + local branch + branch=$(git branch --show-current) + if [[ $(run git status --porcelain --untracked-files=no) ]]; then + log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes" + log_error "Consider cleaning up / renaming this directory." + return 2 + fi + ref_local=$(run git rev-parse "${branch}") + ref_origin=$(run git rev-parse "origin/${branch}") + if [ "x${ref_local}" != "x${ref_origin}" ]; then + log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin." + log_error "Consider cleaning up / renaming this directory." + return 2 + fi +} diff --git a/server/embed/config_logging.go b/server/embed/config_logging.go index 1b0909221..7ab81191e 100644 --- a/server/embed/config_logging.go +++ b/server/embed/config_logging.go @@ -40,6 +40,26 @@ func (cfg Config) GetLogger() *zap.Logger { // for testing var grpcLogOnce = new(sync.Once) +func setupGrpcLogging(debug bool, config zap.Config) { + grpcLogOnce.Do(func() { + // debug true, enable info, warning, error + // debug false, only discard info + if debug { + var gl grpclog.LoggerV2 + gl, err := logutil.NewGRPCLoggerV2(config) + if err == nil { + grpclog.SetLoggerV2(gl) + } + } else { + grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr)) + } + }) +} + +func SetupGrpcLoggingForTest(debug bool) { + setupGrpcLogging(debug, zap.NewDevelopmentConfig()) +} + // setupLogging initializes etcd logging. // Must be called after flag parsing or finishing configuring embed.Config. func (cfg *Config) setupLogging() error { @@ -106,19 +126,7 @@ func (cfg *Config) setupLogging() error { c.loggerConfig = &copied c.loggerCore = nil c.loggerWriteSyncer = nil - grpcLogOnce.Do(func() { - // debug true, enable info, warning, error - // debug false, only discard info - if cfg.LogLevel == "debug" { - var gl grpclog.LoggerV2 - gl, err = logutil.NewGRPCLoggerV2(copied) - if err == nil { - grpclog.SetLoggerV2(gl) - } - } else { - grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr)) - } - }) + setupGrpcLogging(cfg.LogLevel == "debug", copied) return nil } } diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go index dae67c4ae..9bca4647d 100644 --- a/server/etcdserver/api/membership/cluster.go +++ b/server/etcdserver/api/membership/cluster.go @@ -556,7 +556,7 @@ func (c *RaftCluster) IsReadyToAddVotingMember() bool { if nstarted == 1 && nmembers == 2 { // a case of adding a new node to 1-member cluster for restoring cluster data - // https://github.com/etcd-io/etcd/blob/master/Documentation/v2/admin_guide.md#restoring-the-cluster + // https://github.com/etcd-io/website/blob/master/content/docs/v2/admin_guide.md#restoring-the-cluster c.lg.Debug("number of started member is 1; can accept add member request") return true } diff --git a/server/etcdserver/api/v2error/error.go b/server/etcdserver/api/v2error/error.go index 1244290c4..a84572f40 100644 --- a/server/etcdserver/api/v2error/error.go +++ b/server/etcdserver/api/v2error/error.go @@ -13,7 +13,8 @@ // limitations under the License. // Package v2error describes errors in etcd project. When any change happens, -// Documentation/v2/errorcode.md needs to be updated correspondingly. +// https://github.com/etcd-io/website/blob/master/content/docs/v2/errorcode.md +// needs to be updated correspondingly. // To be deprecated in favor of v3 APIs. package v2error diff --git a/server/etcdserver/api/v2store/node_extern_test.go b/server/etcdserver/api/v2store/node_extern_test.go index 067cdc42d..48daeb648 100644 --- a/server/etcdserver/api/v2store/node_extern_test.go +++ b/server/etcdserver/api/v2store/node_extern_test.go @@ -18,7 +18,6 @@ import ( "reflect" "testing" "time" - "unsafe" "go.etcd.io/etcd/pkg/v3/testutil" ) @@ -102,7 +101,7 @@ func TestNodeExternClone(t *testing.T) { } func sameSlice(a, b []*NodeExtern) bool { - ah := (*reflect.SliceHeader)(unsafe.Pointer(&a)) - bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - return *ah == *bh + va := reflect.ValueOf(a) + vb := reflect.ValueOf(b) + return va.Len() == vb.Len() && va.Pointer() == vb.Pointer() } diff --git a/server/etcdserver/api/v3election/v3electionpb/gw/v3election.pb.gw.go b/server/etcdserver/api/v3election/v3electionpb/gw/v3election.pb.gw.go index 6465e26c9..4a6e5e724 100644 --- a/server/etcdserver/api/v3election/v3electionpb/gw/v3election.pb.gw.go +++ b/server/etcdserver/api/v3election/v3electionpb/gw/v3election.pb.gw.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. -// source: etcdserver/api/v3election/v3electionpb/v3election.proto +// source: server/etcdserver/api/v3election/v3electionpb/v3election.proto /* Package v3electionpb is a reverse proxy. diff --git a/server/etcdserver/api/v3lock/v3lockpb/gw/v3lock.pb.gw.go b/server/etcdserver/api/v3lock/v3lockpb/gw/v3lock.pb.gw.go index fd455fdde..dc573d79c 100644 --- a/server/etcdserver/api/v3lock/v3lockpb/gw/v3lock.pb.gw.go +++ b/server/etcdserver/api/v3lock/v3lockpb/gw/v3lock.pb.gw.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. -// source: etcdserver/api/v3lock/v3lockpb/v3lock.proto +// source: server/etcdserver/api/v3lock/v3lockpb/v3lock.proto /* Package v3lockpb is a reverse proxy. diff --git a/server/wal/wal.go b/server/wal/wal.go index bf820a61f..37f6dd136 100644 --- a/server/wal/wal.go +++ b/server/wal/wal.go @@ -419,6 +419,13 @@ func openWALFiles(lg *zap.Logger, dirpath string, names []string, nameIndex int, // TODO: detect not-last-snap error. // TODO: maybe loose the checking of match. // After ReadAll, the WAL will be ready for appending new records. +// +// ReadAll suppresses WAL entries that got overridden (i.e. a newer entry with the same index +// exists in the log). Such a situation can happen in cases described in figure 7. of the +// RAFT paper (http://web.stanford.edu/~ouster/cgi-bin/papers/raft-atc14.pdf). +// +// ReadAll may return uncommitted yet entries, that are subject to be overriden. +// Do not apply entries that have index > state.commit, as they are subject to change. func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb.Entry, err error) { w.mu.Lock() defer w.mu.Unlock() @@ -443,6 +450,7 @@ func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb. // return error before append call causes runtime panic return nil, state, nil, ErrSliceOutOfRange } + // The line below is potentially overriding some 'uncommitted' entries. ents = append(ents[:up], e) } w.enti = e.Index diff --git a/test b/test index ee99a99f2..e47fcb6ef 100755 --- a/test +++ b/test @@ -1,612 +1,6 @@ #!/usr/bin/env bash -# -# Run all etcd tests -# ./test -# ./test -v -# -# -# Run specified test pass -# -# $ PASSES=unit ./test -# $ PASSES=integration ./test -# -# -# Run tests for one package -# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT -# flag for different expectation -# -# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test -# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test -# -# Run specified unit tests in one package -# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew "; -# to run only "TestNew", set "TESTCASE="\bTestNew\b"" -# -# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test -# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test -# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test -# -# -# Run code coverage -# COVERDIR must either be a absolute path or a relative path to the etcd root -# $ COVERDIR=coverage PASSES="build build_cov cov" ./test -# $ go tool cover -html ./coverage/cover.out -set -e -set -o pipefail +echo -e "\e[91mDEPRECATED!!! Use test.sh script instead.\e[0m\n" +sleep 1 -# Consider command as failed when any component of the pipe fails: -# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash -set -o pipefail - -# The test script is not supposed to make any changes to the files -# e.g. add/update missing dependencies. Such divergences should be -# detected and trigger a failure that needs explicit developer's action. -export GOFLAGS=-mod=readonly - -source ./scripts/test_lib.sh -source ./build - -PASSES=${PASSES:-"fmt bom dep build unit"} -PKG=${PKG:-} - -if [ -z "$GOARCH" ]; then - GOARCH=$(go env GOARCH); -fi - -# determine the number of CPUs to use for Go tests -CPU=${CPU:-"4"} - -# determine whether target supports race detection -if [ -z "${RACE}" ] ; then - if [ "$GOARCH" == "amd64" ]; then - RACE="--race" - else - RACE="--race=false" - fi -else - RACE="--race=${RACE:-true}" -fi - -# This options make sense for cases where SUT (System Under Test) is compiled by test. -COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}") -log_callout "Running with ${COMMON_TEST_FLAGS[*]}" - -RUN_ARG=() -if [ -n "${TESTCASE}" ]; then - RUN_ARG=("-run=${TESTCASE}") -fi - -function build_pass { - log_callout "Building etcd" - run_for_modules run go build "${@}" || return 2 - GO_BUILD_FLAGS="-v" etcd_build "${@}" - GO_BUILD_FLAGS="-v" tools_build "${@}" -} - -################# REGULAR TESTS ################################################ - -# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs] -function run_unit_tests { - local pkgs="${1:-./...}" - shift 1 - # shellcheck disable=SC2086 - go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" -} - -function unit_pass { - run_for_modules run_unit_tests "$@" -} - -function integration_extra { - if [ -z "${PKG}" ] ; then - run_for_module "." go_test "./contrib/raftexample" "keep_going" : -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $? - run_for_module "tests" go_test "./integration/v2store/..." "keep_going" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $? - else - log_warning "integration_extra ignored when PKG is specified" - fi -} - -function integration_pass { - local pkgs=${USERPKG:-"./integration/..."} - run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $? - integration_extra "$@" -} - -function e2e_pass { - # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact. - run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@" -} - -function integration_e2e_pass { - run_pass "integration" "${@}" - run_pass "e2e" "${@}" -} - -# generic_checker [cmd...] -# executes given command in the current module, and clearly fails if it -# failed or returned output. -function generic_checker { - local cmd=("$@") - if ! output=$("${cmd[@]}"); then - echo "${output}" - log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)" - return 255 - fi - if [ -n "${output}" ]; then - echo "${output}" - log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)" - return 255 - fi -} - -function functional_pass { - run ./tests/functional/build - - # Clean up any data and logs from previous runs - rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup - - # TODO: These ports should be dynamically allocated instead of hard-coded. - for a in 1 2 3; do - ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 < /dev/null & - pid="$!" - agent_pids="${agent_pids} $pid" - done - - for a in 1 2 3; do - log_callout "Waiting for 'etcd-agent' on ${a}9027..." - while ! nc -z localhost ${a}9027; do - sleep 1 - done - done - - log_callout "functional test START!" - run ./bin/etcd-tester --config ./tests/functional/functional.yaml && log_success "'etcd-tester' succeeded" - local etcd_tester_exit_code=$? - - if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then - log_error "ETCD_TESTER_EXIT_CODE:" ${etcd_tester_exit_code} - fi - - # shellcheck disable=SC2206 - agent_pids=($agent_pids) - kill -s TERM "${agent_pids[@]}" || true - - if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then - log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'" - tail -1000 /tmp/etcd-functional-1/etcd.log - - log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'" - tail -1000 /tmp/etcd-functional-2/etcd.log - - log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'" - tail -1000 /tmp/etcd-functional-3/etcd.log - - log_error "--- FAIL: exit code" ${etcd_tester_exit_code} - return ${etcd_tester_exit_code} - fi - log_success "functional test PASS!" -} - -function grpcproxy_pass { - run_for_module "tests" go_test "./integration/... ./e2e" "fail_fast" : \ - -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@" -} - -################# COVERAGE ##################################################### - -# Builds artifacts used by tests/e2e in coverage mode. -function build_cov_pass { - local out="${BINDIR:-./bin}" - run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test" - run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl" -} - -# pkg_to_coverflag [prefix] [pkgs] -# produces name of .coverprofile file to be used for tests of this package -function pkg_to_coverprofileflag { - local prefix="${1}" - local pkgs="${2}" - local pkgs_normalized - pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+") - echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile" -} - -function cov_pass { - # shellcheck disable=SC2153 - if [ -z "$COVERDIR" ]; then - log_error "COVERDIR undeclared" - return 255 - fi - - if [ ! -f "bin/etcd_test" ]; then - log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test" - return 255 - fi - - local coverdir - coverdir=$(readlink -f "${COVERDIR}") - mkdir -p "${coverdir}" - rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*" - - local covpkgs - covpkgs=$(pkgs_in_module "./...") - local coverpkg_comma - coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',') - local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma") - - local failed="" - - log_callout "Collecting coverage from unit tests ..." - go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \ - "${gocov_build_flags[@]}" "$@" || failed="$failed unit" - - log_callout "Collecting coverage from integration tests ..." - run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration" \ - -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration" - # integration-store-v2 - run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \ - -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3" - # integration_cluster_proxy - run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \ - -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy" - - log_callout "Collecting coverage from e2e tests ..." - # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here, - # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned. - run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e" - - log_callout "Collecting coverage from e2e tests with proxy ..." - run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy" - - log_callout "Merging coverage results ..." - local cover_out_file="${coverdir}/cover.out" - # gocovmerge requires not-empty test to start with: - echo "mode: set" > "${cover_out_file}" - - # incrementally merge to get coverage data even if some coverage files are corrupted - for f in "${coverdir}"/*.coverprofile; do - echo "merging test coverage file ${f}" - run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}" > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f" - if [ -s "${coverdir}"/cover.tmp ]; then - mv "${coverdir}/cover.tmp" "${cover_out_file}" - fi - done - # strip out generated files (using GNU-style sed) - sed --in-place '/generated.go/d' "${cover_out_file}" || true - - # held failures to generate the full coverage file, now fail - if [ -n "$failed" ]; then - for f in $failed; do - log_error "--- FAIL:" "$f" - done - log_warning "Despite failures, you can see partial report:" - log_warning " go tool cover -html ${cover_out_file}" - return 255 - fi - - log_success "done :) [see report: go tool cover -html ${cover_out_file}]" -} - -######### Code formatting checkers ############################################# - -function fmt_pass { - toggle_failpoints disable - - # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret" - # after resolving ore-existing errors. - for p in shellcheck \ - markdown_you \ - goword \ - gofmt \ - govet \ - revive \ - license_header \ - receiver_name \ - mod_tidy \ - dep \ - shellcheck \ - shellws \ - ; do - run_pass "${p}" "${@}" - done -} - -function shellcheck_pass { - if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then - generic_checker run shellcheck -fgcc build test scripts/*.sh - fi -} - -function shellws_pass { - log_callout "Ensuring no tab-based indention in shell scripts" - local files - files=$(find ./ -name '*.sh' -print0 | xargs -0 ) - log_cmd "grep -E -n $'^ *\t' ${files}" - # shellcheck disable=SC2086 - if grep -E -n $'^ *\t' ${files} | sed -s $'s|\t|[\\\\tab]|g'; then - log_error "FAIL: found tab-based indention in bash scripts. Use ' ' (double space)." - return 1 - else - log_success "SUCCESS: no tabulators found." - return 0 - fi -} - -function markdown_you_find_eschew_you { - local find_you_cmd="find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' ! -path './release/*' -exec grep -E --color '[Yy]ou[r]?[ '\''.,;]' {} + || true" - run eval "${find_you_cmd}" -} - -function markdown_you_pass { - generic_checker markdown_you_find_eschew_you -} - -function markdown_marker_pass { - # TODO: check other markdown files when marker handles headers with '[]' - if tool_exists "marker" "https://crates.io/crates/marker"; then - generic_checker run marker --skip-http --root ./Documentation 2>&1 - fi -} - -function govet_pass { - run_for_modules generic_checker run go vet -} - -function govet_shadow_pass { - local shadow - shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow") - run_for_modules generic_checker run go vet -all -vettool="${shadow}" -} - -function unparam_pass { - run_for_modules generic_checker run_go_tool "mvdan.cc/unparam" -} - -function staticcheck_pass { - run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck" -} - -function revive_pass { - run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..." -} - -function unconvert_pass { - run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v -} - -function ineffassign_per_package { - mapfile -t gofiles < <(go_srcs_in_module "$1") - run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}" -} - -function ineffassign_pass { - run_for_modules generic_checker ineffassign_per_package -} - -function nakedret_pass { - run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret" -} - -function license_header_pass { - mapfile -t gofiles < <(go_srcs_in_module "$1") - for file in "${gofiles[@]}"; do - if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then - licRes="${licRes}"$(echo -e " ${file}") - fi - done - if [ -n "${licRes}" ]; then - log_error -e "license header checking failed:\\n${licRes}" - return 255 - fi -} - -function receiver_name_for_package { - mapfile -t gofiles < <(go_srcs_in_module "$1") - recvs=$(grep 'func ([^*]' "${gofiles[@]}" | tr ':' ' ' | \ - awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" | sort | uniq | \ - grep -Ev "(Descriptor|Proto|_)" | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ') - if [ -n "${recvs}" ]; then - # shellcheck disable=SC2206 - recvs=($recvs) - for recv in "${recvs[@]}"; do - log_error "Mismatched receiver for $recv..." - grep "$recv" "${gofiles[@]}" | grep 'func (' - done - return 255 - fi -} - -function receiver_name_pass { - run_for_modules receiver_name_for_package -} - -# goword_for_package package -# checks spelling and comments in the 'package' in the current module -# -function goword_for_package { - mapfile -t gofiles < <(go_srcs_in_module "$1") - local gowordRes - - # spellchecking can be enabled with GOBINARGS="--tags=spell" - # but it requires heavy dependencies installation, like: - # apt-get install libaspell-dev libhunspell-dev hunspell-en-us aspell-en - - # only check for broke exported godocs - if gowordRes=$(run_go_tool "github.com/chzchzchz/goword" -use-spell=false "${gofiles[@]}" | grep godoc-export | sort); then - log_error -e "goword checking failed:\\n${gowordRes}" - return 255 - fi - if [ -n "$gowordRes" ]; then - log_error -e "goword checking returned output:\\n${gowordRes}" - return 255 - fi -} - - -function goword_pass { - run_for_modules goword_for_package || return 255 -} - -function go_fmt_for_package { - # We utilize 'go fmt' to find all files suitable for formatting, - # but reuse full power gofmt to perform just RO check. - go fmt -n "$1" | sed 's| -w | -d |g' | sh -} - -function gofmt_pass { - run_for_modules generic_checker go_fmt_for_package -} - -function bom_pass { - log_callout "Checking bill of materials..." - # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf - # shellcheck disable=SC2207 - modules=($(modules_exp)) - - # Internally license-bill-of-materials tends to modify go.sum - run cp go.sum go.sum.tmp || return 2 - run cp go.mod go.mod.tmp || return 2 - - output=$(GOFLAGS=-mod=mod run_go_tool github.com/coreos/license-bill-of-materials \ - --override-file ./bill-of-materials.override.json \ - "${modules[@]}") - code="$?" - - run cp go.sum.tmp go.sum || return 2 - run cp go.mod.tmp go.mod || return 2 - - if [ "${code}" -ne 0 ] ; then - log_error -e "license-bill-of-materials (code: ${code}) failed with:\n${output}" - return 255 - else - echo "${output}" > "bom-now.json.tmp" - fi - if ! diff ./bill-of-materials.json bom-now.json.tmp; then - log_error "modularized licenses do not match given bill of materials" - return 255 - fi - rm bom-now.json.tmp -} - -######## VARIOUS CHECKERS ###################################################### - -function dump_deps_of_module() { - local module - if ! module=$(run go list -m); then - return 255 - fi - run go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all -} - -# Checks whether dependencies are consistent across modules -function dep_pass { - local all_dependencies - all_dependencies=$(run_for_modules dump_deps_of_module | sort) || return 2 - - local duplicates - duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2 - - for dup in ${duplicates}; do - log_error "FAIL: inconsistent versions for depencency: ${dup}" - echo "${all_dependencies}" | grep "${dup}" | sed "s|\([^,]*\),\([^,]*\),\([^,]*\)| - \1@\2 from: \3|g" - done - if [[ -n "${duplicates}" ]]; then - log_error "FAIL: inconsistent dependencies" - return 2 - else - log_success "SUCCESS: dependencies are consistent across modules" - fi -} - -function release_pass { - rm -f ./bin/etcd-last-release - # to grab latest patch release; bump this up for every minor release - UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1) - if [ -n "$MANUAL_VER" ]; then - # in case, we need to test against different version - UPGRADE_VER=$MANUAL_VER - fi - if [[ -z ${UPGRADE_VER} ]]; then - UPGRADE_VER="v3.3.0" - log_warning "fallback to" ${UPGRADE_VER} - fi - - local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz" - log_callout "Downloading $file" - - set +e - curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file" - local result=$? - set -e - case $result in - 0) ;; - *) log_error "--- FAIL:" ${result} - return $result - ;; - esac - - tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1 - mkdir -p ./bin - mv /tmp/etcd ./bin/etcd-last-release -} - -function mod_tidy_for_module { - # Watch for upstream solution: https://github.com/golang/go/issues/27005 - local tmpModDir - tmpModDir=$(mktemp -d --suffix "etcd-mod") - run cp "./go.mod" "./go.sum" "${tmpModDir}" || return 2 - - # Guarantees keeping go.sum minimal - # If this is causing too much problems, we should - # stop controlling go.sum at all. - rm go.sum - run go mod tidy || return 2 - - set +e - local tmpFileGoModInSync - diff -C 5 "${tmpModDir}/go.mod" "./go.mod" - tmpFileGoModInSync="$?" - - local tmpFileGoSumInSync - diff -C 5 "${tmpModDir}/go.sum" "./go.sum" - tmpFileGoSumInSync="$?" - set -e - - # Bring back initial state - mv "${tmpModDir}/go.mod" "./go.mod" - mv "${tmpModDir}/go.sum" "./go.sum" - - if [ "${tmpFileGoModInSync}" -ne 0 ]; then - log_error "${PWD}/go.mod is not in sync with 'go mod tidy'" - return 255 - fi - if [ "${tmpFileGoSumInSync}" -ne 0 ]; then - log_error "${PWD}/go.sum is not in sync with 'rm go.sum; go mod tidy'" - return 255 - fi -} - -function mod_tidy_pass { - run_for_modules mod_tidy_for_module -} - -########### MAIN ############################################################### - -function run_pass { - local pass="${1}" - shift 1 - log_callout -e "\n'${pass}' started at $(date)" - if "${pass}_pass" "$@" ; then - log_success "'${pass}' completed at $(date)" - else - log_error "FAIL: '${pass}' failed at $(date)" - exit 255 - fi -} - -for pass in $PASSES; do - run_pass "${pass}" "${@}" -done - -log_success "SUCCESS" +source ./test.sh diff --git a/test.sh b/test.sh deleted file mode 120000 index 30d74d258..000000000 --- a/test.sh +++ /dev/null @@ -1 +0,0 @@ -test \ No newline at end of file diff --git a/test.sh b/test.sh new file mode 100755 index 000000000..289102148 --- /dev/null +++ b/test.sh @@ -0,0 +1,628 @@ +#!/usr/bin/env bash +# +# Run all etcd tests +# ./test +# ./test -v +# +# +# Run specified test pass +# +# $ PASSES=unit ./test +# $ PASSES=integration ./test +# +# +# Run tests for one package +# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT +# flag for different expectation +# +# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test +# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test +# +# Run specified unit tests in one package +# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew "; +# to run only "TestNew", set "TESTCASE="\bTestNew\b"" +# +# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test +# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test +# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test +# +# +# Run code coverage +# COVERDIR must either be a absolute path or a relative path to the etcd root +# $ COVERDIR=coverage PASSES="build build_cov cov" ./test +# $ go tool cover -html ./coverage/cover.out +set -e +set -o pipefail + + +# Consider command as failed when any component of the pipe fails: +# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash +set -o pipefail + +# The test script is not supposed to make any changes to the files +# e.g. add/update missing dependencies. Such divergences should be +# detected and trigger a failure that needs explicit developer's action. +export GOFLAGS=-mod=readonly + +source ./scripts/test_lib.sh +source ./build.sh + +PASSES=${PASSES:-"fmt bom dep build unit"} +PKG=${PKG:-} + +if [ -z "$GOARCH" ]; then + GOARCH=$(go env GOARCH); +fi + +# determine the number of CPUs to use for Go tests +CPU=${CPU:-"4"} + +# determine whether target supports race detection +if [ -z "${RACE}" ] ; then + if [ "$GOARCH" == "amd64" ]; then + RACE="--race" + else + RACE="--race=false" + fi +else + RACE="--race=${RACE:-true}" +fi + +# This options make sense for cases where SUT (System Under Test) is compiled by test. +COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}") +log_callout "Running with ${COMMON_TEST_FLAGS[*]}" + +RUN_ARG=() +if [ -n "${TESTCASE}" ]; then + RUN_ARG=("-run=${TESTCASE}") +fi + +function build_pass { + log_callout "Building etcd" + run_for_modules run go build "${@}" || return 2 + GO_BUILD_FLAGS="-v" etcd_build "${@}" + GO_BUILD_FLAGS="-v" tools_build "${@}" +} + +################# REGULAR TESTS ################################################ + +# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs] +function run_unit_tests { + local pkgs="${1:-./...}" + shift 1 + # shellcheck disable=SC2086 + go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" +} + +function unit_pass { + run_for_modules run_unit_tests "$@" +} + +function integration_extra { + if [ -z "${PKG}" ] ; then + run_for_module "." go_test "./contrib/raftexample" "keep_going" : -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $? + run_for_module "tests" go_test "./integration/v2store/..." "keep_going" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $? + else + log_warning "integration_extra ignored when PKG is specified" + fi +} + +function integration_pass { + local pkgs=${USERPKG:-"./integration/..."} + run_for_module "tests" go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $? + integration_extra "$@" +} + +function e2e_pass { + # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact. + run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@" +} + +function integration_e2e_pass { + run_pass "integration" "${@}" + run_pass "e2e" "${@}" +} + +# generic_checker [cmd...] +# executes given command in the current module, and clearly fails if it +# failed or returned output. +function generic_checker { + local cmd=("$@") + if ! output=$("${cmd[@]}"); then + echo "${output}" + log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)" + return 255 + fi + if [ -n "${output}" ]; then + echo "${output}" + log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)" + return 255 + fi +} + +function functional_pass { + run ./tests/functional/build + + # Clean up any data and logs from previous runs + rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup + + # TODO: These ports should be dynamically allocated instead of hard-coded. + for a in 1 2 3; do + ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 < /dev/null & + pid="$!" + agent_pids="${agent_pids} $pid" + done + + for a in 1 2 3; do + log_callout "Waiting for 'etcd-agent' on ${a}9027..." + while ! nc -z localhost ${a}9027; do + sleep 1 + done + done + + log_callout "functional test START!" + run ./bin/etcd-tester --config ./tests/functional/functional.yaml && log_success "'etcd-tester' succeeded" + local etcd_tester_exit_code=$? + + if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then + log_error "ETCD_TESTER_EXIT_CODE:" ${etcd_tester_exit_code} + fi + + # shellcheck disable=SC2206 + agent_pids=($agent_pids) + kill -s TERM "${agent_pids[@]}" || true + + if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then + log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'" + tail -1000 /tmp/etcd-functional-1/etcd.log + + log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'" + tail -1000 /tmp/etcd-functional-2/etcd.log + + log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'" + tail -1000 /tmp/etcd-functional-3/etcd.log + + log_error "--- FAIL: exit code" ${etcd_tester_exit_code} + return ${etcd_tester_exit_code} + fi + log_success "functional test PASS!" +} + +function grpcproxy_pass { + run_for_module "tests" go_test "./integration/... ./e2e" "fail_fast" : \ + -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@" +} + +################# COVERAGE ##################################################### + +# Builds artifacts used by tests/e2e in coverage mode. +function build_cov_pass { + local out="${BINDIR:-./bin}" + run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test" + run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl" +} + +# pkg_to_coverflag [prefix] [pkgs] +# produces name of .coverprofile file to be used for tests of this package +function pkg_to_coverprofileflag { + local prefix="${1}" + local pkgs="${2}" + local pkgs_normalized + pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+") + echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile" +} + +function cov_pass { + # shellcheck disable=SC2153 + if [ -z "$COVERDIR" ]; then + log_error "COVERDIR undeclared" + return 255 + fi + + if [ ! -f "bin/etcd_test" ]; then + log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test" + return 255 + fi + + local coverdir + coverdir=$(readlink -f "${COVERDIR}") + mkdir -p "${coverdir}" + rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*" + + local covpkgs + covpkgs=$(pkgs_in_module "./...") + local coverpkg_comma + coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',') + local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma") + + local failed="" + + log_callout "Collecting coverage from unit tests ..." + go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \ + "${gocov_build_flags[@]}" "$@" || failed="$failed unit" + + log_callout "Collecting coverage from integration tests ..." + run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration" \ + -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration" + # integration-store-v2 + run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \ + -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3" + # integration_cluster_proxy + run_for_module "tests" go_test "./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \ + -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy" + + log_callout "Collecting coverage from e2e tests ..." + # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here, + # as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned. + run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e" + + log_callout "Collecting coverage from e2e tests with proxy ..." + run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy" + + log_callout "Merging coverage results ..." + local cover_out_file="${coverdir}/cover.out" + # gocovmerge requires not-empty test to start with: + echo "mode: set" > "${cover_out_file}" + + # incrementally merge to get coverage data even if some coverage files are corrupted + for f in "${coverdir}"/*.coverprofile; do + echo "merging test coverage file ${f}" + run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}" > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f" + if [ -s "${coverdir}"/cover.tmp ]; then + mv "${coverdir}/cover.tmp" "${cover_out_file}" + fi + done + # strip out generated files (using GNU-style sed) + sed --in-place '/generated.go/d' "${cover_out_file}" || true + + # held failures to generate the full coverage file, now fail + if [ -n "$failed" ]; then + for f in $failed; do + log_error "--- FAIL:" "$f" + done + log_warning "Despite failures, you can see partial report:" + log_warning " go tool cover -html ${cover_out_file}" + return 255 + fi + + log_success "done :) [see report: go tool cover -html ${cover_out_file}]" +} + +######### Code formatting checkers ############################################# + +function fmt_pass { + toggle_failpoints disable + + # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret" + # after resolving ore-existing errors. + for p in shellcheck \ + markdown_you \ + goword \ + gofmt \ + govet \ + revive \ + license_header \ + receiver_name \ + mod_tidy \ + dep \ + shellcheck \ + shellws \ + ; do + run_pass "${p}" "${@}" + done +} + +function shellcheck_pass { + if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then + generic_checker run shellcheck -fgcc build test scripts/*.sh ./*.sh + fi +} + +function shellws_pass { + log_callout "Ensuring no tab-based indention in shell scripts" + local files + files=$(find ./ -name '*.sh' -print0 | xargs -0 ) + # shellcheck disable=SC2206 + files=( ${files[@]} "./scripts/build-binary" "./scripts/build-docker" "./scripts/release" ) + log_cmd "grep -E -n $'^ *\t' ${files[*]}" + # shellcheck disable=SC2086 + if grep -E -n $'^ *\t' "${files[@]}" | sed $'s|\t|[\\\\tab]|g'; then + log_error "FAIL: found tab-based indention in bash scripts. Use ' ' (double space)." + local files_with_tabs + files_with_tabs=$(grep -E -l $'^ *\t' "${files[@]}") + log_warning "Try: sed -i 's|\t| |g' $files_with_tabs" + return 1 + else + log_success "SUCCESS: no tabulators found." + return 0 + fi +} + +function markdown_you_find_eschew_you { + local find_you_cmd="find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' ! -path './release/*' -exec grep -E --color '[Yy]ou[r]?[ '\''.,;]' {} + || true" + run eval "${find_you_cmd}" +} + +function markdown_you_pass { + generic_checker markdown_you_find_eschew_you +} + +function markdown_marker_pass { + # TODO: check other markdown files when marker handles headers with '[]' + if tool_exists "marker" "https://crates.io/crates/marker"; then + generic_checker run marker --skip-http --root ./Documentation 2>&1 + fi +} + +function govet_pass { + run_for_modules generic_checker run go vet +} + +function govet_shadow_pass { + local shadow + shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow") + run_for_modules generic_checker run go vet -all -vettool="${shadow}" +} + +function unparam_pass { + run_for_modules generic_checker run_go_tool "mvdan.cc/unparam" +} + +function staticcheck_pass { + run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck" +} + +function revive_pass { + run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..." +} + +function unconvert_pass { + run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v +} + +function ineffassign_per_package { + # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1") + local gofiles=() + while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1") + run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}" +} + +function ineffassign_pass { + run_for_modules generic_checker ineffassign_per_package +} + +function nakedret_pass { + run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret" +} + +function license_header_pass { + # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1") + local gofiles=() + while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1") + + for file in "${gofiles[@]}"; do + if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then + licRes="${licRes}"$(echo -e " ${file}") + fi + done + if [ -n "${licRes}" ]; then + log_error -e "license header checking failed:\\n${licRes}" + return 255 + fi +} + +function receiver_name_for_package { + # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1") + local gofiles=() + while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1") + + recvs=$(grep 'func ([^*]' "${gofiles[@]}" | tr ':' ' ' | \ + awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" | sort | uniq | \ + grep -Ev "(Descriptor|Proto|_)" | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ') + if [ -n "${recvs}" ]; then + # shellcheck disable=SC2206 + recvs=($recvs) + for recv in "${recvs[@]}"; do + log_error "Mismatched receiver for $recv..." + grep "$recv" "${gofiles[@]}" | grep 'func (' + done + return 255 + fi +} + +function receiver_name_pass { + run_for_modules receiver_name_for_package +} + +# goword_for_package package +# checks spelling and comments in the 'package' in the current module +# +function goword_for_package { + # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1") + local gofiles=() + while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1") + + local gowordRes + + # spellchecking can be enabled with GOBINARGS="--tags=spell" + # but it requires heavy dependencies installation, like: + # apt-get install libaspell-dev libhunspell-dev hunspell-en-us aspell-en + + # only check for broke exported godocs + if gowordRes=$(run_go_tool "github.com/chzchzchz/goword" -use-spell=false "${gofiles[@]}" | grep godoc-export | sort); then + log_error -e "goword checking failed:\\n${gowordRes}" + return 255 + fi + if [ -n "$gowordRes" ]; then + log_error -e "goword checking returned output:\\n${gowordRes}" + return 255 + fi +} + + +function goword_pass { + run_for_modules goword_for_package || return 255 +} + +function go_fmt_for_package { + # We utilize 'go fmt' to find all files suitable for formatting, + # but reuse full power gofmt to perform just RO check. + go fmt -n "$1" | sed 's| -w | -d |g' | sh +} + +function gofmt_pass { + run_for_modules generic_checker go_fmt_for_package +} + +function bom_pass { + log_callout "Checking bill of materials..." + # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf + # shellcheck disable=SC2207 + modules=($(modules_exp)) + + # Internally license-bill-of-materials tends to modify go.sum + run cp go.sum go.sum.tmp || return 2 + run cp go.mod go.mod.tmp || return 2 + + output=$(GOFLAGS=-mod=mod run_go_tool github.com/coreos/license-bill-of-materials \ + --override-file ./bill-of-materials.override.json \ + "${modules[@]}") + code="$?" + + run cp go.sum.tmp go.sum || return 2 + run cp go.mod.tmp go.mod || return 2 + + if [ "${code}" -ne 0 ] ; then + log_error -e "license-bill-of-materials (code: ${code}) failed with:\n${output}" + return 255 + else + echo "${output}" > "bom-now.json.tmp" + fi + if ! diff ./bill-of-materials.json bom-now.json.tmp; then + log_error "modularized licenses do not match given bill of materials" + return 255 + fi + rm bom-now.json.tmp +} + +######## VARIOUS CHECKERS ###################################################### + +function dump_deps_of_module() { + local module + if ! module=$(run go list -m); then + return 255 + fi + run go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all +} + +# Checks whether dependencies are consistent across modules +function dep_pass { + local all_dependencies + all_dependencies=$(run_for_modules dump_deps_of_module | sort) || return 2 + + local duplicates + duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2 + + for dup in ${duplicates}; do + log_error "FAIL: inconsistent versions for depencency: ${dup}" + echo "${all_dependencies}" | grep "${dup}" | sed "s|\([^,]*\),\([^,]*\),\([^,]*\)| - \1@\2 from: \3|g" + done + if [[ -n "${duplicates}" ]]; then + log_error "FAIL: inconsistent dependencies" + return 2 + else + log_success "SUCCESS: dependencies are consistent across modules" + fi +} + +function release_pass { + rm -f ./bin/etcd-last-release + # to grab latest patch release; bump this up for every minor release + UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1) + if [ -n "$MANUAL_VER" ]; then + # in case, we need to test against different version + UPGRADE_VER=$MANUAL_VER + fi + if [[ -z ${UPGRADE_VER} ]]; then + UPGRADE_VER="v3.3.0" + log_warning "fallback to" ${UPGRADE_VER} + fi + + local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz" + log_callout "Downloading $file" + + set +e + curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file" + local result=$? + set -e + case $result in + 0) ;; + *) log_error "--- FAIL:" ${result} + return $result + ;; + esac + + tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1 + mkdir -p ./bin + mv /tmp/etcd ./bin/etcd-last-release +} + +function mod_tidy_for_module { + # Watch for upstream solution: https://github.com/golang/go/issues/27005 + local tmpModDir + tmpModDir=$(mktemp -d -t 'tmpModDir.XXXXXX') + run cp "./go.mod" "./go.sum" "${tmpModDir}" || return 2 + + # Guarantees keeping go.sum minimal + # If this is causing too much problems, we should + # stop controlling go.sum at all. + rm go.sum + run go mod tidy || return 2 + + set +e + local tmpFileGoModInSync + diff -C 5 "${tmpModDir}/go.mod" "./go.mod" + tmpFileGoModInSync="$?" + + local tmpFileGoSumInSync + diff -C 5 "${tmpModDir}/go.sum" "./go.sum" + tmpFileGoSumInSync="$?" + set -e + + # Bring back initial state + mv "${tmpModDir}/go.mod" "./go.mod" + mv "${tmpModDir}/go.sum" "./go.sum" + + if [ "${tmpFileGoModInSync}" -ne 0 ]; then + log_error "${PWD}/go.mod is not in sync with 'go mod tidy'" + return 255 + fi + if [ "${tmpFileGoSumInSync}" -ne 0 ]; then + log_error "${PWD}/go.sum is not in sync with 'rm go.sum; go mod tidy'" + return 255 + fi +} + +function mod_tidy_pass { + run_for_modules mod_tidy_for_module +} + +########### MAIN ############################################################### + +function run_pass { + local pass="${1}" + shift 1 + log_callout -e "\n'${pass}' started at $(date)" + if "${pass}_pass" "$@" ; then + log_success "'${pass}' completed at $(date)" + else + log_error "FAIL: '${pass}' failed at $(date)" + exit 255 + fi +} + +for pass in $PASSES; do + run_pass "${pass}" "${@}" +done + +log_success "SUCCESS" diff --git a/tests/e2e/ctl_v3_auth_test.go b/tests/e2e/ctl_v3_auth_test.go index 79fe3f662..bee66e6d9 100644 --- a/tests/e2e/ctl_v3_auth_test.go +++ b/tests/e2e/ctl_v3_auth_test.go @@ -25,7 +25,9 @@ import ( "go.etcd.io/etcd/client/v3" ) -func TestCtlV3AuthEnable(t *testing.T) { testCtl(t, authEnableTest) } +func TestCtlV3AuthEnable(t *testing.T) { + testCtl(t, authEnableTest) +} func TestCtlV3AuthDisable(t *testing.T) { testCtl(t, authDisableTest) } func TestCtlV3AuthStatus(t *testing.T) { testCtl(t, authStatusTest) } func TestCtlV3AuthWriteKey(t *testing.T) { testCtl(t, authCredWriteKeyTest) } diff --git a/tests/e2e/ctl_v3_elect_test.go b/tests/e2e/ctl_v3_elect_test.go index c3fb90ac1..332ce9708 100644 --- a/tests/e2e/ctl_v3_elect_test.go +++ b/tests/e2e/ctl_v3_elect_test.go @@ -24,10 +24,6 @@ import ( ) func TestCtlV3Elect(t *testing.T) { - oldenv := os.Getenv("EXPECT_DEBUG") - defer os.Setenv("EXPECT_DEBUG", oldenv) - os.Setenv("EXPECT_DEBUG", "1") - testCtl(t, testElect) } diff --git a/tests/e2e/ctl_v3_lock_test.go b/tests/e2e/ctl_v3_lock_test.go index 2bcc5a823..8a1598735 100644 --- a/tests/e2e/ctl_v3_lock_test.go +++ b/tests/e2e/ctl_v3_lock_test.go @@ -24,10 +24,6 @@ import ( ) func TestCtlV3Lock(t *testing.T) { - oldenv := os.Getenv("EXPECT_DEBUG") - defer os.Setenv("EXPECT_DEBUG", oldenv) - os.Setenv("EXPECT_DEBUG", "1") - testCtl(t, testLock) } diff --git a/tests/e2e/ctl_v3_role_test.go b/tests/e2e/ctl_v3_role_test.go index b7c8a5446..fb4e5de60 100644 --- a/tests/e2e/ctl_v3_role_test.go +++ b/tests/e2e/ctl_v3_role_test.go @@ -114,6 +114,7 @@ func ctlV3RoleGrantPermission(cx ctlCtx, rolename string, perm grantingPerm) err if err != nil { return err } + defer proc.Close() expStr := fmt.Sprintf("Role %s updated", rolename) _, err = proc.Expect(expStr) @@ -139,7 +140,7 @@ func ctlV3RoleRevokePermission(cx ctlCtx, rolename string, key, rangeEnd string, if err != nil { return err } - + defer proc.Close() _, err = proc.Expect(expStr) return err } diff --git a/tests/e2e/ctl_v3_snapshot_test.go b/tests/e2e/ctl_v3_snapshot_test.go index 557df0dd3..4bcb3123d 100644 --- a/tests/e2e/ctl_v3_snapshot_test.go +++ b/tests/e2e/ctl_v3_snapshot_test.go @@ -18,7 +18,7 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" + "math/rand" "os" "path/filepath" "strings" @@ -32,6 +32,13 @@ import ( func TestCtlV3Snapshot(t *testing.T) { testCtl(t, snapshotTest) } +// TODO: Replace with testing.T.TestDir() in golang-1.15. +func tempDir(tb testing.TB) string { + dir := filepath.Join(os.TempDir(), tb.Name(), fmt.Sprint(rand.Int())) + os.MkdirAll(dir, 0700) + return dir +} + func snapshotTest(cx ctlCtx) { maintenanceInitKeys(cx) @@ -43,7 +50,7 @@ func snapshotTest(cx ctlCtx) { cx.t.Fatalf("snapshot: ctlV3Put error (%v)", err) } - fpath := "test1.snapshot" + fpath := filepath.Join(tempDir(cx.t), "snapshot") defer os.RemoveAll(fpath) if err = ctlV3SnapshotSave(cx, fpath); err != nil { @@ -65,7 +72,7 @@ func snapshotTest(cx ctlCtx) { func TestCtlV3SnapshotCorrupt(t *testing.T) { testCtl(t, snapshotCorruptTest) } func snapshotCorruptTest(cx ctlCtx) { - fpath := "test2.snapshot" + fpath := filepath.Join(tempDir(cx.t), "snapshot") defer os.RemoveAll(fpath) if err := ctlV3SnapshotSave(cx, fpath); err != nil { @@ -82,10 +89,12 @@ func snapshotCorruptTest(cx ctlCtx) { } f.Close() - defer os.RemoveAll("snap.etcd") + datadir := filepath.Join(tempDir(cx.t), "data") + defer os.RemoveAll(datadir) + serr := spawnWithExpect( append(cx.PrefixArgs(), "snapshot", "restore", - "--data-dir", "snap.etcd", + "--data-dir", datadir, fpath), "expected sha256") @@ -98,7 +107,7 @@ func snapshotCorruptTest(cx ctlCtx) { func TestCtlV3SnapshotStatusBeforeRestore(t *testing.T) { testCtl(t, snapshotStatusBeforeRestoreTest) } func snapshotStatusBeforeRestoreTest(cx ctlCtx) { - fpath := "test3.snapshot" + fpath := filepath.Join(tempDir(cx.t), "snapshot") defer os.RemoveAll(fpath) if err := ctlV3SnapshotSave(cx, fpath); err != nil { @@ -111,10 +120,11 @@ func snapshotStatusBeforeRestoreTest(cx ctlCtx) { cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err) } - defer os.RemoveAll("snap.etcd") + dataDir := filepath.Join(tempDir(cx.t), "data") + defer os.RemoveAll(dataDir) serr := spawnWithExpect( append(cx.PrefixArgs(), "snapshot", "restore", - "--data-dir", "snap.etcd", + "--data-dir", dataDir, fpath), "added member") if serr != nil { @@ -154,6 +164,14 @@ func getSnapshotStatus(cx ctlCtx, fpath string) (snapshot.Status, error) { // TestIssue6361 ensures new member that starts with snapshot correctly // syncs up with other members and serve correct data. func TestIssue6361(t *testing.T) { + { + // This tests is pretty flaky on semaphoreci as of 2021-01-10. + // TODO: Remove when the flakiness source is identified. + oldenv := os.Getenv("EXPECT_DEBUG") + defer os.Setenv("EXPECT_DEBUG", oldenv) + os.Setenv("EXPECT_DEBUG", "1") + } + defer testutil.AfterTest(t) os.Setenv("ETCDCTL_API", "3") defer os.Unsetenv("ETCDCTL_API") @@ -175,7 +193,7 @@ func TestIssue6361(t *testing.T) { dialTimeout := 10 * time.Second prefixArgs := []string{ctlBinPath, "--endpoints", strings.Join(epc.EndpointsV3(), ","), "--dial-timeout", dialTimeout.String()} - // write some keys + t.Log("Writing some keys...") kvs := []kv{{"foo1", "val1"}, {"foo2", "val2"}, {"foo3", "val3"}} for i := range kvs { if err = spawnWithExpect(append(prefixArgs, "put", kvs[i].key, kvs[i].val), "OK"); err != nil { @@ -183,28 +201,29 @@ func TestIssue6361(t *testing.T) { } } - fpath := filepath.Join(os.TempDir(), "test.snapshot") + fpath := filepath.Join(tempDir(t), "snapshot") defer os.RemoveAll(fpath) - // etcdctl save snapshot + t.Log("etcdctl saving snapshot...") if err = spawnWithExpect(append(prefixArgs, "snapshot", "save", fpath), fmt.Sprintf("Snapshot saved at %s", fpath)); err != nil { t.Fatal(err) } + t.Log("Stopping the original server...") if err = epc.procs[0].Stop(); err != nil { t.Fatal(err) } - newDataDir := filepath.Join(os.TempDir(), "test.data") + newDataDir := tempDir(t) defer os.RemoveAll(newDataDir) - // etcdctl restore the snapshot + t.Log("etcdctl restoring the snapshot...") err = spawnWithExpect([]string{ctlBinPath, "snapshot", "restore", fpath, "--name", epc.procs[0].Config().name, "--initial-cluster", epc.procs[0].Config().initialCluster, "--initial-cluster-token", epc.procs[0].Config().initialToken, "--initial-advertise-peer-urls", epc.procs[0].Config().purl.String(), "--data-dir", newDataDir}, "added member") if err != nil { t.Fatal(err) } - // start the etcd member using the restored snapshot + t.Log("(Re)starting the etcd member using the restored snapshot...") epc.procs[0].Config().dataDirPath = newDataDir for i := range epc.procs[0].Config().args { if epc.procs[0].Config().args[i] == "--data-dir" { @@ -215,14 +234,14 @@ func TestIssue6361(t *testing.T) { t.Fatal(err) } - // ensure the restored member has the correct data + t.Log("Ensuring the restored member has the correct data...") for i := range kvs { if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil { t.Fatal(err) } } - // add a new member into the cluster + t.Log("Adding new member into the cluster") clientURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+30) peerURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+31) err = spawnWithExpect(append(prefixArgs, "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)), " added to cluster ") @@ -230,16 +249,13 @@ func TestIssue6361(t *testing.T) { t.Fatal(err) } - var newDataDir2 string - newDataDir2, err = ioutil.TempDir("", "newdata2") - if err != nil { - t.Fatal(err) - } + newDataDir2 := filepath.Join(tempDir(t), "newdata") defer os.RemoveAll(newDataDir2) name2 := "infra2" initialCluster2 := epc.procs[0].Config().initialCluster + fmt.Sprintf(",%s=%s", name2, peerURL) + t.Log("Starting the new member") // start the new member var nepc *expect.ExpectProcess nepc, err = spawnCmd([]string{epc.procs[0].Config().execPath, "--name", name2, @@ -249,19 +265,20 @@ func TestIssue6361(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = nepc.Expect("enabled capabilities for version"); err != nil { + if _, err = nepc.Expect("ready to serve client requests"); err != nil { t.Fatal(err) } prefixArgs = []string{ctlBinPath, "--endpoints", clientURL, "--dial-timeout", dialTimeout.String()} - // ensure added member has data from incoming snapshot + t.Log("Ensuring added member has data from incoming snapshot...") for i := range kvs { if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil { t.Fatal(err) } } + t.Log("Stopping the second member") if err = nepc.Stop(); err != nil { t.Fatal(err) } diff --git a/tests/e2e/ctl_v3_test.go b/tests/e2e/ctl_v3_test.go index 3900446bc..98b93783d 100644 --- a/tests/e2e/ctl_v3_test.go +++ b/tests/e2e/ctl_v3_test.go @@ -88,10 +88,10 @@ func versionTest(cx ctlCtx) { func clusterVersionTest(cx ctlCtx, expected string) { var err error - for i := 0; i < 7; i++ { + for i := 0; i < 35; i++ { if err = cURLGet(cx.epc, cURLReq{endpoint: "/version", expected: expected}); err != nil { cx.t.Logf("#%d: v3 is not ready yet (%v)", i, err) - time.Sleep(time.Second) + time.Sleep(200 * time.Millisecond) continue } break diff --git a/tests/e2e/ctl_v3_user_test.go b/tests/e2e/ctl_v3_user_test.go index db31d8774..8672ae7b6 100644 --- a/tests/e2e/ctl_v3_user_test.go +++ b/tests/e2e/ctl_v3_user_test.go @@ -183,6 +183,7 @@ func ctlV3User(cx ctlCtx, args []string, expStr string, stdIn []string) error { if err != nil { return err } + defer proc.Close() // Send 'stdIn' strings as input. for _, s := range stdIn { diff --git a/tests/e2e/etcd_process.go b/tests/e2e/etcd_process.go index 74ca20d71..a07f8438d 100644 --- a/tests/e2e/etcd_process.go +++ b/tests/e2e/etcd_process.go @@ -24,7 +24,7 @@ import ( ) var ( - etcdServerReadyLines = []string{"enabled capabilities for version", "published"} + etcdServerReadyLines = []string{"enabled capabilities for version", "published", "ready to serve client requests"} binPath string ctlBinPath string ) diff --git a/tests/e2e/v3_curl_test.go b/tests/e2e/v3_curl_test.go index 7c597b8c1..6abe8749e 100644 --- a/tests/e2e/v3_curl_test.go +++ b/tests/e2e/v3_curl_test.go @@ -246,6 +246,7 @@ func testV3CurlAuth(cx ctlCtx) { cmdArgs = cURLPrefixArgs(cx.epc, "POST", cURLReq{endpoint: path.Join(p, "/auth/authenticate"), value: string(authreq)}) proc, err := spawnCmd(cmdArgs) testutil.AssertNil(cx.t, err) + defer proc.Close() cURLRes, err := proc.ExpectFunc(lineFunc) testutil.AssertNil(cx.t, err) diff --git a/tests/functional/agent/handler.go b/tests/functional/agent/handler.go index e59c9038b..9a2962604 100644 --- a/tests/functional/agent/handler.go +++ b/tests/functional/agent/handler.go @@ -665,6 +665,7 @@ func (srv *Server) handle_SIGQUIT_ETCD_AND_ARCHIVE_DATA() (*rpcpb.Response, erro } } + // TODO: Verify whether this cleaning of 'cache pages' is needed. srv.lg.Info("cleaning up page cache") if err := cleanPageCache(); err != nil { srv.lg.Warn("failed to clean up page cache", zap.String("error", err.Error())) diff --git a/tests/functional/agent/utils.go b/tests/functional/agent/utils.go index 45ea66333..d77fce60b 100644 --- a/tests/functional/agent/utils.go +++ b/tests/functional/agent/utils.go @@ -103,6 +103,6 @@ func copyFile(src, dst string) error { func cleanPageCache() error { // https://www.kernel.org/doc/Documentation/sysctl/vm.txt // https://github.com/torvalds/linux/blob/master/fs/drop_caches.c - cmd := exec.Command("/bin/sh", "-c", `echo "echo 1 > /proc/sys/vm/drop_caches" | sudo sh`) + cmd := exec.Command("/bin/sh", "-c", `echo "echo 1 > /proc/sys/vm/drop_caches" | sudo -s -n`) return cmd.Run() } diff --git a/tests/functional/tester/case.go b/tests/functional/tester/case.go index 12c71559e..dbe41aab9 100644 --- a/tests/functional/tester/case.go +++ b/tests/functional/tester/case.go @@ -29,7 +29,7 @@ import ( // 1. implement "Case" interface // 2. define fail case name in "rpcpb.Case" type Case interface { - // Inject injeccts the failure into the testing cluster at the given + // Inject injects the failure into the testing cluster at the given // round. When calling the function, the cluster should be in health. Inject(clus *Cluster) error // Recover recovers the injected failure caused by the injection of the diff --git a/tests/integration/main_test.go b/tests/integration/main_test.go index a4c8fd237..cb47a9d78 100644 --- a/tests/integration/main_test.go +++ b/tests/integration/main_test.go @@ -8,8 +8,10 @@ import ( "testing" "go.etcd.io/etcd/pkg/v3/testutil" + "go.etcd.io/etcd/server/v3/embed" ) func TestMain(m *testing.M) { + embed.SetupGrpcLoggingForTest(true) testutil.MustTestMainWithLeakDetection(m) } diff --git a/tools/mod/go.mod b/tools/mod/go.mod index 52e4ed3b3..0cfdcf349 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -16,6 +16,7 @@ require ( github.com/hexfusion/schwag v0.0.0-20170606222847-b7d0fc9aadaa github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f github.com/mgechev/revive v1.0.2 + github.com/mikefarah/yq/v3 v3.0.0-20201125113350-f42728eef735 github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f // indirect go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 google.golang.org/genproto v0.0.0-20201008135153-289734e2e40c // indirect diff --git a/tools/mod/go.sum b/tools/mod/go.sum index be0dc1b90..ab34dc441 100644 --- a/tools/mod/go.sum +++ b/tools/mod/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -9,34 +10,53 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/akhenakh/hunspellgo v0.0.0-20160221122622-9db38fa26e19 h1:bYOD6QJnBJY79MJQR1i9cyQePG5oNDZXDKL2bhN/uvE= github.com/akhenakh/hunspellgo v0.0.0-20160221122622-9db38fa26e19/go.mod h1:HcqyLXmWoESd/vPSbCPqvgw5l5cMM5PtoqFOnXLjSeM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexkohler/nakedret v1.0.0 h1:S/bzOFhZHYUJp6qPmdXdFHS5nlWGFmLmoc8QOydvotE= github.com/alexkohler/nakedret v1.0.0/go.mod h1:tfDQbtPt67HhBK/6P0yNktIX7peCxfOp0jO9007DrLE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03 h1:0wUHjDfbCAROEAZ96zAJGwcNMkPIheFaIjtQyv3QqfM= github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03/go.mod h1:uFE9hX+zXEwvyUThZ4gDb9vkAwc5DoHUnRSEpH0VrOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e h1:vHRufSa2k8tfkcDdia1vJFa+oiBvvPxW94mg76PPAoA= github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e/go.mod h1:4xMOusJ7xxc84WclVxKT8+lNfGYDwojOUC2OQNCwcj4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.19.10 h1:5BHISBAXOc/aJK25irLZnx2D3s6WyYaY9D4gmuz9fdE= @@ -69,6 +89,8 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.7 h1:VRuXN2EnMSsZdauzdss6JBC29YotDqG59BZ+tdlIL1s= github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -95,12 +117,18 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-yaml v1.8.1 h1:JuZRFlqLM5cWF6A+waL8AKVuCcqvKOuhJtUQI+L3ez0= +github.com/goccy/go-yaml v1.8.1/go.mod h1:wS4gNoLalDSJxo/SpngzPQ2BN4uuZVLCmbM4S3vd4+Y= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -111,6 +139,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -121,28 +150,41 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gordonklaus/ineffassign v0.0.0-20200809085317-e36bfde3bb78 h1:U/zHjaVG/sECz5xhnh7kPH+Fv/maPbhZPcaTquo5sPg= github.com/gordonklaus/ineffassign v0.0.0-20200809085317-e36bfde3bb78/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.6 h1:8ERzHx8aj1Sc47mu9n/AksaKCSWrMchFtkdrS4BIj5o= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/gyuho/gocovmerge v0.0.0-20171205171859-50c7e6afd535 h1:BGeD3v3lyKZy+ocGtprXiDXjIiXvZDfuyII7Lym7GbQ= github.com/gyuho/gocovmerge v0.0.0-20171205171859-50c7e6afd535/go.mod h1:xV7b0Cn2irnP1jU+mMYvqPAPuFPNjtgB+rvKu/dLIz4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hexfusion/schwag v0.0.0-20170606222847-b7d0fc9aadaa h1:oDcxzjIf33MTX7b8Eu7eO3a/z8mlTT+blyEoVxBmUUg= github.com/hexfusion/schwag v0.0.0-20170606222847-b7d0fc9aadaa/go.mod h1:wSgrm+n3LvHOVxUJo2ha5ffLqRmt6+oGoD6J/suB66c= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -150,65 +192,105 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f h1:Kc3s6QFyh9DLgInXpWKuG+8I7R7lXbnP7mcoOVIt6KY= github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f/go.mod h1:AmCV4WB3cDMZqgPk+OUQKumliiQS4ZYsBt3AXekyuAU= github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= github.com/mgechev/revive v1.0.2 h1:v0NxxQ7fSFz/u1NQydPo6EGdq7va0J1BtsZmae6kzUg= github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= +github.com/mikefarah/yq/v3 v3.0.0-20201125113350-f42728eef735 h1:Qn41fatPrqv5qVpDFx+4ABF14LNj9jiNLm/BsrDb01U= +github.com/mikefarah/yq/v3 v3.0.0-20201125113350-f42728eef735/go.mod h1:dYWq+UWoFCDY1TndvFUQuhBbIYmZpjreC8adEAx93zE= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.5.2 h1:qLvObTrvO/XRCqmkKxUlOBc48bI3efyDuAZe25QiF0w= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f h1:92ZQJRegaqnKjz9HY9an696Sw5EmAqRv0eie/U2IE6k= github.com/trustmaster/go-aspell v0.0.0-20200701131845-c2b1f55bec8f/go.mod h1:wxUiQ1klFJmwnM41kQI7IT2g8jjOKbtuL54LdjkxAI0= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.3.0 h1:ew6uUIeJOo+qdUUv7LxFCUhtWmVv7ZV/Xuy4FAUsw2E= go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -225,10 +307,13 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -246,6 +331,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -254,12 +341,18 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -270,7 +363,6 @@ golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -278,8 +370,9 @@ golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b h1:zSzQJAznWxAh9fZxiPy2FZo golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -289,6 +382,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20201008135153-289734e2e40c h1:dvSnJCVti7yKNAZrunMpEE4QLfy5b/UdEKt+UH39BW4= google.golang.org/genproto v0.0.0-20201008135153-289734e2e40c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -302,14 +396,19 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE= +gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -317,6 +416,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/tools/mod/tools.go b/tools/mod/tools.go index c39aee233..86cdaaf40 100644 --- a/tools/mod/tools.go +++ b/tools/mod/tools.go @@ -34,4 +34,5 @@ import ( _ "go.etcd.io/protodoc" _ "honnef.co/go/tools/cmd/staticcheck" _ "mvdan.cc/unparam" + _ "github.com/mikefarah/yq/v3" )