From 85e050a120c3ca933c12337f0ea26b60e0222471 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 07:09:48 -0700 Subject: [PATCH 01/16] *: rename, clean up functional tests Signed-off-by: Gyuho Lee --- ...functional-tester => Dockerfile-functional | 4 +- Makefile | 26 +- .../tester/local-test.yaml => functional.yaml | 13 +- functional/README.md | 31 ++ .../agent/doc.go | 0 .../agent/handler.go | 3 +- .../agent/server.go | 2 +- .../agent/utils.go | 0 .../agent/utils_test.go | 0 {tools/functional-tester => functional}/build | 8 +- .../cmd/etcd-agent/main.go | 2 +- .../cmd/etcd-runner/main.go | 5 +- .../cmd/etcd-tester/main.go | 4 +- .../rpcpb/etcd_config.go | 0 .../rpcpb/etcd_config_test.go | 0 .../rpcpb/member.go | 0 .../rpcpb/rpc.pb.go | 498 +++++++++--------- .../rpcpb/rpc.proto | 18 +- .../runner/election_command.go | 0 .../runner/error.go | 0 .../runner/global.go | 0 .../runner/help.go | 0 .../runner/lease_renewer_command.go | 0 .../runner/lock_racer_command.go | 0 .../runner/root.go | 0 .../runner/watch_command.go | 0 .../scripts/docker-local-agent.sh | 2 +- .../scripts/docker-local-tester.sh | 4 +- .../scripts/genproto.sh | 0 .../tester/checks.go | 2 +- .../tester/cluster.go | 272 +--------- functional/tester/cluster_read_config.go | 244 +++++++++ .../tester/cluster_run.go | 10 +- functional/tester/cluster_shuffle.go | 64 +++ .../tester/cluster_test.go | 19 +- .../tester/doc.go | 0 .../tester/failure.go | 2 +- .../tester/failure_case_delay.go | 0 .../tester/failure_case_external.go | 2 +- .../tester/failure_case_failpoints.go | 2 +- .../tester/failure_case_kill.go | 2 +- .../tester/failure_case_network_blackhole.go | 2 +- .../tester/failure_case_network_delay.go | 2 +- .../tester/failure_case_no_fail.go | 2 +- .../tester/metrics.go | 0 .../tester/stress.go | 2 +- .../tester/stress_composite.go | 0 .../tester/stress_key.go | 2 +- .../tester/stress_lease.go | 2 +- .../tester/stress_runner.go | 0 .../tester/utils.go | 0 test | 4 +- tools/functional-tester/README.md | 33 -- 53 files changed, 668 insertions(+), 620 deletions(-) rename Dockerfile-functional-tester => Dockerfile-functional (89%) rename tools/functional-tester/tester/local-test.yaml => functional.yaml (97%) create mode 100644 functional/README.md rename {tools/functional-tester => functional}/agent/doc.go (100%) rename {tools/functional-tester => functional}/agent/handler.go (99%) rename {tools/functional-tester => functional}/agent/server.go (98%) rename {tools/functional-tester => functional}/agent/utils.go (100%) rename {tools/functional-tester => functional}/agent/utils_test.go (100%) rename {tools/functional-tester => functional}/build (54%) rename {tools/functional-tester => functional}/cmd/etcd-agent/main.go (95%) rename {tools/functional-tester => functional}/cmd/etcd-runner/main.go (80%) rename {tools/functional-tester => functional}/cmd/etcd-tester/main.go (94%) rename {tools/functional-tester => functional}/rpcpb/etcd_config.go (100%) rename {tools/functional-tester => functional}/rpcpb/etcd_config_test.go (100%) rename {tools/functional-tester => functional}/rpcpb/member.go (100%) rename {tools/functional-tester => functional}/rpcpb/rpc.pb.go (85%) rename {tools/functional-tester => functional}/rpcpb/rpc.proto (95%) rename {tools/functional-tester => functional}/runner/election_command.go (100%) rename {tools/functional-tester => functional}/runner/error.go (100%) rename {tools/functional-tester => functional}/runner/global.go (100%) rename {tools/functional-tester => functional}/runner/help.go (100%) rename {tools/functional-tester => functional}/runner/lease_renewer_command.go (100%) rename {tools/functional-tester => functional}/runner/lock_racer_command.go (100%) rename {tools/functional-tester => functional}/runner/root.go (100%) rename {tools/functional-tester => functional}/runner/watch_command.go (100%) rename {tools/functional-tester => functional}/scripts/docker-local-agent.sh (94%) rename {tools/functional-tester => functional}/scripts/docker-local-tester.sh (74%) rename {tools/functional-tester => functional}/scripts/genproto.sh (100%) rename {tools/functional-tester => functional}/tester/checks.go (99%) rename {tools/functional-tester => functional}/tester/cluster.go (66%) create mode 100644 functional/tester/cluster_read_config.go rename tools/functional-tester/tester/cluster_tester.go => functional/tester/cluster_run.go (96%) create mode 100644 functional/tester/cluster_shuffle.go rename {tools/functional-tester => functional}/tester/cluster_test.go (95%) rename {tools/functional-tester => functional}/tester/doc.go (100%) rename {tools/functional-tester => functional}/tester/failure.go (99%) rename {tools/functional-tester => functional}/tester/failure_case_delay.go (100%) rename {tools/functional-tester => functional}/tester/failure_case_external.go (96%) rename {tools/functional-tester => functional}/tester/failure_case_failpoints.go (98%) rename {tools/functional-tester => functional}/tester/failure_case_kill.go (97%) rename {tools/functional-tester => functional}/tester/failure_case_network_blackhole.go (98%) rename {tools/functional-tester => functional}/tester/failure_case_network_delay.go (98%) rename {tools/functional-tester => functional}/tester/failure_case_no_fail.go (97%) rename {tools/functional-tester => functional}/tester/metrics.go (100%) rename {tools/functional-tester => functional}/tester/stress.go (98%) rename {tools/functional-tester => functional}/tester/stress_composite.go (100%) rename {tools/functional-tester => functional}/tester/stress_key.go (99%) rename {tools/functional-tester => functional}/tester/stress_lease.go (99%) rename {tools/functional-tester => functional}/tester/stress_runner.go (100%) rename {tools/functional-tester => functional}/tester/utils.go (100%) delete mode 100644 tools/functional-tester/README.md diff --git a/Dockerfile-functional-tester b/Dockerfile-functional similarity index 89% rename from Dockerfile-functional-tester rename to Dockerfile-functional index e721d6b50..c5751a1a9 100644 --- a/Dockerfile-functional-tester +++ b/Dockerfile-functional @@ -22,7 +22,7 @@ RUN rm -rf ${GOROOT} \ RUN mkdir -p ${GOPATH}/src/github.com/coreos/etcd ADD . ${GOPATH}/src/github.com/coreos/etcd -ADD ./tools/functional-tester/tester/local-test.yaml /local-test.yaml +ADD ./functional.yaml /functional.yaml RUN go get -v github.com/coreos/gofail \ && pushd ${GOPATH}/src/github.com/coreos/etcd \ @@ -32,7 +32,7 @@ RUN go get -v github.com/coreos/gofail \ && cp ./bin/etcdctl /bin/etcdctl \ && GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \ && cp ./bin/etcd /bin/etcd-failpoints \ - && ./tools/functional-tester/build \ + && ./functional/build \ && cp ./bin/etcd-agent /bin/etcd-agent \ && cp ./bin/etcd-tester /bin/etcd-tester \ && cp ./bin/etcd-runner /bin/etcd-runner \ diff --git a/Makefile b/Makefile index ea5f53bc6..d89fbbdcd 100644 --- a/Makefile +++ b/Makefile @@ -477,23 +477,23 @@ build-etcd-test-proxy: # Example: -# make build-docker-functional-tester -# make push-docker-functional-tester -# make pull-docker-functional-tester +# make build-docker-functional +# make push-docker-functional +# make pull-docker-functional -build-docker-functional-tester: +build-docker-functional: $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) - @sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-functional-tester + @sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-functional docker build \ - --tag gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \ - --file ./Dockerfile-functional-tester \ + --tag gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) \ + --file ./Dockerfile-functional \ . - @mv ./Dockerfile-functional-tester.bak ./Dockerfile-functional-tester + @mv ./Dockerfile-functional.bak ./Dockerfile-functional docker run \ --rm \ - gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \ + gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) \ /bin/bash -c "./bin/etcd --version && \ ./bin/etcd-failpoints --version && \ ETCDCTL_API=3 ./bin/etcdctl version && \ @@ -503,12 +503,12 @@ build-docker-functional-tester: ./bin/benchmark --help || true && \ ./bin/etcd-test-proxy -help || true" -push-docker-functional-tester: +push-docker-functional: $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) - gcloud docker -- push gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) + gcloud docker -- push gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) -pull-docker-functional-tester: +pull-docker-functional: $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) - docker pull gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) + docker pull gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) diff --git a/tools/functional-tester/tester/local-test.yaml b/functional.yaml similarity index 97% rename from tools/functional-tester/tester/local-test.yaml rename to functional.yaml index aab38036f..c451592e2 100644 --- a/tools/functional-tester/tester/local-test.yaml +++ b/functional.yaml @@ -106,9 +106,9 @@ agent-configs: initial-corrupt-check: true tester-config: - tester-data-dir: /tmp/etcd-tester-data - tester-network: tcp - tester-addr: 127.0.0.1:9028 + data-dir: /tmp/etcd-tester-data + network: tcp + addr: 127.0.0.1:9028 # slow enough to trigger election delay-latency-ms: 5000 @@ -119,6 +119,8 @@ tester-config: consistency-check: true enable-pprof: true + failure-delay-ms: 7000 + failure-shuffle: true failure-cases: - KILL_ONE_FOLLOWER - KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT @@ -147,14 +149,11 @@ tester-config: - NO_FAIL_WITH_STRESS - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS - failure-delay-ms: 7000 - failure-shuffle: true failpoint-commands: - panic("etcd-tester") - # failpoint-commands: # - panic("etcd-tester"),1*sleep(1000) - runner-exec-path: /etcd-runner + runner-exec-path: ./bin/etcd-runner external-exec-path: "" stress-types: diff --git a/functional/README.md b/functional/README.md new file mode 100644 index 000000000..1c143097e --- /dev/null +++ b/functional/README.md @@ -0,0 +1,31 @@ +# etcd Functional Testing + +`functional` verifies the correct behavior of etcd under various system and network malfunctions. It sets up an etcd cluster under high pressure loads and continuously injects failures into the cluster. Then it expects the etcd cluster to recover within a few seconds. This has been extremely helpful to find critical bugs. + +See [functional.yaml](../functional.yaml) for an example configuration. + +### Run locally + +```bash +PASSES=functional ./test +``` + +### Run with Docker + +```bash +pushd ../.. +make build-docker-functional +popd +``` + +And run [example scripts](./scripts). + +```bash +# run 3 agents for 3-node local etcd cluster +./scripts/docker-local-agent.sh 1 +./scripts/docker-local-agent.sh 2 +./scripts/docker-local-agent.sh 3 + +# to run only 1 tester round +./scripts/docker-local-tester.sh +``` diff --git a/tools/functional-tester/agent/doc.go b/functional/agent/doc.go similarity index 100% rename from tools/functional-tester/agent/doc.go rename to functional/agent/doc.go diff --git a/tools/functional-tester/agent/handler.go b/functional/agent/handler.go similarity index 99% rename from tools/functional-tester/agent/handler.go rename to functional/agent/handler.go index a69e013e7..e80da14f4 100644 --- a/tools/functional-tester/agent/handler.go +++ b/functional/agent/handler.go @@ -25,9 +25,9 @@ import ( "syscall" "time" + "github.com/coreos/etcd/functional/rpcpb" "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/pkg/proxy" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" "go.uber.org/zap" ) @@ -118,7 +118,6 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response, }, nil } -// TODO: support TLS func (srv *Server) startProxy() error { if srv.Member.EtcdClientProxy { advertiseClientURL, advertiseClientURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertiseClientURLs[0]) diff --git a/tools/functional-tester/agent/server.go b/functional/agent/server.go similarity index 98% rename from tools/functional-tester/agent/server.go rename to functional/agent/server.go index f020b5ad4..1c077138d 100644 --- a/tools/functional-tester/agent/server.go +++ b/functional/agent/server.go @@ -21,8 +21,8 @@ import ( "os/exec" "strings" + "github.com/coreos/etcd/functional/rpcpb" "github.com/coreos/etcd/pkg/proxy" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" "go.uber.org/zap" "google.golang.org/grpc" diff --git a/tools/functional-tester/agent/utils.go b/functional/agent/utils.go similarity index 100% rename from tools/functional-tester/agent/utils.go rename to functional/agent/utils.go diff --git a/tools/functional-tester/agent/utils_test.go b/functional/agent/utils_test.go similarity index 100% rename from tools/functional-tester/agent/utils_test.go rename to functional/agent/utils_test.go diff --git a/tools/functional-tester/build b/functional/build similarity index 54% rename from tools/functional-tester/build rename to functional/build index 8087a7a95..60daadcf4 100755 --- a/tools/functional-tester/build +++ b/functional/build @@ -1,10 +1,10 @@ #!/usr/bin/env bash -if ! [[ "$0" =~ "tools/functional-tester/build" ]]; then +if ! [[ "$0" =~ "functional/build" ]]; then echo "must be run from repository root" exit 255 fi -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./tools/functional-tester/cmd/etcd-agent -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./tools/functional-tester/cmd/etcd-tester -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./tools/functional-tester/cmd/etcd-runner +CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./functional/cmd/etcd-agent +CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./functional/cmd/etcd-tester +CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./functional/cmd/etcd-runner diff --git a/tools/functional-tester/cmd/etcd-agent/main.go b/functional/cmd/etcd-agent/main.go similarity index 95% rename from tools/functional-tester/cmd/etcd-agent/main.go rename to functional/cmd/etcd-agent/main.go index 4a797882d..e3f217693 100644 --- a/tools/functional-tester/cmd/etcd-agent/main.go +++ b/functional/cmd/etcd-agent/main.go @@ -18,7 +18,7 @@ package main import ( "flag" - "github.com/coreos/etcd/tools/functional-tester/agent" + "github.com/coreos/etcd/functional/agent" "go.uber.org/zap" ) diff --git a/tools/functional-tester/cmd/etcd-runner/main.go b/functional/cmd/etcd-runner/main.go similarity index 80% rename from tools/functional-tester/cmd/etcd-runner/main.go rename to functional/cmd/etcd-runner/main.go index 41fb0f446..6e3cb16ce 100644 --- a/tools/functional-tester/cmd/etcd-runner/main.go +++ b/functional/cmd/etcd-runner/main.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// etcd-runner is a program for testing etcd clientv3 features against a fault injected cluster. +// etcd-runner is a program for testing etcd clientv3 features +// against a fault injected cluster. package main -import "github.com/coreos/etcd/tools/functional-tester/runner" +import "github.com/coreos/etcd/functional/runner" func main() { runner.Start() diff --git a/tools/functional-tester/cmd/etcd-tester/main.go b/functional/cmd/etcd-tester/main.go similarity index 94% rename from tools/functional-tester/cmd/etcd-tester/main.go rename to functional/cmd/etcd-tester/main.go index c00a28b55..f7966425f 100644 --- a/tools/functional-tester/cmd/etcd-tester/main.go +++ b/functional/cmd/etcd-tester/main.go @@ -18,7 +18,7 @@ package main import ( "flag" - "github.com/coreos/etcd/tools/functional-tester/tester" + "github.com/coreos/etcd/functional/tester" "go.uber.org/zap" ) @@ -56,5 +56,5 @@ func main() { logger.Fatal("WaitHealth failed", zap.Error(err)) } - clus.StartTester() + clus.Run() } diff --git a/tools/functional-tester/rpcpb/etcd_config.go b/functional/rpcpb/etcd_config.go similarity index 100% rename from tools/functional-tester/rpcpb/etcd_config.go rename to functional/rpcpb/etcd_config.go diff --git a/tools/functional-tester/rpcpb/etcd_config_test.go b/functional/rpcpb/etcd_config_test.go similarity index 100% rename from tools/functional-tester/rpcpb/etcd_config_test.go rename to functional/rpcpb/etcd_config_test.go diff --git a/tools/functional-tester/rpcpb/member.go b/functional/rpcpb/member.go similarity index 100% rename from tools/functional-tester/rpcpb/member.go rename to functional/rpcpb/member.go diff --git a/tools/functional-tester/rpcpb/rpc.pb.go b/functional/rpcpb/rpc.pb.go similarity index 85% rename from tools/functional-tester/rpcpb/rpc.pb.go rename to functional/rpcpb/rpc.pb.go index 0f8a3a968..a16522bd7 100644 --- a/tools/functional-tester/rpcpb/rpc.pb.go +++ b/functional/rpcpb/rpc.pb.go @@ -310,9 +310,9 @@ func (*Member) ProtoMessage() {} func (*Member) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } type Tester struct { - TesterDataDir string `protobuf:"bytes,1,opt,name=TesterDataDir,proto3" json:"TesterDataDir,omitempty" yaml:"tester-data-dir"` - TesterNetwork string `protobuf:"bytes,2,opt,name=TesterNetwork,proto3" json:"TesterNetwork,omitempty" yaml:"tester-network"` - TesterAddr string `protobuf:"bytes,3,opt,name=TesterAddr,proto3" json:"TesterAddr,omitempty" yaml:"tester-addr"` + DataDir string `protobuf:"bytes,1,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"` + Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty" yaml:"network"` + Addr string `protobuf:"bytes,3,opt,name=Addr,proto3" json:"Addr,omitempty" yaml:"addr"` // DelayLatencyMsRv is the delay latency in milliseconds, // to inject to simulated slow network. DelayLatencyMs uint32 `protobuf:"varint,11,opt,name=DelayLatencyMs,proto3" json:"DelayLatencyMs,omitempty" yaml:"delay-latency-ms"` @@ -330,15 +330,15 @@ type Tester struct { ConsistencyCheck bool `protobuf:"varint,23,opt,name=ConsistencyCheck,proto3" json:"ConsistencyCheck,omitempty" yaml:"consistency-check"` // EnablePprof is true to enable profiler. EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"` - // FailureCases is the selected test cases to schedule. - // If empty, run all failure cases. - FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"` // FailureDelayMs is the delay duration after failure is injected. // Useful when triggering snapshot or no-op failure cases. - FailureDelayMs uint32 `protobuf:"varint,32,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"` + FailureDelayMs uint32 `protobuf:"varint,31,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"` // FailureShuffle is true to randomize failure injecting order. - FailureShuffle bool `protobuf:"varint,33,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` - // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). + FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` + // FailureCases is the selected test cases to schedule. + // If empty, run all failure cases. + FailureCases []string `protobuf:"bytes,33,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"` + // Failpoinommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000) FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` // RunnerExecPath is a path of etcd-runner binary. RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"` @@ -990,23 +990,23 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.TesterDataDir) > 0 { + if len(m.DataDir) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.TesterDataDir))) - i += copy(dAtA[i:], m.TesterDataDir) + i = encodeVarintRpc(dAtA, i, uint64(len(m.DataDir))) + i += copy(dAtA[i:], m.DataDir) } - if len(m.TesterNetwork) > 0 { + if len(m.Network) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.TesterNetwork))) - i += copy(dAtA[i:], m.TesterNetwork) + i = encodeVarintRpc(dAtA, i, uint64(len(m.Network))) + i += copy(dAtA[i:], m.Network) } - if len(m.TesterAddr) > 0 { + if len(m.Addr) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.TesterAddr))) - i += copy(dAtA[i:], m.TesterAddr) + i = encodeVarintRpc(dAtA, i, uint64(len(m.Addr))) + i += copy(dAtA[i:], m.Addr) } if m.DelayLatencyMs != 0 { dAtA[i] = 0x58 @@ -1066,11 +1066,30 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.FailureDelayMs != 0 { + dAtA[i] = 0xf8 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs)) + } + if m.FailureShuffle { + dAtA[i] = 0x80 + i++ + dAtA[i] = 0x2 + i++ + if m.FailureShuffle { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } if len(m.FailureCases) > 0 { for _, s := range m.FailureCases { - dAtA[i] = 0xfa + dAtA[i] = 0x8a i++ - dAtA[i] = 0x1 + dAtA[i] = 0x2 i++ l = len(s) for l >= 1<<7 { @@ -1083,25 +1102,6 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } - if m.FailureDelayMs != 0 { - dAtA[i] = 0x80 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs)) - } - if m.FailureShuffle { - dAtA[i] = 0x88 - i++ - dAtA[i] = 0x2 - i++ - if m.FailureShuffle { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } if len(m.FailpointCommands) > 0 { for _, s := range m.FailpointCommands { dAtA[i] = 0x92 @@ -1499,15 +1499,15 @@ func (m *Member) Size() (n int) { func (m *Tester) Size() (n int) { var l int _ = l - l = len(m.TesterDataDir) + l = len(m.DataDir) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } - l = len(m.TesterNetwork) + l = len(m.Network) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } - l = len(m.TesterAddr) + l = len(m.Addr) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } @@ -1532,18 +1532,18 @@ func (m *Tester) Size() (n int) { if m.EnablePprof { n += 3 } - if len(m.FailureCases) > 0 { - for _, s := range m.FailureCases { - l = len(s) - n += 2 + l + sovRpc(uint64(l)) - } - } if m.FailureDelayMs != 0 { n += 2 + sovRpc(uint64(m.FailureDelayMs)) } if m.FailureShuffle { n += 3 } + if len(m.FailureCases) > 0 { + for _, s := range m.FailureCases { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } if len(m.FailpointCommands) > 0 { for _, s := range m.FailpointCommands { l = len(s) @@ -3021,7 +3021,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TesterDataDir", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field DataDir", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3046,11 +3046,11 @@ func (m *Tester) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.TesterDataDir = string(dAtA[iNdEx:postIndex]) + m.DataDir = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TesterNetwork", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Network", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3075,11 +3075,11 @@ func (m *Tester) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.TesterNetwork = string(dAtA[iNdEx:postIndex]) + m.Network = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TesterAddr", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Addr", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3104,7 +3104,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.TesterAddr = string(dAtA[iNdEx:postIndex]) + m.Addr = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 11: if wireType != 0 { @@ -3243,6 +3243,45 @@ func (m *Tester) Unmarshal(dAtA []byte) error { } m.EnablePprof = bool(v != 0) case 31: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType) + } + m.FailureDelayMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FailureDelayMs |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 32: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.FailureShuffle = bool(v != 0) + case 33: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field FailureCases", wireType) } @@ -3271,45 +3310,6 @@ func (m *Tester) Unmarshal(dAtA []byte) error { } m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex - case 32: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType) - } - m.FailureDelayMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.FailureDelayMs |= (uint32(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 33: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.FailureShuffle = bool(v != 0) case 34: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType) @@ -3955,160 +3955,160 @@ var ( func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 2480 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0x5b, 0x57, 0xdb, 0xd8, - 0x15, 0x46, 0x38, 0x30, 0x70, 0xb8, 0x99, 0x43, 0x08, 0x4a, 0x32, 0xc1, 0x44, 0x99, 0xa4, 0x84, - 0x56, 0xa4, 0x4d, 0x66, 0xb5, 0x4d, 0xe6, 0x92, 0x18, 0xa3, 0x04, 0x17, 0x61, 0x3b, 0xc7, 0x22, - 0xc9, 0x3c, 0xb9, 0x42, 0x3e, 0xc6, 0x2a, 0x42, 0x72, 0xa4, 0x63, 0xc6, 0xcc, 0x1f, 0xe8, 0x6b, - 0xef, 0xab, 0x0f, 0x5d, 0xfd, 0x05, 0x9d, 0xfe, 0x8e, 0xcc, 0xf4, 0x36, 0x6d, 0xdf, 0xdd, 0x36, - 0x5d, 0xfd, 0x03, 0x5e, 0xbd, 0x4d, 0x9f, 0xba, 0xce, 0xc5, 0xf6, 0x91, 0x6c, 0x43, 0xde, 0x7c, - 0xf6, 0xfe, 0xbe, 0x4f, 0xfb, 0xec, 0x2d, 0x9d, 0xbd, 0x0f, 0x80, 0x85, 0xb0, 0xe1, 0x34, 0x0e, - 0xee, 0x84, 0x0d, 0x67, 0xb3, 0x11, 0x06, 0x24, 0x80, 0x13, 0xcc, 0x70, 0x45, 0x3f, 0x74, 0x49, - 0xbd, 0x79, 0xb0, 0xe9, 0x04, 0xc7, 0x77, 0x0e, 0x83, 0xc3, 0xe0, 0x0e, 0xf3, 0x1e, 0x34, 0x6b, - 0x6c, 0xc5, 0x16, 0xec, 0x17, 0x67, 0x69, 0x5f, 0xce, 0x81, 0x0b, 0x06, 0x71, 0xaa, 0xf0, 0x06, - 0xb8, 0x50, 0xb0, 0x8f, 0xb1, 0xaa, 0xac, 0x29, 0xeb, 0xd3, 0x5b, 0x0b, 0x9d, 0x76, 0x66, 0xe6, - 0xd4, 0x3e, 0xf6, 0x1e, 0x68, 0xbe, 0x7d, 0x8c, 0x35, 0xc4, 0x9c, 0x50, 0x07, 0x6f, 0x6d, 0xdb, - 0xc4, 0xde, 0x76, 0x43, 0x75, 0x9c, 0xe1, 0x96, 0x3a, 0xed, 0xcc, 0x02, 0xc7, 0x55, 0x6d, 0x62, - 0xeb, 0x55, 0x37, 0xd4, 0x50, 0x17, 0x03, 0x37, 0xc0, 0xe4, 0xf3, 0xac, 0x49, 0xd1, 0x29, 0x86, - 0x86, 0x9d, 0x76, 0x66, 0x9e, 0xa3, 0x3f, 0xb6, 0x3d, 0x0e, 0x16, 0x08, 0x58, 0x04, 0x4b, 0x3b, - 0xd8, 0x0e, 0xc9, 0x01, 0xb6, 0x49, 0xde, 0x27, 0x38, 0x3c, 0xb1, 0xbd, 0xbd, 0x48, 0x9d, 0x59, - 0x53, 0xd6, 0x53, 0x5b, 0xd7, 0x3a, 0xed, 0xcc, 0x65, 0x4e, 0xac, 0x77, 0x41, 0xba, 0x2b, 0x50, - 0x1a, 0x1a, 0xc6, 0x84, 0x79, 0xb0, 0x68, 0x78, 0xd8, 0x21, 0x6e, 0xe0, 0x5b, 0xee, 0x31, 0x0e, - 0x9a, 0x64, 0x2f, 0x52, 0x67, 0x99, 0xdc, 0xd5, 0x4e, 0x3b, 0xb3, 0xc2, 0xe5, 0xb0, 0x80, 0xe8, - 0x84, 0x63, 0x34, 0x34, 0xc8, 0x82, 0x79, 0x90, 0x36, 0xdd, 0x88, 0x60, 0x3f, 0xe7, 0xb9, 0xd8, - 0x27, 0xfb, 0xc8, 0x8c, 0xd4, 0xe5, 0xb5, 0xd4, 0xfa, 0xb4, 0x1c, 0x98, 0xc7, 0x10, 0xba, 0xc3, - 0x20, 0x7a, 0x33, 0xf4, 0x22, 0x0d, 0x0d, 0xd0, 0x20, 0x02, 0x4b, 0xd9, 0xea, 0x09, 0x0e, 0x89, - 0x1b, 0x61, 0x49, 0xed, 0x12, 0x53, 0x5b, 0xeb, 0xb4, 0x33, 0x6f, 0x73, 0x35, 0xbb, 0x0b, 0x8a, - 0x0b, 0x0e, 0x23, 0xc3, 0xfb, 0x60, 0x8e, 0xaf, 0xb2, 0x4d, 0x12, 0x58, 0x66, 0x59, 0x5d, 0x59, - 0x53, 0xd6, 0xa7, 0xe4, 0xda, 0xd8, 0x4d, 0x12, 0xe8, 0x84, 0x0a, 0xc4, 0x91, 0x30, 0x07, 0xe6, - 0xb9, 0x21, 0x87, 0x43, 0x6a, 0xac, 0xab, 0x2a, 0xe3, 0x4a, 0x19, 0x12, 0xcf, 0x77, 0x70, 0x48, - 0x74, 0xbb, 0x49, 0xea, 0x1a, 0x4a, 0x50, 0xe0, 0xfb, 0xb2, 0xc8, 0x63, 0xd7, 0xc3, 0xea, 0x65, - 0x56, 0xee, 0x8b, 0x9d, 0x76, 0x26, 0x2d, 0x44, 0x28, 0xbb, 0xe6, 0x7a, 0x38, 0xc6, 0xa6, 0xd8, - 0x7e, 0xf4, 0xbb, 0xf8, 0x94, 0x91, 0xaf, 0x24, 0xdf, 0xac, 0x23, 0x7c, 0x2a, 0xb8, 0x71, 0x24, - 0x34, 0xc1, 0x12, 0x37, 0x58, 0x61, 0x33, 0x22, 0xb8, 0x9a, 0xcb, 0x32, 0x81, 0xab, 0x4c, 0xe0, - 0x4a, 0xa7, 0x9d, 0xb9, 0xc4, 0x05, 0x08, 0x77, 0xeb, 0x8e, 0x2d, 0x74, 0x86, 0xd1, 0x68, 0x2e, - 0x78, 0xb9, 0x4a, 0x18, 0x87, 0xac, 0x2a, 0x19, 0x56, 0x15, 0x29, 0x17, 0xa2, 0xc6, 0x0d, 0x8c, - 0x43, 0x51, 0x90, 0x04, 0x05, 0x5a, 0x60, 0xb1, 0x57, 0xa2, 0x9e, 0xce, 0x1a, 0xd3, 0xb9, 0xd5, - 0x69, 0x67, 0x34, 0xae, 0xe3, 0xfa, 0x2e, 0x71, 0x6d, 0x4f, 0xef, 0x57, 0x59, 0x92, 0x1c, 0x14, - 0x80, 0x0f, 0xc0, 0x0c, 0xfd, 0xdd, 0xad, 0xef, 0x75, 0x56, 0x23, 0xb5, 0xd3, 0xce, 0x5c, 0xe4, - 0x7a, 0x8c, 0xdd, 0x2f, 0xb2, 0x0c, 0x86, 0x25, 0x00, 0xe9, 0x32, 0x51, 0x66, 0x8d, 0x49, 0x48, - 0x2f, 0x1c, 0x93, 0x18, 0xac, 0xf5, 0x10, 0x2e, 0xfc, 0x00, 0xcc, 0x32, 0x6b, 0xb7, 0xda, 0x37, - 0x58, 0xbe, 0x2f, 0x77, 0xda, 0x99, 0x65, 0x59, 0xab, 0x5f, 0xf2, 0x18, 0xbc, 0xbb, 0x99, 0x6e, - 0xb9, 0xdf, 0x61, 0xec, 0xe4, 0x66, 0xfa, 0x35, 0x97, 0xc1, 0x70, 0x0f, 0x2c, 0xd2, 0x65, 0xbc, - 0xde, 0x37, 0x99, 0x42, 0xa6, 0xd3, 0xce, 0x5c, 0x95, 0x14, 0x06, 0x8a, 0x3e, 0xc8, 0x84, 0x5b, - 0x60, 0x3e, 0xcf, 0x4b, 0x91, 0xf3, 0xa8, 0x3d, 0x54, 0x6f, 0x27, 0xdf, 0x9d, 0x6e, 0xa9, 0x1c, - 0x0e, 0xd0, 0x50, 0x82, 0x41, 0xbf, 0xe8, 0xb8, 0xa5, 0x4c, 0x6c, 0x82, 0xd5, 0x0d, 0x26, 0x24, - 0x25, 0x38, 0x21, 0xa4, 0x47, 0x14, 0xa6, 0xa1, 0x61, 0xe4, 0x41, 0x4d, 0x2b, 0x38, 0xc2, 0xbe, - 0xfa, 0xd5, 0xf3, 0x34, 0x09, 0x85, 0x0d, 0x68, 0x32, 0x32, 0x7c, 0x08, 0xe6, 0xca, 0xbe, 0xdd, - 0x88, 0xea, 0x01, 0xc9, 0x05, 0x4d, 0x9f, 0xa8, 0xf7, 0xd8, 0x59, 0x28, 0x95, 0x2d, 0x12, 0x6e, - 0xdd, 0xa1, 0x7e, 0x0d, 0xc5, 0xf1, 0xd0, 0x04, 0x8b, 0x4f, 0x9b, 0x01, 0xb1, 0xb7, 0x6c, 0xe7, - 0x08, 0xfb, 0xd5, 0xad, 0x53, 0x82, 0x23, 0xf5, 0x5d, 0x26, 0xb2, 0xda, 0x69, 0x67, 0xae, 0x70, - 0x91, 0x97, 0x14, 0xa2, 0x1f, 0x70, 0x8c, 0x7e, 0x40, 0x41, 0x1a, 0x1a, 0x24, 0xd2, 0x56, 0x52, - 0x0a, 0xf1, 0xb3, 0x80, 0x60, 0xf5, 0x61, 0xf2, 0xb8, 0x6a, 0x84, 0x58, 0x3f, 0x09, 0x68, 0x76, - 0xba, 0x18, 0x39, 0x23, 0x41, 0x18, 0x36, 0x1b, 0x24, 0x57, 0xc7, 0xce, 0x91, 0xfa, 0x28, 0xf9, - 0x1a, 0xf7, 0x32, 0xc2, 0x51, 0xba, 0x43, 0x61, 0x52, 0x46, 0x24, 0xb2, 0xf6, 0xcb, 0x19, 0x30, - 0xb9, 0x87, 0x8f, 0x0f, 0x70, 0x48, 0x5f, 0x69, 0xda, 0x05, 0x8d, 0x16, 0x76, 0x4a, 0x36, 0xa9, - 0x8b, 0x2e, 0x28, 0xe5, 0x06, 0x13, 0xa7, 0xaa, 0xe3, 0x16, 0x76, 0xf4, 0x86, 0x4d, 0xbf, 0x8b, - 0x18, 0x1c, 0xde, 0x03, 0xd3, 0xd9, 0x43, 0x7a, 0xac, 0x56, 0xab, 0x21, 0x6b, 0x59, 0xd3, 0x5b, - 0xcb, 0x9d, 0x76, 0x66, 0x51, 0x9c, 0xbe, 0xd4, 0xa5, 0xdb, 0xd5, 0x6a, 0xa8, 0xa1, 0x3e, 0x8e, - 0xe6, 0xf3, 0xb1, 0xed, 0x7a, 0x8d, 0xc0, 0xf5, 0xc9, 0x8e, 0x65, 0x95, 0x18, 0x79, 0x96, 0x91, - 0xa5, 0x7c, 0xd6, 0xba, 0x10, 0xbd, 0x4e, 0x48, 0x43, 0xa8, 0x0c, 0x12, 0x69, 0x3e, 0xb7, 0xec, - 0x08, 0xd3, 0x66, 0x8b, 0x93, 0x07, 0xe8, 0x81, 0x1d, 0x61, 0xd1, 0x9a, 0x05, 0x86, 0x7e, 0x84, - 0x74, 0x07, 0x66, 0x70, 0xc8, 0xf6, 0x5b, 0x4b, 0x7e, 0x84, 0x6c, 0xbf, 0x5e, 0x70, 0x28, 0xb6, - 0x2b, 0x83, 0xe1, 0x13, 0xb0, 0x40, 0x97, 0xfc, 0x54, 0x28, 0x85, 0x41, 0xeb, 0x54, 0xfd, 0x4c, - 0x61, 0x85, 0x78, 0xbb, 0xd3, 0xce, 0xa8, 0x92, 0x80, 0x38, 0x4f, 0x1a, 0x14, 0xa3, 0xa1, 0x24, - 0x0b, 0x66, 0xc1, 0x1c, 0x35, 0xd1, 0xef, 0x92, 0xcb, 0x7c, 0xce, 0x65, 0xa4, 0xcf, 0x8f, 0xc9, - 0xb0, 0xef, 0x59, 0x88, 0xc4, 0x19, 0xf4, 0x74, 0xeb, 0xab, 0x1a, 0x7e, 0x95, 0x25, 0x45, 0xfd, - 0x74, 0x3c, 0x79, 0x24, 0xc8, 0xe1, 0x60, 0x01, 0xd3, 0xd0, 0x10, 0x2e, 0xfc, 0x06, 0x1f, 0x88, - 0xd4, 0x5f, 0x53, 0x8d, 0x99, 0xbb, 0x33, 0x9b, 0x6c, 0xae, 0xda, 0xa4, 0x36, 0x79, 0x2c, 0xa2, - 0x82, 0x1a, 0xe2, 0xb3, 0xd3, 0xb6, 0xdc, 0x00, 0xe9, 0xf0, 0xa3, 0xfe, 0x90, 0x0f, 0x3c, 0x23, - 0xda, 0x28, 0x1d, 0x95, 0x62, 0x8d, 0x90, 0x72, 0xe2, 0x2a, 0xac, 0x2a, 0x3f, 0x3a, 0x53, 0x85, - 0x57, 0x26, 0xc1, 0xa1, 0x39, 0xed, 0x35, 0x49, 0x16, 0xca, 0x8f, 0x53, 0xc9, 0x23, 0x4d, 0x88, - 0xd0, 0x23, 0x96, 0x47, 0x12, 0x67, 0xc4, 0x24, 0x58, 0x1c, 0x3f, 0x39, 0x4b, 0x82, 0x87, 0x11, - 0x67, 0x40, 0x6b, 0xa0, 0x33, 0xb3, 0x58, 0x7e, 0xca, 0x85, 0xae, 0x77, 0xda, 0x99, 0x6b, 0x31, - 0x21, 0xe9, 0xb0, 0xe6, 0x21, 0x0d, 0xa3, 0x0f, 0x51, 0x65, 0xe1, 0xfd, 0xec, 0x0d, 0x54, 0x79, - 0x94, 0xc3, 0xe8, 0xf0, 0xc3, 0x7e, 0x3b, 0x63, 0x41, 0xfe, 0x2b, 0x35, 0xba, 0x9f, 0xf1, 0xe0, - 0x62, 0x78, 0x99, 0xcf, 0xc2, 0xf9, 0xf7, 0x19, 0x7c, 0x71, 0x78, 0xc8, 0x78, 0xf8, 0x5e, 0xaf, - 0x1f, 0xb2, 0xc7, 0xff, 0x27, 0x35, 0xb2, 0x21, 0xf2, 0xa7, 0xcb, 0x68, 0x89, 0xcc, 0x9e, 0xfd, - 0xdf, 0xd1, 0x64, 0xf1, 0x21, 0x4b, 0x68, 0x58, 0x48, 0x74, 0x53, 0xf6, 0xfc, 0x2f, 0x53, 0xe7, - 0xb5, 0x53, 0x1e, 0xc6, 0x20, 0x75, 0x40, 0x8f, 0x85, 0xf4, 0xbf, 0x73, 0xf5, 0x78, 0x64, 0x83, - 0x54, 0xed, 0x17, 0x73, 0x60, 0xd2, 0xc2, 0xac, 0xcb, 0x3e, 0x02, 0x73, 0xfc, 0x57, 0xf7, 0xfe, - 0xa1, 0x0c, 0x0c, 0x79, 0xcc, 0xad, 0xf7, 0xaf, 0x21, 0x71, 0x02, 0xed, 0x7f, 0xdc, 0x50, 0xc0, - 0xe4, 0xe3, 0x20, 0x3c, 0x12, 0x37, 0x18, 0xa9, 0x4c, 0x42, 0xc1, 0xe7, 0xfe, 0x9e, 0x80, 0xc0, - 0xc3, 0x6f, 0x02, 0xc0, 0x0d, 0xec, 0xa0, 0xe6, 0xbb, 0xba, 0xd4, 0x69, 0x67, 0x60, 0x8c, 0xcd, - 0x0f, 0x68, 0x09, 0x49, 0xe7, 0xca, 0x6d, 0xec, 0xd9, 0xa7, 0xa6, 0x4d, 0xb0, 0xef, 0x9c, 0x8a, - 0x4b, 0xcd, 0x9c, 0xfc, 0x59, 0x57, 0xa9, 0x5f, 0xf7, 0x38, 0x40, 0x3f, 0xa6, 0x73, 0x65, 0x9c, - 0x02, 0xbf, 0x03, 0xd2, 0x71, 0x0b, 0x3a, 0x61, 0xbd, 0x62, 0x4e, 0xee, 0x15, 0x49, 0x19, 0x3d, - 0x3c, 0xd1, 0xd0, 0x00, 0x0f, 0x7e, 0x04, 0x96, 0xf7, 0x1b, 0x55, 0x9b, 0xe0, 0x6a, 0x22, 0xae, - 0x39, 0x26, 0x78, 0xa3, 0xd3, 0xce, 0x64, 0xb8, 0x60, 0x93, 0xc3, 0xf4, 0xc1, 0xf8, 0x86, 0x2b, - 0xd0, 0x1c, 0xa1, 0xa0, 0xe9, 0x57, 0x4d, 0xf7, 0xd8, 0x25, 0xea, 0xf2, 0x9a, 0xb2, 0x3e, 0x21, - 0xe7, 0x28, 0xa4, 0x3e, 0xdd, 0xa3, 0x4e, 0x0d, 0x49, 0x48, 0x5a, 0x5e, 0xa3, 0xe5, 0x92, 0xa2, - 0x4f, 0x1b, 0x5b, 0x33, 0xc4, 0xea, 0xa5, 0x81, 0x46, 0xd0, 0x72, 0x89, 0x1e, 0xf8, 0x7a, 0x8d, - 0x03, 0x68, 0x23, 0x90, 0x09, 0x70, 0x07, 0xa4, 0x73, 0x81, 0x1f, 0xb1, 0x69, 0xdc, 0x39, 0xe5, - 0xd3, 0xc1, 0x4a, 0xb2, 0x29, 0x39, 0x7d, 0x44, 0x77, 0x32, 0x18, 0x60, 0xc1, 0xfb, 0x60, 0xc6, - 0xf0, 0xed, 0x03, 0x0f, 0x97, 0x1a, 0x61, 0x50, 0x13, 0x17, 0xa2, 0x95, 0x4e, 0x3b, 0xb3, 0x24, - 0x22, 0x61, 0x4e, 0xbd, 0x41, 0xbd, 0xb4, 0x33, 0xf6, 0xb1, 0xf0, 0x7d, 0x30, 0x2b, 0xe2, 0xc9, - 0xd9, 0x11, 0xee, 0x5e, 0x20, 0xa4, 0xaf, 0x51, 0x44, 0xaf, 0x3b, 0xd4, 0xad, 0xa1, 0x18, 0x9a, - 0xbe, 0x28, 0x62, 0xcd, 0xb2, 0xba, 0x47, 0x2f, 0x0e, 0x89, 0x17, 0xa5, 0xcb, 0xe7, 0x05, 0x61, - 0x2f, 0x4a, 0x9c, 0x42, 0x47, 0x5a, 0x61, 0x29, 0xd7, 0x9b, 0xb5, 0x9a, 0x87, 0xc5, 0x6d, 0x41, - 0x4a, 0x65, 0x57, 0x24, 0xe2, 0x80, 0xbe, 0x86, 0x60, 0xc0, 0x5d, 0x69, 0x32, 0xc9, 0x05, 0xc7, - 0xc7, 0xb6, 0x5f, 0x8d, 0x54, 0x2d, 0x79, 0xe1, 0xed, 0x4f, 0x26, 0x8e, 0xc0, 0xc8, 0x83, 0x49, - 0x97, 0x47, 0x77, 0x85, 0x9a, 0xbe, 0x8f, 0xc3, 0xde, 0x70, 0x75, 0x3b, 0xd9, 0xd5, 0x42, 0xe6, - 0x97, 0xc7, 0xab, 0x04, 0x85, 0xde, 0xc0, 0x8d, 0x16, 0xc1, 0xa1, 0x6f, 0x7b, 0x3d, 0x19, 0x3e, - 0x61, 0x4b, 0x01, 0x61, 0x81, 0x90, 0x85, 0x06, 0x68, 0xb4, 0xbc, 0x65, 0x12, 0xe2, 0x28, 0xb2, - 0x4e, 0x1b, 0x38, 0x52, 0x31, 0xdb, 0x96, 0x54, 0xde, 0x88, 0x39, 0x75, 0x42, 0xbd, 0x1a, 0x92, - 0xb1, 0xf4, 0x2d, 0xe5, 0xcb, 0x5d, 0x7c, 0x5a, 0x76, 0x3f, 0xc1, 0x6c, 0x6c, 0x9a, 0x90, 0x53, - 0x2b, 0xc8, 0xf4, 0xbc, 0x8d, 0xdc, 0x4f, 0xe8, 0x5b, 0x1a, 0x23, 0xd0, 0x71, 0x25, 0x66, 0x30, - 0xed, 0xf0, 0x10, 0xab, 0x87, 0x4c, 0x46, 0x9a, 0x62, 0x13, 0x32, 0xba, 0x47, 0x61, 0x1a, 0x1a, - 0xc2, 0x85, 0xcf, 0xc0, 0xc5, 0xbe, 0xb5, 0x59, 0xab, 0xb9, 0x2d, 0x64, 0xfb, 0x87, 0x58, 0xad, - 0x33, 0x4d, 0xad, 0xd3, 0xce, 0xac, 0x0e, 0x6a, 0x32, 0x9c, 0x1e, 0x52, 0xa0, 0x86, 0x86, 0xf2, - 0xe1, 0x77, 0xc1, 0xca, 0x30, 0xbb, 0xd5, 0xf2, 0x55, 0x97, 0x49, 0x4b, 0xd7, 0xd9, 0x11, 0xd2, - 0x3a, 0x69, 0xf9, 0x1a, 0x1a, 0x25, 0x43, 0xc7, 0xc8, 0x9e, 0xcb, 0x6a, 0xf9, 0xc5, 0x46, 0xa4, - 0x7e, 0x8f, 0x29, 0x4b, 0x25, 0x95, 0x94, 0x49, 0xcb, 0xd7, 0x83, 0x46, 0xa4, 0xa1, 0x24, 0xab, - 0x5f, 0x16, 0xde, 0xdd, 0x23, 0x3e, 0x8d, 0x4e, 0xc4, 0xae, 0x36, 0x5c, 0x87, 0xcf, 0x05, 0x51, - 0xaf, 0x2c, 0x82, 0x00, 0xdf, 0x05, 0xd3, 0xdc, 0xf0, 0xb4, 0x54, 0xe6, 0x43, 0xe8, 0x84, 0x3c, - 0xc0, 0x0b, 0xf6, 0x4b, 0xfa, 0xf4, 0x3e, 0x50, 0xfb, 0xbe, 0x02, 0xde, 0x42, 0xf8, 0x65, 0x13, - 0x47, 0x04, 0x6e, 0x82, 0xe9, 0x62, 0x03, 0x87, 0x36, 0x71, 0x03, 0x9f, 0xf5, 0xa6, 0xf9, 0xbb, - 0x69, 0x31, 0x39, 0xf6, 0xec, 0xa8, 0x0f, 0x81, 0x37, 0xbb, 0x57, 0x0f, 0x95, 0x8f, 0x99, 0x73, - 0x02, 0xcc, 0x8d, 0xa8, 0x7b, 0x2f, 0xb9, 0xd9, 0x6d, 0x80, 0xac, 0xdf, 0xf4, 0x61, 0xdc, 0x88, - 0x84, 0x53, 0x73, 0xc0, 0x14, 0xc2, 0x51, 0x23, 0xf0, 0x23, 0x0c, 0x55, 0xf0, 0x56, 0xb9, 0xe9, - 0x38, 0x38, 0x8a, 0x58, 0x1c, 0x53, 0xa8, 0xbb, 0x84, 0x97, 0xc0, 0x24, 0xbd, 0x5e, 0x36, 0x23, - 0xde, 0xfa, 0x90, 0x58, 0x49, 0xb1, 0xa4, 0xce, 0x88, 0x65, 0xe3, 0x2f, 0x8a, 0xb4, 0x47, 0x38, - 0x0f, 0x40, 0x21, 0x20, 0x65, 0x62, 0x87, 0x04, 0x57, 0xd3, 0x63, 0xf0, 0x22, 0x48, 0x8b, 0x3b, - 0x16, 0xb3, 0xd1, 0xb9, 0x38, 0xad, 0xc0, 0x05, 0x30, 0x83, 0x70, 0xd4, 0x33, 0x8c, 0xc3, 0x59, - 0x30, 0xb5, 0xeb, 0x7a, 0x1e, 0x5b, 0xa5, 0xa8, 0x9b, 0x1e, 0x18, 0xd9, 0xd0, 0xa9, 0xbb, 0x27, - 0x38, 0x7d, 0x81, 0xaa, 0x6c, 0xe3, 0x88, 0x84, 0xc1, 0x29, 0x45, 0xb0, 0xbb, 0x52, 0x7a, 0x02, - 0x5e, 0x06, 0xcb, 0x5b, 0x9e, 0xed, 0x1c, 0xd5, 0x03, 0x8f, 0xfd, 0x4d, 0xa4, 0x14, 0x84, 0xc4, - 0x6a, 0xa1, 0x56, 0xba, 0x0a, 0xaf, 0x82, 0x95, 0x7d, 0xff, 0x60, 0xa8, 0x13, 0xc3, 0x65, 0xb0, - 0xc8, 0x8e, 0xc5, 0x98, 0xb9, 0x06, 0x57, 0xc0, 0xd2, 0xbe, 0x5f, 0x1d, 0x70, 0x1c, 0x6e, 0xfc, - 0x63, 0x8a, 0xc7, 0x23, 0x4e, 0x64, 0xca, 0xdf, 0xcd, 0x9b, 0x66, 0xa5, 0x58, 0x30, 0x2a, 0x8f, - 0x8b, 0xa6, 0x59, 0x7c, 0x6e, 0xa0, 0xf4, 0x18, 0xfc, 0x1a, 0x58, 0x1f, 0x30, 0x57, 0xf6, 0x0b, - 0x56, 0xde, 0xac, 0x58, 0x28, 0xff, 0xe4, 0x89, 0x81, 0x2a, 0xe5, 0x42, 0xb6, 0x54, 0xde, 0x29, - 0x5a, 0x3c, 0x05, 0x0c, 0x6d, 0x1a, 0xd9, 0x6d, 0x03, 0xa5, 0xc7, 0xe1, 0x2d, 0xa0, 0x49, 0x86, - 0x51, 0xc4, 0x54, 0x8f, 0xf8, 0x74, 0xbf, 0x88, 0xf6, 0xf7, 0xd2, 0x17, 0x58, 0xee, 0xa8, 0x21, - 0x6b, 0x9a, 0xe9, 0x09, 0xb8, 0x01, 0x6e, 0x6d, 0x99, 0xd9, 0xdc, 0xee, 0x4e, 0xd1, 0x34, 0x2a, - 0x25, 0xc3, 0x40, 0x95, 0x52, 0x11, 0x59, 0x15, 0xeb, 0x45, 0x05, 0xbd, 0x88, 0x47, 0x5c, 0x85, - 0x59, 0xf0, 0xc1, 0x9b, 0x61, 0x47, 0x45, 0x83, 0xe1, 0x3b, 0x60, 0x6d, 0xb4, 0x84, 0xd8, 0x5b, - 0x0d, 0xbe, 0x07, 0xbe, 0x75, 0x1e, 0x6a, 0xd4, 0x23, 0x0e, 0xcf, 0x7e, 0x84, 0xc8, 0x42, 0x1d, - 0x5e, 0x07, 0xd7, 0x46, 0xa3, 0x68, 0x6a, 0x5c, 0xf8, 0x15, 0xa0, 0x6d, 0x1b, 0x66, 0xf6, 0xa3, - 0xb3, 0xd3, 0xf2, 0x4a, 0x81, 0x9b, 0xe0, 0x36, 0xca, 0x16, 0xb6, 0x8b, 0x7b, 0x95, 0x37, 0xc0, - 0x7f, 0xa6, 0xc0, 0x0f, 0xc1, 0xfd, 0xf3, 0x81, 0xa3, 0x36, 0xf8, 0xb9, 0x02, 0x0d, 0xf0, 0xe8, - 0x8d, 0x9f, 0x37, 0x4a, 0xe6, 0x37, 0x0a, 0xbc, 0x0e, 0xde, 0x1e, 0xce, 0x17, 0x75, 0xf8, 0xad, - 0x02, 0xd7, 0xc1, 0x8d, 0x33, 0x9f, 0x24, 0x90, 0xbf, 0x53, 0xe0, 0xb7, 0xc1, 0xbd, 0xb3, 0x20, - 0xa3, 0xc2, 0xf8, 0xbd, 0x02, 0x1f, 0x82, 0x07, 0x6f, 0xf0, 0x8c, 0x51, 0x02, 0x7f, 0x38, 0x63, - 0x1f, 0xa2, 0xd8, 0x5f, 0x9c, 0xbf, 0x0f, 0x81, 0xfc, 0xa3, 0x02, 0x57, 0xc1, 0xe5, 0xe1, 0x10, - 0xfa, 0x4e, 0xfc, 0x49, 0x81, 0x37, 0xc1, 0xda, 0x99, 0x4a, 0x14, 0xf6, 0x67, 0x05, 0xaa, 0x60, - 0xa9, 0x50, 0xac, 0x3c, 0xce, 0xe6, 0xcd, 0xca, 0xf3, 0xbc, 0xb5, 0x53, 0x29, 0x5b, 0xc8, 0x28, - 0x97, 0xd3, 0xbf, 0x1a, 0xa7, 0xa1, 0xc4, 0x3c, 0x85, 0xa2, 0x70, 0x56, 0x1e, 0x17, 0x51, 0xc5, - 0xcc, 0x3f, 0x33, 0x0a, 0x14, 0xf9, 0xe9, 0x38, 0x5c, 0x00, 0x80, 0xc2, 0x4a, 0xc5, 0x7c, 0xc1, - 0x2a, 0xa7, 0x7f, 0x90, 0x82, 0x73, 0x60, 0xca, 0x78, 0x61, 0x19, 0xa8, 0x90, 0x35, 0xd3, 0xff, - 0x4c, 0x6d, 0x04, 0x00, 0xf4, 0xc7, 0x0a, 0x38, 0x09, 0xc6, 0x77, 0x9f, 0xa5, 0xc7, 0xe0, 0x34, - 0x98, 0x30, 0x8d, 0x6c, 0xd9, 0x48, 0x2b, 0x70, 0x09, 0x2c, 0x18, 0xa6, 0x91, 0xb3, 0xf2, 0xc5, - 0x42, 0x05, 0xed, 0x17, 0x0a, 0xec, 0xdc, 0x48, 0x83, 0xd9, 0xe7, 0x59, 0x2b, 0xb7, 0xd3, 0xb5, - 0xa4, 0xe8, 0xf9, 0x64, 0x16, 0x73, 0xbb, 0x15, 0x94, 0xcd, 0x19, 0xa8, 0x6b, 0xbe, 0x40, 0x81, - 0x4c, 0xa8, 0x6b, 0x99, 0xb8, 0xfb, 0x10, 0x4c, 0x5b, 0xa1, 0xed, 0x47, 0x8d, 0x20, 0x24, 0xf0, - 0xae, 0xbc, 0x98, 0x17, 0x67, 0xbd, 0xe8, 0x63, 0x57, 0x16, 0x7a, 0x6b, 0xde, 0x4e, 0xb4, 0xb1, - 0x75, 0xe5, 0xeb, 0xca, 0xd6, 0xc5, 0x57, 0x7f, 0x5b, 0x1d, 0x7b, 0xf5, 0x7a, 0x55, 0xf9, 0xe2, - 0xf5, 0xaa, 0xf2, 0xd7, 0xd7, 0xab, 0xca, 0xcf, 0xff, 0xbe, 0x3a, 0x76, 0x30, 0xc9, 0xfe, 0x87, - 0x74, 0xef, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x10, 0x53, 0x17, 0x6b, 0x8c, 0x1a, 0x00, 0x00, + // 2468 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xdb, 0x76, 0xdb, 0xc6, + 0xd5, 0x16, 0x44, 0x4b, 0x96, 0x46, 0x27, 0x6a, 0x64, 0x59, 0xb0, 0x9d, 0x88, 0x32, 0x1c, 0xe7, + 0x57, 0xf4, 0x07, 0x72, 0x6b, 0x67, 0xb5, 0xb5, 0x73, 0xb0, 0x29, 0x0a, 0xb6, 0x58, 0x41, 0x24, + 0x3d, 0x84, 0x6c, 0xe7, 0x8a, 0x85, 0xc0, 0xa1, 0x88, 0x0a, 0x02, 0x68, 0x60, 0xa8, 0x50, 0x79, + 0x81, 0xde, 0xf6, 0xbc, 0x7a, 0xd5, 0x27, 0x68, 0xfa, 0x1c, 0x4e, 0x7a, 0x4a, 0xdb, 0x7b, 0xb6, + 0x75, 0x56, 0x5f, 0x80, 0xab, 0xa7, 0xf4, 0xaa, 0x6b, 0x0e, 0x24, 0x07, 0x20, 0x29, 0xe9, 0x4e, + 0xd8, 0xfb, 0xfb, 0x3e, 0xec, 0xd9, 0x9b, 0xd8, 0x7b, 0x8f, 0x0d, 0x16, 0xc2, 0x86, 0xd3, 0x38, + 0xb8, 0x13, 0x36, 0x9c, 0xcd, 0x46, 0x18, 0x90, 0x00, 0x4e, 0x30, 0xc3, 0x75, 0xfd, 0xd0, 0x25, + 0xf5, 0xe6, 0xc1, 0xa6, 0x13, 0x1c, 0xdf, 0x39, 0x0c, 0x0e, 0x83, 0x3b, 0xcc, 0x7b, 0xd0, 0xac, + 0xb1, 0x27, 0xf6, 0xc0, 0xfe, 0xe2, 0x2c, 0xed, 0xeb, 0x39, 0x70, 0xc9, 0x20, 0x4e, 0x15, 0xde, + 0x02, 0x97, 0x0a, 0xf6, 0x31, 0x56, 0x95, 0x35, 0x65, 0x7d, 0x7a, 0x6b, 0xa1, 0xd3, 0xce, 0xcc, + 0x9c, 0xda, 0xc7, 0xde, 0x03, 0xcd, 0xb7, 0x8f, 0xb1, 0x86, 0x98, 0x13, 0xea, 0xe0, 0xf2, 0xb6, + 0x4d, 0xec, 0x6d, 0x37, 0x54, 0xc7, 0x19, 0x6e, 0xa9, 0xd3, 0xce, 0x2c, 0x70, 0x5c, 0xd5, 0x26, + 0xb6, 0x5e, 0x75, 0x43, 0x0d, 0x75, 0x31, 0x70, 0x03, 0x4c, 0x3e, 0xcf, 0x9a, 0x14, 0x9d, 0x62, + 0x68, 0xd8, 0x69, 0x67, 0xe6, 0x39, 0xfa, 0x13, 0xdb, 0xe3, 0x60, 0x81, 0x80, 0x45, 0xb0, 0xb4, + 0x83, 0xed, 0x90, 0x1c, 0x60, 0x9b, 0xe4, 0x7d, 0x82, 0xc3, 0x13, 0xdb, 0xdb, 0x8b, 0xd4, 0x99, + 0x35, 0x65, 0x3d, 0xb5, 0xf5, 0x66, 0xa7, 0x9d, 0xb9, 0xc6, 0x89, 0xf5, 0x2e, 0x48, 0x77, 0x05, + 0x4a, 0x43, 0xc3, 0x98, 0x30, 0x0f, 0x16, 0x0d, 0x0f, 0x3b, 0xc4, 0x0d, 0x7c, 0xcb, 0x3d, 0xc6, + 0x41, 0x93, 0xec, 0x45, 0xea, 0x2c, 0x93, 0xbb, 0xd1, 0x69, 0x67, 0x56, 0xb8, 0x1c, 0x16, 0x10, + 0x9d, 0x70, 0x8c, 0x86, 0x06, 0x59, 0x30, 0x0f, 0xd2, 0xa6, 0x1b, 0x11, 0xec, 0xe7, 0x3c, 0x17, + 0xfb, 0x64, 0x1f, 0x99, 0x91, 0xba, 0xbc, 0x96, 0x5a, 0x9f, 0x96, 0x03, 0xf3, 0x18, 0x42, 0x77, + 0x18, 0x44, 0x6f, 0x86, 0x5e, 0xa4, 0xa1, 0x01, 0x1a, 0x44, 0x60, 0x29, 0x5b, 0x3d, 0xc1, 0x21, + 0x71, 0x23, 0x2c, 0xa9, 0x5d, 0x65, 0x6a, 0x6b, 0x9d, 0x76, 0xe6, 0x0d, 0xae, 0x66, 0x77, 0x41, + 0x71, 0xc1, 0x61, 0x64, 0x78, 0x1f, 0xcc, 0xf1, 0xa7, 0x6c, 0x93, 0x04, 0x96, 0x59, 0x56, 0x57, + 0xd6, 0x94, 0xf5, 0x29, 0xb9, 0x36, 0x76, 0x93, 0x04, 0x3a, 0xa1, 0x02, 0x71, 0x24, 0xcc, 0x81, + 0x79, 0x6e, 0xc8, 0xe1, 0x90, 0x1a, 0xeb, 0xaa, 0xca, 0xb8, 0x52, 0x86, 0xc4, 0xfb, 0x1d, 0x1c, + 0x12, 0xdd, 0x6e, 0x92, 0xba, 0x86, 0x12, 0x14, 0xf8, 0x81, 0x2c, 0xf2, 0xd8, 0xf5, 0xb0, 0x7a, + 0x8d, 0x95, 0xfb, 0x4a, 0xa7, 0x9d, 0x49, 0x0b, 0x11, 0xca, 0xae, 0xb9, 0x1e, 0x8e, 0xb1, 0x29, + 0xb6, 0x1f, 0xfd, 0x2e, 0x3e, 0x65, 0xe4, 0xeb, 0xc9, 0x5f, 0xd6, 0x11, 0x3e, 0x15, 0xdc, 0x38, + 0x12, 0x9a, 0x60, 0x89, 0x1b, 0xac, 0xb0, 0x19, 0x11, 0x5c, 0xcd, 0x65, 0x99, 0xc0, 0x0d, 0x26, + 0x70, 0xbd, 0xd3, 0xce, 0x5c, 0xe5, 0x02, 0x84, 0xbb, 0x75, 0xc7, 0x16, 0x3a, 0xc3, 0x68, 0x34, + 0x17, 0xbc, 0x5c, 0x25, 0x8c, 0x43, 0x56, 0x95, 0x0c, 0xab, 0x8a, 0x94, 0x0b, 0x51, 0xe3, 0x06, + 0xc6, 0xa1, 0x28, 0x48, 0x82, 0x02, 0x2d, 0xb0, 0xd8, 0x2b, 0x51, 0x4f, 0x67, 0x8d, 0xe9, 0xbc, + 0xdd, 0x69, 0x67, 0x34, 0xae, 0xe3, 0xfa, 0x2e, 0x71, 0x6d, 0x4f, 0xef, 0x57, 0x59, 0x92, 0x1c, + 0x14, 0x80, 0x0f, 0xc0, 0x0c, 0xfd, 0xbb, 0x5b, 0xdf, 0x9b, 0xac, 0x46, 0x6a, 0xa7, 0x9d, 0xb9, + 0xc2, 0xf5, 0x18, 0xbb, 0x5f, 0x64, 0x19, 0x0c, 0x4b, 0x00, 0xd2, 0xc7, 0x44, 0x99, 0x35, 0x26, + 0x21, 0xfd, 0xe0, 0x98, 0xc4, 0x60, 0xad, 0x87, 0x70, 0xe1, 0x87, 0x60, 0x96, 0x59, 0xbb, 0xd5, + 0xbe, 0xc5, 0xf2, 0x7d, 0xad, 0xd3, 0xce, 0x2c, 0xcb, 0x5a, 0xfd, 0x92, 0xc7, 0xe0, 0xdd, 0xc3, + 0x74, 0xcb, 0xfd, 0x16, 0x63, 0x27, 0x0f, 0xd3, 0xaf, 0xb9, 0x0c, 0x86, 0x7b, 0x60, 0x91, 0x3e, + 0xc6, 0xeb, 0x7d, 0x9b, 0x29, 0x64, 0x3a, 0xed, 0xcc, 0x0d, 0x49, 0x61, 0xa0, 0xe8, 0x83, 0x4c, + 0xb8, 0x05, 0xe6, 0xf3, 0xbc, 0x14, 0x39, 0x8f, 0xda, 0x43, 0xf5, 0x9d, 0xe4, 0x6f, 0xa7, 0x5b, + 0x2a, 0x87, 0x03, 0x34, 0x94, 0x60, 0xd0, 0x2f, 0x3a, 0x6e, 0x29, 0x13, 0x9b, 0x60, 0x75, 0x83, + 0x09, 0x49, 0x09, 0x4e, 0x08, 0xe9, 0x11, 0x85, 0x69, 0x68, 0x18, 0x79, 0x50, 0xd3, 0x0a, 0x8e, + 0xb0, 0xaf, 0xfe, 0xff, 0x79, 0x9a, 0x84, 0xc2, 0x06, 0x34, 0x19, 0x19, 0x3e, 0x04, 0x73, 0x65, + 0xdf, 0x6e, 0x44, 0xf5, 0x80, 0xe4, 0x82, 0xa6, 0x4f, 0xd4, 0x7b, 0xac, 0x17, 0x4a, 0x65, 0x8b, + 0x84, 0x5b, 0x77, 0xa8, 0x5f, 0x43, 0x71, 0x3c, 0x34, 0xc1, 0xe2, 0xd3, 0x66, 0x40, 0xec, 0x2d, + 0xdb, 0x39, 0xc2, 0x7e, 0x75, 0xeb, 0x94, 0xe0, 0x48, 0x7d, 0x8f, 0x89, 0xac, 0x76, 0xda, 0x99, + 0xeb, 0x5c, 0xe4, 0x25, 0x85, 0xe8, 0x07, 0x1c, 0xa3, 0x1f, 0x50, 0x90, 0x86, 0x06, 0x89, 0x74, + 0x94, 0x94, 0x42, 0xfc, 0x2c, 0x20, 0x58, 0x7d, 0x98, 0x6c, 0x57, 0x8d, 0x10, 0xeb, 0x27, 0x01, + 0xcd, 0x4e, 0x17, 0x23, 0x67, 0x24, 0x08, 0xc3, 0x66, 0x83, 0xe4, 0xea, 0xd8, 0x39, 0x52, 0x1f, + 0x25, 0x7f, 0xc6, 0xbd, 0x8c, 0x70, 0x94, 0xee, 0x50, 0x98, 0x94, 0x11, 0x89, 0xac, 0xfd, 0x72, + 0x06, 0x4c, 0xee, 0xe1, 0xe3, 0x03, 0x1c, 0xd2, 0x9f, 0x34, 0x9d, 0x82, 0x46, 0x0b, 0x3b, 0x25, + 0x9b, 0xd4, 0xc5, 0x14, 0x94, 0x72, 0x83, 0x89, 0x53, 0xd5, 0x71, 0x0b, 0x3b, 0x7a, 0xc3, 0xa6, + 0xdf, 0x45, 0x0c, 0x0e, 0xef, 0x81, 0xe9, 0xec, 0x21, 0x6d, 0xab, 0xd5, 0x6a, 0xc8, 0x46, 0xd6, + 0xf4, 0xd6, 0x72, 0xa7, 0x9d, 0x59, 0x14, 0xdd, 0x97, 0xba, 0x74, 0xbb, 0x5a, 0x0d, 0x35, 0xd4, + 0xc7, 0xd1, 0x7c, 0x3e, 0xb6, 0x5d, 0xaf, 0x11, 0xb8, 0x3e, 0xd9, 0xb1, 0xac, 0x12, 0x23, 0xcf, + 0x32, 0xb2, 0x94, 0xcf, 0x5a, 0x17, 0xa2, 0xd7, 0x09, 0x69, 0x08, 0x95, 0x41, 0x22, 0xcd, 0xe7, + 0x96, 0x1d, 0x61, 0x3a, 0x6c, 0x71, 0xb2, 0x81, 0x1e, 0xd8, 0x11, 0x16, 0xa3, 0x59, 0x60, 0xe8, + 0x47, 0x48, 0x4f, 0x60, 0x06, 0x87, 0xec, 0xbc, 0xb5, 0xe4, 0x47, 0xc8, 0xce, 0xeb, 0x05, 0x87, + 0xe2, 0xb8, 0x32, 0x18, 0x3e, 0x01, 0x0b, 0xf4, 0x91, 0x77, 0x85, 0x52, 0x18, 0xb4, 0x4e, 0xd5, + 0xcf, 0x15, 0x56, 0x88, 0x37, 0x3a, 0xed, 0x8c, 0x2a, 0x09, 0x88, 0x7e, 0xd2, 0xa0, 0x18, 0x0d, + 0x25, 0x59, 0x30, 0x0b, 0xe6, 0xa8, 0x89, 0x7e, 0x97, 0x5c, 0xe6, 0x0b, 0x2e, 0x23, 0x7d, 0x7e, + 0x4c, 0x86, 0x7d, 0xcf, 0x42, 0x24, 0xce, 0xa0, 0xdd, 0xad, 0xaf, 0x6a, 0xf8, 0x55, 0x96, 0x14, + 0xf5, 0xb3, 0xf1, 0x64, 0x4b, 0x90, 0xc3, 0xc1, 0x02, 0xa6, 0xa1, 0x21, 0x5c, 0xf8, 0x4d, 0xbe, + 0x10, 0xa9, 0xbf, 0xa6, 0x1a, 0x33, 0x77, 0x67, 0x36, 0xd9, 0x5e, 0xb5, 0x49, 0x6d, 0xf2, 0x5a, + 0x44, 0x05, 0x35, 0xc4, 0x77, 0xa7, 0x6d, 0x79, 0x00, 0xd2, 0xe5, 0x47, 0xfd, 0x11, 0x5f, 0x78, + 0x46, 0x8c, 0x51, 0xba, 0x2a, 0xc5, 0x06, 0x21, 0xe5, 0xc4, 0x55, 0x58, 0x55, 0x7e, 0x7c, 0xa6, + 0x0a, 0xaf, 0x4c, 0x82, 0x43, 0x73, 0xda, 0x1b, 0x92, 0x2c, 0x94, 0x9f, 0xa4, 0x92, 0x2d, 0x4d, + 0x88, 0xd0, 0x16, 0xcb, 0x23, 0x89, 0x33, 0x62, 0x12, 0x2c, 0x8e, 0x9f, 0x9e, 0x25, 0xc1, 0xc3, + 0x88, 0x33, 0xa0, 0x35, 0x30, 0x99, 0x59, 0x2c, 0x3f, 0xe3, 0x42, 0x37, 0x3b, 0xed, 0xcc, 0x9b, + 0x31, 0x21, 0xa9, 0x59, 0xf3, 0x90, 0x86, 0xd1, 0x87, 0xa8, 0xb2, 0xf0, 0x7e, 0x7e, 0x01, 0x55, + 0x1e, 0xe5, 0x30, 0x3a, 0xfc, 0xa8, 0x3f, 0xce, 0x58, 0x90, 0xff, 0x4c, 0x8d, 0x9e, 0x67, 0x3c, + 0xb8, 0x18, 0x5e, 0xe6, 0xb3, 0x70, 0xfe, 0x75, 0x06, 0x5f, 0x34, 0x0f, 0x19, 0x0f, 0xdf, 0xef, + 0xcd, 0x43, 0xf6, 0xfa, 0x7f, 0xa7, 0x46, 0x0e, 0x44, 0xfe, 0x76, 0x19, 0x2d, 0x91, 0xd9, 0xbb, + 0xff, 0x33, 0x9a, 0x2c, 0x3e, 0x64, 0x09, 0x0d, 0x0b, 0x89, 0x69, 0xca, 0xde, 0xff, 0x75, 0xea, + 0xbc, 0x71, 0xca, 0xc3, 0x18, 0xa4, 0x0e, 0xe8, 0xb1, 0x90, 0xfe, 0x7b, 0xae, 0x1e, 0x8f, 0x6c, + 0x90, 0xaa, 0x7d, 0x35, 0x0b, 0x26, 0x2d, 0xcc, 0xa6, 0xac, 0x74, 0xf3, 0x50, 0x2e, 0x70, 0xf3, + 0x78, 0x17, 0x5c, 0x2e, 0x60, 0xf2, 0x49, 0x10, 0x1e, 0x89, 0x8b, 0x8a, 0x74, 0xf5, 0xf0, 0xb9, + 0x43, 0x43, 0x5d, 0x08, 0xbd, 0xfb, 0xb0, 0xe6, 0x9b, 0x4a, 0xde, 0x7d, 0x78, 0xb7, 0x65, 0x4e, + 0xba, 0x1e, 0x6e, 0x63, 0xcf, 0x3e, 0x35, 0x6d, 0x82, 0x7d, 0xe7, 0x54, 0xdc, 0x4d, 0xe6, 0xe4, + 0xaf, 0xb3, 0x4a, 0xfd, 0xba, 0xc7, 0x01, 0xfa, 0x31, 0x5d, 0x0f, 0xe3, 0x14, 0xf8, 0x5d, 0x90, + 0x8e, 0x5b, 0xd0, 0x09, 0x6b, 0xf9, 0x73, 0x72, 0xcb, 0x4f, 0xca, 0xe8, 0xe1, 0x89, 0x86, 0x06, + 0x78, 0xf0, 0x63, 0xb0, 0xbc, 0xdf, 0xa8, 0xda, 0x04, 0x57, 0x13, 0x71, 0xcd, 0x31, 0xc1, 0x5b, + 0x9d, 0x76, 0x26, 0xc3, 0x05, 0x9b, 0x1c, 0xa6, 0x0f, 0xc6, 0x37, 0x5c, 0x01, 0x7e, 0x0b, 0x00, + 0x14, 0x34, 0xfd, 0xaa, 0xe9, 0x1e, 0xbb, 0x44, 0x5d, 0x5e, 0x53, 0xd6, 0x27, 0xb6, 0xae, 0x76, + 0xda, 0x19, 0xc8, 0xf5, 0x42, 0xea, 0xd3, 0x3d, 0xea, 0xd4, 0x90, 0x84, 0x84, 0x8f, 0xc0, 0x9c, + 0xd1, 0x72, 0x49, 0xd1, 0xa7, 0xf3, 0xa9, 0x19, 0x62, 0xf5, 0xea, 0x40, 0x3f, 0x6f, 0xb9, 0x44, + 0x0f, 0x7c, 0xbd, 0xc6, 0x01, 0xb4, 0x9f, 0xcb, 0x04, 0xb8, 0x03, 0xd2, 0xb9, 0xc0, 0x8f, 0xd8, + 0x52, 0xed, 0x9c, 0xf2, 0x21, 0xbf, 0x92, 0x9c, 0x2d, 0x4e, 0x1f, 0xd1, 0x1d, 0xf0, 0x03, 0x2c, + 0x78, 0x1f, 0xcc, 0x18, 0xbe, 0x7d, 0xe0, 0xe1, 0x52, 0x23, 0x0c, 0x6a, 0xe2, 0x5e, 0xb3, 0xd2, + 0x69, 0x67, 0x96, 0x44, 0x24, 0xcc, 0xa9, 0x37, 0xa8, 0x97, 0x0e, 0xb8, 0x3e, 0x96, 0x96, 0x5a, + 0xc4, 0xc3, 0xf2, 0xb2, 0x47, 0x6f, 0x02, 0x89, 0x52, 0x8b, 0xf8, 0x45, 0x4a, 0x59, 0xa9, 0xe3, + 0x14, 0xba, 0x5b, 0x0a, 0x4b, 0xb9, 0xde, 0xac, 0xd5, 0x3c, 0xac, 0xae, 0x25, 0x93, 0xd1, 0x15, + 0x89, 0x38, 0xa0, 0xaf, 0x21, 0x18, 0xf0, 0x03, 0x30, 0x2b, 0x2c, 0x39, 0x3b, 0xc2, 0x91, 0x7a, + 0x93, 0x5d, 0x24, 0xa4, 0xaf, 0xbb, 0xab, 0xe0, 0x50, 0xb7, 0x86, 0x62, 0x68, 0xb8, 0x2b, 0x2d, + 0x18, 0xb9, 0xe0, 0xf8, 0xd8, 0xf6, 0xab, 0x91, 0xaa, 0x25, 0xef, 0xad, 0xfd, 0x05, 0xc3, 0x11, + 0x18, 0x79, 0xbf, 0xe8, 0xf2, 0x68, 0x4e, 0x50, 0xd3, 0xf7, 0x71, 0xd8, 0xdb, 0x91, 0xde, 0x49, + 0x0e, 0xa7, 0x90, 0xf9, 0xe5, 0x2d, 0x29, 0x41, 0xa1, 0x17, 0x69, 0xa3, 0x45, 0x70, 0xe8, 0xdb, + 0x5e, 0x4f, 0x86, 0x2f, 0xca, 0x52, 0x40, 0x58, 0x20, 0x64, 0xa1, 0x01, 0x1a, 0x2d, 0x6f, 0x99, + 0x84, 0x38, 0x8a, 0xac, 0xd3, 0x06, 0x8e, 0x54, 0xcc, 0x8e, 0x25, 0x95, 0x37, 0x62, 0x4e, 0x9d, + 0x50, 0xaf, 0x86, 0x64, 0x2c, 0xfd, 0x95, 0xf2, 0xc7, 0x5d, 0x7c, 0x5a, 0x76, 0x3f, 0xc5, 0x6c, + 0xfb, 0x99, 0x90, 0x0b, 0x23, 0xc8, 0xb4, 0x6d, 0x46, 0xee, 0xa7, 0xf4, 0x57, 0x1a, 0x23, 0xd0, + 0xad, 0x23, 0x66, 0x30, 0xed, 0xf0, 0x10, 0xab, 0x87, 0x4c, 0x46, 0x5a, 0x46, 0x13, 0x32, 0xba, + 0x47, 0x61, 0x1a, 0x1a, 0xc2, 0x85, 0xcf, 0xc0, 0x95, 0xbe, 0xb5, 0x59, 0xab, 0xb9, 0x2d, 0x64, + 0xfb, 0x87, 0x58, 0xad, 0x33, 0x4d, 0xad, 0xd3, 0xce, 0xac, 0x0e, 0x6a, 0x32, 0x9c, 0x1e, 0x52, + 0xa0, 0x86, 0x86, 0xf2, 0xe1, 0xf7, 0xc0, 0xca, 0x30, 0xbb, 0xd5, 0xf2, 0x55, 0x97, 0x49, 0x4b, + 0xb7, 0xd2, 0x11, 0xd2, 0x3a, 0x69, 0xf9, 0x1a, 0x1a, 0x25, 0x43, 0xb7, 0xc1, 0x9e, 0xcb, 0x6a, + 0xf9, 0xc5, 0x46, 0xa4, 0x7e, 0x9f, 0x29, 0x4b, 0x25, 0x95, 0x94, 0x49, 0xcb, 0xd7, 0x83, 0x46, + 0xa4, 0xa1, 0x24, 0xab, 0x5f, 0x16, 0x3e, 0xa4, 0x23, 0xbe, 0x54, 0x4e, 0xc4, 0x6e, 0x28, 0x5c, + 0x87, 0x8f, 0xf7, 0xa8, 0x57, 0x16, 0x41, 0x80, 0xef, 0x81, 0x69, 0x6e, 0x78, 0x5a, 0x2a, 0xf3, + 0x5d, 0x72, 0x42, 0xde, 0xc3, 0x05, 0xfb, 0x25, 0x7d, 0x7b, 0x1f, 0xa8, 0xfd, 0x40, 0x01, 0x97, + 0x11, 0x7e, 0xd9, 0xc4, 0x11, 0x81, 0x9b, 0x60, 0xba, 0xd8, 0xc0, 0xa1, 0x4d, 0xdc, 0xc0, 0x67, + 0x83, 0x66, 0xfe, 0x6e, 0x5a, 0x2c, 0x80, 0x3d, 0x3b, 0xea, 0x43, 0xe0, 0xed, 0xee, 0x0d, 0x42, + 0xe5, 0xdb, 0xe2, 0x9c, 0x00, 0x73, 0x23, 0xea, 0x5e, 0x2f, 0x6e, 0x77, 0xe7, 0x18, 0x1b, 0x31, + 0x7d, 0x18, 0x37, 0x22, 0xe1, 0xd4, 0x1c, 0x30, 0x85, 0x70, 0xd4, 0x08, 0xfc, 0x08, 0x43, 0x15, + 0x5c, 0x2e, 0x37, 0x1d, 0x07, 0x47, 0x11, 0x8b, 0x63, 0x0a, 0x75, 0x1f, 0xe1, 0x55, 0x30, 0x49, + 0x6f, 0x89, 0xcd, 0x88, 0x8f, 0x36, 0x24, 0x9e, 0xa4, 0x58, 0x52, 0x67, 0xc4, 0xb2, 0xf1, 0x17, + 0x45, 0x3a, 0x23, 0x9c, 0x07, 0xa0, 0x10, 0x90, 0x32, 0xb1, 0x43, 0x82, 0xab, 0xe9, 0x31, 0x78, + 0x05, 0xa4, 0xc5, 0x55, 0x89, 0xd9, 0xe8, 0x7a, 0x9b, 0x56, 0xe0, 0x02, 0x98, 0x41, 0x38, 0xea, + 0x19, 0xc6, 0xe1, 0x2c, 0x98, 0xda, 0x75, 0x3d, 0x8f, 0x3d, 0xa5, 0xa8, 0x9b, 0x36, 0x8c, 0x6c, + 0xe8, 0xd4, 0xdd, 0x13, 0x9c, 0xbe, 0x44, 0x55, 0xb6, 0x71, 0x44, 0xc2, 0xe0, 0x94, 0x22, 0xd8, + 0x95, 0x27, 0x3d, 0x01, 0xaf, 0x81, 0xe5, 0x2d, 0xcf, 0x76, 0x8e, 0xea, 0x81, 0xc7, 0xfe, 0x69, + 0xa3, 0x14, 0x84, 0xc4, 0x6a, 0xa1, 0x56, 0xba, 0x0a, 0x6f, 0x80, 0x95, 0x7d, 0xff, 0x60, 0xa8, + 0x13, 0xc3, 0x65, 0xb0, 0xc8, 0x9a, 0x6a, 0xcc, 0x5c, 0x83, 0x2b, 0x60, 0x69, 0xdf, 0xaf, 0x0e, + 0x38, 0x0e, 0x37, 0xfe, 0x3e, 0xc5, 0xe3, 0x11, 0x8d, 0x90, 0xf2, 0x77, 0xf3, 0xa6, 0x59, 0x29, + 0x16, 0x8c, 0xca, 0xe3, 0xa2, 0x69, 0x16, 0x9f, 0x1b, 0x28, 0x3d, 0x06, 0xdf, 0x05, 0xeb, 0x03, + 0xe6, 0xca, 0x7e, 0xc1, 0xca, 0x9b, 0x15, 0x0b, 0xe5, 0x9f, 0x3c, 0x31, 0x50, 0xa5, 0x5c, 0xc8, + 0x96, 0xca, 0x3b, 0x45, 0x8b, 0xa7, 0x80, 0xa1, 0x4d, 0x23, 0xbb, 0x6d, 0xa0, 0xf4, 0x38, 0x7c, + 0x1b, 0x68, 0x92, 0x61, 0x14, 0x31, 0xd5, 0x23, 0x3e, 0xdd, 0x2f, 0xa2, 0xfd, 0xbd, 0xf4, 0x25, + 0x96, 0x3b, 0x6a, 0xc8, 0x9a, 0x66, 0x7a, 0x02, 0x6e, 0x80, 0xb7, 0xb7, 0xcc, 0x6c, 0x6e, 0x77, + 0xa7, 0x68, 0x1a, 0x95, 0x92, 0x61, 0xa0, 0x4a, 0xa9, 0x88, 0xac, 0x8a, 0xf5, 0xa2, 0x82, 0x5e, + 0xc4, 0x23, 0xae, 0xc2, 0x2c, 0xf8, 0xf0, 0x62, 0xd8, 0x51, 0xd1, 0x60, 0xf8, 0x16, 0x58, 0x1b, + 0x2d, 0x21, 0xce, 0x56, 0x83, 0xef, 0x83, 0x6f, 0x9f, 0x87, 0x1a, 0xf5, 0x8a, 0xc3, 0xb3, 0x5f, + 0x21, 0xb2, 0x50, 0x87, 0x37, 0xc1, 0x9b, 0xa3, 0x51, 0x34, 0x35, 0x2e, 0xfc, 0x3f, 0xa0, 0x6d, + 0x1b, 0x66, 0xf6, 0xe3, 0xb3, 0xd3, 0xf2, 0x4a, 0x81, 0x9b, 0xe0, 0x1d, 0x94, 0x2d, 0x6c, 0x17, + 0xf7, 0x2a, 0x17, 0xc0, 0x7f, 0xae, 0xc0, 0x8f, 0xc0, 0xfd, 0xf3, 0x81, 0xa3, 0x0e, 0xf8, 0x85, + 0x02, 0x0d, 0xf0, 0xe8, 0xc2, 0xef, 0x1b, 0x25, 0xf3, 0x1b, 0x05, 0xde, 0x04, 0x6f, 0x0c, 0xe7, + 0x8b, 0x3a, 0xfc, 0x56, 0x81, 0xeb, 0xe0, 0xd6, 0x99, 0x6f, 0x12, 0xc8, 0xdf, 0x29, 0xf0, 0x3b, + 0xe0, 0xde, 0x59, 0x90, 0x51, 0x61, 0xfc, 0x5e, 0x81, 0x0f, 0xc1, 0x83, 0x0b, 0xbc, 0x63, 0x94, + 0xc0, 0x1f, 0xce, 0x38, 0x87, 0x28, 0xf6, 0x97, 0xe7, 0x9f, 0x43, 0x20, 0xff, 0xa8, 0xc0, 0x55, + 0x70, 0x6d, 0x38, 0x84, 0xfe, 0x26, 0xfe, 0xa4, 0xc0, 0xdb, 0x60, 0xed, 0x4c, 0x25, 0x0a, 0xfb, + 0xb3, 0x02, 0x55, 0xb0, 0x54, 0x28, 0x56, 0x1e, 0x67, 0xf3, 0x66, 0xe5, 0x79, 0xde, 0xda, 0xa9, + 0x94, 0x2d, 0x64, 0x94, 0xcb, 0xe9, 0x5f, 0x8d, 0xd3, 0x50, 0x62, 0x9e, 0x42, 0x51, 0x38, 0x2b, + 0x8f, 0x8b, 0xa8, 0x62, 0xe6, 0x9f, 0x19, 0x05, 0x8a, 0xfc, 0x6c, 0x1c, 0x2e, 0x00, 0x40, 0x61, + 0xa5, 0x62, 0xbe, 0x60, 0x95, 0xd3, 0x3f, 0x4c, 0xc1, 0x39, 0x30, 0x65, 0xbc, 0xb0, 0x0c, 0x54, + 0xc8, 0x9a, 0xe9, 0x7f, 0xa4, 0x36, 0x02, 0x00, 0xfa, 0x6b, 0x05, 0x9c, 0x04, 0xe3, 0xbb, 0xcf, + 0xd2, 0x63, 0x70, 0x1a, 0x4c, 0x98, 0x46, 0xb6, 0x6c, 0xa4, 0x15, 0xb8, 0x04, 0x16, 0x0c, 0xd3, + 0xc8, 0x59, 0xf9, 0x62, 0xa1, 0x82, 0xf6, 0x0b, 0x05, 0xd6, 0x37, 0xd2, 0x60, 0xf6, 0x79, 0xd6, + 0xca, 0xed, 0x74, 0x2d, 0x29, 0xda, 0x9f, 0xcc, 0x62, 0x6e, 0xb7, 0x82, 0xb2, 0x39, 0x03, 0x75, + 0xcd, 0x97, 0x28, 0x90, 0x09, 0x75, 0x2d, 0x13, 0x77, 0x1f, 0x82, 0x69, 0x2b, 0xb4, 0xfd, 0xa8, + 0x11, 0x84, 0x04, 0xde, 0x95, 0x1f, 0xe6, 0x45, 0xaf, 0x17, 0x73, 0xec, 0xfa, 0x42, 0xef, 0x99, + 0x8f, 0x13, 0x6d, 0x6c, 0x5d, 0xf9, 0x86, 0xb2, 0x75, 0xe5, 0xd5, 0xdf, 0x56, 0xc7, 0x5e, 0xbd, + 0x5e, 0x55, 0xbe, 0x7c, 0xbd, 0xaa, 0xfc, 0xf5, 0xf5, 0xaa, 0xf2, 0x8b, 0xaf, 0x56, 0xc7, 0x0e, + 0x26, 0xd9, 0x7f, 0x05, 0xdd, 0xfb, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x42, 0xd4, 0x02, + 0x53, 0x1a, 0x00, 0x00, } diff --git a/tools/functional-tester/rpcpb/rpc.proto b/functional/rpcpb/rpc.proto similarity index 95% rename from tools/functional-tester/rpcpb/rpc.proto rename to functional/rpcpb/rpc.proto index 3a0845b46..9d59befbd 100644 --- a/tools/functional-tester/rpcpb/rpc.proto +++ b/functional/rpcpb/rpc.proto @@ -171,9 +171,9 @@ enum StressType { } message Tester { - string TesterDataDir = 1 [(gogoproto.moretags) = "yaml:\"tester-data-dir\""]; - string TesterNetwork = 2 [(gogoproto.moretags) = "yaml:\"tester-network\""]; - string TesterAddr = 3 [(gogoproto.moretags) = "yaml:\"tester-addr\""]; + string DataDir = 1 [(gogoproto.moretags) = "yaml:\"data-dir\""]; + string Network = 2 [(gogoproto.moretags) = "yaml:\"network\""]; + string Addr = 3 [(gogoproto.moretags) = "yaml:\"addr\""]; // DelayLatencyMsRv is the delay latency in milliseconds, // to inject to simulated slow network. @@ -194,15 +194,15 @@ message Tester { // EnablePprof is true to enable profiler. bool EnablePprof = 24 [(gogoproto.moretags) = "yaml:\"enable-pprof\""]; - // FailureCases is the selected test cases to schedule. - // If empty, run all failure cases. - repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; // FailureDelayMs is the delay duration after failure is injected. // Useful when triggering snapshot or no-op failure cases. - uint32 FailureDelayMs = 32 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""]; + uint32 FailureDelayMs = 31 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""]; // FailureShuffle is true to randomize failure injecting order. - bool FailureShuffle = 33 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; - // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). + bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; + // FailureCases is the selected test cases to schedule. + // If empty, run all failure cases. + repeated string FailureCases = 33 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; + // Failpoinommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000) repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; // RunnerExecPath is a path of etcd-runner binary. diff --git a/tools/functional-tester/runner/election_command.go b/functional/runner/election_command.go similarity index 100% rename from tools/functional-tester/runner/election_command.go rename to functional/runner/election_command.go diff --git a/tools/functional-tester/runner/error.go b/functional/runner/error.go similarity index 100% rename from tools/functional-tester/runner/error.go rename to functional/runner/error.go diff --git a/tools/functional-tester/runner/global.go b/functional/runner/global.go similarity index 100% rename from tools/functional-tester/runner/global.go rename to functional/runner/global.go diff --git a/tools/functional-tester/runner/help.go b/functional/runner/help.go similarity index 100% rename from tools/functional-tester/runner/help.go rename to functional/runner/help.go diff --git a/tools/functional-tester/runner/lease_renewer_command.go b/functional/runner/lease_renewer_command.go similarity index 100% rename from tools/functional-tester/runner/lease_renewer_command.go rename to functional/runner/lease_renewer_command.go diff --git a/tools/functional-tester/runner/lock_racer_command.go b/functional/runner/lock_racer_command.go similarity index 100% rename from tools/functional-tester/runner/lock_racer_command.go rename to functional/runner/lock_racer_command.go diff --git a/tools/functional-tester/runner/root.go b/functional/runner/root.go similarity index 100% rename from tools/functional-tester/runner/root.go rename to functional/runner/root.go diff --git a/tools/functional-tester/runner/watch_command.go b/functional/runner/watch_command.go similarity index 100% rename from tools/functional-tester/runner/watch_command.go rename to functional/runner/watch_command.go diff --git a/tools/functional-tester/scripts/docker-local-agent.sh b/functional/scripts/docker-local-agent.sh similarity index 94% rename from tools/functional-tester/scripts/docker-local-agent.sh rename to functional/scripts/docker-local-agent.sh index 8babe1910..355a996f7 100755 --- a/tools/functional-tester/scripts/docker-local-agent.sh +++ b/functional/scripts/docker-local-agent.sh @@ -8,7 +8,7 @@ COMMENT if ! [[ "${0}" =~ "scripts/docker-local-agent.sh" ]]; then - echo "must be run from tools/functional-tester" + echo "must be run from functional" exit 255 fi diff --git a/tools/functional-tester/scripts/docker-local-tester.sh b/functional/scripts/docker-local-tester.sh similarity index 74% rename from tools/functional-tester/scripts/docker-local-tester.sh rename to functional/scripts/docker-local-tester.sh index f95d6650b..8dbea9813 100755 --- a/tools/functional-tester/scripts/docker-local-tester.sh +++ b/functional/scripts/docker-local-tester.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash if ! [[ "${0}" =~ "scripts/docker-local-tester.sh" ]]; then - echo "must be run from tools/functional-tester" + echo "must be run from functional" exit 255 fi @@ -15,4 +15,4 @@ docker run \ --net=host \ --name tester \ gcr.io/etcd-development/etcd-functional-tester:go${GO_VERSION} \ - /bin/bash -c "./bin/etcd-tester --config ./local-test.yaml" + /bin/bash -c "./bin/etcd-tester --config ./functional.yaml" diff --git a/tools/functional-tester/scripts/genproto.sh b/functional/scripts/genproto.sh similarity index 100% rename from tools/functional-tester/scripts/genproto.sh rename to functional/scripts/genproto.sh diff --git a/tools/functional-tester/tester/checks.go b/functional/tester/checks.go similarity index 99% rename from tools/functional-tester/tester/checks.go rename to functional/tester/checks.go index 9788e8649..d628e218f 100644 --- a/tools/functional-tester/tester/checks.go +++ b/functional/tester/checks.go @@ -21,7 +21,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" "google.golang.org/grpc" diff --git a/tools/functional-tester/tester/cluster.go b/functional/tester/cluster.go similarity index 66% rename from tools/functional-tester/tester/cluster.go rename to functional/tester/cluster.go index 68b54bce1..ef2d999c4 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/functional/tester/cluster.go @@ -26,15 +26,14 @@ import ( "strings" "time" + "github.com/coreos/etcd/functional/rpcpb" "github.com/coreos/etcd/pkg/debugutil" "github.com/coreos/etcd/pkg/fileutil" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/zap" "golang.org/x/time/rate" "google.golang.org/grpc" - yaml "gopkg.in/yaml.v2" ) // Cluster defines tester cluster. @@ -62,221 +61,6 @@ type Cluster struct { cs int } -func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) { - bts, err := ioutil.ReadFile(fpath) - if err != nil { - return nil, err - } - lg.Info("opened configuration file", zap.String("path", fpath)) - - clus := &Cluster{lg: lg} - if err = yaml.Unmarshal(bts, clus); err != nil { - return nil, err - } - - for i, mem := range clus.Members { - if mem.BaseDir == "" { - return nil, fmt.Errorf("Members[i].BaseDir cannot be empty (got %q)", mem.BaseDir) - } - if mem.EtcdLogPath == "" { - return nil, fmt.Errorf("Members[i].EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath) - } - - if mem.Etcd.Name == "" { - return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem) - } - if mem.Etcd.DataDir == "" { - return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem) - } - if mem.Etcd.SnapshotCount == 0 { - return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount) - } - if mem.Etcd.DataDir == "" { - return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir) - } - if mem.Etcd.WALDir == "" { - clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal") - } - - if mem.Etcd.HeartbeatIntervalMs == 0 { - return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd) - } - if mem.Etcd.ElectionTimeoutMs == 0 { - return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd) - } - if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs { - return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs) - } - - port := "" - listenClientPorts := make([]string, len(clus.Members)) - for i, u := range mem.Etcd.ListenClientURLs { - if !isValidURL(u) { - return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u) - } - listenClientPorts[i], err = getPort(u) - if err != nil { - return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u) - } - } - for i, u := range mem.Etcd.AdvertiseClientURLs { - if !isValidURL(u) { - return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u) - } - port, err = getPort(u) - if err != nil { - return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u) - } - if mem.EtcdClientProxy && listenClientPorts[i] == port { - return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i]) - } - } - - listenPeerPorts := make([]string, len(clus.Members)) - for i, u := range mem.Etcd.ListenPeerURLs { - if !isValidURL(u) { - return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u) - } - listenPeerPorts[i], err = getPort(u) - if err != nil { - return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u) - } - } - for j, u := range mem.Etcd.AdvertisePeerURLs { - if !isValidURL(u) { - return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u) - } - port, err = getPort(u) - if err != nil { - return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u) - } - if mem.EtcdPeerProxy && listenPeerPorts[j] == port { - return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j]) - } - } - - if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) { - return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath) - } - if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) { - return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir) - } - - // TODO: support separate WALDir that can be handled via failure-archive - if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) { - return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir) - } - - // TODO: only support generated certs with TLS generator - // deprecate auto TLS - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth { - return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'") - } - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" { - return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile) - } - if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" { - return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) - } - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" { - return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile) - } - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" { - return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) - } - if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth { - return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'") - } - if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" { - return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) - } - if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" { - return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) - } - if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" { - return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile) - } - if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" { - return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile) - } - - if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" { - for _, cu := range mem.Etcd.ListenClientURLs { - var u *url.URL - u, err = url.Parse(cu) - if err != nil { - return nil, err - } - if u.Scheme != "https" { // TODO: support unix - return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) - } - } - for _, cu := range mem.Etcd.AdvertiseClientURLs { - var u *url.URL - u, err = url.Parse(cu) - if err != nil { - return nil, err - } - if u.Scheme != "https" { // TODO: support unix - return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) - } - } - } - if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" { - for _, cu := range mem.Etcd.ListenPeerURLs { - var u *url.URL - u, err = url.Parse(cu) - if err != nil { - return nil, err - } - if u.Scheme != "https" { // TODO: support unix - return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) - } - } - for _, cu := range mem.Etcd.AdvertisePeerURLs { - var u *url.URL - u, err = url.Parse(cu) - if err != nil { - return nil, err - } - if u.Scheme != "https" { // TODO: support unix - return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) - } - } - } - } - - if len(clus.Tester.FailureCases) == 0 { - return nil, errors.New("FailureCases not found") - } - if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 { - return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv) - } - if clus.Tester.UpdatedDelayLatencyMs == 0 { - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs - } - - for _, v := range clus.Tester.FailureCases { - if _, ok := rpcpb.FailureCase_value[v]; !ok { - return nil, fmt.Errorf("%q is not defined in 'rpcpb.FailureCase_value'", v) - } - } - - for _, v := range clus.Tester.StressTypes { - if _, ok := rpcpb.StressType_value[v]; !ok { - return nil, fmt.Errorf("StressType is unknown; got %q", v) - } - } - if clus.Tester.StressKeySuffixRangeTxn > 100 { - return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn) - } - if clus.Tester.StressKeyTxnOps > 64 { - return nil, fmt.Errorf("StressKeyTxnOps maximum value is 64, got %v", clus.Tester.StressKeyTxnOps) - } - - return clus, err -} - var dialOpts = []grpc.DialOption{ grpc.WithInsecure(), grpc.WithTimeout(5 * time.Second), @@ -285,7 +69,7 @@ var dialOpts = []grpc.DialOption{ // NewCluster creates a client from a tester configuration. func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) { - clus, err := newCluster(lg, fpath) + clus, err := read(lg, fpath) if err != nil { return nil, err } @@ -320,7 +104,7 @@ func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) { } } clus.testerHTTPServer = &http.Server{ - Addr: clus.Tester.TesterAddr, + Addr: clus.Tester.Addr, Handler: mux, } go clus.serveTesterServer() @@ -340,12 +124,12 @@ func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) { func (clus *Cluster) serveTesterServer() { clus.lg.Info( "started tester HTTP server", - zap.String("tester-address", clus.Tester.TesterAddr), + zap.String("tester-address", clus.Tester.Addr), ) err := clus.testerHTTPServer.ListenAndServe() clus.lg.Info( "tester HTTP server returned", - zap.String("tester-address", clus.Tester.TesterAddr), + zap.String("tester-address", clus.Tester.Addr), zap.Error(err), ) if err != nil && err != http.ErrServerClosed { @@ -444,48 +228,6 @@ func (clus *Cluster) UpdateDelayLatencyMs() { } } -func (clus *Cluster) shuffleFailures() { - rand.Seed(time.Now().UnixNano()) - offset := rand.Intn(1000) - n := len(clus.failures) - cp := coprime(n) - - fs := make([]Failure, n) - for i := 0; i < n; i++ { - fs[i] = clus.failures[(cp*i+offset)%n] - } - clus.failures = fs - clus.lg.Info("shuffled test failure cases", zap.Int("total", n)) -} - -/* -x and y of GCD 1 are coprime to each other - -x1 = ( coprime of n * idx1 + offset ) % n -x2 = ( coprime of n * idx2 + offset ) % n -(x2 - x1) = coprime of n * (idx2 - idx1) % n - = (idx2 - idx1) = 1 - -Consecutive x's are guaranteed to be distinct -*/ -func coprime(n int) int { - coprime := 1 - for i := n / 2; i < n; i++ { - if gcd(i, n) == 1 { - coprime = i - break - } - } - return coprime -} - -func gcd(x, y int) int { - if y == 0 { - return x - } - return gcd(y, x%y) -} - func (clus *Cluster) updateStresserChecker() { cs := &compositeStresser{} for _, m := range clus.Members { @@ -641,7 +383,7 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error { // store TLS assets from agents/servers onto disk if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) { dirClient := filepath.Join( - clus.Tester.TesterDataDir, + clus.Tester.DataDir, clus.Members[idx].Etcd.Name, "fixtures", "client", @@ -717,7 +459,7 @@ func (clus *Cluster) DestroyEtcdAgents() { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) err := clus.testerHTTPServer.Shutdown(ctx) cancel() - clus.lg.Info("closed tester HTTP server", zap.String("tester-address", clus.Tester.TesterAddr), zap.Error(err)) + clus.lg.Info("closed tester HTTP server", zap.String("tester-address", clus.Tester.Addr), zap.Error(err)) } } diff --git a/functional/tester/cluster_read_config.go b/functional/tester/cluster_read_config.go new file mode 100644 index 000000000..ed13f9823 --- /dev/null +++ b/functional/tester/cluster_read_config.go @@ -0,0 +1,244 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "errors" + "fmt" + "io/ioutil" + "net/url" + "path/filepath" + "strings" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" + yaml "gopkg.in/yaml.v2" +) + +func read(lg *zap.Logger, fpath string) (*Cluster, error) { + bts, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + lg.Info("opened configuration file", zap.String("path", fpath)) + + clus := &Cluster{lg: lg} + if err = yaml.Unmarshal(bts, clus); err != nil { + return nil, err + } + + for i, mem := range clus.Members { + if mem.BaseDir == "" { + return nil, fmt.Errorf("BaseDir cannot be empty (got %q)", mem.BaseDir) + } + if mem.EtcdLogPath == "" { + return nil, fmt.Errorf("EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath) + } + + if mem.Etcd.Name == "" { + return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem) + } + if mem.Etcd.DataDir == "" { + return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem) + } + if mem.Etcd.SnapshotCount == 0 { + return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount) + } + if mem.Etcd.DataDir == "" { + return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir) + } + if mem.Etcd.WALDir == "" { + clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal") + } + + if mem.Etcd.HeartbeatIntervalMs == 0 { + return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd) + } + if mem.Etcd.ElectionTimeoutMs == 0 { + return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd) + } + if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs { + return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs) + } + + port := "" + listenClientPorts := make([]string, len(clus.Members)) + for i, u := range mem.Etcd.ListenClientURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u) + } + listenClientPorts[i], err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u) + } + } + for i, u := range mem.Etcd.AdvertiseClientURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u) + } + port, err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u) + } + if mem.EtcdClientProxy && listenClientPorts[i] == port { + return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i]) + } + } + + listenPeerPorts := make([]string, len(clus.Members)) + for i, u := range mem.Etcd.ListenPeerURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u) + } + listenPeerPorts[i], err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u) + } + } + for j, u := range mem.Etcd.AdvertisePeerURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u) + } + port, err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u) + } + if mem.EtcdPeerProxy && listenPeerPorts[j] == port { + return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j]) + } + } + + if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) { + return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath) + } + if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) { + return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir) + } + + // TODO: support separate WALDir that can be handled via failure-archive + if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) { + return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir) + } + + // TODO: only support generated certs with TLS generator + // deprecate auto TLS + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth { + return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'") + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) + } + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth { + return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'") + } + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" { + return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" { + return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile) + } + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile) + } + + if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" { + for _, cu := range mem.Etcd.ListenClientURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) + } + } + for _, cu := range mem.Etcd.AdvertiseClientURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) + } + } + } + if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" { + for _, cu := range mem.Etcd.ListenPeerURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + } + } + for _, cu := range mem.Etcd.AdvertisePeerURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + } + } + } + } + + if len(clus.Tester.FailureCases) == 0 { + return nil, errors.New("FailureCases not found") + } + if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 { + return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv) + } + if clus.Tester.UpdatedDelayLatencyMs == 0 { + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + } + + for _, v := range clus.Tester.FailureCases { + if _, ok := rpcpb.FailureCase_value[v]; !ok { + return nil, fmt.Errorf("%q is not defined in 'rpcpb.FailureCase_value'", v) + } + } + + for _, v := range clus.Tester.StressTypes { + if _, ok := rpcpb.StressType_value[v]; !ok { + return nil, fmt.Errorf("StressType is unknown; got %q", v) + } + } + if clus.Tester.StressKeySuffixRangeTxn > 100 { + return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn) + } + if clus.Tester.StressKeyTxnOps > 64 { + return nil, fmt.Errorf("StressKeyTxnOps maximum value is 64, got %v", clus.Tester.StressKeyTxnOps) + } + + return clus, err +} diff --git a/tools/functional-tester/tester/cluster_tester.go b/functional/tester/cluster_run.go similarity index 96% rename from tools/functional-tester/tester/cluster_tester.go rename to functional/tester/cluster_run.go index b30c568dc..8c8af10ad 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/functional/tester/cluster_run.go @@ -19,8 +19,8 @@ import ( "os" "time" + "github.com/coreos/etcd/functional/rpcpb" "github.com/coreos/etcd/pkg/fileutil" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" "go.uber.org/zap" ) @@ -29,12 +29,12 @@ import ( // Previous tests showed etcd can compact about 60,000 entries per second. const compactQPS = 50000 -// StartTester starts tester. -func (clus *Cluster) StartTester() { - if err := fileutil.TouchDirAll(clus.Tester.TesterDataDir); err != nil { +// Run starts tester. +func (clus *Cluster) Run() { + if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil { clus.lg.Panic( "failed to create test data directory", - zap.String("dir", clus.Tester.TesterDataDir), + zap.String("dir", clus.Tester.DataDir), zap.Error(err), ) } diff --git a/functional/tester/cluster_shuffle.go b/functional/tester/cluster_shuffle.go new file mode 100644 index 000000000..2eaf2737b --- /dev/null +++ b/functional/tester/cluster_shuffle.go @@ -0,0 +1,64 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "math/rand" + "time" + + "go.uber.org/zap" +) + +func (clus *Cluster) shuffleFailures() { + rand.Seed(time.Now().UnixNano()) + offset := rand.Intn(1000) + n := len(clus.failures) + cp := coprime(n) + + fs := make([]Failure, n) + for i := 0; i < n; i++ { + fs[i] = clus.failures[(cp*i+offset)%n] + } + clus.failures = fs + clus.lg.Info("shuffled test failure cases", zap.Int("total", n)) +} + +/* +x and y of GCD 1 are coprime to each other + +x1 = ( coprime of n * idx1 + offset ) % n +x2 = ( coprime of n * idx2 + offset ) % n +(x2 - x1) = coprime of n * (idx2 - idx1) % n + = (idx2 - idx1) = 1 + +Consecutive x's are guaranteed to be distinct +*/ +func coprime(n int) int { + coprime := 1 + for i := n / 2; i < n; i++ { + if gcd(i, n) == 1 { + coprime = i + break + } + } + return coprime +} + +func gcd(x, y int) int { + if y == 0 { + return x + } + return gcd(y, x%y) +} diff --git a/tools/functional-tester/tester/cluster_test.go b/functional/tester/cluster_test.go similarity index 95% rename from tools/functional-tester/tester/cluster_test.go rename to functional/tester/cluster_test.go index e2b34e9e4..323db79d1 100644 --- a/tools/functional-tester/tester/cluster_test.go +++ b/functional/tester/cluster_test.go @@ -19,12 +19,12 @@ import ( "sort" "testing" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" ) -func Test_newCluster(t *testing.T) { +func Test_read(t *testing.T) { exp := &Cluster{ Members: []*rpcpb.Member{ { @@ -143,9 +143,9 @@ func Test_newCluster(t *testing.T) { }, }, Tester: &rpcpb.Tester{ - TesterDataDir: "/tmp/etcd-tester-data", - TesterNetwork: "tcp", - TesterAddr: "127.0.0.1:9028", + DataDir: "/tmp/etcd-tester-data", + Network: "tcp", + Addr: "127.0.0.1:9028", DelayLatencyMs: 5000, DelayLatencyMsRv: 500, UpdatedDelayLatencyMs: 5000, @@ -153,6 +153,8 @@ func Test_newCluster(t *testing.T) { ExitOnFailure: true, ConsistencyCheck: true, EnablePprof: true, + FailureDelayMs: 7000, + FailureShuffle: true, FailureCases: []string{ "KILL_ONE_FOLLOWER", "KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", @@ -181,10 +183,8 @@ func Test_newCluster(t *testing.T) { "NO_FAIL_WITH_STRESS", "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", }, - FailureDelayMs: 7000, - FailureShuffle: true, FailpointCommands: []string{`panic("etcd-tester")`}, - RunnerExecPath: "/etcd-runner", + RunnerExecPath: "./bin/etcd-runner", ExternalExecPath: "", StressTypes: []string{"KV", "LEASE"}, StressKeySize: 100, @@ -203,7 +203,7 @@ func Test_newCluster(t *testing.T) { } defer logger.Sync() - cfg, err := newCluster(logger, "./local-test.yaml") + cfg, err := read(logger, "../../functional.yaml") if err != nil { t.Fatal(err) } @@ -235,6 +235,7 @@ func Test_newCluster(t *testing.T) { sort.Strings(fs1) sort.Strings(fs2) sort.Strings(fs3) + if !reflect.DeepEqual(fs1, fs2) { t.Fatalf("expected %q, got %q", fs1, fs2) } diff --git a/tools/functional-tester/tester/doc.go b/functional/tester/doc.go similarity index 100% rename from tools/functional-tester/tester/doc.go rename to functional/tester/doc.go diff --git a/tools/functional-tester/tester/failure.go b/functional/tester/failure.go similarity index 99% rename from tools/functional-tester/tester/failure.go rename to functional/tester/failure.go index 2a1a20e9d..cb1cef930 100644 --- a/tools/functional-tester/tester/failure.go +++ b/functional/tester/failure.go @@ -19,7 +19,7 @@ import ( "math/rand" "time" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" ) diff --git a/tools/functional-tester/tester/failure_case_delay.go b/functional/tester/failure_case_delay.go similarity index 100% rename from tools/functional-tester/tester/failure_case_delay.go rename to functional/tester/failure_case_delay.go diff --git a/tools/functional-tester/tester/failure_case_external.go b/functional/tester/failure_case_external.go similarity index 96% rename from tools/functional-tester/tester/failure_case_external.go rename to functional/tester/failure_case_external.go index 0d73422b1..09f341d67 100644 --- a/tools/functional-tester/tester/failure_case_external.go +++ b/functional/tester/failure_case_external.go @@ -18,7 +18,7 @@ import ( "fmt" "os/exec" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" ) type failureExternal struct { diff --git a/tools/functional-tester/tester/failure_case_failpoints.go b/functional/tester/failure_case_failpoints.go similarity index 98% rename from tools/functional-tester/tester/failure_case_failpoints.go rename to functional/tester/failure_case_failpoints.go index c32e419b0..675f9be27 100644 --- a/tools/functional-tester/tester/failure_case_failpoints.go +++ b/functional/tester/failure_case_failpoints.go @@ -21,7 +21,7 @@ import ( "strings" "sync" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" ) type failpointStats struct { diff --git a/tools/functional-tester/tester/failure_case_kill.go b/functional/tester/failure_case_kill.go similarity index 97% rename from tools/functional-tester/tester/failure_case_kill.go rename to functional/tester/failure_case_kill.go index 3af1f9da9..8b139b8d5 100644 --- a/tools/functional-tester/tester/failure_case_kill.go +++ b/functional/tester/failure_case_kill.go @@ -14,7 +14,7 @@ package tester -import "github.com/coreos/etcd/tools/functional-tester/rpcpb" +import "github.com/coreos/etcd/functional/rpcpb" func injectKill(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_KillEtcd) diff --git a/tools/functional-tester/tester/failure_case_network_blackhole.go b/functional/tester/failure_case_network_blackhole.go similarity index 98% rename from tools/functional-tester/tester/failure_case_network_blackhole.go rename to functional/tester/failure_case_network_blackhole.go index 469d201bb..79669e989 100644 --- a/tools/functional-tester/tester/failure_case_network_blackhole.go +++ b/functional/tester/failure_case_network_blackhole.go @@ -14,7 +14,7 @@ package tester -import "github.com/coreos/etcd/tools/functional-tester/rpcpb" +import "github.com/coreos/etcd/functional/rpcpb" func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx) diff --git a/tools/functional-tester/tester/failure_case_network_delay.go b/functional/tester/failure_case_network_delay.go similarity index 98% rename from tools/functional-tester/tester/failure_case_network_delay.go rename to functional/tester/failure_case_network_delay.go index e99203768..e8b04157a 100644 --- a/tools/functional-tester/tester/failure_case_network_delay.go +++ b/functional/tester/failure_case_network_delay.go @@ -17,7 +17,7 @@ package tester import ( "time" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" ) diff --git a/tools/functional-tester/tester/failure_case_no_fail.go b/functional/tester/failure_case_no_fail.go similarity index 97% rename from tools/functional-tester/tester/failure_case_no_fail.go rename to functional/tester/failure_case_no_fail.go index 235d8bbc6..fb2b182af 100644 --- a/tools/functional-tester/tester/failure_case_no_fail.go +++ b/functional/tester/failure_case_no_fail.go @@ -17,7 +17,7 @@ package tester import ( "time" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" ) diff --git a/tools/functional-tester/tester/metrics.go b/functional/tester/metrics.go similarity index 100% rename from tools/functional-tester/tester/metrics.go rename to functional/tester/metrics.go diff --git a/tools/functional-tester/tester/stress.go b/functional/tester/stress.go similarity index 98% rename from tools/functional-tester/tester/stress.go rename to functional/tester/stress.go index 281e20d9e..105f09333 100644 --- a/tools/functional-tester/tester/stress.go +++ b/functional/tester/stress.go @@ -18,7 +18,7 @@ import ( "fmt" "time" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" ) diff --git a/tools/functional-tester/tester/stress_composite.go b/functional/tester/stress_composite.go similarity index 100% rename from tools/functional-tester/tester/stress_composite.go rename to functional/tester/stress_composite.go diff --git a/tools/functional-tester/tester/stress_key.go b/functional/tester/stress_key.go similarity index 99% rename from tools/functional-tester/tester/stress_key.go rename to functional/tester/stress_key.go index e8883c145..a589e283c 100644 --- a/tools/functional-tester/tester/stress_key.go +++ b/functional/tester/stress_key.go @@ -25,7 +25,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/etcdserver" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" "golang.org/x/time/rate" diff --git a/tools/functional-tester/tester/stress_lease.go b/functional/tester/stress_lease.go similarity index 99% rename from tools/functional-tester/tester/stress_lease.go rename to functional/tester/stress_lease.go index 5257213d9..9aeab557a 100644 --- a/tools/functional-tester/tester/stress_lease.go +++ b/functional/tester/stress_lease.go @@ -24,7 +24,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" - "github.com/coreos/etcd/tools/functional-tester/rpcpb" + "github.com/coreos/etcd/functional/rpcpb" "go.uber.org/zap" "golang.org/x/time/rate" diff --git a/tools/functional-tester/tester/stress_runner.go b/functional/tester/stress_runner.go similarity index 100% rename from tools/functional-tester/tester/stress_runner.go rename to functional/tester/stress_runner.go diff --git a/tools/functional-tester/tester/utils.go b/functional/tester/utils.go similarity index 100% rename from tools/functional-tester/tester/utils.go rename to functional/tester/utils.go diff --git a/test b/test index 2bee76399..c69f7a62b 100755 --- a/test +++ b/test @@ -37,7 +37,7 @@ source ./build # build before setting up test GOPATH if [[ "${PASSES}" == *"functional"* ]]; then - ./tools/functional-tester/build + ./functional/build fi if [ -z "$PASSES" ]; then @@ -196,7 +196,7 @@ function functional_pass { done echo "Starting 'etcd-tester'" - ./bin/etcd-tester --config ./tools/functional-tester/tester/local-test.yaml && echo "'etcd-tester' succeeded" + ./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded" ETCD_TESTER_EXIT_CODE=$? echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE} diff --git a/tools/functional-tester/README.md b/tools/functional-tester/README.md deleted file mode 100644 index 61e5c08ae..000000000 --- a/tools/functional-tester/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# etcd functional test suite - -etcd functional test suite tests the functionality of an etcd cluster with a focus on failure resistance under high pressure. It sets up an etcd cluster and inject failures into the cluster by killing the process or isolate the network of the process. It expects the etcd cluster to recover within a short amount of time after fixing the fault. - -etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machine, and etcd-tester is a single controller of the test. tester controls agents: start etcd process, stop, terminate, inject failures, and so on. - -### Run locally - -```bash -PASSES=functional ./test -``` - -### Run with Docker - -To run locally, first build tester image: - -```bash -pushd ../.. -make build-docker-functional-tester -popd -``` - -And run [example scripts](./scripts). - -```bash -# run 3 agents for 3-node local etcd cluster -./scripts/docker-local-agent.sh 1 -./scripts/docker-local-agent.sh 2 -./scripts/docker-local-agent.sh 3 - -# to run only 1 tester round -./scripts/docker-local-tester.sh -``` From c57a70caef2225dd6dfaea3cafa2a05a9f931755 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 07:10:28 -0700 Subject: [PATCH 02/16] CHANGELOG-3.4: update Signed-off-by: Gyuho Lee --- CHANGELOG-3.4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-3.4.md b/CHANGELOG-3.4.md index 3e47c8927..119523f60 100644 --- a/CHANGELOG-3.4.md +++ b/CHANGELOG-3.4.md @@ -30,7 +30,7 @@ See [code changes](https://github.com/coreos/etcd/compare/v3.3.0...v3.4.0) and [ - Futhermore, when `--auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute. - Improve [lease expire/revoke operation performance](https://github.com/coreos/etcd/pull/9418), address [lease scalability issue](https://github.com/coreos/etcd/issues/9496). - Make [Lease `Lookup` non-blocking with concurrent `Grant`/`Revoke`](https://github.com/coreos/etcd/pull/9229). -- Improve [functional tester](https://github.com/coreos/etcd/tree/master/tools/functional-tester) coverage: use [proxy layer to run network fault tests in CI](https://github.com/coreos/etcd/pull/9081), enable [TLS both for server and client](https://github.com/coreos/etcd/pull/9534), add [liveness mode](https://github.com/coreos/etcd/issues/9230), and [shuffle test sequence](https://github.com/coreos/etcd/issues/9381). +- Improve [functional tester](https://github.com/coreos/etcd/tree/master/functional) coverage: use [proxy layer to run network fault tests in CI](https://github.com/coreos/etcd/pull/9081), enable [TLS both for server and client](https://github.com/coreos/etcd/pull/9534), add [liveness mode](https://github.com/coreos/etcd/issues/9230), and [shuffle test sequence](https://github.com/coreos/etcd/issues/9381). ### Breaking Changes From ef594eb727db809d4a48827069fa6e818dbf5a92 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 09:23:45 -0700 Subject: [PATCH 03/16] functional/tester: handle static certs Signed-off-by: Gyuho Lee --- functional/tester/cluster_read_config.go | 166 ++++++++++++++++++----- 1 file changed, 129 insertions(+), 37 deletions(-) diff --git a/functional/tester/cluster_read_config.go b/functional/tester/cluster_read_config.go index ed13f9823..1361f9e6e 100644 --- a/functional/tester/cluster_read_config.go +++ b/functional/tester/cluster_read_config.go @@ -135,38 +135,127 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { // TODO: only support generated certs with TLS generator // deprecate auto TLS - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth { - return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'") - } - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" { - return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile) - } - if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" { - return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) - } - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" { - return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile) - } - if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" { - return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) - } - if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth { - return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'") - } if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" { return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) } - if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" { - return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) - } if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" { return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile) } if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" { return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile) } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) + } - if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" { + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerKeyFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerCertFile) + } + // only support self-signed certs + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerTrustedCAFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile != "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerKeyFile != "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile) + } + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerAutoTLS { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth and Etcd.PeerAutoTLS cannot be both 'true'") + } + if (mem.Etcd.PeerCertFile == "") != (mem.Etcd.PeerKeyFile == "") { + return nil, fmt.Errorf("Both Etcd.PeerCertFile %q and Etcd.PeerKeyFile %q must be either empty or non-empty", mem.Etcd.PeerCertFile, mem.Etcd.PeerKeyFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientAutoTLS { + return nil, fmt.Errorf("Etcd.ClientCertAuth and Etcd.ClientAutoTLS cannot be both 'true'") + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientKeyFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientTrustedCAFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) + } + if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile != "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientKeyFile != "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientKeyFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.PeerCertFile) + } + if (mem.Etcd.ClientCertFile == "") != (mem.Etcd.ClientKeyFile == "") { + return nil, fmt.Errorf("Both Etcd.ClientCertFile %q and Etcd.ClientKeyFile %q must be either empty or non-empty", mem.Etcd.ClientCertFile, mem.Etcd.ClientKeyFile) + } + + peerTLS := mem.Etcd.PeerAutoTLS || + (mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile != "" && mem.Etcd.PeerKeyFile != "" && mem.Etcd.PeerTrustedCAFile != "") + if peerTLS { + for _, cu := range mem.Etcd.ListenPeerURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + } + } + for _, cu := range mem.Etcd.AdvertisePeerURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + } + } + clus.Members[i].PeerCertPath = mem.Etcd.PeerCertFile + if mem.Etcd.PeerCertFile != "" { + data, err := ioutil.ReadFile(mem.Etcd.PeerCertFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerCertFile, err) + } + clus.Members[i].PeerCertData = string(data) + } + clus.Members[i].PeerKeyPath = mem.Etcd.PeerKeyFile + if mem.Etcd.PeerKeyFile != "" { + data, err := ioutil.ReadFile(mem.Etcd.PeerKeyFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerKeyFile, err) + } + clus.Members[i].PeerCertData = string(data) + } + clus.Members[i].PeerTrustedCAPath = mem.Etcd.PeerTrustedCAFile + if mem.Etcd.PeerTrustedCAFile != "" { + data, err := ioutil.ReadFile(mem.Etcd.PeerTrustedCAFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerTrustedCAFile, err) + } + clus.Members[i].PeerCertData = string(data) + } + } + + clientTLS := mem.Etcd.ClientAutoTLS || + (mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile != "" && mem.Etcd.ClientKeyFile != "" && mem.Etcd.ClientTrustedCAFile != "") + if clientTLS { for _, cu := range mem.Etcd.ListenClientURLs { var u *url.URL u, err = url.Parse(cu) @@ -187,27 +276,29 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) } } - } - if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" { - for _, cu := range mem.Etcd.ListenPeerURLs { - var u *url.URL - u, err = url.Parse(cu) + clus.Members[i].ClientCertPath = mem.Etcd.ClientCertFile + if mem.Etcd.ClientCertFile != "" { + data, err := ioutil.ReadFile(mem.Etcd.ClientCertFile) if err != nil { - return nil, err - } - if u.Scheme != "https" { // TODO: support unix - return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientCertFile, err) } + clus.Members[i].ClientCertData = string(data) } - for _, cu := range mem.Etcd.AdvertisePeerURLs { - var u *url.URL - u, err = url.Parse(cu) + clus.Members[i].ClientKeyPath = mem.Etcd.ClientKeyFile + if mem.Etcd.ClientKeyFile != "" { + data, err := ioutil.ReadFile(mem.Etcd.ClientKeyFile) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientKeyFile, err) } - if u.Scheme != "https" { // TODO: support unix - return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + clus.Members[i].ClientCertData = string(data) + } + clus.Members[i].ClientTrustedCAPath = mem.Etcd.ClientTrustedCAFile + if mem.Etcd.ClientTrustedCAFile != "" { + data, err := ioutil.ReadFile(mem.Etcd.ClientTrustedCAFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientTrustedCAFile, err) } + clus.Members[i].ClientCertData = string(data) } } } @@ -233,6 +324,7 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { return nil, fmt.Errorf("StressType is unknown; got %q", v) } } + if clus.Tester.StressKeySuffixRangeTxn > 100 { return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn) } From cafa3b92170087331b97082e10a1143d10f50517 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 09:38:40 -0700 Subject: [PATCH 04/16] functional/agent: handle static TLS certs Signed-off-by: Gyuho Lee --- functional/agent/handler.go | 191 ++++++++++++++++++++++-------------- 1 file changed, 116 insertions(+), 75 deletions(-) diff --git a/functional/agent/handler.go b/functional/agent/handler.go index e80da14f4..539b1a6ee 100644 --- a/functional/agent/handler.go +++ b/functional/agent/handler.go @@ -36,11 +36,15 @@ import ( // return status error in response for wrong configuration/operation (e.g. start etcd twice) func (srv *Server) handleTesterRequest(req *rpcpb.Request) (resp *rpcpb.Response, err error) { defer func() { - if err == nil { + if err == nil && req != nil { srv.last = req.Operation srv.lg.Info("handler success", zap.String("operation", req.Operation.String())) } }() + if req != nil { + srv.Member = req.Member + srv.Tester = req.Tester + } switch req.Operation { case rpcpb.Operation_InitialStartEtcd: @@ -78,9 +82,6 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response, }, nil } - srv.Member = req.Member - srv.Tester = req.Tester - err := fileutil.TouchDirAll(srv.Member.BaseDir) if err != nil { return nil, err @@ -235,45 +236,124 @@ func (srv *Server) creatEtcdCmd() { srv.etcdCmd.Stderr = srv.etcdLogFile } +// if started with manual TLS, stores TLS assets +// from tester/client to disk before starting etcd process func (srv *Server) saveTLSAssets() error { - // if started with manual TLS, stores TLS assets - // from tester/client to disk before starting etcd process - // TODO: not implemented yet - if !srv.Member.Etcd.ClientAutoTLS { - if srv.Member.Etcd.ClientCertAuth { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientCertAuth is %v", srv.Member.Etcd.ClientCertAuth) + if srv.Member.PeerCertPath != "" { + if srv.Member.PeerCertData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.PeerCertPath) } - if srv.Member.Etcd.ClientCertFile != "" { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientCertFile is %q", srv.Member.Etcd.ClientCertFile) - } - if srv.Member.Etcd.ClientKeyFile != "" { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientKeyFile is %q", srv.Member.Etcd.ClientKeyFile) - } - if srv.Member.Etcd.ClientTrustedCAFile != "" { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientTrustedCAFile is %q", srv.Member.Etcd.ClientTrustedCAFile) + if err := ioutil.WriteFile(srv.Member.PeerCertPath, []byte(srv.Member.PeerCertData), 0644); err != nil { + return err } } - if !srv.Member.Etcd.PeerAutoTLS { - if srv.Member.Etcd.PeerClientCertAuth { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerClientCertAuth is %v", srv.Member.Etcd.PeerClientCertAuth) + if srv.Member.PeerKeyPath != "" { + if srv.Member.PeerKeyData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.PeerKeyPath) } - if srv.Member.Etcd.PeerCertFile != "" { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerCertFile is %q", srv.Member.Etcd.PeerCertFile) + if err := ioutil.WriteFile(srv.Member.PeerKeyPath, []byte(srv.Member.PeerKeyData), 0644); err != nil { + return err } - if srv.Member.Etcd.PeerKeyFile != "" { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerKeyFile is %q", srv.Member.Etcd.PeerKeyFile) + } + if srv.Member.PeerTrustedCAPath != "" { + if srv.Member.PeerTrustedCAData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.PeerTrustedCAPath) } - if srv.Member.Etcd.PeerTrustedCAFile != "" { - return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerTrustedCAFile is %q", srv.Member.Etcd.PeerTrustedCAFile) + if err := ioutil.WriteFile(srv.Member.PeerTrustedCAPath, []byte(srv.Member.PeerTrustedCAData), 0644); err != nil { + return err } } + if srv.Member.PeerCertPath != "" && + srv.Member.PeerKeyPath != "" && + srv.Member.PeerTrustedCAPath != "" { + srv.lg.Info( + "wrote", + zap.String("peer-cert", srv.Member.PeerCertPath), + zap.String("peer-key", srv.Member.PeerKeyPath), + zap.String("peer-trusted-ca", srv.Member.PeerTrustedCAPath), + ) + } + + if srv.Member.ClientCertPath != "" { + if srv.Member.ClientCertData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.ClientCertPath) + } + if err := ioutil.WriteFile(srv.Member.ClientCertPath, []byte(srv.Member.ClientCertData), 0644); err != nil { + return err + } + } + if srv.Member.ClientKeyPath != "" { + if srv.Member.ClientKeyData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.ClientKeyPath) + } + if err := ioutil.WriteFile(srv.Member.ClientKeyPath, []byte(srv.Member.ClientKeyData), 0644); err != nil { + return err + } + } + if srv.Member.ClientTrustedCAPath != "" { + if srv.Member.ClientTrustedCAData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.ClientTrustedCAPath) + } + if err := ioutil.WriteFile(srv.Member.ClientTrustedCAPath, []byte(srv.Member.ClientTrustedCAData), 0644); err != nil { + return err + } + } + if srv.Member.ClientCertPath != "" && + srv.Member.ClientKeyPath != "" && + srv.Member.ClientTrustedCAPath != "" { + srv.lg.Info( + "wrote", + zap.String("client-cert", srv.Member.ClientCertPath), + zap.String("client-key", srv.Member.ClientKeyPath), + zap.String("client-trusted-ca", srv.Member.ClientTrustedCAPath), + ) + } - // TODO return nil } func (srv *Server) loadAutoTLSAssets() error { - // if started with auto TLS, sends back TLS assets to tester/client + if srv.Member.Etcd.PeerAutoTLS { + // in case of slow disk + time.Sleep(time.Second) + + fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "peer") + + srv.lg.Info( + "loading client auto TLS assets", + zap.String("dir", fdir), + zap.String("endpoint", srv.EtcdClientEndpoint), + ) + + certPath := filepath.Join(fdir, "cert.pem") + if !fileutil.Exist(certPath) { + return fmt.Errorf("cannot find %q", certPath) + } + certData, err := ioutil.ReadFile(certPath) + if err != nil { + return fmt.Errorf("cannot read %q (%v)", certPath, err) + } + srv.Member.PeerCertData = string(certData) + + keyPath := filepath.Join(fdir, "key.pem") + if !fileutil.Exist(keyPath) { + return fmt.Errorf("cannot find %q", keyPath) + } + keyData, err := ioutil.ReadFile(keyPath) + if err != nil { + return fmt.Errorf("cannot read %q (%v)", keyPath, err) + } + srv.Member.PeerKeyData = string(keyData) + + srv.lg.Info( + "loaded peer auto TLS assets", + zap.String("peer-cert-path", certPath), + zap.Int("peer-cert-length", len(certData)), + zap.String("peer-key-path", keyPath), + zap.Int("peer-key-length", len(keyData)), + ) + } + if srv.Member.Etcd.ClientAutoTLS { // in case of slow disk time.Sleep(time.Second) @@ -314,46 +394,7 @@ func (srv *Server) loadAutoTLSAssets() error { zap.Int("peer-key-length", len(keyData)), ) } - if srv.Member.Etcd.ClientAutoTLS { - // in case of slow disk - time.Sleep(time.Second) - fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "peer") - - srv.lg.Info( - "loading client TLS assets", - zap.String("dir", fdir), - zap.String("endpoint", srv.EtcdClientEndpoint), - ) - - certPath := filepath.Join(fdir, "cert.pem") - if !fileutil.Exist(certPath) { - return fmt.Errorf("cannot find %q", certPath) - } - certData, err := ioutil.ReadFile(certPath) - if err != nil { - return fmt.Errorf("cannot read %q (%v)", certPath, err) - } - srv.Member.PeerCertData = string(certData) - - keyPath := filepath.Join(fdir, "key.pem") - if !fileutil.Exist(keyPath) { - return fmt.Errorf("cannot find %q", keyPath) - } - keyData, err := ioutil.ReadFile(keyPath) - if err != nil { - return fmt.Errorf("cannot read %q (%v)", keyPath, err) - } - srv.Member.PeerKeyData = string(keyData) - - srv.lg.Info( - "loaded peer TLS assets", - zap.String("peer-cert-path", certPath), - zap.Int("peer-cert-length", len(certData)), - zap.String("peer-key-path", keyPath), - zap.Int("peer-key-length", len(keyData)), - ) - } return nil } @@ -404,7 +445,7 @@ func (srv *Server) handleKillEtcd() (*rpcpb.Response, error) { return &rpcpb.Response{ Success: true, - Status: "successfully killed etcd!", + Status: "killed etcd", }, nil } @@ -443,7 +484,7 @@ func (srv *Server) handleFailArchive() (*rpcpb.Response, error) { return &rpcpb.Response{ Success: true, - Status: "successfully cleaned up etcd!", + Status: "cleaned up etcd", }, nil } @@ -475,7 +516,7 @@ func (srv *Server) handleDestroyEtcdAgent() (*rpcpb.Response, error) { return &rpcpb.Response{ Success: true, - Status: "successfully destroyed etcd and agent!", + Status: "destroyed etcd and agent", }, nil } @@ -488,7 +529,7 @@ func (srv *Server) handleBlackholePeerPortTxRx() (*rpcpb.Response, error) { } return &rpcpb.Response{ Success: true, - Status: "successfully blackholed peer port tx/rx!", + Status: "blackholed peer port tx/rx", }, nil } @@ -501,7 +542,7 @@ func (srv *Server) handleUnblackholePeerPortTxRx() (*rpcpb.Response, error) { } return &rpcpb.Response{ Success: true, - Status: "successfully unblackholed peer port tx/rx!", + Status: "unblackholed peer port tx/rx", }, nil } @@ -526,7 +567,7 @@ func (srv *Server) handleDelayPeerPortTxRx() (*rpcpb.Response, error) { return &rpcpb.Response{ Success: true, - Status: "successfully delay peer port tx/rx!", + Status: "delayed peer port tx/rx", }, nil } @@ -539,6 +580,6 @@ func (srv *Server) handleUndelayPeerPortTxRx() (*rpcpb.Response, error) { } return &rpcpb.Response{ Success: true, - Status: "successfully undelay peer port tx/rx!", + Status: "undelayed peer port tx/rx", }, nil } From 0e609154c4e1b1b902138e190ad05659b65b7a80 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 10:16:52 -0700 Subject: [PATCH 05/16] functional/tester: clean up "broadcastOperation" Signed-off-by: Gyuho Lee --- functional/tester/cluster.go | 48 +++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go index ef2d999c4..8b15c0193 100644 --- a/functional/tester/cluster.go +++ b/functional/tester/cluster.go @@ -24,6 +24,7 @@ import ( "net/url" "path/filepath" "strings" + "sync" "time" "github.com/coreos/etcd/functional/rpcpb" @@ -302,23 +303,40 @@ func (clus *Cluster) Restart() error { } func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error { + var wg sync.WaitGroup + wg.Add(len(clus.agentStreams)) + + errc := make(chan error, len(clus.agentStreams)) for i := range clus.agentStreams { - err := clus.sendOperation(i, op) - if err != nil { - if op == rpcpb.Operation_DestroyEtcdAgent && - strings.Contains(err.Error(), "rpc error: code = Unavailable desc = transport is closing") { - // agent server has already closed; - // so this error is expected - clus.lg.Info( - "successfully destroyed", - zap.String("member", clus.Members[i].EtcdClientEndpoint), - ) - continue - } - return err - } + go func(idx int, o rpcpb.Operation) { + defer wg.Done() + errc <- clus.sendOperation(idx, o) + }(i, op) } - return nil + wg.Wait() + close(errc) + + errs := []string{} + for err := range errc { + if err == nil { + continue + } + + if err != nil && + op == rpcpb.Operation_DestroyEtcdAgent && + strings.Contains(err.Error(), "rpc error: code = Unavailable desc = transport is closing") { + // agent server has already closed; + // so this error is expected + clus.lg.Info("successfully destroyed all") + continue + } + errs = append(errs, err.Error()) + } + + if len(errs) == 0 { + return nil + } + return errors.New(strings.Join(errs, ", ")) } func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error { From 054721166f17a2f35247500d7ec3ad97ba28852d Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 10:22:17 -0700 Subject: [PATCH 06/16] functional/tester: add "printReport" Signed-off-by: Gyuho Lee --- functional/tester/cluster_run.go | 3 +++ .../tester/{metrics.go => metrics_report.go} | 23 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) rename functional/tester/{metrics.go => metrics_report.go} (83%) diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index 8c8af10ad..05eaa2c9d 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -31,6 +31,8 @@ const compactQPS = 50000 // Run starts tester. func (clus *Cluster) Run() { + defer printReport() + if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil { clus.lg.Panic( "failed to create test data directory", @@ -123,6 +125,7 @@ func (clus *Cluster) doRound() error { for i, fa := range clus.failures { clus.cs = i + caseTotal[fa.Desc()]++ caseTotalCounter.WithLabelValues(fa.Desc()).Inc() caseNow := time.Now() diff --git a/functional/tester/metrics.go b/functional/tester/metrics_report.go similarity index 83% rename from functional/tester/metrics.go rename to functional/tester/metrics_report.go index f8546ddbd..c82e58f5b 100644 --- a/functional/tester/metrics.go +++ b/functional/tester/metrics_report.go @@ -14,9 +14,16 @@ package tester -import "github.com/prometheus/client_golang/prometheus" +import ( + "fmt" + "sort" + + "github.com/prometheus/client_golang/prometheus" +) var ( + caseTotal = make(map[string]int) + caseTotalCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", @@ -60,3 +67,17 @@ func init() { prometheus.MustRegister(roundTotalCounter) prometheus.MustRegister(roundFailedTotalCounter) } + +func printReport() { + rows := make([]string, 0, len(caseTotal)) + for k, v := range caseTotal { + rows = append(rows, fmt.Sprintf("%s: %d", k, v)) + } + sort.Strings(rows) + + println() + for _, row := range rows { + fmt.Println(row) + } + println() +} From 2922116be59f20af2edcbaf103592f370ef6f527 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 10:40:27 -0700 Subject: [PATCH 07/16] functional/tester: clean up runner logging Signed-off-by: Gyuho Lee --- functional/tester/cluster.go | 7 ++----- functional/tester/cluster_run.go | 21 +++++++++++++++---- functional/tester/stress.go | 31 +++++++++++++++++++++++++--- functional/tester/stress_key.go | 12 +++++++---- functional/tester/stress_lease.go | 33 +++++++++++++++++++++++------- functional/tester/stress_runner.go | 24 +++++++++++++++++++++- 6 files changed, 104 insertions(+), 24 deletions(-) diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go index 8b15c0193..763d0a8ec 100644 --- a/functional/tester/cluster.go +++ b/functional/tester/cluster.go @@ -245,11 +245,7 @@ func (clus *Cluster) updateStresserChecker() { clus.checker = newNoChecker() } - clus.lg.Info( - "updated stressers", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - ) + clus.lg.Info("updated stressers") } func (clus *Cluster) checkConsistency() (err error) { @@ -646,6 +642,7 @@ func (clus *Cluster) defrag() error { "defrag ALL PASS", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), ) return nil } diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index 05eaa2c9d..8fc4b36dc 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -51,6 +51,7 @@ func (clus *Cluster) Run() { "round FAIL", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Error(err), ) if clus.cleanup() != nil { @@ -74,6 +75,7 @@ func (clus *Cluster) Run() { "compact START", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Duration("timeout", timeout), ) if err := clus.compact(revToCompact, timeout); err != nil { @@ -81,6 +83,7 @@ func (clus *Cluster) Run() { "compact FAIL", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Error(err), ) if err = clus.cleanup(); err != nil { @@ -88,6 +91,7 @@ func (clus *Cluster) Run() { "cleanup FAIL", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Error(err), ) return @@ -107,6 +111,7 @@ func (clus *Cluster) Run() { "functional-tester PASS", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), ) } @@ -119,8 +124,9 @@ func (clus *Cluster) doRound() error { clus.lg.Info( "round START", zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Strings("failures", clus.failureStrings()), - zap.Int("total-failures", len(clus.failures)), ) for i, fa := range clus.failures { clus.cs = i @@ -133,8 +139,8 @@ func (clus *Cluster) doRound() error { "case START", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.String("desc", fa.Desc()), - zap.Int("total-failures", len(clus.failures)), ) clus.lg.Info("wait health before injecting failures") @@ -149,6 +155,7 @@ func (clus *Cluster) doRound() error { "stresser START", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.String("desc", fa.Desc()), ) if err := clus.stresser.Stress(); err != nil { @@ -161,6 +168,7 @@ func (clus *Cluster) doRound() error { "inject START", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.String("desc", fa.Desc()), ) if err := fa.Inject(clus); err != nil { @@ -174,6 +182,7 @@ func (clus *Cluster) doRound() error { "recover START", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.String("desc", fa.Desc()), ) if err := fa.Recover(clus); err != nil { @@ -218,8 +227,8 @@ func (clus *Cluster) doRound() error { "case PASS", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.String("desc", fa.Desc()), - zap.Int("total-failures", len(clus.failures)), zap.Duration("took", time.Since(caseNow)), ) } @@ -228,7 +237,7 @@ func (clus *Cluster) doRound() error { "round ALL PASS", zap.Int("round", clus.rd), zap.Strings("failures", clus.failureStrings()), - zap.Int("total-failures", len(clus.failures)), + zap.Int("case-total", len(clus.failures)), zap.Duration("took", time.Since(roundNow)), ) return nil @@ -291,6 +300,7 @@ func (clus *Cluster) failed() { "functional-tester FAIL", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), ) clus.DestroyEtcdAgents() os.Exit(2) @@ -310,6 +320,7 @@ func (clus *Cluster) cleanup() error { "closing stressers before archiving failure data", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), ) clus.stresser.Close() @@ -318,6 +329,7 @@ func (clus *Cluster) cleanup() error { "cleanup FAIL", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Error(err), ) return err @@ -327,6 +339,7 @@ func (clus *Cluster) cleanup() error { "restart FAIL", zap.Int("round", clus.rd), zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), zap.Error(err), ) return err diff --git a/functional/tester/stress.go b/functional/tester/stress.go index 105f09333..7671853b3 100644 --- a/functional/tester/stress.go +++ b/functional/tester/stress.go @@ -52,6 +52,7 @@ func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { // TODO: Too intensive stressing clients can panic etcd member with // 'out of memory' error. Put rate limits in server side. stressers[i] = &keyStresser{ + stype: rpcpb.StressType_KV, lg: clus.lg, m: m, keySize: int(clus.Tester.StressKeySize), @@ -65,6 +66,7 @@ func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { case "LEASE": stressers[i] = &leaseStresser{ + stype: rpcpb.StressType_LEASE, lg: clus.lg, m: m, numLeases: 10, // TODO: configurable @@ -84,6 +86,8 @@ func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { "--req-rate", fmt.Sprintf("%v", reqRate), } stressers[i] = newRunnerStresser( + rpcpb.StressType_ELECTION_RUNNER, + clus.lg, clus.Tester.RunnerExecPath, args, clus.rateLimiter, @@ -102,7 +106,14 @@ func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), } - stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, reqRate) + stressers[i] = newRunnerStresser( + rpcpb.StressType_WATCH_RUNNER, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + reqRate, + ) case "LOCK_RACER_RUNNER": reqRate := 100 @@ -114,7 +125,14 @@ func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { "--rounds=0", // runs forever "--req-rate", fmt.Sprintf("%v", reqRate), } - stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, reqRate) + stressers[i] = newRunnerStresser( + rpcpb.StressType_LOCK_RACER_RUNNER, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + reqRate, + ) case "LEASE_RUNNER": args := []string{ @@ -122,7 +140,14 @@ func newStresser(clus *Cluster, m *rpcpb.Member) Stresser { "--ttl=30", "--endpoints", m.EtcdClientEndpoint, } - stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, 0) + stressers[i] = newRunnerStresser( + rpcpb.StressType_LEASE_RUNNER, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + 0, + ) } } return &compositeStresser{stressers} diff --git a/functional/tester/stress_key.go b/functional/tester/stress_key.go index a589e283c..60c016e4d 100644 --- a/functional/tester/stress_key.go +++ b/functional/tester/stress_key.go @@ -34,7 +34,8 @@ import ( ) type keyStresser struct { - lg *zap.Logger + stype rpcpb.StressType + lg *zap.Logger m *rpcpb.Member @@ -102,7 +103,8 @@ func (s *keyStresser) Stress() error { } s.lg.Info( - "key stresser START", + "stress START", + zap.String("stress-type", s.stype.String()), zap.String("endpoint", s.m.EtcdClientEndpoint), ) return nil @@ -156,7 +158,8 @@ func (s *keyStresser) run() { return default: s.lg.Warn( - "key stresser exited with error", + "stress stopped", + zap.String("stress-type", s.stype.String()), zap.String("endpoint", s.m.EtcdClientEndpoint), zap.Error(err), ) @@ -188,7 +191,8 @@ func (s *keyStresser) Close() map[string]int { s.emu.Unlock() s.lg.Info( - "key stresser STOP", + "stress STOP", + zap.String("stress-type", s.stype.String()), zap.String("endpoint", s.m.EtcdClientEndpoint), ) return ess diff --git a/functional/tester/stress_lease.go b/functional/tester/stress_lease.go index 9aeab557a..c3797f47f 100644 --- a/functional/tester/stress_lease.go +++ b/functional/tester/stress_lease.go @@ -38,7 +38,8 @@ const ( ) type leaseStresser struct { - lg *zap.Logger + stype rpcpb.StressType + lg *zap.Logger m *rpcpb.Member cli *clientv3.Client @@ -121,7 +122,8 @@ func (ls *leaseStresser) setupOnce() error { func (ls *leaseStresser) Stress() error { ls.lg.Info( - "lease stresser START", + "stress START", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), ) @@ -159,22 +161,26 @@ func (ls *leaseStresser) run() { } ls.lg.Debug( - "lease stresser is creating leases", + "stress creating leases", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), ) ls.createLeases() ls.lg.Debug( - "lease stresser created leases", + "stress created leases", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), ) ls.lg.Debug( - "lease stresser is dropped leases", + "stress dropped leases", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), ) ls.randomlyDropLeases() ls.lg.Debug( - "lease stresser dropped leases", + "stress dropped leases", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), ) } @@ -243,6 +249,7 @@ func (ls *leaseStresser) createLeaseWithKeys(ttl int64) (int64, error) { if err != nil { ls.lg.Debug( "createLease failed", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.Error(err), ) @@ -251,6 +258,7 @@ func (ls *leaseStresser) createLeaseWithKeys(ttl int64) (int64, error) { ls.lg.Debug( "createLease created lease", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), ) @@ -284,6 +292,7 @@ func (ls *leaseStresser) randomlyDropLeases() { } ls.lg.Debug( "randomlyDropLease dropped a lease", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), ) @@ -313,6 +322,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { case <-ls.ctx.Done(): ls.lg.Debug( "keepLeaseAlive context canceled", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Error(ls.ctx.Err()), @@ -327,6 +337,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { ls.aliveLeases.remove(leaseID) ls.lg.Debug( "keepLeaseAlive lease has not been renewed, dropped it", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), ) @@ -337,6 +348,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { if err != nil { ls.lg.Debug( "keepLeaseAlive lease creates stream error", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Error(err), @@ -350,6 +362,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { if err != nil { ls.lg.Debug( "keepLeaseAlive failed to receive lease keepalive response", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Error(err), @@ -359,6 +372,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { ls.lg.Debug( "keepLeaseAlive waiting on lease stream", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), ) @@ -367,6 +381,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { if respRC == nil { ls.lg.Debug( "keepLeaseAlive received nil lease keepalive response", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), ) @@ -378,6 +393,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { if respRC.TTL <= 0 { ls.lg.Debug( "keepLeaseAlive stream received lease keepalive response TTL <= 0", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Int64("ttl", respRC.TTL), @@ -388,6 +404,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { // renew lease timestamp only if lease is present ls.lg.Debug( "keepLeaseAlive renewed a lease", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), ) @@ -440,6 +457,7 @@ func (ls *leaseStresser) randomlyDropLease(leaseID int64) (bool, error) { ls.lg.Debug( "randomlyDropLease error", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Error(ls.ctx.Err()), @@ -457,7 +475,8 @@ func (ls *leaseStresser) Close() map[string]int { ls.aliveWg.Wait() ls.cli.Close() ls.lg.Info( - "lease stresser STOP", + "stress STOP", + zap.String("stress-type", ls.stype.String()), zap.String("endpoint", ls.m.EtcdClientEndpoint), ) return nil diff --git a/functional/tester/stress_runner.go b/functional/tester/stress_runner.go index 8165db223..d52b94cb8 100644 --- a/functional/tester/stress_runner.go +++ b/functional/tester/stress_runner.go @@ -20,10 +20,16 @@ import ( "os/exec" "syscall" + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" "golang.org/x/time/rate" ) type runnerStresser struct { + stype rpcpb.StressType + lg *zap.Logger + cmd *exec.Cmd cmdStr string args []string @@ -34,9 +40,17 @@ type runnerStresser struct { donec chan struct{} } -func newRunnerStresser(cmdStr string, args []string, rl *rate.Limiter, reqRate int) *runnerStresser { +func newRunnerStresser( + stype rpcpb.StressType, + lg *zap.Logger, + cmdStr string, + args []string, + rl *rate.Limiter, + reqRate int, +) *runnerStresser { rl.SetLimit(rl.Limit() - rate.Limit(reqRate)) return &runnerStresser{ + stype: stype, cmdStr: cmdStr, args: args, rl: rl, @@ -71,6 +85,10 @@ func (rs *runnerStresser) setupOnce() (err error) { } func (rs *runnerStresser) Stress() (err error) { + rs.lg.Info( + "stress START", + zap.String("stress-type", rs.stype.String()), + ) if err = rs.setupOnce(); err != nil { return err } @@ -78,6 +96,10 @@ func (rs *runnerStresser) Stress() (err error) { } func (rs *runnerStresser) Pause() map[string]int { + rs.lg.Info( + "stress STOP", + zap.String("stress-type", rs.stype.String()), + ) syscall.Kill(rs.cmd.Process.Pid, syscall.SIGSTOP) return nil } From 7facfde6fda2dd6e656e50cbeeda3555d0cb8478 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 10:52:19 -0700 Subject: [PATCH 08/16] functional/tester: handle "process already finished" Signed-off-by: Gyuho Lee --- functional/tester/cluster.go | 29 +++++++++++++++++++++-------- functional/tester/cluster_run.go | 9 ++++----- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go index 763d0a8ec..fb081a351 100644 --- a/functional/tester/cluster.go +++ b/functional/tester/cluster.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "io" "io/ioutil" "math/rand" "net/http" @@ -318,15 +319,27 @@ func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error { continue } - if err != nil && - op == rpcpb.Operation_DestroyEtcdAgent && - strings.Contains(err.Error(), "rpc error: code = Unavailable desc = transport is closing") { - // agent server has already closed; - // so this error is expected - clus.lg.Info("successfully destroyed all") - continue + if err != nil { + destroyed := false + if op == rpcpb.Operation_DestroyEtcdAgent { + if err == io.EOF { + destroyed = true + } + if strings.Contains(err.Error(), + "rpc error: code = Unavailable desc = transport is closing") { + // agent server has already closed; + // so this error is expected + destroyed = true + } + if strings.Contains(err.Error(), + "desc = os: process already finished") { + destroyed = true + } + } + if !destroyed { + errs = append(errs, err.Error()) + } } - errs = append(errs, err.Error()) } if len(errs) == 0 { diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index 8fc4b36dc..5cd601ac1 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -292,10 +292,6 @@ func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) { } func (clus *Cluster) failed() { - if !clus.Tester.ExitOnFailure { - return - } - clus.lg.Info( "functional-tester FAIL", zap.Int("round", clus.rd), @@ -303,11 +299,14 @@ func (clus *Cluster) failed() { zap.Int("case-total", len(clus.failures)), ) clus.DestroyEtcdAgents() + os.Exit(2) } func (clus *Cluster) cleanup() error { - defer clus.failed() + if clus.Tester.ExitOnFailure { + defer clus.failed() + } roundFailedTotalCounter.Inc() desc := "compact/defrag" From ecadb0fdf7d6db08cc7629da100a98d2229919c8 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 11:28:11 -0700 Subject: [PATCH 09/16] functional/tester: improve logging Signed-off-by: Gyuho Lee --- functional/tester/cluster_run.go | 26 ++++++++++++++++++++++---- functional/tester/failure.go | 4 ---- functional/tester/stress_key.go | 4 +++- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index 5cd601ac1..c29b81027 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -152,7 +152,7 @@ func (clus *Cluster) doRound() error { fcase := fa.FailureCase() if fcase != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { clus.lg.Info( - "stresser START", + "stress START", zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.Int("case-total", len(clus.failures)), @@ -190,7 +190,13 @@ func (clus *Cluster) doRound() error { } if stressStarted { - clus.lg.Info("stresser PAUSE") + clus.lg.Info( + "stress PAUSE", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), + zap.String("desc", fa.Desc()), + ) ems := clus.stresser.Pause() if fcase == rpcpb.FailureCase_NO_FAIL_WITH_STRESS && len(ems) > 0 { ess := make([]string, 0, len(ems)) @@ -213,12 +219,24 @@ func (clus *Cluster) doRound() error { } } - clus.lg.Info("health check START") + clus.lg.Info( + "health check START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), + zap.String("desc", fa.Desc()), + ) if err := clus.WaitHealth(); err != nil { return fmt.Errorf("wait full health error: %v", err) } - clus.lg.Info("consistency check START") + clus.lg.Info( + "consistency check START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.failures)), + zap.String("desc", fa.Desc()), + ) if err := clus.checkConsistency(); err != nil { return fmt.Errorf("consistency check error (%v)", err) } diff --git a/functional/tester/failure.go b/functional/tester/failure.go index cb1cef930..855cb7f4c 100644 --- a/functional/tester/failure.go +++ b/functional/tester/failure.go @@ -251,8 +251,6 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error { now := time.Now() clus.lg.Info( "trigger snapshot START", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), zap.String("desc", f.Desc()), zap.Int64("etcd-snapshot-count", snapshotCount), ) @@ -283,8 +281,6 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error { if diff > snapshotCount { clus.lg.Info( "trigger snapshot PASS", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), zap.Int("retries", i), zap.String("desc", f.Desc()), zap.Int64("committed-entries", diff), diff --git a/functional/tester/stress_key.go b/functional/tester/stress_key.go index 60c016e4d..509748b8a 100644 --- a/functional/tester/stress_key.go +++ b/functional/tester/stress_key.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "math/rand" + "reflect" "sync" "sync/atomic" "time" @@ -158,9 +159,10 @@ func (s *keyStresser) run() { return default: s.lg.Warn( - "stress stopped", + "stress run exiting", zap.String("stress-type", s.stype.String()), zap.String("endpoint", s.m.EtcdClientEndpoint), + zap.String("error-type", reflect.TypeOf(err).String()), zap.Error(err), ) return From 68adc6e300a3b6fc964f8705054cac4d593843bf Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 13:39:42 -0700 Subject: [PATCH 10/16] functional/rpcpb: document FailureCase Signed-off-by: Gyuho Lee --- functional/rpcpb/rpc.pb.go | 264 ++++++++++++++++++++++++++++++++----- functional/rpcpb/rpc.proto | 229 +++++++++++++++++++++++++++++++- 2 files changed, 458 insertions(+), 35 deletions(-) diff --git a/functional/rpcpb/rpc.pb.go b/functional/rpcpb/rpc.pb.go index a16522bd7..f6b4bc01b 100644 --- a/functional/rpcpb/rpc.pb.go +++ b/functional/rpcpb/rpc.pb.go @@ -52,11 +52,17 @@ const ( // thus need to archive etcd data directories. Operation_FailArchive Operation = 4 // DestroyEtcdAgent destroys etcd process, etcd data, and agent server. - Operation_DestroyEtcdAgent Operation = 5 - Operation_BlackholePeerPortTxRx Operation = 100 + Operation_DestroyEtcdAgent Operation = 5 + // BlackholePeerPortTxRx drops all outgoing/incoming packets from/to the + // peer port on target member's peer port. + Operation_BlackholePeerPortTxRx Operation = 100 + // UnblackholePeerPortTxRx removes outgoing/incoming packet dropping. Operation_UnblackholePeerPortTxRx Operation = 101 - Operation_DelayPeerPortTxRx Operation = 102 - Operation_UndelayPeerPortTxRx Operation = 103 + // DelayPeerPortTxRx delays all outgoing/incoming packets from/to the + // peer port on target member's peer port. + Operation_DelayPeerPortTxRx Operation = 102 + // UndelayPeerPortTxRx removes all outgoing/incoming delays. + Operation_UndelayPeerPortTxRx Operation = 103 ) var Operation_name = map[int32]string{ @@ -89,41 +95,235 @@ func (x Operation) String() string { } func (Operation) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, []int{0} } +// FailureCase defines various system faults in distributed systems, +// in order to verify correct behavior of etcd servers and clients. type FailureCase int32 const ( - FailureCase_KILL_ONE_FOLLOWER FailureCase = 0 - FailureCase_KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 1 - FailureCase_KILL_LEADER FailureCase = 2 - FailureCase_KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 3 - FailureCase_KILL_QUORUM FailureCase = 4 - FailureCase_KILL_ALL FailureCase = 5 - FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 100 - FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 101 - FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER FailureCase = 102 - FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 103 - FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM FailureCase = 104 - FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL FailureCase = 105 - FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 200 - FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 201 - FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 202 + // KILL_ONE_FOLLOWER stops a randomly chosen follower (non-leader) + // but does not delete its data directories on disk for next restart. + // It waits "failure-delay-ms" before recovering this failure. + // The expected behavior is that the follower comes back online + // and rejoins the cluster, and then each member continues to process + // client requests ('Put' request that requires Raft consensus). + FailureCase_KILL_ONE_FOLLOWER FailureCase = 0 + // KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen + // follower but does not delete its data directories on disk for next + // restart. And waits until most up-to-date node (leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that the follower comes back online and + // rejoins the cluster, and then active leader sends snapshot + // to the follower to force it to follow the leader's log. + // As always, after recovery, each member must be able to process + // client requests. + FailureCase_KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 1 + // KILL_LEADER stops the active leader node but does not delete its + // data directories on disk for next restart. Then it waits + // "failure-delay-ms" before recovering this failure, in order to + // trigger election timeouts. + // The expected behavior is that a new leader gets elected, and the + // old leader comes back online and rejoins the cluster as a follower. + // As always, after recovery, each member must be able to process + // client requests. + FailureCase_KILL_LEADER FailureCase = 2 + // KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node + // but does not delete its data directories on disk for next restart. + // And waits until most up-to-date node ("new" leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that cluster elects a new leader, and the + // old leader comes back online and rejoins the cluster as a follower. + // And it receives the snapshot from the new leader to overwrite its + // store. As always, after recovery, each member must be able to + // process client requests. + FailureCase_KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 3 + // KILL_QUORUM stops majority number of nodes to make the whole cluster + // inoperable but does not delete data directories on stopped nodes + // for next restart. And it waits "failure-delay-ms" before recovering + // this failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. + FailureCase_KILL_QUORUM FailureCase = 4 + // KILL_ALL stops the whole cluster but does not delete data directories + // on disk for next restart. And it waits "failure-delay-ms" before + // recovering this failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. + FailureCase_KILL_ALL FailureCase = 5 + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER drops all outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader), and waits for "failure-delay-ms" until recovery. + // The expected behavior is that once dropping operation is undone, + // each member must be able to process client requests. + FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 100 + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT drops + // all outgoing/incoming packets from/to the peer port on a randomly + // chosen follower (non-leader), and waits for most up-to-date node + // (leader) applies the snapshot count of entries since the blackhole + // operation. + // The expected behavior is that once packet drop operation is undone, + // the slow follower tries to catch up, possibly receiving the snapshot + // from the active leader. As always, after recovery, each member must + // be able to process client requests. + FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 101 + // BLACKHOLE_PEER_PORT_TX_RX_LEADER drops all outgoing/incoming packets + // from/to the peer port on the active leader (isolated), and waits for + // "failure-delay-ms" until recovery, in order to trigger election timeout. + // The expected behavior is that after election timeout, a new leader gets + // elected, and once dropping operation is undone, the old leader comes + // back and rejoins the cluster as a follower. As always, after recovery, + // each member must be able to process client requests. + FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER FailureCase = 102 + // BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT drops all + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (leader) applies the snapshot + // count of entries since the blackhole operation. + // The expected behavior is that cluster elects a new leader, and once + // dropping operation is undone, the old leader comes back and rejoins + // the cluster as a follower. The slow follower tries to catch up, likely + // receiving the snapshot from the new active leader. As always, after + // recovery, each member must be able to process client requests. + FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 103 + // BLACKHOLE_PEER_PORT_TX_RX_QUORUM drops all outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, thus losing its + // leader and cluster being inoperable. And it waits for "failure-delay-ms" + // until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM FailureCase = 104 + // BLACKHOLE_PEER_PORT_TX_RX_ALL drops all outgoing/incoming packets + // from/to the peer ports on all nodes, thus making cluster totally + // inoperable. It waits for "failure-delay-ms" until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL FailureCase = 105 + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming packets + // from/to the peer port on a randomly chosen follower (non-leader). + // It waits for "failure-delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // the follower comes back and tries to catch up with latest changes from + // cluster. And as always, after recovery, each member must be able to + // process client requests. + FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 200 + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader) with a randomized time duration (thus isolated). It waits + // for "failure-delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // each member must be able to process client requests. + FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 201 + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader), and waits for most up-to-date node (leader) + // applies the snapshot count of entries since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up possibly receiving snapshot + // from the active leader. As always, after recovery, each member must be + // able to process client requests. + FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 202 + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader) with a randomized time duration, and waits for + // most up-to-date node (leader) applies the snapshot count of entries + // since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up, possibly receiving a + // snapshot from the active leader. As always, after recovery, each member + // must be able to process client requests. FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 203 - FailureCase_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 204 - FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 205 - FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 206 - FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 207 - FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM FailureCase = 208 - FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM FailureCase = 209 - FailureCase_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 210 - FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 211 - // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period - // while stressers are still sending requests. + // DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets from/to + // the peer port on the active leader. And waits for "failure-delay-ms" + // until recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. + FailureCase_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 204 + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets + // from/to the peer port on the active leader with a randomized time + // duration. And waits for "failure-delay-ms" until recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. + FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 205 + // DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (current or new leader) applies the + // snapshot count of entries since the delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. + FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 206 + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // with a randomized time duration. And it waits for most up-to-date node + // (current or new leader) applies the snapshot count of entries since the + // delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. + FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 207 + // DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets from/to + // the peer ports on majority nodes of cluster. And it waits for + // "failure-delay-ms" until recovery, likely to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM FailureCase = 208 + // RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, with randomized + // time durations. And it waits for "failure-delay-ms" until recovery, + // likely to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM FailureCase = 209 + // DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets from/to the + // peer ports on all nodes. And it waits for "failure-delay-ms" until + // recovery, likely to trigger election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. + FailureCase_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 210 + // RANDOM_DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets + // from/to the peer ports on all nodes, with randomized time durations. + // And it waits for "failure-delay-ms" until recovery, likely to trigger + // election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. + FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 211 + // NO_FAIL_WITH_STRESS runs no-op failure injection that does not do + // anything against cluster for "failure-delay-ms" duration, while + // stressers are still sending requests. FailureCase_NO_FAIL_WITH_STRESS FailureCase = 300 // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection - // with all stressers stopped. + // that does not do anything against cluster for "failure-delay-ms" + // duration, while all stressers are stopped. FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS FailureCase = 301 - FailureCase_FAILPOINTS FailureCase = 400 - FailureCase_EXTERNAL FailureCase = 500 + // FAILPOINTS injects failpoints to etcd server runtime, triggering panics + // in critical code paths. + FailureCase_FAILPOINTS FailureCase = 400 + // EXTERNAL runs external failure injection scripts. + FailureCase_EXTERNAL FailureCase = 500 ) var FailureCase_name = map[int32]string{ diff --git a/functional/rpcpb/rpc.proto b/functional/rpcpb/rpc.proto index 9d59befbd..69cbb2c05 100644 --- a/functional/rpcpb/rpc.proto +++ b/functional/rpcpb/rpc.proto @@ -28,9 +28,15 @@ enum Operation { // DestroyEtcdAgent destroys etcd process, etcd data, and agent server. DestroyEtcdAgent = 5; + // BlackholePeerPortTxRx drops all outgoing/incoming packets from/to the + // peer port on target member's peer port. BlackholePeerPortTxRx = 100; + // UnblackholePeerPortTxRx removes outgoing/incoming packet dropping. UnblackholePeerPortTxRx = 101; + // DelayPeerPortTxRx delays all outgoing/incoming packets from/to the + // peer port on target member's peer port. DelayPeerPortTxRx = 102; + // UndelayPeerPortTxRx removes all outgoing/incoming delays. UndelayPeerPortTxRx = 103; } @@ -122,42 +128,259 @@ message Member { string PeerTrustedCAPath = 506 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-path\""]; } +// FailureCase defines various system faults in distributed systems, +// in order to verify correct behavior of etcd servers and clients. enum FailureCase { + // KILL_ONE_FOLLOWER stops a randomly chosen follower (non-leader) + // but does not delete its data directories on disk for next restart. + // It waits "failure-delay-ms" before recovering this failure. + // The expected behavior is that the follower comes back online + // and rejoins the cluster, and then each member continues to process + // client requests ('Put' request that requires Raft consensus). KILL_ONE_FOLLOWER = 0; + + // KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen + // follower but does not delete its data directories on disk for next + // restart. And waits until most up-to-date node (leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that the follower comes back online and + // rejoins the cluster, and then active leader sends snapshot + // to the follower to force it to follow the leader's log. + // As always, after recovery, each member must be able to process + // client requests. KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 1; + + // KILL_LEADER stops the active leader node but does not delete its + // data directories on disk for next restart. Then it waits + // "failure-delay-ms" before recovering this failure, in order to + // trigger election timeouts. + // The expected behavior is that a new leader gets elected, and the + // old leader comes back online and rejoins the cluster as a follower. + // As always, after recovery, each member must be able to process + // client requests. KILL_LEADER = 2; + + // KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node + // but does not delete its data directories on disk for next restart. + // And waits until most up-to-date node ("new" leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that cluster elects a new leader, and the + // old leader comes back online and rejoins the cluster as a follower. + // And it receives the snapshot from the new leader to overwrite its + // store. As always, after recovery, each member must be able to + // process client requests. KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT = 3; + + // KILL_QUORUM stops majority number of nodes to make the whole cluster + // inoperable but does not delete data directories on stopped nodes + // for next restart. And it waits "failure-delay-ms" before recovering + // this failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. KILL_QUORUM = 4; + + // KILL_ALL stops the whole cluster but does not delete data directories + // on disk for next restart. And it waits "failure-delay-ms" before + // recovering this failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. KILL_ALL = 5; + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER drops all outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader), and waits for "failure-delay-ms" until recovery. + // The expected behavior is that once dropping operation is undone, + // each member must be able to process client requests. BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER = 100; + + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT drops + // all outgoing/incoming packets from/to the peer port on a randomly + // chosen follower (non-leader), and waits for most up-to-date node + // (leader) applies the snapshot count of entries since the blackhole + // operation. + // The expected behavior is that once packet drop operation is undone, + // the slow follower tries to catch up, possibly receiving the snapshot + // from the active leader. As always, after recovery, each member must + // be able to process client requests. BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 101; + + // BLACKHOLE_PEER_PORT_TX_RX_LEADER drops all outgoing/incoming packets + // from/to the peer port on the active leader (isolated), and waits for + // "failure-delay-ms" until recovery, in order to trigger election timeout. + // The expected behavior is that after election timeout, a new leader gets + // elected, and once dropping operation is undone, the old leader comes + // back and rejoins the cluster as a follower. As always, after recovery, + // each member must be able to process client requests. BLACKHOLE_PEER_PORT_TX_RX_LEADER = 102; + + // BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT drops all + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (leader) applies the snapshot + // count of entries since the blackhole operation. + // The expected behavior is that cluster elects a new leader, and once + // dropping operation is undone, the old leader comes back and rejoins + // the cluster as a follower. The slow follower tries to catch up, likely + // receiving the snapshot from the new active leader. As always, after + // recovery, each member must be able to process client requests. BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT = 103; + + // BLACKHOLE_PEER_PORT_TX_RX_QUORUM drops all outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, thus losing its + // leader and cluster being inoperable. And it waits for "failure-delay-ms" + // until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. BLACKHOLE_PEER_PORT_TX_RX_QUORUM = 104; + + // BLACKHOLE_PEER_PORT_TX_RX_ALL drops all outgoing/incoming packets + // from/to the peer ports on all nodes, thus making cluster totally + // inoperable. It waits for "failure-delay-ms" until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. BLACKHOLE_PEER_PORT_TX_RX_ALL = 105; + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming packets + // from/to the peer port on a randomly chosen follower (non-leader). + // It waits for "failure-delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // the follower comes back and tries to catch up with latest changes from + // cluster. And as always, after recovery, each member must be able to + // process client requests. DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER = 200; + + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader) with a randomized time duration (thus isolated). It waits + // for "failure-delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // each member must be able to process client requests. RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER = 201; + + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader), and waits for most up-to-date node (leader) + // applies the snapshot count of entries since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up possibly receiving snapshot + // from the active leader. As always, after recovery, each member must be + // able to process client requests. DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 202; + + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader) with a randomized time duration, and waits for + // most up-to-date node (leader) applies the snapshot count of entries + // since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up, possibly receiving a + // snapshot from the active leader. As always, after recovery, each member + // must be able to process client requests. RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 203; + + // DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets from/to + // the peer port on the active leader. And waits for "failure-delay-ms" + // until recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. DELAY_PEER_PORT_TX_RX_LEADER = 204; + + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets + // from/to the peer port on the active leader with a randomized time + // duration. And waits for "failure-delay-ms" until recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. RANDOM_DELAY_PEER_PORT_TX_RX_LEADER = 205; + + // DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (current or new leader) applies the + // snapshot count of entries since the delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT = 206; + + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // with a randomized time duration. And it waits for most up-to-date node + // (current or new leader) applies the snapshot count of entries since the + // delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT = 207; + + // DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets from/to + // the peer ports on majority nodes of cluster. And it waits for + // "failure-delay-ms" until recovery, likely to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. DELAY_PEER_PORT_TX_RX_QUORUM = 208; + + // RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, with randomized + // time durations. And it waits for "failure-delay-ms" until recovery, + // likely to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM = 209; + + // DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets from/to the + // peer ports on all nodes. And it waits for "failure-delay-ms" until + // recovery, likely to trigger election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. DELAY_PEER_PORT_TX_RX_ALL = 210; + + // RANDOM_DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets + // from/to the peer ports on all nodes, with randomized time durations. + // And it waits for "failure-delay-ms" until recovery, likely to trigger + // election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. RANDOM_DELAY_PEER_PORT_TX_RX_ALL = 211; - // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period - // while stressers are still sending requests. + // NO_FAIL_WITH_STRESS runs no-op failure injection that does not do + // anything against cluster for "failure-delay-ms" duration, while + // stressers are still sending requests. NO_FAIL_WITH_STRESS = 300; + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection - // with all stressers stopped. + // that does not do anything against cluster for "failure-delay-ms" + // duration, while all stressers are stopped. NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS = 301; + // FAILPOINTS injects failpoints to etcd server runtime, triggering panics + // in critical code paths. FAILPOINTS = 400; + + // EXTERNAL runs external failure injection scripts. EXTERNAL = 500; } From d8a2d3a209333abe82a297f582f00a87eafd6ba8 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 13:40:00 -0700 Subject: [PATCH 11/16] functional/tester: delay after injecting "kill" to trigger election Signed-off-by: Gyuho Lee --- functional/tester/cluster.go | 84 ++++++++++++------- functional/tester/cluster_read_config.go | 4 + functional/tester/failure.go | 3 - functional/tester/failure_case_delay.go | 4 +- functional/tester/failure_case_kill.go | 40 ++++++--- .../tester/failure_case_network_delay.go | 12 --- 6 files changed, 90 insertions(+), 57 deletions(-) diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go index fb081a351..d2e3325c8 100644 --- a/functional/tester/cluster.go +++ b/functional/tester/cluster.go @@ -143,69 +143,97 @@ func (clus *Cluster) updateFailures() { for _, cs := range clus.Tester.FailureCases { switch cs { case "KILL_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureKillOneFollower()) + clus.failures = append(clus.failures, + newFailureKillOneFollower(clus)) case "KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureKillOneFollowerUntilTriggerSnapshot()) + clus.failures = append(clus.failures, + newFailureKillOneFollowerUntilTriggerSnapshot(clus)) case "KILL_LEADER": - clus.failures = append(clus.failures, newFailureKillLeader()) + clus.failures = append(clus.failures, + newFailureKillLeader(clus)) case "KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureKillLeaderUntilTriggerSnapshot()) + clus.failures = append(clus.failures, + newFailureKillLeaderUntilTriggerSnapshot(clus)) case "KILL_QUORUM": - clus.failures = append(clus.failures, newFailureKillQuorum()) + clus.failures = append(clus.failures, + newFailureKillQuorum(clus)) case "KILL_ALL": - clus.failures = append(clus.failures, newFailureKillAll()) + clus.failures = append(clus.failures, + newFailureKillAll(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus)) + clus.failures = append(clus.failures, + newFailureBlackholePeerPortTxRxOneFollower(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollowerUntilTriggerSnapshot()) + clus.failures = append(clus.failures, + newFailureBlackholePeerPortTxRxOneFollowerUntilTriggerSnapshot()) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus)) + clus.failures = append(clus.failures, + newFailureBlackholePeerPortTxRxLeader(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeaderUntilTriggerSnapshot()) + clus.failures = append(clus.failures, + newFailureBlackholePeerPortTxRxLeaderUntilTriggerSnapshot()) case "BLACKHOLE_PEER_PORT_TX_RX_QUORUM": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxQuorum(clus)) + clus.failures = append(clus.failures, + newFailureBlackholePeerPortTxRxQuorum(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ALL": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus)) + clus.failures = append(clus.failures, + newFailureBlackholePeerPortTxRxAll(clus)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus, false)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxOneFollower(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus, true)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxOneFollower(clus, true)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus, false)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus, true)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus, true)) case "DELAY_PEER_PORT_TX_RX_LEADER": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus, false)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxLeader(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus, true)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxLeader(clus, true)) case "DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus, false)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus, true)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus, true)) case "DELAY_PEER_PORT_TX_RX_QUORUM": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxQuorum(clus, false)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxQuorum(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxQuorum(clus, true)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxQuorum(clus, true)) case "DELAY_PEER_PORT_TX_RX_ALL": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus, false)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxAll(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_ALL": - clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus, true)) + clus.failures = append(clus.failures, + newFailureDelayPeerPortTxRxAll(clus, true)) case "NO_FAIL_WITH_STRESS": - clus.failures = append(clus.failures, newFailureNoFailWithStress(clus)) + clus.failures = append(clus.failures, + newFailureNoFailWithStress(clus)) case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": - clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness(clus)) + clus.failures = append(clus.failures, + newFailureNoFailWithNoStressForLiveness(clus)) case "EXTERNAL": - clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath)) + clus.failures = append(clus.failures, + newFailureExternal(clus.Tester.ExternalExecPath)) case "FAILPOINTS": fpFailures, fperr := failpointFailures(clus) if len(fpFailures) == 0 { clus.lg.Info("no failpoints found!", zap.Error(fperr)) } - clus.failures = append(clus.failures, fpFailures...) + clus.failures = append(clus.failures, + fpFailures...) } } } diff --git a/functional/tester/cluster_read_config.go b/functional/tester/cluster_read_config.go index 1361f9e6e..c45f12169 100644 --- a/functional/tester/cluster_read_config.go +++ b/functional/tester/cluster_read_config.go @@ -40,6 +40,10 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { return nil, err } + if len(clus.Members) < 3 { + return nil, fmt.Errorf("len(clus.Members) expects at least 3, got %d", len(clus.Members)) + } + for i, mem := range clus.Members { if mem.BaseDir == "" { return nil, fmt.Errorf("BaseDir cannot be empty (got %q)", mem.BaseDir) diff --git a/functional/tester/failure.go b/functional/tester/failure.go index 855cb7f4c..2f84959a7 100644 --- a/functional/tester/failure.go +++ b/functional/tester/failure.go @@ -242,9 +242,6 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error { if err := f.Failure.Inject(clus); err != nil { return err } - if len(clus.Members) < 3 { - return nil - } snapshotCount := clus.Members[0].Etcd.SnapshotCount diff --git a/functional/tester/failure_case_delay.go b/functional/tester/failure_case_delay.go index 882a61888..a153b5649 100644 --- a/functional/tester/failure_case_delay.go +++ b/functional/tester/failure_case_delay.go @@ -31,9 +31,9 @@ func (f *failureDelay) Inject(clus *Cluster) error { } if f.delayDuration > 0 { clus.lg.Info( - "sleeping in failureDelay", + "wait after inject", zap.Duration("delay", f.delayDuration), - zap.String("case", f.Failure.Desc()), + zap.String("desc", f.Failure.Desc()), ) time.Sleep(f.delayDuration) } diff --git a/functional/tester/failure_case_kill.go b/functional/tester/failure_case_kill.go index 8b139b8d5..54e717e29 100644 --- a/functional/tester/failure_case_kill.go +++ b/functional/tester/failure_case_kill.go @@ -24,50 +24,66 @@ func recoverKill(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_RestartEtcd) } -func newFailureKillOneFollower() Failure { +func newFailureKillOneFollower(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER, injectMember: injectKill, recoverMember: recoverKill, } - return &failureFollower{ff, -1, -1} + f := &failureFollower{ff, -1, -1} + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } } -func newFailureKillLeader() Failure { +func newFailureKillLeader(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_KILL_LEADER, injectMember: injectKill, recoverMember: recoverKill, } - return &failureLeader{ff, -1, -1} + f := &failureLeader{ff, -1, -1} + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } } -func newFailureKillQuorum() Failure { - return &failureQuorum{ +func newFailureKillQuorum(clus *Cluster) Failure { + f := &failureQuorum{ failureCase: rpcpb.FailureCase_KILL_QUORUM, injectMember: injectKill, recoverMember: recoverKill, } + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } } -func newFailureKillAll() Failure { - return &failureAll{ +func newFailureKillAll(clus *Cluster) Failure { + f := &failureAll{ failureCase: rpcpb.FailureCase_KILL_ALL, injectMember: injectKill, recoverMember: recoverKill, } + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } } -func newFailureKillOneFollowerUntilTriggerSnapshot() Failure { +func newFailureKillOneFollowerUntilTriggerSnapshot(clus *Cluster) Failure { return &failureUntilSnapshot{ failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, - Failure: newFailureKillOneFollower(), + Failure: newFailureKillOneFollower(clus), } } -func newFailureKillLeaderUntilTriggerSnapshot() Failure { +func newFailureKillLeaderUntilTriggerSnapshot(clus *Cluster) Failure { return &failureUntilSnapshot{ failureCase: rpcpb.FailureCase_KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT, - Failure: newFailureKillLeader(), + Failure: newFailureKillLeader(clus), } } diff --git a/functional/tester/failure_case_network_delay.go b/functional/tester/failure_case_network_delay.go index e8b04157a..89cb2ea65 100644 --- a/functional/tester/failure_case_network_delay.go +++ b/functional/tester/failure_case_network_delay.go @@ -52,13 +52,11 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster, random bool) Failure injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { clus.UpdateDelayLatencyMs() ff.failureCase = rpcpb.FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER } - f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, @@ -72,13 +70,11 @@ func newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus *Cluster, r injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { clus.UpdateDelayLatencyMs() ff.failureCase = rpcpb.FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT } - f := &failureFollower{ff, -1, -1} return &failureUntilSnapshot{ failureCase: ff.failureCase, @@ -92,13 +88,11 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster, random bool) Failure { injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { clus.UpdateDelayLatencyMs() ff.failureCase = rpcpb.FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER } - f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, @@ -112,13 +106,11 @@ func newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus *Cluster, random injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { clus.UpdateDelayLatencyMs() ff.failureCase = rpcpb.FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT } - f := &failureLeader{ff, -1, -1} return &failureUntilSnapshot{ failureCase: ff.failureCase, @@ -132,13 +124,11 @@ func newFailureDelayPeerPortTxRxQuorum(clus *Cluster, random bool) Failure { injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { clus.UpdateDelayLatencyMs() f.failureCase = rpcpb.FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM } - return &failureDelay{ Failure: f, delayDuration: clus.GetFailureDelayDuration(), @@ -151,13 +141,11 @@ func newFailureDelayPeerPortTxRxAll(clus *Cluster, random bool) Failure { injectMember: injectDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx, } - clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { clus.UpdateDelayLatencyMs() f.failureCase = rpcpb.FailureCase_RANDOM_DELAY_PEER_PORT_TX_RX_ALL } - return &failureDelay{ Failure: f, delayDuration: clus.GetFailureDelayDuration(), From f53c153fa9648c08efe5fd8fd3173b5924586d25 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Sun, 8 Apr 2018 14:18:09 -0700 Subject: [PATCH 12/16] pkg/stringutil: add tests For later "runner" cleanup Signed-off-by: Gyuho Lee --- pkg/stringutil/doc.go | 16 +++++++++++ pkg/stringutil/{stringutil.go => rand.go} | 35 +++++++++++------------ pkg/stringutil/rand_test.go | 30 +++++++++++++++++++ 3 files changed, 63 insertions(+), 18 deletions(-) create mode 100644 pkg/stringutil/doc.go rename pkg/stringutil/{stringutil.go => rand.go} (64%) create mode 100644 pkg/stringutil/rand_test.go diff --git a/pkg/stringutil/doc.go b/pkg/stringutil/doc.go new file mode 100644 index 000000000..16d2b3834 --- /dev/null +++ b/pkg/stringutil/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package stringutil exports string utility functions. +package stringutil diff --git a/pkg/stringutil/stringutil.go b/pkg/stringutil/rand.go similarity index 64% rename from pkg/stringutil/stringutil.go rename to pkg/stringutil/rand.go index 44d637c20..a15b0de0c 100644 --- a/pkg/stringutil/stringutil.go +++ b/pkg/stringutil/rand.go @@ -1,4 +1,4 @@ -// Copyright 2016 The etcd Authors +// Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,41 +12,40 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package stringutil exports string utility functions. package stringutil -import "math/rand" - -const ( - chars = "abcdefghijklmnopqrstuvwxyz0123456789" +import ( + "math/rand" + "time" ) // UniqueStrings returns a slice of randomly generated unique strings. -func UniqueStrings(maxlen uint, n int) []string { - exist := make(map[string]bool) - ss := make([]string, 0) - +func UniqueStrings(slen uint, n int) (ss []string) { + exist := make(map[string]struct{}) + ss = make([]string, 0, n) for len(ss) < n { - s := randomString(maxlen) - if !exist[s] { - exist[s] = true + s := randString(slen) + if _, ok := exist[s]; !ok { ss = append(ss, s) + exist[s] = struct{}{} } } - return ss } // RandomStrings returns a slice of randomly generated strings. -func RandomStrings(maxlen uint, n int) []string { - ss := make([]string, 0) +func RandomStrings(slen uint, n int) (ss []string) { + ss = make([]string, 0, n) for i := 0; i < n; i++ { - ss = append(ss, randomString(maxlen)) + ss = append(ss, randString(slen)) } return ss } -func randomString(l uint) string { +const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randString(l uint) string { + rand.Seed(time.Now().UnixNano()) s := make([]byte, l) for i := 0; i < int(l); i++ { s[i] = chars[rand.Intn(len(chars))] diff --git a/pkg/stringutil/rand_test.go b/pkg/stringutil/rand_test.go new file mode 100644 index 000000000..1b4a9dfd2 --- /dev/null +++ b/pkg/stringutil/rand_test.go @@ -0,0 +1,30 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stringutil + +import ( + "fmt" + "testing" +) + +func TestUniqueStrings(t *testing.T) { + ss := UniqueStrings(10, 50) + for i := 1; i < len(ss); i++ { + if ss[i-1] == ss[i] { + t.Fatalf("ss[i-1] %q == ss[i] %q", ss[i-1], ss[i]) + } + } + fmt.Println(ss) +} From b8bf42cc5af891680b281f35fd7385a9e67e0e2c Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Sun, 8 Apr 2018 14:24:59 -0700 Subject: [PATCH 13/16] pkg/fileutil: test "Exist" on directory Signed-off-by: Gyuho Lee --- pkg/fileutil/doc.go | 16 ++++++++++++++++ pkg/fileutil/fileutil.go | 2 +- pkg/fileutil/fileutil_test.go | 13 +++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 pkg/fileutil/doc.go diff --git a/pkg/fileutil/doc.go b/pkg/fileutil/doc.go new file mode 100644 index 000000000..69dde5a7d --- /dev/null +++ b/pkg/fileutil/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fileutil implements utility functions related to files and paths. +package fileutil diff --git a/pkg/fileutil/fileutil.go b/pkg/fileutil/fileutil.go index fce5126c6..468d78f65 100644 --- a/pkg/fileutil/fileutil.go +++ b/pkg/fileutil/fileutil.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package fileutil implements utility functions related to files and paths. package fileutil import ( @@ -93,6 +92,7 @@ func CreateDirAll(dir string) error { return err } +// Exist returns true if a file or directory exists. func Exist(name string) bool { _, err := os.Stat(name) return err == nil diff --git a/pkg/fileutil/fileutil_test.go b/pkg/fileutil/fileutil_test.go index cde2f516a..e9497e9b1 100644 --- a/pkg/fileutil/fileutil_test.go +++ b/pkg/fileutil/fileutil_test.go @@ -15,8 +15,10 @@ package fileutil import ( + "fmt" "io" "io/ioutil" + "math/rand" "os" "os/user" "path/filepath" @@ -24,6 +26,7 @@ import ( "runtime" "strings" "testing" + "time" ) func TestIsDirWriteable(t *testing.T) { @@ -104,6 +107,16 @@ func TestCreateDirAll(t *testing.T) { } func TestExist(t *testing.T) { + fdir := filepath.Join(os.TempDir(), fmt.Sprint(time.Now().UnixNano()+rand.Int63n(1000))) + os.RemoveAll(fdir) + if err := os.Mkdir(fdir, 0666); err != nil { + t.Skip(err) + } + defer os.RemoveAll(fdir) + if !Exist(fdir) { + t.Fatalf("expected Exist true, got %v", Exist(fdir)) + } + f, err := ioutil.TempFile(os.TempDir(), "fileutil") if err != nil { t.Fatal(err) From cd4580b4a912e052eb3264f3e1053255870c42eb Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Sun, 8 Apr 2018 14:57:25 -0700 Subject: [PATCH 14/16] functional: move "etcd-test-proxy" Signed-off-by: Gyuho Lee --- Dockerfile-functional | 4 +- Makefile | 24 ++- functional/Procfile-proxy | 14 ++ functional/README.md | 189 +++++++++++++++++- functional/build | 11 +- .../cmd/etcd-proxy}/main.go | 14 +- tools/etcd-test-proxy/Procfile | 14 -- tools/etcd-test-proxy/README.md | 184 ----------------- 8 files changed, 229 insertions(+), 225 deletions(-) create mode 100644 functional/Procfile-proxy rename {tools/etcd-test-proxy => functional/cmd/etcd-proxy}/main.go (94%) delete mode 100644 tools/etcd-test-proxy/Procfile delete mode 100644 tools/etcd-test-proxy/README.md diff --git a/Dockerfile-functional b/Dockerfile-functional index c5751a1a9..fbcc34ec2 100644 --- a/Dockerfile-functional +++ b/Dockerfile-functional @@ -34,9 +34,9 @@ RUN go get -v github.com/coreos/gofail \ && cp ./bin/etcd /bin/etcd-failpoints \ && ./functional/build \ && cp ./bin/etcd-agent /bin/etcd-agent \ - && cp ./bin/etcd-tester /bin/etcd-tester \ + && cp ./bin/etcd-proxy /bin/etcd-proxy \ && cp ./bin/etcd-runner /bin/etcd-runner \ + && cp ./bin/etcd-tester /bin/etcd-tester \ && go build -v -o /bin/benchmark ./tools/benchmark \ - && go build -v -o /bin/etcd-test-proxy ./tools/etcd-test-proxy \ && popd \ && rm -rf ${GOPATH}/src/github.com/coreos/etcd \ No newline at end of file diff --git a/Makefile b/Makefile index d89fbbdcd..ae6cb4aab 100644 --- a/Makefile +++ b/Makefile @@ -469,18 +469,20 @@ docker-dns-srv-test-certs-wildcard-run: # Example: -# make build-etcd-test-proxy - -build-etcd-test-proxy: - go build -v -o ./bin/etcd-test-proxy ./tools/etcd-test-proxy - - - -# Example: +# make build-functional # make build-docker-functional # make push-docker-functional # make pull-docker-functional +build-functional: + $(info GO_VERSION: $(GO_VERSION)) + $(info ETCD_VERSION: $(ETCD_VERSION)) + ./functional/build + ./bin/etcd-agent -help || true && \ + ./bin/etcd-proxy -help || true && \ + ./bin/etcd-runner --help || true && \ + ./bin/etcd-tester -help || true + build-docker-functional: $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) @@ -498,10 +500,10 @@ build-docker-functional: ./bin/etcd-failpoints --version && \ ETCDCTL_API=3 ./bin/etcdctl version && \ ./bin/etcd-agent -help || true && \ - ./bin/etcd-tester -help || true && \ + ./bin/etcd-proxy -help || true && \ ./bin/etcd-runner --help || true && \ - ./bin/benchmark --help || true && \ - ./bin/etcd-test-proxy -help || true" + ./bin/etcd-tester -help || true && \ + ./bin/benchmark --help || true" push-docker-functional: $(info GO_VERSION: $(GO_VERSION)) diff --git a/functional/Procfile-proxy b/functional/Procfile-proxy new file mode 100644 index 000000000..66730ee77 --- /dev/null +++ b/functional/Procfile-proxy @@ -0,0 +1,14 @@ +s1: bin/etcd --name s1 --data-dir /tmp/etcd-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s1-client-proxy: bin/etcd-proxy --from localhost:13790 --to localhost:1379 --http-port 1378 +s1-peer-proxy: bin/etcd-proxy --from localhost:13800 --to localhost:1380 --http-port 1381 + +s2: bin/etcd --name s2 --data-dir /tmp/etcd-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s2-client-proxy: bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 +s2-peer-proxy: bin/etcd-proxy --from localhost:23800 --to localhost:2380 --http-port 2381 + +s3: bin/etcd --name s3 --data-dir /tmp/etcd-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s3-client-proxy: bin/etcd-proxy --from localhost:33790 --to localhost:3379 --http-port 3378 +s3-client-proxy: bin/etcd-proxy --from localhost:33800 --to localhost:3380 --http-port 3381 diff --git a/functional/README.md b/functional/README.md index 1c143097e..3571ce88c 100644 --- a/functional/README.md +++ b/functional/README.md @@ -1,4 +1,4 @@ -# etcd Functional Testing +## etcd Functional Testing `functional` verifies the correct behavior of etcd under various system and network malfunctions. It sets up an etcd cluster under high pressure loads and continuously injects failures into the cluster. Then it expects the etcd cluster to recover within a few seconds. This has been extremely helpful to find critical bugs. @@ -13,7 +13,7 @@ PASSES=functional ./test ### Run with Docker ```bash -pushd ../.. +pushd .. make build-docker-functional popd ``` @@ -29,3 +29,188 @@ And run [example scripts](./scripts). # to run only 1 tester round ./scripts/docker-local-tester.sh ``` + +## etcd Proxy + +Proxy layer that simulates various network conditions. + +Test locally + +```bash +$ ./build +$ ./bin/etcd + +$ make build-functional + +$ ./bin/etcd-proxy --help +$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar +``` + +Proxy overhead per request is under 500μs + +```bash +$ go build -v -o ./bin/benchmark ./tools/benchmark + +$ ./bin/benchmark \ + --endpoints localhost:2379 \ + --conns 5 \ + --clients 15 \ + put \ + --key-size 48 \ + --val-size 50000 \ + --total 10000 + +< tcp://localhost:2379] + +$ ETCDCTL_API=3 ./bin/etcdctl \ + --endpoints localhost:23790 \ + put foo bar +# Error: context deadline exceeded + +$ curl -L http://localhost:2378/pause-tx -X DELETE +# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379] +``` + +Drop client packets + +```bash +$ curl -L http://localhost:2378/blackhole-tx -X PUT +# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379] + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar +# Error: context deadline exceeded + +$ curl -L http://localhost:2378/blackhole-tx -X DELETE +# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379] +``` + +Trigger leader election + +```bash +$ ./build +$ make build-functional + +$ rm -rf /tmp/etcd-proxy-data.s* +$ goreman -f ./functional/Procfile-proxy start + +$ ETCDCTL_API=3 ./bin/etcdctl \ + --endpoints localhost:13790,localhost:23790,localhost:33790 \ + member list + +# isolate s1 when s1 is the current leader +$ curl -L http://localhost:1381/blackhole-tx -X PUT +$ curl -L http://localhost:1381/blackhole-rx -X PUT +# s1 becomes follower after election timeout +``` diff --git a/functional/build b/functional/build index 60daadcf4..47420b769 100755 --- a/functional/build +++ b/functional/build @@ -1,10 +1,11 @@ #!/usr/bin/env bash if ! [[ "$0" =~ "functional/build" ]]; then - echo "must be run from repository root" - exit 255 + echo "must be run from repository root" + exit 255 fi -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./functional/cmd/etcd-agent -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./functional/cmd/etcd-tester -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./functional/cmd/etcd-runner +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-agent ./functional/cmd/etcd-agent +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-proxy ./functional/cmd/etcd-proxy +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-runner ./functional/cmd/etcd-runner +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-tester ./functional/cmd/etcd-tester diff --git a/tools/etcd-test-proxy/main.go b/functional/cmd/etcd-proxy/main.go similarity index 94% rename from tools/etcd-test-proxy/main.go rename to functional/cmd/etcd-proxy/main.go index 61915bfed..24c275e8e 100644 --- a/tools/etcd-test-proxy/main.go +++ b/functional/cmd/etcd-proxy/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// etcd-test-proxy is a proxy layer that simulates various network conditions. +// etcd-proxy is a proxy layer that simulates various network conditions. package main import ( @@ -40,13 +40,13 @@ func main() { // TODO: support TLS flag.StringVar(&from, "from", "localhost:23790", "Address URL to proxy from.") flag.StringVar(&to, "to", "localhost:2379", "Address URL to forward.") - flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-test-proxy API.") + flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-proxy API.") flag.BoolVar(&verbose, "verbose", false, "'true' to run proxy in verbose mode.") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %q:\n", os.Args[0]) fmt.Fprintln(os.Stderr, ` -etcd-test-proxy simulates various network conditions for etcd testing purposes. +etcd-proxy simulates various network conditions for etcd testing purposes. See README.md for more examples. Example: @@ -55,12 +55,12 @@ Example: $ ./build $ ./bin/etcd -# build etcd-test-proxy -$ make build-etcd-test-proxy +# build etcd-proxy +$ make build-etcd-proxy # to test etcd with proxy layer -$ ./bin/etcd-test-proxy --help -$ ./bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose +$ ./bin/etcd-proxy --help +$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose $ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar $ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar`) diff --git a/tools/etcd-test-proxy/Procfile b/tools/etcd-test-proxy/Procfile deleted file mode 100644 index e820c1ea6..000000000 --- a/tools/etcd-test-proxy/Procfile +++ /dev/null @@ -1,14 +0,0 @@ -s1: bin/etcd --name s1 --data-dir /tmp/etcd-test-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new - -s1-client-proxy: bin/etcd-test-proxy --from localhost:13790 --to localhost:1379 --http-port 1378 -s1-peer-proxy: bin/etcd-test-proxy --from localhost:13800 --to localhost:1380 --http-port 1381 - -s2: bin/etcd --name s2 --data-dir /tmp/etcd-test-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new - -s2-client-proxy: bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 -s2-peer-proxy: bin/etcd-test-proxy --from localhost:23800 --to localhost:2380 --http-port 2381 - -s3: bin/etcd --name s3 --data-dir /tmp/etcd-test-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new - -s3-client-proxy: bin/etcd-test-proxy --from localhost:33790 --to localhost:3379 --http-port 3378 -s3-client-proxy: bin/etcd-test-proxy --from localhost:33800 --to localhost:3380 --http-port 3381 diff --git a/tools/etcd-test-proxy/README.md b/tools/etcd-test-proxy/README.md deleted file mode 100644 index bbb2763c4..000000000 --- a/tools/etcd-test-proxy/README.md +++ /dev/null @@ -1,184 +0,0 @@ -#### etcd-test-proxy - -Proxy layer that simulates various network conditions. - -Test locally - -```bash -$ ./build -$ ./bin/etcd - -$ make build-etcd-test-proxy - -$ ./bin/etcd-test-proxy --help -$ ./bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose - -$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar -$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar -``` - -Proxy overhead per request is under 500μs - -```bash -$ go build -v -o ./bin/benchmark ./tools/benchmark - -$ ./bin/benchmark \ - --endpoints localhost:2379 \ - --conns 5 \ - --clients 15 \ - put \ - --key-size 48 \ - --val-size 50000 \ - --total 10000 - -< tcp://localhost:2379] - -$ ETCDCTL_API=3 ./bin/etcdctl \ - --endpoints localhost:23790 \ - put foo bar -# Error: context deadline exceeded - -$ curl -L http://localhost:2378/pause-tx -X DELETE -# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379] -``` - -Drop client packets - -```bash -$ curl -L http://localhost:2378/blackhole-tx -X PUT -# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379] - -$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar -# Error: context deadline exceeded - -$ curl -L http://localhost:2378/blackhole-tx -X DELETE -# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379] -``` - -Trigger leader election - -```bash -$ ./build -$ make build-etcd-test-proxy - -$ rm -rf /tmp/etcd-test-proxy-data.s* -$ goreman -f ./tools/etcd-test-proxy/Procfile start - -$ ETCDCTL_API=3 ./bin/etcdctl \ - --endpoints localhost:13790,localhost:23790,localhost:33790 \ - member list - -# isolate s1 when s1 is the current leader -$ curl -L http://localhost:1381/blackhole-tx -X PUT -$ curl -L http://localhost:1381/blackhole-rx -X PUT -# s1 becomes follower after election timeout -``` From bc1fd9257936184bb638cb2e9be002708a2aa4c9 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 16:18:27 -0700 Subject: [PATCH 15/16] functional: rename to "SIGTERM/SIGQUIT*" Signed-off-by: Gyuho Lee --- functional.yaml | 12 +- functional/agent/handler.go | 107 +- functional/agent/server.go | 2 +- functional/cmd/etcd-tester/main.go | 4 +- functional/rpcpb/rpc.pb.go | 2809 +++++++++-------- functional/rpcpb/rpc.proto | 322 +- functional/tester/cluster.go | 102 +- functional/tester/cluster_run.go | 6 +- functional/tester/cluster_test.go | 12 +- functional/tester/failure_case_external.go | 2 +- functional/tester/failure_case_failpoints.go | 2 +- functional/tester/failure_case_kill.go | 89 - .../tester/failure_case_network_blackhole.go | 44 +- .../tester/failure_case_network_delay.go | 44 +- functional/tester/failure_case_no_fail.go | 4 +- functional/tester/failure_case_sigterm.go | 89 + 16 files changed, 1845 insertions(+), 1805 deletions(-) delete mode 100644 functional/tester/failure_case_kill.go create mode 100644 functional/tester/failure_case_sigterm.go diff --git a/functional.yaml b/functional.yaml index c451592e2..ea69f2443 100644 --- a/functional.yaml +++ b/functional.yaml @@ -122,12 +122,12 @@ tester-config: failure-delay-ms: 7000 failure-shuffle: true failure-cases: - - KILL_ONE_FOLLOWER - - KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT - - KILL_LEADER - - KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT - - KILL_QUORUM - - KILL_ALL + - SIGTERM_ONE_FOLLOWER + - SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT + - SIGTERM_LEADER + - SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT + - SIGTERM_QUORUM + - SIGTERM_ALL - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT - BLACKHOLE_PEER_PORT_TX_RX_LEADER diff --git a/functional/agent/handler.go b/functional/agent/handler.go index 539b1a6ee..5834c602a 100644 --- a/functional/agent/handler.go +++ b/functional/agent/handler.go @@ -47,25 +47,29 @@ func (srv *Server) handleTesterRequest(req *rpcpb.Request) (resp *rpcpb.Response } switch req.Operation { - case rpcpb.Operation_InitialStartEtcd: - return srv.handleInitialStartEtcd(req) - case rpcpb.Operation_RestartEtcd: - return srv.handleRestartEtcd() - case rpcpb.Operation_KillEtcd: - return srv.handleKillEtcd() - case rpcpb.Operation_FailArchive: - return srv.handleFailArchive() - case rpcpb.Operation_DestroyEtcdAgent: - return srv.handleDestroyEtcdAgent() + case rpcpb.Operation_INITIAL_START_ETCD: + return srv.handle_INITIAL_START_ETCD(req) + case rpcpb.Operation_RESTART_ETCD: + return srv.handle_RESTART_ETCD() - case rpcpb.Operation_BlackholePeerPortTxRx: - return srv.handleBlackholePeerPortTxRx() - case rpcpb.Operation_UnblackholePeerPortTxRx: - return srv.handleUnblackholePeerPortTxRx() - case rpcpb.Operation_DelayPeerPortTxRx: - return srv.handleDelayPeerPortTxRx() - case rpcpb.Operation_UndelayPeerPortTxRx: - return srv.handleUndelayPeerPortTxRx() + case rpcpb.Operation_SIGTERM_ETCD: + return srv.handle_SIGTERM_ETCD() + case rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA: + return srv.handle_SIGQUIT_ETCD_AND_REMOVE_DATA() + + case rpcpb.Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA: + return srv.handle_SIGQUIT_ETCD_AND_ARCHIVE_DATA() + case rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT: + return srv.handle_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() + + case rpcpb.Operation_BLACKHOLE_PEER_PORT_TX_RX: + return srv.handle_BLACKHOLE_PEER_PORT_TX_RX() + case rpcpb.Operation_UNBLACKHOLE_PEER_PORT_TX_RX: + return srv.handle_UNBLACKHOLE_PEER_PORT_TX_RX() + case rpcpb.Operation_DELAY_PEER_PORT_TX_RX: + return srv.handle_DELAY_PEER_PORT_TX_RX() + case rpcpb.Operation_UNDELAY_PEER_PORT_TX_RX: + return srv.handle_UNDELAY_PEER_PORT_TX_RX() default: msg := fmt.Sprintf("operation not found (%v)", req.Operation) @@ -73,11 +77,11 @@ func (srv *Server) handleTesterRequest(req *rpcpb.Request) (resp *rpcpb.Response } } -func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response, error) { - if srv.last != rpcpb.Operation_NotStarted { +func (srv *Server) handle_INITIAL_START_ETCD(req *rpcpb.Request) (*rpcpb.Response, error) { + if srv.last != rpcpb.Operation_NOT_STARTED { return &rpcpb.Response{ Success: false, - Status: fmt.Sprintf("%q is not valid; last server operation was %q", rpcpb.Operation_InitialStartEtcd.String(), srv.last.String()), + Status: fmt.Sprintf("%q is not valid; last server operation was %q", rpcpb.Operation_INITIAL_START_ETCD.String(), srv.last.String()), Member: req.Member, }, nil } @@ -403,10 +407,17 @@ func (srv *Server) startEtcdCmd() error { return srv.etcdCmd.Start() } -func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) { +func (srv *Server) handle_RESTART_ETCD() (*rpcpb.Response, error) { + var err error + if !fileutil.Exist(srv.Member.BaseDir) { + err = fileutil.TouchDirAll(srv.Member.BaseDir) + if err != nil { + return nil, err + } + } + srv.creatEtcdCmd() - var err error if err = srv.saveTLSAssets(); err != nil { return nil, err } @@ -434,7 +445,7 @@ func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) { }, nil } -func (srv *Server) handleKillEtcd() (*rpcpb.Response, error) { +func (srv *Server) handle_SIGTERM_ETCD() (*rpcpb.Response, error) { srv.stopProxy() err := stopWithSig(srv.etcdCmd, syscall.SIGTERM) @@ -449,7 +460,28 @@ func (srv *Server) handleKillEtcd() (*rpcpb.Response, error) { }, nil } -func (srv *Server) handleFailArchive() (*rpcpb.Response, error) { +func (srv *Server) handle_SIGQUIT_ETCD_AND_REMOVE_DATA() (*rpcpb.Response, error) { + srv.stopProxy() + + err := stopWithSig(srv.etcdCmd, syscall.SIGQUIT) + if err != nil { + return nil, err + } + srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGQUIT.String())) + + err = os.RemoveAll(srv.Member.BaseDir) + if err != nil { + return nil, err + } + srv.lg.Info("removed base directory", zap.String("dir", srv.Member.BaseDir)) + + return &rpcpb.Response{ + Success: true, + Status: "killed etcd and removed base directory", + }, nil +} + +func (srv *Server) handle_SIGQUIT_ETCD_AND_ARCHIVE_DATA() (*rpcpb.Response, error) { srv.stopProxy() // exit with stackstrace @@ -489,12 +521,14 @@ func (srv *Server) handleFailArchive() (*rpcpb.Response, error) { } // stop proxy, etcd, delete data directory -func (srv *Server) handleDestroyEtcdAgent() (*rpcpb.Response, error) { - err := stopWithSig(srv.etcdCmd, syscall.SIGTERM) +func (srv *Server) handle_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() (*rpcpb.Response, error) { + srv.stopProxy() + + err := stopWithSig(srv.etcdCmd, syscall.SIGQUIT) if err != nil { return nil, err } - srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGTERM.String())) + srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGQUIT.String())) err = os.RemoveAll(srv.Member.BaseDir) if err != nil { @@ -505,22 +539,13 @@ func (srv *Server) handleDestroyEtcdAgent() (*rpcpb.Response, error) { // stop agent server srv.Stop() - for port, px := range srv.advertiseClientPortToProxy { - err := px.Close() - srv.lg.Info("closed proxy", zap.Int("client-port", port), zap.Error(err)) - } - for port, px := range srv.advertisePeerPortToProxy { - err := px.Close() - srv.lg.Info("closed proxy", zap.Int("peer-port", port), zap.Error(err)) - } - return &rpcpb.Response{ Success: true, Status: "destroyed etcd and agent", }, nil } -func (srv *Server) handleBlackholePeerPortTxRx() (*rpcpb.Response, error) { +func (srv *Server) handle_BLACKHOLE_PEER_PORT_TX_RX() (*rpcpb.Response, error) { for port, px := range srv.advertisePeerPortToProxy { srv.lg.Info("blackholing", zap.Int("peer-port", port)) px.BlackholeTx() @@ -533,7 +558,7 @@ func (srv *Server) handleBlackholePeerPortTxRx() (*rpcpb.Response, error) { }, nil } -func (srv *Server) handleUnblackholePeerPortTxRx() (*rpcpb.Response, error) { +func (srv *Server) handle_UNBLACKHOLE_PEER_PORT_TX_RX() (*rpcpb.Response, error) { for port, px := range srv.advertisePeerPortToProxy { srv.lg.Info("unblackholing", zap.Int("peer-port", port)) px.UnblackholeTx() @@ -546,7 +571,7 @@ func (srv *Server) handleUnblackholePeerPortTxRx() (*rpcpb.Response, error) { }, nil } -func (srv *Server) handleDelayPeerPortTxRx() (*rpcpb.Response, error) { +func (srv *Server) handle_DELAY_PEER_PORT_TX_RX() (*rpcpb.Response, error) { lat := time.Duration(srv.Tester.UpdatedDelayLatencyMs) * time.Millisecond rv := time.Duration(srv.Tester.DelayLatencyMsRv) * time.Millisecond @@ -571,7 +596,7 @@ func (srv *Server) handleDelayPeerPortTxRx() (*rpcpb.Response, error) { }, nil } -func (srv *Server) handleUndelayPeerPortTxRx() (*rpcpb.Response, error) { +func (srv *Server) handle_UNDELAY_PEER_PORT_TX_RX() (*rpcpb.Response, error) { for port, px := range srv.advertisePeerPortToProxy { srv.lg.Info("undelaying", zap.Int("peer-port", port)) px.UndelayTx() diff --git a/functional/agent/server.go b/functional/agent/server.go index 1c077138d..d6313d955 100644 --- a/functional/agent/server.go +++ b/functional/agent/server.go @@ -64,7 +64,7 @@ func NewServer( lg: lg, network: network, address: address, - last: rpcpb.Operation_NotStarted, + last: rpcpb.Operation_NOT_STARTED, advertiseClientPortToProxy: make(map[int]proxy.Server), advertisePeerPortToProxy: make(map[int]proxy.Server), } diff --git a/functional/cmd/etcd-tester/main.go b/functional/cmd/etcd-tester/main.go index f7966425f..a0e6eda68 100644 --- a/functional/cmd/etcd-tester/main.go +++ b/functional/cmd/etcd-tester/main.go @@ -44,11 +44,11 @@ func main() { logger.Fatal("failed to create a cluster", zap.Error(err)) } - err = clus.Bootstrap() + err = clus.Send_INITIAL_START_ETCD() if err != nil { logger.Fatal("Bootstrap failed", zap.Error(err)) } - defer clus.DestroyEtcdAgents() + defer clus.Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() logger.Info("wait health after bootstrap") err = clus.WaitHealth() diff --git a/functional/rpcpb/rpc.pb.go b/functional/rpcpb/rpc.pb.go index f6b4bc01b..4d2fe17ac 100644 --- a/functional/rpcpb/rpc.pb.go +++ b/functional/rpcpb/rpc.pb.go @@ -8,11 +8,11 @@ rpcpb/rpc.proto It has these top-level messages: - Etcd - Member - Tester Request Response + Member + Tester + Etcd */ package rpcpb @@ -40,54 +40,61 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Operation int32 const ( - Operation_NotStarted Operation = 0 - // InitialStartEtcd is only called to start etcd very first time. - Operation_InitialStartEtcd Operation = 1 - // RestartEtcd is sent to restart killed etcd. - Operation_RestartEtcd Operation = 2 - // KillEtcd pauses etcd process while keeping data directories + // NOT_STARTED is the agent status before etcd first start. + Operation_NOT_STARTED Operation = 0 + // INITIAL_START_ETCD is only called to start etcd, the very first time. + Operation_INITIAL_START_ETCD Operation = 10 + // RESTART_ETCD is sent to restart killed etcd. + Operation_RESTART_ETCD Operation = 11 + // SIGTERM_ETCD pauses etcd process while keeping data directories // and previous etcd configurations. - Operation_KillEtcd Operation = 3 - // FailArchive is sent when consistency check failed, + Operation_SIGTERM_ETCD Operation = 20 + // SIGQUIT_ETCD_AND_REMOVE_DATA kills etcd process and removes all data + // directories to simulate destroying the whole machine. + Operation_SIGQUIT_ETCD_AND_REMOVE_DATA Operation = 21 + // SIGQUIT_ETCD_AND_ARCHIVE_DATA is sent when consistency check failed, // thus need to archive etcd data directories. - Operation_FailArchive Operation = 4 - // DestroyEtcdAgent destroys etcd process, etcd data, and agent server. - Operation_DestroyEtcdAgent Operation = 5 - // BlackholePeerPortTxRx drops all outgoing/incoming packets from/to the - // peer port on target member's peer port. - Operation_BlackholePeerPortTxRx Operation = 100 - // UnblackholePeerPortTxRx removes outgoing/incoming packet dropping. - Operation_UnblackholePeerPortTxRx Operation = 101 - // DelayPeerPortTxRx delays all outgoing/incoming packets from/to the - // peer port on target member's peer port. - Operation_DelayPeerPortTxRx Operation = 102 - // UndelayPeerPortTxRx removes all outgoing/incoming delays. - Operation_UndelayPeerPortTxRx Operation = 103 + Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA Operation = 30 + // SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT destroys etcd process, + // etcd data, and agent server. + Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT Operation = 31 + // BLACKHOLE_PEER_PORT_TX_RX drops all outgoing/incoming packets from/to + // the peer port on target member's peer port. + Operation_BLACKHOLE_PEER_PORT_TX_RX Operation = 100 + // UNBLACKHOLE_PEER_PORT_TX_RX removes outgoing/incoming packet dropping. + Operation_UNBLACKHOLE_PEER_PORT_TX_RX Operation = 101 + // DELAY_PEER_PORT_TX_RX delays all outgoing/incoming packets from/to + // the peer port on target member's peer port. + Operation_DELAY_PEER_PORT_TX_RX Operation = 200 + // UNDELAY_PEER_PORT_TX_RX removes all outgoing/incoming delays. + Operation_UNDELAY_PEER_PORT_TX_RX Operation = 201 ) var Operation_name = map[int32]string{ - 0: "NotStarted", - 1: "InitialStartEtcd", - 2: "RestartEtcd", - 3: "KillEtcd", - 4: "FailArchive", - 5: "DestroyEtcdAgent", - 100: "BlackholePeerPortTxRx", - 101: "UnblackholePeerPortTxRx", - 102: "DelayPeerPortTxRx", - 103: "UndelayPeerPortTxRx", + 0: "NOT_STARTED", + 10: "INITIAL_START_ETCD", + 11: "RESTART_ETCD", + 20: "SIGTERM_ETCD", + 21: "SIGQUIT_ETCD_AND_REMOVE_DATA", + 30: "SIGQUIT_ETCD_AND_ARCHIVE_DATA", + 31: "SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT", + 100: "BLACKHOLE_PEER_PORT_TX_RX", + 101: "UNBLACKHOLE_PEER_PORT_TX_RX", + 200: "DELAY_PEER_PORT_TX_RX", + 201: "UNDELAY_PEER_PORT_TX_RX", } var Operation_value = map[string]int32{ - "NotStarted": 0, - "InitialStartEtcd": 1, - "RestartEtcd": 2, - "KillEtcd": 3, - "FailArchive": 4, - "DestroyEtcdAgent": 5, - "BlackholePeerPortTxRx": 100, - "UnblackholePeerPortTxRx": 101, - "DelayPeerPortTxRx": 102, - "UndelayPeerPortTxRx": 103, + "NOT_STARTED": 0, + "INITIAL_START_ETCD": 10, + "RESTART_ETCD": 11, + "SIGTERM_ETCD": 20, + "SIGQUIT_ETCD_AND_REMOVE_DATA": 21, + "SIGQUIT_ETCD_AND_ARCHIVE_DATA": 30, + "SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT": 31, + "BLACKHOLE_PEER_PORT_TX_RX": 100, + "UNBLACKHOLE_PEER_PORT_TX_RX": 101, + "DELAY_PEER_PORT_TX_RX": 200, + "UNDELAY_PEER_PORT_TX_RX": 201, } func (x Operation) String() string { @@ -100,14 +107,14 @@ func (Operation) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, [] type FailureCase int32 const ( - // KILL_ONE_FOLLOWER stops a randomly chosen follower (non-leader) + // SIGTERM_ONE_FOLLOWER stops a randomly chosen follower (non-leader) // but does not delete its data directories on disk for next restart. // It waits "failure-delay-ms" before recovering this failure. // The expected behavior is that the follower comes back online // and rejoins the cluster, and then each member continues to process // client requests ('Put' request that requires Raft consensus). - FailureCase_KILL_ONE_FOLLOWER FailureCase = 0 - // KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen + FailureCase_SIGTERM_ONE_FOLLOWER FailureCase = 0 + // SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen // follower but does not delete its data directories on disk for next // restart. And waits until most up-to-date node (leader) applies the // snapshot count of entries since the stop operation. @@ -116,8 +123,8 @@ const ( // to the follower to force it to follow the leader's log. // As always, after recovery, each member must be able to process // client requests. - FailureCase_KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 1 - // KILL_LEADER stops the active leader node but does not delete its + FailureCase_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 1 + // SIGTERM_LEADER stops the active leader node but does not delete its // data directories on disk for next restart. Then it waits // "failure-delay-ms" before recovering this failure, in order to // trigger election timeouts. @@ -125,8 +132,8 @@ const ( // old leader comes back online and rejoins the cluster as a follower. // As always, after recovery, each member must be able to process // client requests. - FailureCase_KILL_LEADER FailureCase = 2 - // KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node + FailureCase_SIGTERM_LEADER FailureCase = 2 + // SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node // but does not delete its data directories on disk for next restart. // And waits until most up-to-date node ("new" leader) applies the // snapshot count of entries since the stop operation. @@ -135,22 +142,22 @@ const ( // And it receives the snapshot from the new leader to overwrite its // store. As always, after recovery, each member must be able to // process client requests. - FailureCase_KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 3 - // KILL_QUORUM stops majority number of nodes to make the whole cluster + FailureCase_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT FailureCase = 3 + // SIGTERM_QUORUM stops majority number of nodes to make the whole cluster // inoperable but does not delete data directories on stopped nodes // for next restart. And it waits "failure-delay-ms" before recovering // this failure. // The expected behavior is that nodes come back online, thus cluster // comes back operative as well. As always, after recovery, each member // must be able to process client requests. - FailureCase_KILL_QUORUM FailureCase = 4 - // KILL_ALL stops the whole cluster but does not delete data directories + FailureCase_SIGTERM_QUORUM FailureCase = 4 + // SIGTERM_ALL stops the whole cluster but does not delete data directories // on disk for next restart. And it waits "failure-delay-ms" before // recovering this failure. // The expected behavior is that nodes come back online, thus cluster // comes back operative as well. As always, after recovery, each member // must be able to process client requests. - FailureCase_KILL_ALL FailureCase = 5 + FailureCase_SIGTERM_ALL FailureCase = 5 // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER drops all outgoing/incoming // packets from/to the peer port on a randomly chosen follower // (non-leader), and waits for "failure-delay-ms" until recovery. @@ -327,12 +334,12 @@ const ( ) var FailureCase_name = map[int32]string{ - 0: "KILL_ONE_FOLLOWER", - 1: "KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", - 2: "KILL_LEADER", - 3: "KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT", - 4: "KILL_QUORUM", - 5: "KILL_ALL", + 0: "SIGTERM_ONE_FOLLOWER", + 1: "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + 2: "SIGTERM_LEADER", + 3: "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT", + 4: "SIGTERM_QUORUM", + 5: "SIGTERM_ALL", 100: "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER", 101: "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", 102: "BLACKHOLE_PEER_PORT_TX_RX_LEADER", @@ -357,12 +364,12 @@ var FailureCase_name = map[int32]string{ 500: "EXTERNAL", } var FailureCase_value = map[string]int32{ - "KILL_ONE_FOLLOWER": 0, - "KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 1, - "KILL_LEADER": 2, - "KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT": 3, - "KILL_QUORUM": 4, - "KILL_ALL": 5, + "SIGTERM_ONE_FOLLOWER": 0, + "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 1, + "SIGTERM_LEADER": 2, + "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT": 3, + "SIGTERM_QUORUM": 4, + "SIGTERM_ALL": 5, "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": 100, "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 101, "BLACKHOLE_PEER_PORT_TX_RX_LEADER": 102, @@ -425,43 +432,30 @@ func (x StressType) String() string { } func (StressType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } -type Etcd struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty" yaml:"name"` - DataDir string `protobuf:"bytes,2,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"` - WALDir string `protobuf:"bytes,3,opt,name=WALDir,proto3" json:"WALDir,omitempty" yaml:"wal-dir"` - // HeartbeatIntervalMs is the time (in milliseconds) of a heartbeat interval. - // Default value is 100, which is 100ms. - HeartbeatIntervalMs int64 `protobuf:"varint,11,opt,name=HeartbeatIntervalMs,proto3" json:"HeartbeatIntervalMs,omitempty" yaml:"heartbeat-interval"` - // ElectionTimeoutMs is the time (in milliseconds) for an election to timeout. - // Default value is 1000, which is 1s. - ElectionTimeoutMs int64 `protobuf:"varint,12,opt,name=ElectionTimeoutMs,proto3" json:"ElectionTimeoutMs,omitempty" yaml:"election-timeout"` - ListenClientURLs []string `protobuf:"bytes,21,rep,name=ListenClientURLs" json:"ListenClientURLs,omitempty" yaml:"listen-client-urls"` - AdvertiseClientURLs []string `protobuf:"bytes,22,rep,name=AdvertiseClientURLs" json:"AdvertiseClientURLs,omitempty" yaml:"advertise-client-urls"` - ClientAutoTLS bool `protobuf:"varint,23,opt,name=ClientAutoTLS,proto3" json:"ClientAutoTLS,omitempty" yaml:"auto-tls"` - ClientCertAuth bool `protobuf:"varint,24,opt,name=ClientCertAuth,proto3" json:"ClientCertAuth,omitempty" yaml:"client-cert-auth"` - ClientCertFile string `protobuf:"bytes,25,opt,name=ClientCertFile,proto3" json:"ClientCertFile,omitempty" yaml:"cert-file"` - ClientKeyFile string `protobuf:"bytes,26,opt,name=ClientKeyFile,proto3" json:"ClientKeyFile,omitempty" yaml:"key-file"` - ClientTrustedCAFile string `protobuf:"bytes,27,opt,name=ClientTrustedCAFile,proto3" json:"ClientTrustedCAFile,omitempty" yaml:"trusted-ca-file"` - ListenPeerURLs []string `protobuf:"bytes,31,rep,name=ListenPeerURLs" json:"ListenPeerURLs,omitempty" yaml:"listen-peer-urls"` - AdvertisePeerURLs []string `protobuf:"bytes,32,rep,name=AdvertisePeerURLs" json:"AdvertisePeerURLs,omitempty" yaml:"initial-advertise-peer-urls"` - PeerAutoTLS bool `protobuf:"varint,33,opt,name=PeerAutoTLS,proto3" json:"PeerAutoTLS,omitempty" yaml:"peer-auto-tls"` - PeerClientCertAuth bool `protobuf:"varint,34,opt,name=PeerClientCertAuth,proto3" json:"PeerClientCertAuth,omitempty" yaml:"peer-client-cert-auth"` - PeerCertFile string `protobuf:"bytes,35,opt,name=PeerCertFile,proto3" json:"PeerCertFile,omitempty" yaml:"peer-cert-file"` - PeerKeyFile string `protobuf:"bytes,36,opt,name=PeerKeyFile,proto3" json:"PeerKeyFile,omitempty" yaml:"peer-key-file"` - PeerTrustedCAFile string `protobuf:"bytes,37,opt,name=PeerTrustedCAFile,proto3" json:"PeerTrustedCAFile,omitempty" yaml:"peer-trusted-ca-file"` - InitialCluster string `protobuf:"bytes,41,opt,name=InitialCluster,proto3" json:"InitialCluster,omitempty" yaml:"initial-cluster"` - InitialClusterState string `protobuf:"bytes,42,opt,name=InitialClusterState,proto3" json:"InitialClusterState,omitempty" yaml:"initial-cluster-state"` - InitialClusterToken string `protobuf:"bytes,43,opt,name=InitialClusterToken,proto3" json:"InitialClusterToken,omitempty" yaml:"initial-cluster-token"` - SnapshotCount int64 `protobuf:"varint,51,opt,name=SnapshotCount,proto3" json:"SnapshotCount,omitempty" yaml:"snapshot-count"` - QuotaBackendBytes int64 `protobuf:"varint,52,opt,name=QuotaBackendBytes,proto3" json:"QuotaBackendBytes,omitempty" yaml:"quota-backend-bytes"` - PreVote bool `protobuf:"varint,63,opt,name=PreVote,proto3" json:"PreVote,omitempty" yaml:"pre-vote"` - InitialCorruptCheck bool `protobuf:"varint,64,opt,name=InitialCorruptCheck,proto3" json:"InitialCorruptCheck,omitempty" yaml:"initial-corrupt-check"` +type Request struct { + Operation Operation `protobuf:"varint,1,opt,name=Operation,proto3,enum=rpcpb.Operation" json:"Operation,omitempty"` + // Member contains the same Member object from tester configuration. + Member *Member `protobuf:"bytes,2,opt,name=Member" json:"Member,omitempty"` + // Tester contains tester configuration. + Tester *Tester `protobuf:"bytes,3,opt,name=Tester" json:"Tester,omitempty"` } -func (m *Etcd) Reset() { *m = Etcd{} } -func (m *Etcd) String() string { return proto.CompactTextString(m) } -func (*Etcd) ProtoMessage() {} -func (*Etcd) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{0} } +func (m *Request) Reset() { *m = Request{} } +func (m *Request) String() string { return proto.CompactTextString(m) } +func (*Request) ProtoMessage() {} +func (*Request) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{0} } + +type Response struct { + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` + Status string `protobuf:"bytes,2,opt,name=Status,proto3" json:"Status,omitempty"` + // Member contains the same Member object from tester request. + Member *Member `protobuf:"bytes,3,opt,name=Member" json:"Member,omitempty"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } type Member struct { // EtcdExecPath is the executable etcd binary path in agent server. @@ -507,7 +501,7 @@ type Member struct { func (m *Member) Reset() { *m = Member{} } func (m *Member) String() string { return proto.CompactTextString(m) } func (*Member) ProtoMessage() {} -func (*Member) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } +func (*Member) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } type Tester struct { DataDir string `protobuf:"bytes,1,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"` @@ -569,39 +563,52 @@ type Tester struct { func (m *Tester) Reset() { *m = Tester{} } func (m *Tester) String() string { return proto.CompactTextString(m) } func (*Tester) ProtoMessage() {} -func (*Tester) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } +func (*Tester) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } -type Request struct { - Operation Operation `protobuf:"varint,1,opt,name=Operation,proto3,enum=rpcpb.Operation" json:"Operation,omitempty"` - // Member contains the same Member object from tester configuration. - Member *Member `protobuf:"bytes,2,opt,name=Member" json:"Member,omitempty"` - // Tester contains tester configuration. - Tester *Tester `protobuf:"bytes,3,opt,name=Tester" json:"Tester,omitempty"` +type Etcd struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty" yaml:"name"` + DataDir string `protobuf:"bytes,2,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"` + WALDir string `protobuf:"bytes,3,opt,name=WALDir,proto3" json:"WALDir,omitempty" yaml:"wal-dir"` + // HeartbeatIntervalMs is the time (in milliseconds) of a heartbeat interval. + // Default value is 100, which is 100ms. + HeartbeatIntervalMs int64 `protobuf:"varint,11,opt,name=HeartbeatIntervalMs,proto3" json:"HeartbeatIntervalMs,omitempty" yaml:"heartbeat-interval"` + // ElectionTimeoutMs is the time (in milliseconds) for an election to timeout. + // Default value is 1000, which is 1s. + ElectionTimeoutMs int64 `protobuf:"varint,12,opt,name=ElectionTimeoutMs,proto3" json:"ElectionTimeoutMs,omitempty" yaml:"election-timeout"` + ListenClientURLs []string `protobuf:"bytes,21,rep,name=ListenClientURLs" json:"ListenClientURLs,omitempty" yaml:"listen-client-urls"` + AdvertiseClientURLs []string `protobuf:"bytes,22,rep,name=AdvertiseClientURLs" json:"AdvertiseClientURLs,omitempty" yaml:"advertise-client-urls"` + ClientAutoTLS bool `protobuf:"varint,23,opt,name=ClientAutoTLS,proto3" json:"ClientAutoTLS,omitempty" yaml:"auto-tls"` + ClientCertAuth bool `protobuf:"varint,24,opt,name=ClientCertAuth,proto3" json:"ClientCertAuth,omitempty" yaml:"client-cert-auth"` + ClientCertFile string `protobuf:"bytes,25,opt,name=ClientCertFile,proto3" json:"ClientCertFile,omitempty" yaml:"cert-file"` + ClientKeyFile string `protobuf:"bytes,26,opt,name=ClientKeyFile,proto3" json:"ClientKeyFile,omitempty" yaml:"key-file"` + ClientTrustedCAFile string `protobuf:"bytes,27,opt,name=ClientTrustedCAFile,proto3" json:"ClientTrustedCAFile,omitempty" yaml:"trusted-ca-file"` + ListenPeerURLs []string `protobuf:"bytes,31,rep,name=ListenPeerURLs" json:"ListenPeerURLs,omitempty" yaml:"listen-peer-urls"` + AdvertisePeerURLs []string `protobuf:"bytes,32,rep,name=AdvertisePeerURLs" json:"AdvertisePeerURLs,omitempty" yaml:"initial-advertise-peer-urls"` + PeerAutoTLS bool `protobuf:"varint,33,opt,name=PeerAutoTLS,proto3" json:"PeerAutoTLS,omitempty" yaml:"peer-auto-tls"` + PeerClientCertAuth bool `protobuf:"varint,34,opt,name=PeerClientCertAuth,proto3" json:"PeerClientCertAuth,omitempty" yaml:"peer-client-cert-auth"` + PeerCertFile string `protobuf:"bytes,35,opt,name=PeerCertFile,proto3" json:"PeerCertFile,omitempty" yaml:"peer-cert-file"` + PeerKeyFile string `protobuf:"bytes,36,opt,name=PeerKeyFile,proto3" json:"PeerKeyFile,omitempty" yaml:"peer-key-file"` + PeerTrustedCAFile string `protobuf:"bytes,37,opt,name=PeerTrustedCAFile,proto3" json:"PeerTrustedCAFile,omitempty" yaml:"peer-trusted-ca-file"` + InitialCluster string `protobuf:"bytes,41,opt,name=InitialCluster,proto3" json:"InitialCluster,omitempty" yaml:"initial-cluster"` + InitialClusterState string `protobuf:"bytes,42,opt,name=InitialClusterState,proto3" json:"InitialClusterState,omitempty" yaml:"initial-cluster-state"` + InitialClusterToken string `protobuf:"bytes,43,opt,name=InitialClusterToken,proto3" json:"InitialClusterToken,omitempty" yaml:"initial-cluster-token"` + SnapshotCount int64 `protobuf:"varint,51,opt,name=SnapshotCount,proto3" json:"SnapshotCount,omitempty" yaml:"snapshot-count"` + QuotaBackendBytes int64 `protobuf:"varint,52,opt,name=QuotaBackendBytes,proto3" json:"QuotaBackendBytes,omitempty" yaml:"quota-backend-bytes"` + PreVote bool `protobuf:"varint,63,opt,name=PreVote,proto3" json:"PreVote,omitempty" yaml:"pre-vote"` + InitialCorruptCheck bool `protobuf:"varint,64,opt,name=InitialCorruptCheck,proto3" json:"InitialCorruptCheck,omitempty" yaml:"initial-corrupt-check"` } -func (m *Request) Reset() { *m = Request{} } -func (m *Request) String() string { return proto.CompactTextString(m) } -func (*Request) ProtoMessage() {} -func (*Request) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } - -type Response struct { - Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` - Status string `protobuf:"bytes,2,opt,name=Status,proto3" json:"Status,omitempty"` - // Member contains the same Member object from tester request. - Member *Member `protobuf:"bytes,3,opt,name=Member" json:"Member,omitempty"` -} - -func (m *Response) Reset() { *m = Response{} } -func (m *Response) String() string { return proto.CompactTextString(m) } -func (*Response) ProtoMessage() {} -func (*Response) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{4} } +func (m *Etcd) Reset() { *m = Etcd{} } +func (m *Etcd) String() string { return proto.CompactTextString(m) } +func (*Etcd) ProtoMessage() {} +func (*Etcd) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{4} } func init() { - proto.RegisterType((*Etcd)(nil), "rpcpb.Etcd") - proto.RegisterType((*Member)(nil), "rpcpb.Member") - proto.RegisterType((*Tester)(nil), "rpcpb.Tester") proto.RegisterType((*Request)(nil), "rpcpb.Request") proto.RegisterType((*Response)(nil), "rpcpb.Response") + proto.RegisterType((*Member)(nil), "rpcpb.Member") + proto.RegisterType((*Tester)(nil), "rpcpb.Tester") + proto.RegisterType((*Etcd)(nil), "rpcpb.Etcd") proto.RegisterEnum("rpcpb.Operation", Operation_name, Operation_value) proto.RegisterEnum("rpcpb.FailureCase", FailureCase_name, FailureCase_value) proto.RegisterEnum("rpcpb.StressType", StressType_name, StressType_value) @@ -711,7 +718,7 @@ var _Transport_serviceDesc = grpc.ServiceDesc{ Metadata: "rpcpb/rpc.proto", } -func (m *Etcd) Marshal() (dAtA []byte, err error) { +func (m *Request) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -721,264 +728,79 @@ func (m *Etcd) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Etcd) MarshalTo(dAtA []byte) (int, error) { +func (m *Request) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa + if m.Operation != 0 { + dAtA[i] = 0x8 i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) + i = encodeVarintRpc(dAtA, i, uint64(m.Operation)) } - if len(m.DataDir) > 0 { + if m.Member != nil { dAtA[i] = 0x12 i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.DataDir))) - i += copy(dAtA[i:], m.DataDir) + i = encodeVarintRpc(dAtA, i, uint64(m.Member.Size())) + n1, err := m.Member.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 } - if len(m.WALDir) > 0 { + if m.Tester != nil { dAtA[i] = 0x1a i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.WALDir))) - i += copy(dAtA[i:], m.WALDir) - } - if m.HeartbeatIntervalMs != 0 { - dAtA[i] = 0x58 - i++ - i = encodeVarintRpc(dAtA, i, uint64(m.HeartbeatIntervalMs)) - } - if m.ElectionTimeoutMs != 0 { - dAtA[i] = 0x60 - i++ - i = encodeVarintRpc(dAtA, i, uint64(m.ElectionTimeoutMs)) - } - if len(m.ListenClientURLs) > 0 { - for _, s := range m.ListenClientURLs { - dAtA[i] = 0xaa - i++ - dAtA[i] = 0x1 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + i = encodeVarintRpc(dAtA, i, uint64(m.Tester.Size())) + n2, err := m.Tester.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } + i += n2 } - if len(m.AdvertiseClientURLs) > 0 { - for _, s := range m.AdvertiseClientURLs { - dAtA[i] = 0xb2 - i++ - dAtA[i] = 0x1 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } + return i, nil +} + +func (m *Response) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err } - if m.ClientAutoTLS { - dAtA[i] = 0xb8 + return dAtA[:n], nil +} + +func (m *Response) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Success { + dAtA[i] = 0x8 i++ - dAtA[i] = 0x1 - i++ - if m.ClientAutoTLS { + if m.Success { dAtA[i] = 1 } else { dAtA[i] = 0 } i++ } - if m.ClientCertAuth { - dAtA[i] = 0xc0 + if len(m.Status) > 0 { + dAtA[i] = 0x12 i++ - dAtA[i] = 0x1 + i = encodeVarintRpc(dAtA, i, uint64(len(m.Status))) + i += copy(dAtA[i:], m.Status) + } + if m.Member != nil { + dAtA[i] = 0x1a i++ - if m.ClientCertAuth { - dAtA[i] = 1 - } else { - dAtA[i] = 0 + i = encodeVarintRpc(dAtA, i, uint64(m.Member.Size())) + n3, err := m.Member.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - i++ - } - if len(m.ClientCertFile) > 0 { - dAtA[i] = 0xca - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientCertFile))) - i += copy(dAtA[i:], m.ClientCertFile) - } - if len(m.ClientKeyFile) > 0 { - dAtA[i] = 0xd2 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientKeyFile))) - i += copy(dAtA[i:], m.ClientKeyFile) - } - if len(m.ClientTrustedCAFile) > 0 { - dAtA[i] = 0xda - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientTrustedCAFile))) - i += copy(dAtA[i:], m.ClientTrustedCAFile) - } - if len(m.ListenPeerURLs) > 0 { - for _, s := range m.ListenPeerURLs { - dAtA[i] = 0xfa - i++ - dAtA[i] = 0x1 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } - } - if len(m.AdvertisePeerURLs) > 0 { - for _, s := range m.AdvertisePeerURLs { - dAtA[i] = 0x82 - i++ - dAtA[i] = 0x2 - i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ - } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) - } - } - if m.PeerAutoTLS { - dAtA[i] = 0x88 - i++ - dAtA[i] = 0x2 - i++ - if m.PeerAutoTLS { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if m.PeerClientCertAuth { - dAtA[i] = 0x90 - i++ - dAtA[i] = 0x2 - i++ - if m.PeerClientCertAuth { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if len(m.PeerCertFile) > 0 { - dAtA[i] = 0x9a - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerCertFile))) - i += copy(dAtA[i:], m.PeerCertFile) - } - if len(m.PeerKeyFile) > 0 { - dAtA[i] = 0xa2 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerKeyFile))) - i += copy(dAtA[i:], m.PeerKeyFile) - } - if len(m.PeerTrustedCAFile) > 0 { - dAtA[i] = 0xaa - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerTrustedCAFile))) - i += copy(dAtA[i:], m.PeerTrustedCAFile) - } - if len(m.InitialCluster) > 0 { - dAtA[i] = 0xca - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialCluster))) - i += copy(dAtA[i:], m.InitialCluster) - } - if len(m.InitialClusterState) > 0 { - dAtA[i] = 0xd2 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialClusterState))) - i += copy(dAtA[i:], m.InitialClusterState) - } - if len(m.InitialClusterToken) > 0 { - dAtA[i] = 0xda - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialClusterToken))) - i += copy(dAtA[i:], m.InitialClusterToken) - } - if m.SnapshotCount != 0 { - dAtA[i] = 0x98 - i++ - dAtA[i] = 0x3 - i++ - i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotCount)) - } - if m.QuotaBackendBytes != 0 { - dAtA[i] = 0xa0 - i++ - dAtA[i] = 0x3 - i++ - i = encodeVarintRpc(dAtA, i, uint64(m.QuotaBackendBytes)) - } - if m.PreVote { - dAtA[i] = 0xf8 - i++ - dAtA[i] = 0x3 - i++ - if m.PreVote { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } - if m.InitialCorruptCheck { - dAtA[i] = 0x80 - i++ - dAtA[i] = 0x4 - i++ - if m.InitialCorruptCheck { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ + i += n3 } return i, nil } @@ -1070,11 +892,11 @@ func (m *Member) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintRpc(dAtA, i, uint64(m.Etcd.Size())) - n1, err := m.Etcd.MarshalTo(dAtA[i:]) + n4, err := m.Etcd.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n1 + i += n4 } if len(m.ClientCertData) > 0 { dAtA[i] = 0x8a @@ -1404,7 +1226,7 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Request) Marshal() (dAtA []byte, err error) { +func (m *Etcd) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1414,79 +1236,264 @@ func (m *Request) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Request) MarshalTo(dAtA []byte) (int, error) { +func (m *Etcd) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.Operation != 0 { - dAtA[i] = 0x8 + if len(m.Name) > 0 { + dAtA[i] = 0xa i++ - i = encodeVarintRpc(dAtA, i, uint64(m.Operation)) + i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) } - if m.Member != nil { + if len(m.DataDir) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintRpc(dAtA, i, uint64(m.Member.Size())) - n2, err := m.Member.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n2 + i = encodeVarintRpc(dAtA, i, uint64(len(m.DataDir))) + i += copy(dAtA[i:], m.DataDir) } - if m.Tester != nil { + if len(m.WALDir) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintRpc(dAtA, i, uint64(m.Tester.Size())) - n3, err := m.Tester.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n3 + i = encodeVarintRpc(dAtA, i, uint64(len(m.WALDir))) + i += copy(dAtA[i:], m.WALDir) } - return i, nil -} - -func (m *Response) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Response) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Success { - dAtA[i] = 0x8 + if m.HeartbeatIntervalMs != 0 { + dAtA[i] = 0x58 i++ - if m.Success { + i = encodeVarintRpc(dAtA, i, uint64(m.HeartbeatIntervalMs)) + } + if m.ElectionTimeoutMs != 0 { + dAtA[i] = 0x60 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.ElectionTimeoutMs)) + } + if len(m.ListenClientURLs) > 0 { + for _, s := range m.ListenClientURLs { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x1 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.AdvertiseClientURLs) > 0 { + for _, s := range m.AdvertiseClientURLs { + dAtA[i] = 0xb2 + i++ + dAtA[i] = 0x1 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.ClientAutoTLS { + dAtA[i] = 0xb8 + i++ + dAtA[i] = 0x1 + i++ + if m.ClientAutoTLS { dAtA[i] = 1 } else { dAtA[i] = 0 } i++ } - if len(m.Status) > 0 { - dAtA[i] = 0x12 + if m.ClientCertAuth { + dAtA[i] = 0xc0 i++ - i = encodeVarintRpc(dAtA, i, uint64(len(m.Status))) - i += copy(dAtA[i:], m.Status) - } - if m.Member != nil { - dAtA[i] = 0x1a + dAtA[i] = 0x1 i++ - i = encodeVarintRpc(dAtA, i, uint64(m.Member.Size())) - n4, err := m.Member.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.ClientCertAuth { + dAtA[i] = 1 + } else { + dAtA[i] = 0 } - i += n4 + i++ + } + if len(m.ClientCertFile) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientCertFile))) + i += copy(dAtA[i:], m.ClientCertFile) + } + if len(m.ClientKeyFile) > 0 { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientKeyFile))) + i += copy(dAtA[i:], m.ClientKeyFile) + } + if len(m.ClientTrustedCAFile) > 0 { + dAtA[i] = 0xda + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientTrustedCAFile))) + i += copy(dAtA[i:], m.ClientTrustedCAFile) + } + if len(m.ListenPeerURLs) > 0 { + for _, s := range m.ListenPeerURLs { + dAtA[i] = 0xfa + i++ + dAtA[i] = 0x1 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.AdvertisePeerURLs) > 0 { + for _, s := range m.AdvertisePeerURLs { + dAtA[i] = 0x82 + i++ + dAtA[i] = 0x2 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.PeerAutoTLS { + dAtA[i] = 0x88 + i++ + dAtA[i] = 0x2 + i++ + if m.PeerAutoTLS { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.PeerClientCertAuth { + dAtA[i] = 0x90 + i++ + dAtA[i] = 0x2 + i++ + if m.PeerClientCertAuth { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.PeerCertFile) > 0 { + dAtA[i] = 0x9a + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerCertFile))) + i += copy(dAtA[i:], m.PeerCertFile) + } + if len(m.PeerKeyFile) > 0 { + dAtA[i] = 0xa2 + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerKeyFile))) + i += copy(dAtA[i:], m.PeerKeyFile) + } + if len(m.PeerTrustedCAFile) > 0 { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerTrustedCAFile))) + i += copy(dAtA[i:], m.PeerTrustedCAFile) + } + if len(m.InitialCluster) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialCluster))) + i += copy(dAtA[i:], m.InitialCluster) + } + if len(m.InitialClusterState) > 0 { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialClusterState))) + i += copy(dAtA[i:], m.InitialClusterState) + } + if len(m.InitialClusterToken) > 0 { + dAtA[i] = 0xda + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialClusterToken))) + i += copy(dAtA[i:], m.InitialClusterToken) + } + if m.SnapshotCount != 0 { + dAtA[i] = 0x98 + i++ + dAtA[i] = 0x3 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotCount)) + } + if m.QuotaBackendBytes != 0 { + dAtA[i] = 0xa0 + i++ + dAtA[i] = 0x3 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.QuotaBackendBytes)) + } + if m.PreVote { + dAtA[i] = 0xf8 + i++ + dAtA[i] = 0x3 + i++ + if m.PreVote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.InitialCorruptCheck { + dAtA[i] = 0x80 + i++ + dAtA[i] = 0x4 + i++ + if m.InitialCorruptCheck { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ } return i, nil } @@ -1500,111 +1507,37 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *Etcd) Size() (n int) { +func (m *Request) Size() (n int) { var l int _ = l - l = len(m.Name) + if m.Operation != 0 { + n += 1 + sovRpc(uint64(m.Operation)) + } + if m.Member != nil { + l = m.Member.Size() + n += 1 + l + sovRpc(uint64(l)) + } + if m.Tester != nil { + l = m.Tester.Size() + n += 1 + l + sovRpc(uint64(l)) + } + return n +} + +func (m *Response) Size() (n int) { + var l int + _ = l + if m.Success { + n += 2 + } + l = len(m.Status) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } - l = len(m.DataDir) - if l > 0 { + if m.Member != nil { + l = m.Member.Size() n += 1 + l + sovRpc(uint64(l)) } - l = len(m.WALDir) - if l > 0 { - n += 1 + l + sovRpc(uint64(l)) - } - if m.HeartbeatIntervalMs != 0 { - n += 1 + sovRpc(uint64(m.HeartbeatIntervalMs)) - } - if m.ElectionTimeoutMs != 0 { - n += 1 + sovRpc(uint64(m.ElectionTimeoutMs)) - } - if len(m.ListenClientURLs) > 0 { - for _, s := range m.ListenClientURLs { - l = len(s) - n += 2 + l + sovRpc(uint64(l)) - } - } - if len(m.AdvertiseClientURLs) > 0 { - for _, s := range m.AdvertiseClientURLs { - l = len(s) - n += 2 + l + sovRpc(uint64(l)) - } - } - if m.ClientAutoTLS { - n += 3 - } - if m.ClientCertAuth { - n += 3 - } - l = len(m.ClientCertFile) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.ClientKeyFile) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.ClientTrustedCAFile) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - if len(m.ListenPeerURLs) > 0 { - for _, s := range m.ListenPeerURLs { - l = len(s) - n += 2 + l + sovRpc(uint64(l)) - } - } - if len(m.AdvertisePeerURLs) > 0 { - for _, s := range m.AdvertisePeerURLs { - l = len(s) - n += 2 + l + sovRpc(uint64(l)) - } - } - if m.PeerAutoTLS { - n += 3 - } - if m.PeerClientCertAuth { - n += 3 - } - l = len(m.PeerCertFile) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.PeerKeyFile) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.PeerTrustedCAFile) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.InitialCluster) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.InitialClusterState) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - l = len(m.InitialClusterToken) - if l > 0 { - n += 2 + l + sovRpc(uint64(l)) - } - if m.SnapshotCount != 0 { - n += 2 + sovRpc(uint64(m.SnapshotCount)) - } - if m.QuotaBackendBytes != 0 { - n += 2 + sovRpc(uint64(m.QuotaBackendBytes)) - } - if m.PreVote { - n += 3 - } - if m.InitialCorruptCheck { - n += 3 - } return n } @@ -1788,37 +1721,111 @@ func (m *Tester) Size() (n int) { return n } -func (m *Request) Size() (n int) { +func (m *Etcd) Size() (n int) { var l int _ = l - if m.Operation != 0 { - n += 1 + sovRpc(uint64(m.Operation)) - } - if m.Member != nil { - l = m.Member.Size() - n += 1 + l + sovRpc(uint64(l)) - } - if m.Tester != nil { - l = m.Tester.Size() - n += 1 + l + sovRpc(uint64(l)) - } - return n -} - -func (m *Response) Size() (n int) { - var l int - _ = l - if m.Success { - n += 2 - } - l = len(m.Status) + l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } - if m.Member != nil { - l = m.Member.Size() + l = len(m.DataDir) + if l > 0 { n += 1 + l + sovRpc(uint64(l)) } + l = len(m.WALDir) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if m.HeartbeatIntervalMs != 0 { + n += 1 + sovRpc(uint64(m.HeartbeatIntervalMs)) + } + if m.ElectionTimeoutMs != 0 { + n += 1 + sovRpc(uint64(m.ElectionTimeoutMs)) + } + if len(m.ListenClientURLs) > 0 { + for _, s := range m.ListenClientURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if len(m.AdvertiseClientURLs) > 0 { + for _, s := range m.AdvertiseClientURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if m.ClientAutoTLS { + n += 3 + } + if m.ClientCertAuth { + n += 3 + } + l = len(m.ClientCertFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientKeyFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientTrustedCAFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if len(m.ListenPeerURLs) > 0 { + for _, s := range m.ListenPeerURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if len(m.AdvertisePeerURLs) > 0 { + for _, s := range m.AdvertisePeerURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if m.PeerAutoTLS { + n += 3 + } + if m.PeerClientCertAuth { + n += 3 + } + l = len(m.PeerCertFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerKeyFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerTrustedCAFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.InitialCluster) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.InitialClusterState) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.InitialClusterToken) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if m.SnapshotCount != 0 { + n += 2 + sovRpc(uint64(m.SnapshotCount)) + } + if m.QuotaBackendBytes != 0 { + n += 2 + sovRpc(uint64(m.QuotaBackendBytes)) + } + if m.PreVote { + n += 3 + } + if m.InitialCorruptCheck { + n += 3 + } return n } @@ -1835,7 +1842,7 @@ func sovRpc(x uint64) (n int) { func sozRpc(x uint64) (n int) { return sovRpc(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *Etcd) Unmarshal(dAtA []byte) error { +func (m *Request) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1858,17 +1865,17 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Etcd: wiretype end group for non-group") + return fmt.Errorf("proto: Request: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Etcd: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Request: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Operation", wireType) } - var stringLen uint64 + m.Operation = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc @@ -1878,26 +1885,16 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + m.Operation |= (Operation(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DataDir", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc @@ -1907,26 +1904,30 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthRpc } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.DataDir = string(dAtA[iNdEx:postIndex]) + if m.Member == nil { + m.Member = &Member{} + } + if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field WALDir", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Tester", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc @@ -1936,120 +1937,78 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthRpc } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.WALDir = string(dAtA[iNdEx:postIndex]) + if m.Tester == nil { + m.Tester = &Tester{} + } + if err := m.Tester.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex - case 11: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field HeartbeatIntervalMs", wireType) + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err } - m.HeartbeatIntervalMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.HeartbeatIntervalMs |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 12: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ElectionTimeoutMs", wireType) - } - m.ElectionTimeoutMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ElectionTimeoutMs |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 21: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ListenClientURLs", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { + if skippy < 0 { return ErrInvalidLengthRpc } - postIndex := iNdEx + intStringLen - if postIndex > l { + if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } - m.ListenClientURLs = append(m.ListenClientURLs, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - case 22: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AdvertiseClientURLs", wireType) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Response) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - m.AdvertiseClientURLs = append(m.AdvertiseClientURLs, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - case 23: + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Response: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Response: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ClientAutoTLS", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -2066,30 +2025,10 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { break } } - m.ClientAutoTLS = bool(v != 0) - case 24: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ClientCertAuth", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.ClientCertAuth = bool(v != 0) - case 25: + m.Success = bool(v != 0) + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ClientCertFile", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2114,13 +2053,13 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.ClientCertFile = string(dAtA[iNdEx:postIndex]) + m.Status = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 26: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ClientKeyFile", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc @@ -2130,400 +2069,25 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthRpc } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.ClientKeyFile = string(dAtA[iNdEx:postIndex]) + if m.Member == nil { + m.Member = &Member{} + } + if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex - case 27: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ClientTrustedCAFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ClientTrustedCAFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 31: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ListenPeerURLs", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ListenPeerURLs = append(m.ListenPeerURLs, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - case 32: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AdvertisePeerURLs", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.AdvertisePeerURLs = append(m.AdvertisePeerURLs, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - case 33: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerAutoTLS", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.PeerAutoTLS = bool(v != 0) - case 34: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerClientCertAuth", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.PeerClientCertAuth = bool(v != 0) - case 35: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerCertFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.PeerCertFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 36: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerKeyFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.PeerKeyFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 37: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PeerTrustedCAFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.PeerTrustedCAFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 41: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field InitialCluster", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.InitialCluster = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 42: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field InitialClusterState", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.InitialClusterState = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 43: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field InitialClusterToken", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.InitialClusterToken = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 51: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SnapshotCount", wireType) - } - m.SnapshotCount = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.SnapshotCount |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 52: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field QuotaBackendBytes", wireType) - } - m.QuotaBackendBytes = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.QuotaBackendBytes |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 63: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field PreVote", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.PreVote = bool(v != 0) - case 64: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field InitialCorruptCheck", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.InitialCorruptCheck = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -3780,7 +3344,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error { } return nil } -func (m *Request) Unmarshal(dAtA []byte) error { +func (m *Etcd) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -3803,170 +3367,15 @@ func (m *Request) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Request: wiretype end group for non-group") + return fmt.Errorf("proto: Etcd: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Request: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Etcd: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Operation", wireType) - } - m.Operation = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Operation |= (Operation(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Member == nil { - m.Member = &Member{} - } - if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Tester", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthRpc - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Tester == nil { - m.Tester = &Tester{} - } - if err := m.Tester.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipRpc(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Response) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Response: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Response: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRpc - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Success = bool(v != 0) - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3991,13 +3400,13 @@ func (m *Response) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Status = string(dAtA[iNdEx:postIndex]) + m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field DataDir", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc @@ -4007,25 +3416,623 @@ func (m *Response) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthRpc } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex > l { return io.ErrUnexpectedEOF } - if m.Member == nil { - m.Member = &Member{} - } - if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.DataDir = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WALDir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WALDir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field HeartbeatIntervalMs", wireType) + } + m.HeartbeatIntervalMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.HeartbeatIntervalMs |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ElectionTimeoutMs", wireType) + } + m.ElectionTimeoutMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ElectionTimeoutMs |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 21: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ListenClientURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ListenClientURLs = append(m.ListenClientURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 22: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AdvertiseClientURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AdvertiseClientURLs = append(m.AdvertiseClientURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 23: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientAutoTLS", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ClientAutoTLS = bool(v != 0) + case 24: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientCertAuth", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ClientCertAuth = bool(v != 0) + case 25: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientCertFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientCertFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 26: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientKeyFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientKeyFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 27: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientTrustedCAFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientTrustedCAFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 31: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ListenPeerURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ListenPeerURLs = append(m.ListenPeerURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 32: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AdvertisePeerURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AdvertisePeerURLs = append(m.AdvertisePeerURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 33: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerAutoTLS", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PeerAutoTLS = bool(v != 0) + case 34: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerClientCertAuth", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PeerClientCertAuth = bool(v != 0) + case 35: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerCertFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerCertFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 36: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerKeyFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerKeyFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 37: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerTrustedCAFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerTrustedCAFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 41: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialCluster", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InitialCluster = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 42: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialClusterState", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InitialClusterState = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 43: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialClusterToken", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InitialClusterToken = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 51: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotCount", wireType) + } + m.SnapshotCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SnapshotCount |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 52: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field QuotaBackendBytes", wireType) + } + m.QuotaBackendBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.QuotaBackendBytes |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 63: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreVote", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PreVote = bool(v != 0) + case 64: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialCorruptCheck", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.InitialCorruptCheck = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -4155,160 +4162,160 @@ var ( func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 2468 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xdb, 0x76, 0xdb, 0xc6, - 0xd5, 0x16, 0x44, 0x4b, 0x96, 0x46, 0x27, 0x6a, 0x64, 0x59, 0xb0, 0x9d, 0x88, 0x32, 0x1c, 0xe7, - 0x57, 0xf4, 0x07, 0x72, 0x6b, 0x67, 0xb5, 0xb5, 0x73, 0xb0, 0x29, 0x0a, 0xb6, 0x58, 0x41, 0x24, - 0x3d, 0x84, 0x6c, 0xe7, 0x8a, 0x85, 0xc0, 0xa1, 0x88, 0x0a, 0x02, 0x68, 0x60, 0xa8, 0x50, 0x79, - 0x81, 0xde, 0xf6, 0xbc, 0x7a, 0xd5, 0x27, 0x68, 0xfa, 0x1c, 0x4e, 0x7a, 0x4a, 0xdb, 0x7b, 0xb6, - 0x75, 0x56, 0x5f, 0x80, 0xab, 0xa7, 0xf4, 0xaa, 0x6b, 0x0e, 0x24, 0x07, 0x20, 0x29, 0xe9, 0x4e, - 0xd8, 0xfb, 0xfb, 0x3e, 0xec, 0xd9, 0x9b, 0xd8, 0x7b, 0x8f, 0x0d, 0x16, 0xc2, 0x86, 0xd3, 0x38, - 0xb8, 0x13, 0x36, 0x9c, 0xcd, 0x46, 0x18, 0x90, 0x00, 0x4e, 0x30, 0xc3, 0x75, 0xfd, 0xd0, 0x25, - 0xf5, 0xe6, 0xc1, 0xa6, 0x13, 0x1c, 0xdf, 0x39, 0x0c, 0x0e, 0x83, 0x3b, 0xcc, 0x7b, 0xd0, 0xac, - 0xb1, 0x27, 0xf6, 0xc0, 0xfe, 0xe2, 0x2c, 0xed, 0xeb, 0x39, 0x70, 0xc9, 0x20, 0x4e, 0x15, 0xde, - 0x02, 0x97, 0x0a, 0xf6, 0x31, 0x56, 0x95, 0x35, 0x65, 0x7d, 0x7a, 0x6b, 0xa1, 0xd3, 0xce, 0xcc, - 0x9c, 0xda, 0xc7, 0xde, 0x03, 0xcd, 0xb7, 0x8f, 0xb1, 0x86, 0x98, 0x13, 0xea, 0xe0, 0xf2, 0xb6, - 0x4d, 0xec, 0x6d, 0x37, 0x54, 0xc7, 0x19, 0x6e, 0xa9, 0xd3, 0xce, 0x2c, 0x70, 0x5c, 0xd5, 0x26, - 0xb6, 0x5e, 0x75, 0x43, 0x0d, 0x75, 0x31, 0x70, 0x03, 0x4c, 0x3e, 0xcf, 0x9a, 0x14, 0x9d, 0x62, - 0x68, 0xd8, 0x69, 0x67, 0xe6, 0x39, 0xfa, 0x13, 0xdb, 0xe3, 0x60, 0x81, 0x80, 0x45, 0xb0, 0xb4, - 0x83, 0xed, 0x90, 0x1c, 0x60, 0x9b, 0xe4, 0x7d, 0x82, 0xc3, 0x13, 0xdb, 0xdb, 0x8b, 0xd4, 0x99, - 0x35, 0x65, 0x3d, 0xb5, 0xf5, 0x66, 0xa7, 0x9d, 0xb9, 0xc6, 0x89, 0xf5, 0x2e, 0x48, 0x77, 0x05, - 0x4a, 0x43, 0xc3, 0x98, 0x30, 0x0f, 0x16, 0x0d, 0x0f, 0x3b, 0xc4, 0x0d, 0x7c, 0xcb, 0x3d, 0xc6, - 0x41, 0x93, 0xec, 0x45, 0xea, 0x2c, 0x93, 0xbb, 0xd1, 0x69, 0x67, 0x56, 0xb8, 0x1c, 0x16, 0x10, - 0x9d, 0x70, 0x8c, 0x86, 0x06, 0x59, 0x30, 0x0f, 0xd2, 0xa6, 0x1b, 0x11, 0xec, 0xe7, 0x3c, 0x17, - 0xfb, 0x64, 0x1f, 0x99, 0x91, 0xba, 0xbc, 0x96, 0x5a, 0x9f, 0x96, 0x03, 0xf3, 0x18, 0x42, 0x77, - 0x18, 0x44, 0x6f, 0x86, 0x5e, 0xa4, 0xa1, 0x01, 0x1a, 0x44, 0x60, 0x29, 0x5b, 0x3d, 0xc1, 0x21, - 0x71, 0x23, 0x2c, 0xa9, 0x5d, 0x65, 0x6a, 0x6b, 0x9d, 0x76, 0xe6, 0x0d, 0xae, 0x66, 0x77, 0x41, - 0x71, 0xc1, 0x61, 0x64, 0x78, 0x1f, 0xcc, 0xf1, 0xa7, 0x6c, 0x93, 0x04, 0x96, 0x59, 0x56, 0x57, - 0xd6, 0x94, 0xf5, 0x29, 0xb9, 0x36, 0x76, 0x93, 0x04, 0x3a, 0xa1, 0x02, 0x71, 0x24, 0xcc, 0x81, - 0x79, 0x6e, 0xc8, 0xe1, 0x90, 0x1a, 0xeb, 0xaa, 0xca, 0xb8, 0x52, 0x86, 0xc4, 0xfb, 0x1d, 0x1c, - 0x12, 0xdd, 0x6e, 0x92, 0xba, 0x86, 0x12, 0x14, 0xf8, 0x81, 0x2c, 0xf2, 0xd8, 0xf5, 0xb0, 0x7a, - 0x8d, 0x95, 0xfb, 0x4a, 0xa7, 0x9d, 0x49, 0x0b, 0x11, 0xca, 0xae, 0xb9, 0x1e, 0x8e, 0xb1, 0x29, - 0xb6, 0x1f, 0xfd, 0x2e, 0x3e, 0x65, 0xe4, 0xeb, 0xc9, 0x5f, 0xd6, 0x11, 0x3e, 0x15, 0xdc, 0x38, - 0x12, 0x9a, 0x60, 0x89, 0x1b, 0xac, 0xb0, 0x19, 0x11, 0x5c, 0xcd, 0x65, 0x99, 0xc0, 0x0d, 0x26, - 0x70, 0xbd, 0xd3, 0xce, 0x5c, 0xe5, 0x02, 0x84, 0xbb, 0x75, 0xc7, 0x16, 0x3a, 0xc3, 0x68, 0x34, - 0x17, 0xbc, 0x5c, 0x25, 0x8c, 0x43, 0x56, 0x95, 0x0c, 0xab, 0x8a, 0x94, 0x0b, 0x51, 0xe3, 0x06, - 0xc6, 0xa1, 0x28, 0x48, 0x82, 0x02, 0x2d, 0xb0, 0xd8, 0x2b, 0x51, 0x4f, 0x67, 0x8d, 0xe9, 0xbc, - 0xdd, 0x69, 0x67, 0x34, 0xae, 0xe3, 0xfa, 0x2e, 0x71, 0x6d, 0x4f, 0xef, 0x57, 0x59, 0x92, 0x1c, - 0x14, 0x80, 0x0f, 0xc0, 0x0c, 0xfd, 0xbb, 0x5b, 0xdf, 0x9b, 0xac, 0x46, 0x6a, 0xa7, 0x9d, 0xb9, - 0xc2, 0xf5, 0x18, 0xbb, 0x5f, 0x64, 0x19, 0x0c, 0x4b, 0x00, 0xd2, 0xc7, 0x44, 0x99, 0x35, 0x26, - 0x21, 0xfd, 0xe0, 0x98, 0xc4, 0x60, 0xad, 0x87, 0x70, 0xe1, 0x87, 0x60, 0x96, 0x59, 0xbb, 0xd5, - 0xbe, 0xc5, 0xf2, 0x7d, 0xad, 0xd3, 0xce, 0x2c, 0xcb, 0x5a, 0xfd, 0x92, 0xc7, 0xe0, 0xdd, 0xc3, - 0x74, 0xcb, 0xfd, 0x16, 0x63, 0x27, 0x0f, 0xd3, 0xaf, 0xb9, 0x0c, 0x86, 0x7b, 0x60, 0x91, 0x3e, - 0xc6, 0xeb, 0x7d, 0x9b, 0x29, 0x64, 0x3a, 0xed, 0xcc, 0x0d, 0x49, 0x61, 0xa0, 0xe8, 0x83, 0x4c, - 0xb8, 0x05, 0xe6, 0xf3, 0xbc, 0x14, 0x39, 0x8f, 0xda, 0x43, 0xf5, 0x9d, 0xe4, 0x6f, 0xa7, 0x5b, - 0x2a, 0x87, 0x03, 0x34, 0x94, 0x60, 0xd0, 0x2f, 0x3a, 0x6e, 0x29, 0x13, 0x9b, 0x60, 0x75, 0x83, - 0x09, 0x49, 0x09, 0x4e, 0x08, 0xe9, 0x11, 0x85, 0x69, 0x68, 0x18, 0x79, 0x50, 0xd3, 0x0a, 0x8e, - 0xb0, 0xaf, 0xfe, 0xff, 0x79, 0x9a, 0x84, 0xc2, 0x06, 0x34, 0x19, 0x19, 0x3e, 0x04, 0x73, 0x65, - 0xdf, 0x6e, 0x44, 0xf5, 0x80, 0xe4, 0x82, 0xa6, 0x4f, 0xd4, 0x7b, 0xac, 0x17, 0x4a, 0x65, 0x8b, - 0x84, 0x5b, 0x77, 0xa8, 0x5f, 0x43, 0x71, 0x3c, 0x34, 0xc1, 0xe2, 0xd3, 0x66, 0x40, 0xec, 0x2d, - 0xdb, 0x39, 0xc2, 0x7e, 0x75, 0xeb, 0x94, 0xe0, 0x48, 0x7d, 0x8f, 0x89, 0xac, 0x76, 0xda, 0x99, - 0xeb, 0x5c, 0xe4, 0x25, 0x85, 0xe8, 0x07, 0x1c, 0xa3, 0x1f, 0x50, 0x90, 0x86, 0x06, 0x89, 0x74, - 0x94, 0x94, 0x42, 0xfc, 0x2c, 0x20, 0x58, 0x7d, 0x98, 0x6c, 0x57, 0x8d, 0x10, 0xeb, 0x27, 0x01, - 0xcd, 0x4e, 0x17, 0x23, 0x67, 0x24, 0x08, 0xc3, 0x66, 0x83, 0xe4, 0xea, 0xd8, 0x39, 0x52, 0x1f, - 0x25, 0x7f, 0xc6, 0xbd, 0x8c, 0x70, 0x94, 0xee, 0x50, 0x98, 0x94, 0x11, 0x89, 0xac, 0xfd, 0x72, - 0x06, 0x4c, 0xee, 0xe1, 0xe3, 0x03, 0x1c, 0xd2, 0x9f, 0x34, 0x9d, 0x82, 0x46, 0x0b, 0x3b, 0x25, - 0x9b, 0xd4, 0xc5, 0x14, 0x94, 0x72, 0x83, 0x89, 0x53, 0xd5, 0x71, 0x0b, 0x3b, 0x7a, 0xc3, 0xa6, - 0xdf, 0x45, 0x0c, 0x0e, 0xef, 0x81, 0xe9, 0xec, 0x21, 0x6d, 0xab, 0xd5, 0x6a, 0xc8, 0x46, 0xd6, - 0xf4, 0xd6, 0x72, 0xa7, 0x9d, 0x59, 0x14, 0xdd, 0x97, 0xba, 0x74, 0xbb, 0x5a, 0x0d, 0x35, 0xd4, - 0xc7, 0xd1, 0x7c, 0x3e, 0xb6, 0x5d, 0xaf, 0x11, 0xb8, 0x3e, 0xd9, 0xb1, 0xac, 0x12, 0x23, 0xcf, - 0x32, 0xb2, 0x94, 0xcf, 0x5a, 0x17, 0xa2, 0xd7, 0x09, 0x69, 0x08, 0x95, 0x41, 0x22, 0xcd, 0xe7, - 0x96, 0x1d, 0x61, 0x3a, 0x6c, 0x71, 0xb2, 0x81, 0x1e, 0xd8, 0x11, 0x16, 0xa3, 0x59, 0x60, 0xe8, - 0x47, 0x48, 0x4f, 0x60, 0x06, 0x87, 0xec, 0xbc, 0xb5, 0xe4, 0x47, 0xc8, 0xce, 0xeb, 0x05, 0x87, - 0xe2, 0xb8, 0x32, 0x18, 0x3e, 0x01, 0x0b, 0xf4, 0x91, 0x77, 0x85, 0x52, 0x18, 0xb4, 0x4e, 0xd5, - 0xcf, 0x15, 0x56, 0x88, 0x37, 0x3a, 0xed, 0x8c, 0x2a, 0x09, 0x88, 0x7e, 0xd2, 0xa0, 0x18, 0x0d, - 0x25, 0x59, 0x30, 0x0b, 0xe6, 0xa8, 0x89, 0x7e, 0x97, 0x5c, 0xe6, 0x0b, 0x2e, 0x23, 0x7d, 0x7e, - 0x4c, 0x86, 0x7d, 0xcf, 0x42, 0x24, 0xce, 0xa0, 0xdd, 0xad, 0xaf, 0x6a, 0xf8, 0x55, 0x96, 0x14, - 0xf5, 0xb3, 0xf1, 0x64, 0x4b, 0x90, 0xc3, 0xc1, 0x02, 0xa6, 0xa1, 0x21, 0x5c, 0xf8, 0x4d, 0xbe, - 0x10, 0xa9, 0xbf, 0xa6, 0x1a, 0x33, 0x77, 0x67, 0x36, 0xd9, 0x5e, 0xb5, 0x49, 0x6d, 0xf2, 0x5a, - 0x44, 0x05, 0x35, 0xc4, 0x77, 0xa7, 0x6d, 0x79, 0x00, 0xd2, 0xe5, 0x47, 0xfd, 0x11, 0x5f, 0x78, - 0x46, 0x8c, 0x51, 0xba, 0x2a, 0xc5, 0x06, 0x21, 0xe5, 0xc4, 0x55, 0x58, 0x55, 0x7e, 0x7c, 0xa6, - 0x0a, 0xaf, 0x4c, 0x82, 0x43, 0x73, 0xda, 0x1b, 0x92, 0x2c, 0x94, 0x9f, 0xa4, 0x92, 0x2d, 0x4d, - 0x88, 0xd0, 0x16, 0xcb, 0x23, 0x89, 0x33, 0x62, 0x12, 0x2c, 0x8e, 0x9f, 0x9e, 0x25, 0xc1, 0xc3, - 0x88, 0x33, 0xa0, 0x35, 0x30, 0x99, 0x59, 0x2c, 0x3f, 0xe3, 0x42, 0x37, 0x3b, 0xed, 0xcc, 0x9b, - 0x31, 0x21, 0xa9, 0x59, 0xf3, 0x90, 0x86, 0xd1, 0x87, 0xa8, 0xb2, 0xf0, 0x7e, 0x7e, 0x01, 0x55, - 0x1e, 0xe5, 0x30, 0x3a, 0xfc, 0xa8, 0x3f, 0xce, 0x58, 0x90, 0xff, 0x4c, 0x8d, 0x9e, 0x67, 0x3c, - 0xb8, 0x18, 0x5e, 0xe6, 0xb3, 0x70, 0xfe, 0x75, 0x06, 0x5f, 0x34, 0x0f, 0x19, 0x0f, 0xdf, 0xef, - 0xcd, 0x43, 0xf6, 0xfa, 0x7f, 0xa7, 0x46, 0x0e, 0x44, 0xfe, 0x76, 0x19, 0x2d, 0x91, 0xd9, 0xbb, - 0xff, 0x33, 0x9a, 0x2c, 0x3e, 0x64, 0x09, 0x0d, 0x0b, 0x89, 0x69, 0xca, 0xde, 0xff, 0x75, 0xea, - 0xbc, 0x71, 0xca, 0xc3, 0x18, 0xa4, 0x0e, 0xe8, 0xb1, 0x90, 0xfe, 0x7b, 0xae, 0x1e, 0x8f, 0x6c, - 0x90, 0xaa, 0x7d, 0x35, 0x0b, 0x26, 0x2d, 0xcc, 0xa6, 0xac, 0x74, 0xf3, 0x50, 0x2e, 0x70, 0xf3, - 0x78, 0x17, 0x5c, 0x2e, 0x60, 0xf2, 0x49, 0x10, 0x1e, 0x89, 0x8b, 0x8a, 0x74, 0xf5, 0xf0, 0xb9, - 0x43, 0x43, 0x5d, 0x08, 0xbd, 0xfb, 0xb0, 0xe6, 0x9b, 0x4a, 0xde, 0x7d, 0x78, 0xb7, 0x65, 0x4e, - 0xba, 0x1e, 0x6e, 0x63, 0xcf, 0x3e, 0x35, 0x6d, 0x82, 0x7d, 0xe7, 0x54, 0xdc, 0x4d, 0xe6, 0xe4, - 0xaf, 0xb3, 0x4a, 0xfd, 0xba, 0xc7, 0x01, 0xfa, 0x31, 0x5d, 0x0f, 0xe3, 0x14, 0xf8, 0x5d, 0x90, - 0x8e, 0x5b, 0xd0, 0x09, 0x6b, 0xf9, 0x73, 0x72, 0xcb, 0x4f, 0xca, 0xe8, 0xe1, 0x89, 0x86, 0x06, - 0x78, 0xf0, 0x63, 0xb0, 0xbc, 0xdf, 0xa8, 0xda, 0x04, 0x57, 0x13, 0x71, 0xcd, 0x31, 0xc1, 0x5b, - 0x9d, 0x76, 0x26, 0xc3, 0x05, 0x9b, 0x1c, 0xa6, 0x0f, 0xc6, 0x37, 0x5c, 0x01, 0x7e, 0x0b, 0x00, - 0x14, 0x34, 0xfd, 0xaa, 0xe9, 0x1e, 0xbb, 0x44, 0x5d, 0x5e, 0x53, 0xd6, 0x27, 0xb6, 0xae, 0x76, - 0xda, 0x19, 0xc8, 0xf5, 0x42, 0xea, 0xd3, 0x3d, 0xea, 0xd4, 0x90, 0x84, 0x84, 0x8f, 0xc0, 0x9c, - 0xd1, 0x72, 0x49, 0xd1, 0xa7, 0xf3, 0xa9, 0x19, 0x62, 0xf5, 0xea, 0x40, 0x3f, 0x6f, 0xb9, 0x44, - 0x0f, 0x7c, 0xbd, 0xc6, 0x01, 0xb4, 0x9f, 0xcb, 0x04, 0xb8, 0x03, 0xd2, 0xb9, 0xc0, 0x8f, 0xd8, - 0x52, 0xed, 0x9c, 0xf2, 0x21, 0xbf, 0x92, 0x9c, 0x2d, 0x4e, 0x1f, 0xd1, 0x1d, 0xf0, 0x03, 0x2c, - 0x78, 0x1f, 0xcc, 0x18, 0xbe, 0x7d, 0xe0, 0xe1, 0x52, 0x23, 0x0c, 0x6a, 0xe2, 0x5e, 0xb3, 0xd2, - 0x69, 0x67, 0x96, 0x44, 0x24, 0xcc, 0xa9, 0x37, 0xa8, 0x97, 0x0e, 0xb8, 0x3e, 0x96, 0x96, 0x5a, - 0xc4, 0xc3, 0xf2, 0xb2, 0x47, 0x6f, 0x02, 0x89, 0x52, 0x8b, 0xf8, 0x45, 0x4a, 0x59, 0xa9, 0xe3, - 0x14, 0xba, 0x5b, 0x0a, 0x4b, 0xb9, 0xde, 0xac, 0xd5, 0x3c, 0xac, 0xae, 0x25, 0x93, 0xd1, 0x15, - 0x89, 0x38, 0xa0, 0xaf, 0x21, 0x18, 0xf0, 0x03, 0x30, 0x2b, 0x2c, 0x39, 0x3b, 0xc2, 0x91, 0x7a, - 0x93, 0x5d, 0x24, 0xa4, 0xaf, 0xbb, 0xab, 0xe0, 0x50, 0xb7, 0x86, 0x62, 0x68, 0xb8, 0x2b, 0x2d, - 0x18, 0xb9, 0xe0, 0xf8, 0xd8, 0xf6, 0xab, 0x91, 0xaa, 0x25, 0xef, 0xad, 0xfd, 0x05, 0xc3, 0x11, - 0x18, 0x79, 0xbf, 0xe8, 0xf2, 0x68, 0x4e, 0x50, 0xd3, 0xf7, 0x71, 0xd8, 0xdb, 0x91, 0xde, 0x49, - 0x0e, 0xa7, 0x90, 0xf9, 0xe5, 0x2d, 0x29, 0x41, 0xa1, 0x17, 0x69, 0xa3, 0x45, 0x70, 0xe8, 0xdb, - 0x5e, 0x4f, 0x86, 0x2f, 0xca, 0x52, 0x40, 0x58, 0x20, 0x64, 0xa1, 0x01, 0x1a, 0x2d, 0x6f, 0x99, - 0x84, 0x38, 0x8a, 0xac, 0xd3, 0x06, 0x8e, 0x54, 0xcc, 0x8e, 0x25, 0x95, 0x37, 0x62, 0x4e, 0x9d, - 0x50, 0xaf, 0x86, 0x64, 0x2c, 0xfd, 0x95, 0xf2, 0xc7, 0x5d, 0x7c, 0x5a, 0x76, 0x3f, 0xc5, 0x6c, - 0xfb, 0x99, 0x90, 0x0b, 0x23, 0xc8, 0xb4, 0x6d, 0x46, 0xee, 0xa7, 0xf4, 0x57, 0x1a, 0x23, 0xd0, - 0xad, 0x23, 0x66, 0x30, 0xed, 0xf0, 0x10, 0xab, 0x87, 0x4c, 0x46, 0x5a, 0x46, 0x13, 0x32, 0xba, - 0x47, 0x61, 0x1a, 0x1a, 0xc2, 0x85, 0xcf, 0xc0, 0x95, 0xbe, 0xb5, 0x59, 0xab, 0xb9, 0x2d, 0x64, - 0xfb, 0x87, 0x58, 0xad, 0x33, 0x4d, 0xad, 0xd3, 0xce, 0xac, 0x0e, 0x6a, 0x32, 0x9c, 0x1e, 0x52, - 0xa0, 0x86, 0x86, 0xf2, 0xe1, 0xf7, 0xc0, 0xca, 0x30, 0xbb, 0xd5, 0xf2, 0x55, 0x97, 0x49, 0x4b, - 0xb7, 0xd2, 0x11, 0xd2, 0x3a, 0x69, 0xf9, 0x1a, 0x1a, 0x25, 0x43, 0xb7, 0xc1, 0x9e, 0xcb, 0x6a, - 0xf9, 0xc5, 0x46, 0xa4, 0x7e, 0x9f, 0x29, 0x4b, 0x25, 0x95, 0x94, 0x49, 0xcb, 0xd7, 0x83, 0x46, - 0xa4, 0xa1, 0x24, 0xab, 0x5f, 0x16, 0x3e, 0xa4, 0x23, 0xbe, 0x54, 0x4e, 0xc4, 0x6e, 0x28, 0x5c, - 0x87, 0x8f, 0xf7, 0xa8, 0x57, 0x16, 0x41, 0x80, 0xef, 0x81, 0x69, 0x6e, 0x78, 0x5a, 0x2a, 0xf3, - 0x5d, 0x72, 0x42, 0xde, 0xc3, 0x05, 0xfb, 0x25, 0x7d, 0x7b, 0x1f, 0xa8, 0xfd, 0x40, 0x01, 0x97, - 0x11, 0x7e, 0xd9, 0xc4, 0x11, 0x81, 0x9b, 0x60, 0xba, 0xd8, 0xc0, 0xa1, 0x4d, 0xdc, 0xc0, 0x67, - 0x83, 0x66, 0xfe, 0x6e, 0x5a, 0x2c, 0x80, 0x3d, 0x3b, 0xea, 0x43, 0xe0, 0xed, 0xee, 0x0d, 0x42, - 0xe5, 0xdb, 0xe2, 0x9c, 0x00, 0x73, 0x23, 0xea, 0x5e, 0x2f, 0x6e, 0x77, 0xe7, 0x18, 0x1b, 0x31, - 0x7d, 0x18, 0x37, 0x22, 0xe1, 0xd4, 0x1c, 0x30, 0x85, 0x70, 0xd4, 0x08, 0xfc, 0x08, 0x43, 0x15, - 0x5c, 0x2e, 0x37, 0x1d, 0x07, 0x47, 0x11, 0x8b, 0x63, 0x0a, 0x75, 0x1f, 0xe1, 0x55, 0x30, 0x49, - 0x6f, 0x89, 0xcd, 0x88, 0x8f, 0x36, 0x24, 0x9e, 0xa4, 0x58, 0x52, 0x67, 0xc4, 0xb2, 0xf1, 0x17, - 0x45, 0x3a, 0x23, 0x9c, 0x07, 0xa0, 0x10, 0x90, 0x32, 0xb1, 0x43, 0x82, 0xab, 0xe9, 0x31, 0x78, - 0x05, 0xa4, 0xc5, 0x55, 0x89, 0xd9, 0xe8, 0x7a, 0x9b, 0x56, 0xe0, 0x02, 0x98, 0x41, 0x38, 0xea, - 0x19, 0xc6, 0xe1, 0x2c, 0x98, 0xda, 0x75, 0x3d, 0x8f, 0x3d, 0xa5, 0xa8, 0x9b, 0x36, 0x8c, 0x6c, - 0xe8, 0xd4, 0xdd, 0x13, 0x9c, 0xbe, 0x44, 0x55, 0xb6, 0x71, 0x44, 0xc2, 0xe0, 0x94, 0x22, 0xd8, - 0x95, 0x27, 0x3d, 0x01, 0xaf, 0x81, 0xe5, 0x2d, 0xcf, 0x76, 0x8e, 0xea, 0x81, 0xc7, 0xfe, 0x69, - 0xa3, 0x14, 0x84, 0xc4, 0x6a, 0xa1, 0x56, 0xba, 0x0a, 0x6f, 0x80, 0x95, 0x7d, 0xff, 0x60, 0xa8, - 0x13, 0xc3, 0x65, 0xb0, 0xc8, 0x9a, 0x6a, 0xcc, 0x5c, 0x83, 0x2b, 0x60, 0x69, 0xdf, 0xaf, 0x0e, - 0x38, 0x0e, 0x37, 0xfe, 0x3e, 0xc5, 0xe3, 0x11, 0x8d, 0x90, 0xf2, 0x77, 0xf3, 0xa6, 0x59, 0x29, - 0x16, 0x8c, 0xca, 0xe3, 0xa2, 0x69, 0x16, 0x9f, 0x1b, 0x28, 0x3d, 0x06, 0xdf, 0x05, 0xeb, 0x03, - 0xe6, 0xca, 0x7e, 0xc1, 0xca, 0x9b, 0x15, 0x0b, 0xe5, 0x9f, 0x3c, 0x31, 0x50, 0xa5, 0x5c, 0xc8, - 0x96, 0xca, 0x3b, 0x45, 0x8b, 0xa7, 0x80, 0xa1, 0x4d, 0x23, 0xbb, 0x6d, 0xa0, 0xf4, 0x38, 0x7c, - 0x1b, 0x68, 0x92, 0x61, 0x14, 0x31, 0xd5, 0x23, 0x3e, 0xdd, 0x2f, 0xa2, 0xfd, 0xbd, 0xf4, 0x25, - 0x96, 0x3b, 0x6a, 0xc8, 0x9a, 0x66, 0x7a, 0x02, 0x6e, 0x80, 0xb7, 0xb7, 0xcc, 0x6c, 0x6e, 0x77, - 0xa7, 0x68, 0x1a, 0x95, 0x92, 0x61, 0xa0, 0x4a, 0xa9, 0x88, 0xac, 0x8a, 0xf5, 0xa2, 0x82, 0x5e, - 0xc4, 0x23, 0xae, 0xc2, 0x2c, 0xf8, 0xf0, 0x62, 0xd8, 0x51, 0xd1, 0x60, 0xf8, 0x16, 0x58, 0x1b, - 0x2d, 0x21, 0xce, 0x56, 0x83, 0xef, 0x83, 0x6f, 0x9f, 0x87, 0x1a, 0xf5, 0x8a, 0xc3, 0xb3, 0x5f, - 0x21, 0xb2, 0x50, 0x87, 0x37, 0xc1, 0x9b, 0xa3, 0x51, 0x34, 0x35, 0x2e, 0xfc, 0x3f, 0xa0, 0x6d, - 0x1b, 0x66, 0xf6, 0xe3, 0xb3, 0xd3, 0xf2, 0x4a, 0x81, 0x9b, 0xe0, 0x1d, 0x94, 0x2d, 0x6c, 0x17, - 0xf7, 0x2a, 0x17, 0xc0, 0x7f, 0xae, 0xc0, 0x8f, 0xc0, 0xfd, 0xf3, 0x81, 0xa3, 0x0e, 0xf8, 0x85, - 0x02, 0x0d, 0xf0, 0xe8, 0xc2, 0xef, 0x1b, 0x25, 0xf3, 0x1b, 0x05, 0xde, 0x04, 0x6f, 0x0c, 0xe7, - 0x8b, 0x3a, 0xfc, 0x56, 0x81, 0xeb, 0xe0, 0xd6, 0x99, 0x6f, 0x12, 0xc8, 0xdf, 0x29, 0xf0, 0x3b, - 0xe0, 0xde, 0x59, 0x90, 0x51, 0x61, 0xfc, 0x5e, 0x81, 0x0f, 0xc1, 0x83, 0x0b, 0xbc, 0x63, 0x94, - 0xc0, 0x1f, 0xce, 0x38, 0x87, 0x28, 0xf6, 0x97, 0xe7, 0x9f, 0x43, 0x20, 0xff, 0xa8, 0xc0, 0x55, - 0x70, 0x6d, 0x38, 0x84, 0xfe, 0x26, 0xfe, 0xa4, 0xc0, 0xdb, 0x60, 0xed, 0x4c, 0x25, 0x0a, 0xfb, - 0xb3, 0x02, 0x55, 0xb0, 0x54, 0x28, 0x56, 0x1e, 0x67, 0xf3, 0x66, 0xe5, 0x79, 0xde, 0xda, 0xa9, - 0x94, 0x2d, 0x64, 0x94, 0xcb, 0xe9, 0x5f, 0x8d, 0xd3, 0x50, 0x62, 0x9e, 0x42, 0x51, 0x38, 0x2b, - 0x8f, 0x8b, 0xa8, 0x62, 0xe6, 0x9f, 0x19, 0x05, 0x8a, 0xfc, 0x6c, 0x1c, 0x2e, 0x00, 0x40, 0x61, - 0xa5, 0x62, 0xbe, 0x60, 0x95, 0xd3, 0x3f, 0x4c, 0xc1, 0x39, 0x30, 0x65, 0xbc, 0xb0, 0x0c, 0x54, - 0xc8, 0x9a, 0xe9, 0x7f, 0xa4, 0x36, 0x02, 0x00, 0xfa, 0x6b, 0x05, 0x9c, 0x04, 0xe3, 0xbb, 0xcf, - 0xd2, 0x63, 0x70, 0x1a, 0x4c, 0x98, 0x46, 0xb6, 0x6c, 0xa4, 0x15, 0xb8, 0x04, 0x16, 0x0c, 0xd3, - 0xc8, 0x59, 0xf9, 0x62, 0xa1, 0x82, 0xf6, 0x0b, 0x05, 0xd6, 0x37, 0xd2, 0x60, 0xf6, 0x79, 0xd6, - 0xca, 0xed, 0x74, 0x2d, 0x29, 0xda, 0x9f, 0xcc, 0x62, 0x6e, 0xb7, 0x82, 0xb2, 0x39, 0x03, 0x75, - 0xcd, 0x97, 0x28, 0x90, 0x09, 0x75, 0x2d, 0x13, 0x77, 0x1f, 0x82, 0x69, 0x2b, 0xb4, 0xfd, 0xa8, - 0x11, 0x84, 0x04, 0xde, 0x95, 0x1f, 0xe6, 0x45, 0xaf, 0x17, 0x73, 0xec, 0xfa, 0x42, 0xef, 0x99, - 0x8f, 0x13, 0x6d, 0x6c, 0x5d, 0xf9, 0x86, 0xb2, 0x75, 0xe5, 0xd5, 0xdf, 0x56, 0xc7, 0x5e, 0xbd, - 0x5e, 0x55, 0xbe, 0x7c, 0xbd, 0xaa, 0xfc, 0xf5, 0xf5, 0xaa, 0xf2, 0x8b, 0xaf, 0x56, 0xc7, 0x0e, - 0x26, 0xd9, 0x7f, 0x05, 0xdd, 0xfb, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x42, 0xd4, 0x02, - 0x53, 0x1a, 0x00, 0x00, + // 2476 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0x5b, 0x77, 0xdb, 0xc6, + 0x11, 0x16, 0x44, 0xcb, 0xb6, 0x56, 0x37, 0x6a, 0x75, 0x31, 0x2c, 0xdb, 0x82, 0x0c, 0xc7, 0xa9, + 0xad, 0x14, 0x72, 0x6b, 0xe7, 0xb4, 0xcd, 0xd5, 0x01, 0x29, 0x58, 0x62, 0x05, 0x91, 0xf4, 0x12, + 0xb2, 0x93, 0x27, 0x16, 0x22, 0x57, 0x12, 0x6a, 0x0a, 0xa0, 0x81, 0xa5, 0x23, 0xe5, 0x0f, 0xf4, + 0xb5, 0xf7, 0xd3, 0xa7, 0xfe, 0x82, 0xa6, 0xbf, 0xc3, 0x4e, 0x6f, 0x69, 0xfb, 0xce, 0xd3, 0x3a, + 0xff, 0x80, 0xa7, 0x6d, 0x9a, 0x3e, 0xf5, 0xec, 0x05, 0xe2, 0x02, 0x20, 0x25, 0xbd, 0x09, 0x33, + 0xdf, 0xf7, 0xed, 0xec, 0xce, 0xee, 0xcc, 0x48, 0x02, 0x33, 0x61, 0xbb, 0xd1, 0xde, 0xbd, 0x17, + 0xb6, 0x1b, 0x6b, 0xed, 0x30, 0x20, 0x01, 0x1c, 0x63, 0x86, 0x25, 0x63, 0xdf, 0x23, 0x07, 0x9d, + 0xdd, 0xb5, 0x46, 0x70, 0x78, 0x6f, 0x3f, 0xd8, 0x0f, 0xee, 0x31, 0xef, 0x6e, 0x67, 0x8f, 0x7d, + 0xb1, 0x0f, 0xf6, 0x13, 0x67, 0xe9, 0x3f, 0x51, 0xc0, 0x25, 0x84, 0x9f, 0x77, 0x70, 0x44, 0xe0, + 0x1a, 0x18, 0xaf, 0xb4, 0x71, 0xe8, 0x12, 0x2f, 0xf0, 0x55, 0x65, 0x45, 0xb9, 0x33, 0x7d, 0x3f, + 0xbf, 0xc6, 0x54, 0xd7, 0x4e, 0xec, 0xa8, 0x0f, 0x81, 0xb7, 0xc1, 0xc5, 0x6d, 0x7c, 0xb8, 0x8b, + 0x43, 0x75, 0x74, 0x45, 0xb9, 0x33, 0x71, 0x7f, 0x4a, 0x80, 0xb9, 0x11, 0x09, 0x27, 0x85, 0x39, + 0x38, 0x22, 0x38, 0x54, 0x73, 0x09, 0x18, 0x37, 0x22, 0xe1, 0xd4, 0x1b, 0xe0, 0x32, 0xc2, 0x51, + 0x3b, 0xf0, 0x23, 0x0c, 0x55, 0x70, 0xa9, 0xd6, 0x69, 0x34, 0x70, 0x14, 0xb1, 0x38, 0x2e, 0xa3, + 0xf8, 0x13, 0x2e, 0x82, 0x8b, 0x35, 0xe2, 0x92, 0x4e, 0xc4, 0xd6, 0x1c, 0x47, 0xe2, 0x4b, 0x8a, + 0x25, 0x77, 0x4a, 0x2c, 0xfa, 0x6f, 0x27, 0x62, 0x1c, 0xfc, 0x00, 0x4c, 0x5a, 0xa4, 0xd1, 0xb4, + 0x8e, 0x70, 0xa3, 0xea, 0x92, 0x03, 0xb6, 0xd0, 0x78, 0xe1, 0x6a, 0xaf, 0xab, 0x2d, 0x1c, 0xbb, + 0x87, 0xad, 0x77, 0x75, 0x4c, 0x1a, 0x4d, 0x03, 0x1f, 0xe1, 0x86, 0xd1, 0x76, 0xc9, 0x81, 0x8e, + 0x12, 0x70, 0xf8, 0x00, 0x8c, 0x9b, 0xfb, 0xd8, 0x27, 0x66, 0xb3, 0x19, 0xaa, 0x13, 0x8c, 0xbb, + 0xd0, 0xeb, 0x6a, 0xb3, 0x9c, 0xeb, 0x52, 0x97, 0xe1, 0x36, 0x9b, 0xa1, 0x8e, 0xfa, 0x38, 0x68, + 0x83, 0xd9, 0x47, 0xae, 0xd7, 0x6a, 0x07, 0x9e, 0x4f, 0x36, 0x1d, 0xa7, 0xca, 0xc8, 0x93, 0x8c, + 0xbc, 0xdc, 0xeb, 0x6a, 0x4b, 0x9c, 0xbc, 0x17, 0x43, 0x8c, 0x03, 0x42, 0xda, 0x42, 0x25, 0x4b, + 0x84, 0x06, 0xb8, 0x54, 0x70, 0x23, 0xbc, 0xee, 0x85, 0x2a, 0x66, 0x1a, 0x73, 0xbd, 0xae, 0x36, + 0xc3, 0x35, 0x76, 0xdd, 0x08, 0x1b, 0x4d, 0x2f, 0xd4, 0x51, 0x8c, 0x81, 0xef, 0x82, 0x09, 0xba, + 0x03, 0x3b, 0xd8, 0x67, 0xfb, 0xdd, 0x63, 0x14, 0xb5, 0xd7, 0xd5, 0xe6, 0xa5, 0xfd, 0xb6, 0x82, + 0x7d, 0xb1, 0x5d, 0x19, 0x0c, 0x37, 0xc0, 0x0c, 0xfd, 0x2c, 0xb6, 0x3c, 0xec, 0x93, 0x6a, 0x18, + 0x1c, 0x1d, 0xab, 0xaf, 0x58, 0x66, 0x0a, 0xd7, 0x7b, 0x5d, 0x4d, 0x95, 0x04, 0x1a, 0x0c, 0x62, + 0xb4, 0x29, 0x46, 0x47, 0x69, 0x16, 0x34, 0xc1, 0x14, 0x35, 0x55, 0x31, 0x0e, 0xb9, 0xcc, 0x17, + 0x5c, 0x66, 0xa9, 0xd7, 0xd5, 0x16, 0x25, 0x99, 0x36, 0xc6, 0x61, 0x2c, 0x92, 0x64, 0xc0, 0x2a, + 0x80, 0x7d, 0x55, 0xcb, 0x6f, 0xb2, 0x43, 0x51, 0x3f, 0x67, 0xf7, 0xa1, 0xa0, 0xf5, 0xba, 0xda, + 0xb5, 0x6c, 0x38, 0x58, 0xc0, 0x74, 0x34, 0x80, 0x0b, 0xbf, 0x0b, 0x2e, 0x50, 0xab, 0xfa, 0x7b, + 0x7e, 0x8f, 0x27, 0xc4, 0xdd, 0xa1, 0xb6, 0xc2, 0x4c, 0xaf, 0xab, 0x4d, 0xf4, 0x05, 0x75, 0xc4, + 0xa0, 0x70, 0x1d, 0x4c, 0x73, 0x91, 0x22, 0x0e, 0xc9, 0xba, 0x4b, 0x5c, 0xf5, 0x67, 0x39, 0x16, + 0xc0, 0xb5, 0x5e, 0x57, 0xbb, 0xc2, 0xf1, 0x62, 0xed, 0x06, 0x0e, 0x89, 0xd1, 0x74, 0x89, 0xab, + 0xa3, 0x14, 0x27, 0xa9, 0xc2, 0xb2, 0xf2, 0xf3, 0x53, 0x55, 0x78, 0x66, 0x52, 0x1c, 0x7a, 0xa6, + 0xdc, 0xb2, 0x85, 0x8f, 0x59, 0x28, 0xbf, 0xe0, 0x22, 0xd2, 0x99, 0x0a, 0x91, 0x67, 0xf8, 0x58, + 0x44, 0x92, 0x64, 0x24, 0x24, 0x58, 0x1c, 0xbf, 0x3c, 0x4d, 0x82, 0x87, 0x91, 0x64, 0x40, 0x07, + 0xcc, 0x71, 0x83, 0x13, 0x76, 0x22, 0x82, 0x9b, 0x45, 0x93, 0xc5, 0xf2, 0x2b, 0x2e, 0x74, 0xb3, + 0xd7, 0xd5, 0x6e, 0x24, 0x84, 0x08, 0x87, 0x19, 0x0d, 0x57, 0x84, 0x34, 0x88, 0x3e, 0x40, 0x95, + 0x85, 0xf7, 0xeb, 0x73, 0xa8, 0xf2, 0x28, 0x07, 0xd1, 0xe1, 0x87, 0x60, 0x92, 0xde, 0xa7, 0x93, + 0xdc, 0xfd, 0x3b, 0x97, 0x7e, 0xfc, 0xec, 0xfe, 0x49, 0x99, 0x4b, 0xe0, 0x65, 0x3e, 0x0b, 0xe7, + 0x3f, 0xa7, 0xf0, 0x45, 0xf1, 0x90, 0xf1, 0xf0, 0x3d, 0x30, 0x41, 0xbf, 0xe3, 0x7c, 0x7d, 0x9d, + 0x4b, 0xbf, 0x45, 0x46, 0xef, 0x67, 0x4b, 0x46, 0x4b, 0x64, 0xb6, 0xf6, 0x7f, 0x87, 0x93, 0xc5, + 0x43, 0x96, 0xd0, 0xb0, 0x0c, 0x66, 0xe9, 0x67, 0x32, 0x47, 0xdf, 0xe4, 0xd2, 0x6f, 0x87, 0x49, + 0x64, 0x32, 0x94, 0xa5, 0x66, 0xf4, 0x58, 0x48, 0xff, 0x3b, 0x53, 0x8f, 0x47, 0x96, 0xa5, 0xea, + 0x5f, 0x4d, 0xc6, 0xdd, 0x82, 0x96, 0x37, 0xba, 0x04, 0x2d, 0x6f, 0x4a, 0xba, 0xbc, 0xd1, 0x78, + 0x44, 0x79, 0x13, 0x18, 0xf8, 0x6d, 0x70, 0xa9, 0x8c, 0xc9, 0xa7, 0x41, 0xf8, 0x8c, 0xb7, 0x86, + 0x02, 0xec, 0x75, 0xb5, 0x69, 0x0e, 0xf7, 0xb9, 0x43, 0x47, 0x31, 0x04, 0xde, 0x02, 0x17, 0x58, + 0xf1, 0xe5, 0x91, 0x4a, 0x8f, 0x9c, 0x57, 0x5b, 0xe6, 0x84, 0x45, 0x30, 0xbd, 0x8e, 0x5b, 0xee, + 0xb1, 0xed, 0x12, 0xec, 0x37, 0x8e, 0xb7, 0x23, 0x56, 0xe8, 0xa7, 0xe4, 0xd7, 0xd9, 0xa4, 0x7e, + 0xa3, 0xc5, 0x01, 0xc6, 0x61, 0xa4, 0xa3, 0x14, 0x05, 0xfe, 0x10, 0xe4, 0x93, 0x16, 0xf4, 0x82, + 0x95, 0xfc, 0x29, 0xb9, 0xe4, 0xa7, 0x65, 0x8c, 0xf0, 0x85, 0x8e, 0x32, 0x3c, 0xf8, 0x09, 0x58, + 0xd8, 0x69, 0x37, 0x5d, 0x82, 0x9b, 0xa9, 0xb8, 0xa6, 0x98, 0xe0, 0xad, 0x5e, 0x57, 0xd3, 0xb8, + 0x60, 0x87, 0xc3, 0x8c, 0x6c, 0x7c, 0x83, 0x15, 0xe0, 0xf7, 0x00, 0x40, 0x41, 0xc7, 0x6f, 0xda, + 0xde, 0xa1, 0x47, 0xd4, 0x85, 0x15, 0xe5, 0xce, 0x58, 0x61, 0xb1, 0xd7, 0xd5, 0x20, 0xd7, 0x0b, + 0xa9, 0xcf, 0x68, 0x51, 0xa7, 0x8e, 0x24, 0x24, 0xfc, 0x08, 0x4c, 0x59, 0x47, 0x1e, 0xa9, 0xf8, + 0xb4, 0x3f, 0x75, 0x42, 0xac, 0x2e, 0x66, 0xea, 0xf9, 0x91, 0x47, 0x8c, 0xc0, 0x37, 0xf6, 0x38, + 0x80, 0xd6, 0x73, 0x99, 0x00, 0x37, 0x41, 0xbe, 0x18, 0xf8, 0x91, 0x17, 0xb1, 0x50, 0x8a, 0x07, + 0xb8, 0xf1, 0x4c, 0xbd, 0x92, 0xee, 0x2d, 0x8d, 0x3e, 0xc2, 0x68, 0x50, 0x88, 0x8e, 0x32, 0x2c, + 0xf8, 0x0e, 0x98, 0xb0, 0x7c, 0x77, 0xb7, 0x85, 0xab, 0xed, 0x30, 0xd8, 0x53, 0x55, 0x26, 0x72, + 0xa5, 0xd7, 0xd5, 0xe6, 0x44, 0x24, 0xcc, 0x69, 0xb4, 0xa9, 0x97, 0x36, 0xb8, 0x3e, 0x96, 0xa6, + 0x5a, 0xc4, 0xc3, 0xce, 0x65, 0x3b, 0x52, 0xb5, 0x74, 0xaa, 0x45, 0xfc, 0xe2, 0x48, 0x59, 0xaa, + 0x93, 0x14, 0x58, 0x38, 0x11, 0xa9, 0x1d, 0x74, 0xf6, 0xf6, 0x5a, 0x58, 0x5d, 0x49, 0x1f, 0x46, + 0x2c, 0x12, 0x71, 0x40, 0x5f, 0x43, 0x30, 0xe0, 0xfb, 0x60, 0x52, 0x58, 0x8a, 0x6e, 0x84, 0x23, + 0xf5, 0xe6, 0x4a, 0x2e, 0xf9, 0xba, 0x63, 0x85, 0x06, 0x75, 0xeb, 0x28, 0x81, 0x86, 0x5b, 0xd2, + 0x80, 0x51, 0x0c, 0x0e, 0x0f, 0x5d, 0xbf, 0x19, 0xa9, 0x3a, 0x93, 0xb8, 0xd1, 0xeb, 0x6a, 0x57, + 0xd3, 0x03, 0x46, 0x43, 0x60, 0xe4, 0xf9, 0x22, 0xe6, 0xd1, 0x33, 0x41, 0x1d, 0xdf, 0xc7, 0xe1, + 0xc9, 0x8c, 0x74, 0x37, 0xdd, 0x9c, 0x42, 0xe6, 0x97, 0xa7, 0xa4, 0x14, 0x05, 0x96, 0x40, 0xde, + 0x3a, 0x22, 0x38, 0xf4, 0xdd, 0xd6, 0x89, 0xcc, 0x2a, 0x93, 0x91, 0x02, 0xc2, 0x02, 0x21, 0x0b, + 0x65, 0x68, 0x34, 0xbd, 0x35, 0x12, 0xe2, 0x28, 0x72, 0x8e, 0xdb, 0x38, 0x52, 0x31, 0xdb, 0x96, + 0x94, 0xde, 0x88, 0x39, 0x0d, 0x42, 0xbd, 0x3a, 0x92, 0xb1, 0xf4, 0x96, 0xf2, 0xcf, 0x2d, 0x7c, + 0x5c, 0xf3, 0x3e, 0xc3, 0x6c, 0xfa, 0x19, 0x93, 0x13, 0x23, 0xc8, 0xb4, 0x6c, 0x46, 0xde, 0x67, + 0xf4, 0x96, 0x26, 0x08, 0x74, 0xea, 0x48, 0x18, 0x6c, 0x37, 0xdc, 0xc7, 0xea, 0x3e, 0x93, 0x59, + 0xe9, 0x75, 0xb5, 0xeb, 0x03, 0x65, 0x8c, 0x16, 0x85, 0xe9, 0x68, 0x00, 0x17, 0x3e, 0x01, 0xf3, + 0x7d, 0x6b, 0x67, 0x6f, 0xcf, 0x3b, 0x42, 0xae, 0xbf, 0x8f, 0xd5, 0x03, 0xa6, 0xa9, 0xf7, 0xba, + 0xda, 0x72, 0x56, 0x93, 0xe1, 0x8c, 0x90, 0x02, 0x75, 0x34, 0x90, 0x0f, 0x7f, 0x04, 0xae, 0x0c, + 0xb2, 0x3b, 0x47, 0xbe, 0xea, 0x31, 0xe9, 0x37, 0x7b, 0x5d, 0x4d, 0x3f, 0x55, 0xda, 0x20, 0x47, + 0xbe, 0x8e, 0x86, 0xc9, 0xd0, 0x69, 0xf0, 0xc4, 0xe5, 0x1c, 0xf9, 0x95, 0x76, 0xa4, 0xfe, 0x98, + 0x29, 0x4b, 0x29, 0x95, 0x94, 0xc9, 0x91, 0x6f, 0x04, 0xed, 0x48, 0x47, 0x69, 0x56, 0x3f, 0x2d, + 0xbc, 0x49, 0x47, 0x7c, 0xa8, 0x1c, 0x93, 0x1b, 0xa9, 0xd0, 0xe1, 0xed, 0x3d, 0x3a, 0x49, 0x8b, + 0x20, 0xc0, 0xb7, 0xc1, 0x38, 0x37, 0x3c, 0xae, 0xd6, 0xf8, 0x2c, 0x39, 0x26, 0xcf, 0xe1, 0x82, + 0xfd, 0x9c, 0xae, 0xde, 0x07, 0xea, 0xdf, 0x4c, 0xf1, 0x89, 0x8f, 0xb6, 0x81, 0xb2, 0x7b, 0x88, + 0x45, 0x83, 0x91, 0xda, 0x80, 0xef, 0x1e, 0x62, 0x1d, 0x31, 0xa7, 0xdc, 0x88, 0x46, 0xcf, 0xd1, + 0x88, 0x56, 0xc1, 0xc5, 0xa7, 0xa6, 0x4d, 0xd1, 0xb9, 0x74, 0x1f, 0xfa, 0xd4, 0x6d, 0x71, 0xb0, + 0x40, 0xc0, 0x0a, 0x98, 0xdb, 0xc4, 0x6e, 0x48, 0x76, 0xb1, 0x4b, 0x4a, 0x3e, 0xc1, 0xe1, 0x0b, + 0xb7, 0x25, 0xda, 0x4c, 0x4e, 0x3e, 0xcd, 0x83, 0x18, 0x64, 0x78, 0x02, 0xa5, 0xa3, 0x41, 0x4c, + 0x58, 0x02, 0xb3, 0x56, 0x0b, 0x37, 0xe8, 0xef, 0x67, 0x8e, 0x77, 0x88, 0x83, 0x0e, 0xd9, 0x8e, + 0x58, 0xbb, 0xc9, 0xc9, 0xcf, 0x16, 0x0b, 0x88, 0x41, 0x38, 0x46, 0x47, 0x59, 0x16, 0x7d, 0xb9, + 0x36, 0x2b, 0xaf, 0xfc, 0xac, 0x77, 0x90, 0x1d, 0xa9, 0x0b, 0xe9, 0x52, 0xd2, 0x62, 0x88, 0x78, + 0xcc, 0xee, 0x84, 0xad, 0x48, 0x47, 0x19, 0x1a, 0x44, 0x60, 0xce, 0x6c, 0xbe, 0xc0, 0x21, 0xf1, + 0x22, 0x2c, 0xa9, 0x2d, 0x32, 0x35, 0xe9, 0xf5, 0xb8, 0x31, 0x28, 0x29, 0x38, 0x88, 0x0c, 0xdf, + 0x89, 0x47, 0x56, 0xb3, 0x43, 0x02, 0xc7, 0xae, 0x89, 0x9e, 0x21, 0xe5, 0xc6, 0xed, 0x90, 0xc0, + 0x20, 0x54, 0x20, 0x89, 0xa4, 0x85, 0xad, 0x3f, 0x42, 0x9b, 0x1d, 0x72, 0x20, 0x5a, 0xc5, 0x90, + 0xa9, 0xdb, 0xed, 0xa4, 0xa6, 0x6e, 0x4a, 0x81, 0xef, 0xcb, 0x22, 0x8f, 0xbc, 0x16, 0x56, 0xaf, + 0xb2, 0x74, 0xcf, 0xf7, 0xba, 0x5a, 0x5e, 0x88, 0x50, 0xf6, 0x9e, 0xc7, 0xca, 0x7c, 0x12, 0xdb, + 0x8f, 0x7e, 0x0b, 0x1f, 0x33, 0xf2, 0x52, 0xfa, 0x66, 0xd1, 0x97, 0xc3, 0xb9, 0x49, 0x24, 0xb4, + 0x33, 0x23, 0x31, 0x13, 0xb8, 0x96, 0x1e, 0xd8, 0xa5, 0x71, 0x8b, 0xeb, 0x0c, 0xa2, 0xd1, 0xb3, + 0xe0, 0xe9, 0xa2, 0xb3, 0x18, 0xcb, 0x8a, 0xc6, 0xb2, 0x22, 0x9d, 0x85, 0xc8, 0x31, 0x9b, 0xe1, + 0x78, 0x42, 0x52, 0x14, 0xe8, 0x80, 0xd9, 0x93, 0x14, 0x9d, 0xe8, 0xac, 0x30, 0x1d, 0xa9, 0xd8, + 0x78, 0xbe, 0x47, 0x3c, 0xb7, 0x65, 0xf4, 0xb3, 0x2c, 0x49, 0x66, 0x05, 0xe8, 0x2f, 0xac, 0xf4, + 0xe7, 0x38, 0xbf, 0x37, 0x59, 0x8e, 0xd2, 0x73, 0x6e, 0x3f, 0xc9, 0x32, 0x98, 0x96, 0x6b, 0x36, + 0x71, 0x27, 0xd3, 0xac, 0x33, 0x09, 0xe9, 0xc2, 0xf1, 0x31, 0x3d, 0x93, 0xeb, 0x01, 0x5c, 0xf8, + 0x41, 0x7f, 0xe6, 0x67, 0xe7, 0x7d, 0x6b, 0xf8, 0xc8, 0xcf, 0x8f, 0x3b, 0x01, 0x8f, 0x37, 0x13, + 0xa7, 0xfb, 0x8d, 0xa1, 0x43, 0x3b, 0x27, 0xcb, 0x60, 0xb8, 0x9d, 0x1a, 0xb2, 0x99, 0xc2, 0xed, + 0xb3, 0x66, 0x6c, 0x2e, 0x94, 0x65, 0xd2, 0x31, 0xa5, 0xc4, 0x53, 0x51, 0x6c, 0x75, 0xd8, 0x1f, + 0x66, 0xee, 0xa6, 0xef, 0x4e, 0x9c, 0xaa, 0x06, 0x07, 0xe8, 0x28, 0xc5, 0xa0, 0x2f, 0x3a, 0x69, + 0xa9, 0x11, 0x97, 0x60, 0xd1, 0xd9, 0xa5, 0x03, 0x4e, 0x09, 0x19, 0x11, 0x85, 0xe9, 0x68, 0x10, + 0x39, 0xab, 0xe9, 0x04, 0xcf, 0xb0, 0xaf, 0xbe, 0x75, 0x96, 0x26, 0xa1, 0xb0, 0x8c, 0x26, 0x23, + 0xc3, 0x87, 0x60, 0xaa, 0xe6, 0xbb, 0xed, 0xe8, 0x20, 0x20, 0xc5, 0xa0, 0xe3, 0x13, 0xf5, 0x01, + 0xab, 0x85, 0x72, 0x83, 0x11, 0x6e, 0xa3, 0x41, 0xfd, 0xb4, 0xc1, 0xc8, 0x78, 0x68, 0x83, 0xd9, + 0xc7, 0x9d, 0x80, 0xb8, 0x05, 0xb7, 0xf1, 0x0c, 0xfb, 0xcd, 0xc2, 0x31, 0xc1, 0x91, 0xfa, 0x36, + 0x13, 0x91, 0xe6, 0xf7, 0xe7, 0x14, 0x62, 0xec, 0x72, 0x8c, 0xb1, 0x4b, 0x41, 0x3a, 0xca, 0x12, + 0x69, 0x2b, 0xa9, 0x86, 0xf8, 0x49, 0x40, 0xb0, 0xfa, 0x30, 0x5d, 0xae, 0xda, 0x21, 0x36, 0x5e, + 0x04, 0xf4, 0x74, 0x62, 0x8c, 0x7c, 0x22, 0x41, 0x18, 0x76, 0xda, 0x84, 0x4f, 0xc7, 0x1f, 0xa5, + 0xaf, 0xf1, 0xc9, 0x89, 0x70, 0x54, 0x3c, 0x21, 0x0f, 0x22, 0xaf, 0xbe, 0x1c, 0x95, 0xfe, 0xcc, + 0x07, 0x67, 0xc0, 0x44, 0xb9, 0xe2, 0xd4, 0x6b, 0x8e, 0x89, 0x1c, 0x6b, 0x3d, 0x3f, 0x02, 0x17, + 0x01, 0x2c, 0x95, 0x4b, 0x4e, 0xc9, 0xb4, 0xb9, 0xb1, 0x6e, 0x39, 0xc5, 0xf5, 0x3c, 0x80, 0x79, + 0x30, 0x89, 0x2c, 0xc9, 0x32, 0x41, 0x2d, 0xb5, 0xd2, 0x86, 0x63, 0xa1, 0x6d, 0x6e, 0x99, 0x87, + 0x2b, 0xe0, 0x7a, 0xad, 0xb4, 0xf1, 0x78, 0xa7, 0xc4, 0x31, 0x75, 0xb3, 0xbc, 0x5e, 0x47, 0xd6, + 0x76, 0xe5, 0x89, 0x55, 0x5f, 0x37, 0x1d, 0x33, 0xbf, 0x00, 0x6f, 0x82, 0x1b, 0x19, 0x84, 0x89, + 0x8a, 0x9b, 0xa5, 0x18, 0xb2, 0x0c, 0xef, 0x81, 0xb7, 0x4e, 0x13, 0x61, 0xdf, 0x35, 0xa7, 0x52, + 0xad, 0x9b, 0x1b, 0x56, 0xd9, 0xc9, 0x6b, 0xf0, 0x06, 0xb8, 0x5a, 0xb0, 0xcd, 0xe2, 0xd6, 0x66, + 0xc5, 0xb6, 0xea, 0x55, 0xcb, 0x42, 0xf5, 0x6a, 0x05, 0x39, 0x75, 0xe7, 0xe3, 0x3a, 0xfa, 0x38, + 0xdf, 0x84, 0x1a, 0xb8, 0xb6, 0x53, 0x1e, 0x0e, 0xc0, 0x70, 0x09, 0x2c, 0xac, 0x5b, 0xb6, 0xf9, + 0x49, 0xc6, 0xf5, 0x52, 0x81, 0xd7, 0xc1, 0x95, 0x9d, 0xf2, 0x60, 0xef, 0x2b, 0x65, 0xf5, 0xeb, + 0xcb, 0x60, 0x42, 0x1a, 0xbf, 0xa1, 0x0a, 0xe6, 0xe3, 0x13, 0xa9, 0x94, 0xad, 0xfa, 0xa3, 0x8a, + 0x6d, 0x57, 0x9e, 0x5a, 0x28, 0x3f, 0x22, 0x36, 0x95, 0xf1, 0xd4, 0x77, 0xca, 0x4e, 0xc9, 0xae, + 0x3b, 0xa8, 0xb4, 0xb1, 0x61, 0xa1, 0x7a, 0xad, 0x6c, 0x56, 0x6b, 0x9b, 0x15, 0x27, 0xaf, 0x40, + 0x08, 0xa6, 0x63, 0x82, 0x6d, 0x99, 0xeb, 0x16, 0xca, 0x8f, 0xc2, 0xbb, 0xe0, 0x76, 0xd2, 0x36, + 0x8c, 0x9e, 0x93, 0xe9, 0x8f, 0x77, 0x2a, 0x68, 0x67, 0x3b, 0x7f, 0x81, 0xa6, 0x3a, 0xb6, 0x99, + 0xb6, 0x9d, 0x1f, 0x83, 0xab, 0xe0, 0xcd, 0xa1, 0xe7, 0x92, 0xdc, 0x40, 0x13, 0x9a, 0xe0, 0x83, + 0xf3, 0x61, 0x87, 0xc5, 0x84, 0xe1, 0x1b, 0x60, 0x65, 0xb8, 0x84, 0xd8, 0xe4, 0x1e, 0x7c, 0x0f, + 0x7c, 0xff, 0x2c, 0xd4, 0xb0, 0x25, 0xf6, 0x4f, 0x5f, 0x42, 0x1c, 0xc4, 0x01, 0xbd, 0x84, 0xc3, + 0x51, 0xf4, 0x68, 0x3c, 0xf8, 0x2d, 0xa0, 0x0f, 0xcc, 0x7a, 0xf2, 0x58, 0x5e, 0x2a, 0x70, 0x0d, + 0xdc, 0x45, 0x66, 0x79, 0xbd, 0xb2, 0x5d, 0x3f, 0x07, 0xfe, 0x95, 0x02, 0x3f, 0x04, 0xef, 0x9c, + 0x0d, 0x1c, 0xb6, 0xc1, 0x2f, 0x14, 0x68, 0x81, 0x8f, 0xce, 0xbd, 0xde, 0x30, 0x99, 0x3f, 0x28, + 0xf0, 0x26, 0xb8, 0x3e, 0x98, 0x2f, 0xf2, 0xf0, 0x47, 0x05, 0xde, 0x01, 0xb7, 0x4e, 0x5d, 0x49, + 0x20, 0xff, 0xa4, 0xc0, 0x1f, 0x80, 0x07, 0xa7, 0x41, 0x86, 0x85, 0xf1, 0x67, 0x05, 0x3e, 0x04, + 0xef, 0x9e, 0x63, 0x8d, 0x61, 0x02, 0x7f, 0x39, 0x65, 0x1f, 0x22, 0xd9, 0x5f, 0x9e, 0xbd, 0x0f, + 0x81, 0xfc, 0xab, 0x02, 0x97, 0xc1, 0xd5, 0xc1, 0x10, 0x7a, 0x27, 0xfe, 0xa6, 0xc0, 0xdb, 0x60, + 0xe5, 0x54, 0x25, 0x0a, 0xfb, 0xbb, 0x02, 0x55, 0x30, 0x57, 0xae, 0xd4, 0x1f, 0x99, 0x25, 0xbb, + 0xfe, 0xb4, 0xe4, 0x6c, 0xd6, 0x6b, 0x0e, 0xb2, 0x6a, 0xb5, 0xfc, 0xef, 0x46, 0x69, 0x28, 0x09, + 0x4f, 0xb9, 0x22, 0x9c, 0xf5, 0x47, 0x15, 0x54, 0xb7, 0x4b, 0x4f, 0xac, 0x32, 0x45, 0x7e, 0x3e, + 0x0a, 0x67, 0x00, 0xa0, 0xb0, 0x6a, 0xa5, 0x54, 0x76, 0x6a, 0xf9, 0x9f, 0xe6, 0xe0, 0x14, 0xb8, + 0x6c, 0x7d, 0xec, 0x58, 0xa8, 0x6c, 0xda, 0xf9, 0x7f, 0xe5, 0x56, 0x03, 0x00, 0xfa, 0xbf, 0xde, + 0xc2, 0x8b, 0x60, 0x74, 0xeb, 0x49, 0x7e, 0x04, 0x8e, 0x83, 0x31, 0xdb, 0x32, 0x6b, 0x56, 0x5e, + 0x81, 0x73, 0x60, 0xc6, 0xb2, 0xad, 0xa2, 0x53, 0xaa, 0x94, 0xeb, 0x68, 0xa7, 0x5c, 0x66, 0x05, + 0x24, 0x0f, 0x26, 0x9f, 0x9a, 0x4e, 0x71, 0x33, 0xb6, 0xe4, 0xe0, 0x02, 0x98, 0xb5, 0x2b, 0xc5, + 0xad, 0x3a, 0x32, 0x8b, 0x16, 0x8a, 0xcd, 0x17, 0x28, 0x90, 0x09, 0xc5, 0x96, 0xb1, 0xfb, 0x0f, + 0xc1, 0xb8, 0x13, 0xba, 0x7e, 0xd4, 0x0e, 0x42, 0x02, 0xef, 0xcb, 0x1f, 0xd3, 0xe2, 0xaf, 0xe5, + 0xe2, 0xbf, 0x48, 0x4b, 0x33, 0x27, 0xdf, 0xfc, 0x9f, 0x39, 0xfa, 0xc8, 0x1d, 0xe5, 0x3b, 0x4a, + 0x61, 0xfe, 0xe5, 0x3f, 0x97, 0x47, 0x5e, 0xbe, 0x5e, 0x56, 0xbe, 0x7c, 0xbd, 0xac, 0xfc, 0xe3, + 0xf5, 0xb2, 0xf2, 0x9b, 0xaf, 0x96, 0x47, 0x76, 0x2f, 0xb2, 0xff, 0x42, 0x3d, 0xf8, 0x7f, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x93, 0xbf, 0x7b, 0xa5, 0xce, 0x1a, 0x00, 0x00, } diff --git a/functional/rpcpb/rpc.proto b/functional/rpcpb/rpc.proto index 69cbb2c05..e8fb36596 100644 --- a/functional/rpcpb/rpc.proto +++ b/functional/rpcpb/rpc.proto @@ -8,77 +8,25 @@ option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (gogoproto.goproto_getters_all) = false; +message Request { + Operation Operation = 1; + // Member contains the same Member object from tester configuration. + Member Member = 2; + // Tester contains tester configuration. + Tester Tester = 3; +} + +message Response { + bool Success = 1; + string Status = 2; + // Member contains the same Member object from tester request. + Member Member = 3; +} + service Transport { rpc Transport(stream Request) returns (stream Response) {} } -enum Operation { - NotStarted = 0; - - // InitialStartEtcd is only called to start etcd very first time. - InitialStartEtcd = 1; - // RestartEtcd is sent to restart killed etcd. - RestartEtcd = 2; - // KillEtcd pauses etcd process while keeping data directories - // and previous etcd configurations. - KillEtcd = 3; - // FailArchive is sent when consistency check failed, - // thus need to archive etcd data directories. - FailArchive = 4; - // DestroyEtcdAgent destroys etcd process, etcd data, and agent server. - DestroyEtcdAgent = 5; - - // BlackholePeerPortTxRx drops all outgoing/incoming packets from/to the - // peer port on target member's peer port. - BlackholePeerPortTxRx = 100; - // UnblackholePeerPortTxRx removes outgoing/incoming packet dropping. - UnblackholePeerPortTxRx = 101; - // DelayPeerPortTxRx delays all outgoing/incoming packets from/to the - // peer port on target member's peer port. - DelayPeerPortTxRx = 102; - // UndelayPeerPortTxRx removes all outgoing/incoming delays. - UndelayPeerPortTxRx = 103; -} - -message Etcd { - string Name = 1 [(gogoproto.moretags) = "yaml:\"name\""]; - string DataDir = 2 [(gogoproto.moretags) = "yaml:\"data-dir\""]; - string WALDir = 3 [(gogoproto.moretags) = "yaml:\"wal-dir\""]; - - // HeartbeatIntervalMs is the time (in milliseconds) of a heartbeat interval. - // Default value is 100, which is 100ms. - int64 HeartbeatIntervalMs = 11 [(gogoproto.moretags) = "yaml:\"heartbeat-interval\""]; - // ElectionTimeoutMs is the time (in milliseconds) for an election to timeout. - // Default value is 1000, which is 1s. - int64 ElectionTimeoutMs = 12 [(gogoproto.moretags) = "yaml:\"election-timeout\""]; - - repeated string ListenClientURLs = 21 [(gogoproto.moretags) = "yaml:\"listen-client-urls\""]; - repeated string AdvertiseClientURLs = 22 [(gogoproto.moretags) = "yaml:\"advertise-client-urls\""]; - bool ClientAutoTLS = 23 [(gogoproto.moretags) = "yaml:\"auto-tls\""]; - bool ClientCertAuth = 24 [(gogoproto.moretags) = "yaml:\"client-cert-auth\""]; - string ClientCertFile = 25 [(gogoproto.moretags) = "yaml:\"cert-file\""]; - string ClientKeyFile = 26 [(gogoproto.moretags) = "yaml:\"key-file\""]; - string ClientTrustedCAFile = 27 [(gogoproto.moretags) = "yaml:\"trusted-ca-file\""]; - - repeated string ListenPeerURLs = 31 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""]; - repeated string AdvertisePeerURLs = 32 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""]; - bool PeerAutoTLS = 33 [(gogoproto.moretags) = "yaml:\"peer-auto-tls\""]; - bool PeerClientCertAuth = 34 [(gogoproto.moretags) = "yaml:\"peer-client-cert-auth\""]; - string PeerCertFile = 35 [(gogoproto.moretags) = "yaml:\"peer-cert-file\""]; - string PeerKeyFile = 36 [(gogoproto.moretags) = "yaml:\"peer-key-file\""]; - string PeerTrustedCAFile = 37 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-file\""]; - - string InitialCluster = 41 [(gogoproto.moretags) = "yaml:\"initial-cluster\""]; - string InitialClusterState = 42 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""]; - string InitialClusterToken = 43 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""]; - - int64 SnapshotCount = 51 [(gogoproto.moretags) = "yaml:\"snapshot-count\""]; - int64 QuotaBackendBytes = 52 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""]; - - bool PreVote = 63 [(gogoproto.moretags) = "yaml:\"pre-vote\""]; - bool InitialCorruptCheck = 64 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""]; -} - message Member { // EtcdExecPath is the executable etcd binary path in agent server. string EtcdExecPath = 1 [(gogoproto.moretags) = "yaml:\"etcd-exec-path\""]; @@ -128,18 +76,156 @@ message Member { string PeerTrustedCAPath = 506 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-path\""]; } +message Tester { + string DataDir = 1 [(gogoproto.moretags) = "yaml:\"data-dir\""]; + string Network = 2 [(gogoproto.moretags) = "yaml:\"network\""]; + string Addr = 3 [(gogoproto.moretags) = "yaml:\"addr\""]; + + // DelayLatencyMsRv is the delay latency in milliseconds, + // to inject to simulated slow network. + uint32 DelayLatencyMs = 11 [(gogoproto.moretags) = "yaml:\"delay-latency-ms\""]; + // DelayLatencyMsRv is the delay latency random variable in milliseconds. + uint32 DelayLatencyMsRv = 12 [(gogoproto.moretags) = "yaml:\"delay-latency-ms-rv\""]; + // UpdatedDelayLatencyMs is the update delay latency in milliseconds, + // to inject to simulated slow network. It's the final latency to apply, + // in case the latency numbers are randomly generated from given delay latency field. + uint32 UpdatedDelayLatencyMs = 13 [(gogoproto.moretags) = "yaml:\"updated-delay-latency-ms\""]; + + // RoundLimit is the limit of rounds to run failure set (-1 to run without limits). + int32 RoundLimit = 21 [(gogoproto.moretags) = "yaml:\"round-limit\""]; + // ExitOnFailure is true, then exit tester on first failure. + bool ExitOnFailure = 22 [(gogoproto.moretags) = "yaml:\"exit-on-failure\""]; + // ConsistencyCheck is true to check consistency (revision, hash). + bool ConsistencyCheck = 23 [(gogoproto.moretags) = "yaml:\"consistency-check\""]; + // EnablePprof is true to enable profiler. + bool EnablePprof = 24 [(gogoproto.moretags) = "yaml:\"enable-pprof\""]; + + // FailureDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + uint32 FailureDelayMs = 31 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""]; + // FailureShuffle is true to randomize failure injecting order. + bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; + // FailureCases is the selected test cases to schedule. + // If empty, run all failure cases. + repeated string FailureCases = 33 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; + // Failpoinommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000) + repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; + + // RunnerExecPath is a path of etcd-runner binary. + string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""]; + // ExternalExecPath is a path of script for enabling/disabling an external fault injector. + string ExternalExecPath = 42 [(gogoproto.moretags) = "yaml:\"external-exec-path\""]; + + // StressTypes is the list of stresser names: + // keys, lease, nop, election-runner, watch-runner, lock-racer-runner, lease-runner. + repeated string StressTypes = 101 [(gogoproto.moretags) = "yaml:\"stress-types\""]; + // StressKeySize is the size of each small key written into etcd. + int32 StressKeySize = 102 [(gogoproto.moretags) = "yaml:\"stress-key-size\""]; + // StressKeySizeLarge is the size of each large key written into etcd. + int32 StressKeySizeLarge = 103 [(gogoproto.moretags) = "yaml:\"stress-key-size-large\""]; + // StressKeySuffixRange is the count of key range written into etcd. + // Stress keys are created with "fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange)". + int32 StressKeySuffixRange = 104 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range\""]; + // StressKeySuffixRangeTxn is the count of key range written into etcd txn (max 100). + // Stress keys are created with "fmt.Sprintf("/k%03d", i)". + int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""]; + // StressKeyTxnOps is the number of operations per a transaction (max 64). + int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""]; + + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + int32 StressClients = 201 [(gogoproto.moretags) = "yaml:\"stress-clients\""]; + // StressQPS is the maximum number of stresser requests per second. + int32 StressQPS = 202 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; +} + +message Etcd { + string Name = 1 [(gogoproto.moretags) = "yaml:\"name\""]; + string DataDir = 2 [(gogoproto.moretags) = "yaml:\"data-dir\""]; + string WALDir = 3 [(gogoproto.moretags) = "yaml:\"wal-dir\""]; + + // HeartbeatIntervalMs is the time (in milliseconds) of a heartbeat interval. + // Default value is 100, which is 100ms. + int64 HeartbeatIntervalMs = 11 [(gogoproto.moretags) = "yaml:\"heartbeat-interval\""]; + // ElectionTimeoutMs is the time (in milliseconds) for an election to timeout. + // Default value is 1000, which is 1s. + int64 ElectionTimeoutMs = 12 [(gogoproto.moretags) = "yaml:\"election-timeout\""]; + + repeated string ListenClientURLs = 21 [(gogoproto.moretags) = "yaml:\"listen-client-urls\""]; + repeated string AdvertiseClientURLs = 22 [(gogoproto.moretags) = "yaml:\"advertise-client-urls\""]; + bool ClientAutoTLS = 23 [(gogoproto.moretags) = "yaml:\"auto-tls\""]; + bool ClientCertAuth = 24 [(gogoproto.moretags) = "yaml:\"client-cert-auth\""]; + string ClientCertFile = 25 [(gogoproto.moretags) = "yaml:\"cert-file\""]; + string ClientKeyFile = 26 [(gogoproto.moretags) = "yaml:\"key-file\""]; + string ClientTrustedCAFile = 27 [(gogoproto.moretags) = "yaml:\"trusted-ca-file\""]; + + repeated string ListenPeerURLs = 31 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""]; + repeated string AdvertisePeerURLs = 32 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""]; + bool PeerAutoTLS = 33 [(gogoproto.moretags) = "yaml:\"peer-auto-tls\""]; + bool PeerClientCertAuth = 34 [(gogoproto.moretags) = "yaml:\"peer-client-cert-auth\""]; + string PeerCertFile = 35 [(gogoproto.moretags) = "yaml:\"peer-cert-file\""]; + string PeerKeyFile = 36 [(gogoproto.moretags) = "yaml:\"peer-key-file\""]; + string PeerTrustedCAFile = 37 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-file\""]; + + string InitialCluster = 41 [(gogoproto.moretags) = "yaml:\"initial-cluster\""]; + string InitialClusterState = 42 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""]; + string InitialClusterToken = 43 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""]; + + int64 SnapshotCount = 51 [(gogoproto.moretags) = "yaml:\"snapshot-count\""]; + int64 QuotaBackendBytes = 52 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""]; + + bool PreVote = 63 [(gogoproto.moretags) = "yaml:\"pre-vote\""]; + bool InitialCorruptCheck = 64 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""]; +} + +enum Operation { + // NOT_STARTED is the agent status before etcd first start. + NOT_STARTED = 0; + + // INITIAL_START_ETCD is only called to start etcd, the very first time. + INITIAL_START_ETCD = 10; + // RESTART_ETCD is sent to restart killed etcd. + RESTART_ETCD = 11; + + // SIGTERM_ETCD pauses etcd process while keeping data directories + // and previous etcd configurations. + SIGTERM_ETCD = 20; + // SIGQUIT_ETCD_AND_REMOVE_DATA kills etcd process and removes all data + // directories to simulate destroying the whole machine. + SIGQUIT_ETCD_AND_REMOVE_DATA = 21; + + // SIGQUIT_ETCD_AND_ARCHIVE_DATA is sent when consistency check failed, + // thus need to archive etcd data directories. + SIGQUIT_ETCD_AND_ARCHIVE_DATA = 30; + // SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT destroys etcd process, + // etcd data, and agent server. + SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT = 31; + + // BLACKHOLE_PEER_PORT_TX_RX drops all outgoing/incoming packets from/to + // the peer port on target member's peer port. + BLACKHOLE_PEER_PORT_TX_RX = 100; + // UNBLACKHOLE_PEER_PORT_TX_RX removes outgoing/incoming packet dropping. + UNBLACKHOLE_PEER_PORT_TX_RX = 101; + + // DELAY_PEER_PORT_TX_RX delays all outgoing/incoming packets from/to + // the peer port on target member's peer port. + DELAY_PEER_PORT_TX_RX = 200; + // UNDELAY_PEER_PORT_TX_RX removes all outgoing/incoming delays. + UNDELAY_PEER_PORT_TX_RX = 201; +} + // FailureCase defines various system faults in distributed systems, // in order to verify correct behavior of etcd servers and clients. enum FailureCase { - // KILL_ONE_FOLLOWER stops a randomly chosen follower (non-leader) + // SIGTERM_ONE_FOLLOWER stops a randomly chosen follower (non-leader) // but does not delete its data directories on disk for next restart. // It waits "failure-delay-ms" before recovering this failure. // The expected behavior is that the follower comes back online // and rejoins the cluster, and then each member continues to process // client requests ('Put' request that requires Raft consensus). - KILL_ONE_FOLLOWER = 0; + SIGTERM_ONE_FOLLOWER = 0; - // KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen + // SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen // follower but does not delete its data directories on disk for next // restart. And waits until most up-to-date node (leader) applies the // snapshot count of entries since the stop operation. @@ -148,9 +234,9 @@ enum FailureCase { // to the follower to force it to follow the leader's log. // As always, after recovery, each member must be able to process // client requests. - KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 1; + SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 1; - // KILL_LEADER stops the active leader node but does not delete its + // SIGTERM_LEADER stops the active leader node but does not delete its // data directories on disk for next restart. Then it waits // "failure-delay-ms" before recovering this failure, in order to // trigger election timeouts. @@ -158,9 +244,9 @@ enum FailureCase { // old leader comes back online and rejoins the cluster as a follower. // As always, after recovery, each member must be able to process // client requests. - KILL_LEADER = 2; + SIGTERM_LEADER = 2; - // KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node + // SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node // but does not delete its data directories on disk for next restart. // And waits until most up-to-date node ("new" leader) applies the // snapshot count of entries since the stop operation. @@ -169,24 +255,24 @@ enum FailureCase { // And it receives the snapshot from the new leader to overwrite its // store. As always, after recovery, each member must be able to // process client requests. - KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT = 3; + SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT = 3; - // KILL_QUORUM stops majority number of nodes to make the whole cluster + // SIGTERM_QUORUM stops majority number of nodes to make the whole cluster // inoperable but does not delete data directories on stopped nodes // for next restart. And it waits "failure-delay-ms" before recovering // this failure. // The expected behavior is that nodes come back online, thus cluster // comes back operative as well. As always, after recovery, each member // must be able to process client requests. - KILL_QUORUM = 4; + SIGTERM_QUORUM = 4; - // KILL_ALL stops the whole cluster but does not delete data directories + // SIGTERM_ALL stops the whole cluster but does not delete data directories // on disk for next restart. And it waits "failure-delay-ms" before // recovering this failure. // The expected behavior is that nodes come back online, thus cluster // comes back operative as well. As always, after recovery, each member // must be able to process client requests. - KILL_ALL = 5; + SIGTERM_ALL = 5; // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER drops all outgoing/incoming // packets from/to the peer port on a randomly chosen follower @@ -392,81 +478,3 @@ enum StressType { LOCK_RACER_RUNNER = 4; LEASE_RUNNER = 5; } - -message Tester { - string DataDir = 1 [(gogoproto.moretags) = "yaml:\"data-dir\""]; - string Network = 2 [(gogoproto.moretags) = "yaml:\"network\""]; - string Addr = 3 [(gogoproto.moretags) = "yaml:\"addr\""]; - - // DelayLatencyMsRv is the delay latency in milliseconds, - // to inject to simulated slow network. - uint32 DelayLatencyMs = 11 [(gogoproto.moretags) = "yaml:\"delay-latency-ms\""]; - // DelayLatencyMsRv is the delay latency random variable in milliseconds. - uint32 DelayLatencyMsRv = 12 [(gogoproto.moretags) = "yaml:\"delay-latency-ms-rv\""]; - // UpdatedDelayLatencyMs is the update delay latency in milliseconds, - // to inject to simulated slow network. It's the final latency to apply, - // in case the latency numbers are randomly generated from given delay latency field. - uint32 UpdatedDelayLatencyMs = 13 [(gogoproto.moretags) = "yaml:\"updated-delay-latency-ms\""]; - - // RoundLimit is the limit of rounds to run failure set (-1 to run without limits). - int32 RoundLimit = 21 [(gogoproto.moretags) = "yaml:\"round-limit\""]; - // ExitOnFailure is true, then exit tester on first failure. - bool ExitOnFailure = 22 [(gogoproto.moretags) = "yaml:\"exit-on-failure\""]; - // ConsistencyCheck is true to check consistency (revision, hash). - bool ConsistencyCheck = 23 [(gogoproto.moretags) = "yaml:\"consistency-check\""]; - // EnablePprof is true to enable profiler. - bool EnablePprof = 24 [(gogoproto.moretags) = "yaml:\"enable-pprof\""]; - - // FailureDelayMs is the delay duration after failure is injected. - // Useful when triggering snapshot or no-op failure cases. - uint32 FailureDelayMs = 31 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""]; - // FailureShuffle is true to randomize failure injecting order. - bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; - // FailureCases is the selected test cases to schedule. - // If empty, run all failure cases. - repeated string FailureCases = 33 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; - // Failpoinommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000) - repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; - - // RunnerExecPath is a path of etcd-runner binary. - string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""]; - // ExternalExecPath is a path of script for enabling/disabling an external fault injector. - string ExternalExecPath = 42 [(gogoproto.moretags) = "yaml:\"external-exec-path\""]; - - // StressTypes is the list of stresser names: - // keys, lease, nop, election-runner, watch-runner, lock-racer-runner, lease-runner. - repeated string StressTypes = 101 [(gogoproto.moretags) = "yaml:\"stress-types\""]; - // StressKeySize is the size of each small key written into etcd. - int32 StressKeySize = 102 [(gogoproto.moretags) = "yaml:\"stress-key-size\""]; - // StressKeySizeLarge is the size of each large key written into etcd. - int32 StressKeySizeLarge = 103 [(gogoproto.moretags) = "yaml:\"stress-key-size-large\""]; - // StressKeySuffixRange is the count of key range written into etcd. - // Stress keys are created with "fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange)". - int32 StressKeySuffixRange = 104 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range\""]; - // StressKeySuffixRangeTxn is the count of key range written into etcd txn (max 100). - // Stress keys are created with "fmt.Sprintf("/k%03d", i)". - int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""]; - // StressKeyTxnOps is the number of operations per a transaction (max 64). - int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""]; - - // StressClients is the number of concurrent stressing clients - // with "one" shared TCP connection. - int32 StressClients = 201 [(gogoproto.moretags) = "yaml:\"stress-clients\""]; - // StressQPS is the maximum number of stresser requests per second. - int32 StressQPS = 202 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; -} - -message Request { - Operation Operation = 1; - // Member contains the same Member object from tester configuration. - Member Member = 2; - // Tester contains tester configuration. - Tester Tester = 3; -} - -message Response { - bool Success = 1; - string Status = 2; - // Member contains the same Member object from tester request. - Member Member = 3; -} diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go index d2e3325c8..827fc500b 100644 --- a/functional/tester/cluster.go +++ b/functional/tester/cluster.go @@ -142,91 +142,91 @@ func (clus *Cluster) serveTesterServer() { func (clus *Cluster) updateFailures() { for _, cs := range clus.Tester.FailureCases { switch cs { - case "KILL_ONE_FOLLOWER": + case "SIGTERM_ONE_FOLLOWER": clus.failures = append(clus.failures, - newFailureKillOneFollower(clus)) - case "KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": + new_FailureCase_SIGTERM_ONE_FOLLOWER(clus)) + case "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureKillOneFollowerUntilTriggerSnapshot(clus)) - case "KILL_LEADER": + new_FailureCase_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus)) + case "SIGTERM_LEADER": clus.failures = append(clus.failures, - newFailureKillLeader(clus)) - case "KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT": + new_FailureCase_SIGTERM_LEADER(clus)) + case "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureKillLeaderUntilTriggerSnapshot(clus)) - case "KILL_QUORUM": + new_FailureCase_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus)) + case "SIGTERM_QUORUM": clus.failures = append(clus.failures, - newFailureKillQuorum(clus)) - case "KILL_ALL": + new_FailureCase_SIGTERM_QUORUM(clus)) + case "SIGTERM_ALL": clus.failures = append(clus.failures, - newFailureKillAll(clus)) + new_FailureCase_SIGTERM_ALL(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": clus.failures = append(clus.failures, - newFailureBlackholePeerPortTxRxOneFollower(clus)) + new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureBlackholePeerPortTxRxOneFollowerUntilTriggerSnapshot()) + new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT()) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": clus.failures = append(clus.failures, - newFailureBlackholePeerPortTxRxLeader(clus)) + new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureBlackholePeerPortTxRxLeaderUntilTriggerSnapshot()) + new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT()) case "BLACKHOLE_PEER_PORT_TX_RX_QUORUM": clus.failures = append(clus.failures, - newFailureBlackholePeerPortTxRxQuorum(clus)) + new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ALL": clus.failures = append(clus.failures, - newFailureBlackholePeerPortTxRxAll(clus)) + new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL(clus)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxOneFollower(clus, false)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxOneFollower(clus, true)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER(clus, true)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus, false)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus, true)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus, true)) case "DELAY_PEER_PORT_TX_RX_LEADER": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxLeader(clus, false)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxLeader(clus, true)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER(clus, true)) case "DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus, false)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus, true)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus, true)) case "DELAY_PEER_PORT_TX_RX_QUORUM": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxQuorum(clus, false)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxQuorum(clus, true)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM(clus, true)) case "DELAY_PEER_PORT_TX_RX_ALL": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxAll(clus, false)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_ALL(clus, false)) case "RANDOM_DELAY_PEER_PORT_TX_RX_ALL": clus.failures = append(clus.failures, - newFailureDelayPeerPortTxRxAll(clus, true)) + new_FailureCase_DELAY_PEER_PORT_TX_RX_ALL(clus, true)) case "NO_FAIL_WITH_STRESS": clus.failures = append(clus.failures, - newFailureNoFailWithStress(clus)) + new_FailureCase_NO_FAIL_WITH_STRESS(clus)) case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": clus.failures = append(clus.failures, - newFailureNoFailWithNoStressForLiveness(clus)) + new_FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS(clus)) case "EXTERNAL": clus.failures = append(clus.failures, - newFailureExternal(clus.Tester.ExternalExecPath)) + new_FailureCase_EXTERNAL(clus.Tester.ExternalExecPath)) case "FAILPOINTS": fpFailures, fperr := failpointFailures(clus) if len(fpFailures) == 0 { @@ -310,24 +310,24 @@ func (clus *Cluster) checkConsistency() (err error) { return err } -// Bootstrap bootstraps etcd cluster the very first time. +// Send_INITIAL_START_ETCD bootstraps etcd cluster the very first time. // After this, just continue to call kill/restart. -func (clus *Cluster) Bootstrap() error { +func (clus *Cluster) Send_INITIAL_START_ETCD() error { // this is the only time that creates request from scratch - return clus.broadcastOperation(rpcpb.Operation_InitialStartEtcd) + return clus.broadcast(rpcpb.Operation_INITIAL_START_ETCD) } -// FailArchive sends "FailArchive" operation. -func (clus *Cluster) FailArchive() error { - return clus.broadcastOperation(rpcpb.Operation_FailArchive) +// send_SIGQUIT_ETCD_AND_ARCHIVE_DATA sends "send_SIGQUIT_ETCD_AND_ARCHIVE_DATA" operation. +func (clus *Cluster) send_SIGQUIT_ETCD_AND_ARCHIVE_DATA() error { + return clus.broadcast(rpcpb.Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA) } -// Restart sends "Restart" operation. -func (clus *Cluster) Restart() error { - return clus.broadcastOperation(rpcpb.Operation_RestartEtcd) +// send_RESTART_ETCD sends restart operation. +func (clus *Cluster) send_RESTART_ETCD() error { + return clus.broadcast(rpcpb.Operation_RESTART_ETCD) } -func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error { +func (clus *Cluster) broadcast(op rpcpb.Operation) error { var wg sync.WaitGroup wg.Add(len(clus.agentStreams)) @@ -335,7 +335,7 @@ func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error { for i := range clus.agentStreams { go func(idx int, o rpcpb.Operation) { defer wg.Done() - errc <- clus.sendOperation(idx, o) + errc <- clus.sendOp(idx, o) }(i, op) } wg.Wait() @@ -349,7 +349,7 @@ func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error { if err != nil { destroyed := false - if op == rpcpb.Operation_DestroyEtcdAgent { + if op == rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT { if err == io.EOF { destroyed = true } @@ -376,8 +376,8 @@ func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error { return errors.New(strings.Join(errs, ", ")) } -func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error { - if op == rpcpb.Operation_InitialStartEtcd { +func (clus *Cluster) sendOp(idx int, op rpcpb.Operation) error { + if op == rpcpb.Operation_INITIAL_START_ETCD { clus.agentRequests[idx] = &rpcpb.Request{ Operation: op, Member: clus.Members[idx], @@ -436,7 +436,7 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error { } // store TLS assets from agents/servers onto disk - if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) { + if secure && (op == rpcpb.Operation_INITIAL_START_ETCD || op == rpcpb.Operation_RESTART_ETCD) { dirClient := filepath.Join( clus.Tester.DataDir, clus.Members[idx].Etcd.Name, @@ -496,9 +496,9 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error { return nil } -// DestroyEtcdAgents terminates all tester connections to agents and etcd servers. -func (clus *Cluster) DestroyEtcdAgents() { - err := clus.broadcastOperation(rpcpb.Operation_DestroyEtcdAgent) +// Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT terminates all tester connections to agents and etcd servers. +func (clus *Cluster) Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() { + err := clus.broadcast(rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT) if err != nil { clus.lg.Warn("destroying etcd/agents FAIL", zap.Error(err)) } else { diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index c29b81027..9185cad0b 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -316,7 +316,7 @@ func (clus *Cluster) failed() { zap.Int("case", clus.cs), zap.Int("case-total", len(clus.failures)), ) - clus.DestroyEtcdAgents() + clus.Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() os.Exit(2) } @@ -341,7 +341,7 @@ func (clus *Cluster) cleanup() error { ) clus.stresser.Close() - if err := clus.FailArchive(); err != nil { + if err := clus.send_SIGQUIT_ETCD_AND_ARCHIVE_DATA(); err != nil { clus.lg.Warn( "cleanup FAIL", zap.Int("round", clus.rd), @@ -351,7 +351,7 @@ func (clus *Cluster) cleanup() error { ) return err } - if err := clus.Restart(); err != nil { + if err := clus.send_RESTART_ETCD(); err != nil { clus.lg.Warn( "restart FAIL", zap.Int("round", clus.rd), diff --git a/functional/tester/cluster_test.go b/functional/tester/cluster_test.go index 323db79d1..a2be1f64f 100644 --- a/functional/tester/cluster_test.go +++ b/functional/tester/cluster_test.go @@ -156,12 +156,12 @@ func Test_read(t *testing.T) { FailureDelayMs: 7000, FailureShuffle: true, FailureCases: []string{ - "KILL_ONE_FOLLOWER", - "KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", - "KILL_LEADER", - "KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT", - "KILL_QUORUM", - "KILL_ALL", + "SIGTERM_ONE_FOLLOWER", + "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + "SIGTERM_LEADER", + "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT", + "SIGTERM_QUORUM", + "SIGTERM_ALL", "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER", "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", "BLACKHOLE_PEER_PORT_TX_RX_LEADER", diff --git a/functional/tester/failure_case_external.go b/functional/tester/failure_case_external.go index 09f341d67..41d779f9f 100644 --- a/functional/tester/failure_case_external.go +++ b/functional/tester/failure_case_external.go @@ -46,7 +46,7 @@ func (f *failureExternal) FailureCase() rpcpb.FailureCase { return f.failureCase } -func newFailureExternal(scriptPath string) Failure { +func new_FailureCase_EXTERNAL(scriptPath string) Failure { return &failureExternal{ desc: fmt.Sprintf("external fault injector (script: %q)", scriptPath), failureCase: rpcpb.FailureCase_EXTERNAL, diff --git a/functional/tester/failure_case_failpoints.go b/functional/tester/failure_case_failpoints.go index 675f9be27..0db8b8965 100644 --- a/functional/tester/failure_case_failpoints.go +++ b/functional/tester/failure_case_failpoints.go @@ -145,7 +145,7 @@ func makeRecoverFailpoint(fp string) recoverMemberFunc { fpStats.mu.Lock() fpStats.crashes[fp]++ fpStats.mu.Unlock() - return recoverKill(clus, idx) + return recover_SIGTERM_ETCD(clus, idx) } } diff --git a/functional/tester/failure_case_kill.go b/functional/tester/failure_case_kill.go deleted file mode 100644 index 54e717e29..000000000 --- a/functional/tester/failure_case_kill.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tester - -import "github.com/coreos/etcd/functional/rpcpb" - -func injectKill(clus *Cluster, idx int) error { - return clus.sendOperation(idx, rpcpb.Operation_KillEtcd) -} - -func recoverKill(clus *Cluster, idx int) error { - return clus.sendOperation(idx, rpcpb.Operation_RestartEtcd) -} - -func newFailureKillOneFollower(clus *Cluster) Failure { - ff := failureByFunc{ - failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER, - injectMember: injectKill, - recoverMember: recoverKill, - } - f := &failureFollower{ff, -1, -1} - return &failureDelay{ - Failure: f, - delayDuration: clus.GetFailureDelayDuration(), - } -} - -func newFailureKillLeader(clus *Cluster) Failure { - ff := failureByFunc{ - failureCase: rpcpb.FailureCase_KILL_LEADER, - injectMember: injectKill, - recoverMember: recoverKill, - } - f := &failureLeader{ff, -1, -1} - return &failureDelay{ - Failure: f, - delayDuration: clus.GetFailureDelayDuration(), - } -} - -func newFailureKillQuorum(clus *Cluster) Failure { - f := &failureQuorum{ - failureCase: rpcpb.FailureCase_KILL_QUORUM, - injectMember: injectKill, - recoverMember: recoverKill, - } - return &failureDelay{ - Failure: f, - delayDuration: clus.GetFailureDelayDuration(), - } -} - -func newFailureKillAll(clus *Cluster) Failure { - f := &failureAll{ - failureCase: rpcpb.FailureCase_KILL_ALL, - injectMember: injectKill, - recoverMember: recoverKill, - } - return &failureDelay{ - Failure: f, - delayDuration: clus.GetFailureDelayDuration(), - } -} - -func newFailureKillOneFollowerUntilTriggerSnapshot(clus *Cluster) Failure { - return &failureUntilSnapshot{ - failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, - Failure: newFailureKillOneFollower(clus), - } -} - -func newFailureKillLeaderUntilTriggerSnapshot(clus *Cluster) Failure { - return &failureUntilSnapshot{ - failureCase: rpcpb.FailureCase_KILL_LEADER_UNTIL_TRIGGER_SNAPSHOT, - Failure: newFailureKillLeader(clus), - } -} diff --git a/functional/tester/failure_case_network_blackhole.go b/functional/tester/failure_case_network_blackhole.go index 79669e989..b75394a78 100644 --- a/functional/tester/failure_case_network_blackhole.go +++ b/functional/tester/failure_case_network_blackhole.go @@ -16,19 +16,19 @@ package tester import "github.com/coreos/etcd/functional/rpcpb" -func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error { - return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx) +func inject_BLACKHOLE_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_BLACKHOLE_PEER_PORT_TX_RX) } -func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error { - return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx) +func recover_BLACKHOLE_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_UNBLACKHOLE_PEER_PORT_TX_RX) } -func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure { +func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER, - injectMember: injectBlackholePeerPortTxRx, - recoverMember: recoverBlackholePeerPortTxRx, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, } f := &failureFollower{ff, -1, -1} return &failureDelay{ @@ -37,11 +37,11 @@ func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure { } } -func newFailureBlackholePeerPortTxRxOneFollowerUntilTriggerSnapshot() Failure { +func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT() Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, - injectMember: injectBlackholePeerPortTxRx, - recoverMember: recoverBlackholePeerPortTxRx, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, } f := &failureFollower{ff, -1, -1} return &failureUntilSnapshot{ @@ -50,11 +50,11 @@ func newFailureBlackholePeerPortTxRxOneFollowerUntilTriggerSnapshot() Failure { } } -func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure { +func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER, - injectMember: injectBlackholePeerPortTxRx, - recoverMember: recoverBlackholePeerPortTxRx, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, } f := &failureLeader{ff, -1, -1} return &failureDelay{ @@ -63,11 +63,11 @@ func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure { } } -func newFailureBlackholePeerPortTxRxLeaderUntilTriggerSnapshot() Failure { +func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT() Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT, - injectMember: injectBlackholePeerPortTxRx, - recoverMember: recoverBlackholePeerPortTxRx, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, } f := &failureLeader{ff, -1, -1} return &failureUntilSnapshot{ @@ -76,11 +76,11 @@ func newFailureBlackholePeerPortTxRxLeaderUntilTriggerSnapshot() Failure { } } -func newFailureBlackholePeerPortTxRxQuorum(clus *Cluster) Failure { +func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM(clus *Cluster) Failure { f := &failureQuorum{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM, - injectMember: injectBlackholePeerPortTxRx, - recoverMember: recoverBlackholePeerPortTxRx, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, } return &failureDelay{ Failure: f, @@ -88,11 +88,11 @@ func newFailureBlackholePeerPortTxRxQuorum(clus *Cluster) Failure { } } -func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure { +func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL(clus *Cluster) Failure { f := &failureAll{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL, - injectMember: injectBlackholePeerPortTxRx, - recoverMember: recoverBlackholePeerPortTxRx, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, } return &failureDelay{ Failure: f, diff --git a/functional/tester/failure_case_network_delay.go b/functional/tester/failure_case_network_delay.go index 89cb2ea65..c2e067c46 100644 --- a/functional/tester/failure_case_network_delay.go +++ b/functional/tester/failure_case_network_delay.go @@ -30,27 +30,27 @@ const ( waitRecover = 5 * time.Second ) -func injectDelayPeerPortTxRx(clus *Cluster, idx int) error { +func inject_DELAY_PEER_PORT_TX_RX(clus *Cluster, idx int) error { clus.lg.Info( "injecting delay latency", zap.Duration("latency", time.Duration(clus.Tester.UpdatedDelayLatencyMs)*time.Millisecond), zap.Duration("latency-rv", time.Duration(clus.Tester.DelayLatencyMsRv)*time.Millisecond), zap.String("endpoint", clus.Members[idx].EtcdClientEndpoint), ) - return clus.sendOperation(idx, rpcpb.Operation_DelayPeerPortTxRx) + return clus.sendOp(idx, rpcpb.Operation_DELAY_PEER_PORT_TX_RX) } -func recoverDelayPeerPortTxRx(clus *Cluster, idx int) error { - err := clus.sendOperation(idx, rpcpb.Operation_UndelayPeerPortTxRx) +func recover_DELAY_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + err := clus.sendOp(idx, rpcpb.Operation_UNDELAY_PEER_PORT_TX_RX) time.Sleep(waitRecover) return err } -func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster, random bool) Failure { +func new_FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER(clus *Cluster, random bool) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER, - injectMember: injectDelayPeerPortTxRx, - recoverMember: recoverDelayPeerPortTxRx, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { @@ -64,11 +64,11 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster, random bool) Failure } } -func newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus *Cluster, random bool) Failure { +func new_FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster, random bool) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, - injectMember: injectDelayPeerPortTxRx, - recoverMember: recoverDelayPeerPortTxRx, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { @@ -82,11 +82,11 @@ func newFailureDelayPeerPortTxRxOneFollowerUntilTriggerSnapshot(clus *Cluster, r } } -func newFailureDelayPeerPortTxRxLeader(clus *Cluster, random bool) Failure { +func new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER(clus *Cluster, random bool) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER, - injectMember: injectDelayPeerPortTxRx, - recoverMember: recoverDelayPeerPortTxRx, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { @@ -100,11 +100,11 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster, random bool) Failure { } } -func newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus *Cluster, random bool) Failure { +func new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster, random bool) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT, - injectMember: injectDelayPeerPortTxRx, - recoverMember: recoverDelayPeerPortTxRx, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { @@ -118,11 +118,11 @@ func newFailureDelayPeerPortTxRxLeaderUntilTriggerSnapshot(clus *Cluster, random } } -func newFailureDelayPeerPortTxRxQuorum(clus *Cluster, random bool) Failure { +func new_FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM(clus *Cluster, random bool) Failure { f := &failureQuorum{ failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM, - injectMember: injectDelayPeerPortTxRx, - recoverMember: recoverDelayPeerPortTxRx, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { @@ -135,11 +135,11 @@ func newFailureDelayPeerPortTxRxQuorum(clus *Cluster, random bool) Failure { } } -func newFailureDelayPeerPortTxRxAll(clus *Cluster, random bool) Failure { +func new_FailureCase_DELAY_PEER_PORT_TX_RX_ALL(clus *Cluster, random bool) Failure { f := &failureAll{ failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ALL, - injectMember: injectDelayPeerPortTxRx, - recoverMember: recoverDelayPeerPortTxRx, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { diff --git a/functional/tester/failure_case_no_fail.go b/functional/tester/failure_case_no_fail.go index fb2b182af..4ae91ea17 100644 --- a/functional/tester/failure_case_no_fail.go +++ b/functional/tester/failure_case_no_fail.go @@ -43,7 +43,7 @@ func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase { return f.failureCase } -func newFailureNoFailWithStress(clus *Cluster) Failure { +func new_FailureCase_NO_FAIL_WITH_STRESS(clus *Cluster) Failure { f := &failureNoFailWithStress{ failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS, } @@ -88,7 +88,7 @@ func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase { return f.failureCase } -func newFailureNoFailWithNoStressForLiveness(clus *Cluster) Failure { +func new_FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS(clus *Cluster) Failure { f := &failureNoFailWithNoStressForLiveness{ failureCase: rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS, } diff --git a/functional/tester/failure_case_sigterm.go b/functional/tester/failure_case_sigterm.go new file mode 100644 index 000000000..e9ed3579d --- /dev/null +++ b/functional/tester/failure_case_sigterm.go @@ -0,0 +1,89 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "github.com/coreos/etcd/functional/rpcpb" + +func inject_SIGTERM_ETCD(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_SIGTERM_ETCD) +} + +func recover_SIGTERM_ETCD(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_RESTART_ETCD) +} + +func new_FailureCase_SIGTERM_ONE_FOLLOWER(clus *Cluster) Failure { + ff := failureByFunc{ + failureCase: rpcpb.FailureCase_SIGTERM_ONE_FOLLOWER, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + f := &failureFollower{ff, -1, -1} + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } +} + +func new_FailureCase_SIGTERM_LEADER(clus *Cluster) Failure { + ff := failureByFunc{ + failureCase: rpcpb.FailureCase_SIGTERM_LEADER, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + f := &failureLeader{ff, -1, -1} + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } +} + +func new_FailureCase_SIGTERM_QUORUM(clus *Cluster) Failure { + f := &failureQuorum{ + failureCase: rpcpb.FailureCase_SIGTERM_QUORUM, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } +} + +func new_FailureCase_SIGTERM_ALL(clus *Cluster) Failure { + f := &failureAll{ + failureCase: rpcpb.FailureCase_SIGTERM_ALL, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + return &failureDelay{ + Failure: f, + delayDuration: clus.GetFailureDelayDuration(), + } +} + +func new_FailureCase_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Failure { + return &failureUntilSnapshot{ + failureCase: rpcpb.FailureCase_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, + Failure: new_FailureCase_SIGTERM_ONE_FOLLOWER(clus), + } +} + +func new_FailureCase_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Failure { + return &failureUntilSnapshot{ + failureCase: rpcpb.FailureCase_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT, + Failure: new_FailureCase_SIGTERM_LEADER(clus), + } +} From c16e411d5188c044f2ce6a1f54f894bf86335658 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 9 Apr 2018 10:19:31 -0700 Subject: [PATCH 16/16] functional/tester: fix shadowed "err" variable Signed-off-by: Gyuho Lee --- functional/tester/cluster_read_config.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/functional/tester/cluster_read_config.go b/functional/tester/cluster_read_config.go index c45f12169..35d618d88 100644 --- a/functional/tester/cluster_read_config.go +++ b/functional/tester/cluster_read_config.go @@ -233,7 +233,8 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { } clus.Members[i].PeerCertPath = mem.Etcd.PeerCertFile if mem.Etcd.PeerCertFile != "" { - data, err := ioutil.ReadFile(mem.Etcd.PeerCertFile) + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.PeerCertFile) if err != nil { return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerCertFile, err) } @@ -241,7 +242,8 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { } clus.Members[i].PeerKeyPath = mem.Etcd.PeerKeyFile if mem.Etcd.PeerKeyFile != "" { - data, err := ioutil.ReadFile(mem.Etcd.PeerKeyFile) + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.PeerKeyFile) if err != nil { return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerKeyFile, err) } @@ -249,7 +251,8 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { } clus.Members[i].PeerTrustedCAPath = mem.Etcd.PeerTrustedCAFile if mem.Etcd.PeerTrustedCAFile != "" { - data, err := ioutil.ReadFile(mem.Etcd.PeerTrustedCAFile) + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.PeerTrustedCAFile) if err != nil { return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerTrustedCAFile, err) } @@ -282,7 +285,8 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { } clus.Members[i].ClientCertPath = mem.Etcd.ClientCertFile if mem.Etcd.ClientCertFile != "" { - data, err := ioutil.ReadFile(mem.Etcd.ClientCertFile) + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.ClientCertFile) if err != nil { return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientCertFile, err) } @@ -290,7 +294,8 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { } clus.Members[i].ClientKeyPath = mem.Etcd.ClientKeyFile if mem.Etcd.ClientKeyFile != "" { - data, err := ioutil.ReadFile(mem.Etcd.ClientKeyFile) + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.ClientKeyFile) if err != nil { return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientKeyFile, err) } @@ -298,7 +303,8 @@ func read(lg *zap.Logger, fpath string) (*Cluster, error) { } clus.Members[i].ClientTrustedCAPath = mem.Etcd.ClientTrustedCAFile if mem.Etcd.ClientTrustedCAFile != "" { - data, err := ioutil.ReadFile(mem.Etcd.ClientTrustedCAFile) + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.ClientTrustedCAFile) if err != nil { return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientTrustedCAFile, err) }