diff --git a/functional.yaml b/functional.yaml new file mode 100644 index 000000000..24f6e50bc --- /dev/null +++ b/functional.yaml @@ -0,0 +1,206 @@ +agent-configs: +- etcd-exec-path: ./bin/etcd + agent-addr: 127.0.0.1:19027 + failpoint-http-addr: http://127.0.0.1:7381 + base-dir: /tmp/etcd-functional-1 + etcd-log-path: /tmp/etcd-functional-1/etcd.log + etcd-client-proxy: false + etcd-peer-proxy: true + etcd-client-endpoint: 127.0.0.1:1379 + etcd: + name: s1 + data-dir: /tmp/etcd-functional-1/etcd.data + wal-dir: /tmp/etcd-functional-1/etcd.data/member/wal + heartbeat-interval: 100 + election-timeout: 1000 + listen-client-urls: ["https://127.0.0.1:1379"] + advertise-client-urls: ["https://127.0.0.1:1379"] + auto-tls: true + client-cert-auth: false + cert-file: "" + key-file: "" + trusted-ca-file: "" + listen-peer-urls: ["https://127.0.0.1:1380"] + initial-advertise-peer-urls: ["https://127.0.0.1:13800"] + peer-auto-tls: true + peer-client-cert-auth: false + peer-cert-file: "" + peer-key-file: "" + peer-trusted-ca-file: "" + initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800 + initial-cluster-state: new + initial-cluster-token: tkn + snapshot-count: 10000 + quota-backend-bytes: 10740000000 # 10 GiB + pre-vote: true + initial-corrupt-check: true + client-cert-data: "" + client-cert-path: "" + client-key-data: "" + client-key-path: "" + client-trusted-ca-data: "" + client-trusted-ca-path: "" + peer-cert-data: "" + peer-cert-path: "" + peer-key-data: "" + peer-key-path: "" + peer-trusted-ca-data: "" + peer-trusted-ca-path: "" + snapshot-path: /tmp/etcd-functional-1.snapshot.db + +- etcd-exec-path: ./bin/etcd + agent-addr: 127.0.0.1:29027 + failpoint-http-addr: http://127.0.0.1:7382 + base-dir: /tmp/etcd-functional-2 + etcd-log-path: /tmp/etcd-functional-2/etcd.log + etcd-client-proxy: false + etcd-peer-proxy: true + etcd-client-endpoint: 127.0.0.1:2379 + etcd: + name: s2 + data-dir: /tmp/etcd-functional-2/etcd.data + wal-dir: /tmp/etcd-functional-2/etcd.data/member/wal + heartbeat-interval: 100 + election-timeout: 1000 + listen-client-urls: ["https://127.0.0.1:2379"] + advertise-client-urls: ["https://127.0.0.1:2379"] + auto-tls: true + client-cert-auth: false + cert-file: "" + key-file: "" + trusted-ca-file: "" + listen-peer-urls: ["https://127.0.0.1:2380"] + initial-advertise-peer-urls: ["https://127.0.0.1:23800"] + peer-auto-tls: true + peer-client-cert-auth: false + peer-cert-file: "" + peer-key-file: "" + peer-trusted-ca-file: "" + initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800 + initial-cluster-state: new + initial-cluster-token: tkn + snapshot-count: 10000 + quota-backend-bytes: 10740000000 # 10 GiB + pre-vote: true + initial-corrupt-check: true + client-cert-data: "" + client-cert-path: "" + client-key-data: "" + client-key-path: "" + client-trusted-ca-data: "" + client-trusted-ca-path: "" + peer-cert-data: "" + peer-cert-path: "" + peer-key-data: "" + peer-key-path: "" + peer-trusted-ca-data: "" + peer-trusted-ca-path: "" + snapshot-path: /tmp/etcd-functional-2.snapshot.db + +- etcd-exec-path: ./bin/etcd + agent-addr: 127.0.0.1:39027 + failpoint-http-addr: http://127.0.0.1:7383 + base-dir: /tmp/etcd-functional-3 + etcd-log-path: /tmp/etcd-functional-3/etcd.log + etcd-client-proxy: false + etcd-peer-proxy: true + etcd-client-endpoint: 127.0.0.1:3379 + etcd: + name: s3 + data-dir: /tmp/etcd-functional-3/etcd.data + wal-dir: /tmp/etcd-functional-3/etcd.data/member/wal + heartbeat-interval: 100 + election-timeout: 1000 + listen-client-urls: ["https://127.0.0.1:3379"] + advertise-client-urls: ["https://127.0.0.1:3379"] + auto-tls: true + client-cert-auth: false + cert-file: "" + key-file: "" + trusted-ca-file: "" + listen-peer-urls: ["https://127.0.0.1:3380"] + initial-advertise-peer-urls: ["https://127.0.0.1:33800"] + peer-auto-tls: true + peer-client-cert-auth: false + peer-cert-file: "" + peer-key-file: "" + peer-trusted-ca-file: "" + initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800 + initial-cluster-state: new + initial-cluster-token: tkn + snapshot-count: 10000 + quota-backend-bytes: 10740000000 # 10 GiB + pre-vote: true + initial-corrupt-check: true + client-cert-data: "" + client-cert-path: "" + client-key-data: "" + client-key-path: "" + client-trusted-ca-data: "" + client-trusted-ca-path: "" + peer-cert-data: "" + peer-cert-path: "" + peer-key-data: "" + peer-key-path: "" + peer-trusted-ca-data: "" + peer-trusted-ca-path: "" + snapshot-path: /tmp/etcd-functional-3.snapshot.db + +tester-config: + data-dir: /tmp/etcd-tester-data + network: tcp + addr: 127.0.0.1:9028 + + # slow enough to trigger election + delay-latency-ms: 5000 + delay-latency-ms-rv: 500 + + round-limit: 1 + exit-on-failure: true + enable-pprof: true + + case-delay-ms: 7000 + case-shuffle: true + + # For full descriptions, + # https://godoc.org/github.com/coreos/etcd/functional/rpcpb#Case + cases: + - SIGTERM_ONE_FOLLOWER + - SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT + - SIGTERM_LEADER + - SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT + - SIGTERM_QUORUM + - SIGTERM_ALL + - SIGQUIT_AND_REMOVE_ONE_FOLLOWER + - SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT + - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER + - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT + - BLACKHOLE_PEER_PORT_TX_RX_LEADER + - BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT + - BLACKHOLE_PEER_PORT_TX_RX_QUORUM + - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER + - DELAY_PEER_PORT_TX_RX_LEADER + - DELAY_PEER_PORT_TX_RX_QUORUM + + failpoint-commands: + - panic("etcd-tester") + + runner-exec-path: ./bin/etcd-runner + external-exec-path: "" + + stressers: + - KV + - LEASE + + checkers: + - KV_HASH + - LEASE_EXPIRE + + stress-key-size: 100 + stress-key-size-large: 32769 + stress-key-suffix-range: 250000 + stress-key-suffix-range-txn: 100 + stress-key-txn-ops: 10 + + stress-clients: 100 + stress-qps: 2000 diff --git a/functional/Dockerfile b/functional/Dockerfile new file mode 100644 index 000000000..fbcc34ec2 --- /dev/null +++ b/functional/Dockerfile @@ -0,0 +1,42 @@ +FROM fedora:28 + +RUN dnf check-update || true \ + && dnf install --assumeyes \ + git curl wget mercurial meld gcc gcc-c++ which \ + gcc automake autoconf dh-autoreconf libtool libtool-ltdl \ + tar unzip gzip \ + && dnf check-update || true \ + && dnf upgrade --assumeyes || true \ + && dnf autoremove --assumeyes || true \ + && dnf clean all || true + +ENV GOROOT /usr/local/go +ENV GOPATH /go +ENV PATH ${GOPATH}/bin:${GOROOT}/bin:${PATH} +ENV GO_VERSION REPLACE_ME_GO_VERSION +ENV GO_DOWNLOAD_URL https://storage.googleapis.com/golang +RUN rm -rf ${GOROOT} \ + && curl -s ${GO_DOWNLOAD_URL}/go${GO_VERSION}.linux-amd64.tar.gz | tar -v -C /usr/local/ -xz \ + && mkdir -p ${GOPATH}/src ${GOPATH}/bin \ + && go version + +RUN mkdir -p ${GOPATH}/src/github.com/coreos/etcd +ADD . ${GOPATH}/src/github.com/coreos/etcd +ADD ./functional.yaml /functional.yaml + +RUN go get -v github.com/coreos/gofail \ + && pushd ${GOPATH}/src/github.com/coreos/etcd \ + && GO_BUILD_FLAGS="-v" ./build \ + && mkdir -p /bin \ + && cp ./bin/etcd /bin/etcd \ + && cp ./bin/etcdctl /bin/etcdctl \ + && GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \ + && cp ./bin/etcd /bin/etcd-failpoints \ + && ./functional/build \ + && cp ./bin/etcd-agent /bin/etcd-agent \ + && cp ./bin/etcd-proxy /bin/etcd-proxy \ + && cp ./bin/etcd-runner /bin/etcd-runner \ + && cp ./bin/etcd-tester /bin/etcd-tester \ + && go build -v -o /bin/benchmark ./tools/benchmark \ + && popd \ + && rm -rf ${GOPATH}/src/github.com/coreos/etcd \ No newline at end of file diff --git a/functional/Procfile-proxy b/functional/Procfile-proxy new file mode 100644 index 000000000..66730ee77 --- /dev/null +++ b/functional/Procfile-proxy @@ -0,0 +1,14 @@ +s1: bin/etcd --name s1 --data-dir /tmp/etcd-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s1-client-proxy: bin/etcd-proxy --from localhost:13790 --to localhost:1379 --http-port 1378 +s1-peer-proxy: bin/etcd-proxy --from localhost:13800 --to localhost:1380 --http-port 1381 + +s2: bin/etcd --name s2 --data-dir /tmp/etcd-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s2-client-proxy: bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 +s2-peer-proxy: bin/etcd-proxy --from localhost:23800 --to localhost:2380 --http-port 2381 + +s3: bin/etcd --name s3 --data-dir /tmp/etcd-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s3-client-proxy: bin/etcd-proxy --from localhost:33790 --to localhost:3379 --http-port 3378 +s3-client-proxy: bin/etcd-proxy --from localhost:33800 --to localhost:3380 --http-port 3381 diff --git a/functional/README.md b/functional/README.md new file mode 100644 index 000000000..f4b8cb1d7 --- /dev/null +++ b/functional/README.md @@ -0,0 +1,218 @@ +## etcd Functional Testing + +[`functional`](https://godoc.org/github.com/coreos/etcd/functional) verifies the correct behavior of etcd under various system and network malfunctions. It sets up an etcd cluster under high pressure loads and continuously injects failures into the cluster. Then it expects the etcd cluster to recover within a few seconds. This has been extremely helpful to find critical bugs. + +See [`rpcpb.Case`](https://godoc.org/github.com/coreos/etcd/functional/rpcpb#Case) for all failure cases. + +See [functional.yaml](https://github.com/coreos/etcd/blob/master/functional.yaml) for an example configuration. + +### Run locally + +```bash +PASSES=functional ./test +``` + +### Run with Docker + +```bash +pushd .. +make build-docker-functional +popd +``` + +And run [example scripts](./scripts). + +```bash +# run 3 agents for 3-node local etcd cluster +./scripts/docker-local-agent.sh 1 +./scripts/docker-local-agent.sh 2 +./scripts/docker-local-agent.sh 3 + +# to run only 1 tester round +./scripts/docker-local-tester.sh +``` + +## etcd Proxy + +Proxy layer that simulates various network conditions. + +Test locally + +```bash +$ ./build +$ ./bin/etcd + +$ make build-functional + +$ ./bin/etcd-proxy --help +$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar +``` + +Proxy overhead per request is under 500μs + +```bash +$ go build -v -o ./bin/benchmark ./tools/benchmark + +$ ./bin/benchmark \ + --endpoints localhost:2379 \ + --conns 5 \ + --clients 15 \ + put \ + --key-size 48 \ + --val-size 50000 \ + --total 10000 + +< tcp://localhost:2379] + +$ ETCDCTL_API=3 ./bin/etcdctl \ + --endpoints localhost:23790 \ + put foo bar +# Error: context deadline exceeded + +$ curl -L http://localhost:2378/pause-tx -X DELETE +# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379] +``` + +Drop client packets + +```bash +$ curl -L http://localhost:2378/blackhole-tx -X PUT +# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379] + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar +# Error: context deadline exceeded + +$ curl -L http://localhost:2378/blackhole-tx -X DELETE +# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379] +``` + +Trigger leader election + +```bash +$ ./build +$ make build-functional + +$ rm -rf /tmp/etcd-proxy-data.s* +$ goreman -f ./functional/Procfile-proxy start + +$ ETCDCTL_API=3 ./bin/etcdctl \ + --endpoints localhost:13790,localhost:23790,localhost:33790 \ + member list + +# isolate s1 when s1 is the current leader +$ curl -L http://localhost:1381/blackhole-tx -X PUT +$ curl -L http://localhost:1381/blackhole-rx -X PUT +# s1 becomes follower after election timeout +``` diff --git a/functional/agent/doc.go b/functional/agent/doc.go new file mode 100644 index 000000000..0195c4c74 --- /dev/null +++ b/functional/agent/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package agent implements functional-tester agent server. +package agent diff --git a/functional/agent/handler.go b/functional/agent/handler.go new file mode 100644 index 000000000..7cd8e6cec --- /dev/null +++ b/functional/agent/handler.go @@ -0,0 +1,698 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "errors" + "fmt" + "io/ioutil" + "net/url" + "os" + "os/exec" + "path/filepath" + "syscall" + "time" + + "github.com/coreos/etcd/functional/rpcpb" + "github.com/coreos/etcd/pkg/fileutil" + "github.com/coreos/etcd/pkg/proxy" + + "go.uber.org/zap" +) + +// return error for system errors (e.g. fail to create files) +// return status error in response for wrong configuration/operation (e.g. start etcd twice) +func (srv *Server) handleTesterRequest(req *rpcpb.Request) (resp *rpcpb.Response, err error) { + defer func() { + if err == nil && req != nil { + srv.last = req.Operation + srv.lg.Info("handler success", zap.String("operation", req.Operation.String())) + } + }() + if req != nil { + srv.Member = req.Member + srv.Tester = req.Tester + } + + switch req.Operation { + case rpcpb.Operation_INITIAL_START_ETCD: + return srv.handle_INITIAL_START_ETCD(req) + case rpcpb.Operation_RESTART_ETCD: + return srv.handle_RESTART_ETCD() + + case rpcpb.Operation_SIGTERM_ETCD: + return srv.handle_SIGTERM_ETCD() + case rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA: + return srv.handle_SIGQUIT_ETCD_AND_REMOVE_DATA() + + case rpcpb.Operation_SAVE_SNAPSHOT: + return srv.handle_SAVE_SNAPSHOT() + case rpcpb.Operation_RESTORE_RESTART_FROM_SNAPSHOT: + return srv.handle_RESTORE_RESTART_FROM_SNAPSHOT() + case rpcpb.Operation_RESTART_FROM_SNAPSHOT: + return srv.handle_RESTART_FROM_SNAPSHOT() + + case rpcpb.Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA: + return srv.handle_SIGQUIT_ETCD_AND_ARCHIVE_DATA() + case rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT: + return srv.handle_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() + + case rpcpb.Operation_BLACKHOLE_PEER_PORT_TX_RX: + return srv.handle_BLACKHOLE_PEER_PORT_TX_RX() + case rpcpb.Operation_UNBLACKHOLE_PEER_PORT_TX_RX: + return srv.handle_UNBLACKHOLE_PEER_PORT_TX_RX() + case rpcpb.Operation_DELAY_PEER_PORT_TX_RX: + return srv.handle_DELAY_PEER_PORT_TX_RX() + case rpcpb.Operation_UNDELAY_PEER_PORT_TX_RX: + return srv.handle_UNDELAY_PEER_PORT_TX_RX() + + default: + msg := fmt.Sprintf("operation not found (%v)", req.Operation) + return &rpcpb.Response{Success: false, Status: msg}, errors.New(msg) + } +} + +func (srv *Server) handle_INITIAL_START_ETCD(req *rpcpb.Request) (*rpcpb.Response, error) { + if srv.last != rpcpb.Operation_NOT_STARTED { + return &rpcpb.Response{ + Success: false, + Status: fmt.Sprintf("%q is not valid; last server operation was %q", rpcpb.Operation_INITIAL_START_ETCD.String(), srv.last.String()), + Member: req.Member, + }, nil + } + + err := fileutil.TouchDirAll(srv.Member.BaseDir) + if err != nil { + return nil, err + } + srv.lg.Info("created base directory", zap.String("path", srv.Member.BaseDir)) + + if err = srv.createEtcdLogFile(); err != nil { + return nil, err + } + + srv.creatEtcdCmd(false) + + if err = srv.saveTLSAssets(); err != nil { + return nil, err + } + if err = srv.startEtcdCmd(); err != nil { + return nil, err + } + srv.lg.Info("started etcd", zap.String("command-path", srv.etcdCmd.Path)) + if err = srv.loadAutoTLSAssets(); err != nil { + return nil, err + } + + // wait some time for etcd listener start + // before setting up proxy + time.Sleep(time.Second) + if err = srv.startProxy(); err != nil { + return nil, err + } + + return &rpcpb.Response{ + Success: true, + Status: "start etcd PASS", + Member: srv.Member, + }, nil +} + +func (srv *Server) startProxy() error { + if srv.Member.EtcdClientProxy { + advertiseClientURL, advertiseClientURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertiseClientURLs[0]) + if err != nil { + return err + } + listenClientURL, _, err := getURLAndPort(srv.Member.Etcd.ListenClientURLs[0]) + if err != nil { + return err + } + + srv.advertiseClientPortToProxy[advertiseClientURLPort] = proxy.NewServer(proxy.ServerConfig{ + Logger: srv.lg, + From: *advertiseClientURL, + To: *listenClientURL, + }) + select { + case err = <-srv.advertiseClientPortToProxy[advertiseClientURLPort].Error(): + return err + case <-time.After(2 * time.Second): + srv.lg.Info("started proxy on client traffic", zap.String("url", advertiseClientURL.String())) + } + } + + if srv.Member.EtcdPeerProxy { + advertisePeerURL, advertisePeerURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertisePeerURLs[0]) + if err != nil { + return err + } + listenPeerURL, _, err := getURLAndPort(srv.Member.Etcd.ListenPeerURLs[0]) + if err != nil { + return err + } + + srv.advertisePeerPortToProxy[advertisePeerURLPort] = proxy.NewServer(proxy.ServerConfig{ + Logger: srv.lg, + From: *advertisePeerURL, + To: *listenPeerURL, + }) + select { + case err = <-srv.advertisePeerPortToProxy[advertisePeerURLPort].Error(): + return err + case <-time.After(2 * time.Second): + srv.lg.Info("started proxy on peer traffic", zap.String("url", advertisePeerURL.String())) + } + } + return nil +} + +func (srv *Server) stopProxy() { + if srv.Member.EtcdClientProxy && len(srv.advertiseClientPortToProxy) > 0 { + for port, px := range srv.advertiseClientPortToProxy { + if err := px.Close(); err != nil { + srv.lg.Warn("failed to close proxy", zap.Int("port", port)) + continue + } + select { + case <-px.Done(): + // enough time to release port + time.Sleep(time.Second) + case <-time.After(time.Second): + } + srv.lg.Info("closed proxy", + zap.Int("port", port), + zap.String("from", px.From()), + zap.String("to", px.To()), + ) + } + srv.advertiseClientPortToProxy = make(map[int]proxy.Server) + } + if srv.Member.EtcdPeerProxy && len(srv.advertisePeerPortToProxy) > 0 { + for port, px := range srv.advertisePeerPortToProxy { + if err := px.Close(); err != nil { + srv.lg.Warn("failed to close proxy", zap.Int("port", port)) + continue + } + select { + case <-px.Done(): + // enough time to release port + time.Sleep(time.Second) + case <-time.After(time.Second): + } + srv.lg.Info("closed proxy", + zap.Int("port", port), + zap.String("from", px.From()), + zap.String("to", px.To()), + ) + } + srv.advertisePeerPortToProxy = make(map[int]proxy.Server) + } +} + +func (srv *Server) createEtcdLogFile() error { + var err error + srv.etcdLogFile, err = os.Create(srv.Member.EtcdLogPath) + if err != nil { + return err + } + srv.lg.Info("created etcd log file", zap.String("path", srv.Member.EtcdLogPath)) + return nil +} + +func (srv *Server) creatEtcdCmd(fromSnapshot bool) { + etcdPath, etcdFlags := srv.Member.EtcdExecPath, srv.Member.Etcd.Flags() + if fromSnapshot { + etcdFlags = srv.Member.EtcdOnSnapshotRestore.Flags() + } + u, _ := url.Parse(srv.Member.FailpointHTTPAddr) + srv.lg.Info("creating etcd command", + zap.String("etcd-exec-path", etcdPath), + zap.Strings("etcd-flags", etcdFlags), + zap.String("failpoint-http-addr", srv.Member.FailpointHTTPAddr), + zap.String("failpoint-addr", u.Host), + ) + srv.etcdCmd = exec.Command(etcdPath, etcdFlags...) + srv.etcdCmd.Env = []string{"GOFAIL_HTTP=" + u.Host} + srv.etcdCmd.Stdout = srv.etcdLogFile + srv.etcdCmd.Stderr = srv.etcdLogFile +} + +// if started with manual TLS, stores TLS assets +// from tester/client to disk before starting etcd process +func (srv *Server) saveTLSAssets() error { + if srv.Member.PeerCertPath != "" { + if srv.Member.PeerCertData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.PeerCertPath) + } + if err := ioutil.WriteFile(srv.Member.PeerCertPath, []byte(srv.Member.PeerCertData), 0644); err != nil { + return err + } + } + if srv.Member.PeerKeyPath != "" { + if srv.Member.PeerKeyData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.PeerKeyPath) + } + if err := ioutil.WriteFile(srv.Member.PeerKeyPath, []byte(srv.Member.PeerKeyData), 0644); err != nil { + return err + } + } + if srv.Member.PeerTrustedCAPath != "" { + if srv.Member.PeerTrustedCAData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.PeerTrustedCAPath) + } + if err := ioutil.WriteFile(srv.Member.PeerTrustedCAPath, []byte(srv.Member.PeerTrustedCAData), 0644); err != nil { + return err + } + } + if srv.Member.PeerCertPath != "" && + srv.Member.PeerKeyPath != "" && + srv.Member.PeerTrustedCAPath != "" { + srv.lg.Info( + "wrote", + zap.String("peer-cert", srv.Member.PeerCertPath), + zap.String("peer-key", srv.Member.PeerKeyPath), + zap.String("peer-trusted-ca", srv.Member.PeerTrustedCAPath), + ) + } + + if srv.Member.ClientCertPath != "" { + if srv.Member.ClientCertData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.ClientCertPath) + } + if err := ioutil.WriteFile(srv.Member.ClientCertPath, []byte(srv.Member.ClientCertData), 0644); err != nil { + return err + } + } + if srv.Member.ClientKeyPath != "" { + if srv.Member.ClientKeyData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.ClientKeyPath) + } + if err := ioutil.WriteFile(srv.Member.ClientKeyPath, []byte(srv.Member.ClientKeyData), 0644); err != nil { + return err + } + } + if srv.Member.ClientTrustedCAPath != "" { + if srv.Member.ClientTrustedCAData == "" { + return fmt.Errorf("got empty data for %q", srv.Member.ClientTrustedCAPath) + } + if err := ioutil.WriteFile(srv.Member.ClientTrustedCAPath, []byte(srv.Member.ClientTrustedCAData), 0644); err != nil { + return err + } + } + if srv.Member.ClientCertPath != "" && + srv.Member.ClientKeyPath != "" && + srv.Member.ClientTrustedCAPath != "" { + srv.lg.Info( + "wrote", + zap.String("client-cert", srv.Member.ClientCertPath), + zap.String("client-key", srv.Member.ClientKeyPath), + zap.String("client-trusted-ca", srv.Member.ClientTrustedCAPath), + ) + } + + return nil +} + +func (srv *Server) loadAutoTLSAssets() error { + if srv.Member.Etcd.PeerAutoTLS { + // in case of slow disk + time.Sleep(time.Second) + + fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "peer") + + srv.lg.Info( + "loading client auto TLS assets", + zap.String("dir", fdir), + zap.String("endpoint", srv.EtcdClientEndpoint), + ) + + certPath := filepath.Join(fdir, "cert.pem") + if !fileutil.Exist(certPath) { + return fmt.Errorf("cannot find %q", certPath) + } + certData, err := ioutil.ReadFile(certPath) + if err != nil { + return fmt.Errorf("cannot read %q (%v)", certPath, err) + } + srv.Member.PeerCertData = string(certData) + + keyPath := filepath.Join(fdir, "key.pem") + if !fileutil.Exist(keyPath) { + return fmt.Errorf("cannot find %q", keyPath) + } + keyData, err := ioutil.ReadFile(keyPath) + if err != nil { + return fmt.Errorf("cannot read %q (%v)", keyPath, err) + } + srv.Member.PeerKeyData = string(keyData) + + srv.lg.Info( + "loaded peer auto TLS assets", + zap.String("peer-cert-path", certPath), + zap.Int("peer-cert-length", len(certData)), + zap.String("peer-key-path", keyPath), + zap.Int("peer-key-length", len(keyData)), + ) + } + + if srv.Member.Etcd.ClientAutoTLS { + // in case of slow disk + time.Sleep(time.Second) + + fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "client") + + srv.lg.Info( + "loading client TLS assets", + zap.String("dir", fdir), + zap.String("endpoint", srv.EtcdClientEndpoint), + ) + + certPath := filepath.Join(fdir, "cert.pem") + if !fileutil.Exist(certPath) { + return fmt.Errorf("cannot find %q", certPath) + } + certData, err := ioutil.ReadFile(certPath) + if err != nil { + return fmt.Errorf("cannot read %q (%v)", certPath, err) + } + srv.Member.ClientCertData = string(certData) + + keyPath := filepath.Join(fdir, "key.pem") + if !fileutil.Exist(keyPath) { + return fmt.Errorf("cannot find %q", keyPath) + } + keyData, err := ioutil.ReadFile(keyPath) + if err != nil { + return fmt.Errorf("cannot read %q (%v)", keyPath, err) + } + srv.Member.ClientKeyData = string(keyData) + + srv.lg.Info( + "loaded client TLS assets", + zap.String("peer-cert-path", certPath), + zap.Int("peer-cert-length", len(certData)), + zap.String("peer-key-path", keyPath), + zap.Int("peer-key-length", len(keyData)), + ) + } + + return nil +} + +// start but do not wait for it to complete +func (srv *Server) startEtcdCmd() error { + return srv.etcdCmd.Start() +} + +func (srv *Server) handle_RESTART_ETCD() (*rpcpb.Response, error) { + var err error + if !fileutil.Exist(srv.Member.BaseDir) { + err = fileutil.TouchDirAll(srv.Member.BaseDir) + if err != nil { + return nil, err + } + } + + srv.creatEtcdCmd(false) + + if err = srv.saveTLSAssets(); err != nil { + return nil, err + } + if err = srv.startEtcdCmd(); err != nil { + return nil, err + } + srv.lg.Info("restarted etcd", zap.String("command-path", srv.etcdCmd.Path)) + if err = srv.loadAutoTLSAssets(); err != nil { + return nil, err + } + + // wait some time for etcd listener start + // before setting up proxy + // TODO: local tests should handle port conflicts + // with clients on restart + time.Sleep(time.Second) + if err = srv.startProxy(); err != nil { + return nil, err + } + + return &rpcpb.Response{ + Success: true, + Status: "restart etcd PASS", + Member: srv.Member, + }, nil +} + +func (srv *Server) handle_SIGTERM_ETCD() (*rpcpb.Response, error) { + srv.stopProxy() + + err := stopWithSig(srv.etcdCmd, syscall.SIGTERM) + if err != nil { + return nil, err + } + srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGTERM.String())) + + return &rpcpb.Response{ + Success: true, + Status: "killed etcd", + }, nil +} + +func (srv *Server) handle_SIGQUIT_ETCD_AND_REMOVE_DATA() (*rpcpb.Response, error) { + srv.stopProxy() + + err := stopWithSig(srv.etcdCmd, syscall.SIGQUIT) + if err != nil { + return nil, err + } + srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGQUIT.String())) + + srv.etcdLogFile.Sync() + srv.etcdLogFile.Close() + + // for debugging purposes, rename instead of removing + if err = os.RemoveAll(srv.Member.BaseDir + ".backup"); err != nil { + return nil, err + } + if err = os.Rename(srv.Member.BaseDir, srv.Member.BaseDir+".backup"); err != nil { + return nil, err + } + srv.lg.Info( + "renamed", + zap.String("base-dir", srv.Member.BaseDir), + zap.String("new-dir", srv.Member.BaseDir+".backup"), + ) + + // create a new log file for next new member restart + if !fileutil.Exist(srv.Member.BaseDir) { + err = fileutil.TouchDirAll(srv.Member.BaseDir) + if err != nil { + return nil, err + } + } + if err = srv.createEtcdLogFile(); err != nil { + return nil, err + } + + return &rpcpb.Response{ + Success: true, + Status: "killed etcd and removed base directory", + }, nil +} + +func (srv *Server) handle_SAVE_SNAPSHOT() (*rpcpb.Response, error) { + err := srv.Member.SaveSnapshot(srv.lg) + if err != nil { + return nil, err + } + return &rpcpb.Response{ + Success: true, + Status: "saved snapshot", + SnapshotInfo: srv.Member.SnapshotInfo, + }, nil +} + +func (srv *Server) handle_RESTORE_RESTART_FROM_SNAPSHOT() (resp *rpcpb.Response, err error) { + err = srv.Member.RestoreSnapshot(srv.lg) + if err != nil { + return nil, err + } + resp, err = srv.handle_RESTART_FROM_SNAPSHOT() + if resp != nil && err == nil { + resp.Status = "restored snapshot and " + resp.Status + } + return resp, err +} + +func (srv *Server) handle_RESTART_FROM_SNAPSHOT() (resp *rpcpb.Response, err error) { + srv.creatEtcdCmd(true) + + if err = srv.saveTLSAssets(); err != nil { + return nil, err + } + if err = srv.startEtcdCmd(); err != nil { + return nil, err + } + srv.lg.Info("restarted etcd", zap.String("command-path", srv.etcdCmd.Path)) + if err = srv.loadAutoTLSAssets(); err != nil { + return nil, err + } + + // wait some time for etcd listener start + // before setting up proxy + // TODO: local tests should handle port conflicts + // with clients on restart + time.Sleep(time.Second) + if err = srv.startProxy(); err != nil { + return nil, err + } + + return &rpcpb.Response{ + Success: true, + Status: "restarted etcd from snapshot", + SnapshotInfo: srv.Member.SnapshotInfo, + }, nil +} + +func (srv *Server) handle_SIGQUIT_ETCD_AND_ARCHIVE_DATA() (*rpcpb.Response, error) { + srv.stopProxy() + + // exit with stackstrace + err := stopWithSig(srv.etcdCmd, syscall.SIGQUIT) + if err != nil { + return nil, err + } + srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGQUIT.String())) + + srv.etcdLogFile.Sync() + srv.etcdLogFile.Close() + + // TODO: support separate WAL directory + if err = archive( + srv.Member.BaseDir, + srv.Member.EtcdLogPath, + srv.Member.Etcd.DataDir, + ); err != nil { + return nil, err + } + srv.lg.Info("archived data", zap.String("base-dir", srv.Member.BaseDir)) + + if err = srv.createEtcdLogFile(); err != nil { + return nil, err + } + + srv.lg.Info("cleaning up page cache") + if err := cleanPageCache(); err != nil { + srv.lg.Warn("failed to clean up page cache", zap.String("error", err.Error())) + } + srv.lg.Info("cleaned up page cache") + + return &rpcpb.Response{ + Success: true, + Status: "cleaned up etcd", + }, nil +} + +// stop proxy, etcd, delete data directory +func (srv *Server) handle_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() (*rpcpb.Response, error) { + srv.stopProxy() + + err := stopWithSig(srv.etcdCmd, syscall.SIGQUIT) + if err != nil { + return nil, err + } + srv.lg.Info("killed etcd", zap.String("signal", syscall.SIGQUIT.String())) + + srv.etcdLogFile.Sync() + srv.etcdLogFile.Close() + + err = os.RemoveAll(srv.Member.BaseDir) + if err != nil { + return nil, err + } + srv.lg.Info("removed base directory", zap.String("dir", srv.Member.BaseDir)) + + // stop agent server + srv.Stop() + + return &rpcpb.Response{ + Success: true, + Status: "destroyed etcd and agent", + }, nil +} + +func (srv *Server) handle_BLACKHOLE_PEER_PORT_TX_RX() (*rpcpb.Response, error) { + for port, px := range srv.advertisePeerPortToProxy { + srv.lg.Info("blackholing", zap.Int("peer-port", port)) + px.BlackholeTx() + px.BlackholeRx() + srv.lg.Info("blackholed", zap.Int("peer-port", port)) + } + return &rpcpb.Response{ + Success: true, + Status: "blackholed peer port tx/rx", + }, nil +} + +func (srv *Server) handle_UNBLACKHOLE_PEER_PORT_TX_RX() (*rpcpb.Response, error) { + for port, px := range srv.advertisePeerPortToProxy { + srv.lg.Info("unblackholing", zap.Int("peer-port", port)) + px.UnblackholeTx() + px.UnblackholeRx() + srv.lg.Info("unblackholed", zap.Int("peer-port", port)) + } + return &rpcpb.Response{ + Success: true, + Status: "unblackholed peer port tx/rx", + }, nil +} + +func (srv *Server) handle_DELAY_PEER_PORT_TX_RX() (*rpcpb.Response, error) { + lat := time.Duration(srv.Tester.UpdatedDelayLatencyMs) * time.Millisecond + rv := time.Duration(srv.Tester.DelayLatencyMsRv) * time.Millisecond + + for port, px := range srv.advertisePeerPortToProxy { + srv.lg.Info("delaying", + zap.Int("peer-port", port), + zap.Duration("latency", lat), + zap.Duration("random-variable", rv), + ) + px.DelayTx(lat, rv) + px.DelayRx(lat, rv) + srv.lg.Info("delayed", + zap.Int("peer-port", port), + zap.Duration("latency", lat), + zap.Duration("random-variable", rv), + ) + } + + return &rpcpb.Response{ + Success: true, + Status: "delayed peer port tx/rx", + }, nil +} + +func (srv *Server) handle_UNDELAY_PEER_PORT_TX_RX() (*rpcpb.Response, error) { + for port, px := range srv.advertisePeerPortToProxy { + srv.lg.Info("undelaying", zap.Int("peer-port", port)) + px.UndelayTx() + px.UndelayRx() + srv.lg.Info("undelayed", zap.Int("peer-port", port)) + } + return &rpcpb.Response{ + Success: true, + Status: "undelayed peer port tx/rx", + }, nil +} diff --git a/functional/agent/server.go b/functional/agent/server.go new file mode 100644 index 000000000..d6313d955 --- /dev/null +++ b/functional/agent/server.go @@ -0,0 +1,166 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "math" + "net" + "os" + "os/exec" + "strings" + + "github.com/coreos/etcd/functional/rpcpb" + "github.com/coreos/etcd/pkg/proxy" + + "go.uber.org/zap" + "google.golang.org/grpc" +) + +// Server implements "rpcpb.TransportServer" +// and other etcd operations as an agent +// no need to lock fields since request operations are +// serialized in tester-side +type Server struct { + grpcServer *grpc.Server + lg *zap.Logger + + network string + address string + ln net.Listener + + rpcpb.TransportServer + last rpcpb.Operation + + *rpcpb.Member + *rpcpb.Tester + + etcdCmd *exec.Cmd + etcdLogFile *os.File + + // forward incoming advertise URLs traffic to listen URLs + advertiseClientPortToProxy map[int]proxy.Server + advertisePeerPortToProxy map[int]proxy.Server +} + +// NewServer returns a new agent server. +func NewServer( + lg *zap.Logger, + network string, + address string, +) *Server { + return &Server{ + lg: lg, + network: network, + address: address, + last: rpcpb.Operation_NOT_STARTED, + advertiseClientPortToProxy: make(map[int]proxy.Server), + advertisePeerPortToProxy: make(map[int]proxy.Server), + } +} + +const ( + maxRequestBytes = 1.5 * 1024 * 1024 + grpcOverheadBytes = 512 * 1024 + maxStreams = math.MaxUint32 + maxSendBytes = math.MaxInt32 +) + +// StartServe starts serving agent server. +func (srv *Server) StartServe() error { + var err error + srv.ln, err = net.Listen(srv.network, srv.address) + if err != nil { + return err + } + + var opts []grpc.ServerOption + opts = append(opts, grpc.MaxRecvMsgSize(int(maxRequestBytes+grpcOverheadBytes))) + opts = append(opts, grpc.MaxSendMsgSize(maxSendBytes)) + opts = append(opts, grpc.MaxConcurrentStreams(maxStreams)) + srv.grpcServer = grpc.NewServer(opts...) + + rpcpb.RegisterTransportServer(srv.grpcServer, srv) + + srv.lg.Info( + "gRPC server started", + zap.String("address", srv.address), + zap.String("listener-address", srv.ln.Addr().String()), + ) + err = srv.grpcServer.Serve(srv.ln) + if err != nil && strings.Contains(err.Error(), "use of closed network connection") { + srv.lg.Info( + "gRPC server is shut down", + zap.String("address", srv.address), + zap.Error(err), + ) + } else { + srv.lg.Warn( + "gRPC server returned with error", + zap.String("address", srv.address), + zap.Error(err), + ) + } + return err +} + +// Stop stops serving gRPC server. +func (srv *Server) Stop() { + srv.lg.Info("gRPC server stopping", zap.String("address", srv.address)) + srv.grpcServer.Stop() + srv.lg.Info("gRPC server stopped", zap.String("address", srv.address)) +} + +// Transport communicates with etcd tester. +func (srv *Server) Transport(stream rpcpb.Transport_TransportServer) (err error) { + errc := make(chan error) + go func() { + for { + var req *rpcpb.Request + req, err = stream.Recv() + if err != nil { + errc <- err + // TODO: handle error and retry + return + } + if req.Member != nil { + srv.Member = req.Member + } + if req.Tester != nil { + srv.Tester = req.Tester + } + + var resp *rpcpb.Response + resp, err = srv.handleTesterRequest(req) + if err != nil { + errc <- err + // TODO: handle error and retry + return + } + + if err = stream.Send(resp); err != nil { + errc <- err + // TODO: handle error and retry + return + } + } + }() + + select { + case err = <-errc: + case <-stream.Context().Done(): + err = stream.Context().Err() + } + return err +} diff --git a/functional/agent/utils.go b/functional/agent/utils.go new file mode 100644 index 000000000..437ffa961 --- /dev/null +++ b/functional/agent/utils.go @@ -0,0 +1,110 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "net" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + "github.com/coreos/etcd/pkg/fileutil" +) + +// TODO: support separate WAL directory +func archive(baseDir, etcdLogPath, dataDir string) error { + dir := filepath.Join(baseDir, "etcd-failure-archive", time.Now().Format(time.RFC3339)) + if existDir(dir) { + dir = filepath.Join(baseDir, "etcd-failure-archive", time.Now().Add(time.Second).Format(time.RFC3339)) + } + if err := fileutil.TouchDirAll(dir); err != nil { + return err + } + + if err := os.Rename(etcdLogPath, filepath.Join(dir, "etcd.log")); err != nil { + if !os.IsNotExist(err) { + return err + } + } + if err := os.Rename(dataDir, filepath.Join(dir, filepath.Base(dataDir))); err != nil { + if !os.IsNotExist(err) { + return err + } + } + + return nil +} + +func existDir(fpath string) bool { + st, err := os.Stat(fpath) + if err != nil { + if os.IsNotExist(err) { + return false + } + } else { + return st.IsDir() + } + return false +} + +func getURLAndPort(addr string) (urlAddr *url.URL, port int, err error) { + urlAddr, err = url.Parse(addr) + if err != nil { + return nil, -1, err + } + var s string + _, s, err = net.SplitHostPort(urlAddr.Host) + if err != nil { + return nil, -1, err + } + port, err = strconv.Atoi(s) + if err != nil { + return nil, -1, err + } + return urlAddr, port, err +} + +func stopWithSig(cmd *exec.Cmd, sig os.Signal) error { + err := cmd.Process.Signal(sig) + if err != nil { + return err + } + + errc := make(chan error) + go func() { + _, ew := cmd.Process.Wait() + errc <- ew + close(errc) + }() + + select { + case <-time.After(5 * time.Second): + cmd.Process.Kill() + case e := <-errc: + return e + } + err = <-errc + return err +} + +func cleanPageCache() error { + // https://www.kernel.org/doc/Documentation/sysctl/vm.txt + // https://github.com/torvalds/linux/blob/master/fs/drop_caches.c + cmd := exec.Command("/bin/sh", "-c", `echo "echo 1 > /proc/sys/vm/drop_caches" | sudo sh`) + return cmd.Run() +} diff --git a/functional/agent/utils_test.go b/functional/agent/utils_test.go new file mode 100644 index 000000000..162300304 --- /dev/null +++ b/functional/agent/utils_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "net/url" + "reflect" + "testing" +) + +func TestGetURLAndPort(t *testing.T) { + addr := "https://127.0.0.1:2379" + urlAddr, port, err := getURLAndPort(addr) + if err != nil { + t.Fatal(err) + } + exp := &url.URL{Scheme: "https", Host: "127.0.0.1:2379"} + if !reflect.DeepEqual(urlAddr, exp) { + t.Fatalf("expected %+v, got %+v", exp, urlAddr) + } + if port != 2379 { + t.Fatalf("port expected 2379, got %d", port) + } +} diff --git a/functional/build b/functional/build new file mode 100755 index 000000000..fd121eaf4 --- /dev/null +++ b/functional/build @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if ! [[ "$0" =~ "functional/build" ]]; then + echo "must be run from repository root" + exit 255 +fi + +CGO_ENABLED=0 go build -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-agent ./functional/cmd/etcd-agent +CGO_ENABLED=0 go build -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-proxy ./functional/cmd/etcd-proxy +CGO_ENABLED=0 go build -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-runner ./functional/cmd/etcd-runner +CGO_ENABLED=0 go build -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-tester ./functional/cmd/etcd-tester diff --git a/functional/cmd/etcd-agent/main.go b/functional/cmd/etcd-agent/main.go new file mode 100644 index 000000000..e3f217693 --- /dev/null +++ b/functional/cmd/etcd-agent/main.go @@ -0,0 +1,46 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// etcd-agent is a program that runs functional-tester agent. +package main + +import ( + "flag" + + "github.com/coreos/etcd/functional/agent" + + "go.uber.org/zap" +) + +var logger *zap.Logger + +func init() { + var err error + logger, err = zap.NewProduction() + if err != nil { + panic(err) + } +} + +func main() { + network := flag.String("network", "tcp", "network to serve agent server") + address := flag.String("address", "127.0.0.1:9027", "address to serve agent server") + flag.Parse() + + defer logger.Sync() + + srv := agent.NewServer(logger, *network, *address) + err := srv.StartServe() + logger.Info("agent exiting", zap.Error(err)) +} diff --git a/functional/cmd/etcd-proxy/main.go b/functional/cmd/etcd-proxy/main.go new file mode 100644 index 000000000..24c275e8e --- /dev/null +++ b/functional/cmd/etcd-proxy/main.go @@ -0,0 +1,216 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// etcd-proxy is a proxy layer that simulates various network conditions. +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + "net/url" + "os" + "os/signal" + "syscall" + "time" + + "github.com/coreos/etcd/pkg/proxy" + + "go.uber.org/zap" +) + +var from string +var to string +var httpPort int +var verbose bool + +func main() { + // TODO: support TLS + flag.StringVar(&from, "from", "localhost:23790", "Address URL to proxy from.") + flag.StringVar(&to, "to", "localhost:2379", "Address URL to forward.") + flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-proxy API.") + flag.BoolVar(&verbose, "verbose", false, "'true' to run proxy in verbose mode.") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage of %q:\n", os.Args[0]) + fmt.Fprintln(os.Stderr, ` +etcd-proxy simulates various network conditions for etcd testing purposes. +See README.md for more examples. + +Example: + +# build etcd +$ ./build +$ ./bin/etcd + +# build etcd-proxy +$ make build-etcd-proxy + +# to test etcd with proxy layer +$ ./bin/etcd-proxy --help +$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar`) + flag.PrintDefaults() + } + + flag.Parse() + + cfg := proxy.ServerConfig{ + From: url.URL{Scheme: "tcp", Host: from}, + To: url.URL{Scheme: "tcp", Host: to}, + } + if verbose { + cfg.Logger = zap.NewExample() + } + p := proxy.NewServer(cfg) + <-p.Ready() + defer p.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + w.Write([]byte(fmt.Sprintf("proxying [%s -> %s]\n", p.From(), p.To()))) + }) + mux.HandleFunc("/delay-tx", func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodGet: + w.Write([]byte(fmt.Sprintf("current send latency %v\n", p.LatencyTx()))) + case http.MethodPut, http.MethodPost: + if err := req.ParseForm(); err != nil { + w.Write([]byte(fmt.Sprintf("wrong form %q\n", err.Error()))) + return + } + lat, err := time.ParseDuration(req.PostForm.Get("latency")) + if err != nil { + w.Write([]byte(fmt.Sprintf("wrong latency form %q\n", err.Error()))) + return + } + rv, err := time.ParseDuration(req.PostForm.Get("random-variable")) + if err != nil { + w.Write([]byte(fmt.Sprintf("wrong random-variable form %q\n", err.Error()))) + return + } + p.DelayTx(lat, rv) + w.Write([]byte(fmt.Sprintf("added send latency %v±%v (current latency %v)\n", lat, rv, p.LatencyTx()))) + case http.MethodDelete: + lat := p.LatencyTx() + p.UndelayTx() + w.Write([]byte(fmt.Sprintf("removed latency %v\n", lat))) + default: + w.Write([]byte(fmt.Sprintf("unsupported method %q\n", req.Method))) + } + }) + mux.HandleFunc("/delay-rx", func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodGet: + w.Write([]byte(fmt.Sprintf("current receive latency %v\n", p.LatencyRx()))) + case http.MethodPut, http.MethodPost: + if err := req.ParseForm(); err != nil { + w.Write([]byte(fmt.Sprintf("wrong form %q\n", err.Error()))) + return + } + lat, err := time.ParseDuration(req.PostForm.Get("latency")) + if err != nil { + w.Write([]byte(fmt.Sprintf("wrong latency form %q\n", err.Error()))) + return + } + rv, err := time.ParseDuration(req.PostForm.Get("random-variable")) + if err != nil { + w.Write([]byte(fmt.Sprintf("wrong random-variable form %q\n", err.Error()))) + return + } + p.DelayRx(lat, rv) + w.Write([]byte(fmt.Sprintf("added receive latency %v±%v (current latency %v)\n", lat, rv, p.LatencyRx()))) + case http.MethodDelete: + lat := p.LatencyRx() + p.UndelayRx() + w.Write([]byte(fmt.Sprintf("removed latency %v\n", lat))) + default: + w.Write([]byte(fmt.Sprintf("unsupported method %q\n", req.Method))) + } + }) + mux.HandleFunc("/pause-tx", func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodPut, http.MethodPost: + p.PauseTx() + w.Write([]byte(fmt.Sprintf("paused forwarding [%s -> %s]\n", p.From(), p.To()))) + case http.MethodDelete: + p.UnpauseTx() + w.Write([]byte(fmt.Sprintf("unpaused forwarding [%s -> %s]\n", p.From(), p.To()))) + default: + w.Write([]byte(fmt.Sprintf("unsupported method %q\n", req.Method))) + } + }) + mux.HandleFunc("/pause-rx", func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodPut, http.MethodPost: + p.PauseRx() + w.Write([]byte(fmt.Sprintf("paused forwarding [%s <- %s]\n", p.From(), p.To()))) + case http.MethodDelete: + p.UnpauseRx() + w.Write([]byte(fmt.Sprintf("unpaused forwarding [%s <- %s]\n", p.From(), p.To()))) + default: + w.Write([]byte(fmt.Sprintf("unsupported method %q\n", req.Method))) + } + }) + mux.HandleFunc("/blackhole-tx", func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodPut, http.MethodPost: + p.BlackholeTx() + w.Write([]byte(fmt.Sprintf("blackholed; dropping packets [%s -> %s]\n", p.From(), p.To()))) + case http.MethodDelete: + p.UnblackholeTx() + w.Write([]byte(fmt.Sprintf("unblackholed; restart forwarding [%s -> %s]\n", p.From(), p.To()))) + default: + w.Write([]byte(fmt.Sprintf("unsupported method %q\n", req.Method))) + } + }) + mux.HandleFunc("/blackhole-rx", func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodPut, http.MethodPost: + p.BlackholeRx() + w.Write([]byte(fmt.Sprintf("blackholed; dropping packets [%s <- %s]\n", p.From(), p.To()))) + case http.MethodDelete: + p.UnblackholeRx() + w.Write([]byte(fmt.Sprintf("unblackholed; restart forwarding [%s <- %s]\n", p.From(), p.To()))) + default: + w.Write([]byte(fmt.Sprintf("unsupported method %q\n", req.Method))) + } + }) + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", httpPort), + Handler: mux, + } + defer srv.Close() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sig) + + go func() { + s := <-sig + fmt.Printf("\n\nreceived signal %q, shutting down HTTP server\n\n", s) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + err := srv.Shutdown(ctx) + cancel() + fmt.Printf("gracefully stopped HTTP server with %v\n\n", err) + os.Exit(0) + }() + + fmt.Printf("\nserving HTTP server http://localhost:%d\n\n", httpPort) + err := srv.ListenAndServe() + fmt.Printf("HTTP server exit with error %v\n", err) +} diff --git a/functional/cmd/etcd-runner/main.go b/functional/cmd/etcd-runner/main.go new file mode 100644 index 000000000..6e3cb16ce --- /dev/null +++ b/functional/cmd/etcd-runner/main.go @@ -0,0 +1,23 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// etcd-runner is a program for testing etcd clientv3 features +// against a fault injected cluster. +package main + +import "github.com/coreos/etcd/functional/runner" + +func main() { + runner.Start() +} diff --git a/functional/cmd/etcd-tester/main.go b/functional/cmd/etcd-tester/main.go new file mode 100644 index 000000000..a0e6eda68 --- /dev/null +++ b/functional/cmd/etcd-tester/main.go @@ -0,0 +1,60 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// etcd-tester is a program that runs functional-tester client. +package main + +import ( + "flag" + + "github.com/coreos/etcd/functional/tester" + + "go.uber.org/zap" +) + +var logger *zap.Logger + +func init() { + var err error + logger, err = zap.NewProduction() + if err != nil { + panic(err) + } +} + +func main() { + config := flag.String("config", "", "path to tester configuration") + flag.Parse() + + defer logger.Sync() + + clus, err := tester.NewCluster(logger, *config) + if err != nil { + logger.Fatal("failed to create a cluster", zap.Error(err)) + } + + err = clus.Send_INITIAL_START_ETCD() + if err != nil { + logger.Fatal("Bootstrap failed", zap.Error(err)) + } + defer clus.Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() + + logger.Info("wait health after bootstrap") + err = clus.WaitHealth() + if err != nil { + logger.Fatal("WaitHealth failed", zap.Error(err)) + } + + clus.Run() +} diff --git a/functional/rpcpb/etcd_config.go b/functional/rpcpb/etcd_config.go new file mode 100644 index 000000000..9d3d8c235 --- /dev/null +++ b/functional/rpcpb/etcd_config.go @@ -0,0 +1,99 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpcpb + +import ( + "fmt" + "reflect" + "strings" +) + +var etcdFields = []string{ + "Name", + "DataDir", + "WALDir", + + "HeartbeatIntervalMs", + "ElectionTimeoutMs", + + "ListenClientURLs", + "AdvertiseClientURLs", + "ClientAutoTLS", + "ClientCertAuth", + "ClientCertFile", + "ClientKeyFile", + "ClientTrustedCAFile", + + "ListenPeerURLs", + "AdvertisePeerURLs", + "PeerAutoTLS", + "PeerClientCertAuth", + "PeerCertFile", + "PeerKeyFile", + "PeerTrustedCAFile", + + "InitialCluster", + "InitialClusterState", + "InitialClusterToken", + + "SnapshotCount", + "QuotaBackendBytes", + + // "PreVote", + // "InitialCorruptCheck", +} + +// Flags returns etcd flags in string slice. +func (cfg *Etcd) Flags() (fs []string) { + tp := reflect.TypeOf(*cfg) + vo := reflect.ValueOf(*cfg) + for _, name := range etcdFields { + field, ok := tp.FieldByName(name) + if !ok { + panic(fmt.Errorf("field %q not found", name)) + } + fv := reflect.Indirect(vo).FieldByName(name) + var sv string + switch fv.Type().Kind() { + case reflect.String: + sv = fv.String() + case reflect.Slice: + n := fv.Len() + sl := make([]string, n) + for i := 0; i < n; i++ { + sl[i] = fv.Index(i).String() + } + sv = strings.Join(sl, ",") + case reflect.Int64: + sv = fmt.Sprintf("%d", fv.Int()) + case reflect.Bool: + sv = fmt.Sprintf("%v", fv.Bool()) + default: + panic(fmt.Errorf("field %q (%v) cannot be parsed", name, fv.Type().Kind())) + } + + fname := field.Tag.Get("yaml") + + // not supported in old etcd + if fname == "pre-vote" || fname == "initial-corrupt-check" { + continue + } + + if sv != "" { + fs = append(fs, fmt.Sprintf("--%s=%s", fname, sv)) + } + } + return fs +} diff --git a/functional/rpcpb/etcd_config_test.go b/functional/rpcpb/etcd_config_test.go new file mode 100644 index 000000000..fce236244 --- /dev/null +++ b/functional/rpcpb/etcd_config_test.go @@ -0,0 +1,84 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpcpb + +import ( + "reflect" + "testing" +) + +func TestEtcdFlags(t *testing.T) { + cfg := &Etcd{ + Name: "s1", + DataDir: "/tmp/etcd-agent-data-1/etcd.data", + WALDir: "/tmp/etcd-agent-data-1/etcd.data/member/wal", + + HeartbeatIntervalMs: 100, + ElectionTimeoutMs: 1000, + + ListenClientURLs: []string{"https://127.0.0.1:1379"}, + AdvertiseClientURLs: []string{"https://127.0.0.1:13790"}, + ClientAutoTLS: true, + ClientCertAuth: false, + ClientCertFile: "", + ClientKeyFile: "", + ClientTrustedCAFile: "", + + ListenPeerURLs: []string{"https://127.0.0.1:1380"}, + AdvertisePeerURLs: []string{"https://127.0.0.1:13800"}, + PeerAutoTLS: true, + PeerClientCertAuth: false, + PeerCertFile: "", + PeerKeyFile: "", + PeerTrustedCAFile: "", + + InitialCluster: "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800", + InitialClusterState: "new", + InitialClusterToken: "tkn", + + SnapshotCount: 10000, + QuotaBackendBytes: 10740000000, + + PreVote: true, + InitialCorruptCheck: true, + } + + exp := []string{ + "--name=s1", + "--data-dir=/tmp/etcd-agent-data-1/etcd.data", + "--wal-dir=/tmp/etcd-agent-data-1/etcd.data/member/wal", + "--heartbeat-interval=100", + "--election-timeout=1000", + "--listen-client-urls=https://127.0.0.1:1379", + "--advertise-client-urls=https://127.0.0.1:13790", + "--auto-tls=true", + "--client-cert-auth=false", + "--listen-peer-urls=https://127.0.0.1:1380", + "--initial-advertise-peer-urls=https://127.0.0.1:13800", + "--peer-auto-tls=true", + "--peer-client-cert-auth=false", + "--initial-cluster=s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800", + "--initial-cluster-state=new", + "--initial-cluster-token=tkn", + "--snapshot-count=10000", + "--quota-backend-bytes=10740000000", + "--pre-vote=true", + "--experimental-initial-corrupt-check=true", + } + fs := cfg.Flags() + if !reflect.DeepEqual(exp, fs) { + t.Fatalf("expected %q, got %q", exp, fs) + } +} diff --git a/functional/rpcpb/member.go b/functional/rpcpb/member.go new file mode 100644 index 000000000..01f684113 --- /dev/null +++ b/functional/rpcpb/member.go @@ -0,0 +1,361 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpcpb + +import ( + "context" + "crypto/tls" + "fmt" + "net/url" + "os" + "time" + + "github.com/coreos/etcd/clientv3" + pb "github.com/coreos/etcd/etcdserver/etcdserverpb" + "github.com/coreos/etcd/pkg/transport" + "github.com/coreos/etcd/snapshot" + + "github.com/dustin/go-humanize" + "go.uber.org/zap" + grpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// ElectionTimeout returns an election timeout duration. +func (m *Member) ElectionTimeout() time.Duration { + return time.Duration(m.Etcd.ElectionTimeoutMs) * time.Millisecond +} + +// DialEtcdGRPCServer creates a raw gRPC connection to an etcd member. +func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) { + dialOpts := []grpc.DialOption{ + grpc.WithTimeout(5 * time.Second), + grpc.WithBlock(), + } + + secure := false + for _, cu := range m.Etcd.AdvertiseClientURLs { + u, err := url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme == "https" { // TODO: handle unix + secure = true + } + } + + if secure { + // assume save TLS assets are already stord on disk + tlsInfo := transport.TLSInfo{ + CertFile: m.ClientCertPath, + KeyFile: m.ClientKeyPath, + TrustedCAFile: m.ClientTrustedCAPath, + + // TODO: remove this with generated certs + // only need it for auto TLS + InsecureSkipVerify: true, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, err + } + creds := credentials.NewTLS(tlsConfig) + dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds)) + } else { + dialOpts = append(dialOpts, grpc.WithInsecure()) + } + dialOpts = append(dialOpts, opts...) + return grpc.Dial(m.EtcdClientEndpoint, dialOpts...) +} + +// CreateEtcdClientConfig creates a client configuration from member. +func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3.Config, err error) { + secure := false + for _, cu := range m.Etcd.AdvertiseClientURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme == "https" { // TODO: handle unix + secure = true + } + } + + cfg = &clientv3.Config{ + Endpoints: []string{m.EtcdClientEndpoint}, + DialTimeout: 10 * time.Second, + DialOptions: opts, + } + if secure { + // assume save TLS assets are already stord on disk + tlsInfo := transport.TLSInfo{ + CertFile: m.ClientCertPath, + KeyFile: m.ClientKeyPath, + TrustedCAFile: m.ClientTrustedCAPath, + + // TODO: remove this with generated certs + // only need it for auto TLS + InsecureSkipVerify: true, + } + var tlsConfig *tls.Config + tlsConfig, err = tlsInfo.ClientConfig() + if err != nil { + return nil, err + } + cfg.TLS = tlsConfig + } + return cfg, err +} + +// CreateEtcdClient creates a client from member. +func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) { + cfg, err := m.CreateEtcdClientConfig(opts...) + if err != nil { + return nil, err + } + return clientv3.New(*cfg) +} + +// CheckCompact ensures that historical data before given revision has been compacted. +func (m *Member) CheckCompact(rev int64) error { + cli, err := m.CreateEtcdClient() + if err != nil { + return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + defer cli.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + wch := cli.Watch(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithRev(rev-1)) + wr, ok := <-wch + cancel() + + if !ok { + return fmt.Errorf("watch channel terminated (endpoint %q)", m.EtcdClientEndpoint) + } + if wr.CompactRevision != rev { + return fmt.Errorf("got compact revision %v, wanted %v (endpoint %q)", wr.CompactRevision, rev, m.EtcdClientEndpoint) + } + + return nil +} + +// Defrag runs defragmentation on this member. +func (m *Member) Defrag() error { + cli, err := m.CreateEtcdClient() + if err != nil { + return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + defer cli.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + _, err = cli.Defragment(ctx, m.EtcdClientEndpoint) + cancel() + return err +} + +// RevHash fetches current revision and hash on this member. +func (m *Member) RevHash() (int64, int64, error) { + conn, err := m.DialEtcdGRPCServer() + if err != nil { + return 0, 0, err + } + defer conn.Close() + + mt := pb.NewMaintenanceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + resp, err := mt.Hash(ctx, &pb.HashRequest{}, grpc.FailFast(false)) + cancel() + + if err != nil { + return 0, 0, err + } + + return resp.Header.Revision, int64(resp.Hash), nil +} + +// Rev fetches current revision on this member. +func (m *Member) Rev(ctx context.Context) (int64, error) { + cli, err := m.CreateEtcdClient() + if err != nil { + return 0, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + defer cli.Close() + + resp, err := cli.Status(ctx, m.EtcdClientEndpoint) + if err != nil { + return 0, err + } + return resp.Header.Revision, nil +} + +// Compact compacts member storage with given revision. +// It blocks until it's physically done. +func (m *Member) Compact(rev int64, timeout time.Duration) error { + cli, err := m.CreateEtcdClient() + if err != nil { + return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + defer cli.Close() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + _, err = cli.Compact(ctx, rev, clientv3.WithCompactPhysical()) + cancel() + return err +} + +// IsLeader returns true if this member is the current cluster leader. +func (m *Member) IsLeader() (bool, error) { + cli, err := m.CreateEtcdClient() + if err != nil { + return false, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + defer cli.Close() + + resp, err := cli.Status(context.Background(), m.EtcdClientEndpoint) + if err != nil { + return false, err + } + return resp.Header.MemberId == resp.Leader, nil +} + +// WriteHealthKey writes a health key to this member. +func (m *Member) WriteHealthKey() error { + cli, err := m.CreateEtcdClient() + if err != nil { + return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + defer cli.Close() + + // give enough time-out in case expensive requests (range/delete) are pending + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err = cli.Put(ctx, "health", "good") + cancel() + if err != nil { + return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + return nil +} + +// SaveSnapshot downloads a snapshot file from this member, locally. +// It's meant to requested remotely, so that local member can store +// snapshot file on local disk. +func (m *Member) SaveSnapshot(lg *zap.Logger) (err error) { + // remove existing snapshot first + if err = os.RemoveAll(m.SnapshotPath); err != nil { + return err + } + + var ccfg *clientv3.Config + ccfg, err = m.CreateEtcdClientConfig() + if err != nil { + return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) + } + + lg.Info( + "snapshot save START", + zap.String("member-name", m.Etcd.Name), + zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs), + zap.String("snapshot-path", m.SnapshotPath), + ) + now := time.Now() + mgr := snapshot.NewV3(lg) + if err = mgr.Save(context.Background(), *ccfg, m.SnapshotPath); err != nil { + return err + } + took := time.Since(now) + + var fi os.FileInfo + fi, err = os.Stat(m.SnapshotPath) + if err != nil { + return err + } + var st snapshot.Status + st, err = mgr.Status(m.SnapshotPath) + if err != nil { + return err + } + m.SnapshotInfo = &SnapshotInfo{ + MemberName: m.Etcd.Name, + MemberClientURLs: m.Etcd.AdvertiseClientURLs, + SnapshotPath: m.SnapshotPath, + SnapshotFileSize: humanize.Bytes(uint64(fi.Size())), + SnapshotTotalSize: humanize.Bytes(uint64(st.TotalSize)), + SnapshotTotalKey: int64(st.TotalKey), + SnapshotHash: int64(st.Hash), + SnapshotRevision: st.Revision, + Took: fmt.Sprintf("%v", took), + } + lg.Info( + "snapshot save END", + zap.String("member-name", m.SnapshotInfo.MemberName), + zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs), + zap.String("snapshot-path", m.SnapshotPath), + zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize), + zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize), + zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey), + zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash), + zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision), + zap.String("took", m.SnapshotInfo.Took), + ) + return nil +} + +// RestoreSnapshot restores a cluster from a given snapshot file on disk. +// It's meant to requested remotely, so that local member can load the +// snapshot file from local disk. +func (m *Member) RestoreSnapshot(lg *zap.Logger) (err error) { + if err = os.RemoveAll(m.EtcdOnSnapshotRestore.DataDir); err != nil { + return err + } + if err = os.RemoveAll(m.EtcdOnSnapshotRestore.WALDir); err != nil { + return err + } + + lg.Info( + "snapshot restore START", + zap.String("member-name", m.Etcd.Name), + zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs), + zap.String("snapshot-path", m.SnapshotPath), + ) + now := time.Now() + mgr := snapshot.NewV3(lg) + err = mgr.Restore(snapshot.RestoreConfig{ + SnapshotPath: m.SnapshotInfo.SnapshotPath, + Name: m.EtcdOnSnapshotRestore.Name, + OutputDataDir: m.EtcdOnSnapshotRestore.DataDir, + OutputWALDir: m.EtcdOnSnapshotRestore.WALDir, + PeerURLs: m.EtcdOnSnapshotRestore.AdvertisePeerURLs, + InitialCluster: m.EtcdOnSnapshotRestore.InitialCluster, + InitialClusterToken: m.EtcdOnSnapshotRestore.InitialClusterToken, + SkipHashCheck: false, + // TODO: set SkipHashCheck it true, to recover from existing db file + }) + took := time.Since(now) + lg.Info( + "snapshot restore END", + zap.String("member-name", m.SnapshotInfo.MemberName), + zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs), + zap.String("snapshot-path", m.SnapshotPath), + zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize), + zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize), + zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey), + zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash), + zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision), + zap.String("took", took.String()), + zap.Error(err), + ) + return err +} diff --git a/functional/rpcpb/rpc.pb.go b/functional/rpcpb/rpc.pb.go new file mode 100644 index 000000000..1c488dc47 --- /dev/null +++ b/functional/rpcpb/rpc.pb.go @@ -0,0 +1,5112 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: rpcpb/rpc.proto + +/* + Package rpcpb is a generated protocol buffer package. + + It is generated from these files: + rpcpb/rpc.proto + + It has these top-level messages: + Request + SnapshotInfo + Response + Member + Tester + Etcd +*/ +package rpcpb + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import context "golang.org/x/net/context" +import grpc "google.golang.org/grpc" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type Operation int32 + +const ( + // NOT_STARTED is the agent status before etcd first start. + Operation_NOT_STARTED Operation = 0 + // INITIAL_START_ETCD is only called to start etcd, the very first time. + Operation_INITIAL_START_ETCD Operation = 10 + // RESTART_ETCD is sent to restart killed etcd. + Operation_RESTART_ETCD Operation = 11 + // SIGTERM_ETCD pauses etcd process while keeping data directories + // and previous etcd configurations. + Operation_SIGTERM_ETCD Operation = 20 + // SIGQUIT_ETCD_AND_REMOVE_DATA kills etcd process and removes all data + // directories to simulate destroying the whole machine. + Operation_SIGQUIT_ETCD_AND_REMOVE_DATA Operation = 21 + // SAVE_SNAPSHOT is sent to trigger local member to download its snapshot + // onto its local disk with the specified path from tester. + Operation_SAVE_SNAPSHOT Operation = 30 + // RESTORE_RESTART_FROM_SNAPSHOT is sent to trigger local member to + // restore a cluster from existing snapshot from disk, and restart + // an etcd instance from recovered data. + Operation_RESTORE_RESTART_FROM_SNAPSHOT Operation = 31 + // RESTART_FROM_SNAPSHOT is sent to trigger local member to restart + // and join an existing cluster that has been recovered from a snapshot. + // Local member joins this cluster with fresh data. + Operation_RESTART_FROM_SNAPSHOT Operation = 32 + // SIGQUIT_ETCD_AND_ARCHIVE_DATA is sent when consistency check failed, + // thus need to archive etcd data directories. + Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA Operation = 40 + // SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT destroys etcd process, + // etcd data, and agent server. + Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT Operation = 41 + // BLACKHOLE_PEER_PORT_TX_RX drops all outgoing/incoming packets from/to + // the peer port on target member's peer port. + Operation_BLACKHOLE_PEER_PORT_TX_RX Operation = 100 + // UNBLACKHOLE_PEER_PORT_TX_RX removes outgoing/incoming packet dropping. + Operation_UNBLACKHOLE_PEER_PORT_TX_RX Operation = 101 + // DELAY_PEER_PORT_TX_RX delays all outgoing/incoming packets from/to + // the peer port on target member's peer port. + Operation_DELAY_PEER_PORT_TX_RX Operation = 200 + // UNDELAY_PEER_PORT_TX_RX removes all outgoing/incoming delays. + Operation_UNDELAY_PEER_PORT_TX_RX Operation = 201 +) + +var Operation_name = map[int32]string{ + 0: "NOT_STARTED", + 10: "INITIAL_START_ETCD", + 11: "RESTART_ETCD", + 20: "SIGTERM_ETCD", + 21: "SIGQUIT_ETCD_AND_REMOVE_DATA", + 30: "SAVE_SNAPSHOT", + 31: "RESTORE_RESTART_FROM_SNAPSHOT", + 32: "RESTART_FROM_SNAPSHOT", + 40: "SIGQUIT_ETCD_AND_ARCHIVE_DATA", + 41: "SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT", + 100: "BLACKHOLE_PEER_PORT_TX_RX", + 101: "UNBLACKHOLE_PEER_PORT_TX_RX", + 200: "DELAY_PEER_PORT_TX_RX", + 201: "UNDELAY_PEER_PORT_TX_RX", +} +var Operation_value = map[string]int32{ + "NOT_STARTED": 0, + "INITIAL_START_ETCD": 10, + "RESTART_ETCD": 11, + "SIGTERM_ETCD": 20, + "SIGQUIT_ETCD_AND_REMOVE_DATA": 21, + "SAVE_SNAPSHOT": 30, + "RESTORE_RESTART_FROM_SNAPSHOT": 31, + "RESTART_FROM_SNAPSHOT": 32, + "SIGQUIT_ETCD_AND_ARCHIVE_DATA": 40, + "SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT": 41, + "BLACKHOLE_PEER_PORT_TX_RX": 100, + "UNBLACKHOLE_PEER_PORT_TX_RX": 101, + "DELAY_PEER_PORT_TX_RX": 200, + "UNDELAY_PEER_PORT_TX_RX": 201, +} + +func (x Operation) String() string { + return proto.EnumName(Operation_name, int32(x)) +} +func (Operation) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, []int{0} } + +// Case defines various system faults or test case in distributed systems, +// in order to verify correct behavior of etcd servers and clients. +type Case int32 + +const ( + // SIGTERM_ONE_FOLLOWER stops a randomly chosen follower (non-leader) + // but does not delete its data directories on disk for next restart. + // It waits "delay-ms" before recovering this failure. + // The expected behavior is that the follower comes back online + // and rejoins the cluster, and then each member continues to process + // client requests ('Put' request that requires Raft consensus). + Case_SIGTERM_ONE_FOLLOWER Case = 0 + // SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen + // follower but does not delete its data directories on disk for next + // restart. And waits until most up-to-date node (leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that the follower comes back online and + // rejoins the cluster, and then active leader sends snapshot + // to the follower to force it to follow the leader's log. + // As always, after recovery, each member must be able to process + // client requests. + Case_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT Case = 1 + // SIGTERM_LEADER stops the active leader node but does not delete its + // data directories on disk for next restart. Then it waits "delay-ms" + // before recovering this failure, in order to trigger election timeouts. + // The expected behavior is that a new leader gets elected, and the + // old leader comes back online and rejoins the cluster as a follower. + // As always, after recovery, each member must be able to process + // client requests. + Case_SIGTERM_LEADER Case = 2 + // SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node + // but does not delete its data directories on disk for next restart. + // And waits until most up-to-date node ("new" leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that cluster elects a new leader, and the + // old leader comes back online and rejoins the cluster as a follower. + // And it receives the snapshot from the new leader to overwrite its + // store. As always, after recovery, each member must be able to + // process client requests. + Case_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT Case = 3 + // SIGTERM_QUORUM stops majority number of nodes to make the whole cluster + // inoperable but does not delete data directories on stopped nodes + // for next restart. And it waits "delay-ms" before recovering failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. + Case_SIGTERM_QUORUM Case = 4 + // SIGTERM_ALL stops the whole cluster but does not delete data directories + // on disk for next restart. And it waits "delay-ms" before recovering + // this failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. + Case_SIGTERM_ALL Case = 5 + // SIGQUIT_AND_REMOVE_ONE_FOLLOWER stops a randomly chosen follower + // (non-leader), deletes its data directories on disk, and removes + // this member from cluster (membership reconfiguration). On recovery, + // tester adds a new member, and this member joins the existing cluster + // with fresh data. It waits "delay-ms" before recovering this + // failure. This simulates destroying one follower machine, where operator + // needs to add a new member from a fresh machine. + // The expected behavior is that a new member joins the existing cluster, + // and then each member continues to process client requests. + Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER Case = 10 + // SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly + // chosen follower, deletes its data directories on disk, and removes + // this member from cluster (membership reconfiguration). On recovery, + // tester adds a new member, and this member joins the existing cluster + // restart. On member remove, cluster waits until most up-to-date node + // (leader) applies the snapshot count of entries since the stop operation. + // This simulates destroying a leader machine, where operator needs to add + // a new member from a fresh machine. + // The expected behavior is that a new member joins the existing cluster, + // and receives a snapshot from the active leader. As always, after + // recovery, each member must be able to process client requests. + Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT Case = 11 + // SIGQUIT_AND_REMOVE_LEADER stops the active leader node, deletes its + // data directories on disk, and removes this member from cluster. + // On recovery, tester adds a new member, and this member joins the + // existing cluster with fresh data. It waits "delay-ms" before + // recovering this failure. This simulates destroying a leader machine, + // where operator needs to add a new member from a fresh machine. + // The expected behavior is that a new member joins the existing cluster, + // and then each member continues to process client requests. + Case_SIGQUIT_AND_REMOVE_LEADER Case = 12 + // SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader, + // deletes its data directories on disk, and removes this member from + // cluster (membership reconfiguration). On recovery, tester adds a new + // member, and this member joins the existing cluster restart. On member + // remove, cluster waits until most up-to-date node (new leader) applies + // the snapshot count of entries since the stop operation. This simulates + // destroying a leader machine, where operator needs to add a new member + // from a fresh machine. + // The expected behavior is that on member remove, cluster elects a new + // leader, and a new member joins the existing cluster and receives a + // snapshot from the newly elected leader. As always, after recovery, each + // member must be able to process client requests. + Case_SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT Case = 13 + // SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH first + // stops majority number of nodes, deletes data directories on those quorum + // nodes, to make the whole cluster inoperable. Now that quorum and their + // data are totally destroyed, cluster cannot even remove unavailable nodes + // (e.g. 2 out of 3 are lost, so no leader can be elected). + // Let's assume 3-node cluster of node A, B, and C. One day, node A and B + // are destroyed and all their data are gone. The only viable solution is + // to recover from C's latest snapshot. + // + // To simulate: + // 1. Assume node C is the current leader with most up-to-date data. + // 2. Download snapshot from node C, before destroying node A and B. + // 3. Destroy node A and B, and make the whole cluster inoperable. + // 4. Now node C cannot operate either. + // 5. SIGTERM node C and remove its data directories. + // 6. Restore a new seed member from node C's latest snapshot file. + // 7. Add another member to establish 2-node cluster. + // 8. Add another member to establish 3-node cluster. + // 9. Add more if any. + // + // The expected behavior is that etcd successfully recovers from such + // disastrous situation as only 1-node survives out of 3-node cluster, + // new members joins the existing cluster, and previous data from snapshot + // are still preserved after recovery process. As always, after recovery, + // each member must be able to process client requests. + Case_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH Case = 14 + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER drops all outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader), and waits for "delay-ms" until recovery. + // The expected behavior is that once dropping operation is undone, + // each member must be able to process client requests. + Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER Case = 100 + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT drops + // all outgoing/incoming packets from/to the peer port on a randomly + // chosen follower (non-leader), and waits for most up-to-date node + // (leader) applies the snapshot count of entries since the blackhole + // operation. + // The expected behavior is that once packet drop operation is undone, + // the slow follower tries to catch up, possibly receiving the snapshot + // from the active leader. As always, after recovery, each member must + // be able to process client requests. + Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT Case = 101 + // BLACKHOLE_PEER_PORT_TX_RX_LEADER drops all outgoing/incoming packets + // from/to the peer port on the active leader (isolated), and waits for + // "delay-ms" until recovery, in order to trigger election timeout. + // The expected behavior is that after election timeout, a new leader gets + // elected, and once dropping operation is undone, the old leader comes + // back and rejoins the cluster as a follower. As always, after recovery, + // each member must be able to process client requests. + Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER Case = 102 + // BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT drops all + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (leader) applies the snapshot + // count of entries since the blackhole operation. + // The expected behavior is that cluster elects a new leader, and once + // dropping operation is undone, the old leader comes back and rejoins + // the cluster as a follower. The slow follower tries to catch up, likely + // receiving the snapshot from the new active leader. As always, after + // recovery, each member must be able to process client requests. + Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT Case = 103 + // BLACKHOLE_PEER_PORT_TX_RX_QUORUM drops all outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, thus losing its + // leader and cluster being inoperable. And it waits for "delay-ms" + // until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + Case_BLACKHOLE_PEER_PORT_TX_RX_QUORUM Case = 104 + // BLACKHOLE_PEER_PORT_TX_RX_ALL drops all outgoing/incoming packets + // from/to the peer ports on all nodes, thus making cluster totally + // inoperable. It waits for "delay-ms" until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + Case_BLACKHOLE_PEER_PORT_TX_RX_ALL Case = 105 + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming packets + // from/to the peer port on a randomly chosen follower (non-leader). + // It waits for "delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // the follower comes back and tries to catch up with latest changes from + // cluster. And as always, after recovery, each member must be able to + // process client requests. + Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER Case = 200 + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader) with a randomized time duration (thus isolated). It + // waits for "delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // each member must be able to process client requests. + Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER Case = 201 + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader), and waits for most up-to-date node (leader) + // applies the snapshot count of entries since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up possibly receiving snapshot + // from the active leader. As always, after recovery, each member must be + // able to process client requests. + Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT Case = 202 + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader) with a randomized time duration, and waits for + // most up-to-date node (leader) applies the snapshot count of entries + // since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up, possibly receiving a + // snapshot from the active leader. As always, after recovery, each member + // must be able to process client requests. + Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT Case = 203 + // DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets from/to + // the peer port on the active leader. And waits for "delay-ms" until + // recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. + Case_DELAY_PEER_PORT_TX_RX_LEADER Case = 204 + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets + // from/to the peer port on the active leader with a randomized time + // duration. And waits for "delay-ms" until recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. + Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER Case = 205 + // DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (current or new leader) applies the + // snapshot count of entries since the delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. + Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT Case = 206 + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // with a randomized time duration. And it waits for most up-to-date node + // (current or new leader) applies the snapshot count of entries since the + // delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. + Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT Case = 207 + // DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets from/to + // the peer ports on majority nodes of cluster. And it waits for + // "delay-ms" until recovery, likely to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + Case_DELAY_PEER_PORT_TX_RX_QUORUM Case = 208 + // RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, with randomized + // time durations. And it waits for "delay-ms" until recovery, likely + // to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + Case_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM Case = 209 + // DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets from/to the + // peer ports on all nodes. And it waits for "delay-ms" until recovery, + // likely to trigger election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. + Case_DELAY_PEER_PORT_TX_RX_ALL Case = 210 + // RANDOM_DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets + // from/to the peer ports on all nodes, with randomized time durations. + // And it waits for "delay-ms" until recovery, likely to trigger + // election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. + Case_RANDOM_DELAY_PEER_PORT_TX_RX_ALL Case = 211 + // NO_FAIL_WITH_STRESS stops injecting failures while testing the + // consistency and correctness under pressure loads, for the duration of + // "delay-ms". Goal is to ensure cluster be still making progress + // on recovery, and verify system does not deadlock following a sequence + // of failure injections. + // The expected behavior is that cluster remains fully operative in healthy + // condition. As always, after recovery, each member must be able to process + // client requests. + Case_NO_FAIL_WITH_STRESS Case = 300 + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS neither injects failures nor + // sends stressig client requests to the cluster, for the duration of + // "delay-ms". Goal is to ensure cluster be still making progress + // on recovery, and verify system does not deadlock following a sequence + // of failure injections. + // The expected behavior is that cluster remains fully operative in healthy + // condition, and clients requests during liveness period succeed without + // errors. + // Note: this is how Google Chubby does failure injection testing + // https://static.googleusercontent.com/media/research.google.com/en//archive/paxos_made_live.pdf. + Case_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS Case = 301 + // FAILPOINTS injects failpoints to etcd server runtime, triggering panics + // in critical code paths. + Case_FAILPOINTS Case = 400 + // EXTERNAL runs external failure injection scripts. + Case_EXTERNAL Case = 500 +) + +var Case_name = map[int32]string{ + 0: "SIGTERM_ONE_FOLLOWER", + 1: "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + 2: "SIGTERM_LEADER", + 3: "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT", + 4: "SIGTERM_QUORUM", + 5: "SIGTERM_ALL", + 10: "SIGQUIT_AND_REMOVE_ONE_FOLLOWER", + 11: "SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + 12: "SIGQUIT_AND_REMOVE_LEADER", + 13: "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT", + 14: "SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH", + 100: "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER", + 101: "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + 102: "BLACKHOLE_PEER_PORT_TX_RX_LEADER", + 103: "BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT", + 104: "BLACKHOLE_PEER_PORT_TX_RX_QUORUM", + 105: "BLACKHOLE_PEER_PORT_TX_RX_ALL", + 200: "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", + 201: "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", + 202: "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + 203: "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT", + 204: "DELAY_PEER_PORT_TX_RX_LEADER", + 205: "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER", + 206: "DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT", + 207: "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT", + 208: "DELAY_PEER_PORT_TX_RX_QUORUM", + 209: "RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM", + 210: "DELAY_PEER_PORT_TX_RX_ALL", + 211: "RANDOM_DELAY_PEER_PORT_TX_RX_ALL", + 300: "NO_FAIL_WITH_STRESS", + 301: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", + 400: "FAILPOINTS", + 500: "EXTERNAL", +} +var Case_value = map[string]int32{ + "SIGTERM_ONE_FOLLOWER": 0, + "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 1, + "SIGTERM_LEADER": 2, + "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT": 3, + "SIGTERM_QUORUM": 4, + "SIGTERM_ALL": 5, + "SIGQUIT_AND_REMOVE_ONE_FOLLOWER": 10, + "SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 11, + "SIGQUIT_AND_REMOVE_LEADER": 12, + "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT": 13, + "SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH": 14, + "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": 100, + "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 101, + "BLACKHOLE_PEER_PORT_TX_RX_LEADER": 102, + "BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": 103, + "BLACKHOLE_PEER_PORT_TX_RX_QUORUM": 104, + "BLACKHOLE_PEER_PORT_TX_RX_ALL": 105, + "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": 200, + "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": 201, + "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 202, + "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": 203, + "DELAY_PEER_PORT_TX_RX_LEADER": 204, + "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER": 205, + "DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": 206, + "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": 207, + "DELAY_PEER_PORT_TX_RX_QUORUM": 208, + "RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM": 209, + "DELAY_PEER_PORT_TX_RX_ALL": 210, + "RANDOM_DELAY_PEER_PORT_TX_RX_ALL": 211, + "NO_FAIL_WITH_STRESS": 300, + "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 301, + "FAILPOINTS": 400, + "EXTERNAL": 500, +} + +func (x Case) String() string { + return proto.EnumName(Case_name, int32(x)) +} +func (Case) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } + +type Stresser int32 + +const ( + Stresser_KV Stresser = 0 + Stresser_LEASE Stresser = 1 + Stresser_ELECTION_RUNNER Stresser = 2 + Stresser_WATCH_RUNNER Stresser = 3 + Stresser_LOCK_RACER_RUNNER Stresser = 4 + Stresser_LEASE_RUNNER Stresser = 5 +) + +var Stresser_name = map[int32]string{ + 0: "KV", + 1: "LEASE", + 2: "ELECTION_RUNNER", + 3: "WATCH_RUNNER", + 4: "LOCK_RACER_RUNNER", + 5: "LEASE_RUNNER", +} +var Stresser_value = map[string]int32{ + "KV": 0, + "LEASE": 1, + "ELECTION_RUNNER": 2, + "WATCH_RUNNER": 3, + "LOCK_RACER_RUNNER": 4, + "LEASE_RUNNER": 5, +} + +func (x Stresser) String() string { + return proto.EnumName(Stresser_name, int32(x)) +} +func (Stresser) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } + +type Checker int32 + +const ( + Checker_KV_HASH Checker = 0 + Checker_LEASE_EXPIRE Checker = 1 + Checker_RUNNER Checker = 2 + Checker_NO_CHECK Checker = 3 +) + +var Checker_name = map[int32]string{ + 0: "KV_HASH", + 1: "LEASE_EXPIRE", + 2: "RUNNER", + 3: "NO_CHECK", +} +var Checker_value = map[string]int32{ + "KV_HASH": 0, + "LEASE_EXPIRE": 1, + "RUNNER": 2, + "NO_CHECK": 3, +} + +func (x Checker) String() string { + return proto.EnumName(Checker_name, int32(x)) +} +func (Checker) EnumDescriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } + +type Request struct { + Operation Operation `protobuf:"varint,1,opt,name=Operation,proto3,enum=rpcpb.Operation" json:"Operation,omitempty"` + // Member contains the same Member object from tester configuration. + Member *Member `protobuf:"bytes,2,opt,name=Member" json:"Member,omitempty"` + // Tester contains tester configuration. + Tester *Tester `protobuf:"bytes,3,opt,name=Tester" json:"Tester,omitempty"` +} + +func (m *Request) Reset() { *m = Request{} } +func (m *Request) String() string { return proto.CompactTextString(m) } +func (*Request) ProtoMessage() {} +func (*Request) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{0} } + +// SnapshotInfo contains SAVE_SNAPSHOT request results. +type SnapshotInfo struct { + MemberName string `protobuf:"bytes,1,opt,name=MemberName,proto3" json:"MemberName,omitempty"` + MemberClientURLs []string `protobuf:"bytes,2,rep,name=MemberClientURLs" json:"MemberClientURLs,omitempty"` + SnapshotPath string `protobuf:"bytes,3,opt,name=SnapshotPath,proto3" json:"SnapshotPath,omitempty"` + SnapshotFileSize string `protobuf:"bytes,4,opt,name=SnapshotFileSize,proto3" json:"SnapshotFileSize,omitempty"` + SnapshotTotalSize string `protobuf:"bytes,5,opt,name=SnapshotTotalSize,proto3" json:"SnapshotTotalSize,omitempty"` + SnapshotTotalKey int64 `protobuf:"varint,6,opt,name=SnapshotTotalKey,proto3" json:"SnapshotTotalKey,omitempty"` + SnapshotHash int64 `protobuf:"varint,7,opt,name=SnapshotHash,proto3" json:"SnapshotHash,omitempty"` + SnapshotRevision int64 `protobuf:"varint,8,opt,name=SnapshotRevision,proto3" json:"SnapshotRevision,omitempty"` + Took string `protobuf:"bytes,9,opt,name=Took,proto3" json:"Took,omitempty"` +} + +func (m *SnapshotInfo) Reset() { *m = SnapshotInfo{} } +func (m *SnapshotInfo) String() string { return proto.CompactTextString(m) } +func (*SnapshotInfo) ProtoMessage() {} +func (*SnapshotInfo) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } + +type Response struct { + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` + Status string `protobuf:"bytes,2,opt,name=Status,proto3" json:"Status,omitempty"` + // Member contains the same Member object from tester request. + Member *Member `protobuf:"bytes,3,opt,name=Member" json:"Member,omitempty"` + // SnapshotInfo contains SAVE_SNAPSHOT request results. + SnapshotInfo *SnapshotInfo `protobuf:"bytes,4,opt,name=SnapshotInfo" json:"SnapshotInfo,omitempty"` +} + +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } + +type Member struct { + // EtcdExecPath is the executable etcd binary path in agent server. + EtcdExecPath string `protobuf:"bytes,1,opt,name=EtcdExecPath,proto3" json:"EtcdExecPath,omitempty" yaml:"etcd-exec-path"` + // AgentAddr is the agent HTTP server address. + AgentAddr string `protobuf:"bytes,11,opt,name=AgentAddr,proto3" json:"AgentAddr,omitempty" yaml:"agent-addr"` + // FailpointHTTPAddr is the agent's failpoints HTTP server address. + FailpointHTTPAddr string `protobuf:"bytes,12,opt,name=FailpointHTTPAddr,proto3" json:"FailpointHTTPAddr,omitempty" yaml:"failpoint-http-addr"` + // BaseDir is the base directory where all logs and etcd data are stored. + BaseDir string `protobuf:"bytes,101,opt,name=BaseDir,proto3" json:"BaseDir,omitempty" yaml:"base-dir"` + // EtcdLogPath is the log file to store current etcd server logs. + EtcdLogPath string `protobuf:"bytes,102,opt,name=EtcdLogPath,proto3" json:"EtcdLogPath,omitempty" yaml:"etcd-log-path"` + // EtcdClientProxy is true when client traffic needs to be proxied. + // If true, listen client URL port must be different than advertise client URL port. + EtcdClientProxy bool `protobuf:"varint,201,opt,name=EtcdClientProxy,proto3" json:"EtcdClientProxy,omitempty" yaml:"etcd-client-proxy"` + // EtcdPeerProxy is true when peer traffic needs to be proxied. + // If true, listen peer URL port must be different than advertise peer URL port. + EtcdPeerProxy bool `protobuf:"varint,202,opt,name=EtcdPeerProxy,proto3" json:"EtcdPeerProxy,omitempty" yaml:"etcd-peer-proxy"` + // EtcdClientEndpoint is the etcd client endpoint. + EtcdClientEndpoint string `protobuf:"bytes,301,opt,name=EtcdClientEndpoint,proto3" json:"EtcdClientEndpoint,omitempty" yaml:"etcd-client-endpoint"` + // Etcd defines etcd binary configuration flags. + Etcd *Etcd `protobuf:"bytes,302,opt,name=Etcd" json:"Etcd,omitempty" yaml:"etcd"` + // EtcdOnSnapshotRestore defines one-time use configuration during etcd + // snapshot recovery process. + EtcdOnSnapshotRestore *Etcd `protobuf:"bytes,303,opt,name=EtcdOnSnapshotRestore" json:"EtcdOnSnapshotRestore,omitempty"` + // ClientCertData contains cert file contents from this member's etcd server. + ClientCertData string `protobuf:"bytes,401,opt,name=ClientCertData,proto3" json:"ClientCertData,omitempty" yaml:"client-cert-data"` + ClientCertPath string `protobuf:"bytes,402,opt,name=ClientCertPath,proto3" json:"ClientCertPath,omitempty" yaml:"client-cert-path"` + // ClientKeyData contains key file contents from this member's etcd server. + ClientKeyData string `protobuf:"bytes,403,opt,name=ClientKeyData,proto3" json:"ClientKeyData,omitempty" yaml:"client-key-data"` + ClientKeyPath string `protobuf:"bytes,404,opt,name=ClientKeyPath,proto3" json:"ClientKeyPath,omitempty" yaml:"client-key-path"` + // ClientTrustedCAData contains trusted CA file contents from this member's etcd server. + ClientTrustedCAData string `protobuf:"bytes,405,opt,name=ClientTrustedCAData,proto3" json:"ClientTrustedCAData,omitempty" yaml:"client-trusted-ca-data"` + ClientTrustedCAPath string `protobuf:"bytes,406,opt,name=ClientTrustedCAPath,proto3" json:"ClientTrustedCAPath,omitempty" yaml:"client-trusted-ca-path"` + // PeerCertData contains cert file contents from this member's etcd server. + PeerCertData string `protobuf:"bytes,501,opt,name=PeerCertData,proto3" json:"PeerCertData,omitempty" yaml:"peer-cert-data"` + PeerCertPath string `protobuf:"bytes,502,opt,name=PeerCertPath,proto3" json:"PeerCertPath,omitempty" yaml:"peer-cert-path"` + // PeerKeyData contains key file contents from this member's etcd server. + PeerKeyData string `protobuf:"bytes,503,opt,name=PeerKeyData,proto3" json:"PeerKeyData,omitempty" yaml:"peer-key-data"` + PeerKeyPath string `protobuf:"bytes,504,opt,name=PeerKeyPath,proto3" json:"PeerKeyPath,omitempty" yaml:"peer-key-path"` + // PeerTrustedCAData contains trusted CA file contents from this member's etcd server. + PeerTrustedCAData string `protobuf:"bytes,505,opt,name=PeerTrustedCAData,proto3" json:"PeerTrustedCAData,omitempty" yaml:"peer-trusted-ca-data"` + PeerTrustedCAPath string `protobuf:"bytes,506,opt,name=PeerTrustedCAPath,proto3" json:"PeerTrustedCAPath,omitempty" yaml:"peer-trusted-ca-path"` + // SnapshotPath is the snapshot file path to store or restore from. + SnapshotPath string `protobuf:"bytes,601,opt,name=SnapshotPath,proto3" json:"SnapshotPath,omitempty" yaml:"snapshot-path"` + // SnapshotInfo contains last SAVE_SNAPSHOT request results. + SnapshotInfo *SnapshotInfo `protobuf:"bytes,602,opt,name=SnapshotInfo" json:"SnapshotInfo,omitempty"` +} + +func (m *Member) Reset() { *m = Member{} } +func (m *Member) String() string { return proto.CompactTextString(m) } +func (*Member) ProtoMessage() {} +func (*Member) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } + +type Tester struct { + DataDir string `protobuf:"bytes,1,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"` + Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty" yaml:"network"` + Addr string `protobuf:"bytes,3,opt,name=Addr,proto3" json:"Addr,omitempty" yaml:"addr"` + // DelayLatencyMsRv is the delay latency in milliseconds, + // to inject to simulated slow network. + DelayLatencyMs uint32 `protobuf:"varint,11,opt,name=DelayLatencyMs,proto3" json:"DelayLatencyMs,omitempty" yaml:"delay-latency-ms"` + // DelayLatencyMsRv is the delay latency random variable in milliseconds. + DelayLatencyMsRv uint32 `protobuf:"varint,12,opt,name=DelayLatencyMsRv,proto3" json:"DelayLatencyMsRv,omitempty" yaml:"delay-latency-ms-rv"` + // UpdatedDelayLatencyMs is the update delay latency in milliseconds, + // to inject to simulated slow network. It's the final latency to apply, + // in case the latency numbers are randomly generated from given delay latency field. + UpdatedDelayLatencyMs uint32 `protobuf:"varint,13,opt,name=UpdatedDelayLatencyMs,proto3" json:"UpdatedDelayLatencyMs,omitempty" yaml:"updated-delay-latency-ms"` + // RoundLimit is the limit of rounds to run failure set (-1 to run without limits). + RoundLimit int32 `protobuf:"varint,21,opt,name=RoundLimit,proto3" json:"RoundLimit,omitempty" yaml:"round-limit"` + // ExitOnCaseFail is true, then exit tester on first failure. + ExitOnCaseFail bool `protobuf:"varint,22,opt,name=ExitOnCaseFail,proto3" json:"ExitOnCaseFail,omitempty" yaml:"exit-on-failure"` + // EnablePprof is true to enable profiler. + EnablePprof bool `protobuf:"varint,23,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"` + // CaseDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + CaseDelayMs uint32 `protobuf:"varint,31,opt,name=CaseDelayMs,proto3" json:"CaseDelayMs,omitempty" yaml:"case-delay-ms"` + // CaseShuffle is true to randomize failure injecting order. + CaseShuffle bool `protobuf:"varint,32,opt,name=CaseShuffle,proto3" json:"CaseShuffle,omitempty" yaml:"case-shuffle"` + // Cases is the selected test cases to schedule. + // If empty, run all failure cases. + Cases []string `protobuf:"bytes,33,rep,name=Cases" json:"Cases,omitempty" yaml:"cases"` + // FailpointCommands is the list of "gofail" commands + // (e.g. panic("etcd-tester"),1*sleep(1000). + FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` + // RunnerExecPath is a path of etcd-runner binary. + RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"` + // ExternalExecPath is a path of script for enabling/disabling an external fault injector. + ExternalExecPath string `protobuf:"bytes,42,opt,name=ExternalExecPath,proto3" json:"ExternalExecPath,omitempty" yaml:"external-exec-path"` + // Stressers is the list of stresser types: + // KV, LEASE, ELECTION_RUNNER, WATCH_RUNNER, LOCK_RACER_RUNNER, LEASE_RUNNER. + Stressers []string `protobuf:"bytes,101,rep,name=Stressers" json:"Stressers,omitempty" yaml:"stressers"` + // Checkers is the list of consistency checker types: + // KV_HASH, LEASE_EXPIRE, NO_CHECK, RUNNER. + // Leave empty to skip consistency checks. + Checkers []string `protobuf:"bytes,102,rep,name=Checkers" json:"Checkers,omitempty" yaml:"checkers"` + // StressKeySize is the size of each small key written into etcd. + StressKeySize int32 `protobuf:"varint,201,opt,name=StressKeySize,proto3" json:"StressKeySize,omitempty" yaml:"stress-key-size"` + // StressKeySizeLarge is the size of each large key written into etcd. + StressKeySizeLarge int32 `protobuf:"varint,202,opt,name=StressKeySizeLarge,proto3" json:"StressKeySizeLarge,omitempty" yaml:"stress-key-size-large"` + // StressKeySuffixRange is the count of key range written into etcd. + // Stress keys are created with "fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange)". + StressKeySuffixRange int32 `protobuf:"varint,203,opt,name=StressKeySuffixRange,proto3" json:"StressKeySuffixRange,omitempty" yaml:"stress-key-suffix-range"` + // StressKeySuffixRangeTxn is the count of key range written into etcd txn (max 100). + // Stress keys are created with "fmt.Sprintf("/k%03d", i)". + StressKeySuffixRangeTxn int32 `protobuf:"varint,204,opt,name=StressKeySuffixRangeTxn,proto3" json:"StressKeySuffixRangeTxn,omitempty" yaml:"stress-key-suffix-range-txn"` + // StressKeyTxnOps is the number of operations per a transaction (max 64). + StressKeyTxnOps int32 `protobuf:"varint,205,opt,name=StressKeyTxnOps,proto3" json:"StressKeyTxnOps,omitempty" yaml:"stress-key-txn-ops"` + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + StressClients int32 `protobuf:"varint,301,opt,name=StressClients,proto3" json:"StressClients,omitempty" yaml:"stress-clients"` + // StressQPS is the maximum number of stresser requests per second. + StressQPS int32 `protobuf:"varint,302,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"` +} + +func (m *Tester) Reset() { *m = Tester{} } +func (m *Tester) String() string { return proto.CompactTextString(m) } +func (*Tester) ProtoMessage() {} +func (*Tester) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{4} } + +type Etcd struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty" yaml:"name"` + DataDir string `protobuf:"bytes,2,opt,name=DataDir,proto3" json:"DataDir,omitempty" yaml:"data-dir"` + WALDir string `protobuf:"bytes,3,opt,name=WALDir,proto3" json:"WALDir,omitempty" yaml:"wal-dir"` + // HeartbeatIntervalMs is the time (in milliseconds) of a heartbeat interval. + // Default value is 100, which is 100ms. + HeartbeatIntervalMs int64 `protobuf:"varint,11,opt,name=HeartbeatIntervalMs,proto3" json:"HeartbeatIntervalMs,omitempty" yaml:"heartbeat-interval"` + // ElectionTimeoutMs is the time (in milliseconds) for an election to timeout. + // Default value is 1000, which is 1s. + ElectionTimeoutMs int64 `protobuf:"varint,12,opt,name=ElectionTimeoutMs,proto3" json:"ElectionTimeoutMs,omitempty" yaml:"election-timeout"` + ListenClientURLs []string `protobuf:"bytes,21,rep,name=ListenClientURLs" json:"ListenClientURLs,omitempty" yaml:"listen-client-urls"` + AdvertiseClientURLs []string `protobuf:"bytes,22,rep,name=AdvertiseClientURLs" json:"AdvertiseClientURLs,omitempty" yaml:"advertise-client-urls"` + ClientAutoTLS bool `protobuf:"varint,23,opt,name=ClientAutoTLS,proto3" json:"ClientAutoTLS,omitempty" yaml:"auto-tls"` + ClientCertAuth bool `protobuf:"varint,24,opt,name=ClientCertAuth,proto3" json:"ClientCertAuth,omitempty" yaml:"client-cert-auth"` + ClientCertFile string `protobuf:"bytes,25,opt,name=ClientCertFile,proto3" json:"ClientCertFile,omitempty" yaml:"cert-file"` + ClientKeyFile string `protobuf:"bytes,26,opt,name=ClientKeyFile,proto3" json:"ClientKeyFile,omitempty" yaml:"key-file"` + ClientTrustedCAFile string `protobuf:"bytes,27,opt,name=ClientTrustedCAFile,proto3" json:"ClientTrustedCAFile,omitempty" yaml:"trusted-ca-file"` + ListenPeerURLs []string `protobuf:"bytes,31,rep,name=ListenPeerURLs" json:"ListenPeerURLs,omitempty" yaml:"listen-peer-urls"` + AdvertisePeerURLs []string `protobuf:"bytes,32,rep,name=AdvertisePeerURLs" json:"AdvertisePeerURLs,omitempty" yaml:"initial-advertise-peer-urls"` + PeerAutoTLS bool `protobuf:"varint,33,opt,name=PeerAutoTLS,proto3" json:"PeerAutoTLS,omitempty" yaml:"peer-auto-tls"` + PeerClientCertAuth bool `protobuf:"varint,34,opt,name=PeerClientCertAuth,proto3" json:"PeerClientCertAuth,omitempty" yaml:"peer-client-cert-auth"` + PeerCertFile string `protobuf:"bytes,35,opt,name=PeerCertFile,proto3" json:"PeerCertFile,omitempty" yaml:"peer-cert-file"` + PeerKeyFile string `protobuf:"bytes,36,opt,name=PeerKeyFile,proto3" json:"PeerKeyFile,omitempty" yaml:"peer-key-file"` + PeerTrustedCAFile string `protobuf:"bytes,37,opt,name=PeerTrustedCAFile,proto3" json:"PeerTrustedCAFile,omitempty" yaml:"peer-trusted-ca-file"` + InitialCluster string `protobuf:"bytes,41,opt,name=InitialCluster,proto3" json:"InitialCluster,omitempty" yaml:"initial-cluster"` + InitialClusterState string `protobuf:"bytes,42,opt,name=InitialClusterState,proto3" json:"InitialClusterState,omitempty" yaml:"initial-cluster-state"` + InitialClusterToken string `protobuf:"bytes,43,opt,name=InitialClusterToken,proto3" json:"InitialClusterToken,omitempty" yaml:"initial-cluster-token"` + SnapshotCount int64 `protobuf:"varint,51,opt,name=SnapshotCount,proto3" json:"SnapshotCount,omitempty" yaml:"snapshot-count"` + QuotaBackendBytes int64 `protobuf:"varint,52,opt,name=QuotaBackendBytes,proto3" json:"QuotaBackendBytes,omitempty" yaml:"quota-backend-bytes"` + PreVote bool `protobuf:"varint,63,opt,name=PreVote,proto3" json:"PreVote,omitempty" yaml:"pre-vote"` + InitialCorruptCheck bool `protobuf:"varint,64,opt,name=InitialCorruptCheck,proto3" json:"InitialCorruptCheck,omitempty" yaml:"initial-corrupt-check"` +} + +func (m *Etcd) Reset() { *m = Etcd{} } +func (m *Etcd) String() string { return proto.CompactTextString(m) } +func (*Etcd) ProtoMessage() {} +func (*Etcd) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{5} } + +func init() { + proto.RegisterType((*Request)(nil), "rpcpb.Request") + proto.RegisterType((*SnapshotInfo)(nil), "rpcpb.SnapshotInfo") + proto.RegisterType((*Response)(nil), "rpcpb.Response") + proto.RegisterType((*Member)(nil), "rpcpb.Member") + proto.RegisterType((*Tester)(nil), "rpcpb.Tester") + proto.RegisterType((*Etcd)(nil), "rpcpb.Etcd") + proto.RegisterEnum("rpcpb.Operation", Operation_name, Operation_value) + proto.RegisterEnum("rpcpb.Case", Case_name, Case_value) + proto.RegisterEnum("rpcpb.Stresser", Stresser_name, Stresser_value) + proto.RegisterEnum("rpcpb.Checker", Checker_name, Checker_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Transport service + +type TransportClient interface { + Transport(ctx context.Context, opts ...grpc.CallOption) (Transport_TransportClient, error) +} + +type transportClient struct { + cc *grpc.ClientConn +} + +func NewTransportClient(cc *grpc.ClientConn) TransportClient { + return &transportClient{cc} +} + +func (c *transportClient) Transport(ctx context.Context, opts ...grpc.CallOption) (Transport_TransportClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Transport_serviceDesc.Streams[0], c.cc, "/rpcpb.Transport/Transport", opts...) + if err != nil { + return nil, err + } + x := &transportTransportClient{stream} + return x, nil +} + +type Transport_TransportClient interface { + Send(*Request) error + Recv() (*Response, error) + grpc.ClientStream +} + +type transportTransportClient struct { + grpc.ClientStream +} + +func (x *transportTransportClient) Send(m *Request) error { + return x.ClientStream.SendMsg(m) +} + +func (x *transportTransportClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// Server API for Transport service + +type TransportServer interface { + Transport(Transport_TransportServer) error +} + +func RegisterTransportServer(s *grpc.Server, srv TransportServer) { + s.RegisterService(&_Transport_serviceDesc, srv) +} + +func _Transport_Transport_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TransportServer).Transport(&transportTransportServer{stream}) +} + +type Transport_TransportServer interface { + Send(*Response) error + Recv() (*Request, error) + grpc.ServerStream +} + +type transportTransportServer struct { + grpc.ServerStream +} + +func (x *transportTransportServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *transportTransportServer) Recv() (*Request, error) { + m := new(Request) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _Transport_serviceDesc = grpc.ServiceDesc{ + ServiceName: "rpcpb.Transport", + HandlerType: (*TransportServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Transport", + Handler: _Transport_Transport_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "rpcpb/rpc.proto", +} + +func (m *Request) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Request) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Operation != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.Operation)) + } + if m.Member != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.Member.Size())) + n1, err := m.Member.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + } + if m.Tester != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.Tester.Size())) + n2, err := m.Tester.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + } + return i, nil +} + +func (m *SnapshotInfo) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SnapshotInfo) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.MemberName) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.MemberName))) + i += copy(dAtA[i:], m.MemberName) + } + if len(m.MemberClientURLs) > 0 { + for _, s := range m.MemberClientURLs { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.SnapshotPath) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.SnapshotPath))) + i += copy(dAtA[i:], m.SnapshotPath) + } + if len(m.SnapshotFileSize) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.SnapshotFileSize))) + i += copy(dAtA[i:], m.SnapshotFileSize) + } + if len(m.SnapshotTotalSize) > 0 { + dAtA[i] = 0x2a + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.SnapshotTotalSize))) + i += copy(dAtA[i:], m.SnapshotTotalSize) + } + if m.SnapshotTotalKey != 0 { + dAtA[i] = 0x30 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotTotalKey)) + } + if m.SnapshotHash != 0 { + dAtA[i] = 0x38 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotHash)) + } + if m.SnapshotRevision != 0 { + dAtA[i] = 0x40 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotRevision)) + } + if len(m.Took) > 0 { + dAtA[i] = 0x4a + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.Took))) + i += copy(dAtA[i:], m.Took) + } + return i, nil +} + +func (m *Response) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Response) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Success { + dAtA[i] = 0x8 + i++ + if m.Success { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.Status) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.Status))) + i += copy(dAtA[i:], m.Status) + } + if m.Member != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.Member.Size())) + n3, err := m.Member.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n3 + } + if m.SnapshotInfo != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotInfo.Size())) + n4, err := m.SnapshotInfo.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + return i, nil +} + +func (m *Member) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Member) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.EtcdExecPath) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.EtcdExecPath))) + i += copy(dAtA[i:], m.EtcdExecPath) + } + if len(m.AgentAddr) > 0 { + dAtA[i] = 0x5a + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.AgentAddr))) + i += copy(dAtA[i:], m.AgentAddr) + } + if len(m.FailpointHTTPAddr) > 0 { + dAtA[i] = 0x62 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.FailpointHTTPAddr))) + i += copy(dAtA[i:], m.FailpointHTTPAddr) + } + if len(m.BaseDir) > 0 { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x6 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.BaseDir))) + i += copy(dAtA[i:], m.BaseDir) + } + if len(m.EtcdLogPath) > 0 { + dAtA[i] = 0xb2 + i++ + dAtA[i] = 0x6 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.EtcdLogPath))) + i += copy(dAtA[i:], m.EtcdLogPath) + } + if m.EtcdClientProxy { + dAtA[i] = 0xc8 + i++ + dAtA[i] = 0xc + i++ + if m.EtcdClientProxy { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.EtcdPeerProxy { + dAtA[i] = 0xd0 + i++ + dAtA[i] = 0xc + i++ + if m.EtcdPeerProxy { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.EtcdClientEndpoint) > 0 { + dAtA[i] = 0xea + i++ + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.EtcdClientEndpoint))) + i += copy(dAtA[i:], m.EtcdClientEndpoint) + } + if m.Etcd != nil { + dAtA[i] = 0xf2 + i++ + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.Etcd.Size())) + n5, err := m.Etcd.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + } + if m.EtcdOnSnapshotRestore != nil { + dAtA[i] = 0xfa + i++ + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.EtcdOnSnapshotRestore.Size())) + n6, err := m.EtcdOnSnapshotRestore.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n6 + } + if len(m.ClientCertData) > 0 { + dAtA[i] = 0x8a + i++ + dAtA[i] = 0x19 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientCertData))) + i += copy(dAtA[i:], m.ClientCertData) + } + if len(m.ClientCertPath) > 0 { + dAtA[i] = 0x92 + i++ + dAtA[i] = 0x19 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientCertPath))) + i += copy(dAtA[i:], m.ClientCertPath) + } + if len(m.ClientKeyData) > 0 { + dAtA[i] = 0x9a + i++ + dAtA[i] = 0x19 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientKeyData))) + i += copy(dAtA[i:], m.ClientKeyData) + } + if len(m.ClientKeyPath) > 0 { + dAtA[i] = 0xa2 + i++ + dAtA[i] = 0x19 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientKeyPath))) + i += copy(dAtA[i:], m.ClientKeyPath) + } + if len(m.ClientTrustedCAData) > 0 { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x19 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientTrustedCAData))) + i += copy(dAtA[i:], m.ClientTrustedCAData) + } + if len(m.ClientTrustedCAPath) > 0 { + dAtA[i] = 0xb2 + i++ + dAtA[i] = 0x19 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientTrustedCAPath))) + i += copy(dAtA[i:], m.ClientTrustedCAPath) + } + if len(m.PeerCertData) > 0 { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x1f + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerCertData))) + i += copy(dAtA[i:], m.PeerCertData) + } + if len(m.PeerCertPath) > 0 { + dAtA[i] = 0xb2 + i++ + dAtA[i] = 0x1f + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerCertPath))) + i += copy(dAtA[i:], m.PeerCertPath) + } + if len(m.PeerKeyData) > 0 { + dAtA[i] = 0xba + i++ + dAtA[i] = 0x1f + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerKeyData))) + i += copy(dAtA[i:], m.PeerKeyData) + } + if len(m.PeerKeyPath) > 0 { + dAtA[i] = 0xc2 + i++ + dAtA[i] = 0x1f + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerKeyPath))) + i += copy(dAtA[i:], m.PeerKeyPath) + } + if len(m.PeerTrustedCAData) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x1f + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerTrustedCAData))) + i += copy(dAtA[i:], m.PeerTrustedCAData) + } + if len(m.PeerTrustedCAPath) > 0 { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x1f + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerTrustedCAPath))) + i += copy(dAtA[i:], m.PeerTrustedCAPath) + } + if len(m.SnapshotPath) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x25 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.SnapshotPath))) + i += copy(dAtA[i:], m.SnapshotPath) + } + if m.SnapshotInfo != nil { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x25 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotInfo.Size())) + n7, err := m.SnapshotInfo.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + } + return i, nil +} + +func (m *Tester) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Tester) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DataDir) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.DataDir))) + i += copy(dAtA[i:], m.DataDir) + } + if len(m.Network) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.Network))) + i += copy(dAtA[i:], m.Network) + } + if len(m.Addr) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.Addr))) + i += copy(dAtA[i:], m.Addr) + } + if m.DelayLatencyMs != 0 { + dAtA[i] = 0x58 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.DelayLatencyMs)) + } + if m.DelayLatencyMsRv != 0 { + dAtA[i] = 0x60 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.DelayLatencyMsRv)) + } + if m.UpdatedDelayLatencyMs != 0 { + dAtA[i] = 0x68 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.UpdatedDelayLatencyMs)) + } + if m.RoundLimit != 0 { + dAtA[i] = 0xa8 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.RoundLimit)) + } + if m.ExitOnCaseFail { + dAtA[i] = 0xb0 + i++ + dAtA[i] = 0x1 + i++ + if m.ExitOnCaseFail { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.EnablePprof { + dAtA[i] = 0xb8 + i++ + dAtA[i] = 0x1 + i++ + if m.EnablePprof { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.CaseDelayMs != 0 { + dAtA[i] = 0xf8 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.CaseDelayMs)) + } + if m.CaseShuffle { + dAtA[i] = 0x80 + i++ + dAtA[i] = 0x2 + i++ + if m.CaseShuffle { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.Cases) > 0 { + for _, s := range m.Cases { + dAtA[i] = 0x8a + i++ + dAtA[i] = 0x2 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.FailpointCommands) > 0 { + for _, s := range m.FailpointCommands { + dAtA[i] = 0x92 + i++ + dAtA[i] = 0x2 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.RunnerExecPath) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.RunnerExecPath))) + i += copy(dAtA[i:], m.RunnerExecPath) + } + if len(m.ExternalExecPath) > 0 { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ExternalExecPath))) + i += copy(dAtA[i:], m.ExternalExecPath) + } + if len(m.Stressers) > 0 { + for _, s := range m.Stressers { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x6 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.Checkers) > 0 { + for _, s := range m.Checkers { + dAtA[i] = 0xb2 + i++ + dAtA[i] = 0x6 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.StressKeySize != 0 { + dAtA[i] = 0xc8 + i++ + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressKeySize)) + } + if m.StressKeySizeLarge != 0 { + dAtA[i] = 0xd0 + i++ + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressKeySizeLarge)) + } + if m.StressKeySuffixRange != 0 { + dAtA[i] = 0xd8 + i++ + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressKeySuffixRange)) + } + if m.StressKeySuffixRangeTxn != 0 { + dAtA[i] = 0xe0 + i++ + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressKeySuffixRangeTxn)) + } + if m.StressKeyTxnOps != 0 { + dAtA[i] = 0xe8 + i++ + dAtA[i] = 0xc + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressKeyTxnOps)) + } + if m.StressClients != 0 { + dAtA[i] = 0xe8 + i++ + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressClients)) + } + if m.StressQPS != 0 { + dAtA[i] = 0xf0 + i++ + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.StressQPS)) + } + return i, nil +} + +func (m *Etcd) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Etcd) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.DataDir) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.DataDir))) + i += copy(dAtA[i:], m.DataDir) + } + if len(m.WALDir) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.WALDir))) + i += copy(dAtA[i:], m.WALDir) + } + if m.HeartbeatIntervalMs != 0 { + dAtA[i] = 0x58 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.HeartbeatIntervalMs)) + } + if m.ElectionTimeoutMs != 0 { + dAtA[i] = 0x60 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.ElectionTimeoutMs)) + } + if len(m.ListenClientURLs) > 0 { + for _, s := range m.ListenClientURLs { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x1 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.AdvertiseClientURLs) > 0 { + for _, s := range m.AdvertiseClientURLs { + dAtA[i] = 0xb2 + i++ + dAtA[i] = 0x1 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.ClientAutoTLS { + dAtA[i] = 0xb8 + i++ + dAtA[i] = 0x1 + i++ + if m.ClientAutoTLS { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.ClientCertAuth { + dAtA[i] = 0xc0 + i++ + dAtA[i] = 0x1 + i++ + if m.ClientCertAuth { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.ClientCertFile) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientCertFile))) + i += copy(dAtA[i:], m.ClientCertFile) + } + if len(m.ClientKeyFile) > 0 { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientKeyFile))) + i += copy(dAtA[i:], m.ClientKeyFile) + } + if len(m.ClientTrustedCAFile) > 0 { + dAtA[i] = 0xda + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientTrustedCAFile))) + i += copy(dAtA[i:], m.ClientTrustedCAFile) + } + if len(m.ListenPeerURLs) > 0 { + for _, s := range m.ListenPeerURLs { + dAtA[i] = 0xfa + i++ + dAtA[i] = 0x1 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.AdvertisePeerURLs) > 0 { + for _, s := range m.AdvertisePeerURLs { + dAtA[i] = 0x82 + i++ + dAtA[i] = 0x2 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.PeerAutoTLS { + dAtA[i] = 0x88 + i++ + dAtA[i] = 0x2 + i++ + if m.PeerAutoTLS { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.PeerClientCertAuth { + dAtA[i] = 0x90 + i++ + dAtA[i] = 0x2 + i++ + if m.PeerClientCertAuth { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.PeerCertFile) > 0 { + dAtA[i] = 0x9a + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerCertFile))) + i += copy(dAtA[i:], m.PeerCertFile) + } + if len(m.PeerKeyFile) > 0 { + dAtA[i] = 0xa2 + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerKeyFile))) + i += copy(dAtA[i:], m.PeerKeyFile) + } + if len(m.PeerTrustedCAFile) > 0 { + dAtA[i] = 0xaa + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerTrustedCAFile))) + i += copy(dAtA[i:], m.PeerTrustedCAFile) + } + if len(m.InitialCluster) > 0 { + dAtA[i] = 0xca + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialCluster))) + i += copy(dAtA[i:], m.InitialCluster) + } + if len(m.InitialClusterState) > 0 { + dAtA[i] = 0xd2 + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialClusterState))) + i += copy(dAtA[i:], m.InitialClusterState) + } + if len(m.InitialClusterToken) > 0 { + dAtA[i] = 0xda + i++ + dAtA[i] = 0x2 + i++ + i = encodeVarintRpc(dAtA, i, uint64(len(m.InitialClusterToken))) + i += copy(dAtA[i:], m.InitialClusterToken) + } + if m.SnapshotCount != 0 { + dAtA[i] = 0x98 + i++ + dAtA[i] = 0x3 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.SnapshotCount)) + } + if m.QuotaBackendBytes != 0 { + dAtA[i] = 0xa0 + i++ + dAtA[i] = 0x3 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.QuotaBackendBytes)) + } + if m.PreVote { + dAtA[i] = 0xf8 + i++ + dAtA[i] = 0x3 + i++ + if m.PreVote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.InitialCorruptCheck { + dAtA[i] = 0x80 + i++ + dAtA[i] = 0x4 + i++ + if m.InitialCorruptCheck { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *Request) Size() (n int) { + var l int + _ = l + if m.Operation != 0 { + n += 1 + sovRpc(uint64(m.Operation)) + } + if m.Member != nil { + l = m.Member.Size() + n += 1 + l + sovRpc(uint64(l)) + } + if m.Tester != nil { + l = m.Tester.Size() + n += 1 + l + sovRpc(uint64(l)) + } + return n +} + +func (m *SnapshotInfo) Size() (n int) { + var l int + _ = l + l = len(m.MemberName) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if len(m.MemberClientURLs) > 0 { + for _, s := range m.MemberClientURLs { + l = len(s) + n += 1 + l + sovRpc(uint64(l)) + } + } + l = len(m.SnapshotPath) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.SnapshotFileSize) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.SnapshotTotalSize) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if m.SnapshotTotalKey != 0 { + n += 1 + sovRpc(uint64(m.SnapshotTotalKey)) + } + if m.SnapshotHash != 0 { + n += 1 + sovRpc(uint64(m.SnapshotHash)) + } + if m.SnapshotRevision != 0 { + n += 1 + sovRpc(uint64(m.SnapshotRevision)) + } + l = len(m.Took) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + return n +} + +func (m *Response) Size() (n int) { + var l int + _ = l + if m.Success { + n += 2 + } + l = len(m.Status) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if m.Member != nil { + l = m.Member.Size() + n += 1 + l + sovRpc(uint64(l)) + } + if m.SnapshotInfo != nil { + l = m.SnapshotInfo.Size() + n += 1 + l + sovRpc(uint64(l)) + } + return n +} + +func (m *Member) Size() (n int) { + var l int + _ = l + l = len(m.EtcdExecPath) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.AgentAddr) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.FailpointHTTPAddr) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.BaseDir) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.EtcdLogPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if m.EtcdClientProxy { + n += 3 + } + if m.EtcdPeerProxy { + n += 3 + } + l = len(m.EtcdClientEndpoint) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if m.Etcd != nil { + l = m.Etcd.Size() + n += 2 + l + sovRpc(uint64(l)) + } + if m.EtcdOnSnapshotRestore != nil { + l = m.EtcdOnSnapshotRestore.Size() + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientCertData) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientCertPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientKeyData) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientKeyPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientTrustedCAData) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientTrustedCAPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerCertData) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerCertPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerKeyData) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerKeyPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerTrustedCAData) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerTrustedCAPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.SnapshotPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if m.SnapshotInfo != nil { + l = m.SnapshotInfo.Size() + n += 2 + l + sovRpc(uint64(l)) + } + return n +} + +func (m *Tester) Size() (n int) { + var l int + _ = l + l = len(m.DataDir) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.Network) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.Addr) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if m.DelayLatencyMs != 0 { + n += 1 + sovRpc(uint64(m.DelayLatencyMs)) + } + if m.DelayLatencyMsRv != 0 { + n += 1 + sovRpc(uint64(m.DelayLatencyMsRv)) + } + if m.UpdatedDelayLatencyMs != 0 { + n += 1 + sovRpc(uint64(m.UpdatedDelayLatencyMs)) + } + if m.RoundLimit != 0 { + n += 2 + sovRpc(uint64(m.RoundLimit)) + } + if m.ExitOnCaseFail { + n += 3 + } + if m.EnablePprof { + n += 3 + } + if m.CaseDelayMs != 0 { + n += 2 + sovRpc(uint64(m.CaseDelayMs)) + } + if m.CaseShuffle { + n += 3 + } + if len(m.Cases) > 0 { + for _, s := range m.Cases { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if len(m.FailpointCommands) > 0 { + for _, s := range m.FailpointCommands { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + l = len(m.RunnerExecPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ExternalExecPath) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if len(m.Stressers) > 0 { + for _, s := range m.Stressers { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if len(m.Checkers) > 0 { + for _, s := range m.Checkers { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if m.StressKeySize != 0 { + n += 2 + sovRpc(uint64(m.StressKeySize)) + } + if m.StressKeySizeLarge != 0 { + n += 2 + sovRpc(uint64(m.StressKeySizeLarge)) + } + if m.StressKeySuffixRange != 0 { + n += 2 + sovRpc(uint64(m.StressKeySuffixRange)) + } + if m.StressKeySuffixRangeTxn != 0 { + n += 2 + sovRpc(uint64(m.StressKeySuffixRangeTxn)) + } + if m.StressKeyTxnOps != 0 { + n += 2 + sovRpc(uint64(m.StressKeyTxnOps)) + } + if m.StressClients != 0 { + n += 2 + sovRpc(uint64(m.StressClients)) + } + if m.StressQPS != 0 { + n += 2 + sovRpc(uint64(m.StressQPS)) + } + return n +} + +func (m *Etcd) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.DataDir) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + l = len(m.WALDir) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } + if m.HeartbeatIntervalMs != 0 { + n += 1 + sovRpc(uint64(m.HeartbeatIntervalMs)) + } + if m.ElectionTimeoutMs != 0 { + n += 1 + sovRpc(uint64(m.ElectionTimeoutMs)) + } + if len(m.ListenClientURLs) > 0 { + for _, s := range m.ListenClientURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if len(m.AdvertiseClientURLs) > 0 { + for _, s := range m.AdvertiseClientURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if m.ClientAutoTLS { + n += 3 + } + if m.ClientCertAuth { + n += 3 + } + l = len(m.ClientCertFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientKeyFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.ClientTrustedCAFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if len(m.ListenPeerURLs) > 0 { + for _, s := range m.ListenPeerURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if len(m.AdvertisePeerURLs) > 0 { + for _, s := range m.AdvertisePeerURLs { + l = len(s) + n += 2 + l + sovRpc(uint64(l)) + } + } + if m.PeerAutoTLS { + n += 3 + } + if m.PeerClientCertAuth { + n += 3 + } + l = len(m.PeerCertFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerKeyFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.PeerTrustedCAFile) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.InitialCluster) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.InitialClusterState) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + l = len(m.InitialClusterToken) + if l > 0 { + n += 2 + l + sovRpc(uint64(l)) + } + if m.SnapshotCount != 0 { + n += 2 + sovRpc(uint64(m.SnapshotCount)) + } + if m.QuotaBackendBytes != 0 { + n += 2 + sovRpc(uint64(m.QuotaBackendBytes)) + } + if m.PreVote { + n += 3 + } + if m.InitialCorruptCheck { + n += 3 + } + return n +} + +func sovRpc(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozRpc(x uint64) (n int) { + return sovRpc(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Request) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Request: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Request: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Operation", wireType) + } + m.Operation = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Operation |= (Operation(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Member == nil { + m.Member = &Member{} + } + if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tester", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Tester == nil { + m.Tester = &Tester{} + } + if err := m.Tester.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SnapshotInfo) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SnapshotInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SnapshotInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MemberName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MemberName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MemberClientURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MemberClientURLs = append(m.MemberClientURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SnapshotPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotFileSize", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SnapshotFileSize = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotTotalSize", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SnapshotTotalSize = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotTotalKey", wireType) + } + m.SnapshotTotalKey = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SnapshotTotalKey |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotHash", wireType) + } + m.SnapshotHash = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SnapshotHash |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotRevision", wireType) + } + m.SnapshotRevision = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SnapshotRevision |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Took", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Took = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Response) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Response: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Response: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Success = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Status = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Member == nil { + m.Member = &Member{} + } + if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SnapshotInfo == nil { + m.SnapshotInfo = &SnapshotInfo{} + } + if err := m.SnapshotInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Member) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Member: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Member: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EtcdExecPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EtcdExecPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AgentAddr", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AgentAddr = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FailpointHTTPAddr", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FailpointHTTPAddr = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 101: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BaseDir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BaseDir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 102: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EtcdLogPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EtcdLogPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 201: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EtcdClientProxy", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.EtcdClientProxy = bool(v != 0) + case 202: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EtcdPeerProxy", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.EtcdPeerProxy = bool(v != 0) + case 301: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EtcdClientEndpoint", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EtcdClientEndpoint = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 302: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Etcd", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Etcd == nil { + m.Etcd = &Etcd{} + } + if err := m.Etcd.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 303: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EtcdOnSnapshotRestore", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.EtcdOnSnapshotRestore == nil { + m.EtcdOnSnapshotRestore = &Etcd{} + } + if err := m.EtcdOnSnapshotRestore.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 401: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientCertData", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientCertData = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 402: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientCertPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientCertPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 403: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientKeyData", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientKeyData = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 404: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientKeyPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientKeyPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 405: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientTrustedCAData", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientTrustedCAData = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 406: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientTrustedCAPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientTrustedCAPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 501: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerCertData", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerCertData = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 502: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerCertPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerCertPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 503: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerKeyData", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerKeyData = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 504: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerKeyPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerKeyPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 505: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerTrustedCAData", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerTrustedCAData = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 506: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerTrustedCAPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerTrustedCAPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 601: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SnapshotPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 602: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SnapshotInfo == nil { + m.SnapshotInfo = &SnapshotInfo{} + } + if err := m.SnapshotInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Tester) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Tester: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Tester: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataDir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DataDir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Network", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Network = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Addr", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Addr = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DelayLatencyMs", wireType) + } + m.DelayLatencyMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DelayLatencyMs |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DelayLatencyMsRv", wireType) + } + m.DelayLatencyMsRv = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DelayLatencyMsRv |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedDelayLatencyMs", wireType) + } + m.UpdatedDelayLatencyMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdatedDelayLatencyMs |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 21: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RoundLimit", wireType) + } + m.RoundLimit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RoundLimit |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 22: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExitOnCaseFail", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ExitOnCaseFail = bool(v != 0) + case 23: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EnablePprof", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.EnablePprof = bool(v != 0) + case 31: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CaseDelayMs", wireType) + } + m.CaseDelayMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CaseDelayMs |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 32: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CaseShuffle", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.CaseShuffle = bool(v != 0) + case 33: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Cases", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Cases = append(m.Cases, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 34: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FailpointCommands = append(m.FailpointCommands, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 41: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RunnerExecPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RunnerExecPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 42: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExternalExecPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ExternalExecPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 101: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Stressers", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Stressers = append(m.Stressers, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 102: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Checkers", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Checkers = append(m.Checkers, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 201: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressKeySize", wireType) + } + m.StressKeySize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressKeySize |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 202: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressKeySizeLarge", wireType) + } + m.StressKeySizeLarge = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressKeySizeLarge |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 203: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressKeySuffixRange", wireType) + } + m.StressKeySuffixRange = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressKeySuffixRange |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 204: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressKeySuffixRangeTxn", wireType) + } + m.StressKeySuffixRangeTxn = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressKeySuffixRangeTxn |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 205: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressKeyTxnOps", wireType) + } + m.StressKeyTxnOps = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressKeyTxnOps |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 301: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressClients", wireType) + } + m.StressClients = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressClients |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 302: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StressQPS", wireType) + } + m.StressQPS = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StressQPS |= (int32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Etcd) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Etcd: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Etcd: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataDir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DataDir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WALDir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WALDir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field HeartbeatIntervalMs", wireType) + } + m.HeartbeatIntervalMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.HeartbeatIntervalMs |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ElectionTimeoutMs", wireType) + } + m.ElectionTimeoutMs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ElectionTimeoutMs |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 21: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ListenClientURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ListenClientURLs = append(m.ListenClientURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 22: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AdvertiseClientURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AdvertiseClientURLs = append(m.AdvertiseClientURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 23: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientAutoTLS", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ClientAutoTLS = bool(v != 0) + case 24: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientCertAuth", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ClientCertAuth = bool(v != 0) + case 25: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientCertFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientCertFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 26: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientKeyFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientKeyFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 27: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientTrustedCAFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClientTrustedCAFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 31: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ListenPeerURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ListenPeerURLs = append(m.ListenPeerURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 32: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AdvertisePeerURLs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AdvertisePeerURLs = append(m.AdvertisePeerURLs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 33: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerAutoTLS", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PeerAutoTLS = bool(v != 0) + case 34: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerClientCertAuth", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PeerClientCertAuth = bool(v != 0) + case 35: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerCertFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerCertFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 36: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerKeyFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerKeyFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 37: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PeerTrustedCAFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PeerTrustedCAFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 41: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialCluster", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InitialCluster = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 42: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialClusterState", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InitialClusterState = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 43: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialClusterToken", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InitialClusterToken = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 51: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SnapshotCount", wireType) + } + m.SnapshotCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SnapshotCount |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 52: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field QuotaBackendBytes", wireType) + } + m.QuotaBackendBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.QuotaBackendBytes |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 63: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreVote", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.PreVote = bool(v != 0) + case 64: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InitialCorruptCheck", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.InitialCorruptCheck = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipRpc(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRpc + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRpc + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRpc + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthRpc + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRpc + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipRpc(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthRpc = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowRpc = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } + +var fileDescriptorRpc = []byte{ + // 2808 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xdb, 0x73, 0xdb, 0xc6, + 0xf5, 0x16, 0x44, 0x5d, 0x57, 0x37, 0x68, 0x65, 0xd9, 0xf0, 0x4d, 0x90, 0xe1, 0x38, 0x3f, 0x59, + 0x09, 0xec, 0xfc, 0xec, 0x4c, 0x2e, 0x4e, 0x13, 0x07, 0xa4, 0x20, 0x8b, 0x15, 0x44, 0xd2, 0x4b, + 0xc8, 0x76, 0x9e, 0x38, 0x10, 0xb9, 0x92, 0x30, 0xa6, 0x00, 0x06, 0x58, 0x2a, 0x52, 0xfe, 0x81, + 0xbe, 0xf6, 0x3e, 0xed, 0x4c, 0x9f, 0xfa, 0xdc, 0xb4, 0xff, 0x86, 0x73, 0x6b, 0xd3, 0xf6, 0xa9, + 0xed, 0x0c, 0xa7, 0x4d, 0x5f, 0xfa, 0xd4, 0x07, 0x4e, 0x6f, 0xe9, 0x53, 0x67, 0x77, 0x01, 0x71, + 0x01, 0x90, 0x92, 0x9e, 0xa4, 0x3d, 0xe7, 0xfb, 0xbe, 0x3d, 0xbb, 0x67, 0xb1, 0xe7, 0x00, 0x04, + 0x73, 0x41, 0xab, 0xde, 0xda, 0xb9, 0x1b, 0xb4, 0xea, 0x77, 0x5a, 0x81, 0x4f, 0x7c, 0x38, 0xca, + 0x0c, 0x57, 0xf4, 0x3d, 0x97, 0xec, 0xb7, 0x77, 0xee, 0xd4, 0xfd, 0x83, 0xbb, 0x7b, 0xfe, 0x9e, + 0x7f, 0x97, 0x79, 0x77, 0xda, 0xbb, 0x6c, 0xc4, 0x06, 0xec, 0x3f, 0xce, 0xd2, 0xbe, 0x23, 0x81, + 0x71, 0x84, 0x3f, 0x6c, 0xe3, 0x90, 0xc0, 0x3b, 0x60, 0xb2, 0xdc, 0xc2, 0x81, 0x43, 0x5c, 0xdf, + 0x53, 0xa4, 0x65, 0x69, 0x65, 0xf6, 0x9e, 0x7c, 0x87, 0xa9, 0xde, 0x39, 0xb1, 0xa3, 0x1e, 0x04, + 0xde, 0x02, 0x63, 0x5b, 0xf8, 0x60, 0x07, 0x07, 0xca, 0xf0, 0xb2, 0xb4, 0x32, 0x75, 0x6f, 0x26, + 0x02, 0x73, 0x23, 0x8a, 0x9c, 0x14, 0x66, 0xe3, 0x90, 0xe0, 0x40, 0xc9, 0x25, 0x60, 0xdc, 0x88, + 0x22, 0xa7, 0xf6, 0xb7, 0x61, 0x30, 0x5d, 0xf5, 0x9c, 0x56, 0xb8, 0xef, 0x93, 0xa2, 0xb7, 0xeb, + 0xc3, 0x25, 0x00, 0xb8, 0x42, 0xc9, 0x39, 0xc0, 0x2c, 0x9e, 0x49, 0x24, 0x58, 0xe0, 0x2a, 0x90, + 0xf9, 0xa8, 0xd0, 0x74, 0xb1, 0x47, 0xb6, 0x91, 0x15, 0x2a, 0xc3, 0xcb, 0xb9, 0x95, 0x49, 0x94, + 0xb1, 0x43, 0xad, 0xa7, 0x5d, 0x71, 0xc8, 0x3e, 0x8b, 0x64, 0x12, 0x25, 0x6c, 0x54, 0x2f, 0x1e, + 0xaf, 0xbb, 0x4d, 0x5c, 0x75, 0x3f, 0xc6, 0xca, 0x08, 0xc3, 0x65, 0xec, 0xf0, 0x55, 0x30, 0x1f, + 0xdb, 0x6c, 0x9f, 0x38, 0x4d, 0x06, 0x1e, 0x65, 0xe0, 0xac, 0x43, 0x54, 0x66, 0xc6, 0x4d, 0x7c, + 0xac, 0x8c, 0x2d, 0x4b, 0x2b, 0x39, 0x94, 0xb1, 0x8b, 0x91, 0x6e, 0x38, 0xe1, 0xbe, 0x32, 0xce, + 0x70, 0x09, 0x9b, 0xa8, 0x87, 0xf0, 0xa1, 0x1b, 0xd2, 0x7c, 0x4d, 0x24, 0xf5, 0x62, 0x3b, 0x84, + 0x60, 0xc4, 0xf6, 0xfd, 0xe7, 0xca, 0x24, 0x0b, 0x8e, 0xfd, 0xaf, 0xfd, 0x4c, 0x02, 0x13, 0x08, + 0x87, 0x2d, 0xdf, 0x0b, 0x31, 0x54, 0xc0, 0x78, 0xb5, 0x5d, 0xaf, 0xe3, 0x30, 0x64, 0x7b, 0x3c, + 0x81, 0xe2, 0x21, 0xbc, 0x08, 0xc6, 0xaa, 0xc4, 0x21, 0xed, 0x90, 0xe5, 0x77, 0x12, 0x45, 0x23, + 0x21, 0xef, 0xb9, 0xd3, 0xf2, 0xfe, 0x66, 0x32, 0x9f, 0x6c, 0x2f, 0xa7, 0xee, 0x2d, 0x44, 0x60, + 0xd1, 0x85, 0x12, 0x40, 0xed, 0x4f, 0xd3, 0xf1, 0x04, 0xf0, 0x5d, 0x30, 0x6d, 0x92, 0x7a, 0xc3, + 0x3c, 0xc2, 0x75, 0x96, 0x37, 0x76, 0x0a, 0xf2, 0x97, 0xbb, 0x1d, 0x75, 0xf1, 0xd8, 0x39, 0x68, + 0x3e, 0xd0, 0x30, 0xa9, 0x37, 0x74, 0x7c, 0x84, 0xeb, 0x7a, 0xcb, 0x21, 0xfb, 0x1a, 0x4a, 0xc0, + 0xe1, 0x7d, 0x30, 0x69, 0xec, 0x61, 0x8f, 0x18, 0x8d, 0x46, 0xa0, 0x4c, 0x31, 0xee, 0x62, 0xb7, + 0xa3, 0xce, 0x73, 0xae, 0x43, 0x5d, 0xba, 0xd3, 0x68, 0x04, 0x1a, 0xea, 0xe1, 0xa0, 0x05, 0xe6, + 0xd7, 0x1d, 0xb7, 0xd9, 0xf2, 0x5d, 0x8f, 0x6c, 0xd8, 0x76, 0x85, 0x91, 0xa7, 0x19, 0x79, 0xa9, + 0xdb, 0x51, 0xaf, 0x70, 0xf2, 0x6e, 0x0c, 0xd1, 0xf7, 0x09, 0x69, 0x45, 0x2a, 0x59, 0x22, 0xd4, + 0xc1, 0x78, 0xde, 0x09, 0xf1, 0x9a, 0x1b, 0x28, 0x98, 0x69, 0x2c, 0x74, 0x3b, 0xea, 0x1c, 0xd7, + 0xd8, 0x71, 0x42, 0xac, 0x37, 0xdc, 0x40, 0x43, 0x31, 0x06, 0x3e, 0x00, 0x53, 0x74, 0x05, 0x96, + 0xbf, 0xc7, 0xd6, 0xbb, 0xcb, 0x28, 0x4a, 0xb7, 0xa3, 0x5e, 0x10, 0xd6, 0xdb, 0xf4, 0xf7, 0xa2, + 0xe5, 0x8a, 0x60, 0xf8, 0x08, 0xcc, 0xd1, 0x21, 0x3f, 0xf6, 0x95, 0xc0, 0x3f, 0x3a, 0x56, 0x3e, + 0x65, 0x29, 0xcd, 0x5f, 0xeb, 0x76, 0x54, 0x45, 0x10, 0xa8, 0x33, 0x88, 0xde, 0xa2, 0x18, 0x0d, + 0xa5, 0x59, 0xd0, 0x00, 0x33, 0xd4, 0x54, 0xc1, 0x38, 0xe0, 0x32, 0x9f, 0x71, 0x99, 0x2b, 0xdd, + 0x8e, 0x7a, 0x51, 0x90, 0x69, 0x61, 0x1c, 0xc4, 0x22, 0x49, 0x06, 0xac, 0x00, 0xd8, 0x53, 0x35, + 0xbd, 0x06, 0xdb, 0x14, 0xe5, 0x13, 0x76, 0x90, 0xf2, 0x6a, 0xb7, 0xa3, 0x5e, 0xcd, 0x86, 0x83, + 0x23, 0x98, 0x86, 0xfa, 0x70, 0xe1, 0xff, 0x83, 0x11, 0x6a, 0x55, 0x7e, 0xc9, 0x2f, 0x9b, 0xa9, + 0xe8, 0x1c, 0x51, 0x5b, 0x7e, 0xae, 0xdb, 0x51, 0xa7, 0x7a, 0x82, 0x1a, 0x62, 0x50, 0x98, 0x07, + 0x8b, 0xf4, 0x6f, 0xd9, 0xeb, 0x3d, 0x15, 0x21, 0xf1, 0x03, 0xac, 0xfc, 0x2a, 0xab, 0x81, 0xfa, + 0x43, 0xe1, 0x1a, 0x98, 0xe5, 0x81, 0x14, 0x70, 0x40, 0xd6, 0x1c, 0xe2, 0x28, 0xdf, 0x63, 0x97, + 0x47, 0xfe, 0x6a, 0xb7, 0xa3, 0x5e, 0xe2, 0x73, 0x46, 0xf1, 0xd7, 0x71, 0x40, 0xf4, 0x86, 0x43, + 0x1c, 0x0d, 0xa5, 0x38, 0x49, 0x15, 0x96, 0xd9, 0xef, 0x9f, 0xaa, 0xc2, 0xb3, 0x9b, 0xe2, 0xd0, + 0xbc, 0x70, 0xcb, 0x26, 0x3e, 0x66, 0xa1, 0xfc, 0x80, 0x8b, 0x08, 0x79, 0x89, 0x44, 0x9e, 0xe3, + 0xe3, 0x28, 0x92, 0x24, 0x23, 0x21, 0xc1, 0xe2, 0xf8, 0xe1, 0x69, 0x12, 0x3c, 0x8c, 0x24, 0x03, + 0xda, 0x60, 0x81, 0x1b, 0xec, 0xa0, 0x1d, 0x12, 0xdc, 0x28, 0x18, 0x2c, 0x96, 0x1f, 0x71, 0xa1, + 0x1b, 0xdd, 0x8e, 0x7a, 0x3d, 0x21, 0x44, 0x38, 0x4c, 0xaf, 0x3b, 0x51, 0x48, 0xfd, 0xe8, 0x7d, + 0x54, 0x59, 0x78, 0x3f, 0x3e, 0x87, 0x2a, 0x8f, 0xb2, 0x1f, 0x1d, 0xbe, 0x07, 0xa6, 0xe9, 0x99, + 0x3c, 0xc9, 0xdd, 0x3f, 0x73, 0xe9, 0x0b, 0x84, 0x9d, 0x61, 0x21, 0x73, 0x09, 0xbc, 0xc8, 0x67, + 0xe1, 0xfc, 0xeb, 0x14, 0x7e, 0x74, 0x01, 0x89, 0x78, 0xf8, 0x0e, 0x98, 0xa2, 0xe3, 0x38, 0x5f, + 0xff, 0xce, 0xa5, 0x9f, 0x67, 0x46, 0xef, 0x65, 0x4b, 0x44, 0x0b, 0x64, 0x36, 0xf7, 0x7f, 0x06, + 0x93, 0xa3, 0xcb, 0x40, 0x40, 0xc3, 0x12, 0x98, 0xa7, 0xc3, 0x64, 0x8e, 0xbe, 0xc9, 0xa5, 0x9f, + 0x3f, 0x26, 0x91, 0xc9, 0x50, 0x96, 0x9a, 0xd1, 0x63, 0x21, 0xfd, 0xf7, 0x4c, 0x3d, 0x1e, 0x59, + 0x96, 0x4a, 0x6f, 0xf6, 0x44, 0x45, 0xfe, 0xc3, 0x48, 0x7a, 0x75, 0x61, 0xe4, 0x8e, 0x37, 0x36, + 0x51, 0xac, 0xdf, 0x4a, 0x15, 0x97, 0x3f, 0x9e, 0xbb, 0xba, 0xfc, 0x7c, 0x3a, 0xee, 0x47, 0xe8, + 0xdd, 0x4c, 0xd7, 0x46, 0xef, 0x66, 0x29, 0x7d, 0x37, 0xd3, 0x8d, 0x88, 0xee, 0xe6, 0x08, 0x03, + 0x5f, 0x05, 0xe3, 0x25, 0x4c, 0x3e, 0xf2, 0x83, 0xe7, 0xbc, 0x20, 0xe6, 0x61, 0xb7, 0xa3, 0xce, + 0x72, 0xb8, 0xc7, 0x1d, 0x1a, 0x8a, 0x21, 0xf0, 0x26, 0x18, 0x61, 0x95, 0x83, 0x6f, 0x91, 0x70, + 0x43, 0xf1, 0x52, 0xc1, 0x9c, 0xb0, 0x00, 0x66, 0xd7, 0x70, 0xd3, 0x39, 0xb6, 0x1c, 0x82, 0xbd, + 0xfa, 0xf1, 0x56, 0xc8, 0xaa, 0xd4, 0x8c, 0x78, 0x2d, 0x34, 0xa8, 0x5f, 0x6f, 0x72, 0x80, 0x7e, + 0x10, 0x6a, 0x28, 0x45, 0x81, 0xdf, 0x06, 0x72, 0xd2, 0x82, 0x0e, 0x59, 0xbd, 0x9a, 0x11, 0xeb, + 0x55, 0x5a, 0x46, 0x0f, 0x0e, 0x35, 0x94, 0xe1, 0xc1, 0x0f, 0xc0, 0xe2, 0x76, 0xab, 0xe1, 0x10, + 0xdc, 0x48, 0xc5, 0x35, 0xc3, 0x04, 0x6f, 0x76, 0x3b, 0xaa, 0xca, 0x05, 0xdb, 0x1c, 0xa6, 0x67, + 0xe3, 0xeb, 0xaf, 0x00, 0xdf, 0x00, 0x00, 0xf9, 0x6d, 0xaf, 0x61, 0xb9, 0x07, 0x2e, 0x51, 0x16, + 0x97, 0xa5, 0x95, 0xd1, 0xfc, 0xc5, 0x6e, 0x47, 0x85, 0x5c, 0x2f, 0xa0, 0x3e, 0xbd, 0x49, 0x9d, + 0x1a, 0x12, 0x90, 0x30, 0x0f, 0x66, 0xcd, 0x23, 0x97, 0x94, 0xbd, 0x82, 0x13, 0x62, 0x5a, 0x60, + 0x95, 0x8b, 0x99, 0x6a, 0x74, 0xe4, 0x12, 0xdd, 0xf7, 0x74, 0x5a, 0x94, 0xdb, 0x01, 0xd6, 0x50, + 0x8a, 0x01, 0xdf, 0x06, 0x53, 0xa6, 0xe7, 0xec, 0x34, 0x71, 0xa5, 0x15, 0xf8, 0xbb, 0xca, 0x25, + 0x26, 0x70, 0xa9, 0xdb, 0x51, 0x17, 0x22, 0x01, 0xe6, 0xd4, 0x5b, 0xd4, 0x4b, 0xab, 0x6a, 0x0f, + 0x4b, 0x2b, 0x32, 0x95, 0x61, 0x8b, 0xd9, 0x0a, 0x15, 0x95, 0xed, 0x83, 0x70, 0x4c, 0xeb, 0xac, + 0x88, 0xb3, 0x4d, 0xa0, 0x8b, 0x17, 0xc1, 0x74, 0x5a, 0x3a, 0xac, 0xee, 0xb7, 0x77, 0x77, 0x9b, + 0x58, 0x59, 0x4e, 0x4f, 0xcb, 0xb8, 0x21, 0xf7, 0x46, 0xd4, 0x08, 0x0b, 0x5f, 0x06, 0xa3, 0x74, + 0x18, 0x2a, 0x37, 0x68, 0x4b, 0x9b, 0x97, 0xbb, 0x1d, 0x75, 0xba, 0x47, 0x0a, 0x35, 0xc4, 0xdd, + 0x70, 0x53, 0xe8, 0x56, 0x0a, 0xfe, 0xc1, 0x81, 0xe3, 0x35, 0x42, 0x45, 0x63, 0x9c, 0xeb, 0xdd, + 0x8e, 0x7a, 0x39, 0xdd, 0xad, 0xd4, 0x23, 0x8c, 0xd8, 0xac, 0xc4, 0x3c, 0x7a, 0x1c, 0x51, 0xdb, + 0xf3, 0x70, 0x70, 0xd2, 0x70, 0xdd, 0x4e, 0x57, 0xa9, 0x80, 0xf9, 0xc5, 0x96, 0x2b, 0x45, 0x81, + 0x45, 0x20, 0x9b, 0x47, 0x04, 0x07, 0x9e, 0xd3, 0x3c, 0x91, 0x59, 0x65, 0x32, 0x42, 0x40, 0x38, + 0x42, 0x88, 0x42, 0x19, 0x1a, 0xbc, 0x07, 0x26, 0xab, 0x24, 0xc0, 0x61, 0x88, 0x83, 0x50, 0xc1, + 0x6c, 0x51, 0x17, 0xba, 0x1d, 0x55, 0x8e, 0x2e, 0x88, 0xd8, 0xa5, 0xa1, 0x1e, 0x0c, 0xde, 0x05, + 0x13, 0x85, 0x7d, 0x5c, 0x7f, 0x4e, 0x29, 0xbb, 0x8c, 0x22, 0x3c, 0xd5, 0xf5, 0xc8, 0xa3, 0xa1, + 0x13, 0x10, 0x2d, 0x89, 0x9c, 0xbd, 0x89, 0x8f, 0x59, 0x1f, 0xcf, 0x9a, 0xa6, 0x51, 0xf1, 0x7c, + 0xf1, 0x99, 0xd8, 0x55, 0x1b, 0xba, 0x1f, 0x63, 0x0d, 0x25, 0x19, 0xf0, 0x31, 0x80, 0x09, 0x83, + 0xe5, 0x04, 0x7b, 0x98, 0x77, 0x4d, 0xa3, 0xf9, 0xe5, 0x6e, 0x47, 0xbd, 0xd6, 0x57, 0x47, 0x6f, + 0x52, 0x9c, 0x86, 0xfa, 0x90, 0xe1, 0x53, 0x70, 0xa1, 0x67, 0x6d, 0xef, 0xee, 0xba, 0x47, 0xc8, + 0xf1, 0xf6, 0xb0, 0xf2, 0x39, 0x17, 0xd5, 0xba, 0x1d, 0x75, 0x29, 0x2b, 0xca, 0x80, 0x7a, 0x40, + 0x91, 0x1a, 0xea, 0x2b, 0x00, 0x1d, 0x70, 0xa9, 0x9f, 0xdd, 0x3e, 0xf2, 0x94, 0x2f, 0xb8, 0xf6, + 0xcb, 0xdd, 0x8e, 0xaa, 0x9d, 0xaa, 0xad, 0x93, 0x23, 0x4f, 0x43, 0x83, 0x74, 0xe0, 0x06, 0x98, + 0x3b, 0x71, 0xd9, 0x47, 0x5e, 0xb9, 0x15, 0x2a, 0x5f, 0x72, 0x69, 0xe1, 0x04, 0x08, 0xd2, 0xe4, + 0xc8, 0xd3, 0xfd, 0x56, 0xa8, 0xa1, 0x34, 0x0d, 0xbe, 0x1f, 0xe7, 0x86, 0x17, 0xf7, 0x90, 0x77, + 0x90, 0xa3, 0x62, 0x01, 0x8e, 0x74, 0x78, 0x5b, 0x10, 0x9e, 0xa4, 0x26, 0x22, 0xc0, 0xd7, 0xe3, + 0x23, 0xf4, 0xb8, 0x52, 0xe5, 0xbd, 0xe3, 0xa8, 0xf8, 0x0e, 0x10, 0xb1, 0x3f, 0x6c, 0xf5, 0x0e, + 0xd1, 0xe3, 0x4a, 0x55, 0xfb, 0x66, 0x86, 0x77, 0x9b, 0xf4, 0x16, 0xef, 0xbd, 0x7e, 0x8a, 0xb7, + 0xb8, 0xe7, 0x1c, 0x60, 0x0d, 0x31, 0xa7, 0x58, 0x47, 0x86, 0xcf, 0x51, 0x47, 0x56, 0xc1, 0xd8, + 0x53, 0xc3, 0xa2, 0xe8, 0x5c, 0xba, 0x8c, 0x7c, 0xe4, 0x34, 0x39, 0x38, 0x42, 0xc0, 0x32, 0x58, + 0xd8, 0xc0, 0x4e, 0x40, 0x76, 0xb0, 0x43, 0x8a, 0x1e, 0xc1, 0xc1, 0xa1, 0xd3, 0x8c, 0xaa, 0x44, + 0x4e, 0xdc, 0xcd, 0xfd, 0x18, 0xa4, 0xbb, 0x11, 0x4a, 0x43, 0xfd, 0x98, 0xb0, 0x08, 0xe6, 0xcd, + 0x26, 0xae, 0xd3, 0x17, 0x78, 0xdb, 0x3d, 0xc0, 0x7e, 0x9b, 0x6c, 0x85, 0xac, 0x5a, 0xe4, 0xc4, + 0xa7, 0x1c, 0x47, 0x10, 0x9d, 0x70, 0x8c, 0x86, 0xb2, 0x2c, 0xfa, 0xa0, 0x5b, 0x6e, 0x48, 0xb0, + 0x27, 0xbc, 0x80, 0x2f, 0xa6, 0x6f, 0x9e, 0x26, 0x43, 0xc4, 0x2d, 0x7e, 0x3b, 0x68, 0x86, 0x1a, + 0xca, 0xd0, 0x20, 0x02, 0x0b, 0x46, 0xe3, 0x10, 0x07, 0xc4, 0x0d, 0xb1, 0xa0, 0x76, 0x91, 0xa9, + 0x09, 0x0f, 0x90, 0x13, 0x83, 0x92, 0x82, 0xfd, 0xc8, 0xf0, 0xed, 0xb8, 0xd5, 0x35, 0xda, 0xc4, + 0xb7, 0xad, 0x6a, 0x74, 0xeb, 0x0b, 0xb9, 0x71, 0xda, 0xc4, 0xd7, 0x09, 0x15, 0x48, 0x22, 0xe9, + 0x3d, 0xd8, 0x6b, 0xbd, 0x8d, 0x36, 0xd9, 0x57, 0x14, 0xc6, 0x1d, 0xd0, 0xad, 0x3b, 0xed, 0x54, + 0xb7, 0x4e, 0x29, 0xf0, 0x5b, 0xa2, 0xc8, 0xba, 0xdb, 0xc4, 0xca, 0x65, 0x96, 0x6e, 0xe1, 0x06, + 0x63, 0xec, 0x5d, 0x97, 0x5e, 0xfe, 0x29, 0x6c, 0x2f, 0xfa, 0x4d, 0x7c, 0xcc, 0xc8, 0x57, 0xd2, + 0x27, 0x8b, 0x3e, 0x39, 0x9c, 0x9b, 0x44, 0x42, 0x2b, 0xd3, 0x4a, 0x33, 0x81, 0xab, 0xe9, 0x46, + 0x5f, 0x68, 0xd3, 0xb8, 0x4e, 0x3f, 0x1a, 0xdd, 0x0b, 0x9e, 0x2e, 0xda, 0xc3, 0xb1, 0xac, 0xa8, + 0x2c, 0x2b, 0xc2, 0x5e, 0x44, 0x39, 0x66, 0xbd, 0x1f, 0x4f, 0x48, 0x8a, 0x02, 0x6d, 0x30, 0x7f, + 0x92, 0xa2, 0x13, 0x9d, 0x65, 0xa6, 0x23, 0xdc, 0x36, 0xae, 0xe7, 0x12, 0xd7, 0x69, 0xea, 0xbd, + 0x2c, 0x0b, 0x92, 0x59, 0x01, 0x5a, 0x9a, 0xe9, 0xff, 0x71, 0x7e, 0x6f, 0xb0, 0x1c, 0xa5, 0xfb, + 0xe3, 0x5e, 0x92, 0x45, 0x30, 0x7d, 0x41, 0x65, 0x9d, 0x7a, 0x32, 0xcd, 0x1a, 0x93, 0x10, 0x0e, + 0x1c, 0x6f, 0xef, 0x33, 0xb9, 0xee, 0xc3, 0xa5, 0x1d, 0x6d, 0xdc, 0xfb, 0xb3, 0xfd, 0xbe, 0x39, + 0xf8, 0x55, 0x81, 0x6f, 0x77, 0x02, 0x1e, 0x2f, 0x26, 0x4e, 0xf7, 0x4b, 0x03, 0x9b, 0x7d, 0x4e, + 0x16, 0xc1, 0x70, 0x2b, 0xd5, 0x9c, 0x33, 0x85, 0x5b, 0x67, 0xf5, 0xe6, 0x5c, 0x28, 0xcb, 0xa4, + 0x1d, 0x57, 0x91, 0xa7, 0xa2, 0xd0, 0x6c, 0xb3, 0x2f, 0x77, 0xb7, 0xd3, 0x67, 0x27, 0x4e, 0x55, + 0x9d, 0x03, 0x34, 0x94, 0x62, 0xd0, 0x27, 0x3a, 0x69, 0xa9, 0x12, 0x87, 0xe0, 0xa8, 0x11, 0x10, + 0x36, 0x38, 0x25, 0xa4, 0x87, 0x14, 0xa6, 0xa1, 0x7e, 0xe4, 0xac, 0xa6, 0xed, 0x3f, 0xc7, 0x9e, + 0xf2, 0xca, 0x59, 0x9a, 0x84, 0xc2, 0x32, 0x9a, 0x8c, 0x0c, 0x1f, 0x82, 0x99, 0xf8, 0xf5, 0xa0, + 0xe0, 0xb7, 0x3d, 0xa2, 0xdc, 0x67, 0x77, 0xa1, 0x58, 0x60, 0xe2, 0xf7, 0x90, 0x3a, 0xf5, 0xd3, + 0x02, 0x23, 0xe2, 0xa1, 0x05, 0xe6, 0x1f, 0xb7, 0x7d, 0xe2, 0xe4, 0x9d, 0xfa, 0x73, 0xec, 0x35, + 0xf2, 0xc7, 0x04, 0x87, 0xca, 0xeb, 0x4c, 0x44, 0x68, 0xbf, 0x3f, 0xa4, 0x10, 0x7d, 0x87, 0x63, + 0xf4, 0x1d, 0x0a, 0xd2, 0x50, 0x96, 0x48, 0x4b, 0x49, 0x25, 0xc0, 0x4f, 0x7c, 0x82, 0x95, 0x87, + 0xe9, 0xeb, 0xaa, 0x15, 0x60, 0xfd, 0xd0, 0xa7, 0xbb, 0x13, 0x63, 0xc4, 0x1d, 0xf1, 0x83, 0xa0, + 0xdd, 0x22, 0xac, 0xab, 0x51, 0xde, 0x4f, 0x1f, 0xe3, 0x93, 0x1d, 0xe1, 0x28, 0x9d, 0xf5, 0x41, + 0xc2, 0x8e, 0x08, 0xe4, 0xd5, 0x9f, 0xe6, 0x84, 0xef, 0xc0, 0x70, 0x0e, 0x4c, 0x95, 0xca, 0x76, + 0xad, 0x6a, 0x1b, 0xc8, 0x36, 0xd7, 0xe4, 0x21, 0x78, 0x11, 0xc0, 0x62, 0xa9, 0x68, 0x17, 0x0d, + 0x8b, 0x1b, 0x6b, 0xa6, 0x5d, 0x58, 0x93, 0x01, 0x94, 0xc1, 0x34, 0x32, 0x05, 0xcb, 0x14, 0xb5, + 0x54, 0x8b, 0x8f, 0x6c, 0x13, 0x6d, 0x71, 0xcb, 0x05, 0xb8, 0x0c, 0xae, 0x55, 0x8b, 0x8f, 0x1e, + 0x6f, 0x17, 0x39, 0xa6, 0x66, 0x94, 0xd6, 0x6a, 0xc8, 0xdc, 0x2a, 0x3f, 0x31, 0x6b, 0x6b, 0x86, + 0x6d, 0xc8, 0x8b, 0x70, 0x1e, 0xcc, 0x54, 0x8d, 0x27, 0x66, 0xad, 0x5a, 0x32, 0x2a, 0xd5, 0x8d, + 0xb2, 0x2d, 0x2f, 0xc1, 0x1b, 0xe0, 0x3a, 0x15, 0x2e, 0x23, 0xb3, 0x16, 0x4f, 0xb0, 0x8e, 0xca, + 0x5b, 0x3d, 0x88, 0x0a, 0x2f, 0x83, 0xc5, 0xfe, 0xae, 0x65, 0xca, 0xce, 0x4c, 0x69, 0xa0, 0xc2, + 0x46, 0x31, 0x9e, 0x73, 0x05, 0xde, 0x05, 0xaf, 0x9c, 0x16, 0x15, 0x1b, 0x57, 0xed, 0x72, 0xa5, + 0x66, 0x3c, 0x32, 0x4b, 0xb6, 0x7c, 0x1b, 0x5e, 0x07, 0x97, 0xf3, 0x96, 0x51, 0xd8, 0xdc, 0x28, + 0x5b, 0x66, 0xad, 0x62, 0x9a, 0xa8, 0x56, 0x29, 0x23, 0xbb, 0x66, 0x3f, 0xab, 0xa1, 0x67, 0x72, + 0x03, 0xaa, 0xe0, 0xea, 0x76, 0x69, 0x30, 0x00, 0xc3, 0x2b, 0x60, 0x71, 0xcd, 0xb4, 0x8c, 0x0f, + 0x32, 0xae, 0x17, 0x12, 0xbc, 0x06, 0x2e, 0x6d, 0x97, 0xfa, 0x7b, 0x3f, 0x95, 0x56, 0xff, 0x0e, + 0xc0, 0x08, 0xed, 0xfb, 0xa1, 0x02, 0x2e, 0xc4, 0x7b, 0x5b, 0x2e, 0x99, 0xb5, 0xf5, 0xb2, 0x65, + 0x95, 0x9f, 0x9a, 0x48, 0x1e, 0x8a, 0x56, 0x93, 0xf1, 0xd4, 0xb6, 0x4b, 0x76, 0xd1, 0xaa, 0xd9, + 0xa8, 0xf8, 0xe8, 0x91, 0x89, 0x7a, 0x3b, 0x24, 0x41, 0x08, 0x66, 0x63, 0x82, 0x65, 0x1a, 0x6b, + 0x26, 0x92, 0x87, 0xe1, 0x6d, 0x70, 0x2b, 0x69, 0x1b, 0x44, 0xcf, 0x89, 0xf4, 0xc7, 0xdb, 0x65, + 0xb4, 0xbd, 0x25, 0x8f, 0xd0, 0x43, 0x13, 0xdb, 0x0c, 0xcb, 0x92, 0x47, 0xe1, 0x4d, 0xa0, 0xc6, + 0x5b, 0x2c, 0xec, 0x6e, 0x22, 0x72, 0x00, 0x1f, 0x80, 0x37, 0xce, 0x00, 0x0d, 0x8a, 0x62, 0x8a, + 0xa6, 0xa4, 0x0f, 0x37, 0x5a, 0xcf, 0x34, 0x7c, 0x1d, 0xbc, 0x36, 0xd0, 0x3d, 0x48, 0x74, 0x06, + 0xae, 0x83, 0x7c, 0x1f, 0x16, 0x5f, 0x65, 0x64, 0xe1, 0xe7, 0x32, 0x12, 0x8a, 0xa9, 0xd1, 0x21, + 0x2c, 0x20, 0xc3, 0x2e, 0x6c, 0xc8, 0xb3, 0x70, 0x15, 0xbc, 0x3c, 0xf0, 0x38, 0x24, 0x37, 0xa1, + 0x01, 0x0d, 0xf0, 0xee, 0xf9, 0xb0, 0x83, 0xc2, 0xc6, 0xf0, 0x25, 0xb0, 0x3c, 0x58, 0x22, 0xda, + 0x92, 0x5d, 0xf8, 0x0e, 0x78, 0xf3, 0x2c, 0xd4, 0xa0, 0x29, 0xf6, 0x4e, 0x9f, 0x22, 0x3a, 0x06, + 0xfb, 0xf4, 0xd9, 0x1b, 0x8c, 0xa2, 0x07, 0xc3, 0x85, 0xff, 0x07, 0xb4, 0xbe, 0x87, 0x3d, 0xb9, + 0x2d, 0x2f, 0x24, 0x78, 0x07, 0xdc, 0x46, 0x46, 0x69, 0xad, 0xbc, 0x55, 0x3b, 0x07, 0xfe, 0x53, + 0x09, 0xbe, 0x07, 0xde, 0x3e, 0x1b, 0x38, 0x68, 0x81, 0x9f, 0x49, 0xd0, 0x04, 0xef, 0x9f, 0x7b, + 0xbe, 0x41, 0x32, 0x9f, 0x4b, 0xf0, 0x06, 0xb8, 0xd6, 0x9f, 0x1f, 0xe5, 0xe1, 0x0b, 0x09, 0xae, + 0x80, 0x9b, 0xa7, 0xce, 0x14, 0x21, 0xbf, 0x94, 0xe0, 0x5b, 0xe0, 0xfe, 0x69, 0x90, 0x41, 0x61, + 0xfc, 0x5a, 0x82, 0x0f, 0xc1, 0x83, 0x73, 0xcc, 0x31, 0x48, 0xe0, 0x37, 0xa7, 0xac, 0x23, 0x4a, + 0xf6, 0x57, 0x67, 0xaf, 0x23, 0x42, 0xfe, 0x56, 0x82, 0x4b, 0xe0, 0x72, 0x7f, 0x08, 0x3d, 0x13, + 0xbf, 0x93, 0xe0, 0x2d, 0xb0, 0x7c, 0xaa, 0x12, 0x85, 0xfd, 0x5e, 0x82, 0x0a, 0x58, 0x28, 0x95, + 0x6b, 0xeb, 0x46, 0xd1, 0xaa, 0x3d, 0x2d, 0xda, 0x1b, 0xb5, 0xaa, 0x8d, 0xcc, 0x6a, 0x55, 0xfe, + 0xc5, 0x30, 0x0d, 0x25, 0xe1, 0x29, 0x95, 0x23, 0x67, 0x6d, 0xbd, 0x8c, 0x6a, 0x56, 0xf1, 0x89, + 0x59, 0xa2, 0xc8, 0x4f, 0x86, 0xe1, 0x1c, 0x00, 0x14, 0x56, 0x29, 0x17, 0x4b, 0x76, 0x55, 0xfe, + 0x6e, 0x0e, 0xce, 0x80, 0x09, 0xf3, 0x99, 0x6d, 0xa2, 0x92, 0x61, 0xc9, 0xff, 0xc8, 0xad, 0x1e, + 0x80, 0x89, 0xf8, 0xd3, 0x02, 0x1c, 0x03, 0xc3, 0x9b, 0x4f, 0xe4, 0x21, 0x38, 0x09, 0x46, 0x2d, + 0xd3, 0xa8, 0x9a, 0xb2, 0x04, 0x17, 0xc0, 0x9c, 0x69, 0x99, 0x05, 0xbb, 0x58, 0x2e, 0xd5, 0xd0, + 0x76, 0xa9, 0xc4, 0x2e, 0x4f, 0x19, 0x4c, 0x3f, 0xa5, 0x4f, 0x7e, 0x6c, 0xc9, 0xc1, 0x45, 0x30, + 0x6f, 0x95, 0x0b, 0x9b, 0x35, 0x64, 0x14, 0x4c, 0x14, 0x9b, 0x47, 0x28, 0x90, 0x09, 0xc5, 0x96, + 0xd1, 0xd5, 0x3c, 0x18, 0x8f, 0xbe, 0x4b, 0xc0, 0x29, 0x30, 0xbe, 0xf9, 0xa4, 0xb6, 0x61, 0x54, + 0x37, 0xe4, 0xa1, 0x1e, 0xd2, 0x7c, 0x56, 0x29, 0x22, 0x3a, 0x33, 0x00, 0x63, 0x27, 0x13, 0x4e, + 0x83, 0x89, 0x52, 0xb9, 0x56, 0xd8, 0x30, 0x0b, 0x9b, 0x72, 0xee, 0xde, 0x43, 0x30, 0x69, 0x07, + 0x8e, 0x17, 0xb6, 0xfc, 0x80, 0xc0, 0x7b, 0xe2, 0x60, 0x36, 0xfa, 0x3a, 0x1a, 0xfd, 0xe0, 0x7b, + 0x65, 0xee, 0x64, 0xcc, 0x7f, 0x0b, 0xd4, 0x86, 0x56, 0xa4, 0xd7, 0xa4, 0xfc, 0x85, 0x17, 0x7f, + 0x59, 0x1a, 0x7a, 0xf1, 0xf5, 0x92, 0xf4, 0xd5, 0xd7, 0x4b, 0xd2, 0x9f, 0xbf, 0x5e, 0x92, 0x7e, + 0xf2, 0xd7, 0xa5, 0xa1, 0x9d, 0x31, 0xf6, 0x83, 0xf1, 0xfd, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, + 0x5c, 0x9f, 0x8c, 0x37, 0x79, 0x1e, 0x00, 0x00, +} diff --git a/functional/rpcpb/rpc.proto b/functional/rpcpb/rpc.proto new file mode 100644 index 000000000..c7f6ea003 --- /dev/null +++ b/functional/rpcpb/rpc.proto @@ -0,0 +1,612 @@ +syntax = "proto3"; +package rpcpb; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (gogoproto.goproto_getters_all) = false; + +message Request { + Operation Operation = 1; + // Member contains the same Member object from tester configuration. + Member Member = 2; + // Tester contains tester configuration. + Tester Tester = 3; +} + +// SnapshotInfo contains SAVE_SNAPSHOT request results. +message SnapshotInfo { + string MemberName = 1; + repeated string MemberClientURLs = 2; + string SnapshotPath = 3; + string SnapshotFileSize = 4; + string SnapshotTotalSize = 5; + int64 SnapshotTotalKey = 6; + int64 SnapshotHash = 7; + int64 SnapshotRevision = 8; + string Took = 9; +} + +message Response { + bool Success = 1; + string Status = 2; + + // Member contains the same Member object from tester request. + Member Member = 3; + + // SnapshotInfo contains SAVE_SNAPSHOT request results. + SnapshotInfo SnapshotInfo = 4; +} + +service Transport { + rpc Transport(stream Request) returns (stream Response) {} +} + +message Member { + // EtcdExecPath is the executable etcd binary path in agent server. + string EtcdExecPath = 1 [(gogoproto.moretags) = "yaml:\"etcd-exec-path\""]; + // TODO: support embedded etcd + + // AgentAddr is the agent HTTP server address. + string AgentAddr = 11 [(gogoproto.moretags) = "yaml:\"agent-addr\""]; + // FailpointHTTPAddr is the agent's failpoints HTTP server address. + string FailpointHTTPAddr = 12 [(gogoproto.moretags) = "yaml:\"failpoint-http-addr\""]; + + // BaseDir is the base directory where all logs and etcd data are stored. + string BaseDir = 101 [(gogoproto.moretags) = "yaml:\"base-dir\""]; + // EtcdLogPath is the log file to store current etcd server logs. + string EtcdLogPath = 102 [(gogoproto.moretags) = "yaml:\"etcd-log-path\""]; + + // EtcdClientProxy is true when client traffic needs to be proxied. + // If true, listen client URL port must be different than advertise client URL port. + bool EtcdClientProxy = 201 [(gogoproto.moretags) = "yaml:\"etcd-client-proxy\""]; + // EtcdPeerProxy is true when peer traffic needs to be proxied. + // If true, listen peer URL port must be different than advertise peer URL port. + bool EtcdPeerProxy = 202 [(gogoproto.moretags) = "yaml:\"etcd-peer-proxy\""]; + + // EtcdClientEndpoint is the etcd client endpoint. + string EtcdClientEndpoint = 301 [(gogoproto.moretags) = "yaml:\"etcd-client-endpoint\""]; + // Etcd defines etcd binary configuration flags. + Etcd Etcd = 302 [(gogoproto.moretags) = "yaml:\"etcd\""]; + // EtcdOnSnapshotRestore defines one-time use configuration during etcd + // snapshot recovery process. + Etcd EtcdOnSnapshotRestore = 303; + + // ClientCertData contains cert file contents from this member's etcd server. + string ClientCertData = 401 [(gogoproto.moretags) = "yaml:\"client-cert-data\""]; + string ClientCertPath = 402 [(gogoproto.moretags) = "yaml:\"client-cert-path\""]; + // ClientKeyData contains key file contents from this member's etcd server. + string ClientKeyData = 403 [(gogoproto.moretags) = "yaml:\"client-key-data\""]; + string ClientKeyPath = 404 [(gogoproto.moretags) = "yaml:\"client-key-path\""]; + // ClientTrustedCAData contains trusted CA file contents from this member's etcd server. + string ClientTrustedCAData = 405 [(gogoproto.moretags) = "yaml:\"client-trusted-ca-data\""]; + string ClientTrustedCAPath = 406 [(gogoproto.moretags) = "yaml:\"client-trusted-ca-path\""]; + + // PeerCertData contains cert file contents from this member's etcd server. + string PeerCertData = 501 [(gogoproto.moretags) = "yaml:\"peer-cert-data\""]; + string PeerCertPath = 502 [(gogoproto.moretags) = "yaml:\"peer-cert-path\""]; + // PeerKeyData contains key file contents from this member's etcd server. + string PeerKeyData = 503 [(gogoproto.moretags) = "yaml:\"peer-key-data\""]; + string PeerKeyPath = 504 [(gogoproto.moretags) = "yaml:\"peer-key-path\""]; + // PeerTrustedCAData contains trusted CA file contents from this member's etcd server. + string PeerTrustedCAData = 505 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-data\""]; + string PeerTrustedCAPath = 506 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-path\""]; + + // SnapshotPath is the snapshot file path to store or restore from. + string SnapshotPath = 601 [(gogoproto.moretags) = "yaml:\"snapshot-path\""]; + // SnapshotInfo contains last SAVE_SNAPSHOT request results. + SnapshotInfo SnapshotInfo = 602; +} + +message Tester { + string DataDir = 1 [(gogoproto.moretags) = "yaml:\"data-dir\""]; + string Network = 2 [(gogoproto.moretags) = "yaml:\"network\""]; + string Addr = 3 [(gogoproto.moretags) = "yaml:\"addr\""]; + + // DelayLatencyMsRv is the delay latency in milliseconds, + // to inject to simulated slow network. + uint32 DelayLatencyMs = 11 [(gogoproto.moretags) = "yaml:\"delay-latency-ms\""]; + // DelayLatencyMsRv is the delay latency random variable in milliseconds. + uint32 DelayLatencyMsRv = 12 [(gogoproto.moretags) = "yaml:\"delay-latency-ms-rv\""]; + // UpdatedDelayLatencyMs is the update delay latency in milliseconds, + // to inject to simulated slow network. It's the final latency to apply, + // in case the latency numbers are randomly generated from given delay latency field. + uint32 UpdatedDelayLatencyMs = 13 [(gogoproto.moretags) = "yaml:\"updated-delay-latency-ms\""]; + + // RoundLimit is the limit of rounds to run failure set (-1 to run without limits). + int32 RoundLimit = 21 [(gogoproto.moretags) = "yaml:\"round-limit\""]; + // ExitOnCaseFail is true, then exit tester on first failure. + bool ExitOnCaseFail = 22 [(gogoproto.moretags) = "yaml:\"exit-on-failure\""]; + // EnablePprof is true to enable profiler. + bool EnablePprof = 23 [(gogoproto.moretags) = "yaml:\"enable-pprof\""]; + + // CaseDelayMs is the delay duration after failure is injected. + // Useful when triggering snapshot or no-op failure cases. + uint32 CaseDelayMs = 31 [(gogoproto.moretags) = "yaml:\"case-delay-ms\""]; + // CaseShuffle is true to randomize failure injecting order. + bool CaseShuffle = 32 [(gogoproto.moretags) = "yaml:\"case-shuffle\""]; + // Cases is the selected test cases to schedule. + // If empty, run all failure cases. + repeated string Cases = 33 [(gogoproto.moretags) = "yaml:\"cases\""]; + // FailpointCommands is the list of "gofail" commands + // (e.g. panic("etcd-tester"),1*sleep(1000). + repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; + + // RunnerExecPath is a path of etcd-runner binary. + string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""]; + // ExternalExecPath is a path of script for enabling/disabling an external fault injector. + string ExternalExecPath = 42 [(gogoproto.moretags) = "yaml:\"external-exec-path\""]; + + // Stressers is the list of stresser types: + // KV, LEASE, ELECTION_RUNNER, WATCH_RUNNER, LOCK_RACER_RUNNER, LEASE_RUNNER. + repeated string Stressers = 101 [(gogoproto.moretags) = "yaml:\"stressers\""]; + // Checkers is the list of consistency checker types: + // KV_HASH, LEASE_EXPIRE, NO_CHECK, RUNNER. + // Leave empty to skip consistency checks. + repeated string Checkers = 102 [(gogoproto.moretags) = "yaml:\"checkers\""]; + + // StressKeySize is the size of each small key written into etcd. + int32 StressKeySize = 201 [(gogoproto.moretags) = "yaml:\"stress-key-size\""]; + // StressKeySizeLarge is the size of each large key written into etcd. + int32 StressKeySizeLarge = 202 [(gogoproto.moretags) = "yaml:\"stress-key-size-large\""]; + // StressKeySuffixRange is the count of key range written into etcd. + // Stress keys are created with "fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange)". + int32 StressKeySuffixRange = 203 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range\""]; + // StressKeySuffixRangeTxn is the count of key range written into etcd txn (max 100). + // Stress keys are created with "fmt.Sprintf("/k%03d", i)". + int32 StressKeySuffixRangeTxn = 204 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""]; + // StressKeyTxnOps is the number of operations per a transaction (max 64). + int32 StressKeyTxnOps = 205 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""]; + + // StressClients is the number of concurrent stressing clients + // with "one" shared TCP connection. + int32 StressClients = 301 [(gogoproto.moretags) = "yaml:\"stress-clients\""]; + // StressQPS is the maximum number of stresser requests per second. + int32 StressQPS = 302 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; +} + +message Etcd { + string Name = 1 [(gogoproto.moretags) = "yaml:\"name\""]; + string DataDir = 2 [(gogoproto.moretags) = "yaml:\"data-dir\""]; + string WALDir = 3 [(gogoproto.moretags) = "yaml:\"wal-dir\""]; + + // HeartbeatIntervalMs is the time (in milliseconds) of a heartbeat interval. + // Default value is 100, which is 100ms. + int64 HeartbeatIntervalMs = 11 [(gogoproto.moretags) = "yaml:\"heartbeat-interval\""]; + // ElectionTimeoutMs is the time (in milliseconds) for an election to timeout. + // Default value is 1000, which is 1s. + int64 ElectionTimeoutMs = 12 [(gogoproto.moretags) = "yaml:\"election-timeout\""]; + + repeated string ListenClientURLs = 21 [(gogoproto.moretags) = "yaml:\"listen-client-urls\""]; + repeated string AdvertiseClientURLs = 22 [(gogoproto.moretags) = "yaml:\"advertise-client-urls\""]; + bool ClientAutoTLS = 23 [(gogoproto.moretags) = "yaml:\"auto-tls\""]; + bool ClientCertAuth = 24 [(gogoproto.moretags) = "yaml:\"client-cert-auth\""]; + string ClientCertFile = 25 [(gogoproto.moretags) = "yaml:\"cert-file\""]; + string ClientKeyFile = 26 [(gogoproto.moretags) = "yaml:\"key-file\""]; + string ClientTrustedCAFile = 27 [(gogoproto.moretags) = "yaml:\"trusted-ca-file\""]; + + repeated string ListenPeerURLs = 31 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""]; + repeated string AdvertisePeerURLs = 32 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""]; + bool PeerAutoTLS = 33 [(gogoproto.moretags) = "yaml:\"peer-auto-tls\""]; + bool PeerClientCertAuth = 34 [(gogoproto.moretags) = "yaml:\"peer-client-cert-auth\""]; + string PeerCertFile = 35 [(gogoproto.moretags) = "yaml:\"peer-cert-file\""]; + string PeerKeyFile = 36 [(gogoproto.moretags) = "yaml:\"peer-key-file\""]; + string PeerTrustedCAFile = 37 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-file\""]; + + string InitialCluster = 41 [(gogoproto.moretags) = "yaml:\"initial-cluster\""]; + string InitialClusterState = 42 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""]; + string InitialClusterToken = 43 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""]; + + int64 SnapshotCount = 51 [(gogoproto.moretags) = "yaml:\"snapshot-count\""]; + int64 QuotaBackendBytes = 52 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""]; + + bool PreVote = 63 [(gogoproto.moretags) = "yaml:\"pre-vote\""]; + bool InitialCorruptCheck = 64 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""]; +} + +enum Operation { + // NOT_STARTED is the agent status before etcd first start. + NOT_STARTED = 0; + + // INITIAL_START_ETCD is only called to start etcd, the very first time. + INITIAL_START_ETCD = 10; + // RESTART_ETCD is sent to restart killed etcd. + RESTART_ETCD = 11; + + // SIGTERM_ETCD pauses etcd process while keeping data directories + // and previous etcd configurations. + SIGTERM_ETCD = 20; + // SIGQUIT_ETCD_AND_REMOVE_DATA kills etcd process and removes all data + // directories to simulate destroying the whole machine. + SIGQUIT_ETCD_AND_REMOVE_DATA = 21; + + // SAVE_SNAPSHOT is sent to trigger local member to download its snapshot + // onto its local disk with the specified path from tester. + SAVE_SNAPSHOT = 30; + // RESTORE_RESTART_FROM_SNAPSHOT is sent to trigger local member to + // restore a cluster from existing snapshot from disk, and restart + // an etcd instance from recovered data. + RESTORE_RESTART_FROM_SNAPSHOT = 31; + // RESTART_FROM_SNAPSHOT is sent to trigger local member to restart + // and join an existing cluster that has been recovered from a snapshot. + // Local member joins this cluster with fresh data. + RESTART_FROM_SNAPSHOT = 32; + + // SIGQUIT_ETCD_AND_ARCHIVE_DATA is sent when consistency check failed, + // thus need to archive etcd data directories. + SIGQUIT_ETCD_AND_ARCHIVE_DATA = 40; + // SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT destroys etcd process, + // etcd data, and agent server. + SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT = 41; + + // BLACKHOLE_PEER_PORT_TX_RX drops all outgoing/incoming packets from/to + // the peer port on target member's peer port. + BLACKHOLE_PEER_PORT_TX_RX = 100; + // UNBLACKHOLE_PEER_PORT_TX_RX removes outgoing/incoming packet dropping. + UNBLACKHOLE_PEER_PORT_TX_RX = 101; + + // DELAY_PEER_PORT_TX_RX delays all outgoing/incoming packets from/to + // the peer port on target member's peer port. + DELAY_PEER_PORT_TX_RX = 200; + // UNDELAY_PEER_PORT_TX_RX removes all outgoing/incoming delays. + UNDELAY_PEER_PORT_TX_RX = 201; +} + +// Case defines various system faults or test case in distributed systems, +// in order to verify correct behavior of etcd servers and clients. +enum Case { + // SIGTERM_ONE_FOLLOWER stops a randomly chosen follower (non-leader) + // but does not delete its data directories on disk for next restart. + // It waits "delay-ms" before recovering this failure. + // The expected behavior is that the follower comes back online + // and rejoins the cluster, and then each member continues to process + // client requests ('Put' request that requires Raft consensus). + SIGTERM_ONE_FOLLOWER = 0; + + // SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly chosen + // follower but does not delete its data directories on disk for next + // restart. And waits until most up-to-date node (leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that the follower comes back online and + // rejoins the cluster, and then active leader sends snapshot + // to the follower to force it to follow the leader's log. + // As always, after recovery, each member must be able to process + // client requests. + SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 1; + + // SIGTERM_LEADER stops the active leader node but does not delete its + // data directories on disk for next restart. Then it waits "delay-ms" + // before recovering this failure, in order to trigger election timeouts. + // The expected behavior is that a new leader gets elected, and the + // old leader comes back online and rejoins the cluster as a follower. + // As always, after recovery, each member must be able to process + // client requests. + SIGTERM_LEADER = 2; + + // SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader node + // but does not delete its data directories on disk for next restart. + // And waits until most up-to-date node ("new" leader) applies the + // snapshot count of entries since the stop operation. + // The expected behavior is that cluster elects a new leader, and the + // old leader comes back online and rejoins the cluster as a follower. + // And it receives the snapshot from the new leader to overwrite its + // store. As always, after recovery, each member must be able to + // process client requests. + SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT = 3; + + // SIGTERM_QUORUM stops majority number of nodes to make the whole cluster + // inoperable but does not delete data directories on stopped nodes + // for next restart. And it waits "delay-ms" before recovering failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. + SIGTERM_QUORUM = 4; + + // SIGTERM_ALL stops the whole cluster but does not delete data directories + // on disk for next restart. And it waits "delay-ms" before recovering + // this failure. + // The expected behavior is that nodes come back online, thus cluster + // comes back operative as well. As always, after recovery, each member + // must be able to process client requests. + SIGTERM_ALL = 5; + + // SIGQUIT_AND_REMOVE_ONE_FOLLOWER stops a randomly chosen follower + // (non-leader), deletes its data directories on disk, and removes + // this member from cluster (membership reconfiguration). On recovery, + // tester adds a new member, and this member joins the existing cluster + // with fresh data. It waits "delay-ms" before recovering this + // failure. This simulates destroying one follower machine, where operator + // needs to add a new member from a fresh machine. + // The expected behavior is that a new member joins the existing cluster, + // and then each member continues to process client requests. + SIGQUIT_AND_REMOVE_ONE_FOLLOWER = 10; + + // SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT stops a randomly + // chosen follower, deletes its data directories on disk, and removes + // this member from cluster (membership reconfiguration). On recovery, + // tester adds a new member, and this member joins the existing cluster + // restart. On member remove, cluster waits until most up-to-date node + // (leader) applies the snapshot count of entries since the stop operation. + // This simulates destroying a leader machine, where operator needs to add + // a new member from a fresh machine. + // The expected behavior is that a new member joins the existing cluster, + // and receives a snapshot from the active leader. As always, after + // recovery, each member must be able to process client requests. + SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 11; + + // SIGQUIT_AND_REMOVE_LEADER stops the active leader node, deletes its + // data directories on disk, and removes this member from cluster. + // On recovery, tester adds a new member, and this member joins the + // existing cluster with fresh data. It waits "delay-ms" before + // recovering this failure. This simulates destroying a leader machine, + // where operator needs to add a new member from a fresh machine. + // The expected behavior is that a new member joins the existing cluster, + // and then each member continues to process client requests. + SIGQUIT_AND_REMOVE_LEADER = 12; + + // SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT stops the active leader, + // deletes its data directories on disk, and removes this member from + // cluster (membership reconfiguration). On recovery, tester adds a new + // member, and this member joins the existing cluster restart. On member + // remove, cluster waits until most up-to-date node (new leader) applies + // the snapshot count of entries since the stop operation. This simulates + // destroying a leader machine, where operator needs to add a new member + // from a fresh machine. + // The expected behavior is that on member remove, cluster elects a new + // leader, and a new member joins the existing cluster and receives a + // snapshot from the newly elected leader. As always, after recovery, each + // member must be able to process client requests. + SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT = 13; + + // SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH first + // stops majority number of nodes, deletes data directories on those quorum + // nodes, to make the whole cluster inoperable. Now that quorum and their + // data are totally destroyed, cluster cannot even remove unavailable nodes + // (e.g. 2 out of 3 are lost, so no leader can be elected). + // Let's assume 3-node cluster of node A, B, and C. One day, node A and B + // are destroyed and all their data are gone. The only viable solution is + // to recover from C's latest snapshot. + // + // To simulate: + // 1. Assume node C is the current leader with most up-to-date data. + // 2. Download snapshot from node C, before destroying node A and B. + // 3. Destroy node A and B, and make the whole cluster inoperable. + // 4. Now node C cannot operate either. + // 5. SIGTERM node C and remove its data directories. + // 6. Restore a new seed member from node C's latest snapshot file. + // 7. Add another member to establish 2-node cluster. + // 8. Add another member to establish 3-node cluster. + // 9. Add more if any. + // + // The expected behavior is that etcd successfully recovers from such + // disastrous situation as only 1-node survives out of 3-node cluster, + // new members joins the existing cluster, and previous data from snapshot + // are still preserved after recovery process. As always, after recovery, + // each member must be able to process client requests. + SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH = 14; + + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER drops all outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader), and waits for "delay-ms" until recovery. + // The expected behavior is that once dropping operation is undone, + // each member must be able to process client requests. + BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER = 100; + + // BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT drops + // all outgoing/incoming packets from/to the peer port on a randomly + // chosen follower (non-leader), and waits for most up-to-date node + // (leader) applies the snapshot count of entries since the blackhole + // operation. + // The expected behavior is that once packet drop operation is undone, + // the slow follower tries to catch up, possibly receiving the snapshot + // from the active leader. As always, after recovery, each member must + // be able to process client requests. + BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 101; + + // BLACKHOLE_PEER_PORT_TX_RX_LEADER drops all outgoing/incoming packets + // from/to the peer port on the active leader (isolated), and waits for + // "delay-ms" until recovery, in order to trigger election timeout. + // The expected behavior is that after election timeout, a new leader gets + // elected, and once dropping operation is undone, the old leader comes + // back and rejoins the cluster as a follower. As always, after recovery, + // each member must be able to process client requests. + BLACKHOLE_PEER_PORT_TX_RX_LEADER = 102; + + // BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT drops all + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (leader) applies the snapshot + // count of entries since the blackhole operation. + // The expected behavior is that cluster elects a new leader, and once + // dropping operation is undone, the old leader comes back and rejoins + // the cluster as a follower. The slow follower tries to catch up, likely + // receiving the snapshot from the new active leader. As always, after + // recovery, each member must be able to process client requests. + BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT = 103; + + // BLACKHOLE_PEER_PORT_TX_RX_QUORUM drops all outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, thus losing its + // leader and cluster being inoperable. And it waits for "delay-ms" + // until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + BLACKHOLE_PEER_PORT_TX_RX_QUORUM = 104; + + // BLACKHOLE_PEER_PORT_TX_RX_ALL drops all outgoing/incoming packets + // from/to the peer ports on all nodes, thus making cluster totally + // inoperable. It waits for "delay-ms" until recovery. + // The expected behavior is that once packet drop operation is undone, + // nodes come back online, thus cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + BLACKHOLE_PEER_PORT_TX_RX_ALL = 105; + + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming packets + // from/to the peer port on a randomly chosen follower (non-leader). + // It waits for "delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // the follower comes back and tries to catch up with latest changes from + // cluster. And as always, after recovery, each member must be able to + // process client requests. + DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER = 200; + + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER delays outgoing/incoming + // packets from/to the peer port on a randomly chosen follower + // (non-leader) with a randomized time duration (thus isolated). It + // waits for "delay-ms" until recovery. + // The expected behavior is that once packet delay operation is undone, + // each member must be able to process client requests. + RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER = 201; + + // DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader), and waits for most up-to-date node (leader) + // applies the snapshot count of entries since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up possibly receiving snapshot + // from the active leader. As always, after recovery, each member must be + // able to process client requests. + DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 202; + + // RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on a randomly chosen + // follower (non-leader) with a randomized time duration, and waits for + // most up-to-date node (leader) applies the snapshot count of entries + // since the delay operation. + // The expected behavior is that the delayed follower gets isolated + // and behind the current active leader, and once delay operation is undone, + // the slow follower comes back and catches up, possibly receiving a + // snapshot from the active leader. As always, after recovery, each member + // must be able to process client requests. + RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT = 203; + + // DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets from/to + // the peer port on the active leader. And waits for "delay-ms" until + // recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. + DELAY_PEER_PORT_TX_RX_LEADER = 204; + + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER delays outgoing/incoming packets + // from/to the peer port on the active leader with a randomized time + // duration. And waits for "delay-ms" until recovery. + // The expected behavior is that cluster may elect a new leader, and + // once packet delay operation is undone, the (old) leader comes back + // and tries to catch up with latest changes from cluster. As always, + // after recovery, each member must be able to process client requests. + RANDOM_DELAY_PEER_PORT_TX_RX_LEADER = 205; + + // DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // and waits for most up-to-date node (current or new leader) applies the + // snapshot count of entries since the delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. + DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT = 206; + + // RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT delays + // outgoing/incoming packets from/to the peer port on the active leader, + // with a randomized time duration. And it waits for most up-to-date node + // (current or new leader) applies the snapshot count of entries since the + // delay operation. + // The expected behavior is that cluster may elect a new leader, and + // the old leader gets isolated and behind the current active leader, + // and once delay operation is undone, the slow follower comes back + // and catches up, likely receiving a snapshot from the active leader. + // As always, after recovery, each member must be able to process client + // requests. + RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT = 207; + + // DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets from/to + // the peer ports on majority nodes of cluster. And it waits for + // "delay-ms" until recovery, likely to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + DELAY_PEER_PORT_TX_RX_QUORUM = 208; + + // RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM delays outgoing/incoming packets + // from/to the peer ports on majority nodes of cluster, with randomized + // time durations. And it waits for "delay-ms" until recovery, likely + // to trigger election timeouts. + // The expected behavior is that cluster may elect a new leader, while + // quorum of nodes struggle with slow networks, and once delay operation + // is undone, nodes come back and cluster comes back operative. As always, + // after recovery, each member must be able to process client requests. + RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM = 209; + + // DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets from/to the + // peer ports on all nodes. And it waits for "delay-ms" until recovery, + // likely to trigger election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. + DELAY_PEER_PORT_TX_RX_ALL = 210; + + // RANDOM_DELAY_PEER_PORT_TX_RX_ALL delays outgoing/incoming packets + // from/to the peer ports on all nodes, with randomized time durations. + // And it waits for "delay-ms" until recovery, likely to trigger + // election timeouts. + // The expected behavior is that cluster may become totally inoperable, + // struggling with slow networks across the whole cluster. Once delay + // operation is undone, nodes come back and cluster comes back operative. + // As always, after recovery, each member must be able to process client + // requests. + RANDOM_DELAY_PEER_PORT_TX_RX_ALL = 211; + + // NO_FAIL_WITH_STRESS stops injecting failures while testing the + // consistency and correctness under pressure loads, for the duration of + // "delay-ms". Goal is to ensure cluster be still making progress + // on recovery, and verify system does not deadlock following a sequence + // of failure injections. + // The expected behavior is that cluster remains fully operative in healthy + // condition. As always, after recovery, each member must be able to process + // client requests. + NO_FAIL_WITH_STRESS = 300; + + // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS neither injects failures nor + // sends stressig client requests to the cluster, for the duration of + // "delay-ms". Goal is to ensure cluster be still making progress + // on recovery, and verify system does not deadlock following a sequence + // of failure injections. + // The expected behavior is that cluster remains fully operative in healthy + // condition, and clients requests during liveness period succeed without + // errors. + // Note: this is how Google Chubby does failure injection testing + // https://static.googleusercontent.com/media/research.google.com/en//archive/paxos_made_live.pdf. + NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS = 301; + + // FAILPOINTS injects failpoints to etcd server runtime, triggering panics + // in critical code paths. + FAILPOINTS = 400; + + // EXTERNAL runs external failure injection scripts. + EXTERNAL = 500; +} + +enum Stresser { + KV = 0; + LEASE = 1; + ELECTION_RUNNER = 2; + WATCH_RUNNER = 3; + LOCK_RACER_RUNNER = 4; + LEASE_RUNNER = 5; +} + +enum Checker { + KV_HASH = 0; + LEASE_EXPIRE = 1; + RUNNER = 2; + NO_CHECK = 3; +} diff --git a/functional/runner/election_command.go b/functional/runner/election_command.go new file mode 100644 index 000000000..b2bc99a16 --- /dev/null +++ b/functional/runner/election_command.go @@ -0,0 +1,144 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "context" + "errors" + "fmt" + + "github.com/coreos/etcd/clientv3/concurrency" + + "github.com/spf13/cobra" +) + +// NewElectionCommand returns the cobra command for "election runner". +func NewElectionCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "election [election name (defaults to 'elector')]", + Short: "Performs election operation", + Run: runElectionFunc, + } + cmd.Flags().IntVar(&totalClientConnections, "total-client-connections", 10, "total number of client connections") + return cmd +} + +func runElectionFunc(cmd *cobra.Command, args []string) { + election := "elector" + if len(args) == 1 { + election = args[0] + } + if len(args) > 1 { + ExitWithError(ExitBadArgs, errors.New("election takes at most one argument")) + } + + rcs := make([]roundClient, totalClientConnections) + validatec := make(chan struct{}, len(rcs)) + // nextc closes when election is ready for next round. + nextc := make(chan struct{}) + eps := endpointsFromFlag(cmd) + + for i := range rcs { + v := fmt.Sprintf("%d", i) + observedLeader := "" + validateWaiters := 0 + var rcNextc chan struct{} + setRcNextc := func() { + rcNextc = nextc + } + + rcs[i].c = newClient(eps, dialTimeout) + var ( + s *concurrency.Session + err error + ) + for { + s, err = concurrency.NewSession(rcs[i].c) + if err == nil { + break + } + } + + e := concurrency.NewElection(s, election) + rcs[i].acquire = func() (err error) { + ctx, cancel := context.WithCancel(context.Background()) + donec := make(chan struct{}) + go func() { + defer close(donec) + for ctx.Err() == nil { + if ol, ok := <-e.Observe(ctx); ok { + observedLeader = string(ol.Kvs[0].Value) + break + } + } + if observedLeader != v { + cancel() + } + }() + err = e.Campaign(ctx, v) + cancel() + <-donec + if err == nil { + observedLeader = v + } + if observedLeader == v { + validateWaiters = len(rcs) + } + select { + case <-ctx.Done(): + return nil + default: + return err + } + } + rcs[i].validate = func() error { + l, err := e.Leader(context.TODO()) + if err == nil && string(l.Kvs[0].Value) != observedLeader { + return fmt.Errorf("expected leader %q, got %q", observedLeader, l.Kvs[0].Value) + } + if err != nil { + return err + } + setRcNextc() + validatec <- struct{}{} + return nil + } + rcs[i].release = func() error { + for validateWaiters > 0 { + select { + case <-validatec: + validateWaiters-- + default: + return fmt.Errorf("waiting on followers") + } + } + if err := e.Resign(context.TODO()); err != nil { + return err + } + if observedLeader == v { + oldNextc := nextc + nextc = make(chan struct{}) + close(oldNextc) + + } + <-rcNextc + observedLeader = "" + return nil + } + } + // each client creates 1 key from Campaign() and delete it from Resign() + // a round involves in 2*len(rcs) requests. + doRounds(rcs, rounds, 2*len(rcs)) +} diff --git a/functional/runner/error.go b/functional/runner/error.go new file mode 100644 index 000000000..335e85cb7 --- /dev/null +++ b/functional/runner/error.go @@ -0,0 +1,42 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "fmt" + "os" + + "github.com/coreos/etcd/client" +) + +const ( + // http://tldp.org/LDP/abs/html/exitcodes.html + ExitSuccess = iota + ExitError + ExitBadConnection + ExitInvalidInput // for txn, watch command + ExitBadFeature // provided a valid flag with an unsupported value + ExitInterrupted + ExitIO + ExitBadArgs = 128 +) + +func ExitWithError(code int, err error) { + fmt.Fprintln(os.Stderr, "Error: ", err) + if cerr, ok := err.(*client.ClusterError); ok { + fmt.Fprintln(os.Stderr, cerr.Detail()) + } + os.Exit(code) +} diff --git a/functional/runner/global.go b/functional/runner/global.go new file mode 100644 index 000000000..94a3a2aae --- /dev/null +++ b/functional/runner/global.go @@ -0,0 +1,114 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "context" + "fmt" + "log" + "sync" + "time" + + "github.com/coreos/etcd/clientv3" + + "github.com/spf13/cobra" + "golang.org/x/time/rate" +) + +// shared flags +var ( + totalClientConnections int // total number of client connections to be made with server + endpoints []string + dialTimeout time.Duration + rounds int // total number of rounds to run; set to <= 0 to run forever. + reqRate int // maximum number of requests per second. +) + +type roundClient struct { + c *clientv3.Client + progress int + acquire func() error + validate func() error + release func() error +} + +func newClient(eps []string, timeout time.Duration) *clientv3.Client { + c, err := clientv3.New(clientv3.Config{ + Endpoints: eps, + DialTimeout: time.Duration(timeout) * time.Second, + }) + if err != nil { + log.Fatal(err) + } + return c +} + +func doRounds(rcs []roundClient, rounds int, requests int) { + var wg sync.WaitGroup + + wg.Add(len(rcs)) + finished := make(chan struct{}) + limiter := rate.NewLimiter(rate.Limit(reqRate), reqRate) + for i := range rcs { + go func(rc *roundClient) { + defer wg.Done() + for rc.progress < rounds || rounds <= 0 { + if err := limiter.WaitN(context.Background(), requests/len(rcs)); err != nil { + log.Panicf("rate limiter error %v", err) + } + + for rc.acquire() != nil { /* spin */ + } + + if err := rc.validate(); err != nil { + log.Fatal(err) + } + + time.Sleep(10 * time.Millisecond) + rc.progress++ + finished <- struct{}{} + + for rc.release() != nil { /* spin */ + } + } + }(&rcs[i]) + } + + start := time.Now() + for i := 1; i < len(rcs)*rounds+1 || rounds <= 0; i++ { + select { + case <-finished: + if i%100 == 0 { + fmt.Printf("finished %d, took %v\n", i, time.Since(start)) + start = time.Now() + } + case <-time.After(time.Minute): + log.Panic("no progress after 1 minute!") + } + } + wg.Wait() + + for _, rc := range rcs { + rc.c.Close() + } +} + +func endpointsFromFlag(cmd *cobra.Command) []string { + eps, err := cmd.Flags().GetStringSlice("endpoints") + if err != nil { + ExitWithError(ExitError, err) + } + return eps +} diff --git a/functional/runner/help.go b/functional/runner/help.go new file mode 100644 index 000000000..63e9815ab --- /dev/null +++ b/functional/runner/help.go @@ -0,0 +1,175 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// copied from https://github.com/rkt/rkt/blob/master/rkt/help.go + +package runner + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + "text/template" + + "github.com/coreos/etcd/version" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var ( + commandUsageTemplate *template.Template + templFuncs = template.FuncMap{ + "descToLines": func(s string) []string { + // trim leading/trailing whitespace and split into slice of lines + return strings.Split(strings.Trim(s, "\n\t "), "\n") + }, + "cmdName": func(cmd *cobra.Command, startCmd *cobra.Command) string { + parts := []string{cmd.Name()} + for cmd.HasParent() && cmd.Parent().Name() != startCmd.Name() { + cmd = cmd.Parent() + parts = append([]string{cmd.Name()}, parts...) + } + return strings.Join(parts, " ") + }, + } +) + +func init() { + commandUsage := ` +{{ $cmd := .Cmd }}\ +{{ $cmdname := cmdName .Cmd .Cmd.Root }}\ +NAME: +{{ if not .Cmd.HasParent }}\ +{{printf "\t%s - %s" .Cmd.Name .Cmd.Short}} +{{else}}\ +{{printf "\t%s - %s" $cmdname .Cmd.Short}} +{{end}}\ + +USAGE: +{{printf "\t%s" .Cmd.UseLine}} +{{ if not .Cmd.HasParent }}\ + +VERSION: +{{printf "\t%s" .Version}} +{{end}}\ +{{if .Cmd.HasSubCommands}}\ + +API VERSION: +{{printf "\t%s" .APIVersion}} +{{end}}\ +{{if .Cmd.HasSubCommands}}\ + + +COMMANDS: +{{range .SubCommands}}\ +{{ $cmdname := cmdName . $cmd }}\ +{{ if .Runnable }}\ +{{printf "\t%s\t%s" $cmdname .Short}} +{{end}}\ +{{end}}\ +{{end}}\ +{{ if .Cmd.Long }}\ + +DESCRIPTION: +{{range $line := descToLines .Cmd.Long}}{{printf "\t%s" $line}} +{{end}}\ +{{end}}\ +{{if .Cmd.HasLocalFlags}}\ + +OPTIONS: +{{.LocalFlags}}\ +{{end}}\ +{{if .Cmd.HasInheritedFlags}}\ + +GLOBAL OPTIONS: +{{.GlobalFlags}}\ +{{end}} +`[1:] + + commandUsageTemplate = template.Must(template.New("command_usage").Funcs(templFuncs).Parse(strings.Replace(commandUsage, "\\\n", "", -1))) +} + +func etcdFlagUsages(flagSet *pflag.FlagSet) string { + x := new(bytes.Buffer) + + flagSet.VisitAll(func(flag *pflag.Flag) { + if len(flag.Deprecated) > 0 { + return + } + var format string + if len(flag.Shorthand) > 0 { + format = " -%s, --%s" + } else { + format = " %s --%s" + } + if len(flag.NoOptDefVal) > 0 { + format = format + "[" + } + if flag.Value.Type() == "string" { + // put quotes on the value + format = format + "=%q" + } else { + format = format + "=%s" + } + if len(flag.NoOptDefVal) > 0 { + format = format + "]" + } + format = format + "\t%s\n" + shorthand := flag.Shorthand + fmt.Fprintf(x, format, shorthand, flag.Name, flag.DefValue, flag.Usage) + }) + + return x.String() +} + +func getSubCommands(cmd *cobra.Command) []*cobra.Command { + var subCommands []*cobra.Command + for _, subCmd := range cmd.Commands() { + subCommands = append(subCommands, subCmd) + subCommands = append(subCommands, getSubCommands(subCmd)...) + } + return subCommands +} + +func usageFunc(cmd *cobra.Command) error { + subCommands := getSubCommands(cmd) + tabOut := getTabOutWithWriter(os.Stdout) + commandUsageTemplate.Execute(tabOut, struct { + Cmd *cobra.Command + LocalFlags string + GlobalFlags string + SubCommands []*cobra.Command + Version string + APIVersion string + }{ + cmd, + etcdFlagUsages(cmd.LocalFlags()), + etcdFlagUsages(cmd.InheritedFlags()), + subCommands, + version.Version, + version.APIVersion, + }) + tabOut.Flush() + return nil +} + +func getTabOutWithWriter(writer io.Writer) *tabwriter.Writer { + aTabOut := new(tabwriter.Writer) + aTabOut.Init(writer, 0, 8, 1, '\t', 0) + return aTabOut +} diff --git a/functional/runner/lease_renewer_command.go b/functional/runner/lease_renewer_command.go new file mode 100644 index 000000000..a57c53f27 --- /dev/null +++ b/functional/runner/lease_renewer_command.go @@ -0,0 +1,91 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/coreos/etcd/clientv3" + + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + leaseTTL int64 +) + +// NewLeaseRenewerCommand returns the cobra command for "lease-renewer runner". +func NewLeaseRenewerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "lease-renewer", + Short: "Performs lease renew operation", + Run: runLeaseRenewerFunc, + } + cmd.Flags().Int64Var(&leaseTTL, "ttl", 5, "lease's ttl") + return cmd +} + +func runLeaseRenewerFunc(cmd *cobra.Command, args []string) { + if len(args) > 0 { + ExitWithError(ExitBadArgs, errors.New("lease-renewer does not take any argument")) + } + + eps := endpointsFromFlag(cmd) + c := newClient(eps, dialTimeout) + ctx := context.Background() + + for { + var ( + l *clientv3.LeaseGrantResponse + lk *clientv3.LeaseKeepAliveResponse + err error + ) + for { + l, err = c.Lease.Grant(ctx, leaseTTL) + if err == nil { + break + } + } + expire := time.Now().Add(time.Duration(l.TTL-1) * time.Second) + + for { + lk, err = c.Lease.KeepAliveOnce(ctx, l.ID) + if ev, ok := status.FromError(err); ok && ev.Code() == codes.NotFound { + if time.Since(expire) < 0 { + log.Fatalf("bad renew! exceeded: %v", time.Since(expire)) + for { + lk, err = c.Lease.KeepAliveOnce(ctx, l.ID) + fmt.Println(lk, err) + time.Sleep(time.Second) + } + } + log.Fatalf("lost lease %d, expire: %v\n", l.ID, expire) + break + } + if err != nil { + continue + } + expire = time.Now().Add(time.Duration(lk.TTL-1) * time.Second) + log.Printf("renewed lease %d, expire: %v\n", lk.ID, expire) + time.Sleep(time.Duration(lk.TTL-2) * time.Second) + } + } +} diff --git a/functional/runner/lock_racer_command.go b/functional/runner/lock_racer_command.go new file mode 100644 index 000000000..18b10e403 --- /dev/null +++ b/functional/runner/lock_racer_command.go @@ -0,0 +1,94 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/coreos/etcd/clientv3/concurrency" + + "github.com/spf13/cobra" +) + +// NewLockRacerCommand returns the cobra command for "lock-racer runner". +func NewLockRacerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "lock-racer [name of lock (defaults to 'racers')]", + Short: "Performs lock race operation", + Run: runRacerFunc, + } + cmd.Flags().IntVar(&totalClientConnections, "total-client-connections", 10, "total number of client connections") + return cmd +} + +func runRacerFunc(cmd *cobra.Command, args []string) { + racers := "racers" + if len(args) == 1 { + racers = args[0] + } + + if len(args) > 1 { + ExitWithError(ExitBadArgs, errors.New("lock-racer takes at most one argument")) + } + + rcs := make([]roundClient, totalClientConnections) + ctx := context.Background() + // mu ensures validate and release funcs are atomic. + var mu sync.Mutex + cnt := 0 + + eps := endpointsFromFlag(cmd) + + for i := range rcs { + var ( + s *concurrency.Session + err error + ) + + rcs[i].c = newClient(eps, dialTimeout) + + for { + s, err = concurrency.NewSession(rcs[i].c) + if err == nil { + break + } + } + m := concurrency.NewMutex(s, racers) + rcs[i].acquire = func() error { return m.Lock(ctx) } + rcs[i].validate = func() error { + mu.Lock() + defer mu.Unlock() + if cnt++; cnt != 1 { + return fmt.Errorf("bad lock; count: %d", cnt) + } + return nil + } + rcs[i].release = func() error { + mu.Lock() + defer mu.Unlock() + if err := m.Unlock(ctx); err != nil { + return err + } + cnt = 0 + return nil + } + } + // each client creates 1 key from NewMutex() and delete it from Unlock() + // a round involves in 2*len(rcs) requests. + doRounds(rcs, rounds, 2*len(rcs)) +} diff --git a/functional/runner/root.go b/functional/runner/root.go new file mode 100644 index 000000000..abd74af1b --- /dev/null +++ b/functional/runner/root.go @@ -0,0 +1,70 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runner implements individual etcd-runner commands for the etcd-runner utility. +package runner + +import ( + "log" + "math/rand" + "time" + + "github.com/spf13/cobra" +) + +const ( + cliName = "etcd-runner" + cliDescription = "Stress tests using clientv3 functionality.." + + defaultDialTimeout = 2 * time.Second +) + +var ( + rootCmd = &cobra.Command{ + Use: cliName, + Short: cliDescription, + SuggestFor: []string{"etcd-runner"}, + } +) + +func init() { + cobra.EnablePrefixMatching = true + + rand.Seed(time.Now().UnixNano()) + + log.SetFlags(log.Lmicroseconds) + + rootCmd.PersistentFlags().StringSliceVar(&endpoints, "endpoints", []string{"127.0.0.1:2379"}, "gRPC endpoints") + rootCmd.PersistentFlags().DurationVar(&dialTimeout, "dial-timeout", defaultDialTimeout, "dial timeout for client connections") + rootCmd.PersistentFlags().IntVar(&reqRate, "req-rate", 30, "maximum number of requests per second") + rootCmd.PersistentFlags().IntVar(&rounds, "rounds", 100, "number of rounds to run; 0 to run forever") + + rootCmd.AddCommand( + NewElectionCommand(), + NewLeaseRenewerCommand(), + NewLockRacerCommand(), + NewWatchCommand(), + ) +} + +func Start() { + rootCmd.SetUsageFunc(usageFunc) + + // Make help just show the usage + rootCmd.SetHelpTemplate(`{{.UsageString}}`) + + if err := rootCmd.Execute(); err != nil { + ExitWithError(ExitError, err) + } +} diff --git a/functional/runner/watch_command.go b/functional/runner/watch_command.go new file mode 100644 index 000000000..646092ad0 --- /dev/null +++ b/functional/runner/watch_command.go @@ -0,0 +1,210 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/pkg/stringutil" + + "github.com/spf13/cobra" + "golang.org/x/time/rate" +) + +var ( + runningTime time.Duration // time for which operation should be performed + noOfPrefixes int // total number of prefixes which will be watched upon + watchPerPrefix int // number of watchers per prefix + watchPrefix string // prefix append to keys in watcher + totalKeys int // total number of keys for operation +) + +// NewWatchCommand returns the cobra command for "watcher runner". +func NewWatchCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "watcher", + Short: "Performs watch operation", + Run: runWatcherFunc, + } + cmd.Flags().DurationVar(&runningTime, "running-time", 60, "number of seconds to run") + cmd.Flags().StringVar(&watchPrefix, "prefix", "", "the prefix to append on all keys") + cmd.Flags().IntVar(&noOfPrefixes, "total-prefixes", 10, "total no of prefixes to use") + cmd.Flags().IntVar(&watchPerPrefix, "watch-per-prefix", 10, "number of watchers per prefix") + cmd.Flags().IntVar(&totalKeys, "total-keys", 1000, "total number of keys to watch") + + return cmd +} + +func runWatcherFunc(cmd *cobra.Command, args []string) { + if len(args) > 0 { + ExitWithError(ExitBadArgs, errors.New("watcher does not take any argument")) + } + + ctx := context.Background() + for round := 0; round < rounds || rounds <= 0; round++ { + fmt.Println("round", round) + performWatchOnPrefixes(ctx, cmd, round) + } +} + +func performWatchOnPrefixes(ctx context.Context, cmd *cobra.Command, round int) { + keyPerPrefix := totalKeys / noOfPrefixes + prefixes := stringutil.UniqueStrings(5, noOfPrefixes) + keys := stringutil.RandomStrings(10, keyPerPrefix) + + roundPrefix := fmt.Sprintf("%16x", round) + + eps := endpointsFromFlag(cmd) + + var ( + revision int64 + wg sync.WaitGroup + gr *clientv3.GetResponse + err error + ) + + client := newClient(eps, dialTimeout) + defer client.Close() + + gr, err = getKey(ctx, client, "non-existent") + if err != nil { + log.Fatalf("failed to get the initial revision: %v", err) + } + revision = gr.Header.Revision + + ctxt, cancel := context.WithDeadline(ctx, time.Now().Add(runningTime*time.Second)) + defer cancel() + + // generate and put keys in cluster + limiter := rate.NewLimiter(rate.Limit(reqRate), reqRate) + + go func() { + for _, key := range keys { + for _, prefix := range prefixes { + if err = limiter.Wait(ctxt); err != nil { + return + } + if err = putKeyAtMostOnce(ctxt, client, watchPrefix+"-"+roundPrefix+"-"+prefix+"-"+key); err != nil { + log.Fatalf("failed to put key: %v", err) + return + } + } + } + }() + + ctxc, cancelc := context.WithCancel(ctx) + + wcs := make([]clientv3.WatchChan, 0) + rcs := make([]*clientv3.Client, 0) + + for _, prefix := range prefixes { + for j := 0; j < watchPerPrefix; j++ { + rc := newClient(eps, dialTimeout) + rcs = append(rcs, rc) + + wprefix := watchPrefix + "-" + roundPrefix + "-" + prefix + + wc := rc.Watch(ctxc, wprefix, clientv3.WithPrefix(), clientv3.WithRev(revision)) + wcs = append(wcs, wc) + + wg.Add(1) + go func() { + defer wg.Done() + checkWatchResponse(wc, wprefix, keys) + }() + } + } + wg.Wait() + + cancelc() + + // verify all watch channels are closed + for e, wc := range wcs { + if _, ok := <-wc; ok { + log.Fatalf("expected wc to be closed, but received %v", e) + } + } + + for _, rc := range rcs { + rc.Close() + } + + if err = deletePrefix(ctx, client, watchPrefix); err != nil { + log.Fatalf("failed to clean up keys after test: %v", err) + } +} + +func checkWatchResponse(wc clientv3.WatchChan, prefix string, keys []string) { + for n := 0; n < len(keys); { + wr, more := <-wc + if !more { + log.Fatalf("expect more keys (received %d/%d) for %s", n, len(keys), prefix) + } + for _, event := range wr.Events { + expectedKey := prefix + "-" + keys[n] + receivedKey := string(event.Kv.Key) + if expectedKey != receivedKey { + log.Fatalf("expected key %q, got %q for prefix : %q\n", expectedKey, receivedKey, prefix) + } + n++ + } + } +} + +func putKeyAtMostOnce(ctx context.Context, client *clientv3.Client, key string) error { + gr, err := getKey(ctx, client, key) + if err != nil { + return err + } + + var modrev int64 + if len(gr.Kvs) > 0 { + modrev = gr.Kvs[0].ModRevision + } + + for ctx.Err() == nil { + _, err := client.Txn(ctx).If(clientv3.Compare(clientv3.ModRevision(key), "=", modrev)).Then(clientv3.OpPut(key, key)).Commit() + + if err == nil { + return nil + } + } + + return ctx.Err() +} + +func deletePrefix(ctx context.Context, client *clientv3.Client, key string) error { + for ctx.Err() == nil { + if _, err := client.Delete(ctx, key, clientv3.WithPrefix()); err == nil { + return nil + } + } + return ctx.Err() +} + +func getKey(ctx context.Context, client *clientv3.Client, key string) (*clientv3.GetResponse, error) { + for ctx.Err() == nil { + if gr, err := client.Get(ctx, key); err == nil { + return gr, nil + } + } + return nil, ctx.Err() +} diff --git a/functional/scripts/docker-local-agent.sh b/functional/scripts/docker-local-agent.sh new file mode 100755 index 000000000..355a996f7 --- /dev/null +++ b/functional/scripts/docker-local-agent.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +< snapshotCount { + clus.lg.Info( + "trigger snapshot PASS", + zap.Int("retries", i), + zap.String("desc", c.Desc()), + zap.Int64("committed-entries", dicc), + zap.Int64("etcd-snapshot-count", snapshotCount), + zap.Int64("last-revision", lastRev), + zap.Duration("took", time.Since(now)), + ) + return nil + } + + clus.lg.Info( + "trigger snapshot PROGRESS", + zap.Int("retries", i), + zap.Int64("committed-entries", dicc), + zap.Int64("etcd-snapshot-count", snapshotCount), + zap.Int64("last-revision", lastRev), + zap.Duration("took", time.Since(now)), + ) + time.Sleep(time.Second) + } + + return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries) +} + +func (c *caseUntilSnapshot) Desc() string { + if c.desc != "" { + return c.desc + } + if c.rpcpbCase.String() != "" { + return c.rpcpbCase.String() + } + return c.Case.Desc() +} + +func (c *caseUntilSnapshot) TestCase() rpcpb.Case { + return c.rpcpbCase +} diff --git a/functional/tester/case_delay.go b/functional/tester/case_delay.go new file mode 100644 index 000000000..d06d1d65d --- /dev/null +++ b/functional/tester/case_delay.go @@ -0,0 +1,41 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "time" + + "go.uber.org/zap" +) + +type caseDelay struct { + Case + delayDuration time.Duration +} + +func (c *caseDelay) Inject(clus *Cluster) error { + if err := c.Case.Inject(clus); err != nil { + return err + } + if c.delayDuration > 0 { + clus.lg.Info( + "wait after inject", + zap.Duration("delay", c.delayDuration), + zap.String("desc", c.Case.Desc()), + ) + time.Sleep(c.delayDuration) + } + return nil +} diff --git a/functional/tester/case_external.go b/functional/tester/case_external.go new file mode 100644 index 000000000..79d2a3717 --- /dev/null +++ b/functional/tester/case_external.go @@ -0,0 +1,55 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "os/exec" + + "github.com/coreos/etcd/functional/rpcpb" +) + +type caseExternal struct { + Case + + desc string + rpcpbCase rpcpb.Case + + scriptPath string +} + +func (c *caseExternal) Inject(clus *Cluster) error { + return exec.Command(c.scriptPath, "enable", fmt.Sprintf("%d", clus.rd)).Run() +} + +func (c *caseExternal) Recover(clus *Cluster) error { + return exec.Command(c.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run() +} + +func (c *caseExternal) Desc() string { + return c.desc +} + +func (c *caseExternal) TestCase() rpcpb.Case { + return c.rpcpbCase +} + +func new_Case_EXTERNAL(scriptPath string) Case { + return &caseExternal{ + desc: fmt.Sprintf("external fault injector (script: %q)", scriptPath), + rpcpbCase: rpcpb.Case_EXTERNAL, + scriptPath: scriptPath, + } +} diff --git a/functional/tester/case_failpoints.go b/functional/tester/case_failpoints.go new file mode 100644 index 000000000..4d26c8a8d --- /dev/null +++ b/functional/tester/case_failpoints.go @@ -0,0 +1,181 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + "sync" + + "github.com/coreos/etcd/functional/rpcpb" +) + +type failpointStats struct { + mu sync.Mutex + // crashes counts the number of crashes for a failpoint + crashes map[string]int +} + +var fpStats failpointStats + +func failpointFailures(clus *Cluster) (ret []Case, err error) { + var fps []string + fps, err = failpointPaths(clus.Members[0].FailpointHTTPAddr) + if err != nil { + return nil, err + } + // create failure objects for all failpoints + for _, fp := range fps { + if len(fp) == 0 { + continue + } + + fpFails := casesFromFailpoint(fp, clus.Tester.FailpointCommands) + + // wrap in delays so failpoint has time to trigger + for i, fpf := range fpFails { + if strings.Contains(fp, "Snap") { + // hack to trigger snapshot failpoints + fpFails[i] = &caseUntilSnapshot{ + desc: fpf.Desc(), + rpcpbCase: rpcpb.Case_FAILPOINTS, + Case: fpf, + } + } else { + fpFails[i] = &caseDelay{ + Case: fpf, + delayDuration: clus.GetCaseDelayDuration(), + } + } + } + ret = append(ret, fpFails...) + } + fpStats.crashes = make(map[string]int) + return ret, err +} + +func failpointPaths(endpoint string) ([]string, error) { + resp, err := http.Get(endpoint) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, rerr := ioutil.ReadAll(resp.Body) + if rerr != nil { + return nil, rerr + } + var fps []string + for _, l := range strings.Split(string(body), "\n") { + fp := strings.Split(l, "=")[0] + fps = append(fps, fp) + } + return fps, nil +} + +// failpoints follows FreeBSD FAIL_POINT syntax. +// e.g. panic("etcd-tester"),1*sleep(1000)->panic("etcd-tester") +func casesFromFailpoint(fp string, failpointCommands []string) (fs []Case) { + recov := makeRecoverFailpoint(fp) + for _, fcmd := range failpointCommands { + inject := makeInjectFailpoint(fp, fcmd) + fs = append(fs, []Case{ + &caseFollower{ + caseByFunc: caseByFunc{ + desc: fmt.Sprintf("failpoint %q (one: %q)", fp, fcmd), + rpcpbCase: rpcpb.Case_FAILPOINTS, + injectMember: inject, + recoverMember: recov, + }, + last: -1, + lead: -1, + }, + &caseLeader{ + caseByFunc: caseByFunc{ + desc: fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd), + rpcpbCase: rpcpb.Case_FAILPOINTS, + injectMember: inject, + recoverMember: recov, + }, + last: -1, + lead: -1, + }, + &caseQuorum{ + caseByFunc: caseByFunc{ + desc: fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd), + rpcpbCase: rpcpb.Case_FAILPOINTS, + injectMember: inject, + recoverMember: recov, + }, + injected: make(map[int]struct{}), + }, + &caseAll{ + desc: fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd), + rpcpbCase: rpcpb.Case_FAILPOINTS, + injectMember: inject, + recoverMember: recov, + }, + }...) + } + return fs +} + +func makeInjectFailpoint(fp, val string) injectMemberFunc { + return func(clus *Cluster, idx int) (err error) { + return putFailpoint(clus.Members[idx].FailpointHTTPAddr, fp, val) + } +} + +func makeRecoverFailpoint(fp string) recoverMemberFunc { + return func(clus *Cluster, idx int) error { + if err := delFailpoint(clus.Members[idx].FailpointHTTPAddr, fp); err == nil { + return nil + } + // node not responding, likely dead from fp panic; restart + fpStats.mu.Lock() + fpStats.crashes[fp]++ + fpStats.mu.Unlock() + return recover_SIGTERM_ETCD(clus, idx) + } +} + +func putFailpoint(ep, fp, val string) error { + req, _ := http.NewRequest(http.MethodPut, ep+"/"+fp, strings.NewReader(val)) + c := http.Client{} + resp, err := c.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if resp.StatusCode/100 != 2 { + return fmt.Errorf("failed to PUT %s=%s at %s (%v)", fp, val, ep, resp.Status) + } + return nil +} + +func delFailpoint(ep, fp string) error { + req, _ := http.NewRequest(http.MethodDelete, ep+"/"+fp, strings.NewReader("")) + c := http.Client{} + resp, err := c.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if resp.StatusCode/100 != 2 { + return fmt.Errorf("failed to DELETE %s at %s (%v)", fp, ep, resp.Status) + } + return nil +} diff --git a/functional/tester/case_network_blackhole.go b/functional/tester/case_network_blackhole.go new file mode 100644 index 000000000..0d496eade --- /dev/null +++ b/functional/tester/case_network_blackhole.go @@ -0,0 +1,104 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "github.com/coreos/etcd/functional/rpcpb" + +func inject_BLACKHOLE_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_BLACKHOLE_PEER_PORT_TX_RX) +} + +func recover_BLACKHOLE_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_UNBLACKHOLE_PEER_PORT_TX_RX) +} + +func new_Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER(clus *Cluster) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + } + c := &caseFollower{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT() Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + } + c := &caseFollower{cc, -1, -1} + return &caseUntilSnapshot{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, + Case: c, + } +} + +func new_Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER(clus *Cluster) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + } + c := &caseLeader{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT() Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + } + c := &caseLeader{cc, -1, -1} + return &caseUntilSnapshot{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT, + Case: c, + } +} + +func new_Case_BLACKHOLE_PEER_PORT_TX_RX_QUORUM(clus *Cluster) Case { + c := &caseQuorum{ + caseByFunc: caseByFunc{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_QUORUM, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + }, + injected: make(map[int]struct{}), + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_BLACKHOLE_PEER_PORT_TX_RX_ALL(clus *Cluster) Case { + c := &caseAll{ + rpcpbCase: rpcpb.Case_BLACKHOLE_PEER_PORT_TX_RX_ALL, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} diff --git a/functional/tester/case_network_delay.go b/functional/tester/case_network_delay.go new file mode 100644 index 000000000..39a471702 --- /dev/null +++ b/functional/tester/case_network_delay.go @@ -0,0 +1,156 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "time" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" +) + +const ( + // Wait more when it recovers from slow network, because network layer + // needs extra time to propagate traffic control (tc command) change. + // Otherwise, we get different hash values from the previous revision. + // For more detail, please see https://github.com/coreos/etcd/issues/5121. + waitRecover = 5 * time.Second +) + +func inject_DELAY_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + clus.lg.Info( + "injecting delay latency", + zap.Duration("latency", time.Duration(clus.Tester.UpdatedDelayLatencyMs)*time.Millisecond), + zap.Duration("latency-rv", time.Duration(clus.Tester.DelayLatencyMsRv)*time.Millisecond), + zap.String("endpoint", clus.Members[idx].EtcdClientEndpoint), + ) + return clus.sendOp(idx, rpcpb.Operation_DELAY_PEER_PORT_TX_RX) +} + +func recover_DELAY_PEER_PORT_TX_RX(clus *Cluster, idx int) error { + err := clus.sendOp(idx, rpcpb.Operation_UNDELAY_PEER_PORT_TX_RX) + time.Sleep(waitRecover) + return err +} + +func new_Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER(clus *Cluster, random bool) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + } + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + if random { + clus.UpdateDelayLatencyMs() + cc.rpcpbCase = rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER + } + c := &caseFollower{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster, random bool) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + } + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + if random { + clus.UpdateDelayLatencyMs() + cc.rpcpbCase = rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT + } + c := &caseFollower{cc, -1, -1} + return &caseUntilSnapshot{ + rpcpbCase: cc.rpcpbCase, + Case: c, + } +} + +func new_Case_DELAY_PEER_PORT_TX_RX_LEADER(clus *Cluster, random bool) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_DELAY_PEER_PORT_TX_RX_LEADER, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + } + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + if random { + clus.UpdateDelayLatencyMs() + cc.rpcpbCase = rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER + } + c := &caseLeader{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster, random bool) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + } + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + if random { + clus.UpdateDelayLatencyMs() + cc.rpcpbCase = rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT + } + c := &caseLeader{cc, -1, -1} + return &caseUntilSnapshot{ + rpcpbCase: cc.rpcpbCase, + Case: c, + } +} + +func new_Case_DELAY_PEER_PORT_TX_RX_QUORUM(clus *Cluster, random bool) Case { + c := &caseQuorum{ + caseByFunc: caseByFunc{ + rpcpbCase: rpcpb.Case_DELAY_PEER_PORT_TX_RX_QUORUM, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + }, + injected: make(map[int]struct{}), + } + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + if random { + clus.UpdateDelayLatencyMs() + c.rpcpbCase = rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_DELAY_PEER_PORT_TX_RX_ALL(clus *Cluster, random bool) Case { + c := &caseAll{ + rpcpbCase: rpcpb.Case_DELAY_PEER_PORT_TX_RX_ALL, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + } + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + if random { + clus.UpdateDelayLatencyMs() + c.rpcpbCase = rpcpb.Case_RANDOM_DELAY_PEER_PORT_TX_RX_ALL + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} diff --git a/functional/tester/case_no_fail.go b/functional/tester/case_no_fail.go new file mode 100644 index 000000000..e85bef93c --- /dev/null +++ b/functional/tester/case_no_fail.go @@ -0,0 +1,99 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "time" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" +) + +type caseNoFailWithStress caseByFunc + +func (c *caseNoFailWithStress) Inject(clus *Cluster) error { + return nil +} + +func (c *caseNoFailWithStress) Recover(clus *Cluster) error { + return nil +} + +func (c *caseNoFailWithStress) Desc() string { + if c.desc != "" { + return c.desc + } + return c.rpcpbCase.String() +} + +func (c *caseNoFailWithStress) TestCase() rpcpb.Case { + return c.rpcpbCase +} + +func new_Case_NO_FAIL_WITH_STRESS(clus *Cluster) Case { + c := &caseNoFailWithStress{ + rpcpbCase: rpcpb.Case_NO_FAIL_WITH_STRESS, + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +type caseNoFailWithNoStressForLiveness caseByFunc + +func (c *caseNoFailWithNoStressForLiveness) Inject(clus *Cluster) error { + clus.lg.Info( + "extra delay for liveness mode with no stresser", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", c.Desc()), + ) + time.Sleep(clus.GetCaseDelayDuration()) + + clus.lg.Info( + "wait health in liveness mode", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.String("desc", c.Desc()), + ) + return clus.WaitHealth() +} + +func (c *caseNoFailWithNoStressForLiveness) Recover(clus *Cluster) error { + return nil +} + +func (c *caseNoFailWithNoStressForLiveness) Desc() string { + if c.desc != "" { + return c.desc + } + return c.rpcpbCase.String() +} + +func (c *caseNoFailWithNoStressForLiveness) TestCase() rpcpb.Case { + return c.rpcpbCase +} + +func new_Case_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS(clus *Cluster) Case { + c := &caseNoFailWithNoStressForLiveness{ + rpcpbCase: rpcpb.Case_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS, + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} diff --git a/functional/tester/case_sigquit_remove.go b/functional/tester/case_sigquit_remove.go new file mode 100644 index 000000000..13fe68f4e --- /dev/null +++ b/functional/tester/case_sigquit_remove.go @@ -0,0 +1,229 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" +) + +func inject_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error { + cli1, err := clus.Members[idx1].CreateEtcdClient() + if err != nil { + return err + } + defer cli1.Close() + + var mresp *clientv3.MemberListResponse + mresp, err = cli1.MemberList(context.Background()) + mss := []string{} + if err == nil && mresp != nil { + mss = describeMembers(mresp) + } + clus.lg.Info( + "member list before disastrous machine failure", + zap.String("request-to", clus.Members[idx1].EtcdClientEndpoint), + zap.Strings("members", mss), + zap.Error(err), + ) + if err != nil { + return err + } + + sresp, serr := cli1.Status(context.Background(), clus.Members[idx1].EtcdClientEndpoint) + if serr != nil { + return serr + } + id1 := sresp.Header.MemberId + is1 := fmt.Sprintf("%016x", id1) + + clus.lg.Info( + "disastrous machine failure START", + zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), + zap.String("target-member-id", is1), + zap.Error(err), + ) + err = clus.sendOp(idx1, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA) + clus.lg.Info( + "disastrous machine failure END", + zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), + zap.String("target-member-id", is1), + zap.Error(err), + ) + if err != nil { + return err + } + + time.Sleep(2 * time.Second) + + idx2 := (idx1 + 1) % len(clus.Members) + var cli2 *clientv3.Client + cli2, err = clus.Members[idx2].CreateEtcdClient() + if err != nil { + return err + } + defer cli2.Close() + + // FIXME(bug): this may block forever during + // "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT" + // is the new leader too busy with snapshotting? + // is raft proposal dropped? + // enable client keepalive for failover? + clus.lg.Info( + "member remove after disaster START", + zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), + zap.String("target-member-id", is1), + zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint), + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + _, err = cli2.MemberRemove(ctx, id1) + cancel() + clus.lg.Info( + "member remove after disaster END", + zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), + zap.String("target-member-id", is1), + zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint), + zap.Error(err), + ) + if err != nil { + return err + } + + time.Sleep(2 * time.Second) + + mresp, err = cli2.MemberList(context.Background()) + mss = []string{} + if err == nil && mresp != nil { + mss = describeMembers(mresp) + } + clus.lg.Info( + "member list after member remove", + zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint), + zap.Strings("members", mss), + zap.Error(err), + ) + return err +} + +func recover_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error { + idx2 := (idx1 + 1) % len(clus.Members) + cli2, err := clus.Members[idx2].CreateEtcdClient() + if err != nil { + return err + } + defer cli2.Close() + + _, err = cli2.MemberAdd(context.Background(), clus.Members[idx1].Etcd.AdvertisePeerURLs) + clus.lg.Info( + "member add before fresh restart", + zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), + zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint), + zap.Error(err), + ) + if err != nil { + return err + } + + time.Sleep(2 * time.Second) + + clus.Members[idx1].Etcd.InitialClusterState = "existing" + err = clus.sendOp(idx1, rpcpb.Operation_RESTART_ETCD) + clus.lg.Info( + "fresh restart after member add", + zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), + zap.Error(err), + ) + if err != nil { + return err + } + + time.Sleep(2 * time.Second) + + var mresp *clientv3.MemberListResponse + mresp, err = cli2.MemberList(context.Background()) + mss := []string{} + if err == nil && mresp != nil { + mss = describeMembers(mresp) + } + clus.lg.Info( + "member list after member add", + zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint), + zap.Strings("members", mss), + zap.Error(err), + ) + return err +} + +func new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER(clus *Cluster) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER, + injectMember: inject_SIGQUIT_ETCD_AND_REMOVE_DATA, + recoverMember: recover_SIGQUIT_ETCD_AND_REMOVE_DATA, + } + c := &caseFollower{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Case { + return &caseUntilSnapshot{ + rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, + Case: new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER(clus), + } +} + +func new_Case_SIGQUIT_AND_REMOVE_LEADER(clus *Cluster) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_LEADER, + injectMember: inject_SIGQUIT_ETCD_AND_REMOVE_DATA, + recoverMember: recover_SIGQUIT_ETCD_AND_REMOVE_DATA, + } + c := &caseLeader{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Case { + return &caseUntilSnapshot{ + rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT, + Case: new_Case_SIGQUIT_AND_REMOVE_LEADER(clus), + } +} + +func describeMembers(mresp *clientv3.MemberListResponse) (ss []string) { + ss = make([]string, len(mresp.Members)) + for i, m := range mresp.Members { + ss[i] = fmt.Sprintf("Name %s / ID %016x / ClientURLs %s / PeerURLs %s", + m.Name, + m.ID, + strings.Join(m.ClientURLs, ","), + strings.Join(m.PeerURLs, ","), + ) + } + sort.Strings(ss) + return ss +} diff --git a/functional/tester/case_sigquit_remove_quorum.go b/functional/tester/case_sigquit_remove_quorum.go new file mode 100644 index 000000000..9653de10d --- /dev/null +++ b/functional/tester/case_sigquit_remove_quorum.go @@ -0,0 +1,275 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" +) + +type fetchSnapshotCaseQuorum struct { + desc string + rpcpbCase rpcpb.Case + injected map[int]struct{} + snapshotted int +} + +func (c *fetchSnapshotCaseQuorum) Inject(clus *Cluster) error { + // 1. Assume node C is the current leader with most up-to-date data. + lead, err := clus.GetLeader() + if err != nil { + return err + } + c.snapshotted = lead + + // 2. Download snapshot from node C, before destroying node A and B. + clus.lg.Info( + "save snapshot on leader node START", + zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint), + ) + var resp *rpcpb.Response + resp, err = clus.sendOpWithResp(lead, rpcpb.Operation_SAVE_SNAPSHOT) + if resp == nil || (resp != nil && !resp.Success) || err != nil { + clus.lg.Info( + "save snapshot on leader node FAIL", + zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint), + zap.Error(err), + ) + return err + } + clus.lg.Info( + "save snapshot on leader node SUCCESS", + zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint), + zap.String("member-name", resp.SnapshotInfo.MemberName), + zap.Strings("member-client-urls", resp.SnapshotInfo.MemberClientURLs), + zap.String("snapshot-path", resp.SnapshotInfo.SnapshotPath), + zap.String("snapshot-file-size", resp.SnapshotInfo.SnapshotFileSize), + zap.String("snapshot-total-size", resp.SnapshotInfo.SnapshotTotalSize), + zap.Int64("snapshot-total-key", resp.SnapshotInfo.SnapshotTotalKey), + zap.Int64("snapshot-hash", resp.SnapshotInfo.SnapshotHash), + zap.Int64("snapshot-revision", resp.SnapshotInfo.SnapshotRevision), + zap.String("took", resp.SnapshotInfo.Took), + zap.Error(err), + ) + if err != nil { + return err + } + clus.Members[lead].SnapshotInfo = resp.SnapshotInfo + + leaderc, err := clus.Members[lead].CreateEtcdClient() + if err != nil { + return err + } + defer leaderc.Close() + var mresp *clientv3.MemberListResponse + mresp, err = leaderc.MemberList(context.Background()) + mss := []string{} + if err == nil && mresp != nil { + mss = describeMembers(mresp) + } + clus.lg.Info( + "member list before disastrous machine failure", + zap.String("request-to", clus.Members[lead].EtcdClientEndpoint), + zap.Strings("members", mss), + zap.Error(err), + ) + if err != nil { + return err + } + + // simulate real life; machine failures may happen + // after some time since last snapshot save + time.Sleep(time.Second) + + // 3. Destroy node A and B, and make the whole cluster inoperable. + for { + c.injected = pickQuorum(len(clus.Members)) + if _, ok := c.injected[lead]; !ok { + break + } + } + for idx := range c.injected { + clus.lg.Info( + "disastrous machine failure to quorum START", + zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint), + ) + err = clus.sendOp(idx, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA) + clus.lg.Info( + "disastrous machine failure to quorum END", + zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint), + zap.Error(err), + ) + if err != nil { + return err + } + } + + // 4. Now node C cannot operate either. + // 5. SIGTERM node C and remove its data directories. + clus.lg.Info( + "disastrous machine failure to old leader START", + zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint), + ) + err = clus.sendOp(lead, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA) + clus.lg.Info( + "disastrous machine failure to old leader END", + zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint), + zap.Error(err), + ) + return err +} + +func (c *fetchSnapshotCaseQuorum) Recover(clus *Cluster) error { + // 6. Restore a new seed member from node C's latest snapshot file. + oldlead := c.snapshotted + + // configuration on restart from recovered snapshot + // seed member's configuration is all the same as previous one + // except initial cluster string is now a single-node cluster + clus.Members[oldlead].EtcdOnSnapshotRestore = clus.Members[oldlead].Etcd + clus.Members[oldlead].EtcdOnSnapshotRestore.InitialClusterState = "existing" + name := clus.Members[oldlead].Etcd.Name + initClus := []string{} + for _, u := range clus.Members[oldlead].Etcd.AdvertisePeerURLs { + initClus = append(initClus, fmt.Sprintf("%s=%s", name, u)) + } + clus.Members[oldlead].EtcdOnSnapshotRestore.InitialCluster = strings.Join(initClus, ",") + + clus.lg.Info( + "restore snapshot and restart from snapshot request START", + zap.String("target-endpoint", clus.Members[oldlead].EtcdClientEndpoint), + zap.Strings("initial-cluster", initClus), + ) + err := clus.sendOp(oldlead, rpcpb.Operation_RESTORE_RESTART_FROM_SNAPSHOT) + clus.lg.Info( + "restore snapshot and restart from snapshot request END", + zap.String("target-endpoint", clus.Members[oldlead].EtcdClientEndpoint), + zap.Strings("initial-cluster", initClus), + zap.Error(err), + ) + if err != nil { + return err + } + + leaderc, err := clus.Members[oldlead].CreateEtcdClient() + if err != nil { + return err + } + defer leaderc.Close() + + // 7. Add another member to establish 2-node cluster. + // 8. Add another member to establish 3-node cluster. + // 9. Add more if any. + idxs := make([]int, 0, len(c.injected)) + for idx := range c.injected { + idxs = append(idxs, idx) + } + clus.lg.Info("member add START", zap.Int("members-to-add", len(idxs))) + for i, idx := range idxs { + clus.lg.Info( + "member add request SENT", + zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint), + zap.Strings("peer-urls", clus.Members[idx].Etcd.AdvertisePeerURLs), + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + _, err := leaderc.MemberAdd(ctx, clus.Members[idx].Etcd.AdvertisePeerURLs) + cancel() + clus.lg.Info( + "member add request DONE", + zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint), + zap.Strings("peer-urls", clus.Members[idx].Etcd.AdvertisePeerURLs), + zap.Error(err), + ) + if err != nil { + return err + } + + // start the added(new) member with fresh data + clus.Members[idx].EtcdOnSnapshotRestore = clus.Members[idx].Etcd + clus.Members[idx].EtcdOnSnapshotRestore.InitialClusterState = "existing" + name := clus.Members[idx].Etcd.Name + for _, u := range clus.Members[idx].Etcd.AdvertisePeerURLs { + initClus = append(initClus, fmt.Sprintf("%s=%s", name, u)) + } + clus.Members[idx].EtcdOnSnapshotRestore.InitialCluster = strings.Join(initClus, ",") + clus.lg.Info( + "restart from snapshot request SENT", + zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint), + zap.Strings("initial-cluster", initClus), + ) + err = clus.sendOp(idx, rpcpb.Operation_RESTART_FROM_SNAPSHOT) + clus.lg.Info( + "restart from snapshot request DONE", + zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint), + zap.Strings("initial-cluster", initClus), + zap.Error(err), + ) + if err != nil { + return err + } + + if i != len(c.injected)-1 { + // wait until membership reconfiguration entry gets applied + // TODO: test concurrent member add + dur := 5 * clus.Members[idx].ElectionTimeout() + clus.lg.Info( + "waiting after restart from snapshot request", + zap.Int("i", i), + zap.Int("idx", idx), + zap.Duration("sleep", dur), + ) + time.Sleep(dur) + } else { + clus.lg.Info( + "restart from snapshot request ALL END", + zap.Int("i", i), + zap.Int("idx", idx), + ) + } + } + return nil +} + +func (c *fetchSnapshotCaseQuorum) Desc() string { + if c.desc != "" { + return c.desc + } + return c.rpcpbCase.String() +} + +func (c *fetchSnapshotCaseQuorum) TestCase() rpcpb.Case { + return c.rpcpbCase +} + +func new_Case_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH(clus *Cluster) Case { + c := &fetchSnapshotCaseQuorum{ + rpcpbCase: rpcpb.Case_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH, + injected: make(map[int]struct{}), + snapshotted: -1, + } + // simulate real life; machine replacements may happen + // after some time since disaster + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} diff --git a/functional/tester/case_sigterm.go b/functional/tester/case_sigterm.go new file mode 100644 index 000000000..f5d472afc --- /dev/null +++ b/functional/tester/case_sigterm.go @@ -0,0 +1,92 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "github.com/coreos/etcd/functional/rpcpb" + +func inject_SIGTERM_ETCD(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_SIGTERM_ETCD) +} + +func recover_SIGTERM_ETCD(clus *Cluster, idx int) error { + return clus.sendOp(idx, rpcpb.Operation_RESTART_ETCD) +} + +func new_Case_SIGTERM_ONE_FOLLOWER(clus *Cluster) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_SIGTERM_ONE_FOLLOWER, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + c := &caseFollower{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Case { + return &caseUntilSnapshot{ + rpcpbCase: rpcpb.Case_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT, + Case: new_Case_SIGTERM_ONE_FOLLOWER(clus), + } +} + +func new_Case_SIGTERM_LEADER(clus *Cluster) Case { + cc := caseByFunc{ + rpcpbCase: rpcpb.Case_SIGTERM_LEADER, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + c := &caseLeader{cc, -1, -1} + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Case { + return &caseUntilSnapshot{ + rpcpbCase: rpcpb.Case_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT, + Case: new_Case_SIGTERM_LEADER(clus), + } +} + +func new_Case_SIGTERM_QUORUM(clus *Cluster) Case { + c := &caseQuorum{ + caseByFunc: caseByFunc{ + rpcpbCase: rpcpb.Case_SIGTERM_QUORUM, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + }, + injected: make(map[int]struct{}), + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} + +func new_Case_SIGTERM_ALL(clus *Cluster) Case { + c := &caseAll{ + rpcpbCase: rpcpb.Case_SIGTERM_ALL, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + } + return &caseDelay{ + Case: c, + delayDuration: clus.GetCaseDelayDuration(), + } +} diff --git a/functional/tester/checker.go b/functional/tester/checker.go new file mode 100644 index 000000000..48e98cb0d --- /dev/null +++ b/functional/tester/checker.go @@ -0,0 +1,28 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "github.com/coreos/etcd/functional/rpcpb" + +// Checker checks cluster consistency. +type Checker interface { + // Type returns the checker type. + Type() rpcpb.Checker + // EtcdClientEndpoints returns the client endpoints of + // all checker target nodes.. + EtcdClientEndpoints() []string + // Check returns an error if the system fails a consistency check. + Check() error +} diff --git a/functional/tester/checker_kv_hash.go b/functional/tester/checker_kv_hash.go new file mode 100644 index 000000000..586ad89bd --- /dev/null +++ b/functional/tester/checker_kv_hash.go @@ -0,0 +1,89 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "time" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" +) + +const retries = 7 + +type kvHashChecker struct { + ctype rpcpb.Checker + clus *Cluster +} + +func newKVHashChecker(clus *Cluster) Checker { + return &kvHashChecker{ + ctype: rpcpb.Checker_KV_HASH, + clus: clus, + } +} + +func (hc *kvHashChecker) checkRevAndHashes() (err error) { + var ( + revs map[string]int64 + hashes map[string]int64 + ) + // retries in case of transient failure or etcd cluster has not stablized yet. + for i := 0; i < retries; i++ { + revs, hashes, err = hc.clus.getRevisionHash() + if err != nil { + hc.clus.lg.Warn( + "failed to get revision and hash", + zap.Int("retries", i), + zap.Error(err), + ) + } else { + sameRev := getSameValue(revs) + sameHashes := getSameValue(hashes) + if sameRev && sameHashes { + return nil + } + hc.clus.lg.Warn( + "retrying; etcd cluster is not stable", + zap.Int("retries", i), + zap.Bool("same-revisions", sameRev), + zap.Bool("same-hashes", sameHashes), + zap.String("revisions", fmt.Sprintf("%+v", revs)), + zap.String("hashes", fmt.Sprintf("%+v", hashes)), + ) + } + time.Sleep(time.Second) + } + + if err != nil { + return fmt.Errorf("failed revision and hash check (%v)", err) + } + + return fmt.Errorf("etcd cluster is not stable: [revisions: %v] and [hashes: %v]", revs, hashes) +} + +func (hc *kvHashChecker) Type() rpcpb.Checker { + return hc.ctype +} + +func (hc *kvHashChecker) EtcdClientEndpoints() []string { + return hc.clus.EtcdClientEndpoints() +} + +func (hc *kvHashChecker) Check() error { + return hc.checkRevAndHashes() +} diff --git a/functional/tester/checker_lease_expire.go b/functional/tester/checker_lease_expire.go new file mode 100644 index 000000000..a89742128 --- /dev/null +++ b/functional/tester/checker_lease_expire.go @@ -0,0 +1,238 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "fmt" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type leaseExpireChecker struct { + ctype rpcpb.Checker + lg *zap.Logger + m *rpcpb.Member + ls *leaseStresser + cli *clientv3.Client +} + +func newLeaseExpireChecker(ls *leaseStresser) Checker { + return &leaseExpireChecker{ + ctype: rpcpb.Checker_LEASE_EXPIRE, + lg: ls.lg, + m: ls.m, + ls: ls, + } +} + +func (lc *leaseExpireChecker) Type() rpcpb.Checker { + return lc.ctype +} + +func (lc *leaseExpireChecker) EtcdClientEndpoints() []string { + return []string{lc.m.EtcdClientEndpoint} +} + +func (lc *leaseExpireChecker) Check() error { + if lc.ls == nil { + return nil + } + if lc.ls != nil && + (lc.ls.revokedLeases == nil || + lc.ls.aliveLeases == nil || + lc.ls.shortLivedLeases == nil) { + return nil + } + + cli, err := lc.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(time.Second)) + if err != nil { + return fmt.Errorf("%v (%q)", err, lc.m.EtcdClientEndpoint) + } + defer func() { + if cli != nil { + cli.Close() + } + }() + lc.cli = cli + + if err := lc.check(true, lc.ls.revokedLeases.leases); err != nil { + return err + } + if err := lc.check(false, lc.ls.aliveLeases.leases); err != nil { + return err + } + return lc.checkShortLivedLeases() +} + +const leaseExpireCheckerTimeout = 10 * time.Second + +// checkShortLivedLeases ensures leases expire. +func (lc *leaseExpireChecker) checkShortLivedLeases() error { + ctx, cancel := context.WithTimeout(context.Background(), leaseExpireCheckerTimeout) + errc := make(chan error) + defer cancel() + for leaseID := range lc.ls.shortLivedLeases.leases { + go func(id int64) { + errc <- lc.checkShortLivedLease(ctx, id) + }(leaseID) + } + + var errs []error + for range lc.ls.shortLivedLeases.leases { + if err := <-errc; err != nil { + errs = append(errs, err) + } + } + return errsToError(errs) +} + +func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID int64) (err error) { + // retry in case of transient failure or lease is expired but not yet revoked due to the fact that etcd cluster didn't have enought time to delete it. + var resp *clientv3.LeaseTimeToLiveResponse + for i := 0; i < retries; i++ { + resp, err = lc.getLeaseByID(ctx, leaseID) + // lease not found, for ~v3.1 compatibilities, check ErrLeaseNotFound + if (err == nil && resp.TTL == -1) || (err != nil && rpctypes.Error(err) == rpctypes.ErrLeaseNotFound) { + return nil + } + if err != nil { + lc.lg.Debug( + "retrying; Lease TimeToLive failed", + zap.Int("retries", i), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(err), + ) + continue + } + if resp.TTL > 0 { + dur := time.Duration(resp.TTL) * time.Second + lc.lg.Debug( + "lease has not been expired, wait until expire", + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Int64("ttl", resp.TTL), + zap.Duration("wait-duration", dur), + ) + time.Sleep(dur) + } else { + lc.lg.Debug( + "lease expired but not yet revoked", + zap.Int("retries", i), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Int64("ttl", resp.TTL), + zap.Duration("wait-duration", time.Second), + ) + time.Sleep(time.Second) + } + if err = lc.checkLease(ctx, false, leaseID); err != nil { + continue + } + return nil + } + return err +} + +func (lc *leaseExpireChecker) checkLease(ctx context.Context, expired bool, leaseID int64) error { + keysExpired, err := lc.hasKeysAttachedToLeaseExpired(ctx, leaseID) + if err != nil { + lc.lg.Warn( + "hasKeysAttachedToLeaseExpired failed", + zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.Error(err), + ) + return err + } + leaseExpired, err := lc.hasLeaseExpired(ctx, leaseID) + if err != nil { + lc.lg.Warn( + "hasLeaseExpired failed", + zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.Error(err), + ) + return err + } + if leaseExpired != keysExpired { + return fmt.Errorf("lease %v expiration mismatch (lease expired=%v, keys expired=%v)", leaseID, leaseExpired, keysExpired) + } + if leaseExpired != expired { + return fmt.Errorf("lease %v expected expired=%v, got %v", leaseID, expired, leaseExpired) + } + return nil +} + +func (lc *leaseExpireChecker) check(expired bool, leases map[int64]time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), leaseExpireCheckerTimeout) + defer cancel() + for leaseID := range leases { + if err := lc.checkLease(ctx, expired, leaseID); err != nil { + return err + } + } + return nil +} + +// TODO: handle failures from "grpc.FailFast(false)" +func (lc *leaseExpireChecker) getLeaseByID(ctx context.Context, leaseID int64) (*clientv3.LeaseTimeToLiveResponse, error) { + return lc.cli.TimeToLive( + ctx, + clientv3.LeaseID(leaseID), + clientv3.WithAttachedKeys(), + ) +} + +func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64) (bool, error) { + // keep retrying until lease's state is known or ctx is being canceled + for ctx.Err() == nil { + resp, err := lc.getLeaseByID(ctx, leaseID) + if err != nil { + // for ~v3.1 compatibilities + if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound { + return true, nil + } + } else { + return resp.TTL == -1, nil + } + lc.lg.Warn( + "hasLeaseExpired getLeaseByID failed", + zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(err), + ) + } + return false, ctx.Err() +} + +// The keys attached to the lease has the format of "_" where idx is the ordering key creation +// Since the format of keys contains about leaseID, finding keys base on "" prefix +// determines whether the attached keys for a given leaseID has been deleted or not +func (lc *leaseExpireChecker) hasKeysAttachedToLeaseExpired(ctx context.Context, leaseID int64) (bool, error) { + resp, err := lc.cli.Get(ctx, fmt.Sprintf("%d", leaseID), clientv3.WithPrefix()) + if err != nil { + lc.lg.Warn( + "hasKeysAttachedToLeaseExpired failed", + zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(err), + ) + return false, err + } + return len(resp.Kvs) == 0, nil +} diff --git a/functional/tester/checker_no_check.go b/functional/tester/checker_no_check.go new file mode 100644 index 000000000..d36702319 --- /dev/null +++ b/functional/tester/checker_no_check.go @@ -0,0 +1,24 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "github.com/coreos/etcd/functional/rpcpb" + +type noCheck struct{} + +func newNoChecker() Checker { return &noCheck{} } +func (nc *noCheck) Type() rpcpb.Checker { return rpcpb.Checker_NO_CHECK } +func (nc *noCheck) EtcdClientEndpoints() []string { return nil } +func (nc *noCheck) Check() error { return nil } diff --git a/functional/tester/checker_runner.go b/functional/tester/checker_runner.go new file mode 100644 index 000000000..a5b7ff4d1 --- /dev/null +++ b/functional/tester/checker_runner.go @@ -0,0 +1,48 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "github.com/coreos/etcd/functional/rpcpb" + +type runnerChecker struct { + ctype rpcpb.Checker + etcdClientEndpoint string + errc chan error +} + +func newRunnerChecker(ep string, errc chan error) Checker { + return &runnerChecker{ + ctype: rpcpb.Checker_RUNNER, + etcdClientEndpoint: ep, + errc: errc, + } +} + +func (rc *runnerChecker) Type() rpcpb.Checker { + return rc.ctype +} + +func (rc *runnerChecker) EtcdClientEndpoints() []string { + return []string{rc.etcdClientEndpoint} +} + +func (rc *runnerChecker) Check() error { + select { + case err := <-rc.errc: + return err + default: + return nil + } +} diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go new file mode 100644 index 000000000..b18084d48 --- /dev/null +++ b/functional/tester/cluster.go @@ -0,0 +1,761 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/coreos/etcd/functional/rpcpb" + "github.com/coreos/etcd/pkg/debugutil" + "github.com/coreos/etcd/pkg/fileutil" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" + "golang.org/x/time/rate" + "google.golang.org/grpc" +) + +// Cluster defines tester cluster. +type Cluster struct { + lg *zap.Logger + + agentConns []*grpc.ClientConn + agentClients []rpcpb.TransportClient + agentStreams []rpcpb.Transport_TransportClient + agentRequests []*rpcpb.Request + + testerHTTPServer *http.Server + + Members []*rpcpb.Member `yaml:"agent-configs"` + Tester *rpcpb.Tester `yaml:"tester-config"` + + cases []Case + + rateLimiter *rate.Limiter + stresser Stresser + checkers []Checker + + currentRevision int64 + rd int + cs int +} + +var dialOpts = []grpc.DialOption{ + grpc.WithInsecure(), + grpc.WithTimeout(5 * time.Second), + grpc.WithBlock(), +} + +// NewCluster creates a client from a tester configuration. +func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) { + clus, err := read(lg, fpath) + if err != nil { + return nil, err + } + + clus.agentConns = make([]*grpc.ClientConn, len(clus.Members)) + clus.agentClients = make([]rpcpb.TransportClient, len(clus.Members)) + clus.agentStreams = make([]rpcpb.Transport_TransportClient, len(clus.Members)) + clus.agentRequests = make([]*rpcpb.Request, len(clus.Members)) + clus.cases = make([]Case, 0) + + for i, ap := range clus.Members { + var err error + clus.agentConns[i], err = grpc.Dial(ap.AgentAddr, dialOpts...) + if err != nil { + return nil, err + } + clus.agentClients[i] = rpcpb.NewTransportClient(clus.agentConns[i]) + clus.lg.Info("connected", zap.String("agent-address", ap.AgentAddr)) + + clus.agentStreams[i], err = clus.agentClients[i].Transport(context.Background()) + if err != nil { + return nil, err + } + clus.lg.Info("created stream", zap.String("agent-address", ap.AgentAddr)) + } + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + if clus.Tester.EnablePprof { + for p, h := range debugutil.PProfHandlers() { + mux.Handle(p, h) + } + } + clus.testerHTTPServer = &http.Server{ + Addr: clus.Tester.Addr, + Handler: mux, + } + go clus.serveTesterServer() + + clus.updateCases() + + clus.rateLimiter = rate.NewLimiter( + rate.Limit(int(clus.Tester.StressQPS)), + int(clus.Tester.StressQPS), + ) + + clus.setStresserChecker() + + return clus, nil +} + +// EtcdClientEndpoints returns all etcd client endpoints. +func (clus *Cluster) EtcdClientEndpoints() (css []string) { + css = make([]string, len(clus.Members)) + for i := range clus.Members { + css[i] = clus.Members[i].EtcdClientEndpoint + } + return css +} + +func (clus *Cluster) serveTesterServer() { + clus.lg.Info( + "started tester HTTP server", + zap.String("tester-address", clus.Tester.Addr), + ) + err := clus.testerHTTPServer.ListenAndServe() + clus.lg.Info( + "tester HTTP server returned", + zap.String("tester-address", clus.Tester.Addr), + zap.Error(err), + ) + if err != nil && err != http.ErrServerClosed { + clus.lg.Fatal("tester HTTP errored", zap.Error(err)) + } +} + +func (clus *Cluster) updateCases() { + for _, cs := range clus.Tester.Cases { + switch cs { + case "SIGTERM_ONE_FOLLOWER": + clus.cases = append(clus.cases, + new_Case_SIGTERM_ONE_FOLLOWER(clus)) + case "SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_SIGTERM_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus)) + case "SIGTERM_LEADER": + clus.cases = append(clus.cases, + new_Case_SIGTERM_LEADER(clus)) + case "SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus)) + case "SIGTERM_QUORUM": + clus.cases = append(clus.cases, + new_Case_SIGTERM_QUORUM(clus)) + case "SIGTERM_ALL": + clus.cases = append(clus.cases, + new_Case_SIGTERM_ALL(clus)) + + case "SIGQUIT_AND_REMOVE_ONE_FOLLOWER": + clus.cases = append(clus.cases, + new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER(clus)) + case "SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_SIGQUIT_AND_REMOVE_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus)) + case "SIGQUIT_AND_REMOVE_LEADER": + clus.cases = append(clus.cases, + new_Case_SIGQUIT_AND_REMOVE_LEADER(clus)) + case "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus)) + case "SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH": + clus.cases = append(clus.cases, + new_Case_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH(clus)) + + case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": + clus.cases = append(clus.cases, + new_Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER(clus)) + case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT()) + case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": + clus.cases = append(clus.cases, + new_Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER(clus)) + case "BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT()) + case "BLACKHOLE_PEER_PORT_TX_RX_QUORUM": + clus.cases = append(clus.cases, + new_Case_BLACKHOLE_PEER_PORT_TX_RX_QUORUM(clus)) + case "BLACKHOLE_PEER_PORT_TX_RX_ALL": + clus.cases = append(clus.cases, + new_Case_BLACKHOLE_PEER_PORT_TX_RX_ALL(clus)) + + case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER(clus, false)) + case "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER(clus, true)) + case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus, false)) + case "RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT(clus, true)) + case "DELAY_PEER_PORT_TX_RX_LEADER": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_LEADER(clus, false)) + case "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_LEADER(clus, true)) + case "DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus, false)) + case "RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus, true)) + case "DELAY_PEER_PORT_TX_RX_QUORUM": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_QUORUM(clus, false)) + case "RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_QUORUM(clus, true)) + case "DELAY_PEER_PORT_TX_RX_ALL": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_ALL(clus, false)) + case "RANDOM_DELAY_PEER_PORT_TX_RX_ALL": + clus.cases = append(clus.cases, + new_Case_DELAY_PEER_PORT_TX_RX_ALL(clus, true)) + + case "NO_FAIL_WITH_STRESS": + clus.cases = append(clus.cases, + new_Case_NO_FAIL_WITH_STRESS(clus)) + case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": + clus.cases = append(clus.cases, + new_Case_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS(clus)) + + case "EXTERNAL": + clus.cases = append(clus.cases, + new_Case_EXTERNAL(clus.Tester.ExternalExecPath)) + case "FAILPOINTS": + fpFailures, fperr := failpointFailures(clus) + if len(fpFailures) == 0 { + clus.lg.Info("no failpoints found!", zap.Error(fperr)) + } + clus.cases = append(clus.cases, + fpFailures...) + } + } +} + +func (clus *Cluster) listCases() (css []string) { + css = make([]string, len(clus.cases)) + for i := range clus.cases { + css[i] = clus.cases[i].Desc() + } + return css +} + +// UpdateDelayLatencyMs updates delay latency with random value +// within election timeout. +func (clus *Cluster) UpdateDelayLatencyMs() { + rand.Seed(time.Now().UnixNano()) + clus.Tester.UpdatedDelayLatencyMs = uint32(rand.Int63n(clus.Members[0].Etcd.ElectionTimeoutMs)) + + minLatRv := clus.Tester.DelayLatencyMsRv + clus.Tester.DelayLatencyMsRv/5 + if clus.Tester.UpdatedDelayLatencyMs <= minLatRv { + clus.Tester.UpdatedDelayLatencyMs += minLatRv + } +} + +func (clus *Cluster) setStresserChecker() { + css := &compositeStresser{} + lss := []*leaseStresser{} + rss := []*runnerStresser{} + for _, m := range clus.Members { + sss := newStresser(clus, m) + css.stressers = append(css.stressers, &compositeStresser{sss}) + for _, s := range sss { + if v, ok := s.(*leaseStresser); ok { + lss = append(lss, v) + clus.lg.Info("added lease stresser", zap.String("endpoint", m.EtcdClientEndpoint)) + } + if v, ok := s.(*runnerStresser); ok { + rss = append(rss, v) + clus.lg.Info("added lease stresser", zap.String("endpoint", m.EtcdClientEndpoint)) + } + } + } + clus.stresser = css + + for _, cs := range clus.Tester.Checkers { + switch cs { + case "KV_HASH": + clus.checkers = append(clus.checkers, newKVHashChecker(clus)) + + case "LEASE_EXPIRE": + for _, ls := range lss { + clus.checkers = append(clus.checkers, newLeaseExpireChecker(ls)) + } + + case "RUNNER": + for _, rs := range rss { + clus.checkers = append(clus.checkers, newRunnerChecker(rs.etcdClientEndpoint, rs.errc)) + } + + case "NO_CHECK": + clus.checkers = append(clus.checkers, newNoChecker()) + } + } + clus.lg.Info("updated stressers") +} + +func (clus *Cluster) runCheckers(exceptions ...rpcpb.Checker) (err error) { + defer func() { + if err != nil { + return + } + if err = clus.updateRevision(); err != nil { + clus.lg.Warn( + "updateRevision failed", + zap.Error(err), + ) + return + } + }() + + exs := make(map[rpcpb.Checker]struct{}) + for _, e := range exceptions { + exs[e] = struct{}{} + } + for _, chk := range clus.checkers { + clus.lg.Warn( + "consistency check START", + zap.String("checker", chk.Type().String()), + zap.Strings("client-endpoints", chk.EtcdClientEndpoints()), + ) + err = chk.Check() + clus.lg.Warn( + "consistency check END", + zap.String("checker", chk.Type().String()), + zap.Strings("client-endpoints", chk.EtcdClientEndpoints()), + zap.Error(err), + ) + if err != nil { + _, ok := exs[chk.Type()] + if !ok { + return err + } + clus.lg.Warn( + "consistency check SKIP FAIL", + zap.String("checker", chk.Type().String()), + zap.Strings("client-endpoints", chk.EtcdClientEndpoints()), + zap.Error(err), + ) + } + } + return nil +} + +// Send_INITIAL_START_ETCD bootstraps etcd cluster the very first time. +// After this, just continue to call kill/restart. +func (clus *Cluster) Send_INITIAL_START_ETCD() error { + // this is the only time that creates request from scratch + return clus.broadcast(rpcpb.Operation_INITIAL_START_ETCD) +} + +// send_SIGQUIT_ETCD_AND_ARCHIVE_DATA sends "send_SIGQUIT_ETCD_AND_ARCHIVE_DATA" operation. +func (clus *Cluster) send_SIGQUIT_ETCD_AND_ARCHIVE_DATA() error { + return clus.broadcast(rpcpb.Operation_SIGQUIT_ETCD_AND_ARCHIVE_DATA) +} + +// send_RESTART_ETCD sends restart operation. +func (clus *Cluster) send_RESTART_ETCD() error { + return clus.broadcast(rpcpb.Operation_RESTART_ETCD) +} + +func (clus *Cluster) broadcast(op rpcpb.Operation) error { + var wg sync.WaitGroup + wg.Add(len(clus.agentStreams)) + + errc := make(chan error, len(clus.agentStreams)) + for i := range clus.agentStreams { + go func(idx int, o rpcpb.Operation) { + defer wg.Done() + errc <- clus.sendOp(idx, o) + }(i, op) + } + wg.Wait() + close(errc) + + errs := []string{} + for err := range errc { + if err == nil { + continue + } + + if err != nil { + destroyed := false + if op == rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT { + if err == io.EOF { + destroyed = true + } + if strings.Contains(err.Error(), + "rpc error: code = Unavailable desc = transport is closing") { + // agent server has already closed; + // so this error is expected + destroyed = true + } + if strings.Contains(err.Error(), + "desc = os: process already finished") { + destroyed = true + } + } + if !destroyed { + errs = append(errs, err.Error()) + } + } + } + + if len(errs) == 0 { + return nil + } + return errors.New(strings.Join(errs, ", ")) +} + +func (clus *Cluster) sendOp(idx int, op rpcpb.Operation) error { + _, err := clus.sendOpWithResp(idx, op) + return err +} + +func (clus *Cluster) sendOpWithResp(idx int, op rpcpb.Operation) (*rpcpb.Response, error) { + // maintain the initial member object + // throughout the test time + clus.agentRequests[idx] = &rpcpb.Request{ + Operation: op, + Member: clus.Members[idx], + Tester: clus.Tester, + } + + err := clus.agentStreams[idx].Send(clus.agentRequests[idx]) + clus.lg.Info( + "sent request", + zap.String("operation", op.String()), + zap.String("to", clus.Members[idx].EtcdClientEndpoint), + zap.Error(err), + ) + if err != nil { + return nil, err + } + + resp, err := clus.agentStreams[idx].Recv() + if resp != nil { + clus.lg.Info( + "received response", + zap.String("operation", op.String()), + zap.String("from", clus.Members[idx].EtcdClientEndpoint), + zap.Bool("success", resp.Success), + zap.String("status", resp.Status), + zap.Error(err), + ) + } else { + clus.lg.Info( + "received empty response", + zap.String("operation", op.String()), + zap.String("from", clus.Members[idx].EtcdClientEndpoint), + zap.Error(err), + ) + } + if err != nil { + return nil, err + } + + if !resp.Success { + return nil, errors.New(resp.Status) + } + + m, secure := clus.Members[idx], false + for _, cu := range m.Etcd.AdvertiseClientURLs { + u, err := url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme == "https" { // TODO: handle unix + secure = true + } + } + + // store TLS assets from agents/servers onto disk + if secure && (op == rpcpb.Operation_INITIAL_START_ETCD || op == rpcpb.Operation_RESTART_ETCD) { + dirClient := filepath.Join( + clus.Tester.DataDir, + clus.Members[idx].Etcd.Name, + "fixtures", + "client", + ) + if err = fileutil.TouchDirAll(dirClient); err != nil { + return nil, err + } + + clientCertData := []byte(resp.Member.ClientCertData) + if len(clientCertData) == 0 { + return nil, fmt.Errorf("got empty client cert from %q", m.EtcdClientEndpoint) + } + clientCertPath := filepath.Join(dirClient, "cert.pem") + if err = ioutil.WriteFile(clientCertPath, clientCertData, 0644); err != nil { // overwrite if exists + return nil, err + } + resp.Member.ClientCertPath = clientCertPath + clus.lg.Info( + "saved client cert file", + zap.String("path", clientCertPath), + ) + + clientKeyData := []byte(resp.Member.ClientKeyData) + if len(clientKeyData) == 0 { + return nil, fmt.Errorf("got empty client key from %q", m.EtcdClientEndpoint) + } + clientKeyPath := filepath.Join(dirClient, "key.pem") + if err = ioutil.WriteFile(clientKeyPath, clientKeyData, 0644); err != nil { // overwrite if exists + return nil, err + } + resp.Member.ClientKeyPath = clientKeyPath + clus.lg.Info( + "saved client key file", + zap.String("path", clientKeyPath), + ) + + clientTrustedCAData := []byte(resp.Member.ClientTrustedCAData) + if len(clientTrustedCAData) != 0 { + // TODO: disable this when auto TLS is deprecated + clientTrustedCAPath := filepath.Join(dirClient, "ca.pem") + if err = ioutil.WriteFile(clientTrustedCAPath, clientTrustedCAData, 0644); err != nil { // overwrite if exists + return nil, err + } + resp.Member.ClientTrustedCAPath = clientTrustedCAPath + clus.lg.Info( + "saved client trusted CA file", + zap.String("path", clientTrustedCAPath), + ) + } + + // no need to store peer certs for tester clients + + clus.Members[idx] = resp.Member + } + + return resp, nil +} + +// Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT terminates all tester connections to agents and etcd servers. +func (clus *Cluster) Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() { + err := clus.broadcast(rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT) + if err != nil { + clus.lg.Warn("destroying etcd/agents FAIL", zap.Error(err)) + } else { + clus.lg.Info("destroying etcd/agents PASS") + } + + for i, conn := range clus.agentConns { + err := conn.Close() + clus.lg.Info("closed connection to agent", zap.String("agent-address", clus.Members[i].AgentAddr), zap.Error(err)) + } + + if clus.testerHTTPServer != nil { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + err := clus.testerHTTPServer.Shutdown(ctx) + cancel() + clus.lg.Info("closed tester HTTP server", zap.String("tester-address", clus.Tester.Addr), zap.Error(err)) + } +} + +// WaitHealth ensures all members are healthy +// by writing a test key to etcd cluster. +func (clus *Cluster) WaitHealth() error { + var err error + // wait 60s to check cluster health. + // TODO: set it to a reasonable value. It is set that high because + // follower may use long time to catch up the leader when reboot under + // reasonable workload (https://github.com/coreos/etcd/issues/2698) + for i := 0; i < 60; i++ { + for _, m := range clus.Members { + if err = m.WriteHealthKey(); err != nil { + clus.lg.Warn( + "health check FAIL", + zap.Int("retries", i), + zap.String("endpoint", m.EtcdClientEndpoint), + zap.Error(err), + ) + break + } + clus.lg.Info( + "health check PASS", + zap.Int("retries", i), + zap.String("endpoint", m.EtcdClientEndpoint), + ) + } + if err == nil { + clus.lg.Info("health check ALL PASS") + return nil + } + time.Sleep(time.Second) + } + return err +} + +// GetLeader returns the index of leader and error if any. +func (clus *Cluster) GetLeader() (int, error) { + for i, m := range clus.Members { + isLeader, err := m.IsLeader() + if isLeader || err != nil { + return i, err + } + } + return 0, fmt.Errorf("no leader found") +} + +// maxRev returns the maximum revision found on the cluster. +func (clus *Cluster) maxRev() (rev int64, err error) { + ctx, cancel := context.WithTimeout(context.TODO(), time.Second) + defer cancel() + revc, errc := make(chan int64, len(clus.Members)), make(chan error, len(clus.Members)) + for i := range clus.Members { + go func(m *rpcpb.Member) { + mrev, merr := m.Rev(ctx) + revc <- mrev + errc <- merr + }(clus.Members[i]) + } + for i := 0; i < len(clus.Members); i++ { + if merr := <-errc; merr != nil { + err = merr + } + if mrev := <-revc; mrev > rev { + rev = mrev + } + } + return rev, err +} + +func (clus *Cluster) getRevisionHash() (map[string]int64, map[string]int64, error) { + revs := make(map[string]int64) + hashes := make(map[string]int64) + for _, m := range clus.Members { + rev, hash, err := m.RevHash() + if err != nil { + return nil, nil, err + } + revs[m.EtcdClientEndpoint] = rev + hashes[m.EtcdClientEndpoint] = hash + } + return revs, hashes, nil +} + +func (clus *Cluster) compactKV(rev int64, timeout time.Duration) (err error) { + if rev <= 0 { + return nil + } + + for i, m := range clus.Members { + clus.lg.Info( + "compact START", + zap.String("endpoint", m.EtcdClientEndpoint), + zap.Int64("compact-revision", rev), + zap.Duration("timeout", timeout), + ) + now := time.Now() + cerr := m.Compact(rev, timeout) + succeed := true + if cerr != nil { + if strings.Contains(cerr.Error(), "required revision has been compacted") && i > 0 { + clus.lg.Info( + "compact error is ignored", + zap.String("endpoint", m.EtcdClientEndpoint), + zap.Int64("compact-revision", rev), + zap.Error(cerr), + ) + } else { + clus.lg.Warn( + "compact FAIL", + zap.String("endpoint", m.EtcdClientEndpoint), + zap.Int64("compact-revision", rev), + zap.Error(cerr), + ) + err = cerr + succeed = false + } + } + + if succeed { + clus.lg.Info( + "compact PASS", + zap.String("endpoint", m.EtcdClientEndpoint), + zap.Int64("compact-revision", rev), + zap.Duration("timeout", timeout), + zap.Duration("took", time.Since(now)), + ) + } + } + return err +} + +func (clus *Cluster) checkCompact(rev int64) error { + if rev == 0 { + return nil + } + for _, m := range clus.Members { + if err := m.CheckCompact(rev); err != nil { + return err + } + } + return nil +} + +func (clus *Cluster) defrag() error { + for _, m := range clus.Members { + if err := m.Defrag(); err != nil { + clus.lg.Warn( + "defrag FAIL", + zap.String("endpoint", m.EtcdClientEndpoint), + zap.Error(err), + ) + return err + } + clus.lg.Info( + "defrag PASS", + zap.String("endpoint", m.EtcdClientEndpoint), + ) + } + clus.lg.Info( + "defrag ALL PASS", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + ) + return nil +} + +// GetCaseDelayDuration computes failure delay duration. +func (clus *Cluster) GetCaseDelayDuration() time.Duration { + return time.Duration(clus.Tester.CaseDelayMs) * time.Millisecond +} + +// Report reports the number of modified keys. +func (clus *Cluster) Report() int64 { + return clus.stresser.ModifiedKeys() +} diff --git a/functional/tester/cluster_read_config.go b/functional/tester/cluster_read_config.go new file mode 100644 index 000000000..223265e66 --- /dev/null +++ b/functional/tester/cluster_read_config.go @@ -0,0 +1,358 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "errors" + "fmt" + "io/ioutil" + "net/url" + "path/filepath" + "strings" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" + yaml "gopkg.in/yaml.v2" +) + +func read(lg *zap.Logger, fpath string) (*Cluster, error) { + bts, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + lg.Info("opened configuration file", zap.String("path", fpath)) + + clus := &Cluster{lg: lg} + if err = yaml.Unmarshal(bts, clus); err != nil { + return nil, err + } + + if len(clus.Members) < 3 { + return nil, fmt.Errorf("len(clus.Members) expects at least 3, got %d", len(clus.Members)) + } + + for i, mem := range clus.Members { + if mem.BaseDir == "" { + return nil, fmt.Errorf("BaseDir cannot be empty (got %q)", mem.BaseDir) + } + if mem.EtcdLogPath == "" { + return nil, fmt.Errorf("EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath) + } + + if mem.Etcd.Name == "" { + return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem) + } + if mem.Etcd.DataDir == "" { + return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem) + } + if mem.Etcd.SnapshotCount == 0 { + return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount) + } + if mem.Etcd.DataDir == "" { + return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir) + } + if mem.Etcd.WALDir == "" { + clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal") + } + + switch mem.Etcd.InitialClusterState { + case "new": + case "existing": + default: + return nil, fmt.Errorf("'--initial-cluster-state' got %q", mem.Etcd.InitialClusterState) + } + + if mem.Etcd.HeartbeatIntervalMs == 0 { + return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd) + } + if mem.Etcd.ElectionTimeoutMs == 0 { + return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd) + } + if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs { + return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs) + } + + port := "" + listenClientPorts := make([]string, len(clus.Members)) + for i, u := range mem.Etcd.ListenClientURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u) + } + listenClientPorts[i], err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u) + } + } + for i, u := range mem.Etcd.AdvertiseClientURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u) + } + port, err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u) + } + if mem.EtcdClientProxy && listenClientPorts[i] == port { + return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i]) + } + } + + listenPeerPorts := make([]string, len(clus.Members)) + for i, u := range mem.Etcd.ListenPeerURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u) + } + listenPeerPorts[i], err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u) + } + } + for j, u := range mem.Etcd.AdvertisePeerURLs { + if !isValidURL(u) { + return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u) + } + port, err = getPort(u) + if err != nil { + return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u) + } + if mem.EtcdPeerProxy && listenPeerPorts[j] == port { + return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j]) + } + } + + if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) { + return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath) + } + if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) { + return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir) + } + + // TODO: support separate WALDir that can be handled via failure-archive + if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) { + return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir) + } + + // TODO: only support generated certs with TLS generator + // deprecate auto TLS + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" { + return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" { + return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile) + } + if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile) + } + if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) + } + + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerKeyFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerCertFile) + } + // only support self-signed certs + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerTrustedCAFile == "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile != "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerKeyFile != "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'false', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile) + } + if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerAutoTLS { + return nil, fmt.Errorf("Etcd.PeerClientCertAuth and Etcd.PeerAutoTLS cannot be both 'true'") + } + if (mem.Etcd.PeerCertFile == "") != (mem.Etcd.PeerKeyFile == "") { + return nil, fmt.Errorf("Both Etcd.PeerCertFile %q and Etcd.PeerKeyFile %q must be either empty or non-empty", mem.Etcd.PeerCertFile, mem.Etcd.PeerKeyFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientAutoTLS { + return nil, fmt.Errorf("Etcd.ClientCertAuth and Etcd.ClientAutoTLS cannot be both 'true'") + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientKeyFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.PeerCertFile) + } + if mem.Etcd.ClientCertAuth && mem.Etcd.ClientTrustedCAFile == "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile) + } + if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile != "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientKeyFile != "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientKeyFile is %q", mem.Etcd.PeerCertFile) + } + if !mem.Etcd.ClientCertAuth && mem.Etcd.ClientTrustedCAFile != "" { + return nil, fmt.Errorf("Etcd.ClientCertAuth 'false', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.PeerCertFile) + } + if (mem.Etcd.ClientCertFile == "") != (mem.Etcd.ClientKeyFile == "") { + return nil, fmt.Errorf("Both Etcd.ClientCertFile %q and Etcd.ClientKeyFile %q must be either empty or non-empty", mem.Etcd.ClientCertFile, mem.Etcd.ClientKeyFile) + } + + peerTLS := mem.Etcd.PeerAutoTLS || + (mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile != "" && mem.Etcd.PeerKeyFile != "" && mem.Etcd.PeerTrustedCAFile != "") + if peerTLS { + for _, cu := range mem.Etcd.ListenPeerURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + } + } + for _, cu := range mem.Etcd.AdvertisePeerURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu) + } + } + clus.Members[i].PeerCertPath = mem.Etcd.PeerCertFile + if mem.Etcd.PeerCertFile != "" { + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.PeerCertFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerCertFile, err) + } + clus.Members[i].PeerCertData = string(data) + } + clus.Members[i].PeerKeyPath = mem.Etcd.PeerKeyFile + if mem.Etcd.PeerKeyFile != "" { + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.PeerKeyFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerKeyFile, err) + } + clus.Members[i].PeerCertData = string(data) + } + clus.Members[i].PeerTrustedCAPath = mem.Etcd.PeerTrustedCAFile + if mem.Etcd.PeerTrustedCAFile != "" { + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.PeerTrustedCAFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.PeerTrustedCAFile, err) + } + clus.Members[i].PeerCertData = string(data) + } + } + + clientTLS := mem.Etcd.ClientAutoTLS || + (mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile != "" && mem.Etcd.ClientKeyFile != "" && mem.Etcd.ClientTrustedCAFile != "") + if clientTLS { + for _, cu := range mem.Etcd.ListenClientURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) + } + } + for _, cu := range mem.Etcd.AdvertiseClientURLs { + var u *url.URL + u, err = url.Parse(cu) + if err != nil { + return nil, err + } + if u.Scheme != "https" { // TODO: support unix + return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu) + } + } + clus.Members[i].ClientCertPath = mem.Etcd.ClientCertFile + if mem.Etcd.ClientCertFile != "" { + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.ClientCertFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientCertFile, err) + } + clus.Members[i].ClientCertData = string(data) + } + clus.Members[i].ClientKeyPath = mem.Etcd.ClientKeyFile + if mem.Etcd.ClientKeyFile != "" { + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.ClientKeyFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientKeyFile, err) + } + clus.Members[i].ClientCertData = string(data) + } + clus.Members[i].ClientTrustedCAPath = mem.Etcd.ClientTrustedCAFile + if mem.Etcd.ClientTrustedCAFile != "" { + var data []byte + data, err = ioutil.ReadFile(mem.Etcd.ClientTrustedCAFile) + if err != nil { + return nil, fmt.Errorf("failed to read %q (%v)", mem.Etcd.ClientTrustedCAFile, err) + } + clus.Members[i].ClientCertData = string(data) + } + } + } + + if len(clus.Tester.Cases) == 0 { + return nil, errors.New("Cases not found") + } + if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 { + return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv) + } + if clus.Tester.UpdatedDelayLatencyMs == 0 { + clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs + } + + for _, v := range clus.Tester.Cases { + if _, ok := rpcpb.Case_value[v]; !ok { + return nil, fmt.Errorf("%q is not defined in 'rpcpb.Case_value'", v) + } + } + + for _, v := range clus.Tester.Stressers { + if _, ok := rpcpb.Stresser_value[v]; !ok { + return nil, fmt.Errorf("Stresser is unknown; got %q", v) + } + } + for _, v := range clus.Tester.Checkers { + if _, ok := rpcpb.Checker_value[v]; !ok { + return nil, fmt.Errorf("Checker is unknown; got %q", v) + } + } + + if clus.Tester.StressKeySuffixRangeTxn > 100 { + return nil, fmt.Errorf("StressKeySuffixRangeTxn maximum value is 100, got %v", clus.Tester.StressKeySuffixRangeTxn) + } + if clus.Tester.StressKeyTxnOps > 64 { + return nil, fmt.Errorf("StressKeyTxnOps maximum value is 64, got %v", clus.Tester.StressKeyTxnOps) + } + + return clus, err +} diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go new file mode 100644 index 000000000..6dd002106 --- /dev/null +++ b/functional/tester/cluster_run.go @@ -0,0 +1,373 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "os" + "time" + + "github.com/coreos/etcd/functional/rpcpb" + "github.com/coreos/etcd/pkg/fileutil" + + "go.uber.org/zap" +) + +// compactQPS is rough number of compact requests per second. +// Previous tests showed etcd can compact about 60,000 entries per second. +const compactQPS = 50000 + +// Run starts tester. +func (clus *Cluster) Run() { + defer printReport() + + if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil { + clus.lg.Panic( + "failed to create test data directory", + zap.String("dir", clus.Tester.DataDir), + zap.Error(err), + ) + } + + var preModifiedKey int64 + for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { + roundTotalCounter.Inc() + clus.rd = round + + if err := clus.doRound(); err != nil { + clus.lg.Warn( + "round FAIL", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Error(err), + ) + if clus.cleanup() != nil { + return + } + // reset preModifiedKey after clean up + preModifiedKey = 0 + continue + } + + // -1 so that logPrefix doesn't print out 'case' + clus.cs = -1 + + revToCompact := max(0, clus.currentRevision-10000) + currentModifiedKey := clus.stresser.ModifiedKeys() + modifiedKey := currentModifiedKey - preModifiedKey + preModifiedKey = currentModifiedKey + timeout := 10 * time.Second + timeout += time.Duration(modifiedKey/compactQPS) * time.Second + clus.lg.Info( + "compact START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Duration("timeout", timeout), + ) + if err := clus.compact(revToCompact, timeout); err != nil { + clus.lg.Warn( + "compact FAIL", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Error(err), + ) + if err = clus.cleanup(); err != nil { + clus.lg.Warn( + "cleanup FAIL", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Error(err), + ) + return + } + // reset preModifiedKey after clean up + preModifiedKey = 0 + } + if round > 0 && round%500 == 0 { // every 500 rounds + if err := clus.defrag(); err != nil { + clus.failed() + return + } + } + } + + clus.lg.Info( + "functional-tester PASS", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + ) +} + +func (clus *Cluster) doRound() error { + if clus.Tester.CaseShuffle { + clus.shuffleCases() + } + + roundNow := time.Now() + clus.lg.Info( + "round START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Strings("cases", clus.listCases()), + ) + for i, fa := range clus.cases { + clus.cs = i + + caseTotal[fa.Desc()]++ + caseTotalCounter.WithLabelValues(fa.Desc()).Inc() + + caseNow := time.Now() + clus.lg.Info( + "case START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + + clus.lg.Info("wait health before injecting failures") + if err := clus.WaitHealth(); err != nil { + return fmt.Errorf("wait full health error: %v", err) + } + + stressStarted := false + fcase := fa.TestCase() + if fcase != rpcpb.Case_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS { + clus.lg.Info( + "stress START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + if err := clus.stresser.Stress(); err != nil { + return fmt.Errorf("start stresser error: %v", err) + } + stressStarted = true + } + + clus.lg.Info( + "inject START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + if err := fa.Inject(clus); err != nil { + return fmt.Errorf("injection error: %v", err) + } + + // if run local, recovering server may conflict + // with stressing client ports + // TODO: use unix for local tests + clus.lg.Info( + "recover START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + if err := fa.Recover(clus); err != nil { + return fmt.Errorf("recovery error: %v", err) + } + + if stressStarted { + clus.lg.Info( + "stress PAUSE", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + ems := clus.stresser.Pause() + if fcase == rpcpb.Case_NO_FAIL_WITH_STRESS && len(ems) > 0 { + ess := make([]string, 0, len(ems)) + cnt := 0 + for k, v := range ems { + ess = append(ess, fmt.Sprintf("%s (count: %d)", k, v)) + cnt += v + } + clus.lg.Warn( + "expected no errors", + zap.String("desc", fa.Desc()), + zap.Strings("errors", ess), + ) + + // with network delay, some ongoing requests may fail + // only return error, if more than 10% of QPS requests fail + if cnt > int(clus.Tester.StressQPS)/10 { + return fmt.Errorf("expected no error in %q, got %q", fcase.String(), ess) + } + } + } + + clus.lg.Info( + "health check START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + if err := clus.WaitHealth(); err != nil { + return fmt.Errorf("wait full health error: %v", err) + } + + checkerFailExceptions := []rpcpb.Checker{} + switch fcase { + case rpcpb.Case_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH: + // TODO: restore from snapshot + checkerFailExceptions = append(checkerFailExceptions, rpcpb.Checker_LEASE_EXPIRE) + } + + clus.lg.Info( + "consistency check START", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + ) + if err := clus.runCheckers(checkerFailExceptions...); err != nil { + return fmt.Errorf("consistency check error (%v)", err) + } + clus.lg.Info( + "consistency check PASS", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.String("desc", fa.Desc()), + zap.Duration("took", time.Since(caseNow)), + ) + } + + clus.lg.Info( + "round ALL PASS", + zap.Int("round", clus.rd), + zap.Strings("cases", clus.listCases()), + zap.Int("case-total", len(clus.cases)), + zap.Duration("took", time.Since(roundNow)), + ) + return nil +} + +func (clus *Cluster) updateRevision() error { + revs, _, err := clus.getRevisionHash() + for _, rev := range revs { + clus.currentRevision = rev + break // just need get one of the current revisions + } + + clus.lg.Info( + "updated current revision", + zap.Int64("current-revision", clus.currentRevision), + ) + return err +} + +func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) { + if err = clus.compactKV(rev, timeout); err != nil { + clus.lg.Warn( + "compact FAIL", + zap.Int64("current-revision", clus.currentRevision), + zap.Int64("compact-revision", rev), + zap.Error(err), + ) + return err + } + clus.lg.Info( + "compact DONE", + zap.Int64("current-revision", clus.currentRevision), + zap.Int64("compact-revision", rev), + ) + + if err = clus.checkCompact(rev); err != nil { + clus.lg.Warn( + "check compact FAIL", + zap.Int64("current-revision", clus.currentRevision), + zap.Int64("compact-revision", rev), + zap.Error(err), + ) + return err + } + clus.lg.Info( + "check compact DONE", + zap.Int64("current-revision", clus.currentRevision), + zap.Int64("compact-revision", rev), + ) + + return nil +} + +func (clus *Cluster) failed() { + clus.lg.Info( + "functional-tester FAIL", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + ) + clus.Send_SIGQUIT_ETCD_AND_REMOVE_DATA_AND_STOP_AGENT() + + os.Exit(2) +} + +func (clus *Cluster) cleanup() error { + if clus.Tester.ExitOnCaseFail { + defer clus.failed() + } + + roundFailedTotalCounter.Inc() + desc := "compact/defrag" + if clus.cs != -1 { + desc = clus.cases[clus.cs].Desc() + } + caseFailedTotalCounter.WithLabelValues(desc).Inc() + + clus.lg.Info( + "closing stressers before archiving failure data", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + ) + clus.stresser.Close() + + if err := clus.send_SIGQUIT_ETCD_AND_ARCHIVE_DATA(); err != nil { + clus.lg.Warn( + "cleanup FAIL", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Error(err), + ) + return err + } + if err := clus.send_RESTART_ETCD(); err != nil { + clus.lg.Warn( + "restart FAIL", + zap.Int("round", clus.rd), + zap.Int("case", clus.cs), + zap.Int("case-total", len(clus.cases)), + zap.Error(err), + ) + return err + } + + clus.setStresserChecker() + return nil +} diff --git a/functional/tester/cluster_shuffle.go b/functional/tester/cluster_shuffle.go new file mode 100644 index 000000000..16c79b2f6 --- /dev/null +++ b/functional/tester/cluster_shuffle.go @@ -0,0 +1,64 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "math/rand" + "time" + + "go.uber.org/zap" +) + +func (clus *Cluster) shuffleCases() { + rand.Seed(time.Now().UnixNano()) + offset := rand.Intn(1000) + n := len(clus.cases) + cp := coprime(n) + + css := make([]Case, n) + for i := 0; i < n; i++ { + css[i] = clus.cases[(cp*i+offset)%n] + } + clus.cases = css + clus.lg.Info("shuffled test failure cases", zap.Int("total", n)) +} + +/* +x and y of GCD 1 are coprime to each other + +x1 = ( coprime of n * idx1 + offset ) % n +x2 = ( coprime of n * idx2 + offset ) % n +(x2 - x1) = coprime of n * (idx2 - idx1) % n + = (idx2 - idx1) = 1 + +Consecutive x's are guaranteed to be distinct +*/ +func coprime(n int) int { + coprime := 1 + for i := n / 2; i < n; i++ { + if gcd(i, n) == 1 { + coprime = i + break + } + } + return coprime +} + +func gcd(x, y int) int { + if y == 0 { + return x + } + return gcd(y, x%y) +} diff --git a/functional/tester/doc.go b/functional/tester/doc.go new file mode 100644 index 000000000..d1e23e941 --- /dev/null +++ b/functional/tester/doc.go @@ -0,0 +1,16 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tester implements functional-tester tester server. +package tester diff --git a/functional/tester/metrics_report.go b/functional/tester/metrics_report.go new file mode 100644 index 000000000..c82e58f5b --- /dev/null +++ b/functional/tester/metrics_report.go @@ -0,0 +1,83 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "sort" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + caseTotal = make(map[string]int) + + caseTotalCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "funcational_tester", + Name: "case_total", + Help: "Total number of finished test cases", + }, + []string{"desc"}, + ) + + caseFailedTotalCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "funcational_tester", + Name: "case_failed_total", + Help: "Total number of failed test cases", + }, + []string{"desc"}, + ) + + roundTotalCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "funcational_tester", + Name: "round_total", + Help: "Total number of finished test rounds.", + }) + + roundFailedTotalCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "funcational_tester", + Name: "round_failed_total", + Help: "Total number of failed test rounds.", + }) +) + +func init() { + prometheus.MustRegister(caseTotalCounter) + prometheus.MustRegister(caseFailedTotalCounter) + prometheus.MustRegister(roundTotalCounter) + prometheus.MustRegister(roundFailedTotalCounter) +} + +func printReport() { + rows := make([]string, 0, len(caseTotal)) + for k, v := range caseTotal { + rows = append(rows, fmt.Sprintf("%s: %d", k, v)) + } + sort.Strings(rows) + + println() + for _, row := range rows { + fmt.Println(row) + } + println() +} diff --git a/functional/tester/stresser.go b/functional/tester/stresser.go new file mode 100644 index 000000000..b74b84b15 --- /dev/null +++ b/functional/tester/stresser.go @@ -0,0 +1,156 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "time" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" +) + +// Stresser defines stressing client operations. +type Stresser interface { + // Stress starts to stress the etcd cluster + Stress() error + // Pause stops the stresser from sending requests to etcd. Resume by calling Stress. + Pause() map[string]int + // Close releases all of the Stresser's resources. + Close() map[string]int + // ModifiedKeys reports the number of keys created and deleted by stresser + ModifiedKeys() int64 +} + +// newStresser creates stresser from a comma separated list of stresser types. +func newStresser(clus *Cluster, m *rpcpb.Member) (stressers []Stresser) { + stressers = make([]Stresser, len(clus.Tester.Stressers)) + for i, stype := range clus.Tester.Stressers { + clus.lg.Info( + "creating stresser", + zap.String("type", stype), + zap.String("endpoint", m.EtcdClientEndpoint), + ) + + switch stype { + case "KV": + // TODO: Too intensive stressing clients can panic etcd member with + // 'out of memory' error. Put rate limits in server side. + stressers[i] = &keyStresser{ + stype: rpcpb.Stresser_KV, + lg: clus.lg, + m: m, + keySize: int(clus.Tester.StressKeySize), + keyLargeSize: int(clus.Tester.StressKeySizeLarge), + keySuffixRange: int(clus.Tester.StressKeySuffixRange), + keyTxnSuffixRange: int(clus.Tester.StressKeySuffixRangeTxn), + keyTxnOps: int(clus.Tester.StressKeyTxnOps), + clientsN: int(clus.Tester.StressClients), + rateLimiter: clus.rateLimiter, + } + + case "LEASE": + stressers[i] = &leaseStresser{ + stype: rpcpb.Stresser_LEASE, + lg: clus.lg, + m: m, + numLeases: 10, // TODO: configurable + keysPerLease: 10, // TODO: configurable + rateLimiter: clus.rateLimiter, + } + + case "ELECTION_RUNNER": + reqRate := 100 + args := []string{ + "election", + fmt.Sprintf("%v", time.Now().UnixNano()), // election name as current nano time + "--dial-timeout=10s", + "--endpoints", m.EtcdClientEndpoint, + "--total-client-connections=10", + "--rounds=0", // runs forever + "--req-rate", fmt.Sprintf("%v", reqRate), + } + stressers[i] = newRunnerStresser( + rpcpb.Stresser_ELECTION_RUNNER, + m.EtcdClientEndpoint, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + reqRate, + ) + + case "WATCH_RUNNER": + reqRate := 100 + args := []string{ + "watcher", + "--prefix", fmt.Sprintf("%v", time.Now().UnixNano()), // prefix all keys with nano time + "--total-keys=1", + "--total-prefixes=1", + "--watch-per-prefix=1", + "--endpoints", m.EtcdClientEndpoint, + "--rounds=0", // runs forever + "--req-rate", fmt.Sprintf("%v", reqRate), + } + stressers[i] = newRunnerStresser( + rpcpb.Stresser_WATCH_RUNNER, + m.EtcdClientEndpoint, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + reqRate, + ) + + case "LOCK_RACER_RUNNER": + reqRate := 100 + args := []string{ + "lock-racer", + fmt.Sprintf("%v", time.Now().UnixNano()), // locker name as current nano time + "--endpoints", m.EtcdClientEndpoint, + "--total-client-connections=10", + "--rounds=0", // runs forever + "--req-rate", fmt.Sprintf("%v", reqRate), + } + stressers[i] = newRunnerStresser( + rpcpb.Stresser_LOCK_RACER_RUNNER, + m.EtcdClientEndpoint, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + reqRate, + ) + + case "LEASE_RUNNER": + args := []string{ + "lease-renewer", + "--ttl=30", + "--endpoints", m.EtcdClientEndpoint, + } + stressers[i] = newRunnerStresser( + rpcpb.Stresser_LEASE_RUNNER, + m.EtcdClientEndpoint, + clus.lg, + clus.Tester.RunnerExecPath, + args, + clus.rateLimiter, + 0, + ) + } + } + return stressers +} diff --git a/functional/tester/stresser_composite.go b/functional/tester/stresser_composite.go new file mode 100644 index 000000000..09dcb55ff --- /dev/null +++ b/functional/tester/stresser_composite.go @@ -0,0 +1,82 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "sync" + +// compositeStresser implements a Stresser that runs a slice of +// stressing clients concurrently. +type compositeStresser struct { + stressers []Stresser +} + +func (cs *compositeStresser) Stress() error { + for i, s := range cs.stressers { + if err := s.Stress(); err != nil { + for j := 0; j < i; j++ { + cs.stressers[j].Close() + } + return err + } + } + return nil +} + +func (cs *compositeStresser) Pause() (ems map[string]int) { + var emu sync.Mutex + ems = make(map[string]int) + var wg sync.WaitGroup + wg.Add(len(cs.stressers)) + for i := range cs.stressers { + go func(s Stresser) { + defer wg.Done() + errs := s.Pause() + for k, v := range errs { + emu.Lock() + ems[k] += v + emu.Unlock() + } + }(cs.stressers[i]) + } + wg.Wait() + return ems +} + +func (cs *compositeStresser) Close() (ems map[string]int) { + var emu sync.Mutex + ems = make(map[string]int) + var wg sync.WaitGroup + wg.Add(len(cs.stressers)) + for i := range cs.stressers { + go func(s Stresser) { + defer wg.Done() + errs := s.Close() + for k, v := range errs { + emu.Lock() + ems[k] += v + emu.Unlock() + } + }(cs.stressers[i]) + } + wg.Wait() + return ems +} + +func (cs *compositeStresser) ModifiedKeys() (modifiedKey int64) { + for _, stress := range cs.stressers { + modifiedKey += stress.ModifiedKeys() + } + return modifiedKey +} diff --git a/functional/tester/stresser_key.go b/functional/tester/stresser_key.go new file mode 100644 index 000000000..2fc1bf2b0 --- /dev/null +++ b/functional/tester/stresser_key.go @@ -0,0 +1,344 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "fmt" + "math/rand" + "reflect" + "sync" + "sync/atomic" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/etcdserver" + "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" + "golang.org/x/time/rate" + "google.golang.org/grpc" + "google.golang.org/grpc/transport" +) + +type keyStresser struct { + stype rpcpb.Stresser + lg *zap.Logger + + m *rpcpb.Member + + keySize int + keyLargeSize int + keySuffixRange int + keyTxnSuffixRange int + keyTxnOps int + + rateLimiter *rate.Limiter + + wg sync.WaitGroup + clientsN int + + ctx context.Context + cancel func() + cli *clientv3.Client + + emu sync.RWMutex + ems map[string]int + paused bool + + // atomicModifiedKeys records the number of keys created and deleted by the stresser. + atomicModifiedKeys int64 + + stressTable *stressTable +} + +func (s *keyStresser) Stress() error { + var err error + s.cli, err = s.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(1 * time.Second)) + if err != nil { + return fmt.Errorf("%v (%q)", err, s.m.EtcdClientEndpoint) + } + s.ctx, s.cancel = context.WithCancel(context.Background()) + + s.wg.Add(s.clientsN) + var stressEntries = []stressEntry{ + {weight: 0.7, f: newStressPut(s.cli, s.keySuffixRange, s.keySize)}, + { + weight: 0.7 * float32(s.keySize) / float32(s.keyLargeSize), + f: newStressPut(s.cli, s.keySuffixRange, s.keyLargeSize), + }, + {weight: 0.07, f: newStressRange(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressRangeInterval(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressDelete(s.cli, s.keySuffixRange)}, + {weight: 0.07, f: newStressDeleteInterval(s.cli, s.keySuffixRange)}, + } + if s.keyTxnSuffixRange > 0 { + // adjust to make up ±70% of workloads with writes + stressEntries[0].weight = 0.35 + stressEntries = append(stressEntries, stressEntry{ + weight: 0.35, + f: newStressTxn(s.cli, s.keyTxnSuffixRange, s.keyTxnOps), + }) + } + s.stressTable = createStressTable(stressEntries) + + s.emu.Lock() + s.paused = false + s.ems = make(map[string]int, 100) + s.emu.Unlock() + for i := 0; i < s.clientsN; i++ { + go s.run() + } + + s.lg.Info( + "stress START", + zap.String("stress-type", s.stype.String()), + zap.String("endpoint", s.m.EtcdClientEndpoint), + ) + return nil +} + +func (s *keyStresser) run() { + defer s.wg.Done() + + for { + if err := s.rateLimiter.Wait(s.ctx); err == context.Canceled { + return + } + + // TODO: 10-second is enough timeout to cover leader failure + // and immediate leader election. Find out what other cases this + // could be timed out. + sctx, scancel := context.WithTimeout(s.ctx, 10*time.Second) + err, modifiedKeys := s.stressTable.choose()(sctx) + scancel() + if err == nil { + atomic.AddInt64(&s.atomicModifiedKeys, modifiedKeys) + continue + } + + switch rpctypes.ErrorDesc(err) { + case context.DeadlineExceeded.Error(): + // This retries when request is triggered at the same time as + // leader failure. When we terminate the leader, the request to + // that leader cannot be processed, and times out. Also requests + // to followers cannot be forwarded to the old leader, so timing out + // as well. We want to keep stressing until the cluster elects a + // new leader and start processing requests again. + case etcdserver.ErrTimeoutDueToLeaderFail.Error(), etcdserver.ErrTimeout.Error(): + // This retries when request is triggered at the same time as + // leader failure and follower nodes receive time out errors + // from losing their leader. Followers should retry to connect + // to the new leader. + case etcdserver.ErrStopped.Error(): + // one of the etcd nodes stopped from failure injection + case transport.ErrConnClosing.Desc: + // server closed the transport (failure injected node) + case rpctypes.ErrNotCapable.Error(): + // capability check has not been done (in the beginning) + case rpctypes.ErrTooManyRequests.Error(): + // hitting the recovering member. + case context.Canceled.Error(): + // from stresser.Cancel method: + return + case grpc.ErrClientConnClosing.Error(): + // from stresser.Cancel method: + return + default: + s.lg.Warn( + "stress run exiting", + zap.String("stress-type", s.stype.String()), + zap.String("endpoint", s.m.EtcdClientEndpoint), + zap.String("error-type", reflect.TypeOf(err).String()), + zap.Error(err), + ) + return + } + + // only record errors before pausing stressers + s.emu.Lock() + if !s.paused { + s.ems[err.Error()]++ + } + s.emu.Unlock() + } +} + +func (s *keyStresser) Pause() map[string]int { + return s.Close() +} + +func (s *keyStresser) Close() map[string]int { + s.cancel() + s.cli.Close() + s.wg.Wait() + + s.emu.Lock() + s.paused = true + ess := s.ems + s.ems = make(map[string]int, 100) + s.emu.Unlock() + + s.lg.Info( + "stress STOP", + zap.String("stress-type", s.stype.String()), + zap.String("endpoint", s.m.EtcdClientEndpoint), + ) + return ess +} + +func (s *keyStresser) ModifiedKeys() int64 { + return atomic.LoadInt64(&s.atomicModifiedKeys) +} + +type stressFunc func(ctx context.Context) (err error, modifiedKeys int64) + +type stressEntry struct { + weight float32 + f stressFunc +} + +type stressTable struct { + entries []stressEntry + sumWeights float32 +} + +func createStressTable(entries []stressEntry) *stressTable { + st := stressTable{entries: entries} + for _, entry := range st.entries { + st.sumWeights += entry.weight + } + return &st +} + +func (st *stressTable) choose() stressFunc { + v := rand.Float32() * st.sumWeights + var sum float32 + var idx int + for i := range st.entries { + sum += st.entries[i].weight + if sum >= v { + idx = i + break + } + } + return st.entries[idx].f +} + +func newStressPut(cli *clientv3.Client, keySuffixRange, keySize int) stressFunc { + return func(ctx context.Context) (error, int64) { + _, err := cli.Put( + ctx, + fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange)), + string(randBytes(keySize)), + ) + return err, 1 + } +} + +func newStressTxn(cli *clientv3.Client, keyTxnSuffixRange, txnOps int) stressFunc { + keys := make([]string, keyTxnSuffixRange) + for i := range keys { + keys[i] = fmt.Sprintf("/k%03d", i) + } + return writeTxn(cli, keys, txnOps) +} + +func writeTxn(cli *clientv3.Client, keys []string, txnOps int) stressFunc { + return func(ctx context.Context) (error, int64) { + ks := make(map[string]struct{}, txnOps) + for len(ks) != txnOps { + ks[keys[rand.Intn(len(keys))]] = struct{}{} + } + selected := make([]string, 0, txnOps) + for k := range ks { + selected = append(selected, k) + } + com, delOp, putOp := getTxnOps(selected[0], "bar00") + thenOps := []clientv3.Op{delOp} + elseOps := []clientv3.Op{putOp} + for i := 1; i < txnOps; i++ { // nested txns + k, v := selected[i], fmt.Sprintf("bar%02d", i) + com, delOp, putOp = getTxnOps(k, v) + txnOp := clientv3.OpTxn( + []clientv3.Cmp{com}, + []clientv3.Op{delOp}, + []clientv3.Op{putOp}, + ) + thenOps = append(thenOps, txnOp) + elseOps = append(elseOps, txnOp) + } + _, err := cli.Txn(ctx). + If(com). + Then(thenOps...). + Else(elseOps...). + Commit() + return err, int64(txnOps) + } +} + +func getTxnOps(k, v string) ( + cmp clientv3.Cmp, + dop clientv3.Op, + pop clientv3.Op) { + // if key exists (version > 0) + cmp = clientv3.Compare(clientv3.Version(k), ">", 0) + dop = clientv3.OpDelete(k) + pop = clientv3.OpPut(k, v) + return cmp, dop, pop +} + +func newStressRange(cli *clientv3.Client, keySuffixRange int) stressFunc { + return func(ctx context.Context) (error, int64) { + _, err := cli.Get(ctx, fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange))) + return err, 0 + } +} + +func newStressRangeInterval(cli *clientv3.Client, keySuffixRange int) stressFunc { + return func(ctx context.Context) (error, int64) { + start := rand.Intn(keySuffixRange) + end := start + 500 + _, err := cli.Get( + ctx, + fmt.Sprintf("foo%016x", start), + clientv3.WithRange(fmt.Sprintf("foo%016x", end)), + ) + return err, 0 + } +} + +func newStressDelete(cli *clientv3.Client, keySuffixRange int) stressFunc { + return func(ctx context.Context) (error, int64) { + _, err := cli.Delete(ctx, fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange))) + return err, 1 + } +} + +func newStressDeleteInterval(cli *clientv3.Client, keySuffixRange int) stressFunc { + return func(ctx context.Context) (error, int64) { + start := rand.Intn(keySuffixRange) + end := start + 500 + resp, err := cli.Delete(ctx, + fmt.Sprintf("foo%016x", start), + clientv3.WithRange(fmt.Sprintf("foo%016x", end)), + ) + if err == nil { + return nil, resp.Deleted + } + return err, 0 + } +} diff --git a/functional/tester/stresser_lease.go b/functional/tester/stresser_lease.go new file mode 100644 index 000000000..8510a0765 --- /dev/null +++ b/functional/tester/stresser_lease.go @@ -0,0 +1,487 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" + "golang.org/x/time/rate" + "google.golang.org/grpc" +) + +const ( + // time to live for lease + defaultTTL = 120 + defaultTTLShort = 2 +) + +type leaseStresser struct { + stype rpcpb.Stresser + lg *zap.Logger + + m *rpcpb.Member + cli *clientv3.Client + ctx context.Context + cancel func() + + rateLimiter *rate.Limiter + // atomicModifiedKey records the number of keys created and deleted during a test case + atomicModifiedKey int64 + numLeases int + keysPerLease int + + aliveLeases *atomicLeases + revokedLeases *atomicLeases + shortLivedLeases *atomicLeases + + runWg sync.WaitGroup + aliveWg sync.WaitGroup +} + +type atomicLeases struct { + // rwLock is used to protect read/write access of leases map + // which are accessed and modified by different go routines. + rwLock sync.RWMutex + leases map[int64]time.Time +} + +func (al *atomicLeases) add(leaseID int64, t time.Time) { + al.rwLock.Lock() + al.leases[leaseID] = t + al.rwLock.Unlock() +} + +func (al *atomicLeases) update(leaseID int64, t time.Time) { + al.rwLock.Lock() + _, ok := al.leases[leaseID] + if ok { + al.leases[leaseID] = t + } + al.rwLock.Unlock() +} + +func (al *atomicLeases) read(leaseID int64) (rv time.Time, ok bool) { + al.rwLock.RLock() + rv, ok = al.leases[leaseID] + al.rwLock.RUnlock() + return rv, ok +} + +func (al *atomicLeases) remove(leaseID int64) { + al.rwLock.Lock() + delete(al.leases, leaseID) + al.rwLock.Unlock() +} + +func (al *atomicLeases) getLeasesMap() map[int64]time.Time { + leasesCopy := make(map[int64]time.Time) + al.rwLock.RLock() + for k, v := range al.leases { + leasesCopy[k] = v + } + al.rwLock.RUnlock() + return leasesCopy +} + +func (ls *leaseStresser) setupOnce() error { + if ls.aliveLeases != nil { + return nil + } + if ls.numLeases == 0 { + panic("expect numLeases to be set") + } + if ls.keysPerLease == 0 { + panic("expect keysPerLease to be set") + } + + ls.aliveLeases = &atomicLeases{leases: make(map[int64]time.Time)} + return nil +} + +func (ls *leaseStresser) Stress() error { + ls.lg.Info( + "stress START", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + ) + + if err := ls.setupOnce(); err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + ls.ctx = ctx + ls.cancel = cancel + + cli, err := ls.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(1 * time.Second)) + if err != nil { + return fmt.Errorf("%v (%s)", err, ls.m.EtcdClientEndpoint) + } + ls.cli = cli + + ls.revokedLeases = &atomicLeases{leases: make(map[int64]time.Time)} + ls.shortLivedLeases = &atomicLeases{leases: make(map[int64]time.Time)} + + ls.runWg.Add(1) + go ls.run() + return nil +} + +func (ls *leaseStresser) run() { + defer ls.runWg.Done() + ls.restartKeepAlives() + for { + // the number of keys created and deleted is roughly 2x the number of created keys for an iteration. + // the rateLimiter therefore consumes 2x ls.numLeases*ls.keysPerLease tokens where each token represents a create/delete operation for key. + err := ls.rateLimiter.WaitN(ls.ctx, 2*ls.numLeases*ls.keysPerLease) + if err == context.Canceled { + return + } + + ls.lg.Debug( + "stress creating leases", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + ) + ls.createLeases() + ls.lg.Debug( + "stress created leases", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + ) + + ls.lg.Debug( + "stress dropped leases", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + ) + ls.randomlyDropLeases() + ls.lg.Debug( + "stress dropped leases", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + ) + } +} + +func (ls *leaseStresser) restartKeepAlives() { + for leaseID := range ls.aliveLeases.getLeasesMap() { + ls.aliveWg.Add(1) + go func(id int64) { + ls.keepLeaseAlive(id) + }(leaseID) + } +} + +func (ls *leaseStresser) createLeases() { + ls.createAliveLeases() + ls.createShortLivedLeases() +} + +func (ls *leaseStresser) createAliveLeases() { + neededLeases := ls.numLeases - len(ls.aliveLeases.getLeasesMap()) + var wg sync.WaitGroup + for i := 0; i < neededLeases; i++ { + wg.Add(1) + go func() { + defer wg.Done() + leaseID, err := ls.createLeaseWithKeys(defaultTTL) + if err != nil { + ls.lg.Debug( + "createLeaseWithKeys failed", + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.Error(err), + ) + return + } + ls.aliveLeases.add(leaseID, time.Now()) + // keep track of all the keep lease alive go routines + ls.aliveWg.Add(1) + go ls.keepLeaseAlive(leaseID) + }() + } + wg.Wait() +} + +func (ls *leaseStresser) createShortLivedLeases() { + // one round of createLeases() might not create all the short lived leases we want due to falures. + // thus, we want to create remaining short lived leases in the future round. + neededLeases := ls.numLeases - len(ls.shortLivedLeases.getLeasesMap()) + var wg sync.WaitGroup + for i := 0; i < neededLeases; i++ { + wg.Add(1) + go func() { + defer wg.Done() + leaseID, err := ls.createLeaseWithKeys(defaultTTLShort) + if err != nil { + return + } + ls.shortLivedLeases.add(leaseID, time.Now()) + }() + } + wg.Wait() +} + +func (ls *leaseStresser) createLeaseWithKeys(ttl int64) (int64, error) { + leaseID, err := ls.createLease(ttl) + if err != nil { + ls.lg.Debug( + "createLease failed", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.Error(err), + ) + return -1, err + } + + ls.lg.Debug( + "createLease created lease", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + ) + if err := ls.attachKeysWithLease(leaseID); err != nil { + return -1, err + } + return leaseID, nil +} + +func (ls *leaseStresser) randomlyDropLeases() { + var wg sync.WaitGroup + for l := range ls.aliveLeases.getLeasesMap() { + wg.Add(1) + go func(leaseID int64) { + defer wg.Done() + dropped, err := ls.randomlyDropLease(leaseID) + // if randomlyDropLease encountered an error such as context is cancelled, remove the lease from aliveLeases + // because we can't tell whether the lease is dropped or not. + if err != nil { + ls.lg.Debug( + "randomlyDropLease failed", + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(err), + ) + ls.aliveLeases.remove(leaseID) + return + } + if !dropped { + return + } + ls.lg.Debug( + "randomlyDropLease dropped a lease", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + ) + ls.revokedLeases.add(leaseID, time.Now()) + ls.aliveLeases.remove(leaseID) + }(l) + } + wg.Wait() +} + +func (ls *leaseStresser) createLease(ttl int64) (int64, error) { + resp, err := ls.cli.Grant(ls.ctx, ttl) + if err != nil { + return -1, err + } + return int64(resp.ID), nil +} + +func (ls *leaseStresser) keepLeaseAlive(leaseID int64) { + defer ls.aliveWg.Done() + ctx, cancel := context.WithCancel(ls.ctx) + stream, err := ls.cli.KeepAlive(ctx, clientv3.LeaseID(leaseID)) + defer func() { cancel() }() + for { + select { + case <-time.After(500 * time.Millisecond): + case <-ls.ctx.Done(): + ls.lg.Debug( + "keepLeaseAlive context canceled", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(ls.ctx.Err()), + ) + // it is possible that lease expires at invariant checking phase but not at keepLeaseAlive() phase. + // this scenerio is possible when alive lease is just about to expire when keepLeaseAlive() exists and expires at invariant checking phase. + // to circumvent that scenerio, we check each lease before keepalive loop exist to see if it has been renewed in last TTL/2 duration. + // if it is renewed, this means that invariant checking have at least ttl/2 time before lease exipres which is long enough for the checking to finish. + // if it is not renewed, we remove the lease from the alive map so that the lease doesn't exipre during invariant checking + renewTime, ok := ls.aliveLeases.read(leaseID) + if ok && renewTime.Add(defaultTTL/2*time.Second).Before(time.Now()) { + ls.aliveLeases.remove(leaseID) + ls.lg.Debug( + "keepLeaseAlive lease has not been renewed, dropped it", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + ) + } + return + } + + if err != nil { + ls.lg.Debug( + "keepLeaseAlive lease creates stream error", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(err), + ) + cancel() + ctx, cancel = context.WithCancel(ls.ctx) + stream, err = ls.cli.KeepAlive(ctx, clientv3.LeaseID(leaseID)) + cancel() + continue + } + if err != nil { + ls.lg.Debug( + "keepLeaseAlive failed to receive lease keepalive response", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(err), + ) + continue + } + + ls.lg.Debug( + "keepLeaseAlive waiting on lease stream", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + ) + leaseRenewTime := time.Now() + respRC := <-stream + if respRC == nil { + ls.lg.Debug( + "keepLeaseAlive received nil lease keepalive response", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + ) + continue + } + + // lease expires after TTL become 0 + // don't send keepalive if the lease has expired + if respRC.TTL <= 0 { + ls.lg.Debug( + "keepLeaseAlive stream received lease keepalive response TTL <= 0", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Int64("ttl", respRC.TTL), + ) + ls.aliveLeases.remove(leaseID) + return + } + // renew lease timestamp only if lease is present + ls.lg.Debug( + "keepLeaseAlive renewed a lease", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + ) + ls.aliveLeases.update(leaseID, leaseRenewTime) + } +} + +// attachKeysWithLease function attaches keys to the lease. +// the format of key is the concat of leaseID + '_' + '' +// e.g 5186835655248304152_0 for first created key and 5186835655248304152_1 for second created key +func (ls *leaseStresser) attachKeysWithLease(leaseID int64) error { + var txnPuts []clientv3.Op + for j := 0; j < ls.keysPerLease; j++ { + txnput := clientv3.OpPut( + fmt.Sprintf("%d%s%d", leaseID, "_", j), + fmt.Sprintf("bar"), + clientv3.WithLease(clientv3.LeaseID(leaseID)), + ) + txnPuts = append(txnPuts, txnput) + } + // keep retrying until lease is not found or ctx is being canceled + for ls.ctx.Err() == nil { + _, err := ls.cli.Txn(ls.ctx).Then(txnPuts...).Commit() + if err == nil { + // since all created keys will be deleted too, the number of operations on keys will be roughly 2x the number of created keys + atomic.AddInt64(&ls.atomicModifiedKey, 2*int64(ls.keysPerLease)) + return nil + } + if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound { + return err + } + } + return ls.ctx.Err() +} + +// randomlyDropLease drops the lease only when the rand.Int(2) returns 1. +// This creates a 50/50 percents chance of dropping a lease +func (ls *leaseStresser) randomlyDropLease(leaseID int64) (bool, error) { + if rand.Intn(2) != 0 { + return false, nil + } + + // keep retrying until a lease is dropped or ctx is being canceled + for ls.ctx.Err() == nil { + _, err := ls.cli.Revoke(ls.ctx, clientv3.LeaseID(leaseID)) + if err == nil || rpctypes.Error(err) == rpctypes.ErrLeaseNotFound { + return true, nil + } + } + + ls.lg.Debug( + "randomlyDropLease error", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), + zap.Error(ls.ctx.Err()), + ) + return false, ls.ctx.Err() +} + +func (ls *leaseStresser) Pause() map[string]int { + return ls.Close() +} + +func (ls *leaseStresser) Close() map[string]int { + ls.cancel() + ls.runWg.Wait() + ls.aliveWg.Wait() + ls.cli.Close() + ls.lg.Info( + "stress STOP", + zap.String("stress-type", ls.stype.String()), + zap.String("endpoint", ls.m.EtcdClientEndpoint), + ) + return nil +} + +func (ls *leaseStresser) ModifiedKeys() int64 { + return atomic.LoadInt64(&ls.atomicModifiedKey) +} diff --git a/functional/tester/stresser_runner.go b/functional/tester/stresser_runner.go new file mode 100644 index 000000000..18487f402 --- /dev/null +++ b/functional/tester/stresser_runner.go @@ -0,0 +1,120 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "io/ioutil" + "os/exec" + "syscall" + + "github.com/coreos/etcd/functional/rpcpb" + + "go.uber.org/zap" + "golang.org/x/time/rate" +) + +type runnerStresser struct { + stype rpcpb.Stresser + etcdClientEndpoint string + lg *zap.Logger + + cmd *exec.Cmd + cmdStr string + args []string + rl *rate.Limiter + reqRate int + + errc chan error + donec chan struct{} +} + +func newRunnerStresser( + stype rpcpb.Stresser, + ep string, + lg *zap.Logger, + cmdStr string, + args []string, + rl *rate.Limiter, + reqRate int, +) *runnerStresser { + rl.SetLimit(rl.Limit() - rate.Limit(reqRate)) + return &runnerStresser{ + stype: stype, + etcdClientEndpoint: ep, + cmdStr: cmdStr, + args: args, + rl: rl, + reqRate: reqRate, + errc: make(chan error, 1), + donec: make(chan struct{}), + } +} + +func (rs *runnerStresser) setupOnce() (err error) { + if rs.cmd != nil { + return nil + } + + rs.cmd = exec.Command(rs.cmdStr, rs.args...) + stderr, err := rs.cmd.StderrPipe() + if err != nil { + return err + } + + go func() { + defer close(rs.donec) + out, err := ioutil.ReadAll(stderr) + if err != nil { + rs.errc <- err + } else { + rs.errc <- fmt.Errorf("(%v %v) stderr %v", rs.cmdStr, rs.args, string(out)) + } + }() + + return rs.cmd.Start() +} + +func (rs *runnerStresser) Stress() (err error) { + rs.lg.Info( + "stress START", + zap.String("stress-type", rs.stype.String()), + ) + if err = rs.setupOnce(); err != nil { + return err + } + return syscall.Kill(rs.cmd.Process.Pid, syscall.SIGCONT) +} + +func (rs *runnerStresser) Pause() map[string]int { + rs.lg.Info( + "stress STOP", + zap.String("stress-type", rs.stype.String()), + ) + syscall.Kill(rs.cmd.Process.Pid, syscall.SIGSTOP) + return nil +} + +func (rs *runnerStresser) Close() map[string]int { + syscall.Kill(rs.cmd.Process.Pid, syscall.SIGINT) + rs.cmd.Wait() + <-rs.donec + rs.rl.SetLimit(rs.rl.Limit() + rate.Limit(rs.reqRate)) + return nil +} + +func (rs *runnerStresser) ModifiedKeys() int64 { + return 1 +} diff --git a/functional/tester/utils.go b/functional/tester/utils.go new file mode 100644 index 000000000..74e34146d --- /dev/null +++ b/functional/tester/utils.go @@ -0,0 +1,79 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "math/rand" + "net" + "net/url" + "strings" +) + +func isValidURL(u string) bool { + _, err := url.Parse(u) + return err == nil +} + +func getPort(addr string) (port string, err error) { + urlAddr, err := url.Parse(addr) + if err != nil { + return "", err + } + _, port, err = net.SplitHostPort(urlAddr.Host) + if err != nil { + return "", err + } + return port, nil +} + +func getSameValue(vals map[string]int64) bool { + var rv int64 + for _, v := range vals { + if rv == 0 { + rv = v + } + if rv != v { + return false + } + } + return true +} + +func max(n1, n2 int64) int64 { + if n1 > n2 { + return n1 + } + return n2 +} + +func errsToError(errs []error) error { + if len(errs) == 0 { + return nil + } + stringArr := make([]string, len(errs)) + for i, err := range errs { + stringArr[i] = err.Error() + } + return fmt.Errorf(strings.Join(stringArr, ", ")) +} + +func randBytes(size int) []byte { + data := make([]byte, size) + for i := 0; i < size; i++ { + data[i] = byte(int('a') + rand.Intn(26)) + } + return data +} diff --git a/test b/test index 9c89ca5ae..bd6df0305 100755 --- a/test +++ b/test @@ -618,7 +618,6 @@ function build_cov_pass { function build_pass { echo "Checking build..." GO_BUILD_FLAGS="-v" etcd_build - GO_BUILD_FLAGS="-v" tools_build } for pass in $PASSES; do