diff --git a/Dockerfile-functional b/Dockerfile-functional index c5751a1a9..fbcc34ec2 100644 --- a/Dockerfile-functional +++ b/Dockerfile-functional @@ -34,9 +34,9 @@ RUN go get -v github.com/coreos/gofail \ && cp ./bin/etcd /bin/etcd-failpoints \ && ./functional/build \ && cp ./bin/etcd-agent /bin/etcd-agent \ - && cp ./bin/etcd-tester /bin/etcd-tester \ + && cp ./bin/etcd-proxy /bin/etcd-proxy \ && cp ./bin/etcd-runner /bin/etcd-runner \ + && cp ./bin/etcd-tester /bin/etcd-tester \ && go build -v -o /bin/benchmark ./tools/benchmark \ - && go build -v -o /bin/etcd-test-proxy ./tools/etcd-test-proxy \ && popd \ && rm -rf ${GOPATH}/src/github.com/coreos/etcd \ No newline at end of file diff --git a/Makefile b/Makefile index d89fbbdcd..ae6cb4aab 100644 --- a/Makefile +++ b/Makefile @@ -469,18 +469,20 @@ docker-dns-srv-test-certs-wildcard-run: # Example: -# make build-etcd-test-proxy - -build-etcd-test-proxy: - go build -v -o ./bin/etcd-test-proxy ./tools/etcd-test-proxy - - - -# Example: +# make build-functional # make build-docker-functional # make push-docker-functional # make pull-docker-functional +build-functional: + $(info GO_VERSION: $(GO_VERSION)) + $(info ETCD_VERSION: $(ETCD_VERSION)) + ./functional/build + ./bin/etcd-agent -help || true && \ + ./bin/etcd-proxy -help || true && \ + ./bin/etcd-runner --help || true && \ + ./bin/etcd-tester -help || true + build-docker-functional: $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) @@ -498,10 +500,10 @@ build-docker-functional: ./bin/etcd-failpoints --version && \ ETCDCTL_API=3 ./bin/etcdctl version && \ ./bin/etcd-agent -help || true && \ - ./bin/etcd-tester -help || true && \ + ./bin/etcd-proxy -help || true && \ ./bin/etcd-runner --help || true && \ - ./bin/benchmark --help || true && \ - ./bin/etcd-test-proxy -help || true" + ./bin/etcd-tester -help || true && \ + ./bin/benchmark --help || true" push-docker-functional: $(info GO_VERSION: $(GO_VERSION)) diff --git a/functional/Procfile-proxy b/functional/Procfile-proxy new file mode 100644 index 000000000..66730ee77 --- /dev/null +++ b/functional/Procfile-proxy @@ -0,0 +1,14 @@ +s1: bin/etcd --name s1 --data-dir /tmp/etcd-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s1-client-proxy: bin/etcd-proxy --from localhost:13790 --to localhost:1379 --http-port 1378 +s1-peer-proxy: bin/etcd-proxy --from localhost:13800 --to localhost:1380 --http-port 1381 + +s2: bin/etcd --name s2 --data-dir /tmp/etcd-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s2-client-proxy: bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 +s2-peer-proxy: bin/etcd-proxy --from localhost:23800 --to localhost:2380 --http-port 2381 + +s3: bin/etcd --name s3 --data-dir /tmp/etcd-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new + +s3-client-proxy: bin/etcd-proxy --from localhost:33790 --to localhost:3379 --http-port 3378 +s3-client-proxy: bin/etcd-proxy --from localhost:33800 --to localhost:3380 --http-port 3381 diff --git a/functional/README.md b/functional/README.md index 1c143097e..3571ce88c 100644 --- a/functional/README.md +++ b/functional/README.md @@ -1,4 +1,4 @@ -# etcd Functional Testing +## etcd Functional Testing `functional` verifies the correct behavior of etcd under various system and network malfunctions. It sets up an etcd cluster under high pressure loads and continuously injects failures into the cluster. Then it expects the etcd cluster to recover within a few seconds. This has been extremely helpful to find critical bugs. @@ -13,7 +13,7 @@ PASSES=functional ./test ### Run with Docker ```bash -pushd ../.. +pushd .. make build-docker-functional popd ``` @@ -29,3 +29,188 @@ And run [example scripts](./scripts). # to run only 1 tester round ./scripts/docker-local-tester.sh ``` + +## etcd Proxy + +Proxy layer that simulates various network conditions. + +Test locally + +```bash +$ ./build +$ ./bin/etcd + +$ make build-functional + +$ ./bin/etcd-proxy --help +$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar +``` + +Proxy overhead per request is under 500μs + +```bash +$ go build -v -o ./bin/benchmark ./tools/benchmark + +$ ./bin/benchmark \ + --endpoints localhost:2379 \ + --conns 5 \ + --clients 15 \ + put \ + --key-size 48 \ + --val-size 50000 \ + --total 10000 + +< tcp://localhost:2379] + +$ ETCDCTL_API=3 ./bin/etcdctl \ + --endpoints localhost:23790 \ + put foo bar +# Error: context deadline exceeded + +$ curl -L http://localhost:2378/pause-tx -X DELETE +# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379] +``` + +Drop client packets + +```bash +$ curl -L http://localhost:2378/blackhole-tx -X PUT +# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379] + +$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar +# Error: context deadline exceeded + +$ curl -L http://localhost:2378/blackhole-tx -X DELETE +# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379] +``` + +Trigger leader election + +```bash +$ ./build +$ make build-functional + +$ rm -rf /tmp/etcd-proxy-data.s* +$ goreman -f ./functional/Procfile-proxy start + +$ ETCDCTL_API=3 ./bin/etcdctl \ + --endpoints localhost:13790,localhost:23790,localhost:33790 \ + member list + +# isolate s1 when s1 is the current leader +$ curl -L http://localhost:1381/blackhole-tx -X PUT +$ curl -L http://localhost:1381/blackhole-rx -X PUT +# s1 becomes follower after election timeout +``` diff --git a/functional/build b/functional/build index 60daadcf4..47420b769 100755 --- a/functional/build +++ b/functional/build @@ -1,10 +1,11 @@ #!/usr/bin/env bash if ! [[ "$0" =~ "functional/build" ]]; then - echo "must be run from repository root" - exit 255 + echo "must be run from repository root" + exit 255 fi -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./functional/cmd/etcd-agent -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./functional/cmd/etcd-tester -CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./functional/cmd/etcd-runner +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-agent ./functional/cmd/etcd-agent +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-proxy ./functional/cmd/etcd-proxy +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-runner ./functional/cmd/etcd-runner +CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-tester ./functional/cmd/etcd-tester diff --git a/tools/etcd-test-proxy/main.go b/functional/cmd/etcd-proxy/main.go similarity index 94% rename from tools/etcd-test-proxy/main.go rename to functional/cmd/etcd-proxy/main.go index 61915bfed..24c275e8e 100644 --- a/tools/etcd-test-proxy/main.go +++ b/functional/cmd/etcd-proxy/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// etcd-test-proxy is a proxy layer that simulates various network conditions. +// etcd-proxy is a proxy layer that simulates various network conditions. package main import ( @@ -40,13 +40,13 @@ func main() { // TODO: support TLS flag.StringVar(&from, "from", "localhost:23790", "Address URL to proxy from.") flag.StringVar(&to, "to", "localhost:2379", "Address URL to forward.") - flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-test-proxy API.") + flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-proxy API.") flag.BoolVar(&verbose, "verbose", false, "'true' to run proxy in verbose mode.") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %q:\n", os.Args[0]) fmt.Fprintln(os.Stderr, ` -etcd-test-proxy simulates various network conditions for etcd testing purposes. +etcd-proxy simulates various network conditions for etcd testing purposes. See README.md for more examples. Example: @@ -55,12 +55,12 @@ Example: $ ./build $ ./bin/etcd -# build etcd-test-proxy -$ make build-etcd-test-proxy +# build etcd-proxy +$ make build-etcd-proxy # to test etcd with proxy layer -$ ./bin/etcd-test-proxy --help -$ ./bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose +$ ./bin/etcd-proxy --help +$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose $ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar $ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar`) diff --git a/tools/etcd-test-proxy/Procfile b/tools/etcd-test-proxy/Procfile deleted file mode 100644 index e820c1ea6..000000000 --- a/tools/etcd-test-proxy/Procfile +++ /dev/null @@ -1,14 +0,0 @@ -s1: bin/etcd --name s1 --data-dir /tmp/etcd-test-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new - -s1-client-proxy: bin/etcd-test-proxy --from localhost:13790 --to localhost:1379 --http-port 1378 -s1-peer-proxy: bin/etcd-test-proxy --from localhost:13800 --to localhost:1380 --http-port 1381 - -s2: bin/etcd --name s2 --data-dir /tmp/etcd-test-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new - -s2-client-proxy: bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 -s2-peer-proxy: bin/etcd-test-proxy --from localhost:23800 --to localhost:2380 --http-port 2381 - -s3: bin/etcd --name s3 --data-dir /tmp/etcd-test-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new - -s3-client-proxy: bin/etcd-test-proxy --from localhost:33790 --to localhost:3379 --http-port 3378 -s3-client-proxy: bin/etcd-test-proxy --from localhost:33800 --to localhost:3380 --http-port 3381 diff --git a/tools/etcd-test-proxy/README.md b/tools/etcd-test-proxy/README.md deleted file mode 100644 index bbb2763c4..000000000 --- a/tools/etcd-test-proxy/README.md +++ /dev/null @@ -1,184 +0,0 @@ -#### etcd-test-proxy - -Proxy layer that simulates various network conditions. - -Test locally - -```bash -$ ./build -$ ./bin/etcd - -$ make build-etcd-test-proxy - -$ ./bin/etcd-test-proxy --help -$ ./bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose - -$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar -$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar -``` - -Proxy overhead per request is under 500μs - -```bash -$ go build -v -o ./bin/benchmark ./tools/benchmark - -$ ./bin/benchmark \ - --endpoints localhost:2379 \ - --conns 5 \ - --clients 15 \ - put \ - --key-size 48 \ - --val-size 50000 \ - --total 10000 - -< tcp://localhost:2379] - -$ ETCDCTL_API=3 ./bin/etcdctl \ - --endpoints localhost:23790 \ - put foo bar -# Error: context deadline exceeded - -$ curl -L http://localhost:2378/pause-tx -X DELETE -# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379] -``` - -Drop client packets - -```bash -$ curl -L http://localhost:2378/blackhole-tx -X PUT -# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379] - -$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar -# Error: context deadline exceeded - -$ curl -L http://localhost:2378/blackhole-tx -X DELETE -# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379] -``` - -Trigger leader election - -```bash -$ ./build -$ make build-etcd-test-proxy - -$ rm -rf /tmp/etcd-test-proxy-data.s* -$ goreman -f ./tools/etcd-test-proxy/Procfile start - -$ ETCDCTL_API=3 ./bin/etcdctl \ - --endpoints localhost:13790,localhost:23790,localhost:33790 \ - member list - -# isolate s1 when s1 is the current leader -$ curl -L http://localhost:1381/blackhole-tx -X PUT -$ curl -L http://localhost:1381/blackhole-rx -X PUT -# s1 becomes follower after election timeout -```