Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

functional: move "etcd-test-proxy"

Browse Source 
Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
This commit is contained in:
Gyuho Lee 2018-04-08 14:57:25 -07:00
parent b8bf42cc5a
commit cd4580b4a9
8 changed files with 229 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile-functional
View File

@ -34,9 +34,9 @@ RUN go get -v github.com/coreos/gofail \
&& cp ./bin/etcd /bin/etcd-failpoints \
&& ./functional/build \
&& cp ./bin/etcd-agent /bin/etcd-agent \
&& cp ./bin/etcd-tester /bin/etcd-tester \
&& cp ./bin/etcd-proxy /bin/etcd-proxy \
&& cp ./bin/etcd-runner /bin/etcd-runner \
&& cp ./bin/etcd-tester /bin/etcd-tester \
&& go build -v -o /bin/benchmark ./tools/benchmark \
&& go build -v -o /bin/etcd-test-proxy ./tools/etcd-test-proxy \
&& popd \
&& rm -rf ${GOPATH}/src/github.com/coreos/etcd

24
Makefile
View File

@ -469,18 +469,20 @@ docker-dns-srv-test-certs-wildcard-run:
# Example:
# make build-etcd-test-proxy
build-etcd-test-proxy:
go build -v -o ./bin/etcd-test-proxy ./tools/etcd-test-proxy
# Example:
# make build-functional
# make build-docker-functional
# make push-docker-functional
# make pull-docker-functional
build-functional:
$(info GO_VERSION: $(GO_VERSION))
$(info ETCD_VERSION: $(ETCD_VERSION))
./functional/build
./bin/etcd-agent -help || true && \
./bin/etcd-proxy -help || true && \
./bin/etcd-runner --help || true && \
./bin/etcd-tester -help || true
build-docker-functional:
$(info GO_VERSION: $(GO_VERSION))
$(info ETCD_VERSION: $(ETCD_VERSION))
@ -498,10 +500,10 @@ build-docker-functional:
./bin/etcd-failpoints --version && \
ETCDCTL_API=3 ./bin/etcdctl version && \
./bin/etcd-agent -help || true && \
./bin/etcd-tester -help || true && \
./bin/etcd-proxy -help || true && \
./bin/etcd-runner --help || true && \
./bin/benchmark --help || true && \
./bin/etcd-test-proxy -help || true"
./bin/etcd-tester -help || true && \
./bin/benchmark --help || true"
push-docker-functional:
$(info GO_VERSION: $(GO_VERSION))

14
functional/Procfile-proxy Normal file
View File

@ -0,0 +1,14 @@
s1: bin/etcd --name s1 --data-dir /tmp/etcd-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new
s1-client-proxy: bin/etcd-proxy --from localhost:13790 --to localhost:1379 --http-port 1378
s1-peer-proxy: bin/etcd-proxy --from localhost:13800 --to localhost:1380 --http-port 1381
s2: bin/etcd --name s2 --data-dir /tmp/etcd-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new
s2-client-proxy: bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378
s2-peer-proxy: bin/etcd-proxy --from localhost:23800 --to localhost:2380 --http-port 2381
s3: bin/etcd --name s3 --data-dir /tmp/etcd-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new
s3-client-proxy: bin/etcd-proxy --from localhost:33790 --to localhost:3379 --http-port 3378
s3-client-proxy: bin/etcd-proxy --from localhost:33800 --to localhost:3380 --http-port 3381

189
functional/README.md
View File

@ -1,4 +1,4 @@
# etcd Functional Testing
## etcd Functional Testing
`functional` verifies the correct behavior of etcd under various system and network malfunctions. It sets up an etcd cluster under high pressure loads and continuously injects failures into the cluster. Then it expects the etcd cluster to recover within a few seconds. This has been extremely helpful to find critical bugs.
@ -13,7 +13,7 @@ PASSES=functional ./test
### Run with Docker
```bash
pushd ../..
pushd ..
make build-docker-functional
popd
```
@ -29,3 +29,188 @@ And run [example scripts](./scripts).
# to run only 1 tester round
./scripts/docker-local-tester.sh
```
## etcd Proxy
Proxy layer that simulates various network conditions.
Test locally
```bash
$ ./build
$ ./bin/etcd
$ make build-functional
$ ./bin/etcd-proxy --help
$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar
```
Proxy overhead per request is under 500μs
```bash
$ go build -v -o ./bin/benchmark ./tools/benchmark
$ ./bin/benchmark \
--endpoints localhost:2379 \
--conns 5 \
--clients 15 \
put \
--key-size 48 \
--val-size 50000 \
--total 10000
<<COMMENT
Summary:
Total: 8.4611 secs.
Slowest: 0.1324 secs.
Fastest: 0.0011 secs.
Average: 0.0121 secs.
Stddev: 0.0125 secs.
Requests/sec: 1181.8758
Response time histogram:
0.0011 [1] |
0.0142 [7899] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
0.0273 [1339] |∎∎∎∎∎∎
0.0405 [543] |∎∎
0.0536 [67] |
0.0667 [49] |
0.0798 [9] |
0.0930 [15] |
0.1061 [42] |
0.1192 [21] |
0.1324 [15] |
Latency distribution:
10% in 0.0049 secs.
25% in 0.0064 secs.
50% in 0.0085 secs.
75% in 0.0126 secs.
90% in 0.0243 secs.
95% in 0.0307 secs.
99% in 0.0686 secs.
99.9% in 0.1294 secs.
COMMENT
$ ./bin/benchmark \
--endpoints localhost:23790 \
--conns 5 \
--clients 15 \
put \
--key-size 48 \
--val-size 50000 \
--total 10000
<<COMMENT
Summary:
Total: 9.1128 secs.
Slowest: 0.1363 secs.
Fastest: 0.0015 secs.
Average: 0.0131 secs.
Stddev: 0.0113 secs.
Requests/sec: 1097.3613
Response time histogram:
0.0015 [1] |
0.0150 [7407] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
0.0285 [2017] |∎∎∎∎∎∎∎∎∎∎
0.0419 [440] |∎∎
0.0554 [30] |
0.0689 [13] |
0.0824 [12] |
0.0959 [48] |
0.1093 [2] |
0.1228 [16] |
0.1363 [14] |
Latency distribution:
10% in 0.0054 secs.
25% in 0.0071 secs.
50% in 0.0100 secs.
75% in 0.0153 secs.
90% in 0.0241 secs.
95% in 0.0297 secs.
99% in 0.0584 secs.
99.9% in 0.1312 secs.
COMMENT
```
Delay client transmit
```bash
$ curl -L http://localhost:2378/delay-tx -X PUT \
-d "latency=5s&random-variable=100ms"
# added send latency 5s±100ms (current latency 4.92143955s)
$ curl -L http://localhost:2378/delay-tx
# current send latency 4.92143955s
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:23790 \
--command-timeout=3s \
put foo bar
# Error: context deadline exceeded
$ curl -L http://localhost:2378/delay-tx -X DELETE
# removed latency 4.92143955s
$ curl -L http://localhost:2378/delay-tx
# current send latency 0s
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:23790 \
--command-timeout=3s \
put foo bar
# OK
```
Pause client transmit
```bash
$ curl -L http://localhost:2378/pause-tx -X PUT
# paused forwarding [tcp://localhost:23790 -> tcp://localhost:2379]
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:23790 \
put foo bar
# Error: context deadline exceeded
$ curl -L http://localhost:2378/pause-tx -X DELETE
# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379]
```
Drop client packets
```bash
$ curl -L http://localhost:2378/blackhole-tx -X PUT
# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379]
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar
# Error: context deadline exceeded
$ curl -L http://localhost:2378/blackhole-tx -X DELETE
# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379]
```
Trigger leader election
```bash
$ ./build
$ make build-functional
$ rm -rf /tmp/etcd-proxy-data.s*
$ goreman -f ./functional/Procfile-proxy start
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:13790,localhost:23790,localhost:33790 \
member list
# isolate s1 when s1 is the current leader
$ curl -L http://localhost:1381/blackhole-tx -X PUT
$ curl -L http://localhost:1381/blackhole-rx -X PUT
# s1 becomes follower after election timeout
```

11
functional/build
View File

@ -1,10 +1,11 @@
#!/usr/bin/env bash
if ! [[ "$0" =~ "functional/build" ]]; then
echo "must be run from repository root"
exit 255
echo "must be run from repository root"
exit 255
fi
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-agent ./functional/cmd/etcd-agent
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-tester ./functional/cmd/etcd-tester
CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-s" -o bin/etcd-runner ./functional/cmd/etcd-runner
CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-agent ./functional/cmd/etcd-agent
CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-proxy ./functional/cmd/etcd-proxy
CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-runner ./functional/cmd/etcd-runner
CGO_ENABLED=0 go build -a -v -installsuffix cgo -ldflags "-s" -o ./bin/etcd-tester ./functional/cmd/etcd-tester

14
tools/etcd-test-proxy/main.go → functional/cmd/etcd-proxy/main.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// etcd-test-proxy is a proxy layer that simulates various network conditions.
// etcd-proxy is a proxy layer that simulates various network conditions.
package main
import (
@ -40,13 +40,13 @@ func main() {
// TODO: support TLS
flag.StringVar(&from, "from", "localhost:23790", "Address URL to proxy from.")
flag.StringVar(&to, "to", "localhost:2379", "Address URL to forward.")
flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-test-proxy API.")
flag.IntVar(&httpPort, "http-port", 2378, "Port to serve etcd-proxy API.")
flag.BoolVar(&verbose, "verbose", false, "'true' to run proxy in verbose mode.")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage of %q:\n", os.Args[0])
fmt.Fprintln(os.Stderr, `
etcd-test-proxy simulates various network conditions for etcd testing purposes.
etcd-proxy simulates various network conditions for etcd testing purposes.
See README.md for more examples.
Example:
@ -55,12 +55,12 @@ Example:
$ ./build
$ ./bin/etcd
# build etcd-test-proxy
$ make build-etcd-test-proxy
# build etcd-proxy
$ make build-etcd-proxy
# to test etcd with proxy layer
$ ./bin/etcd-test-proxy --help
$ ./bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose
$ ./bin/etcd-proxy --help
$ ./bin/etcd-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar`)

14
tools/etcd-test-proxy/Procfile
View File

@ -1,14 +0,0 @@
s1: bin/etcd --name s1 --data-dir /tmp/etcd-test-proxy-data.s1 --listen-client-urls http://127.0.0.1:1379 --advertise-client-urls http://127.0.0.1:13790 --listen-peer-urls http://127.0.0.1:1380 --initial-advertise-peer-urls http://127.0.0.1:13800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new
s1-client-proxy: bin/etcd-test-proxy --from localhost:13790 --to localhost:1379 --http-port 1378
s1-peer-proxy: bin/etcd-test-proxy --from localhost:13800 --to localhost:1380 --http-port 1381
s2: bin/etcd --name s2 --data-dir /tmp/etcd-test-proxy-data.s2 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:23790 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:23800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new
s2-client-proxy: bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378
s2-peer-proxy: bin/etcd-test-proxy --from localhost:23800 --to localhost:2380 --http-port 2381
s3: bin/etcd --name s3 --data-dir /tmp/etcd-test-proxy-data.s3 --listen-client-urls http://127.0.0.1:3379 --advertise-client-urls http://127.0.0.1:33790 --listen-peer-urls http://127.0.0.1:3380 --initial-advertise-peer-urls http://127.0.0.1:33800 --initial-cluster-token tkn --initial-cluster 's1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800' --initial-cluster-state new
s3-client-proxy: bin/etcd-test-proxy --from localhost:33790 --to localhost:3379 --http-port 3378
s3-client-proxy: bin/etcd-test-proxy --from localhost:33800 --to localhost:3380 --http-port 3381

184
tools/etcd-test-proxy/README.md
View File

@ -1,184 +0,0 @@
#### etcd-test-proxy
Proxy layer that simulates various network conditions.
Test locally
```bash
$ ./build
$ ./bin/etcd
$ make build-etcd-test-proxy
$ ./bin/etcd-test-proxy --help
$ ./bin/etcd-test-proxy --from localhost:23790 --to localhost:2379 --http-port 2378 --verbose
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:2379 put foo bar
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar
```
Proxy overhead per request is under 500μs
```bash
$ go build -v -o ./bin/benchmark ./tools/benchmark
$ ./bin/benchmark \
--endpoints localhost:2379 \
--conns 5 \
--clients 15 \
put \
--key-size 48 \
--val-size 50000 \
--total 10000
<<COMMENT
Summary:
Total: 8.4611 secs.
Slowest: 0.1324 secs.
Fastest: 0.0011 secs.
Average: 0.0121 secs.
Stddev: 0.0125 secs.
Requests/sec: 1181.8758
Response time histogram:
0.0011 [1] |
0.0142 [7899] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
0.0273 [1339] |∎∎∎∎∎∎
0.0405 [543] |∎∎
0.0536 [67] |
0.0667 [49] |
0.0798 [9] |
0.0930 [15] |
0.1061 [42] |
0.1192 [21] |
0.1324 [15] |
Latency distribution:
10% in 0.0049 secs.
25% in 0.0064 secs.
50% in 0.0085 secs.
75% in 0.0126 secs.
90% in 0.0243 secs.
95% in 0.0307 secs.
99% in 0.0686 secs.
99.9% in 0.1294 secs.
COMMENT
$ ./bin/benchmark \
--endpoints localhost:23790 \
--conns 5 \
--clients 15 \
put \
--key-size 48 \
--val-size 50000 \
--total 10000
<<COMMENT
Summary:
Total: 9.1128 secs.
Slowest: 0.1363 secs.
Fastest: 0.0015 secs.
Average: 0.0131 secs.
Stddev: 0.0113 secs.
Requests/sec: 1097.3613
Response time histogram:
0.0015 [1] |
0.0150 [7407] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
0.0285 [2017] |∎∎∎∎∎∎∎∎∎∎
0.0419 [440] |∎∎
0.0554 [30] |
0.0689 [13] |
0.0824 [12] |
0.0959 [48] |
0.1093 [2] |
0.1228 [16] |
0.1363 [14] |
Latency distribution:
10% in 0.0054 secs.
25% in 0.0071 secs.
50% in 0.0100 secs.
75% in 0.0153 secs.
90% in 0.0241 secs.
95% in 0.0297 secs.
99% in 0.0584 secs.
99.9% in 0.1312 secs.
COMMENT
```
Delay client transmit
```bash
$ curl -L http://localhost:2378/delay-tx -X PUT \
-d "latency=5s&random-variable=100ms"
# added send latency 5s±100ms (current latency 4.92143955s)
$ curl -L http://localhost:2378/delay-tx
# current send latency 4.92143955s
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:23790 \
--command-timeout=3s \
put foo bar
# Error: context deadline exceeded
$ curl -L http://localhost:2378/delay-tx -X DELETE
# removed latency 4.92143955s
$ curl -L http://localhost:2378/delay-tx
# current send latency 0s
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:23790 \
--command-timeout=3s \
put foo bar
# OK
```
Pause client transmit
```bash
$ curl -L http://localhost:2378/pause-tx -X PUT
# paused forwarding [tcp://localhost:23790 -> tcp://localhost:2379]
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:23790 \
put foo bar
# Error: context deadline exceeded
$ curl -L http://localhost:2378/pause-tx -X DELETE
# unpaused forwarding [tcp://localhost:23790 -> tcp://localhost:2379]
```
Drop client packets
```bash
$ curl -L http://localhost:2378/blackhole-tx -X PUT
# blackholed; dropping packets [tcp://localhost:23790 -> tcp://localhost:2379]
$ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar
# Error: context deadline exceeded
$ curl -L http://localhost:2378/blackhole-tx -X DELETE
# unblackholed; restart forwarding [tcp://localhost:23790 -> tcp://localhost:2379]
```
Trigger leader election
```bash
$ ./build
$ make build-etcd-test-proxy
$ rm -rf /tmp/etcd-test-proxy-data.s*
$ goreman -f ./tools/etcd-test-proxy/Procfile start
$ ETCDCTL_API=3 ./bin/etcdctl \
--endpoints localhost:13790,localhost:23790,localhost:33790 \
member list
# isolate s1 when s1 is the current leader
$ curl -L http://localhost:1381/blackhole-tx -X PUT
$ curl -L http://localhost:1381/blackhole-rx -X PUT
# s1 becomes follower after election timeout
```