Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9528 from gyuho/no-op

Browse Source 
functional-tester: add liveness mode
This commit is contained in:
Gyuho Lee 2018-04-04 15:47:53 -07:00 committed by GitHub
commit 49d4fbdde9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 614 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test
View File

@ -234,7 +234,7 @@ function cov_pass {
mkdir -p "$COVERDIR" mkdir -p "$COVERDIR"
# run code coverage for unit and integration tests # run code coverage for unit and integration tests
GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 15m" GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 20m"
# shellcheck disable=SC2206 # shellcheck disable=SC2206
GOCOVFLAGS=($GOCOVFLAGS) GOCOVFLAGS=($GOCOVFLAGS)
failed="" failed=""
@ -292,7 +292,7 @@ function e2e_pass {
# expectation could be different # expectation could be different
USERTIMEOUT="" USERTIMEOUT=""
if [ -z "${TIMEOUT}" ]; then if [ -z "${TIMEOUT}" ]; then
USERTIMEOUT="15m" USERTIMEOUT="20m"
else else
USERTIMEOUT="${TIMEOUT}" USERTIMEOUT="${TIMEOUT}"
fi fi
@ -303,9 +303,9 @@ function e2e_pass {
function integration_e2e_pass { function integration_e2e_pass {
echo "Running integration and e2e tests..." echo "Running integration and e2e tests..."
go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" & go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
e2epid="$!" e2epid="$!"
go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" & go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
intpid="$!" intpid="$!"
wait $e2epid wait $e2epid
wait $intpid wait $intpid
@ -315,7 +315,7 @@ function integration_e2e_pass {
function grpcproxy_pass { function grpcproxy_pass {
go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration" go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration" go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
go test -timeout 15m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e" go test -timeout 20m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e"
} }
function release_pass { function release_pass {

366
tools/functional-tester/rpcpb/rpc.pb.go
View File

@ -104,9 +104,14 @@ const (
FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 9 FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER FailureCase = 9
FailureCase_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 10 FailureCase_DELAY_PEER_PORT_TX_RX_LEADER FailureCase = 10
FailureCase_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 11 FailureCase_DELAY_PEER_PORT_TX_RX_ALL FailureCase = 11
FailureCase_FAILPOINTS FailureCase = 100 // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period
FailureCase_NO_FAIL FailureCase = 200 // while stressers are still sending requests.
FailureCase_EXTERNAL FailureCase = 300 FailureCase_NO_FAIL_WITH_STRESS FailureCase = 100
// NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection
// with all stressers stopped.
FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS FailureCase = 101
FailureCase_FAILPOINTS FailureCase = 200
FailureCase_EXTERNAL FailureCase = 300
) )
var FailureCase_name = map[int32]string{ var FailureCase_name = map[int32]string{
@ -122,8 +127,9 @@ var FailureCase_name = map[int32]string{
9: "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", 9: "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
10: "DELAY_PEER_PORT_TX_RX_LEADER", 10: "DELAY_PEER_PORT_TX_RX_LEADER",
11: "DELAY_PEER_PORT_TX_RX_ALL", 11: "DELAY_PEER_PORT_TX_RX_ALL",
100: "FAILPOINTS", 100: "NO_FAIL_WITH_STRESS",
200: "NO_FAIL", 101: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
200: "FAILPOINTS",
300: "EXTERNAL", 300: "EXTERNAL",
} }
var FailureCase_value = map[string]int32{ var FailureCase_value = map[string]int32{
@ -139,8 +145,9 @@ var FailureCase_value = map[string]int32{
"DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": 9, "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": 9,
"DELAY_PEER_PORT_TX_RX_LEADER": 10, "DELAY_PEER_PORT_TX_RX_LEADER": 10,
"DELAY_PEER_PORT_TX_RX_ALL": 11, "DELAY_PEER_PORT_TX_RX_ALL": 11,
"FAILPOINTS": 100, "NO_FAIL_WITH_STRESS": 100,
"NO_FAIL": 200, "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 101,
"FAILPOINTS": 200,
"EXTERNAL": 300, "EXTERNAL": 300,
} }
@ -154,30 +161,27 @@ type StressType int32
const ( const (
StressType_KV StressType = 0 StressType_KV StressType = 0
StressType_LEASE StressType = 1 StressType_LEASE StressType = 1
StressType_NO_STRESS StressType = 2 StressType_ELECTION_RUNNER StressType = 2
StressType_ELECTION_RUNNER StressType = 3 StressType_WATCH_RUNNER StressType = 3
StressType_WATCH_RUNNER StressType = 4 StressType_LOCK_RACER_RUNNER StressType = 4
StressType_LOCK_RACER_RUNNER StressType = 5 StressType_LEASE_RUNNER StressType = 5
StressType_LEASE_RUNNER StressType = 6
) )
var StressType_name = map[int32]string{ var StressType_name = map[int32]string{
0: "KV", 0: "KV",
1: "LEASE", 1: "LEASE",
2: "NO_STRESS", 2: "ELECTION_RUNNER",
3: "ELECTION_RUNNER", 3: "WATCH_RUNNER",
4: "WATCH_RUNNER", 4: "LOCK_RACER_RUNNER",
5: "LOCK_RACER_RUNNER", 5: "LEASE_RUNNER",
6: "LEASE_RUNNER",
} }
var StressType_value = map[string]int32{ var StressType_value = map[string]int32{
"KV": 0, "KV": 0,
"LEASE": 1, "LEASE": 1,
"NO_STRESS": 2, "ELECTION_RUNNER": 2,
"ELECTION_RUNNER": 3, "WATCH_RUNNER": 3,
"WATCH_RUNNER": 4, "LOCK_RACER_RUNNER": 4,
"LOCK_RACER_RUNNER": 5, "LEASE_RUNNER": 5,
"LEASE_RUNNER": 6,
} }
func (x StressType) String() string { func (x StressType) String() string {
@ -255,12 +259,14 @@ type Tester struct {
EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"` EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"`
// FailureCases is the selected test cases to schedule. // FailureCases is the selected test cases to schedule.
// If empty, run all failure cases. // If empty, run all failure cases.
// TODO: support no-op
FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"` FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"`
// FailureDelayMs is the delay duration after failure is injected.
// Useful when triggering snapshot or no-op failure cases.
FailureDelayMs uint32 `protobuf:"varint,32,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"`
// FailureShuffle is true to randomize failure injecting order. // FailureShuffle is true to randomize failure injecting order.
FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"` FailureShuffle bool `protobuf:"varint,33,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"`
// FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
FailpointCommands []string `protobuf:"bytes,33,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"` FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"`
// RunnerExecPath is a path of etcd-runner binary. // RunnerExecPath is a path of etcd-runner binary.
RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"` RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"`
// ExternalExecPath is a path of script for enabling/disabling an external fault injector. // ExternalExecPath is a path of script for enabling/disabling an external fault injector.
@ -280,8 +286,11 @@ type Tester struct {
StressKeySuffixRangeTxn int32 `protobuf:"varint,105,opt,name=StressKeySuffixRangeTxn,proto3" json:"StressKeySuffixRangeTxn,omitempty" yaml:"stress-key-suffix-range-txn"` StressKeySuffixRangeTxn int32 `protobuf:"varint,105,opt,name=StressKeySuffixRangeTxn,proto3" json:"StressKeySuffixRangeTxn,omitempty" yaml:"stress-key-suffix-range-txn"`
// StressKeyTxnOps is the number of operations per a transaction (max 64). // StressKeyTxnOps is the number of operations per a transaction (max 64).
StressKeyTxnOps int32 `protobuf:"varint,106,opt,name=StressKeyTxnOps,proto3" json:"StressKeyTxnOps,omitempty" yaml:"stress-key-txn-ops"` StressKeyTxnOps int32 `protobuf:"varint,106,opt,name=StressKeyTxnOps,proto3" json:"StressKeyTxnOps,omitempty" yaml:"stress-key-txn-ops"`
// StressClients is the number of concurrent stressing clients
// with "one" shared TCP connection.
StressClients int32 `protobuf:"varint,201,opt,name=StressClients,proto3" json:"StressClients,omitempty" yaml:"stress-clients"`
// StressQPS is the maximum number of stresser requests per second. // StressQPS is the maximum number of stresser requests per second.
StressQPS int32 `protobuf:"varint,107,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"` StressQPS int32 `protobuf:"varint,202,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"`
} }
func (m *Tester) Reset() { *m = Tester{} } func (m *Tester) Reset() { *m = Tester{} }
@ -774,11 +783,18 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
i += copy(dAtA[i:], s) i += copy(dAtA[i:], s)
} }
} }
if m.FailureShuffle { if m.FailureDelayMs != 0 {
dAtA[i] = 0x80 dAtA[i] = 0x80
i++ i++
dAtA[i] = 0x2 dAtA[i] = 0x2
i++ i++
i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs))
}
if m.FailureShuffle {
dAtA[i] = 0x88
i++
dAtA[i] = 0x2
i++
if m.FailureShuffle { if m.FailureShuffle {
dAtA[i] = 1 dAtA[i] = 1
} else { } else {
@ -788,7 +804,7 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
} }
if len(m.FailpointCommands) > 0 { if len(m.FailpointCommands) > 0 {
for _, s := range m.FailpointCommands { for _, s := range m.FailpointCommands {
dAtA[i] = 0x8a dAtA[i] = 0x92
i++ i++
dAtA[i] = 0x2 dAtA[i] = 0x2
i++ i++
@ -871,10 +887,17 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
i++ i++
i = encodeVarintRpc(dAtA, i, uint64(m.StressKeyTxnOps)) i = encodeVarintRpc(dAtA, i, uint64(m.StressKeyTxnOps))
} }
if m.StressQPS != 0 { if m.StressClients != 0 {
dAtA[i] = 0xd8 dAtA[i] = 0xc8
i++ i++
dAtA[i] = 0x6 dAtA[i] = 0xc
i++
i = encodeVarintRpc(dAtA, i, uint64(m.StressClients))
}
if m.StressQPS != 0 {
dAtA[i] = 0xd0
i++
dAtA[i] = 0xc
i++ i++
i = encodeVarintRpc(dAtA, i, uint64(m.StressQPS)) i = encodeVarintRpc(dAtA, i, uint64(m.StressQPS))
} }
@ -1111,6 +1134,9 @@ func (m *Tester) Size() (n int) {
n += 2 + l + sovRpc(uint64(l)) n += 2 + l + sovRpc(uint64(l))
} }
} }
if m.FailureDelayMs != 0 {
n += 2 + sovRpc(uint64(m.FailureDelayMs))
}
if m.FailureShuffle { if m.FailureShuffle {
n += 3 n += 3
} }
@ -1149,6 +1175,9 @@ func (m *Tester) Size() (n int) {
if m.StressKeyTxnOps != 0 { if m.StressKeyTxnOps != 0 {
n += 2 + sovRpc(uint64(m.StressKeyTxnOps)) n += 2 + sovRpc(uint64(m.StressKeyTxnOps))
} }
if m.StressClients != 0 {
n += 2 + sovRpc(uint64(m.StressClients))
}
if m.StressQPS != 0 { if m.StressQPS != 0 {
n += 2 + sovRpc(uint64(m.StressQPS)) n += 2 + sovRpc(uint64(m.StressQPS))
} }
@ -2167,6 +2196,25 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex])) m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex iNdEx = postIndex
case 32: case 32:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType)
}
m.FailureDelayMs = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.FailureDelayMs |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 33:
if wireType != 0 { if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType) return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType)
} }
@ -2186,7 +2234,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
} }
} }
m.FailureShuffle = bool(v != 0) m.FailureShuffle = bool(v != 0)
case 33: case 34:
if wireType != 2 { if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType) return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType)
} }
@ -2397,7 +2445,26 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
break break
} }
} }
case 107: case 201:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field StressClients", wireType)
}
m.StressClients = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.StressClients |= (int32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 202:
if wireType != 0 { if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field StressQPS", wireType) return fmt.Errorf("proto: wrong wireType = %d for field StressQPS", wireType)
} }
@ -2779,119 +2846,122 @@ var (
func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) } func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) }
var fileDescriptorRpc = []byte{ var fileDescriptorRpc = []byte{
// 1813 bytes of a gzipped FileDescriptorProto // 1870 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x57, 0x51, 0x73, 0xda, 0xca, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x58, 0x7f, 0x72, 0xdb, 0xb8,
0x15, 0x36, 0xb6, 0xc1, 0x66, 0x6d, 0x6c, 0xdd, 0xb5, 0x1d, 0x2b, 0x4e, 0x62, 0x71, 0xd5, 0xde, 0x15, 0xb6, 0x6c, 0xcb, 0xb6, 0xe0, 0x5f, 0x0c, 0x6c, 0xc7, 0x8c, 0x93, 0x98, 0x5e, 0x6e, 0x37,
0x8c, 0xeb, 0x19, 0x39, 0x6d, 0xee, 0x4c, 0x67, 0x7a, 0xe7, 0x76, 0x52, 0xc0, 0xca, 0x8d, 0x6b, 0x75, 0x3d, 0x43, 0xa7, 0xcd, 0x76, 0x3a, 0xd3, 0x9d, 0xed, 0xa4, 0x92, 0xcc, 0xac, 0x5d, 0x33,
0x05, 0xc8, 0x82, 0x93, 0xf4, 0x89, 0x0a, 0x69, 0x01, 0xd5, 0x42, 0x52, 0x56, 0x8b, 0x2f, 0xdc, 0x92, 0x02, 0xc9, 0x49, 0xfa, 0x97, 0x4a, 0x51, 0x90, 0xc4, 0x9a, 0x22, 0x19, 0x00, 0xf2, 0x4a,
0x87, 0xbe, 0x76, 0xfa, 0x0f, 0xda, 0xff, 0xd0, 0xfe, 0x8f, 0xb4, 0xbd, 0x0f, 0xfd, 0x05, 0xb4, 0x7b, 0x81, 0x4e, 0x6f, 0xd0, 0x43, 0xb4, 0xf7, 0x48, 0xdb, 0x9d, 0x4e, 0x4f, 0xa0, 0xb6, 0xe9,
0x4d, 0xff, 0x01, 0x2f, 0x7d, 0xed, 0xec, 0xae, 0x80, 0x15, 0xe0, 0xf6, 0x8d, 0x3d, 0xe7, 0xfb, 0x0d, 0x74, 0x81, 0x76, 0x00, 0x50, 0x12, 0x28, 0xc9, 0xed, 0x7f, 0xc2, 0x7b, 0xdf, 0xf7, 0xf1,
0xbe, 0x3d, 0x7b, 0x74, 0xce, 0xd9, 0x05, 0xec, 0x93, 0xc8, 0x89, 0xda, 0xcf, 0x48, 0xe4, 0x5c, 0xe1, 0x01, 0xef, 0x3d, 0xd8, 0x60, 0x97, 0xc4, 0x5e, 0xdc, 0x78, 0x4e, 0x62, 0xef, 0x3c, 0x26,
0x44, 0x24, 0xa4, 0x21, 0xcc, 0x72, 0xc3, 0x89, 0xd1, 0xf5, 0x68, 0x6f, 0xd0, 0xbe, 0x70, 0xc2, 0x11, 0x8b, 0x60, 0x56, 0x18, 0x8e, 0xac, 0xb6, 0xcf, 0x3a, 0xbd, 0xc6, 0xb9, 0x17, 0x75, 0x9f,
0xfe, 0xb3, 0x6e, 0xd8, 0x0d, 0x9f, 0x71, 0x6f, 0x7b, 0xd0, 0xe1, 0x2b, 0xbe, 0xe0, 0xbf, 0x04, 0xb7, 0xa3, 0x76, 0xf4, 0x5c, 0x78, 0x1b, 0xbd, 0x96, 0x58, 0x89, 0x85, 0xf8, 0x25, 0x59, 0xe6,
0x4b, 0xff, 0x7e, 0x0b, 0x6c, 0x9a, 0xd4, 0x71, 0xe1, 0x0f, 0xc0, 0x66, 0xd5, 0xee, 0x63, 0x35, 0xf7, 0xeb, 0x60, 0xd5, 0x66, 0x5e, 0x13, 0x7e, 0x0e, 0x56, 0x4b, 0x6e, 0x17, 0xeb, 0x99, 0x93,
0x53, 0xcc, 0x9c, 0xe5, 0xcb, 0xfb, 0x93, 0xb1, 0xb6, 0x33, 0xb2, 0xfb, 0xfe, 0x57, 0x7a, 0x60, 0xcc, 0x69, 0xae, 0xb0, 0x3b, 0x1a, 0x1a, 0x9b, 0x03, 0xb7, 0x1b, 0x7c, 0x65, 0x86, 0x6e, 0x17,
0xf7, 0xb1, 0x8e, 0xb8, 0x13, 0x1a, 0x60, 0xeb, 0xd2, 0xa6, 0xf6, 0xa5, 0x47, 0xd4, 0x75, 0x8e, 0x9b, 0x48, 0x38, 0xa1, 0x05, 0xd6, 0x2f, 0x5c, 0xe6, 0x5e, 0xf8, 0x44, 0x5f, 0x16, 0xb8, 0xbd,
0x3b, 0x98, 0x8c, 0xb5, 0x7d, 0x81, 0x73, 0x6d, 0x6a, 0x1b, 0xae, 0x47, 0x74, 0x34, 0xc5, 0xc0, 0xd1, 0xd0, 0xd8, 0x95, 0xb8, 0xa6, 0xcb, 0x5c, 0xab, 0xe9, 0x13, 0x13, 0x8d, 0x31, 0xf0, 0x0c,
0x73, 0x90, 0x7b, 0x57, 0xb2, 0x18, 0x7a, 0x83, 0xa3, 0xe1, 0x64, 0xac, 0xed, 0x09, 0xf4, 0xb7, 0xac, 0xbd, 0xcb, 0x3b, 0x1c, 0xbd, 0x22, 0xd0, 0x70, 0x34, 0x34, 0x76, 0x24, 0xfa, 0x5b, 0x37,
0xb6, 0x2f, 0xc0, 0x09, 0x02, 0x5e, 0x01, 0xc5, 0xf2, 0x62, 0x8a, 0x83, 0x8a, 0xef, 0xe1, 0x80, 0x90, 0xe0, 0x04, 0x01, 0xaf, 0x80, 0xe6, 0xf8, 0x94, 0xe1, 0xb0, 0x18, 0xf8, 0x38, 0x64, 0x37,
0xde, 0x20, 0x2b, 0x56, 0x37, 0x8b, 0x1b, 0x67, 0xf9, 0xf2, 0x93, 0xc9, 0x58, 0x7b, 0x28, 0x58, 0xc8, 0xa1, 0xfa, 0xea, 0xc9, 0xca, 0x69, 0xae, 0xf0, 0x74, 0x34, 0x34, 0x1e, 0x49, 0x56, 0x20,
0x3e, 0x47, 0x18, 0x0e, 0x87, 0x18, 0x03, 0xe2, 0xc7, 0x3a, 0x5a, 0xa2, 0x41, 0x04, 0x0e, 0x4a, 0x10, 0x96, 0x27, 0x20, 0x56, 0x8f, 0x04, 0xd4, 0x44, 0x73, 0x34, 0x88, 0xc0, 0x5e, 0xbe, 0x79,
0xee, 0x1d, 0x26, 0xd4, 0x8b, 0xb1, 0xa4, 0x96, 0xe5, 0x6a, 0xc5, 0xc9, 0x58, 0x7b, 0x2c, 0xd4, 0x87, 0x09, 0xf3, 0x29, 0x56, 0xd4, 0xb2, 0x42, 0xed, 0x64, 0x34, 0x34, 0x9e, 0x48, 0x35, 0x77,
0xec, 0x29, 0x28, 0x2d, 0xb8, 0x8a, 0x0c, 0x2b, 0x60, 0x4f, 0xec, 0x53, 0xc7, 0x98, 0x70, 0xb9, 0x0c, 0x4a, 0x0b, 0x2e, 0x22, 0xc3, 0x22, 0xd8, 0x91, 0xdf, 0xa9, 0x60, 0x4c, 0x84, 0xdc, 0x9a,
0x1c, 0x97, 0x7b, 0x34, 0x19, 0x6b, 0xc7, 0xa9, 0xe0, 0x22, 0x8c, 0x49, 0xa2, 0xb4, 0x40, 0x81, 0x90, 0x7b, 0x3c, 0x1a, 0x1a, 0x87, 0xa9, 0xe0, 0x62, 0x8c, 0x49, 0xa2, 0x34, 0x43, 0x81, 0x0d,
0x6d, 0xa0, 0x5e, 0x05, 0x1e, 0xf5, 0x6c, 0x7f, 0xb6, 0xc5, 0x4c, 0x6e, 0x8b, 0xcb, 0x3d, 0x9d, 0xa0, 0x5f, 0x85, 0x3e, 0xf3, 0xdd, 0x60, 0xf2, 0x89, 0x89, 0xdc, 0xba, 0x90, 0x7b, 0x36, 0x1a,
0x8c, 0x35, 0x5d, 0xc8, 0x79, 0x02, 0x69, 0xcc, 0xa3, 0x94, 0x94, 0xef, 0xd5, 0x81, 0x65, 0xb0, 0x1a, 0xa6, 0x94, 0xf3, 0x25, 0xd2, 0x9a, 0x46, 0xa9, 0x28, 0xdf, 0xab, 0x03, 0x0b, 0x60, 0x27,
0x97, 0xf8, 0x2a, 0xfe, 0x20, 0xa6, 0x98, 0xa8, 0xdb, 0x3c, 0xf7, 0x27, 0x93, 0xb1, 0xf6, 0x20, 0xf1, 0x15, 0x83, 0x1e, 0x65, 0x98, 0xe8, 0x1b, 0x22, 0xf7, 0x47, 0xa3, 0xa1, 0xf1, 0x30, 0xad,
0xad, 0xec, 0x08, 0x80, 0x8e, 0x16, 0x18, 0x2c, 0x81, 0x69, 0x4b, 0x83, 0xda, 0x14, 0xab, 0x79, 0xec, 0x49, 0x80, 0x89, 0x66, 0x18, 0x3c, 0x81, 0x69, 0x4b, 0x95, 0xb9, 0x0c, 0xeb, 0x39, 0x21,
0x2e, 0x24, 0x25, 0x70, 0x41, 0xc8, 0x88, 0x19, 0x4c, 0x47, 0xab, 0xc8, 0xcb, 0x9a, 0xcd, 0xf0, 0xa4, 0x24, 0x70, 0x46, 0xc8, 0xa2, 0x1c, 0x66, 0xa2, 0x45, 0xe4, 0x79, 0xcd, 0x5a, 0x74, 0x8b,
0x16, 0x07, 0x2a, 0xf8, 0x7f, 0x9a, 0x94, 0xc1, 0x96, 0x34, 0x39, 0x19, 0xbe, 0x00, 0x85, 0x46, 0x43, 0x1d, 0xfc, 0x3f, 0x4d, 0xc6, 0x61, 0x73, 0x9a, 0x82, 0x0c, 0x5f, 0x82, 0xed, 0x6a, 0xe8,
0x60, 0x47, 0x71, 0x2f, 0xa4, 0x95, 0x70, 0x10, 0x50, 0x75, 0xa7, 0x98, 0x39, 0xdb, 0x28, 0x3f, 0xc6, 0xb4, 0x13, 0xb1, 0x62, 0xd4, 0x0b, 0x99, 0xbe, 0x79, 0x92, 0x39, 0x5d, 0x29, 0x3c, 0x1a,
0x9c, 0x8c, 0xb5, 0x23, 0xa1, 0x16, 0x27, 0x6e, 0xc3, 0x61, 0x7e, 0x1d, 0xa5, 0xf1, 0xd0, 0x02, 0x0d, 0x8d, 0x03, 0xa9, 0x46, 0x13, 0xb7, 0xe5, 0x71, 0xbf, 0x89, 0xd2, 0x78, 0xe8, 0x80, 0x07,
0x9f, 0xbd, 0x19, 0x84, 0xd4, 0x2e, 0xdb, 0xce, 0x2d, 0x0e, 0xdc, 0xf2, 0x88, 0xe2, 0x58, 0xdd, 0x6f, 0x7a, 0x11, 0x73, 0x0b, 0xae, 0x77, 0x8b, 0xc3, 0x66, 0x61, 0xc0, 0x30, 0xd5, 0xb7, 0x84,
0xe5, 0x22, 0xa7, 0x93, 0xb1, 0x76, 0x22, 0x44, 0x3e, 0x30, 0x88, 0xd1, 0x16, 0x18, 0xa3, 0xcd, 0xc8, 0xf1, 0x68, 0x68, 0x1c, 0x49, 0x91, 0x0f, 0x1c, 0x62, 0x35, 0x24, 0xc6, 0x6a, 0x70, 0x90,
0x40, 0x3a, 0x5a, 0x26, 0xb2, 0xee, 0xa8, 0x13, 0xfc, 0x36, 0xa4, 0x58, 0x2d, 0x14, 0x33, 0x67, 0x89, 0xe6, 0x89, 0xbc, 0x3a, 0x2a, 0x04, 0xbf, 0x8d, 0x18, 0xd6, 0xb7, 0x4f, 0x32, 0xa7, 0x1b,
0xdb, 0x72, 0x77, 0x44, 0x04, 0x1b, 0x77, 0x21, 0xcb, 0xce, 0x14, 0x23, 0x67, 0x24, 0x24, 0x64, 0x6a, 0x75, 0xc4, 0x04, 0x5b, 0x77, 0x11, 0xcf, 0xce, 0x18, 0xa3, 0x66, 0x24, 0x22, 0xa4, 0x17,
0x10, 0xd1, 0x4a, 0x0f, 0x3b, 0xb7, 0xea, 0x1e, 0xa7, 0xae, 0xca, 0x88, 0x40, 0x19, 0x0e, 0x83, 0xb3, 0x62, 0x07, 0x7b, 0xb7, 0xfa, 0x8e, 0xa0, 0x2e, 0xca, 0x88, 0x44, 0x59, 0x1e, 0x87, 0x29,
0x49, 0x19, 0x91, 0xc8, 0xfa, 0xef, 0xb3, 0x20, 0xf7, 0x1a, 0xf7, 0xdb, 0x98, 0xc0, 0x9f, 0x83, 0x19, 0x51, 0xc8, 0xe6, 0xef, 0xb3, 0x60, 0xed, 0x35, 0xee, 0x36, 0x30, 0x81, 0xbf, 0x00, 0x5b,
0x5d, 0xd6, 0xd8, 0xe6, 0x10, 0x3b, 0x75, 0x9b, 0xf6, 0x92, 0xc6, 0x96, 0x72, 0x83, 0xa9, 0xe3, 0xbc, 0xb0, 0xed, 0x3e, 0xf6, 0x2a, 0x2e, 0xeb, 0x24, 0x85, 0xad, 0xe4, 0x06, 0x33, 0xaf, 0x69,
0x1a, 0x78, 0x88, 0x1d, 0x23, 0xb2, 0x69, 0x4f, 0x47, 0x29, 0x38, 0xfc, 0x12, 0xe4, 0x4b, 0x5d, 0xe1, 0x3e, 0xf6, 0xac, 0xd8, 0x65, 0x1d, 0x13, 0xa5, 0xe0, 0xf0, 0x4b, 0x90, 0xcb, 0xb7, 0x71,
0x1c, 0xd0, 0x92, 0xeb, 0x12, 0x9e, 0xd7, 0x7c, 0xf9, 0x68, 0x32, 0xd6, 0x3e, 0x4b, 0x5a, 0x87, 0xc8, 0xf2, 0xcd, 0x26, 0x11, 0x79, 0xcd, 0x15, 0x0e, 0x46, 0x43, 0xe3, 0x41, 0x52, 0x3a, 0xdc,
0xb9, 0x0c, 0xdb, 0x75, 0x89, 0x8e, 0xe6, 0x38, 0x96, 0xcf, 0x97, 0xb6, 0xe7, 0x47, 0xa1, 0x17, 0x65, 0xb9, 0xcd, 0x26, 0x31, 0xd1, 0x14, 0xc7, 0xf3, 0xf9, 0xca, 0xf5, 0x83, 0x38, 0xf2, 0x43,
0xd0, 0x57, 0xcd, 0x66, 0x9d, 0x93, 0x77, 0x39, 0x59, 0xca, 0x67, 0x67, 0x0a, 0x31, 0x7a, 0x94, 0x76, 0x59, 0xab, 0x55, 0x04, 0x79, 0x4b, 0x90, 0x95, 0x7c, 0xb6, 0xc6, 0x10, 0xab, 0xc3, 0x58,
0x46, 0x89, 0xca, 0x32, 0x91, 0xe5, 0xb3, 0x6c, 0xc7, 0x98, 0xcd, 0x0f, 0xbc, 0x38, 0x6d, 0xda, 0x9c, 0xa8, 0xcc, 0x13, 0x79, 0x3e, 0x0b, 0x2e, 0xc5, 0xbc, 0x7f, 0xe0, 0xd9, 0x6e, 0xd3, 0x70,
0x76, 0x8c, 0x93, 0x69, 0x93, 0x60, 0xe0, 0x57, 0x60, 0x87, 0x9d, 0xc0, 0x0a, 0xbb, 0xfc, 0xbc, 0x29, 0x4e, 0xba, 0x4d, 0x82, 0x81, 0x5f, 0x81, 0x4d, 0xbe, 0x03, 0x27, 0x6a, 0x8b, 0xfd, 0xb6,
0x1d, 0x4e, 0x51, 0x27, 0x63, 0xed, 0x50, 0x3a, 0xaf, 0x1f, 0x76, 0x93, 0xe3, 0xca, 0x60, 0x58, 0x04, 0x45, 0x1f, 0x0d, 0x8d, 0x7d, 0x65, 0xbf, 0x41, 0xd4, 0x4e, 0xb6, 0xab, 0x82, 0x61, 0x1e,
0x02, 0x05, 0xb6, 0x14, 0x0d, 0xdf, 0xb4, 0x1a, 0xea, 0x5f, 0x32, 0xfc, 0x33, 0x48, 0x5d, 0xc3, 0x6c, 0xf3, 0xa5, 0x2c, 0xf8, 0x9a, 0x53, 0xd5, 0xff, 0x9c, 0x11, 0xc7, 0xa0, 0x54, 0x8d, 0xa0,
0xe9, 0xc9, 0xa0, 0xa0, 0xac, 0x07, 0xd3, 0x0c, 0xf8, 0x0d, 0xd8, 0x9f, 0x1b, 0xea, 0x24, 0x1c, 0x27, 0x8d, 0x82, 0xf1, 0x1a, 0x4c, 0x33, 0xe0, 0x37, 0x60, 0x77, 0x6a, 0xa8, 0x90, 0xa8, 0x3f,
0x8e, 0xd4, 0xbf, 0x0a, 0x91, 0xc7, 0x93, 0xb1, 0xa6, 0x2e, 0x8b, 0x44, 0x0c, 0xa3, 0xa3, 0x45, 0xd0, 0xff, 0x22, 0x45, 0x9e, 0x8c, 0x86, 0x86, 0x3e, 0x2f, 0x12, 0x73, 0x8c, 0x89, 0x66, 0x59,
0xd6, 0x34, 0x16, 0xd6, 0xd1, 0x42, 0xe6, 0x6f, 0xab, 0x63, 0xe1, 0xe3, 0x20, 0x11, 0x49, 0x33, 0xe3, 0x58, 0x78, 0x45, 0x4b, 0x99, 0xbf, 0x2e, 0x8e, 0x45, 0xb4, 0x83, 0x44, 0x24, 0xcd, 0x80,
0x60, 0x1d, 0xc0, 0xb9, 0xaa, 0x19, 0xb8, 0x3c, 0xaf, 0xea, 0xf7, 0xa2, 0x04, 0xb4, 0xc9, 0x58, 0x15, 0x00, 0xa7, 0xaa, 0x76, 0xd8, 0x14, 0x79, 0xd5, 0xbf, 0x97, 0x57, 0xc0, 0x18, 0x0d, 0x8d,
0x7b, 0xb4, 0x1c, 0x0e, 0x4e, 0x60, 0x3a, 0x5a, 0xc1, 0x85, 0x3f, 0x11, 0xd7, 0x84, 0xfa, 0x67, 0xc7, 0xf3, 0xe1, 0xe0, 0x04, 0x66, 0xa2, 0x05, 0x5c, 0xf8, 0x13, 0x39, 0x26, 0xf4, 0x3f, 0xf1,
0x36, 0xf7, 0x77, 0x9e, 0xef, 0x5c, 0xf0, 0xdb, 0xe6, 0x82, 0xd9, 0xe4, 0xcb, 0x82, 0x09, 0xea, 0xbe, 0xbf, 0xf9, 0x62, 0xf3, 0x5c, 0x4c, 0x9b, 0x73, 0x6e, 0x53, 0x87, 0x05, 0x17, 0x34, 0x91,
0x88, 0x43, 0xf5, 0xff, 0x00, 0x90, 0x6b, 0x62, 0x3e, 0x50, 0x5e, 0x80, 0x82, 0xf8, 0x55, 0xc5, 0x80, 0x9a, 0xff, 0xd9, 0x04, 0x6b, 0x35, 0x2c, 0x1a, 0xca, 0x4b, 0xb0, 0x2d, 0x7f, 0x95, 0x30,
0xf4, 0xdb, 0x90, 0xdc, 0x2e, 0x17, 0x23, 0xe5, 0x6e, 0x23, 0x10, 0x7e, 0x1d, 0xa5, 0xf1, 0xf0, 0xfb, 0x36, 0x22, 0xb7, 0xf3, 0x97, 0x91, 0x09, 0xb7, 0x15, 0x4a, 0xbf, 0x89, 0xd2, 0x78, 0xf8,
0xa7, 0x00, 0x08, 0x03, 0xaf, 0x28, 0x71, 0xf7, 0x3c, 0x98, 0x8c, 0x35, 0x98, 0x62, 0x8b, 0x4a, 0x33, 0x00, 0xa4, 0x41, 0xdc, 0x28, 0x39, 0x7b, 0x1e, 0x8e, 0x86, 0x06, 0x4c, 0xb1, 0xe5, 0x4d,
0x92, 0x90, 0x6c, 0x6c, 0x5f, 0x62, 0xdf, 0x1e, 0x59, 0x36, 0xc5, 0x81, 0x33, 0x7a, 0x1d, 0xf3, 0x52, 0x90, 0xbc, 0x6d, 0x5f, 0xe0, 0xc0, 0x1d, 0x38, 0x2e, 0xc3, 0xa1, 0x37, 0x78, 0x4d, 0xc5,
0x52, 0x2e, 0xc8, 0x63, 0xdb, 0x65, 0x7e, 0xc3, 0x17, 0x00, 0xa3, 0xcf, 0xc6, 0x76, 0x9a, 0x02, 0x55, 0xde, 0x56, 0xdb, 0x76, 0x93, 0xfb, 0xad, 0x40, 0x02, 0xac, 0x2e, 0x6f, 0xdb, 0x69, 0x0a,
0x7f, 0x09, 0x94, 0xb4, 0x05, 0xdd, 0xf1, 0xa2, 0x2e, 0xc8, 0x45, 0xbd, 0x28, 0x63, 0x90, 0x3b, 0xfc, 0x15, 0xd0, 0xd2, 0x16, 0x74, 0x27, 0x2e, 0xf5, 0xb6, 0x7a, 0xa9, 0x67, 0x65, 0x2c, 0x72,
0x1d, 0x2d, 0xf1, 0xd8, 0x41, 0x50, 0x38, 0x08, 0x5c, 0xcb, 0xeb, 0x7b, 0x54, 0x3d, 0x2a, 0x66, 0x67, 0xa2, 0x39, 0x1e, 0xdf, 0x08, 0x8a, 0x7a, 0x61, 0xd3, 0xf1, 0xbb, 0x3e, 0xd3, 0x0f, 0x4e,
0xce, 0xb2, 0xf2, 0x41, 0x08, 0xf3, 0x19, 0x3e, 0x73, 0xea, 0x48, 0x42, 0xc2, 0x5f, 0x80, 0x82, 0x32, 0xa7, 0x59, 0x75, 0x23, 0x84, 0xfb, 0xac, 0x80, 0x3b, 0x4d, 0xa4, 0x20, 0xe1, 0x2f, 0xc1,
0x39, 0xf4, 0x68, 0x2d, 0x60, 0x6d, 0x32, 0x20, 0x58, 0x7d, 0xb0, 0x54, 0x13, 0x43, 0x8f, 0x1a, 0xb6, 0xdd, 0xf7, 0x59, 0x39, 0xe4, 0x65, 0xd2, 0x23, 0x58, 0x7f, 0x38, 0x77, 0x27, 0xfa, 0x3e,
0x61, 0x60, 0x74, 0x04, 0x80, 0xd5, 0x84, 0x4c, 0x80, 0xaf, 0x80, 0x52, 0x09, 0x83, 0x98, 0xdf, 0xb3, 0xa2, 0xd0, 0x6a, 0x49, 0x00, 0xbf, 0x13, 0x2a, 0x01, 0x5e, 0x02, 0xad, 0x18, 0x85, 0x54,
0x48, 0xce, 0x48, 0xcc, 0x9a, 0xe3, 0xc5, 0xfa, 0x74, 0xe6, 0x88, 0xe9, 0x9c, 0x59, 0x62, 0xc1, 0x4c, 0x24, 0x6f, 0x20, 0x7b, 0xcd, 0xe1, 0xec, 0xfd, 0xf4, 0xa6, 0x88, 0x71, 0x9f, 0x99, 0x63,
0x9f, 0x81, 0x1d, 0x33, 0xb0, 0xdb, 0x3e, 0xae, 0x47, 0x24, 0xec, 0xa8, 0x2a, 0x17, 0x39, 0x9e, 0xc1, 0x9f, 0x83, 0x4d, 0x3b, 0x74, 0x1b, 0x01, 0xae, 0xc4, 0x24, 0x6a, 0xe9, 0xba, 0x10, 0x39,
0x8c, 0xb5, 0x83, 0x24, 0x12, 0xee, 0x34, 0x22, 0xe6, 0x65, 0x7d, 0x36, 0xc7, 0xc2, 0xaf, 0xc1, 0x1c, 0x0d, 0x8d, 0xbd, 0x24, 0x12, 0xe1, 0xb4, 0x62, 0xee, 0xe5, 0x75, 0x36, 0xc5, 0xc2, 0xaf,
0x6e, 0x12, 0x4f, 0xc5, 0x8e, 0x71, 0xac, 0x6a, 0xfc, 0xd6, 0x93, 0x9a, 0x34, 0x89, 0xde, 0x70, 0xc1, 0x56, 0x12, 0x4f, 0xd1, 0xa5, 0x98, 0xea, 0x86, 0x98, 0x7a, 0x4a, 0x91, 0x26, 0xd1, 0x5b,
0x98, 0x5b, 0x47, 0x29, 0x34, 0xbb, 0xdb, 0x92, 0x75, 0xa3, 0x37, 0xe8, 0x74, 0x7c, 0xac, 0x16, 0x1e, 0x77, 0x9b, 0x28, 0x85, 0xe6, 0xa7, 0x99, 0xac, 0x45, 0x5e, 0x5f, 0x53, 0xfd, 0x64, 0xf6,
0x17, 0xb3, 0x30, 0xe5, 0xc7, 0x02, 0xa0, 0xa3, 0x05, 0x06, 0xbc, 0x96, 0x46, 0x54, 0x25, 0xec, 0x34, 0xc7, 0x7c, 0x79, 0x1c, 0xe2, 0x34, 0xd3, 0x14, 0x3e, 0x20, 0x13, 0x4b, 0xb5, 0xd3, 0x6b,
0xf7, 0xed, 0xc0, 0x8d, 0xd5, 0xcf, 0x17, 0x1f, 0x1a, 0xf3, 0x11, 0xe5, 0x24, 0x18, 0x79, 0x42, 0xb5, 0x02, 0xac, 0x7f, 0x36, 0x9b, 0xca, 0xb1, 0x08, 0x95, 0x80, 0xa9, 0x46, 0xc2, 0x80, 0xd7,
0x4d, 0x79, 0xac, 0xbc, 0xd0, 0x20, 0x08, 0x30, 0x99, 0x4d, 0xd9, 0x1f, 0xf1, 0xd2, 0x94, 0xca, 0x4a, 0x9f, 0x2b, 0x46, 0xdd, 0xae, 0x1b, 0x36, 0xa9, 0x6e, 0xce, 0xbe, 0x56, 0xa6, 0x7d, 0xce,
0x8b, 0x70, 0xbf, 0x3c, 0x67, 0x17, 0x28, 0xec, 0xe5, 0x63, 0x0e, 0x29, 0x26, 0x81, 0xed, 0xcf, 0x4b, 0x30, 0x6a, 0x9b, 0x1b, 0xf3, 0xf8, 0xae, 0x50, 0x2f, 0x0c, 0x31, 0x99, 0xb4, 0xea, 0x1f,
0x64, 0xce, 0xb9, 0x8c, 0x14, 0x10, 0x4e, 0x10, 0xb2, 0xd0, 0x12, 0x8d, 0x7d, 0x99, 0x06, 0x25, 0x89, 0xfb, 0xad, 0xec, 0x8a, 0x08, 0xbf, 0xda, 0xac, 0x67, 0x28, 0xfc, 0xf9, 0x64, 0xf7, 0x19,
0x38, 0x8e, 0x9b, 0xa3, 0x08, 0xc7, 0x2a, 0xe6, 0xc7, 0x92, 0xbe, 0x4c, 0xcc, 0x9d, 0x06, 0x65, 0x26, 0xa1, 0x1b, 0x4c, 0x64, 0xce, 0x84, 0x8c, 0x12, 0x10, 0x4e, 0x10, 0xaa, 0xd0, 0x1c, 0x8d,
0x5e, 0x1d, 0xc9, 0x58, 0x56, 0x60, 0x62, 0x79, 0x8d, 0x47, 0x0d, 0xef, 0x3b, 0xcc, 0xe7, 0x67, 0x1f, 0x6f, 0x95, 0x11, 0x4c, 0x69, 0x6d, 0x10, 0x63, 0xaa, 0x63, 0xb1, 0x2d, 0xe5, 0x78, 0xa9,
0x56, 0x4e, 0x6d, 0x42, 0xbe, 0xc5, 0x23, 0x23, 0xf6, 0xbe, 0x63, 0x05, 0x96, 0x22, 0xb0, 0xa1, 0x70, 0x5a, 0x8c, 0x7b, 0x4d, 0xa4, 0x62, 0xf9, 0x2d, 0x95, 0xcb, 0x6b, 0x3c, 0xa8, 0xfa, 0xdf,
0x93, 0x32, 0x58, 0x36, 0xe9, 0x62, 0xb5, 0xcb, 0x65, 0xa4, 0xeb, 0x6c, 0x41, 0xc6, 0xf0, 0x19, 0x61, 0xd1, 0x84, 0xb3, 0x6a, 0x6a, 0x13, 0xf2, 0x2d, 0x1e, 0x58, 0xd4, 0xff, 0x8e, 0xdf, 0xd2,
0x4c, 0x47, 0x2b, 0xb8, 0xf0, 0x2d, 0x38, 0x9c, 0x5b, 0x07, 0x9d, 0x8e, 0x37, 0x44, 0x76, 0xd0, 0x14, 0x81, 0x77, 0xae, 0x94, 0xc1, 0x71, 0x49, 0x1b, 0xeb, 0x6d, 0x21, 0xa3, 0xcc, 0xc4, 0x19,
0xc5, 0x6a, 0x8f, 0x6b, 0xea, 0x93, 0xb1, 0x76, 0xba, 0xac, 0xc9, 0x71, 0x06, 0x61, 0x40, 0x1d, 0x19, 0x2b, 0xe0, 0x30, 0x13, 0x2d, 0xe0, 0xc2, 0xb7, 0x60, 0x7f, 0x6a, 0xed, 0xb5, 0x5a, 0x7e,
0xad, 0xe4, 0xc3, 0x5f, 0x83, 0xe3, 0x55, 0xf6, 0xe6, 0x30, 0x50, 0x3d, 0x2e, 0x2d, 0x3d, 0xc3, 0x1f, 0xb9, 0x61, 0x1b, 0xeb, 0x1d, 0xa1, 0x69, 0x8e, 0x86, 0xc6, 0xf1, 0xbc, 0xa6, 0xc0, 0x59,
0xee, 0x91, 0x36, 0xe8, 0x30, 0xd0, 0xd1, 0x7d, 0x32, 0xec, 0x32, 0x98, 0xb9, 0x9a, 0xc3, 0xa0, 0x84, 0x03, 0x4d, 0xb4, 0x90, 0x0f, 0x7f, 0x03, 0x0e, 0x17, 0xd9, 0x6b, 0xfd, 0x50, 0xf7, 0x85,
0x16, 0xc5, 0xea, 0x6f, 0xb8, 0xb2, 0xf4, 0x49, 0x25, 0x65, 0x3a, 0x0c, 0x8c, 0x30, 0x8a, 0x75, 0xb4, 0xf2, 0x96, 0xbb, 0x47, 0xda, 0x62, 0xfd, 0xd0, 0x44, 0xf7, 0xc9, 0xf0, 0x89, 0x32, 0x71,
0xb4, 0xc8, 0x62, 0xd7, 0xb0, 0x30, 0xbd, 0xa9, 0x37, 0xd4, 0x5b, 0x2e, 0x21, 0x5d, 0xc3, 0x89, 0xd5, 0xfa, 0x61, 0x39, 0xa6, 0xfa, 0x6f, 0x85, 0xb2, 0x72, 0xa4, 0x8a, 0x32, 0xeb, 0x87, 0x56,
0xc4, 0x07, 0x46, 0x9d, 0xe3, 0xf4, 0xdf, 0x65, 0xc0, 0x16, 0xc2, 0x1f, 0x06, 0x38, 0xa6, 0xf0, 0x14, 0x53, 0x13, 0xcd, 0xb2, 0xa6, 0xc7, 0x22, 0x9b, 0x3a, 0x95, 0xd3, 0x2d, 0x9b, 0x7a, 0x28,
0x02, 0xe4, 0x6b, 0x11, 0x26, 0x36, 0xf5, 0xc2, 0x80, 0x8f, 0xdd, 0xbd, 0xe7, 0x4a, 0x32, 0xbc, 0x49, 0x1d, 0x39, 0x0b, 0xe8, 0xe4, 0x58, 0x12, 0x02, 0xfc, 0x29, 0xc8, 0x49, 0xc3, 0x9b, 0x4a,
0x67, 0x76, 0x34, 0x87, 0xc0, 0x2f, 0xa6, 0x0f, 0x08, 0x55, 0x4c, 0xfa, 0x42, 0x02, 0x16, 0x46, 0x55, 0x8e, 0xb5, 0xac, 0xfa, 0x1c, 0x48, 0xd8, 0x1f, 0xf8, 0xd7, 0xa7, 0x40, 0xf3, 0x77, 0x19,
0x34, 0x7d, 0x5d, 0x7c, 0x31, 0x9d, 0xed, 0xfc, 0x69, 0x3f, 0x87, 0x09, 0x23, 0x4a, 0x9c, 0xfa, 0xb0, 0x8e, 0xf0, 0x87, 0x1e, 0xa6, 0x0c, 0x9e, 0x83, 0x5c, 0x39, 0xc6, 0xc4, 0x65, 0x7e, 0x14,
0xd7, 0x60, 0x1b, 0xe1, 0x38, 0x0a, 0x83, 0x18, 0x43, 0x15, 0x6c, 0x35, 0x06, 0x8e, 0x83, 0xe3, 0x8a, 0xf6, 0xbf, 0xf3, 0x42, 0x4b, 0x86, 0xc8, 0xc4, 0x8e, 0xa6, 0x10, 0xf8, 0xc5, 0xf8, 0x21,
0x98, 0xc7, 0xb1, 0x8d, 0xa6, 0x4b, 0xf8, 0x00, 0xe4, 0xd8, 0x23, 0x71, 0x10, 0x8b, 0xc9, 0x8e, 0xa3, 0xcb, 0x89, 0xb3, 0x9d, 0x80, 0xa5, 0x11, 0x8d, 0x5f, 0x39, 0x5f, 0x8c, 0x67, 0x8c, 0xf8,
0x92, 0xd5, 0xf9, 0x3f, 0x32, 0x52, 0xf0, 0x70, 0x0f, 0x80, 0x6a, 0x48, 0x1b, 0xd4, 0x26, 0x14, 0x13, 0x63, 0x0a, 0x93, 0x46, 0x94, 0x38, 0xcd, 0xaf, 0xc1, 0x06, 0xc2, 0x34, 0x8e, 0x42, 0x8a,
0xbb, 0xca, 0x1a, 0x3c, 0x04, 0x4a, 0xf2, 0x04, 0xe2, 0x36, 0x76, 0xe7, 0x28, 0x19, 0xb8, 0x0f, 0xa1, 0x0e, 0xd6, 0xab, 0x3d, 0xcf, 0xc3, 0x94, 0x8a, 0x38, 0x36, 0xd0, 0x78, 0x09, 0x1f, 0x82,
0x76, 0x10, 0x8e, 0x67, 0x86, 0x75, 0xb8, 0x0b, 0xb6, 0xaf, 0x3d, 0xdf, 0xe7, 0xab, 0x0d, 0xe6, 0x35, 0xfe, 0x58, 0xed, 0x51, 0x39, 0x61, 0x50, 0xb2, 0x3a, 0xfb, 0x47, 0x46, 0x09, 0x1e, 0xee,
0x66, 0x6d, 0x5c, 0x22, 0x4e, 0xcf, 0xbb, 0xc3, 0xca, 0x26, 0x53, 0xb9, 0xc4, 0x31, 0x25, 0xe1, 0x00, 0x50, 0x8a, 0x58, 0x95, 0xb9, 0x84, 0xe1, 0xa6, 0xb6, 0x04, 0xf7, 0x81, 0x96, 0x3c, 0xc5,
0x88, 0x21, 0xf8, 0x53, 0x46, 0xc9, 0xc2, 0x87, 0xe0, 0xa8, 0xec, 0xdb, 0xce, 0x6d, 0x2f, 0xf4, 0x84, 0x8d, 0xcf, 0x3e, 0x2d, 0x03, 0x77, 0xc1, 0x26, 0xc2, 0x74, 0x62, 0x58, 0x86, 0x5b, 0x60,
0xf9, 0xd3, 0xba, 0x1e, 0x12, 0xda, 0x1c, 0xa2, 0xa1, 0xe2, 0xc2, 0x47, 0xe0, 0xf8, 0x26, 0x68, 0xe3, 0xda, 0x0f, 0x02, 0xb1, 0x5a, 0xe1, 0x6e, 0xde, 0x09, 0xf2, 0xc4, 0xeb, 0xf8, 0x77, 0x58,
0xaf, 0x74, 0x62, 0x78, 0x04, 0x3e, 0xe3, 0x23, 0x3f, 0x65, 0xee, 0xc0, 0x63, 0x70, 0x70, 0x13, 0x5b, 0xe5, 0x2a, 0x17, 0x98, 0x32, 0x12, 0x0d, 0x38, 0x42, 0x3c, 0xa9, 0xb4, 0x2c, 0x7c, 0x04,
0xb8, 0x4b, 0x8e, 0xee, 0xf9, 0x1f, 0x37, 0x44, 0x3c, 0xc9, 0x88, 0x63, 0xfc, 0xeb, 0x2b, 0xcb, 0x0e, 0x0a, 0x81, 0xeb, 0xdd, 0x76, 0xa2, 0x40, 0x3c, 0xf1, 0x2b, 0x11, 0x61, 0xb5, 0x3e, 0xea,
0x6a, 0xd5, 0xaa, 0x66, 0xeb, 0x65, 0xcd, 0xb2, 0x6a, 0xef, 0x4c, 0xa4, 0xac, 0xb1, 0xa8, 0xb9, 0x6b, 0x4d, 0xf8, 0x18, 0x1c, 0xde, 0x84, 0x8d, 0x85, 0x4e, 0x0c, 0x0f, 0xc0, 0x03, 0xd1, 0xef,
0xd9, 0x32, 0x4b, 0x97, 0x26, 0x52, 0x32, 0xf0, 0x14, 0x9c, 0x2c, 0xe1, 0x5a, 0x2f, 0x6b, 0xa8, 0x52, 0xe6, 0x16, 0x3c, 0x04, 0x7b, 0x37, 0x61, 0x73, 0xce, 0xd1, 0x3e, 0xfb, 0xdb, 0x8a, 0x8c,
0x65, 0xd5, 0xaa, 0xdf, 0x28, 0xeb, 0x50, 0x05, 0x87, 0x12, 0x61, 0xee, 0xd9, 0x98, 0x49, 0xbd, 0x27, 0x69, 0xb5, 0x9c, 0x7f, 0x7d, 0xe5, 0x38, 0xf5, 0x72, 0xc9, 0xae, 0xbf, 0x2a, 0x3b, 0x4e,
0xb9, 0xa9, 0xa1, 0x9b, 0xd7, 0xca, 0x26, 0xcf, 0x0f, 0x33, 0x94, 0x2c, 0x4b, 0xc9, 0xc2, 0x73, 0xf9, 0x9d, 0x8d, 0xb4, 0x25, 0x1e, 0xb5, 0x30, 0x3b, 0x76, 0xfe, 0xc2, 0x46, 0x5a, 0x06, 0x1e,
0xf0, 0xb4, 0x6c, 0x95, 0x2a, 0xd7, 0xaf, 0x6a, 0x96, 0xd9, 0xaa, 0x9b, 0x26, 0x6a, 0xd5, 0x6b, 0x83, 0xa3, 0x39, 0x5c, 0xfd, 0x55, 0x19, 0xd5, 0x9d, 0x72, 0xe9, 0x1b, 0x6d, 0x19, 0xea, 0x60,
0xa8, 0xd9, 0x6a, 0xbe, 0x6f, 0xa1, 0xf7, 0xe9, 0xa8, 0x72, 0xf0, 0x87, 0xa0, 0x78, 0x3f, 0x36, 0x5f, 0x21, 0x4c, 0x3d, 0x2b, 0x13, 0xa9, 0x37, 0x37, 0x65, 0x74, 0xf3, 0x5a, 0x5b, 0x15, 0xf9,
0x09, 0x75, 0x0b, 0x7e, 0x0e, 0x9e, 0xdc, 0x8f, 0x62, 0x9b, 0x6e, 0xc3, 0xa7, 0x40, 0xbf, 0x34, 0xe1, 0x86, 0xbc, 0xe3, 0x68, 0x59, 0x78, 0x06, 0x9e, 0x15, 0x9c, 0x7c, 0xf1, 0xfa, 0xb2, 0xec,
0xad, 0xd2, 0xaf, 0xfe, 0xf7, 0x86, 0x79, 0x58, 0x04, 0x8f, 0x57, 0xe3, 0x92, 0xcd, 0x00, 0x7c, 0xd8, 0xf5, 0x8a, 0x6d, 0xa3, 0x7a, 0xa5, 0x8c, 0x6a, 0xf5, 0xda, 0xfb, 0x3a, 0x7a, 0x9f, 0x8e,
0x02, 0x1e, 0xae, 0x46, 0xb0, 0x8d, 0x76, 0x58, 0x09, 0xbd, 0x2c, 0x5d, 0x59, 0xf5, 0xda, 0x55, 0x6a, 0x0d, 0xfe, 0x00, 0x9c, 0xdc, 0x8f, 0x4d, 0x42, 0x5d, 0x87, 0x9f, 0x81, 0xa7, 0xf7, 0xa3,
0xb5, 0xd9, 0x50, 0x5c, 0xb8, 0x0b, 0xb6, 0xaa, 0xb5, 0x16, 0x33, 0x29, 0x1f, 0x33, 0xb0, 0x00, 0xf8, 0x47, 0x37, 0xe0, 0x33, 0x60, 0x5e, 0xd8, 0x4e, 0xfe, 0xd7, 0xff, 0xfb, 0x83, 0x39, 0x78,
0xb6, 0xcd, 0xf7, 0x4d, 0x13, 0x55, 0x4b, 0x96, 0xf2, 0xa7, 0xf5, 0xf3, 0xdf, 0x02, 0x30, 0x1f, 0x02, 0x9e, 0x2c, 0xc6, 0x25, 0x1f, 0x03, 0xf0, 0x29, 0x78, 0xb4, 0x18, 0xc1, 0x3f, 0xb4, 0xc9,
0x90, 0x30, 0x07, 0xd6, 0xaf, 0xdf, 0x2a, 0x6b, 0x30, 0x0f, 0xb2, 0x96, 0x59, 0x6a, 0x98, 0x0a, 0xcf, 0xa1, 0x54, 0xae, 0xbf, 0xca, 0x5f, 0x39, 0xf5, 0x77, 0x57, 0xb5, 0xcb, 0x7a, 0xb5, 0x86,
0xc3, 0xe7, 0xab, 0xb5, 0x56, 0xa3, 0x89, 0xcc, 0x46, 0x43, 0x59, 0x87, 0x07, 0x60, 0xdf, 0xb4, 0xec, 0x6a, 0x55, 0x6b, 0xc2, 0x1f, 0x82, 0xcf, 0x53, 0x8e, 0x52, 0x39, 0xf1, 0xc9, 0xd4, 0x5d,
0xcc, 0x4a, 0xf3, 0xaa, 0x56, 0x6d, 0xa1, 0x9b, 0x6a, 0xd5, 0x44, 0xca, 0x06, 0x54, 0xc0, 0xee, 0xbd, 0xb5, 0x4b, 0x1c, 0x88, 0xe1, 0x2e, 0x00, 0x1c, 0x55, 0x29, 0x5f, 0x95, 0x6a, 0x55, 0xed,
0xbb, 0x52, 0xb3, 0xf2, 0x6a, 0x6a, 0xd9, 0x64, 0x9f, 0xd8, 0xaa, 0x55, 0xae, 0x5b, 0xa8, 0x54, 0x63, 0x06, 0x6e, 0x83, 0x0d, 0xfb, 0x7d, 0xcd, 0x46, 0xa5, 0xbc, 0xa3, 0xfd, 0x71, 0xf9, 0x2c,
0x31, 0xd1, 0xd4, 0x9c, 0x65, 0x40, 0xae, 0x3b, 0xb5, 0xe4, 0x9e, 0xbf, 0x00, 0xf9, 0x26, 0xb1, 0x02, 0x60, 0xda, 0x98, 0xe1, 0x1a, 0x58, 0xbe, 0x7e, 0xab, 0x2d, 0xc1, 0x1c, 0xc8, 0x3a, 0x76,
0x83, 0x38, 0x0a, 0x09, 0x85, 0xcf, 0xe5, 0xc5, 0x5e, 0xd2, 0x6c, 0x49, 0x8f, 0x9f, 0xec, 0xcf, 0xbe, 0x6a, 0x6b, 0x19, 0xb8, 0x07, 0x76, 0x6d, 0xc7, 0x2e, 0xd6, 0xae, 0xca, 0xa5, 0x3a, 0xba,
0xd6, 0xa2, 0xd5, 0xf4, 0xb5, 0xb3, 0xcc, 0x8f, 0x33, 0xe5, 0xc3, 0x8f, 0xff, 0x3a, 0x5d, 0xfb, 0x29, 0x95, 0x6c, 0xa4, 0x2d, 0x43, 0x0d, 0x6c, 0xbd, 0xcb, 0xd7, 0x8a, 0x97, 0x63, 0xcb, 0x0a,
0xf8, 0xe9, 0x34, 0xf3, 0xf7, 0x4f, 0xa7, 0x99, 0x7f, 0x7e, 0x3a, 0xcd, 0xfc, 0xe1, 0xdf, 0xa7, 0xbf, 0x08, 0x4e, 0xb9, 0x78, 0x5d, 0x47, 0xf9, 0xa2, 0x8d, 0xc6, 0xe6, 0x55, 0x0e, 0x14, 0x42,
0x6b, 0xed, 0x1c, 0xff, 0xe3, 0xff, 0xe5, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc2, 0x76, 0x9b, 0x63, 0x4b, 0xf6, 0xc5, 0x4b, 0x90, 0xab, 0x11, 0x37, 0xa4, 0x71, 0x44, 0x18, 0x7c, 0xa1, 0x2e,
0x6a, 0x41, 0x10, 0x00, 0x00, 0x76, 0x92, 0x92, 0x4c, 0x3a, 0xc1, 0xd1, 0xee, 0x64, 0x2d, 0x0b, 0xd2, 0x5c, 0x3a, 0xcd, 0xfc,
0x38, 0x53, 0xd8, 0xff, 0xf8, 0xaf, 0xe3, 0xa5, 0x8f, 0x9f, 0x8e, 0x33, 0x7f, 0xff, 0x74, 0x9c,
0xf9, 0xe7, 0xa7, 0xe3, 0xcc, 0x1f, 0xfe, 0x7d, 0xbc, 0xd4, 0x58, 0x13, 0xff, 0xa6, 0xf8, 0xf2,
0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x33, 0x62, 0x54, 0x24, 0xef, 0x10, 0x00, 0x00,
} }

35
tools/functional-tester/rpcpb/rpc.proto
View File

@ -100,23 +100,24 @@ enum FailureCase {
DELAY_PEER_PORT_TX_RX_LEADER = 10; DELAY_PEER_PORT_TX_RX_LEADER = 10;
DELAY_PEER_PORT_TX_RX_ALL = 11; DELAY_PEER_PORT_TX_RX_ALL = 11;
FAILPOINTS = 100; // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period
// while stressers are still sending requests.
NO_FAIL = 200; NO_FAIL_WITH_STRESS = 100;
// TODO: support no-op of liveness duration // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection
// NO_FAIL_LIVENESS = 201; // with all stressers stopped.
NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS = 101;
FAILPOINTS = 200;
EXTERNAL = 300; EXTERNAL = 300;
} }
enum StressType { enum StressType {
KV = 0; KV = 0;
LEASE = 1; LEASE = 1;
NO_STRESS = 2; ELECTION_RUNNER = 2;
ELECTION_RUNNER = 3; WATCH_RUNNER = 3;
WATCH_RUNNER = 4; LOCK_RACER_RUNNER = 4;
LOCK_RACER_RUNNER = 5; LEASE_RUNNER = 5;
LEASE_RUNNER = 6;
} }
message Tester { message Tester {
@ -140,12 +141,14 @@ message Tester {
// FailureCases is the selected test cases to schedule. // FailureCases is the selected test cases to schedule.
// If empty, run all failure cases. // If empty, run all failure cases.
// TODO: support no-op
repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""]; repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""];
// FailureDelayMs is the delay duration after failure is injected.
// Useful when triggering snapshot or no-op failure cases.
uint32 FailureDelayMs = 32 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""];
// FailureShuffle is true to randomize failure injecting order. // FailureShuffle is true to randomize failure injecting order.
bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""]; bool FailureShuffle = 33 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
// FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)). // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
repeated string FailpointCommands = 33 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""]; repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""];
// RunnerExecPath is a path of etcd-runner binary. // RunnerExecPath is a path of etcd-runner binary.
string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""]; string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""];
@ -167,8 +170,12 @@ message Tester {
int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""]; int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""];
// StressKeyTxnOps is the number of operations per a transaction (max 64). // StressKeyTxnOps is the number of operations per a transaction (max 64).
int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""]; int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""];
// StressClients is the number of concurrent stressing clients
// with "one" shared TCP connection.
int32 StressClients = 201 [(gogoproto.moretags) = "yaml:\"stress-clients\""];
// StressQPS is the maximum number of stresser requests per second. // StressQPS is the maximum number of stresser requests per second.
int32 StressQPS = 107 [(gogoproto.moretags) = "yaml:\"stress-qps\""]; int32 StressQPS = 202 [(gogoproto.moretags) = "yaml:\"stress-qps\""];
} }
message Request { message Request {

74
tools/functional-tester/tester/cluster.go
View File

@ -276,27 +276,29 @@ func (clus *Cluster) updateFailures() {
case "KILL_ALL": case "KILL_ALL":
clus.failures = append(clus.failures, newFailureKillAll()) clus.failures = append(clus.failures, newFailureKillAll())
case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER":
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower()) clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus))
case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": case "BLACKHOLE_PEER_PORT_TX_RX_LEADER":
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader()) clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus))
case "BLACKHOLE_PEER_PORT_TX_RX_ALL": case "BLACKHOLE_PEER_PORT_TX_RX_ALL":
clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll()) clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus))
case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER":
clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus)) clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus))
case "DELAY_PEER_PORT_TX_RX_LEADER": case "DELAY_PEER_PORT_TX_RX_LEADER":
clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus)) clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus))
case "DELAY_PEER_PORT_TX_RX_ALL": case "DELAY_PEER_PORT_TX_RX_ALL":
clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus)) clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus))
case "NO_FAIL_WITH_STRESS":
clus.failures = append(clus.failures, newFailureNoFailWithStress(clus))
case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS":
clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness(clus))
case "EXTERNAL":
clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath))
case "FAILPOINTS": case "FAILPOINTS":
fpFailures, fperr := failpointFailures(clus) fpFailures, fperr := failpointFailures(clus)
if len(fpFailures) == 0 { if len(fpFailures) == 0 {
clus.lg.Info("no failpoints found!", zap.Error(fperr)) clus.lg.Info("no failpoints found!", zap.Error(fperr))
} }
clus.failures = append(clus.failures, fpFailures...) clus.failures = append(clus.failures, fpFailures...)
case "NO_FAIL":
clus.failures = append(clus.failures, newFailureNoOp())
case "EXTERNAL":
clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath))
} }
} }
} }
@ -360,8 +362,8 @@ func (clus *Cluster) updateStresserChecker() {
) )
cs := &compositeStresser{} cs := &compositeStresser{}
for idx := range clus.Members { for _, m := range clus.Members {
cs.stressers = append(cs.stressers, newStresser(clus, idx)) cs.stressers = append(cs.stressers, newStresser(clus, m))
} }
clus.stresser = cs clus.stresser = cs
@ -381,49 +383,6 @@ func (clus *Cluster) updateStresserChecker() {
) )
} }
func (clus *Cluster) startStresser() (err error) {
clus.lg.Info(
"starting stressers",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
err = clus.stresser.Stress()
clus.lg.Info(
"started stressers",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
return err
}
func (clus *Cluster) closeStresser() {
clus.lg.Info(
"closing stressers",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
clus.stresser.Close()
clus.lg.Info(
"closed stressers",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
}
func (clus *Cluster) pauseStresser() {
clus.lg.Info(
"pausing stressers",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
clus.stresser.Pause()
clus.lg.Info(
"paused stressers",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
}
func (clus *Cluster) checkConsistency() (err error) { func (clus *Cluster) checkConsistency() (err error) {
defer func() { defer func() {
if err != nil { if err != nil {
@ -436,7 +395,6 @@ func (clus *Cluster) checkConsistency() (err error) {
) )
return return
} }
err = clus.startStresser()
}() }()
clus.lg.Info( clus.lg.Info(
@ -759,4 +717,12 @@ func (clus *Cluster) defrag() error {
return nil return nil
} }
func (clus *Cluster) Report() int64 { return clus.stresser.ModifiedKeys() } // GetFailureDelayDuration computes failure delay duration.
func (clus *Cluster) GetFailureDelayDuration() time.Duration {
return time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond
}
// Report reports the number of modified keys.
func (clus *Cluster) Report() int64 {
return clus.stresser.ModifiedKeys()
}

4
tools/functional-tester/tester/cluster_test.go
View File

@ -131,7 +131,10 @@ func Test_newCluster(t *testing.T) {
"DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER", "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
"DELAY_PEER_PORT_TX_RX_LEADER", "DELAY_PEER_PORT_TX_RX_LEADER",
"DELAY_PEER_PORT_TX_RX_ALL", "DELAY_PEER_PORT_TX_RX_ALL",
"NO_FAIL_WITH_STRESS",
"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
}, },
FailureDelayMs: 7000,
FailureShuffle: true, FailureShuffle: true,
FailpointCommands: []string{`panic("etcd-tester")`}, FailpointCommands: []string{`panic("etcd-tester")`},
RunnerExecPath: "/etcd-runner", RunnerExecPath: "/etcd-runner",
@ -142,6 +145,7 @@ func Test_newCluster(t *testing.T) {
StressKeySuffixRange: 250000, StressKeySuffixRange: 250000,
StressKeySuffixRangeTxn: 100, StressKeySuffixRangeTxn: 100,
StressKeyTxnOps: 10, StressKeyTxnOps: 10,
StressClients: 100,
StressQPS: 1000, StressQPS: 1000,
}, },
} }

84
tools/functional-tester/tester/cluster_tester.go
View File

@ -19,6 +19,8 @@ import (
"os" "os"
"time" "time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -29,7 +31,6 @@ const compactQPS = 50000
// StartTester starts tester. // StartTester starts tester.
func (clus *Cluster) StartTester() { func (clus *Cluster) StartTester() {
// TODO: upate status // TODO: upate status
clus.startStresser()
var preModifiedKey int64 var preModifiedKey int64
for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
@ -116,68 +117,80 @@ func (clus *Cluster) doRound() error {
zap.Int("round", clus.rd), zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()), zap.Strings("failures", clus.failureStrings()),
) )
for i, f := range clus.failures {
for i, fa := range clus.failures {
clus.cs = i clus.cs = i
caseTotalCounter.WithLabelValues(f.Desc()).Inc() caseTotalCounter.WithLabelValues(fa.Desc()).Inc()
clus.lg.Info(
"failure case START",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", fa.Desc()),
)
clus.lg.Info("wait health before injecting failures") clus.lg.Info("wait health before injecting failures")
if err := clus.WaitHealth(); err != nil { if err := clus.WaitHealth(); err != nil {
return fmt.Errorf("wait full health error: %v", err) return fmt.Errorf("wait full health error: %v", err)
} }
stressStarted := false
if fa.FailureCase() != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS {
clus.lg.Info(
"starting stressers before injecting failures",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", fa.Desc()),
)
if err := clus.stresser.Stress(); err != nil {
return fmt.Errorf("start stresser error: %v", err)
}
stressStarted = true
}
clus.lg.Info( clus.lg.Info(
"injecting failure", "injecting",
zap.Int("round", clus.rd), zap.Int("round", clus.rd),
zap.Int("case", clus.cs), zap.Int("case", clus.cs),
zap.String("desc", f.Desc()), zap.String("desc", fa.Desc()),
) )
if err := f.Inject(clus); err != nil { if err := fa.Inject(clus); err != nil {
return fmt.Errorf("injection error: %v", err) return fmt.Errorf("injection error: %v", err)
} }
clus.lg.Info(
"injected failure",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", f.Desc()),
)
// if run local, recovering server may conflict // if run local, recovering server may conflict
// with stressing client ports // with stressing client ports
// TODO: use unix for local tests // TODO: use unix for local tests
clus.lg.Info( clus.lg.Info(
"recovering failure", "recovering",
zap.Int("round", clus.rd), zap.Int("round", clus.rd),
zap.Int("case", clus.cs), zap.Int("case", clus.cs),
zap.String("desc", f.Desc()), zap.String("desc", fa.Desc()),
) )
if err := f.Recover(clus); err != nil { if err := fa.Recover(clus); err != nil {
return fmt.Errorf("recovery error: %v", err) return fmt.Errorf("recovery error: %v", err)
} }
clus.lg.Info(
"recovered failure",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", f.Desc()),
)
clus.lg.Info("pausing stresser after failure recovery, before wait health") if stressStarted {
clus.pauseStresser() clus.lg.Info("pausing stresser after failure recovery, before wait health")
clus.stresser.Pause()
}
clus.lg.Info("wait health after recovering failures") clus.lg.Info("wait health after recover")
if err := clus.WaitHealth(); err != nil { if err := clus.WaitHealth(); err != nil {
return fmt.Errorf("wait full health error: %v", err) return fmt.Errorf("wait full health error: %v", err)
} }
clus.lg.Info("check consistency after recovering failures")
clus.lg.Info("check consistency after recover")
if err := clus.checkConsistency(); err != nil { if err := clus.checkConsistency(); err != nil {
return fmt.Errorf("tt.checkConsistency error (%v)", err) return fmt.Errorf("tt.checkConsistency error (%v)", err)
} }
clus.lg.Info( clus.lg.Info(
"failure case passed", "failure case PASS",
zap.Int("round", clus.rd), zap.Int("round", clus.rd),
zap.Int("case", clus.cs), zap.Int("case", clus.cs),
zap.String("desc", f.Desc()), zap.String("desc", fa.Desc()),
) )
} }
@ -186,6 +199,7 @@ func (clus *Cluster) doRound() error {
zap.Int("round", clus.rd), zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()), zap.Strings("failures", clus.failureStrings()),
) )
return nil return nil
} }
@ -204,14 +218,6 @@ func (clus *Cluster) updateRevision() error {
} }
func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) { func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) {
clus.lg.Info("pausing stresser before compact")
clus.pauseStresser()
defer func() {
if err == nil {
err = clus.startStresser()
}
}()
clus.lg.Info( clus.lg.Info(
"compacting storage", "compacting storage",
zap.Int64("current-revision", clus.currentRevision), zap.Int64("current-revision", clus.currentRevision),
@ -273,7 +279,13 @@ func (clus *Cluster) cleanup() error {
} }
caseFailedTotalCounter.WithLabelValues(desc).Inc() caseFailedTotalCounter.WithLabelValues(desc).Inc()
clus.closeStresser() clus.lg.Info(
"closing stressers before archiving failure data",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
)
clus.stresser.Close()
if err := clus.FailArchive(); err != nil { if err := clus.FailArchive(); err != nil {
clus.lg.Warn( clus.lg.Warn(
"cleanup failed", "cleanup failed",

120
tools/functional-tester/tester/failure.go
View File

@ -18,6 +18,8 @@ import (
"fmt" "fmt"
"math/rand" "math/rand"
"time" "time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
) )
// Failure defines failure injection interface. // Failure defines failure injection interface.
@ -33,28 +35,32 @@ type Failure interface {
Recover(clus *Cluster) error Recover(clus *Cluster) error
// Desc returns a description of the failure // Desc returns a description of the failure
Desc() string Desc() string
// FailureCase returns "rpcpb.FailureCase" enum type.
FailureCase() rpcpb.FailureCase
} }
type description string
func (d description) Desc() string { return string(d) }
type injectMemberFunc func(*Cluster, int) error type injectMemberFunc func(*Cluster, int) error
type recoverMemberFunc func(*Cluster, int) error type recoverMemberFunc func(*Cluster, int) error
type failureByFunc struct { type failureByFunc struct {
description desc
failureCase rpcpb.FailureCase
injectMember injectMemberFunc injectMember injectMemberFunc
recoverMember recoverMemberFunc recoverMember recoverMemberFunc
} }
type failureFollower struct { func (f *failureByFunc) Desc() string {
failureByFunc if string(f.desc) != "" {
last int return string(f.desc)
lead int }
return f.failureCase.String()
} }
type failureLeader struct { func (f *failureByFunc) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
type failureFollower struct {
failureByFunc failureByFunc
last int last int
lead int lead int
@ -82,22 +88,6 @@ func (f *failureFollower) updateIndex(clus *Cluster) error {
return nil return nil
} }
func (f *failureLeader) updateIndex(clus *Cluster) error {
idx, err := clus.GetLeader()
if err != nil {
return err
}
f.lead = idx
f.last = idx
return nil
}
type failureQuorum failureByFunc
type failureAll failureByFunc
// failureUntilSnapshot injects a failure and waits for a snapshot event
type failureUntilSnapshot struct{ Failure }
func (f *failureFollower) Inject(clus *Cluster) error { func (f *failureFollower) Inject(clus *Cluster) error {
if err := f.updateIndex(clus); err != nil { if err := f.updateIndex(clus); err != nil {
return err return err
@ -109,6 +99,33 @@ func (f *failureFollower) Recover(clus *Cluster) error {
return f.recoverMember(clus, f.last) return f.recoverMember(clus, f.last)
} }
func (f *failureFollower) Desc() string {
if string(f.desc) != "" {
return string(f.desc)
}
return f.failureCase.String()
}
func (f *failureFollower) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
type failureLeader struct {
failureByFunc
last int
lead int
}
func (f *failureLeader) updateIndex(clus *Cluster) error {
idx, err := clus.GetLeader()
if err != nil {
return err
}
f.lead = idx
f.last = idx
return nil
}
func (f *failureLeader) Inject(clus *Cluster) error { func (f *failureLeader) Inject(clus *Cluster) error {
if err := f.updateIndex(clus); err != nil { if err := f.updateIndex(clus); err != nil {
return err return err
@ -120,6 +137,12 @@ func (f *failureLeader) Recover(clus *Cluster) error {
return f.recoverMember(clus, f.last) return f.recoverMember(clus, f.last)
} }
func (f *failureLeader) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
type failureQuorum failureByFunc
func (f *failureQuorum) Inject(clus *Cluster) error { func (f *failureQuorum) Inject(clus *Cluster) error {
for i := range killMap(len(clus.Members), clus.rd) { for i := range killMap(len(clus.Members), clus.rd) {
if err := f.injectMember(clus, i); err != nil { if err := f.injectMember(clus, i); err != nil {
@ -138,6 +161,19 @@ func (f *failureQuorum) Recover(clus *Cluster) error {
return nil return nil
} }
func (f *failureQuorum) Desc() string {
if string(f.desc) != "" {
return string(f.desc)
}
return f.failureCase.String()
}
func (f *failureQuorum) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
type failureAll failureByFunc
func (f *failureAll) Inject(clus *Cluster) error { func (f *failureAll) Inject(clus *Cluster) error {
for i := range clus.Members { for i := range clus.Members {
if err := f.injectMember(clus, i); err != nil { if err := f.injectMember(clus, i); err != nil {
@ -156,6 +192,25 @@ func (f *failureAll) Recover(clus *Cluster) error {
return nil return nil
} }
func (f *failureAll) Desc() string {
if string(f.desc) != "" {
return string(f.desc)
}
return f.failureCase.String()
}
func (f *failureAll) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
// failureUntilSnapshot injects a failure and waits for a snapshot event
type failureUntilSnapshot struct {
desc desc
failureCase rpcpb.FailureCase
Failure
}
const snapshotCount = 10000 const snapshotCount = 10000
func (f *failureUntilSnapshot) Inject(clus *Cluster) error { func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
@ -190,7 +245,14 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
} }
func (f *failureUntilSnapshot) Desc() string { func (f *failureUntilSnapshot) Desc() string {
return f.Failure.Desc() + " for a long time and expect it to recover from an incoming snapshot" if f.desc.Desc() != "" {
return f.desc.Desc()
}
return f.failureCase.String()
}
func (f *failureUntilSnapshot) FailureCase() rpcpb.FailureCase {
return f.failureCase
} }
func killMap(size int, seed int) map[int]bool { func killMap(size int, seed int) map[int]bool {
@ -204,3 +266,7 @@ func killMap(size int, seed int) map[int]bool {
} }
} }
} }
type desc string
func (d desc) Desc() string { return string(d) }

19
tools/functional-tester/tester/failure_case_external.go
View File

@ -17,13 +17,17 @@ package tester
import ( import (
"fmt" "fmt"
"os/exec" "os/exec"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
) )
type failureExternal struct { type failureExternal struct {
Failure Failure
description string desc string
scriptPath string failureCase rpcpb.FailureCase
scriptPath string
} }
func (f *failureExternal) Inject(clus *Cluster) error { func (f *failureExternal) Inject(clus *Cluster) error {
@ -34,11 +38,18 @@ func (f *failureExternal) Recover(clus *Cluster) error {
return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run() return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run()
} }
func (f *failureExternal) Desc() string { return f.description } func (f *failureExternal) Desc() string {
return f.desc
}
func (f *failureExternal) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
func newFailureExternal(scriptPath string) Failure { func newFailureExternal(scriptPath string) Failure {
return &failureExternal{ return &failureExternal{
description: fmt.Sprintf("external fault injector (script: %q)", scriptPath), desc: fmt.Sprintf("external fault injector (script: %q)", scriptPath),
failureCase: rpcpb.FailureCase_EXTERNAL,
scriptPath: scriptPath, scriptPath: scriptPath,
} }
} }

28
tools/functional-tester/tester/failure_case_failpoints.go
View File

@ -20,7 +20,8 @@ import (
"net/http" "net/http"
"strings" "strings"
"sync" "sync"
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
) )
type failpointStats struct { type failpointStats struct {
@ -42,14 +43,23 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) {
if len(fp) == 0 { if len(fp) == 0 {
continue continue
} }
fpFails := failuresFromFailpoint(fp, clus.Tester.FailpointCommands) fpFails := failuresFromFailpoint(fp, clus.Tester.FailpointCommands)
// wrap in delays so failpoint has time to trigger // wrap in delays so failpoint has time to trigger
for i, fpf := range fpFails { for i, fpf := range fpFails {
if strings.Contains(fp, "Snap") { if strings.Contains(fp, "Snap") {
// hack to trigger snapshot failpoints // hack to trigger snapshot failpoints
fpFails[i] = &failureUntilSnapshot{fpf} fpFails[i] = &failureUntilSnapshot{
desc: desc(fpf.Desc()),
failureCase: rpcpb.FailureCase_FAILPOINTS,
Failure: fpf,
}
} else { } else {
fpFails[i] = &failureDelay{fpf, 3 * time.Second} fpFails[i] = &failureDelay{
Failure: fpf,
delayDuration: clus.GetFailureDelayDuration(),
}
} }
} }
ret = append(ret, fpFails...) ret = append(ret, fpFails...)
@ -85,7 +95,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
fs = append(fs, []Failure{ fs = append(fs, []Failure{
&failureFollower{ &failureFollower{
failureByFunc: failureByFunc{ failureByFunc: failureByFunc{
description: description(fmt.Sprintf("failpoint %s (one: %s)", fp, fcmd)), desc: desc(fmt.Sprintf("failpoint %q (one: %q)", fp, fcmd)),
failureCase: rpcpb.FailureCase_FAILPOINTS,
injectMember: inject, injectMember: inject,
recoverMember: recov, recoverMember: recov,
}, },
@ -94,7 +105,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
}, },
&failureLeader{ &failureLeader{
failureByFunc: failureByFunc{ failureByFunc: failureByFunc{
description: description(fmt.Sprintf("failpoint %s (leader: %s)", fp, fcmd)), desc: desc(fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd)),
failureCase: rpcpb.FailureCase_FAILPOINTS,
injectMember: inject, injectMember: inject,
recoverMember: recov, recoverMember: recov,
}, },
@ -102,12 +114,14 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
lead: -1, lead: -1,
}, },
&failureQuorum{ &failureQuorum{
description: description(fmt.Sprintf("failpoint %s (quorum: %s)", fp, fcmd)), desc: desc(fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd)),
failureCase: rpcpb.FailureCase_FAILPOINTS,
injectMember: inject, injectMember: inject,
recoverMember: recov, recoverMember: recov,
}, },
&failureAll{ &failureAll{
description: description(fmt.Sprintf("failpoint %s (all: %s)", fp, fcmd)), desc: desc(fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd)),
failureCase: rpcpb.FailureCase_FAILPOINTS,
injectMember: inject, injectMember: inject,
recoverMember: recov, recoverMember: recov,
}, },

18
tools/functional-tester/tester/failure_case_kill.go
View File

@ -26,7 +26,7 @@ func recoverKill(clus *Cluster, idx int) error {
func newFailureKillOneFollower() Failure { func newFailureKillOneFollower() Failure {
ff := failureByFunc{ ff := failureByFunc{
description: "kill one follower", failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER,
injectMember: injectKill, injectMember: injectKill,
recoverMember: recoverKill, recoverMember: recoverKill,
} }
@ -35,7 +35,7 @@ func newFailureKillOneFollower() Failure {
func newFailureKillLeader() Failure { func newFailureKillLeader() Failure {
ff := failureByFunc{ ff := failureByFunc{
description: "kill leader", failureCase: rpcpb.FailureCase_KILL_LEADER,
injectMember: injectKill, injectMember: injectKill,
recoverMember: recoverKill, recoverMember: recoverKill,
} }
@ -44,7 +44,7 @@ func newFailureKillLeader() Failure {
func newFailureKillQuorum() Failure { func newFailureKillQuorum() Failure {
return &failureQuorum{ return &failureQuorum{
description: "kill quorum", failureCase: rpcpb.FailureCase_KILL_QUORUM,
injectMember: injectKill, injectMember: injectKill,
recoverMember: recoverKill, recoverMember: recoverKill,
} }
@ -52,16 +52,22 @@ func newFailureKillQuorum() Failure {
func newFailureKillAll() Failure { func newFailureKillAll() Failure {
return &failureAll{ return &failureAll{
description: "kill all", failureCase: rpcpb.FailureCase_KILL_ALL,
injectMember: injectKill, injectMember: injectKill,
recoverMember: recoverKill, recoverMember: recoverKill,
} }
} }
func newFailureKillOneFollowerForLongTime() Failure { func newFailureKillOneFollowerForLongTime() Failure {
return &failureUntilSnapshot{newFailureKillOneFollower()} return &failureUntilSnapshot{
failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER_FOR_LONG,
Failure: newFailureKillOneFollower(),
}
} }
func newFailureKillLeaderForLongTime() Failure { func newFailureKillLeaderForLongTime() Failure {
return &failureUntilSnapshot{newFailureKillLeader()} return &failureUntilSnapshot{
failureCase: rpcpb.FailureCase_KILL_LEADER_FOR_LONG,
Failure: newFailureKillLeader(),
}
} }

22
tools/functional-tester/tester/failure_case_network_blackhole.go
View File

@ -14,7 +14,9 @@
package tester package tester
import "github.com/coreos/etcd/tools/functional-tester/rpcpb" import (
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
)
func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error { func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error {
return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx) return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx)
@ -24,40 +26,40 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error {
return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx) return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx)
} }
func newFailureBlackholePeerPortTxRxOneFollower() Failure { func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure {
ff := failureByFunc{ ff := failureByFunc{
description: "blackhole peer port on one follower", failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER,
injectMember: injectBlackholePeerPortTxRx, injectMember: injectBlackholePeerPortTxRx,
recoverMember: recoverBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx,
} }
f := &failureFollower{ff, -1, -1} f := &failureFollower{ff, -1, -1}
return &failureDelay{ return &failureDelay{
Failure: f, Failure: f,
delayDuration: triggerElectionDur, delayDuration: clus.GetFailureDelayDuration(),
} }
} }
func newFailureBlackholePeerPortTxRxLeader() Failure { func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure {
ff := failureByFunc{ ff := failureByFunc{
description: "blackhole peer port on leader", failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER,
injectMember: injectBlackholePeerPortTxRx, injectMember: injectBlackholePeerPortTxRx,
recoverMember: recoverBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx,
} }
f := &failureLeader{ff, -1, -1} f := &failureLeader{ff, -1, -1}
return &failureDelay{ return &failureDelay{
Failure: f, Failure: f,
delayDuration: triggerElectionDur, delayDuration: clus.GetFailureDelayDuration(),
} }
} }
func newFailureBlackholePeerPortTxRxAll() Failure { func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure {
f := &failureAll{ f := &failureAll{
description: "blackhole peer port on all", failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL,
injectMember: injectBlackholePeerPortTxRx, injectMember: injectBlackholePeerPortTxRx,
recoverMember: recoverBlackholePeerPortTxRx, recoverMember: recoverBlackholePeerPortTxRx,
} }
return &failureDelay{ return &failureDelay{
Failure: f, Failure: f,
delayDuration: triggerElectionDur, delayDuration: clus.GetFailureDelayDuration(),
} }
} }

19
tools/functional-tester/tester/failure_case_network_slow.go
View File

@ -15,16 +15,12 @@
package tester package tester
import ( import (
"fmt"
"time" "time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb" "github.com/coreos/etcd/tools/functional-tester/rpcpb"
) )
const ( const (
// delay duration to trigger leader election (default election timeout 1s)
triggerElectionDur = 5 * time.Second
// Wait more when it recovers from slow network, because network layer // Wait more when it recovers from slow network, because network layer
// needs extra time to propagate traffic control (tc command) change. // needs extra time to propagate traffic control (tc command) change.
// Otherwise, we get different hash values from the previous revision. // Otherwise, we get different hash values from the previous revision.
@ -43,42 +39,39 @@ func recoverDelayPeerPortTxRx(clus *Cluster, idx int) error {
} }
func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure {
desc := fmt.Sprintf("delay follower peer port by %d ms", clus.Tester.DelayLatencyMs)
ff := failureByFunc{ ff := failureByFunc{
description: description(desc), failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER,
injectMember: injectDelayPeerPortTxRx, injectMember: injectDelayPeerPortTxRx,
recoverMember: recoverDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx,
} }
f := &failureFollower{ff, -1, -1} f := &failureFollower{ff, -1, -1}
return &failureDelay{ return &failureDelay{
Failure: f, Failure: f,
delayDuration: triggerElectionDur, delayDuration: clus.GetFailureDelayDuration(),
} }
} }
func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure {
desc := fmt.Sprintf("delay leader peer port by %d ms", clus.Tester.DelayLatencyMs)
ff := failureByFunc{ ff := failureByFunc{
description: description(desc), failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER,
injectMember: injectDelayPeerPortTxRx, injectMember: injectDelayPeerPortTxRx,
recoverMember: recoverDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx,
} }
f := &failureLeader{ff, -1, -1} f := &failureLeader{ff, -1, -1}
return &failureDelay{ return &failureDelay{
Failure: f, Failure: f,
delayDuration: triggerElectionDur, delayDuration: clus.GetFailureDelayDuration(),
} }
} }
func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure { func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure {
desc := fmt.Sprintf("delay all peer port by %d ms", clus.Tester.DelayLatencyMs)
f := &failureAll{ f := &failureAll{
description: description(desc), failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ALL,
injectMember: injectDelayPeerPortTxRx, injectMember: injectDelayPeerPortTxRx,
recoverMember: recoverDelayPeerPortTxRx, recoverMember: recoverDelayPeerPortTxRx,
} }
return &failureDelay{ return &failureDelay{
Failure: f, Failure: f,
delayDuration: triggerElectionDur, delayDuration: clus.GetFailureDelayDuration(),
} }
} }

99
tools/functional-tester/tester/failure_case_no_fail.go Normal file
View File

@ -0,0 +1,99 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
import (
"time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap"
)
type failureNoFailWithStress failureByFunc
func (f *failureNoFailWithStress) Inject(clus *Cluster) error {
return nil
}
func (f *failureNoFailWithStress) Recover(clus *Cluster) error {
return nil
}
func (f *failureNoFailWithStress) Desc() string {
if f.desc.Desc() != "" {
return f.desc.Desc()
}
return f.failureCase.String()
}
func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
func newFailureNoFailWithStress(clus *Cluster) Failure {
f := &failureNoFailWithStress{
failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS,
}
return &failureDelay{
Failure: f,
delayDuration: clus.GetFailureDelayDuration(),
}
}
type failureNoFailWithNoStressForLiveness failureByFunc
func (f *failureNoFailWithNoStressForLiveness) Inject(clus *Cluster) error {
clus.lg.Info(
"extra delay for liveness mode with no stresser",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", f.Desc()),
)
time.Sleep(clus.GetFailureDelayDuration())
clus.lg.Info(
"wait health in liveness mode",
zap.Int("round", clus.rd),
zap.Int("case", clus.cs),
zap.String("desc", f.Desc()),
)
return clus.WaitHealth()
}
func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error {
return nil
}
func (f *failureNoFailWithNoStressForLiveness) Desc() string {
if f.desc.Desc() != "" {
return f.desc.Desc()
}
return f.failureCase.String()
}
func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase {
return f.failureCase
}
func newFailureNoFailWithNoStressForLiveness(clus *Cluster) Failure {
f := &failureNoFailWithNoStressForLiveness{
failureCase: rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS,
}
return &failureDelay{
Failure: f,
delayDuration: clus.GetFailureDelayDuration(),
}
}

26
tools/functional-tester/tester/failure_case_no_op.go
View File

@ -1,26 +0,0 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
type failureNoOp failureByFunc
func (f *failureNoOp) Inject(clus *Cluster) error { return nil }
func (f *failureNoOp) Recover(clus *Cluster) error { return nil }
func newFailureNoOp() Failure {
return &failureNoOp{
description: "no failure",
}
}

6
tools/functional-tester/tester/local-test.yaml
View File

@ -97,7 +97,10 @@ tester-config:
- DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER
- DELAY_PEER_PORT_TX_RX_LEADER - DELAY_PEER_PORT_TX_RX_LEADER
- DELAY_PEER_PORT_TX_RX_ALL - DELAY_PEER_PORT_TX_RX_ALL
- NO_FAIL_WITH_STRESS
- NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS
failure-delay-ms: 7000
failure-shuffle: true failure-shuffle: true
failpoint-commands: failpoint-commands:
- panic("etcd-tester") - panic("etcd-tester")
@ -110,7 +113,6 @@ tester-config:
stress-types: stress-types:
- KV - KV
- LEASE - LEASE
# - NO_STRESS
# - ELECTION_RUNNER # - ELECTION_RUNNER
# - WATCH_RUNNER # - WATCH_RUNNER
# - LOCK_RACER_RUNNER # - LOCK_RACER_RUNNER
@ -121,4 +123,6 @@ tester-config:
stress-key-suffix-range: 250000 stress-key-suffix-range: 250000
stress-key-suffix-range-txn: 100 stress-key-suffix-range-txn: 100
stress-key-txn-ops: 10 stress-key-txn-ops: 10
stress-clients: 100
stress-qps: 1000 stress-qps: 1000

21
tools/functional-tester/tester/stress.go
View File

@ -18,6 +18,8 @@ import (
"fmt" "fmt"
"time" "time"
"github.com/coreos/etcd/tools/functional-tester/rpcpb"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -36,34 +38,31 @@ type Stresser interface {
} }
// newStresser creates stresser from a comma separated list of stresser types. // newStresser creates stresser from a comma separated list of stresser types.
func newStresser(clus *Cluster, idx int) Stresser { func newStresser(clus *Cluster, m *rpcpb.Member) Stresser {
stressers := make([]Stresser, len(clus.Tester.StressTypes)) stressers := make([]Stresser, len(clus.Tester.StressTypes))
for i, stype := range clus.Tester.StressTypes { for i, stype := range clus.Tester.StressTypes {
clus.lg.Info("creating stresser", zap.String("type", stype)) clus.lg.Info("creating stresser", zap.String("type", stype))
switch stype { switch stype {
case "NO_STRESS":
stressers[i] = &nopStresser{start: time.Now(), qps: int(clus.rateLimiter.Limit())}
case "KV": case "KV":
// TODO: Too intensive stressing clients can panic etcd member with // TODO: Too intensive stressing clients can panic etcd member with
// 'out of memory' error. Put rate limits in server side. // 'out of memory' error. Put rate limits in server side.
stressers[i] = &keyStresser{ stressers[i] = &keyStresser{
lg: clus.lg, lg: clus.lg,
m: clus.Members[idx], m: m,
keySize: int(clus.Tester.StressKeySize), keySize: int(clus.Tester.StressKeySize),
keyLargeSize: int(clus.Tester.StressKeySizeLarge), keyLargeSize: int(clus.Tester.StressKeySizeLarge),
keySuffixRange: int(clus.Tester.StressKeySuffixRange), keySuffixRange: int(clus.Tester.StressKeySuffixRange),
keyTxnSuffixRange: int(clus.Tester.StressKeySuffixRangeTxn), keyTxnSuffixRange: int(clus.Tester.StressKeySuffixRangeTxn),
keyTxnOps: int(clus.Tester.StressKeyTxnOps), keyTxnOps: int(clus.Tester.StressKeyTxnOps),
N: 100, clientsN: int(clus.Tester.StressClients),
rateLimiter: clus.rateLimiter, rateLimiter: clus.rateLimiter,
} }
case "LEASE": case "LEASE":
stressers[i] = &leaseStresser{ stressers[i] = &leaseStresser{
lg: clus.lg, lg: clus.lg,
m: clus.Members[idx], m: m,
numLeases: 10, // TODO: configurable numLeases: 10, // TODO: configurable
keysPerLease: 10, // TODO: configurable keysPerLease: 10, // TODO: configurable
rateLimiter: clus.rateLimiter, rateLimiter: clus.rateLimiter,
@ -75,7 +74,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
"election", "election",
fmt.Sprintf("%v", time.Now().UnixNano()), // election name as current nano time fmt.Sprintf("%v", time.Now().UnixNano()), // election name as current nano time
"--dial-timeout=10s", "--dial-timeout=10s",
"--endpoints", clus.Members[idx].EtcdClientEndpoint, "--endpoints", m.EtcdClientEndpoint,
"--total-client-connections=10", "--total-client-connections=10",
"--rounds=0", // runs forever "--rounds=0", // runs forever
"--req-rate", fmt.Sprintf("%v", reqRate), "--req-rate", fmt.Sprintf("%v", reqRate),
@ -95,7 +94,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
"--total-keys=1", "--total-keys=1",
"--total-prefixes=1", "--total-prefixes=1",
"--watch-per-prefix=1", "--watch-per-prefix=1",
"--endpoints", clus.Members[idx].EtcdClientEndpoint, "--endpoints", m.EtcdClientEndpoint,
"--rounds=0", // runs forever "--rounds=0", // runs forever
"--req-rate", fmt.Sprintf("%v", reqRate), "--req-rate", fmt.Sprintf("%v", reqRate),
} }
@ -106,7 +105,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
args := []string{ args := []string{
"lock-racer", "lock-racer",
fmt.Sprintf("%v", time.Now().UnixNano()), // locker name as current nano time fmt.Sprintf("%v", time.Now().UnixNano()), // locker name as current nano time
"--endpoints", clus.Members[idx].EtcdClientEndpoint, "--endpoints", m.EtcdClientEndpoint,
"--total-client-connections=10", "--total-client-connections=10",
"--rounds=0", // runs forever "--rounds=0", // runs forever
"--req-rate", fmt.Sprintf("%v", reqRate), "--req-rate", fmt.Sprintf("%v", reqRate),
@ -117,7 +116,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
args := []string{ args := []string{
"lease-renewer", "lease-renewer",
"--ttl=30", "--ttl=30",
"--endpoints", clus.Members[idx].EtcdClientEndpoint, "--endpoints", m.EtcdClientEndpoint,
} }
stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, 0) stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, 0)
} }

42
tools/functional-tester/tester/stress_key.go
View File

@ -44,14 +44,15 @@ type keyStresser struct {
keyTxnSuffixRange int keyTxnSuffixRange int
keyTxnOps int keyTxnOps int
N int
rateLimiter *rate.Limiter rateLimiter *rate.Limiter
wg sync.WaitGroup wg sync.WaitGroup
clientsN int
ctx context.Context
cancel func() cancel func()
cli *clientv3.Client cli *clientv3.Client
// atomicModifiedKeys records the number of keys created and deleted by the stresser. // atomicModifiedKeys records the number of keys created and deleted by the stresser.
atomicModifiedKeys int64 atomicModifiedKeys int64
@ -59,40 +60,37 @@ type keyStresser struct {
} }
func (s *keyStresser) Stress() error { func (s *keyStresser) Stress() error {
// TODO: add backoff option var err error
cli, err := s.m.CreateEtcdClient() s.cli, err = s.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(1 * time.Second))
if err != nil { if err != nil {
return fmt.Errorf("%v (%q)", err, s.m.EtcdClientEndpoint) return fmt.Errorf("%v (%q)", err, s.m.EtcdClientEndpoint)
} }
ctx, cancel := context.WithCancel(context.Background()) s.ctx, s.cancel = context.WithCancel(context.Background())
s.wg.Add(s.N)
s.cli = cli
s.cancel = cancel
s.wg.Add(s.clientsN)
var stressEntries = []stressEntry{ var stressEntries = []stressEntry{
{weight: 0.7, f: newStressPut(cli, s.keySuffixRange, s.keySize)}, {weight: 0.7, f: newStressPut(s.cli, s.keySuffixRange, s.keySize)},
{ {
weight: 0.7 * float32(s.keySize) / float32(s.keyLargeSize), weight: 0.7 * float32(s.keySize) / float32(s.keyLargeSize),
f: newStressPut(cli, s.keySuffixRange, s.keyLargeSize), f: newStressPut(s.cli, s.keySuffixRange, s.keyLargeSize),
}, },
{weight: 0.07, f: newStressRange(cli, s.keySuffixRange)}, {weight: 0.07, f: newStressRange(s.cli, s.keySuffixRange)},
{weight: 0.07, f: newStressRangeInterval(cli, s.keySuffixRange)}, {weight: 0.07, f: newStressRangeInterval(s.cli, s.keySuffixRange)},
{weight: 0.07, f: newStressDelete(cli, s.keySuffixRange)}, {weight: 0.07, f: newStressDelete(s.cli, s.keySuffixRange)},
{weight: 0.07, f: newStressDeleteInterval(cli, s.keySuffixRange)}, {weight: 0.07, f: newStressDeleteInterval(s.cli, s.keySuffixRange)},
} }
if s.keyTxnSuffixRange > 0 { if s.keyTxnSuffixRange > 0 {
// adjust to make up ±70% of workloads with writes // adjust to make up ±70% of workloads with writes
stressEntries[0].weight = 0.35 stressEntries[0].weight = 0.35
stressEntries = append(stressEntries, stressEntry{ stressEntries = append(stressEntries, stressEntry{
weight: 0.35, weight: 0.35,
f: newStressTxn(cli, s.keyTxnSuffixRange, s.keyTxnOps), f: newStressTxn(s.cli, s.keyTxnSuffixRange, s.keyTxnOps),
}) })
} }
s.stressTable = createStressTable(stressEntries) s.stressTable = createStressTable(stressEntries)
for i := 0; i < s.N; i++ { for i := 0; i < s.clientsN; i++ {
go s.run(ctx) go s.run()
} }
s.lg.Info( s.lg.Info(
@ -102,18 +100,18 @@ func (s *keyStresser) Stress() error {
return nil return nil
} }
func (s *keyStresser) run(ctx context.Context) { func (s *keyStresser) run() {
defer s.wg.Done() defer s.wg.Done()
for { for {
if err := s.rateLimiter.Wait(ctx); err == context.Canceled { if err := s.rateLimiter.Wait(s.ctx); err == context.Canceled {
return return
} }
// TODO: 10-second is enough timeout to cover leader failure // TODO: 10-second is enough timeout to cover leader failure
// and immediate leader election. Find out what other cases this // and immediate leader election. Find out what other cases this
// could be timed out. // could be timed out.
sctx, scancel := context.WithTimeout(ctx, 10*time.Second) sctx, scancel := context.WithTimeout(s.ctx, 10*time.Second)
err, modifiedKeys := s.stressTable.choose()(sctx) err, modifiedKeys := s.stressTable.choose()(sctx)
scancel() scancel()
if err == nil { if err == nil {

10
tools/functional-tester/tester/stress_lease.go
View File

@ -33,8 +33,8 @@ import (
const ( const (
// time to live for lease // time to live for lease
TTL = 120 defaultTTL = 120
TTLShort = 2 defaultTTLShort = 2
) )
type leaseStresser struct { type leaseStresser struct {
@ -201,7 +201,7 @@ func (ls *leaseStresser) createAliveLeases() {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
leaseID, err := ls.createLeaseWithKeys(TTL) leaseID, err := ls.createLeaseWithKeys(defaultTTL)
if err != nil { if err != nil {
ls.lg.Debug( ls.lg.Debug(
"createLeaseWithKeys failed", "createLeaseWithKeys failed",
@ -228,7 +228,7 @@ func (ls *leaseStresser) createShortLivedLeases() {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
leaseID, err := ls.createLeaseWithKeys(TTLShort) leaseID, err := ls.createLeaseWithKeys(defaultTTLShort)
if err != nil { if err != nil {
return return
} }
@ -323,7 +323,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) {
// if it is renewed, this means that invariant checking have at least ttl/2 time before lease exipres which is long enough for the checking to finish. // if it is renewed, this means that invariant checking have at least ttl/2 time before lease exipres which is long enough for the checking to finish.
// if it is not renewed, we remove the lease from the alive map so that the lease doesn't exipre during invariant checking // if it is not renewed, we remove the lease from the alive map so that the lease doesn't exipre during invariant checking
renewTime, ok := ls.aliveLeases.read(leaseID) renewTime, ok := ls.aliveLeases.read(leaseID)
if ok && renewTime.Add(TTL/2*time.Second).Before(time.Now()) { if ok && renewTime.Add(defaultTTL/2*time.Second).Before(time.Now()) {
ls.aliveLeases.remove(leaseID) ls.aliveLeases.remove(leaseID)
ls.lg.Debug( ls.lg.Debug(
"keepLeaseAlive lease has not been renewed, dropped it", "keepLeaseAlive lease has not been renewed, dropped it",

31
tools/functional-tester/tester/stress_nop.go
View File

@ -1,31 +0,0 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
import "time"
// nopStresser implements Stresser that does nothing
type nopStresser struct {
start time.Time
qps int
}
func (s *nopStresser) Stress() error { return nil }
func (s *nopStresser) Pause() {}
func (s *nopStresser) Close() {}
func (s *nopStresser) ModifiedKeys() int64 {
return 0
}
func (s *nopStresser) Checker() Checker { return nil }