diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 176525e84..5abaf5b4b 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -276,11 +276,11 @@ func (clus *Cluster) updateFailures() { case "KILL_ALL": clus.failures = append(clus.failures, newFailureKillAll()) case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_LEADER": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus)) case "BLACKHOLE_PEER_PORT_TX_RX_ALL": - clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll()) + clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus)) case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER": clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus)) case "DELAY_PEER_PORT_TX_RX_LEADER": diff --git a/tools/functional-tester/tester/cluster_test.go b/tools/functional-tester/tester/cluster_test.go index 2653544b0..e0627453e 100644 --- a/tools/functional-tester/tester/cluster_test.go +++ b/tools/functional-tester/tester/cluster_test.go @@ -134,6 +134,7 @@ func Test_newCluster(t *testing.T) { "NO_FAIL_WITH_STRESS", "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", }, + FailureDelayMs: 7000, FailureShuffle: true, FailpointCommands: []string{`panic("etcd-tester")`}, RunnerExecPath: "/etcd-runner", diff --git a/tools/functional-tester/tester/failure_case_failpoints.go b/tools/functional-tester/tester/failure_case_failpoints.go index f5df8138b..f468a53c2 100644 --- a/tools/functional-tester/tester/failure_case_failpoints.go +++ b/tools/functional-tester/tester/failure_case_failpoints.go @@ -59,7 +59,7 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) { } else { fpFails[i] = &failureDelay{ Failure: fpf, - delayDuration: 3 * time.Second, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } } diff --git a/tools/functional-tester/tester/failure_case_network_blackhole.go b/tools/functional-tester/tester/failure_case_network_blackhole.go index f57c20477..c3a0f00c5 100644 --- a/tools/functional-tester/tester/failure_case_network_blackhole.go +++ b/tools/functional-tester/tester/failure_case_network_blackhole.go @@ -14,7 +14,11 @@ package tester -import "github.com/coreos/etcd/tools/functional-tester/rpcpb" +import ( + "time" + + "github.com/coreos/etcd/tools/functional-tester/rpcpb" +) func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx) @@ -24,7 +28,7 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error { return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx) } -func newFailureBlackholePeerPortTxRxOneFollower() Failure { +func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER, injectMember: injectBlackholePeerPortTxRx, @@ -33,11 +37,11 @@ func newFailureBlackholePeerPortTxRxOneFollower() Failure { f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } -func newFailureBlackholePeerPortTxRxLeader() Failure { +func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure { ff := failureByFunc{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER, injectMember: injectBlackholePeerPortTxRx, @@ -46,11 +50,11 @@ func newFailureBlackholePeerPortTxRxLeader() Failure { f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } -func newFailureBlackholePeerPortTxRxAll() Failure { +func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure { f := &failureAll{ failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL, injectMember: injectBlackholePeerPortTxRx, @@ -58,6 +62,6 @@ func newFailureBlackholePeerPortTxRxAll() Failure { } return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } diff --git a/tools/functional-tester/tester/failure_case_network_slow.go b/tools/functional-tester/tester/failure_case_network_slow.go index 9fadab67c..db28206ad 100644 --- a/tools/functional-tester/tester/failure_case_network_slow.go +++ b/tools/functional-tester/tester/failure_case_network_slow.go @@ -21,9 +21,6 @@ import ( ) const ( - // delay duration to trigger leader election (default election timeout 1s) - triggerElectionDur = 5 * time.Second - // Wait more when it recovers from slow network, because network layer // needs extra time to propagate traffic control (tc command) change. // Otherwise, we get different hash values from the previous revision. @@ -50,7 +47,7 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure { f := &failureFollower{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } @@ -63,7 +60,7 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure { f := &failureLeader{ff, -1, -1} return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } @@ -75,6 +72,6 @@ func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure { } return &failureDelay{ Failure: f, - delayDuration: triggerElectionDur, + delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond, } } diff --git a/tools/functional-tester/tester/local-test.yaml b/tools/functional-tester/tester/local-test.yaml index 09f631cfc..f5068c236 100644 --- a/tools/functional-tester/tester/local-test.yaml +++ b/tools/functional-tester/tester/local-test.yaml @@ -100,6 +100,7 @@ tester-config: - NO_FAIL_WITH_STRESS - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS + failure-delay-ms: 7000 failure-shuffle: true failpoint-commands: - panic("etcd-tester")