diff --git a/functional/tester/failure.go b/functional/tester/failure.go index 2f84959a7..2feb59931 100644 --- a/functional/tester/failure.go +++ b/functional/tester/failure.go @@ -143,11 +143,15 @@ func (f *failureLeader) FailureCase() rpcpb.FailureCase { return f.failureCase } -type failureQuorum failureByFunc +type failureQuorum struct { + failureByFunc + injected map[int]struct{} +} func (f *failureQuorum) Inject(clus *Cluster) error { - for i := range killMap(len(clus.Members), clus.rd) { - if err := f.injectMember(clus, i); err != nil { + f.injected = pickQuorum(len(clus.Members)) + for idx := range f.injected { + if err := f.injectMember(clus, idx); err != nil { return err } } @@ -155,8 +159,8 @@ func (f *failureQuorum) Inject(clus *Cluster) error { } func (f *failureQuorum) Recover(clus *Cluster) error { - for i := range killMap(len(clus.Members), clus.rd) { - if err := f.recoverMember(clus, i); err != nil { + for idx := range f.injected { + if err := f.recoverMember(clus, idx); err != nil { return err } } @@ -174,16 +178,15 @@ func (f *failureQuorum) FailureCase() rpcpb.FailureCase { return f.failureCase } -func killMap(size int, seed int) map[int]bool { - m := make(map[int]bool) - r := rand.New(rand.NewSource(int64(seed))) - majority := size/2 + 1 - for { - m[r.Intn(size)] = true - if len(m) >= majority { - return m - } +func pickQuorum(size int) (picked map[int]struct{}) { + picked = make(map[int]struct{}) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + quorum := size/2 + 1 + for len(picked) < quorum { + idx := r.Intn(size) + picked[idx] = struct{}{} } + return picked } type failureAll failureByFunc diff --git a/functional/tester/failure_case_failpoints.go b/functional/tester/failure_case_failpoints.go index 0db8b8965..e827d74f5 100644 --- a/functional/tester/failure_case_failpoints.go +++ b/functional/tester/failure_case_failpoints.go @@ -114,10 +114,13 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure) lead: -1, }, &failureQuorum{ - desc: fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd), - failureCase: rpcpb.FailureCase_FAILPOINTS, - injectMember: inject, - recoverMember: recov, + failureByFunc: failureByFunc{ + desc: fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd), + failureCase: rpcpb.FailureCase_FAILPOINTS, + injectMember: inject, + recoverMember: recov, + }, + injected: make(map[int]struct{}), }, &failureAll{ desc: fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd), diff --git a/functional/tester/failure_case_network_blackhole.go b/functional/tester/failure_case_network_blackhole.go index b75394a78..18764cf78 100644 --- a/functional/tester/failure_case_network_blackhole.go +++ b/functional/tester/failure_case_network_blackhole.go @@ -78,9 +78,12 @@ func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT() F func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM(clus *Cluster) Failure { f := &failureQuorum{ - failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM, - injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, - recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + failureByFunc: failureByFunc{ + failureCase: rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM, + injectMember: inject_BLACKHOLE_PEER_PORT_TX_RX, + recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX, + }, + injected: make(map[int]struct{}), } return &failureDelay{ Failure: f, diff --git a/functional/tester/failure_case_network_delay.go b/functional/tester/failure_case_network_delay.go index c2e067c46..6c8d95260 100644 --- a/functional/tester/failure_case_network_delay.go +++ b/functional/tester/failure_case_network_delay.go @@ -120,9 +120,12 @@ func new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *C func new_FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM(clus *Cluster, random bool) Failure { f := &failureQuorum{ - failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM, - injectMember: inject_DELAY_PEER_PORT_TX_RX, - recoverMember: recover_DELAY_PEER_PORT_TX_RX, + failureByFunc: failureByFunc{ + failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM, + injectMember: inject_DELAY_PEER_PORT_TX_RX, + recoverMember: recover_DELAY_PEER_PORT_TX_RX, + }, + injected: make(map[int]struct{}), } clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs if random { diff --git a/functional/tester/failure_case_sigterm.go b/functional/tester/failure_case_sigterm.go index 78afec006..7186f8d74 100644 --- a/functional/tester/failure_case_sigterm.go +++ b/functional/tester/failure_case_sigterm.go @@ -66,9 +66,12 @@ func new_FailureCase_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Failur func new_FailureCase_SIGTERM_QUORUM(clus *Cluster) Failure { f := &failureQuorum{ - failureCase: rpcpb.FailureCase_SIGTERM_QUORUM, - injectMember: inject_SIGTERM_ETCD, - recoverMember: recover_SIGTERM_ETCD, + failureByFunc: failureByFunc{ + failureCase: rpcpb.FailureCase_SIGTERM_QUORUM, + injectMember: inject_SIGTERM_ETCD, + recoverMember: recover_SIGTERM_ETCD, + }, + injected: make(map[int]struct{}), } return &failureDelay{ Failure: f,