diff --git a/tools/functional-tester/tester/cluster_tester.go b/tools/functional-tester/tester/cluster_tester.go index 0adb0ad92..8e2b4662c 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/tools/functional-tester/tester/cluster_tester.go @@ -34,9 +34,9 @@ func (clus *Cluster) StartTester() { var preModifiedKey int64 for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { roundTotalCounter.Inc() - clus.rd = round - if err := clus.doRound(round); err != nil { + + if err := clus.doRound(); err != nil { clus.logger.Warn( "doRound failed; returning", zap.Int("round", clus.rd), @@ -50,6 +50,7 @@ func (clus *Cluster) StartTester() { preModifiedKey = 0 continue } + // -1 so that logPrefix doesn't print out 'case' clus.cs = -1 @@ -105,11 +106,16 @@ func (clus *Cluster) StartTester() { ) } -func (clus *Cluster) doRound(round int) error { +func (clus *Cluster) doRound() error { if clus.Tester.FailureShuffle { clus.shuffleFailures() } + clus.logger.Info( + "starting round", + zap.Int("round", clus.rd), + zap.Strings("failures", clus.failureStrings()), + ) for i, f := range clus.failures { clus.cs = i @@ -126,7 +132,7 @@ func (clus *Cluster) doRound(round int) error { zap.Int("case", clus.cs), zap.String("desc", f.Desc()), ) - if err := f.Inject(clus, round); err != nil { + if err := f.Inject(clus); err != nil { return fmt.Errorf("injection error: %v", err) } clus.logger.Info( @@ -145,7 +151,7 @@ func (clus *Cluster) doRound(round int) error { zap.Int("case", clus.cs), zap.String("desc", f.Desc()), ) - if err := f.Recover(clus, round); err != nil { + if err := f.Recover(clus); err != nil { return fmt.Errorf("recovery error: %v", err) } clus.logger.Info( @@ -174,6 +180,12 @@ func (clus *Cluster) doRound(round int) error { zap.String("desc", f.Desc()), ) } + + clus.logger.Info( + "finished round", + zap.Int("round", clus.rd), + zap.Strings("failures", clus.failureStrings()), + ) return nil } diff --git a/tools/functional-tester/tester/failure.go b/tools/functional-tester/tester/failure.go index 4947c86ae..5ff1e856e 100644 --- a/tools/functional-tester/tester/failure.go +++ b/tools/functional-tester/tester/failure.go @@ -21,10 +21,10 @@ package tester type Failure interface { // Inject injeccts the failure into the testing cluster at the given // round. When calling the function, the cluster should be in health. - Inject(clus *Cluster, round int) error + Inject(clus *Cluster) error // Recover recovers the injected failure caused by the injection of the // given round and wait for the recovery of the testing cluster. - Recover(clus *Cluster, round int) error + Recover(clus *Cluster) error // Desc returns a description of the failure Desc() string } diff --git a/tools/functional-tester/tester/failure_case_delay.go b/tools/functional-tester/tester/failure_case_delay.go index 3a7c975a1..ba6d5a291 100644 --- a/tools/functional-tester/tester/failure_case_delay.go +++ b/tools/functional-tester/tester/failure_case_delay.go @@ -25,8 +25,8 @@ type failureDelay struct { delayDuration time.Duration } -func (f *failureDelay) Inject(clus *Cluster, round int) error { - if err := f.Failure.Inject(clus, round); err != nil { +func (f *failureDelay) Inject(clus *Cluster) error { + if err := f.Failure.Inject(clus); err != nil { return err } if f.delayDuration > 0 { diff --git a/tools/functional-tester/tester/failure_case_external.go b/tools/functional-tester/tester/failure_case_external.go index 4f88b60f3..e5b3b6a18 100644 --- a/tools/functional-tester/tester/failure_case_external.go +++ b/tools/functional-tester/tester/failure_case_external.go @@ -26,12 +26,12 @@ type failureExternal struct { scriptPath string } -func (f *failureExternal) Inject(clus *Cluster, round int) error { - return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", round)).Run() +func (f *failureExternal) Inject(clus *Cluster) error { + return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", clus.rd)).Run() } -func (f *failureExternal) Recover(clus *Cluster, round int) error { - return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", round)).Run() +func (f *failureExternal) Recover(clus *Cluster) error { + return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run() } func (f *failureExternal) Desc() string { return f.description } diff --git a/tools/functional-tester/tester/failure_case_kill.go b/tools/functional-tester/tester/failure_case_kill.go index 9982dcc21..ddbfab350 100644 --- a/tools/functional-tester/tester/failure_case_kill.go +++ b/tools/functional-tester/tester/failure_case_kill.go @@ -99,19 +99,19 @@ type failureLeader struct { // failureUntilSnapshot injects a failure and waits for a snapshot event type failureUntilSnapshot struct{ Failure } -func (f *failureOne) Inject(clus *Cluster, round int) error { - return f.injectMember(clus, round%len(clus.Members)) +func (f *failureOne) Inject(clus *Cluster) error { + return f.injectMember(clus, clus.rd%len(clus.Members)) } -func (f *failureOne) Recover(clus *Cluster, round int) error { - if err := f.recoverMember(clus, round%len(clus.Members)); err != nil { +func (f *failureOne) Recover(clus *Cluster) error { + if err := f.recoverMember(clus, clus.rd%len(clus.Members)); err != nil { return err } clus.logger.Info("wait health after recovering failureOne") return clus.WaitHealth() } -func (f *failureAll) Inject(clus *Cluster, round int) error { +func (f *failureAll) Inject(clus *Cluster) error { for i := range clus.Members { if err := f.injectMember(clus, i); err != nil { return err @@ -120,7 +120,7 @@ func (f *failureAll) Inject(clus *Cluster, round int) error { return nil } -func (f *failureAll) Recover(clus *Cluster, round int) error { +func (f *failureAll) Recover(clus *Cluster) error { for i := range clus.Members { if err := f.recoverMember(clus, i); err != nil { return err @@ -130,8 +130,8 @@ func (f *failureAll) Recover(clus *Cluster, round int) error { return clus.WaitHealth() } -func (f *failureQuorum) Inject(clus *Cluster, round int) error { - for i := range killMap(len(clus.Members), round) { +func (f *failureQuorum) Inject(clus *Cluster) error { + for i := range killMap(len(clus.Members), clus.rd) { if err := f.injectMember(clus, i); err != nil { return err } @@ -139,8 +139,8 @@ func (f *failureQuorum) Inject(clus *Cluster, round int) error { return nil } -func (f *failureQuorum) Recover(clus *Cluster, round int) error { - for i := range killMap(len(clus.Members), round) { +func (f *failureQuorum) Recover(clus *Cluster) error { + for i := range killMap(len(clus.Members), clus.rd) { if err := f.recoverMember(clus, i); err != nil { return err } @@ -148,7 +148,7 @@ func (f *failureQuorum) Recover(clus *Cluster, round int) error { return nil } -func (f *failureLeader) Inject(clus *Cluster, round int) error { +func (f *failureLeader) Inject(clus *Cluster) error { idx, err := clus.GetLeader() if err != nil { return err @@ -157,7 +157,7 @@ func (f *failureLeader) Inject(clus *Cluster, round int) error { return f.injectMember(clus, idx) } -func (f *failureLeader) Recover(clus *Cluster, round int) error { +func (f *failureLeader) Recover(clus *Cluster) error { if err := f.recoverMember(clus, f.idx); err != nil { return err } @@ -165,8 +165,8 @@ func (f *failureLeader) Recover(clus *Cluster, round int) error { return clus.WaitHealth() } -func (f *failureUntilSnapshot) Inject(clus *Cluster, round int) error { - if err := f.Failure.Inject(clus, round); err != nil { +func (f *failureUntilSnapshot) Inject(clus *Cluster) error { + if err := f.Failure.Inject(clus); err != nil { return err } if len(clus.Members) < 3 { diff --git a/tools/functional-tester/tester/failure_case_no_op.go b/tools/functional-tester/tester/failure_case_no_op.go index 719430290..fdcddd306 100644 --- a/tools/functional-tester/tester/failure_case_no_op.go +++ b/tools/functional-tester/tester/failure_case_no_op.go @@ -16,8 +16,8 @@ package tester type failureNoOp failureByFunc -func (f *failureNoOp) Inject(clus *Cluster, round int) error { return nil } -func (f *failureNoOp) Recover(clus *Cluster, round int) error { return nil } +func (f *failureNoOp) Inject(clus *Cluster) error { return nil } +func (f *failureNoOp) Recover(clus *Cluster) error { return nil } func newFailureNoOp() Failure { return &failureNoOp{