From 63f78bf7c8f49ed799191c07873ff13ca34873ff Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Mon, 27 Jun 2016 16:42:33 -0700 Subject: [PATCH] etcd-tester: refactor round loop --- tools/functional-tester/etcd-tester/tester.go | 134 ++++++++---------- 1 file changed, 59 insertions(+), 75 deletions(-) diff --git a/tools/functional-tester/etcd-tester/tester.go b/tools/functional-tester/etcd-tester/tester.go index 7cd601e59..fdc264a78 100644 --- a/tools/functional-tester/etcd-tester/tester.go +++ b/tools/functional-tester/etcd-tester/tester.go @@ -41,80 +41,15 @@ func (tt *tester) runLoop() { tt.status.Failures = append(tt.status.Failures, f.Desc()) } - var ( - round int - prevCompactRev int64 - ) - for { + var prevCompactRev int64 + for round := 0; round < tt.limit || tt.limit == -1; round++ { tt.status.setRound(round) - tt.status.setCase(-1) // -1 so that logPrefix doesn't print out 'case' roundTotalCounter.Inc() - var failed bool - for j, f := range tt.failures { - caseTotalCounter.WithLabelValues(f.Desc()).Inc() - tt.status.setCase(j) - - if err := tt.cluster.WaitHealth(); err != nil { - plog.Printf("%s wait full health error: %v", tt.logPrefix(), err) - if err := tt.cleanup(); err != nil { - return - } - failed = true - break - } - - plog.Printf("%s injecting failure %q", tt.logPrefix(), f.Desc()) - if err := f.Inject(tt.cluster, round); err != nil { - plog.Printf("%s injection error: %v", tt.logPrefix(), err) - if err := tt.cleanup(); err != nil { - return - } - failed = true - break - } - plog.Printf("%s injected failure", tt.logPrefix()) - - plog.Printf("%s recovering failure %q", tt.logPrefix(), f.Desc()) - if err := f.Recover(tt.cluster, round); err != nil { - plog.Printf("%s recovery error: %v", tt.logPrefix(), err) - if err := tt.cleanup(); err != nil { - return - } - failed = true - break - } - plog.Printf("%s recovered failure", tt.logPrefix()) - - if tt.cluster.v2Only { - plog.Printf("%s succeed!", tt.logPrefix()) - continue - } - - if !tt.consistencyCheck { - if err := tt.updateRevision(); err != nil { - plog.Warningf("%s functional-tester returning with tt.updateRevision error (%v)", tt.logPrefix(), err) - return - } - continue - } - - var err error - failed, err = tt.checkConsistency() - if err != nil { - plog.Warningf("%s functional-tester returning with tt.checkConsistency error (%v)", tt.logPrefix(), err) + if ok, err := tt.doRound(round); !ok { + if err != nil || tt.cleanup() != nil { return } - if failed { - break - } - plog.Printf("%s succeed!", tt.logPrefix()) - } - - // -1 so that logPrefix doesn't print out 'case' - tt.status.setCase(-1) - - if failed { continue } @@ -139,13 +74,62 @@ func (tt *tester) runLoop() { return } } - - round++ - if round == tt.limit { - plog.Printf("%s functional-tester is finished", tt.logPrefix()) - break - } } + + plog.Printf("%s functional-tester is finished", tt.logPrefix()) +} + +func (tt *tester) doRound(round int) (bool, error) { + // -1 so that logPrefix doesn't print out 'case' + defer tt.status.setCase(-1) + + for j, f := range tt.failures { + caseTotalCounter.WithLabelValues(f.Desc()).Inc() + tt.status.setCase(j) + + if err := tt.cluster.WaitHealth(); err != nil { + plog.Printf("%s wait full health error: %v", tt.logPrefix(), err) + return false, nil + } + + plog.Printf("%s injecting failure %q", tt.logPrefix(), f.Desc()) + if err := f.Inject(tt.cluster, round); err != nil { + plog.Printf("%s injection error: %v", tt.logPrefix(), err) + return false, nil + } + plog.Printf("%s injected failure", tt.logPrefix()) + + plog.Printf("%s recovering failure %q", tt.logPrefix(), f.Desc()) + if err := f.Recover(tt.cluster, round); err != nil { + plog.Printf("%s recovery error: %v", tt.logPrefix(), err) + return false, nil + } + plog.Printf("%s recovered failure", tt.logPrefix()) + + if tt.cluster.v2Only { + plog.Printf("%s succeed!", tt.logPrefix()) + continue + } + + if !tt.consistencyCheck { + if err := tt.updateRevision(); err != nil { + plog.Warningf("%s functional-tester returning with tt.updateRevision error (%v)", tt.logPrefix(), err) + return false, err + } + continue + } + + failed, err := tt.checkConsistency() + if err != nil { + plog.Warningf("%s functional-tester returning with tt.checkConsistency error (%v)", tt.logPrefix(), err) + return false, err + } + if failed { + return false, nil + } + plog.Printf("%s succeed!", tt.logPrefix()) + } + return true, nil } func (tt *tester) updateRevision() error {