From 94ea82c00d38e773e76ef21a321e1077f28178f8 Mon Sep 17 00:00:00 2001 From: fanmin shi Date: Wed, 19 Oct 2016 17:54:09 -0700 Subject: [PATCH] functional-tester: add logic to ensure etcd node is alive after fault recovery returns failure recovery needs to wait etcd node to become alive before returning FIX #6654 --- tools/functional-tester/etcd-tester/checks.go | 6 +++--- tools/functional-tester/etcd-tester/tester.go | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/functional-tester/etcd-tester/checks.go b/tools/functional-tester/etcd-tester/checks.go index 1d3d8342a..d7f450374 100644 --- a/tools/functional-tester/etcd-tester/checks.go +++ b/tools/functional-tester/etcd-tester/checks.go @@ -46,9 +46,8 @@ func (hc *hashChecker) Check() (err error) { hashes map[string]int64 ok bool ) - for i := 0; i < 7; i++ { - time.Sleep(time.Second) - + // retry in case of transient failure + for i := 0; i < 3; i++ { revs, hashes, err = hc.hrg.getRevisionHash() if err != nil { plog.Printf("#%d failed to get current revisions (%v)", i, err) @@ -59,6 +58,7 @@ func (hc *hashChecker) Check() (err error) { } plog.Printf("#%d inconsistent current revisions %+v", i, revs) + time.Sleep(time.Second) } if !ok || err != nil { return fmt.Errorf("checking current revisions failed [err: %v, revisions: %v]", err, revs) diff --git a/tools/functional-tester/etcd-tester/tester.go b/tools/functional-tester/etcd-tester/tester.go index c439e6251..2d038aef3 100644 --- a/tools/functional-tester/etcd-tester/tester.go +++ b/tools/functional-tester/etcd-tester/tester.go @@ -104,6 +104,11 @@ func (tt *tester) doRound(round int) (bool, error) { plog.Printf("%s recovery error: %v", tt.logPrefix(), err) return false, nil } + plog.Printf("%s wait until cluster is healthy", tt.logPrefix()) + if err := tt.cluster.WaitHealth(); err != nil { + plog.Printf("%s wait full health error: %v", tt.logPrefix(), err) + return false, nil + } plog.Printf("%s recovered failure", tt.logPrefix()) if err := tt.checkConsistency(); err != nil {