Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

functional-tester: add logic to ensure etcd node is alive after fault recovery returns

Browse Source 
failure recovery needs to wait etcd node to become alive before returning

FIX #6654
This commit is contained in:
fanmin shi 2016-10-19 17:54:09 -07:00
parent 6b1b13eabb
commit 94ea82c00d
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/functional-tester/etcd-tester/checks.go
View File

@ -46,9 +46,8 @@ func (hc *hashChecker) Check() (err error) {
hashes map[string]int64 hashes map[string]int64
ok bool ok bool
) )
for i := 0; i < 7; i++ { // retry in case of transient failure
time.Sleep(time.Second) for i := 0; i < 3; i++ {
revs, hashes, err = hc.hrg.getRevisionHash() revs, hashes, err = hc.hrg.getRevisionHash()
if err != nil { if err != nil {
plog.Printf("#%d failed to get current revisions (%v)", i, err) plog.Printf("#%d failed to get current revisions (%v)", i, err)
@ -59,6 +58,7 @@ func (hc *hashChecker) Check() (err error) {
} }
plog.Printf("#%d inconsistent current revisions %+v", i, revs) plog.Printf("#%d inconsistent current revisions %+v", i, revs)
time.Sleep(time.Second)
} }
if !ok || err != nil { if !ok || err != nil {
return fmt.Errorf("checking current revisions failed [err: %v, revisions: %v]", err, revs) return fmt.Errorf("checking current revisions failed [err: %v, revisions: %v]", err, revs)

5
tools/functional-tester/etcd-tester/tester.go
View File

@ -104,6 +104,11 @@ func (tt *tester) doRound(round int) (bool, error) {
plog.Printf("%s recovery error: %v", tt.logPrefix(), err) plog.Printf("%s recovery error: %v", tt.logPrefix(), err)
return false, nil return false, nil
} }
plog.Printf("%s wait until cluster is healthy", tt.logPrefix())
if err := tt.cluster.WaitHealth(); err != nil {
plog.Printf("%s wait full health error: %v", tt.logPrefix(), err)
return false, nil
}
plog.Printf("%s recovered failure", tt.logPrefix()) plog.Printf("%s recovered failure", tt.logPrefix())
if err := tt.checkConsistency(); err != nil { if err := tt.checkConsistency(); err != nil {