From d4eff5381c10a5d97aa06d0276d485561871094b Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Mon, 19 Sep 2016 20:32:30 -0700 Subject: [PATCH] etcd-tester: split out consistency checking code from tester --- tools/functional-tester/etcd-tester/checks.go | 71 ++++++++++++++++++ tools/functional-tester/etcd-tester/main.go | 12 ++- tools/functional-tester/etcd-tester/tester.go | 73 +++++-------------- 3 files changed, 98 insertions(+), 58 deletions(-) create mode 100644 tools/functional-tester/etcd-tester/checks.go diff --git a/tools/functional-tester/etcd-tester/checks.go b/tools/functional-tester/etcd-tester/checks.go new file mode 100644 index 000000000..9b4381ef6 --- /dev/null +++ b/tools/functional-tester/etcd-tester/checks.go @@ -0,0 +1,71 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "time" +) + +type Checker interface { + // Check returns an error if the system fails a consistency check. + Check() error +} + +type hashChecker struct { + tt *tester +} + +func newHashChecker(tt *tester) Checker { return &hashChecker{tt} } + +func (hc *hashChecker) Check() (err error) { + plog.Printf("%s fetching current revisions...", hc.tt.logPrefix()) + var ( + revs map[string]int64 + hashes map[string]int64 + ok bool + ) + for i := 0; i < 7; i++ { + time.Sleep(time.Second) + + revs, hashes, err = hc.tt.cluster.getRevisionHash() + if err != nil { + plog.Printf("%s #%d failed to get current revisions (%v)", hc.tt.logPrefix(), i, err) + continue + } + if _, ok = getSameValue(revs); ok { + break + } + + plog.Printf("%s #%d inconsistent current revisions %+v", hc.tt.logPrefix(), i, revs) + } + if !ok || err != nil { + return fmt.Errorf("checking current revisions failed [err: %v, revisions: %v]", err, revs) + } + plog.Printf("%s all members are consistent with current revisions [revisions: %v]", hc.tt.logPrefix(), revs) + + plog.Printf("%s checking current storage hashes...", hc.tt.logPrefix()) + if _, ok = getSameValue(hashes); !ok { + return fmt.Errorf("inconsistent hashes [%v]", hashes) + } + + plog.Printf("%s all members are consistent with storage hashes", hc.tt.logPrefix()) + return nil +} + +type noChecker struct{} + +func newNoChecker() Checker { return &noChecker{} } +func (nc *noChecker) Check() error { return nil } diff --git a/tools/functional-tester/etcd-tester/main.go b/tools/functional-tester/etcd-tester/main.go index f36396292..b31165325 100644 --- a/tools/functional-tester/etcd-tester/main.go +++ b/tools/functional-tester/etcd-tester/main.go @@ -114,10 +114,14 @@ func main() { } t := &tester{ - failures: schedule, - cluster: c, - limit: *limit, - consistencyCheck: *consistencyCheck, + failures: schedule, + cluster: c, + limit: *limit, + checker: newNoChecker(), + } + + if *consistencyCheck && !c.v2Only { + t.checker = newHashChecker(t) } sh := statusHandler{status: &t.status} diff --git a/tools/functional-tester/etcd-tester/tester.go b/tools/functional-tester/etcd-tester/tester.go index 8a16de0a8..61e604490 100644 --- a/tools/functional-tester/etcd-tester/tester.go +++ b/tools/functional-tester/etcd-tester/tester.go @@ -20,10 +20,10 @@ import ( ) type tester struct { - failures []failure - cluster *cluster - limit int - consistencyCheck bool + failures []failure + cluster *cluster + limit int + checker Checker status Status currentRevision int64 @@ -109,82 +109,47 @@ func (tt *tester) doRound(round int) (bool, error) { } plog.Printf("%s recovered failure", tt.logPrefix()) - if tt.cluster.v2Only { - plog.Printf("%s succeed!", tt.logPrefix()) - continue - } - - if !tt.consistencyCheck { - if err := tt.updateRevision(); err != nil { - plog.Warningf("%s functional-tester returning with tt.updateRevision error (%v)", tt.logPrefix(), err) - return false, err - } - continue - } - if err := tt.checkConsistency(); err != nil { plog.Warningf("%s functional-tester returning with tt.checkConsistency error (%v)", tt.logPrefix(), err) return false, err } + plog.Printf("%s succeed!", tt.logPrefix()) } return true, nil } func (tt *tester) updateRevision() error { + if tt.cluster.v2Only { + return nil + } + revs, _, err := tt.cluster.getRevisionHash() for _, rev := range revs { tt.currentRevision = rev break // just need get one of the current revisions } + + plog.Printf("%s updated current revision to %d", tt.logPrefix(), tt.currentRevision) return err } func (tt *tester) checkConsistency() (err error) { tt.cancelStressers() defer func() { - if err == nil { - err = tt.startStressers() - } - }() - - plog.Printf("%s updating current revisions...", tt.logPrefix()) - var ( - revs map[string]int64 - hashes map[string]int64 - ok bool - ) - for i := 0; i < 7; i++ { - time.Sleep(time.Second) - - revs, hashes, err = tt.cluster.getRevisionHash() if err != nil { - plog.Printf("%s #%d failed to get current revisions (%v)", tt.logPrefix(), i, err) - continue + return } - if tt.currentRevision, ok = getSameValue(revs); ok { - break + if err = tt.updateRevision(); err != nil { + plog.Warningf("%s functional-tester returning with tt.updateRevision error (%v)", tt.logPrefix(), err) + return } - - plog.Printf("%s #%d inconsistent current revisions %+v", tt.logPrefix(), i, revs) - } - if !ok || err != nil { - err = fmt.Errorf("checking current revisions failed [err: %v, revisions: %v]", err, revs) + err = tt.startStressers() + }() + if err = tt.checker.Check(); err != nil { plog.Printf("%s %v", tt.logPrefix(), err) - return } - plog.Printf("%s all members are consistent with current revisions [revisions: %v]", tt.logPrefix(), revs) - - plog.Printf("%s checking current storage hashes...", tt.logPrefix()) - if _, ok = getSameValue(hashes); !ok { - err = fmt.Errorf("inconsistent hashes [%v]", hashes) - plog.Printf("%s %v", tt.logPrefix(), err) - return - } - plog.Printf("%s all members are consistent with storage hashes", tt.logPrefix()) - - plog.Printf("%s updated current revision to %d", tt.logPrefix(), tt.currentRevision) - return nil + return err } func (tt *tester) compact(rev int64, timeout time.Duration) (err error) {