From d275437c05c8f6ceaf8196a77858afdbae7d511d Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 11 Apr 2018 19:31:16 -0700 Subject: [PATCH] functional/tester: whitelist lease expire checker on snapshot restore Signed-off-by: Gyuho Lee --- functional/tester/cluster.go | 39 ++++++++++++++++++++------------ functional/tester/cluster_run.go | 12 +++++++--- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/functional/tester/cluster.go b/functional/tester/cluster.go index 5bd7a1709..5cd863d14 100644 --- a/functional/tester/cluster.go +++ b/functional/tester/cluster.go @@ -325,7 +325,7 @@ func (clus *Cluster) setStresserChecker() { clus.lg.Info("updated stressers") } -func (clus *Cluster) runCheckers() (err error) { +func (clus *Cluster) runCheckers(exceptions ...rpcpb.Checker) (err error) { defer func() { if err != nil { return @@ -339,28 +339,37 @@ func (clus *Cluster) runCheckers() (err error) { } }() + exs := make(map[rpcpb.Checker]struct{}) + for _, e := range exceptions { + exs[e] = struct{}{} + } for _, chk := range clus.checkers { - if err = chk.Check(); err != nil { + clus.lg.Warn( + "consistency check START", + zap.String("checker", chk.Type().String()), + zap.Strings("client-endpoints", chk.EtcdClientEndpoints()), + ) + err = chk.Check() + clus.lg.Warn( + "consistency check END", + zap.String("checker", chk.Type().String()), + zap.Strings("client-endpoints", chk.EtcdClientEndpoints()), + zap.Error(err), + ) + if err != nil { + _, ok := exs[chk.Type()] + if !ok { + return err + } clus.lg.Warn( - "consistency check FAIL", + "consistency check SKIP FAIL", zap.String("checker", chk.Type().String()), zap.Strings("client-endpoints", chk.EtcdClientEndpoints()), - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), zap.Error(err), ) - return err } } - - clus.lg.Info( - "consistency check ALL PASS", - zap.Int("round", clus.rd), - zap.Int("case", clus.cs), - zap.String("desc", clus.cases[clus.cs].Desc()), - ) - - return err + return nil } // Send_INITIAL_START_ETCD bootstraps etcd cluster the very first time. diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index f016d9088..6ef47fd90 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -230,6 +230,13 @@ func (clus *Cluster) doRound() error { return fmt.Errorf("wait full health error: %v", err) } + checkerFailExceptions := []rpcpb.Checker{} + switch fcase { + case rpcpb.Case_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH: + // TODO: restore from snapshot + checkerFailExceptions = append(checkerFailExceptions, rpcpb.Checker_LEASE_EXPIRE) + } + clus.lg.Info( "consistency check START", zap.Int("round", clus.rd), @@ -237,12 +244,11 @@ func (clus *Cluster) doRound() error { zap.Int("case-total", len(clus.cases)), zap.String("desc", fa.Desc()), ) - if err := clus.runCheckers(); err != nil { + if err := clus.runCheckers(checkerFailExceptions...); err != nil { return fmt.Errorf("consistency check error (%v)", err) } - clus.lg.Info( - "case PASS", + "consistency check PASS", zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.Int("case-total", len(clus.cases)),