Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

functional-tester: improve logging, remove "round" parameter from

Browse Source 
"Failure"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>
This commit is contained in:
Gyuho Lee 2018-04-02 09:37:50 -07:00
parent 3ced4443ba
commit 94a96e59e0
6 changed files with 41 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tools/functional-tester/tester/cluster_tester.go
View File

@ -34,9 +34,9 @@ func (clus *Cluster) StartTester() {
var preModifiedKey int64 var preModifiedKey int64
for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
roundTotalCounter.Inc() roundTotalCounter.Inc()
clus.rd = round clus.rd = round
if err := clus.doRound(round); err != nil {
if err := clus.doRound(); err != nil {
clus.logger.Warn( clus.logger.Warn(
"doRound failed; returning", "doRound failed; returning",
zap.Int("round", clus.rd), zap.Int("round", clus.rd),
@ -50,6 +50,7 @@ func (clus *Cluster) StartTester() {
preModifiedKey = 0 preModifiedKey = 0
continue continue
} }
// -1 so that logPrefix doesn't print out 'case' // -1 so that logPrefix doesn't print out 'case'
clus.cs = -1 clus.cs = -1
@ -105,11 +106,16 @@ func (clus *Cluster) StartTester() {
) )
} }
func (clus *Cluster) doRound(round int) error { func (clus *Cluster) doRound() error {
if clus.Tester.FailureShuffle { if clus.Tester.FailureShuffle {
clus.shuffleFailures() clus.shuffleFailures()
} }
clus.logger.Info(
"starting round",
zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()),
)
for i, f := range clus.failures { for i, f := range clus.failures {
clus.cs = i clus.cs = i
@ -126,7 +132,7 @@ func (clus *Cluster) doRound(round int) error {
zap.Int("case", clus.cs), zap.Int("case", clus.cs),
zap.String("desc", f.Desc()), zap.String("desc", f.Desc()),
) )
if err := f.Inject(clus, round); err != nil { if err := f.Inject(clus); err != nil {
return fmt.Errorf("injection error: %v", err) return fmt.Errorf("injection error: %v", err)
} }
clus.logger.Info( clus.logger.Info(
@ -145,7 +151,7 @@ func (clus *Cluster) doRound(round int) error {
zap.Int("case", clus.cs), zap.Int("case", clus.cs),
zap.String("desc", f.Desc()), zap.String("desc", f.Desc()),
) )
if err := f.Recover(clus, round); err != nil { if err := f.Recover(clus); err != nil {
return fmt.Errorf("recovery error: %v", err) return fmt.Errorf("recovery error: %v", err)
} }
clus.logger.Info( clus.logger.Info(
@ -174,6 +180,12 @@ func (clus *Cluster) doRound(round int) error {
zap.String("desc", f.Desc()), zap.String("desc", f.Desc()),
) )
} }
clus.logger.Info(
"finished round",
zap.Int("round", clus.rd),
zap.Strings("failures", clus.failureStrings()),
)
return nil return nil
} }

4
tools/functional-tester/tester/failure.go
View File

@ -21,10 +21,10 @@ package tester
type Failure interface { type Failure interface {
// Inject injeccts the failure into the testing cluster at the given // Inject injeccts the failure into the testing cluster at the given
// round. When calling the function, the cluster should be in health. // round. When calling the function, the cluster should be in health.
Inject(clus *Cluster, round int) error Inject(clus *Cluster) error
// Recover recovers the injected failure caused by the injection of the // Recover recovers the injected failure caused by the injection of the
// given round and wait for the recovery of the testing cluster. // given round and wait for the recovery of the testing cluster.
Recover(clus *Cluster, round int) error Recover(clus *Cluster) error
// Desc returns a description of the failure // Desc returns a description of the failure
Desc() string Desc() string
} }

4
tools/functional-tester/tester/failure_case_delay.go
View File

@ -25,8 +25,8 @@ type failureDelay struct {
delayDuration time.Duration delayDuration time.Duration
} }
func (f *failureDelay) Inject(clus *Cluster, round int) error { func (f *failureDelay) Inject(clus *Cluster) error {
if err := f.Failure.Inject(clus, round); err != nil { if err := f.Failure.Inject(clus); err != nil {
return err return err
} }
if f.delayDuration > 0 { if f.delayDuration > 0 {

8
tools/functional-tester/tester/failure_case_external.go
View File

@ -26,12 +26,12 @@ type failureExternal struct {
scriptPath string scriptPath string
} }
func (f *failureExternal) Inject(clus *Cluster, round int) error { func (f *failureExternal) Inject(clus *Cluster) error {
return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", round)).Run() return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", clus.rd)).Run()
} }
func (f *failureExternal) Recover(clus *Cluster, round int) error { func (f *failureExternal) Recover(clus *Cluster) error {
return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", round)).Run() return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run()
} }
func (f *failureExternal) Desc() string { return f.description } func (f *failureExternal) Desc() string { return f.description }

28
tools/functional-tester/tester/failure_case_kill.go
View File

@ -99,19 +99,19 @@ type failureLeader struct {
// failureUntilSnapshot injects a failure and waits for a snapshot event // failureUntilSnapshot injects a failure and waits for a snapshot event
type failureUntilSnapshot struct{ Failure } type failureUntilSnapshot struct{ Failure }
func (f *failureOne) Inject(clus *Cluster, round int) error { func (f *failureOne) Inject(clus *Cluster) error {
return f.injectMember(clus, round%len(clus.Members)) return f.injectMember(clus, clus.rd%len(clus.Members))
} }
func (f *failureOne) Recover(clus *Cluster, round int) error { func (f *failureOne) Recover(clus *Cluster) error {
if err := f.recoverMember(clus, round%len(clus.Members)); err != nil { if err := f.recoverMember(clus, clus.rd%len(clus.Members)); err != nil {
return err return err
} }
clus.logger.Info("wait health after recovering failureOne") clus.logger.Info("wait health after recovering failureOne")
return clus.WaitHealth() return clus.WaitHealth()
} }
func (f *failureAll) Inject(clus *Cluster, round int) error { func (f *failureAll) Inject(clus *Cluster) error {
for i := range clus.Members { for i := range clus.Members {
if err := f.injectMember(clus, i); err != nil { if err := f.injectMember(clus, i); err != nil {
return err return err
@ -120,7 +120,7 @@ func (f *failureAll) Inject(clus *Cluster, round int) error {
return nil return nil
} }
func (f *failureAll) Recover(clus *Cluster, round int) error { func (f *failureAll) Recover(clus *Cluster) error {
for i := range clus.Members { for i := range clus.Members {
if err := f.recoverMember(clus, i); err != nil { if err := f.recoverMember(clus, i); err != nil {
return err return err
@ -130,8 +130,8 @@ func (f *failureAll) Recover(clus *Cluster, round int) error {
return clus.WaitHealth() return clus.WaitHealth()
} }
func (f *failureQuorum) Inject(clus *Cluster, round int) error { func (f *failureQuorum) Inject(clus *Cluster) error {
for i := range killMap(len(clus.Members), round) { for i := range killMap(len(clus.Members), clus.rd) {
if err := f.injectMember(clus, i); err != nil { if err := f.injectMember(clus, i); err != nil {
return err return err
} }
@ -139,8 +139,8 @@ func (f *failureQuorum) Inject(clus *Cluster, round int) error {
return nil return nil
} }
func (f *failureQuorum) Recover(clus *Cluster, round int) error { func (f *failureQuorum) Recover(clus *Cluster) error {
for i := range killMap(len(clus.Members), round) { for i := range killMap(len(clus.Members), clus.rd) {
if err := f.recoverMember(clus, i); err != nil { if err := f.recoverMember(clus, i); err != nil {
return err return err
} }
@ -148,7 +148,7 @@ func (f *failureQuorum) Recover(clus *Cluster, round int) error {
return nil return nil
} }
func (f *failureLeader) Inject(clus *Cluster, round int) error { func (f *failureLeader) Inject(clus *Cluster) error {
idx, err := clus.GetLeader() idx, err := clus.GetLeader()
if err != nil { if err != nil {
return err return err
@ -157,7 +157,7 @@ func (f *failureLeader) Inject(clus *Cluster, round int) error {
return f.injectMember(clus, idx) return f.injectMember(clus, idx)
} }
func (f *failureLeader) Recover(clus *Cluster, round int) error { func (f *failureLeader) Recover(clus *Cluster) error {
if err := f.recoverMember(clus, f.idx); err != nil { if err := f.recoverMember(clus, f.idx); err != nil {
return err return err
} }
@ -165,8 +165,8 @@ func (f *failureLeader) Recover(clus *Cluster, round int) error {
return clus.WaitHealth() return clus.WaitHealth()
} }
func (f *failureUntilSnapshot) Inject(clus *Cluster, round int) error { func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
if err := f.Failure.Inject(clus, round); err != nil { if err := f.Failure.Inject(clus); err != nil {
return err return err
} }
if len(clus.Members) < 3 { if len(clus.Members) < 3 {

4
tools/functional-tester/tester/failure_case_no_op.go
View File

@ -16,8 +16,8 @@ package tester
type failureNoOp failureByFunc type failureNoOp failureByFunc
func (f *failureNoOp) Inject(clus *Cluster, round int) error { return nil } func (f *failureNoOp) Inject(clus *Cluster) error { return nil }
func (f *failureNoOp) Recover(clus *Cluster, round int) error { return nil } func (f *failureNoOp) Recover(clus *Cluster) error { return nil }
func newFailureNoOp() Failure { func newFailureNoOp() Failure {
return &failureNoOp{ return &failureNoOp{