Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Abort if failpoint injecton failed

Browse Source 
If one of nodes is unhealthy the test would never finish as watchers
would never reach max revision.

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz 2023-12-03 17:24:53 +01:00
parent 9d18fb0c6c
commit 5175652a8e
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/robustness/failpoint/failpoint.go
View File

@ -78,7 +78,7 @@ func Validate(clus *e2e.EtcdProcessCluster, failpoint Failpoint) error {
return nil
}
func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint) {
func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint) error {
ctx, cancel := context.WithTimeout(ctx, triggerTimeout)
defer cancel()
var err error
@ -89,8 +89,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
lg.Info("Verifying cluster health before failpoint", zap.String("failpoint", failpoint.Name()))
if err = verifyClusterHealth(ctx, t, clus); err != nil {
t.Errorf("failed to verify cluster health before failpoint injection, err: %v", err)
return
return fmt.Errorf("failed to verify cluster health before failpoint injection, err: %v", err)
}
lg.Info("Triggering failpoint", zap.String("failpoint", failpoint.Name()))
@ -98,8 +97,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
if err != nil {
select {
case <-ctx.Done():
t.Errorf("Triggering failpoints timed out, err: %v", ctx.Err())
return
return fmt.Errorf("Triggering failpoints timed out, err: %v", ctx.Err())
default:
}
lg.Info("Failed to trigger failpoint", zap.String("failpoint", failpoint.Name()), zap.Error(err))
@ -109,8 +107,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
lg.Info("Verifying cluster health after failpoint", zap.String("failpoint", failpoint.Name()))
if err = verifyClusterHealth(ctx, t, clus); err != nil {
t.Errorf("failed to verify cluster health after failpoint injection, err: %v", err)
return
return fmt.Errorf("failed to verify cluster health after failpoint injection, err: %v", err)
}
successes++
@ -119,7 +116,7 @@ func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdPro
t.Errorf("failed to trigger failpoints enough times, err: %v", err)
}
return
return nil
}
func verifyClusterHealth(ctx context.Context, _ *testing.T, clus *e2e.EtcdProcessCluster) error {

13
tests/robustness/main_test.go
View File

@ -88,6 +88,8 @@ func testRobustness(ctx context.Context, t *testing.T, lg *zap.Logger, s testSce
}
func (s testScenario) run(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) (reports []report.ClientReport) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
g := errgroup.Group{}
var operationReport, watchReport []report.ClientReport
finishTraffic := make(chan struct{})
@ -98,15 +100,22 @@ func (s testScenario) run(ctx context.Context, t *testing.T, lg *zap.Logger, clu
ids := identity.NewIdProvider()
g.Go(func() error {
defer close(finishTraffic)
failpoint.Inject(ctx, t, lg, clus, s.failpoint)
err := failpoint.Inject(ctx, t, lg, clus, s.failpoint)
if err != nil {
t.Error(err)
cancel()
}
time.Sleep(time.Second)
lg.Info("Finished injecting failures")
return nil
})
maxRevisionChan := make(chan int64, 1)
g.Go(func() error {
defer close(maxRevisionChan)
operationReport = traffic.SimulateTraffic(ctx, t, lg, clus, s.profile, s.traffic, finishTraffic, baseTime, ids)
maxRevisionChan <- operationsMaxRevision(operationReport)
maxRevision := operationsMaxRevision(operationReport)
maxRevisionChan <- maxRevision
lg.Info("Finished simulating traffic", zap.Int64("max-revision", maxRevision))
return nil
})
g.Go(func() error {