Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17989 from serathius/robustness-pick-snapshot

Browse Source 
Prevent picking a failpoint that waiting till snapshot that doesn't support lower snapshot catchup entries but allow reproducing issue #15271
This commit is contained in:
Marek Siarkowicz 2024-05-13 14:33:02 +02:00 committed by GitHub
commit d0ea231f8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 17 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tests/robustness/failpoint/network.go
View File

@ -57,6 +57,11 @@ func (tb triggerBlackhole) Trigger(ctx context.Context, t *testing.T, member e2e
}
func (tb triggerBlackhole) Available(config e2e.EtcdProcessClusterConfig, process e2e.EtcdProcess) bool {
// Avoid triggering failpoint if waiting for failpoint would take too long to fit into timeout.
// Number of required entries for snapshot depends on etcd configuration.
if tb.waitTillSnapshot && entriesToGuaranteeSnapshot(config) > 200 {
return false
}
return config.ClusterSize > 1 && process.PeerProxy() != nil
}
@ -127,9 +132,7 @@ func waitTillSnapshot(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCl
}
t.Logf("clusterRevision: %d, blackholedMemberRevision: %d", clusterRevision, blackholedMemberRevision)
// Blackholed member has to be sufficiently behind to trigger snapshot transfer.
// Need to make sure leader compacted latest revBlackholedMem inside EtcdServer.snapshot.
// That's why we wait for clus.Cfg.SnapshotCount (to trigger snapshot) + clus.Cfg.SnapshotCatchUpEntries (EtcdServer.snapshot compaction offset)
if clusterRevision-blackholedMemberRevision > int64(clus.Cfg.ServerConfig.SnapshotCount+clus.Cfg.ServerConfig.SnapshotCatchUpEntries) {
if clusterRevision-blackholedMemberRevision > int64(entriesToGuaranteeSnapshot(*clus.Cfg)) {
break
}
time.Sleep(100 * time.Millisecond)
@ -137,6 +140,12 @@ func waitTillSnapshot(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCl
return nil
}
func entriesToGuaranteeSnapshot(config e2e.EtcdProcessClusterConfig) uint64 {
// Need to make sure leader compacted latest revBlackholedMem inside EtcdServer.snapshot.
// That's why we wait for clus.Cfg.SnapshotCount (to trigger snapshot) + clus.Cfg.SnapshotCatchUpEntries (EtcdServer.snapshot compaction offset)
return config.ServerConfig.SnapshotCount + config.ServerConfig.SnapshotCatchUpEntries
}
// latestRevisionForEndpoint gets latest revision of the first endpoint in Client.Endpoints list
func latestRevisionForEndpoint(ctx context.Context, c *clientv3.Client) (int64, error) {
resp, err := c.Status(ctx, c.Endpoints()[0])

4
tests/robustness/main_test.go
View File

@ -78,10 +78,6 @@ func testRobustness(ctx context.Context, t *testing.T, lg *zap.Logger, s testSce
t.Fatal(err)
}
}
err = failpoint.Validate(r.Cluster, s.failpoint)
if err != nil {
t.Fatal(err)
}
// t.Failed() returns false during panicking. We need to forcibly
// save data on panicking.

8
tests/robustness/scenarios.go
View File

@ -81,6 +81,11 @@ func exploratoryScenarios(t *testing.T) []testScenario {
e2e.WithCompactionBatchLimit(100),
e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
}
// snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808
v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13}
if v.Compare(v3_5_13) >= 0 {
baseOptions = append(baseOptions, e2e.WithSnapshotCatchUpEntries(100))
}
scenarios := []testScenario{}
for _, tp := range trafficProfiles {
name := filepath.Join(tp.Traffic.Name(), tp.Profile.Name, "ClusterOfSize1")
@ -104,9 +109,6 @@ func exploratoryScenarios(t *testing.T) []testScenario {
clusterOfSize3Options := baseOptions
clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true))
clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true))
if !v.LessThan(version.V3_6) {
clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithSnapshotCatchUpEntries(100))
}
scenarios = append(scenarios, testScenario{
name: name,
traffic: tp.Traffic,