robustness: add mix version option in exploratoryScenarios.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

tests/e2e/etcd_mix_versions_test.go

@@ -130,12 +130,13 @@ func mixVersionsSnapshotTestByMockPartition(t *testing.T, cfg *e2e.EtcdProcessCl
 		t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease)
 	}
 
+	clusterOptions := []e2e.EPClusterOption{e2e.WithConfig(cfg), e2e.WithSnapshotCount(10)}
+	// TODO: remove version check after 3.5.14 release.
+	if cfg.Version == e2e.CurrentVersion {
+		clusterOptions = append(clusterOptions, e2e.WithSnapshotCatchUpEntries(10))
+	}
 	t.Logf("Create an etcd cluster with %d member", cfg.ClusterSize)
-	epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t,
+	epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t, clusterOptions...)
-		e2e.WithConfig(cfg),
-		e2e.WithSnapshotCount(10),
-		e2e.WithSnapshotCatchUpEntries(10),
-	)
 	require.NoError(t, err, "failed to start etcd cluster: %v", err)
 	defer func() {
 		derr := epc.Close()
@@ -161,7 +162,7 @@ func mixVersionsSnapshotTestByMockPartition(t *testing.T, cfg *e2e.EtcdProcessCl
 	assertKVHash(t, epc)
 
 	leaderEPC = epc.Procs[epc.WaitLeader(t)]
-	if leaderEPC.Config().ExecPath == e2e.BinPath.Etcd {
+	if cfg.Version == e2e.CurrentVersion {
 		t.Log("Verify logs to check snapshot be sent from leader to follower")
 		e2e.AssertProcessLogs(t, leaderEPC, "sent database snapshot")
 	}

tests/framework/e2e/cluster.go

@@ -570,27 +570,6 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		args = append(args, "--discovery="+cfg.Discovery)
 	}
 
-	defaultValues := values(*embed.NewConfig())
-	overrideValues := values(cfg.ServerConfig)
-	for flag, value := range overrideValues {
-		if defaultValue := defaultValues[flag]; value == "" || value == defaultValue {
-			continue
-		}
-		if flag == "experimental-snapshot-catchup-entries" && !(cfg.Version == CurrentVersion || (cfg.Version == MinorityLastVersion && i <= cfg.ClusterSize/2) || (cfg.Version == QuorumLastVersion && i > cfg.ClusterSize/2)) {
-			continue
-		}
-		args = append(args, fmt.Sprintf("--%s=%s", flag, value))
-	}
-	envVars := map[string]string{}
-	for key, value := range cfg.EnvVars {
-		envVars[key] = value
-	}
-	var gofailPort int
-	if cfg.GoFailEnabled {
-		gofailPort = (i+1)*10000 + 2381
-		envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort)
-	}
-
 	var execPath string
 	switch cfg.Version {
 	case CurrentVersion:
@@ -613,6 +592,27 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		panic(fmt.Sprintf("Unknown cluster version %v", cfg.Version))
 	}
 
+	defaultValues := values(*embed.NewConfig())
+	overrideValues := values(cfg.ServerConfig)
+	for flag, value := range overrideValues {
+		if defaultValue := defaultValues[flag]; value == "" || value == defaultValue {
+			continue
+		}
+		if flag == "experimental-snapshot-catchup-entries" && !CouldSetSnapshotCatchupEntries(execPath) {
+			continue
+		}
+		args = append(args, fmt.Sprintf("--%s=%s", flag, value))
+	}
+	envVars := map[string]string{}
+	for key, value := range cfg.EnvVars {
+		envVars[key] = value
+	}
+	var gofailPort int
+	if cfg.GoFailEnabled {
+		gofailPort = (i+1)*10000 + 2381
+		envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort)
+	}
+
 	return &EtcdServerProcessConfig{
 		lg:                  cfg.Logger,
 		ExecPath:            execPath,

tests/framework/e2e/cluster_test.go

@@ -74,13 +74,6 @@ func TestEtcdServerProcessConfig(t *testing.T) {
 				"--experimental-snapshot-catchup-entries=100",
 			},
 		},
-		{
-			name:   "CatchUpEntriesLastVersion",
-			config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)),
-			expectArgsNotContain: []string{
-				"--experimental-snapshot-catchup-entries=100",
-			},
-		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {

tests/framework/e2e/etcd_process.go

@@ -508,3 +508,18 @@ func GetVersionFromBinary(binaryPath string) (*semver.Version, error) {
 
 	return nil, fmt.Errorf("could not find version in binary output of %s, lines outputted were %v", binaryPath, lines)
 }
+
+func CouldSetSnapshotCatchupEntries(execPath string) bool {
+	if !fileutil.Exist(execPath) {
+		// default to true if the binary does not exist, because binary might not exist for unit test,
+		// which does not really matter because "experimental-snapshot-catchup-entries" can be set for v3.6 and v3.5.
+		return true
+	}
+	v, err := GetVersionFromBinary(execPath)
+	if err != nil {
+		panic(err)
+	}
+	// snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808
+	v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13}
+	return v.Compare(v3_5_13) >= 0
+}

tests/robustness/failpoint/cluster.go

@@ -135,8 +135,9 @@ func (f memberReplace) Name() string {
 	return "MemberReplace"
 }
 
-func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, _ e2e.EtcdProcess) bool {
+func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess) bool {
-	return config.ClusterSize > 1
+	// a lower etcd version may not be able to join a cluster with higher cluster version.
+	return config.ClusterSize > 1 && member.Config().ExecPath != e2e.BinPath.EtcdLastRelease
 }
 
 func getID(ctx context.Context, cc clientv3.Cluster, name string) (id uint64, found bool, err error) {

tests/robustness/failpoint/network.go

@@ -59,7 +59,7 @@ func (tb triggerBlackhole) Trigger(ctx context.Context, t *testing.T, member e2e
 func (tb triggerBlackhole) Available(config e2e.EtcdProcessClusterConfig, process e2e.EtcdProcess) bool {
 	// Avoid triggering failpoint if waiting for failpoint would take too long to fit into timeout.
 	// Number of required entries for snapshot depends on etcd configuration.
-	if tb.waitTillSnapshot && entriesToGuaranteeSnapshot(config) > 200 {
+	if tb.waitTillSnapshot && (entriesToGuaranteeSnapshot(config) > 200 || !e2e.CouldSetSnapshotCatchupEntries(process.Config().ExecPath)) {
 		return false
 	}
 	return config.ClusterSize > 1 && process.PeerProxy() != nil

tests/robustness/options/server_config_options.go

@@ -49,3 +49,7 @@ func WithExperimentalWatchProgressNotifyInterval(input ...time.Duration) e2e.EPC
 		c.ServerConfig.ExperimentalWatchProgressNotifyInterval = input[internalRand.Intn(len(input))]
 	}
 }
+
+func WithVersion(input ...e2e.ClusterVersion) e2e.EPClusterOption {
+	return func(c *e2e.EtcdProcessClusterConfig) { c.Version = input[internalRand.Intn(len(input))] }
+}

tests/robustness/scenarios.go

@@ -19,9 +19,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/go-semver/semver"
-
 	"go.etcd.io/etcd/api/v3/version"
+	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
 	"go.etcd.io/etcd/tests/v3/robustness/failpoint"
 	"go.etcd.io/etcd/tests/v3/robustness/options"
@@ -61,11 +60,7 @@ type testScenario struct {
 	watch     watchConfig
 }
 
-func exploratoryScenarios(t *testing.T) []testScenario {
+func exploratoryScenarios(_ *testing.T) []testScenario {
-	v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
-	if err != nil {
-		t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err)
-	}
 	enableLazyFS := e2e.BinPath.LazyFSAvailable()
 	randomizableOptions := []e2e.EPClusterOption{
 		options.WithClusterOptionGroups(
@@ -74,6 +69,10 @@ func exploratoryScenarios(t *testing.T) []testScenario {
 			options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}),
 	}
 
+	// 66% current version, 33% MinorityLastVersion and QuorumLastVersion
+	mixedVersionOption := options.WithVersion(e2e.CurrentVersion, e2e.CurrentVersion, e2e.CurrentVersion,
+		e2e.CurrentVersion, e2e.MinorityLastVersion, e2e.QuorumLastVersion)
+
 	baseOptions := []e2e.EPClusterOption{
 		options.WithSnapshotCount(50, 100, 1000),
 		options.WithSubsetOptions(randomizableOptions...),
@@ -81,9 +80,8 @@ func exploratoryScenarios(t *testing.T) []testScenario {
 		e2e.WithCompactionBatchLimit(100),
 		e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
 	}
-	// snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808
+
-	v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13}
+	if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
-	if v.Compare(v3_5_13) >= 0 {
 		baseOptions = append(baseOptions, e2e.WithSnapshotCatchUpEntries(100))
 	}
 	scenarios := []testScenario{}
@@ -109,6 +107,9 @@ func exploratoryScenarios(t *testing.T) []testScenario {
 		clusterOfSize3Options := baseOptions
 		clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true))
 		clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true))
+		if fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
+			clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption)
+		}
 		scenarios = append(scenarios, testScenario{
 			name:    name,
 			traffic: tp.Traffic,
@@ -172,8 +173,7 @@ func regressionScenarios(t *testing.T) []testScenario {
 			e2e.WithPeerProxy(true),
 			e2e.WithIsPeerTLS(true),
 		}
-		v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13}
+		if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
-		if v.Compare(v3_5_13) >= 0 {
 			opts = append(opts, e2e.WithSnapshotCatchUpEntries(100))
 		}
 		scenarios = append(scenarios, testScenario{