Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

robustness: add mix version scenario with fixed leader.

Browse Source 
Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang 2024-05-16 22:14:39 +00:00
parent b54d7552a7
commit 0f94c2ca4f
7 changed files with 131 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
tests/framework/e2e/cluster.go
View File

@ -151,8 +151,11 @@ type EtcdProcessClusterConfig struct {
// Cluster setup config
ClusterSize int
RollingStart bool
ClusterSize int
// InitialLeaderIndex makes sure the leader is the ith proc
// when the cluster starts if it is specified (>=0).
InitialLeaderIndex int
RollingStart bool
// BaseDataDirPath specifies the data-dir for the members. If test cases
// do not specify `BaseDataDirPath`, then e2e framework creates a
// temporary directory for each member; otherwise, it creates a
@ -180,10 +183,10 @@ type EtcdProcessClusterConfig struct {
func DefaultConfig() *EtcdProcessClusterConfig {
cfg := &EtcdProcessClusterConfig{
ClusterSize: 3,
CN: true,
ServerConfig: *embed.NewConfig(),
ClusterSize: 3,
CN: true,
InitialLeaderIndex: -1,
ServerConfig: *embed.NewConfig(),
}
cfg.ServerConfig.InitialClusterToken = "new"
return cfg
@ -207,6 +210,10 @@ func WithVersion(version ClusterVersion) EPClusterOption {
return func(c *EtcdProcessClusterConfig) { c.Version = version }
}
func WithInitialLeaderIndex(i int) EPClusterOption {
return func(c *EtcdProcessClusterConfig) { c.InitialLeaderIndex = i }
}
func WithDataDirPath(path string) EPClusterOption {
return func(c *EtcdProcessClusterConfig) { c.BaseDataDirPath = path }
}
@ -398,6 +405,16 @@ func InitEtcdProcessCluster(t testing.TB, cfg *EtcdProcessClusterConfig) (*EtcdP
cfg.ServerConfig.SnapshotCount = etcdserver.DefaultSnapshotCount
}
// validate SnapshotCatchUpEntries could be set for at least one member
if cfg.ServerConfig.SnapshotCatchUpEntries != etcdserver.DefaultSnapshotCatchUpEntries {
if !CouldSetSnapshotCatchupEntries(BinPath.Etcd) {
return nil, fmt.Errorf("cannot set SnapshotCatchUpEntries for current etcd version: %s", BinPath.Etcd)
}
if cfg.Version == LastVersion && !CouldSetSnapshotCatchupEntries(BinPath.EtcdLastRelease) {
return nil, fmt.Errorf("cannot set SnapshotCatchUpEntries for last etcd version: %s", BinPath.EtcdLastRelease)
}
}
etcdCfgs := cfg.EtcdAllServerProcessConfigs(t)
epc := &EtcdProcessCluster{
Cfg: cfg,
@ -437,7 +454,11 @@ func StartEtcdProcessCluster(ctx context.Context, t testing.TB, epc *EtcdProcess
t.Skip("please run 'make gofail-enable && make build' before running the test")
}
}
if cfg.InitialLeaderIndex >= 0 {
if err := epc.MoveLeader(ctx, t, cfg.InitialLeaderIndex); err != nil {
return nil, fmt.Errorf("failed to move leader: %v", err)
}
}
return epc, nil
}
@ -1050,3 +1071,31 @@ func (epc *EtcdProcessCluster) WaitMembersForLeader(ctx context.Context, t testi
t.Fatal("impossible path of execution")
return -1
}
// MoveLeader moves the leader to the ith process.
func (epc *EtcdProcessCluster) MoveLeader(ctx context.Context, t testing.TB, i int) error {
if i < 0 || i >= len(epc.Procs) {
return fmt.Errorf("invalid index: %d, must between 0 and %d", i, len(epc.Procs)-1)
}
t.Logf("moving leader to Procs[%d]", i)
oldLeader := epc.WaitMembersForLeader(ctx, t, epc.Procs)
if oldLeader == i {
t.Logf("Procs[%d] is already the leader", i)
return nil
}
resp, err := epc.Procs[i].Etcdctl().Status(ctx)
if err != nil {
return err
}
memberID := resp[0].Header.MemberId
err = epc.Procs[oldLeader].Etcdctl().MoveLeader(ctx, memberID)
if err != nil {
return err
}
newLeader := epc.WaitMembersForLeader(ctx, t, epc.Procs)
if newLeader != i {
t.Fatalf("expect new leader to be Procs[%d] but got Procs[%d]", i, newLeader)
}
t.Logf("moved leader from Procs[%d] to Procs[%d]", oldLeader, i)
return nil
}

32
tests/framework/e2e/cluster_test.go
View File

@ -15,17 +15,22 @@
package e2e
import (
"fmt"
"testing"
"github.com/coreos/go-semver/semver"
"github.com/stretchr/testify/assert"
)
func TestEtcdServerProcessConfig(t *testing.T) {
v3_5_12 := semver.Version{Major: 3, Minor: 5, Patch: 12}
v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13}
tcs := []struct {
name string
config *EtcdProcessClusterConfig
expectArgsNotContain []string
expectArgsContain []string
mockBinaryVersion *semver.Version
}{
{
name: "Default",
@ -73,10 +78,37 @@ func TestEtcdServerProcessConfig(t *testing.T) {
expectArgsContain: []string{
"--experimental-snapshot-catchup-entries=100",
},
mockBinaryVersion: &v3_5_13,
},
{
name: "CatchUpEntriesNoVersion",
config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)),
expectArgsNotContain: []string{
"--experimental-snapshot-catchup-entries=100",
},
},
{
name: "CatchUpEntriesOldVersion",
config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)),
expectArgsNotContain: []string{
"--experimental-snapshot-catchup-entries=100",
},
mockBinaryVersion: &v3_5_12,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
var mockGetVersionFromBinary func(binaryPath string) (*semver.Version, error)
if tc.mockBinaryVersion == nil {
mockGetVersionFromBinary = func(binaryPath string) (*semver.Version, error) {
return nil, fmt.Errorf("could not get binary version")
}
} else {
mockGetVersionFromBinary = func(binaryPath string) (*semver.Version, error) {
return tc.mockBinaryVersion, nil
}
}
setGetVersionFromBinary(t, mockGetVersionFromBinary)
args := tc.config.EtcdServerProcessConfig(t, 0).Args
if len(tc.expectArgsContain) != 0 {
assert.Subset(t, args, tc.expectArgsContain)

21
tests/framework/e2e/etcd_process.go
View File

@ -485,7 +485,10 @@ func parseFailpointsBody(body io.Reader) (map[string]string, error) {
return failpoints, nil
}
func GetVersionFromBinary(binaryPath string) (*semver.Version, error) {
var GetVersionFromBinary = func(binaryPath string) (*semver.Version, error) {
if !fileutil.Exist(binaryPath) {
return nil, fmt.Errorf("binary path does not exist: %s", binaryPath)
}
lines, err := RunUtilCompletion([]string{binaryPath, "--version"}, nil)
if err != nil {
return nil, fmt.Errorf("could not find binary version from %s, err: %w", binaryPath, err)
@ -509,15 +512,19 @@ func GetVersionFromBinary(binaryPath string) (*semver.Version, error) {
return nil, fmt.Errorf("could not find version in binary output of %s, lines outputted were %v", binaryPath, lines)
}
// setGetVersionFromBinary changes the GetVersionFromBinary function to a mock in testing.
func setGetVersionFromBinary(tb testing.TB, f func(binaryPath string) (*semver.Version, error)) {
origGetVersionFromBinary := GetVersionFromBinary
GetVersionFromBinary = f
tb.Cleanup(func() {
GetVersionFromBinary = origGetVersionFromBinary
})
}
func CouldSetSnapshotCatchupEntries(execPath string) bool {
if !fileutil.Exist(execPath) {
// default to true if the binary does not exist, because binary might not exist for unit test,
// which does not really matter because "experimental-snapshot-catchup-entries" can be set for v3.6 and v3.5.
return true
}
v, err := GetVersionFromBinary(execPath)
if err != nil {
panic(err)
return false
}
// snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808
v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13}

7
tests/framework/e2e/etcdctl.go
View File

@ -312,6 +312,13 @@ func (ctl *EtcdctlV3) MemberPromote(ctx context.Context, id uint64) (*clientv3.M
return &resp, err
}
// MoveLeader requests current leader to transfer its leadership to the transferee.
// Request must be made to the leader.
func (ctl *EtcdctlV3) MoveLeader(ctx context.Context, transfereeID uint64) error {
_, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("move-leader", fmt.Sprintf("%x", transfereeID)), nil, expect.ExpectedResponse{Value: "Leadership transferred"})
return err
}
func (ctl *EtcdctlV3) cmdArgs(args ...string) []string {
cmdArgs := []string{BinPath.Etcdctl}
for k, v := range ctl.flags() {

2
tests/robustness/failpoint/cluster.go
View File

@ -137,7 +137,7 @@ func (f memberReplace) Name() string {
func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess) bool {
// a lower etcd version may not be able to join a cluster with higher cluster version.
return config.ClusterSize > 1 && member.Config().ExecPath != e2e.BinPath.EtcdLastRelease
return config.ClusterSize > 1 && (config.Version == e2e.QuorumLastVersion || member.Config().ExecPath == e2e.BinPath.Etcd)
}
func getID(ctx context.Context, cc clientv3.Cluster, name string) (id uint64, found bool, err error) {

4
tests/robustness/options/server_config_options.go
View File

@ -53,3 +53,7 @@ func WithExperimentalWatchProgressNotifyInterval(input ...time.Duration) e2e.EPC
func WithVersion(input ...e2e.ClusterVersion) e2e.EPClusterOption {
return func(c *e2e.EtcdProcessClusterConfig) { c.Version = input[internalRand.Intn(len(input))] }
}
func WithInitialLeaderIndex(input ...int) e2e.EPClusterOption {
return func(c *e2e.EtcdProcessClusterConfig) { c.InitialLeaderIndex = input[internalRand.Intn(len(input))] }
}

20
tests/robustness/scenarios.go
View File

@ -69,9 +69,23 @@ func exploratoryScenarios(_ *testing.T) []testScenario {
options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}),
}
// 66% current version, 33% MinorityLastVersion and QuorumLastVersion
mixedVersionOption := options.WithVersion(e2e.CurrentVersion, e2e.CurrentVersion, e2e.CurrentVersion,
e2e.CurrentVersion, e2e.MinorityLastVersion, e2e.QuorumLastVersion)
mixedVersionOption := options.WithClusterOptionGroups(
// 60% with all members of current version
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)},
// 10% with 2 members of current version, 1 member last version, leader is current version
options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)},
// 10% with 2 members of current version, 1 member last version, leader is last version
options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)},
// 10% with 2 members of last version, 1 member current version, leader is last version
options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)},
// 10% with 2 members of last version, 1 member current version, leader is current version
options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)},
)
baseOptions := []e2e.EPClusterOption{
options.WithSnapshotCount(50, 100, 1000),