From 3f05f7de92a6b6243d7425033aa8622d6f23fe9e Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sun, 13 Feb 2022 23:26:53 +0100 Subject: [PATCH 1/8] tests: Use restart in downgrade tests --- tests/e2e/cluster_downgrade_test.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 06031ea48..2e362d200 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -27,7 +27,7 @@ import ( ) func TestDowngradeUpgrade(t *testing.T) { - currentEtcdBinary := "" + currentEtcdBinary := e2e.BinDir + "/etcd" lastReleaseBinary := e2e.BinDir + "/etcd-last-release" if !fileutil.Exist(lastReleaseBinary) { t.Skipf("%q does not exist", lastReleaseBinary) @@ -40,24 +40,24 @@ func TestDowngradeUpgrade(t *testing.T) { e2e.BeforeTest(t) dataDirPath := t.TempDir() - epc := startEtcd(t, currentEtcdBinary, dataDirPath) + epc := newCluster(t, currentEtcdBinary, dataDirPath) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) downgradeEnable(t, epc, lastVersion) expectLog(t, epc, "The server is ready to downgrade") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) - stopEtcd(t, epc) - epc = startEtcd(t, lastReleaseBinary, dataDirPath) + stopEtcd(t, epc.Procs[0]) + startEtcd(t, epc, lastReleaseBinary) expectLog(t, epc, "the cluster has been downgraded") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) - stopEtcd(t, epc) - epc = startEtcd(t, currentEtcdBinary, dataDirPath) + stopEtcd(t, epc.Procs[0]) + startEtcd(t, epc, currentEtcdBinary) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) } -func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessCluster { +func newCluster(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ ExecPath: execPath, DataDirPath: dataDirPath, @@ -76,6 +76,14 @@ func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessClust return epc } +func startEtcd(t *testing.T, epc *e2e.EtcdProcessCluster, execPath string) { + epc.Procs[0].Config().ExecPath = execPath + err := epc.Procs[0].Restart() + if err != nil { + t.Fatalf("could not start etcd process cluster (%v)", err) + } +} + func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) { t.Log("etcdctl downgrade...") c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) @@ -87,9 +95,9 @@ func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Versi }) } -func stopEtcd(t *testing.T, epc *e2e.EtcdProcessCluster) { +func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { t.Log("Stopping the server...") - if err := epc.Procs[0].Stop(); err != nil { + if err := ep.Stop(); err != nil { t.Fatal(err) } } From 923096a2bb629ecdeae80725769ceb5718d3ac90 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sun, 13 Feb 2022 23:31:08 +0100 Subject: [PATCH 2/8] tests: Simplify downgrade tests --- tests/e2e/cluster_downgrade_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 2e362d200..e5e74528a 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -38,9 +38,8 @@ func TestDowngradeUpgrade(t *testing.T) { lastVersionStr := fmt.Sprintf("%d.%d", lastVersion.Major, lastVersion.Minor) e2e.BeforeTest(t) - dataDirPath := t.TempDir() - epc := newCluster(t, currentEtcdBinary, dataDirPath) + epc := newCluster(t, currentEtcdBinary) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) downgradeEnable(t, epc, lastVersion) @@ -57,10 +56,9 @@ func TestDowngradeUpgrade(t *testing.T) { validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) } -func newCluster(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessCluster { +func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ ExecPath: execPath, - DataDirPath: dataDirPath, ClusterSize: 1, InitialToken: "new", KeepDataDir: true, From 75449c075df341e580acf2f2333ec854b4c75826 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 10:04:50 +0100 Subject: [PATCH 3/8] tests: Allow sepecting member to in curl commands --- tests/e2e/cluster_downgrade_test.go | 5 +++-- tests/e2e/v3_curl_test.go | 5 +++-- tests/framework/e2e/curl.go | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index e5e74528a..84ed7183c 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -16,6 +16,7 @@ package e2e import ( "fmt" + "math/rand" "testing" "time" @@ -105,13 +106,13 @@ func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.V // Two separate calls to expect as it doesn't support multiple matches on the same line testutils.ExecuteWithTimeout(t, 20*time.Second, func() { if expect.Server != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) if err != nil { t.Fatal(err) } } if expect.Cluster != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) if err != nil { t.Fatal(err) } diff --git a/tests/e2e/v3_curl_test.go b/tests/e2e/v3_curl_test.go index 17fe6590b..35450bbe4 100644 --- a/tests/e2e/v3_curl_test.go +++ b/tests/e2e/v3_curl_test.go @@ -18,6 +18,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math/rand" "path" "strconv" "testing" @@ -243,7 +244,7 @@ func testV3CurlAuth(cx ctlCtx) { lineFunc = func(txt string) bool { return true } ) - cmdArgs = e2e.CURLPrefixArgs(cx.epc, "POST", e2e.CURLReq{Endpoint: path.Join(p, "/auth/authenticate"), Value: string(authreq)}) + cmdArgs = e2e.CURLPrefixArgs(cx.epc.Cfg, cx.epc.Procs[rand.Intn(cx.epc.Cfg.ClusterSize)], "POST", e2e.CURLReq{Endpoint: path.Join(p, "/auth/authenticate"), Value: string(authreq)}) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) testutil.AssertNil(cx.t, err) defer proc.Close() @@ -282,7 +283,7 @@ func testV3CurlCampaign(cx ctlCtx) { if err != nil { cx.t.Fatal(err) } - cargs := e2e.CURLPrefixArgs(cx.epc, "POST", e2e.CURLReq{ + cargs := e2e.CURLPrefixArgs(cx.epc.Cfg, cx.epc.Procs[rand.Intn(cx.epc.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: path.Join(cx.apiPrefix, "/election/campaign"), Value: string(cdata), }) diff --git a/tests/framework/e2e/curl.go b/tests/framework/e2e/curl.go index 284b49aaa..a3b11de85 100644 --- a/tests/framework/e2e/curl.go +++ b/tests/framework/e2e/curl.go @@ -40,20 +40,20 @@ type CURLReq struct { // CURLPrefixArgs builds the beginning of a curl command for a given key // addressed to a random URL in the given cluster. -func CURLPrefixArgs(clus *EtcdProcessCluster, method string, req CURLReq) []string { +func CURLPrefixArgs(cfg *EtcdProcessClusterConfig, member EtcdProcess, method string, req CURLReq) []string { var ( cmdArgs = []string{"curl"} - acurl = clus.Procs[rand.Intn(clus.Cfg.ClusterSize)].Config().Acurl + acurl = member.Config().Acurl ) if req.MetricsURLScheme != "https" { if req.IsTLS { - if clus.Cfg.ClientTLS != ClientTLSAndNonTLS { + if cfg.ClientTLS != ClientTLSAndNonTLS { panic("should not use cURLPrefixArgsUseTLS when serving only TLS or non-TLS") } cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) - acurl = ToTLS(clus.Procs[rand.Intn(clus.Cfg.ClusterSize)].Config().Acurl) - } else if clus.Cfg.ClientTLS == ClientTLS { - if !clus.Cfg.NoCN { + acurl = ToTLS(member.Config().Acurl) + } else if cfg.ClientTLS == ClientTLS { + if !cfg.NoCN { cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) } else { cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath3, "--key", PrivateKeyPath3) @@ -61,7 +61,7 @@ func CURLPrefixArgs(clus *EtcdProcessCluster, method string, req CURLReq) []stri } } if req.MetricsURLScheme != "" { - acurl = clus.Procs[rand.Intn(clus.Cfg.ClusterSize)].EndpointsMetrics()[0] + acurl = member.EndpointsMetrics()[0] } ep := acurl + req.Endpoint @@ -94,13 +94,13 @@ func CURLPrefixArgs(clus *EtcdProcessCluster, method string, req CURLReq) []stri } func CURLPost(clus *EtcdProcessCluster, req CURLReq) error { - return SpawnWithExpect(CURLPrefixArgs(clus, "POST", req), req.Expected) + return SpawnWithExpect(CURLPrefixArgs(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", req), req.Expected) } func CURLPut(clus *EtcdProcessCluster, req CURLReq) error { - return SpawnWithExpect(CURLPrefixArgs(clus, "PUT", req), req.Expected) + return SpawnWithExpect(CURLPrefixArgs(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "PUT", req), req.Expected) } func CURLGet(clus *EtcdProcessCluster, req CURLReq) error { - return SpawnWithExpect(CURLPrefixArgs(clus, "GET", req), req.Expected) + return SpawnWithExpect(CURLPrefixArgs(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "GET", req), req.Expected) } From 7b04674d178c17c6145b440acf3b30c1b1ff0cc8 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 10:30:36 +0100 Subject: [PATCH 4/8] server: Move fatal calls out of internal function to make stacktrace cleaner --- tests/e2e/cluster_downgrade_test.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 84ed7183c..be20a21c1 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -104,28 +104,33 @@ func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.Versions) { t.Log("Validate version") // Two separate calls to expect as it doesn't support multiple matches on the same line + var err error testutils.ExecuteWithTimeout(t, 20*time.Second, func() { if expect.Server != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) if err != nil { - t.Fatal(err) + return } } if expect.Cluster != "" { - err := e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) if err != nil { - t.Fatal(err) + return } } }) + if err != nil { + t.Fatal(err) + } } func expectLog(t *testing.T, epc *e2e.EtcdProcessCluster, expectLog string) { t.Helper() + var err error testutils.ExecuteWithTimeout(t, 30*time.Second, func() { - _, err := epc.Procs[0].Logs().Expect(expectLog) - if err != nil { - t.Fatal(err) - } + _, err = epc.Procs[0].Logs().Expect(expectLog) }) + if err != nil { + t.Fatal(err) + } } From c9af75dd8da7913886617ad249fc07c70102f903 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 10:40:23 +0100 Subject: [PATCH 5/8] tests: Improve logging in downgrade tests --- tests/e2e/cluster_downgrade_test.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index be20a21c1..6b38cdb67 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -40,21 +40,33 @@ func TestDowngradeUpgrade(t *testing.T) { e2e.BeforeTest(t) + t.Logf("Create cluster with version %s", currentVersionStr) epc := newCluster(t, currentEtcdBinary) validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + t.Logf("Cluster created") + t.Logf("etcdctl downgrade enable %s", lastVersion) downgradeEnable(t, epc, lastVersion) + + t.Log("Downgrade enabled, validating if cluster is ready for downgrade") expectLog(t, epc, "The server is ready to downgrade") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + t.Log("Cluster is ready for downgrade") + t.Log("Starting downgrade process") stopEtcd(t, epc.Procs[0]) startEtcd(t, epc, lastReleaseBinary) expectLog(t, epc, "the cluster has been downgraded") + t.Log("All members downgraded, validating downgrade") validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) + t.Log("Downgrade complete") + t.Log("Starting upgrade process") stopEtcd(t, epc.Procs[0]) startEtcd(t, epc, currentEtcdBinary) + t.Log("All members upgraded, validating upgrade") validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + t.Log("Upgrade complete") } func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { @@ -84,7 +96,6 @@ func startEtcd(t *testing.T, epc *e2e.EtcdProcessCluster, execPath string) { } func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) { - t.Log("etcdctl downgrade...") c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) testutils.ExecuteWithTimeout(t, 20*time.Second, func() { err := c.DowngradeEnable(ver.String()) @@ -95,14 +106,12 @@ func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Versi } func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { - t.Log("Stopping the server...") if err := ep.Stop(); err != nil { t.Fatal(err) } } func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.Versions) { - t.Log("Validate version") // Two separate calls to expect as it doesn't support multiple matches on the same line var err error testutils.ExecuteWithTimeout(t, 20*time.Second, func() { From c50fbb9ad6008e32c456a37069029c67d805d14e Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 12:09:19 +0100 Subject: [PATCH 6/8] tests: Retry validating version during downgrades --- tests/e2e/cluster_downgrade_test.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 6b38cdb67..180293361 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -115,17 +115,22 @@ func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.V // Two separate calls to expect as it doesn't support multiple matches on the same line var err error testutils.ExecuteWithTimeout(t, 20*time.Second, func() { - if expect.Server != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) - if err != nil { - return + for { + if expect.Server != "" { + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + if err != nil { + time.Sleep(time.Second) + continue + } } - } - if expect.Cluster != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) - if err != nil { - return + if expect.Cluster != "" { + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + if err != nil { + time.Sleep(time.Second) + continue + } } + break } }) if err != nil { From a65f7da5cb9f3eb4f86ff7384ddca5e27004fb34 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Feb 2022 13:02:06 +0100 Subject: [PATCH 7/8] server: Implement HA downgrade test --- tests/e2e/cluster_downgrade_test.go | 77 ++++++++++++++++++++--------- tests/framework/e2e/cluster.go | 24 +++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 180293361..9458ec78e 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -16,7 +16,6 @@ package e2e import ( "fmt" - "math/rand" "testing" "time" @@ -27,7 +26,15 @@ import ( "go.etcd.io/etcd/tests/v3/framework/testutils" ) -func TestDowngradeUpgrade(t *testing.T) { +func TestDowngradeUpgradeClusterOf1(t *testing.T) { + testDowngradeUpgrade(t, 1) +} + +func TestDowngradeUpgradeClusterOf3(t *testing.T) { + testDowngradeUpgrade(t, 3) +} + +func testDowngradeUpgrade(t *testing.T, clusterSize int) { currentEtcdBinary := e2e.BinDir + "/etcd" lastReleaseBinary := e2e.BinDir + "/etcd-last-release" if !fileutil.Exist(lastReleaseBinary) { @@ -41,38 +48,52 @@ func TestDowngradeUpgrade(t *testing.T) { e2e.BeforeTest(t) t.Logf("Create cluster with version %s", currentVersionStr) - epc := newCluster(t, currentEtcdBinary) - validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + epc := newCluster(t, currentEtcdBinary, clusterSize) + for i := 0; i < len(epc.Procs); i++ { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + } t.Logf("Cluster created") t.Logf("etcdctl downgrade enable %s", lastVersion) downgradeEnable(t, epc, lastVersion) t.Log("Downgrade enabled, validating if cluster is ready for downgrade") - expectLog(t, epc, "The server is ready to downgrade") - validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + for i := 0; i < len(epc.Procs); i++ { + expectLog(t, epc.Procs[i], "The server is ready to downgrade") + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + } t.Log("Cluster is ready for downgrade") t.Log("Starting downgrade process") - stopEtcd(t, epc.Procs[0]) - startEtcd(t, epc, lastReleaseBinary) - expectLog(t, epc, "the cluster has been downgraded") + for i := 0; i < len(epc.Procs); i++ { + t.Logf("Downgrading member %d", i) + stopEtcd(t, epc.Procs[i]) + startEtcd(t, epc.Procs[i], lastReleaseBinary) + } t.Log("All members downgraded, validating downgrade") - validateVersion(t, epc, version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) + expectLog(t, leader(t, epc), "the cluster has been downgraded") + for i := 0; i < len(epc.Procs); i++ { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: lastVersionStr, Server: lastVersionStr}) + } t.Log("Downgrade complete") t.Log("Starting upgrade process") - stopEtcd(t, epc.Procs[0]) - startEtcd(t, epc, currentEtcdBinary) + for i := 0; i < len(epc.Procs); i++ { + t.Logf("Upgrading member %d", i) + stopEtcd(t, epc.Procs[i]) + startEtcd(t, epc.Procs[i], currentEtcdBinary) + } t.Log("All members upgraded, validating upgrade") - validateVersion(t, epc, version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + for i := 0; i < len(epc.Procs); i++ { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: currentVersionStr, Server: currentVersionStr}) + } t.Log("Upgrade complete") } -func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { +func newCluster(t *testing.T, execPath string, clusterSize int) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{ ExecPath: execPath, - ClusterSize: 1, + ClusterSize: clusterSize, InitialToken: "new", KeepDataDir: true, }) @@ -87,9 +108,9 @@ func newCluster(t *testing.T, execPath string) *e2e.EtcdProcessCluster { return epc } -func startEtcd(t *testing.T, epc *e2e.EtcdProcessCluster, execPath string) { - epc.Procs[0].Config().ExecPath = execPath - err := epc.Procs[0].Restart() +func startEtcd(t *testing.T, ep e2e.EtcdProcess, execPath string) { + ep.Config().ExecPath = execPath + err := ep.Restart() if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } @@ -111,20 +132,20 @@ func stopEtcd(t *testing.T, ep e2e.EtcdProcess) { } } -func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.Versions) { +func validateVersion(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, expect version.Versions) { // Two separate calls to expect as it doesn't support multiple matches on the same line var err error testutils.ExecuteWithTimeout(t, 20*time.Second, func() { for { if expect.Server != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(cfg, member, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server) if err != nil { time.Sleep(time.Second) continue } } if expect.Cluster != "" { - err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(epc.Cfg, epc.Procs[rand.Intn(epc.Cfg.ClusterSize)], "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) + err = e2e.SpawnWithExpects(e2e.CURLPrefixArgs(cfg, member, "GET", e2e.CURLReq{Endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster) if err != nil { time.Sleep(time.Second) continue @@ -138,13 +159,23 @@ func validateVersion(t *testing.T, epc *e2e.EtcdProcessCluster, expect version.V } } -func expectLog(t *testing.T, epc *e2e.EtcdProcessCluster, expectLog string) { +func expectLog(t *testing.T, ep e2e.EtcdProcess, expectLog string) { t.Helper() var err error testutils.ExecuteWithTimeout(t, 30*time.Second, func() { - _, err = epc.Procs[0].Logs().Expect(expectLog) + _, err = ep.Logs().Expect(expectLog) }) if err != nil { t.Fatal(err) } } + +func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + leader, err := epc.Leader(ctx) + cancel() + if err != nil { + t.Fatal(err) + } + return leader +} diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 4b1daf93d..fe608f744 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -15,6 +15,7 @@ package e2e import ( + "context" "fmt" "net/url" "os" @@ -23,6 +24,7 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -513,3 +515,25 @@ func (epc *EtcdProcessCluster) WithStopSignal(sig os.Signal) (ret os.Signal) { } return ret } +func (epc *EtcdProcessCluster) Leader(ctx context.Context) (EtcdProcess, error) { + for i := 0; i < len(epc.Procs); i++ { + endpoints := epc.Procs[i].EndpointsV3() + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 3 * time.Second, + }) + if err != nil { + return nil, err + } + defer cli.Close() + resp, err := cli.Status(ctx, endpoints[0]) + if err != nil { + return nil, err + } + if resp.Header.GetMemberId() == resp.Leader { + return epc.Procs[i], nil + } + } + + return nil, fmt.Errorf("Leader not found") +} From e292e5b0b4c35d6840cf1b86ecfbc48bfe737fa1 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 21 Feb 2022 15:46:36 +0100 Subject: [PATCH 8/8] tests: Apply suggestions from code-review --- tests/e2e/cluster_downgrade_test.go | 38 ++++++++++++++++++++++------- tests/framework/e2e/cluster.go | 24 ------------------ 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 9458ec78e..eecf4c862 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -15,6 +15,7 @@ package e2e import ( + "context" "fmt" "testing" "time" @@ -22,6 +23,7 @@ import ( "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -54,7 +56,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int) { } t.Logf("Cluster created") - t.Logf("etcdctl downgrade enable %s", lastVersion) + t.Logf("etcdctl downgrade enable %s", lastVersionStr) downgradeEnable(t, epc, lastVersion) t.Log("Downgrade enabled, validating if cluster is ready for downgrade") @@ -64,9 +66,9 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int) { } t.Log("Cluster is ready for downgrade") - t.Log("Starting downgrade process") + t.Logf("Starting downgrade process to %q", lastVersionStr) for i := 0; i < len(epc.Procs); i++ { - t.Logf("Downgrading member %d", i) + t.Logf("Downgrading member %d by running %s binary", i, lastReleaseBinary) stopEtcd(t, epc.Procs[i]) startEtcd(t, epc.Procs[i], lastReleaseBinary) } @@ -77,11 +79,14 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int) { } t.Log("Downgrade complete") - t.Log("Starting upgrade process") + t.Logf("Starting upgrade process to %q", currentVersionStr) for i := 0; i < len(epc.Procs); i++ { t.Logf("Upgrading member %d", i) stopEtcd(t, epc.Procs[i]) startEtcd(t, epc.Procs[i], currentEtcdBinary) + if i+1 < len(epc.Procs) { + validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{Cluster: lastVersionStr, Server: currentVersionStr}) + } } t.Log("All members upgraded, validating upgrade") for i := 0; i < len(epc.Procs); i++ { @@ -172,10 +177,25 @@ func expectLog(t *testing.T, ep e2e.EtcdProcess, expectLog string) { func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - leader, err := epc.Leader(ctx) - cancel() - if err != nil { - t.Fatal(err) + defer cancel() + for i := 0; i < len(epc.Procs); i++ { + endpoints := epc.Procs[i].EndpointsV3() + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 3 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + resp, err := cli.Status(ctx, endpoints[0]) + if err != nil { + t.Fatal(err) + } + if resp.Header.GetMemberId() == resp.Leader { + return epc.Procs[i] + } } - return leader + t.Fatal("Leader not found") + return nil } diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index fe608f744..4b1daf93d 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -15,7 +15,6 @@ package e2e import ( - "context" "fmt" "net/url" "os" @@ -24,7 +23,6 @@ import ( "testing" "time" - clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -515,25 +513,3 @@ func (epc *EtcdProcessCluster) WithStopSignal(sig os.Signal) (ret os.Signal) { } return ret } -func (epc *EtcdProcessCluster) Leader(ctx context.Context) (EtcdProcess, error) { - for i := 0; i < len(epc.Procs); i++ { - endpoints := epc.Procs[i].EndpointsV3() - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 3 * time.Second, - }) - if err != nil { - return nil, err - } - defer cli.Close() - resp, err := cli.Status(ctx, endpoints[0]) - if err != nil { - return nil, err - } - if resp.Header.GetMemberId() == resp.Leader { - return epc.Procs[i], nil - } - } - - return nil, fmt.Errorf("Leader not found") -}