From 3ffa253516b7c05aefe913a7c606bf67c3cd708c Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 2 Mar 2022 17:09:22 +0100 Subject: [PATCH] tests: Add tests for snapshot compatibility and recovery between versions --- client/pkg/fileutil/fileutil.go | 14 +++ tests/e2e/v2store_deprecation_test.go | 158 +++++++++++++++++++++++++- tests/framework/e2e/etcdctl.go | 42 +++++++ 3 files changed, 213 insertions(+), 1 deletion(-) diff --git a/client/pkg/fileutil/fileutil.go b/client/pkg/fileutil/fileutil.go index 582b56231..9719f0327 100644 --- a/client/pkg/fileutil/fileutil.go +++ b/client/pkg/fileutil/fileutil.go @@ -17,6 +17,7 @@ package fileutil import ( "fmt" "io" + "io/fs" "os" "path/filepath" @@ -165,3 +166,16 @@ func RemoveMatchFile(lg *zap.Logger, dir string, matchFunc func(fileName string) } return nil } + +// ListFiles lists files if matchFunc is true on an existing dir +// Returns error if the dir does not exist +func ListFiles(dir string, matchFunc func(fileName string) bool) ([]string, error) { + var files []string + err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { + if matchFunc(path) { + files = append(files, path) + } + return nil + }) + return files, err +} diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go index a8afde5aa..6ed4dca62 100644 --- a/tests/e2e/v2store_deprecation_test.go +++ b/tests/e2e/v2store_deprecation_test.go @@ -15,12 +15,20 @@ package e2e import ( + "bytes" "fmt" + "sort" + "strings" "testing" "github.com/stretchr/testify/assert" "go.etcd.io/etcd/client/pkg/v3/fileutil" + "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" + "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" + "go.uber.org/zap/zaptest" ) func createV2store(t testing.TB, lastReleaseBinary string, dataDirPath string) { @@ -62,7 +70,7 @@ func assertVerifyCannotStartV2deprecationNotYet(t testing.TB, dataDirPath string assert.NoError(t, err) } -func TestV2Deprecation(t *testing.T) { +func TestV2DeprecationFlags(t *testing.T) { e2e.BeforeTest(t) dataDirPath := t.TempDir() @@ -84,3 +92,151 @@ func TestV2Deprecation(t *testing.T) { }) } + +func TestV2DeprecationSnapshotMatches(t *testing.T) { + e2e.BeforeTest(t) + lastReleaseData := t.TempDir() + currentReleaseData := t.TempDir() + + lastReleaseBinary := e2e.BinDir + "/etcd-last-release" + currentReleaseBinary := e2e.BinDir + "/etcd" + + if !fileutil.Exist(lastReleaseBinary) { + t.Skipf("%q does not exist", lastReleaseBinary) + } + snapshotCount := 10 + epc := runEtcdAndCreateSnapshot(t, lastReleaseBinary, lastReleaseData, snapshotCount) + members1 := addAndRemoveKeysAndMembers(t, e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()), snapshotCount) + assert.NoError(t, epc.Close()) + epc = runEtcdAndCreateSnapshot(t, currentReleaseBinary, currentReleaseData, snapshotCount) + members2 := addAndRemoveKeysAndMembers(t, e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()), snapshotCount) + assert.NoError(t, epc.Close()) + + assertSnapshotsMatch(t, lastReleaseData, currentReleaseData, func(data []byte) []byte { + // Patch cluster version + data = bytes.Replace(data, []byte("3.5.0"), []byte("X.X.X"), -1) + data = bytes.Replace(data, []byte("3.6.0"), []byte("X.X.X"), -1) + // Patch members ids + for i, mid := range members1 { + data = bytes.Replace(data, []byte(fmt.Sprintf("%x", mid)), []byte(fmt.Sprintf("member%d", i+1)), -1) + } + for i, mid := range members2 { + data = bytes.Replace(data, []byte(fmt.Sprintf("%x", mid)), []byte(fmt.Sprintf("member%d", i+1)), -1) + } + return data + }) +} + +func TestV2DeprecationSnapshotRecover(t *testing.T) { + e2e.BeforeTest(t) + dataDir := t.TempDir() + + lastReleaseBinary := e2e.BinDir + "/etcd-last-release" + currentReleaseBinary := e2e.BinDir + "/etcd" + + if !fileutil.Exist(lastReleaseBinary) { + t.Skipf("%q does not exist", lastReleaseBinary) + } + epc := runEtcdAndCreateSnapshot(t, lastReleaseBinary, dataDir, 10) + + cc := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) + + lastReleaseGetResponse, err := cc.Get("", config.GetOptions{Prefix: true}) + assert.NoError(t, err) + + lastReleaseMemberListResponse, err := cc.MemberList() + assert.NoError(t, err) + + assert.NoError(t, epc.Close()) + cfg := e2e.ConfigStandalone(e2e.EtcdProcessClusterConfig{ExecPath: currentReleaseBinary, DataDirPath: dataDir}) + epc, err = e2e.NewEtcdProcessCluster(t, cfg) + assert.NoError(t, err) + + cc = e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) + currentReleaseGetResponse, err := cc.Get("", config.GetOptions{Prefix: true}) + assert.NoError(t, err) + + currentReleaseMemberListResponse, err := cc.MemberList() + assert.NoError(t, err) + + assert.Equal(t, lastReleaseGetResponse.Kvs, currentReleaseGetResponse.Kvs) + assert.Equal(t, lastReleaseMemberListResponse.Members, currentReleaseMemberListResponse.Members) + assert.NoError(t, epc.Close()) +} + +func runEtcdAndCreateSnapshot(t testing.TB, binary, dataDir string, snapshotCount int) *e2e.EtcdProcessCluster { + cfg := e2e.ConfigStandalone(e2e.EtcdProcessClusterConfig{ExecPath: binary, DataDirPath: dataDir, SnapshotCount: snapshotCount, KeepDataDir: true}) + epc, err := e2e.NewEtcdProcessCluster(t, cfg) + assert.NoError(t, err) + return epc +} + +func addAndRemoveKeysAndMembers(t testing.TB, cc *e2e.EtcdctlV3, snapshotCount int) (members []uint64) { + // Execute some non-trivial key&member operation + for i := 0; i < snapshotCount*3; i++ { + err := cc.Put(fmt.Sprintf("%d", i), "1", config.PutOptions{}) + assert.NoError(t, err) + } + member1, err := cc.MemberAddAsLearner("member1", []string{"http://127.0.0.1:2000"}) + assert.NoError(t, err) + members = append(members, member1.Member.ID) + + for i := 0; i < snapshotCount*2; i++ { + _, err = cc.Delete(fmt.Sprintf("%d", i), config.DeleteOptions{}) + assert.NoError(t, err) + } + _, err = cc.MemberRemove(member1.Member.ID) + assert.NoError(t, err) + + for i := 0; i < snapshotCount; i++ { + err = cc.Put(fmt.Sprintf("%d", i), "2", config.PutOptions{}) + assert.NoError(t, err) + } + member2, err := cc.MemberAddAsLearner("member2", []string{"http://127.0.0.1:2001"}) + assert.NoError(t, err) + members = append(members, member2.Member.ID) + + for i := 0; i < snapshotCount/2; i++ { + err = cc.Put(fmt.Sprintf("%d", i), "3", config.PutOptions{}) + assert.NoError(t, err) + } + return members +} + +func filterSnapshotFiles(path string) bool { + return strings.HasSuffix(path, ".snap") +} + +func assertSnapshotsMatch(t testing.TB, firstDataDir, secondDataDir string, patch func([]byte) []byte) { + lg := zaptest.NewLogger(t) + firstFiles, err := fileutil.ListFiles(firstDataDir, filterSnapshotFiles) + if err != nil { + t.Fatal(err) + } + secondFiles, err := fileutil.ListFiles(secondDataDir, filterSnapshotFiles) + if err != nil { + t.Fatal(err) + } + assert.NotEmpty(t, firstFiles) + assert.NotEmpty(t, secondFiles) + assert.Equal(t, len(firstFiles), len(secondFiles)) + sort.Strings(firstFiles) + sort.Strings(secondFiles) + for i := 0; i < len(firstFiles); i++ { + firstSnapshot, err := snap.Read(lg, firstFiles[i]) + if err != nil { + t.Fatal(err) + } + secondSnapshot, err := snap.Read(lg, secondFiles[i]) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, openSnap(patch(firstSnapshot.Data)), openSnap(patch(secondSnapshot.Data))) + } +} + +func openSnap(data []byte) v2store.Store { + st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) + st.Recovery(data) + return st +} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 6d697582f..4ef8f1fe9 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -144,6 +144,48 @@ func (ctl *EtcdctlV3) Delete(key string, o config.DeleteOptions) (*clientv3.Dele return &resp, err } +func (ctl *EtcdctlV3) MemberList() (*clientv3.MemberListResponse, error) { + cmd, err := SpawnCmd(ctl.cmdArgs("member", "list", "-w", "json"), nil) + if err != nil { + return nil, err + } + var resp clientv3.MemberListResponse + line, err := cmd.Expect("header") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} + +func (ctl *EtcdctlV3) MemberAddAsLearner(name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { + cmd, err := SpawnCmd(ctl.cmdArgs("member", "add", name, "--learner", "--peer-urls", strings.Join(peerAddrs, ","), "-w", "json"), nil) + if err != nil { + return nil, err + } + var resp clientv3.MemberAddResponse + line, err := cmd.Expect("header") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} + +func (ctl *EtcdctlV3) MemberRemove(id uint64) (*clientv3.MemberRemoveResponse, error) { + cmd, err := SpawnCmd(ctl.cmdArgs("member", "remove", fmt.Sprintf("%x", id), "-w", "json"), nil) + if err != nil { + return nil, err + } + var resp clientv3.MemberRemoveResponse + line, err := cmd.Expect("header") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} + func (ctl *EtcdctlV3) cmdArgs(args ...string) []string { cmdArgs := []string{CtlBinPath + "3"} for k, v := range ctl.flags() {