From 2f8d7d0381129d476fd0a95e9bf5c4e954d6c1be Mon Sep 17 00:00:00 2001 From: ZhouJianMS Date: Wed, 10 Jan 2024 10:46:23 +0800 Subject: [PATCH] member replace e2e test Signed-off-by: ZhouJianMS --- tests/e2e/ctl_v3_member_no_proxy_test.go | 106 +++++++++++++++++++++++ tests/e2e/etcdctl.go | 18 ++++ tests/e2e/utils.go | 27 ++++++ 3 files changed, 151 insertions(+) create mode 100644 tests/e2e/ctl_v3_member_no_proxy_test.go diff --git a/tests/e2e/ctl_v3_member_no_proxy_test.go b/tests/e2e/ctl_v3_member_no_proxy_test.go new file mode 100644 index 000000000..b26b8f514 --- /dev/null +++ b/tests/e2e/ctl_v3_member_no_proxy_test.go @@ -0,0 +1,106 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !cluster_proxy + +package e2e + +import ( + "context" + "math/rand" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd/etcdserver" + "go.etcd.io/etcd/pkg/testutil" +) + +func TestMemberReplace(t *testing.T) { + defer testutil.AfterTest(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + epc, err := newEtcdProcessCluster(t, &etcdProcessClusterConfig{ + clusterSize: 3, + keepDataDir: true, + corruptCheckTime: time.Second, + }) + require.NoError(t, err) + defer epc.Close() + + memberIdx := rand.Int() % len(epc.procs) + member := epc.procs[memberIdx] + memberName := member.Config().name + var endpoints []string + for i := 1; i < len(epc.procs); i++ { + endpoints = append(endpoints, epc.procs[(memberIdx+i)%len(epc.procs)].EndpointsGRPC()...) + } + cc := NewEtcdctl(endpoints, clientNonTLS, false, false) + + memberID, found, err := getMemberIdByName(ctx, cc, memberName) + require.NoError(t, err) + require.Equal(t, found, true, "Member not found") + + // Need to wait health interval for cluster to accept member changes + time.Sleep(etcdserver.HealthInterval) + + t.Logf("Removing member %s", memberName) + _, err = cc.MemberRemove(memberID) + require.NoError(t, err) + _, found, err = getMemberIdByName(ctx, cc, memberName) + require.NoError(t, err) + require.Equal(t, found, false, "Expected member to be removed") + for member.IsRunning() { + member.Close() + time.Sleep(10 * time.Millisecond) + } + + t.Logf("Removing member %s data", memberName) + err = os.RemoveAll(member.Config().dataDirPath) + require.NoError(t, err) + + t.Logf("Adding member %s back", memberName) + removedMemberPeerUrl := member.Config().purl.String() + // 3.4 has probability that the leader election is still in progress. + // So we need to retry until the member is added back. + for { + _, err = cc.MemberAdd(memberName, []string{removedMemberPeerUrl}) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + require.NoError(t, err) + member.Config().args = patchArgs(member.Config().args, "initial-cluster-state", "existing") + require.NoError(t, err) + + // Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687. + time.Sleep(100 * time.Millisecond) + t.Logf("Starting member %s", memberName) + err = member.Start() + require.NoError(t, err) + executeUntil(ctx, t, func() { + for { + _, found, err := getMemberIdByName(ctx, cc, memberName) + if err != nil || !found { + time.Sleep(10 * time.Millisecond) + continue + } + break + } + }) +} diff --git a/tests/e2e/etcdctl.go b/tests/e2e/etcdctl.go index cd1d2b8ec..1fcba4102 100644 --- a/tests/e2e/etcdctl.go +++ b/tests/e2e/etcdctl.go @@ -115,6 +115,24 @@ func (ctl *Etcdctl) MemberList() (*clientv3.MemberListResponse, error) { return &resp, err } +func (ctl *Etcdctl) MemberAdd(name string, peerURLs []string) (*clientv3.MemberAddResponse, error) { + if ctl.v2 { + panic("Unsupported method for v2") + } + var resp clientv3.MemberAddResponse + err := ctl.spawnJsonCmd(&resp, "", "member", "add", name, "--peer-urls", strings.Join(peerURLs, ",")) + return &resp, err +} + +func (ctl *Etcdctl) MemberRemove(id uint64) (*clientv3.MemberRemoveResponse, error) { + if ctl.v2 { + panic("Unsupported method for v2") + } + var resp clientv3.MemberRemoveResponse + err := ctl.spawnJsonCmd(&resp, "", "member", "remove", fmt.Sprintf("%x", id)) + return &resp, err +} + func (ctl *Etcdctl) Compact(rev int64) (*clientv3.CompactResponse, error) { if ctl.v2 { panic("Unsupported method for v2") diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go index 79baaf7a2..995d44db5 100644 --- a/tests/e2e/utils.go +++ b/tests/e2e/utils.go @@ -17,6 +17,7 @@ package e2e import ( "context" "fmt" + "strings" "testing" "time" @@ -129,3 +130,29 @@ func fillEtcdWithData(ctx context.Context, c *clientv3.Client, dbSize int) error } return g.Wait() } + +func getMemberIdByName(ctx context.Context, c *Etcdctl, name string) (id uint64, found bool, err error) { + resp, err := c.MemberList() + if err != nil { + return 0, false, err + } + for _, member := range resp.Members { + if name == member.Name { + return member.ID, true, nil + } + } + return 0, false, nil +} + +// Different implementations here since 3.4 e2e test framework does not have "initial-cluster-state" as a default argument +// Append new flag if not exist, otherwise replace the value +func patchArgs(args []string, flag, newValue string) []string { + for i, arg := range args { + if strings.Contains(arg, flag) { + args[i] = fmt.Sprintf("--%s=%s", flag, newValue) + return args + } + } + args = append(args, fmt.Sprintf("--%s=%s", flag, newValue)) + return args +}