Merge pull request #17079 from ZhouJianMS/member-replace

Add member replace e2e test

tests/e2e/corrupt_test.go

@@ -218,15 +218,10 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) {
 		assert.NoError(t, err, "error on put")
 	}
 
-	members, err := cc.MemberList(ctx, false)
+	memberID, found, err := getMemberIdByName(ctx, cc, epc.Procs[0].Config().Name)
 	assert.NoError(t, err, "error on member list")
-	var memberID uint64
-	for _, m := range members.Members {
-		if m.Name == epc.Procs[0].Config().Name {
-			memberID = m.ID
-		}
-	}
-	assert.NotZero(t, memberID, "member not found")
+	assert.Equal(t, found, true, "member not found")
+
 	epc.Procs[0].Stop()
 	err = testutil.CorruptBBolt(datadir.ToBackendFileName(epc.Procs[0].Config().DataDirPath))
 	assert.NoError(t, err)
@@ -263,14 +258,9 @@ func TestCompactHashCheckDetectCorruption(t *testing.T) {
 		err = cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i), config.PutOptions{})
 		assert.NoError(t, err, "error on put")
 	}
-	members, err := cc.MemberList(ctx, false)
+	memberID, found, err := getMemberIdByName(ctx, cc, epc.Procs[0].Config().Name)
 	assert.NoError(t, err, "error on member list")
-	var memberID uint64
-	for _, m := range members.Members {
-		if m.Name == epc.Procs[0].Config().Name {
-			memberID = m.ID
-		}
-	}
+	assert.Equal(t, found, true, "member not found")
 
 	epc.Procs[0].Stop()
 	err = testutil.CorruptBBolt(datadir.ToBackendFileName(epc.Procs[0].Config().DataDirPath))

tests/e2e/ctl_v3_member_no_proxy_test.go

@@ -0,0 +1,98 @@
+// Copyright 2023 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !cluster_proxy
+
+package e2e
+
+import (
+	"context"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"go.etcd.io/etcd/server/v3/etcdserver"
+	"go.etcd.io/etcd/tests/v3/framework/e2e"
+	"go.etcd.io/etcd/tests/v3/framework/testutils"
+)
+
+func TestMemberReplace(t *testing.T) {
+	e2e.BeforeTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	epc, err := e2e.NewEtcdProcessCluster(ctx, t)
+	require.NoError(t, err)
+	defer epc.Close()
+
+	memberId := rand.Int() % len(epc.Procs)
+	member := epc.Procs[memberId]
+	memberName := member.Config().Name
+	var endpoints []string
+	for i := 1; i < len(epc.Procs); i++ {
+		endpoints = append(endpoints, epc.Procs[(memberId+i)%len(epc.Procs)].EndpointsGRPC()...)
+	}
+	cc, err := e2e.NewEtcdctl(epc.Cfg.Client, endpoints)
+	require.NoError(t, err)
+
+	c := epc.Etcdctl()
+	memberID, found, err := getMemberIdByName(ctx, c, memberName)
+	require.NoError(t, err)
+	require.Equal(t, found, true, "Member not found")
+
+	// Need to wait health interval for cluster to accept member changes
+	time.Sleep(etcdserver.HealthInterval)
+
+	t.Logf("Removing member %s", memberName)
+	_, err = c.MemberRemove(ctx, memberID)
+	require.NoError(t, err)
+	_, found, err = getMemberIdByName(ctx, c, memberName)
+	require.NoError(t, err)
+	require.Equal(t, found, false, "Expected member to be removed")
+	for member.IsRunning() {
+		err = member.Wait(ctx)
+		if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
+			t.Fatalf("member didn't exit as expected: %v", err)
+		}
+	}
+
+	t.Logf("Removing member %s data", memberName)
+	err = os.RemoveAll(member.Config().DataDirPath)
+	require.NoError(t, err)
+
+	t.Logf("Adding member %s back", memberName)
+	removedMemberPeerUrl := member.Config().PeerURL.String()
+	_, err = cc.MemberAdd(ctx, memberName, []string{removedMemberPeerUrl})
+	require.NoError(t, err)
+	err = patchArgs(member.Config().Args, "initial-cluster-state", "existing")
+	require.NoError(t, err)
+
+	t.Logf("Starting member %s", memberName)
+	err = member.Start(ctx)
+	require.NoError(t, err)
+	testutils.ExecuteUntil(ctx, t, func() {
+		for {
+			_, found, err := getMemberIdByName(ctx, c, memberName)
+			if err != nil || !found {
+				time.Sleep(10 * time.Millisecond)
+				continue
+			}
+			break
+		}
+	})
+}

tests/e2e/utils.go

@@ -122,3 +122,26 @@ func runCommandAndReadJsonOutput(args []string) (map[string]any, error) {
 
 	return resp, nil
 }
+
+func getMemberIdByName(ctx context.Context, c *e2e.EtcdctlV3, name string) (id uint64, found bool, err error) {
+	resp, err := c.MemberList(ctx, false)
+	if err != nil {
+		return 0, false, err
+	}
+	for _, member := range resp.Members {
+		if name == member.Name {
+			return member.ID, true, nil
+		}
+	}
+	return 0, false, nil
+}
+
+func patchArgs(args []string, flag, newValue string) error {
+	for i, arg := range args {
+		if strings.Contains(arg, flag) {
+			args[i] = fmt.Sprintf("--%s=%s", flag, newValue)
+			return nil
+		}
+	}
+	return fmt.Errorf("--%s flag not found", flag)
+}