member replace e2e test

Signed-off-by: ZhouJianMS <zhoujian@microsoft.com>

tests/e2e/corrupt_test.go

@@ -218,15 +218,10 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) {
 		assert.NoError(t, err, "error on put")
 	}
 
-	members, err := cc.MemberList()
+	_, found, err := getMemberIdByName(context.Background(), cc, epc.Procs[0].Config().Name)
 	assert.NoError(t, err, "error on member list")
-	var memberID uint64
-	for _, m := range members.Members {
-		if m.Name == epc.Procs[0].Config().Name {
-			memberID = m.ID
-		}
-	}
-	assert.NotZero(t, memberID, "member not found")
+	assert.Equal(t, found, true, "member not found")
+
 	epc.Procs[0].Stop()
 	err = testutil.CorruptBBolt(datadir.ToBackendFileName(epc.Procs[0].Config().DataDirPath))
 	assert.NoError(t, err)

tests/e2e/ctl_v3_member_no_proxy_test.go

@@ -0,0 +1,98 @@
+// Copyright 2023 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !cluster_proxy
+
+package e2e
+
+import (
+	"context"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"go.etcd.io/etcd/server/v3/etcdserver"
+	"go.etcd.io/etcd/tests/v3/framework/e2e"
+)
+
+func TestMemberReplace(t *testing.T) {
+	e2e.BeforeTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	epc, err := e2e.NewEtcdProcessCluster(t, &e2e.EtcdProcessClusterConfig{
+		ClusterSize:      3,
+		KeepDataDir:      true,
+		CorruptCheckTime: time.Second,
+	})
+	require.NoError(t, err)
+	defer epc.Close()
+
+	memberIdx := rand.Int() % len(epc.Procs)
+	member := epc.Procs[memberIdx]
+	memberName := member.Config().Name
+	var endpoints []string
+	for i := 1; i < len(epc.Procs); i++ {
+		endpoints = append(endpoints, epc.Procs[(memberIdx+i)%len(epc.Procs)].EndpointsGRPC()...)
+	}
+	cc := NewEtcdctl(endpoints, e2e.ClientNonTLS, false, false)
+
+	memberID, found, err := getMemberIdByName(ctx, cc, memberName)
+	require.NoError(t, err)
+	require.Equal(t, found, true, "Member not found")
+
+	// Need to wait health interval for cluster to accept member changes
+	time.Sleep(etcdserver.HealthInterval)
+
+	t.Logf("Removing member %s", memberName)
+	_, err = cc.MemberRemove(memberID)
+	require.NoError(t, err)
+	_, found, err = getMemberIdByName(ctx, cc, memberName)
+	require.NoError(t, err)
+	require.Equal(t, found, false, "Expected member to be removed")
+	for member.IsRunning() {
+		member.Close()
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Logf("Removing member %s data", memberName)
+	err = os.RemoveAll(member.Config().DataDirPath)
+	require.NoError(t, err)
+
+	t.Logf("Adding member %s back", memberName)
+	removedMemberPeerUrl := member.Config().Purl.String()
+	_, err = cc.MemberAdd(memberName, []string{removedMemberPeerUrl})
+	require.NoError(t, err)
+	member.Config().Args = patchArgs(member.Config().Args, "initial-cluster-state", "existing")
+	require.NoError(t, err)
+
+	// Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687.
+	time.Sleep(100 * time.Millisecond)
+	t.Logf("Starting member %s", memberName)
+	err = member.Start()
+	require.NoError(t, err)
+	e2e.ExecuteUntil(ctx, t, func() {
+		for {
+			_, found, err := getMemberIdByName(ctx, cc, memberName)
+			if err != nil || !found {
+				time.Sleep(10 * time.Millisecond)
+				continue
+			}
+			break
+		}
+	})
+}

tests/e2e/etcdctl.go

@@ -125,6 +125,24 @@ func (ctl *Etcdctl) MemberList() (*clientv3.MemberListResponse, error) {
 	return &resp, err
 }
 
+func (ctl *Etcdctl) MemberAdd(name string, peerURLs []string) (*clientv3.MemberAddResponse, error) {
+	if ctl.v2 {
+		panic("Unsupported method for v2")
+	}
+	var resp clientv3.MemberAddResponse
+	err := ctl.spawnJsonCmd(&resp, "member", "add", name, "--peer-urls", strings.Join(peerURLs, ","))
+	return &resp, err
+}
+
+func (ctl *Etcdctl) MemberRemove(id uint64) (*clientv3.MemberRemoveResponse, error) {
+	if ctl.v2 {
+		panic("Unsupported method for v2")
+	}
+	var resp clientv3.MemberRemoveResponse
+	err := ctl.spawnJsonCmd(&resp, "member", "remove", fmt.Sprintf("%x", id))
+	return &resp, err
+}
+
 func (ctl *Etcdctl) Compact(rev int64) (*clientv3.CompactResponse, error) {
 	if ctl.v2 {
 		panic("Unsupported method for v2")

tests/e2e/utils.go

@@ -17,6 +17,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -114,3 +115,29 @@ func fillEtcdWithData(ctx context.Context, c *clientv3.Client, dbSize int) error
 	}
 	return g.Wait()
 }
+
+func getMemberIdByName(ctx context.Context, c *Etcdctl, name string) (id uint64, found bool, err error) {
+	resp, err := c.MemberList()
+	if err != nil {
+		return 0, false, err
+	}
+	for _, member := range resp.Members {
+		if name == member.Name {
+			return member.ID, true, nil
+		}
+	}
+	return 0, false, nil
+}
+
+// Different implementations here since 3.5 e2e test framework does not have "initial-cluster-state" as a default argument
+// Append new flag if not exist, otherwise replace the value
+func patchArgs(args []string, flag, newValue string) []string {
+	for i, arg := range args {
+		if strings.Contains(arg, flag) {
+			args[i] = fmt.Sprintf("--%s=%s", flag, newValue)
+			return args
+		}
+	}
+	args = append(args, fmt.Sprintf("--%s=%s", flag, newValue))
+	return args
+}