etcd/tests/e2e/runtime_reconfiguration_test.go

// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build !cluster_proxy

package e2e

import (
	"context"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"go.etcd.io/etcd/server/v3/etcdserver"
	"go.etcd.io/etcd/tests/v3/framework/e2e"
)

// TestRuntimeReconfigGrowClusterSize ensures growing cluster size with two phases
// Phase 1 - Inform cluster of new configuration
// Phase 2 - Start new member
func TestRuntimeReconfigGrowClusterSize(t *testing.T) {
	e2e.BeforeTest(t)

	tcs := []struct {
		name        string
		clusterSize int
		asLearner   bool
	}{
		{
			name:        "grow cluster size from 1 to 3",
			clusterSize: 1,
		},
		{
			name:        "grow cluster size from 3 to 5",
			clusterSize: 3,
		},
		{
			name:        "grow cluster size from 1 to 3 with learner",
			clusterSize: 1,
			asLearner:   true,
		},
		{
			name:        "grow cluster size from 3 to 5 with learner",
			clusterSize: 3,
			asLearner:   true,
		},
	}
	for _, tc := range tcs {
		t.Run(tc.name, func(t *testing.T) {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()

			epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(tc.clusterSize))
			require.NoError(t, err)
			require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx))
			defer func() {
				err := epc.Close()
				require.NoError(t, err, "failed to close etcd cluster: %v", err)
			}()

			for i := 0; i < 2; i++ {
				time.Sleep(etcdserver.HealthInterval)
				if !tc.asLearner {
					addMember(ctx, t, epc)
				} else {
					addMemberAsLearnerAndPromote(ctx, t, epc)
				}
			}
		})
	}
}

func TestRuntimeReconfigDecreaseClusterSize(t *testing.T) {
	e2e.BeforeTest(t)

	tcs := []struct {
		name        string
		clusterSize int
		asLearner   bool
	}{
		{
			name:        "decrease cluster size from 3 to 1",
			clusterSize: 3,
		},
		{
			name:        "decrease cluster size from 5 to 3",
			clusterSize: 5,
		},
	}
	for _, tc := range tcs {
		t.Run(tc.name, func(t *testing.T) {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()

			epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(tc.clusterSize))
			require.NoError(t, err)
			require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx))
			defer func() {
				err := epc.Close()
				require.NoError(t, err, "failed to close etcd cluster: %v", err)
			}()

			for i := 0; i < 2; i++ {
				time.Sleep(etcdserver.HealthInterval)
				removeFirstMember(ctx, t, epc)
			}
		})
	}
}

func TestRuntimeReconfigRollingUpgrade(t *testing.T) {
	e2e.BeforeTest(t)

	tcs := []struct {
		name        string
		withLearner bool
	}{
		{
			name:        "with learner",
			withLearner: true,
		},
		{
			name:        "without learner",
			withLearner: false,
		},
	}

	for _, tc := range tcs {
		t.Run(tc.name, func(t *testing.T) {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()

			epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(3))
			require.NoError(t, err)
			require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx))
			defer func() {
				err := epc.Close()
				require.NoError(t, err, "failed to close etcd cluster: %v", err)
			}()

			for i := 0; i < 2; i++ {
				time.Sleep(etcdserver.HealthInterval)
				removeFirstMember(ctx, t, epc)
				epc.WaitLeader(t)
				// if we do not wait for leader, without the fix of notify raft Advance,
				// have to wait 1 sec to pass the test stably.
				if tc.withLearner {
					addMemberAsLearnerAndPromote(ctx, t, epc)
				} else {
					addMember(ctx, t, epc)
				}
			}
		})
	}
}

func addMember(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) {
	_, err := epc.StartNewProc(ctx, nil, t, false /* addAsLearner */)
	require.NoError(t, err)
	require.NoError(t, epc.Procs[len(epc.Procs)-1].Etcdctl().Health(ctx))
}

func addMemberAsLearnerAndPromote(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) {
	endpoints := epc.EndpointsGRPC()

	id, err := epc.StartNewProc(ctx, nil, t, true /* addAsLearner */)
	require.NoError(t, err)
	_, err = epc.Etcdctl(e2e.WithEndpoints(endpoints)).MemberPromote(ctx, id)
	require.NoError(t, err)

	newLearnerMemberProc := epc.Procs[len(epc.Procs)-1]
	require.NoError(t, newLearnerMemberProc.Etcdctl().Health(ctx))
}

func removeFirstMember(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) {
	// avoid tearing down the last etcd process
	if len(epc.Procs) == 1 {
		return
	}

	firstProc := epc.Procs[0]
	sts, err := firstProc.Etcdctl().Status(ctx)
	require.NoError(t, err)
	memberIDToRemove := sts[0].Header.MemberId

	epc.Procs = epc.Procs[1:]
	_, err = epc.Etcdctl().MemberRemove(ctx, memberIDToRemove)
	require.NoError(t, err)
	require.NoError(t, firstProc.Stop())
	require.NoError(t, firstProc.Close())
}