Merge pull request #13687 from serathius/etcdctl

Add downgrade commands

client/v3/maintenance.go

@@ -33,6 +33,14 @@ type (
 	HashKVResponse     pb.HashKVResponse
 	MoveLeaderResponse pb.MoveLeaderResponse
 	DowngradeResponse  pb.DowngradeResponse
+
+	DowngradeAction pb.DowngradeRequest_DowngradeAction
+)
+
+const (
+	DowngradeValidate = DowngradeAction(pb.DowngradeRequest_VALIDATE)
+	DowngradeEnable   = DowngradeAction(pb.DowngradeRequest_ENABLE)
+	DowngradeCancel   = DowngradeAction(pb.DowngradeRequest_CANCEL)
 )
 
 type Maintenance interface {
@@ -76,12 +84,8 @@ type Maintenance interface {
 
 	// Downgrade requests downgrades, verifies feasibility or cancels downgrade
 	// on the cluster version.
-	// action is one of the following:
-	// VALIDATE = 0;
-	// ENABLE = 1;
-	// CANCEL = 2;
 	// Supported since etcd 3.5.
-	Downgrade(ctx context.Context, action int32, version string) (*DowngradeResponse, error)
+	Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error)
 }
 
 // SnapshotResponse is aggregated response from the snapshot stream.
@@ -337,14 +341,14 @@ func (m *maintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*Mov
 	return (*MoveLeaderResponse)(resp), toErr(ctx, err)
 }
 
-func (m *maintenance) Downgrade(ctx context.Context, action int32, version string) (*DowngradeResponse, error) {
+func (m *maintenance) Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) {
-	actionType := pb.DowngradeRequest_VALIDATE
+	var actionType pb.DowngradeRequest_DowngradeAction
 	switch action {
-	case 0:
+	case DowngradeValidate:
 		actionType = pb.DowngradeRequest_VALIDATE
-	case 1:
+	case DowngradeEnable:
 		actionType = pb.DowngradeRequest_ENABLE
-	case 2:
+	case DowngradeCancel:
 		actionType = pb.DowngradeRequest_CANCEL
 	default:
 		return nil, errors.New("etcdclient: unknown downgrade action")

etcdctl/README.md

@@ -1083,6 +1083,69 @@ echo ${transferee_id}
 # Leadership transferred from 45ddc0e800e20b93 to c89feb932daef420
 ```
 
+### DOWNGRADE \<subcommand\>
+
+NOTICE: Downgrades is an experimental feature in v3.6 and is not recommended for production clusters.
+
+Downgrade provides commands to downgrade cluster.
+Normally etcd members cannot be downgraded due to cluster version mechanism.
+
+After initial bootstrap, cluster members agree on the cluster version. Every 5 seconds, leader checks versions of all members and picks lowers minor version.
+New members will refuse joining cluster with cluster version newer than theirs, thus preventing cluster from downgrading.
+Downgrade commands allow cluster administrator to force cluster version to be lowered to previous minor version, thus allowing to downgrade the cluster.
+
+Downgrade should be is executed in stages:
+1. Verify that cluster is ready be downgraded by running `etcdctl downgrade validate <TARGET_VERSION>`
+2. Start the downgrade process by running `etcdctl downgrade enable <TARGET_VERSION>`
+3. For each cluster member:
+   1. Ensure that member is ready for downgrade by confirming that it wrote `The server is ready to downgrade` log.
+   2. Replace member binary with one with older version.
+   3. Confirm that member has correctly started and joined the cluster.
+4. Ensure that downgrade process has succeeded by checking leader log for `the cluster has been downgraded`
+
+Downgrade can be canceled by running `etcdctl downgrade cancel` command.
+
+In case of downgrade being canceled, cluster version will return to its normal behavior (pick the lowest member minor version).
+If no members were downgraded, cluster version will return to original value.
+If at least one member was downgraded, cluster version will stay at the `<TARGET_VALUE>` until downgraded members are upgraded back.
+
+### DOWNGRADE VALIDATE \<TARGET_VERSION\>
+
+DOWNGRADE VALIDATE validate downgrade capability before starting downgrade.
+
+#### Example
+
+```bash
+./etcdctl downgrade validate 3.5
+Downgrade validate success, cluster version 3.6
+
+./etcdctl downgrade validate 3.4
+Error: etcdserver: invalid downgrade target version
+
+```
+
+### DOWNGRADE ENABLE \<TARGET_VERSION\>
+
+DOWNGRADE ENABLE starts a downgrade action to cluster.
+
+#### Example
+
+```bash
+./etcdctl downgrade enable 3.5
+Downgrade enable success, cluster version 3.6
+```
+
+### DOWNGRADE CANCEL \<TARGET_VERSION\>
+
+DOWNGRADE CANCEL cancels the ongoing downgrade action to cluster.
+
+#### Example
+
+```bash
+./etcdctl downgrade cancel
+Downgrade cancel success, cluster version 3.5
+```
+
 ## Concurrency commands
 
 ### LOCK [options] \<lockname\> [command arg1 arg2 ...]

etcdctl/ctlv3/command/downgrade_command.go

@@ -0,0 +1,136 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"errors"
+
+	"github.com/spf13/cobra"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/pkg/v3/cobrautl"
+)
+
+// NewDowngradeCommand returns the cobra command for "downgrade".
+func NewDowngradeCommand() *cobra.Command {
+	dc := &cobra.Command{
+		Use:   "downgrade <TARGET_VERSION>",
+		Short: "Downgrade related commands",
+	}
+
+	dc.AddCommand(NewDowngradeValidateCommand())
+	dc.AddCommand(NewDowngradeEnableCommand())
+	dc.AddCommand(NewDowngradeCancelCommand())
+
+	return dc
+}
+
+// NewDowngradeValidateCommand returns the cobra command for "downgrade validate".
+func NewDowngradeValidateCommand() *cobra.Command {
+	cc := &cobra.Command{
+		Use:   "validate <TARGET_VERSION>",
+		Short: "Validate downgrade capability before starting downgrade",
+
+		Run: downgradeValidateCommandFunc,
+	}
+	return cc
+}
+
+// NewDowngradeEnableCommand returns the cobra command for "downgrade enable".
+func NewDowngradeEnableCommand() *cobra.Command {
+	cc := &cobra.Command{
+		Use:   "enable <TARGET_VERSION>",
+		Short: "Start a downgrade action to cluster",
+
+		Run: downgradeEnableCommandFunc,
+	}
+	return cc
+}
+
+// NewDowngradeCancelCommand returns the cobra command for "downgrade cancel".
+func NewDowngradeCancelCommand() *cobra.Command {
+	cc := &cobra.Command{
+		Use:   "cancel",
+		Short: "Cancel the ongoing downgrade action to cluster",
+
+		Run: downgradeCancelCommandFunc,
+	}
+	return cc
+}
+
+// downgradeValidateCommandFunc executes the "downgrade validate" command.
+func downgradeValidateCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) < 1 {
+		cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided"))
+	}
+	if len(args) > 1 {
+		cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments"))
+	}
+	targetVersion := args[0]
+
+	if len(targetVersion) == 0 {
+		cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided"))
+	}
+
+	ctx, cancel := commandCtx(cmd)
+	cli := mustClientFromCmd(cmd)
+
+	resp, err := cli.Downgrade(ctx, clientv3.DowngradeValidate, targetVersion)
+	cancel()
+	if err != nil {
+		cobrautl.ExitWithError(cobrautl.ExitError, err)
+	}
+
+	display.DowngradeValidate(*resp)
+}
+
+// downgradeEnableCommandFunc executes the "downgrade enable" command.
+func downgradeEnableCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) < 1 {
+		cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided"))
+	}
+	if len(args) > 1 {
+		cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments"))
+	}
+	targetVersion := args[0]
+
+	if len(targetVersion) == 0 {
+		cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided"))
+	}
+
+	ctx, cancel := commandCtx(cmd)
+	cli := mustClientFromCmd(cmd)
+
+	resp, err := cli.Downgrade(ctx, clientv3.DowngradeEnable, targetVersion)
+	cancel()
+	if err != nil {
+		cobrautl.ExitWithError(cobrautl.ExitError, err)
+	}
+
+	display.DowngradeEnable(*resp)
+}
+
+// downgradeCancelCommandFunc executes the "downgrade cancel" command.
+func downgradeCancelCommandFunc(cmd *cobra.Command, args []string) {
+	ctx, cancel := commandCtx(cmd)
+	cli := mustClientFromCmd(cmd)
+
+	resp, err := cli.Downgrade(ctx, clientv3.DowngradeCancel, "")
+	cancel()
+	if err != nil {
+		cobrautl.ExitWithError(cobrautl.ExitError, err)
+	}
+
+	display.DowngradeCancel(*resp)
+}

etcdctl/ctlv3/command/printer.go

@@ -50,6 +50,10 @@ type printer interface {
 	EndpointHashKV([]epHashKV)
 	MoveLeader(leader, target uint64, r v3.MoveLeaderResponse)
 
+	DowngradeValidate(r v3.DowngradeResponse)
+	DowngradeEnable(r v3.DowngradeResponse)
+	DowngradeCancel(r v3.DowngradeResponse)
+
 	Alarm(v3.AlarmResponse)
 
 	RoleAdd(role string, r v3.AuthRoleAddResponse)
@@ -118,6 +122,9 @@ func (p *printerRPC) Alarm(r v3.AlarmResponse)           { p.p((*pb.AlarmRespons
 func (p *printerRPC) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) {
 	p.p((*pb.MoveLeaderResponse)(&r))
 }
+func (p *printerRPC) DowngradeValidate(r v3.DowngradeResponse)   { p.p((*pb.DowngradeResponse)(&r)) }
+func (p *printerRPC) DowngradeEnable(r v3.DowngradeResponse)     { p.p((*pb.DowngradeResponse)(&r)) }
+func (p *printerRPC) DowngradeCancel(r v3.DowngradeResponse)     { p.p((*pb.DowngradeResponse)(&r)) }
 func (p *printerRPC) RoleAdd(_ string, r v3.AuthRoleAddResponse) { p.p((*pb.AuthRoleAddResponse)(&r)) }
 func (p *printerRPC) RoleGet(_ string, r v3.AuthRoleGetResponse) { p.p((*pb.AuthRoleGetResponse)(&r)) }
 func (p *printerRPC) RoleDelete(_ string, r v3.AuthRoleDeleteResponse) {
@@ -163,6 +170,9 @@ func (p *printerUnsupported) EndpointStatus([]epStatus) { p.p(nil) }
 func (p *printerUnsupported) EndpointHashKV([]epHashKV) { p.p(nil) }
 
 func (p *printerUnsupported) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p(nil) }
+func (p *printerUnsupported) DowngradeValidate(r v3.DowngradeResponse)                  { p.p(nil) }
+func (p *printerUnsupported) DowngradeEnable(r v3.DowngradeResponse)                    { p.p(nil) }
+func (p *printerUnsupported) DowngradeCancel(r v3.DowngradeResponse)                    { p.p(nil) }
 
 func makeMemberListTable(r v3.MemberListResponse) (hdr []string, rows [][]string) {
 	hdr = []string{"ID", "Status", "Name", "Peer Addrs", "Client Addrs", "Is Learner"}

etcdctl/ctlv3/command/printer_simple.go

@@ -176,6 +176,16 @@ func (s *simplePrinter) MoveLeader(leader, target uint64, r v3.MoveLeaderRespons
 	fmt.Printf("Leadership transferred from %s to %s\n", types.ID(leader), types.ID(target))
 }
 
+func (s *simplePrinter) DowngradeValidate(r v3.DowngradeResponse) {
+	fmt.Printf("Downgrade validate success, cluster version %s\n", r.Version)
+}
+func (s *simplePrinter) DowngradeEnable(r v3.DowngradeResponse) {
+	fmt.Printf("Downgrade enable success, cluster version %s\n", r.Version)
+}
+func (s *simplePrinter) DowngradeCancel(r v3.DowngradeResponse) {
+	fmt.Printf("Downgrade cancel success, cluster version %s\n", r.Version)
+}
+
 func (s *simplePrinter) RoleAdd(role string, r v3.AuthRoleAddResponse) {
 	fmt.Printf("Role %s created\n", role)
 }

etcdctl/ctlv3/ctl.go

@@ -97,6 +97,7 @@ func init() {
 		command.NewRoleCommand(),
 		command.NewCheckCommand(),
 		command.NewCompletionCommand(),
+		command.NewDowngradeCommand(),
 	)
 }
 

server/etcdserver/v3_server.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/pkg/v3/traceutil"
 	"go.etcd.io/etcd/raft/v3"
 	"go.etcd.io/etcd/server/v3/auth"
@@ -922,7 +923,7 @@ func (s *EtcdServer) downgradeValidate(ctx context.Context, v string) (*pb.Downg
 	if cv == nil {
 		return nil, ErrClusterVersionUnavailable
 	}
-	resp.Version = cv.String()
+	resp.Version = version.Cluster(cv.String())
 	err = s.Version().DowngradeValidate(ctx, targetVersion)
 	if err != nil {
 		return nil, err
@@ -943,7 +944,7 @@ func (s *EtcdServer) downgradeEnable(ctx context.Context, r *pb.DowngradeRequest
 		lg.Warn("reject downgrade request", zap.Error(err))
 		return nil, err
 	}
-	resp := pb.DowngradeResponse{Version: s.ClusterVersion().String()}
+	resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())}
 	return &resp, nil
 }
 
@@ -952,6 +953,6 @@ func (s *EtcdServer) downgradeCancel(ctx context.Context) (*pb.DowngradeResponse
 	if err != nil {
 		s.lg.Warn("failed to cancel downgrade", zap.Error(err))
 	}
-	resp := pb.DowngradeResponse{Version: s.ClusterVersion().String()}
+	resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())}
 	return &resp, nil
 }

tests/e2e/cluster_downgrade_test.go

@@ -15,7 +15,6 @@
 package e2e
 
 import (
-	"context"
 	"fmt"
 	"testing"
 	"time"
@@ -23,7 +22,6 @@ import (
 	"github.com/coreos/go-semver/semver"
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/fileutil"
-	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
 )
 
@@ -79,20 +77,13 @@ func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessClust
 
 func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) {
 	t.Log("etcdctl downgrade...")
-	c, err := clientv3.New(clientv3.Config{
+	c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3())
-		Endpoints: epc.EndpointsV3(),
+	e2e.ExecuteWithTimeout(t, 20*time.Second, func() {
+		err := c.DowngradeEnable(ver.String())
+		if err != nil {
+			t.Fatal(err)
+		}
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer c.Close()
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	_, err = c.Downgrade(ctx, 1, ver.String())
-	if err != nil {
-		t.Fatal(err)
-	}
-	cancel()
-
 }
 
 func stopEtcd(t *testing.T, epc *e2e.EtcdProcessCluster) {

tests/e2e/ctl_v3_grpc_test.go

@@ -98,7 +98,7 @@ func TestAuthority(t *testing.T) {
 				defer epc.Close()
 				endpoints := templateEndpoints(t, tc.clientURLPattern, epc)
 
-				client := clusterEtcdctlV3(cfg, endpoints)
+				client := e2e.NewEtcdctl(cfg, endpoints)
 				err = client.Put("foo", "bar")
 				if err != nil {
 					t.Fatal(err)
@@ -152,48 +152,3 @@ func firstMatch(t *testing.T, expectLine string, logs ...e2e.LogsExpect) string
 	}
 	return <-match
 }
-
-type etcdctlV3 struct {
-	cfg       *e2e.EtcdProcessClusterConfig
-	endpoints []string
-}
-
-func clusterEtcdctlV3(cfg *e2e.EtcdProcessClusterConfig, endpoints []string) *etcdctlV3 {
-	return &etcdctlV3{
-		cfg:       cfg,
-		endpoints: endpoints,
-	}
-}
-
-func (ctl *etcdctlV3) Put(key, value string) error {
-	return ctl.runCmd("put", key, value)
-}
-
-func (ctl *etcdctlV3) runCmd(args ...string) error {
-	cmdArgs := []string{e2e.CtlBinPath + "3"}
-	for k, v := range ctl.flags() {
-		cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
-	}
-	cmdArgs = append(cmdArgs, args...)
-	return e2e.SpawnWithExpect(cmdArgs, "OK")
-}
-
-func (ctl *etcdctlV3) flags() map[string]string {
-	fmap := make(map[string]string)
-	if ctl.cfg.ClientTLS == e2e.ClientTLS {
-		if ctl.cfg.IsClientAutoTLS {
-			fmap["insecure-transport"] = "false"
-			fmap["insecure-skip-tls-verify"] = "true"
-		} else if ctl.cfg.IsClientCRL {
-			fmap["cacert"] = e2e.CaPath
-			fmap["cert"] = e2e.RevokedCertPath
-			fmap["key"] = e2e.RevokedPrivateKeyPath
-		} else {
-			fmap["cacert"] = e2e.CaPath
-			fmap["cert"] = e2e.CertPath
-			fmap["key"] = e2e.PrivateKeyPath
-		}
-	}
-	fmap["endpoints"] = strings.Join(ctl.endpoints, ",")
-	return fmap
-}

tests/framework/e2e/etcdctl.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+)
+
+type etcdctlV3 struct {
+	cfg       *EtcdProcessClusterConfig
+	endpoints []string
+}
+
+func NewEtcdctl(cfg *EtcdProcessClusterConfig, endpoints []string) *etcdctlV3 {
+	return &etcdctlV3{
+		cfg:       cfg,
+		endpoints: endpoints,
+	}
+}
+
+func (ctl *etcdctlV3) Put(key, value string) error {
+	return SpawnWithExpect(ctl.cmdArgs("put", key, value), "OK")
+}
+
+func (ctl *etcdctlV3) DowngradeEnable(version string) error {
+	return SpawnWithExpect(ctl.cmdArgs("downgrade", "enable", version), "Downgrade enable success")
+}
+
+func (ctl *etcdctlV3) cmdArgs(args ...string) []string {
+	cmdArgs := []string{CtlBinPath + "3"}
+	for k, v := range ctl.flags() {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
+	}
+	return append(cmdArgs, args...)
+}
+
+func (ctl *etcdctlV3) flags() map[string]string {
+	fmap := make(map[string]string)
+	if ctl.cfg.ClientTLS == ClientTLS {
+		if ctl.cfg.IsClientAutoTLS {
+			fmap["insecure-transport"] = "false"
+			fmap["insecure-skip-tls-verify"] = "true"
+		} else if ctl.cfg.IsClientCRL {
+			fmap["cacert"] = CaPath
+			fmap["cert"] = RevokedCertPath
+			fmap["key"] = RevokedPrivateKeyPath
+		} else {
+			fmap["cacert"] = CaPath
+			fmap["cert"] = CertPath
+			fmap["key"] = PrivateKeyPath
+		}
+	}
+	fmap["endpoints"] = strings.Join(ctl.endpoints, ",")
+	return fmap
+}