diff --git a/client/v3/maintenance.go b/client/v3/maintenance.go index 59fdcfd9f..25ff135ff 100644 --- a/client/v3/maintenance.go +++ b/client/v3/maintenance.go @@ -33,6 +33,14 @@ type ( HashKVResponse pb.HashKVResponse MoveLeaderResponse pb.MoveLeaderResponse DowngradeResponse pb.DowngradeResponse + + DowngradeAction pb.DowngradeRequest_DowngradeAction +) + +const ( + DowngradeValidate = DowngradeAction(pb.DowngradeRequest_VALIDATE) + DowngradeEnable = DowngradeAction(pb.DowngradeRequest_ENABLE) + DowngradeCancel = DowngradeAction(pb.DowngradeRequest_CANCEL) ) type Maintenance interface { @@ -76,12 +84,8 @@ type Maintenance interface { // Downgrade requests downgrades, verifies feasibility or cancels downgrade // on the cluster version. - // action is one of the following: - // VALIDATE = 0; - // ENABLE = 1; - // CANCEL = 2; // Supported since etcd 3.5. - Downgrade(ctx context.Context, action int32, version string) (*DowngradeResponse, error) + Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) } // SnapshotResponse is aggregated response from the snapshot stream. @@ -337,14 +341,14 @@ func (m *maintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*Mov return (*MoveLeaderResponse)(resp), toErr(ctx, err) } -func (m *maintenance) Downgrade(ctx context.Context, action int32, version string) (*DowngradeResponse, error) { - actionType := pb.DowngradeRequest_VALIDATE +func (m *maintenance) Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) { + var actionType pb.DowngradeRequest_DowngradeAction switch action { - case 0: + case DowngradeValidate: actionType = pb.DowngradeRequest_VALIDATE - case 1: + case DowngradeEnable: actionType = pb.DowngradeRequest_ENABLE - case 2: + case DowngradeCancel: actionType = pb.DowngradeRequest_CANCEL default: return nil, errors.New("etcdclient: unknown downgrade action") diff --git a/etcdctl/README.md b/etcdctl/README.md index 8dc9d2a13..66dfc619f 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -1083,6 +1083,69 @@ echo ${transferee_id} # Leadership transferred from 45ddc0e800e20b93 to c89feb932daef420 ``` +### DOWNGRADE \ + +NOTICE: Downgrades is an experimental feature in v3.6 and is not recommended for production clusters. + +Downgrade provides commands to downgrade cluster. +Normally etcd members cannot be downgraded due to cluster version mechanism. + +After initial bootstrap, cluster members agree on the cluster version. Every 5 seconds, leader checks versions of all members and picks lowers minor version. +New members will refuse joining cluster with cluster version newer than theirs, thus preventing cluster from downgrading. +Downgrade commands allow cluster administrator to force cluster version to be lowered to previous minor version, thus allowing to downgrade the cluster. + +Downgrade should be is executed in stages: +1. Verify that cluster is ready be downgraded by running `etcdctl downgrade validate ` +2. Start the downgrade process by running `etcdctl downgrade enable ` +3. For each cluster member: + 1. Ensure that member is ready for downgrade by confirming that it wrote `The server is ready to downgrade` log. + 2. Replace member binary with one with older version. + 3. Confirm that member has correctly started and joined the cluster. +4. Ensure that downgrade process has succeeded by checking leader log for `the cluster has been downgraded` + +Downgrade can be canceled by running `etcdctl downgrade cancel` command. + +In case of downgrade being canceled, cluster version will return to its normal behavior (pick the lowest member minor version). +If no members were downgraded, cluster version will return to original value. +If at least one member was downgraded, cluster version will stay at the `` until downgraded members are upgraded back. + +### DOWNGRADE VALIDATE \ + +DOWNGRADE VALIDATE validate downgrade capability before starting downgrade. + +#### Example + +```bash +./etcdctl downgrade validate 3.5 +Downgrade validate success, cluster version 3.6 + +./etcdctl downgrade validate 3.4 +Error: etcdserver: invalid downgrade target version + +``` + +### DOWNGRADE ENABLE \ + +DOWNGRADE ENABLE starts a downgrade action to cluster. + +#### Example + +```bash +./etcdctl downgrade enable 3.5 +Downgrade enable success, cluster version 3.6 +``` + +### DOWNGRADE CANCEL \ + +DOWNGRADE CANCEL cancels the ongoing downgrade action to cluster. + +#### Example + +```bash +./etcdctl downgrade cancel +Downgrade cancel success, cluster version 3.5 +``` + ## Concurrency commands ### LOCK [options] \ [command arg1 arg2 ...] diff --git a/etcdctl/ctlv3/command/downgrade_command.go b/etcdctl/ctlv3/command/downgrade_command.go new file mode 100644 index 000000000..bccae16c3 --- /dev/null +++ b/etcdctl/ctlv3/command/downgrade_command.go @@ -0,0 +1,136 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "errors" + + "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/pkg/v3/cobrautl" +) + +// NewDowngradeCommand returns the cobra command for "downgrade". +func NewDowngradeCommand() *cobra.Command { + dc := &cobra.Command{ + Use: "downgrade ", + Short: "Downgrade related commands", + } + + dc.AddCommand(NewDowngradeValidateCommand()) + dc.AddCommand(NewDowngradeEnableCommand()) + dc.AddCommand(NewDowngradeCancelCommand()) + + return dc +} + +// NewDowngradeValidateCommand returns the cobra command for "downgrade validate". +func NewDowngradeValidateCommand() *cobra.Command { + cc := &cobra.Command{ + Use: "validate ", + Short: "Validate downgrade capability before starting downgrade", + + Run: downgradeValidateCommandFunc, + } + return cc +} + +// NewDowngradeEnableCommand returns the cobra command for "downgrade enable". +func NewDowngradeEnableCommand() *cobra.Command { + cc := &cobra.Command{ + Use: "enable ", + Short: "Start a downgrade action to cluster", + + Run: downgradeEnableCommandFunc, + } + return cc +} + +// NewDowngradeCancelCommand returns the cobra command for "downgrade cancel". +func NewDowngradeCancelCommand() *cobra.Command { + cc := &cobra.Command{ + Use: "cancel", + Short: "Cancel the ongoing downgrade action to cluster", + + Run: downgradeCancelCommandFunc, + } + return cc +} + +// downgradeValidateCommandFunc executes the "downgrade validate" command. +func downgradeValidateCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 1 { + cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided")) + } + if len(args) > 1 { + cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments")) + } + targetVersion := args[0] + + if len(targetVersion) == 0 { + cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided")) + } + + ctx, cancel := commandCtx(cmd) + cli := mustClientFromCmd(cmd) + + resp, err := cli.Downgrade(ctx, clientv3.DowngradeValidate, targetVersion) + cancel() + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + + display.DowngradeValidate(*resp) +} + +// downgradeEnableCommandFunc executes the "downgrade enable" command. +func downgradeEnableCommandFunc(cmd *cobra.Command, args []string) { + if len(args) < 1 { + cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided")) + } + if len(args) > 1 { + cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments")) + } + targetVersion := args[0] + + if len(targetVersion) == 0 { + cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided")) + } + + ctx, cancel := commandCtx(cmd) + cli := mustClientFromCmd(cmd) + + resp, err := cli.Downgrade(ctx, clientv3.DowngradeEnable, targetVersion) + cancel() + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + + display.DowngradeEnable(*resp) +} + +// downgradeCancelCommandFunc executes the "downgrade cancel" command. +func downgradeCancelCommandFunc(cmd *cobra.Command, args []string) { + ctx, cancel := commandCtx(cmd) + cli := mustClientFromCmd(cmd) + + resp, err := cli.Downgrade(ctx, clientv3.DowngradeCancel, "") + cancel() + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + + display.DowngradeCancel(*resp) +} diff --git a/etcdctl/ctlv3/command/printer.go b/etcdctl/ctlv3/command/printer.go index 7a4086acb..287f88984 100644 --- a/etcdctl/ctlv3/command/printer.go +++ b/etcdctl/ctlv3/command/printer.go @@ -50,6 +50,10 @@ type printer interface { EndpointHashKV([]epHashKV) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) + DowngradeValidate(r v3.DowngradeResponse) + DowngradeEnable(r v3.DowngradeResponse) + DowngradeCancel(r v3.DowngradeResponse) + Alarm(v3.AlarmResponse) RoleAdd(role string, r v3.AuthRoleAddResponse) @@ -118,6 +122,9 @@ func (p *printerRPC) Alarm(r v3.AlarmResponse) { p.p((*pb.AlarmRespons func (p *printerRPC) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p((*pb.MoveLeaderResponse)(&r)) } +func (p *printerRPC) DowngradeValidate(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) } +func (p *printerRPC) DowngradeEnable(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) } +func (p *printerRPC) DowngradeCancel(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) } func (p *printerRPC) RoleAdd(_ string, r v3.AuthRoleAddResponse) { p.p((*pb.AuthRoleAddResponse)(&r)) } func (p *printerRPC) RoleGet(_ string, r v3.AuthRoleGetResponse) { p.p((*pb.AuthRoleGetResponse)(&r)) } func (p *printerRPC) RoleDelete(_ string, r v3.AuthRoleDeleteResponse) { @@ -163,6 +170,9 @@ func (p *printerUnsupported) EndpointStatus([]epStatus) { p.p(nil) } func (p *printerUnsupported) EndpointHashKV([]epHashKV) { p.p(nil) } func (p *printerUnsupported) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p(nil) } +func (p *printerUnsupported) DowngradeValidate(r v3.DowngradeResponse) { p.p(nil) } +func (p *printerUnsupported) DowngradeEnable(r v3.DowngradeResponse) { p.p(nil) } +func (p *printerUnsupported) DowngradeCancel(r v3.DowngradeResponse) { p.p(nil) } func makeMemberListTable(r v3.MemberListResponse) (hdr []string, rows [][]string) { hdr = []string{"ID", "Status", "Name", "Peer Addrs", "Client Addrs", "Is Learner"} diff --git a/etcdctl/ctlv3/command/printer_simple.go b/etcdctl/ctlv3/command/printer_simple.go index 14028c614..32f8cac60 100644 --- a/etcdctl/ctlv3/command/printer_simple.go +++ b/etcdctl/ctlv3/command/printer_simple.go @@ -176,6 +176,16 @@ func (s *simplePrinter) MoveLeader(leader, target uint64, r v3.MoveLeaderRespons fmt.Printf("Leadership transferred from %s to %s\n", types.ID(leader), types.ID(target)) } +func (s *simplePrinter) DowngradeValidate(r v3.DowngradeResponse) { + fmt.Printf("Downgrade validate success, cluster version %s\n", r.Version) +} +func (s *simplePrinter) DowngradeEnable(r v3.DowngradeResponse) { + fmt.Printf("Downgrade enable success, cluster version %s\n", r.Version) +} +func (s *simplePrinter) DowngradeCancel(r v3.DowngradeResponse) { + fmt.Printf("Downgrade cancel success, cluster version %s\n", r.Version) +} + func (s *simplePrinter) RoleAdd(role string, r v3.AuthRoleAddResponse) { fmt.Printf("Role %s created\n", role) } diff --git a/etcdctl/ctlv3/ctl.go b/etcdctl/ctlv3/ctl.go index 8de7a7768..bfe8f8674 100644 --- a/etcdctl/ctlv3/ctl.go +++ b/etcdctl/ctlv3/ctl.go @@ -97,6 +97,7 @@ func init() { command.NewRoleCommand(), command.NewCheckCommand(), command.NewCompletionCommand(), + command.NewDowngradeCommand(), ) } diff --git a/server/etcdserver/v3_server.go b/server/etcdserver/v3_server.go index 9885fc01c..3e868bebd 100644 --- a/server/etcdserver/v3_server.go +++ b/server/etcdserver/v3_server.go @@ -23,6 +23,7 @@ import ( "time" pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/raft/v3" "go.etcd.io/etcd/server/v3/auth" @@ -922,7 +923,7 @@ func (s *EtcdServer) downgradeValidate(ctx context.Context, v string) (*pb.Downg if cv == nil { return nil, ErrClusterVersionUnavailable } - resp.Version = cv.String() + resp.Version = version.Cluster(cv.String()) err = s.Version().DowngradeValidate(ctx, targetVersion) if err != nil { return nil, err @@ -943,7 +944,7 @@ func (s *EtcdServer) downgradeEnable(ctx context.Context, r *pb.DowngradeRequest lg.Warn("reject downgrade request", zap.Error(err)) return nil, err } - resp := pb.DowngradeResponse{Version: s.ClusterVersion().String()} + resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())} return &resp, nil } @@ -952,6 +953,6 @@ func (s *EtcdServer) downgradeCancel(ctx context.Context) (*pb.DowngradeResponse if err != nil { s.lg.Warn("failed to cancel downgrade", zap.Error(err)) } - resp := pb.DowngradeResponse{Version: s.ClusterVersion().String()} + resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())} return &resp, nil } diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 5e2eb9ff2..93a54884b 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -15,7 +15,6 @@ package e2e import ( - "context" "fmt" "testing" "time" @@ -23,7 +22,6 @@ import ( "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" - clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) @@ -79,20 +77,13 @@ func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessClust func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) { t.Log("etcdctl downgrade...") - c, err := clientv3.New(clientv3.Config{ - Endpoints: epc.EndpointsV3(), + c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3()) + e2e.ExecuteWithTimeout(t, 20*time.Second, func() { + err := c.DowngradeEnable(ver.String()) + if err != nil { + t.Fatal(err) + } }) - if err != nil { - t.Fatal(err) - } - defer c.Close() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - _, err = c.Downgrade(ctx, 1, ver.String()) - if err != nil { - t.Fatal(err) - } - cancel() - } func stopEtcd(t *testing.T, epc *e2e.EtcdProcessCluster) { diff --git a/tests/e2e/ctl_v3_grpc_test.go b/tests/e2e/ctl_v3_grpc_test.go index 39211e7dc..d3cc101a6 100644 --- a/tests/e2e/ctl_v3_grpc_test.go +++ b/tests/e2e/ctl_v3_grpc_test.go @@ -98,7 +98,7 @@ func TestAuthority(t *testing.T) { defer epc.Close() endpoints := templateEndpoints(t, tc.clientURLPattern, epc) - client := clusterEtcdctlV3(cfg, endpoints) + client := e2e.NewEtcdctl(cfg, endpoints) err = client.Put("foo", "bar") if err != nil { t.Fatal(err) @@ -152,48 +152,3 @@ func firstMatch(t *testing.T, expectLine string, logs ...e2e.LogsExpect) string } return <-match } - -type etcdctlV3 struct { - cfg *e2e.EtcdProcessClusterConfig - endpoints []string -} - -func clusterEtcdctlV3(cfg *e2e.EtcdProcessClusterConfig, endpoints []string) *etcdctlV3 { - return &etcdctlV3{ - cfg: cfg, - endpoints: endpoints, - } -} - -func (ctl *etcdctlV3) Put(key, value string) error { - return ctl.runCmd("put", key, value) -} - -func (ctl *etcdctlV3) runCmd(args ...string) error { - cmdArgs := []string{e2e.CtlBinPath + "3"} - for k, v := range ctl.flags() { - cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v)) - } - cmdArgs = append(cmdArgs, args...) - return e2e.SpawnWithExpect(cmdArgs, "OK") -} - -func (ctl *etcdctlV3) flags() map[string]string { - fmap := make(map[string]string) - if ctl.cfg.ClientTLS == e2e.ClientTLS { - if ctl.cfg.IsClientAutoTLS { - fmap["insecure-transport"] = "false" - fmap["insecure-skip-tls-verify"] = "true" - } else if ctl.cfg.IsClientCRL { - fmap["cacert"] = e2e.CaPath - fmap["cert"] = e2e.RevokedCertPath - fmap["key"] = e2e.RevokedPrivateKeyPath - } else { - fmap["cacert"] = e2e.CaPath - fmap["cert"] = e2e.CertPath - fmap["key"] = e2e.PrivateKeyPath - } - } - fmap["endpoints"] = strings.Join(ctl.endpoints, ",") - return fmap -} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go new file mode 100644 index 000000000..86940eee7 --- /dev/null +++ b/tests/framework/e2e/etcdctl.go @@ -0,0 +1,68 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "strings" +) + +type etcdctlV3 struct { + cfg *EtcdProcessClusterConfig + endpoints []string +} + +func NewEtcdctl(cfg *EtcdProcessClusterConfig, endpoints []string) *etcdctlV3 { + return &etcdctlV3{ + cfg: cfg, + endpoints: endpoints, + } +} + +func (ctl *etcdctlV3) Put(key, value string) error { + return SpawnWithExpect(ctl.cmdArgs("put", key, value), "OK") +} + +func (ctl *etcdctlV3) DowngradeEnable(version string) error { + return SpawnWithExpect(ctl.cmdArgs("downgrade", "enable", version), "Downgrade enable success") +} + +func (ctl *etcdctlV3) cmdArgs(args ...string) []string { + cmdArgs := []string{CtlBinPath + "3"} + for k, v := range ctl.flags() { + cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v)) + } + return append(cmdArgs, args...) +} + +func (ctl *etcdctlV3) flags() map[string]string { + fmap := make(map[string]string) + if ctl.cfg.ClientTLS == ClientTLS { + if ctl.cfg.IsClientAutoTLS { + fmap["insecure-transport"] = "false" + fmap["insecure-skip-tls-verify"] = "true" + } else if ctl.cfg.IsClientCRL { + fmap["cacert"] = CaPath + fmap["cert"] = RevokedCertPath + fmap["key"] = RevokedPrivateKeyPath + } else { + fmap["cacert"] = CaPath + fmap["cert"] = CertPath + fmap["key"] = PrivateKeyPath + } + } + fmap["endpoints"] = strings.Join(ctl.endpoints, ",") + return fmap +}