From 7cc3f5f97e6dfa76d5b718efa8b57940ebc7258c Mon Sep 17 00:00:00 2001 From: xakdwch Date: Fri, 12 Aug 2022 16:35:36 +0800 Subject: [PATCH] etcdctl: add --max-txn-ops flag to make-mirror command --max-txn-ops flag allows users to define the maximum number of operations permitted in a transaction during syncing updates.if unlimited syncing may fail when number of txn ops exceeds the maximum number of server side. Signed-off-by: xakdwch --- CHANGELOG/CHANGELOG-3.6.md | 1 + etcdctl/README.md | 2 ++ etcdctl/ctlv3/command/make_mirror_command.go | 15 +++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index cdb15de51..1f785a52c 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -29,6 +29,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - When print endpoint status, [show db size in use](https://github.com/etcd-io/etcd/pull/13639) - [Always print the raft_term in decimal](https://github.com/etcd-io/etcd/pull/13711) when displaying member list in json. - [Add one more field `storageVersion`](https://github.com/etcd-io/etcd/pull/13773) into the response of command `etcdctl endpoint status`. +- Add [`--max-txn-ops`](https://github.com/etcd-io/etcd/pull/14340) flag to make-mirror command. ### etcdutl v3 diff --git a/etcdctl/README.md b/etcdctl/README.md index 013a49766..1a5a75da5 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -1466,6 +1466,8 @@ RPC: UserRevokeRole - dest-insecure-transport -- Disable transport security for client connections +- max-txn-ops -- Maximum number of operations permitted in a transaction during syncing updates + #### Output The approximate total number of keys transferred to the destination cluster, updated every 30 seconds. diff --git a/etcdctl/ctlv3/command/make_mirror_command.go b/etcdctl/ctlv3/command/make_mirror_command.go index e93e108ea..e81f9732a 100644 --- a/etcdctl/ctlv3/command/make_mirror_command.go +++ b/etcdctl/ctlv3/command/make_mirror_command.go @@ -33,6 +33,10 @@ import ( "github.com/spf13/cobra" ) +const ( + defaultMaxTxnOps = uint(128) +) + var ( mminsecureTr bool mmcert string @@ -44,6 +48,7 @@ var ( mmpassword string mmnodestprefix bool mmrev int64 + mmmaxTxnOps uint ) // NewMakeMirrorCommand returns the cobra command for "makeMirror". @@ -56,6 +61,7 @@ func NewMakeMirrorCommand() *cobra.Command { c.Flags().StringVar(&mmprefix, "prefix", "", "Key-value prefix to mirror") c.Flags().Int64Var(&mmrev, "rev", 0, "Specify the kv revision to start to mirror") + c.Flags().UintVar(&mmmaxTxnOps, "max-txn-ops", defaultMaxTxnOps, "Maximum number of operations permitted in a transaction during syncing updates.") c.Flags().StringVar(&mmdestprefix, "dest-prefix", "", "destination prefix to mirror a prefix to a different prefix in the destination cluster") c.Flags().BoolVar(&mmnodestprefix, "no-dest-prefix", false, "mirror key-values to the root of the destination cluster") c.Flags().StringVar(&mmcert, "dest-cert", "", "Identify secure client using this TLS certificate file for the destination cluster") @@ -197,6 +203,15 @@ func makeMirror(ctx context.Context, c *clientv3.Client, dc *clientv3.Client) er ops = []clientv3.Op{} } lastRev = nextRev + + if len(ops) == int(mmmaxTxnOps) { + _, err := dc.Txn(ctx).Then(ops...).Commit() + if err != nil { + return err + } + ops = []clientv3.Op{} + } + switch ev.Type { case mvccpb.PUT: ops = append(ops, clientv3.OpPut(modifyPrefix(string(ev.Kv.Key)), string(ev.Kv.Value)))