From 859e336d6890562151afb340a3734acbfe5ddccd Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Mon, 27 Jun 2016 11:50:05 -0700 Subject: [PATCH 1/3] clientv3: configurable physical in compact --- clientv3/compact_op.go | 53 +++++++++++++++++++++++++++++++++++++ clientv3/compact_op_test.go | 30 +++++++++++++++++++++ clientv3/kv.go | 6 ++--- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 clientv3/compact_op.go create mode 100644 clientv3/compact_op_test.go diff --git a/clientv3/compact_op.go b/clientv3/compact_op.go new file mode 100644 index 000000000..264f4037b --- /dev/null +++ b/clientv3/compact_op.go @@ -0,0 +1,53 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientv3 + +import ( + pb "github.com/coreos/etcd/etcdserver/etcdserverpb" +) + +// CompactOp represents a compact Operation. +type CompactOp struct { + revision int64 + physical bool +} + +// CompactOption configures compact operation. +type CompactOption func(*CompactOp) + +func (op *CompactOp) applyCompactOpts(opts []CompactOption) { + for _, opt := range opts { + opt(op) + } +} + +// OpCompact wraps slice CompactOption to create a CompactOp. +func OpCompact(rev int64, opts ...CompactOption) CompactOp { + ret := CompactOp{revision: rev} + ret.applyCompactOpts(opts) + return ret +} + +func (op CompactOp) toRequest() *pb.CompactionRequest { + return &pb.CompactionRequest{Revision: op.revision, Physical: op.physical} +} + +// WithCompactPhysical makes compact RPC call wait until +// the compaction is physically applied to the local database +// such that compacted entries are totally removed from the +// backend database. +func WithCompactPhysical() CompactOption { + return func(op *CompactOp) { op.physical = true } +} diff --git a/clientv3/compact_op_test.go b/clientv3/compact_op_test.go new file mode 100644 index 000000000..6c91d3ab0 --- /dev/null +++ b/clientv3/compact_op_test.go @@ -0,0 +1,30 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientv3 + +import ( + "reflect" + "testing" + + "github.com/coreos/etcd/etcdserver/etcdserverpb" +) + +func TestCompactOp(t *testing.T) { + req1 := OpCompact(100, WithCompactPhysical()).toRequest() + req2 := &etcdserverpb.CompactionRequest{Revision: 100, Physical: true} + if !reflect.DeepEqual(req1, req2) { + t.Fatalf("expected %+v, got %+v", req2, req1) + } +} diff --git a/clientv3/kv.go b/clientv3/kv.go index 583c880cc..6ba836db8 100644 --- a/clientv3/kv.go +++ b/clientv3/kv.go @@ -47,7 +47,7 @@ type KV interface { Delete(ctx context.Context, key string, opts ...OpOption) (*DeleteResponse, error) // Compact compacts etcd KV history before the given rev. - Compact(ctx context.Context, rev int64) error + Compact(ctx context.Context, rev int64, opts ...CompactOption) error // Do applies a single Op on KV without a transaction. // Do is useful when declaring operations to be issued at a later time @@ -98,8 +98,8 @@ func (kv *kv) Delete(ctx context.Context, key string, opts ...OpOption) (*Delete return r.del, toErr(ctx, err) } -func (kv *kv) Compact(ctx context.Context, rev int64) error { - if _, err := kv.remote.Compact(ctx, &pb.CompactionRequest{Revision: rev}); err != nil { +func (kv *kv) Compact(ctx context.Context, rev int64, opts ...CompactOption) error { + if _, err := kv.remote.Compact(ctx, OpCompact(rev, opts...).toRequest()); err != nil { return toErr(ctx, err) } return nil From 76e2bf03b888a6522de88a9881b792ea98549d7f Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Mon, 27 Jun 2016 11:50:28 -0700 Subject: [PATCH 2/3] etcdctl: v3 compact with physical flag --- clientv3/compact_op.go | 2 +- etcdctl/ctlv3/command/compaction_command.go | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/clientv3/compact_op.go b/clientv3/compact_op.go index 264f4037b..32d97eb0c 100644 --- a/clientv3/compact_op.go +++ b/clientv3/compact_op.go @@ -18,7 +18,7 @@ import ( pb "github.com/coreos/etcd/etcdserver/etcdserverpb" ) -// CompactOp represents a compact Operation. +// CompactOp represents a compact operation. type CompactOp struct { revision int64 physical bool diff --git a/etcdctl/ctlv3/command/compaction_command.go b/etcdctl/ctlv3/command/compaction_command.go index a0ae92a35..d1296aaf1 100644 --- a/etcdctl/ctlv3/command/compaction_command.go +++ b/etcdctl/ctlv3/command/compaction_command.go @@ -18,16 +18,21 @@ import ( "fmt" "strconv" + "github.com/coreos/etcd/clientv3" "github.com/spf13/cobra" ) +var compactPhysical bool + // NewCompactionCommand returns the cobra command for "compaction". func NewCompactionCommand() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "compaction ", Short: "Compaction compacts the event history in etcd.", Run: compactionCommandFunc, } + cmd.Flags().BoolVar(&compactPhysical, "physical", false, "'true' to wait for compaction to physically remove all old revisions.") + return cmd } // compactionCommandFunc executes the "compaction" command. @@ -41,9 +46,14 @@ func compactionCommandFunc(cmd *cobra.Command, args []string) { ExitWithError(ExitError, err) } + var opts []clientv3.CompactOption + if compactPhysical { + opts = append(opts, clientv3.WithCompactPhysical()) + } + c := mustClientFromCmd(cmd) ctx, cancel := commandCtx(cmd) - cerr := c.Compact(ctx, rev) + cerr := c.Compact(ctx, rev, opts...) cancel() if cerr != nil { ExitWithError(ExitError, cerr) From f63e6875bdad674c363d5a615b190821f1ea967d Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Mon, 27 Jun 2016 11:50:50 -0700 Subject: [PATCH 3/3] e2e: test 'physical' flag in compact cmd --- e2e/ctl_v3_compact_test.go | 15 ++++++++++----- e2e/ctl_v3_defrag_test.go | 2 +- e2e/ctl_v3_test.go | 7 +++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/e2e/ctl_v3_compact_test.go b/e2e/ctl_v3_compact_test.go index d06e0fb07..5b0c51eb4 100644 --- a/e2e/ctl_v3_compact_test.go +++ b/e2e/ctl_v3_compact_test.go @@ -20,10 +20,12 @@ import ( "testing" ) -func TestCtlV3Compact(t *testing.T) { testCtl(t, compactTest) } +func TestCtlV3Compact(t *testing.T) { testCtl(t, compactTest) } +func TestCtlV3CompactPhysical(t *testing.T) { testCtl(t, compactTest, withCompactPhysical()) } func compactTest(cx ctlCtx) { - if err := ctlV3Compact(cx, 2); err != nil { + compactPhysical := cx.compactPhysical + if err := ctlV3Compact(cx, 2, compactPhysical); err != nil { if !strings.Contains(err.Error(), "required revision is a future revision") { cx.t.Fatal(err) } @@ -42,7 +44,7 @@ func compactTest(cx ctlCtx) { cx.t.Errorf("compactTest: ctlV3Get error (%v)", err) } - if err := ctlV3Compact(cx, 4); err != nil { + if err := ctlV3Compact(cx, 4, compactPhysical); err != nil { cx.t.Fatal(err) } @@ -54,7 +56,7 @@ func compactTest(cx ctlCtx) { cx.t.Fatalf("expected '...has been compacted' error, got ") } - if err := ctlV3Compact(cx, 2); err != nil { + if err := ctlV3Compact(cx, 2, compactPhysical); err != nil { if !strings.Contains(err.Error(), "required revision has been compacted") { cx.t.Fatal(err) } @@ -63,8 +65,11 @@ func compactTest(cx ctlCtx) { } } -func ctlV3Compact(cx ctlCtx, rev int64) error { +func ctlV3Compact(cx ctlCtx, rev int64, physical bool) error { rs := strconv.FormatInt(rev, 10) cmdArgs := append(cx.PrefixArgs(), "compact", rs) + if physical { + cmdArgs = append(cmdArgs, "--physical") + } return spawnWithExpect(cmdArgs, "compacted revision "+rs) } diff --git a/e2e/ctl_v3_defrag_test.go b/e2e/ctl_v3_defrag_test.go index 5505f71ee..ccaa845f2 100644 --- a/e2e/ctl_v3_defrag_test.go +++ b/e2e/ctl_v3_defrag_test.go @@ -26,7 +26,7 @@ func defragTest(cx ctlCtx) { } } - if err := ctlV3Compact(cx, 4); err != nil { + if err := ctlV3Compact(cx, 4, cx.compactPhysical); err != nil { cx.t.Fatal(err) } diff --git a/e2e/ctl_v3_test.go b/e2e/ctl_v3_test.go index 9ea4ea069..93d1c868b 100644 --- a/e2e/ctl_v3_test.go +++ b/e2e/ctl_v3_test.go @@ -51,6 +51,9 @@ type ctlCtx struct { user string pass string + + // for compaction + compactPhysical bool } type ctlOption func(*ctlCtx) @@ -81,6 +84,10 @@ func withQuota(b int64) ctlOption { return func(cx *ctlCtx) { cx.quotaBackendBytes = b } } +func withCompactPhysical() ctlOption { + return func(cx *ctlCtx) { cx.compactPhysical = true } +} + func testCtl(t *testing.T, testFunc func(ctlCtx), opts ...ctlOption) { defer testutil.AfterTest(t)