server: Move server files to 'server' directory.

26  git mv mvcc wal auth etcdserver etcdmain proxy embed/ lease/ server
   36  git mv go.mod go.sum server

server/etcdserver/api/v3compactor/compactor.go

@@ -0,0 +1,73 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3compactor
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+
+	"github.com/jonboulle/clockwork"
+	"go.uber.org/zap"
+)
+
+const (
+	ModePeriodic = "periodic"
+	ModeRevision = "revision"
+)
+
+// Compactor purges old log from the storage periodically.
+type Compactor interface {
+	// Run starts the main loop of the compactor in background.
+	// Use Stop() to halt the loop and release the resource.
+	Run()
+	// Stop halts the main loop of the compactor.
+	Stop()
+	// Pause temporally suspend the compactor not to run compaction. Resume() to unpose.
+	Pause()
+	// Resume restarts the compactor suspended by Pause().
+	Resume()
+}
+
+type Compactable interface {
+	Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error)
+}
+
+type RevGetter interface {
+	Rev() int64
+}
+
+// New returns a new Compactor based on given "mode".
+func New(
+	lg *zap.Logger,
+	mode string,
+	retention time.Duration,
+	rg RevGetter,
+	c Compactable,
+) (Compactor, error) {
+	if lg == nil {
+		lg = zap.NewNop()
+	}
+	switch mode {
+	case ModePeriodic:
+		return newPeriodic(lg, clockwork.NewRealClock(), retention, rg, c), nil
+	case ModeRevision:
+		return newRevision(lg, clockwork.NewRealClock(), int64(retention), rg, c), nil
+	default:
+		return nil, fmt.Errorf("unsupported compaction mode %s", mode)
+	}
+}

server/etcdserver/api/v3compactor/compactor_test.go

@@ -0,0 +1,47 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3compactor
+
+import (
+	"context"
+	"sync/atomic"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/pkg/v3/testutil"
+)
+
+type fakeCompactable struct {
+	testutil.Recorder
+}
+
+func (fc *fakeCompactable) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
+	fc.Record(testutil.Action{Name: "c", Params: []interface{}{r}})
+	return &pb.CompactionResponse{}, nil
+}
+
+type fakeRevGetter struct {
+	testutil.Recorder
+	rev int64
+}
+
+func (fr *fakeRevGetter) Rev() int64 {
+	fr.Record(testutil.Action{Name: "g"})
+	rev := atomic.AddInt64(&fr.rev, 1)
+	return rev
+}
+
+func (fr *fakeRevGetter) SetRev(rev int64) {
+	atomic.StoreInt64(&fr.rev, rev)
+}

server/etcdserver/api/v3compactor/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package v3compactor implements automated policies for compacting etcd's mvcc storage.
+package v3compactor

server/etcdserver/api/v3compactor/periodic.go

@@ -0,0 +1,205 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3compactor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/v3/mvcc"
+
+	"github.com/jonboulle/clockwork"
+	"go.uber.org/zap"
+)
+
+// Periodic compacts the log by purging revisions older than
+// the configured retention time.
+type Periodic struct {
+	lg     *zap.Logger
+	clock  clockwork.Clock
+	period time.Duration
+
+	rg RevGetter
+	c  Compactable
+
+	revs   []int64
+	ctx    context.Context
+	cancel context.CancelFunc
+
+	// mu protects paused
+	mu     sync.RWMutex
+	paused bool
+}
+
+// newPeriodic creates a new instance of Periodic compactor that purges
+// the log older than h Duration.
+func newPeriodic(lg *zap.Logger, clock clockwork.Clock, h time.Duration, rg RevGetter, c Compactable) *Periodic {
+	pc := &Periodic{
+		lg:     lg,
+		clock:  clock,
+		period: h,
+		rg:     rg,
+		c:      c,
+		revs:   make([]int64, 0),
+	}
+	pc.ctx, pc.cancel = context.WithCancel(context.Background())
+	return pc
+}
+
+/*
+Compaction period 1-hour:
+  1. compute compaction period, which is 1-hour
+  2. record revisions for every 1/10 of 1-hour (6-minute)
+  3. keep recording revisions with no compaction for first 1-hour
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 1-hour (6-minute)
+
+Compaction period 24-hour:
+  1. compute compaction period, which is 1-hour
+  2. record revisions for every 1/10 of 1-hour (6-minute)
+  3. keep recording revisions with no compaction for first 24-hour
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 1-hour (6-minute)
+
+Compaction period 59-min:
+  1. compute compaction period, which is 59-min
+  2. record revisions for every 1/10 of 59-min (5.9-min)
+  3. keep recording revisions with no compaction for first 59-min
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 59-min (5.9-min)
+
+Compaction period 5-sec:
+  1. compute compaction period, which is 5-sec
+  2. record revisions for every 1/10 of 5-sec (0.5-sec)
+  3. keep recording revisions with no compaction for first 5-sec
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 5-sec (0.5-sec)
+*/
+
+// Run runs periodic compactor.
+func (pc *Periodic) Run() {
+	compactInterval := pc.getCompactInterval()
+	retryInterval := pc.getRetryInterval()
+	retentions := pc.getRetentions()
+
+	go func() {
+		lastSuccess := pc.clock.Now()
+		baseInterval := pc.period
+		for {
+			pc.revs = append(pc.revs, pc.rg.Rev())
+			if len(pc.revs) > retentions {
+				pc.revs = pc.revs[1:] // pc.revs[0] is always the rev at pc.period ago
+			}
+
+			select {
+			case <-pc.ctx.Done():
+				return
+			case <-pc.clock.After(retryInterval):
+				pc.mu.Lock()
+				p := pc.paused
+				pc.mu.Unlock()
+				if p {
+					continue
+				}
+			}
+
+			if pc.clock.Now().Sub(lastSuccess) < baseInterval {
+				continue
+			}
+
+			// wait up to initial given period
+			if baseInterval == pc.period {
+				baseInterval = compactInterval
+			}
+			rev := pc.revs[0]
+
+			pc.lg.Info(
+				"starting auto periodic compaction",
+				zap.Int64("revision", rev),
+				zap.Duration("compact-period", pc.period),
+			)
+			startTime := pc.clock.Now()
+			_, err := pc.c.Compact(pc.ctx, &pb.CompactionRequest{Revision: rev})
+			if err == nil || err == mvcc.ErrCompacted {
+				pc.lg.Info(
+					"completed auto periodic compaction",
+					zap.Int64("revision", rev),
+					zap.Duration("compact-period", pc.period),
+					zap.Duration("took", pc.clock.Now().Sub(startTime)),
+				)
+				lastSuccess = pc.clock.Now()
+			} else {
+				pc.lg.Warn(
+					"failed auto periodic compaction",
+					zap.Int64("revision", rev),
+					zap.Duration("compact-period", pc.period),
+					zap.Duration("retry-interval", retryInterval),
+					zap.Error(err),
+				)
+			}
+		}
+	}()
+}
+
+// if given compaction period x is <1-hour, compact every x duration.
+// (e.g. --auto-compaction-mode 'periodic' --auto-compaction-retention='10m', then compact every 10-minute)
+// if given compaction period x is >1-hour, compact every hour.
+// (e.g. --auto-compaction-mode 'periodic' --auto-compaction-retention='2h', then compact every 1-hour)
+func (pc *Periodic) getCompactInterval() time.Duration {
+	itv := pc.period
+	if itv > time.Hour {
+		itv = time.Hour
+	}
+	return itv
+}
+
+func (pc *Periodic) getRetentions() int {
+	return int(pc.period/pc.getRetryInterval()) + 1
+}
+
+const retryDivisor = 10
+
+func (pc *Periodic) getRetryInterval() time.Duration {
+	itv := pc.period
+	if itv > time.Hour {
+		itv = time.Hour
+	}
+	return itv / retryDivisor
+}
+
+// Stop stops periodic compactor.
+func (pc *Periodic) Stop() {
+	pc.cancel()
+}
+
+// Pause pauses periodic compactor.
+func (pc *Periodic) Pause() {
+	pc.mu.Lock()
+	pc.paused = true
+	pc.mu.Unlock()
+}
+
+// Resume resumes periodic compactor.
+func (pc *Periodic) Resume() {
+	pc.mu.Lock()
+	pc.paused = false
+	pc.mu.Unlock()
+}

server/etcdserver/api/v3compactor/periodic_test.go

@@ -0,0 +1,174 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3compactor
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/pkg/v3/testutil"
+
+	"github.com/jonboulle/clockwork"
+	"go.uber.org/zap"
+)
+
+func TestPeriodicHourly(t *testing.T) {
+	retentionHours := 2
+	retentionDuration := time.Duration(retentionHours) * time.Hour
+
+	fc := clockwork.NewFakeClock()
+	// TODO: Do not depand or real time (Recorder.Wait) in unit tests.
+	rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond), 0}
+	compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)}
+	tb := newPeriodic(zap.NewExample(), fc, retentionDuration, rg, compactable)
+
+	tb.Run()
+	defer tb.Stop()
+
+	initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10
+
+	// compaction doesn't happen til 2 hours elapse
+	for i := 0; i < initialIntervals; i++ {
+		rg.Wait(1)
+		fc.Advance(tb.getRetryInterval())
+	}
+
+	// very first compaction
+	a, err := compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedRevision := int64(1)
+	if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+	}
+
+	// simulate 3 hours
+	// now compactor kicks in, every hour
+	for i := 0; i < 3; i++ {
+		// advance one hour, one revision for each interval
+		for j := 0; j < intervalsPerPeriod; j++ {
+			rg.Wait(1)
+			fc.Advance(tb.getRetryInterval())
+		}
+
+		a, err = compactable.Wait(1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expectedRevision = int64((i + 1) * 10)
+		if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+			t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+		}
+	}
+}
+
+func TestPeriodicMinutes(t *testing.T) {
+	retentionMinutes := 5
+	retentionDuration := time.Duration(retentionMinutes) * time.Minute
+
+	fc := clockwork.NewFakeClock()
+	rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond), 0}
+	compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)}
+	tb := newPeriodic(zap.NewExample(), fc, retentionDuration, rg, compactable)
+
+	tb.Run()
+	defer tb.Stop()
+
+	initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10
+
+	// compaction doesn't happen til 5 minutes elapse
+	for i := 0; i < initialIntervals; i++ {
+		rg.Wait(1)
+		fc.Advance(tb.getRetryInterval())
+	}
+
+	// very first compaction
+	a, err := compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedRevision := int64(1)
+	if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+	}
+
+	// compaction happens at every interval
+	for i := 0; i < 5; i++ {
+		// advance 5-minute, one revision for each interval
+		for j := 0; j < intervalsPerPeriod; j++ {
+			rg.Wait(1)
+			fc.Advance(tb.getRetryInterval())
+		}
+
+		a, err := compactable.Wait(1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expectedRevision = int64((i + 1) * 10)
+		if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+			t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+		}
+	}
+}
+
+func TestPeriodicPause(t *testing.T) {
+	fc := clockwork.NewFakeClock()
+	retentionDuration := time.Hour
+	rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond), 0}
+	compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)}
+	tb := newPeriodic(zap.NewExample(), fc, retentionDuration, rg, compactable)
+
+	tb.Run()
+	tb.Pause()
+
+	n := tb.getRetentions()
+
+	// tb will collect 3 hours of revisions but not compact since paused
+	for i := 0; i < n*3; i++ {
+		rg.Wait(1)
+		fc.Advance(tb.getRetryInterval())
+	}
+	// t.revs = [21 22 23 24 25 26 27 28 29 30]
+
+	select {
+	case a := <-compactable.Chan():
+		t.Fatalf("unexpected action %v", a)
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// tb resumes to being blocked on the clock
+	tb.Resume()
+	rg.Wait(1)
+
+	// unblock clock, will kick off a compaction at T=3h6m by retry
+	fc.Advance(tb.getRetryInterval())
+
+	// T=3h6m
+	a, err := compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// compact the revision from hour 2:06
+	wreq := &pb.CompactionRequest{Revision: int64(1 + 2*n + 1)}
+	if !reflect.DeepEqual(a[0].Params[0], wreq) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], wreq.Revision)
+	}
+}

server/etcdserver/api/v3compactor/revision.go

@@ -0,0 +1,130 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3compactor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/v3/mvcc"
+
+	"github.com/jonboulle/clockwork"
+	"go.uber.org/zap"
+)
+
+// Revision compacts the log by purging revisions older than
+// the configured reivison number. Compaction happens every 5 minutes.
+type Revision struct {
+	lg *zap.Logger
+
+	clock     clockwork.Clock
+	retention int64
+
+	rg RevGetter
+	c  Compactable
+
+	ctx    context.Context
+	cancel context.CancelFunc
+
+	mu     sync.Mutex
+	paused bool
+}
+
+// newRevision creates a new instance of Revisonal compactor that purges
+// the log older than retention revisions from the current revision.
+func newRevision(lg *zap.Logger, clock clockwork.Clock, retention int64, rg RevGetter, c Compactable) *Revision {
+	rc := &Revision{
+		lg:        lg,
+		clock:     clock,
+		retention: retention,
+		rg:        rg,
+		c:         c,
+	}
+	rc.ctx, rc.cancel = context.WithCancel(context.Background())
+	return rc
+}
+
+const revInterval = 5 * time.Minute
+
+// Run runs revision-based compactor.
+func (rc *Revision) Run() {
+	prev := int64(0)
+	go func() {
+		for {
+			select {
+			case <-rc.ctx.Done():
+				return
+			case <-rc.clock.After(revInterval):
+				rc.mu.Lock()
+				p := rc.paused
+				rc.mu.Unlock()
+				if p {
+					continue
+				}
+			}
+
+			rev := rc.rg.Rev() - rc.retention
+			if rev <= 0 || rev == prev {
+				continue
+			}
+
+			now := time.Now()
+			rc.lg.Info(
+				"starting auto revision compaction",
+				zap.Int64("revision", rev),
+				zap.Int64("revision-compaction-retention", rc.retention),
+			)
+			_, err := rc.c.Compact(rc.ctx, &pb.CompactionRequest{Revision: rev})
+			if err == nil || err == mvcc.ErrCompacted {
+				prev = rev
+				rc.lg.Info(
+					"completed auto revision compaction",
+					zap.Int64("revision", rev),
+					zap.Int64("revision-compaction-retention", rc.retention),
+					zap.Duration("took", time.Since(now)),
+				)
+			} else {
+				rc.lg.Warn(
+					"failed auto revision compaction",
+					zap.Int64("revision", rev),
+					zap.Int64("revision-compaction-retention", rc.retention),
+					zap.Duration("retry-interval", revInterval),
+					zap.Error(err),
+				)
+			}
+		}
+	}()
+}
+
+// Stop stops revision-based compactor.
+func (rc *Revision) Stop() {
+	rc.cancel()
+}
+
+// Pause pauses revision-based compactor.
+func (rc *Revision) Pause() {
+	rc.mu.Lock()
+	rc.paused = true
+	rc.mu.Unlock()
+}
+
+// Resume resumes revision-based compactor.
+func (rc *Revision) Resume() {
+	rc.mu.Lock()
+	rc.paused = false
+	rc.mu.Unlock()
+}

server/etcdserver/api/v3compactor/revision_test.go

@@ -0,0 +1,108 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3compactor
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/pkg/v3/testutil"
+
+	"github.com/jonboulle/clockwork"
+	"go.uber.org/zap"
+)
+
+func TestRevision(t *testing.T) {
+	fc := clockwork.NewFakeClock()
+	rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond), 0}
+	compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)}
+	tb := newRevision(zap.NewExample(), fc, 10, rg, compactable)
+
+	tb.Run()
+	defer tb.Stop()
+
+	fc.Advance(revInterval)
+	rg.Wait(1)
+	// nothing happens
+
+	rg.SetRev(99) // will be 100
+	expectedRevision := int64(90)
+	fc.Advance(revInterval)
+	rg.Wait(1)
+	a, err := compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+	}
+
+	// skip the same revision
+	rg.SetRev(99) // will be 100
+	rg.Wait(1)
+	// nothing happens
+
+	rg.SetRev(199) // will be 200
+	expectedRevision = int64(190)
+	fc.Advance(revInterval)
+	rg.Wait(1)
+	a, err = compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+	}
+}
+
+func TestRevisionPause(t *testing.T) {
+	fc := clockwork.NewFakeClock()
+	rg := &fakeRevGetter{testutil.NewRecorderStream(), 99} // will be 100
+	compactable := &fakeCompactable{testutil.NewRecorderStream()}
+	tb := newRevision(zap.NewExample(), fc, 10, rg, compactable)
+
+	tb.Run()
+	tb.Pause()
+
+	// tb will collect 3 hours of revisions but not compact since paused
+	n := int(time.Hour / revInterval)
+	for i := 0; i < 3*n; i++ {
+		fc.Advance(revInterval)
+	}
+	// tb ends up waiting for the clock
+
+	select {
+	case a := <-compactable.Chan():
+		t.Fatalf("unexpected action %v", a)
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// tb resumes to being blocked on the clock
+	tb.Resume()
+
+	// unblock clock, will kick off a compaction at hour 3:05
+	fc.Advance(revInterval)
+	rg.Wait(1)
+	a, err := compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wreq := &pb.CompactionRequest{Revision: int64(90)}
+	if !reflect.DeepEqual(a[0].Params[0], wreq) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], wreq.Revision)
+	}
+}