etcdserver, lease: tie lease min ttl to election timeout

etcdserver/server.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"expvar"
 	"fmt"
+	"math"
 	"math/rand"
 	"net/http"
 	"os"
@@ -398,11 +399,9 @@ func NewServer(cfg *ServerConfig) (srv *EtcdServer, err error) {
 	srv.applyV2 = &applierV2store{store: srv.store, cluster: srv.cluster}
 
 	srv.be = be
-	srv.lessor = lease.NewLessor(srv.be)
+	minTTL := time.Duration((3*cfg.ElectionTicks)/2) * time.Duration(cfg.TickMs) * time.Millisecond
+	srv.lessor = lease.NewLessor(srv.be, int64(math.Ceil(minTTL.Seconds())))
 
-	// always recover lessor before kv. When we recover the mvcc.KV it will reattach keys to its leases.
-	// If we recover mvcc.KV first, it will attach the keys to the wrong lessor before it recovers.
-	srv.lessor = lease.NewLessor(srv.be)
 	srv.kv = mvcc.New(srv.be, srv.lessor, &srv.consistIndex)
 	if beExist {
 		kvindex := srv.kv.ConsistentIndex()

lease/lessor.go

@@ -31,8 +31,6 @@ const (
 )
 
 var (
-	minLeaseTTL = int64(5)
-
 	leaseBucketName = []byte("lease")
 	// do not use maxInt64 since it can overflow time which will add
 	// the offset of unix time (1970yr to seconds).
@@ -143,6 +141,10 @@ type lessor struct {
 	// The leased items can be recovered by iterating all the keys in kv.
 	b backend.Backend
 
+	// minLeaseTTL is the minimum lease TTL that can be granted for a lease. Any
+	// requests for shorter TTLs are extended to the minimum TTL.
+	minLeaseTTL int64
+
 	expiredC chan []*Lease
 	// stopC is a channel whose closure indicates that the lessor should be stopped.
 	stopC chan struct{}
@@ -150,14 +152,15 @@ type lessor struct {
 	doneC chan struct{}
 }
 
-func NewLessor(b backend.Backend) Lessor {
+func NewLessor(b backend.Backend, minLeaseTTL int64) Lessor {
-	return newLessor(b)
+	return newLessor(b, minLeaseTTL)
 }
 
-func newLessor(b backend.Backend) *lessor {
+func newLessor(b backend.Backend, minLeaseTTL int64) *lessor {
 	l := &lessor{
 		leaseMap:    make(map[LeaseID]*Lease),
 		b:           b,
+		minLeaseTTL: minLeaseTTL,
 		// expiredC is a small buffered chan to avoid unnecessary blocking.
 		expiredC: make(chan []*Lease, 16),
 		stopC:    make(chan struct{}),
@@ -193,6 +196,10 @@ func (le *lessor) Grant(id LeaseID, ttl int64) (*Lease, error) {
 		return nil, ErrLeaseExists
 	}
 
+	if l.TTL < le.minLeaseTTL {
+		l.TTL = le.minLeaseTTL
+	}
+
 	if le.primary {
 		l.refresh(0)
 	} else {
@@ -425,6 +432,9 @@ func (le *lessor) initAndRecover() {
 			panic("failed to unmarshal lease proto item")
 		}
 		ID := LeaseID(lpb.ID)
+		if lpb.TTL < le.minLeaseTTL {
+			lpb.TTL = le.minLeaseTTL
+		}
 		le.leaseMap[ID] = &Lease{
 			ID:  ID,
 			TTL: lpb.TTL,
@@ -464,19 +474,11 @@ func (l Lease) persistTo(b backend.Backend) {
 
 // refresh refreshes the expiry of the lease.
 func (l *Lease) refresh(extend time.Duration) {
-	if l.TTL < minLeaseTTL {
-		l.TTL = minLeaseTTL
-	}
 	l.expiry = time.Now().Add(extend + time.Second*time.Duration(l.TTL))
 }
 
 // forever sets the expiry of lease to be forever.
-func (l *Lease) forever() {
+func (l *Lease) forever() { l.expiry = forever }
-	if l.TTL < minLeaseTTL {
-		l.TTL = minLeaseTTL
-	}
-	l.expiry = forever
-}
 
 type LeaseItem struct {
 	Key string

lease/lessor_test.go

@@ -26,6 +26,8 @@ import (
 	"github.com/coreos/etcd/mvcc/backend"
 )
 
+const minLeaseTTL = int64(5)
+
 // TestLessorGrant ensures Lessor can grant wanted lease.
 // The granted lease should have a unique ID with a term
 // that is greater than minLeaseTTL.
@@ -34,7 +36,7 @@ func TestLessorGrant(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer be.Close()
 
-	le := newLessor(be)
+	le := newLessor(be, minLeaseTTL)
 	le.Promote(0)
 
 	l, err := le.Grant(1, 1)
@@ -82,7 +84,7 @@ func TestLessorRevoke(t *testing.T) {
 
 	fd := &fakeDeleter{}
 
-	le := newLessor(be)
+	le := newLessor(be, minLeaseTTL)
 	le.SetRangeDeleter(fd)
 
 	// grant a lease with long term (100 seconds) to
@@ -129,10 +131,10 @@ func TestLessorRenew(t *testing.T) {
 	defer be.Close()
 	defer os.RemoveAll(dir)
 
-	le := newLessor(be)
+	le := newLessor(be, minLeaseTTL)
 	le.Promote(0)
 
-	l, err := le.Grant(1, 5)
+	l, err := le.Grant(1, minLeaseTTL)
 	if err != nil {
 		t.Fatalf("failed to grant lease (%v)", err)
 	}
@@ -160,7 +162,7 @@ func TestLessorDetach(t *testing.T) {
 
 	fd := &fakeDeleter{}
 
-	le := newLessor(be)
+	le := newLessor(be, minLeaseTTL)
 	le.SetRangeDeleter(fd)
 
 	// grant a lease with long term (100 seconds) to
@@ -199,7 +201,7 @@ func TestLessorRecover(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer be.Close()
 
-	le := newLessor(be)
+	le := newLessor(be, minLeaseTTL)
 	l1, err1 := le.Grant(1, 10)
 	l2, err2 := le.Grant(2, 20)
 	if err1 != nil || err2 != nil {
@@ -207,7 +209,7 @@ func TestLessorRecover(t *testing.T) {
 	}
 
 	// Create a new lessor with the same backend
-	nle := newLessor(be)
+	nle := newLessor(be, minLeaseTTL)
 	nl1 := nle.get(l1.ID)
 	if nl1 == nil || nl1.TTL != l1.TTL {
 		t.Errorf("nl1 = %v, want nl1.TTL= %d", nl1.TTL, l1.TTL)