Merge pull request #7877 from fanminshi/warning_on_snapshotting

backend: print snapshotting duration warning every 30s

mvcc/backend/backend.go

@@ -41,6 +41,8 @@ var (
 	initialMmapSize = uint64(10 * 1024 * 1024 * 1024)
 
 	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "mvcc/backend")
+
+	snapshotWarningTimeout = 30 * time.Second
 )
 
 type Backend interface {
@@ -163,6 +165,22 @@ func (b *backend) ForceCommit() {
 }
 
 func (b *backend) Snapshot() Snapshot {
+	stopc, donec := make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(donec)
+		start := time.Now()
+		ticker := time.NewTicker(snapshotWarningTimeout)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				plog.Warningf("snapshotting is taking more than %v seconds to finish [started at %v]", time.Since(start).Seconds(), start)
+			case <-stopc:
+				return
+			}
+		}
+	}()
+
 	b.batchTx.Commit()
 
 	b.mu.RLock()
@@ -171,7 +189,7 @@ func (b *backend) Snapshot() Snapshot {
 	if err != nil {
 		plog.Fatalf("cannot begin tx (%s)", err)
 	}
-	return &snapshot{tx}
+	return &snapshot{tx, stopc, donec}
 }
 
 type IgnoreKey struct {
@@ -403,6 +421,12 @@ func NewDefaultTmpBackend() (*backend, string) {
 
 type snapshot struct {
 	*bolt.Tx
+	stopc chan struct{}
+	donec chan struct{}
 }
 
-func (s *snapshot) Close() error { return s.Tx.Rollback() }
+func (s *snapshot) Close() error {
+	close(s.stopc)
+	<-s.donec
+	return s.Tx.Rollback()
+}