From c20cc05fc54849b193c11702afda17782a1d9f7f Mon Sep 17 00:00:00 2001 From: Ben Cox Date: Thu, 27 Aug 2020 01:27:10 +0200 Subject: [PATCH] mvcc: Export a "Last DB compaction" timestamp metric (#12176) This is to aid with debugging the effectiveness of systems that manually take care of cluster compaction, and have greater visibity into recent compactions. It can be handy to alert on the exactly how long it was since a compaction (and also to put on dashboards) had happened. --- Tested using a test cluster, the final result looks like this: ``` root@etcd-1:~# ETCDCTL_API=3 /tmp/test-etcd/etcdctl --endpoints=192.168.232.10:2379 compact 1012 compacted revision 1012 root@etcd-1:~# curl -s 192.168.232.10:2379/metrics | grep last # HELP etcd_debugging_mvcc_db_compaction_last The unix time since the last db compaction. Resets to 0 on start. # TYPE etcd_debugging_mvcc_db_compaction_last gauge etcd_debugging_mvcc_db_compaction_last 1.595873939e+09 ``` --- mvcc/kvstore_compaction.go | 1 + mvcc/metrics.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/mvcc/kvstore_compaction.go b/mvcc/kvstore_compaction.go index 0074c59e2..de806f5e8 100644 --- a/mvcc/kvstore_compaction.go +++ b/mvcc/kvstore_compaction.go @@ -26,6 +26,7 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc defer func() { dbCompactionTotalMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) }() keyCompactions := 0 defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }() + defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }() end := make([]byte, 8) binary.BigEndian.PutUint64(end, uint64(compactMainRev+1)) diff --git a/mvcc/metrics.go b/mvcc/metrics.go index 42932c40d..580e66fd8 100644 --- a/mvcc/metrics.go +++ b/mvcc/metrics.go @@ -167,6 +167,14 @@ var ( Buckets: prometheus.ExponentialBuckets(100, 2, 14), }) + dbCompactionLast = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "etcd_debugging", + Subsystem: "mvcc", + Name: "db_compaction_last", + Help: "The unix time of the last db compaction. Resets to 0 on start.", + }) + dbCompactionKeysCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", @@ -324,6 +332,7 @@ func init() { prometheus.MustRegister(indexCompactionPauseMs) prometheus.MustRegister(dbCompactionPauseMs) prometheus.MustRegister(dbCompactionTotalMs) + prometheus.MustRegister(dbCompactionLast) prometheus.MustRegister(dbCompactionKeysCounter) prometheus.MustRegister(dbTotalSize) prometheus.MustRegister(dbTotalSizeDebug)