mvcc: Export a "Last DB compaction" timestamp metric (#12176)

This is to aid with debugging the effectiveness of systems that
manually take care of cluster compaction, and have greater visibity
into recent compactions.

It can be handy to alert on the exactly how long it was since a
compaction (and also to put on dashboards) had happened.

---

Tested using a test cluster, the final result looks like this:

```
	root@etcd-1:~# ETCDCTL_API=3 /tmp/test-etcd/etcdctl --endpoints=192.168.232.10:2379 compact 1012
	compacted revision 1012

	root@etcd-1:~# curl -s 192.168.232.10:2379/metrics | grep last
	# HELP etcd_debugging_mvcc_db_compaction_last The unix time since the last db compaction.  Resets to 0 on start.
	# TYPE etcd_debugging_mvcc_db_compaction_last gauge
	etcd_debugging_mvcc_db_compaction_last 1.595873939e+09
```
This commit is contained in:
Ben Cox 2020-08-27 01:27:10 +02:00 committed by GitHub
parent facd0c9460
commit c20cc05fc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 0 deletions

View File

@ -26,6 +26,7 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc
defer func() { dbCompactionTotalMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) }()
keyCompactions := 0
defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }()
defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }()
end := make([]byte, 8)
binary.BigEndian.PutUint64(end, uint64(compactMainRev+1))

View File

@ -167,6 +167,14 @@ var (
Buckets: prometheus.ExponentialBuckets(100, 2, 14),
})
dbCompactionLast = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "etcd_debugging",
Subsystem: "mvcc",
Name: "db_compaction_last",
Help: "The unix time of the last db compaction. Resets to 0 on start.",
})
dbCompactionKeysCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "etcd_debugging",
@ -324,6 +332,7 @@ func init() {
prometheus.MustRegister(indexCompactionPauseMs)
prometheus.MustRegister(dbCompactionPauseMs)
prometheus.MustRegister(dbCompactionTotalMs)
prometheus.MustRegister(dbCompactionLast)
prometheus.MustRegister(dbCompactionKeysCounter)
prometheus.MustRegister(dbTotalSize)
prometheus.MustRegister(dbTotalSizeDebug)