diff --git a/etcdserver/metrics.go b/etcdserver/metrics.go index 4e1d189eb..6bdc0a7a6 100644 --- a/etcdserver/metrics.go +++ b/etcdserver/metrics.go @@ -48,6 +48,12 @@ var ( Name: "heartbeat_failures_total", Help: "The total number of heartbeat send failures (likely overloaded from slow disk).", }) + slowApplies = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "slow_apply_total", + Help: "The total number of slow apply requests (likely overloaded from slow disk).", + }) proposalsCommitted = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", @@ -92,6 +98,7 @@ func init() { prometheus.MustRegister(isLeader) prometheus.MustRegister(leaderChanges) prometheus.MustRegister(heartbeatFailures) + prometheus.MustRegister(slowApplies) prometheus.MustRegister(proposalsCommitted) prometheus.MustRegister(proposalsApplied) prometheus.MustRegister(proposalsPending) diff --git a/etcdserver/util.go b/etcdserver/util.go index 3a12a4be4..e05884f59 100644 --- a/etcdserver/util.go +++ b/etcdserver/util.go @@ -113,7 +113,7 @@ func warnOfExpensiveGenericRequest(lg *zap.Logger, now time.Time, stringer fmt.S if d > warnApplyDuration { if lg != nil { lg.Warn( - "request took too long", + "apply request took too long", zap.Duration("took", d), zap.Duration("expected-duration", warnApplyDuration), zap.String("prefix", prefix), @@ -122,5 +122,6 @@ func warnOfExpensiveGenericRequest(lg *zap.Logger, now time.Time, stringer fmt.S } else { plog.Warningf("%srequest %q took too long (%v) to execute", prefix, stringer.String(), d) } + slowApplies.Inc() } }