From e43224c3b648806b314a10f65bf1a85752127d2d Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 23 May 2018 12:01:38 -0700 Subject: [PATCH] etcdserver: add "etcd_server_slow_apply_total" {"level":"warn","ts":1527101858.6985068,"caller":"etcdserver/util.go:115","msg":"apply request took too long","took":0.114101529,"expected-duration":0.1,"prefix":"","request":"header: put: --- etcdserver/metrics.go | 7 +++++++ etcdserver/util.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/etcdserver/metrics.go b/etcdserver/metrics.go index 6c228b721..80798bc1b 100644 --- a/etcdserver/metrics.go +++ b/etcdserver/metrics.go @@ -47,6 +47,12 @@ var ( Name: "heartbeat_failures_total", Help: "The total number of heartbeat send failures (likely overloaded from slow disk).", }) + slowApplies = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "slow_apply_total", + Help: "The total number of slow apply requests (likely overloaded from slow disk).", + }) proposalsCommitted = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", @@ -103,6 +109,7 @@ func init() { prometheus.MustRegister(isLeader) prometheus.MustRegister(leaderChanges) prometheus.MustRegister(heartbeatFailures) + prometheus.MustRegister(slowApplies) prometheus.MustRegister(proposalsCommitted) prometheus.MustRegister(proposalsApplied) prometheus.MustRegister(proposalsPending) diff --git a/etcdserver/util.go b/etcdserver/util.go index 708fee4cb..a321f4ff5 100644 --- a/etcdserver/util.go +++ b/etcdserver/util.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/rafthttp" "github.com/golang/protobuf/proto" + "github.com/gogo/protobuf/plugin/stringer" ) // isConnectedToQuorumSince checks whether the local member is connected to the @@ -146,6 +147,7 @@ func warnOfExpensiveGenericRequest(now time.Time, reqStringer fmt.Stringer, pref result = resp } plog.Warningf("%srequest %q with result %q took too long (%v) to execute", prefix, reqStringer.String(), result, d) + slowApplies.Inc() } }