From 4481238224bfa9bf5abf78316009372e07789e69 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 2 Jul 2018 12:53:01 -0700 Subject: [PATCH] etcdserver: add "etcd_server_slow_read_indexes_total" Signed-off-by: Gyuho Lee --- etcdserver/metrics.go | 7 +++++++ etcdserver/v3_server.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/etcdserver/metrics.go b/etcdserver/metrics.go index 980e3ebaf..79b5b1148 100644 --- a/etcdserver/metrics.go +++ b/etcdserver/metrics.go @@ -71,6 +71,12 @@ var ( Name: "lease_expired_total", Help: "The total number of expired leases.", }) + slowReadIndex = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "slow_read_indexes_total", + Help: "The total number of pending read indexes not in sync with leader's or timed out read index requests.", + }) currentVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", @@ -89,6 +95,7 @@ func init() { prometheus.MustRegister(proposalsPending) prometheus.MustRegister(proposalsFailed) prometheus.MustRegister(leaseExpired) + prometheus.MustRegister(slowReadIndex) prometheus.MustRegister(currentVersion) currentVersion.With(prometheus.Labels{ diff --git a/etcdserver/v3_server.go b/etcdserver/v3_server.go index bdef77034..8d5961bb0 100644 --- a/etcdserver/v3_server.go +++ b/etcdserver/v3_server.go @@ -655,12 +655,14 @@ func (s *EtcdServer) linearizableReadLoop() { id2 = binary.BigEndian.Uint64(rs.RequestCtx) } plog.Warningf("ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader (request ID want %d, got %d)", id1, id2) + slowReadIndex.Inc() } case <-time.After(s.Cfg.ReqTimeout()): plog.Warningf("timed out waiting for read index response") nr.notify(ErrTimeout) timeout = true + slowReadIndex.Inc() case <-s.stopping: return