From 37000cc4b8cd521ad37e5eb232113cf98c487fc9 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 2 Jul 2018 12:53:01 -0700 Subject: [PATCH] etcdserver: add "etcd_server_slow_read_indexes_total" Signed-off-by: Gyuho Lee --- etcdserver/metrics.go | 7 +++++++ etcdserver/v3_server.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/etcdserver/metrics.go b/etcdserver/metrics.go index 60e5395a3..733de4741 100644 --- a/etcdserver/metrics.go +++ b/etcdserver/metrics.go @@ -78,6 +78,12 @@ var ( Name: "proposals_failed_total", Help: "The total number of failed proposals seen.", }) + slowReadIndex = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "slow_read_indexes_total", + Help: "The total number of pending read indexes not in sync with leader's or timed out read index requests.", + }) leaseExpired = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "server", @@ -109,6 +115,7 @@ func init() { prometheus.MustRegister(proposalsApplied) prometheus.MustRegister(proposalsPending) prometheus.MustRegister(proposalsFailed) + prometheus.MustRegister(slowReadIndex) prometheus.MustRegister(leaseExpired) prometheus.MustRegister(quotaBackendBytes) prometheus.MustRegister(currentVersion) diff --git a/etcdserver/v3_server.go b/etcdserver/v3_server.go index 5826af2fe..48c345d72 100644 --- a/etcdserver/v3_server.go +++ b/etcdserver/v3_server.go @@ -689,6 +689,7 @@ func (s *EtcdServer) linearizableReadLoop() { } else { plog.Warningf("ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader (request ID want %d, got %d)", id1, id2) } + slowReadIndex.Inc() } case <-time.After(s.Cfg.ReqTimeout()): if lg != nil { @@ -698,6 +699,7 @@ func (s *EtcdServer) linearizableReadLoop() { } nr.notify(ErrTimeout) timeout = true + slowReadIndex.Inc() case <-s.stopping: return }