Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

etcdserver/api: add "etcd_network_snapshot_send_inflights_total", "etcd_network_snapshot_receive_inflights_total"

Browse Source 
Useful for deciding when to terminate the unhealthy follower.
If the follower is receiving a leader snapshot, operator may wait.

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
This commit is contained in:
Gyuho Lee 2019-08-08 12:39:27 -07:00
parent 43ce2eefaa
commit 46bddacacb
3 changed files with 30 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
etcdserver/api/rafthttp/http.go
View File

@ -258,6 +258,11 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return return
} }
snapshotReceiveInflights.WithLabelValues(from).Inc()
defer func() {
snapshotReceiveInflights.WithLabelValues(from).Dec()
}()
if h.lg != nil { if h.lg != nil {
h.lg.Info( h.lg.Info(
"receiving database snapshot", "receiving database snapshot",

20
etcdserver/api/rafthttp/metrics.go
View File

@ -80,6 +80,15 @@ var (
[]string{"To"}, []string{"To"},
) )
snapshotSendInflights = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_send_inflights_total",
Help: "Total number of inflight snapshot sends",
},
[]string{"To"},
)
snapshotSendFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ snapshotSendFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "etcd", Namespace: "etcd",
Subsystem: "network", Subsystem: "network",
@ -111,6 +120,15 @@ var (
[]string{"From"}, []string{"From"},
) )
snapshotReceiveInflights = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_receive_inflights_total",
Help: "Total number of inflight snapshot receives",
},
[]string{"From"},
)
snapshotReceiveFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ snapshotReceiveFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "etcd", Namespace: "etcd",
Subsystem: "network", Subsystem: "network",
@ -156,9 +174,11 @@ func init() {
prometheus.MustRegister(recvFailures) prometheus.MustRegister(recvFailures)
prometheus.MustRegister(snapshotSend) prometheus.MustRegister(snapshotSend)
prometheus.MustRegister(snapshotSendInflights)
prometheus.MustRegister(snapshotSendFailures) prometheus.MustRegister(snapshotSendFailures)
prometheus.MustRegister(snapshotSendSeconds) prometheus.MustRegister(snapshotSendSeconds)
prometheus.MustRegister(snapshotReceive) prometheus.MustRegister(snapshotReceive)
prometheus.MustRegister(snapshotReceiveInflights)
prometheus.MustRegister(snapshotReceiveFailures) prometheus.MustRegister(snapshotReceiveFailures)
prometheus.MustRegister(snapshotReceiveSeconds) prometheus.MustRegister(snapshotReceiveSeconds)

6
etcdserver/api/rafthttp/snapshot_sender.go
View File

@ -90,6 +90,11 @@ func (s *snapshotSender) send(merged snap.Message) {
plog.Infof("start to send database snapshot [index: %d, to %s]...", m.Snapshot.Metadata.Index, types.ID(m.To)) plog.Infof("start to send database snapshot [index: %d, to %s]...", m.Snapshot.Metadata.Index, types.ID(m.To))
} }
snapshotSendInflights.WithLabelValues(to).Inc()
defer func() {
snapshotSendInflights.WithLabelValues(to).Dec()
}()
err := s.post(req) err := s.post(req)
defer merged.CloseWithError(err) defer merged.CloseWithError(err)
if err != nil { if err != nil {
@ -139,7 +144,6 @@ func (s *snapshotSender) send(merged snap.Message) {
} }
sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize)) sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize))
snapshotSend.WithLabelValues(to).Inc() snapshotSend.WithLabelValues(to).Inc()
snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds()) snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds())
} }