Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

etcdserver/api/rafthttp: display roundtripper name in warnings

Browse Source 
Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
This commit is contained in:
Gyuho Lee 2018-08-17 16:22:00 -07:00
parent 47cff4dfe5
commit 4a239070c8
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
etcdserver/api/rafthttp/probing_status.go
View File

@ -17,10 +17,16 @@ package rafthttp
import ( import (
"time" "time"
"github.com/prometheus/client_golang/prometheus"
"github.com/xiang90/probing" "github.com/xiang90/probing"
"go.uber.org/zap" "go.uber.org/zap"
) )
const (
// RoundTripperNameSnapshot is the name of round-tripper that sends merged snapshot message.
RoundTripperNameSnapshot = "ROUND_TRIPPER_SNAPSHOT"
)
var ( var (
// proberInterval must be shorter than read timeout. // proberInterval must be shorter than read timeout.
// Or the connection will time-out. // Or the connection will time-out.
@ -29,7 +35,7 @@ var (
statusErrorInterval = 5 * time.Second statusErrorInterval = 5 * time.Second
) )
func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string) { func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string, roundTripperName string, rttSecProm *prometheus.HistogramVec) {
hus := make([]string, len(us)) hus := make([]string, len(us))
for i := range us { for i := range us {
hus[i] = us[i] + ProbingPrefix hus[i] = us[i] + ProbingPrefix
@ -47,10 +53,10 @@ func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string) {
return return
} }
go monitorProbingStatus(lg, s, id) go monitorProbingStatus(lg, s, id, roundTripperName, rttSecProm)
} }
func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) { func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string, roundTripperName string, rttSecProm *prometheus.HistogramVec) {
// set the first interval short to log error early. // set the first interval short to log error early.
interval := statusErrorInterval interval := statusErrorInterval
for { for {
@ -60,6 +66,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
if lg != nil { if lg != nil {
lg.Warn( lg.Warn(
"prober detected unhealthy status", "prober detected unhealthy status",
zap.String("round-tripper-name", roundTripperName),
zap.String("remote-peer-id", id), zap.String("remote-peer-id", id),
zap.Duration("rtt", s.SRTT()), zap.Duration("rtt", s.SRTT()),
zap.Error(s.Err()), zap.Error(s.Err()),
@ -75,6 +82,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
if lg != nil { if lg != nil {
lg.Warn( lg.Warn(
"prober found high clock drift", "prober found high clock drift",
zap.String("round-tripper-name", roundTripperName),
zap.String("remote-peer-id", id), zap.String("remote-peer-id", id),
zap.Duration("clock-drift", s.SRTT()), zap.Duration("clock-drift", s.SRTT()),
zap.Duration("rtt", s.ClockDiff()), zap.Duration("rtt", s.ClockDiff()),
@ -84,7 +92,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
plog.Warningf("the clock difference against peer %s is too high [%v > %v]", id, s.ClockDiff(), time.Second) plog.Warningf("the clock difference against peer %s is too high [%v > %v]", id, s.ClockDiff(), time.Second)
} }
} }
rttSec.WithLabelValues(id).Observe(s.SRTT().Seconds()) rttSecProm.WithLabelValues(id).Observe(s.SRTT().Seconds())
case <-s.StopNotify(): case <-s.StopNotify():
return return

4
etcdserver/api/rafthttp/transport.go
View File

@ -317,7 +317,7 @@ func (t *Transport) AddPeer(id types.ID, us []string) {
} }
fs := t.LeaderStats.Follower(id.String()) fs := t.LeaderStats.Follower(id.String())
t.peers[id] = startPeer(t, urls, id, fs) t.peers[id] = startPeer(t, urls, id, fs)
addPeerToProber(t.Logger, t.pipelineProber, id.String(), us) addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec)
if t.Logger != nil { if t.Logger != nil {
t.Logger.Info( t.Logger.Info(
@ -389,7 +389,7 @@ func (t *Transport) UpdatePeer(id types.ID, us []string) {
t.peers[id].update(urls) t.peers[id].update(urls)
t.pipelineProber.Remove(id.String()) t.pipelineProber.Remove(id.String())
addPeerToProber(t.Logger, t.pipelineProber, id.String(), us) addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec)
if t.Logger != nil { if t.Logger != nil {
t.Logger.Info( t.Logger.Info(