Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

rafthttp: record the number of failed messages

Browse Source
This commit is contained in:
Xiang Li 2015-03-04 16:09:50 -08:00
parent c3f32504ec
commit ab33c068b7
3 changed files with 22 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
rafthttp/metrics.go
View File

@ -30,10 +30,18 @@ var (
}, },
[]string{"channel", "remoteID", "msgType"}, []string{"channel", "remoteID", "msgType"},
) )
msgWriteFailed = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "rafthttp_message_sent_failed_total",
Help: "The total number of failed messages sent.",
},
[]string{"channel", "remoteID", "msgType"},
)
) )
func init() { func init() {
prometheus.MustRegister(msgWriteDuration) prometheus.MustRegister(msgWriteDuration)
prometheus.MustRegister(msgWriteFailed)
} }
func reportSendingDuration(channel string, m raftpb.Message, duration time.Duration) { func reportSendingDuration(channel string, m raftpb.Message, duration time.Duration) {
@ -43,3 +51,11 @@ func reportSendingDuration(channel string, m raftpb.Message, duration time.Durat
} }
msgWriteDuration.WithLabelValues(channel, types.ID(m.To).String(), typ).Observe(float64(duration.Nanoseconds() / int64(time.Microsecond))) msgWriteDuration.WithLabelValues(channel, types.ID(m.To).String(), typ).Observe(float64(duration.Nanoseconds() / int64(time.Microsecond)))
} }
func reportMessageFailure(channel string, m raftpb.Message) {
typ := m.Type.String()
if isLinkHeartbeatMessage(m) {
typ = "MsgLinkHeartbeat"
}
msgWriteFailed.WithLabelValues(channel, types.ID(m.To).String(), typ).Inc()
}

2
rafthttp/pipeline.go
View File

@ -94,6 +94,8 @@ func (p *pipeline) handle() {
p.Lock() p.Lock()
if err != nil { if err != nil {
reportMessageFailure(pipelineMsg, m)
if p.errored == nil || p.errored.Error() != err.Error() { if p.errored == nil || p.errored.Error() != err.Error() {
log.Printf("pipeline: error posting to %s: %v", p.id, err) log.Printf("pipeline: error posting to %s: %v", p.id, err)
p.errored = err p.errored = err

4
rafthttp/stream.go
View File

@ -102,6 +102,8 @@ func (cw *streamWriter) run() {
case <-heartbeatc: case <-heartbeatc:
start := time.Now() start := time.Now()
if err := enc.encode(linkHeartbeatMessage); err != nil { if err := enc.encode(linkHeartbeatMessage); err != nil {
reportMessageFailure(string(t), linkHeartbeatMessage)
log.Printf("rafthttp: failed to heartbeat on stream %s due to %v. waiting for a new stream to be established.", t, err) log.Printf("rafthttp: failed to heartbeat on stream %s due to %v. waiting for a new stream to be established.", t, err)
cw.resetCloser() cw.resetCloser()
heartbeatc, msgc = nil, nil heartbeatc, msgc = nil, nil
@ -120,6 +122,8 @@ func (cw *streamWriter) run() {
} }
start := time.Now() start := time.Now()
if err := enc.encode(m); err != nil { if err := enc.encode(m); err != nil {
reportMessageFailure(string(t), m)
log.Printf("rafthttp: failed to send message on stream %s due to %v. waiting for a new stream to be established.", t, err) log.Printf("rafthttp: failed to send message on stream %s due to %v. waiting for a new stream to be established.", t, err)
cw.resetCloser() cw.resetCloser()
heartbeatc, msgc = nil, nil heartbeatc, msgc = nil, nil