Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

server: Refactor health checks

Browse Source
This commit is contained in:
Marek Siarkowicz 2022-01-27 16:34:06 +01:00
parent 191aed645e
commit e9dec74ded
1 changed files with 31 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
server/etcdserver/api/etcdhttp/metrics.go
View File

@ -40,7 +40,15 @@ const (
// HandleMetricsHealth registers metrics and health handlers. // HandleMetricsHealth registers metrics and health handlers.
func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.ServerV2) { func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.ServerV2) {
mux.Handle(PathMetrics, promhttp.Handler()) mux.Handle(PathMetrics, promhttp.Handler())
mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet, serializable bool) Health { return checkV2Health(lg, srv, excludedAlarms) })) mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet, serializable bool) Health {
if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" {
return h
}
if h := checkLeader(lg, srv, serializable); h.Health != "true" {
return h
}
return checkV2API(lg, srv)
}))
} }
// HandleMetricsHealthForV3 registers metrics and health handlers. it checks health by using v3 range request // HandleMetricsHealthForV3 registers metrics and health handlers. it checks health by using v3 range request
@ -48,7 +56,13 @@ func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.Serv
func HandleMetricsHealthForV3(lg *zap.Logger, mux *http.ServeMux, srv *etcdserver.EtcdServer) { func HandleMetricsHealthForV3(lg *zap.Logger, mux *http.ServeMux, srv *etcdserver.EtcdServer) {
mux.Handle(PathMetrics, promhttp.Handler()) mux.Handle(PathMetrics, promhttp.Handler())
mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet, serializable bool) Health { mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet, serializable bool) Health {
return checkV3Health(lg, srv, excludedAlarms, serializable) if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" {
return h
}
if h := checkLeader(lg, srv, serializable); h.Health != "true" {
return h
}
return checkV3API(lg, srv, serializable)
})) }))
} }
@ -141,9 +155,8 @@ func getSerializableFlag(r *http.Request) bool {
// TODO: etcdserver.ErrNoLeader in health API // TODO: etcdserver.ErrNoLeader in health API
func checkHealth(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSet, serializable bool) Health { func checkAlarms(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSet) Health {
h := Health{} h := Health{Health: "true"}
h.Health = "true"
as := srv.Alarms() as := srv.Alarms()
if len(as) > 0 { if len(as) > 0 {
for _, v := range as { for _, v := range as {
@ -167,19 +180,21 @@ func checkHealth(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSe
} }
} }
return h
}
func checkLeader(lg *zap.Logger, srv etcdserver.ServerV2, serializable bool) Health {
h := Health{Health: "true"}
if !serializable && (uint64(srv.Leader()) == raft.None) { if !serializable && (uint64(srv.Leader()) == raft.None) {
h.Health = "false" h.Health = "false"
h.Reason = "RAFT NO LEADER" h.Reason = "RAFT NO LEADER"
lg.Warn("serving /health false; no leader") lg.Warn("serving /health false; no leader")
return h
} }
return h return h
} }
func checkV2Health(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSet) (h Health) { func checkV2API(lg *zap.Logger, srv etcdserver.ServerV2) Health {
if h = checkHealth(lg, srv, excludedAlarms, false); h.Health != "true" { h := Health{Health: "true"}
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second) ctx, cancel := context.WithTimeout(context.Background(), time.Second)
_, err := srv.Do(ctx, etcdserverpb.Request{Method: "QGET"}) _, err := srv.Do(ctx, etcdserverpb.Request{Method: "QGET"})
cancel() cancel()
@ -187,16 +202,14 @@ func checkV2Health(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms Alarm
h.Health = "false" h.Health = "false"
h.Reason = fmt.Sprintf("QGET ERROR:%s", err) h.Reason = fmt.Sprintf("QGET ERROR:%s", err)
lg.Warn("serving /health false; QGET fails", zap.Error(err)) lg.Warn("serving /health false; QGET fails", zap.Error(err))
return return h
} }
lg.Debug("serving /health true") lg.Debug("serving /health true")
return return h
} }
func checkV3Health(lg *zap.Logger, srv *etcdserver.EtcdServer, excludedAlarms AlarmSet, serializable bool) (h Health) { func checkV3API(lg *zap.Logger, srv *etcdserver.EtcdServer, serializable bool) Health {
if h = checkHealth(lg, srv, excludedAlarms, serializable); h.Health != "true" { h := Health{Health: "true"}
return
}
ctx, cancel := context.WithTimeout(context.Background(), srv.Cfg.ReqTimeout()) ctx, cancel := context.WithTimeout(context.Background(), srv.Cfg.ReqTimeout())
_, err := srv.Range(ctx, &etcdserverpb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) _, err := srv.Range(ctx, &etcdserverpb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable})
cancel() cancel()
@ -204,8 +217,8 @@ func checkV3Health(lg *zap.Logger, srv *etcdserver.EtcdServer, excludedAlarms Al
h.Health = "false" h.Health = "false"
h.Reason = fmt.Sprintf("RANGE ERROR:%s", err) h.Reason = fmt.Sprintf("RANGE ERROR:%s", err)
lg.Warn("serving /health false; Range fails", zap.Error(err)) lg.Warn("serving /health false; Range fails", zap.Error(err))
return return h
} }
lg.Debug("serving /health true") lg.Debug("serving /health true")
return return h
} }