mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
Merge pull request #17120 from siyuanfoundation/livez-bp-3.4
[3.4] Backport healthcheck code cleanup
This commit is contained in:
commit
963af731bd
@ -773,6 +773,8 @@ func (e *Etcd) serveClients() (err error) {
|
||||
} else {
|
||||
mux := http.NewServeMux()
|
||||
etcdhttp.HandleBasic(mux, e.Server)
|
||||
etcdhttp.HandleMetrics(mux)
|
||||
etcdhttp.HandleHealth(mux, e.Server)
|
||||
h = mux
|
||||
}
|
||||
|
||||
@ -861,7 +863,8 @@ func (e *Etcd) serveMetrics() (err error) {
|
||||
|
||||
if len(e.cfg.ListenMetricsUrls) > 0 {
|
||||
metricsMux := http.NewServeMux()
|
||||
etcdhttp.HandleMetricsHealth(metricsMux, e.Server)
|
||||
etcdhttp.HandleMetrics(metricsMux)
|
||||
etcdhttp.HandleHealth(metricsMux, e.Server)
|
||||
|
||||
for _, murl := range e.cfg.ListenMetricsUrls {
|
||||
tlsInfo := &e.cfg.ClientTLSInfo
|
||||
|
@ -542,7 +542,7 @@ func startProxy(cfg *config) error {
|
||||
plog.Infof("v2 proxy started listening on client requests on %q", host)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
etcdhttp.HandlePrometheus(mux) // v2 proxy just uses the same port
|
||||
etcdhttp.HandleMetrics(mux) // v2 proxy just uses the same port
|
||||
mux.Handle("/", ph)
|
||||
plog.Fatal(http.Serve(l, mux))
|
||||
}()
|
||||
|
@ -51,7 +51,6 @@ func HandleBasic(mux *http.ServeMux, server etcdserver.ServerPeer) {
|
||||
// TODO: deprecate '/config/local/log' in v3.5
|
||||
mux.HandleFunc(configPath+"/local/log", logHandleFunc)
|
||||
|
||||
HandleMetricsHealth(mux, server)
|
||||
mux.HandleFunc(versionPath, versionHandler(server.Cluster(), serveVersion))
|
||||
}
|
||||
|
||||
|
223
etcdserver/api/etcdhttp/health.go
Normal file
223
etcdserver/api/etcdhttp/health.go
Normal file
@ -0,0 +1,223 @@
|
||||
// Copyright 2017 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package etcdhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/auth"
|
||||
"go.etcd.io/etcd/etcdserver"
|
||||
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
|
||||
"go.etcd.io/etcd/pkg/types"
|
||||
"go.etcd.io/etcd/raft"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
const (
|
||||
PathHealth = "/health"
|
||||
)
|
||||
|
||||
type ServerHealth interface {
|
||||
serverHealthV2V3
|
||||
Range(context.Context, *pb.RangeRequest) (*pb.RangeResponse, error)
|
||||
Config() etcdserver.ServerConfig
|
||||
}
|
||||
|
||||
type serverHealthV2V3 interface {
|
||||
Alarms() []*pb.AlarmMember
|
||||
Leader() types.ID
|
||||
}
|
||||
|
||||
// HandleHealthForV2 registers metrics and health handlers for v2.
|
||||
func HandleHealthForV2(mux *http.ServeMux, srv etcdserver.ServerV2) {
|
||||
mux.Handle(PathHealth, NewHealthHandler(func(excludedAlarms AlarmSet, serializable bool) Health {
|
||||
if h := checkAlarms(srv, excludedAlarms); h.Health != "true" {
|
||||
return h
|
||||
}
|
||||
if h := checkLeader(srv, serializable); h.Health != "true" {
|
||||
return h
|
||||
}
|
||||
return checkV2API(srv)
|
||||
}))
|
||||
}
|
||||
|
||||
// HandleHealth registers metrics and health handlers. it checks health by using v3 range request
|
||||
// and its corresponding timeout.
|
||||
func HandleHealth(mux *http.ServeMux, srv ServerHealth) {
|
||||
mux.Handle(PathHealth, NewHealthHandler(func(excludedAlarms AlarmSet, serializable bool) Health {
|
||||
if h := checkAlarms(srv, excludedAlarms); h.Health != "true" {
|
||||
return h
|
||||
}
|
||||
if h := checkLeader(srv, serializable); h.Health != "true" {
|
||||
return h
|
||||
}
|
||||
return checkAPI(srv, serializable)
|
||||
}))
|
||||
}
|
||||
|
||||
// NewHealthHandler handles '/health' requests.
|
||||
func NewHealthHandler(hfunc func(excludedAlarms AlarmSet, serializable bool) Health) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
w.Header().Set("Allow", http.MethodGet)
|
||||
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||
plog.Warningf("/health error (status code %d)", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
excludedAlarms := getExcludedAlarms(r)
|
||||
// Passing the query parameter "serializable=true" ensures that the
|
||||
// health of the local etcd is checked vs the health of the cluster.
|
||||
// This is useful for probes attempting to validate the liveness of
|
||||
// the etcd process vs readiness of the cluster to serve requests.
|
||||
serializableFlag := getSerializableFlag(r)
|
||||
h := hfunc(excludedAlarms, serializableFlag)
|
||||
defer func() {
|
||||
if h.Health == "true" {
|
||||
healthSuccess.Inc()
|
||||
} else {
|
||||
healthFailed.Inc()
|
||||
}
|
||||
}()
|
||||
d, _ := json.Marshal(h)
|
||||
if h.Health != "true" {
|
||||
http.Error(w, string(d), http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(d)
|
||||
plog.Debugf("/health OK (status code %d)", http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "etcd",
|
||||
Subsystem: "server",
|
||||
Name: "health_success",
|
||||
Help: "The total number of successful health checks",
|
||||
})
|
||||
healthFailed = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "etcd",
|
||||
Subsystem: "server",
|
||||
Name: "health_failures",
|
||||
Help: "The total number of failed health checks",
|
||||
})
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(healthSuccess)
|
||||
prometheus.MustRegister(healthFailed)
|
||||
}
|
||||
|
||||
// Health defines etcd server health status.
|
||||
// TODO: remove manual parsing in etcdctl cluster-health
|
||||
type Health struct {
|
||||
Health string `json:"health"`
|
||||
Reason string `json:"-"`
|
||||
}
|
||||
|
||||
type AlarmSet map[string]struct{}
|
||||
|
||||
func getExcludedAlarms(r *http.Request) (alarms AlarmSet) {
|
||||
alarms = make(map[string]struct{}, 2)
|
||||
alms, found := r.URL.Query()["exclude"]
|
||||
if found {
|
||||
for _, alm := range alms {
|
||||
if len(alms) == 0 {
|
||||
continue
|
||||
}
|
||||
alarms[alm] = struct{}{}
|
||||
}
|
||||
}
|
||||
return alarms
|
||||
}
|
||||
|
||||
func getSerializableFlag(r *http.Request) bool {
|
||||
return r.URL.Query().Get("serializable") == "true"
|
||||
}
|
||||
|
||||
// TODO: etcdserver.ErrNoLeader in health API
|
||||
|
||||
func checkAlarms(srv serverHealthV2V3, excludedAlarms AlarmSet) Health {
|
||||
h := Health{Health: "true"}
|
||||
as := srv.Alarms()
|
||||
if len(as) > 0 {
|
||||
for _, v := range as {
|
||||
alarmName := v.Alarm.String()
|
||||
if _, found := excludedAlarms[alarmName]; found {
|
||||
plog.Debugf("/health excluded alarm %s", v.String())
|
||||
continue
|
||||
}
|
||||
|
||||
h.Health = "false"
|
||||
switch v.Alarm {
|
||||
case pb.AlarmType_NOSPACE:
|
||||
h.Reason = "ALARM NOSPACE"
|
||||
case pb.AlarmType_CORRUPT:
|
||||
h.Reason = "ALARM CORRUPT"
|
||||
default:
|
||||
h.Reason = "ALARM UNKNOWN"
|
||||
}
|
||||
plog.Warningf("/health error due to %s", v.String())
|
||||
return h
|
||||
}
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func checkLeader(srv serverHealthV2V3, serializable bool) Health {
|
||||
h := Health{Health: "true"}
|
||||
if !serializable && (uint64(srv.Leader()) == raft.None) {
|
||||
h.Health = "false"
|
||||
h.Reason = "RAFT NO LEADER"
|
||||
plog.Warningf("/health error; no leader (status code %d)", http.StatusServiceUnavailable)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func checkV2API(srv etcdserver.ServerV2) Health {
|
||||
h := Health{Health: "true"}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
_, err := srv.Do(ctx, pb.Request{Method: "QGET"})
|
||||
cancel()
|
||||
if err != nil {
|
||||
h.Health = "false"
|
||||
h.Reason = fmt.Sprintf("QGET ERROR:%s", err)
|
||||
plog.Warningf("/health error; QGET failed %v (status code %d)", err, http.StatusServiceUnavailable)
|
||||
return h
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func checkAPI(srv ServerHealth, serializable bool) Health {
|
||||
h := Health{Health: "true"}
|
||||
cfg := srv.Config()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.ReqTimeout())
|
||||
_, err := srv.Range(ctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable})
|
||||
cancel()
|
||||
if err != nil && err != auth.ErrUserEmpty && err != auth.ErrPermissionDenied {
|
||||
h.Health = "false"
|
||||
h.Reason = fmt.Sprintf("RANGE ERROR:%s", err)
|
||||
plog.Warningf("serving /health false; Range failed %v (status code %d)", err, http.StatusServiceUnavailable)
|
||||
return h
|
||||
}
|
||||
return h
|
||||
}
|
189
etcdserver/api/etcdhttp/health_test.go
Normal file
189
etcdserver/api/etcdhttp/health_test.go
Normal file
@ -0,0 +1,189 @@
|
||||
// Copyright 2021 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package etcdhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go.etcd.io/etcd/auth"
|
||||
"go.etcd.io/etcd/etcdserver"
|
||||
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
|
||||
"go.etcd.io/etcd/pkg/testutil"
|
||||
"go.etcd.io/etcd/pkg/types"
|
||||
"go.etcd.io/etcd/raft"
|
||||
)
|
||||
|
||||
type fakeHealthServer struct {
|
||||
fakeServer
|
||||
health string
|
||||
apiError error
|
||||
}
|
||||
|
||||
func (s *fakeHealthServer) Range(ctx context.Context, request *pb.RangeRequest) (*pb.RangeResponse, error) {
|
||||
return nil, s.apiError
|
||||
}
|
||||
|
||||
func (s *fakeHealthServer) Config() etcdserver.ServerConfig {
|
||||
return etcdserver.ServerConfig{}
|
||||
}
|
||||
|
||||
func (s *fakeHealthServer) Leader() types.ID {
|
||||
if s.health == "true" {
|
||||
return 1
|
||||
}
|
||||
return types.ID(raft.None)
|
||||
}
|
||||
func (s *fakeHealthServer) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) {
|
||||
if s.health == "true" {
|
||||
return etcdserver.Response{}, nil
|
||||
}
|
||||
return etcdserver.Response{}, fmt.Errorf("fail health check")
|
||||
}
|
||||
func (s *fakeHealthServer) ClientCertAuthEnabled() bool { return false }
|
||||
|
||||
func TestHealthHandler(t *testing.T) {
|
||||
// define the input and expected output
|
||||
// input: alarms, and healthCheckURL
|
||||
tests := []struct {
|
||||
name string
|
||||
alarms []*pb.AlarmMember
|
||||
healthCheckURL string
|
||||
apiError error
|
||||
|
||||
expectStatusCode int
|
||||
expectHealth string
|
||||
}{
|
||||
{
|
||||
name: "Healthy if no alarm",
|
||||
alarms: []*pb.AlarmMember{},
|
||||
healthCheckURL: "/health",
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Unhealthy if NOSPACE alarm is on",
|
||||
alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}},
|
||||
healthCheckURL: "/health",
|
||||
expectStatusCode: http.StatusServiceUnavailable,
|
||||
expectHealth: "false",
|
||||
},
|
||||
{
|
||||
name: "Healthy if NOSPACE alarm is on and excluded",
|
||||
alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}},
|
||||
healthCheckURL: "/health?exclude=NOSPACE",
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Healthy if NOSPACE alarm is excluded",
|
||||
alarms: []*pb.AlarmMember{},
|
||||
healthCheckURL: "/health?exclude=NOSPACE",
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Healthy if multiple NOSPACE alarms are on and excluded",
|
||||
alarms: []*pb.AlarmMember{{MemberID: uint64(1), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(2), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(3), Alarm: pb.AlarmType_NOSPACE}},
|
||||
healthCheckURL: "/health?exclude=NOSPACE",
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Unhealthy if NOSPACE alarms is excluded and CORRUPT is on",
|
||||
alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}},
|
||||
healthCheckURL: "/health?exclude=NOSPACE",
|
||||
expectStatusCode: http.StatusServiceUnavailable,
|
||||
expectHealth: "false",
|
||||
},
|
||||
{
|
||||
name: "Unhealthy if both NOSPACE and CORRUPT are on and excluded",
|
||||
alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}},
|
||||
healthCheckURL: "/health?exclude=NOSPACE&exclude=CORRUPT",
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Healthy even if authentication failed",
|
||||
healthCheckURL: "/health",
|
||||
apiError: auth.ErrUserEmpty,
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Healthy even if authorization failed",
|
||||
healthCheckURL: "/health",
|
||||
apiError: auth.ErrPermissionDenied,
|
||||
expectStatusCode: http.StatusOK,
|
||||
expectHealth: "true",
|
||||
},
|
||||
{
|
||||
name: "Unhealthy if api is not available",
|
||||
healthCheckURL: "/health",
|
||||
apiError: fmt.Errorf("Unexpected error"),
|
||||
expectStatusCode: http.StatusServiceUnavailable,
|
||||
expectHealth: "false",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
HandleHealth(mux, &fakeHealthServer{
|
||||
fakeServer: fakeServer{alarms: tt.alarms},
|
||||
health: tt.expectHealth,
|
||||
apiError: tt.apiError,
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
defer ts.Close()
|
||||
|
||||
res, err := ts.Client().Do(&http.Request{Method: http.MethodGet, URL: testutil.MustNewURL(t, ts.URL+tt.healthCheckURL)})
|
||||
if err != nil {
|
||||
t.Errorf("fail serve http request %s %v", tt.healthCheckURL, err)
|
||||
}
|
||||
if res == nil {
|
||||
t.Errorf("got nil http response with http request %s", tt.healthCheckURL)
|
||||
return
|
||||
}
|
||||
if res.StatusCode != tt.expectStatusCode {
|
||||
t.Errorf("want statusCode %d but got %d", tt.expectStatusCode, res.StatusCode)
|
||||
}
|
||||
health, err := parseHealthOutput(res.Body)
|
||||
if err != nil {
|
||||
t.Errorf("fail parse health check output %v", err)
|
||||
}
|
||||
if health.Health != tt.expectHealth {
|
||||
t.Errorf("want health %s but got %s", tt.expectHealth, health.Health)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func parseHealthOutput(body io.Reader) (Health, error) {
|
||||
obj := Health{}
|
||||
d, derr := io.ReadAll(body)
|
||||
if derr != nil {
|
||||
return obj, derr
|
||||
}
|
||||
if err := json.Unmarshal(d, &obj); err != nil {
|
||||
return obj, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
@ -15,139 +15,16 @@
|
||||
package etcdhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/etcdserver"
|
||||
"go.etcd.io/etcd/etcdserver/etcdserverpb"
|
||||
"go.etcd.io/etcd/raft"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
const (
|
||||
PathMetrics = "/metrics"
|
||||
PathHealth = "/health"
|
||||
)
|
||||
|
||||
// HandleMetricsHealth registers metrics and health handlers.
|
||||
func HandleMetricsHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
|
||||
mux.Handle(PathMetrics, promhttp.Handler())
|
||||
mux.Handle(PathHealth, NewHealthHandler(func(excludedAlarms AlarmSet) Health { return checkHealth(srv, excludedAlarms) }))
|
||||
}
|
||||
|
||||
// HandlePrometheus registers prometheus handler on '/metrics'.
|
||||
func HandlePrometheus(mux *http.ServeMux) {
|
||||
// HandleMetrics registers prometheus handler on '/metrics'.
|
||||
func HandleMetrics(mux *http.ServeMux) {
|
||||
mux.Handle(PathMetrics, promhttp.Handler())
|
||||
}
|
||||
|
||||
// NewHealthHandler handles '/health' requests.
|
||||
func NewHealthHandler(hfunc func(excludedAlarms AlarmSet) Health) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
w.Header().Set("Allow", http.MethodGet)
|
||||
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||
plog.Warningf("/health error (status code %d)", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
excludedAlarms := getExcludedAlarms(r)
|
||||
h := hfunc(excludedAlarms)
|
||||
d, _ := json.Marshal(h)
|
||||
if h.Health != "true" {
|
||||
http.Error(w, string(d), http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(d)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "etcd",
|
||||
Subsystem: "server",
|
||||
Name: "health_success",
|
||||
Help: "The total number of successful health checks",
|
||||
})
|
||||
healthFailed = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "etcd",
|
||||
Subsystem: "server",
|
||||
Name: "health_failures",
|
||||
Help: "The total number of failed health checks",
|
||||
})
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(healthSuccess)
|
||||
prometheus.MustRegister(healthFailed)
|
||||
}
|
||||
|
||||
// Health defines etcd server health status.
|
||||
// TODO: remove manual parsing in etcdctl cluster-health
|
||||
type Health struct {
|
||||
Health string `json:"health"`
|
||||
}
|
||||
|
||||
type AlarmSet map[string]struct{}
|
||||
|
||||
func getExcludedAlarms(r *http.Request) (alarms AlarmSet) {
|
||||
alarms = make(map[string]struct{}, 2)
|
||||
alms, found := r.URL.Query()["exclude"]
|
||||
if found {
|
||||
for _, alm := range alms {
|
||||
if len(alms) == 0 {
|
||||
continue
|
||||
}
|
||||
alarms[alm] = struct{}{}
|
||||
}
|
||||
}
|
||||
return alarms
|
||||
}
|
||||
|
||||
// TODO: server NOSPACE, etcdserver.ErrNoLeader in health API
|
||||
|
||||
func checkHealth(srv etcdserver.ServerV2, excludedAlarms AlarmSet) Health {
|
||||
h := Health{Health: "true"}
|
||||
|
||||
as := srv.Alarms()
|
||||
if len(as) > 0 {
|
||||
for _, v := range as {
|
||||
alarmName := v.Alarm.String()
|
||||
if _, found := excludedAlarms[alarmName]; found {
|
||||
plog.Debugf("/health excluded alarm %s", v.String())
|
||||
continue
|
||||
}
|
||||
h.Health = "false"
|
||||
plog.Warningf("/health error due to %s", v.String())
|
||||
return h
|
||||
}
|
||||
}
|
||||
|
||||
if h.Health == "true" {
|
||||
if uint64(srv.Leader()) == raft.None {
|
||||
h.Health = "false"
|
||||
plog.Warningf("/health error; no leader (status code %d)", http.StatusServiceUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
if h.Health == "true" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
_, err := srv.Do(ctx, etcdserverpb.Request{Method: "QGET"})
|
||||
cancel()
|
||||
if err != nil {
|
||||
h.Health = "false"
|
||||
plog.Warningf("/health error; QGET failed %v (status code %d)", err, http.StatusServiceUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
if h.Health == "true" {
|
||||
healthSuccess.Inc()
|
||||
plog.Debugf("/health OK (status code %d)", http.StatusOK)
|
||||
} else {
|
||||
healthFailed.Inc()
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
@ -1,157 +0,0 @@
|
||||
// Copyright 2021 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package etcdhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go.etcd.io/etcd/etcdserver"
|
||||
stats "go.etcd.io/etcd/etcdserver/api/v2stats"
|
||||
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
|
||||
"go.etcd.io/etcd/pkg/testutil"
|
||||
"go.etcd.io/etcd/pkg/types"
|
||||
"go.etcd.io/etcd/raft"
|
||||
)
|
||||
|
||||
type fakeStats struct{}
|
||||
|
||||
func (s *fakeStats) SelfStats() []byte { return nil }
|
||||
func (s *fakeStats) LeaderStats() []byte { return nil }
|
||||
func (s *fakeStats) StoreStats() []byte { return nil }
|
||||
|
||||
type fakeServerV2 struct {
|
||||
fakeServer
|
||||
stats.Stats
|
||||
health string
|
||||
}
|
||||
|
||||
func (s *fakeServerV2) Leader() types.ID {
|
||||
if s.health == "true" {
|
||||
return 1
|
||||
}
|
||||
return types.ID(raft.None)
|
||||
}
|
||||
func (s *fakeServerV2) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) {
|
||||
if s.health == "true" {
|
||||
return etcdserver.Response{}, nil
|
||||
}
|
||||
return etcdserver.Response{}, fmt.Errorf("fail health check")
|
||||
}
|
||||
func (s *fakeServerV2) ClientCertAuthEnabled() bool { return false }
|
||||
|
||||
func TestHealthHandler(t *testing.T) {
|
||||
// define the input and expected output
|
||||
// input: alarms, and healthCheckURL
|
||||
tests := []struct {
|
||||
alarms []*pb.AlarmMember
|
||||
healthCheckURL string
|
||||
statusCode int
|
||||
health string
|
||||
}{
|
||||
{
|
||||
[]*pb.AlarmMember{},
|
||||
"/health",
|
||||
http.StatusOK,
|
||||
"true",
|
||||
},
|
||||
{
|
||||
[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}},
|
||||
"/health",
|
||||
http.StatusServiceUnavailable,
|
||||
"false",
|
||||
},
|
||||
{
|
||||
[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}},
|
||||
"/health?exclude=NOSPACE",
|
||||
http.StatusOK,
|
||||
"true",
|
||||
},
|
||||
{
|
||||
[]*pb.AlarmMember{},
|
||||
"/health?exclude=NOSPACE",
|
||||
http.StatusOK,
|
||||
"true",
|
||||
},
|
||||
{
|
||||
[]*pb.AlarmMember{{MemberID: uint64(1), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(2), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(3), Alarm: pb.AlarmType_NOSPACE}},
|
||||
"/health?exclude=NOSPACE",
|
||||
http.StatusOK,
|
||||
"true",
|
||||
},
|
||||
{
|
||||
[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}},
|
||||
"/health?exclude=NOSPACE",
|
||||
http.StatusServiceUnavailable,
|
||||
"false",
|
||||
},
|
||||
{
|
||||
[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}},
|
||||
"/health?exclude=NOSPACE&exclude=CORRUPT",
|
||||
http.StatusOK,
|
||||
"true",
|
||||
},
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
func() {
|
||||
mux := http.NewServeMux()
|
||||
HandleMetricsHealth(mux, &fakeServerV2{
|
||||
fakeServer: fakeServer{alarms: tt.alarms},
|
||||
Stats: &fakeStats{},
|
||||
health: tt.health,
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
defer ts.Close()
|
||||
|
||||
res, err := ts.Client().Do(&http.Request{Method: http.MethodGet, URL: testutil.MustNewURL(t, ts.URL+tt.healthCheckURL)})
|
||||
if err != nil {
|
||||
t.Errorf("fail serve http request %s %v in test case #%d", tt.healthCheckURL, err, i+1)
|
||||
}
|
||||
if res == nil {
|
||||
t.Errorf("got nil http response with http request %s in test case #%d", tt.healthCheckURL, i+1)
|
||||
return
|
||||
}
|
||||
if res.StatusCode != tt.statusCode {
|
||||
t.Errorf("want statusCode %d but got %d in test case #%d", tt.statusCode, res.StatusCode, i+1)
|
||||
}
|
||||
health, err := parseHealthOutput(res.Body)
|
||||
if err != nil {
|
||||
t.Errorf("fail parse health check output %v", err)
|
||||
}
|
||||
if health.Health != tt.health {
|
||||
t.Errorf("want health %s but got %s", tt.health, health.Health)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func parseHealthOutput(body io.Reader) (Health, error) {
|
||||
obj := Health{}
|
||||
d, derr := ioutil.ReadAll(body)
|
||||
if derr != nil {
|
||||
return obj, derr
|
||||
}
|
||||
if err := json.Unmarshal(d, &obj); err != nil {
|
||||
return obj, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
@ -55,6 +55,8 @@ const (
|
||||
func NewClientHandler(lg *zap.Logger, server etcdserver.ServerPeer, timeout time.Duration) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
etcdhttp.HandleBasic(mux, server)
|
||||
etcdhttp.HandleMetrics(mux)
|
||||
etcdhttp.HandleHealthForV2(mux, server)
|
||||
handleV2(lg, mux, server, timeout)
|
||||
return requestLogger(lg, mux)
|
||||
}
|
||||
|
@ -667,6 +667,10 @@ func (s *EtcdServer) getLogger() *zap.Logger {
|
||||
return l
|
||||
}
|
||||
|
||||
func (s *EtcdServer) Config() ServerConfig {
|
||||
return s.Cfg
|
||||
}
|
||||
|
||||
func tickToDur(ticks int, tickMs uint) string {
|
||||
return fmt.Sprintf("%v", time.Duration(ticks)*time.Duration(tickMs)*time.Millisecond)
|
||||
}
|
||||
|
@ -26,7 +26,7 @@ import (
|
||||
|
||||
// HandleHealth registers health handler on '/health'.
|
||||
func HandleHealth(mux *http.ServeMux, c *clientv3.Client) {
|
||||
mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(func(excludedAlarms etcdhttp.AlarmSet) etcdhttp.Health { return checkHealth(c) }))
|
||||
mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(func(excludedAlarms etcdhttp.AlarmSet, serializable bool) etcdhttp.Health { return checkHealth(c) }))
|
||||
}
|
||||
|
||||
func checkHealth(c *clientv3.Client) etcdhttp.Health {
|
||||
|
Loading…
x
Reference in New Issue
Block a user