Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

http health check bug fixes

Browse Source 
Signed-off-by: Chao Chen <chaochn@amazon.com>
This commit is contained in:
Chao Chen 2023-10-05 23:28:33 -07:00 committed by Siyuan Zhang
parent 963af731bd
commit d4861d660b
3 changed files with 30 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
etcdserver/api/etcdhttp/health.go
View File

@ -38,6 +38,7 @@ type ServerHealth interface {
serverHealthV2V3
Range(context.Context, *pb.RangeRequest) (*pb.RangeResponse, error)
Config() etcdserver.ServerConfig
AuthStore() auth.AuthStore
}
type serverHealthV2V3 interface {
@ -47,33 +48,33 @@ type serverHealthV2V3 interface {
// HandleHealthForV2 registers metrics and health handlers for v2.
func HandleHealthForV2(mux *http.ServeMux, srv etcdserver.ServerV2) {
mux.Handle(PathHealth, NewHealthHandler(func(excludedAlarms AlarmSet, serializable bool) Health {
mux.Handle(PathHealth, NewHealthHandler(func(ctx context.Context, excludedAlarms AlarmSet, serializable bool) Health {
if h := checkAlarms(srv, excludedAlarms); h.Health != "true" {
return h
}
if h := checkLeader(srv, serializable); h.Health != "true" {
return h
}
return checkV2API(srv)
return checkV2API(ctx, srv)
}))
}
// HandleHealth registers metrics and health handlers. it checks health by using v3 range request
// and its corresponding timeout.
func HandleHealth(mux *http.ServeMux, srv ServerHealth) {
mux.Handle(PathHealth, NewHealthHandler(func(excludedAlarms AlarmSet, serializable bool) Health {
mux.Handle(PathHealth, NewHealthHandler(func(ctx context.Context, excludedAlarms AlarmSet, serializable bool) Health {
if h := checkAlarms(srv, excludedAlarms); h.Health != "true" {
return h
}
if h := checkLeader(srv, serializable); h.Health != "true" {
return h
}
return checkAPI(srv, serializable)
return checkAPI(ctx, srv, serializable)
}))
}
// NewHealthHandler handles '/health' requests.
func NewHealthHandler(hfunc func(excludedAlarms AlarmSet, serializable bool) Health) http.HandlerFunc {
func NewHealthHandler(hfunc func(ctx context.Context, excludedAlarms AlarmSet, serializable bool) Health) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Allow", http.MethodGet)
@ -87,7 +88,7 @@ func NewHealthHandler(hfunc func(excludedAlarms AlarmSet, serializable bool) Hea
// This is useful for probes attempting to validate the liveness of
// the etcd process vs readiness of the cluster to serve requests.
serializableFlag := getSerializableFlag(r)
h := hfunc(excludedAlarms, serializableFlag)
h := hfunc(r.Context(), excludedAlarms, serializableFlag)
defer func() {
if h.Health == "true" {
healthSuccess.Inc()
@ -193,9 +194,9 @@ func checkLeader(srv serverHealthV2V3, serializable bool) Health {
return h
}
func checkV2API(srv etcdserver.ServerV2) Health {
func checkV2API(ctx context.Context, srv etcdserver.ServerV2) Health {
h := Health{Health: "true"}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
ctx, cancel := context.WithTimeout(ctx, time.Second)
_, err := srv.Do(ctx, pb.Request{Method: "QGET"})
cancel()
if err != nil {
@ -207,13 +208,14 @@ func checkV2API(srv etcdserver.ServerV2) Health {
return h
}
func checkAPI(srv ServerHealth, serializable bool) Health {
func checkAPI(ctx context.Context, srv ServerHealth, serializable bool) Health {
h := Health{Health: "true"}
cfg := srv.Config()
ctx, cancel := context.WithTimeout(context.Background(), cfg.ReqTimeout())
_, err := srv.Range(ctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable})
ctx = srv.AuthStore().WithRoot(ctx)
cctx, cancel := context.WithTimeout(ctx, cfg.ReqTimeout())
_, err := srv.Range(cctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable})
cancel()
if err != nil && err != auth.ErrUserEmpty && err != auth.ErrPermissionDenied {
if err != nil {
h.Health = "false"
h.Reason = fmt.Sprintf("RANGE ERROR:%s", err)
plog.Warningf("serving /health false; Range failed %v (status code %d)", err, http.StatusServiceUnavailable)

31
etcdserver/api/etcdhttp/health_test.go
View File

@ -23,9 +23,12 @@ import (
"net/http/httptest"
"testing"
"go.uber.org/zap/zaptest"
"go.etcd.io/etcd/auth"
"go.etcd.io/etcd/etcdserver"
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
betesting "go.etcd.io/etcd/mvcc/backend"
"go.etcd.io/etcd/pkg/testutil"
"go.etcd.io/etcd/pkg/types"
"go.etcd.io/etcd/raft"
@ -33,11 +36,12 @@ import (
type fakeHealthServer struct {
fakeServer
health string
apiError error
health string
apiError error
authStore auth.AuthStore
}
func (s *fakeHealthServer) Range(ctx context.Context, request *pb.RangeRequest) (*pb.RangeResponse, error) {
func (s *fakeHealthServer) Range(_ context.Context, _ *pb.RangeRequest) (*pb.RangeResponse, error) {
return nil, s.apiError
}
@ -51,12 +55,13 @@ func (s *fakeHealthServer) Leader() types.ID {
}
return types.ID(raft.None)
}
func (s *fakeHealthServer) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) {
func (s *fakeHealthServer) Do(_ context.Context, _ pb.Request) (etcdserver.Response, error) {
if s.health == "true" {
return etcdserver.Response{}, nil
}
return etcdserver.Response{}, fmt.Errorf("fail health check")
}
func (s *fakeHealthServer) AuthStore() auth.AuthStore { return s.authStore }
func (s *fakeHealthServer) ClientCertAuthEnabled() bool { return false }
func TestHealthHandler(t *testing.T) {
@ -120,20 +125,6 @@ func TestHealthHandler(t *testing.T) {
expectStatusCode: http.StatusOK,
expectHealth: "true",
},
{
name: "Healthy even if authentication failed",
healthCheckURL: "/health",
apiError: auth.ErrUserEmpty,
expectStatusCode: http.StatusOK,
expectHealth: "true",
},
{
name: "Healthy even if authorization failed",
healthCheckURL: "/health",
apiError: auth.ErrPermissionDenied,
expectStatusCode: http.StatusOK,
expectHealth: "true",
},
{
name: "Unhealthy if api is not available",
healthCheckURL: "/health",
@ -146,10 +137,14 @@ func TestHealthHandler(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mux := http.NewServeMux()
lg := zaptest.NewLogger(t)
be, _ := betesting.NewDefaultTmpBackend()
defer be.Close()
HandleHealth(mux, &fakeHealthServer{
fakeServer: fakeServer{alarms: tt.alarms},
health: tt.expectHealth,
apiError: tt.apiError,
authStore: auth.NewAuthStore(lg, be, nil, 0),
})
ts := httptest.NewServer(mux)
defer ts.Close()

4
proxy/grpcproxy/health.go
View File

@ -26,7 +26,9 @@ import (
// HandleHealth registers health handler on '/health'.
func HandleHealth(mux *http.ServeMux, c *clientv3.Client) {
mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(func(excludedAlarms etcdhttp.AlarmSet, serializable bool) etcdhttp.Health { return checkHealth(c) }))
mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(func(ctx context.Context, excludedAlarms etcdhttp.AlarmSet, serializable bool) etcdhttp.Health {
return checkHealth(c)
}))
}
func checkHealth(c *clientv3.Client) etcdhttp.Health {