Merge pull request #11983 from tangcong/add_reason_field_for_health

etcdserver/api/etcdhttp: add reason field for /health response

CHANGELOG-3.5.md

@@ -125,6 +125,7 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
   - See https://github.com/etcd-io/etcd/issues/11918.
 - Improve logging around snapshot send and receive.
 - [Push down RangeOptions.limit argv into index tree to reduce memory overhead](https://github.com/etcd-io/etcd/pull/11990).
+- Add [reason field for /health response](https://github.com/etcd-io/etcd/pull/11983).
 
 ### Package `embed`
 

etcdserver/api/etcdhttp/metrics.go

@@ -90,6 +90,7 @@ func init() {
 // TODO: remove manual parsing in etcdctl cluster-health
 type Health struct {
 	Health string `json:"health"`
+	Reason string `json:"reason"`
 }
 
 // TODO: server NOSPACE, etcdserver.ErrNoLeader in health API
@@ -109,6 +110,14 @@ func checkHealth(lg *zap.Logger, srv etcdserver.ServerV2) (h Health) {
 	if len(as) > 0 {
 		h.Health = "false"
 		for _, v := range as {
+			switch v.Alarm {
+			case etcdserverpb.AlarmType_NOSPACE:
+				h.Reason = "ALARM NOSPACE"
+			case etcdserverpb.AlarmType_CORRUPT:
+				h.Reason = "ALARM CORRUPT"
+			default:
+				h.Reason = "ALARM UNKNOWN"
+			}
 			lg.Warn("serving /health false due to an alarm", zap.String("alarm", v.String()))
 		}
 		return
@@ -116,6 +125,7 @@ func checkHealth(lg *zap.Logger, srv etcdserver.ServerV2) (h Health) {
 
 	if uint64(srv.Leader()) == raft.None {
 		h.Health = "false"
+		h.Reason = "RAFT NO LEADER"
 		lg.Warn("serving /health false; no leader")
 		return
 	}
@@ -125,6 +135,7 @@ func checkHealth(lg *zap.Logger, srv etcdserver.ServerV2) (h Health) {
 	cancel()
 	if err != nil {
 		h.Health = "false"
+		h.Reason = "QGET ERROR"
 		lg.Warn("serving /health false; QGET fails", zap.Error(err))
 	}
 

tests/e2e/ctl_v3_alarm_test.go

@@ -53,7 +53,7 @@ func alarmTest(cx ctlCtx) {
 	}
 
 	// '/health' handler should return 'false'
-	if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"false"}`}); err != nil {
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"false","reason":"ALARM NOSPACE"}`}); err != nil {
 		cx.t.Fatalf("failed get with curl (%v)", err)
 	}
 

tests/e2e/metrics_test.go

@@ -49,7 +49,7 @@ func metricsTest(cx ctlCtx) {
 		{"/metrics", fmt.Sprintf("etcd_mvcc_delete_total 3")},
 		{"/metrics", fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version)},
 		{"/metrics", fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version))},
-		{"/health", `{"health":"true"}`},
+		{"/health", `{"health":"true","reason":""}`},
 	} {
 		i++
 		if err := ctlV3Put(cx, fmt.Sprintf("%d", i), "v", ""); err != nil {