Merge pull request #8322 from gyuho/health-grpc-proxy

*: add /health endpoint to grpc-proxy

e2e/metrics_test.go

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !cluster_proxy
-
 package e2e
 
 import (

etcdmain/grpc_proxy.go

@@ -154,7 +154,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 
 	client := mustNewClient()
 
-	srvhttp, httpl := mustHTTPListener(m, tlsinfo)
+	srvhttp, httpl := mustHTTPListener(m, tlsinfo, client)
 	errc := make(chan error)
 	go func() { errc <- newGRPCProxyServer(client).Serve(grpcl) }()
 	go func() { errc <- srvhttp.Serve(httpl) }()
@@ -164,6 +164,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 		go func() {
 			mux := http.NewServeMux()
 			etcdhttp.HandlePrometheus(mux)
+			grpcproxy.HandleHealth(mux, client)
 			plog.Fatal(http.Serve(mhttpl, mux))
 		}()
 	}
@@ -310,10 +311,11 @@ func newGRPCProxyServer(client *clientv3.Client) *grpc.Server {
 	return server
 }
 
-func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo) (*http.Server, net.Listener) {
+func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo, c *clientv3.Client) (*http.Server, net.Listener) {
 	httpmux := http.NewServeMux()
 	httpmux.HandleFunc("/", http.NotFound)
 	etcdhttp.HandlePrometheus(httpmux)
+	grpcproxy.HandleHealth(httpmux, c)
 	if grpcProxyEnablePprof {
 		for p, h := range debugutil.PProfHandlers() {
 			httpmux.Handle(p, h)

etcdserver/api/etcdhttp/metrics.go

@@ -29,13 +29,13 @@ import (
 
 const (
 	pathMetrics = "/metrics"
-	pathHealth  = "/health"
+	PathHealth  = "/health"
 )
 
 // HandleMetricsHealth registers metrics and health handlers.
 func HandleMetricsHealth(mux *http.ServeMux, srv *etcdserver.EtcdServer) {
 	mux.Handle(pathMetrics, prometheus.Handler())
-	mux.Handle(pathHealth, newHealthHandler(srv))
+	mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
 }
 
 // HandlePrometheus registers prometheus handler on '/metrics'.
@@ -45,18 +45,18 @@ func HandlePrometheus(mux *http.ServeMux) {
 
 // HandleHealth registers health handler on '/health'.
 func HandleHealth(mux *http.ServeMux, srv *etcdserver.EtcdServer) {
-	mux.Handle(pathHealth, newHealthHandler(srv))
+	mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
 }
 
-// newHealthHandler handles '/health' requests.
+// NewHealthHandler handles '/health' requests.
-func newHealthHandler(srv *etcdserver.EtcdServer) http.HandlerFunc {
+func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodGet {
 			w.Header().Set("Allow", http.MethodGet)
 			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
 			return
 		}
-		h := checkHealth(srv)
+		h := hfunc()
 		d, _ := json.Marshal(h)
 		if !h.Health {
 			http.Error(w, string(d), http.StatusServiceUnavailable)
@@ -67,14 +67,15 @@ func newHealthHandler(srv *etcdserver.EtcdServer) http.HandlerFunc {
 	}
 }
 
+// Health defines etcd server health status.
 // TODO: remove manual parsing in etcdctl cluster-health
-type health struct {
+type Health struct {
 	Health bool     `json:"health"`
 	Errors []string `json:"errors,omitempty"`
 }
 
-func checkHealth(srv *etcdserver.EtcdServer) health {
+func checkHealth(srv *etcdserver.EtcdServer) Health {
-	h := health{Health: false}
+	h := Health{Health: false}
 
 	as := srv.Alarms()
 	if len(as) > 0 {

proxy/grpcproxy/health.go

@@ -0,0 +1,43 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpcproxy
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/etcdserver/api/etcdhttp"
+	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+
+	"golang.org/x/net/context"
+)
+
+// HandleHealth registers health handler on '/health'.
+func HandleHealth(mux *http.ServeMux, c *clientv3.Client) {
+	mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(func() etcdhttp.Health { return checkHealth(c) }))
+}
+
+func checkHealth(c *clientv3.Client) etcdhttp.Health {
+	h := etcdhttp.Health{Health: false}
+	ctx, cancel := context.WithTimeout(c.Ctx(), time.Second)
+	_, err := c.Get(ctx, "a")
+	cancel()
+	h.Health = err == nil || err == rpctypes.ErrPermissionDenied
+	if !h.Health {
+		h.Errors = append(h.Errors, err.Error())
+	}
+	return h
+}