e2e: test /metrics, /health endpoints

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

e2e/cluster_proxy_test.go

@@ -54,6 +54,9 @@ func (p *proxyEtcdProcess) Config() *etcdServerProcessConfig { return p.etcdProc
 
 func (p *proxyEtcdProcess) EndpointsV2() []string { return p.proxyV2.endpoints() }
 func (p *proxyEtcdProcess) EndpointsV3() []string { return p.proxyV3.endpoints() }
+func (p *proxyEtcdProcess) EndpointsMetrics() []string {
+	panic("not implemented; proxy doesn't provide health information")
+}
 
 func (p *proxyEtcdProcess) Start() error {
 	if err := p.etcdProc.Start(); err != nil {
@@ -113,6 +116,7 @@ type proxyProc struct {
 	execPath string
 	args     []string
 	ep       string
+	murl     string
 	donec    chan struct{}
 
 	proc *expect.ExpectProcess
@@ -232,6 +236,11 @@ func newProxyV3Proc(cfg *etcdServerProcessConfig) *proxyV3Proc {
 		// pass-through member RPCs
 		"--advertise-client-url", "",
 	}
+	murl := ""
+	if cfg.murl != "" {
+		murl = proxyListenURL(cfg, 4)
+		args = append(args, "--metrics-addr", murl)
+	}
 	tlsArgs := []string{}
 	for i := 0; i < len(cfg.tlsArgs); i++ {
 		switch cfg.tlsArgs[i] {
@@ -258,6 +267,7 @@ func newProxyV3Proc(cfg *etcdServerProcessConfig) *proxyV3Proc {
 			execPath: cfg.execPath,
 			args:     append(args, tlsArgs...),
 			ep:       listenAddr,
+			murl:     murl,
 			donec:    make(chan struct{}),
 		},
 	}

e2e/cluster_test.go

@@ -101,6 +101,8 @@ type etcdProcessClusterConfig struct {
 	baseScheme string
 	basePort   int
 
+	metricsURLScheme string
+
 	snapCount int // default is 10000
 
 	clientTLS             clientConnType
@@ -175,7 +177,7 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs() []*etcdServerPro
 	for i := 0; i < cfg.clusterSize; i++ {
 		var curls []string
 		var curl, curltls string
-		port := cfg.basePort + 4*i
+		port := cfg.basePort + 5*i
 		curlHost := fmt.Sprintf("localhost:%d", port)
 
 		switch cfg.clientTLS {
@@ -221,6 +223,14 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs() []*etcdServerPro
 		if cfg.noStrictReconfig {
 			args = append(args, "--strict-reconfig-check=false")
 		}
+		var murl string
+		if cfg.metricsURLScheme != "" {
+			murl = (&url.URL{
+				Scheme: cfg.metricsURLScheme,
+				Host:   fmt.Sprintf("localhost:%d", port+2),
+			}).String()
+			args = append(args, "--listen-metrics-urls", murl)
+		}
 
 		args = append(args, cfg.tlsArgs()...)
 		etcdCfgs[i] = &etcdServerProcessConfig{
@@ -232,6 +242,7 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs() []*etcdServerPro
 			name:         name,
 			purl:         purl,
 			acurl:        curl,
+			murl:         murl,
 			initialToken: cfg.initialToken,
 		}
 	}

e2e/etcd_process.go

@@ -29,6 +29,7 @@ var etcdServerReadyLines = []string{"enabled capabilities for version", "publish
 type etcdProcess interface {
 	EndpointsV2() []string
 	EndpointsV3() []string
+	EndpointsMetrics() []string
 
 	Start() error
 	Restart() error
@@ -57,6 +58,7 @@ type etcdServerProcessConfig struct {
 	purl url.URL
 
 	acurl string
+	murl  string
 
 	initialToken   string
 	initialCluster string
@@ -74,8 +76,9 @@ func newEtcdServerProcess(cfg *etcdServerProcessConfig) (*etcdServerProcess, err
 	return &etcdServerProcess{cfg: cfg, donec: make(chan struct{})}, nil
 }
 
-func (ep *etcdServerProcess) EndpointsV2() []string { return []string{ep.cfg.acurl} }
+func (ep *etcdServerProcess) EndpointsV2() []string      { return []string{ep.cfg.acurl} }
-func (ep *etcdServerProcess) EndpointsV3() []string { return ep.EndpointsV2() }
+func (ep *etcdServerProcess) EndpointsV3() []string      { return ep.EndpointsV2() }
+func (ep *etcdServerProcess) EndpointsMetrics() []string { return []string{ep.cfg.murl} }
 
 func (ep *etcdServerProcess) Start() error {
 	if ep.proc != nil {

e2e/metrics_test.go

@@ -0,0 +1,47 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !cluster_proxy
+
+package e2e
+
+import (
+	"testing"
+)
+
+func TestV3MetricsSecure(t *testing.T) {
+	cfg := configTLS
+	cfg.clusterSize = 1
+	cfg.metricsURLScheme = "https"
+	testCtl(t, metricsTest)
+}
+
+func TestV3MetricsInsecure(t *testing.T) {
+	cfg := configTLS
+	cfg.clusterSize = 1
+	cfg.metricsURLScheme = "http"
+	testCtl(t, metricsTest)
+}
+
+func metricsTest(cx ctlCtx) {
+	if err := ctlV3Put(cx, "k", "v", ""); err != nil {
+		cx.t.Fatal(err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: `etcd_debugging_mvcc_keys_total 1`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health": "true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+}

e2e/v2_curl_test.go

@@ -125,6 +125,8 @@ type cURLReq struct {
 	value    string
 	expected string
 	header   string
+
+	metricsURLScheme string
 }
 
 // cURLPrefixArgs builds the beginning of a curl command for a given key
@@ -134,14 +136,19 @@ func cURLPrefixArgs(clus *etcdProcessCluster, method string, req cURLReq) []stri
 		cmdArgs = []string{"curl"}
 		acurl   = clus.procs[rand.Intn(clus.cfg.clusterSize)].Config().acurl
 	)
-	if req.isTLS {
+	if req.metricsURLScheme != "https" {
-		if clus.cfg.clientTLS != clientTLSAndNonTLS {
+		if req.isTLS {
-			panic("should not use cURLPrefixArgsUseTLS when serving only TLS or non-TLS")
+			if clus.cfg.clientTLS != clientTLSAndNonTLS {
+				panic("should not use cURLPrefixArgsUseTLS when serving only TLS or non-TLS")
+			}
+			cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", privateKeyPath)
+			acurl = toTLS(clus.procs[rand.Intn(clus.cfg.clusterSize)].Config().acurl)
+		} else if clus.cfg.clientTLS == clientTLS {
+			cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", privateKeyPath)
 		}
-		cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", privateKeyPath)
+	}
-		acurl = toTLS(clus.procs[rand.Intn(clus.cfg.clusterSize)].Config().acurl)
+	if req.metricsURLScheme != "" {
-	} else if clus.cfg.clientTLS == clientTLS {
+		acurl = clus.procs[rand.Intn(clus.cfg.clusterSize)].EndpointsMetrics()[0]
-		cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", privateKeyPath)
 	}
 	ep := acurl + req.endpoint