Merge pull request #8242 from gyuho/ppp

*: support additional '/metrics' endpoints

embed/config.go

@@ -112,10 +112,12 @@ type Config struct {
 
 	// debug
 
-	Debug        bool   `json:"debug"`
-	LogPkgLevels string `json:"log-package-levels"`
-	EnablePprof  bool
-	Metrics      string `json:"metrics"`
+	Debug                 bool   `json:"debug"`
+	LogPkgLevels          string `json:"log-package-levels"`
+	EnablePprof           bool
+	Metrics               string `json:"metrics"`
+	ListenMetricsUrls     []url.URL
+	ListenMetricsUrlsJSON string `json:"listen-metrics-urls"`
 
 	// ForceNewCluster starts a new cluster even if previously started; unsafe.
 	ForceNewCluster bool `json:"force-new-cluster"`
@@ -255,6 +257,14 @@ func (cfg *configYAML) configFromFile(path string) error {
 		cfg.ACUrls = []url.URL(u)
 	}
 
+	if cfg.ListenMetricsUrlsJSON != "" {
+		u, err := types.NewURLs(strings.Split(cfg.ListenMetricsUrlsJSON, ","))
+		if err != nil {
+			plog.Fatalf("unexpected error setting up listen-metrics-urls: %v", err)
+		}
+		cfg.ListenMetricsUrls = []url.URL(u)
+	}
+
 	// If a discovery flag is set, clear default initial cluster set by InitialClusterFromName
 	if (cfg.Durl != "" || cfg.DNSCluster != "") && cfg.InitialCluster == defaultInitialCluster {
 		cfg.InitialCluster = ""
@@ -285,6 +295,9 @@ func (cfg *Config) Validate() error {
 	if err := checkBindURLs(cfg.LCUrls); err != nil {
 		return err
 	}
+	if err := checkBindURLs(cfg.ListenMetricsUrls); err != nil {
+		return err
+	}
 
 	// Check if conflicting flags are passed.
 	nSet := 0

embed/etcd.go

@@ -21,6 +21,7 @@ import (
 	defaultLog "log"
 	"net"
 	"net/http"
+	"net/url"
 	"path/filepath"
 	"sync"
 	"time"
@@ -35,6 +36,7 @@ import (
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/rafthttp"
 	"github.com/coreos/pkg/capnslog"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "embed")
@@ -55,9 +57,10 @@ const (
 
 // Etcd contains a running etcd server and its listeners.
 type Etcd struct {
-	Peers   []*peerListener
-	Clients []net.Listener
-	Server  *etcdserver.EtcdServer
+	Peers            []*peerListener
+	Clients          []net.Listener
+	metricsListeners []net.Listener
+	Server           *etcdserver.EtcdServer
 
 	cfg   Config
 	stopc chan struct{}
@@ -205,6 +208,9 @@ func (e *Etcd) Close() {
 			e.Clients[i].Close()
 		}
 	}
+	for i := range e.metricsListeners {
+		e.metricsListeners[i].Close()
+	}
 
 	// close rafthttp transports
 	if e.Server != nil {
@@ -400,6 +406,25 @@ func (e *Etcd) serve() (err error) {
 			e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler))
 		}(sctx)
 	}
+
+	if len(e.cfg.ListenMetricsUrls) > 0 {
+		// TODO: maybe etcdhttp.MetricsPath or get the path from the user-provided URL
+		metricsMux := http.NewServeMux()
+		metricsMux.Handle("/metrics", prometheus.Handler())
+
+		for _, murl := range e.cfg.ListenMetricsUrls {
+			ml, err := transport.NewListener(murl.Host, murl.Scheme, &e.cfg.ClientTLSInfo)
+			if err != nil {
+				return err
+			}
+			e.metricsListeners = append(e.metricsListeners, ml)
+			go func(u url.URL, ln net.Listener) {
+				plog.Info("listening for metrics on ", u.String())
+				e.errHandler(http.Serve(ln, metricsMux))
+			}(murl, ml)
+		}
+	}
+
 	return nil
 }
 

etcdmain/config.go

@@ -20,12 +20,14 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
 
 	"github.com/coreos/etcd/embed"
 	"github.com/coreos/etcd/pkg/flags"
+	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/version"
 	"github.com/ghodss/yaml"
 )
@@ -131,6 +133,7 @@ func newConfig() *config {
 	fs.StringVar(&cfg.WalDir, "wal-dir", cfg.WalDir, "Path to the dedicated wal directory.")
 	fs.Var(flags.NewURLsValue(embed.DefaultListenPeerURLs), "listen-peer-urls", "List of URLs to listen on for peer traffic.")
 	fs.Var(flags.NewURLsValue(embed.DefaultListenClientURLs), "listen-client-urls", "List of URLs to listen on for client traffic.")
+	fs.StringVar(&cfg.ListenMetricsUrlsJSON, "listen-metrics-urls", "", "List of URLs to listen on for metrics.")
 	fs.UintVar(&cfg.MaxSnapFiles, "max-snapshots", cfg.MaxSnapFiles, "Maximum number of snapshot files to retain (0 is unlimited).")
 	fs.UintVar(&cfg.MaxWalFiles, "max-wals", cfg.MaxWalFiles, "Maximum number of wal files to retain (0 is unlimited).")
 	fs.StringVar(&cfg.Name, "name", cfg.Name, "Human-readable name for this member.")
@@ -262,6 +265,15 @@ func (cfg *config) configFromCmdLine() error {
 	cfg.APUrls = flags.URLsFromFlag(cfg.FlagSet, "initial-advertise-peer-urls")
 	cfg.LCUrls = flags.URLsFromFlag(cfg.FlagSet, "listen-client-urls")
 	cfg.ACUrls = flags.URLsFromFlag(cfg.FlagSet, "advertise-client-urls")
+
+	if len(cfg.ListenMetricsUrlsJSON) > 0 {
+		u, err := types.NewURLs(strings.Split(cfg.ListenMetricsUrlsJSON, ","))
+		if err != nil {
+			plog.Fatalf("unexpected error setting up listen-metrics-urls: %v", err)
+		}
+		cfg.ListenMetricsUrls = []url.URL(u)
+	}
+
 	cfg.ClusterState = cfg.clusterState.String()
 	cfg.Fallback = cfg.fallback.String()
 	cfg.Proxy = cfg.proxy.String()

etcdmain/etcd.go

@@ -313,7 +313,7 @@ func startProxy(cfg *config) error {
 		go func() {
 			plog.Info("proxy: listening for client requests on ", host)
 			mux := http.NewServeMux()
-			mux.Handle("/metrics", prometheus.Handler())
+			mux.Handle("/metrics", prometheus.Handler()) // v2 proxy just uses the same port
 			mux.Handle("/", ph)
 			plog.Fatal(http.Serve(l, mux))
 		}()

etcdmain/grpc_proxy.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"time"
 
@@ -40,6 +41,7 @@ import (
 
 var (
 	grpcProxyListenAddr        string
+	grpcProxyMetricsListenAddr string
 	grpcProxyEndpoints         []string
 	grpcProxyDNSCluster        string
 	grpcProxyInsecureDiscovery bool
@@ -80,6 +82,7 @@ func newGRPCProxyStartCommand() *cobra.Command {
 
 	cmd.Flags().StringVar(&grpcProxyListenAddr, "listen-addr", "127.0.0.1:23790", "listen address")
 	cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster")
+	cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for /metrics requests on an additional interface")
 	cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records")
 	cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints")
 	cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
@@ -129,7 +132,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 	}()
 	m := cmux.New(l)
 
-	cfg, err := newClientCfg()
+	cfg, cfgtls, err := newClientCfg()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
@@ -202,6 +205,27 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 
 	go func() { errc <- m.Serve() }()
 
+	if len(grpcProxyMetricsListenAddr) > 0 {
+		murl, err := url.Parse(grpcProxyMetricsListenAddr)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "cannot parse %q", grpcProxyMetricsListenAddr)
+			os.Exit(1)
+		}
+		ml, err := transport.NewListener(murl.Host, murl.Scheme, cfgtls)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		mux := http.NewServeMux()
+		mux.Handle("/metrics", prometheus.Handler())
+
+		go func() {
+			plog.Info("grpc-proxy: listening for metrics on ", murl.String())
+			plog.Fatal(http.Serve(ml, mux))
+		}()
+	}
+
 	// grpc-proxy is initialized, ready to serve
 	notifySystemd()
 
@@ -209,7 +233,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 	os.Exit(1)
 }
 
-func newClientCfg() (*clientv3.Config, error) {
+func newClientCfg() (*clientv3.Config, *transport.TLSInfo, error) {
 	// set tls if any one tls option set
 	var cfgtls *transport.TLSInfo
 	tlsinfo := transport.TLSInfo{}
@@ -235,12 +259,12 @@ func newClientCfg() (*clientv3.Config, error) {
 	if cfgtls != nil {
 		clientTLS, err := cfgtls.ClientConfig()
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		cfg.TLS = clientTLS
 	}
 
 	// TODO: support insecure tls
 
-	return &cfg, nil
+	return &cfg, cfgtls, nil
 }

etcdmain/help.go

@@ -66,7 +66,7 @@ member flags:
 		comma-separated whitelist of origins for CORS (cross-origin resource sharing).
 	--quota-backend-bytes '0'
 		raise alarms when backend size exceeds the given quota (0 defaults to low space quota).
-	--max-txn-ops '128' 
+	--max-txn-ops '128'
 		maximum number of operations permitted in a transaction.
 	--max-request-bytes '1572864'
 		maximum client request size in bytes the server will accept.
@@ -172,7 +172,9 @@ profiling flags:
 	--enable-pprof 'false'
 		Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/"
 	--metrics 'basic'
-	  Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
+		Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
+	--listen-metrics-urls ''
+		List of URLs to listen on for metrics.
 
 auth flags:
 	--auth-token 'simple'