server: Move server files to 'server' directory.

26  git mv mvcc wal auth etcdserver etcdmain proxy embed/ lease/ server
   36  git mv go.mod go.sum server

server/proxy/httpproxy/director.go

@@ -0,0 +1,179 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"math/rand"
+	"net/url"
+	"sync"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+// defaultRefreshInterval is the default proxyRefreshIntervalMs value
+// as in etcdmain/config.go.
+const defaultRefreshInterval = 30000 * time.Millisecond
+
+var once sync.Once
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func newDirector(lg *zap.Logger, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) *director {
+	if lg == nil {
+		lg = zap.NewNop()
+	}
+	d := &director{
+		lg:          lg,
+		uf:          urlsFunc,
+		failureWait: failureWait,
+	}
+	d.refresh()
+	go func() {
+		// In order to prevent missing proxy endpoints in the first try:
+		// when given refresh interval of defaultRefreshInterval or greater
+		// and whenever there is no available proxy endpoints,
+		// give 1-second refreshInterval.
+		for {
+			es := d.endpoints()
+			ri := refreshInterval
+			if ri >= defaultRefreshInterval {
+				if len(es) == 0 {
+					ri = time.Second
+				}
+			}
+			if len(es) > 0 {
+				once.Do(func() {
+					var sl []string
+					for _, e := range es {
+						sl = append(sl, e.URL.String())
+					}
+					lg.Info("endpoints found", zap.Strings("endpoints", sl))
+				})
+			}
+			time.Sleep(ri)
+			d.refresh()
+		}
+	}()
+	return d
+}
+
+type director struct {
+	sync.Mutex
+	lg          *zap.Logger
+	ep          []*endpoint
+	uf          GetProxyURLs
+	failureWait time.Duration
+}
+
+func (d *director) refresh() {
+	urls := d.uf()
+	d.Lock()
+	defer d.Unlock()
+	var endpoints []*endpoint
+	for _, u := range urls {
+		uu, err := url.Parse(u)
+		if err != nil {
+			d.lg.Info("upstream URL invalid", zap.Error(err))
+			continue
+		}
+		endpoints = append(endpoints, newEndpoint(d.lg, *uu, d.failureWait))
+	}
+
+	// shuffle array to avoid connections being "stuck" to a single endpoint
+	for i := range endpoints {
+		j := rand.Intn(i + 1)
+		endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
+	}
+
+	d.ep = endpoints
+}
+
+func (d *director) endpoints() []*endpoint {
+	d.Lock()
+	defer d.Unlock()
+	filtered := make([]*endpoint, 0)
+	for _, ep := range d.ep {
+		if ep.Available {
+			filtered = append(filtered, ep)
+		}
+	}
+
+	return filtered
+}
+
+func newEndpoint(lg *zap.Logger, u url.URL, failureWait time.Duration) *endpoint {
+	ep := endpoint{
+		lg:        lg,
+		URL:       u,
+		Available: true,
+		failFunc:  timedUnavailabilityFunc(failureWait),
+	}
+
+	return &ep
+}
+
+type endpoint struct {
+	sync.Mutex
+
+	lg        *zap.Logger
+	URL       url.URL
+	Available bool
+
+	failFunc func(ep *endpoint)
+}
+
+func (ep *endpoint) Failed() {
+	ep.Lock()
+	if !ep.Available {
+		ep.Unlock()
+		return
+	}
+
+	ep.Available = false
+	ep.Unlock()
+
+	if ep.lg != nil {
+		ep.lg.Info("marked endpoint unavailable", zap.String("endpoint", ep.URL.String()))
+	}
+
+	if ep.failFunc == nil {
+		if ep.lg != nil {
+			ep.lg.Info(
+				"no failFunc defined, endpoint will be unavailable forever",
+				zap.String("endpoint", ep.URL.String()),
+			)
+		}
+		return
+	}
+
+	ep.failFunc(ep)
+}
+
+func timedUnavailabilityFunc(wait time.Duration) func(*endpoint) {
+	return func(ep *endpoint) {
+		time.AfterFunc(wait, func() {
+			ep.Available = true
+			if ep.lg != nil {
+				ep.lg.Info(
+					"marked endpoint available, to retest connectivity",
+					zap.String("endpoint", ep.URL.String()),
+				)
+			}
+		})
+	}
+}

server/proxy/httpproxy/director_test.go

@@ -0,0 +1,97 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"net/url"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+func TestNewDirectorScheme(t *testing.T) {
+	tests := []struct {
+		urls []string
+		want []string
+	}{
+		{
+			urls: []string{"http://192.0.2.8:4002", "http://example.com:8080"},
+			want: []string{"http://192.0.2.8:4002", "http://example.com:8080"},
+		},
+		{
+			urls: []string{"https://192.0.2.8:4002", "https://example.com:8080"},
+			want: []string{"https://192.0.2.8:4002", "https://example.com:8080"},
+		},
+
+		// accept urls without a port
+		{
+			urls: []string{"http://192.0.2.8"},
+			want: []string{"http://192.0.2.8"},
+		},
+
+		// accept urls even if they are garbage
+		{
+			urls: []string{"http://."},
+			want: []string{"http://."},
+		},
+	}
+
+	for i, tt := range tests {
+		uf := func() []string {
+			return tt.urls
+		}
+		got := newDirector(zap.NewExample(), uf, time.Minute, time.Minute)
+
+		var gep []string
+		for _, ep := range got.ep {
+			gep = append(gep, ep.URL.String())
+		}
+		sort.Strings(tt.want)
+		sort.Strings(gep)
+		if !reflect.DeepEqual(tt.want, gep) {
+			t.Errorf("#%d: want endpoints = %#v, got = %#v", i, tt.want, gep)
+		}
+	}
+}
+
+func TestDirectorEndpointsFiltering(t *testing.T) {
+	d := director{
+		ep: []*endpoint{
+			{
+				URL:       url.URL{Scheme: "http", Host: "192.0.2.5:5050"},
+				Available: false,
+			},
+			{
+				URL:       url.URL{Scheme: "http", Host: "192.0.2.4:4000"},
+				Available: true,
+			},
+		},
+	}
+
+	got := d.endpoints()
+	want := []*endpoint{
+		{
+			URL:       url.URL{Scheme: "http", Host: "192.0.2.4:4000"},
+			Available: true,
+		},
+	}
+
+	if !reflect.DeepEqual(want, got) {
+		t.Fatalf("directed to incorrect endpoint: want = %#v, got = %#v", want, got)
+	}
+}

server/proxy/httpproxy/doc.go

@@ -0,0 +1,18 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package httpproxy implements etcd httpproxy. The etcd proxy acts as a reverse
+// http proxy forwarding client requests to active etcd cluster members, and does
+// not participate in consensus.
+package httpproxy

server/proxy/httpproxy/metrics.go

@@ -0,0 +1,90 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	requestsIncoming = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "proxy",
+			Name:      "requests_total",
+			Help:      "Counter requests incoming by method.",
+		}, []string{"method"})
+
+	requestsHandled = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "proxy",
+			Name:      "handled_total",
+			Help:      "Counter of requests fully handled (by authoratitave servers)",
+		}, []string{"method", "code"})
+
+	requestsDropped = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "proxy",
+			Name:      "dropped_total",
+			Help:      "Counter of requests dropped on the proxy.",
+		}, []string{"method", "proxying_error"})
+
+	requestsHandlingSec = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "etcd",
+			Subsystem: "proxy",
+			Name:      "handling_duration_seconds",
+			Help:      "Bucketed histogram of handling time of successful events (non-watches), by method (GET/PUT etc.).",
+
+			// lowest bucket start of upper bound 0.0005 sec (0.5 ms) with factor 2
+			// highest bucket start of 0.0005 sec * 2^12 == 2.048 sec
+			Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13),
+		}, []string{"method"})
+)
+
+type forwardingError string
+
+const (
+	zeroEndpoints         forwardingError = "zero_endpoints"
+	failedSendingRequest  forwardingError = "failed_sending_request"
+	failedGettingResponse forwardingError = "failed_getting_response"
+)
+
+func init() {
+	prometheus.MustRegister(requestsIncoming)
+	prometheus.MustRegister(requestsHandled)
+	prometheus.MustRegister(requestsDropped)
+	prometheus.MustRegister(requestsHandlingSec)
+}
+
+func reportIncomingRequest(request *http.Request) {
+	requestsIncoming.WithLabelValues(request.Method).Inc()
+}
+
+func reportRequestHandled(request *http.Request, response *http.Response, startTime time.Time) {
+	method := request.Method
+	requestsHandled.WithLabelValues(method, strconv.Itoa(response.StatusCode)).Inc()
+	requestsHandlingSec.WithLabelValues(method).Observe(time.Since(startTime).Seconds())
+}
+
+func reportRequestDropped(request *http.Request, err forwardingError) {
+	requestsDropped.WithLabelValues(request.Method, string(err)).Inc()
+}

server/proxy/httpproxy/proxy.go

@@ -0,0 +1,121 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	"go.uber.org/zap"
+	"golang.org/x/net/http2"
+)
+
+const (
+	// DefaultMaxIdleConnsPerHost indicates the default maximum idle connection
+	// count maintained between proxy and each member. We set it to 128 to
+	// let proxy handle 128 concurrent requests in long term smoothly.
+	// If the number of concurrent requests is bigger than this value,
+	// proxy needs to create one new connection when handling each request in
+	// the delta, which is bad because the creation consumes resource and
+	// may eat up ephemeral ports.
+	DefaultMaxIdleConnsPerHost = 128
+)
+
+// GetProxyURLs is a function which should return the current set of URLs to
+// which client requests should be proxied. This function will be queried
+// periodically by the proxy Handler to refresh the set of available
+// backends.
+type GetProxyURLs func() []string
+
+// NewHandler creates a new HTTP handler, listening on the given transport,
+// which will proxy requests to an etcd cluster.
+// The handler will periodically update its view of the cluster.
+func NewHandler(lg *zap.Logger, t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler {
+	if lg == nil {
+		lg = zap.NewNop()
+	}
+	if t.TLSClientConfig != nil {
+		// Enable http2, see Issue 5033.
+		err := http2.ConfigureTransport(t)
+		if err != nil {
+			lg.Info("Error enabling Transport HTTP/2 support", zap.Error(err))
+		}
+	}
+
+	p := &reverseProxy{
+		lg:        lg,
+		director:  newDirector(lg, urlsFunc, failureWait, refreshInterval),
+		transport: t,
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle("/", p)
+	mux.HandleFunc("/v2/config/local/proxy", p.configHandler)
+
+	return mux
+}
+
+// NewReadonlyHandler wraps the given HTTP handler to allow only GET requests
+func NewReadonlyHandler(hdlr http.Handler) http.Handler {
+	readonly := readonlyHandlerFunc(hdlr)
+	return http.HandlerFunc(readonly)
+}
+
+func readonlyHandlerFunc(next http.Handler) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		if req.Method != "GET" {
+			w.WriteHeader(http.StatusNotImplemented)
+			return
+		}
+
+		next.ServeHTTP(w, req)
+	}
+}
+
+func (p *reverseProxy) configHandler(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET") {
+		return
+	}
+
+	eps := p.director.endpoints()
+	epstr := make([]string, len(eps))
+	for i, e := range eps {
+		epstr[i] = e.URL.String()
+	}
+
+	proxyConfig := struct {
+		Endpoints []string `json:"endpoints"`
+	}{
+		Endpoints: epstr,
+	}
+
+	json.NewEncoder(w).Encode(proxyConfig)
+}
+
+// allowMethod verifies that the given method is one of the allowed methods,
+// and if not, it writes an error to w.  A boolean is returned indicating
+// whether or not the method is allowed.
+func allowMethod(w http.ResponseWriter, m string, ms ...string) bool {
+	for _, meth := range ms {
+		if m == meth {
+			return true
+		}
+	}
+	w.Header().Set("Allow", strings.Join(ms, ","))
+	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+	return false
+}

server/proxy/httpproxy/proxy_test.go

@@ -0,0 +1,103 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+func TestReadonlyHandler(t *testing.T) {
+	fixture := func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}
+	hdlrFunc := readonlyHandlerFunc(http.HandlerFunc(fixture))
+
+	tests := []struct {
+		method string
+		want   int
+	}{
+		// GET is only passing method
+		{"GET", http.StatusOK},
+
+		// everything but GET is StatusNotImplemented
+		{"POST", http.StatusNotImplemented},
+		{"PUT", http.StatusNotImplemented},
+		{"PATCH", http.StatusNotImplemented},
+		{"DELETE", http.StatusNotImplemented},
+		{"FOO", http.StatusNotImplemented},
+	}
+
+	for i, tt := range tests {
+		req, _ := http.NewRequest(tt.method, "http://example.com", nil)
+		rr := httptest.NewRecorder()
+		hdlrFunc(rr, req)
+
+		if tt.want != rr.Code {
+			t.Errorf("#%d: incorrect HTTP status code: method=%s want=%d got=%d", i, tt.method, tt.want, rr.Code)
+		}
+	}
+}
+
+func TestConfigHandlerGET(t *testing.T) {
+	var err error
+	us := make([]*url.URL, 3)
+	us[0], err = url.Parse("http://example1.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	us[1], err = url.Parse("http://example2.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	us[2], err = url.Parse("http://example3.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lg := zap.NewExample()
+	rp := reverseProxy{
+		lg: lg,
+		director: &director{
+			lg: lg,
+			ep: []*endpoint{
+				newEndpoint(lg, *us[0], 1*time.Second),
+				newEndpoint(lg, *us[1], 1*time.Second),
+				newEndpoint(lg, *us[2], 1*time.Second),
+			},
+		},
+	}
+
+	req, _ := http.NewRequest("GET", "http://example.com//v2/config/local/proxy", nil)
+	rr := httptest.NewRecorder()
+	rp.configHandler(rr, req)
+
+	wbody := "{\"endpoints\":[\"http://example1.com\",\"http://example2.com\",\"http://example3.com\"]}\n"
+
+	body, err := ioutil.ReadAll(rr.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(body) != wbody {
+		t.Errorf("body = %s, want %s", string(body), wbody)
+	}
+}

server/proxy/httpproxy/reverse.go

@@ -0,0 +1,227 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"go.etcd.io/etcd/v3/etcdserver/api/v2http/httptypes"
+
+	"go.uber.org/zap"
+)
+
+var (
+	// Hop-by-hop headers. These are removed when sent to the backend.
+	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
+	// This list of headers borrowed from stdlib httputil.ReverseProxy
+	singleHopHeaders = []string{
+		"Connection",
+		"Keep-Alive",
+		"Proxy-Authenticate",
+		"Proxy-Authorization",
+		"Te", // canonicalized version of "TE"
+		"Trailers",
+		"Transfer-Encoding",
+		"Upgrade",
+	}
+)
+
+func removeSingleHopHeaders(hdrs *http.Header) {
+	for _, h := range singleHopHeaders {
+		hdrs.Del(h)
+	}
+}
+
+type reverseProxy struct {
+	lg        *zap.Logger
+	director  *director
+	transport http.RoundTripper
+}
+
+func (p *reverseProxy) ServeHTTP(rw http.ResponseWriter, clientreq *http.Request) {
+	reportIncomingRequest(clientreq)
+	proxyreq := new(http.Request)
+	*proxyreq = *clientreq
+	startTime := time.Now()
+
+	var (
+		proxybody []byte
+		err       error
+	)
+
+	if clientreq.Body != nil {
+		proxybody, err = ioutil.ReadAll(clientreq.Body)
+		if err != nil {
+			msg := fmt.Sprintf("failed to read request body: %v", err)
+			p.lg.Info("failed to read request body", zap.Error(err))
+			e := httptypes.NewHTTPError(http.StatusInternalServerError, "httpproxy: "+msg)
+			if we := e.WriteTo(rw); we != nil {
+				p.lg.Debug(
+					"error writing HTTPError to remote addr",
+					zap.String("remote-addr", clientreq.RemoteAddr),
+					zap.Error(we),
+				)
+			}
+			return
+		}
+	}
+
+	// deep-copy the headers, as these will be modified below
+	proxyreq.Header = make(http.Header)
+	copyHeader(proxyreq.Header, clientreq.Header)
+
+	normalizeRequest(proxyreq)
+	removeSingleHopHeaders(&proxyreq.Header)
+	maybeSetForwardedFor(proxyreq)
+
+	endpoints := p.director.endpoints()
+	if len(endpoints) == 0 {
+		msg := "zero endpoints currently available"
+		reportRequestDropped(clientreq, zeroEndpoints)
+
+		// TODO: limit the rate of the error logging.
+		p.lg.Info(msg)
+		e := httptypes.NewHTTPError(http.StatusServiceUnavailable, "httpproxy: "+msg)
+		if we := e.WriteTo(rw); we != nil {
+			p.lg.Debug(
+				"error writing HTTPError to remote addr",
+				zap.String("remote-addr", clientreq.RemoteAddr),
+				zap.Error(we),
+			)
+		}
+		return
+	}
+
+	var requestClosed int32
+	completeCh := make(chan bool, 1)
+	closeNotifier, ok := rw.(http.CloseNotifier)
+	ctx, cancel := context.WithCancel(context.Background())
+	proxyreq = proxyreq.WithContext(ctx)
+	defer cancel()
+	if ok {
+		closeCh := closeNotifier.CloseNotify()
+		go func() {
+			select {
+			case <-closeCh:
+				atomic.StoreInt32(&requestClosed, 1)
+				p.lg.Info(
+					"client closed request prematurely",
+					zap.String("remote-addr", clientreq.RemoteAddr),
+				)
+				cancel()
+			case <-completeCh:
+			}
+		}()
+
+		defer func() {
+			completeCh <- true
+		}()
+	}
+
+	var res *http.Response
+
+	for _, ep := range endpoints {
+		if proxybody != nil {
+			proxyreq.Body = ioutil.NopCloser(bytes.NewBuffer(proxybody))
+		}
+		redirectRequest(proxyreq, ep.URL)
+
+		res, err = p.transport.RoundTrip(proxyreq)
+		if atomic.LoadInt32(&requestClosed) == 1 {
+			return
+		}
+		if err != nil {
+			reportRequestDropped(clientreq, failedSendingRequest)
+			p.lg.Info(
+				"failed to direct request",
+				zap.String("url", ep.URL.String()),
+				zap.Error(err),
+			)
+			ep.Failed()
+			continue
+		}
+
+		break
+	}
+
+	if res == nil {
+		// TODO: limit the rate of the error logging.
+		msg := fmt.Sprintf("unable to get response from %d endpoint(s)", len(endpoints))
+		reportRequestDropped(clientreq, failedGettingResponse)
+		p.lg.Info(msg)
+		e := httptypes.NewHTTPError(http.StatusBadGateway, "httpproxy: "+msg)
+		if we := e.WriteTo(rw); we != nil {
+			p.lg.Debug(
+				"error writing HTTPError to remote addr",
+				zap.String("remote-addr", clientreq.RemoteAddr),
+				zap.Error(we),
+			)
+		}
+		return
+	}
+
+	defer res.Body.Close()
+	reportRequestHandled(clientreq, res, startTime)
+	removeSingleHopHeaders(&res.Header)
+	copyHeader(rw.Header(), res.Header)
+
+	rw.WriteHeader(res.StatusCode)
+	io.Copy(rw, res.Body)
+}
+
+func copyHeader(dst, src http.Header) {
+	for k, vv := range src {
+		for _, v := range vv {
+			dst.Add(k, v)
+		}
+	}
+}
+
+func redirectRequest(req *http.Request, loc url.URL) {
+	req.URL.Scheme = loc.Scheme
+	req.URL.Host = loc.Host
+}
+
+func normalizeRequest(req *http.Request) {
+	req.Proto = "HTTP/1.1"
+	req.ProtoMajor = 1
+	req.ProtoMinor = 1
+	req.Close = false
+}
+
+func maybeSetForwardedFor(req *http.Request) {
+	clientIP, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		return
+	}
+
+	// If we aren't the first proxy retain prior
+	// X-Forwarded-For information as a comma+space
+	// separated list and fold multiple headers into one.
+	if prior, ok := req.Header["X-Forwarded-For"]; ok {
+		clientIP = strings.Join(prior, ", ") + ", " + clientIP
+	}
+	req.Header.Set("X-Forwarded-For", clientIP)
+}

server/proxy/httpproxy/reverse_test.go

@@ -0,0 +1,249 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpproxy
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"go.uber.org/zap"
+)
+
+type staticRoundTripper struct {
+	res *http.Response
+	err error
+}
+
+func (srt *staticRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
+	return srt.res, srt.err
+}
+
+func TestReverseProxyServe(t *testing.T) {
+	u := url.URL{Scheme: "http", Host: "192.0.2.3:4040"}
+	lg := zap.NewExample()
+
+	tests := []struct {
+		eps  []*endpoint
+		rt   http.RoundTripper
+		want int
+	}{
+		// no endpoints available so no requests are even made
+		{
+			eps: []*endpoint{},
+			rt: &staticRoundTripper{
+				res: &http.Response{
+					StatusCode: http.StatusCreated,
+					Body:       ioutil.NopCloser(&bytes.Reader{}),
+				},
+			},
+			want: http.StatusServiceUnavailable,
+		},
+
+		// error is returned from one endpoint that should be available
+		{
+			eps:  []*endpoint{{URL: u, Available: true}},
+			rt:   &staticRoundTripper{err: errors.New("what a bad trip")},
+			want: http.StatusBadGateway,
+		},
+
+		// endpoint is available and returns success
+		{
+			eps: []*endpoint{{URL: u, Available: true}},
+			rt: &staticRoundTripper{
+				res: &http.Response{
+					StatusCode: http.StatusCreated,
+					Body:       ioutil.NopCloser(&bytes.Reader{}),
+					Header:     map[string][]string{"Content-Type": {"application/json"}},
+				},
+			},
+			want: http.StatusCreated,
+		},
+	}
+
+	for i, tt := range tests {
+		rp := reverseProxy{
+			lg:        lg,
+			director:  &director{lg: lg, ep: tt.eps},
+			transport: tt.rt,
+		}
+
+		req, _ := http.NewRequest("GET", "http://192.0.2.2:2379", nil)
+		rr := httptest.NewRecorder()
+		rp.ServeHTTP(rr, req)
+
+		if rr.Code != tt.want {
+			t.Errorf("#%d: unexpected HTTP status code: want = %d, got = %d", i, tt.want, rr.Code)
+		}
+		if gct := rr.Header().Get("Content-Type"); gct != "application/json" {
+			t.Errorf("#%d: Content-Type = %s, want %s", i, gct, "application/json")
+		}
+	}
+}
+
+func TestRedirectRequest(t *testing.T) {
+	loc := url.URL{
+		Scheme: "http",
+		Host:   "bar.example.com",
+	}
+
+	req := &http.Request{
+		Method: "GET",
+		Host:   "foo.example.com",
+		URL: &url.URL{
+			Host: "foo.example.com",
+			Path: "/v2/keys/baz",
+		},
+	}
+
+	redirectRequest(req, loc)
+
+	want := &http.Request{
+		Method: "GET",
+		// this field must not change
+		Host: "foo.example.com",
+		URL: &url.URL{
+			// the Scheme field is updated to that of the provided URL
+			Scheme: "http",
+			// the Host field is updated to that of the provided URL
+			Host: "bar.example.com",
+			Path: "/v2/keys/baz",
+		},
+	}
+
+	if !reflect.DeepEqual(want, req) {
+		t.Fatalf("HTTP request does not match expected criteria: want=%#v got=%#v", want, req)
+	}
+}
+
+func TestMaybeSetForwardedFor(t *testing.T) {
+	tests := []struct {
+		raddr  string
+		fwdFor string
+		want   string
+	}{
+		{"192.0.2.3:8002", "", "192.0.2.3"},
+		{"192.0.2.3:8002", "192.0.2.2", "192.0.2.2, 192.0.2.3"},
+		{"192.0.2.3:8002", "192.0.2.1, 192.0.2.2", "192.0.2.1, 192.0.2.2, 192.0.2.3"},
+		{"example.com:8002", "", "example.com"},
+
+		// While these cases look valid, golang net/http will not let it happen
+		// The RemoteAddr field will always be a valid host:port
+		{":8002", "", ""},
+		{"192.0.2.3", "", ""},
+
+		// blatantly invalid host w/o a port
+		{"12", "", ""},
+		{"12", "192.0.2.3", "192.0.2.3"},
+	}
+
+	for i, tt := range tests {
+		req := &http.Request{
+			RemoteAddr: tt.raddr,
+			Header:     make(http.Header),
+		}
+
+		if tt.fwdFor != "" {
+			req.Header.Set("X-Forwarded-For", tt.fwdFor)
+		}
+
+		maybeSetForwardedFor(req)
+		got := req.Header.Get("X-Forwarded-For")
+		if tt.want != got {
+			t.Errorf("#%d: incorrect header: want = %q, got = %q", i, tt.want, got)
+		}
+	}
+}
+
+func TestRemoveSingleHopHeaders(t *testing.T) {
+	hdr := http.Header(map[string][]string{
+		// single-hop headers that should be removed
+		"Connection":          {"close"},
+		"Keep-Alive":          {"foo"},
+		"Proxy-Authenticate":  {"Basic realm=example.com"},
+		"Proxy-Authorization": {"foo"},
+		"Te":                  {"deflate,gzip"},
+		"Trailers":            {"ETag"},
+		"Transfer-Encoding":   {"chunked"},
+		"Upgrade":             {"WebSocket"},
+
+		// headers that should persist
+		"Accept": {"application/json"},
+		"X-Foo":  {"Bar"},
+	})
+
+	removeSingleHopHeaders(&hdr)
+
+	want := http.Header(map[string][]string{
+		"Accept": {"application/json"},
+		"X-Foo":  {"Bar"},
+	})
+
+	if !reflect.DeepEqual(want, hdr) {
+		t.Fatalf("unexpected result: want = %#v, got = %#v", want, hdr)
+	}
+}
+
+func TestCopyHeader(t *testing.T) {
+	tests := []struct {
+		src  http.Header
+		dst  http.Header
+		want http.Header
+	}{
+		{
+			src: http.Header(map[string][]string{
+				"Foo": {"bar", "baz"},
+			}),
+			dst: http.Header(map[string][]string{}),
+			want: http.Header(map[string][]string{
+				"Foo": {"bar", "baz"},
+			}),
+		},
+		{
+			src: http.Header(map[string][]string{
+				"Foo":  {"bar"},
+				"Ping": {"pong"},
+			}),
+			dst: http.Header(map[string][]string{}),
+			want: http.Header(map[string][]string{
+				"Foo":  {"bar"},
+				"Ping": {"pong"},
+			}),
+		},
+		{
+			src: http.Header(map[string][]string{
+				"Foo": {"bar", "baz"},
+			}),
+			dst: http.Header(map[string][]string{
+				"Foo": {"qux"},
+			}),
+			want: http.Header(map[string][]string{
+				"Foo": {"qux", "bar", "baz"},
+			}),
+		},
+	}
+
+	for i, tt := range tests {
+		copyHeader(tt.dst, tt.src)
+		if !reflect.DeepEqual(tt.dst, tt.want) {
+			t.Errorf("#%d: unexpected headers: want = %v, got = %v", i, tt.want, tt.dst)
+		}
+	}
+}