From 1264dbe24d2d1612ae0da05e31ee978b104c2188 Mon Sep 17 00:00:00 2001 From: Wolfgang Ebner Date: Fri, 12 Jun 2015 20:35:38 +0200 Subject: [PATCH 1/4] proxy: added endpoint refresh and timeout configuration values the default dial timeout was set to 30 seconds this made the proxy a pain to use in failure scenarios. fixes 2862 --- Documentation/configuration.md | 21 +++++++++++++++++++++ etcdmain/config.go | 12 +++++++++++- etcdmain/etcd.go | 6 +++--- etcdmain/help.go | 10 ++++++++++ proxy/director.go | 29 ++++++++++++----------------- proxy/director_test.go | 3 ++- proxy/proxy.go | 5 +++-- 7 files changed, 62 insertions(+), 24 deletions(-) diff --git a/Documentation/configuration.md b/Documentation/configuration.md index f55dc1d2e..fa90173e1 100644 --- a/Documentation/configuration.md +++ b/Documentation/configuration.md @@ -105,6 +105,27 @@ To start etcd automatically using custom settings at startup in Linux, using a [ + Proxy mode setting ("off", "readonly" or "on"). + default: "off" +##### -proxy-failure-wait ++ Time (in milliseconds) an endpoint will be held in a failed state. ++ default: 5000 + +##### -proxy-refresh-interval ++ Time (in milliseconds) of the endpoints refresh interval. ++ default: 30000 + +##### -proxy-dial-timeout ++ Time (in milliseconds) for a dial to timeout. ++ default: 1000 + +##### -proxy-write-timeout ++ Time (in milliseconds) for a write to timeout. ++ default: 5000 + +##### -proxy-read-timeout ++ Time (in milliseconds) for a read to timeout. ++ Don't change this value if you use watches because they are using long polling requests. ++ default: 0 + ### Security Flags The security flags help to [build a secure etcd cluster][security]. diff --git a/etcdmain/config.go b/etcdmain/config.go index 8069e637a..eb00a5532 100644 --- a/etcdmain/config.go +++ b/etcdmain/config.go @@ -92,7 +92,12 @@ type config struct { initialClusterToken string // proxy - proxy *flags.StringsFlag + proxy *flags.StringsFlag + proxyFailureWaitMs uint + proxyRefreshIntervalMs uint + proxyDialTimeoutMs uint + proxyWriteTimeoutMs uint + proxyReadTimeoutMs uint // security clientTLSInfo, peerTLSInfo transport.TLSInfo @@ -172,6 +177,11 @@ func NewConfig() *config { // Should never happen. plog.Panicf("unexpected error setting up proxyFlag: %v", err) } + fs.UintVar(&cfg.proxyFailureWaitMs, "proxy-failure-wait", 5000, "Time (in milliseconds) an endpoint will be held in a failed state.") + fs.UintVar(&cfg.proxyRefreshIntervalMs, "proxy-refresh-interval", 30000, "Time (in milliseconds) of the endpoints refresh interval.") + fs.UintVar(&cfg.proxyDialTimeoutMs, "proxy-dial-timeout", 1000, "Time (in milliseconds) for a dial to timeout.") + fs.UintVar(&cfg.proxyWriteTimeoutMs, "proxy-write-timeout", 5000, "Time (in milliseconds) for a write to timeout.") + fs.UintVar(&cfg.proxyReadTimeoutMs, "proxy-read-timeout", 0, "Time (in milliseconds) for a read to timeout.") // security fs.StringVar(&cfg.clientTLSInfo.CAFile, "ca-file", "", "DEPRECATED: Path to the client server TLS CA file.") diff --git a/etcdmain/etcd.go b/etcdmain/etcd.go index c4619492c..e8fcd7dd7 100644 --- a/etcdmain/etcd.go +++ b/etcdmain/etcd.go @@ -248,13 +248,13 @@ func startProxy(cfg *config) error { } } - pt, err := transport.NewTransport(cfg.clientTLSInfo) + pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Duration(cfg.proxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyWriteTimeoutMs)*time.Millisecond) pt.MaxIdleConnsPerHost = proxy.DefaultMaxIdleConnsPerHost if err != nil { return err } - tr, err := transport.NewTransport(cfg.peerTLSInfo) + tr, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, time.Duration(cfg.proxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.proxyWriteTimeoutMs)*time.Millisecond) if err != nil { return err } @@ -323,7 +323,7 @@ func startProxy(cfg *config) error { return clientURLs } - ph := proxy.NewHandler(pt, uf) + ph := proxy.NewHandler(pt, uf, time.Duration(cfg.proxyFailureWaitMs)*time.Millisecond, time.Duration(cfg.proxyRefreshIntervalMs)*time.Millisecond) ph = &cors.CORSHandler{ Handler: ph, Info: cfg.corsInfo, diff --git a/etcdmain/help.go b/etcdmain/help.go index 5e0aa863d..b8a3095d8 100644 --- a/etcdmain/help.go +++ b/etcdmain/help.go @@ -72,6 +72,16 @@ proxy flags: --proxy 'off' proxy mode setting ('off', 'readonly' or 'on'). + --proxy-failure-wait 5000 + time (in milliseconds) an endpoint will be held in a failed state. + --proxy-refresh-interval 30000 + time (in milliseconds) of the endpoints refresh interval. + --proxy-dial-timeout 1000 + time (in milliseconds) for a dial to timeout. + --proxy-write-timeout 5000 + time (in milliseconds) for a write to timeout. + --proxy-read-timeout 0 + time (in milliseconds) for a read to timeout. security flags: diff --git a/proxy/director.go b/proxy/director.go index e795b27fb..429698e75 100644 --- a/proxy/director.go +++ b/proxy/director.go @@ -22,24 +22,17 @@ import ( "time" ) -const ( - // amount of time an endpoint will be held in a failed - // state before being reconsidered for proxied requests - endpointFailureWait = 5 * time.Second - - // how often the proxy will attempt to refresh its set of endpoints - refreshEndpoints = 30 * time.Second -) - -func newDirector(urlsFunc GetProxyURLs) *director { +func newDirector(urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) *director { d := &director{ - uf: urlsFunc, + uf: urlsFunc, + failureWait: failureWait, + refreshInterval: refreshInterval, } d.refresh() go func() { for { select { - case <-time.After(refreshEndpoints): + case <-time.After(refreshInterval): d.refresh() } } @@ -49,8 +42,10 @@ func newDirector(urlsFunc GetProxyURLs) *director { type director struct { sync.Mutex - ep []*endpoint - uf GetProxyURLs + ep []*endpoint + uf GetProxyURLs + failureWait time.Duration + refreshInterval time.Duration } func (d *director) refresh() { @@ -64,7 +59,7 @@ func (d *director) refresh() { log.Printf("proxy: upstream URL invalid: %v", err) continue } - endpoints = append(endpoints, newEndpoint(*uu)) + endpoints = append(endpoints, newEndpoint(*uu, d.failureWait)) } // shuffle array to avoid connections being "stuck" to a single endpoint @@ -89,11 +84,11 @@ func (d *director) endpoints() []*endpoint { return filtered } -func newEndpoint(u url.URL) *endpoint { +func newEndpoint(u url.URL, failureWait time.Duration) *endpoint { ep := endpoint{ URL: u, Available: true, - failFunc: timedUnavailabilityFunc(endpointFailureWait), + failFunc: timedUnavailabilityFunc(failureWait), } return &ep diff --git a/proxy/director_test.go b/proxy/director_test.go index 6287091bf..0d56ea961 100644 --- a/proxy/director_test.go +++ b/proxy/director_test.go @@ -19,6 +19,7 @@ import ( "reflect" "sort" "testing" + "time" ) func TestNewDirectorScheme(t *testing.T) { @@ -52,7 +53,7 @@ func TestNewDirectorScheme(t *testing.T) { uf := func() []string { return tt.urls } - got := newDirector(uf) + got := newDirector(uf, time.Minute, time.Minute) var gep []string for _, ep := range got.ep { diff --git a/proxy/proxy.go b/proxy/proxy.go index 295fede53..f28102f78 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -16,6 +16,7 @@ package proxy import ( "net/http" + "time" ) const ( @@ -38,9 +39,9 @@ type GetProxyURLs func() []string // NewHandler creates a new HTTP handler, listening on the given transport, // which will proxy requests to an etcd cluster. // The handler will periodically update its view of the cluster. -func NewHandler(t *http.Transport, urlsFunc GetProxyURLs) http.Handler { +func NewHandler(t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler { return &reverseProxy{ - director: newDirector(urlsFunc), + director: newDirector(urlsFunc, failureWait, refreshInterval), transport: t, } } From 5854d0e8a95a8ab746b145910b1b6637b5d8b757 Mon Sep 17 00:00:00 2001 From: Wolfgang Ebner Date: Tue, 16 Jun 2015 12:17:08 +0200 Subject: [PATCH 2/4] proxy: removed unused refreshInterval variable in director structure --- proxy/director.go | 1 - 1 file changed, 1 deletion(-) diff --git a/proxy/director.go b/proxy/director.go index 429698e75..9a91c723a 100644 --- a/proxy/director.go +++ b/proxy/director.go @@ -26,7 +26,6 @@ func newDirector(urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterv d := &director{ uf: urlsFunc, failureWait: failureWait, - refreshInterval: refreshInterval, } d.refresh() go func() { From aeeae25d87eff68fdfde8a792937708f0286147b Mon Sep 17 00:00:00 2001 From: Wolfgang Ebner Date: Tue, 16 Jun 2015 12:18:16 +0200 Subject: [PATCH 3/4] proxy: documentation for disabling the proxy timeout --- Documentation/configuration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/configuration.md b/Documentation/configuration.md index fa90173e1..812fedf3b 100644 --- a/Documentation/configuration.md +++ b/Documentation/configuration.md @@ -106,7 +106,7 @@ To start etcd automatically using custom settings at startup in Linux, using a [ + default: "off" ##### -proxy-failure-wait -+ Time (in milliseconds) an endpoint will be held in a failed state. ++ Time (in milliseconds) an endpoint will be held in a failed state before being reconsidered for proxied requests. + default: 5000 ##### -proxy-refresh-interval @@ -114,15 +114,15 @@ To start etcd automatically using custom settings at startup in Linux, using a [ + default: 30000 ##### -proxy-dial-timeout -+ Time (in milliseconds) for a dial to timeout. ++ Time (in milliseconds) for a dial to timeout or 0 to disable the timeout + default: 1000 ##### -proxy-write-timeout -+ Time (in milliseconds) for a write to timeout. ++ Time (in milliseconds) for a write to timeout or 0 to disable the timeout. + default: 5000 ##### -proxy-read-timeout -+ Time (in milliseconds) for a read to timeout. ++ Time (in milliseconds) for a read to timeout or 0 to disable the timeout. + Don't change this value if you use watches because they are using long polling requests. + default: 0 From 954e416bf6ea0b1e08bb6fd8f61fe51b46ae2204 Mon Sep 17 00:00:00 2001 From: Wolfgang Ebner Date: Fri, 3 Jul 2015 14:11:40 +0200 Subject: [PATCH 4/4] proxy: fixed director.go formatting --- proxy/director.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/director.go b/proxy/director.go index 9a91c723a..0cfe37d89 100644 --- a/proxy/director.go +++ b/proxy/director.go @@ -24,8 +24,8 @@ import ( func newDirector(urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) *director { d := &director{ - uf: urlsFunc, - failureWait: failureWait, + uf: urlsFunc, + failureWait: failureWait, } d.refresh() go func() {