Merge branch 'master' of https://github.com/coreos/etcd into proxy

Conflicts:
	config/config.go
	server/peer_server.go
	server/transporter.go
	tests/server_utils.go

server/transporter.go

@@ -11,6 +11,11 @@ import (
 
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/third_party/github.com/goraft/raft"
+	httpclient "github.com/coreos/etcd/third_party/github.com/mreiferson/go-httpclient"
+)
+
+const (
+	snapshotTimeout = time.Second * 120
 )
 
 // Transporter layer for communication between raft nodes
@@ -20,8 +25,10 @@ type transporter struct {
 	serverStats    *raftServerStats
 	registry       *Registry
 
-	client    *http.Client
-	transport *http.Transport
+	client            *http.Client
+	transport         *httpclient.Transport
+	snapshotClient    *http.Client
+	snapshotTransport *httpclient.Transport
 }
 
 type dialer func(network, addr string) (net.Conn, error)
@@ -30,20 +37,38 @@ type dialer func(network, addr string) (net.Conn, error)
 // Create http or https transporter based on
 // whether the user give the server cert and key
 func NewTransporter(followersStats *raftFollowersStats, serverStats *raftServerStats, registry *Registry, dialTimeout, requestTimeout, responseHeaderTimeout time.Duration) *transporter {
-	tr := &http.Transport{
-		Dial: func(network, addr string) (net.Conn, error) {
-			return net.DialTimeout(network, addr, dialTimeout)
-		},
+	tr := &httpclient.Transport{
 		ResponseHeaderTimeout: responseHeaderTimeout,
+		// This is a workaround for Transport.CancelRequest doesn't work on
+		// HTTPS connections blocked. The patch for it is in progress,
+		// and would be available in Go1.3
+		// More: https://codereview.appspot.com/69280043/
+		ConnectTimeout:   dialTimeout,
+		RequestTimeout:   dialTimeout + responseHeaderTimeout,
+		ReadWriteTimeout: responseHeaderTimeout,
+	}
+
+	// Sending snapshot might take a long time so we use a different HTTP transporter
+	// Timeout is set to 120s (Around 100MB if the bandwidth is 10Mbits/s)
+	// This timeout is not calculated by heartbeat time.
+	// TODO(xiangl1) we can actually calculate the max bandwidth if we know
+	// average RTT.
+	// It should be equal to (TCP max window size/RTT).
+	sTr := &httpclient.Transport{
+		ConnectTimeout:   dialTimeout,
+		RequestTimeout:   snapshotTimeout,
+		ReadWriteTimeout: snapshotTimeout,
 	}
 
 	t := transporter{
-		client:         &http.Client{Transport: tr},
-		transport:      tr,
-		requestTimeout: requestTimeout,
-		followersStats: followersStats,
-		serverStats:    serverStats,
-		registry:       registry,
+		client:            &http.Client{Transport: tr},
+		transport:         tr,
+		snapshotClient:    &http.Client{Transport: sTr},
+		snapshotTransport: sTr,
+		requestTimeout:    requestTimeout,
+		followersStats:    followersStats,
+		serverStats:       serverStats,
+		registry:          registry,
 	}
 
 	return &t
@@ -52,6 +77,9 @@ func NewTransporter(followersStats *raftFollowersStats, serverStats *raftServerS
 func (t *transporter) SetTLSConfig(tlsConf tls.Config) {
 	t.transport.TLSClientConfig = &tlsConf
 	t.transport.DisableCompression = true
+
+	t.snapshotTransport.TLSClientConfig = &tlsConf
+	t.snapshotTransport.DisableCompression = true
 }
 
 // Sends AppendEntries RPCs to a peer when the server is the leader.
@@ -81,7 +109,7 @@ func (t *transporter) SendAppendEntriesRequest(server raft.Server, peer *raft.Pe
 
 	start := time.Now()
 
-	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/log/append", u), &b)
+	resp, _, err := t.Post(fmt.Sprintf("%s/log/append", u), &b)
 
 	end := time.Now()
 
@@ -100,8 +128,6 @@ func (t *transporter) SendAppendEntriesRequest(server raft.Server, peer *raft.Pe
 	if resp != nil {
 		defer resp.Body.Close()
 
-		t.CancelWhenTimeout(httpRequest)
-
 		aeresp := &raft.AppendEntriesResponse{}
 		if _, err = aeresp.Decode(resp.Body); err != nil && err != io.EOF {
 			log.Warn("transporter.ae.decoding.error:", err)
@@ -125,7 +151,7 @@ func (t *transporter) SendVoteRequest(server raft.Server, peer *raft.Peer, req *
 	u, _ := t.registry.PeerURL(peer.Name)
 	log.Debugf("Send Vote from %s to %s", server.Name(), u)
 
-	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/vote", u), &b)
+	resp, _, err := t.Post(fmt.Sprintf("%s/vote", u), &b)
 
 	if err != nil {
 		log.Debugf("Cannot send VoteRequest to %s : %s", u, err)
@@ -134,8 +160,6 @@ func (t *transporter) SendVoteRequest(server raft.Server, peer *raft.Peer, req *
 	if resp != nil {
 		defer resp.Body.Close()
 
-		t.CancelWhenTimeout(httpRequest)
-
 		rvrsp := &raft.RequestVoteResponse{}
 		if _, err = rvrsp.Decode(resp.Body); err != nil && err != io.EOF {
 			log.Warn("transporter.vr.decoding.error:", err)
@@ -158,7 +182,7 @@ func (t *transporter) SendSnapshotRequest(server raft.Server, peer *raft.Peer, r
 	u, _ := t.registry.PeerURL(peer.Name)
 	log.Debugf("Send Snapshot Request from %s to %s", server.Name(), u)
 
-	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/snapshot", u), &b)
+	resp, _, err := t.Post(fmt.Sprintf("%s/snapshot", u), &b)
 
 	if err != nil {
 		log.Debugf("Cannot send Snapshot Request to %s : %s", u, err)
@@ -167,8 +191,6 @@ func (t *transporter) SendSnapshotRequest(server raft.Server, peer *raft.Peer, r
 	if resp != nil {
 		defer resp.Body.Close()
 
-		t.CancelWhenTimeout(httpRequest)
-
 		ssrsp := &raft.SnapshotResponse{}
 		if _, err = ssrsp.Decode(resp.Body); err != nil && err != io.EOF {
 			log.Warn("transporter.ss.decoding.error:", err)
@@ -191,7 +213,7 @@ func (t *transporter) SendSnapshotRecoveryRequest(server raft.Server, peer *raft
 	u, _ := t.registry.PeerURL(peer.Name)
 	log.Debugf("Send Snapshot Recovery from %s to %s", server.Name(), u)
 
-	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/snapshotRecovery", u), &b)
+	resp, err := t.PostSnapshot(fmt.Sprintf("%s/snapshotRecovery", u), &b)
 
 	if err != nil {
 		log.Debugf("Cannot send Snapshot Recovery to %s : %s", u, err)
@@ -200,8 +222,6 @@ func (t *transporter) SendSnapshotRecoveryRequest(server raft.Server, peer *raft
 	if resp != nil {
 		defer resp.Body.Close()
 
-		t.CancelWhenTimeout(httpRequest)
-
 		ssrrsp := &raft.SnapshotRecoveryResponse{}
 		if _, err = ssrrsp.Decode(resp.Body); err != nil && err != io.EOF {
 			log.Warn("transporter.ssr.decoding.error:", err)
@@ -227,10 +247,8 @@ func (t *transporter) Get(urlStr string) (*http.Response, *http.Request, error)
 	return resp, req, err
 }
 
-// Cancel the on fly HTTP transaction when timeout happens.
-func (t *transporter) CancelWhenTimeout(req *http.Request) {
-	go func() {
-		time.Sleep(t.requestTimeout)
-		t.transport.CancelRequest(req)
-	}()
+// PostSnapshot posts a json format snapshot to the given url
+// The underlying HTTP transport has a minute level timeout
+func (t *transporter) PostSnapshot(url string, body io.Reader) (*http.Response, error) {
+	return t.snapshotClient.Post(url, "application/json", body)
 }