client: return cluster error if the etcd cluster is not avaliable

Add a new ClusterError type. It contians all encountered errors and
return ClusterNotAvailable as the error string.

Conflicts:
	client/client.go
	discovery/discovery.go

client/client.go

@@ -30,6 +30,7 @@ import (
 var (
 	ErrNoEndpoints           = errors.New("client: no endpoints available")
 	ErrTooManyRedirects      = errors.New("client: too many redirects")
+	ErrClusterUnavailable    = errors.New("client: etcd cluster is unavailable or misconfigured")
 	errTooManyRedirectChecks = errors.New("client: too many redirect checks")
 )
 
@@ -223,23 +224,27 @@ func (c *httpClusterClient) Do(ctx context.Context, act httpAction) (*http.Respo
 	var resp *http.Response
 	var body []byte
 	var err error
+	cerr := &ClusterError{}
 
 	for _, ep := range eps {
 		hc := c.clientFactory(ep)
 		resp, body, err = hc.Do(ctx, action)
 		if err != nil {
+			cerr.Errors = append(cerr.Errors, err)
 			if err == context.DeadlineExceeded || err == context.Canceled {
-				return nil, nil, err
+				return nil, nil, cerr
 			}
 			continue
 		}
 		if resp.StatusCode/100 == 5 {
+			// TODO: make sure this is a no leader response
+			cerr.Errors = append(cerr.Errors, fmt.Errorf("client: etcd member %s has no leader", ep.String()))
 			continue
 		}
-		break
+		return resp, body, nil
 	}
 
-	return resp, body, err
+	return nil, nil, cerr
 }
 
 func (c *httpClusterClient) Endpoints() []string {

client/client_test.go

@@ -342,7 +342,7 @@ func TestHTTPClusterClientDo(t *testing.T) {
 					},
 				),
 			},
-			wantErr: context.DeadlineExceeded,
+			wantErr: &ClusterError{Errors: []error{context.DeadlineExceeded}},
 		},
 
 		// context.Canceled short-circuits Do
@@ -356,7 +356,7 @@ func TestHTTPClusterClientDo(t *testing.T) {
 					},
 				),
 			},
-			wantErr: context.Canceled,
+			wantErr: &ClusterError{Errors: []error{context.Canceled}},
 		},
 
 		// return err if there are no endpoints
@@ -379,7 +379,7 @@ func TestHTTPClusterClientDo(t *testing.T) {
 					},
 				),
 			},
-			wantErr: fakeErr,
+			wantErr: &ClusterError{Errors: []error{fakeErr, fakeErr}},
 		},
 
 		// 500-level errors cause Do to fallthrough to next endpoint

client/cluster_error.go

@@ -0,0 +1,23 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+type ClusterError struct {
+	Errors []error
+}
+
+func (ce *ClusterError) Error() string {
+	return ErrClusterUnavailable.Error()
+}

discovery/discovery.go

@@ -216,7 +216,7 @@ func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) {
 		if eerr, ok := err.(*client.Error); ok && eerr.Code == client.ErrorCodeKeyNotFound {
 			return nil, 0, 0, ErrSizeNotFound
 		}
-		if err == context.DeadlineExceeded {
+		if _, ok := err.(*client.ClusterError); ok {
 			return d.checkClusterRetry()
 		}
 		return nil, 0, 0, err
@@ -230,7 +230,7 @@ func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) {
 	resp, err = d.c.Get(ctx, d.cluster, nil)
 	cancel()
 	if err != nil {
-		if err == context.DeadlineExceeded {
+		if _, ok := err.(*client.ClusterError); ok {
 			return d.checkClusterRetry()
 		}
 		return nil, 0, 0, err
@@ -306,7 +306,7 @@ func (d *discovery) waitNodes(nodes []*client.Node, size int, index uint64) ([]*
 		plog.Noticef("found %d peer(s), waiting for %d more", len(all), size-len(all))
 		resp, err := w.Next(context.Background())
 		if err != nil {
-			if err == context.DeadlineExceeded {
+			if _, ok := err.(*client.ClusterError); ok {
 				return d.waitNodesRetry()
 			}
 			return nil, err

discovery/discovery_test.go

@@ -488,7 +488,7 @@ type clientWithRetry struct {
 func (c *clientWithRetry) Create(ctx context.Context, key string, value string) (*client.Response, error) {
 	if c.failCount < c.failTimes {
 		c.failCount++
-		return nil, context.DeadlineExceeded
+		return nil, &client.ClusterError{Errors: []error{context.DeadlineExceeded}}
 	}
 	return c.clientWithResp.Create(ctx, key, value)
 }
@@ -496,7 +496,7 @@ func (c *clientWithRetry) Create(ctx context.Context, key string, value string)
 func (c *clientWithRetry) Get(ctx context.Context, key string, opts *client.GetOptions) (*client.Response, error) {
 	if c.failCount < c.failTimes {
 		c.failCount++
-		return nil, context.DeadlineExceeded
+		return nil, &client.ClusterError{Errors: []error{context.DeadlineExceeded}}
 	}
 	return c.clientWithResp.Get(ctx, key, opts)
 }
@@ -511,7 +511,7 @@ type watcherWithRetry struct {
 func (w *watcherWithRetry) Next(context.Context) (*client.Response, error) {
 	if w.failCount < w.failTimes {
 		w.failCount++
-		return nil, context.DeadlineExceeded
+		return nil, &client.ClusterError{Errors: []error{context.DeadlineExceeded}}
 	}
 	if len(w.rs) == 0 {
 		return &client.Response{}, nil