test retry logic

discovery/discovery.go

@@ -41,6 +41,9 @@ type discovery struct {
 	c       client.Client
 	retries uint
 	url     *url.URL
+
+	// Injectable for testing. nil means Seconds.
+	timeoutTimescale time.Duration
 }
 
 func New(durl string, id int64, config string) (Discoverer, error) {
@@ -63,6 +66,7 @@ func New(durl string, id int64, config string) (Discoverer, error) {
 		config:           config,
 		c:                c,
 		url:              u,
+		timeoutTimescale: time.Second,
 	}, nil
 }
 
@@ -156,7 +160,7 @@ func (d *discovery) checkCluster() (client.Nodes, int, error) {
 
 func (d *discovery) logAndBackoffForRetry(step string) {
 	d.retries++
-	retryTime := time.Second * (0x1 << d.retries)
+	retryTime := d.timeoutTimescale * (0x1 << d.retries)
 	log.Println("discovery: during", step, "connection to", d.url, "timed out, retrying in", retryTime)
 	time.Sleep(retryTime)
 }

discovery/discovery_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"math/rand"
 	"sort"
+	"strconv"
 
 	"reflect"
 	"testing"
@@ -107,6 +108,11 @@ func TestCheckCluster(t *testing.T) {
 		c := &clientWithResp{rs: rs}
 		d := discovery{cluster: cluster, id: 1, c: c}
 
+		cRetry := &clientWithRetry{}
+		cRetry.rs = rs
+		dRetry := discovery{cluster: cluster, id: 1, c: cRetry, timeoutTimescale: time.Millisecond * 2}
+
+		for _, d := range []discovery{d, dRetry} {
 			ns, size, err := d.checkCluster()
 			if err != tt.werr {
 				t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
@@ -119,6 +125,7 @@ func TestCheckCluster(t *testing.T) {
 			}
 		}
 	}
+}
 
 func TestWaitNodes(t *testing.T) {
 	all := client.Nodes{
@@ -173,8 +180,35 @@ func TestWaitNodes(t *testing.T) {
 	}
 
 	for i, tt := range tests {
+		// Basic case
 		c := &clientWithResp{nil, &watcherWithResp{tt.rs}}
 		d := &discovery{cluster: "1000", c: c}
+
+		// Retry case
+		retryScanResp := make([]*client.Response, 0)
+		if len(tt.nodes) > 0 {
+			retryScanResp = append(retryScanResp, &client.Response{
+				Node: &client.Node{
+					Key:   "1000",
+					Value: strconv.Itoa(tt.size),
+				},
+			})
+			retryScanResp = append(retryScanResp, &client.Response{
+				Node: &client.Node{
+					Nodes: tt.nodes,
+				},
+			})
+		}
+		cRetry := &clientWithRetry{}
+		cRetry.rs = retryScanResp
+		cRetry.w = &watcherWithRetry{tt.rs, false}
+		dRetry := &discovery{
+			cluster:          "1000",
+			c:                cRetry,
+			timeoutTimescale: time.Millisecond * 2,
+		}
+
+		for _, d := range []*discovery{d, dRetry} {
 			g, err := d.waitNodes(tt.nodes, tt.size)
 			if err != tt.werr {
 				t.Errorf("#%d: err = %v, want %v", i, err, tt.werr)
@@ -184,6 +218,7 @@ func TestWaitNodes(t *testing.T) {
 			}
 		}
 	}
+}
 
 func TestCreateSelf(t *testing.T) {
 	rs := []*client.Response{{Node: &client.Node{Key: "1000/1", CreatedIndex: 2}}}
@@ -330,3 +365,53 @@ type watcherWithErr struct {
 func (w *watcherWithErr) Next() (*client.Response, error) {
 	return &client.Response{}, w.err
 }
+
+// Fails every other time
+type clientWithRetry struct {
+	clientWithResp
+	haveFailed bool
+}
+
+func (c *clientWithRetry) Create(key string, value string, ttl time.Duration) (*client.Response, error) {
+	if !c.haveFailed {
+		c.haveFailed = true
+		return nil, client.ErrTimeout
+	}
+	if len(c.rs) == 0 {
+		return &client.Response{}, nil
+	}
+	r := c.rs[0]
+	c.rs = c.rs[1:]
+	return r, nil
+}
+
+func (c *clientWithRetry) Get(key string) (*client.Response, error) {
+	if !c.haveFailed {
+		c.haveFailed = true
+		return nil, client.ErrTimeout
+	}
+	if len(c.rs) == 0 {
+		return &client.Response{}, client.ErrKeyNoExist
+	}
+	r := c.rs[0]
+	c.rs = c.rs[1:]
+	return r, nil
+}
+
+type watcherWithRetry struct {
+	rs         []*client.Response
+	haveFailed bool
+}
+
+func (w *watcherWithRetry) Next() (*client.Response, error) {
+	if !w.haveFailed {
+		w.haveFailed = true
+		return nil, client.ErrTimeout
+	}
+	if len(w.rs) == 0 {
+		return &client.Response{}, nil
+	}
+	r := w.rs[0]
+	w.rs = w.rs[1:]
+	return r, nil
+}