diff --git a/server/v2/tests/delete_handler_test.go b/server/v2/tests/delete_handler_test.go index 4907e102d..c18b402f5 100644 --- a/server/v2/tests/delete_handler_test.go +++ b/server/v2/tests/delete_handler_test.go @@ -24,6 +24,6 @@ func TestV2DeleteKey(t *testing.T) { resp, err = tests.DeleteForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/bar"), url.Values{}) body := tests.ReadBody(resp) assert.Nil(t, err, "") - assert.Equal(t, string(body), `{"action":"delete","node":{"key":"/foo/bar","modifiedIndex":2,"createdIndex":1}}`, "") + assert.Equal(t, string(body), `{"action":"delete","node":{"key":"/foo/bar","prevValue":"XXX","modifiedIndex":2,"createdIndex":1}}`, "") }) } diff --git a/tests/functional/multi_node_kill_all_and_recovery_test.go b/tests/functional/multi_node_kill_all_and_recovery_test.go index 75fe22974..c7e710636 100644 --- a/tests/functional/multi_node_kill_all_and_recovery_test.go +++ b/tests/functional/multi_node_kill_all_and_recovery_test.go @@ -65,7 +65,7 @@ func TestMultiNodeKillAllAndRecovery(t *testing.T) { t.Fatalf("Recovery error: %s", err) } - if result.ModifiedIndex != 16 { - t.Fatalf("recovery failed! [%d/16]", result.ModifiedIndex) + if result.Node.ModifiedIndex != 16 { + t.Fatalf("recovery failed! [%d/16]", result.Node.ModifiedIndex) } } diff --git a/tests/functional/remove_node_test.go b/tests/functional/remove_node_test.go index dbd9ebd71..9793b22de 100644 --- a/tests/functional/remove_node_test.go +++ b/tests/functional/remove_node_test.go @@ -35,13 +35,13 @@ func TestRemoveNode(t *testing.T) { fmt.Println("send remove to node3 and wait for its exiting") etcds[2].Wait() - resp, err := c.Get("_etcd/machines", false) + resp, err := c.Get("_etcd/machines", false, false) if err != nil { panic(err) } - if len(resp.Kvs) != 2 { + if len(resp.Node.Nodes) != 2 { t.Fatal("cannot remove peer") } @@ -59,14 +59,14 @@ func TestRemoveNode(t *testing.T) { time.Sleep(time.Second) - resp, err = c.Get("_etcd/machines", false) + resp, err = c.Get("_etcd/machines", false, false) if err != nil { panic(err) } - if len(resp.Kvs) != 3 { - t.Fatalf("add peer fails #1 (%d != 3)", len(resp.Kvs)) + if len(resp.Node.Nodes) != 3 { + t.Fatalf("add peer fails #1 (%d != 3)", len(resp.Node.Nodes)) } } @@ -78,13 +78,13 @@ func TestRemoveNode(t *testing.T) { client.Do(rmReq) - resp, err := c.Get("_etcd/machines", false) + resp, err := c.Get("_etcd/machines", false, false) if err != nil { panic(err) } - if len(resp.Kvs) != 2 { + if len(resp.Node.Nodes) != 2 { t.Fatal("cannot remove peer") } @@ -102,14 +102,14 @@ func TestRemoveNode(t *testing.T) { time.Sleep(time.Second) - resp, err = c.Get("_etcd/machines", false) + resp, err = c.Get("_etcd/machines", false, false) if err != nil { panic(err) } - if len(resp.Kvs) != 3 { - t.Fatalf("add peer fails #2 (%d != 3)", len(resp.Kvs)) + if len(resp.Node.Nodes) != 3 { + t.Fatalf("add peer fails #2 (%d != 3)", len(resp.Node.Nodes)) } } } diff --git a/tests/functional/simple_multi_node_test.go b/tests/functional/simple_multi_node_test.go index 7afe0a35f..0506b95aa 100644 --- a/tests/functional/simple_multi_node_test.go +++ b/tests/functional/simple_multi_node_test.go @@ -39,24 +39,26 @@ func templateTestSimpleMultiNode(t *testing.T, tls bool) { // Test Set result, err := c.Set("foo", "bar", 100) + node := result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } - t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set 1 failed with %s %s %v", node.Key, node.Value, node.TTL) } time.Sleep(time.Second) result, err = c.Set("foo", "bar", 100) + node = result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL < 95 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.PrevValue != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } - t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set 2 failed with %s %s %v", node.Key, node.Value, node.TTL) } } diff --git a/tests/functional/simple_snapshot_test.go b/tests/functional/simple_snapshot_test.go index 2ca14cdfe..29872d648 100644 --- a/tests/functional/simple_snapshot_test.go +++ b/tests/functional/simple_snapshot_test.go @@ -30,13 +30,14 @@ func TestSimpleSnapshot(t *testing.T) { // issue first 501 commands for i := 0; i < 501; i++ { result, err := c.Set("foo", "bar", 100) + node := result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } - t.Fatalf("Set failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set failed with %s %s %v", node.Key, node.Value, node.TTL) } } @@ -62,13 +63,14 @@ func TestSimpleSnapshot(t *testing.T) { // issue second 501 commands for i := 0; i < 501; i++ { result, err := c.Set("foo", "bar", 100) + node := result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } - t.Fatalf("Set failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set failed with %s %s %v", node.Key, node.Value, node.TTL) } } diff --git a/tests/functional/single_node_recovery_test.go b/tests/functional/single_node_recovery_test.go index dde9e16ca..632b767de 100644 --- a/tests/functional/single_node_recovery_test.go +++ b/tests/functional/single_node_recovery_test.go @@ -28,13 +28,14 @@ func TestSingleNodeRecovery(t *testing.T) { c.SyncCluster() // Test Set result, err := c.Set("foo", "bar", 100) + node := result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } - t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set 1 failed with %s %s %v", node.Key, node.Value, node.TTL) } time.Sleep(time.Second) @@ -50,16 +51,18 @@ func TestSingleNodeRecovery(t *testing.T) { time.Sleep(time.Second) - result, err = c.Get("foo", false) + result, err = c.Get("foo", false, false) + node = result.Node + if err != nil { t.Fatal("get fail: " + err.Error()) return } - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL > 99 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL > 99 { if err != nil { t.Fatal(err) } - t.Fatalf("Recovery Get failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Recovery Get failed with %s %s %v", node.Key, node.Value, node.TTL) } } diff --git a/tests/functional/single_node_test.go b/tests/functional/single_node_test.go index 87d297eea..9294e2f96 100644 --- a/tests/functional/single_node_test.go +++ b/tests/functional/single_node_test.go @@ -28,36 +28,39 @@ func TestSingleNode(t *testing.T) { c.SyncCluster() // Test Set result, err := c.Set("foo", "bar", 100) + node := result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal("Set 1: ", err) } - t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set 1 failed with %s %s %v", node.Key, node.Value, node.TTL) } time.Sleep(time.Second) result, err = c.Set("foo", "bar", 100) + node = result.Node - if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 100 { + if err != nil || node.Key != "/foo" || node.Value != "bar" || node.PrevValue != "bar" || node.TTL != 100 { if err != nil { t.Fatal("Set 2: ", err) } - t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set 2 failed with %s %s %v", node.Key, node.Value, node.TTL) } // Add a test-and-set test // First, we'll test we can change the value if we get it write result, err = c.CompareAndSwap("foo", "foobar", 100, "bar", 0) + node = result.Node - if err != nil || result.Key != "/foo" || result.Value != "foobar" || result.PrevValue != "bar" || result.TTL != 100 { + if err != nil || node.Key != "/foo" || node.Value != "foobar" || node.PrevValue != "bar" || node.TTL != 100 { if err != nil { t.Fatal(err) } - t.Fatalf("Set 3 failed with %s %s %v", result.Key, result.Value, result.TTL) + t.Fatalf("Set 3 failed with %s %s %v", node.Key, node.Value, node.TTL) } // Next, we'll make sure we can't set it without the correct prior value diff --git a/tests/functional/util.go b/tests/functional/util.go index eb0173c76..9ceff0e70 100644 --- a/tests/functional/util.go +++ b/tests/functional/util.go @@ -44,7 +44,7 @@ func Set(stop chan bool) { result, err := c.Set(key, "bar", 0) - if err != nil || result.Key != "/"+key || result.Value != "bar" { + if err != nil || result.Node.Key != "/"+key || result.Node.Value != "bar" { select { case <-stop: stopSet = true diff --git a/tests/functional/v1_migration_test.go b/tests/functional/v1_migration_test.go index 506f48cc7..9f4fbeeea 100644 --- a/tests/functional/v1_migration_test.go +++ b/tests/functional/v1_migration_test.go @@ -99,5 +99,5 @@ func TestV1ClusterMigration(t *testing.T) { body = tests.ReadBody(resp) assert.Nil(t, err, "") assert.Equal(t, resp.StatusCode, 200, "") - assert.Equal(t, string(body), `{"action":"get","key":"/foo","value":"one","modifiedIndex":9}`) + assert.Equal(t, string(body), `{"action":"get","node":{"key":"/foo","value":"one","modifiedIndex":9,"createdIndex":9}}`) } diff --git a/third_party/github.com/coreos/go-etcd/etcd/add_child.go b/third_party/github.com/coreos/go-etcd/etcd/add_child.go index f275599c5..53af6d285 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/add_child.go +++ b/third_party/github.com/coreos/go-etcd/etcd/add_child.go @@ -2,10 +2,22 @@ package etcd // Add a new directory with a random etcd-generated key under the given path. func (c *Client) AddChildDir(key string, ttl uint64) (*Response, error) { - return c.post(key, "", ttl) + raw, err := c.post(key, "", ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } // Add a new file with a random etcd-generated key under the given path. func (c *Client) AddChild(key string, value string, ttl uint64) (*Response, error) { - return c.post(key, value, ttl) + raw, err := c.post(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } diff --git a/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go b/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go index efe155467..b5a5bd971 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/add_child_test.go @@ -5,8 +5,8 @@ import "testing" func TestAddChild(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("fooDir") - c.DeleteAll("nonexistentDir") + c.Delete("fooDir", true) + c.Delete("nonexistentDir", true) }() c.SetDir("fooDir", 5) @@ -21,10 +21,10 @@ func TestAddChild(t *testing.T) { t.Fatal(err) } - resp, err := c.Get("fooDir", true) + resp, err := c.Get("fooDir", true, false) // The child with v0 should proceed the child with v1 because it's added // earlier, so it should have a lower key. - if !(len(resp.Kvs) == 2 && (resp.Kvs[0].Value == "v0" && resp.Kvs[1].Value == "v1")) { + if !(len(resp.Node.Nodes) == 2 && (resp.Node.Nodes[0].Value == "v0" && resp.Node.Nodes[1].Value == "v1")) { t.Fatalf("AddChild 1 failed. There should be two chlidren whose values are v0 and v1, respectively."+ " The response was: %#v", resp) } @@ -40,8 +40,8 @@ func TestAddChild(t *testing.T) { func TestAddChildDir(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("fooDir") - c.DeleteAll("nonexistentDir") + c.Delete("fooDir", true) + c.Delete("nonexistentDir", true) }() c.SetDir("fooDir", 5) @@ -56,10 +56,10 @@ func TestAddChildDir(t *testing.T) { t.Fatal(err) } - resp, err := c.Get("fooDir", true) + resp, err := c.Get("fooDir", true, false) // The child with v0 should proceed the child with v1 because it's added // earlier, so it should have a lower key. - if !(len(resp.Kvs) == 2 && (len(resp.Kvs[0].KVPairs) == 0 && len(resp.Kvs[1].KVPairs) == 0)) { + if !(len(resp.Node.Nodes) == 2 && (len(resp.Node.Nodes[0].Nodes) == 0 && len(resp.Node.Nodes[1].Nodes) == 0)) { t.Fatalf("AddChildDir 1 failed. There should be two chlidren whose values are v0 and v1, respectively."+ " The response was: %#v", resp) } diff --git a/third_party/github.com/coreos/go-etcd/etcd/client.go b/third_party/github.com/coreos/go-etcd/etcd/client.go index 9b9efce0a..e03719e72 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/client.go +++ b/third_party/github.com/coreos/go-etcd/etcd/client.go @@ -11,7 +11,6 @@ import ( "net/url" "os" "path" - "reflect" "strings" "time" ) @@ -30,6 +29,10 @@ const ( WEAK_CONSISTENCY = "WEAK" ) +const ( + defaultBufferSize = 10 +) + type Cluster struct { Leader string `json:"leader"` Machines []string `json:"machines"` @@ -48,14 +51,9 @@ type Client struct { config Config `json:"config"` httpClient *http.Client persistence io.Writer + cURLch chan string } -type options map[string]interface{} - -// An internally-used data structure that represents a mapping -// between valid options and their kinds -type validOptions map[string]reflect.Kind - // NewClient create a basic client that is configured to be used // with the given machine list. func NewClient(machines []string) *Client { @@ -333,9 +331,7 @@ func dialTimeout(network, addr string) (net.Conn, error) { return net.DialTimeout(network, addr, time.Second) } -func (c *Client) updateLeader(httpPath string) { - u, _ := url.Parse(httpPath) - +func (c *Client) updateLeader(u *url.URL) { var leader string if u.Scheme == "" { leader = "http://" + u.Host @@ -347,3 +343,32 @@ func (c *Client) updateLeader(httpPath string) { c.cluster.Leader = leader c.saveConfig() } + +// switchLeader switch the current leader to machines[num] +func (c *Client) switchLeader(num int) { + logger.Debugf("switch.leader[from %v to %v]", + c.cluster.Leader, c.cluster.Machines[num]) + + c.cluster.Leader = c.cluster.Machines[num] +} + +func (c *Client) OpenCURL() { + c.cURLch = make(chan string, defaultBufferSize) +} + +func (c *Client) CloseCURL() { + c.cURLch = nil +} + +func (c *Client) sendCURL(command string) { + go func() { + select { + case c.cURLch <- command: + default: + } + }() +} + +func (c *Client) RecvCURL() string { + return <-c.cURLch +} diff --git a/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap.go b/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap.go index 565a03ef1..4099d1348 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap.go +++ b/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap.go @@ -14,5 +14,12 @@ func (c *Client) CompareAndSwap(key string, value string, ttl uint64, prevValue if prevIndex != 0 { options["prevIndex"] = prevIndex } - return c.put(key, value, ttl, options) + + raw, err := c.put(key, value, ttl, options) + + if err != nil { + return nil, err + } + + return raw.toResponse() } diff --git a/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go b/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go index bc452a910..56f8e4a8e 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go @@ -7,7 +7,7 @@ import ( func TestCompareAndSwap(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") + c.Delete("foo", true) }() c.Set("foo", "bar", 5) @@ -17,8 +17,8 @@ func TestCompareAndSwap(t *testing.T) { if err != nil { t.Fatal(err) } - if !(resp.Value == "bar2" && resp.PrevValue == "bar" && - resp.Key == "/foo" && resp.TTL == 5) { + if !(resp.Node.Value == "bar2" && resp.Node.PrevValue == "bar" && + resp.Node.Key == "/foo" && resp.Node.TTL == 5) { t.Fatalf("CompareAndSwap 1 failed: %#v", resp) } @@ -34,12 +34,12 @@ func TestCompareAndSwap(t *testing.T) { } // This should succeed - resp, err = c.CompareAndSwap("foo", "bar2", 5, "", resp.ModifiedIndex) + resp, err = c.CompareAndSwap("foo", "bar2", 5, "", resp.Node.ModifiedIndex) if err != nil { t.Fatal(err) } - if !(resp.Value == "bar2" && resp.PrevValue == "bar" && - resp.Key == "/foo" && resp.TTL == 5) { + if !(resp.Node.Value == "bar2" && resp.Node.PrevValue == "bar" && + resp.Node.Key == "/foo" && resp.Node.TTL == 5) { t.Fatalf("CompareAndSwap 1 failed: %#v", resp) } diff --git a/third_party/github.com/coreos/go-etcd/etcd/debug.go b/third_party/github.com/coreos/go-etcd/etcd/debug.go index bd6739881..44e59ef9e 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/debug.go +++ b/third_party/github.com/coreos/go-etcd/etcd/debug.go @@ -1,16 +1,15 @@ package etcd import ( - "github.com/coreos/go-log/log" "os" + + "github.com/coreos/go-log/log" ) var logger *log.Logger func init() { setLogger(log.PriErr) - // Uncomment the following line if you want to see lots of logs - // OpenDebug() } func OpenDebug() { diff --git a/third_party/github.com/coreos/go-etcd/etcd/delete.go b/third_party/github.com/coreos/go-etcd/etcd/delete.go index 00348f6ba..869b88e34 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/delete.go +++ b/third_party/github.com/coreos/go-etcd/etcd/delete.go @@ -1,17 +1,26 @@ package etcd -// DeleteAll deletes everything under the given key. If the key -// points to a file, the file will be deleted. If the key points -// to a directory, then everything under the directory, include +// Delete deletes the given key. +// When recursive set to false If the key points to a +// directory, the method will fail. +// When recursive set to true, if the key points to a file, +// the file will be deleted. If the key points +// to a directory, then everything under the directory, including // all child directories, will be deleted. -func (c *Client) DeleteAll(key string) (*Response, error) { - return c.delete(key, options{ - "recursive": true, - }) +func (c *Client) Delete(key string, recursive bool) (*Response, error) { + raw, err := c.DeleteRaw(key, recursive) + + if err != nil { + return nil, err + } + + return raw.toResponse() } -// Delete deletes the given key. If the key points to a -// directory, the method will fail. -func (c *Client) Delete(key string) (*Response, error) { - return c.delete(key, nil) +func (c *Client) DeleteRaw(key string, recursive bool) (*RawResponse, error) { + ops := options{ + "recursive": recursive, + } + + return c.delete(key, ops) } diff --git a/third_party/github.com/coreos/go-etcd/etcd/delete_test.go b/third_party/github.com/coreos/go-etcd/etcd/delete_test.go index 0f8475a23..30089aac4 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/delete_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/delete_test.go @@ -7,21 +7,21 @@ import ( func TestDelete(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") + c.Delete("foo", true) }() c.Set("foo", "bar", 5) - resp, err := c.Delete("foo") + resp, err := c.Delete("foo", false) if err != nil { t.Fatal(err) } - if !(resp.PrevValue == "bar" && resp.Value == "") { - t.Fatalf("Delete failed with %s %s", resp.PrevValue, - resp.Value) + if !(resp.Node.PrevValue == "bar" && resp.Node.Value == "") { + t.Fatalf("Delete failed with %s %s", resp.Node.PrevValue, + resp.Node.Value) } - resp, err = c.Delete("foo") + resp, err = c.Delete("foo", false) if err == nil { t.Fatalf("Delete should have failed because the key foo did not exist. "+ "The response was: %v", resp) @@ -31,32 +31,32 @@ func TestDelete(t *testing.T) { func TestDeleteAll(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") - c.DeleteAll("fooDir") + c.Delete("foo", true) + c.Delete("fooDir", true) }() c.Set("foo", "bar", 5) - resp, err := c.DeleteAll("foo") + resp, err := c.Delete("foo", true) if err != nil { t.Fatal(err) } - if !(resp.PrevValue == "bar" && resp.Value == "") { + if !(resp.Node.PrevValue == "bar" && resp.Node.Value == "") { t.Fatalf("DeleteAll 1 failed: %#v", resp) } c.SetDir("fooDir", 5) c.Set("fooDir/foo", "bar", 5) - resp, err = c.DeleteAll("fooDir") + resp, err = c.Delete("fooDir", true) if err != nil { t.Fatal(err) } - if !(resp.PrevValue == "" && resp.Value == "") { + if !(resp.Node.PrevValue == "" && resp.Node.Value == "") { t.Fatalf("DeleteAll 2 failed: %#v", resp) } - resp, err = c.DeleteAll("foo") + resp, err = c.Delete("foo", true) if err == nil { t.Fatalf("DeleteAll should have failed because the key foo did not exist. "+ "The response was: %v", resp) diff --git a/third_party/github.com/coreos/go-etcd/etcd/get.go b/third_party/github.com/coreos/go-etcd/etcd/get.go index d42a83c7d..7988f1a80 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/get.go +++ b/third_party/github.com/coreos/go-etcd/etcd/get.go @@ -1,23 +1,27 @@ package etcd -// GetDir gets the all contents under the given key. -// If the key points to a file, the file is returned. -// If the key points to a directory, everything under it is returnd, -// including all contents under all child directories. -func (c *Client) GetAll(key string, sort bool) (*Response, error) { - return c.get(key, options{ - "recursive": true, - "sorted": sort, - }) -} - // Get gets the file or directory associated with the given key. // If the key points to a directory, files and directories under // it will be returned in sorted or unsorted order, depending on -// the sort flag. Note that contents under child directories -// will not be returned. To get those contents, use GetAll. -func (c *Client) Get(key string, sort bool) (*Response, error) { - return c.get(key, options{ - "sorted": sort, - }) +// the sort flag. +// If recursive is set to false, contents under child directories +// will not be returned. +// If recursive is set to true, all the contents will be returned. +func (c *Client) Get(key string, sort, recursive bool) (*Response, error) { + raw, err := c.RawGet(key, sort, recursive) + + if err != nil { + return nil, err + } + + return raw.toResponse() +} + +func (c *Client) RawGet(key string, sort, recursive bool) (*RawResponse, error) { + ops := options{ + "recursive": recursive, + "sorted": sort, + } + + return c.get(key, ops) } diff --git a/third_party/github.com/coreos/go-etcd/etcd/get_test.go b/third_party/github.com/coreos/go-etcd/etcd/get_test.go index a34946c7e..1f56d4ab5 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/get_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/get_test.go @@ -8,22 +8,22 @@ import ( func TestGet(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") + c.Delete("foo", true) }() c.Set("foo", "bar", 5) - result, err := c.Get("foo", false) + result, err := c.Get("foo", false, false) if err != nil { t.Fatal(err) } - if result.Key != "/foo" || result.Value != "bar" { - t.Fatalf("Get failed with %s %s %v", result.Key, result.Value, result.TTL) + if result.Node.Key != "/foo" || result.Node.Value != "bar" { + t.Fatalf("Get failed with %s %s %v", result.Node.Key, result.Node.Value, result.Node.TTL) } - result, err = c.Get("goo", false) + result, err = c.Get("goo", false, false) if err == nil { t.Fatalf("should not be able to get non-exist key") } @@ -32,7 +32,7 @@ func TestGet(t *testing.T) { func TestGetAll(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("fooDir") + c.Delete("fooDir", true) }() c.SetDir("fooDir", 5) @@ -40,25 +40,36 @@ func TestGetAll(t *testing.T) { c.Set("fooDir/k1", "v1", 5) // Return kv-pairs in sorted order - result, err := c.Get("fooDir", true) + result, err := c.Get("fooDir", true, false) if err != nil { t.Fatal(err) } - expected := kvPairs{ - KeyValuePair{ - Key: "/fooDir/k0", - Value: "v0", + expected := Nodes{ + Node{ + Key: "/fooDir/k0", + Value: "v0", + TTL: 5, + ModifiedIndex: 31, + CreatedIndex: 31, }, - KeyValuePair{ - Key: "/fooDir/k1", - Value: "v1", + Node{ + Key: "/fooDir/k1", + Value: "v1", + TTL: 5, + ModifiedIndex: 32, + CreatedIndex: 32, }, } - if !reflect.DeepEqual(result.Kvs, expected) { - t.Fatalf("(actual) %v != (expected) %v", result.Kvs, expected) + // do not check expiration time, too hard to fake + for i, _ := range result.Node.Nodes { + result.Node.Nodes[i].Expiration = nil + } + + if !reflect.DeepEqual(result.Node.Nodes, expected) { + t.Fatalf("(actual) %v != (expected) %v", result.Node.Nodes, expected) } // Test the `recursive` option @@ -66,34 +77,44 @@ func TestGetAll(t *testing.T) { c.Set("fooDir/childDir/k2", "v2", 5) // Return kv-pairs in sorted order - result, err = c.GetAll("fooDir", true) + result, err = c.Get("fooDir", true, true) + + // do not check expiration time, too hard to fake + result.Node.Expiration = nil + for i, _ := range result.Node.Nodes { + result.Node.Nodes[i].Expiration = nil + } if err != nil { t.Fatal(err) } - expected = kvPairs{ - KeyValuePair{ + expected = Nodes{ + Node{ Key: "/fooDir/childDir", Dir: true, - KVPairs: kvPairs{ - KeyValuePair{ + Nodes: Nodes{ + Node{ Key: "/fooDir/childDir/k2", Value: "v2", + TTL: 5, }, }, + TTL: 5, }, - KeyValuePair{ + Node{ Key: "/fooDir/k0", Value: "v0", + TTL: 5, }, - KeyValuePair{ + Node{ Key: "/fooDir/k1", Value: "v1", + TTL: 5, }, } - if !reflect.DeepEqual(result.Kvs, expected) { - t.Fatalf("(actual) %v != (expected) %v", result.Kvs) + if !reflect.DeepEqual(result.Node.Nodes, expected) { + t.Fatalf("(actual) %v != (expected) %v", result.Node.Nodes, expected) } } diff --git a/third_party/github.com/coreos/go-etcd/etcd/options.go b/third_party/github.com/coreos/go-etcd/etcd/options.go new file mode 100644 index 000000000..31fe80288 --- /dev/null +++ b/third_party/github.com/coreos/go-etcd/etcd/options.go @@ -0,0 +1,68 @@ +package etcd + +import ( + "fmt" + "net/url" + "reflect" +) + +type options map[string]interface{} + +// An internally-used data structure that represents a mapping +// between valid options and their kinds +type validOptions map[string]reflect.Kind + +// Valid options for GET, PUT, POST, DELETE +// Using CAPITALIZED_UNDERSCORE to emphasize that these +// values are meant to be used as constants. +var ( + VALID_GET_OPTIONS = validOptions{ + "recursive": reflect.Bool, + "consistent": reflect.Bool, + "sorted": reflect.Bool, + "wait": reflect.Bool, + "waitIndex": reflect.Uint64, + } + + VALID_PUT_OPTIONS = validOptions{ + "prevValue": reflect.String, + "prevIndex": reflect.Uint64, + "prevExist": reflect.Bool, + } + + VALID_POST_OPTIONS = validOptions{} + + VALID_DELETE_OPTIONS = validOptions{ + "recursive": reflect.Bool, + } +) + +// Convert options to a string of HTML parameters +func (ops options) toParameters(validOps validOptions) (string, error) { + p := "?" + values := url.Values{} + + if ops == nil { + return "", nil + } + + for k, v := range ops { + // Check if the given option is valid (that it exists) + kind := validOps[k] + if kind == reflect.Invalid { + return "", fmt.Errorf("Invalid option: %v", k) + } + + // Check if the given option is of the valid type + t := reflect.TypeOf(v) + if kind != t.Kind() { + return "", fmt.Errorf("Option %s should be of %v kind, not of %v kind.", + k, kind, t.Kind()) + } + + values.Set(k, fmt.Sprintf("%v", v)) + } + + p += values.Encode() + return p, nil +} diff --git a/third_party/github.com/coreos/go-etcd/etcd/requests.go b/third_party/github.com/coreos/go-etcd/etcd/requests.go index 83e3b519e..7385bfacd 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/requests.go +++ b/third_party/github.com/coreos/go-etcd/etcd/requests.go @@ -1,54 +1,18 @@ package etcd import ( - "encoding/json" - "errors" "fmt" "io/ioutil" "math/rand" "net/http" "net/url" "path" - "reflect" "strings" "time" ) -// Valid options for GET, PUT, POST, DELETE -// Using CAPITALIZED_UNDERSCORE to emphasize that these -// values are meant to be used as constants. -var ( - VALID_GET_OPTIONS = validOptions{ - "recursive": reflect.Bool, - "consistent": reflect.Bool, - "sorted": reflect.Bool, - "wait": reflect.Bool, - "waitIndex": reflect.Uint64, - } - - VALID_PUT_OPTIONS = validOptions{ - "prevValue": reflect.String, - "prevIndex": reflect.Uint64, - "prevExist": reflect.Bool, - } - - VALID_POST_OPTIONS = validOptions{} - - VALID_DELETE_OPTIONS = validOptions{ - "recursive": reflect.Bool, - } - - curlChan chan string -) - -// SetCurlChan sets a channel to which cURL commands which can be used to -// re-produce requests are sent. This is useful for debugging. -func SetCurlChan(c chan string) { - curlChan = c -} - // get issues a GET request -func (c *Client) get(key string, options options) (*Response, error) { +func (c *Client) get(key string, options options) (*RawResponse, error) { logger.Debugf("get %s [%s]", key, c.cluster.Leader) p := path.Join("keys", key) @@ -57,15 +21,14 @@ func (c *Client) get(key string, options options) (*Response, error) { if c.config.Consistency == STRONG_CONSISTENCY { options["consistent"] = true } - if options != nil { - str, err := optionsToString(options, VALID_GET_OPTIONS) - if err != nil { - return nil, err - } - p += str - } - resp, err := c.sendRequest("GET", p, url.Values{}) + str, err := options.toParameters(VALID_GET_OPTIONS) + if err != nil { + return nil, err + } + p += str + + resp, err := c.sendRequest("GET", p, nil) if err != nil { return nil, err @@ -75,28 +38,19 @@ func (c *Client) get(key string, options options) (*Response, error) { } // put issues a PUT request -func (c *Client) put(key string, value string, ttl uint64, options options) (*Response, error) { +func (c *Client) put(key string, value string, ttl uint64, + options options) (*RawResponse, error) { + logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) - v := url.Values{} - - if value != "" { - v.Set("value", value) - } - - if ttl > 0 { - v.Set("ttl", fmt.Sprintf("%v", ttl)) - } - p := path.Join("keys", key) - if options != nil { - str, err := optionsToString(options, VALID_PUT_OPTIONS) - if err != nil { - return nil, err - } - p += str - } - resp, err := c.sendRequest("PUT", p, v) + str, err := options.toParameters(VALID_PUT_OPTIONS) + if err != nil { + return nil, err + } + p += str + + resp, err := c.sendRequest("PUT", p, buildValues(value, ttl)) if err != nil { return nil, err @@ -106,19 +60,11 @@ func (c *Client) put(key string, value string, ttl uint64, options options) (*Re } // post issues a POST request -func (c *Client) post(key string, value string, ttl uint64) (*Response, error) { +func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) { logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) - v := url.Values{} + p := path.Join("keys", key) - if value != "" { - v.Set("value", value) - } - - if ttl > 0 { - v.Set("ttl", fmt.Sprintf("%v", ttl)) - } - - resp, err := c.sendRequest("POST", path.Join("keys", key), v) + resp, err := c.sendRequest("POST", p, buildValues(value, ttl)) if err != nil { return nil, err @@ -128,20 +74,18 @@ func (c *Client) post(key string, value string, ttl uint64) (*Response, error) { } // delete issues a DELETE request -func (c *Client) delete(key string, options options) (*Response, error) { +func (c *Client) delete(key string, options options) (*RawResponse, error) { logger.Debugf("delete %s [%s]", key, c.cluster.Leader) - v := url.Values{} p := path.Join("keys", key) - if options != nil { - str, err := optionsToString(options, VALID_DELETE_OPTIONS) - if err != nil { - return nil, err - } - p += str - } - resp, err := c.sendRequest("DELETE", p, v) + str, err := options.toParameters(VALID_DELETE_OPTIONS) + if err != nil { + return nil, err + } + p += str + + resp, err := c.sendRequest("DELETE", p, nil) if err != nil { return nil, err @@ -151,126 +95,128 @@ func (c *Client) delete(key string, options options) (*Response, error) { } // sendRequest sends a HTTP request and returns a Response as defined by etcd -func (c *Client) sendRequest(method string, _path string, values url.Values) (*Response, error) { - var body string = values.Encode() - var resp *http.Response - var req *http.Request +func (c *Client) sendRequest(method string, relativePath string, + values url.Values) (*RawResponse, error) { + + var req *http.Request + var resp *http.Response + var httpPath string + var err error + var b []byte + + trial := 0 - retry := 0 // if we connect to a follower, we will retry until we found a leader for { - var httpPath string - - // If _path has schema already, then it's assumed to be - // a complete URL and therefore needs no further processing. - u, err := url.Parse(_path) - if err != nil { - return nil, err + trial++ + logger.Debug("begin trail ", trial) + if trial > 2*len(c.cluster.Machines) { + return nil, fmt.Errorf("Cannot reach servers after %v time", trial) } - if u.Scheme != "" { - httpPath = _path + if method == "GET" && c.config.Consistency == WEAK_CONSISTENCY { + // If it's a GET and consistency level is set to WEAK, + // then use a random machine. + httpPath = c.getHttpPath(true, relativePath) } else { - if method == "GET" && c.config.Consistency == WEAK_CONSISTENCY { - // If it's a GET and consistency level is set to WEAK, - // then use a random machine. - httpPath = c.getHttpPath(true, _path) - } else { - // Else use the leader. - httpPath = c.getHttpPath(false, _path) - } + // Else use the leader. + httpPath = c.getHttpPath(false, relativePath) } // Return a cURL command if curlChan is set - if curlChan != nil { + if c.cURLch != nil { command := fmt.Sprintf("curl -X %s %s", method, httpPath) for key, value := range values { command += fmt.Sprintf(" -d %s=%s", key, value[0]) } - curlChan <- command + c.sendCURL(command) } logger.Debug("send.request.to ", httpPath, " | method ", method) - if body == "" { + if values == nil { req, _ = http.NewRequest(method, httpPath, nil) - } else { - req, _ = http.NewRequest(method, httpPath, strings.NewReader(body)) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") + req, _ = http.NewRequest(method, httpPath, + strings.NewReader(values.Encode())) + + req.Header.Set("Content-Type", + "application/x-www-form-urlencoded; param=value") } - resp, err = c.httpClient.Do(req) - - logger.Debug("recv.response.from ", httpPath) // network error, change a machine! - if err != nil { - retry++ - if retry > 2*len(c.cluster.Machines) { - return nil, errors.New("Cannot reach servers") - } - num := retry % len(c.cluster.Machines) - logger.Debug("update.leader[", c.cluster.Leader, ",", c.cluster.Machines[num], "]") - c.cluster.Leader = c.cluster.Machines[num] + if resp, err = c.httpClient.Do(req); err != nil { + c.switchLeader(trial % len(c.cluster.Machines)) time.Sleep(time.Millisecond * 200) continue } if resp != nil { - if resp.StatusCode == http.StatusTemporaryRedirect { - httpPath := resp.Header.Get("Location") + logger.Debug("recv.response.from ", httpPath) - resp.Body.Close() + var ok bool + ok, b = c.handleResp(resp) - if httpPath == "" { - return nil, errors.New("Cannot get redirection location") - } - - c.updateLeader(httpPath) - logger.Debug("send.redirect") - // try to connect the leader + if !ok { continue - } else if resp.StatusCode == http.StatusInternalServerError { - resp.Body.Close() - - retry++ - if retry > 2*len(c.cluster.Machines) { - return nil, errors.New("Cannot reach servers") - } - continue - } else { - logger.Debug("send.return.response ", httpPath) - break } + logger.Debug("recv.success.", httpPath) + break } - logger.Debug("error.from ", httpPath, " ", err.Error()) + + // should not reach here + // err and resp should not be nil at the same time + logger.Debug("error.from ", httpPath) return nil, err } - // Convert HTTP response to etcd response - b, err := ioutil.ReadAll(resp.Body) - - resp.Body.Close() - - if err != nil { - return nil, err + r := &RawResponse{ + StatusCode: resp.StatusCode, + Body: b, + Header: resp.Header, } - if !(resp.StatusCode == http.StatusOK || - resp.StatusCode == http.StatusCreated) { - return nil, handleError(b) + return r, nil +} + +// handleResp handles the responses from the etcd server +// If status code is OK, read the http body and return it as byte array +// If status code is TemporaryRedirect, update leader. +// If status code is InternalServerError, sleep for 200ms. +func (c *Client) handleResp(resp *http.Response) (bool, []byte) { + defer resp.Body.Close() + + code := resp.StatusCode + + if code == http.StatusTemporaryRedirect { + u, err := resp.Location() + + if err != nil { + logger.Warning(err) + } else { + c.updateLeader(u) + } + + return false, nil + + } else if code == http.StatusInternalServerError { + time.Sleep(time.Millisecond * 200) + + } else if code == http.StatusOK || + code == http.StatusCreated || + code == http.StatusBadRequest { + b, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return false, nil + } + + return true, b } - var result Response - - err = json.Unmarshal(b, &result) - - if err != nil { - return nil, err - } - - return &result, nil + logger.Warning("bad status code ", resp.StatusCode) + return false, nil } func (c *Client) getHttpPath(random bool, s ...string) string { @@ -288,3 +234,18 @@ func (c *Client) getHttpPath(random bool, s ...string) string { return fullPath } + +// buildValues builds a url.Values map according to the given value and ttl +func buildValues(value string, ttl uint64) url.Values { + v := url.Values{} + + if value != "" { + v.Set("value", value) + } + + if ttl > 0 { + v.Set("ttl", fmt.Sprintf("%v", ttl)) + } + + return v +} diff --git a/third_party/github.com/coreos/go-etcd/etcd/response.go b/third_party/github.com/coreos/go-etcd/etcd/response.go index a8bb21a68..746a7c6e5 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/response.go +++ b/third_party/github.com/coreos/go-etcd/etcd/response.go @@ -1,51 +1,68 @@ package etcd import ( + "encoding/json" + "net/http" "time" ) -// The response object from the server. +const ( + rawResponse = iota + normalResponse +) + +type responseType int + +type RawResponse struct { + StatusCode int + Body []byte + Header http.Header +} + +func (rr *RawResponse) toResponse() (*Response, error) { + if rr.StatusCode == http.StatusBadRequest { + return nil, handleError(rr.Body) + } + + resp := new(Response) + + err := json.Unmarshal(rr.Body, resp) + + if err != nil { + return nil, err + } + + return resp, nil +} + type Response struct { - Action string `json:"action"` - Key string `json:"key"` - Dir bool `json:"dir,omitempty"` - PrevValue string `json:"prevValue,omitempty"` - Value string `json:"value,omitempty"` - Kvs kvPairs `json:"kvs,omitempty"` - - // If the key did not exist before the action, - // this field should be set to true - NewKey bool `json:"newKey,omitempty"` - - Expiration *time.Time `json:"expiration,omitempty"` - - // Time to live in second - TTL int64 `json:"ttl,omitempty"` - - // The command index of the raft machine when the command is executed - ModifiedIndex uint64 `json:"modifiedIndex"` + Action string `json:"action"` + Node *Node `json:"node,omitempty"` } -// When user list a directory, we add all the node into key-value pair slice -type KeyValuePair struct { - Key string `json:"key, omitempty"` - Value string `json:"value,omitempty"` - Dir bool `json:"dir,omitempty"` - KVPairs kvPairs `json:"kvs,omitempty"` - TTL int64 `json:"ttl,omitempty"` +type Node struct { + Key string `json:"key, omitempty"` + PrevValue string `json:"prevValue,omitempty"` + Value string `json:"value,omitempty"` + Dir bool `json:"dir,omitempty"` + Expiration *time.Time `json:"expiration,omitempty"` + TTL int64 `json:"ttl,omitempty"` + Nodes Nodes `json:"nodes,omitempty"` + ModifiedIndex uint64 `json:"modifiedIndex,omitempty"` + CreatedIndex uint64 `json:"createdIndex,omitempty"` } -type kvPairs []KeyValuePair +type Nodes []Node // interfaces for sorting -func (kvs kvPairs) Len() int { - return len(kvs) +func (ns Nodes) Len() int { + return len(ns) } -func (kvs kvPairs) Less(i, j int) bool { - return kvs[i].Key < kvs[j].Key +func (ns Nodes) Less(i, j int) bool { + return ns[i].Key < ns[j].Key } -func (kvs kvPairs) Swap(i, j int) { - kvs[i], kvs[j] = kvs[j], kvs[i] +func (ns Nodes) Swap(i, j int) { + ns[i], ns[j] = ns[j], ns[i] } diff --git a/third_party/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go b/third_party/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go index 6d0633109..756e31781 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go @@ -7,12 +7,11 @@ import ( func TestSetCurlChan(t *testing.T) { c := NewClient(nil) - defer func() { - c.DeleteAll("foo") - }() + c.OpenCURL() - curlChan := make(chan string, 1) - SetCurlChan(curlChan) + defer func() { + c.Delete("foo", true) + }() _, err := c.Set("foo", "bar", 5) if err != nil { @@ -21,21 +20,21 @@ func TestSetCurlChan(t *testing.T) { expected := fmt.Sprintf("curl -X PUT %s/v2/keys/foo -d value=bar -d ttl=5", c.cluster.Leader) - actual := <-curlChan + actual := c.RecvCURL() if expected != actual { t.Fatalf(`Command "%s" is not equal to expected value "%s"`, actual, expected) } c.SetConsistency(STRONG_CONSISTENCY) - _, err = c.Get("foo", false) + _, err = c.Get("foo", false, false) if err != nil { t.Fatal(err) } - expected = fmt.Sprintf("curl -X GET %s/v2/keys/foo?consistent=true&sorted=false", + expected = fmt.Sprintf("curl -X GET %s/v2/keys/foo?consistent=true&recursive=false&sorted=false", c.cluster.Leader) - actual = <-curlChan + actual = c.RecvCURL() if expected != actual { t.Fatalf(`Command "%s" is not equal to expected value "%s"`, actual, expected) diff --git a/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go b/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go index 281cd577c..8558db1ac 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go +++ b/third_party/github.com/coreos/go-etcd/etcd/set_update_create.go @@ -2,42 +2,110 @@ package etcd // SetDir sets the given key to a directory. func (c *Client) SetDir(key string, ttl uint64) (*Response, error) { - return c.put(key, "", ttl, nil) + raw, err := c.RawSetDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } // UpdateDir updates the given key to a directory. It succeeds only if the // given key already exists. func (c *Client) UpdateDir(key string, ttl uint64) (*Response, error) { - return c.put(key, "", ttl, options{ - "prevExist": true, - }) + raw, err := c.RawUpdateDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } // UpdateDir creates a directory under the given key. It succeeds only if // the given key does not yet exist. func (c *Client) CreateDir(key string, ttl uint64) (*Response, error) { - return c.put(key, "", ttl, options{ - "prevExist": false, - }) + raw, err := c.RawCreateDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } // Set sets the given key to the given value. func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) { - return c.put(key, value, ttl, nil) + raw, err := c.RawSet(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } // Update updates the given key to the given value. It succeeds only if the // given key already exists. func (c *Client) Update(key string, value string, ttl uint64) (*Response, error) { - return c.put(key, value, ttl, options{ - "prevExist": true, - }) + raw, err := c.RawUpdate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() } // Create creates a file with the given value under the given key. It succeeds // only if the given key does not yet exist. func (c *Client) Create(key string, value string, ttl uint64) (*Response, error) { - return c.put(key, value, ttl, options{ - "prevExist": false, - }) + raw, err := c.RawCreate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.toResponse() +} + +func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) { + return c.put(key, "", ttl, nil) +} + +func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) { + ops := options{ + "prevExist": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawCreateDir(key string, ttl uint64) (*RawResponse, error) { + ops := options{ + "prevExist": false, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawSet(key string, value string, ttl uint64) (*RawResponse, error) { + return c.put(key, value, ttl, nil) +} + +func (c *Client) RawUpdate(key string, value string, ttl uint64) (*RawResponse, error) { + ops := options{ + "prevExist": true, + } + + return c.put(key, value, ttl, ops) +} + +func (c *Client) RawCreate(key string, value string, ttl uint64) (*RawResponse, error) { + ops := options{ + "prevExist": false, + } + + return c.put(key, value, ttl, ops) } diff --git a/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go b/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go index 6f27fdfa6..215622b9e 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/set_update_create_test.go @@ -7,14 +7,14 @@ import ( func TestSet(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") + c.Delete("foo", true) }() resp, err := c.Set("foo", "bar", 5) if err != nil { t.Fatal(err) } - if resp.Key != "/foo" || resp.Value != "bar" || resp.TTL != 5 { + if resp.Node.Key != "/foo" || resp.Node.Value != "bar" || resp.Node.TTL != 5 { t.Fatalf("Set 1 failed: %#v", resp) } @@ -22,8 +22,8 @@ func TestSet(t *testing.T) { if err != nil { t.Fatal(err) } - if !(resp.Key == "/foo" && resp.Value == "bar2" && - resp.PrevValue == "bar" && resp.TTL == 5) { + if !(resp.Node.Key == "/foo" && resp.Node.Value == "bar2" && + resp.Node.PrevValue == "bar" && resp.Node.TTL == 5) { t.Fatalf("Set 2 failed: %#v", resp) } } @@ -31,12 +31,12 @@ func TestSet(t *testing.T) { func TestUpdate(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") - c.DeleteAll("nonexistent") + c.Delete("foo", true) + c.Delete("nonexistent", true) }() resp, err := c.Set("foo", "bar", 5) - t.Logf("%#v", resp) + if err != nil { t.Fatal(err) } @@ -47,8 +47,8 @@ func TestUpdate(t *testing.T) { t.Fatal(err) } - if !(resp.Action == "update" && resp.Key == "/foo" && - resp.PrevValue == "bar" && resp.TTL == 5) { + if !(resp.Action == "update" && resp.Node.Key == "/foo" && + resp.Node.PrevValue == "bar" && resp.Node.TTL == 5) { t.Fatalf("Update 1 failed: %#v", resp) } @@ -56,14 +56,14 @@ func TestUpdate(t *testing.T) { resp, err = c.Update("nonexistent", "whatever", 5) if err == nil { t.Fatalf("The key %v did not exist, so the update should have failed."+ - "The response was: %#v", resp.Key, resp) + "The response was: %#v", resp.Node.Key, resp) } } func TestCreate(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("newKey") + c.Delete("newKey", true) }() newKey := "/newKey" @@ -75,8 +75,8 @@ func TestCreate(t *testing.T) { t.Fatal(err) } - if !(resp.Action == "create" && resp.Key == newKey && - resp.Value == newValue && resp.PrevValue == "" && resp.TTL == 5) { + if !(resp.Action == "create" && resp.Node.Key == newKey && + resp.Node.Value == newValue && resp.Node.PrevValue == "" && resp.Node.TTL == 5) { t.Fatalf("Create 1 failed: %#v", resp) } @@ -84,22 +84,22 @@ func TestCreate(t *testing.T) { resp, err = c.Create(newKey, newValue, 5) if err == nil { t.Fatalf("The key %v did exist, so the creation should have failed."+ - "The response was: %#v", resp.Key, resp) + "The response was: %#v", resp.Node.Key, resp) } } func TestSetDir(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("foo") - c.DeleteAll("fooDir") + c.Delete("foo", true) + c.Delete("fooDir", true) }() resp, err := c.SetDir("fooDir", 5) if err != nil { t.Fatal(err) } - if !(resp.Key == "/fooDir" && resp.Value == "" && resp.TTL == 5) { + if !(resp.Node.Key == "/fooDir" && resp.Node.Value == "" && resp.Node.TTL == 5) { t.Fatalf("SetDir 1 failed: %#v", resp) } @@ -120,8 +120,8 @@ func TestSetDir(t *testing.T) { if err != nil { t.Fatal(err) } - if !(resp.Key == "/foo" && resp.Value == "" && - resp.PrevValue == "bar" && resp.TTL == 5) { + if !(resp.Node.Key == "/foo" && resp.Node.Value == "" && + resp.Node.PrevValue == "bar" && resp.Node.TTL == 5) { t.Fatalf("SetDir 2 failed: %#v", resp) } } @@ -129,11 +129,10 @@ func TestSetDir(t *testing.T) { func TestUpdateDir(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("fooDir") + c.Delete("fooDir", true) }() resp, err := c.SetDir("fooDir", 5) - t.Logf("%#v", resp) if err != nil { t.Fatal(err) } @@ -144,8 +143,8 @@ func TestUpdateDir(t *testing.T) { t.Fatal(err) } - if !(resp.Action == "update" && resp.Key == "/fooDir" && - resp.Value == "" && resp.PrevValue == "" && resp.TTL == 5) { + if !(resp.Action == "update" && resp.Node.Key == "/fooDir" && + resp.Node.Value == "" && resp.Node.PrevValue == "" && resp.Node.TTL == 5) { t.Fatalf("UpdateDir 1 failed: %#v", resp) } @@ -153,14 +152,14 @@ func TestUpdateDir(t *testing.T) { resp, err = c.UpdateDir("nonexistentDir", 5) if err == nil { t.Fatalf("The key %v did not exist, so the update should have failed."+ - "The response was: %#v", resp.Key, resp) + "The response was: %#v", resp.Node.Key, resp) } } func TestCreateDir(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("fooDir") + c.Delete("fooDir", true) }() // This should succeed @@ -169,8 +168,8 @@ func TestCreateDir(t *testing.T) { t.Fatal(err) } - if !(resp.Action == "create" && resp.Key == "/fooDir" && - resp.Value == "" && resp.PrevValue == "" && resp.TTL == 5) { + if !(resp.Action == "create" && resp.Node.Key == "/fooDir" && + resp.Node.Value == "" && resp.Node.PrevValue == "" && resp.Node.TTL == 5) { t.Fatalf("CreateDir 1 failed: %#v", resp) } @@ -178,6 +177,6 @@ func TestCreateDir(t *testing.T) { resp, err = c.CreateDir("fooDir", 5) if err == nil { t.Fatalf("The key %v did exist, so the creation should have failed."+ - "The response was: %#v", resp.Key, resp) + "The response was: %#v", resp.Node.Key, resp) } } diff --git a/third_party/github.com/coreos/go-etcd/etcd/watch.go b/third_party/github.com/coreos/go-etcd/etcd/watch.go index bbce2039b..1e1ac74a5 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/watch.go +++ b/third_party/github.com/coreos/go-etcd/etcd/watch.go @@ -9,44 +9,74 @@ var ( ErrWatchStoppedByUser = errors.New("Watch stopped by the user via stop channel") ) -// WatchAll returns the first change under the given prefix since the given index. To -// watch for the latest change, set waitIndex = 0. +// If recursive is set to true the watch returns the first change under the given +// prefix since the given index. // -// If the prefix points to a directory, any change under it, including all child directories, -// will be returned. +// If recursive is set to false the watch returns the first change to the given key +// since the given index. +// +// To watch for the latest change, set waitIndex = 0. // // If a receiver channel is given, it will be a long-term watch. Watch will block at the -// channel. And after someone receive the channel, it will go on to watch that prefix. -// If a stop channel is given, client can close long-term watch using the stop channel -func (c *Client) WatchAll(prefix string, waitIndex uint64, receiver chan *Response, stop chan bool) (*Response, error) { - return c.watch(prefix, waitIndex, true, receiver, stop) -} - -// Watch returns the first change to the given key since the given index. To -// watch for the latest change, set waitIndex = 0. -// -// If a receiver channel is given, it will be a long-term watch. Watch will block at the -// channel. And after someone receive the channel, it will go on to watch that -// prefix. If a stop channel is given, client can close long-term watch using -// the stop channel -func (c *Client) Watch(key string, waitIndex uint64, receiver chan *Response, stop chan bool) (*Response, error) { - return c.watch(key, waitIndex, false, receiver, stop) -} - -func (c *Client) watch(prefix string, waitIndex uint64, recursive bool, receiver chan *Response, stop chan bool) (*Response, error) { +//channel. After someone receives the channel, it will go on to watch that +// prefix. If a stop channel is given, the client can close long-term watch using +// the stop channel. +func (c *Client) Watch(prefix string, waitIndex uint64, recursive bool, + receiver chan *Response, stop chan bool) (*Response, error) { logger.Debugf("watch %s [%s]", prefix, c.cluster.Leader) if receiver == nil { - return c.watchOnce(prefix, waitIndex, recursive, stop) - } else { - for { - resp, err := c.watchOnce(prefix, waitIndex, recursive, stop) - if resp != nil { - waitIndex = resp.ModifiedIndex + 1 - receiver <- resp - } else { - return nil, err - } + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err } + + return raw.toResponse() + } + + for { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + resp, err := raw.toResponse() + + if err != nil { + return nil, err + } + + waitIndex = resp.Node.ModifiedIndex + 1 + receiver <- resp + } + + return nil, nil +} + +func (c *Client) RawWatch(prefix string, waitIndex uint64, recursive bool, + receiver chan *RawResponse, stop chan bool) (*RawResponse, error) { + + logger.Debugf("rawWatch %s [%s]", prefix, c.cluster.Leader) + if receiver == nil { + return c.watchOnce(prefix, waitIndex, recursive, stop) + } + + for { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + resp, err := raw.toResponse() + + if err != nil { + return nil, err + } + + waitIndex = resp.Node.ModifiedIndex + 1 + receiver <- raw } return nil, nil @@ -54,9 +84,9 @@ func (c *Client) watch(prefix string, waitIndex uint64, recursive bool, receiver // helper func // return when there is change under the given prefix -func (c *Client) watchOnce(key string, waitIndex uint64, recursive bool, stop chan bool) (*Response, error) { +func (c *Client) watchOnce(key string, waitIndex uint64, recursive bool, stop chan bool) (*RawResponse, error) { - respChan := make(chan *Response) + respChan := make(chan *RawResponse, 1) errChan := make(chan error) go func() { @@ -74,6 +104,7 @@ func (c *Client) watchOnce(key string, waitIndex uint64, recursive bool, stop ch if err != nil { errChan <- err + return } respChan <- resp diff --git a/third_party/github.com/coreos/go-etcd/etcd/watch_test.go b/third_party/github.com/coreos/go-etcd/etcd/watch_test.go index 10fc2b6b5..9b466489c 100644 --- a/third_party/github.com/coreos/go-etcd/etcd/watch_test.go +++ b/third_party/github.com/coreos/go-etcd/etcd/watch_test.go @@ -9,26 +9,26 @@ import ( func TestWatch(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("watch_foo") + c.Delete("watch_foo", true) }() go setHelper("watch_foo", "bar", c) - resp, err := c.Watch("watch_foo", 0, nil, nil) + resp, err := c.Watch("watch_foo", 0, false, nil, nil) if err != nil { t.Fatal(err) } - if !(resp.Key == "/watch_foo" && resp.Value == "bar") { + if !(resp.Node.Key == "/watch_foo" && resp.Node.Value == "bar") { t.Fatalf("Watch 1 failed: %#v", resp) } go setHelper("watch_foo", "bar", c) - resp, err = c.Watch("watch_foo", resp.ModifiedIndex, nil, nil) + resp, err = c.Watch("watch_foo", resp.Node.ModifiedIndex+1, false, nil, nil) if err != nil { t.Fatal(err) } - if !(resp.Key == "/watch_foo" && resp.Value == "bar") { + if !(resp.Node.Key == "/watch_foo" && resp.Node.Value == "bar") { t.Fatalf("Watch 2 failed: %#v", resp) } @@ -39,7 +39,7 @@ func TestWatch(t *testing.T) { go receiver(ch, stop) - _, err = c.Watch("watch_foo", 0, ch, stop) + _, err = c.Watch("watch_foo", 0, false, ch, stop) if err != ErrWatchStoppedByUser { t.Fatalf("Watch returned a non-user stop error") } @@ -48,26 +48,26 @@ func TestWatch(t *testing.T) { func TestWatchAll(t *testing.T) { c := NewClient(nil) defer func() { - c.DeleteAll("watch_foo") + c.Delete("watch_foo", true) }() go setHelper("watch_foo/foo", "bar", c) - resp, err := c.WatchAll("watch_foo", 0, nil, nil) + resp, err := c.Watch("watch_foo", 0, true, nil, nil) if err != nil { t.Fatal(err) } - if !(resp.Key == "/watch_foo/foo" && resp.Value == "bar") { + if !(resp.Node.Key == "/watch_foo/foo" && resp.Node.Value == "bar") { t.Fatalf("WatchAll 1 failed: %#v", resp) } go setHelper("watch_foo/foo", "bar", c) - resp, err = c.WatchAll("watch_foo", resp.ModifiedIndex, nil, nil) + resp, err = c.Watch("watch_foo", resp.Node.ModifiedIndex+1, true, nil, nil) if err != nil { t.Fatal(err) } - if !(resp.Key == "/watch_foo/foo" && resp.Value == "bar") { + if !(resp.Node.Key == "/watch_foo/foo" && resp.Node.Value == "bar") { t.Fatalf("WatchAll 2 failed: %#v", resp) } @@ -78,7 +78,7 @@ func TestWatchAll(t *testing.T) { go receiver(ch, stop) - _, err = c.WatchAll("watch_foo", 0, ch, stop) + _, err = c.Watch("watch_foo", 0, true, ch, stop) if err != ErrWatchStoppedByUser { t.Fatalf("Watch returned a non-user stop error") } diff --git a/third_party/github.com/gorilla/context/README.md b/third_party/github.com/gorilla/context/README.md index 8ee62b426..c60a31b05 100644 --- a/third_party/github.com/gorilla/context/README.md +++ b/third_party/github.com/gorilla/context/README.md @@ -1,5 +1,6 @@ context ======= +[![Build Status](https://travis-ci.org/gorilla/context.png?branch=master)](https://travis-ci.org/gorilla/context) gorilla/context is a general purpose registry for global request variables. diff --git a/third_party/github.com/gorilla/context/context.go b/third_party/github.com/gorilla/context/context.go index 35d65561f..12accb114 100644 --- a/third_party/github.com/gorilla/context/context.go +++ b/third_party/github.com/gorilla/context/context.go @@ -92,7 +92,7 @@ func Purge(maxAge int) int { datat = make(map[*http.Request]int64) } else { min := time.Now().Unix() - int64(maxAge) - for r, _ := range data { + for r := range data { if datat[r] < min { clear(r) count++ diff --git a/third_party/github.com/gorilla/mux/README.md b/third_party/github.com/gorilla/mux/README.md index f6db41ad8..e60301b03 100644 --- a/third_party/github.com/gorilla/mux/README.md +++ b/third_party/github.com/gorilla/mux/README.md @@ -1,5 +1,6 @@ mux === +[![Build Status](https://travis-ci.org/gorilla/mux.png?branch=master)](https://travis-ci.org/gorilla/mux) gorilla/mux is a powerful URL router and dispatcher. diff --git a/third_party/github.com/gorilla/mux/mux.go b/third_party/github.com/gorilla/mux/mux.go index ddc1acc63..ca51a011d 100644 --- a/third_party/github.com/gorilla/mux/mux.go +++ b/third_party/github.com/gorilla/mux/mux.go @@ -67,6 +67,14 @@ func (r *Router) Match(req *http.Request, match *RouteMatch) bool { func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { // Clean path to canonical form and redirect. if p := cleanPath(req.URL.Path); p != req.URL.Path { + + // Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query. + // This matches with fix in go 1.2 r.c. 4 for same problem. Go Issue: + // http://code.google.com/p/go/issues/detail?id=5252 + url := *req.URL + url.Path = p + p = url.String() + w.Header().Set("Location", p) w.WriteHeader(http.StatusMovedPermanently) return diff --git a/third_party/github.com/gorilla/mux/mux_test.go b/third_party/github.com/gorilla/mux/mux_test.go index 8789697f5..1a2a092df 100644 --- a/third_party/github.com/gorilla/mux/mux_test.go +++ b/third_party/github.com/gorilla/mux/mux_test.go @@ -22,6 +22,7 @@ type routeTest struct { shouldMatch bool // whether the request is expected to match the route at all } + func TestHost(t *testing.T) { // newRequestHost a new request with a method, url, and host header newRequestHost := func(method, url, host string) *http.Request { @@ -416,6 +417,15 @@ func TestQueries(t *testing.T) { path: "", shouldMatch: true, }, + { + title: "Queries route, match with a query string", + route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"), + request: newRequest("GET", "http://www.example.com/api?foo=bar&baz=ding"), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: true, + }, { title: "Queries route, bad query", route: new(Route).Queries("foo", "bar", "baz", "ding"), @@ -663,7 +673,7 @@ func testRoute(t *testing.T, test routeTest) { func TestKeepContext(t *testing.T) { func1 := func(w http.ResponseWriter, r *http.Request) {} - r := NewRouter() + r:= NewRouter() r.HandleFunc("/", func1).Name("func1") req, _ := http.NewRequest("GET", "http://localhost/", nil) @@ -688,6 +698,47 @@ func TestKeepContext(t *testing.T) { } + +type TestA301ResponseWriter struct { + hh http.Header + status int +} + +func (ho TestA301ResponseWriter) Header() http.Header { + return http.Header(ho.hh) +} + +func (ho TestA301ResponseWriter) Write( b []byte) (int, error) { + return 0, nil +} + +func (ho TestA301ResponseWriter) WriteHeader( code int ) { + ho.status = code +} + +func Test301Redirect(t *testing.T) { + m := make(http.Header) + + func1 := func(w http.ResponseWriter, r *http.Request) {} + func2 := func(w http.ResponseWriter, r *http.Request) {} + + r:= NewRouter() + r.HandleFunc("/api/", func2).Name("func2") + r.HandleFunc("/", func1).Name("func1") + + req, _ := http.NewRequest("GET", "http://localhost//api/?abc=def", nil) + + res := TestA301ResponseWriter{ + hh: m, + status : 0, + } + r.ServeHTTP(&res, req) + + if "http://localhost/api/?abc=def" != res.hh["Location"][0] { + t.Errorf("Should have complete URL with query string") + } +} + // https://plus.google.com/101022900381697718949/posts/eWy6DjFJ6uW func TestSubrouterHeader(t *testing.T) { expected := "func1 response" diff --git a/third_party/github.com/gorilla/mux/old_test.go b/third_party/github.com/gorilla/mux/old_test.go index 7e266bb69..42530590e 100644 --- a/third_party/github.com/gorilla/mux/old_test.go +++ b/third_party/github.com/gorilla/mux/old_test.go @@ -96,8 +96,8 @@ func TestRouteMatchers(t *testing.T) { method = "GET" headers = map[string]string{"X-Requested-With": "XMLHttpRequest"} resultVars = map[bool]map[string]string{ - true: map[string]string{"var1": "www", "var2": "product", "var3": "42"}, - false: map[string]string{}, + true: {"var1": "www", "var2": "product", "var3": "42"}, + false: {}, } } @@ -110,8 +110,8 @@ func TestRouteMatchers(t *testing.T) { method = "POST" headers = map[string]string{"Content-Type": "application/json"} resultVars = map[bool]map[string]string{ - true: map[string]string{"var4": "google", "var5": "product", "var6": "42"}, - false: map[string]string{}, + true: {"var4": "google", "var5": "product", "var6": "42"}, + false: {}, } }