From 880cd71df56729f8442ff2991eb6e9622ab09a80 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 09:43:14 -0700 Subject: [PATCH 01/10] add http timeout --- etcd.go | 20 +++++++++++++++++++- transporter.go | 13 ++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/etcd.go b/etcd.go index a2d840cc8..4f1a4112b 100644 --- a/etcd.go +++ b/etcd.go @@ -13,6 +13,7 @@ import ( "github.com/coreos/go-raft" "io/ioutil" "log" + "net" "net/http" "os" "strings" @@ -89,6 +90,7 @@ const ( const ( ELECTIONTIMTOUT = 200 * time.Millisecond HEARTBEATTIMEOUT = 50 * time.Millisecond + HTTPTIMEOUT = time.Second ) //------------------------------------------------------------------------------ @@ -263,12 +265,22 @@ func createTransporter(st int) transporter { switch st { case HTTP: - t.client = nil + t.https = false + + tr := &http.Transport{ + Dial: dialTimeout, + } + + t.client = &http.Client{ + Transport: tr, + } return t case HTTPS: fallthrough case HTTPSANDVERIFY: + t.https = true + tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile) if err != nil { @@ -280,6 +292,7 @@ func createTransporter(st int) transporter { Certificates: []tls.Certificate{tlsCert}, InsecureSkipVerify: true, }, + Dial: dialTimeout, DisableCompression: true, } @@ -291,6 +304,11 @@ func createTransporter(st int) transporter { return transporter{} } +// Dial with timeout +func dialTimeout(network, addr string) (net.Conn, error) { + return net.DialTimeout(network, addr, HTTPTIMEOUT) +} + // Start to listen and response raft command func startRaftTransport(port int, st int) { diff --git a/transporter.go b/transporter.go index 0eb6c5599..f49047372 100644 --- a/transporter.go +++ b/transporter.go @@ -12,10 +12,9 @@ import ( // Transporter layer for communication between raft nodes type transporter struct { - name string - // If https is used for server internal communcation, - // we will have a http client. Or it will be nil. client *http.Client + // https + https bool } // Sends AppendEntries RPCs to a peer when the server is the leader. @@ -104,22 +103,22 @@ func (t transporter) GetLeaderClientAddress() string { // Send server side POST request func (t transporter) Post(path string, body io.Reader) (*http.Response, error) { - if t.client != nil { + if t.https { resp, err := t.client.Post("https://"+path, "application/json", body) return resp, err } else { - resp, err := http.Post("http://"+path, "application/json", body) + resp, err := t.client.Post("http://"+path, "application/json", body) return resp, err } } // Send server side GET request func (t transporter) Get(path string) (*http.Response, error) { - if t.client != nil { + if t.https { resp, err := t.client.Get("https://" + path) return resp, err } else { - resp, err := http.Get("http://" + path) + resp, err := t.client.Get("http://" + path) return resp, err } } From 1349c377f27f65df20d19242c42a8b9b5bece69f Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 11:51:13 -0700 Subject: [PATCH 02/10] Create README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..21d2034fd --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +etcd +==== +## Overview + +## Start with examples + +### Set a node up and say hello world From ffa4d1a3c0f2bf5ce6b95709f5870592fc6d4b9c Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 14:51:18 -0700 Subject: [PATCH 03/10] add comment for HTTPTIMEOUT constant --- etcd.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etcd.go b/etcd.go index 4f1a4112b..c3cb06e9a 100644 --- a/etcd.go +++ b/etcd.go @@ -90,7 +90,8 @@ const ( const ( ELECTIONTIMTOUT = 200 * time.Millisecond HEARTBEATTIMEOUT = 50 * time.Millisecond - HTTPTIMEOUT = time.Second + // Timeout for internal raft http connection + HTTPTIMEOUT = time.Second ) //------------------------------------------------------------------------------ From c135c1e299cb19d6e9a22f1bae36f503bbe58341 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 14:56:16 -0700 Subject: [PATCH 04/10] add comment for HTTPTIMEOUT constant --- etcd.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etcd.go b/etcd.go index c3cb06e9a..45522d1ff 100644 --- a/etcd.go +++ b/etcd.go @@ -91,6 +91,8 @@ const ( ELECTIONTIMTOUT = 200 * time.Millisecond HEARTBEATTIMEOUT = 50 * time.Millisecond // Timeout for internal raft http connection + // The origin timeout for http is 45 seconds + // which is too long for our usage. HTTPTIMEOUT = time.Second ) From 8164d688cbe822e7395a708862d83fb41557b6a5 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 22:00:39 -0700 Subject: [PATCH 05/10] Update README.md --- README.md | 260 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 21d2034fd..43df187ee 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,262 @@ etcd ## Start with examples -### Set a node up and say hello world +### Setting up a node + +./etcd +This will bring up a node, which will be listening on internal port 7001 (for server communication) and external port 4001 (for client communication) + +Setting and Retrieving Values + +Let’s set the first key-value pair to the node. In this case our key is “/message” and our value is “Hello world”. + +#### Setting the value to a key + +```sh +curl http://127.0.0.1:4001/v1/keys/message -d value="Hello world" +``` + +```json +{"action":"SET","key":"/message","value":"Hello world","newKey":true,"index":3} +``` + +This response contains five fields. We will introduce three more fields as we try more commands. +1. The action of the request; we set the value via a post request, thus the action is “SET” + +2. The key of the request; we set “/message” to “Hello world!”, so the key field is “/message”. +Notice we use a file system like structure to represent the key-value pairs. So each key starts with “/”. + +3. The current value of the key; we set the value to “Hello world!”. + +4. If we set a new key; “/message” did not exist before, so this is a new key. + +5. Index field is the unique request index of the set request. Each sensitive request we send to the server will have a unique request index. The current sensitive request are “SET”, “DELETE” and “TESTANDSET”. All of these request will change the state of the key-value store system, thus they are sensitive. “GET”, “LIST” and “WATCH” are non-sensitive commands. Those commands will not change the state of the key-value store system. +You may notice that in this example the index is 3, although it is the first request you sent to the server. This is because there are some internal commands that also change the state of the server, we also need to assign them command indexes(Command used to add a server and sync the servers). + +#### Getting the value of a key + +```sh +curl http://127.0.0.1:4001/v1/keys/message +``` + +You should receive the response as +{"action":"GET","key":"/message","value":"Hello world","index":3} + +#### Changing the value of a key + +We change the value of “/message” from “Hello world” to “Hello etcd” + +```sh +curl http://127.0.0.1:4001/v1/keys/message -d value="Hello etcd" +``` + +```json +{"action":"SET","key":"/message","prevValue":"Hello world","value":"Hello etcd","index":4} +``` + +There is a new field in the response: prevValue. It is the value of the key before the change happened. + +#### Deleting a key + +```sh +curl http://127.0.0.1:4001/v1/keys/message -X DELETE +``` + +You should see the response as + +```json +{"action":"DELETE","key":"/message","prevValue":"Hello etcd","index":5} +``` + +#### Using time to live key + +```sh +curl http://127.0.0.1:4001/v1/keys/foo -d value=bar -d ttl=5 +``` + +You should see the similar response as (not exact same, they should have different expiration time) + +```json +{"action":"SET","key":"/foo","value":"bar","newKey":true,"expiration":"2013-07-11T20:31:12.156146039-07:00","ttl":4,"index":6} +``` + +There are the last two new fields in response. + +Expiration field is the time that this key will expire and be deleted. + +Ttl field is the time to live of the key, it can be derived from current time and expiration time. + +Now you can try to get the key by sending + +```sh +curl http://127.0.0.1:4001/v1/keys/foo +``` +You can expect the ttl is counting down and after 5 seconds you should see this, + +404 page not found + +which indicates the key has expired and was deleted. + +#### Watching a prefix + +Watch command can watch as a prefix path and get notification if any key changes after the prefix. + +In one terminal, we send a watch request: +```sh +curl http://127.0.0.1:4001/v1/watch/foo +``` + +Now, we are watching at the path prefix “/foo” and wait for any changes under this path. + +In another terminal, we set a key “/foo/foo” to “barbar” to see what will happen: + +```sh +curl http://127.0.0.1:4001/v1/keys/foo/foo -d value=barbar +``` + +The first terminal should get the notification and return with the same response as the set request. + +```json +{"action":"SET","key":"/foo/foo","value":"barbar","newKey":true,"index":7} +``` + +OK. Watch command can do more than this. We have index and in etcd we store the most recent 1000 responses by default, which allow us to watch for previous commands. + +Let us try to watch for the set command of index 6 again. + +```sh +curl http://127.0.0.1:4001/v1/watch/foo -d index=7 +``` + +You should see the watch command return immediately with the same response as previous. + +#### Trying TestAndSet + +Etcd servers will process all the command in sequence atomically, thus it can be used as a centralized decision making cluster. + +TestAndSet is the most basic operation to build distributed lock service and more interesting stuff. + +What it does is to test whether the given previous value is equal to the value of the key, if equal etcd will change the value of the key to the given value. + +Here is a simple example. +Let us create a key first. testAndSet=one +```sh +curl http://127.0.0.1:4001/v1/keys/testAndSet -d value=one +``` + +Let us try this +```sh +curl http://127.0.0.1:4001/v1/testAndSet/testAndSet -d prevValue=two -d value=three +``` + +This will try to test if the previous of the key is two, it is change it to three. + +The response should be +status code 400 +Test one==two fails + +which means testAndSet fails. + +Let try again + +```sh +curl http://127.0.0.1:4001/v1/testAndSet/testAndSet -d prevValue=one -d value=two +``` + +The response should be + +```json +{"action":"SET","key":"/testAndSet","prevValue":"one","value":"two","index":10} +``` + +We successfully change the value from “one” to “two”, since we give the correct previous value. + + +#### Listing directory +Last we provide a simple List command to list all the keys under a prefix path. + +Let us create some keys first. + +We already have /foo/foo=barbar + +We create another one /foo/foo_dir/foo=barbarbar + +```sh +http://127.0.0.1:4001/v1/keys/foo/foo_dir/bar -d value=barbarbar +``` + +Let us list them next. + +```sh +curl http://127.0.0.1:4001/v1/list/foo/ +``` + +We should see the response as + +```json +{"Key":"foo","Value":"barbar","Type":"f"},{"Key":"foo_dir","Value":".","Type":"d"}, +``` + +which meas foo is a key under /foo and the value is “barbar” and a key foo_dir is a directory. + +### Setting up a cluster of three machines +Next we can explore the power of etcd cluster. We use go-raft as the underlay distributed protocol which provide consistency and persistence of all the machines in the cluster. The will allow if the minor machine dies, the cluster will still be able to performance correctly. Also if most of the machines dead and restart, we will recover from the previous state of the cluster. + +Let us create 3 new machines. +The first one will be +We use -s to specify server port and -c to specify client port and -d to specify the directory to store the log and info of the node in the cluster + +```sh +./etcd -s 7001 -c 4001 -d nodes/node1 +``` + +We use -C to specify the Cluster + +Let the second one join it. +```sh +./etcd -c 4002 -s 7002 -C 127.0.0.1:7001 -d nod/node2 +``` + +And the third one: +```sh +./etcd -c 4003 -s 7003 -C 127.0.0.1:7001 -d nod/node3 +``` + +Let us add a key to the cluster of 3 nodes. + +```sh +curl http://127.0.0.1:4001/v1/keys/foo -d value=bar +``` + +```json +{"action":"SET","key":"/foo","value":"bar","newKey":true,"index":5} +``` + +Let us kill the leader of the cluster to see what will happen. + +Kill the first node which is the current leader + +Try to get the value from the other machine + +```sh +curl http://127.0.0.1:4002/v1/keys/foo +``` + +You should be able to see this + +```json +{"action":"GET","key":"/foo","value":"bar","index":5} +``` + +It succeed! + +OK. Next let us kill all the nodes to test persistence. And restart all the nodes use the same command before. + +Try +```sh +curl http://127.0.0.1:4002/v1/keys/foo again. +``` +You should able to see +```json +{"action":"GET","key":"/foo","value":"bar","index":5} +``` From c4ac81ea62938fc24ebc60a60cf8e7ce4ddd80a1 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 22:08:41 -0700 Subject: [PATCH 06/10] Update README.md --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 43df187ee..352149d4e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,10 @@ etcd ### Setting up a node +```sh ./etcd +``` + This will bring up a node, which will be listening on internal port 7001 (for server communication) and external port 4001 (for client communication) Setting and Retrieving Values @@ -96,7 +99,9 @@ curl http://127.0.0.1:4001/v1/keys/foo ``` You can expect the ttl is counting down and after 5 seconds you should see this, +```html 404 page not found +``` which indicates the key has expired and was deleted. @@ -155,8 +160,9 @@ curl http://127.0.0.1:4001/v1/testAndSet/testAndSet -d prevValue=two -d value=th This will try to test if the previous of the key is two, it is change it to three. The response should be -status code 400 +```html Test one==two fails +``` which means testAndSet fails. From 6ef625faec9c26a599bf3aecd8a9fff0bf70e7aa Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 22:10:23 -0700 Subject: [PATCH 07/10] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 352149d4e..4f512a84b 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,10 @@ etcd This will bring up a node, which will be listening on internal port 7001 (for server communication) and external port 4001 (for client communication) -Setting and Retrieving Values +#### Setting the value to a key Let’s set the first key-value pair to the node. In this case our key is “/message” and our value is “Hello world”. -#### Setting the value to a key - ```sh curl http://127.0.0.1:4001/v1/keys/message -d value="Hello world" ``` From 57bb8bf3f544220d358f8e43d369d670b586a234 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 11 Jul 2013 22:32:40 -0700 Subject: [PATCH 08/10] Update README.md --- README.md | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4f512a84b..696574783 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This will bring up a node, which will be listening on internal port 7001 (for se #### Setting the value to a key -Let’s set the first key-value pair to the node. In this case our key is “/message” and our value is “Hello world”. +Let’s set the first key-value pair to the node. In this case the key is `/message` and the value is `Hello world`. ```sh curl http://127.0.0.1:4001/v1/keys/message -d value="Hello world" @@ -25,16 +25,16 @@ curl http://127.0.0.1:4001/v1/keys/message -d value="Hello world" ``` This response contains five fields. We will introduce three more fields as we try more commands. -1. The action of the request; we set the value via a post request, thus the action is “SET” +1. The action of the request; we set the value via a POST request, thus the action is `SET`. -2. The key of the request; we set “/message” to “Hello world!”, so the key field is “/message”. -Notice we use a file system like structure to represent the key-value pairs. So each key starts with “/”. +2. The key of the request; we set `/message` to `Hello world!`, so the key field is `/message`. +Notice we use a file system like structure to represent the key-value pairs. So each key starts with `/`. -3. The current value of the key; we set the value to “Hello world!”. +3. The current value of the key; we set the value to`Hello world`. -4. If we set a new key; “/message” did not exist before, so this is a new key. +4. If we set a new key; `/message` did not exist before, so this is a new key. -5. Index field is the unique request index of the set request. Each sensitive request we send to the server will have a unique request index. The current sensitive request are “SET”, “DELETE” and “TESTANDSET”. All of these request will change the state of the key-value store system, thus they are sensitive. “GET”, “LIST” and “WATCH” are non-sensitive commands. Those commands will not change the state of the key-value store system. +5. Index field is the unique request index of the set request. Each sensitive request we send to the server will have a unique request index. The current sensitive request are `SET`, `DELETE` and `TESTANDSET`. All of these request will change the state of the key-value store system, thus they are sensitive. `GET`, `LIST` and `WATCH` are non-sensitive commands. Those commands will not change the state of the key-value store system. You may notice that in this example the index is 3, although it is the first request you sent to the server. This is because there are some internal commands that also change the state of the server, we also need to assign them command indexes(Command used to add a server and sync the servers). #### Getting the value of a key @@ -44,11 +44,12 @@ curl http://127.0.0.1:4001/v1/keys/message ``` You should receive the response as +```json {"action":"GET","key":"/message","value":"Hello world","index":3} - +``` #### Changing the value of a key -We change the value of “/message” from “Hello world” to “Hello etcd” +We change the value of `/message` from `Hello world` to `Hello etcd` ```sh curl http://127.0.0.1:4001/v1/keys/message -d value="Hello etcd" @@ -112,9 +113,9 @@ In one terminal, we send a watch request: curl http://127.0.0.1:4001/v1/watch/foo ``` -Now, we are watching at the path prefix “/foo” and wait for any changes under this path. +Now, we are watching at the path prefix `/foo` and wait for any changes under this path. -In another terminal, we set a key “/foo/foo” to “barbar” to see what will happen: +In another terminal, we set a key `/foo/foo` to `barbar` to see what will happen: ```sh curl http://127.0.0.1:4001/v1/keys/foo/foo -d value=barbar @@ -138,19 +139,19 @@ You should see the watch command return immediately with the same response as pr #### Trying TestAndSet -Etcd servers will process all the command in sequence atomically, thus it can be used as a centralized decision making cluster. +Etcd servers will process all the command in sequence atomically, thus it can be used as a centralized decision making cluster. TestAndSet is the most basic operation to build distributed lock service and more interesting stuff. What it does is to test whether the given previous value is equal to the value of the key, if equal etcd will change the value of the key to the given value. Here is a simple example. -Let us create a key first. testAndSet=one +Let us create a key-value pair first: `testAndSet=one`. ```sh curl http://127.0.0.1:4001/v1/keys/testAndSet -d value=one ``` -Let us try this +Let us try a invaild `TestAndSet` command. ```sh curl http://127.0.0.1:4001/v1/testAndSet/testAndSet -d prevValue=two -d value=three ``` @@ -162,9 +163,9 @@ The response should be Test one==two fails ``` -which means testAndSet fails. +which means `testAndSet` failed. -Let try again +Let us try a vaild one. ```sh curl http://127.0.0.1:4001/v1/testAndSet/testAndSet -d prevValue=one -d value=two @@ -184,9 +185,9 @@ Last we provide a simple List command to list all the keys under a prefix path. Let us create some keys first. -We already have /foo/foo=barbar +We already have `/foo/foo=barbar` -We create another one /foo/foo_dir/foo=barbarbar +We create another one `/foo/foo_dir/foo=barbarbar` ```sh http://127.0.0.1:4001/v1/keys/foo/foo_dir/bar -d value=barbarbar @@ -201,16 +202,18 @@ curl http://127.0.0.1:4001/v1/list/foo/ We should see the response as ```json -{"Key":"foo","Value":"barbar","Type":"f"},{"Key":"foo_dir","Value":".","Type":"d"}, +{"Key":"foo","Value":"barbar","Type":"f"} {"Key":"foo_dir","Value":".","Type":"d"} ``` -which meas foo is a key under /foo and the value is “barbar” and a key foo_dir is a directory. +which meas `foo=barbar` is a key-value pair under `/foo` and `foo_dir` is a directory. ### Setting up a cluster of three machines Next we can explore the power of etcd cluster. We use go-raft as the underlay distributed protocol which provide consistency and persistence of all the machines in the cluster. The will allow if the minor machine dies, the cluster will still be able to performance correctly. Also if most of the machines dead and restart, we will recover from the previous state of the cluster. Let us create 3 new machines. + The first one will be + We use -s to specify server port and -c to specify client port and -d to specify the directory to store the log and info of the node in the cluster ```sh @@ -261,7 +264,7 @@ OK. Next let us kill all the nodes to test persistence. And restart all the node Try ```sh -curl http://127.0.0.1:4002/v1/keys/foo again. +curl http://127.0.0.1:4002/v1/keys/foo ``` You should able to see ```json From e6354cdccf924fa10ce02610895696acf5838926 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Fri, 12 Jul 2013 08:35:09 -0700 Subject: [PATCH 09/10] Fix a typo origin->original --- etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etcd.go b/etcd.go index 45522d1ff..d3a964ac7 100644 --- a/etcd.go +++ b/etcd.go @@ -91,7 +91,7 @@ const ( ELECTIONTIMTOUT = 200 * time.Millisecond HEARTBEATTIMEOUT = 50 * time.Millisecond // Timeout for internal raft http connection - // The origin timeout for http is 45 seconds + // The original timeout for http is 45 seconds // which is too long for our usage. HTTPTIMEOUT = time.Second ) From 89bacedf3e88810388ba657e95f86f323691fa68 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Fri, 12 Jul 2013 08:44:40 -0700 Subject: [PATCH 10/10] change https bool to scheme string --- etcd.go | 4 ++-- transporter.go | 19 ++++--------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/etcd.go b/etcd.go index d3a964ac7..fbf868a70 100644 --- a/etcd.go +++ b/etcd.go @@ -268,7 +268,7 @@ func createTransporter(st int) transporter { switch st { case HTTP: - t.https = false + t.scheme = "http://" tr := &http.Transport{ Dial: dialTimeout, @@ -282,7 +282,7 @@ func createTransporter(st int) transporter { case HTTPS: fallthrough case HTTPSANDVERIFY: - t.https = true + t.scheme = "https://" tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile) diff --git a/transporter.go b/transporter.go index f49047372..c68f062da 100644 --- a/transporter.go +++ b/transporter.go @@ -13,8 +13,8 @@ import ( // Transporter layer for communication between raft nodes type transporter struct { client *http.Client - // https - https bool + // scheme + scheme string } // Sends AppendEntries RPCs to a peer when the server is the leader. @@ -102,23 +102,12 @@ func (t transporter) GetLeaderClientAddress() string { // Send server side POST request func (t transporter) Post(path string, body io.Reader) (*http.Response, error) { - - if t.https { - resp, err := t.client.Post("https://"+path, "application/json", body) + resp, err := t.client.Post(t.scheme + path, "application/json", body) return resp, err - } else { - resp, err := t.client.Post("http://"+path, "application/json", body) - return resp, err - } } // Send server side GET request func (t transporter) Get(path string) (*http.Response, error) { - if t.https { - resp, err := t.client.Get("https://" + path) + resp, err := t.client.Get(t.scheme + path) return resp, err - } else { - resp, err := t.client.Get("http://" + path) - return resp, err - } }