diff --git a/Documentation/api.md b/Documentation/api.md new file mode 100644 index 000000000..32d65bc32 --- /dev/null +++ b/Documentation/api.md @@ -0,0 +1,797 @@ +# etcd API + +## Running a Single Machine Cluster + +These examples will use a single machine cluster to show you the basics of the etcd REST API. +Let's start etcd: + +```sh +./etcd -data-dir machine0 -name machine0 +``` + +This will bring up etcd listening on port 4001 for client communication and on port 7001 for server-to-server communication. +The `-data-dir machine0` argument tells etcd to write machine configuration, logs and snapshots to the `./machine0/` directory. +The `-name machine` tells the rest of the cluster that this machine is named machine0. + +## Key Space Operations + +The primary API of etcd is hierarchical key space. +There are directories and keys which are generically referred to as "nodes". + +### Setting the value to a key + +Let’s set the first key-value pair to the datastore. +In this case the key is `/message` and the value is `Hello world`. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/message -X PUT -d value="Hello world" +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 2, + "key": "/message", + "modifiedIndex": 2, + "value": "Hello world" + } +} +``` + +The response object contains several attributes: + +1. `action`: the action of the request that was just made. +The request attempted to modify `node.value` via a `PUT` HTTP request, thus the value of action is `set`. + +2. `node.key`: the HTTP path the to which the request was made. +We set `/message` to `Hello world`, so the key field is `/message`. +Etcd uses a file-system-like structure to represent the key-value pairs, therefore all keys start with `/`. + +3. `node.value`: the value of the key after resolving the request. +In this case, a successful request was made that attempted to change the node's value to `Hello world`. + +4. `node.createdIndex`: an index is a unique, monotonically-incrementing integer created for each change to etcd. +This specific index reflects at which point in the etcd state machine a given key was created. +You may notice that in this example the index is `2` even though it is the first request you sent to the server. +This is because there are internal commands that also change the state behind the scenes like adding and syncing servers. + +5. `node.modifiedIndex`: like `node.createdIndex`, this attribute is also an etcd index. +Actions that cause the value to change include `set`, `delete`, `update`, `create` and `compareAndSwap`. +Since the `get` and `watch` commands do not change state in the store, they do not change the value of `node.modifiedIndex`. + + +### Response Headers + +etcd includes a few HTTP headers that provide global information about the etcd cluster that serviced a request: + +``` +X-Etcd-Index: 35 +X-Raft-Index: 5398 +X-Raft-Term: 0 +``` + +- `X-Etcd-Index` is the current etcd index as explained above. +- `X-Raft-Index` is similar to the etcd index but is for the underlying raft protocol +- `X-Raft-Term` this number will increase when an etcd master election happens. If this number is increasing rapdily you may need to tune the election timeout. See the [tuning][tuning] section for details. + +[tuning]: #tuning + +### Get the value of a key + +We can get the value that we just set in `/message` by issuing a `GET` request: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/message +``` + +```json +{ + "action": "get", + "node": { + "createdIndex": 2, + "key": "/message", + "modifiedIndex": 2, + "value": "Hello world" + } +} +``` + + +### Changing the value of a key + +You can change the value of `/message` from `Hello world` to `Hello etcd` with another `PUT` request to the key: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello etcd" +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 3, + "key": "/message", + "modifiedIndex": 3, + "value": "Hello etcd" + } +} +``` + +### Deleting a key + +You can remove the `/message` key with a `DELETE` request: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/message -XDELETE +``` + +```json +{ + "action": "delete", + "node": { + "createdIndex": 3, + "key": "/message", + "modifiedIndex": 4 + } +} +``` + + +### Using key TTL + +Keys in etcd can be set to expire after a specified number of seconds. +You can do this by setting a TTL (time to live) on the key when send a `PUT` request: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -d ttl=5 +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 5, + "expiration": "2013-12-04T12:01:21.874888581-08:00", + "key": "/foo", + "modifiedIndex": 5, + "ttl": 5, + "value": "bar" + } +} +``` + +Note the two new fields in response: + +1. The `expiration` is the time that this key will expire and be deleted. + +2. The `ttl` is the time to live for the key, in seconds. + +_NOTE_: Keys can only be expired by a cluster leader so if a machine gets disconnected from the cluster, its keys will not expire until it rejoins. + +Now you can try to get the key by sending a `GET` request: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo +``` + +If the TTL has expired, the key will be deleted, and you will be returned a 100. + +```json +{ + "cause": "/foo", + "errorCode": 100, + "index": 6, + "message": "Key Not Found" +} +``` + +### Waiting for a change + +We can watch for a change on a key and receive a notification by using long polling. +This also works for child keys by passing `recursive=true` in curl. + +In one terminal, we send a get request with `wait=true` : + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo?wait=true +``` + +Now we are waiting for any changes at path `/foo`. + +In another terminal, we set a key `/foo` with value `bar`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar +``` + +The first terminal should get the notification and return with the same response as the set request. + +```json +{ + "action": "set", + "node": { + "createdIndex": 7, + "key": "/foo", + "modifiedIndex": 7, + "value": "bar" + } +} +``` + +However, the watch command can do more than this. +Using the the index we can watch for commands that has happened in the past. +This is useful for ensuring you don't miss events between watch commands. + +Let's try to watch for the set command of index 7 again: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo?wait=true\&waitIndex=7 +``` + +The watch command returns immediately with the same response as previous. + + +### Atomically Creating In-Order Keys + +Using the `POST` on a directory you can create keys with key names that are created in-order. +This can be used in a variety of useful patterns like implementing queues of keys that need to be processed in strict order. +An example use case is the [locking module][lockmod] which uses it to ensure clients get fair access to a mutex. + +Creating an in-order key is easy + +```sh +curl -X POST http://127.0.0.1:4001/v2/keys/queue -d value=Job1 +``` + +```json +{ + "action": "create", + "node": { + "createdIndex": 6, + "key": "/queue/6", + "modifiedIndex": 6, + "value": "Job1" + } +} +``` + +If you create another entry some time later it is guaranteed to have a key name that is greater than the previous key. +Also note the key names use the global etcd index so the next key can be more than `previous + 1`. + +```sh +curl -X POST http://127.0.0.1:4001/v2/keys/queue -d value=Job2 +``` + +```json +{ + "action": "create", + "node": { + "createdIndex": 29, + "key": "/queue/29", + "modifiedIndex": 29, + "value": "Job2" + } +} +``` + +To enumerate the in-order keys as a sorted list, use the "sorted" parameter. + +```sh +curl -s -X GET 'http://127.0.0.1:4001/v2/keys/queue?recursive=true&sorted=true' +``` + +```json +{ + "action": "get", + "node": { + "createdIndex": 2, + "dir": true, + "key": "/queue", + "modifiedIndex": 2, + "nodes": [ + { + "createdIndex": 2, + "key": "/queue/2", + "modifiedIndex": 2, + "value": "Job1" + }, + { + "createdIndex": 3, + "key": "/queue/3", + "modifiedIndex": 3, + "value": "Job2" + } + ] + } +} +``` + +[lockmod]: #lock + + +### Using a directory TTL + +Like keys, directories in etcd can be set to expire after a specified number of seconds. +You can do this by setting a TTL (time to live) on a directory when it is created with a `PUT`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d ttl=30 -d dir=true +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 17, + "dir": true, + "expiration": "2013-12-11T10:37:33.689275857-08:00", + "key": "/newdir", + "modifiedIndex": 17, + "ttl": 30 + } +} +``` + +The directories TTL can be refreshed by making an update. +You can do this by making a PUT with `prevExist=true` and a new TTL. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d ttl=30 -d dir=true -d prevExist=true +``` + +Keys that are under this directory work as usual, but when the directory expires a watcher on a key under the directory will get an expire event: + +```sh +curl -X GET http://127.0.0.1:4001/v2/keys/dir/asdf\?consistent\=true\&wait\=true +``` + +```json +{ + "action": "expire", + "node": { + "createdIndex": 8, + "key": "/dir", + "modifiedIndex": 15 + } +} +``` + +### Atomic Compare-and-Swap + +Etcd can be used as a centralized coordination service in a cluster and `CompareAndSwap` (CAS) is the most basic operation used to build a distributed lock service. + +This command will set the value of a key only if the client-provided conditions are equal to the current conditions. + +The current comparable conditions are: + +1. `prevValue` - checks the previous value of the key. + +2. `prevIndex` - checks the previous index of the key. + +3. `prevExist` - checks existence of the key: if `prevExist` is true, it is a `update` request; if prevExist is `false`, it is a `create` request. + +Here is a simple example. +Let's create a key-value pair first: `foo=one`. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=one +``` + +Let's try some invalid `CompareAndSwap` commands first. + +Trying to set this existing key with `prevExist=false` fails as expected: +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo?prevExist=false -XPUT -d value=three +``` + +The error code explains the problem: + +```json +{ + "cause": "/foo", + "errorCode": 105, + "index": 39776, + "message": "Already exists" +} +``` + +Now lets provide a `prevValue` parameter: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo?prevValue=two -XPUT -d value=three +``` + +This will try to compare the previous value of the key and the previous value we provided. If they are equal, the value of the key will change to three. + +```json +{ + "cause": "[two != one] [0 != 8]", + "errorCode": 101, + "index": 8, + "message": "Test Failed" +} +``` + +which means `CompareAndSwap` failed. + +Let's try a valid condition: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo?prevValue=one -XPUT -d value=two +``` + +The response should be + +```json +{ + "action": "compareAndSwap", + "node": { + "createdIndex": 8, + "key": "/foo", + "modifiedIndex": 9, + "value": "two" + } +} +``` + +We successfully changed the value from "one" to "two" since we gave the correct previous value. + +### Creating Directories + +In most cases directories for a key are automatically created. +But, there are cases where you will want to create a directory or remove one. + +Creating a directory is just like a key only you cannot provide a value and must add the `dir=true` parameter. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d dir=true +``` +```json +{ + "action": "set", + "node": { + "createdIndex": 30, + "dir": true, + "key": "/dir", + "modifiedIndex": 30 + } +} +``` + +### Listing a directory + +In etcd we can store two types of things: keys and directories. +Keys store a single string value. +Directories store a set of keys and/or other directories. + +In this example, let's first create some keys: + +We already have `/foo=two` so now we'll create another one called `/foo_dir/foo` with the value of `bar`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo_dir/foo -XPUT -d value=bar +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 2, + "key": "/foo_dir/foo", + "modifiedIndex": 2, + "value": "bar" + } +} +``` + +Now we can list the keys under root `/`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/ +``` + +We should see the response as an array of items: + +```json +{ + "action": "get", + "node": { + "dir": true, + "key": "/", + "nodes": [ + { + "createdIndex": 2, + "dir": true, + "key": "/foo_dir", + "modifiedIndex": 2 + } + ] + } +} +``` + +Here we can see `/foo` is a key-value pair under `/` and `/foo_dir` is a directory. +We can also recursively get all the contents under a directory by adding `recursive=true`. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/?recursive=true +``` + +```json +{ + "action": "get", + "node": { + "dir": true, + "key": "/", + "nodes": [ + { + "createdIndex": 2, + "dir": true, + "key": "/foo_dir", + "modifiedIndex": 2, + "nodes": [ + { + "createdIndex": 2, + "key": "/foo_dir/foo", + "modifiedIndex": 2, + "value": "bar" + } + ] + } + ] + } +} +``` + + +### Deleting a Directory + +Now let's try to delete the directory `/foo_dir`. + +You can remove an empty directory using the `DELETE` verb and the `dir=true` parameter. + +```sh +curl -L -X DELETE 'http://127.0.0.1:4001/v2/keys/dir?dir=true' +``` +```json +{ + "action": "delete", + "node": { + "createdIndex": 30, + "dir": true, + "key": "/dir", + "modifiedIndex": 31 + } +} +``` + +To delete a directory that holds keys, you must add `recursive=true`. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/dir?recursive=true -XDELETE +``` + +```json +{ + "action": "delete", + "node": { + "createdIndex": 10, + "dir": true, + "key": "/dir", + "modifiedIndex": 11 + } +} +``` + +### Creating a hidden node + +We can create a hidden key-value pair or directory by add a `_` prefix. +The hidden item will not be listed when sending a `GET` request for a directory. + +First we'll add a hidden key named `/_message`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/_message -XPUT -d value="Hello hidden world" +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 3, + "key": "/_message", + "modifiedIndex": 3, + "value": "Hello hidden world" + } +} +``` + + +Next we'll add a regular key named `/message`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello world" +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 4, + "key": "/message", + "modifiedIndex": 4, + "value": "Hello world" + } +} +``` + +Now let's try to get a listing of keys under the root directory, `/`: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/ +``` + +```json +{ + "action": "get", + "node": { + "dir": true, + "key": "/", + "nodes": [ + { + "createdIndex": 2, + "dir": true, + "key": "/foo_dir", + "modifiedIndex": 2 + }, + { + "createdIndex": 4, + "key": "/message", + "modifiedIndex": 4, + "value": "Hello world" + } + ] + } +} +``` + +Here we see the `/message` key but our hidden `/_message` key is not returned. + +## Statistics + +An etcd cluster keeps track of a number of stastics including latency, bandwidth and uptime. +These statistics are used in the `/mod/dashboard` to generate tables and graphs about the cluster state. + +### Leader Statistics + +The leader has a view of the entire cluster and keeps track of two interesting statistics: latency to each peer in the cluster and the number of failed and successful Raft RPC requests. +You can find grab these stastistics from the `/v2/stats/leader` endpoint: + +```sh +curl -L 127.0.0.1:4001/v2/stats/leader +``` + +```json +{ + "followers": { + "etcd-node1": { + "counts": { + "fail": 1212, + "success": 4163176 + }, + "latency": { + "average": 2.7206299430775007, + "current": 1.486487, + "maximum": 2018.410279, + "minimum": 1.011763, + "standardDeviation": 6.246990702203536 + } + }, + "etcd-node3": { + "counts": { + "fail": 1378, + "success": 4164598 + }, + "latency": { + "average": 2.707100125761001, + "current": 1.666258, + "maximum": 1409.054765, + "minimum": 0.998415, + "standardDeviation": 5.910089773061448 + } + } + }, + "leader": "etcd-node2" +} +``` + +### Self Statistics + +Each node keeps a number of internal statistics: + +- `leaderInfo.leader`: name of the current leader machine +- `leaderInfo.uptime`: amount of time the leader has been leader +- `name`: this machine's name +- `recvAppendRequestCnt`: number of append requests this node has processed +- `recvBandwidthRate`: number of bytes per second this node is receiving (follower only) +- `recvPkgRate`: number of requests per second this node is receiving (follower only) +- `sendAppendRequestCnt`: number of requests that this node has sent +- `sendBandwidthRate`: number of bytes per second this node is receiving (leader only) +- `sendPkgRate`: number of requests per second this node is receiving (leader only) +- `state`: either leader or folower +- `startTime`: the time when this node was started + +This is an example response from a follower machine: + +```sh +curl -L 127.0.0.1:4001/v2/stats/self +``` + +```json +{ + "leaderInfo": { + "leader": "machine1", + "uptime": "1m18.544996775s" + }, + "name": "machine0", + "recvAppendRequestCnt": 5871307, + "recvBandwidthRate": 630.3121596542599, + "recvPkgRate": 19.272654323628185, + "sendAppendRequestCnt": 3175763, + "startTime": "2014-01-01T15:26:24.96569404Z", + "state": "follower" +} +``` + +And this is an example response from a leader machine: + +```sh +curl -L 127.0.0.1:4001/v2/stats/self +``` + +``` +{ + "leaderInfo": { + "leader": "machine0", + "uptime": "24.648619798s" + }, + "name": "machine0", + "recvAppendRequestCnt": 5901116, + "sendAppendRequestCnt": 3212344, + "sendBandwidthRate": 1254.3151237301615, + "sendPkgRate": 38.71342974475808, + "startTime": "2014-01-01T15:26:24.96569404Z", + "state": "leader" +} +``` + +### Store Statistics + +The store statistics include information about the operations that this node has handled. + +Operations that modify the store's state like create, delete, set and update are seen by the entire cluster and the number will increase on all nodes. +Operations like get and watch are node local and will only be seen on this node. + +```sh +curl -L 127.0.0.1:4001/v2/stats/store +``` + +```json +{ + "compareAndSwapFail": 0, + "compareAndSwapSuccess": 0, + "createFail": 0, + "createSuccess": 2, + "deleteFail": 0, + "deleteSuccess": 0, + "expireCount": 0, + "getsFail": 4, + "getsSuccess": 75, + "setsFail": 2, + "setsSuccess": 4, + "updateFail": 0, + "updateSuccess": 0, + "watchers": 0 +} +``` diff --git a/Documentation/clients-matrix.md b/Documentation/clients-matrix.md new file mode 100644 index 000000000..e622062d3 --- /dev/null +++ b/Documentation/clients-matrix.md @@ -0,0 +1,46 @@ +# Client libraries support matrix for etcd + +As etcd features support is really uneven between client libraries, a compatibility matrix can be important. +We will consider in detail only the features of clients supporting the v2 API. Clients still supporting the v1 API *only* are listed below. + +## v1-only clients + +Clients supporting only the API version 1 + +- [justinsb/jetcd](https://github.com/justinsb/jetcd) Java +- [transitorykris/etcd-py](https://github.com/transitorykris/etcd-py) Python +- [russellhaering/txetcd](https://github.com/russellhaering/txetcd) Python +- [iconara/etcd-rb](https://github.com/iconara/etcd-rb) Ruby +- [jpfuentes2/etcd-ruby](https://github.com/jpfuentes2/etcd-ruby) Ruby +- [aterreno/etcd-clojure](https://github.com/aterreno/etcd-clojure) Clojure +- [marshall-lee/etcd.erl](https://github.com/marshall-lee/etcd.erl) Erlang + + +## v2 clients + +The v2 API has a lot of features, we will categorize them in a few categories: + +- **HTTPS Auth**: Support for SSL-certificate based authentication +- **Reconnect**: If the client is able to reconnect automatically to another server if one fails. +- **Mod/Lock**: Support for the locking module +- **Mod/Leader**: Support for the leader election module +- **GET,PUT,POST,DEL Features**: Support for all the modifiers when calling the etcd server with said HTTP method. + + +### Supported features matrix + +| Client| [go-etcd](https://github.com/coreos/go-etcd) | [jetcd](https://github.com/diwakergupta/jetcd) | [python-etcd](https://github.com/jplana/python-etcd) | [python-etcd-client](https://github.com/dsoprea/PythonEtcdClient) | [node-etcd](https://github.com/stianeikeland/node-etcd) | [nodejs-etcd](https://github.com/lavagetto/nodejs-etcd) | [etcd-ruby](https://github.com/ranjib/etcd-ruby) | [etcd-api](https://github.com/jdarcy/etcd-api) | [cetcd](https://github.com/dwwoelfel/cetcd) | [clj-etcd](https://github.com/rthomas/clj-etcd) | +| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| **HTTPS Auth** | Y | Y | Y | - | Y | Y | - | - | - | - | +| **Reconnect** | Y | - | Y | Y | - | - | - | Y | - | - | +| **Mod/Lock** | Y | - | Y | Y | - | - | - | - | - | - | +| **Mod/Leader** | Y | - | - | Y | - | - | - | - | - | - | +| **GET Features** | F | B | F | F | F | F | F | B | F | G | +| **PUT Features** | F | B | F | F | F | F | F | G | F | G | +| **POST Features** | F | - | F | F | - | F | F | - | - | - | +| **DEL Features** | F | B | F | F | F | F | F | B | G | B | + +**Legend** + +**F**: Full support **G**: Good support **B**: Basic support +**Y**: Feature supported **-**: Feature not supported diff --git a/Documentation/clustering.md b/Documentation/clustering.md new file mode 100644 index 000000000..e9ffe8d18 --- /dev/null +++ b/Documentation/clustering.md @@ -0,0 +1,174 @@ +## Clustering + +### Example cluster of three machines + +Let's explore the use of etcd clustering. +We use Raft as the underlying distributed protocol which provides consistency and persistence of the data across all of the etcd instances. + +Let start by creating 3 new etcd instances. + +We use `-peer-addr` to specify server port and `-addr` to specify client port and `-data-dir` to specify the directory to store the log and info of the machine in the cluster: + +```sh +./etcd -peer-addr 127.0.0.1:7001 -addr 127.0.0.1:4001 -data-dir machines/machine1 -name machine1 +``` + +**Note:** If you want to run etcd on an external IP address and still have access locally, you'll need to add `-bind-addr 0.0.0.0` so that it will listen on both external and localhost addresses. +A similar argument `-peer-bind-addr` is used to setup the listening address for the server port. + +Let's join two more machines to this cluster using the `-peers` argument: + +```sh +./etcd -peer-addr 127.0.0.1:7002 -addr 127.0.0.1:4002 -peers 127.0.0.1:7001 -data-dir machines/machine2 -name machine2 +./etcd -peer-addr 127.0.0.1:7003 -addr 127.0.0.1:4003 -peers 127.0.0.1:7001 -data-dir machines/machine3 -name machine3 +``` + +We can retrieve a list of machines in the cluster using the HTTP API: + +```sh +curl -L http://127.0.0.1:4001/v2/machines +``` + +We should see there are three machines in the cluster + +``` +http://127.0.0.1:4001, http://127.0.0.1:4002, http://127.0.0.1:4003 +``` + +The machine list is also available via the main key API: + +```sh +curl -L http://127.0.0.1:4001/v2/keys/_etcd/machines +``` + +```json +{ + "action": "get", + "node": { + "createdIndex": 1, + "dir": true, + "key": "/_etcd/machines", + "modifiedIndex": 1, + "nodes": [ + { + "createdIndex": 1, + "key": "/_etcd/machines/machine1", + "modifiedIndex": 1, + "value": "raft=http://127.0.0.1:7001&etcd=http://127.0.0.1:4001" + }, + { + "createdIndex": 2, + "key": "/_etcd/machines/machine2", + "modifiedIndex": 2, + "value": "raft=http://127.0.0.1:7002&etcd=http://127.0.0.1:4002" + }, + { + "createdIndex": 3, + "key": "/_etcd/machines/machine3", + "modifiedIndex": 3, + "value": "raft=http://127.0.0.1:7003&etcd=http://127.0.0.1:4003" + } + ] + } +} +``` + +We can also get the current leader in the cluster: + +``` +curl -L http://127.0.0.1:4001/v2/leader +``` + +The first server we set up should still be the leader unless it has died during these commands. + +``` +http://127.0.0.1:7001 +``` + +Now we can do normal SET and GET operations on keys as we explored earlier. + +```sh +curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar +``` + +```json +{ + "action": "set", + "node": { + "createdIndex": 4, + "key": "/foo", + "modifiedIndex": 4, + "value": "bar" + } +} +``` + + +### Killing Nodes in the Cluster + +Now if we kill the leader of the cluster, we can get the value from one of the other two machines: + +```sh +curl -L http://127.0.0.1:4002/v2/keys/foo +``` + +We can also see that a new leader has been elected: + +``` +curl -L http://127.0.0.1:4002/v2/leader +``` + +``` +http://127.0.0.1:7002 +``` + +or + +``` +http://127.0.0.1:7003 +``` + + +### Testing Persistence + +Next we'll kill all the machines to test persistence. +Type `CTRL-C` on each terminal and then rerun the same command you used to start each machine. + +Your request for the `foo` key will return the correct value: + +```sh +curl -L http://127.0.0.1:4002/v2/keys/foo +``` + +```json +{ + "action": "get", + "node": { + "createdIndex": 4, + "key": "/foo", + "modifiedIndex": 4, + "value": "bar" + } +} +``` + + +### Using HTTPS between servers + +In the previous example we showed how to use SSL client certs for client-to-server communication. +Etcd can also do internal server-to-server communication using SSL client certs. +To do this just change the `-*-file` flags to `-peer-*-file`. + +If you are using SSL for server-to-server communication, you must use it on all instances of etcd. + + +### What size cluster should I use? + +Every command the client sends to the master is broadcast to all of the followers. +The command is not committed until the majority of the cluster peers receive that command. + +Because of this majority voting property, the ideal cluster should be kept small to keep speed up and be made up of an odd number of peers. + +Odd numbers are good because if you have 8 peers the majority will be 5 and if you have 9 peers the majority will still be 5. +The result is that an 8 peer cluster can tolerate 3 peer failures and a 9 peer cluster can tolerate 4 machine failures. +And in the best case when all 9 peers are responding the cluster will perform at the speed of the fastest 5 machines. diff --git a/Documentation/external-documentation.md b/Documentation/configuration.md similarity index 97% rename from Documentation/external-documentation.md rename to Documentation/configuration.md index e48140ce8..e265f522f 100644 --- a/Documentation/external-documentation.md +++ b/Documentation/configuration.md @@ -41,7 +41,6 @@ configuration files. * `-v` - Enable verbose logging. Defaults to `false`. * `-vv` - Enable very verbose logging. Defaults to `false`. * `-version` - Print the version and exit. -* `-web-url` - The hostname:port of web interface. ## Configuration File @@ -66,7 +65,6 @@ name = "default-name" snapshot = false verbose = false very_verbose = false -web_url = "" [peer] addr = "127.0.0.1:7001" @@ -96,7 +94,6 @@ key_file = "" * `ETCD_SNAPSHOT` * `ETCD_VERBOSE` * `ETCD_VERY_VERBOSE` - * `ETCD_WEB_URL` * `ETCD_PEER_ADDR` * `ETCD_PEER_BIND_ADDR` * `ETCD_PEER_CA_FILE` diff --git a/Documentation/libraries-and-tools.md b/Documentation/libraries-and-tools.md new file mode 100644 index 000000000..dc364d314 --- /dev/null +++ b/Documentation/libraries-and-tools.md @@ -0,0 +1,75 @@ +## Libraries and Tools + +**Tools** + +- [etcdctl](https://github.com/coreos/etcdctl) - A command line client for etcd + +**Go libraries** + +- [go-etcd](https://github.com/coreos/go-etcd) - Supports v2 + +**Java libraries** + +- [justinsb/jetcd](https://github.com/justinsb/jetcd) +- [diwakergupta/jetcd](https://github.com/diwakergupta/jetcd) - Supports v2 + +**Python libraries** + +- [transitorykris/etcd-py](https://github.com/transitorykris/etcd-py) +- [jplana/python-etcd](https://github.com/jplana/python-etcd) - Supports v2 +- [russellhaering/txetcd](https://github.com/russellhaering/txetcd) - a Twisted Python library + +**Node libraries** + +- [stianeikeland/node-etcd](https://github.com/stianeikeland/node-etcd) - Supports v2 (w Coffeescript) +- [lavagetto/nodejs-etcd](https://github.com/lavagetto/nodejs-etcd) - Supports v2 + +**Ruby libraries** + +- [iconara/etcd-rb](https://github.com/iconara/etcd-rb) +- [jpfuentes2/etcd-ruby](https://github.com/jpfuentes2/etcd-ruby) +- [ranjib/etcd-ruby](https://github.com/ranjib/etcd-ruby) - Supports v2 + +**C libraries** + +- [jdarcy/etcd-api](https://github.com/jdarcy/etcd-api) - Supports v2 + +**Clojure libraries** + +- [aterreno/etcd-clojure](https://github.com/aterreno/etcd-clojure) +- [dwwoelfel/cetcd](https://github.com/dwwoelfel/cetcd) - Supports v2 +- [rthomas/clj-etcd](https://github.com/rthomas/clj-etcd) - Supports v2 + +**Erlang libraries** + +- [marshall-lee/etcd.erl](https://github.com/marshall-lee/etcd.erl) + +A detailed recap of client functionalities can be found in the [clients compatibility matrix][clients-matrix.md]. + +[clients-matrix.md]: https://github.com/coreos/etcd/blob/master/Documentation/clients-matrix.md + +**Chef Integration** + +- [coderanger/etcd-chef](https://github.com/coderanger/etcd-chef) + +**Chef Cookbook** + +- [spheromak/etcd-cookbook](https://github.com/spheromak/etcd-cookbook) + +**BOSH Releases** + +- [cloudfoundry-community/etcd-boshrelease](https://github.com/cloudfoundry-community/etcd-boshrelease) +- [cloudfoundry/cf-release](https://github.com/cloudfoundry/cf-release/tree/master/jobs/etcd) + +**Projects using etcd** + +- [binocarlos/yoda](https://github.com/binocarlos/yoda) - etcd + ZeroMQ +- [calavera/active-proxy](https://github.com/calavera/active-proxy) - HTTP Proxy configured with etcd +- [derekchiang/etcdplus](https://github.com/derekchiang/etcdplus) - A set of distributed synchronization primitives built upon etcd +- [go-discover](https://github.com/flynn/go-discover) - service discovery in Go +- [gleicon/goreman](https://github.com/gleicon/goreman/tree/etcd) - Branch of the Go Foreman clone with etcd support +- [garethr/hiera-etcd](https://github.com/garethr/hiera-etcd) - Puppet hiera backend using etcd +- [mattn/etcd-vim](https://github.com/mattn/etcd-vim) - SET and GET keys from inside vim +- [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration +- [kelseyhightower/confd](https://github.com/kelseyhightower/confd) - Manage local app config files using templates and data from etcd +- [configdb](https://git.autistici.org/ai/configdb/tree/master) - A REST relational abstraction on top of arbitrary database backends, aimed at storing configs and inventories. diff --git a/Documentation/modules.md b/Documentation/modules.md new file mode 100644 index 000000000..441dd9b72 --- /dev/null +++ b/Documentation/modules.md @@ -0,0 +1,102 @@ +## Modules + +etcd has a number of modules that are built on top of the core etcd API. +These modules provide things like dashboards, locks and leader election. + +### Dashboard + +An HTML dashboard can be found at `http://127.0.0.1:4001/mod/dashboard/` + +### Lock + +The Lock module implements a fair lock that can be used when lots of clients want access to a single resource. +A lock can be associated with a value. +The value is unique so if a lock tries to request a value that is already queued for a lock then it will find it and watch until that value obtains the lock. +If you lock the same value on a key from two separate curl sessions they'll both return at the same time. + +Here's the API: + +**Acquire a lock (with no value) for "customer1"** + +```sh +curl -X POST http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 +``` + +**Acquire a lock for "customer1" that is associated with the value "bar"** + +```sh +curl -X POST http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -d value=bar +``` + +**Renew the TTL on the "customer1" lock for index 2** + +```sh +curl -X PUT http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -d index=2 +``` + +**Renew the TTL on the "customer1" lock for value "customer1"** + +```sh +curl -X PUT http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -d value=bar +``` + +**Retrieve the current value for the "customer1" lock.** + +```sh +curl http://127.0.0.1:4001/mod/v2/lock/customer1 +``` + +**Retrieve the current index for the "customer1" lock** + +```sh +curl http://127.0.0.1:4001/mod/v2/lock/customer1?field=index +``` + +**Delete the "customer1" lock with the index 2** + +```sh +curl -X DELETE http://127.0.0.1:4001/mod/v2/lock/customer1?index=2 +``` + +**Delete the "customer1" lock with the value "bar"** + +```sh +curl -X DELETE http://127.0.0.1:4001/mod/v2/lock/customer1?value=bar +``` + + +### Leader Election + +The Leader Election module wraps the Lock module to allow clients to come to consensus on a single value. +This is useful when you want one server to process at a time but allow other servers to fail over. +The API is similar to the Lock module but is limited to simple strings values. + +Here's the API: + +**Attempt to set a value for the "order_processing" leader key:** + +```sh +curl -X PUT http://127.0.0.1:4001/mod/v2/leader/order_processing?ttl=60 -d name=myserver1.foo.com +``` + +**Retrieve the current value for the "order_processing" leader key:** + +```sh +curl http://127.0.0.1:4001/mod/v2/leader/order_processing +myserver1.foo.com +``` + +**Remove a value from the "order_processing" leader key:** + +```sh +curl -X DELETE http://127.0.0.1:4001/mod/v2/leader/order_processing?name=myserver1.foo.com +``` + +If multiple clients attempt to set the value for a key then only one will succeed. +The other clients will hang until the current value is removed because of TTL or because of a `DELETE` operation. +Multiple clients can submit the same value and will all be notified when that value succeeds. + +To update the TTL of a value simply reissue the same `PUT` command that you used to set the value. + + + diff --git a/Documentation/optimal-cluster-size.md b/Documentation/optimal-cluster-size.md new file mode 100644 index 000000000..53cf67378 --- /dev/null +++ b/Documentation/optimal-cluster-size.md @@ -0,0 +1,30 @@ +# Optimal etcd Cluster Size + +etcd's Raft consensus algorithm is most efficient in small clusters between 3 and 9 peers. Let's briefly explore how etcd works internally to understand why. + +## Writing to etcd + +Writes to an etcd peer are always redirected to the leader of the cluster and distributed to all of the peers immediately. A write is only considered successful when a majority of the peers acknowledge the write. + +For example, in a 5 node cluster, a write operation is only as fast as the 3rd fastest machine. This is the main reason for keeping your etcd cluster below 9 nodes. In practice, you only need to worry about write performance in high latency environments such as a cluster spanning multiple data centers. + +## Leader Election + +The leader election process is similar to writing a key — a majority of the cluster must acknowledge the new leader before cluster operations can continue. The longer each node takes to elect a new leader means you have to wait longer before you can write to the cluster again. In low latency environments this process takes milliseconds. + +## Odd Cluster Size + +The other important cluster optimization is to always have an odd cluster size. Adding an odd node to the cluster doesn't change the size of the majority and therefore doesn't increase the total latency of the majority as described above. But you do gain a higher tolerance for peer failure by adding the extra machine. You can see this in practice when comparing two even and odd sized clusters: + +| Cluster Size | Majority | Failure Tolerance | +|--------------|------------|-------------------| +| 8 machines | 5 machines | 3 machines | +| 9 machines | 5 machines | **4 machines** | + +As you can see, adding another node to bring the cluster up to an odd size is always worth it. During a network partition, an odd cluster size also guarantees that there will almost always be a majority of the cluster that can continue to operate and be the source of truth when the partition ends. + +## Cluster Management + +Currently, each CoreOS machine is an etcd peer — if you have 30 CoreOS machines, you have 30 etcd peers and end up with a cluster size that is way too large. If desired, you may manually stop some of these etcd instances to increase cluster performance. + +Functionality is being developed to expose two different types of followers: active and benched followers. Active followers will influence operations within the cluster. Benched followers will not participate, but will transparently proxy etcd traffic to an active follower. This allows every CoreOS machine to expose etcd on port 4001 for ease of use. Benched followers will have the ability to transition into an active follower if needed. diff --git a/Documentation/security.md b/Documentation/security.md new file mode 100644 index 000000000..19fb71906 --- /dev/null +++ b/Documentation/security.md @@ -0,0 +1,130 @@ +## Advanced Usage + +### Transport security with HTTPS + +Etcd supports SSL/TLS and client cert authentication for clients to server, as well as server to server communication. + +First, you need to have a CA cert `clientCA.crt` and signed key pair `client.crt`, `client.key`. +This site has a good reference for how to generate self-signed key pairs: +http://www.g-loaded.eu/2005/11/10/be-your-own-ca/ + +For testing you can use the certificates in the `fixtures/ca` directory. + +Let's configure etcd to use this keypair: + +```sh +./etcd -f -name machine0 -data-dir machine0 -cert-file=./fixtures/ca/server.crt -key-file=./fixtures/ca/server.key.insecure +``` + +There are a few new options we're using: + +* `-f` - forces a new machine configuration, even if an existing configuration is found. (WARNING: data loss!) +* `-cert-file` and `-key-file` specify the location of the cert and key files to be used for for transport layer security between the client and server. + +You can now test the configuration using HTTPS: + +```sh +curl --cacert ./fixtures/ca/server-chain.pem https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v +``` + +You should be able to see the handshake succeed. + +**OSX 10.9+ Users**: curl 7.30.0 on OSX 10.9+ doesn't understand certificates passed in on the command line. +Instead you must import the dummy ca.crt directly into the keychain or add the `-k` flag to curl to ignore errors. +If you want to test without the `-k` flag run `open ./fixtures/ca/ca.crt` and follow the prompts. +Please remove this certificate after you are done testing! +If you know of a workaround let us know. + +``` +... +SSLv3, TLS handshake, Finished (20): +... +``` + +And also the response from the etcd server: + +```json +{ + "action": "set", + "key": "/foo", + "modifiedIndex": 3, + "value": "bar" +} +``` + + +### Authentication with HTTPS client certificates + +We can also do authentication using CA certs. +The clients will provide their cert to the server and the server will check whether the cert is signed by the CA and decide whether to serve the request. + +```sh +./etcd -f -name machine0 -data-dir machine0 -ca-file=./fixtures/ca/ca.crt -cert-file=./fixtures/ca/server.crt -key-file=./fixtures/ca/server.key.insecure +``` + +```-ca-file``` is the path to the CA cert. + +Try the same request to this server: + +```sh +curl --cacert ./fixtures/ca/server-chain.pem https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v +``` + +The request should be rejected by the server. + +``` +... +routines:SSL3_READ_BYTES:sslv3 alert bad certificate +... +``` + +We need to give the CA signed cert to the server. + +```sh +curl --key ./fixtures/ca/server2.key.insecure --cert ./fixtures/ca/server2.crt --cacert ./fixtures/ca/server-chain.pem -L https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v +``` + +You should able to see: + +``` +... +SSLv3, TLS handshake, CERT verify (15): +... +TLS handshake, Finished (20) +``` + +And also the response from the server: + +```json +{ + "action": "set", + "node": { + "createdIndex": 12, + "key": "/foo", + "modifiedIndex": 12, + "value": "bar" + } +} +``` + +### Why SSLv3 alert handshake failure when using SSL client auth? + +The `crypto/tls` package of `golang` checks the key usage of the certificate public key before using it. +To use the certificate public key to do client auth, we need to add `clientAuth` to `Extended Key Usage` when creating the certificate public key. + +Here is how to do it: + +Add the following section to your openssl.cnf: + +``` +[ ssl_client ] +... + extendedKeyUsage = clientAuth +... +``` + +When creating the cert be sure to reference it in the `-extensions` flag: + +``` +openssl ca -config openssl.cnf -policy policy_anything -extensions ssl_client -out certs/machine.crt -infiles machine.csr +``` diff --git a/Documentation/tuning.md b/Documentation/tuning.md new file mode 100644 index 000000000..607f16b42 --- /dev/null +++ b/Documentation/tuning.md @@ -0,0 +1,45 @@ +## Tuning + +The default settings in etcd should work well for installations on a local network where the average network latency is low. +However, when using etcd across multiple data centers or over networks with high latency you may need to tweak the heartbeat and election timeout settings. + +The underlying distributed consensus protocol relies on two separate timeouts to ensure that nodes can handoff leadership if one stalls or goes offline. +The first timeout is called the *Heartbeat Timeout*. +This is the frequency with which the leader will notify followers that it is still the leader. +etcd batches commands together for higher throughput so this heartbeat timeout is also a delay for how long it takes for commands to be committed. +By default, etcd uses a `50ms` heartbeat timeout. + +The second timeout is the *Election Timeout*. +This timeout is how long a follower node will go without hearing a heartbeat before attempting to become leader itself. +By default, etcd uses a `200ms` election timeout. + +Adjusting these values is a trade off. +Lowering the heartbeat timeout will cause individual commands to be committed faster but it will lower the overall throughput of etcd. +If your etcd instances have low utilization then lowering the heartbeat timeout can improve your command response time. + +The election timeout should be set based on the heartbeat timeout and your network ping time between nodes. +Election timeouts should be at least 10 times your ping time so it can account for variance in your network. +For example, if the ping time between your nodes is 10ms then you should have at least a 100ms election timeout. + +You should also set your election timeout to at least 4 to 5 times your heartbeat timeout to account for variance in leader replication. +For a heartbeat timeout of 50ms you should set your election timeout to at least 200ms - 250ms. + +You can override the default values on the command line: + +```sh +# Command line arguments: +$ etcd -peer-heartbeat-timeout=100 -peer-election-timeout=500 + +# Environment variables: +$ ETCD_PEER_HEARTBEAT_TIMEOUT=100 ETCD_PEER_ELECTION_TIMEOUT=500 etcd +``` + +Or you can set the values within the configuration file: + +```toml +[peer] +heartbeat_timeout = 100 +election_timeout = 100 +``` + +The values are specified in milliseconds. diff --git a/README.md b/README.md index 22bde6678..0e67f34e9 100644 --- a/README.md +++ b/README.md @@ -55,1199 +55,47 @@ _NOTE_: you need go 1.1+. Please check your installation with go version ``` +### Running -### Running a single machine - -These examples will use a single machine cluster to show you the basics of the etcd REST API. -Let's start etcd: +First start a single machine cluster of etcd: ```sh -./etcd -data-dir machine0 -name machine0 +./etcd ``` This will bring up etcd listening on port 4001 for client communication and on port 7001 for server-to-server communication. -The `-data-dir machine0` argument tells etcd to write machine configuration, logs and snapshots to the `./machine0/` directory. -The `-name machine` tells the rest of the cluster that this machine is named machine0. +Next lets set a single key and then retrieve it: - -## Usage - -### Setting the value to a key - -Let’s set the first key-value pair to the datastore. -In this case the key is `/message` and the value is `Hello world`. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/message -X PUT -d value="Hello world" -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 2, - "key": "/message", - "modifiedIndex": 2, - "value": "Hello world" - } -} -``` - -The response object contains several attributes: - -1. `action`: the action of the request that was just made. -The request attempted to modify `node.value` via a `PUT` HTTP request, thus the value of action is `set`. - -2. `node.key`: the HTTP path the to which the request was made. -We set `/message` to `Hello world`, so the key field is `/message`. -Etcd uses a file-system-like structure to represent the key-value pairs, therefore all keys start with `/`. - -3. `node.value`: the value of the key after resolving the request. -In this case, a successful request was made that attempted to change the node's value to `Hello world`. - -4. `node.createdIndex`: an index is a unique, monotonically-incrementing integer created for each change to etcd. -This specific index reflects at which point in the etcd state machine a given key was created. -You may notice that in this example the index is `2` even though it is the first request you sent to the server. -This is because there are internal commands that also change the state behind the scenes like adding and syncing servers. - -5. `node.modifiedIndex`: like `node.createdIndex`, this attribute is also an etcd index. -Actions that cause the value to change include `set`, `delete`, `update`, `create` and `compareAndSwap`. -Since the `get` and `watch` commands do not change state in the store, they do not change the value of `node.modifiedIndex`. - - -### Response Headers - -etcd includes a few HTTP headers that provide global information about the etcd cluster that serviced a request: - -``` -X-Etcd-Index: 35 -X-Raft-Index: 5398 -X-Raft-Term: 0 -``` - -- `X-Etcd-Index` is the current etcd index as explained above. -- `X-Raft-Index` is similar to the etcd index but is for the underlying raft protocol -- `X-Raft-Term` this number will increase when an etcd master election happens. If this number is increasing rapdily you may need to tune the election timeout. See the [tuning][tuning] section for details. - -[tuning]: #tuning - -### Get the value of a key - -We can get the value that we just set in `/message` by issuing a `GET` request: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/message -``` - -```json -{ - "action": "get", - "node": { - "createdIndex": 2, - "key": "/message", - "modifiedIndex": 2, - "value": "Hello world" - } -} -``` - - -### Changing the value of a key - -You can change the value of `/message` from `Hello world` to `Hello etcd` with another `PUT` request to the key: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello etcd" -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 3, - "key": "/message", - "modifiedIndex": 3, - "value": "Hello etcd" - } -} -``` - -### Deleting a key - -You can remove the `/message` key with a `DELETE` request: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/message -XDELETE -``` - -```json -{ - "action": "delete", - "node": { - "createdIndex": 3, - "key": "/message", - "modifiedIndex": 4 - } -} -``` - - -### Using key TTL - -Keys in etcd can be set to expire after a specified number of seconds. -You can do this by setting a TTL (time to live) on the key when send a `PUT` request: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -d ttl=5 -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 5, - "expiration": "2013-12-04T12:01:21.874888581-08:00", - "key": "/foo", - "modifiedIndex": 5, - "ttl": 5, - "value": "bar" - } -} -``` - -Note the two new fields in response: - -1. The `expiration` is the time that this key will expire and be deleted. - -2. The `ttl` is the time to live for the key, in seconds. - -_NOTE_: Keys can only be expired by a cluster leader so if a machine gets disconnected from the cluster, its keys will not expire until it rejoins. - -Now you can try to get the key by sending a `GET` request: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo -``` - -If the TTL has expired, the key will be deleted, and you will be returned a 100. - -```json -{ - "cause": "/foo", - "errorCode": 100, - "index": 6, - "message": "Key Not Found" -} -``` - -### Waiting for a change - -We can watch for a change on a key and receive a notification by using long polling. -This also works for child keys by passing `recursive=true` in curl. - -In one terminal, we send a get request with `wait=true` : - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo?wait=true -``` - -Now we are waiting for any changes at path `/foo`. - -In another terminal, we set a key `/foo` with value `bar`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -``` - -The first terminal should get the notification and return with the same response as the set request. - -```json -{ - "action": "set", - "node": { - "createdIndex": 7, - "key": "/foo", - "modifiedIndex": 7, - "value": "bar" - } -} -``` - -However, the watch command can do more than this. -Using the the index we can watch for commands that has happened in the past. -This is useful for ensuring you don't miss events between watch commands. - -Let's try to watch for the set command of index 7 again: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo?wait=true\&waitIndex=7 -``` - -The watch command returns immediately with the same response as previous. - - -### Atomically Creating In-Order Keys - -Using the `POST` on a directory you can create keys with key names that are created in-order. -This can be used in a variety of useful patterns like implementing queues of keys that need to be processed in strict order. -An example use case is the [locking module][lockmod] which uses it to ensure clients get fair access to a mutex. - -Creating an in-order key is easy - -```sh -curl -X POST http://127.0.0.1:4001/v2/keys/queue -d value=Job1 -``` - -```json -{ - "action": "create", - "node": { - "createdIndex": 6, - "key": "/queue/6", - "modifiedIndex": 6, - "value": "Job1" - } -} -``` - -If you create another entry some time later it is guaranteed to have a key name that is greater than the previous key. -Also note the key names use the global etcd index so the next key can be more than `previous + 1`. - -```sh -curl -X POST http://127.0.0.1:4001/v2/keys/queue -d value=Job2 -``` - -```json -{ - "action": "create", - "node": { - "createdIndex": 29, - "key": "/queue/29", - "modifiedIndex": 29, - "value": "Job2" - } -} -``` - -To enumerate the in-order keys as a sorted list, use the "sorted" parameter. - -```sh -curl -s -X GET 'http://127.0.0.1:4001/v2/keys/queue?recursive=true&sorted=true' -``` - -```json -{ - "action": "get", - "node": { - "createdIndex": 2, - "dir": true, - "key": "/queue", - "modifiedIndex": 2, - "nodes": [ - { - "createdIndex": 2, - "key": "/queue/2", - "modifiedIndex": 2, - "value": "Job1" - }, - { - "createdIndex": 3, - "key": "/queue/3", - "modifiedIndex": 3, - "value": "Job2" - } - ] - } -} -``` - -[lockmod]: #lock - - -### Using a directory TTL - -Like keys, directories in etcd can be set to expire after a specified number of seconds. -You can do this by setting a TTL (time to live) on a directory when it is created with a `PUT`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d ttl=30 -d dir=true -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 17, - "dir": true, - "expiration": "2013-12-11T10:37:33.689275857-08:00", - "key": "/newdir", - "modifiedIndex": 17, - "ttl": 30 - } -} -``` - -The directories TTL can be refreshed by making an update. -You can do this by making a PUT with `prevExist=true` and a new TTL. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d ttl=30 -d dir=true -d prevExist=true -``` - -Keys that are under this directory work as usual, but when the directory expires a watcher on a key under the directory will get an expire event: - -```sh -curl -X GET http://127.0.0.1:4001/v2/keys/dir/asdf\?consistent\=true\&wait\=true -``` - -```json -{ - "action": "expire", - "node": { - "createdIndex": 8, - "key": "/dir", - "modifiedIndex": 15 - } -} -``` - - -### Atomic Compare-and-Swap (CAS) - -Etcd can be used as a centralized coordination service in a cluster and `CompareAndSwap` is the most basic operation used to build a distributed lock service. - -This command will set the value of a key only if the client-provided conditions are equal to the current conditions. - -The current comparable conditions are: - -1. `prevValue` - checks the previous value of the key. - -2. `prevIndex` - checks the previous index of the key. - -3. `prevExist` - checks existence of the key: if `prevExist` is true, it is a `update` request; if prevExist is `false`, it is a `create` request. - -Here is a simple example. -Let's create a key-value pair first: `foo=one`. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=one -``` - -Let's try some invalid `CompareAndSwap` commands first. - -Trying to set this existing key with `prevExist=false` fails as expected: -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo?prevExist=false -XPUT -d value=three -``` - -The error code explains the problem: - -```json -{ - "cause": "/foo", - "errorCode": 105, - "index": 39776, - "message": "Already exists" -} -``` - -Now lets provide a `prevValue` parameter: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo?prevValue=two -XPUT -d value=three -``` - -This will try to compare the previous value of the key and the previous value we provided. If they are equal, the value of the key will change to three. - -```json -{ - "cause": "[two != one] [0 != 8]", - "errorCode": 101, - "index": 8, - "message": "Test Failed" -} -``` - -which means `CompareAndSwap` failed. - -Let's try a valid condition: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo?prevValue=one -XPUT -d value=two -``` - -The response should be - -```json -{ - "action": "compareAndSwap", - "node": { - "createdIndex": 8, - "key": "/foo", - "modifiedIndex": 9, - "value": "two" - } -} -``` - -We successfully changed the value from "one" to "two" since we gave the correct previous value. - -### Creating Directories - -In most cases directories for a key are automatically created. -But, there are cases where you will want to create a directory or remove one. - -Creating a directory is just like a key only you cannot provide a value and must add the `dir=true` parameter. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d dir=true -``` -```json -{ - "action": "set", - "node": { - "createdIndex": 30, - "dir": true, - "key": "/dir", - "modifiedIndex": 30 - } -} -``` - -### Listing a directory - -In etcd we can store two types of things: keys and directories. -Keys store a single string value. -Directories store a set of keys and/or other directories. - -In this example, let's first create some keys: - -We already have `/foo=two` so now we'll create another one called `/foo_dir/foo` with the value of `bar`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo_dir/foo -XPUT -d value=bar -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 2, - "key": "/foo_dir/foo", - "modifiedIndex": 2, - "value": "bar" - } -} -``` - -Now we can list the keys under root `/`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/ -``` - -We should see the response as an array of items: - -```json -{ - "action": "get", - "node": { - "dir": true, - "key": "/", - "nodes": [ - { - "createdIndex": 2, - "dir": true, - "key": "/foo_dir", - "modifiedIndex": 2 - } - ] - } -} -``` - -Here we can see `/foo` is a key-value pair under `/` and `/foo_dir` is a directory. -We can also recursively get all the contents under a directory by adding `recursive=true`. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/?recursive=true -``` - -```json -{ - "action": "get", - "node": { - "dir": true, - "key": "/", - "nodes": [ - { - "createdIndex": 2, - "dir": true, - "key": "/foo_dir", - "modifiedIndex": 2, - "nodes": [ - { - "createdIndex": 2, - "key": "/foo_dir/foo", - "modifiedIndex": 2, - "value": "bar" - } - ] - } - ] - } -} -``` - - -### Deleting a Directory - -Now let's try to delete the directory `/foo_dir`. - -You can remove an empty directory using the `DELETE` verb and the `dir=true` parameter. - -```sh -curl -L -X DELETE 'http://127.0.0.1:4001/v2/keys/dir?dir=true' -``` -```json -{ - "action": "delete", - "node": { - "createdIndex": 30, - "dir": true, - "key": "/dir", - "modifiedIndex": 31 - } -} -``` - -To delete a directory that holds keys, you must add `recursive=true`. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/dir?recursive=true -XDELETE ``` - -```json -{ - "action": "delete", - "node": { - "createdIndex": 10, - "dir": true, - "key": "/dir", - "modifiedIndex": 11 - } -} -``` - - -### Creating a hidden node - -We can create a hidden key-value pair or directory by add a `_` prefix. -The hidden item will not be listed when sending a `GET` request for a directory. - -First we'll add a hidden key named `/_message`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/_message -XPUT -d value="Hello hidden world" -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 3, - "key": "/_message", - "modifiedIndex": 3, - "value": "Hello hidden world" - } -} -``` - - -Next we'll add a regular key named `/message`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello world" -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 4, - "key": "/message", - "modifiedIndex": 4, - "value": "Hello world" - } -} -``` - -Now let's try to get a listing of keys under the root directory, `/`: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/ -``` - -```json -{ - "action": "get", - "node": { - "dir": true, - "key": "/", - "nodes": [ - { - "createdIndex": 2, - "dir": true, - "key": "/foo_dir", - "modifiedIndex": 2 - }, - { - "createdIndex": 4, - "key": "/message", - "modifiedIndex": 4, - "value": "Hello world" - } - ] - } -} -``` - -Here we see the `/message` key but our hidden `/_message` key is not returned. - -## Advanced Usage - -### Transport security with HTTPS - -Etcd supports SSL/TLS and client cert authentication for clients to server, as well as server to server communication. - -First, you need to have a CA cert `clientCA.crt` and signed key pair `client.crt`, `client.key`. -This site has a good reference for how to generate self-signed key pairs: -http://www.g-loaded.eu/2005/11/10/be-your-own-ca/ - -For testing you can use the certificates in the `fixtures/ca` directory. - -Let's configure etcd to use this keypair: - -```sh -./etcd -f -name machine0 -data-dir machine0 -cert-file=./fixtures/ca/server.crt -key-file=./fixtures/ca/server.key.insecure -``` - -There are a few new options we're using: - -* `-f` - forces a new machine configuration, even if an existing configuration is found. (WARNING: data loss!) -* `-cert-file` and `-key-file` specify the location of the cert and key files to be used for for transport layer security between the client and server. - -You can now test the configuration using HTTPS: - -```sh -curl --cacert ./fixtures/ca/server-chain.pem https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v -``` - -You should be able to see the handshake succeed. - -**OSX 10.9+ Users**: curl 7.30.0 on OSX 10.9+ doesn't understand certificates passed in on the command line. -Instead you must import the dummy ca.crt directly into the keychain or add the `-k` flag to curl to ignore errors. -If you want to test without the `-k` flag run `open ./fixtures/ca/ca.crt` and follow the prompts. -Please remove this certificate after you are done testing! -If you know of a workaround let us know. - -``` -... -SSLv3, TLS handshake, Finished (20): -... -``` - -And also the response from the etcd server: - -```json -{ - "action": "set", - "key": "/foo", - "modifiedIndex": 3, - "value": "bar" -} -``` - - -### Authentication with HTTPS client certificates - -We can also do authentication using CA certs. -The clients will provide their cert to the server and the server will check whether the cert is signed by the CA and decide whether to serve the request. - -```sh -./etcd -f -name machine0 -data-dir machine0 -ca-file=./fixtures/ca/ca.crt -cert-file=./fixtures/ca/server.crt -key-file=./fixtures/ca/server.key.insecure -``` - -```-ca-file``` is the path to the CA cert. - -Try the same request to this server: - -```sh -curl --cacert ./fixtures/ca/server-chain.pem https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v -``` - -The request should be rejected by the server. - -``` -... -routines:SSL3_READ_BYTES:sslv3 alert bad certificate -... -``` - -We need to give the CA signed cert to the server. - -```sh -curl --key ./fixtures/ca/server2.key.insecure --cert ./fixtures/ca/server2.crt --cacert ./fixtures/ca/server-chain.pem -L https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v -``` - -You should able to see: - -``` -... -SSLv3, TLS handshake, CERT verify (15): -... -TLS handshake, Finished (20) -``` - -And also the response from the server: - -```json -{ - "action": "set", - "node": { - "createdIndex": 12, - "key": "/foo", - "modifiedIndex": 12, - "value": "bar" - } -} -``` - - -## Clustering - -### Example cluster of three machines - -Let's explore the use of etcd clustering. -We use Raft as the underlying distributed protocol which provides consistency and persistence of the data across all of the etcd instances. - -Let start by creating 3 new etcd instances. - -We use `-peer-addr` to specify server port and `-addr` to specify client port and `-data-dir` to specify the directory to store the log and info of the machine in the cluster: - -```sh -./etcd -peer-addr 127.0.0.1:7001 -addr 127.0.0.1:4001 -data-dir machines/machine1 -name machine1 -``` - -**Note:** If you want to run etcd on an external IP address and still have access locally, you'll need to add `-bind-addr 0.0.0.0` so that it will listen on both external and localhost addresses. -A similar argument `-peer-bind-addr` is used to setup the listening address for the server port. - -Let's join two more machines to this cluster using the `-peers` argument: - -```sh -./etcd -peer-addr 127.0.0.1:7002 -addr 127.0.0.1:4002 -peers 127.0.0.1:7001 -data-dir machines/machine2 -name machine2 -./etcd -peer-addr 127.0.0.1:7003 -addr 127.0.0.1:4003 -peers 127.0.0.1:7001 -data-dir machines/machine3 -name machine3 -``` - -We can retrieve a list of machines in the cluster using the HTTP API: - -```sh -curl -L http://127.0.0.1:4001/v2/machines -``` - -We should see there are three machines in the cluster - -``` -http://127.0.0.1:4001, http://127.0.0.1:4002, http://127.0.0.1:4003 -``` - -The machine list is also available via the main key API: - -```sh -curl -L http://127.0.0.1:4001/v2/keys/_etcd/machines +curl -L http://127.0.0.1:4001/v2/keys/mykey -XPUT -d value="this is awesome" +curl -L http://127.0.0.1:4001/v2/keys/mykey ``` -```json -{ - "action": "get", - "node": { - "createdIndex": 1, - "dir": true, - "key": "/_etcd/machines", - "modifiedIndex": 1, - "nodes": [ - { - "createdIndex": 1, - "key": "/_etcd/machines/machine1", - "modifiedIndex": 1, - "value": "raft=http://127.0.0.1:7001&etcd=http://127.0.0.1:4001" - }, - { - "createdIndex": 2, - "key": "/_etcd/machines/machine2", - "modifiedIndex": 2, - "value": "raft=http://127.0.0.1:7002&etcd=http://127.0.0.1:4002" - }, - { - "createdIndex": 3, - "key": "/_etcd/machines/machine3", - "modifiedIndex": 3, - "value": "raft=http://127.0.0.1:7003&etcd=http://127.0.0.1:4003" - } - ] - } -} -``` - -We can also get the current leader in the cluster: - -``` -curl -L http://127.0.0.1:4001/v2/leader -``` - -The first server we set up should still be the leader unless it has died during these commands. - -``` -http://127.0.0.1:7001 -``` - -Now we can do normal SET and GET operations on keys as we explored earlier. - -```sh -curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -``` - -```json -{ - "action": "set", - "node": { - "createdIndex": 4, - "key": "/foo", - "modifiedIndex": 4, - "value": "bar" - } -} -``` - - -### Killing Nodes in the Cluster - -Now if we kill the leader of the cluster, we can get the value from one of the other two machines: - -```sh -curl -L http://127.0.0.1:4002/v2/keys/foo -``` - -We can also see that a new leader has been elected: - -``` -curl -L http://127.0.0.1:4002/v2/leader -``` - -``` -http://127.0.0.1:7002 -``` - -or - -``` -http://127.0.0.1:7003 -``` - - -### Testing Persistence - -Next we'll kill all the machines to test persistence. -Type `CTRL-C` on each terminal and then rerun the same command you used to start each machine. - -Your request for the `foo` key will return the correct value: - -```sh -curl -L http://127.0.0.1:4002/v2/keys/foo -``` - -```json -{ - "action": "get", - "node": { - "createdIndex": 4, - "key": "/foo", - "modifiedIndex": 4, - "value": "bar" - } -} -``` - - -### Using HTTPS between servers - -In the previous example we showed how to use SSL client certs for client-to-server communication. -Etcd can also do internal server-to-server communication using SSL client certs. -To do this just change the `-*-file` flags to `-peer-*-file`. - -If you are using SSL for server-to-server communication, you must use it on all instances of etcd. - -## Modules - -etcd has a number of modules that are built on top of the core etcd API. -These modules provide things like dashboards, locks and leader election. - -### Dashboard - -An HTML dashboard can be found at `http://127.0.0.1:4001/mod/dashboard/` - -### Lock - -The Lock module implements a fair lock that can be used when lots of clients want access to a single resource. -A lock can be associated with a value. -The value is unique so if a lock tries to request a value that is already queued for a lock then it will find it and watch until that value obtains the lock. -If you lock the same value on a key from two separate curl sessions they'll both return at the same time. - -Here's the API: - -**Acquire a lock (with no value) for "customer1"** - -```sh -curl -X POST http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -``` - -**Acquire a lock for "customer1" that is associated with the value "bar"** - -```sh -curl -X POST http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -d value=bar -``` - -**Renew the TTL on the "customer1" lock for index 2** - -```sh -curl -X PUT http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -d index=2 -``` - -**Renew the TTL on the "customer1" lock for value "customer1"** - -```sh -curl -X PUT http://127.0.0.1:4001/mod/v2/lock/customer1?ttl=60 -d value=bar -``` - -**Retrieve the current value for the "customer1" lock.** - -```sh -curl http://127.0.0.1:4001/mod/v2/lock/customer1 -``` - -**Retrieve the current index for the "customer1" lock** - -```sh -curl http://127.0.0.1:4001/mod/v2/lock/customer1?field=index -``` - -**Delete the "customer1" lock with the index 2** - -```sh -curl -X DELETE http://127.0.0.1:4001/mod/v2/lock/customer1?index=2 -``` - -**Delete the "customer1" lock with the value "bar"** - -```sh -curl -X DELETE http://127.0.0.1:4001/mod/v2/lock/customer1?value=bar -``` - - -### Leader Election - -The Leader Election module wraps the Lock module to allow clients to come to consensus on a single value. -This is useful when you want one server to process at a time but allow other servers to fail over. -The API is similar to the Lock module but is limited to simple strings values. - -Here's the API: - -**Attempt to set a value for the "order_processing" leader key:** - -```sh -curl -X PUT http://127.0.0.1:4001/mod/v2/leader/order_processing?ttl=60 -d name=myserver1.foo.com -``` - -**Retrieve the current value for the "order_processing" leader key:** - -```sh -curl http://127.0.0.1:4001/mod/v2/leader/order_processing -myserver1.foo.com -``` - -**Remove a value from the "order_processing" leader key:** - -```sh -curl -X DELETE http://127.0.0.1:4001/mod/v2/leader/order_processing?name=myserver1.foo.com -``` +You have successfully started an etcd on a single machine and written a key to the store. Now it time to dig into the full etcd API and other guides. -If multiple clients attempt to set the value for a key then only one will succeed. -The other clients will hang until the current value is removed because of TTL or because of a `DELETE` operation. -Multiple clients can submit the same value and will all be notified when that value succeeds. +### Next Steps -To update the TTL of a value simply reissue the same `PUT` command that you used to set the value. +- Explore the full [API][api.md]. +- Setup a [multi-machine cluster][clustering.md]. +- Learn the [config format, env variables and flags][configuration.md]. +- Find [language bindings and tools][libraries-and-tools.md]. +- Learn about the dashboard, lock and leader election [modules][modules.md]. +- Use TLS to [secure an etcd cluster][security.md]. +- [Tune etcd][tuning.md]. +[api.md]: https://github.com/coreos/etcd/blob/master/Documentation/api.md +[clustering.md]: https://github.com/coreos/etcd/blob/master/Documentation/clustering.md +[configuration.md]: https://github.com/coreos/etcd/blob/master/Documentation/configuration.md +[libraries-and-tools.md]: https://github.com/coreos/etcd/blob/master/Documentation/libraries-and-tools.md +[modules.md]: https://github.com/coreos/etcd/blob/master/Documentation/modules.md +[security.md]: https://github.com/coreos/etcd/blob/master/Documentation/security.md +[tuning.md]: https://github.com/coreos/etcd/blob/master/Documentation/tuning.md ## Contributing See [CONTRIBUTING](https://github.com/coreos/etcd/blob/master/CONTRIBUTING.md) for details on submitting patches and contacting developers via IRC and mailing lists. - -## Libraries and Tools - -**Tools** - -- [etcdctl](https://github.com/coreos/etcdctl) - A command line client for etcd - -**Go libraries** - -- [go-etcd](https://github.com/coreos/go-etcd) - -**Java libraries** - -- [justinsb/jetcd](https://github.com/justinsb/jetcd) -- [diwakergupta/jetcd](https://github.com/diwakergupta/jetcd) - -**Python libraries** - -- [transitorykris/etcd-py](https://github.com/transitorykris/etcd-py) -- [jplana/python-etcd](https://github.com/jplana/python-etcd) -- [russellhaering/txetcd](https://github.com/russellhaering/txetcd) - a Twisted Python library - -**Node libraries** - -- [stianeikeland/node-etcd](https://github.com/stianeikeland/node-etcd) - -**Ruby libraries** - -- [iconara/etcd-rb](https://github.com/iconara/etcd-rb) -- [jpfuentes2/etcd-ruby](https://github.com/jpfuentes2/etcd-ruby) -- [ranjib/etcd-ruby](https://github.com/ranjib/etcd-ruby) - -**C libraries** - -- [jdarcy/etcd-api](https://github.com/jdarcy/etcd-api) - -**Clojure libraries** - -- [aterreno/etcd-clojure](https://github.com/aterreno/etcd-clojure) -- [dwwoelfel/cetcd](https://github.com/dwwoelfel/cetcd) -- [rthomas/clj-etcd](https://github.com/rthomas/clj-etcd) - -**Erlang libraries** - -- [marshall-lee/etcd.erl](https://github.com/marshall-lee/etcd.erl) - -**Chef Integration** - -- [coderanger/etcd-chef](https://github.com/coderanger/etcd-chef) - -**Chef Cookbook** - -- [spheromak/etcd-cookbook](https://github.com/spheromak/etcd-cookbook) - -**BOSH Releases** - -- [cloudfoundry-community/etcd-boshrelease](https://github.com/cloudfoundry-community/etcd-boshrelease) -- [cloudfoundry/cf-release](https://github.com/cloudfoundry/cf-release/tree/master/jobs/etcd) - -**Projects using etcd** - -- [binocarlos/yoda](https://github.com/binocarlos/yoda) - etcd + ZeroMQ -- [calavera/active-proxy](https://github.com/calavera/active-proxy) - HTTP Proxy configured with etcd -- [derekchiang/etcdplus](https://github.com/derekchiang/etcdplus) - A set of distributed synchronization primitives built upon etcd -- [go-discover](https://github.com/flynn/go-discover) - service discovery in Go -- [gleicon/goreman](https://github.com/gleicon/goreman/tree/etcd) - Branch of the Go Foreman clone with etcd support -- [garethr/hiera-etcd](https://github.com/garethr/hiera-etcd) - Puppet hiera backend using etcd -- [mattn/etcd-vim](https://github.com/mattn/etcd-vim) - SET and GET keys from inside vim -- [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration -- [kelseyhightower/confd](https://github.com/kelseyhightower/confd) - Manage local app config files using templates and data from etcd - - -## FAQ - -### What size cluster should I use? - -Every command the client sends to the master is broadcast to all of the followers. -The command is not committed until the majority of the cluster peers receive that command. - -Because of this majority voting property, the ideal cluster should be kept small to keep speed up and be made up of an odd number of peers. - -Odd numbers are good because if you have 8 peers the majority will be 5 and if you have 9 peers the majority will still be 5. -The result is that an 8 peer cluster can tolerate 3 peer failures and a 9 peer cluster can tolerate 4 machine failures. -And in the best case when all 9 peers are responding the cluster will perform at the speed of the fastest 5 machines. - - -### Why SSLv3 alert handshake failure when using SSL client auth? - -The `crypto/tls` package of `golang` checks the key usage of the certificate public key before using it. -To use the certificate public key to do client auth, we need to add `clientAuth` to `Extended Key Usage` when creating the certificate public key. - -Here is how to do it: - -Add the following section to your openssl.cnf: - -``` -[ ssl_client ] -... - extendedKeyUsage = clientAuth -... -``` - -When creating the cert be sure to reference it in the `-extensions` flag: - -``` -openssl ca -config openssl.cnf -policy policy_anything -extensions ssl_client -out certs/machine.crt -infiles machine.csr -``` - -### Tuning - -The default settings in etcd should work well for installations on a local network where the average network latency is low. -However, when using etcd across multiple data centers or over networks with high latency you may need to tweak the heartbeat and election timeout settings. - -The underlying distributed consensus protocol relies on two separate timeouts to ensure that nodes can handoff leadership if one stalls or goes offline. -The first timeout is called the *Heartbeat Timeout*. -This is the frequency with which the leader will notify followers that it is still the leader. -etcd batches commands together for higher throughput so this heartbeat timeout is also a delay for how long it takes for commands to be committed. -By default, etcd uses a `50ms` heartbeat timeout. - -The second timeout is the *Election Timeout*. -This timeout is how long a follower node will go without hearing a heartbeat before attempting to become leader itself. -By default, etcd uses a `200ms` election timeout. - -Adjusting these values is a trade off. -Lowering the heartbeat timeout will cause individual commands to be committed faster but it will lower the overall throughput of etcd. -If your etcd instances have low utilization then lowering the heartbeat timeout can improve your command response time. - -The election timeout should be set based on the heartbeat timeout and your network ping time between nodes. -Election timeouts should be at least 10 times your ping time so it can account for variance in your network. -For example, if the ping time between your nodes is 10ms then you should have at least a 100ms election timeout. - -You should also set your election timeout to at least 4 to 5 times your heartbeat timeout to account for variance in leader replication. -For a heartbeat timeout of 50ms you should set your election timeout to at least 200ms - 250ms. - -You can override the default values on the command line: - -```sh -# Command line arguments: -$ etcd -peer-heartbeat-timeout=100 -peer-election-timeout=500 - -# Environment variables: -$ ETCD_PEER_HEARTBEAT_TIMEOUT=100 ETCD_PEER_ELECTION_TIMEOUT=500 etcd -``` - -Or you can set the values within the configuration file: - -```toml -[peer] -heartbeat_timeout = 100 -election_timeout = 100 -``` - -The values are specified in milliseconds. - - ## Project Details ### Versioning diff --git a/server/config_test.go b/server/config_test.go index 2cab7d610..5a2047375 100644 --- a/server/config_test.go +++ b/server/config_test.go @@ -29,7 +29,6 @@ func TestConfigTOML(t *testing.T) { snapshot = true verbose = true very_verbose = true - web_url = "/web" [peer] addr = "127.0.0.1:7002" @@ -82,7 +81,6 @@ func TestConfigEnv(t *testing.T) { os.Setenv("ETCD_SNAPSHOT", "true") os.Setenv("ETCD_VERBOSE", "1") os.Setenv("ETCD_VERY_VERBOSE", "yes") - os.Setenv("ETCD_WEB_URL", "/web") os.Setenv("ETCD_PEER_ADDR", "127.0.0.1:7002") os.Setenv("ETCD_PEER_CA_FILE", "/tmp/peer/file.ca") os.Setenv("ETCD_PEER_CERT_FILE", "/tmp/peer/file.cert") diff --git a/server/peer_server.go b/server/peer_server.go index 93568ac6a..dc2de40e6 100644 --- a/server/peer_server.go +++ b/server/peer_server.go @@ -77,6 +77,7 @@ func NewPeerServer(name string, path string, url string, bindAddr string, tlsCon Followers: make(map[string]*raftFollowerStats), }, serverStats: &raftServerStats{ + Name: name, StartTime: time.Now(), sendRateQueue: &statsQueue{ back: -1, @@ -422,6 +423,12 @@ func (s *PeerServer) joinByPeer(server raft.Server, peer string, scheme string) func (s *PeerServer) Stats() []byte { s.serverStats.LeaderInfo.Uptime = time.Now().Sub(s.serverStats.LeaderInfo.startTime).String() + // TODO: register state listener to raft to change this field + // rather than compare the state each time Stats() is called. + if s.RaftServer().State() == raft.Leader { + s.serverStats.LeaderInfo.Name = s.RaftServer().Name() + } + queue := s.serverStats.sendRateQueue s.serverStats.SendingPkgRate, s.serverStats.SendingBandwidthRate = queue.Rate()