Refactor commands.

command.go

@@ -1,255 +0,0 @@
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"os"
-	"path"
-	"time"
-
-	etcdErr "github.com/coreos/etcd/error"
-	"github.com/coreos/etcd/store"
-	"github.com/coreos/go-raft"
-)
-
-const commandPrefix = "etcd:"
-
-func commandName(name string) string {
-	return commandPrefix + name
-}
-
-// A command represents an action to be taken on the replicated state machine.
-type Command interface {
-	CommandName() string
-	Apply(server *raft.Server) (interface{}, error)
-}
-
-// Create command
-type CreateCommand struct {
-	Key               string    `json:"key"`
-	Value             string    `json:"value"`
-	ExpireTime        time.Time `json:"expireTime"`
-	IncrementalSuffix bool      `json:"incrementalSuffix"`
-	Force             bool      `json:"force"`
-}
-
-// The name of the create command in the log
-func (c *CreateCommand) CommandName() string {
-	return commandName("create")
-}
-
-// Create node
-func (c *CreateCommand) Apply(server *raft.Server) (interface{}, error) {
-	s, _ := server.StateMachine().(*store.Store)
-
-	e, err := s.Create(c.Key, c.Value, c.IncrementalSuffix, c.Force, c.ExpireTime, server.CommitIndex(), server.Term())
-
-	if err != nil {
-		debug(err)
-		return nil, err
-	}
-
-	return e, nil
-}
-
-// Update command
-type UpdateCommand struct {
-	Key        string    `json:"key"`
-	Value      string    `json:"value"`
-	ExpireTime time.Time `json:"expireTime"`
-}
-
-// The name of the update command in the log
-func (c *UpdateCommand) CommandName() string {
-	return commandName("update")
-}
-
-// Update node
-func (c *UpdateCommand) Apply(server *raft.Server) (interface{}, error) {
-	s, _ := server.StateMachine().(*store.Store)
-
-	e, err := s.Update(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
-
-	if err != nil {
-		debug(err)
-		return nil, err
-	}
-
-	return e, nil
-}
-
-// TestAndSet command
-type TestAndSetCommand struct {
-	Key        string    `json:"key"`
-	Value      string    `json:"value"`
-	ExpireTime time.Time `json:"expireTime"`
-	PrevValue  string    `json: prevValue`
-	PrevIndex  uint64    `json: prevValue`
-}
-
-// The name of the testAndSet command in the log
-func (c *TestAndSetCommand) CommandName() string {
-	return commandName("testAndSet")
-}
-
-// Set the key-value pair if the current value of the key equals to the given prevValue
-func (c *TestAndSetCommand) Apply(server *raft.Server) (interface{}, error) {
-	s, _ := server.StateMachine().(*store.Store)
-
-	e, err := s.TestAndSet(c.Key, c.PrevValue, c.PrevIndex,
-		c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
-
-	if err != nil {
-		debug(err)
-		return nil, err
-	}
-
-	return e, nil
-}
-
-// Delete command
-type DeleteCommand struct {
-	Key       string `json:"key"`
-	Recursive bool   `json:"recursive"`
-}
-
-// The name of the delete command in the log
-func (c *DeleteCommand) CommandName() string {
-	return commandName("delete")
-}
-
-// Delete the key
-func (c *DeleteCommand) Apply(server *raft.Server) (interface{}, error) {
-	s, _ := server.StateMachine().(*store.Store)
-
-	e, err := s.Delete(c.Key, c.Recursive, server.CommitIndex(), server.Term())
-
-	if err != nil {
-		debug(err)
-		return nil, err
-	}
-
-	return e, nil
-}
-
-// JoinCommand
-type JoinCommand struct {
-	RaftVersion string `json:"raftVersion"`
-	Name        string `json:"name"`
-	RaftURL     string `json:"raftURL"`
-	EtcdURL     string `json:"etcdURL"`
-}
-
-func newJoinCommand(version, name, raftUrl, etcdUrl string) *JoinCommand {
-	return &JoinCommand{
-		RaftVersion: version,
-		Name:        name,
-		RaftURL:     raftUrl,
-		EtcdURL:     etcdUrl,
-	}
-}
-
-// The name of the join command in the log
-func (c *JoinCommand) CommandName() string {
-	return commandName("join")
-}
-
-// Join a server to the cluster
-func (c *JoinCommand) Apply(server *raft.Server) (interface{}, error) {
-	s, _ := server.StateMachine().(*store.Store)
-	r, _ := server.Context().(*raftServer)
-
-	// check if the join command is from a previous machine, who lost all its previous log.
-	e, _ := s.Get(path.Join("/_etcd/machines", c.Name), false, false, server.CommitIndex(), server.Term())
-
-	b := make([]byte, 8)
-	binary.PutUvarint(b, server.CommitIndex())
-
-	if e != nil {
-		return b, nil
-	}
-
-	// check machine number in the cluster
-	num := machineNum()
-	if num == maxClusterSize {
-		debug("Reject join request from ", c.Name)
-		return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMoreMachine, "", server.CommitIndex(), server.Term())
-	}
-
-	addNameToURL(c.Name, c.RaftVersion, c.RaftURL, c.EtcdURL)
-
-	// add peer in raft
-	err := server.AddPeer(c.Name, "")
-
-	// add machine in etcd storage
-	key := path.Join("_etcd/machines", c.Name)
-	value := fmt.Sprintf("raft=%s&etcd=%s&raftVersion=%s", c.RaftURL, c.EtcdURL, c.RaftVersion)
-	s.Create(key, value, false, false, store.Permanent, server.CommitIndex(), server.Term())
-
-	// add peer stats
-	if c.Name != r.Name() {
-		r.followersStats.Followers[c.Name] = &raftFollowerStats{}
-		r.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63
-	}
-
-	return b, err
-}
-
-func (c *JoinCommand) NodeName() string {
-	return c.Name
-}
-
-// RemoveCommand
-type RemoveCommand struct {
-	Name string `json:"name"`
-}
-
-// The name of the remove command in the log
-func (c *RemoveCommand) CommandName() string {
-	return commandName("remove")
-}
-
-// Remove a server from the cluster
-func (c *RemoveCommand) Apply(server *raft.Server) (interface{}, error) {
-	s, _ := server.StateMachine().(*store.Store)
-	r, _ := server.Context().(*raftServer)
-
-	// remove machine in etcd storage
-	key := path.Join("_etcd/machines", c.Name)
-
-	_, err := s.Delete(key, false, server.CommitIndex(), server.Term())
-	// delete from stats
-	delete(r.followersStats.Followers, c.Name)
-
-	if err != nil {
-		return []byte{0}, err
-	}
-
-	// remove peer in raft
-	err = server.RemovePeer(c.Name)
-
-	if err != nil {
-		return []byte{0}, err
-	}
-
-	if c.Name == server.Name() {
-		// the removed node is this node
-
-		// if the node is not replaying the previous logs
-		// and the node has sent out a join request in this
-		// start. It is sure that this node received a new remove
-		// command and need to be removed
-		if server.CommitIndex() > r.joinIndex && r.joinIndex != 0 {
-			debugf("server [%s] is removed", server.Name())
-			os.Exit(0)
-		} else {
-			// else ignore remove
-			debugf("ignore previous remove command.")
-		}
-	}
-
-	b := make([]byte, 8)
-	binary.PutUvarint(b, server.CommitIndex())
-
-	return b, err
-}

command/command.go

@@ -0,0 +1,19 @@
+package command
+
+import (
+	"github.com/coreos/go-raft"
+)
+
+// A command represents an action to be taken on the replicated state machine.
+type Command interface {
+	CommandName() string
+	Apply(server *raft.Server) (interface{}, error)
+}
+
+// Registers commands to the Raft library.
+func Register() {
+	raft.RegisterCommand(&DeleteCommand{})
+	raft.RegisterCommand(&TestAndSetCommand{})
+	raft.RegisterCommand(&CreateCommand{})
+	raft.RegisterCommand(&UpdateCommand{})
+}

command/create_command.go

@@ -0,0 +1,36 @@
+package command
+
+import (
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"time"
+)
+
+// Create command
+type CreateCommand struct {
+	Key               string    `json:"key"`
+	Value             string    `json:"value"`
+	ExpireTime        time.Time `json:"expireTime"`
+	IncrementalSuffix bool      `json:"incrementalSuffix"`
+	Force             bool      `json:"force"`
+}
+
+// The name of the create command in the log
+func (c *CreateCommand) CommandName() string {
+	return "etcd:create"
+}
+
+// Create node
+func (c *CreateCommand) Apply(server *raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(*store.Store)
+
+	e, err := s.Create(c.Key, c.Value, c.IncrementalSuffix, c.Force, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}

command/delete_command.go

@@ -0,0 +1,32 @@
+package command
+
+import (
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+)
+
+// The DeleteCommand removes a key from the Store.
+type DeleteCommand struct {
+	Key       string `json:"key"`
+	Recursive bool   `json:"recursive"`
+}
+
+// The name of the delete command in the log
+func (c *DeleteCommand) CommandName() string {
+	return "etcd:delete"
+}
+
+// Delete the key
+func (c *DeleteCommand) Apply(server *raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(*store.Store)
+
+	e, err := s.Delete(c.Key, c.Recursive, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}

command/test_and_set_command.go

@@ -0,0 +1,38 @@
+package command
+
+import (
+	"time"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+)
+
+// The TestAndSetCommand performs a conditional update on a key in the store.
+type TestAndSetCommand struct {
+	Key        string    `json:"key"`
+	Value      string    `json:"value"`
+	ExpireTime time.Time `json:"expireTime"`
+	PrevValue  string    `json: prevValue`
+	PrevIndex  uint64    `json: prevIndex`
+}
+
+// The name of the testAndSet command in the log
+func (c *TestAndSetCommand) CommandName() string {
+	return "etcd:testAndSet"
+}
+
+// Set the key-value pair if the current value of the key equals to the given prevValue
+func (c *TestAndSetCommand) Apply(server *raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(*store.Store)
+
+	e, err := s.TestAndSet(c.Key, c.PrevValue, c.PrevIndex,
+		c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}

command/update_command.go

@@ -0,0 +1,35 @@
+package command
+
+import (
+	"time"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+)
+
+// The UpdateCommand updates the value of a key in the Store.
+type UpdateCommand struct {
+	Key        string    `json:"key"`
+	Value      string    `json:"value"`
+	ExpireTime time.Time `json:"expireTime"`
+}
+
+// The name of the update command in the log
+func (c *UpdateCommand) CommandName() string {
+	return "etcd:update"
+}
+
+// Update node
+func (c *UpdateCommand) Apply(server *raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(*store.Store)
+
+	e, err := s.Update(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}

etcd.go

@@ -8,8 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/coreos/etcd/store"
+	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/server"
+	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 )
 
@@ -20,7 +21,6 @@ import (
 //------------------------------------------------------------------------------
 
 var (
-	verbose     bool
 	veryVerbose bool
 
 	machines     string
@@ -47,7 +47,7 @@ var (
 )
 
 func init() {
-	flag.BoolVar(&verbose, "v", false, "verbose logging")
+	flag.BoolVar(&log.Verbose, "v", false, "verbose logging")
 	flag.BoolVar(&veryVerbose, "vv", false, "very verbose logging")
 
 	flag.StringVar(&machines, "C", "", "the ip address and port of a existing machines in the cluster, sepearate by comma")
@@ -97,12 +97,6 @@ const (
 //
 //------------------------------------------------------------------------------
 
-type TLSInfo struct {
-	CertFile string `json:"CertFile"`
-	KeyFile  string `json:"KeyFile"`
-	CAFile   string `json:"CAFile"`
-}
-
 type Info struct {
 	Name string `json:"name"`
 
@@ -117,12 +111,6 @@ type Info struct {
 	EtcdTLS TLSInfo `json:"etcdTLS"`
 }
 
-type TLSConfig struct {
-	Scheme string
-	Server tls.Config
-	Client tls.Config
-}
-
 //------------------------------------------------------------------------------
 //
 // Variables
@@ -199,6 +187,7 @@ func main() {
 
 	// Create etcd and raft server
 	r := newRaftServer(info.Name, info.RaftURL, info.RaftListenHost, &raftTLSConfig, &info.RaftTLS)
+	r.MaxClusterSize = maxClusterSize
 	snapConf = r.newSnapshotConf()
 
 	s := server.New(info.Name, info.EtcdURL, info.EtcdListenHost, &etcdTLSConfig, &info.EtcdTLS, r)
@@ -209,4 +198,3 @@ func main() {
 	r.ListenAndServe()
 	s.ListenAndServe()
 }
-

log/log.go

@@ -0,0 +1,44 @@
+package log
+
+import (
+	golog "github.com/coreos/go-log/log"
+	"os"
+)
+
+// The Verbose flag turns on verbose logging.
+var Verbose bool = false
+
+var logger *golog.Logger = golog.New("etcd", false,
+	golog.CombinedSink(os.Stdout, "[%s] %s %-9s | %s\n", []string{"prefix", "time", "priority", "message"}))
+
+func Infof(format string, v ...interface{}) {
+	logger.Infof(format, v...)
+}
+
+func Debugf(format string, v ...interface{}) {
+	if Verbose {
+		logger.Debugf(format, v...)
+	}
+}
+
+func Debug(v ...interface{}) {
+	if Verbose {
+		logger.Debug(v...)
+	}
+}
+
+func Warnf(format string, v ...interface{}) {
+	logger.Warningf(format, v...)
+}
+
+func Warn(v ...interface{}) {
+	logger.Warning(v...)
+}
+
+func Fatalf(format string, v ...interface{}) {
+	logger.Fatalf(format, v...)
+}
+
+func Fatal(v ...interface{}) {
+	logger.Fatalln(v...)
+}

machines.go

@@ -1,16 +1,5 @@
 package main
 
-// machineNum returns the number of machines in the cluster
-func machineNum() int {
-	e, err := etcdStore.Get("/_etcd/machines", false, false, 0, 0)
-
-	if err != nil {
-		return 0
-	}
-
-	return len(e.KVPairs)
-}
-
 // getMachines gets the current machines in the cluster
 func (r *raftServer) getMachines(toURL func(string) (string, bool)) []string {
 	peers := r.Peers()

raft_server.go

@@ -11,10 +11,15 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/coreos/etcd/command"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/go-raft"
 )
 
+func init() {
+	command.Register()
+}
+
 type raftServer struct {
 	*raft.Server
 	version        string
@@ -26,10 +31,9 @@ type raftServer struct {
 	tlsInfo        *TLSInfo
 	followersStats *raftFollowersStats
 	serverStats    *raftServerStats
+	MaxClusterSize int
 }
 
-//var r *raftServer
-
 func newRaftServer(name string, url string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo) *raftServer {
 
 	raftWrapper := &raftServer{
@@ -68,9 +72,6 @@ func newRaftServer(name string, url string, listenHost string, tlsConf *TLSConfi
 
 // Start the raft server
 func (r *raftServer) ListenAndServe() {
-	// Setup commands.
-	registerCommands()
-
 	// LoadSnapshot
 	if snapshot {
 		err := r.LoadSnapshot()
@@ -314,16 +315,3 @@ func (r *raftServer) PeerStats() []byte {
 	}
 	return nil
 }
-
-// Register commands to raft server
-func registerCommands() {
-	raft.RegisterCommand(&JoinCommand{})
-	raft.RegisterCommand(&RemoveCommand{})
-	raft.RegisterCommand(&GetCommand{})
-	raft.RegisterCommand(&DeleteCommand{})
-	raft.RegisterCommand(&WatchCommand{})
-	raft.RegisterCommand(&TestAndSetCommand{})
-
-	raft.RegisterCommand(&CreateCommand{})
-	raft.RegisterCommand(&UpdateCommand{})
-}

raft_stats.go

@@ -1,210 +0,0 @@
-package main
-
-import (
-	"math"
-	"sync"
-	"time"
-
-	"github.com/coreos/go-raft"
-)
-
-const (
-	queueCapacity = 200
-)
-
-// packageStats represent the stats we need for a package.
-// It has sending time and the size of the package.
-type packageStats struct {
-	sendingTime time.Time
-	size        int
-}
-
-// NewPackageStats creates a pacakgeStats and return the pointer to it.
-func NewPackageStats(now time.Time, size int) *packageStats {
-	return &packageStats{
-		sendingTime: now,
-		size:        size,
-	}
-}
-
-// Time return the sending time of the package.
-func (ps *packageStats) Time() time.Time {
-	return ps.sendingTime
-}
-
-type raftServerStats struct {
-	Name      string    `json:"name"`
-	State     string    `json:"state"`
-	StartTime time.Time `json:"startTime"`
-
-	LeaderInfo struct {
-		Name      string `json:"leader"`
-		Uptime    string `json:"uptime"`
-		startTime time.Time
-	} `json:"leaderInfo"`
-
-	RecvAppendRequestCnt uint64  `json:"recvAppendRequestCnt,"`
-	RecvingPkgRate       float64 `json:"recvPkgRate,omitempty"`
-	RecvingBandwidthRate float64 `json:"recvBandwidthRate,omitempty"`
-
-	SendAppendRequestCnt uint64  `json:"sendAppendRequestCnt"`
-	SendingPkgRate       float64 `json:"sendPkgRate,omitempty"`
-	SendingBandwidthRate float64 `json:"sendBandwidthRate,omitempty"`
-
-	sendRateQueue *statsQueue
-	recvRateQueue *statsQueue
-}
-
-func (ss *raftServerStats) RecvAppendReq(leaderName string, pkgSize int) {
-	ss.State = raft.Follower
-	if leaderName != ss.LeaderInfo.Name {
-		ss.LeaderInfo.Name = leaderName
-		ss.LeaderInfo.startTime = time.Now()
-	}
-
-	ss.recvRateQueue.Insert(NewPackageStats(time.Now(), pkgSize))
-	ss.RecvAppendRequestCnt++
-}
-
-func (ss *raftServerStats) SendAppendReq(pkgSize int) {
-	now := time.Now()
-
-	if ss.State != raft.Leader {
-		ss.State = raft.Leader
-		ss.LeaderInfo.Name = ss.Name
-		ss.LeaderInfo.startTime = now
-	}
-
-	ss.sendRateQueue.Insert(NewPackageStats(now, pkgSize))
-
-	ss.SendAppendRequestCnt++
-}
-
-type raftFollowersStats struct {
-	Leader    string                        `json:"leader"`
-	Followers map[string]*raftFollowerStats `json:"followers"`
-}
-
-type raftFollowerStats struct {
-	Latency struct {
-		Current           float64 `json:"current"`
-		Average           float64 `json:"average"`
-		averageSquare     float64
-		StandardDeviation float64 `json:"standardDeviation"`
-		Minimum           float64 `json:"minimum"`
-		Maximum           float64 `json:"maximum"`
-	} `json:"latency"`
-
-	Counts struct {
-		Fail    uint64 `json:"fail"`
-		Success uint64 `json:"success"`
-	} `json:"counts"`
-}
-
-// Succ function update the raftFollowerStats with a successful send
-func (ps *raftFollowerStats) Succ(d time.Duration) {
-	total := float64(ps.Counts.Success) * ps.Latency.Average
-	totalSquare := float64(ps.Counts.Success) * ps.Latency.averageSquare
-
-	ps.Counts.Success++
-
-	ps.Latency.Current = float64(d) / (1000000.0)
-
-	if ps.Latency.Current > ps.Latency.Maximum {
-		ps.Latency.Maximum = ps.Latency.Current
-	}
-
-	if ps.Latency.Current < ps.Latency.Minimum {
-		ps.Latency.Minimum = ps.Latency.Current
-	}
-
-	ps.Latency.Average = (total + ps.Latency.Current) / float64(ps.Counts.Success)
-	ps.Latency.averageSquare = (totalSquare + ps.Latency.Current*ps.Latency.Current) / float64(ps.Counts.Success)
-
-	// sdv = sqrt(avg(x^2) - avg(x)^2)
-	ps.Latency.StandardDeviation = math.Sqrt(ps.Latency.averageSquare - ps.Latency.Average*ps.Latency.Average)
-}
-
-// Fail function update the raftFollowerStats with a unsuccessful send
-func (ps *raftFollowerStats) Fail() {
-	ps.Counts.Fail++
-}
-
-type statsQueue struct {
-	items        [queueCapacity]*packageStats
-	size         int
-	front        int
-	back         int
-	totalPkgSize int
-	rwl          sync.RWMutex
-}
-
-func (q *statsQueue) Len() int {
-	return q.size
-}
-
-func (q *statsQueue) PkgSize() int {
-	return q.totalPkgSize
-}
-
-// FrontAndBack gets the front and back elements in the queue
-// We must grab front and back together with the protection of the lock
-func (q *statsQueue) frontAndBack() (*packageStats, *packageStats) {
-	q.rwl.RLock()
-	defer q.rwl.RUnlock()
-	if q.size != 0 {
-		return q.items[q.front], q.items[q.back]
-	}
-	return nil, nil
-}
-
-// Insert function insert a packageStats into the queue and update the records
-func (q *statsQueue) Insert(p *packageStats) {
-	q.rwl.Lock()
-	defer q.rwl.Unlock()
-
-	q.back = (q.back + 1) % queueCapacity
-
-	if q.size == queueCapacity { //dequeue
-		q.totalPkgSize -= q.items[q.front].size
-		q.front = (q.back + 1) % queueCapacity
-	} else {
-		q.size++
-	}
-
-	q.items[q.back] = p
-	q.totalPkgSize += q.items[q.back].size
-
-}
-
-// Rate function returns the package rate and byte rate
-func (q *statsQueue) Rate() (float64, float64) {
-	front, back := q.frontAndBack()
-
-	if front == nil || back == nil {
-		return 0, 0
-	}
-
-	if time.Now().Sub(back.Time()) > time.Second {
-		q.Clear()
-		return 0, 0
-	}
-
-	sampleDuration := back.Time().Sub(front.Time())
-
-	pr := float64(q.Len()) / float64(sampleDuration) * float64(time.Second)
-
-	br := float64(q.PkgSize()) / float64(sampleDuration) * float64(time.Second)
-
-	return pr, br
-}
-
-// Clear function clear up the statsQueue
-func (q *statsQueue) Clear() {
-	q.rwl.Lock()
-	defer q.rwl.Unlock()
-	q.back = -1
-	q.front = 0
-	q.size = 0
-	q.totalPkgSize = 0
-}

server/join_command.go

@@ -0,0 +1,84 @@
+package server
+
+import (
+	"encoding/binary"
+	"fmt"
+	"path"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+)
+
+func init() {
+	raft.RegisterCommand(&JoinCommand{})
+}
+
+// The JoinCommand adds a node to the cluster.
+type JoinCommand struct {
+	RaftVersion    string `json:"raftVersion"`
+	Name           string `json:"name"`
+	RaftURL        string `json:"raftURL"`
+	EtcdURL        string `json:"etcdURL"`
+	MaxClusterSize int    `json:"maxClusterSize"`
+}
+
+func NewJoinCommand(version, name, raftUrl, etcdUrl string, maxClusterSize int) *JoinCommand {
+	return &JoinCommand{
+		RaftVersion:    version,
+		Name:           name,
+		RaftURL:        raftUrl,
+		EtcdURL:        etcdUrl,
+		MaxClusterSize: maxClusterSize,
+	}
+}
+
+// The name of the join command in the log
+func (c *JoinCommand) CommandName() string {
+	return "etcd:join"
+}
+
+// Join a server to the cluster
+func (c *JoinCommand) Apply(server *raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(*store.Store)
+	r, _ := server.Context().(*RaftServer)
+
+	// check if the join command is from a previous machine, who lost all its previous log.
+	e, _ := s.Get(path.Join("/_etcd/machines", c.Name), false, false, server.CommitIndex(), server.Term())
+
+	b := make([]byte, 8)
+	binary.PutUvarint(b, server.CommitIndex())
+
+	if e != nil {
+		return b, nil
+	}
+
+	// check machine number in the cluster
+	if s.MachineCount() == c.MaxClusterSize {
+		log.Debug("Reject join request from ", c.Name)
+		return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMoreMachine, "", server.CommitIndex(), server.Term())
+	}
+
+	addNameToURL(c.Name, c.RaftVersion, c.RaftURL, c.EtcdURL)
+
+	// add peer in raft
+	err := server.AddPeer(c.Name, "")
+
+	// add machine in etcd storage
+	key := path.Join("_etcd/machines", c.Name)
+	value := fmt.Sprintf("raft=%s&etcd=%s&raftVersion=%s", c.RaftURL, c.EtcdURL, c.RaftVersion)
+	s.Create(key, value, false, false, store.Permanent, server.CommitIndex(), server.Term())
+
+	// add peer stats
+	if c.Name != r.Name() {
+		r.followersStats.Followers[c.Name] = &raftFollowerStats{}
+		r.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63
+	}
+
+	return b, err
+}
+
+func (c *JoinCommand) NodeName() string {
+	return c.Name
+}

server/package_stats.go

@@ -0,0 +1,25 @@
+package server
+
+import (
+	"time"
+)
+
+// packageStats represent the stats we need for a package.
+// It has sending time and the size of the package.
+type packageStats struct {
+	sendingTime time.Time
+	size        int
+}
+
+// NewPackageStats creates a pacakgeStats and return the pointer to it.
+func NewPackageStats(now time.Time, size int) *packageStats {
+	return &packageStats{
+		sendingTime: now,
+		size:        size,
+	}
+}
+
+// Time return the sending time of the package.
+func (ps *packageStats) Time() time.Time {
+	return ps.sendingTime
+}

server/raft_follower_stats.go

@@ -0,0 +1,56 @@
+package server
+
+import (
+	"math"
+	"time"
+)
+
+type raftFollowersStats struct {
+	Leader    string                        `json:"leader"`
+	Followers map[string]*raftFollowerStats `json:"followers"`
+}
+
+type raftFollowerStats struct {
+	Latency struct {
+		Current           float64 `json:"current"`
+		Average           float64 `json:"average"`
+		averageSquare     float64
+		StandardDeviation float64 `json:"standardDeviation"`
+		Minimum           float64 `json:"minimum"`
+		Maximum           float64 `json:"maximum"`
+	} `json:"latency"`
+
+	Counts struct {
+		Fail    uint64 `json:"fail"`
+		Success uint64 `json:"success"`
+	} `json:"counts"`
+}
+
+// Succ function update the raftFollowerStats with a successful send
+func (ps *raftFollowerStats) Succ(d time.Duration) {
+	total := float64(ps.Counts.Success) * ps.Latency.Average
+	totalSquare := float64(ps.Counts.Success) * ps.Latency.averageSquare
+
+	ps.Counts.Success++
+
+	ps.Latency.Current = float64(d) / (1000000.0)
+
+	if ps.Latency.Current > ps.Latency.Maximum {
+		ps.Latency.Maximum = ps.Latency.Current
+	}
+
+	if ps.Latency.Current < ps.Latency.Minimum {
+		ps.Latency.Minimum = ps.Latency.Current
+	}
+
+	ps.Latency.Average = (total + ps.Latency.Current) / float64(ps.Counts.Success)
+	ps.Latency.averageSquare = (totalSquare + ps.Latency.Current*ps.Latency.Current) / float64(ps.Counts.Success)
+
+	// sdv = sqrt(avg(x^2) - avg(x)^2)
+	ps.Latency.StandardDeviation = math.Sqrt(ps.Latency.averageSquare - ps.Latency.Average*ps.Latency.Average)
+}
+
+// Fail function update the raftFollowerStats with a unsuccessful send
+func (ps *raftFollowerStats) Fail() {
+	ps.Counts.Fail++
+}

server/raft_server_stats.go

@@ -0,0 +1,55 @@
+package server
+
+import (
+	"time"
+
+	"github.com/coreos/go-raft"
+)
+
+type raftServerStats struct {
+	Name      string    `json:"name"`
+	State     string    `json:"state"`
+	StartTime time.Time `json:"startTime"`
+
+	LeaderInfo struct {
+		Name      string `json:"leader"`
+		Uptime    string `json:"uptime"`
+		startTime time.Time
+	} `json:"leaderInfo"`
+
+	RecvAppendRequestCnt uint64  `json:"recvAppendRequestCnt,"`
+	RecvingPkgRate       float64 `json:"recvPkgRate,omitempty"`
+	RecvingBandwidthRate float64 `json:"recvBandwidthRate,omitempty"`
+
+	SendAppendRequestCnt uint64  `json:"sendAppendRequestCnt"`
+	SendingPkgRate       float64 `json:"sendPkgRate,omitempty"`
+	SendingBandwidthRate float64 `json:"sendBandwidthRate,omitempty"`
+
+	sendRateQueue *statsQueue
+	recvRateQueue *statsQueue
+}
+
+func (ss *raftServerStats) RecvAppendReq(leaderName string, pkgSize int) {
+	ss.State = raft.Follower
+	if leaderName != ss.LeaderInfo.Name {
+		ss.LeaderInfo.Name = leaderName
+		ss.LeaderInfo.startTime = time.Now()
+	}
+
+	ss.recvRateQueue.Insert(NewPackageStats(time.Now(), pkgSize))
+	ss.RecvAppendRequestCnt++
+}
+
+func (ss *raftServerStats) SendAppendReq(pkgSize int) {
+	now := time.Now()
+
+	if ss.State != raft.Leader {
+		ss.State = raft.Leader
+		ss.LeaderInfo.Name = ss.Name
+		ss.LeaderInfo.startTime = now
+	}
+
+	ss.sendRateQueue.Insert(NewPackageStats(now, pkgSize))
+
+	ss.SendAppendRequestCnt++
+}

server/remove_command.go

@@ -0,0 +1,68 @@
+package server
+
+import (
+	"encoding/binary"
+	"path"
+
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+)
+
+func init() {
+	raft.RegisterCommand(&RemoveCommand{})
+}
+
+// The RemoveCommand removes a server from the cluster.
+type RemoveCommand struct {
+	Name string `json:"name"`
+}
+
+// The name of the remove command in the log
+func (c *RemoveCommand) CommandName() string {
+	return "etcd:remove"
+}
+
+// Remove a server from the cluster
+func (c *RemoveCommand) Apply(server *raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(*store.Store)
+	r, _ := server.Context().(*RaftServer)
+
+	// remove machine in etcd storage
+	key := path.Join("_etcd/machines", c.Name)
+
+	_, err := s.Delete(key, false, server.CommitIndex(), server.Term())
+	// delete from stats
+	delete(r.followersStats.Followers, c.Name)
+
+	if err != nil {
+		return []byte{0}, err
+	}
+
+	// remove peer in raft
+	err = server.RemovePeer(c.Name)
+
+	if err != nil {
+		return []byte{0}, err
+	}
+
+	if c.Name == server.Name() {
+		// the removed node is this node
+
+		// if the node is not replaying the previous logs
+		// and the node has sent out a join request in this
+		// start. It is sure that this node received a new remove
+		// command and need to be removed
+		if server.CommitIndex() > r.joinIndex && r.joinIndex != 0 {
+			debugf("server [%s] is removed", server.Name())
+			os.Exit(0)
+		} else {
+			// else ignore remove
+			debugf("ignore previous remove command.")
+		}
+	}
+
+	b := make([]byte, 8)
+	binary.PutUvarint(b, server.CommitIndex())
+
+	return b, err
+}

server/server.go

@@ -1,16 +1,19 @@
 package server
 
 import (
-	"github.com/gorilla/mux"
 	"net/http"
 	"net/url"
+
+	"github.com/coreos/etcd/command"
+	"github.com/coreos/go-raft"
+	"github.com/gorilla/mux"
 )
 
 // The Server provides an HTTP interface to the underlying store.
 type Server interface {
 	CommitIndex() uint64
 	Term() uint64
-    Dispatch(Command, http.ResponseWriter, *http.Request)
+	Dispatch(command.Command, http.ResponseWriter, *http.Request)
 }
 
 // This is the default implementation of the Server interface.
@@ -55,11 +58,6 @@ func (s *server) Term() uint64 {
 	return c.raftServer.Term()
 }
 
-// Executes a command against the Raft server.
-func (s *server) Do(c Command, localOnly bool) (interface{}, error) {
-	return c.raftServer.Do(s.RaftServer().Server)
-}
-
 func (s *server) installV1() {
 	s.handleFunc("/v1/keys/{key:.*}", v1.GetKeyHandler).Methods("GET")
 	s.handleFunc("/v1/keys/{key:.*}", v1.SetKeyHandler).Methods("POST", "PUT")

server/stats_queue.go

@@ -0,0 +1,88 @@
+package server
+
+import (
+	"sync"
+)
+
+const (
+	queueCapacity = 200
+)
+
+type statsQueue struct {
+	items        [queueCapacity]*packageStats
+	size         int
+	front        int
+	back         int
+	totalPkgSize int
+	rwl          sync.RWMutex
+}
+
+func (q *statsQueue) Len() int {
+	return q.size
+}
+
+func (q *statsQueue) PkgSize() int {
+	return q.totalPkgSize
+}
+
+// FrontAndBack gets the front and back elements in the queue
+// We must grab front and back together with the protection of the lock
+func (q *statsQueue) frontAndBack() (*packageStats, *packageStats) {
+	q.rwl.RLock()
+	defer q.rwl.RUnlock()
+	if q.size != 0 {
+		return q.items[q.front], q.items[q.back]
+	}
+	return nil, nil
+}
+
+// Insert function insert a packageStats into the queue and update the records
+func (q *statsQueue) Insert(p *packageStats) {
+	q.rwl.Lock()
+	defer q.rwl.Unlock()
+
+	q.back = (q.back + 1) % queueCapacity
+
+	if q.size == queueCapacity { //dequeue
+		q.totalPkgSize -= q.items[q.front].size
+		q.front = (q.back + 1) % queueCapacity
+	} else {
+		q.size++
+	}
+
+	q.items[q.back] = p
+	q.totalPkgSize += q.items[q.back].size
+
+}
+
+// Rate function returns the package rate and byte rate
+func (q *statsQueue) Rate() (float64, float64) {
+	front, back := q.frontAndBack()
+
+	if front == nil || back == nil {
+		return 0, 0
+	}
+
+	if time.Now().Sub(back.Time()) > time.Second {
+		q.Clear()
+		return 0, 0
+	}
+
+	sampleDuration := back.Time().Sub(front.Time())
+
+	pr := float64(q.Len()) / float64(sampleDuration) * float64(time.Second)
+
+	br := float64(q.PkgSize()) / float64(sampleDuration) * float64(time.Second)
+
+	return pr, br
+}
+
+// Clear function clear up the statsQueue
+func (q *statsQueue) Clear() {
+	q.rwl.Lock()
+	defer q.rwl.Unlock()
+	q.back = -1
+	q.front = 0
+	q.size = 0
+	q.totalPkgSize = 0
+}

server/tls_config.go

@@ -0,0 +1,11 @@
+package server
+
+import (
+	"crypto/tls"
+)
+
+type TLSConfig struct {
+	Scheme string
+	Server tls.Config
+	Client tls.Config
+}

server/tls_info.go

@@ -0,0 +1,7 @@
+package server
+
+type TLSInfo struct {
+	CertFile string `json:"CertFile"`
+	KeyFile  string `json:"KeyFile"`
+	CAFile   string `json:"CAFile"`
+}

store/store.go

@@ -401,6 +401,16 @@ func (s *Store) checkDir(parent *Node, dirName string) (*Node, *etcdErr.Error) {
 	return n, nil
 }
 
+// Returns the number of machines in the cluster.
+func (s *Store) MachineCount() int {
+	e, err := s.Get("/_etcd/machines", false, false, 0, 0)
+	if err != nil {
+		return 0
+	}
+
+	return len(e.KVPairs)
+}
+
 // Save function saves the static state of the store system.
 // Save function will not be able to save the state of watchers.
 // Save function will not save the parent field of the node. Or there will

util.go

@@ -15,7 +15,6 @@ import (
 
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/store"
-	"github.com/coreos/go-log/log"
 	"github.com/coreos/go-raft"
 )
 
@@ -172,45 +171,6 @@ func getNodePath(urlPath string) string {
 	return urlPath[pathPrefixLen:]
 }
 
-//--------------------------------------
-// Log
-//--------------------------------------
-
-var logger *log.Logger = log.New("etcd", false,
-	log.CombinedSink(os.Stdout, "[%s] %s %-9s | %s\n", []string{"prefix", "time", "priority", "message"}))
-
-func infof(format string, v ...interface{}) {
-	logger.Infof(format, v...)
-}
-
-func debugf(format string, v ...interface{}) {
-	if verbose {
-		logger.Debugf(format, v...)
-	}
-}
-
-func debug(v ...interface{}) {
-	if verbose {
-		logger.Debug(v...)
-	}
-}
-
-func warnf(format string, v ...interface{}) {
-	logger.Warningf(format, v...)
-}
-
-func warn(v ...interface{}) {
-	logger.Warning(v...)
-}
-
-func fatalf(format string, v ...interface{}) {
-	logger.Fatalf(format, v...)
-}
-
-func fatal(v ...interface{}) {
-	logger.Fatalln(v...)
-}
-
 //--------------------------------------
 // CPU profile
 //--------------------------------------