Proxy promotion.

server/cluster_config.go

@@ -21,7 +21,7 @@ type ClusterConfig struct {
 
 	// PromoteDelay is the amount of time, in seconds, after a node is
 	// unreachable that it will be swapped out for a proxy node, if available.
-	PromoteDelay int `json:"PromoteDelay"`
+	PromoteDelay int `json:"promoteDelay"`
 }
 
 // NewClusterConfig returns a cluster configuration with default settings.

server/join_command.go

@@ -62,6 +62,11 @@ func (c *JoinCommand) Apply(context raft.Context) (interface{}, error) {
 		return buf.Bytes(), nil
 	}
 
+	// Remove it as a proxy if it is one.
+	if ps.registry.ProxyExists(c.Name) {
+		ps.registry.UnregisterProxy(c.Name)
+	}
+
 	// Add to shared peer registry.
 	ps.registry.RegisterPeer(c.Name, c.RaftURL, c.EtcdURL)
 

server/peer_server.go

@@ -141,7 +141,6 @@ func (s *PeerServer) ClusterConfig() *ClusterConfig {
 // SetClusterConfig updates the current cluster configuration.
 // Adjusting the active size will 
 func (s *PeerServer) SetClusterConfig(c *ClusterConfig) error {
-	prevActiveSize := s.clusterConfig.ActiveSize
 	s.clusterConfig = c
 
 	// Validate configuration.
@@ -294,9 +293,10 @@ func (s *PeerServer) HTTPHandler() http.Handler {
 	router.HandleFunc("/version/{version:[0-9]+}/check", s.VersionCheckHttpHandler)
 	router.HandleFunc("/upgrade", s.UpgradeHttpHandler)
 	router.HandleFunc("/join", s.JoinHttpHandler)
+	router.HandleFunc("/promote", s.PromoteHttpHandler).Methods("POST")
 	router.HandleFunc("/remove/{name:.+}", s.RemoveHttpHandler)
 	router.HandleFunc("/config", s.getClusterConfigHttpHandler).Methods("GET")
-	router.HandleFunc("/config", s.setClusterConfigHttpHandler).Methods("POST")
+	router.HandleFunc("/config", s.setClusterConfigHttpHandler).Methods("PUT")
 	router.HandleFunc("/vote", s.VoteHttpHandler)
 	router.HandleFunc("/log", s.GetLogHttpHandler)
 	router.HandleFunc("/log/append", s.AppendEntriesHttpHandler)
@@ -632,18 +632,45 @@ func (s *PeerServer) monitorActive(closeChan chan bool) {
 		peerCount := s.registry.PeerCount()
 		proxies := s.registry.Proxies()
 		peers := s.registry.Peers()
+		fmt.Println("active.3»", peers)
 		if index := sort.SearchStrings(peers, s.Config.Name); index < len(peers) && peers[index] == s.Config.Name {
 			peers = append(peers[:index], peers[index+1:]...)
 		}
 
+		fmt.Println("active.1»", activeSize, peerCount)
+		fmt.Println("active.2»", proxies)
+
 		// If we have more active nodes than we should then demote.
 		if peerCount > activeSize {
 			peer := peers[rand.Intn(len(peers))]
+			fmt.Println("active.demote»", peer)
 			if _, err := s.raftServer.Do(&RemoveCommand{Name: peer}); err != nil {
 				log.Infof("%s: warning: demotion error: %v", s.Config.Name, err)
 			}
 			continue
 		}
+
+		// If we don't have enough active nodes then try to promote a proxy.
+		if peerCount < activeSize && len(proxies) > 0 {
+			proxy := proxies[rand.Intn(len(proxies))]
+			proxyPeerURL, _ := s.registry.ProxyPeerURL(proxy)
+			log.Infof("%s: promoting: %v (%s)", s.Config.Name, proxy, proxyPeerURL)
+
+			// Notify proxy to promote itself.
+			client := &http.Client{
+				Transport: &http.Transport{
+					DisableKeepAlives: false,
+					ResponseHeaderTimeout: ActiveMonitorTimeout,
+				},
+			}
+			resp, err := client.Post(fmt.Sprintf("%s/promote", proxyPeerURL), "application/json", nil)
+			if err != nil {
+				log.Infof("%s: warning: promotion error: %v", s.Config.Name, err)
+			} else if resp.StatusCode != http.StatusOK {
+				log.Infof("%s: warning: promotion failure: %v", s.Config.Name, resp.StatusCode)
+			}
+			continue
+		}
 	}
 }
 

server/peer_server_handlers.go

@@ -3,6 +3,7 @@ package server
 import (
 	"encoding/json"
 	"net/http"
+	"net/url"
 	"strconv"
 	"time"
 
@@ -171,6 +172,25 @@ func (ps *PeerServer) JoinHttpHandler(w http.ResponseWriter, req *http.Request)
 	}
 }
 
+// Attempt to rejoin the cluster as a peer.
+func (ps *PeerServer) PromoteHttpHandler(w http.ResponseWriter, req *http.Request) {
+	log.Infof("%s attempting to promote in cluster: %s", ps.Config.Name, ps.proxyPeerURL)
+	url, err := url.Parse(ps.proxyPeerURL)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	err = ps.joinByPeer(ps.raftServer, url.Host, ps.Config.Scheme)
+	if err != nil {
+		log.Infof("%s error while promoting: %v", ps.Config.Name, err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	log.Infof("%s promoted in the cluster", ps.Config.Name)
+	w.WriteHeader(http.StatusOK)
+}
+
 // Response to remove request
 func (ps *PeerServer) RemoveHttpHandler(w http.ResponseWriter, req *http.Request) {
 	if req.Method != "DELETE" {

server/promote_command.go

@@ -1,64 +0,0 @@
-package server
-
-import (
-	"github.com/coreos/etcd/log"
-	"github.com/coreos/etcd/third_party/github.com/coreos/raft"
-)
-
-func init() {
-	raft.RegisterCommand(&PromoteCommand{})
-}
-
-// PromoteCommand represents a Raft command for converting a proxy to a peer.
-type PromoteCommand struct {
-	Name string `json:"name"`
-}
-
-// CommandName returns the name of the command.
-func (c *PromoteCommand) CommandName() string {
-	return "etcd:promote"
-}
-
-// Apply promotes a named proxy to a peer.
-func (c *PromoteCommand) Apply(context raft.Context) (interface{}, error) {
-	ps, _ := context.Server().Context().(*PeerServer)
-	config := ps.ClusterConfig()
-
-	// If cluster size is larger than max cluster size then return an error.
-	if ps.registry.PeerCount() >= config.ActiveSize {
-		return etcdErr.NewError(etcdErr.EcodePromoteError, "", 0)
-	}
-
-	// If proxy doesn't exist then return an error.
-	if !ps.registry.ProxyExists(c.Name) {
-		return etcdErr.NewError(etcdErr.EcodePromoteError, "", 0)
-	}
-
-	// Retrieve proxy settings.
-	proxyClientURL := ps.registry.ProxyClientURL()
-	proxyPeerURL := ps.registry.ProxyPeerURL()
-
-	// Remove from registry as a proxy.
-	if err := ps.registry.UnregisterProxy(c.Name); err != nil {
-		log.Info("Cannot remove proxy: ", c.Name)
-		return nil, err
-	}
-
-	// Add to shared peer registry.
-	ps.registry.RegisterPeer(c.Name, c.RaftURL, c.EtcdURL)
-
-	// Add peer in raft
-	err := context.Server().AddPeer(c.Name, "")
-
-	// Add peer stats
-	if c.Name != ps.RaftServer().Name() {
-		ps.followersStats.Followers[c.Name] = &raftFollowerStats{}
-		ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63
-	}
-
-	return nil, err
-}
-
-func (c *JoinCommand) NodeName() string {
-	return c.Name
-}

server/registry.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 
@@ -48,6 +49,7 @@ func (r *Registry) Peers() []string {
 	for name, _ := range r.peers {
 		names = append(names, name)
 	}
+	sort.Sort(sort.StringSlice(names))
 	return names
 }
 
@@ -57,6 +59,7 @@ func (r *Registry) Proxies() []string {
 	for name, _ := range r.proxies {
 		names = append(names, name)
 	}
+	sort.Sort(sort.StringSlice(names))
 	return names
 }
 
@@ -70,7 +73,11 @@ func (r *Registry) RegisterPeer(name string, peerURL string, machURL string) err
 // RegisterProxy adds a proxy to the registry.
 func (r *Registry) RegisterProxy(name string, peerURL string, machURL string) error {
 	// TODO(benbjohnson): Disallow proxies that are already peers.
-	return r.register(RegistryProxyKey, name, peerURL, machURL)
+	if err := r.register(RegistryProxyKey, name, peerURL, machURL); err != nil {
+		return err
+	}
+	r.proxies[name] = r.load(RegistryProxyKey, name)
+	return nil
 }
 
 func (r *Registry) register(key, name string, peerURL string, machURL string) error {
@@ -153,7 +160,9 @@ func (r *Registry) ClientURL(name string) (string, bool) {
 
 func (r *Registry) clientURL(key, name string) (string, bool) {
 	if r.peers[name] == nil {
-		r.peers[name] = r.load(key, name)
+		if node := r.load(key, name); node != nil {
+			r.peers[name] = node
+		}
 	}
 
 	if node := r.peers[name]; node != nil {
@@ -184,7 +193,9 @@ func (r *Registry) PeerURL(name string) (string, bool) {
 
 func (r *Registry) peerURL(key, name string) (string, bool) {
 	if r.peers[name] == nil {
-		r.peers[name] = r.load(key, name)
+		if node := r.load(key, name); node != nil {
+			r.peers[name] = node
+		}
 	}
 
 	if node := r.peers[name]; node != nil {
@@ -203,7 +214,9 @@ func (r *Registry) ProxyClientURL(name string) (string, bool) {
 
 func (r *Registry) proxyClientURL(key, name string) (string, bool) {
 	if r.proxies[name] == nil {
-		r.proxies[name] = r.load(key, name)
+		if node := r.load(key, name); node != nil {
+			r.proxies[name] = node
+		}
 	}
 	if node := r.proxies[name]; node != nil {
 		return node.url, true
@@ -215,12 +228,14 @@ func (r *Registry) proxyClientURL(key, name string) (string, bool) {
 func (r *Registry) ProxyPeerURL(name string) (string, bool) {
 	r.Lock()
 	defer r.Unlock()
-	return r.proxyPeerURL(RegistryProxyKey,name)
+	return r.proxyPeerURL(RegistryProxyKey, name)
 }
 
 func (r *Registry) proxyPeerURL(key, name string) (string, bool) {
 	if r.proxies[name] == nil {
-		r.proxies[name] = r.load(key, name)
+		if node := r.load(key, name); node != nil {
+			r.proxies[name] = node
+		}
 	}
 	if node := r.proxies[name]; node != nil {
 		return node.peerURL, true
@@ -278,7 +293,7 @@ func (r *Registry) load(key, name string) *node {
 	}
 
 	// Retrieve from store.
-	e, err := r.store.Get(path.Join(RegistryPeerKey, name), false, false)
+	e, err := r.store.Get(path.Join(key, name), false, false)
 	if err != nil {
 		return nil
 	}