hack(server): notify removed peers when they try to become candidates

A peer might be removed during a network partiton. When it comes back it
will not have received any of the log entries that would have notified
it of its removal and go onto propose a vote. This will disrupt the
cluster and the cluster should give the machine feedback that it is no
longer a member.

The term of a denied vote is MaxUint64. The notification of the removal
is a raft event. These two modification are quick heck.

In reaction to this notification the machine should shutdown. In this
case the shutdown just moves it towards becoming a standby server.

server/peer_server.go

@@ -122,6 +122,8 @@ func (s *PeerServer) SetRaftServer(raftServer raft.Server, snapshot bool) {
 
 	raftServer.AddEventListener(raft.HeartbeatEventType, s.recordMetricEvent)
 
+	raftServer.AddEventListener(raft.RemovedEventType, s.removedEvent)
+
 	s.raftServer = raftServer
 	s.removedInLog = false
 
@@ -644,6 +646,14 @@ func (s *PeerServer) PeerStats() []byte {
 	return nil
 }
 
+// removedEvent handles the case where a machine has been removed from the
+// cluster and is notified when it tries to become a candidate.
+func (s *PeerServer) removedEvent(event raft.Event) {
+	// HACK(philips): we need to find a better notification for this.
+	log.Infof("removed during cluster re-configuration")
+	s.asyncRemove()
+}
+
 // raftEventLogger converts events from the Raft server into log messages.
 func (s *PeerServer) raftEventLogger(event raft.Event) {
 	value := event.Value()