*: add proposalsCommitted metrics

Documentation/metrics.md

@@ -24,6 +24,7 @@ All these metrics are prefixed with `etcd_server_`
 |---------------------------|----------------------------------------------------------|---------|
 | has_leader                | Whether or not a leader exists. 1 is existence, 0 is not.| Gauge   |
 | leader_changes_seen_total | The number of leader changes seen.                       | Counter |
+| proposals_committed_total | The total number of consensus proposals committed.       | Gauge   |
 
 
 `has_leader` indicates whether the member has a leader. If a member does not have a leader, it is
@@ -32,6 +33,8 @@ is totally unavailable.
 
 `leader_changes_seen_total` counts the number of leader changes the member has seen since its start. Rapid leadership changes impact the performance of etcd significantly. It also signals that the leader is unstable, perhaps due to network connectivity issues or excessive load hitting the etcd cluster.
 
+`proposals_committed_total` records the total number of consensus proposals committed. This gauge should increase over time if the cluster is healthy. Several healthy members of an etcd cluster may have different total committed proposals at once. This discrepancy may be due to recovering from peers after starting, lagging behind the leader, or being the leader and therefore having the most commits. It is important to monitor this metric across all the members in the cluster; a consistently large lag between a single member and its leader indicates that member is slow or unhealthy.
+
 ### network
 
 These metrics describe the status of the network.

etcdserver/metrics.go

@@ -58,6 +58,12 @@ var (
 		Name:      "leader_changes_seen_total",
 		Help:      "The number of leader changes seen.",
 	})
+	proposalsCommitted = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "proposals_committed_total",
+		Help:      "The total number of consensus proposals committed.",
+	})
 )
 
 func init() {
@@ -66,6 +72,7 @@ func init() {
 	prometheus.MustRegister(proposeFailed)
 	prometheus.MustRegister(hasLeader)
 	prometheus.MustRegister(leaderChanges)
+	prometheus.MustRegister(proposalsCommitted)
 }
 
 func monitorFileDescriptor(done <-chan struct{}) {

etcdserver/raft.go

@@ -226,6 +226,10 @@ func (r *raftNode) start(s *EtcdServer) {
 				if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
 					plog.Fatalf("raft save state and entries error: %v", err)
 				}
+				if !raft.IsEmptyHardState(rd.HardState) {
+					proposalsCommitted.Set(float64(rd.HardState.Commit))
+				}
+
 				r.raftStorage.Append(rd.Entries)
 
 				if !islead {