Merge pull request #12588 from ptabor/20201230-raft-docs

Raft: Expand raft documentation, in particular point on the godocs

api/etcdserverpb/rpc.proto

@@ -238,7 +238,9 @@ service Maintenance {
     };
   }
 
-  // Downgrade requests downgrade, cancel downgrade on the cluster version.
+  // Downgrade requests downgrades, verifies feasibility or cancels downgrade
+  // on the cluster version.
+  // Supported since etcd 3.5.
   rpc Downgrade(DowngradeRequest) returns (DowngradeResponse) {
     option (google.api.http) = {
       post: "/v3/maintenance/downgrade"

raft/README.md

@@ -195,3 +195,7 @@ This implementation is up to date with the final Raft thesis (https://github.com
 To ensure there is no attempt to commit two membership changes at once by matching log positions (which would be unsafe since they should have different quorum requirements), any proposed membership change is simply disallowed while any uncommitted change appears in the leader's log.
 
 This approach introduces a problem when removing a member from a two-member cluster: If one of the members dies before the other one receives the commit of the confchange entry, then the member cannot be removed any more since the cluster cannot make progress. For this reason it is highly recommended to use three or more nodes in every cluster.
+
+## Go docs
+
+More detailed development documentation can be found in go docs: https://pkg.go.dev/go.etcd.io/etcd/raft/v3.

raft/raft.go

@@ -397,7 +397,8 @@ func (r *raft) hardState() pb.HardState {
 	}
 }
 
-// send persists state to stable storage and then sends to its mailbox.
+// send schedules persisting state to a stable storage and AFTER that
+// sending the message (as part of next Ready message processing).
 func (r *raft) send(m pb.Message) {
 	if m.From == None {
 		m.From = r.id

raft/raftpb/raft.proto

@@ -33,6 +33,8 @@ message Snapshot {
 	optional SnapshotMetadata metadata = 2 [(gogoproto.nullable) = false];
 }
 
+// For description of different message types, see:
+// https://pkg.go.dev/go.etcd.io/etcd/raft/v3#hdr-MessageType
 enum MessageType {
 	MsgHup             = 0;
 	MsgBeat            = 1;

server/wal/wal.go

@@ -419,6 +419,13 @@ func openWALFiles(lg *zap.Logger, dirpath string, names []string, nameIndex int,
 // TODO: detect not-last-snap error.
 // TODO: maybe loose the checking of match.
 // After ReadAll, the WAL will be ready for appending new records.
+//
+// ReadAll suppresses WAL entries that got overridden (i.e. a newer entry with the same index
+// exists in the log). Such a situation can happen in cases described in figure 7. of the
+// RAFT paper (http://web.stanford.edu/~ouster/cgi-bin/papers/raft-atc14.pdf).
+//
+// ReadAll may return uncommitted yet entries, that are subject to be overriden.
+// Do not apply entries that have index > state.commit, as they are subject to change.
 func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb.Entry, err error) {
 	w.mu.Lock()
 	defer w.mu.Unlock()
@@ -443,6 +450,7 @@ func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb.
 					// return error before append call causes runtime panic
 					return nil, state, nil, ErrSliceOutOfRange
 				}
+				// The line below is potentially overriding some 'uncommitted' entries.
 				ents = append(ents[:up], e)
 			}
 			w.enti = e.Index