diff --git a/raft/quorum/joint.go b/raft/quorum/joint.go index 9f8f484dc..e3741e0b0 100644 --- a/raft/quorum/joint.go +++ b/raft/quorum/joint.go @@ -18,6 +18,13 @@ package quorum // majority configurations. Decisions require the support of both majorities. type JointConfig [2]MajorityConfig +func (c JointConfig) String() string { + if len(c[1]) > 0 { + return c[0].String() + "&&" + c[1].String() + } + return c[0].String() +} + // IDs returns a newly initialized map representing the set of voters present // in the joint configuration. func (c JointConfig) IDs() map[uint64]struct{} { diff --git a/raft/quorum/majority.go b/raft/quorum/majority.go index 3d7bf8233..5eba50344 100644 --- a/raft/quorum/majority.go +++ b/raft/quorum/majority.go @@ -24,6 +24,24 @@ import ( // MajorityConfig is a set of IDs that uses majority quorums to make decisions. type MajorityConfig map[uint64]struct{} +func (c MajorityConfig) String() string { + sl := make([]uint64, 0, len(c)) + for id := range c { + sl = append(sl, id) + } + sort.Slice(sl, func(i, j int) bool { return sl[i] < sl[j] }) + var buf strings.Builder + buf.WriteByte('(') + for i := range sl { + if i > 0 { + buf.WriteByte(' ') + } + fmt.Fprint(&buf, sl[i]) + } + buf.WriteByte(')') + return buf.String() +} + // Describe returns a (multi-line) representation of the commit indexes for the // given lookuper. func (c MajorityConfig) Describe(l AckedIndexer) string { diff --git a/raft/quorum/quorum.go b/raft/quorum/quorum.go index ff9c6f48d..2899e46c9 100644 --- a/raft/quorum/quorum.go +++ b/raft/quorum/quorum.go @@ -19,6 +19,7 @@ import ( "strconv" ) +// Index is a Raft log position. type Index uint64 func (i Index) String() string { diff --git a/raft/raft.go b/raft/raft.go index 4edf9616c..a42bb4e63 100644 --- a/raft/raft.go +++ b/raft/raft.go @@ -1461,6 +1461,7 @@ func (r *raft) applyConfChange(cc pb.ConfChange) pb.ConfState { } } + r.logger.Infof("%x switched to configuration %s", r.id, r.prs.Config) // Now that the configuration is updated, handle any side effects. cs := pb.ConfState{Nodes: r.prs.VoterNodes(), Learners: r.prs.LearnerNodes()} diff --git a/raft/tracker/tracker.go b/raft/tracker/tracker.go index 2d162c6de..4b3396fbe 100644 --- a/raft/tracker/tracker.go +++ b/raft/tracker/tracker.go @@ -21,12 +21,72 @@ import ( "go.etcd.io/etcd/raft/quorum" ) +// Config reflects the configuration tracked in a ProgressTracker. +type Config struct { + Voters quorum.JointConfig + // Learners is a set of IDs corresponding to the learners active in the + // current configuration. + // + // Invariant: Learners and Voters does not intersect, i.e. if a peer is in + // either half of the joint config, it can't be a learner; if it is a + // learner it can't be in either half of the joint config. This invariant + // simplifies the implementation since it allows peers to have clarity about + // its current role without taking into account joint consensus. + Learners map[uint64]struct{} + // TODO(tbg): when we actually carry out joint consensus changes and turn a + // voter into a learner, we cannot add the learner when entering the joint + // state. This is because this would violate the invariant that the inter- + // section of voters and learners is empty. For example, assume a Voter is + // removed and immediately re-added as a learner (or in other words, it is + // demoted). + // + // Initially, the configuration will be + // + // voters: {1 2 3} + // learners: {} + // + // and we want to demote 3. Entering the joint configuration, we naively get + // + // voters: {1 2} & {1 2 3} + // learners: {3} + // + // but this violates the invariant (3 is both voter and learner). Instead, + // we have + // + // voters: {1 2} & {1 2 3} + // learners: {} + // next_learners: {3} + // + // Where 3 is now still purely a voter, but we are remembering the intention + // to make it a learner upon transitioning into the final configuration: + // + // voters: {1 2} + // learners: {3} + // next_learners: {} + // + // Note that next_learners is not used while adding a learner that is not + // also a voter in the joint config. In this case, the learner is added + // to Learners right away when entering the joint configuration, so that it + // is caught up as soon as possible. + // + // NextLearners map[uint64]struct{} +} + +func (c *Config) String() string { + if len(c.Learners) == 0 { + return fmt.Sprintf("voters=%s", c.Voters) + } + return fmt.Sprintf( + "voters=%s learners=%s", + c.Voters, quorum.MajorityConfig(c.Learners).String(), + ) +} + // ProgressTracker tracks the currently active configuration and the information // known about the nodes and learners in it. In particular, it tracks the match // index for each peer which in turn allows reasoning about the committed index. type ProgressTracker struct { - Voters quorum.JointConfig - Learners map[uint64]struct{} + Config Progress map[uint64]*Progress @@ -39,11 +99,15 @@ type ProgressTracker struct { func MakeProgressTracker(maxInflight int) ProgressTracker { p := ProgressTracker{ MaxInflight: maxInflight, - Voters: quorum.JointConfig{ - quorum.MajorityConfig{}, - quorum.MajorityConfig{}, + Config: Config{ + Voters: quorum.JointConfig{ + quorum.MajorityConfig{}, + // TODO(tbg): this will be mostly empty, so make it a nil pointer + // in the common case. + quorum.MajorityConfig{}, + }, + Learners: map[uint64]struct{}{}, }, - Learners: map[uint64]struct{}{}, Votes: map[uint64]bool{}, Progress: map[uint64]*Progress{}, }