mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00

It has often been tedious to test the interactions between multi-member Raft groups, especially when many steps were required to reach a certain scenario. Often, this boilerplate was as boring as it is hard to write and hard to maintain, making it attractive to resort to shortcuts whenever possible, which in turn tended to undercut how meaningful and maintainable the tests ended up being - that is, if the tests were even written, which sometimes they weren't. This change introduces a datadriven framework specifically for testing deterministically the interaction between multiple members of a raft group with the goal of reducing the friction for writing these tests to near zero. In the near term, this will be used to add thorough testing for joint consensus (which is already available today, but wildly undertested), but just converting an existing test into this framework has shown that the concise representation and built-in inspection of log messages highlights unexpected behavior much more readily than the previous unit tests did (the test in question is `snapshot_succeed_via_app_resp`; the reader is invited to compare the old and new version of it). The main building block is `InteractionEnv`, which holds on to the state of the whole system and exposes various relevant methods for manipulating it, including but not limited to adding nodes, delivering and dropping messages, and proposing configuration changes. All of this is extensible so that in the future I hope to use it to explore the phenomena discussed in https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263 which requires injecting appropriate "crash points" in the Ready handling loop. Discussions of the "what if X happened in state Y" can quickly be made concrete by "scripting up an interaction test". Additionally, this framework is intentionally not kept internal to the raft package.. Though this is in its infancy, a goal is that it should be possible for a suite of interaction tests to allow applications to validate that their Storage implementation behaves accordingly, simply by running a raft-provided interaction suite against their Storage.
171 lines
4.9 KiB
Go
171 lines
4.9 KiB
Go
// Copyright 2019 The etcd Authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package raftpb
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/gogo/protobuf/proto"
|
|
)
|
|
|
|
// ConfChangeI abstracts over ConfChangeV2 and (legacy) ConfChange to allow
|
|
// treating them in a unified manner.
|
|
type ConfChangeI interface {
|
|
AsV2() ConfChangeV2
|
|
AsV1() (ConfChange, bool)
|
|
}
|
|
|
|
// MarshalConfChange calls Marshal on the underlying ConfChange or ConfChangeV2
|
|
// and returns the result along with the corresponding EntryType.
|
|
func MarshalConfChange(c ConfChangeI) (EntryType, []byte, error) {
|
|
var typ EntryType
|
|
var ccdata []byte
|
|
var err error
|
|
if ccv1, ok := c.AsV1(); ok {
|
|
typ = EntryConfChange
|
|
ccdata, err = ccv1.Marshal()
|
|
} else {
|
|
ccv2 := c.AsV2()
|
|
typ = EntryConfChangeV2
|
|
ccdata, err = ccv2.Marshal()
|
|
}
|
|
return typ, ccdata, err
|
|
}
|
|
|
|
// AsV2 returns a V2 configuration change carrying out the same operation.
|
|
func (c ConfChange) AsV2() ConfChangeV2 {
|
|
return ConfChangeV2{
|
|
Changes: []ConfChangeSingle{{
|
|
Type: c.Type,
|
|
NodeID: c.NodeID,
|
|
}},
|
|
Context: c.Context,
|
|
}
|
|
}
|
|
|
|
// AsV1 returns the ConfChange and true.
|
|
func (c ConfChange) AsV1() (ConfChange, bool) {
|
|
return c, true
|
|
}
|
|
|
|
// AsV2 is the identity.
|
|
func (c ConfChangeV2) AsV2() ConfChangeV2 { return c }
|
|
|
|
// AsV1 returns ConfChange{} and false.
|
|
func (c ConfChangeV2) AsV1() (ConfChange, bool) { return ConfChange{}, false }
|
|
|
|
// EnterJoint returns two bools. The second bool is true if and only if this
|
|
// config change will use Joint Consensus, which is the case if it contains more
|
|
// than one change or if the use of Joint Consensus was requested explicitly.
|
|
// The first bool can only be true if second one is, and indicates whether the
|
|
// Joint State will be left automatically.
|
|
func (c *ConfChangeV2) EnterJoint() (autoLeave bool, ok bool) {
|
|
// NB: in theory, more config changes could qualify for the "simple"
|
|
// protocol but it depends on the config on top of which the changes apply.
|
|
// For example, adding two learners is not OK if both nodes are part of the
|
|
// base config (i.e. two voters are turned into learners in the process of
|
|
// applying the conf change). In practice, these distinctions should not
|
|
// matter, so we keep it simple and use Joint Consensus liberally.
|
|
if c.Transition != ConfChangeTransitionAuto || len(c.Changes) > 1 {
|
|
// Use Joint Consensus.
|
|
var autoLeave bool
|
|
switch c.Transition {
|
|
case ConfChangeTransitionAuto:
|
|
autoLeave = true
|
|
case ConfChangeTransitionJointImplicit:
|
|
autoLeave = true
|
|
case ConfChangeTransitionJointExplicit:
|
|
default:
|
|
panic(fmt.Sprintf("unknown transition: %+v", c))
|
|
}
|
|
return autoLeave, true
|
|
}
|
|
return false, false
|
|
}
|
|
|
|
// LeaveJoint is true if the configuration change leaves a joint configuration.
|
|
// This is the case if the ConfChangeV2 is zero, with the possible exception of
|
|
// the Context field.
|
|
func (c *ConfChangeV2) LeaveJoint() bool {
|
|
cpy := *c
|
|
cpy.Context = nil
|
|
return proto.Equal(&cpy, &ConfChangeV2{})
|
|
}
|
|
|
|
// ConfChangesFromString parses a Space-delimited sequence of operations into a
|
|
// slice of ConfChangeSingle. The supported operations are:
|
|
// - vn: make n a voter,
|
|
// - ln: make n a learner,
|
|
// - rn: remove n, and
|
|
// - un: update n.
|
|
func ConfChangesFromString(s string) ([]ConfChangeSingle, error) {
|
|
var ccs []ConfChangeSingle
|
|
toks := strings.Split(strings.TrimSpace(s), " ")
|
|
if toks[0] == "" {
|
|
toks = nil
|
|
}
|
|
for _, tok := range toks {
|
|
if len(tok) < 2 {
|
|
return nil, fmt.Errorf("unknown token %s", tok)
|
|
}
|
|
var cc ConfChangeSingle
|
|
switch tok[0] {
|
|
case 'v':
|
|
cc.Type = ConfChangeAddNode
|
|
case 'l':
|
|
cc.Type = ConfChangeAddLearnerNode
|
|
case 'r':
|
|
cc.Type = ConfChangeRemoveNode
|
|
case 'u':
|
|
cc.Type = ConfChangeUpdateNode
|
|
default:
|
|
return nil, fmt.Errorf("unknown input: %s", tok)
|
|
}
|
|
id, err := strconv.ParseUint(tok[1:], 10, 64)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cc.NodeID = id
|
|
ccs = append(ccs, cc)
|
|
}
|
|
return ccs, nil
|
|
}
|
|
|
|
// ConfChangesToString is the inverse to ConfChangesFromString.
|
|
func ConfChangesToString(ccs []ConfChangeSingle) string {
|
|
var buf strings.Builder
|
|
for i, cc := range ccs {
|
|
if i > 0 {
|
|
buf.WriteByte(' ')
|
|
}
|
|
switch cc.Type {
|
|
case ConfChangeAddNode:
|
|
buf.WriteByte('v')
|
|
case ConfChangeAddLearnerNode:
|
|
buf.WriteByte('l')
|
|
case ConfChangeRemoveNode:
|
|
buf.WriteByte('r')
|
|
case ConfChangeUpdateNode:
|
|
buf.WriteByte('u')
|
|
default:
|
|
buf.WriteString("unknown")
|
|
}
|
|
fmt.Fprintf(&buf, "%d", cc.NodeID)
|
|
}
|
|
return buf.String()
|
|
}
|