diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go index 9d905e61f..9bb3e8155 100644 --- a/server/etcdserver/bootstrap.go +++ b/server/etcdserver/bootstrap.go @@ -72,7 +72,6 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) { if terr := fileutil.TouchDirAll(cfg.MemberDir()); terr != nil { return nil, fmt.Errorf("cannot access member directory: %v", terr) } - haveWAL := wal.Exist(cfg.WALDir()) s, err := bootstrapStorage(cfg, haveWAL, ss, prt) if err != nil { @@ -84,17 +83,23 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) { s.backend.be.Close() return nil, err } + raft := bootstrapRaft(cfg, haveWAL, cluster.cl, cluster.wal) + if !haveWAL { + cluster.cl.SetID(cluster.nodeID, cluster.cl.ID()) + } return &bootstrappedServer{ prt: prt, ss: ss, storage: s, cluster: cluster, + raft: raft, }, nil } type bootstrappedServer struct { storage *bootstrappedStorage cluster *bootstrapedCluster + raft *bootstrappedRaft prt http.RoundTripper ss *snap.Snapshotter } @@ -113,7 +118,6 @@ type bootstrappedBackend struct { } type bootstrapedCluster struct { - raft *bootstrappedRaft remotes []*membership.Member wal *bootstrappedWAL cl *membership.RaftCluster @@ -232,14 +236,13 @@ func maybeDefragBackend(cfg config.ServerConfig, be backend.Backend) error { func bootstrapCluster(cfg config.ServerConfig, haveWAL bool, storage *bootstrappedStorage, prt http.RoundTripper, ss *snap.Snapshotter) (c *bootstrapedCluster, err error) { c = &bootstrapedCluster{} var ( - meta *snapshotMetadata bwal *bootstrappedWAL ) if haveWAL { if err = fileutil.IsDirWriteable(cfg.WALDir()); err != nil { return nil, fmt.Errorf("cannot write to WAL directory: %v", err) } - bwal, meta = bootstrapWALFromSnapshot(cfg, storage.backend.snapshot) + bwal = bootstrapWALFromSnapshot(cfg, storage.backend.snapshot) } switch { @@ -256,7 +259,7 @@ func bootstrapCluster(cfg config.ServerConfig, haveWAL bool, storage *bootstrapp } c.wal = bootstrapNewWAL(cfg, c.cl, c.nodeID) case haveWAL: - c, err = bootstrapClusterWithWAL(cfg, storage, meta) + c, err = bootstrapClusterWithWAL(cfg, storage, bwal.meta) if err != nil { return nil, err } @@ -264,18 +267,6 @@ func bootstrapCluster(cfg config.ServerConfig, haveWAL bool, storage *bootstrapp default: return nil, fmt.Errorf("unsupported bootstrap config") } - switch { - case !haveWAL && !cfg.NewCluster: - c.raft = bootstrapRaftFromCluster(cfg, c.cl, nil, c.wal) - c.cl.SetID(c.nodeID, c.cl.ID()) - case !haveWAL && cfg.NewCluster: - c.raft = bootstrapRaftFromCluster(cfg, c.cl, c.cl.MemberIDs(), c.wal) - c.cl.SetID(c.nodeID, c.cl.ID()) - case haveWAL: - c.raft = bootstrapRaftFromSnapshot(cfg, c.wal, meta) - default: - return nil, fmt.Errorf("unsupported bootstrap config") - } return c, nil } @@ -437,6 +428,20 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe return snapshot, be, nil } +func bootstrapRaft(cfg config.ServerConfig, haveWAL bool, cl *membership.RaftCluster, bwal *bootstrappedWAL) *bootstrappedRaft { + switch { + case !haveWAL && !cfg.NewCluster: + return bootstrapRaftFromCluster(cfg, cl, nil, bwal) + case !haveWAL && cfg.NewCluster: + return bootstrapRaftFromCluster(cfg, cl, cl.MemberIDs(), bwal) + case haveWAL: + return bootstrapRaftFromWAL(cfg, bwal) + default: + cfg.Logger.Panic("unsupported bootstrap config") + return nil + } +} + func bootstrapRaftFromCluster(cfg config.ServerConfig, cl *membership.RaftCluster, ids []types.ID, bwal *bootstrappedWAL) *bootstrappedRaft { member := cl.MemberByName(cfg.Name) peers := make([]raft.Peer, len(ids)) @@ -463,12 +468,12 @@ func bootstrapRaftFromCluster(cfg config.ServerConfig, cl *membership.RaftCluste } } -func bootstrapRaftFromSnapshot(cfg config.ServerConfig, bwal *bootstrappedWAL, meta *snapshotMetadata) *bootstrappedRaft { +func bootstrapRaftFromWAL(cfg config.ServerConfig, bwal *bootstrappedWAL) *bootstrappedRaft { s := bwal.MemoryStorage() return &bootstrappedRaft{ lg: cfg.Logger, heartbeat: time.Duration(cfg.TickMs) * time.Millisecond, - config: raftConfig(cfg, uint64(meta.nodeID), s), + config: raftConfig(cfg, uint64(bwal.meta.nodeID), s), storage: s, } } @@ -509,7 +514,7 @@ func (b *bootstrappedRaft) newRaftNode(ss *snap.Snapshotter, wal *wal.WAL, cl *m ) } -func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot) (*bootstrappedWAL, *snapshotMetadata) { +func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot) *bootstrappedWAL { wal, st, ents, snap, meta := openWALFromSnapshot(cfg, snapshot) bwal := &bootstrappedWAL{ lg: cfg.Logger, @@ -517,12 +522,13 @@ func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot st: st, ents: ents, snapshot: snap, + meta: meta, } if cfg.ForceNewCluster { // discard the previously uncommitted entries bwal.ents = bwal.CommitedEntries() - entries := bwal.ConfigChangeEntries(meta) + entries := bwal.ConfigChangeEntries() // force commit config change entries bwal.AppendAndCommitEntries(entries) cfg.Logger.Info( @@ -539,7 +545,7 @@ func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot zap.Uint64("commit-index", bwal.st.Commit), ) } - return bwal, meta + return bwal } // openWALFromSnapshot reads the WAL at the given snap and returns the wal, its latest HardState and cluster ID, and all entries that appear @@ -614,6 +620,7 @@ type bootstrappedWAL struct { st *raftpb.HardState ents []raftpb.Entry snapshot *raftpb.Snapshot + meta *snapshotMetadata } func (wal *bootstrappedWAL) MemoryStorage() *raft.MemoryStorage { @@ -645,11 +652,11 @@ func (wal *bootstrappedWAL) CommitedEntries() []raftpb.Entry { return wal.ents } -func (wal *bootstrappedWAL) ConfigChangeEntries(meta *snapshotMetadata) []raftpb.Entry { +func (wal *bootstrappedWAL) ConfigChangeEntries() []raftpb.Entry { return serverstorage.CreateConfigChangeEnts( wal.lg, serverstorage.GetIDs(wal.lg, wal.snapshot, wal.ents), - uint64(meta.nodeID), + uint64(wal.meta.nodeID), wal.st.Term, wal.st.Commit, ) diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 421f7d059..8d55295e1 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -320,7 +320,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { errorc: make(chan error, 1), v2store: b.storage.st, snapshotter: b.ss, - r: *b.cluster.raft.newRaftNode(b.ss, b.cluster.wal.w, b.cluster.cl), + r: *b.raft.newRaftNode(b.ss, b.cluster.wal.w, b.cluster.cl), id: b.cluster.nodeID, attributes: membership.Attributes{Name: cfg.Name, ClientURLs: cfg.ClientURLs.StringSlice()}, cluster: b.cluster.cl,