From 0d541e633872e78f8d007b096edc16d59a9d30a8 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Fri, 7 Nov 2014 12:53:32 -0800 Subject: [PATCH] *: add ctl backup support --- etcdctl/command/backup_command.go | 87 +++++++++++++++++++++++++++++++ etcdctl/main.go | 1 + etcdserver/force_cluster.go | 27 ++++++++++ etcdserver/force_cluster_test.go | 22 ++++++++ 4 files changed, 137 insertions(+) create mode 100644 etcdctl/command/backup_command.go diff --git a/etcdctl/command/backup_command.go b/etcdctl/command/backup_command.go new file mode 100644 index 000000000..4245d957c --- /dev/null +++ b/etcdctl/command/backup_command.go @@ -0,0 +1,87 @@ +/* + Copyright 2014 CoreOS, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package command + +import ( + "log" + "math/rand" + "path" + "time" + + "github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli" + "github.com/coreos/etcd/etcdserver" + "github.com/coreos/etcd/etcdserver/etcdserverpb" + "github.com/coreos/etcd/pkg/pbutil" + "github.com/coreos/etcd/snap" + "github.com/coreos/etcd/wal" +) + +func NewBackupCommand() cli.Command { + return cli.Command{ + Name: "backup", + Usage: "backup an etcd directory", + Flags: []cli.Flag{ + cli.StringFlag{Name: "data-dir", Value: "", Usage: "Path to the etcd data dir"}, + cli.StringFlag{Name: "backup-dir", Value: "", Usage: "Path to the backup dir"}, + }, + Action: handleBackup, + } +} + +// handleBackup handles a request that intends to do a backup. +func handleBackup(c *cli.Context) { + srcSnap := path.Join(c.String("data-dir"), "snap") + destSnap := path.Join(c.String("backup-dir"), "snap") + srcWAL := path.Join(c.String("data-dir"), "wal") + destWAL := path.Join(c.String("backup-dir"), "wal") + + ss := snap.New(srcSnap) + snapshot, err := ss.Load() + if err != nil && err != snap.ErrNoSnapshot { + log.Fatal(err) + } + var index uint64 + if snapshot != nil { + index = snapshot.Index + newss := snap.New(destSnap) + newss.SaveSnap(*snapshot) + } + + w, err := wal.OpenAtIndex(srcWAL, index) + if err != nil { + log.Fatal(err) + } + defer w.Close() + wmetadata, state, ents, err := w.ReadAll() + if err != nil { + log.Fatal(err) + } + var metadata etcdserverpb.Metadata + pbutil.MustUnmarshal(&metadata, wmetadata) + rand.Seed(time.Now().UnixNano()) + metadata.NodeID = etcdserver.GenID() + metadata.ClusterID = etcdserver.GenID() + + neww, err := wal.Create(destWAL, pbutil.MustMarshal(&metadata)) + if err != nil { + log.Fatal(err) + } + defer neww.Close() + if err := neww.Save(state, ents); err != nil { + log.Fatal(err) + } +} diff --git a/etcdctl/main.go b/etcdctl/main.go index e2b67625b..21abd5fdd 100644 --- a/etcdctl/main.go +++ b/etcdctl/main.go @@ -37,6 +37,7 @@ func main() { cli.StringFlag{Name: "peers, C", Value: "", Usage: "a comma-delimited list of machine addresses in the cluster (default: \"127.0.0.1:4001\")"}, } app.Commands = []cli.Command{ + command.NewBackupCommand(), command.NewMakeCommand(), command.NewMakeDirCommand(), command.NewRemoveCommand(), diff --git a/etcdserver/force_cluster.go b/etcdserver/force_cluster.go index d5f4064e0..d6c40a487 100644 --- a/etcdserver/force_cluster.go +++ b/etcdserver/force_cluster.go @@ -17,6 +17,7 @@ package etcdserver import ( + "encoding/json" "log" "sort" @@ -101,11 +102,15 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 { // createConfigChangeEnts creates a series of Raft entries (i.e. // EntryConfChange) to remove the set of given IDs from the cluster. The ID // `self` is _not_ removed, even if present in the set. +// If `self` is not inside the given ids, it creates a Raft entry to add a +// default member with the given `self`. func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raftpb.Entry { ents := make([]raftpb.Entry, 0) next := index + 1 + found := false for _, id := range ids { if id == self { + found = true continue } cc := &raftpb.ConfChange{ @@ -121,5 +126,27 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf ents = append(ents, e) next++ } + if !found { + m := Member{ + ID: types.ID(self), + RaftAttributes: RaftAttributes{PeerURLs: []string{"http://localhost:7001", "http://localhost:2380"}}, + } + ctx, err := json.Marshal(m) + if err != nil { + log.Panicf("marshal member should never fail: %v", err) + } + cc := &raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: self, + Context: ctx, + } + e := raftpb.Entry{ + Type: raftpb.EntryConfChange, + Data: pbutil.MustMarshal(cc), + Term: term, + Index: next, + } + ents = append(ents, e) + } return ents } diff --git a/etcdserver/force_cluster_test.go b/etcdserver/force_cluster_test.go index f4673f857..368ef889b 100644 --- a/etcdserver/force_cluster_test.go +++ b/etcdserver/force_cluster_test.go @@ -17,10 +17,12 @@ package etcdserver import ( + "encoding/json" "reflect" "testing" "github.com/coreos/etcd/pkg/pbutil" + "github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/raft/raftpb" ) @@ -54,6 +56,15 @@ func TestGetIDs(t *testing.T) { } func TestCreateConfigChangeEnts(t *testing.T) { + m := Member{ + ID: types.ID(1), + RaftAttributes: RaftAttributes{PeerURLs: []string{"http://localhost:7001", "http://localhost:2380"}}, + } + ctx, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + addcc1 := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx} removecc2 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2} removecc3 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 3} tests := []struct { @@ -103,6 +114,17 @@ func TestCreateConfigChangeEnts(t *testing.T) { {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)}, }, }, + { + []uint64{2, 3}, + 1, + 2, 2, + + []raftpb.Entry{ + {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}, + {Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)}, + {Term: 2, Index: 5, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(addcc1)}, + }, + }, } for i, tt := range tests {