From af4d4ab16a09a2113c4fbf85eecf86c16e77e83a Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 18 Dec 2017 14:13:14 -0800 Subject: [PATCH 1/2] tools/etcd-dump-db: add "iterate-bucket --decode" flag Signed-off-by: Gyuho Lee --- tools/etcd-dump-db/backend.go | 59 +++++++++++++++++++++++++++++++++-- tools/etcd-dump-db/main.go | 22 +++++-------- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/tools/etcd-dump-db/backend.go b/tools/etcd-dump-db/backend.go index aff90fc49..f610b4cd9 100644 --- a/tools/etcd-dump-db/backend.go +++ b/tools/etcd-dump-db/backend.go @@ -15,12 +15,15 @@ package main import ( + "encoding/binary" "fmt" "path/filepath" bolt "github.com/coreos/bbolt" + "github.com/coreos/etcd/lease/leasepb" "github.com/coreos/etcd/mvcc" "github.com/coreos/etcd/mvcc/backend" + "github.com/coreos/etcd/mvcc/mvccpb" ) func snapDir(dataDir string) string { @@ -43,7 +46,53 @@ func getBuckets(dbPath string) (buckets []string, err error) { return buckets, err } -func iterateBucket(dbPath, bucket string, limit uint64) (err error) { +// TODO: import directly from packages, rather than copy&paste + +type decoder func(k, v []byte) + +var decoders = map[string]decoder{ + "key": keyDecoder, + "lease": leaseDecoder, +} + +type revision struct { + main int64 + sub int64 +} + +func bytesToRev(bytes []byte) revision { + return revision{ + main: int64(binary.BigEndian.Uint64(bytes[0:8])), + sub: int64(binary.BigEndian.Uint64(bytes[9:])), + } +} + +func keyDecoder(k, v []byte) { + rev := bytesToRev(k) + var kv mvccpb.KeyValue + if err := kv.Unmarshal(v); err != nil { + panic(err) + } + fmt.Printf("rev=%+v, value=[key %q | val %q | created %d | mod %d | ver %d]\n", rev, string(kv.Key), string(kv.Value), kv.CreateRevision, kv.ModRevision, kv.Version) +} + +func bytesToLeaseID(bytes []byte) int64 { + if len(bytes) != 8 { + panic(fmt.Errorf("lease ID must be 8-byte")) + } + return int64(binary.BigEndian.Uint64(bytes)) +} + +func leaseDecoder(k, v []byte) { + leaseID := bytesToLeaseID(k) + var lpb leasepb.Lease + if err := lpb.Unmarshal(v); err != nil { + panic(err) + } + fmt.Printf("lease ID=%016x, TTL=%ds\n", leaseID, lpb.TTL) +} + +func iterateBucket(dbPath, bucket string, limit uint64, decode bool) (err error) { db, derr := bolt.Open(dbPath, 0600, &bolt.Options{}) if derr != nil { return derr @@ -60,7 +109,13 @@ func iterateBucket(dbPath, bucket string, limit uint64) (err error) { // iterate in reverse order (use First() and Next() for ascending order) for k, v := c.Last(); k != nil; k, v = c.Prev() { - fmt.Printf("key=%q, value=%q\n", k, v) + // TODO: remove sensitive information + // (https://github.com/coreos/etcd/issues/7620) + if dec, ok := decoders[bucket]; decode && ok { + dec(k, v) + } else { + fmt.Printf("key=%q, value=%q\n", k, v) + } limit-- if limit == 0 { diff --git a/tools/etcd-dump-db/main.go b/tools/etcd-dump-db/main.go index b5262c90b..dd7b46ee3 100644 --- a/tools/etcd-dump-db/main.go +++ b/tools/etcd-dump-db/main.go @@ -35,7 +35,7 @@ var ( Run: listBucketCommandFunc, } iterateBucketCommand = &cobra.Command{ - Use: "iterate-bucket [data dir or db file path]", + Use: "iterate-bucket [data dir or db file path] [bucket name]", Short: "iterate-bucket lists key-value pairs in reverse order.", Run: iterateBucketCommandFunc, } @@ -46,14 +46,12 @@ var ( } ) -var ( - iterateBucketName string - iterateBucketLimit uint64 -) +var iterateBucketLimit uint64 +var iterateBucketDecode bool func init() { - iterateBucketCommand.PersistentFlags().StringVar(&iterateBucketName, "bucket", "", "bucket name to iterate") iterateBucketCommand.PersistentFlags().Uint64Var(&iterateBucketLimit, "limit", 0, "max number of key-value pairs to iterate (0< to iterate all)") + iterateBucketCommand.PersistentFlags().BoolVar(&iterateBucketDecode, "decode", false, "true to decode Protocol Buffer encoded data") rootCommand.AddCommand(listBucketCommand) rootCommand.AddCommand(iterateBucketCommand) @@ -89,8 +87,8 @@ func listBucketCommandFunc(cmd *cobra.Command, args []string) { } func iterateBucketCommandFunc(cmd *cobra.Command, args []string) { - if len(args) < 1 { - log.Fatalf("Must provide at least 1 argument (got %v)", args) + if len(args) != 2 { + log.Fatalf("Must provide 2 arguments (got %v)", args) } dp := args[0] if !strings.HasSuffix(dp, "db") { @@ -99,12 +97,8 @@ func iterateBucketCommandFunc(cmd *cobra.Command, args []string) { if !existFileOrDir(dp) { log.Fatalf("%q does not exist", dp) } - - if iterateBucketName == "" { - log.Fatal("got empty bucket name") - } - - err := iterateBucket(dp, iterateBucketName, iterateBucketLimit) + bucket := args[1] + err := iterateBucket(dp, bucket, iterateBucketLimit, iterateBucketDecode) if err != nil { log.Fatal(err) } From 931dfd7da88b8f7fb56bca63e694d9f0e60f5af0 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Mon, 18 Dec 2017 14:19:45 -0800 Subject: [PATCH 2/2] etcd-dump-logs: take data-dir as argument, add TODO Signed-off-by: Gyuho Lee --- tools/etcd-dump-logs/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/etcd-dump-logs/main.go b/tools/etcd-dump-logs/main.go index 17343180a..fb72be100 100644 --- a/tools/etcd-dump-logs/main.go +++ b/tools/etcd-dump-logs/main.go @@ -31,13 +31,15 @@ import ( ) func main() { - from := flag.String("data-dir", "", "") snapfile := flag.String("start-snap", "", "The base name of snapshot file to start dumping") index := flag.Uint64("start-index", 0, "The index to start dumping") flag.Parse() - if *from == "" { - log.Fatal("Must provide -data-dir flag.") + + if len(flag.Args()) != 1 { + log.Fatalf("Must provide data-dir argument (got %+v)", flag.Args()) } + dataDir := flag.Args()[0] + if *snapfile != "" && *index != 0 { log.Fatal("start-snap and start-index flags cannot be used together.") } @@ -55,10 +57,10 @@ func main() { walsnap.Index = *index } else { if *snapfile == "" { - ss := snap.New(snapDir(*from)) + ss := snap.New(snapDir(dataDir)) snapshot, err = ss.Load() } else { - snapshot, err = snap.Read(filepath.Join(snapDir(*from), *snapfile)) + snapshot, err = snap.Read(filepath.Join(snapDir(dataDir), *snapfile)) } switch err { @@ -75,7 +77,7 @@ func main() { fmt.Println("Start dupmping log entries from snapshot.") } - w, err := wal.OpenForRead(walDir(*from), walsnap) + w, err := wal.OpenForRead(walDir(dataDir), walsnap) if err != nil { log.Fatalf("Failed opening WAL: %v", err) } @@ -104,6 +106,8 @@ func main() { break } + // TODO: remove sensitive information + // (https://github.com/coreos/etcd/issues/7620) var r etcdserverpb.Request if err := r.Unmarshal(e.Data); err == nil { switch r.Method {