Merge pull request #9041 from gyuho/etcd-tooling

tools/etcd-dump-*: add TODOs, --decode flag

tools/etcd-dump-db/backend.go

@@ -15,12 +15,15 @@
 package main
 
 import (
+	"encoding/binary"
 	"fmt"
 	"path/filepath"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/coreos/etcd/lease/leasepb"
 	"github.com/coreos/etcd/mvcc"
 	"github.com/coreos/etcd/mvcc/backend"
+	"github.com/coreos/etcd/mvcc/mvccpb"
 )
 
 func snapDir(dataDir string) string {
@@ -43,7 +46,53 @@ func getBuckets(dbPath string) (buckets []string, err error) {
 	return buckets, err
 }
 
-func iterateBucket(dbPath, bucket string, limit uint64) (err error) {
+// TODO: import directly from packages, rather than copy&paste
+
+type decoder func(k, v []byte)
+
+var decoders = map[string]decoder{
+	"key":   keyDecoder,
+	"lease": leaseDecoder,
+}
+
+type revision struct {
+	main int64
+	sub  int64
+}
+
+func bytesToRev(bytes []byte) revision {
+	return revision{
+		main: int64(binary.BigEndian.Uint64(bytes[0:8])),
+		sub:  int64(binary.BigEndian.Uint64(bytes[9:])),
+	}
+}
+
+func keyDecoder(k, v []byte) {
+	rev := bytesToRev(k)
+	var kv mvccpb.KeyValue
+	if err := kv.Unmarshal(v); err != nil {
+		panic(err)
+	}
+	fmt.Printf("rev=%+v, value=[key %q | val %q | created %d | mod %d | ver %d]\n", rev, string(kv.Key), string(kv.Value), kv.CreateRevision, kv.ModRevision, kv.Version)
+}
+
+func bytesToLeaseID(bytes []byte) int64 {
+	if len(bytes) != 8 {
+		panic(fmt.Errorf("lease ID must be 8-byte"))
+	}
+	return int64(binary.BigEndian.Uint64(bytes))
+}
+
+func leaseDecoder(k, v []byte) {
+	leaseID := bytesToLeaseID(k)
+	var lpb leasepb.Lease
+	if err := lpb.Unmarshal(v); err != nil {
+		panic(err)
+	}
+	fmt.Printf("lease ID=%016x, TTL=%ds\n", leaseID, lpb.TTL)
+}
+
+func iterateBucket(dbPath, bucket string, limit uint64, decode bool) (err error) {
 	db, derr := bolt.Open(dbPath, 0600, &bolt.Options{})
 	if derr != nil {
 		return derr
@@ -60,7 +109,13 @@ func iterateBucket(dbPath, bucket string, limit uint64) (err error) {
 
 		// iterate in reverse order (use First() and Next() for ascending order)
 		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			// TODO: remove sensitive information
+			// (https://github.com/coreos/etcd/issues/7620)
+			if dec, ok := decoders[bucket]; decode && ok {
+				dec(k, v)
+			} else {
 				fmt.Printf("key=%q, value=%q\n", k, v)
+			}
 
 			limit--
 			if limit == 0 {

tools/etcd-dump-db/main.go

@@ -35,7 +35,7 @@ var (
 		Run:   listBucketCommandFunc,
 	}
 	iterateBucketCommand = &cobra.Command{
-		Use:   "iterate-bucket [data dir or db file path]",
+		Use:   "iterate-bucket [data dir or db file path] [bucket name]",
 		Short: "iterate-bucket lists key-value pairs in reverse order.",
 		Run:   iterateBucketCommandFunc,
 	}
@@ -46,14 +46,12 @@ var (
 	}
 )
 
-var (
+var iterateBucketLimit uint64
-	iterateBucketName  string
+var iterateBucketDecode bool
-	iterateBucketLimit uint64
-)
 
 func init() {
-	iterateBucketCommand.PersistentFlags().StringVar(&iterateBucketName, "bucket", "", "bucket name to iterate")
 	iterateBucketCommand.PersistentFlags().Uint64Var(&iterateBucketLimit, "limit", 0, "max number of key-value pairs to iterate (0< to iterate all)")
+	iterateBucketCommand.PersistentFlags().BoolVar(&iterateBucketDecode, "decode", false, "true to decode Protocol Buffer encoded data")
 
 	rootCommand.AddCommand(listBucketCommand)
 	rootCommand.AddCommand(iterateBucketCommand)
@@ -89,8 +87,8 @@ func listBucketCommandFunc(cmd *cobra.Command, args []string) {
 }
 
 func iterateBucketCommandFunc(cmd *cobra.Command, args []string) {
-	if len(args) < 1 {
+	if len(args) != 2 {
-		log.Fatalf("Must provide at least 1 argument (got %v)", args)
+		log.Fatalf("Must provide 2 arguments (got %v)", args)
 	}
 	dp := args[0]
 	if !strings.HasSuffix(dp, "db") {
@@ -99,12 +97,8 @@ func iterateBucketCommandFunc(cmd *cobra.Command, args []string) {
 	if !existFileOrDir(dp) {
 		log.Fatalf("%q does not exist", dp)
 	}
-
+	bucket := args[1]
-	if iterateBucketName == "" {
+	err := iterateBucket(dp, bucket, iterateBucketLimit, iterateBucketDecode)
-		log.Fatal("got empty bucket name")
-	}
-
-	err := iterateBucket(dp, iterateBucketName, iterateBucketLimit)
 	if err != nil {
 		log.Fatal(err)
 	}

tools/etcd-dump-logs/main.go

@@ -31,13 +31,15 @@ import (
 )
 
 func main() {
-	from := flag.String("data-dir", "", "")
 	snapfile := flag.String("start-snap", "", "The base name of snapshot file to start dumping")
 	index := flag.Uint64("start-index", 0, "The index to start dumping")
 	flag.Parse()
-	if *from == "" {
+
-		log.Fatal("Must provide -data-dir flag.")
+	if len(flag.Args()) != 1 {
+		log.Fatalf("Must provide data-dir argument (got %+v)", flag.Args())
 	}
+	dataDir := flag.Args()[0]
+
 	if *snapfile != "" && *index != 0 {
 		log.Fatal("start-snap and start-index flags cannot be used together.")
 	}
@@ -55,10 +57,10 @@ func main() {
 		walsnap.Index = *index
 	} else {
 		if *snapfile == "" {
-			ss := snap.New(snapDir(*from))
+			ss := snap.New(snapDir(dataDir))
 			snapshot, err = ss.Load()
 		} else {
-			snapshot, err = snap.Read(filepath.Join(snapDir(*from), *snapfile))
+			snapshot, err = snap.Read(filepath.Join(snapDir(dataDir), *snapfile))
 		}
 
 		switch err {
@@ -75,7 +77,7 @@ func main() {
 		fmt.Println("Start dupmping log entries from snapshot.")
 	}
 
-	w, err := wal.OpenForRead(walDir(*from), walsnap)
+	w, err := wal.OpenForRead(walDir(dataDir), walsnap)
 	if err != nil {
 		log.Fatalf("Failed opening WAL: %v", err)
 	}
@@ -104,6 +106,8 @@ func main() {
 				break
 			}
 
+			// TODO: remove sensitive information
+			// (https://github.com/coreos/etcd/issues/7620)
 			var r etcdserverpb.Request
 			if err := r.Unmarshal(e.Data); err == nil {
 				switch r.Method {