mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
etcd-dump-logs: add decoder support
This commit is contained in:
Wenjia Zhang
Gyuho Lee
parent
6d773ffe96
commit
29121a8cee
129
tools/etcd-dump-logs/README.md
Normal file
129
tools/etcd-dump-logs/README.md
Normal file
@ -0,0 +1,129 @@
|
||||
### etcd-dump-logs
|
||||
|
||||
etcd-dump-logs inspects etcd db files.
|
||||
|
||||
```
|
||||
Usage:
|
||||
|
||||
etcd-dump-logs [data dir]
|
||||
* Data dir is where the snapshots and WAL logs are located. The structure of the data dir should look like this:
|
||||
- data_dir/member
|
||||
- data_dir/member/snap
|
||||
- data_dir/member/wal
|
||||
- data_dir/member/wal/0000000000000000-0000000000000000.wal
|
||||
|
||||
Flags:
|
||||
|
||||
-entry-type string
|
||||
If set, filters output by entry type. Must be one or more than one of:
|
||||
ConfigChange, Normal, Request, InternalRaftRequest,
|
||||
IRRRange, IRRPut, IRRDeleteRange, IRRTxn,
|
||||
IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke
|
||||
-start-index uint
|
||||
The index to start dumping
|
||||
-start-snap string
|
||||
The base name of snapshot file to start dumping
|
||||
-stream-decoder string
|
||||
The name and arguments of an executable decoding tool, the executable
|
||||
must process hex encoded lines of binary input (from etcd-dump-logs)
|
||||
and output a hex encoded line of binary for each input line
|
||||
```
|
||||
#### etcd-dump-logs -entry-type <ENTRY_TYPE_NAME(S)> [data dir]
|
||||
|
||||
Lists all the interested entries from WAL log.
|
||||
|
||||
```
|
||||
$ etcd-dump-logs -entry-type IRRTxn /tmp/datadir
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data
|
||||
7 13 norm ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"k8s\000\n\025\n\002v1\022\017RangeAllocation\022#\n\022\n\000\022\000\032\000\"\000*\0002\0008\000B\000z\000\022\01310.0.0.0/16\032\000\032\000\"\000" > > failure:<request_delete_range:<key:"a" range_end:"k8s\000\n\025\n\002v1\022\017RangeAllocation\022#\n\022\n\000\022\000\032\000\"\000*\0002\0008\000B\000z\000\022\01310.0.0.0/16\032\000\032\000\"\000" > > >
|
||||
|
||||
Entry types (IRRTxn) count is : 1
|
||||
|
||||
$ etcd-dump-logs -entry-type ConfigChange,IRRCompaction /tmp/datadir
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data
|
||||
1 1 conf method=ConfChangeAddNode id=2
|
||||
2 2 conf method=ConfChangeRemoveNode id=2
|
||||
2 3 conf method=ConfChangeUpdateNode id=2
|
||||
2 4 conf method=ConfChangeAddLearnerNode id=3
|
||||
8 14 norm ID:9 compaction:<physical:true >
|
||||
|
||||
Entry types (ConfigChange,IRRCompaction) count is : 5
|
||||
```
|
||||
#### etcd-dump-logs -stream-decoder <EXECUTABLE_DECODER> [data dir]
|
||||
|
||||
Decode each entry based on logic in the passed decoder. Decoded status and decoded data are listed in separated tab/columns in the ouput. For parsing purpose, the output from decoder are expected to be in format of "<DECODING_STATUS>|<DECODED_DATA". Please refer to [decoder_correctoutputformat.sh] as an example.
|
||||
|
||||
However, if the decoder output format is not as expected, "decoder_status" will be "decoder output format is not right, print output anyway", and all output from decoder will be considered as "decoded_data"
|
||||
|
||||
```
|
||||
$ etcd-dump-logs -stream-decoder decoder_wrongoutputformat.sh /tmp/datadir
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data decoder_status decoded_data
|
||||
1 1 conf method=ConfChangeAddNode id=2 decoder output format is not right, print output anyway jhjaajjjahjbbbjj
|
||||
3 2 norm noop decoder output format is not right, print output anyway jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 3 norm method=QGET path="/path1" decoder output format is not right, print output anyway jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
7 4 norm ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > decoder output format is not right, print output anyway jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
|
||||
8 5 norm ID:9 compaction:<physical:true > decoder output format is not right, print output anyway jhjicajbajja
|
||||
9 6 norm ID:10 lease_grant:<TTL:1 ID:1 > decoder output format is not right, print output anyway jhjadbjdjhjaajja
|
||||
12 7 norm ID:13 auth_enable:<> decoder output format is not right, print output anyway jhjdcbcejj
|
||||
27 8 norm ??? decoder output format is not right, print output anyway cf
|
||||
Entry types () count is : 8
|
||||
|
||||
$ etcd-dump-logs -stream-decoder decoder_wrongoutputformat.sh /tmp/datadir
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data decoder_status decoded_data
|
||||
1 1 conf method=ConfChangeAddNode id=2 ERROR jhjaajjjahjbbbjj
|
||||
3 2 norm noop OK jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 3 norm method=QGET path="/path1" OK jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
7 4 norm ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > OK jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
|
||||
8 5 norm ID:9 compaction:<physical:true > ERROR jhjicajbajja
|
||||
9 6 norm ID:10 lease_grant:<TTL:1 ID:1 > ERROR jhjadbjdjhjaajja
|
||||
12 7 norm ID:13 auth_enable:<> ERROR jhjdcbcejj
|
||||
27 8 norm ??? ERROR cf
|
||||
Entry types () count is : 8
|
||||
```
|
||||
#### etcd-dump-logs -start-index <INDEX NUMBER> [data dir]
|
||||
|
||||
Only shows WAL log entries after the specified start-index number, exclusively.
|
||||
|
||||
```
|
||||
$ etcd-dump-logs -start-index 30 /tmp/datadir
|
||||
Start dumping log entries from index 30.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data
|
||||
25 31 norm ID:26 auth_role_get:<role:"role3" >
|
||||
26 32 norm ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > >
|
||||
27 33 norm ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" >
|
||||
27 34 norm ???
|
||||
Entry types () count is : 4
|
||||
```
|
||||
[decoder_correctoutputformat.sh]: ./testdecoder/decoder_correctoutputformat.sh
|
||||
@ -45,6 +45,9 @@ func TestEtcdDumpLogEntryType(t *testing.T) {
|
||||
t.Skipf("%q does not exist", dumpLogsBinary)
|
||||
}
|
||||
|
||||
decoder_correctoutputformat := filepath.Join(binDir, "/testdecoder/decoder_correctoutputformat.sh")
|
||||
decoder_wrongoutputformat := filepath.Join(binDir, "/testdecoder/decoder_wrongoutputformat.sh")
|
||||
|
||||
p, err := ioutil.TempDir(os.TempDir(), "etcddumplogstest")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@ -102,6 +105,8 @@ func TestEtcdDumpLogEntryType(t *testing.T) {
|
||||
{"lease grant entry-type", []string{"-entry-type", "IRRLeaseGrant", p}, "expectedoutput/listIRRLeaseGrant.output"},
|
||||
{"lease revoke entry-type", []string{"-entry-type", "IRRLeaseRevoke", p}, "expectedoutput/listIRRLeaseRevoke.output"},
|
||||
{"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,IRRCompaction", p}, "expectedoutput/listConfigChangeIRRCompaction.output"},
|
||||
{"decoder_correctoutputformat", []string{"-stream-decoder", decoder_correctoutputformat, p}, "expectedoutput/decoder_correctoutputformat.output"},
|
||||
{"decoder_wrongoutputformat", []string{"-stream-decoder", decoder_wrongoutputformat, p}, "expectedoutput/decoder_wrongoutputformat.output"},
|
||||
}
|
||||
|
||||
for _, argtest := range argtests {
|
||||
|
||||
@ -0,0 +1,44 @@
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data decoder_status decoded_data
|
||||
1 1 conf method=ConfChangeAddNode id=2 ERROR jhjaajjjahjbbbjj
|
||||
2 2 conf method=ConfChangeRemoveNode id=2 ERROR jhjbajjaahjbbbjj
|
||||
2 3 conf method=ConfChangeUpdateNode id=2 ERROR jhjcajjbahjbbbjj
|
||||
2 4 conf method=ConfChangeAddLearnerNode id=3 ERROR jhjdajjcahjcbbjj
|
||||
3 5 norm noop OK jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 6 norm method=QGET path="/path1" OK jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" OK jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 8 norm method=DELETE path="/path3" OK jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" OK jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
4 10 norm ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > OK jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja
|
||||
5 11 norm ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > OK jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja
|
||||
6 12 norm ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > OK jhjgbajhjajacjabjaciahja
|
||||
7 13 norm ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > OK jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
|
||||
8 14 norm ID:9 compaction:<physical:true > ERROR jhjicajbajja
|
||||
9 15 norm ID:10 lease_grant:<TTL:1 ID:1 > ERROR jhjadbjdjhjaajja
|
||||
10 16 norm ID:11 lease_revoke:<ID:2 > ERROR jhjbdajbjhjb
|
||||
11 17 norm ID:12 alarm:<action:3 memberID:4 alarm:5 > OK jhjcebjfjhjcajjdahje
|
||||
12 18 norm ID:13 auth_enable:<> ERROR jhjdcbcejj
|
||||
13 19 norm ID:14 auth_disable:<> ERROR jhjeiacfjj
|
||||
14 20 norm ID:15 authenticate:<name:"myname" password:"password" simple_token:"token" > OK jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe
|
||||
15 21 norm ID:16 auth_user_add:<name:"name1" password:"pass1" > OK jhajebddjejajefefafdfecaabjegjfagcgcca
|
||||
16 22 norm ID:17 auth_user_delete:<name:"name1" > OK jhaaeaddjgjajefefafdfeca
|
||||
17 23 norm ID:18 auth_user_get:<name:"name1" > OK jhabfbddjgjajefefafdfeca
|
||||
18 24 norm ID:19 auth_user_change_password:<name:"name1" password:"pass2" > OK jhacfaddjejajefefafdfecaabjegjfagcgccb
|
||||
19 25 norm ID:20 auth_user_grant_role:<user:"user1" role:"role1" > OK jhadhbdejejajegegcfegbcaabjegbfffcfeca
|
||||
20 26 norm ID:21 auth_user_revoke_role:<name:"user2" role:"role2" > OK jhaehadejejajegegcfegbcbabjegbfffcfecb
|
||||
21 27 norm ID:22 auth_user_list:<> ERROR jhafibdejj
|
||||
22 28 norm ID:23 auth_role_list:<> ERROR jhagiadejj
|
||||
23 29 norm ID:24 auth_role_add:<name:"role2" > OK jhahhbdbjgjajegbfffcfecb
|
||||
24 30 norm ID:25 auth_role_delete:<role:"role1" > OK jhaihadbjgjajegbfffcfeca
|
||||
25 31 norm ID:26 auth_role_get:<role:"role3" > OK jhaaibdbjgjajegbfffcfecc
|
||||
26 32 norm ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > > OK jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd
|
||||
27 33 norm ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" > OK jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd
|
||||
27 34 norm ??? ERROR cf
|
||||
|
||||
Entry types () count is : 34
|
||||
@ -0,0 +1,44 @@
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data decoder_status decoded_data
|
||||
1 1 conf method=ConfChangeAddNode id=2 decoder output format is not right, print output anyway jhjaajjjahjbbbjj
|
||||
2 2 conf method=ConfChangeRemoveNode id=2 decoder output format is not right, print output anyway jhjbajjaahjbbbjj
|
||||
2 3 conf method=ConfChangeUpdateNode id=2 decoder output format is not right, print output anyway jhjcajjbahjbbbjj
|
||||
2 4 conf method=ConfChangeAddLearnerNode id=3 decoder output format is not right, print output anyway jhjdajjcahjcbbjj
|
||||
3 5 norm noop decoder output format is not right, print output anyway jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 6 norm method=QGET path="/path1" decoder output format is not right, print output anyway jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" decoder output format is not right, print output anyway jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 8 norm method=DELETE path="/path3" decoder output format is not right, print output anyway jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" decoder output format is not right, print output anyway jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
|
||||
4 10 norm ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > decoder output format is not right, print output anyway jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja
|
||||
5 11 norm ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > decoder output format is not right, print output anyway jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja
|
||||
6 12 norm ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > decoder output format is not right, print output anyway jhjgbajhjajacjabjaciahja
|
||||
7 13 norm ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > decoder output format is not right, print output anyway jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
|
||||
8 14 norm ID:9 compaction:<physical:true > decoder output format is not right, print output anyway jhjicajbajja
|
||||
9 15 norm ID:10 lease_grant:<TTL:1 ID:1 > decoder output format is not right, print output anyway jhjadbjdjhjaajja
|
||||
10 16 norm ID:11 lease_revoke:<ID:2 > decoder output format is not right, print output anyway jhjbdajbjhjb
|
||||
11 17 norm ID:12 alarm:<action:3 memberID:4 alarm:5 > decoder output format is not right, print output anyway jhjcebjfjhjcajjdahje
|
||||
12 18 norm ID:13 auth_enable:<> decoder output format is not right, print output anyway jhjdcbcejj
|
||||
13 19 norm ID:14 auth_disable:<> decoder output format is not right, print output anyway jhjeiacfjj
|
||||
14 20 norm ID:15 authenticate:<name:"myname" password:"password" simple_token:"token" > decoder output format is not right, print output anyway jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe
|
||||
15 21 norm ID:16 auth_user_add:<name:"name1" password:"pass1" > decoder output format is not right, print output anyway jhajebddjejajefefafdfecaabjegjfagcgcca
|
||||
16 22 norm ID:17 auth_user_delete:<name:"name1" > decoder output format is not right, print output anyway jhaaeaddjgjajefefafdfeca
|
||||
17 23 norm ID:18 auth_user_get:<name:"name1" > decoder output format is not right, print output anyway jhabfbddjgjajefefafdfeca
|
||||
18 24 norm ID:19 auth_user_change_password:<name:"name1" password:"pass2" > decoder output format is not right, print output anyway jhacfaddjejajefefafdfecaabjegjfagcgccb
|
||||
19 25 norm ID:20 auth_user_grant_role:<user:"user1" role:"role1" > decoder output format is not right, print output anyway jhadhbdejejajegegcfegbcaabjegbfffcfeca
|
||||
20 26 norm ID:21 auth_user_revoke_role:<name:"user2" role:"role2" > decoder output format is not right, print output anyway jhaehadejejajegegcfegbcbabjegbfffcfecb
|
||||
21 27 norm ID:22 auth_user_list:<> decoder output format is not right, print output anyway jhafibdejj
|
||||
22 28 norm ID:23 auth_role_list:<> decoder output format is not right, print output anyway jhagiadejj
|
||||
23 29 norm ID:24 auth_role_add:<name:"role2" > decoder output format is not right, print output anyway jhahhbdbjgjajegbfffcfecb
|
||||
24 30 norm ID:25 auth_role_delete:<role:"role1" > decoder output format is not right, print output anyway jhaihadbjgjajegbfffcfeca
|
||||
25 31 norm ID:26 auth_role_get:<role:"role3" > decoder output format is not right, print output anyway jhaaibdbjgjajegbfffcfecc
|
||||
26 32 norm ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > > decoder output format is not right, print output anyway jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd
|
||||
27 33 norm ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" > decoder output format is not right, print output anyway jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd
|
||||
27 34 norm ??? decoder output format is not right, print output anyway cf
|
||||
|
||||
Entry types () count is : 34
|
||||
@ -1,11 +0,0 @@
|
||||
Snapshot:
|
||||
empty
|
||||
Start dupmping log entries from snapshot.
|
||||
WAL metadata:
|
||||
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
|
||||
WAL entries:
|
||||
lastIndex=34
|
||||
term index type data
|
||||
10 16 norm ID:11 lease_revoke:<ID:2 >
|
||||
|
||||
Entry types (IRRLeaseRevoke) count is : 1
|
||||
@ -15,9 +15,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
@ -40,6 +46,9 @@ func main() {
|
||||
ConfigChange, Normal, Request, InternalRaftRequest,
|
||||
IRRRange, IRRPut, IRRDeleteRange, IRRTxn,
|
||||
IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`)
|
||||
streamdecoder := flag.String("stream-decoder", "", `The name of an executable decoding tool, the executable must process
|
||||
hex encoded lines of binary input (from etcd-dump-logs)
|
||||
and output a hex encoded line of binary for each input line`)
|
||||
|
||||
flag.Parse()
|
||||
|
||||
@ -101,8 +110,14 @@ func main() {
|
||||
|
||||
fmt.Printf("WAL entries:\n")
|
||||
fmt.Printf("lastIndex=%d\n", ents[len(ents)-1].Index)
|
||||
fmt.Printf("%4s\t%10s\ttype\tdata\n", "term", "index")
|
||||
listEntriesType(*entrytype, ents)
|
||||
|
||||
fmt.Printf("%4s\t%10s\ttype\tdata", "term", "index")
|
||||
if *streamdecoder != "" {
|
||||
fmt.Printf("\tdecoder_status\tdecoded_data")
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
listEntriesType(*entrytype, *streamdecoder, ents)
|
||||
}
|
||||
|
||||
func walDir(dataDir string) string { return filepath.Join(dataDir, "member", "wal") }
|
||||
@ -203,12 +218,12 @@ type EntryPrinter func(e raftpb.Entry)
|
||||
func printInternalRaftRequest(entry raftpb.Entry) {
|
||||
var rr etcdserverpb.InternalRaftRequest
|
||||
if err := rr.Unmarshal(entry.Data); err == nil {
|
||||
fmt.Printf("%4d\t%10d\tnorm\t%s\n", entry.Term, entry.Index, rr.String())
|
||||
fmt.Printf("%4d\t%10d\tnorm\t%s", entry.Term, entry.Index, rr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func printUnknownNormal(entry raftpb.Entry) {
|
||||
fmt.Printf("%4d\t%10d\tnorm\t???\n", entry.Term, entry.Index)
|
||||
fmt.Printf("%4d\t%10d\tnorm\t???", entry.Term, entry.Index)
|
||||
}
|
||||
|
||||
func printConfChange(entry raftpb.Entry) {
|
||||
@ -216,9 +231,9 @@ func printConfChange(entry raftpb.Entry) {
|
||||
fmt.Printf("\tconf")
|
||||
var r raftpb.ConfChange
|
||||
if err := r.Unmarshal(entry.Data); err != nil {
|
||||
fmt.Printf("\t???\n")
|
||||
fmt.Printf("\t???")
|
||||
} else {
|
||||
fmt.Printf("\tmethod=%s id=%s\n", r.Type, types.ID(r.NodeID))
|
||||
fmt.Printf("\tmethod=%s id=%s", r.Type, types.ID(r.NodeID))
|
||||
}
|
||||
}
|
||||
|
||||
@ -228,13 +243,13 @@ func printRequest(entry raftpb.Entry) {
|
||||
fmt.Printf("%4d\t%10d\tnorm", entry.Term, entry.Index)
|
||||
switch r.Method {
|
||||
case "":
|
||||
fmt.Printf("\tnoop\n")
|
||||
fmt.Printf("\tnoop")
|
||||
case "SYNC":
|
||||
fmt.Printf("\tmethod=SYNC time=%q\n", time.Unix(0, r.Time))
|
||||
fmt.Printf("\tmethod=SYNC time=%q", time.Unix(0, r.Time))
|
||||
case "QGET", "DELETE":
|
||||
fmt.Printf("\tmethod=%s path=%s\n", r.Method, excerpt(r.Path, 64, 64))
|
||||
fmt.Printf("\tmethod=%s path=%s", r.Method, excerpt(r.Path, 64, 64))
|
||||
default:
|
||||
fmt.Printf("\tmethod=%s path=%s val=%s\n", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0))
|
||||
fmt.Printf("\tmethod=%s path=%s val=%s", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -281,13 +296,33 @@ IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`, et)
|
||||
}
|
||||
|
||||
// listEntriesType filters and prints entries based on the entry-type flag,
|
||||
func listEntriesType(entrytype string, ents []raftpb.Entry) {
|
||||
func listEntriesType(entrytype string, streamdecoder string, ents []raftpb.Entry) {
|
||||
entryFilters := evaluateEntrytypeFlag(entrytype)
|
||||
printerMap := map[string]EntryPrinter{"InternalRaftRequest": printInternalRaftRequest,
|
||||
"Request": printRequest,
|
||||
"ConfigChange": printConfChange,
|
||||
"UnknownNormal": printUnknownNormal}
|
||||
var stderr bytes.Buffer
|
||||
args := strings.Split(streamdecoder, " ")
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
cmd.Stderr = &stderr
|
||||
if streamdecoder != "" {
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
cnt := 0
|
||||
|
||||
for _, e := range ents {
|
||||
passed := false
|
||||
currtype := ""
|
||||
@ -301,7 +336,55 @@ func listEntriesType(entrytype string, ents []raftpb.Entry) {
|
||||
if passed {
|
||||
printer := printerMap[currtype]
|
||||
printer(e)
|
||||
if streamdecoder == "" {
|
||||
fmt.Println()
|
||||
continue
|
||||
}
|
||||
|
||||
// if decoder is set, pass the e.Data to stdin and read the stdout from decoder
|
||||
io.WriteString(stdin, hex.EncodeToString(e.Data))
|
||||
io.WriteString(stdin, "\n")
|
||||
outputReader := bufio.NewReader(stdout)
|
||||
decoderoutput, currerr := outputReader.ReadString('\n')
|
||||
if currerr != nil {
|
||||
fmt.Println(currerr)
|
||||
return
|
||||
}
|
||||
|
||||
decoder_status, decoded_data := parseDecoderOutput(decoderoutput)
|
||||
|
||||
fmt.Printf("\t%s\t%s", decoder_status, decoded_data)
|
||||
}
|
||||
}
|
||||
|
||||
stdin.Close()
|
||||
err = cmd.Wait()
|
||||
if streamdecoder != "" {
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
if stderr.String() != "" {
|
||||
os.Stderr.WriteString("decoder stderr: " + stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\nEntry types (%s) count is : %d", entrytype, cnt)
|
||||
}
|
||||
|
||||
func parseDecoderOutput(decoderoutput string) (string, string) {
|
||||
var decoder_status string
|
||||
var decoded_data string
|
||||
output := strings.Split(decoderoutput, "|")
|
||||
switch len(output) {
|
||||
case 1:
|
||||
decoder_status = "decoder output format is not right, print output anyway"
|
||||
decoded_data = decoderoutput
|
||||
case 2:
|
||||
decoder_status = output[0]
|
||||
decoded_data = output[1]
|
||||
default:
|
||||
decoder_status = output[0] + "(*WARNING: data might contain deliminator used by etcd-dump-logs)"
|
||||
decoded_data = strings.Join(output[1:], "")
|
||||
}
|
||||
return decoder_status, decoded_data
|
||||
}
|
||||
|
||||
10
tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh
Executable file
10
tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh
Executable file
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
while read line
|
||||
do
|
||||
LEN=$(echo ${#line})
|
||||
if [ $LEN -ge 20 ]; then
|
||||
echo "OK|$line" | tr 1234567890 abcdefghij
|
||||
else
|
||||
echo "ERROR|$line" | tr 1234567890 abcdefghij
|
||||
fi
|
||||
done < "${1:-/dev/stdin}"
|
||||
5
tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh
Executable file
5
tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh
Executable file
@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
while read line
|
||||
do
|
||||
echo "$line" | tr 1234567890 abcdefghij
|
||||
done < "${1:-/dev/stdin}"
|
||||
Loading…
x
Reference in New Issue
Block a user