From 44ab66d8584684952cc69f739ed542cb17c69045 Mon Sep 17 00:00:00 2001 From: Yicheng Qin Date: Thu, 18 Sep 2014 23:13:08 -0700 Subject: [PATCH 1/4] etcdserver: add TestRecvSnapshot --- etcdserver/server_test.go | 65 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go index 1caedb230..6c5dbc647 100644 --- a/etcdserver/server_test.go +++ b/etcdserver/server_test.go @@ -472,9 +472,6 @@ func TestSnapshot(t *testing.T) { } // Applied > SnapCount should trigger a SaveSnap event -// TODO: receive a snapshot from raft leader should also be able -// to trigger snapSave and also trigger a store.Recover. -// We need fake node! func TestTriggerSnap(t *testing.T) { ctx := context.Background() n := raft.StartNode(0xBAD0, []int64{0xBAD0}, 10, 1) @@ -507,6 +504,34 @@ func TestTriggerSnap(t *testing.T) { } } +// TestRecvSnapshot tests when it receives a snapshot from raft leader, +// it should trigger storage.SaveSnap and also store.Recover. +func TestRecvSnapshot(t *testing.T) { + n := newReadyNode(raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}}) + st := &storeRecorder{} + p := &storageRecorder{} + s := &EtcdServer{ + Store: st, + Send: func(_ []raftpb.Message) {}, + Storage: p, + Node: n, + } + + s.Start() + // make goroutines move forward to receive snapshot + forceGosched() + s.Stop() + + waction := []string{"Recovery"} + if g := st.Action(); !reflect.DeepEqual(g, waction) { + t.Errorf("store action = %v, want %v", g, waction) + } + waction = []string{"Save", "SaveSnap"} + if g := p.Action(); !reflect.DeepEqual(g, waction) { + t.Errorf("storage action = %v, want %v", g, waction) + } +} + // TODO: test wait trigger correctness in multi-server case func TestGetBool(t *testing.T) { @@ -590,7 +615,10 @@ func (s *storeRecorder) Save() ([]byte, error) { s.record("Save") return nil, nil } -func (s *storeRecorder) Recovery(b []byte) error { return nil } +func (s *storeRecorder) Recovery(b []byte) error { + s.record("Recovery") + return nil +} func (s *storeRecorder) TotalTransactions() uint64 { return 0 } func (s *storeRecorder) JsonStats() []byte { return nil } func (s *storeRecorder) DeleteExpiredKeys(cutoff time.Time) { @@ -636,6 +664,26 @@ func (p *storageRecorder) SaveSnap(st raftpb.Snapshot) { p.record("SaveSnap") } +type readyNode struct { + readyc chan raft.Ready +} + +func newReadyNode(ready raft.Ready) *readyNode { + readyc := make(chan raft.Ready, 1) + readyc <- ready + return &readyNode{readyc: readyc} +} +func (n *readyNode) Tick() {} +func (n *readyNode) Campaign(ctx context.Context) error { return nil } +func (n *readyNode) Propose(ctx context.Context, data []byte) error { return nil } +func (n *readyNode) Configure(ctx context.Context, data []byte) error { return nil } +func (n *readyNode) Step(ctx context.Context, msg raftpb.Message) error { return nil } +func (n *readyNode) Ready() <-chan raft.Ready { return n.readyc } +func (n *readyNode) Stop() {} +func (n *readyNode) Compact(d []byte) {} +func (n *readyNode) AddNode(id int64) {} +func (n *readyNode) RemoveNode(id int64) {} + func TestGenID(t *testing.T) { // Sanity check that the GenID function has been seeded appropriately // (math/rand is seeded with 1 by default) @@ -648,3 +696,12 @@ func TestGenID(t *testing.T) { t.Fatalf("GenID's rand seeded with 1!") } } + +// WARNING: This is a hack. +// Remove this when we are able to block/check the status of the go-routines. +func forceGosched() { + // possibility enough to sched upto 10 go routines. + for i := 0; i < 10000; i++ { + runtime.Gosched() + } +} From f2ebd64a1bcb55af95ecd8ca45239878fbec6ec7 Mon Sep 17 00:00:00 2001 From: Yicheng Qin Date: Fri, 19 Sep 2014 14:32:38 -0700 Subject: [PATCH 2/4] *: add testutil pkg --- etcdserver/server_test.go | 12 ++---------- raft/node_test.go | 21 ++++++--------------- testutil/testutil.go | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 25 deletions(-) create mode 100644 testutil/testutil.go diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go index 6c5dbc647..3aa47e57e 100644 --- a/etcdserver/server_test.go +++ b/etcdserver/server_test.go @@ -13,6 +13,7 @@ import ( "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" "github.com/coreos/etcd/store" + "github.com/coreos/etcd/testutil" "github.com/coreos/etcd/third_party/code.google.com/p/go.net/context" ) @@ -519,7 +520,7 @@ func TestRecvSnapshot(t *testing.T) { s.Start() // make goroutines move forward to receive snapshot - forceGosched() + testutil.ForceGosched() s.Stop() waction := []string{"Recovery"} @@ -696,12 +697,3 @@ func TestGenID(t *testing.T) { t.Fatalf("GenID's rand seeded with 1!") } } - -// WARNING: This is a hack. -// Remove this when we are able to block/check the status of the go-routines. -func forceGosched() { - // possibility enough to sched upto 10 go routines. - for i := 0; i < 10000; i++ { - runtime.Gosched() - } -} diff --git a/raft/node_test.go b/raft/node_test.go index 92c23552a..a3f2470ce 100644 --- a/raft/node_test.go +++ b/raft/node_test.go @@ -2,11 +2,11 @@ package raft import ( "reflect" - "runtime" "testing" "time" "github.com/coreos/etcd/raft/raftpb" + "github.com/coreos/etcd/testutil" "github.com/coreos/etcd/third_party/code.google.com/p/go.net/context" ) @@ -96,7 +96,7 @@ func TestBlockProposal(t *testing.T) { errc <- n.Propose(context.TODO(), []byte("somedata")) }() - forceGosched() + testutil.ForceGosched() select { case err := <-errc: t.Errorf("err = %v, want blocking", err) @@ -104,7 +104,7 @@ func TestBlockProposal(t *testing.T) { } n.Campaign(context.TODO()) - forceGosched() + testutil.ForceGosched() select { case err := <-errc: if err != nil { @@ -216,7 +216,7 @@ func TestCompact(t *testing.T) { Nodes: []int64{1}, } - forceGosched() + testutil.ForceGosched() select { case <-n.Ready(): default: @@ -224,7 +224,7 @@ func TestCompact(t *testing.T) { } n.Compact(w.Data) - forceGosched() + testutil.ForceGosched() select { case rd := <-n.Ready(): if !reflect.DeepEqual(rd.Snapshot, w) { @@ -233,7 +233,7 @@ func TestCompact(t *testing.T) { default: t.Fatalf("unexpected compact failure: unable to create a snapshot") } - forceGosched() + testutil.ForceGosched() // TODO: this test the run updates the snapi correctly... should be tested // separately with other kinds of updates select { @@ -265,12 +265,3 @@ func TestIsStateEqual(t *testing.T) { } } } - -// WARNING: This is a hack. -// Remove this when we are able to block/check the status of the go-routines. -func forceGosched() { - // possibility enough to sched upto 10 go routines. - for i := 0; i < 10000; i++ { - runtime.Gosched() - } -} diff --git a/testutil/testutil.go b/testutil/testutil.go new file mode 100644 index 000000000..78ee24989 --- /dev/null +++ b/testutil/testutil.go @@ -0,0 +1,14 @@ +package testutil + +import ( + "runtime" +) + +// WARNING: This is a hack. +// Remove this when we are able to block/check the status of the go-routines. +func ForceGosched() { + // possibility enough to sched upto 10 go routines. + for i := 0; i < 10000; i++ { + runtime.Gosched() + } +} From b8e59a3c6a519798c73134c5d145f0e8d2faac4e Mon Sep 17 00:00:00 2001 From: Yicheng Qin Date: Fri, 19 Sep 2014 14:40:11 -0700 Subject: [PATCH 3/4] etcdserver: not push ready to readyNode automatically --- etcdserver/server_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go index 3aa47e57e..0645a764f 100644 --- a/etcdserver/server_test.go +++ b/etcdserver/server_test.go @@ -508,7 +508,7 @@ func TestTriggerSnap(t *testing.T) { // TestRecvSnapshot tests when it receives a snapshot from raft leader, // it should trigger storage.SaveSnap and also store.Recover. func TestRecvSnapshot(t *testing.T) { - n := newReadyNode(raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}}) + n := newReadyNode() st := &storeRecorder{} p := &storageRecorder{} s := &EtcdServer{ @@ -519,6 +519,7 @@ func TestRecvSnapshot(t *testing.T) { } s.Start() + n.readyc <- raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}} // make goroutines move forward to receive snapshot testutil.ForceGosched() s.Stop() @@ -669,9 +670,8 @@ type readyNode struct { readyc chan raft.Ready } -func newReadyNode(ready raft.Ready) *readyNode { +func newReadyNode() *readyNode { readyc := make(chan raft.Ready, 1) - readyc <- ready return &readyNode{readyc: readyc} } func (n *readyNode) Tick() {} From fac38aad3339688e3c015d04d381a49d2b57efea Mon Sep 17 00:00:00 2001 From: Yicheng Qin Date: Fri, 19 Sep 2014 15:09:04 -0700 Subject: [PATCH 4/4] etcdserver: add TestRecvSlowSnapshot --- etcdserver/server_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go index 0645a764f..fd455519a 100644 --- a/etcdserver/server_test.go +++ b/etcdserver/server_test.go @@ -534,6 +534,34 @@ func TestRecvSnapshot(t *testing.T) { } } +// TestRecvSlowSnapshot tests that slow snapshot will not be applied +// to store. +func TestRecvSlowSnapshot(t *testing.T) { + n := newReadyNode() + st := &storeRecorder{} + s := &EtcdServer{ + Store: st, + Send: func(_ []raftpb.Message) {}, + Storage: &storageRecorder{}, + Node: n, + } + + s.Start() + n.readyc <- raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}} + // make goroutines move forward to receive snapshot + testutil.ForceGosched() + action := st.Action() + + n.readyc <- raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}} + // make goroutines move forward to receive snapshot + testutil.ForceGosched() + s.Stop() + + if g := st.Action(); !reflect.DeepEqual(g, action) { + t.Errorf("store action = %v, want %v", g, action) + } +} + // TODO: test wait trigger correctness in multi-server case func TestGetBool(t *testing.T) {