From 6534525cf842b64c61dc590506fee8631a0ba4d5 Mon Sep 17 00:00:00 2001 From: Yicheng Qin Date: Tue, 22 Jul 2014 17:16:15 -0700 Subject: [PATCH] server: add bootstrap tests --- etcd/discovery.go | 8 +- etcd/etcd_start_test.go | 186 +++++++++++++++++++++++++++++ etcd/participant.go | 4 +- tests/functional/discovery_test.go | 181 ---------------------------- 4 files changed, 192 insertions(+), 187 deletions(-) create mode 100644 etcd/etcd_start_test.go diff --git a/etcd/discovery.go b/etcd/discovery.go index dca7d6947..e73fc3e00 100644 --- a/etcd/discovery.go +++ b/etcd/discovery.go @@ -53,7 +53,7 @@ func newDiscoverer(u *url.URL, name, raftPubAddr string) *discoverer { u.Path = "" // Connect to a scheme://host not a full URL with path - log.Println("Discovery via %s using prefix %s.", u.String(), d.prefix) + log.Printf("Discovery via %s using prefix %s.\n", u.String(), d.prefix) d.client = etcd.NewClient([]string{u.String()}) if !strings.HasPrefix(oldPath, "/v2/keys") { @@ -64,7 +64,7 @@ func newDiscoverer(u *url.URL, name, raftPubAddr string) *discoverer { func (d *discoverer) discover() ([]string, error) { if _, err := d.client.Set(path.Join(d.prefix, d.name), d.addr, defaultTTL); err != nil { - return nil, err + return nil, fmt.Errorf("discovery service error: %v", err) } // Attempt to take the leadership role, if there is no error we are it! @@ -72,7 +72,7 @@ func (d *discoverer) discover() ([]string, error) { // Bail out on unexpected errors if err != nil { if clientErr, ok := err.(*etcd.EtcdError); !ok || clientErr.ErrorCode != etcdErr.EcodeNodeExist { - return nil, err + return nil, fmt.Errorf("discovery service error: %v", err) } } @@ -90,7 +90,7 @@ func (d *discoverer) discover() ([]string, error) { func (d *discoverer) findPeers() (peers []string, err error) { resp, err := d.client.Get(path.Join(d.prefix), false, true) if err != nil { - return nil, err + return nil, fmt.Errorf("discovery service error: %v", err) } node := resp.Node diff --git a/etcd/etcd_start_test.go b/etcd/etcd_start_test.go new file mode 100644 index 000000000..3ea412ad1 --- /dev/null +++ b/etcd/etcd_start_test.go @@ -0,0 +1,186 @@ +/* +Copyright 2014 CoreOS Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/coreos/etcd/config" +) + +const ( + bootstrapId = 0xBEEF +) + +type garbageHandler struct { + t *testing.T + success bool + sync.Mutex +} + +func (g *garbageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, client") + wp := fmt.Sprint("/v2/keys/_etcd/registry/1/", bootstrapId) + if gp := r.URL.String(); gp != wp { + g.t.Fatalf("url = %s, want %s", gp, wp) + } + g.Lock() + defer g.Unlock() + + g.success = true +} + +func TestBadDiscoveryService(t *testing.T) { + g := garbageHandler{t: t} + ts := httptest.NewServer(&g) + defer ts.Close() + + c := config.New() + c.Discovery = ts.URL + "/v2/keys/_etcd/registry/1" + _, _, err := buildServer(c, bootstrapId) + w := `discovery service error` + if err == nil || !strings.HasPrefix(err.Error(), w) { + t.Errorf("err = %v, want %s prefix", err, w) + } + + g.Lock() + defer g.Unlock() + if !g.success { + t.Fatal("Discovery server never called") + } +} + +func TestRunByAdvisedPeers(t *testing.T) { + es, hs := buildCluster(1, false) + waitCluster(t, es) + + c := config.New() + c.Peers = []string{hs[0].URL} + e, h, err := buildServer(c, bootstrapId) + if err != nil { + t.Fatalf("build server err = %v, want nil", err) + } + w := es[0].id + if g, _ := waitLeader(append(es, e)); g != w { + t.Errorf("leader = %d, want %d", g, w) + } + + destroyServer(e, h) + for i := range hs { + es[len(hs)-i-1].Stop() + } + for i := range hs { + hs[len(hs)-i-1].Close() + } +} + +func TestBadDiscoveryServiceWithAdvisedPeers(t *testing.T) { + g := garbageHandler{t: t} + ts := httptest.NewServer(&g) + defer ts.Close() + + es, hs := buildCluster(1, false) + waitCluster(t, es) + + c := config.New() + c.Discovery = ts.URL + "/v2/keys/_etcd/registry/1" + c.Peers = []string{hs[0].URL} + _, _, err := buildServer(c, bootstrapId) + w := `discovery service error` + if err == nil || !strings.HasPrefix(err.Error(), w) { + t.Errorf("err = %v, want %s prefix", err, w) + } + + for i := range hs { + es[len(hs)-i-1].Stop() + } + for i := range hs { + hs[len(hs)-i-1].Close() + } +} + +func TestBootstrapByDiscoveryService(t *testing.T) { + de, dh, _ := buildServer(config.New(), genId()) + + c := config.New() + c.Discovery = dh.URL + "/v2/keys/_etcd/registry/1" + e, h, err := buildServer(c, bootstrapId) + if err != nil { + t.Fatalf("build server err = %v, want nil", err) + } + + destroyServer(e, h) + destroyServer(de, dh) +} + +func TestRunByDiscoveryService(t *testing.T) { + de, dh, _ := buildServer(config.New(), genId()) + + tc := NewTestClient() + v := url.Values{} + v.Set("value", "started") + resp, _ := tc.PutForm(fmt.Sprintf("%s%s", dh.URL, "/v2/keys/_etcd/registry/1/_state"), v) + if g := resp.StatusCode; g != http.StatusCreated { + t.Fatalf("put status = %d, want %d", g, http.StatusCreated) + } + v.Set("value", dh.URL) + resp, _ = tc.PutForm(fmt.Sprintf("%s%s%d", dh.URL, "/v2/keys/_etcd/registry/1/", de.id), v) + if g := resp.StatusCode; g != http.StatusCreated { + t.Fatalf("put status = %d, want %d", g, http.StatusCreated) + } + + c := config.New() + c.Discovery = dh.URL + "/v2/keys/_etcd/registry/1" + e, h, err := buildServer(c, bootstrapId) + if err != nil { + t.Fatalf("build server err = %v, want nil", err) + } + w := de.id + if g, _ := waitLeader([]*Server{e, de}); g != w { + t.Errorf("leader = %d, want %d", g, w) + } + + destroyServer(e, h) + destroyServer(de, dh) +} + +func buildServer(c *config.Config, id int64) (e *Server, h *httptest.Server, err error) { + e, h = initTestServer(c, id, false) + go func() { err = e.Run() }() + for { + if e.mode.Get() == participantMode { + break + } + if err != nil { + return nil, nil, err + } + time.Sleep(10 * time.Millisecond) + } + return e, h, nil +} + +func destroyServer(e *Server, h *httptest.Server) { + e.Stop() + h.Close() +} diff --git a/etcd/participant.go b/etcd/participant.go index c1f3dc3b3..d393b0d8d 100644 --- a/etcd/participant.go +++ b/etcd/participant.go @@ -167,13 +167,13 @@ func (p *participant) run() int64 { case <-v2SyncTicker.C: node.Sync() case <-p.stopc: - log.Printf("Participant %d stopped\n", p.id) + log.Printf("Participant %x stopped\n", p.id) return stopMode } p.apply(node.Next()) p.send(node.Msgs()) if node.IsRemoved() { - log.Printf("Participant %d return\n", p.id) + log.Printf("Participant %x return\n", p.id) p.stop() return standbyMode } diff --git a/tests/functional/discovery_test.go b/tests/functional/discovery_test.go index a769244f7..b444adce8 100644 --- a/tests/functional/discovery_test.go +++ b/tests/functional/discovery_test.go @@ -35,91 +35,6 @@ func (g *garbageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { g.success = true } -// TestDiscoveryDownNoBackupPeers ensures that etcd stops if it is started with a -// bad discovery URL and no backups. -func TestDiscoveryDownNoBackupPeers(t *testing.T) { - g := garbageHandler{t: t} - ts := httptest.NewServer(&g) - defer ts.Close() - - discover := ts.URL + "/v2/keys/_etcd/registry/1" - proc, err := startServer([]string{"-discovery", discover}) - - if err != nil { - t.Fatal(err.Error()) - } - defer stopServer(proc) - - client := http.Client{} - err = assertServerNotUp(client, "http") - if err != nil { - t.Fatal(err.Error()) - } - - g.Lock() - defer g.Unlock() - if !g.success { - t.Fatal("Discovery server never called") - } -} - -// TestDiscoveryDownWithBackupPeers ensures that etcd runs if it is started with a -// bad discovery URL and a peer list. -func TestDiscoveryDownWithBackupPeers(t *testing.T) { - etcdtest.RunServer(func(s *server.Server) { - g := garbageHandler{t: t} - ts := httptest.NewServer(&g) - defer ts.Close() - - discover := ts.URL + "/v2/keys/_etcd/registry/1" - u, ok := s.PeerHost("ETCDTEST") - if !ok { - t.Fatalf("Couldn't find the URL") - } - proc, err := startServer([]string{"-discovery", discover, "-peers", u}) - - if err != nil { - t.Fatal(err.Error()) - } - defer stopServer(proc) - - client := http.Client{} - err = assertServerFunctional(client, "http") - if err != nil { - t.Fatal(err.Error()) - } - - g.Lock() - defer g.Unlock() - if !g.success { - t.Fatal("Discovery server never called") - } - }) -} - -// TestDiscoveryNoWithBackupPeers ensures that etcd runs if it is started with -// no discovery URL and a peer list. -func TestDiscoveryNoWithBackupPeers(t *testing.T) { - etcdtest.RunServer(func(s *server.Server) { - u, ok := s.PeerHost("ETCDTEST") - if !ok { - t.Fatalf("Couldn't find the URL") - } - proc, err := startServer([]string{"-peers", u}) - - if err != nil { - t.Fatal(err.Error()) - } - defer stopServer(proc) - - client := http.Client{} - err = assertServerFunctional(client, "http") - if err != nil { - t.Fatal(err.Error()) - } - }) -} - // TestDiscoveryDownNoBackupPeersWithDataDir ensures that etcd runs if it is // started with a bad discovery URL, no backups and valid data dir. func TestDiscoveryDownNoBackupPeersWithDataDir(t *testing.T) { @@ -173,47 +88,6 @@ func TestDiscoveryDownNoBackupPeersWithDataDir(t *testing.T) { }) } -// TestDiscoveryFirstPeer ensures that etcd starts as the leader if it -// registers as the first peer. -func TestDiscoveryFirstPeer(t *testing.T) { - etcdtest.RunServer(func(s *server.Server) { - proc, err := startServer([]string{"-discovery", s.URL() + "/v2/keys/_etcd/registry/2"}) - if err != nil { - t.Fatal(err.Error()) - } - defer stopServer(proc) - - client := http.Client{} - err = assertServerFunctional(client, "http") - if err != nil { - t.Fatal(err.Error()) - } - }) -} - -// TestDiscoverySecondPeerFirstDown ensures that etcd stops if it is started with a -// correct discovery URL but no active machines are found. -func TestDiscoverySecondPeerFirstDown(t *testing.T) { - etcdtest.RunServer(func(s *server.Server) { - v := url.Values{} - v.Set("value", "started") - resp, err := etcdtest.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/_etcd/registry/2/_state"), v) - assert.Equal(t, resp.StatusCode, http.StatusCreated) - - proc, err := startServer([]string{"-discovery", s.URL() + "/v2/keys/_etcd/registry/2"}) - if err != nil { - t.Fatal(err.Error()) - } - defer stopServer(proc) - - client := http.Client{} - err = assertServerNotUp(client, "http") - if err != nil { - t.Fatal(err.Error()) - } - }) -} - // TestDiscoverySecondPeerFirstNoResponse ensures that if the first etcd // machine stops after heartbeating that the second machine fails too. func TestDiscoverySecondPeerFirstNoResponse(t *testing.T) { @@ -246,61 +120,6 @@ func TestDiscoverySecondPeerFirstNoResponse(t *testing.T) { }) } -// TestDiscoverySecondPeerUp ensures that a second peer joining a discovery -// cluster works. -func TestDiscoverySecondPeerUp(t *testing.T) { - etcdtest.RunServer(func(s *server.Server) { - v := url.Values{} - v.Set("value", "started") - resp, err := etcdtest.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/_etcd/registry/3/_state"), v) - assert.Equal(t, resp.StatusCode, http.StatusCreated) - - u, ok := s.PeerURL("ETCDTEST") - if !ok { - t.Fatalf("Couldn't find the URL") - } - - wc := goetcd.NewClient([]string{s.URL()}) - testResp, err := wc.Set("test", "0", 0) - - if err != nil { - t.Fatalf("Couldn't set a test key on the leader %v", err) - } - - v = url.Values{} - v.Set("value", u) - resp, err = etcdtest.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/_etcd/registry/3/ETCDTEST"), v) - assert.Equal(t, resp.StatusCode, http.StatusCreated) - - proc, err := startServer([]string{"-discovery", s.URL() + "/v2/keys/_etcd/registry/3"}) - if err != nil { - t.Fatal(err.Error()) - } - defer stopServer(proc) - - watch := fmt.Sprintf("%s%s%d", s.URL(), "/v2/keys/_etcd/registry/3/node1?wait=true&waitIndex=", testResp.EtcdIndex) - resp, err = http.Get(watch) - if err != nil { - t.Fatal(err.Error()) - } - - // TODO(bp): need to have a better way of knowing a machine is up - for i := 0; i < 10; i++ { - time.Sleep(1 * time.Second) - - etcdc := goetcd.NewClient(nil) - _, err = etcdc.Set("foobar", "baz", 0) - if err == nil { - break - } - } - - if err != nil { - t.Fatal(err.Error()) - } - }) -} - // TestDiscoveryRestart ensures that a discovery cluster could be restarted. func TestDiscoveryRestart(t *testing.T) { etcdtest.RunServer(func(s *server.Server) {