From d0f6f490854fd40f9d3bc705c3f537f4291bb096 Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Sat, 5 Mar 2016 13:26:02 -0800 Subject: [PATCH] e2e: quorum cleanup If a test gets data without quorum, it should only have one node in the test cluster to avoid reading stale data. Fixes #4694 --- e2e/etcd_test.go | 38 +++++---- e2e/etcdctl_test.go | 183 ++++++++++++++++++++------------------------ 2 files changed, 106 insertions(+), 115 deletions(-) diff --git a/e2e/etcd_test.go b/e2e/etcd_test.go index a47488bc1..2819e7e22 100644 --- a/e2e/etcd_test.go +++ b/e2e/etcd_test.go @@ -35,49 +35,49 @@ const ( ) var ( - defaultConfig = etcdProcessClusterConfig{ + configNoTLS = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 0, isClientTLS: false, isPeerTLS: false, initialToken: "new", } - defaultConfigTLS = etcdProcessClusterConfig{ + configTLS = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 0, isClientTLS: true, isPeerTLS: true, initialToken: "new", } - defaultConfigClientTLS = etcdProcessClusterConfig{ + configClientTLS = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 0, isClientTLS: true, isPeerTLS: false, initialToken: "new", } - defaultConfigPeerTLS = etcdProcessClusterConfig{ + configPeerTLS = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 0, isClientTLS: false, isPeerTLS: true, initialToken: "new", } - defaultConfigWithProxy = etcdProcessClusterConfig{ + configWithProxy = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 1, isClientTLS: false, isPeerTLS: false, initialToken: "new", } - defaultConfigWithProxyTLS = etcdProcessClusterConfig{ + configWithProxyTLS = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 1, isClientTLS: true, isPeerTLS: true, initialToken: "new", } - defaultConfigWithProxyPeerTLS = etcdProcessClusterConfig{ + configWithProxyPeerTLS = etcdProcessClusterConfig{ clusterSize: 3, proxySize: 1, isClientTLS: false, @@ -86,17 +86,27 @@ var ( } ) -func TestBasicOpsNoTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfig) } -func TestBasicOpsAllTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigTLS) } -func TestBasicOpsPeerTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigPeerTLS) } -func TestBasicOpsClientTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigClientTLS) } -func TestBasicOpsProxyNoTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigWithProxy) } -func TestBasicOpsProxyTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigWithProxyTLS) } -func TestBasicOpsProxyPeerTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigWithProxyPeerTLS) } +func configStandalone(cfg etcdProcessClusterConfig) *etcdProcessClusterConfig { + ret := cfg + ret.clusterSize = 1 + return &ret +} + +func TestBasicOpsNoTLS(t *testing.T) { testBasicOpsPutGet(t, &configNoTLS) } +func TestBasicOpsAllTLS(t *testing.T) { testBasicOpsPutGet(t, &configTLS) } +func TestBasicOpsPeerTLS(t *testing.T) { testBasicOpsPutGet(t, &configPeerTLS) } +func TestBasicOpsClientTLS(t *testing.T) { testBasicOpsPutGet(t, &configClientTLS) } +func TestBasicOpsProxyNoTLS(t *testing.T) { testBasicOpsPutGet(t, &configWithProxy) } +func TestBasicOpsProxyTLS(t *testing.T) { testBasicOpsPutGet(t, &configWithProxyTLS) } +func TestBasicOpsProxyPeerTLS(t *testing.T) { testBasicOpsPutGet(t, &configWithProxyPeerTLS) } func testBasicOpsPutGet(t *testing.T, cfg *etcdProcessClusterConfig) { defer testutil.AfterTest(t) + // test doesn't use quorum gets, so ensure there are no followers to avoid + // stale reads that will break the test + cfg = configStandalone(*cfg) + epc, err := newEtcdProcessCluster(cfg) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) diff --git a/e2e/etcdctl_test.go b/e2e/etcdctl_test.go index 75b5f92bc..c79142f9d 100644 --- a/e2e/etcdctl_test.go +++ b/e2e/etcdctl_test.go @@ -23,19 +23,15 @@ import ( "github.com/coreos/etcd/pkg/testutil" ) -func TestCtlV2Set(t *testing.T) { testCtlV2Set(t, &defaultConfig, false) } -func TestCtlV2SetClientTLS(t *testing.T) { testCtlV2Set(t, &defaultConfigClientTLS, false) } -func TestCtlV2SetPeerTLS(t *testing.T) { testCtlV2Set(t, &defaultConfigPeerTLS, false) } -func TestCtlV2SetTLS(t *testing.T) { testCtlV2Set(t, &defaultConfigTLS, false) } -func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { +func TestCtlV2Set(t *testing.T) { testCtlV2Set(t, &configNoTLS, false) } +func TestCtlV2SetQuorum(t *testing.T) { testCtlV2Set(t, &configNoTLS, true) } +func TestCtlV2SetClientTLS(t *testing.T) { testCtlV2Set(t, &configClientTLS, false) } +func TestCtlV2SetPeerTLS(t *testing.T) { testCtlV2Set(t, &configPeerTLS, false) } +func TestCtlV2SetTLS(t *testing.T) { testCtlV2Set(t, &configTLS, false) } +func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, errC := newEtcdProcessCluster(cfg) - if errC != nil { - t.Fatalf("could not start etcd process cluster (%v)", errC) - } + epc := setupEtcdctlTest(t, cfg, quorum) defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) @@ -44,26 +40,22 @@ func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { key, value := "foo", "bar" - if err := etcdctlSet(epc, key, value, noSync); err != nil { + if err := etcdctlSet(epc, key, value); err != nil { t.Fatalf("failed set (%v)", err) } - if err := etcdctlGet(epc, key, value, noSync); err != nil { + if err := etcdctlGet(epc, key, value, quorum); err != nil { t.Fatalf("failed get (%v)", err) } } -func TestCtlV2Mk(t *testing.T) { testCtlV2Mk(t, &defaultConfig, false) } -func TestCtlV2MkTLS(t *testing.T) { testCtlV2Mk(t, &defaultConfigTLS, false) } -func testCtlV2Mk(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { +func TestCtlV2Mk(t *testing.T) { testCtlV2Mk(t, &configNoTLS, false) } +func TestCtlV2MkQuorum(t *testing.T) { testCtlV2Mk(t, &configNoTLS, true) } +func TestCtlV2MkTLS(t *testing.T) { testCtlV2Mk(t, &configTLS, false) } +func testCtlV2Mk(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, errC := newEtcdProcessCluster(cfg) - if errC != nil { - t.Fatalf("could not start etcd process cluster (%v)", errC) - } + epc := setupEtcdctlTest(t, cfg, quorum) defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) @@ -72,29 +64,24 @@ func testCtlV2Mk(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { key, value := "foo", "bar" - if err := etcdctlMk(epc, key, value, true, noSync); err != nil { + if err := etcdctlMk(epc, key, value, true); err != nil { t.Fatalf("failed mk (%v)", err) } - if err := etcdctlMk(epc, key, value, false, noSync); err != nil { + if err := etcdctlMk(epc, key, value, false); err != nil { t.Fatalf("failed mk (%v)", err) } - if err := etcdctlGet(epc, key, value, noSync); err != nil { + if err := etcdctlGet(epc, key, value, quorum); err != nil { t.Fatalf("failed get (%v)", err) } } -func TestCtlV2Rm(t *testing.T) { testCtlV2Rm(t, &defaultConfig, false) } -func TestCtlV2RmTLS(t *testing.T) { testCtlV2Rm(t, &defaultConfigTLS, false) } -func testCtlV2Rm(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { +func TestCtlV2Rm(t *testing.T) { testCtlV2Rm(t, &configNoTLS) } +func TestCtlV2RmTLS(t *testing.T) { testCtlV2Rm(t, &configTLS) } +func testCtlV2Rm(t *testing.T, cfg *etcdProcessClusterConfig) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, errC := newEtcdProcessCluster(cfg) - if errC != nil { - t.Fatalf("could not start etcd process cluster (%v)", errC) - } + epc := setupEtcdctlTest(t, cfg, true) defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) @@ -103,29 +90,25 @@ func testCtlV2Rm(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { key, value := "foo", "bar" - if err := etcdctlSet(epc, key, value, noSync); err != nil { + if err := etcdctlSet(epc, key, value); err != nil { t.Fatalf("failed set (%v)", err) } - if err := etcdctlRm(epc, key, value, true, noSync); err != nil { + if err := etcdctlRm(epc, key, value, true); err != nil { t.Fatalf("failed rm (%v)", err) } - if err := etcdctlRm(epc, key, value, false, noSync); err != nil { + if err := etcdctlRm(epc, key, value, false); err != nil { t.Fatalf("failed rm (%v)", err) } } -func TestCtlV2Ls(t *testing.T) { testCtlV2Ls(t, &defaultConfig, false) } -func TestCtlV2LsTLS(t *testing.T) { testCtlV2Ls(t, &defaultConfigTLS, false) } -func testCtlV2Ls(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { +func TestCtlV2Ls(t *testing.T) { testCtlV2Ls(t, &configNoTLS, false) } +func TestCtlV2LsQuorum(t *testing.T) { testCtlV2Ls(t, &configNoTLS, true) } +func TestCtlV2LsTLS(t *testing.T) { testCtlV2Ls(t, &configTLS, false) } +func testCtlV2Ls(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, errC := newEtcdProcessCluster(cfg) - if errC != nil { - t.Fatalf("could not start etcd process cluster (%v)", errC) - } + epc := setupEtcdctlTest(t, cfg, quorum) defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) @@ -134,28 +117,23 @@ func testCtlV2Ls(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { key, value := "foo", "bar" - if err := etcdctlSet(epc, key, value, noSync); err != nil { + if err := etcdctlSet(epc, key, value); err != nil { t.Fatalf("failed set (%v)", err) } - if err := etcdctlLs(epc, key, noSync); err != nil { + if err := etcdctlLs(epc, key, quorum); err != nil { t.Fatalf("failed ls (%v)", err) } } -func TestCtlV2Watch(t *testing.T) { testCtlV2Watch(t, &defaultConfig, false) } -func TestCtlV2WatchTLS(t *testing.T) { testCtlV2Watch(t, &defaultConfigTLS, false) } -func TestCtlV2WatchWithProxy(t *testing.T) { testCtlV2Watch(t, &defaultConfigWithProxy, false) } -func TestCtlV2WatchWithProxyNoSync(t *testing.T) { testCtlV2Watch(t, &defaultConfigWithProxy, true) } +func TestCtlV2Watch(t *testing.T) { testCtlV2Watch(t, &configNoTLS, false) } +func TestCtlV2WatchTLS(t *testing.T) { testCtlV2Watch(t, &configTLS, false) } +func TestCtlV2WatchWithProxy(t *testing.T) { testCtlV2Watch(t, &configWithProxy, false) } +func TestCtlV2WatchWithProxyNoSync(t *testing.T) { testCtlV2Watch(t, &configWithProxy, true) } func testCtlV2Watch(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, errC := newEtcdProcessCluster(cfg) - if errC != nil { - t.Fatalf("could not start etcd process cluster (%v)", errC) - } + epc := setupEtcdctlTest(t, cfg, true) defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) @@ -164,7 +142,7 @@ func testCtlV2Watch(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { key, value := "foo", "bar" errc := etcdctlWatch(epc, key, value, noSync) - if err := etcdctlSet(epc, key, value, noSync); err != nil { + if err := etcdctlSet(epc, key, value); err != nil { t.Fatalf("failed set (%v)", err) } @@ -178,18 +156,13 @@ func testCtlV2Watch(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) { } } -func TestCtlV2GetRoleUser(t *testing.T) { testCtlV2GetRoleUser(t, &defaultConfig) } -func TestCtlV2GetRoleUserWithProxy(t *testing.T) { testCtlV2GetRoleUser(t, &defaultConfigWithProxy) } +func TestCtlV2GetRoleUser(t *testing.T) { testCtlV2GetRoleUser(t, &configNoTLS) } +func TestCtlV2GetRoleUserWithProxy(t *testing.T) { testCtlV2GetRoleUser(t, &configWithProxy) } func testCtlV2GetRoleUser(t *testing.T, cfg *etcdProcessClusterConfig) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, cerr := newEtcdProcessCluster(cfg) - if cerr != nil { - t.Fatalf("could not start etcd process cluster (%v)", cerr) - } + epc := setupEtcdctlTest(t, cfg, true) defer func() { if err := epc.Close(); err != nil { t.Fatalf("error closing etcd processes (%v)", err) @@ -217,12 +190,7 @@ func testCtlV2GetRoleUser(t *testing.T, cfg *etcdProcessClusterConfig) { func TestCtlV2UserList(t *testing.T) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, cerr := newEtcdProcessCluster(&defaultConfigWithProxy) - if cerr != nil { - t.Fatalf("could not start etcd process cluster (%v)", cerr) - } + epc := setupEtcdctlTest(t, &configWithProxy, false) defer func() { if err := epc.Close(); err != nil { t.Fatalf("error closing etcd processes (%v)", err) @@ -240,12 +208,7 @@ func TestCtlV2UserList(t *testing.T) { func TestCtlV2RoleList(t *testing.T) { defer testutil.AfterTest(t) - mustEtcdctl(t) - - epc, cerr := newEtcdProcessCluster(&defaultConfigWithProxy) - if cerr != nil { - t.Fatalf("could not start etcd process cluster (%v)", cerr) - } + epc := setupEtcdctlTest(t, &configWithProxy, false) defer func() { if err := epc.Close(); err != nil { t.Fatalf("error closing etcd processes (%v)", err) @@ -260,7 +223,7 @@ func TestCtlV2RoleList(t *testing.T) { } } -func etcdctlPrefixArgs(clus *etcdProcessCluster, noSync bool) []string { +func etcdctlPrefixArgs(clus *etcdProcessCluster) []string { endpoints := "" if proxies := clus.proxies(); len(proxies) != 0 { endpoints = proxies[0].cfg.acurl.String() @@ -272,48 +235,54 @@ func etcdctlPrefixArgs(clus *etcdProcessCluster, noSync bool) []string { endpoints = strings.Join(es, ",") } cmdArgs := []string{"../bin/etcdctl", "--endpoints", endpoints} - if noSync { - cmdArgs = append(cmdArgs, "--no-sync") - } if clus.cfg.isClientTLS { cmdArgs = append(cmdArgs, "--ca-file", caPath, "--cert-file", certPath, "--key-file", privateKeyPath) } return cmdArgs } -func etcdctlSet(clus *etcdProcessCluster, key, value string, noSync bool) error { - cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "set", key, value) - return spawnWithExpect(cmdArgs, value) +func etcdctlSet(clus *etcdProcessCluster, key, value string) error { + cmdArgs := append(etcdctlPrefixArgs(clus), "set", key, value) + return spawnWithExpectedString(cmdArgs, value) } -func etcdctlMk(clus *etcdProcessCluster, key, value string, first, noSync bool) error { - cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "mk", key, value) +func etcdctlMk(clus *etcdProcessCluster, key, value string, first bool) error { + cmdArgs := append(etcdctlPrefixArgs(clus), "mk", key, value) if first { return spawnWithExpectedString(cmdArgs, value) } return spawnWithExpectedString(cmdArgs, "Error: 105: Key already exists") } -func etcdctlGet(clus *etcdProcessCluster, key, value string, noSync bool) error { - cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "get", key) +func etcdctlGet(clus *etcdProcessCluster, key, value string, quorum bool) error { + cmdArgs := append(etcdctlPrefixArgs(clus), "get", key) + if quorum { + cmdArgs = append(cmdArgs, "--quorum") + } return spawnWithExpectedString(cmdArgs, value) } -func etcdctlRm(clus *etcdProcessCluster, key, value string, first, noSync bool) error { - cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "rm", key) +func etcdctlRm(clus *etcdProcessCluster, key, value string, first bool) error { + cmdArgs := append(etcdctlPrefixArgs(clus), "rm", key) if first { return spawnWithExpectedString(cmdArgs, "PrevNode.Value: "+value) } return spawnWithExpect(cmdArgs, "Error: 100: Key not found") } -func etcdctlLs(clus *etcdProcessCluster, key string, noSync bool) error { - cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "ls") +func etcdctlLs(clus *etcdProcessCluster, key string, quorum bool) error { + cmdArgs := append(etcdctlPrefixArgs(clus), "ls") + if quorum { + cmdArgs = append(cmdArgs, "--quorum") + } return spawnWithExpect(cmdArgs, key) } func etcdctlWatch(clus *etcdProcessCluster, key, value string, noSync bool) <-chan error { - cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "watch", "--after-index 1", key) + cmdArgs := append(etcdctlPrefixArgs(clus), "watch", "--after-index 1", key) + if noSync { + cmdArgs = append(cmdArgs, "--no-sync") + } errc := make(chan error, 1) go func() { errc <- spawnWithExpect(cmdArgs, value) @@ -322,32 +291,32 @@ func etcdctlWatch(clus *etcdProcessCluster, key, value string, noSync bool) <-ch } func etcdctlRoleAdd(clus *etcdProcessCluster, role string) error { - cmdArgs := append(etcdctlPrefixArgs(clus, false), "role", "add", role) + cmdArgs := append(etcdctlPrefixArgs(clus), "role", "add", role) return spawnWithExpectedString(cmdArgs, role) } func etcdctlRoleList(clus *etcdProcessCluster, expectedRole string) error { - cmdArgs := append(etcdctlPrefixArgs(clus, false), "role", "list") + cmdArgs := append(etcdctlPrefixArgs(clus), "role", "list") return spawnWithExpectedString(cmdArgs, expectedRole) } func etcdctlUserAdd(clus *etcdProcessCluster, user, pass string) error { - cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "add", user+":"+pass) + cmdArgs := append(etcdctlPrefixArgs(clus), "user", "add", user+":"+pass) return spawnWithExpectedString(cmdArgs, "User "+user+" created") } func etcdctlUserGrant(clus *etcdProcessCluster, user, role string) error { - cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "grant", "--roles", role, user) + cmdArgs := append(etcdctlPrefixArgs(clus), "user", "grant", "--roles", role, user) return spawnWithExpectedString(cmdArgs, "User "+user+" updated") } func etcdctlUserGet(clus *etcdProcessCluster, user string) error { - cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "get", user) + cmdArgs := append(etcdctlPrefixArgs(clus), "user", "get", user) return spawnWithExpectedString(cmdArgs, "User: "+user) } func etcdctlUserList(clus *etcdProcessCluster, expectedUser string) error { - cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "list") + cmdArgs := append(etcdctlPrefixArgs(clus), "user", "list") return spawnWithExpectedString(cmdArgs, expectedUser) } @@ -356,3 +325,15 @@ func mustEtcdctl(t *testing.T) { t.Fatalf("could not find etcdctl binary") } } + +func setupEtcdctlTest(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) *etcdProcessCluster { + mustEtcdctl(t) + if !quorum { + cfg = configStandalone(*cfg) + } + epc, err := newEtcdProcessCluster(cfg) + if err != nil { + t.Fatalf("could not start etcd process cluster (%v)", err) + } + return epc +}