Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix tests flakiness: in particular TestIssue6361.

Browse Source 
The root reason of flakes, was that server was considered as ready to
early.
In particular:
```
../../bin/etcd-2456648: {"level":"info","ts":"2021-01-11T09:56:44.474+0100","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"ed5f620d34a8e61b","remote-peer-id":"ca50e9357181d758"}
../../bin/etcd-2456648: {"level":"warn","ts":"2021-01-11T09:56:49.040+0100","caller":"etcdserver/server.go:1942","msg":"failed to publish local member to cluster through raft","local-member-id":"ed5f620d34a8e61b","local-member-attributes":"{Name:infra2 ClientURLs:[http://localhost:20030]}","request-path":"/0/members/ed5f620d34a8e61b/attributes","publish-timeout":"7s","error":"etcdserver: request timed out, possibly due to connection lost"}
../../bin/etcd-2456648: {"level":"info","ts":"2021-01-11T09:56:49.049+0100","caller":"etcdserver/server.go:1921","msg":"published local member to cluster through raft","local-member-id":"ed5f620d34a8e61b","local-member-attributes":"{Name:infra2 ClientURLs:[http://localhost:20030]}","request-path":"/0/members/ed5f620d34a8e61b/attributes","cluster-id":"34f27e83b3bc2ff","publish-timeout":"7s"}
```
was taking 5s.   If this was happening concurrently with etcdctl, the
etcdctl could timeout.

The fix, requires servers to report 'ready to serve client requests' to consider them up.

Fixed also some whitelisted 'goroutines'.
This commit is contained in:
Piotr Tabor 2021-01-11 12:01:22 +01:00
parent 74274f4417
commit 70b5ef1d3a
6 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/testutil/leak.go
View File

@ -111,10 +111,12 @@ func interestingGoroutines() (gs []string) {
if stack == "" ||
strings.Contains(stack, "sync.(*WaitGroup).Done") ||
strings.Contains(stack, "os.(*file).close") ||
strings.Contains(stack, "os.(*Process).Release") ||
strings.Contains(stack, "created by os/signal.init") ||
strings.Contains(stack, "runtime/panic.go") ||
strings.Contains(stack, "created by testing.RunTests") ||
strings.Contains(stack, "created by testing.runTests") ||
strings.Contains(stack, "created by testing.(*T).Run") ||
strings.Contains(stack, "testing.Main(") ||
strings.Contains(stack, "runtime.goexit") ||
strings.Contains(stack, "go.etcd.io/etcd/pkg/v3/testutil.interestingGoroutines") ||

2
tests/e2e/ctl_v3_auth_test.go
View File

@ -25,7 +25,7 @@ import (
"go.etcd.io/etcd/client/v3"
)
func TestCtlV3AuthEnable(t *testing.T) {
func TestCtlV3AuthEnable(t *testing.T) {
testCtl(t, authEnableTest)
}
func TestCtlV3AuthDisable(t *testing.T) { testCtl(t, authDisableTest) }

6
tests/e2e/ctl_v3_snapshot_test.go
View File

@ -223,7 +223,7 @@ func TestIssue6361(t *testing.T) {
t.Fatal(err)
}
t.Log("Start the etcd member using the restored snapshot...")
t.Log("(Re)starting the etcd member using the restored snapshot...")
epc.procs[0].Config().dataDirPath = newDataDir
for i := range epc.procs[0].Config().args {
if epc.procs[0].Config().args[i] == "--data-dir" {
@ -255,7 +255,7 @@ func TestIssue6361(t *testing.T) {
name2 := "infra2"
initialCluster2 := epc.procs[0].Config().initialCluster + fmt.Sprintf(",%s=%s", name2, peerURL)
t.Log("Starting new member")
t.Log("Starting the new member")
// start the new member
var nepc *expect.ExpectProcess
nepc, err = spawnCmd([]string{epc.procs[0].Config().execPath, "--name", name2,
@ -265,7 +265,7 @@ func TestIssue6361(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err = nepc.Expect("enabled capabilities for version"); err != nil {
if _, err = nepc.Expect("ready to serve client requests"); err != nil {
t.Fatal(err)
}

4
tests/e2e/ctl_v3_test.go
View File

@ -88,10 +88,10 @@ func versionTest(cx ctlCtx) {
func clusterVersionTest(cx ctlCtx, expected string) {
var err error
for i := 0; i < 7; i++ {
for i := 0; i < 35; i++ {
if err = cURLGet(cx.epc, cURLReq{endpoint: "/version", expected: expected}); err != nil {
cx.t.Logf("#%d: v3 is not ready yet (%v)", i, err)
time.Sleep(time.Second)
time.Sleep(200 * time.Millisecond)
continue
}
break

2
tests/e2e/etcd_process.go
View File

@ -24,7 +24,7 @@ import (
)
var (
etcdServerReadyLines = []string{"enabled capabilities for version", "published"}
etcdServerReadyLines = []string{"enabled capabilities for version", "published", "ready to serve client requests"}
binPath string
ctlBinPath string
)

2
tests/e2e/v3_curl_test.go
View File

@ -246,7 +246,7 @@ func testV3CurlAuth(cx ctlCtx) {
cmdArgs = cURLPrefixArgs(cx.epc, "POST", cURLReq{endpoint: path.Join(p, "/auth/authenticate"), value: string(authreq)})
proc, err := spawnCmd(cmdArgs)
testutil.AssertNil(cx.t, err)
defer proc.Close()
defer proc.Close()
cURLRes, err := proc.ExpectFunc(lineFunc)
testutil.AssertNil(cx.t, err)