From 59d232adf903b449973ba33ad6f182555af75044 Mon Sep 17 00:00:00 2001 From: Anthony Romano Date: Thu, 31 Aug 2017 23:36:12 -0700 Subject: [PATCH] integration: retry remove in TestDecreaseClusterSize Cluster may go through a second leader election if test machine is overloaded. Retry remove until it passes without error. Fixes #8225 --- integration/cluster.go | 5 +++-- integration/cluster_test.go | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/integration/cluster.go b/integration/cluster.go index 7dcfdcdd4..01a2aa6ad 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -303,10 +303,11 @@ func (c *cluster) removeMember(t *testing.T, id uint64) error { cc := MustNewHTTPClient(t, c.URLs(), c.cfg.ClientTLS) ma := client.NewMembersAPI(cc) ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - if err := ma.Remove(ctx, types.ID(id).String()); err != nil { + err := ma.Remove(ctx, types.ID(id).String()) + cancel() + if err != nil { return err } - cancel() newMembers := make([]*member, 0) for _, m := range c.Members { if uint64(m.s.ID()) != id { diff --git a/integration/cluster_test.go b/integration/cluster_test.go index 5907d6841..14549dac0 100644 --- a/integration/cluster_test.go +++ b/integration/cluster_test.go @@ -151,7 +151,15 @@ func testDecreaseClusterSize(t *testing.T, size int) { // TODO: remove the last but one member for i := 0; i < size-1; i++ { id := c.Members[len(c.Members)-1].s.ID() - c.RemoveMember(t, uint64(id)) + // may hit second leader election on slow machines + if err := c.removeMember(t, uint64(id)); err != nil { + if strings.Contains(err.Error(), "no leader") { + t.Logf("got leader error (%v)", err) + i-- + continue + } + t.Fatal(err) + } c.waitLeader(t, c.Members) } clusterMustProgress(t, c.Members)