From 11711fc00732a1b29c65b232f3836c2dd2abc0c3 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Fri, 2 Dec 2022 15:37:25 +0100 Subject: [PATCH 1/4] tests: Recreate cluster for every failpoint trigger This shortens operation history and avoids having to many failed requests. Failed requests are problematic as too many of them can cause linearizability verification complexity to become exponential. Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 9749c5b8a..51ebfb261 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -74,20 +74,22 @@ func TestLinearizability(t *testing.T) { }, } for _, tc := range tcs { - t.Run(tc.name, func(t *testing.T) { - failpoint := FailpointConfig{ - failpoint: tc.failpoint, - count: failpointTriggersCount, - waitBetweenTriggers: waitBetweenFailpointTriggers, - } - traffic := trafficConfig{ - minimalQPS: minimalQPS, - maximalQPS: maximalQPS, - clientCount: 8, - traffic: DefaultTraffic, - } - testLinearizability(context.Background(), t, tc.config, failpoint, traffic) - }) + for i := 0; i < failpointTriggersCount; i++ { + t.Run(tc.name, func(t *testing.T) { + failpoint := FailpointConfig{ + failpoint: tc.failpoint, + count: 1, + waitBetweenTriggers: waitBetweenFailpointTriggers, + } + traffic := trafficConfig{ + minimalQPS: minimalQPS, + maximalQPS: maximalQPS, + clientCount: 8, + traffic: DefaultTraffic, + } + testLinearizability(context.Background(), t, tc.config, failpoint, traffic) + }) + } } } From df2d075e1aee3c11e4564742cb89197ba5f946c9 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Fri, 2 Dec 2022 00:41:47 +0100 Subject: [PATCH 2/4] tests: Sleep between failpoint failure Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 51ebfb261..1ef4ef91e 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -119,8 +119,8 @@ func triggerFailpoints(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessC var err error successes := 0 failures := 0 - time.Sleep(config.waitBetweenTriggers) for successes < config.count && failures < config.count { + time.Sleep(config.waitBetweenTriggers) err = config.failpoint.Trigger(t, ctx, clus) if err != nil { t.Logf("Failed to trigger failpoint %q, err: %v\n", config.failpoint.Name(), err) @@ -128,8 +128,8 @@ func triggerFailpoints(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessC continue } successes++ - time.Sleep(config.waitBetweenTriggers) } + time.Sleep(config.waitBetweenTriggers) if successes < config.count || failures >= config.count { return fmt.Errorf("failed to trigger failpoints enough times, err: %v", err) } From 7f70b8cf760c11eb43103ae1696acb142eeeb58f Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Fri, 2 Dec 2022 16:05:56 +0100 Subject: [PATCH 3/4] tests: Separate retries count from failure trigger count Signed-off-by: Marek Siarkowicz --- tests/linearizability/linearizability_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index 1ef4ef91e..ff3ef3479 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -79,6 +79,7 @@ func TestLinearizability(t *testing.T) { failpoint := FailpointConfig{ failpoint: tc.failpoint, count: 1, + retries: 3, waitBetweenTriggers: waitBetweenFailpointTriggers, } traffic := trafficConfig{ @@ -119,7 +120,7 @@ func triggerFailpoints(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessC var err error successes := 0 failures := 0 - for successes < config.count && failures < config.count { + for successes < config.count && failures < config.retries { time.Sleep(config.waitBetweenTriggers) err = config.failpoint.Trigger(t, ctx, clus) if err != nil { @@ -129,16 +130,17 @@ func triggerFailpoints(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessC } successes++ } - time.Sleep(config.waitBetweenTriggers) - if successes < config.count || failures >= config.count { + if successes < config.count || failures >= config.retries { return fmt.Errorf("failed to trigger failpoints enough times, err: %v", err) } + time.Sleep(config.waitBetweenTriggers) return nil } type FailpointConfig struct { failpoint Failpoint count int + retries int waitBetweenTriggers time.Duration } From a573d8af697fd073d3a97262667e912736546175 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sat, 3 Dec 2022 11:10:08 +0100 Subject: [PATCH 4/4] tests: Use golang count to repeat tests Signed-off-by: Marek Siarkowicz --- .github/workflows/linearizability.yaml | 2 +- tests/linearizability/linearizability_test.go | 34 ++++++++----------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/linearizability.yaml b/.github/workflows/linearizability.yaml index 7303a3509..4938d6a00 100644 --- a/.github/workflows/linearizability.yaml +++ b/.github/workflows/linearizability.yaml @@ -13,7 +13,7 @@ jobs: make build mkdir -p /tmp/linearizability cat server/etcdserver/raft.fail.go - EXPECT_DEBUG=true GO_TEST_FLAGS=-v RESULTS_DIR=/tmp/linearizability make test-linearizability + EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability - uses: actions/upload-artifact@v2 if: always() with: diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index ff3ef3479..70ac4cb4a 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -34,8 +34,6 @@ const ( minimalQPS = 100.0 // maximalQPS limits number of requests send to etcd to avoid linearizability analysis taking too long. maximalQPS = 200.0 - // failpointTriggersCount - failpointTriggersCount = 60 // waitBetweenFailpointTriggers waitBetweenFailpointTriggers = time.Second ) @@ -74,23 +72,21 @@ func TestLinearizability(t *testing.T) { }, } for _, tc := range tcs { - for i := 0; i < failpointTriggersCount; i++ { - t.Run(tc.name, func(t *testing.T) { - failpoint := FailpointConfig{ - failpoint: tc.failpoint, - count: 1, - retries: 3, - waitBetweenTriggers: waitBetweenFailpointTriggers, - } - traffic := trafficConfig{ - minimalQPS: minimalQPS, - maximalQPS: maximalQPS, - clientCount: 8, - traffic: DefaultTraffic, - } - testLinearizability(context.Background(), t, tc.config, failpoint, traffic) - }) - } + t.Run(tc.name, func(t *testing.T) { + failpoint := FailpointConfig{ + failpoint: tc.failpoint, + count: 1, + retries: 3, + waitBetweenTriggers: waitBetweenFailpointTriggers, + } + traffic := trafficConfig{ + minimalQPS: minimalQPS, + maximalQPS: maximalQPS, + clientCount: 8, + traffic: DefaultTraffic, + } + testLinearizability(context.Background(), t, tc.config, failpoint, traffic) + }) } }