diff --git a/.github/workflows/robustness-nightly.yaml b/.github/workflows/robustness-nightly.yaml index 53ebf436f..9a9a99708 100644 --- a/.github/workflows/robustness-nightly.yaml +++ b/.github/workflows/robustness-nightly.yaml @@ -16,39 +16,44 @@ jobs: uses: ./.github/workflows/robustness-template.yaml with: etcdBranch: main - count: 80 + count: 150 testTimeout: 200m artifactName: main runs-on: "['ubuntu-latest-8-cores']" + scenario: TestRobustnessExploratory main-arm64: uses: ./.github/workflows/robustness-template.yaml with: etcdBranch: main - count: 80 + count: 150 testTimeout: 200m artifactName: main-arm64 runs-on: "['actuated-arm64-8cpu-8gb']" + scenario: TestRobustnessExploratory release-35: uses: ./.github/workflows/robustness-template.yaml with: etcdBranch: release-3.5 - count: 80 + count: 150 testTimeout: 200m artifactName: release-35 runs-on: "['ubuntu-latest-8-cores']" + scenario: TestRobustnessExploratory release-35-arm64: uses: ./.github/workflows/robustness-template.yaml with: etcdBranch: release-3.5 - count: 100 + count: 150 testTimeout: 200m artifactName: release-35-arm64 runs-on: "['actuated-arm64-8cpu-8gb']" + scenario: TestRobustnessExploratory release-34: uses: ./.github/workflows/robustness-template.yaml with: etcdBranch: release-3.4 - count: 80 + count: 150 testTimeout: 200m artifactName: release-34 runs-on: "['ubuntu-latest-8-cores']" + scenario: TestRobustnessExploratory diff --git a/.github/workflows/robustness-template.yaml b/.github/workflows/robustness-template.yaml index 2267cd788..d7d6ebe45 100644 --- a/.github/workflows/robustness-template.yaml +++ b/.github/workflows/robustness-template.yaml @@ -20,6 +20,9 @@ on: required: false type: string default: "['ubuntu-latest']" + scenario: + required: true + type: string permissions: read-all jobs: @@ -45,7 +48,7 @@ jobs: sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf make install-lazyfs # Use --failfast to avoid overriding report generated by failed test - GO_TEST_FLAGS="-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness" + GO_TEST_FLAGS="-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run ${{ inputs.scenario }}" case "${ETCD_BRANCH}" in release-3.5) EXPECT_DEBUG=true GO_TEST_FLAGS=${GO_TEST_FLAGS} RESULTS_DIR=/tmp/results make test-robustness-release-3.5 diff --git a/.github/workflows/robustness.yaml b/.github/workflows/robustness.yaml index b31a7c1c7..c137b3743 100644 --- a/.github/workflows/robustness.yaml +++ b/.github/workflows/robustness.yaml @@ -11,6 +11,7 @@ jobs: testTimeout: 30m artifactName: main runs-on: "['ubuntu-latest-8-cores']" + scenario: TestRobustness main-arm64: uses: ./.github/workflows/robustness-template.yaml with: @@ -19,3 +20,4 @@ jobs: testTimeout: 30m artifactName: main-arm64 runs-on: "['actuated-arm64-8cpu-8gb']" + scenario: TestRobustness diff --git a/tests/robustness/main_test.go b/tests/robustness/main_test.go index b674c3ed7..68a972681 100644 --- a/tests/robustness/main_test.go +++ b/tests/robustness/main_test.go @@ -39,9 +39,21 @@ func TestMain(m *testing.M) { testRunner.TestMain(m) } -func TestRobustness(t *testing.T) { +func TestRobustnessExploratory(t *testing.T) { testRunner.BeforeTest(t) - for _, scenario := range scenarios(t) { + for _, scenario := range exploratoryScenarios(t) { + t.Run(scenario.name, func(t *testing.T) { + lg := zaptest.NewLogger(t) + scenario.cluster.Logger = lg + ctx := context.Background() + testRobustness(ctx, t, lg, scenario) + }) + } +} + +func TestRobustnessRegression(t *testing.T) { + testRunner.BeforeTest(t) + for _, scenario := range regressionScenarios(t) { t.Run(scenario.name, func(t *testing.T) { lg := zaptest.NewLogger(t) scenario.cluster.Logger = lg diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk index 957b58f92..381db6301 100644 --- a/tests/robustness/makefile.mk +++ b/tests/robustness/makefile.mk @@ -12,22 +12,22 @@ test-robustness-release-3.4: /tmp/etcd-release-3.4-failpoints/bin .PHONY: test-robustness-issue14370 test-robustness-issue14370: /tmp/etcd-v3.5.4-failpoints/bin - GO_TEST_FLAGS='-v --run=TestRobustness/Issue14370 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.4-failpoints/bin' make test-robustness && \ + GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue14370 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.4-failpoints/bin' make test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue13766 test-robustness-issue13766: /tmp/etcd-v3.5.2-failpoints/bin - GO_TEST_FLAGS='-v --run=TestRobustness/Issue13766 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.2-failpoints/bin' make test-robustness && \ + GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue13766 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.2-failpoints/bin' make test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue14685 test-robustness-issue14685: /tmp/etcd-v3.5.5-failpoints/bin - GO_TEST_FLAGS='-v --run=TestRobustness/Issue14685 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.5-failpoints/bin' make test-robustness && \ + GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue14685 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.5-failpoints/bin' make test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue15271 test-robustness-issue15271: /tmp/etcd-v3.5.7-failpoints/bin - GO_TEST_FLAGS='-v --run=TestRobustness/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \ + GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" # Failpoints diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go index 8450bc1b1..01cd00c8b 100644 --- a/tests/robustness/scenarios.go +++ b/tests/robustness/scenarios.go @@ -59,7 +59,7 @@ type testScenario struct { watch watchConfig } -func scenarios(t *testing.T) []testScenario { +func exploratoryScenarios(t *testing.T) []testScenario { v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) if err != nil { t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err) @@ -112,13 +112,22 @@ func scenarios(t *testing.T) []testScenario { cluster: *e2e.NewConfig(clusterOfSize3Options...), }) } + return scenarios +} + +func regressionScenarios(t *testing.T) []testScenario { + v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) + if err != nil { + t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err) + } + + scenarios := []testScenario{} scenarios = append(scenarios, testScenario{ name: "Issue14370", failpoint: failpoint.RaftBeforeSavePanic, profile: traffic.LowTraffic, traffic: traffic.EtcdPutDeleteLease, cluster: *e2e.NewConfig( - options.WithSubsetOptions(randomizableOptions...), e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), @@ -129,7 +138,6 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.LowTraffic, traffic: traffic.EtcdPutDeleteLease, cluster: *e2e.NewConfig( - options.WithSubsetOptions(randomizableOptions...), e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), @@ -151,7 +159,6 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.LowTraffic, traffic: traffic.EtcdPutDeleteLease, cluster: *e2e.NewConfig( - options.WithSubsetOptions(randomizableOptions...), e2e.WithClusterSize(1), ), }) @@ -163,7 +170,6 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.HighTrafficProfile, traffic: traffic.EtcdPut, cluster: *e2e.NewConfig( - options.WithSubsetOptions(randomizableOptions...), e2e.WithSnapshotCatchUpEntries(100), e2e.WithSnapshotCount(100), e2e.WithPeerProxy(true),