From 834fac9fb2c3e5881bb0925f474f2afe7232b28c Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Mon, 13 Nov 2023 10:43:50 -0800 Subject: [PATCH] robustness test: add with functions of randomizable config params in robustness test Signed-off-by: Siyuan Zhang --- tests/robustness/options/cluster_options.go | 50 ++++++++ .../options/cluster_options_test.go | 108 ++++++++++++++++++ .../options/server_config_options.go | 51 +++++++++ tests/robustness/scenarios.go | 15 ++- 4 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 tests/robustness/options/cluster_options.go create mode 100644 tests/robustness/options/cluster_options_test.go create mode 100644 tests/robustness/options/server_config_options.go diff --git a/tests/robustness/options/cluster_options.go b/tests/robustness/options/cluster_options.go new file mode 100644 index 000000000..01031e218 --- /dev/null +++ b/tests/robustness/options/cluster_options.go @@ -0,0 +1,50 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "math/rand" + "time" + + "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +var internalRand = rand.New(rand.NewSource(time.Now().UnixNano())) + +type ClusterOptions []e2e.EPClusterOption + +// WithClusterOptionGroups takes an array of EPClusterOption arrays, and randomly picks one EPClusterOption array when constructing the config. +// This function is mainly used to group strongly coupled config options together, so that we can dynamically test different groups of options. +func WithClusterOptionGroups(input ...ClusterOptions) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + optsPicked := input[internalRand.Intn(len(input))] + for _, opt := range optsPicked { + opt(c) + } + } +} + +// WithSubsetOptions randomly select a subset of input options, and apply the subset to the cluster config. +func WithSubsetOptions(input ...e2e.EPClusterOption) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + // selects random subsetLen (0 to len(input)) elements from the input array. + subsetLen := internalRand.Intn(len(input) + 1) + perm := internalRand.Perm(len(input)) + for i := 0; i < subsetLen; i++ { + opt := input[perm[i]] + opt(c) + } + } +} diff --git a/tests/robustness/options/cluster_options_test.go b/tests/robustness/options/cluster_options_test.go new file mode 100644 index 000000000..6c58c04ff --- /dev/null +++ b/tests/robustness/options/cluster_options_test.go @@ -0,0 +1,108 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "math/rand" + "testing" + + "go.etcd.io/etcd/server/v3/embed" + "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +func resetRand() { + internalRand = rand.New(rand.NewSource(1)) +} + +func init() { + resetRand() +} + +func TestWithClusterOptionGroups(t *testing.T) { + defer resetRand() + tickOptions1 := ClusterOptions{WithTickMs(101), WithElectionMs(1001)} + tickOptions2 := ClusterOptions{WithTickMs(202), WithElectionMs(2002)} + tickOptions3 := ClusterOptions{WithTickMs(303), WithElectionMs(3003)} + opts := ClusterOptions{ + WithSnapshotCount(100, 150, 200), + WithClusterOptionGroups(tickOptions1, tickOptions2, tickOptions3), + WithSnapshotCatchUpEntries(100), + } + + expectedServerConfigs := []embed.Config{ + embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001}, + embed.Config{SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002}, + embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002}, + embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001}, + embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001}, + embed.Config{SnapshotCount: 150, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002}, + } + for i, tt := range expectedServerConfigs { + cluster := *e2e.NewConfig(opts...) + if cluster.ServerConfig.SnapshotCount != tt.SnapshotCount { + t.Errorf("Test case %d: SnapshotCount = %v, want %v\n", i, cluster.ServerConfig.SnapshotCount, tt.SnapshotCount) + } + if cluster.ServerConfig.SnapshotCatchUpEntries != tt.SnapshotCatchUpEntries { + t.Errorf("Test case %d: SnapshotCatchUpEntries = %v, want %v\n", i, cluster.ServerConfig.SnapshotCatchUpEntries, tt.SnapshotCatchUpEntries) + } + if cluster.ServerConfig.TickMs != tt.TickMs { + t.Errorf("Test case %d: TickMs = %v, want %v\n", i, cluster.ServerConfig.TickMs, tt.TickMs) + } + if cluster.ServerConfig.ElectionMs != tt.ElectionMs { + t.Errorf("Test case %d: ElectionMs = %v, want %v\n", i, cluster.ServerConfig.ElectionMs, tt.ElectionMs) + } + } +} + +func TestWithOptionsSubset(t *testing.T) { + defer resetRand() + tickOptions := ClusterOptions{WithTickMs(50), WithElectionMs(500)} + opts := ClusterOptions{ + WithSnapshotCatchUpEntries(100), + WithSubsetOptions(WithSnapshotCount(100, 150, 200), WithClusterOptionGroups(tickOptions)), + } + + expectedServerConfigs := []embed.Config{ + embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, + embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, + embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, + // both SnapshotCount and TickMs&ElectionMs are not default values. + embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, + embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, + // only TickMs&ElectionMs are not default values. + embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, + // both SnapshotCount and TickMs&ElectionMs are not default values. + embed.Config{SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, + // both SnapshotCount and TickMs&ElectionMs are not default values. + embed.Config{SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, + // only SnapshotCount is not default value. + embed.Config{SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, + } + for i, tt := range expectedServerConfigs { + cluster := *e2e.NewConfig(opts...) + if cluster.ServerConfig.SnapshotCount != tt.SnapshotCount { + t.Errorf("Test case %d: SnapshotCount = %v, want %v\n", i, cluster.ServerConfig.SnapshotCount, tt.SnapshotCount) + } + if cluster.ServerConfig.SnapshotCatchUpEntries != tt.SnapshotCatchUpEntries { + t.Errorf("Test case %d: SnapshotCatchUpEntries = %v, want %v\n", i, cluster.ServerConfig.SnapshotCatchUpEntries, tt.SnapshotCatchUpEntries) + } + if cluster.ServerConfig.TickMs != tt.TickMs { + t.Errorf("Test case %d: TickMs = %v, want %v\n", i, cluster.ServerConfig.TickMs, tt.TickMs) + } + if cluster.ServerConfig.ElectionMs != tt.ElectionMs { + t.Errorf("Test case %d: ElectionMs = %v, want %v\n", i, cluster.ServerConfig.ElectionMs, tt.ElectionMs) + } + } +} diff --git a/tests/robustness/options/server_config_options.go b/tests/robustness/options/server_config_options.go new file mode 100644 index 000000000..5018d1bf9 --- /dev/null +++ b/tests/robustness/options/server_config_options.go @@ -0,0 +1,51 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "time" + + e2e "go.etcd.io/etcd/tests/v3/framework/e2e" +) + +func WithSnapshotCount(input ...uint64) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + c.ServerConfig.SnapshotCount = input[internalRand.Intn(len(input))] + } +} + +func WithSnapshotCatchUpEntries(input ...uint64) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + c.ServerConfig.SnapshotCatchUpEntries = input[internalRand.Intn(len(input))] + } +} + +func WithTickMs(input ...uint) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + c.ServerConfig.TickMs = input[internalRand.Intn(len(input))] + } +} + +func WithElectionMs(input ...uint) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + c.ServerConfig.ElectionMs = input[internalRand.Intn(len(input))] + } +} + +func WithExperimentalWatchProgressNotifyInterval(input ...time.Duration) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { + c.ServerConfig.ExperimentalWatchProgressNotifyInterval = input[internalRand.Intn(len(input))] + } +} diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go index 7ada046f9..8450bc1b1 100644 --- a/tests/robustness/scenarios.go +++ b/tests/robustness/scenarios.go @@ -22,6 +22,7 @@ import ( "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/failpoint" + "go.etcd.io/etcd/tests/v3/robustness/options" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) @@ -64,8 +65,16 @@ func scenarios(t *testing.T) []testScenario { t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err) } enableLazyFS := e2e.BinPath.LazyFSAvailable() + randomizableOptions := []e2e.EPClusterOption{ + options.WithClusterOptionGroups( + options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)}, + options.ClusterOptions{options.WithTickMs(101), options.WithElectionMs(521)}, + options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}), + } + baseOptions := []e2e.EPClusterOption{ - e2e.WithSnapshotCount(100), + options.WithSnapshotCount(50, 100, 1000), + options.WithSubsetOptions(randomizableOptions...), e2e.WithGoFailEnabled(true), e2e.WithCompactionBatchLimit(100), e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), @@ -109,6 +118,7 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.LowTraffic, traffic: traffic.EtcdPutDeleteLease, cluster: *e2e.NewConfig( + options.WithSubsetOptions(randomizableOptions...), e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), @@ -119,6 +129,7 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.LowTraffic, traffic: traffic.EtcdPutDeleteLease, cluster: *e2e.NewConfig( + options.WithSubsetOptions(randomizableOptions...), e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), @@ -140,6 +151,7 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.LowTraffic, traffic: traffic.EtcdPutDeleteLease, cluster: *e2e.NewConfig( + options.WithSubsetOptions(randomizableOptions...), e2e.WithClusterSize(1), ), }) @@ -151,6 +163,7 @@ func scenarios(t *testing.T) []testScenario { profile: traffic.HighTrafficProfile, traffic: traffic.EtcdPut, cluster: *e2e.NewConfig( + options.WithSubsetOptions(randomizableOptions...), e2e.WithSnapshotCatchUpEntries(100), e2e.WithSnapshotCount(100), e2e.WithPeerProxy(true),