Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14611 from serathius/issue14370

Browse Source 
tests: Add linearizability tests scenario for #14370
This commit is contained in:
Marek Siarkowicz 2022-10-25 14:03:39 +02:00 committed by GitHub
commit 8ce81a1624
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 101 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/linearizability.yaml vendored
View File

@ -9,7 +9,10 @@ jobs:
with:
go-version: "1.19.1"
- run: |
pushd tools/mod; go install go.etcd.io/gofail; popd
mkdir -p /tmp/linearizability
FAILPOINTS=true make build
cat server/etcdserver/raft.fail.go
EXPECT_DEBUG=true GO_TEST_FLAGS=-v RESULTS_DIR=/tmp/linearizability make test-linearizability
- uses: actions/upload-artifact@v2
if: always()

4
Makefile
View File

@ -30,8 +30,8 @@ test-e2e-release: build
PASSES="release e2e" ./scripts/test.sh $(GO_TEST_FLAGS)
.PHONY: test-linearizability
test-linearizability: build
PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS)
test-linearizability:
FAILPOINTS=true PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS)
.PHONY: fuzz
fuzz:

16
scripts/build_lib.sh
View File

@ -19,6 +19,22 @@ toggle_failpoints() {
mode="$1"
if command -v gofail >/dev/null 2>&1; then
run gofail "$mode" server/etcdserver/ server/storage/backend/
(
cd ./server
run go get go.etcd.io/gofail/runtime
) || exit 2
(
cd ./etcdutl
run go get go.etcd.io/gofail/runtime
) || exit 2
(
cd ./etcdctl
run go get go.etcd.io/gofail/runtime
) || exit 2
(
cd ./tests
run go get go.etcd.io/gofail/runtime
) || exit 2
elif [[ "$mode" != "disable" ]]; then
log_error "FAILPOINTS set but gofail not found"
exit 1

13
tests/framework/e2e/cluster.go
View File

@ -182,6 +182,7 @@ type EtcdProcessClusterConfig struct {
CorruptCheckTime time.Duration
CompactHashCheckEnabled bool
CompactHashCheckTime time.Duration
GoFailEnabled bool
}
// NewEtcdProcessCluster launches a new cluster from etcd processes, returning
@ -388,12 +389,21 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
if cfg.CompactHashCheckTime != 0 {
args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String())
}
envVars := map[string]string{}
for key, value := range cfg.EnvVars {
envVars[key] = value
}
var gofailPort int
if cfg.GoFailEnabled {
gofailPort = (i+1)*10000 + 2381
envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort)
}
return &EtcdServerProcessConfig{
lg: cfg.Logger,
ExecPath: cfg.ExecPath,
Args: args,
EnvVars: cfg.EnvVars,
EnvVars: envVars,
TlsArgs: cfg.TlsArgs(),
DataDirPath: dataDirPath,
KeepDataDir: cfg.KeepDataDir,
@ -402,6 +412,7 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
Acurl: curl,
Murl: murl,
InitialToken: cfg.InitialToken,
GoFailPort: gofailPort,
}
}

1
tests/framework/e2e/etcd_process.go
View File

@ -80,6 +80,7 @@ type EtcdServerProcessConfig struct {
InitialToken string
InitialCluster string
GoFailPort int
}
func NewEtcdServerProcess(cfg *EtcdServerProcessConfig) (*EtcdServerProcess, error) {

56
tests/linearizability/failpoints.go
View File

@ -15,14 +15,20 @@
package linearizability
import (
"bytes"
"context"
"fmt"
"math/rand"
"net/http"
"net/url"
"time"
"go.etcd.io/etcd/tests/v3/framework/e2e"
)
var (
KillFailpoint Failpoint = killFailpoint{}
KillFailpoint Failpoint = killFailpoint{}
RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic"}
)
type Failpoint interface {
@ -47,3 +53,51 @@ func (f killFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster
}
return nil
}
type goFailpoint struct {
failpoint string
payload string
}
func (f goFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error {
member := clus.Procs[rand.Int()%len(clus.Procs)]
address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
err := triggerGoFailpoint(address, f.failpoint, f.payload)
if err != nil {
return fmt.Errorf("failed to trigger failpoint %q, err: %v", f.failpoint, err)
}
err = clus.Procs[0].Wait()
if err != nil {
return err
}
err = clus.Procs[0].Start(ctx)
if err != nil {
return err
}
return nil
}
func triggerGoFailpoint(host, failpoint, payload string) error {
failpointUrl := url.URL{
Scheme: "http",
Host: host,
Path: failpoint,
}
r, err := http.NewRequest("PUT", failpointUrl.String(), bytes.NewBuffer([]byte(payload)))
if err != nil {
return err
}
resp, err := httpClient.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
return fmt.Errorf("bad status code: %d", resp.StatusCode)
}
return nil
}
var httpClient = http.Client{
Timeout: 10 * time.Millisecond,
}

8
tests/linearizability/linearizability_test.go
View File

@ -60,6 +60,14 @@ func TestLinearizability(t *testing.T) {
ClusterSize: 3,
},
},
{
name: "Issue14370",
failpoint: RaftBeforeSavePanic,
config: e2e.EtcdProcessClusterConfig{
ClusterSize: 1,
GoFailEnabled: true,
},
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {

1
tools/mod/go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f
github.com/mgechev/revive v1.2.1
github.com/mikefarah/yq/v4 v4.24.2
go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0
go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
gotest.tools/gotestsum v1.7.0
gotest.tools/v3 v3.1.0

2
tools/mod/go.sum
View File

@ -238,6 +238,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964=
go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4=
go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ=
go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw=
go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs=

1
tools/mod/tools.go
View File

@ -34,6 +34,7 @@ import (
_ "github.com/mdempsky/unconvert"
_ "github.com/mgechev/revive"
_ "github.com/mikefarah/yq/v4"
_ "go.etcd.io/gofail"
_ "go.etcd.io/protodoc"
_ "gotest.tools/gotestsum"
_ "gotest.tools/v3"