Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

etcd-tester: add '-failpoints' to configure gofail

Browse Source 
Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
This commit is contained in:
Gyu-Ho Lee 2017-05-18 09:44:13 -07:00
parent e7d705b25f
commit 62b44a85f8
2 changed files with 38 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
tools/functional-tester/etcd-tester/failpoint.go
View File

@ -32,7 +32,7 @@ type failpointStats struct {
var fpStats failpointStats
func failpointFailures(c *cluster) (ret []failure, err error) {
func failpointFailures(c *cluster, failpoints []string) (ret []failure, err error) {
var fps []string
fps, err = failpointPaths(c.Members[0].FailpointURL)
if err != nil {
@ -43,7 +43,7 @@ func failpointFailures(c *cluster) (ret []failure, err error) {
if len(fp) == 0 {
continue
}
fpFails := failuresFromFailpoint(fp)
fpFails := failuresFromFailpoint(fp, failpoints)
// wrap in delays so failpoint has time to trigger
for i, fpf := range fpFails {
if strings.Contains(fp, "Snap") {
@ -77,34 +77,39 @@ func failpointPaths(endpoint string) ([]string, error) {
return fps, nil
}
func failuresFromFailpoint(fp string) []failure {
inject := makeInjectFailpoint(fp, `panic("etcd-tester")`)
// failpoints follows FreeBSD KFAIL_POINT syntax.
// e.g. panic("etcd-tester"),1*sleep(1000)->panic("etcd-tester")
func failuresFromFailpoint(fp string, failpoints []string) (fs []failure) {
recov := makeRecoverFailpoint(fp)
return []failure{
&failureOne{
description: description("failpoint " + fp + " panic one"),
injectMember: inject,
recoverMember: recov,
},
&failureAll{
description: description("failpoint " + fp + " panic all"),
injectMember: inject,
recoverMember: recov,
},
&failureMajority{
description: description("failpoint " + fp + " panic majority"),
injectMember: inject,
recoverMember: recov,
},
&failureLeader{
failureByFunc{
description: description("failpoint " + fp + " panic leader"),
for _, failpoint := range failpoints {
inject := makeInjectFailpoint(fp, failpoint)
fs = append(fs, []failure{
&failureOne{
description: description(fmt.Sprintf("failpoint %s (one: %s)", fp, failpoint)),
injectMember: inject,
recoverMember: recov,
},
0,
},
&failureAll{
description: description(fmt.Sprintf("failpoint %s (all: %s)", fp, failpoint)),
injectMember: inject,
recoverMember: recov,
},
&failureMajority{
description: description(fmt.Sprintf("failpoint %s (majority: %s)", fp, failpoint)),
injectMember: inject,
recoverMember: recov,
},
&failureLeader{
failureByFunc{
description: description(fmt.Sprintf("failpoint %s (leader: %s)", fp, failpoint)),
injectMember: inject,
recoverMember: recov,
},
0,
},
}...)
}
return fs
}
func makeInjectFailpoint(fp, val string) injectMemberFunc {

16
tools/functional-tester/etcd-tester/main.go
View File

@ -54,6 +54,7 @@ func main() {
stresserType := flag.String("stresser", "keys,lease", "comma separated list of stressers (keys, lease, v2keys, nop, election-runner, watch-runner, lock-racer-runner, lease-runner).")
etcdRunnerPath := flag.String("etcd-runner", "", "specify a path of etcd runner binary")
failureTypes := flag.String("failures", "default,failpoints", "specify failures (concat of \"default\" and \"failpoints\").")
failpoints := flag.String("failpoints", `panic("etcd-tester")`, `comma separated list of failpoint terms to inject (e.g. 'panic("etcd-tester"),1*sleep(1000)')`)
externalFailures := flag.String("external-failures", "", "specify a path of script for enabling/disabling an external fault injector")
enablePprof := flag.Bool("enable-pprof", false, "true to enable pprof")
flag.Parse()
@ -83,7 +84,8 @@ func main() {
var failures []failure
if failureTypes != nil && *failureTypes != "" {
failures = makeFailures(*failureTypes, c)
types, failpoints := strings.Split(*failureTypes, ","), strings.Split(*failpoints, ",")
failures = makeFailures(types, failpoints, c)
}
if externalFailures != nil && *externalFailures != "" {
@ -172,12 +174,10 @@ func portsFromArg(arg string, n, defaultPort int) []int {
return ret
}
func makeFailures(types string, c *cluster) []failure {
func makeFailures(types, failpoints []string, c *cluster) []failure {
var failures []failure
fails := strings.Split(types, ",")
for i := range fails {
switch fails[i] {
for i := range types {
switch types[i] {
case "default":
defaultFailures := []failure{
newFailureKillAll(),
@ -195,14 +195,14 @@ func makeFailures(types string, c *cluster) []failure {
failures = append(failures, defaultFailures...)
case "failpoints":
fpFailures, fperr := failpointFailures(c)
fpFailures, fperr := failpointFailures(c, failpoints)
if len(fpFailures) == 0 {
plog.Infof("no failpoints found (%v)", fperr)
}
failures = append(failures, fpFailures...)
default:
plog.Errorf("unknown failure: %s\n", fails[i])
plog.Errorf("unknown failure: %s\n", types[i])
os.Exit(1)
}
}