Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

tests: Allow failpoint requests to fail assuming that process exists within 1 second

Browse Source 
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz 2022-12-05 13:52:30 +01:00
parent a5cfc089fa
commit 738ee3687a
3 changed files with 73 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/framework/e2e/cluster_proxy.go
View File

@ -116,8 +116,12 @@ func (p *proxyEtcdProcess) Kill() error {
return p.etcdProc.Kill()
}
func (p *proxyEtcdProcess) Wait() error {
return p.etcdProc.Wait()
func (p *proxyEtcdProcess) IsRunning() bool {
return p.etcdProc.IsRunning()
}
func (p *proxyEtcdProcess) Wait(ctx context.Context) error {
return p.etcdProc.Wait(ctx)
}
type proxyProc struct {

33
tests/framework/e2e/etcd_process.go
View File

@ -42,7 +42,8 @@ type EtcdProcess interface {
EndpointsMetrics() []string
Client(opts ...config.ClientOption) *EtcdctlV3
Wait() error
IsRunning() bool
Wait(ctx context.Context) error
Start(ctx context.Context) error
Restart(ctx context.Context) error
Stop() error
@ -201,11 +202,35 @@ func (ep *EtcdServerProcess) Kill() error {
return ep.proc.Signal(syscall.SIGKILL)
}
func (ep *EtcdServerProcess) Wait() error {
ep.proc.Wait()
func (ep *EtcdServerProcess) Wait(ctx context.Context) error {
ch := make(chan struct{})
go func() {
defer close(ch)
if ep.proc != nil {
ep.proc.Wait()
ep.cfg.lg.Info("server exited", zap.String("name", ep.cfg.Name))
}
}()
select {
case <-ch:
ep.proc = nil
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (ep *EtcdServerProcess) IsRunning() bool {
if ep.proc == nil {
return false
}
_, err := ep.proc.ExitCode()
if err == expect.ErrProcessRunning {
return true
}
ep.cfg.lg.Info("server exited", zap.String("name", ep.cfg.Name))
ep.proc = nil
return nil
return false
}
func AssertProcessLogs(t *testing.T, ep EtcdProcess, expectLog string) {

60
tests/linearizability/failpoints.go
View File

@ -31,6 +31,10 @@ import (
"go.etcd.io/etcd/tests/v3/framework/e2e"
)
const (
triggerTimeout = time.Second
)
var (
KillFailpoint Failpoint = killFailpoint{}
DefragBeforeCopyPanic Failpoint = goFailpoint{"backend/defragBeforeCopy", "panic", triggerDefrag, AnyMember}
@ -81,15 +85,21 @@ type killFailpoint struct{}
func (f killFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdProcessCluster) error {
member := clus.Procs[rand.Int()%len(clus.Procs)]
err := member.Kill()
if err != nil {
return err
killCtx, cancel := context.WithTimeout(ctx, triggerTimeout)
defer cancel()
for member.IsRunning() {
err := member.Kill()
if err != nil {
t.Logf("sending kill signal failed: %v", err)
}
err = member.Wait(killCtx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return fmt.Errorf("failed to kill the process within %s, err: %w", triggerTimeout, err)
}
}
err = member.Wait()
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return err
}
err = member.Start(ctx)
err := member.Start(ctx)
if err != nil {
return err
}
@ -118,21 +128,27 @@ func (f goFailpoint) Trigger(t *testing.T, ctx context.Context, clus *e2e.EtcdPr
member := f.pickMember(t, clus)
address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
err := setupGoFailpoint(address, f.failpoint, f.payload)
if err != nil {
return fmt.Errorf("gofailpoint setup failed: %w", err)
}
if f.trigger != nil {
err = f.trigger(ctx, member)
triggerCtx, cancel := context.WithTimeout(ctx, triggerTimeout)
defer cancel()
for member.IsRunning() {
err := setupGoFailpoint(triggerCtx, address, f.failpoint, f.payload)
if err != nil {
return fmt.Errorf("triggering gofailpoint failed: %w", err)
t.Logf("gofailpoint setup failed: %v", err)
}
if f.trigger != nil {
err = f.trigger(triggerCtx, member)
if err != nil {
t.Logf("triggering gofailpoint failed: %v", err)
}
}
err = member.Wait(triggerCtx)
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return fmt.Errorf("failed to trigger a process panic within %s, err: %w", triggerTimeout, err)
}
}
err = member.Wait()
if err != nil && !strings.Contains(err.Error(), "unexpected exit code") {
return err
}
err = member.Start(ctx)
err := member.Start(ctx)
if err != nil {
return err
}
@ -150,13 +166,13 @@ func (f goFailpoint) pickMember(t *testing.T, clus *e2e.EtcdProcessCluster) e2e.
}
}
func setupGoFailpoint(host, failpoint, payload string) error {
func setupGoFailpoint(ctx context.Context, host, failpoint, payload string) error {
failpointUrl := url.URL{
Scheme: "http",
Host: host,
Path: failpoint,
}
r, err := http.NewRequest("PUT", failpointUrl.String(), bytes.NewBuffer([]byte(payload)))
r, err := http.NewRequestWithContext(ctx, "PUT", failpointUrl.String(), bytes.NewBuffer([]byte(payload)))
if err != nil {
return err
}