Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement random packet dropping

Browse Source 
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz 2023-10-17 14:33:51 +02:00
parent be83fffedb
commit aa28a69ce0
2 changed files with 38 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/robustness/failpoint/failpoint.go
View File

@ -47,6 +47,7 @@ var (
RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot, RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
beforeApplyOneConfChangeSleep, beforeApplyOneConfChangeSleep,
MemberReplace, MemberReplace,
DropPeerNetwork,
} }
) )

39
tests/robustness/failpoint/network.go
View File

@ -30,6 +30,7 @@ var (
BlackholePeerNetwork Failpoint = blackholePeerNetworkFailpoint{triggerBlackhole{waitTillSnapshot: false}} BlackholePeerNetwork Failpoint = blackholePeerNetworkFailpoint{triggerBlackhole{waitTillSnapshot: false}}
BlackholeUntilSnapshot Failpoint = blackholePeerNetworkFailpoint{triggerBlackhole{waitTillSnapshot: true}} BlackholeUntilSnapshot Failpoint = blackholePeerNetworkFailpoint{triggerBlackhole{waitTillSnapshot: true}}
DelayPeerNetwork Failpoint = delayPeerNetworkFailpoint{duration: time.Second, baseLatency: 75 * time.Millisecond, randomizedLatency: 50 * time.Millisecond} DelayPeerNetwork Failpoint = delayPeerNetworkFailpoint{duration: time.Second, baseLatency: 75 * time.Millisecond, randomizedLatency: 50 * time.Millisecond}
DropPeerNetwork Failpoint = dropPeerNetworkFailpoint{duration: time.Second, dropProbabilityPercent: 50}
) )
type blackholePeerNetworkFailpoint struct { type blackholePeerNetworkFailpoint struct {
@ -42,7 +43,7 @@ func (f blackholePeerNetworkFailpoint) Inject(ctx context.Context, t *testing.T,
} }
func (f blackholePeerNetworkFailpoint) Name() string { func (f blackholePeerNetworkFailpoint) Name() string {
return "blackhole" return "blackholePeerNetwork"
} }
type triggerBlackhole struct { type triggerBlackhole struct {
@ -167,9 +168,43 @@ func (f delayPeerNetworkFailpoint) Inject(ctx context.Context, t *testing.T, lg
} }
func (f delayPeerNetworkFailpoint) Name() string { func (f delayPeerNetworkFailpoint) Name() string {
return "delay" return "delayPeerNetwork"
} }
func (f delayPeerNetworkFailpoint) Available(config e2e.EtcdProcessClusterConfig, clus e2e.EtcdProcess) bool { func (f delayPeerNetworkFailpoint) Available(config e2e.EtcdProcessClusterConfig, clus e2e.EtcdProcess) bool {
return config.ClusterSize > 1 && clus.PeerProxy() != nil return config.ClusterSize > 1 && clus.PeerProxy() != nil
} }
type dropPeerNetworkFailpoint struct {
duration time.Duration
dropProbabilityPercent int
}
func (f dropPeerNetworkFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error {
member := clus.Procs[rand.Int()%len(clus.Procs)]
proxy := member.PeerProxy()
proxy.ModifyRx(f.modifyPacket)
proxy.ModifyTx(f.modifyPacket)
lg.Info("Dropping traffic from and to member", zap.String("member", member.Config().Name), zap.Int("probability", f.dropProbabilityPercent))
time.Sleep(f.duration)
lg.Info("Traffic drop removed", zap.String("member", member.Config().Name))
proxy.UnmodifyRx()
proxy.UnmodifyTx()
return nil
}
func (f dropPeerNetworkFailpoint) modifyPacket(data []byte) []byte {
if rand.Intn(100) < f.dropProbabilityPercent {
return nil
}
return data
}
func (f dropPeerNetworkFailpoint) Name() string {
return "dropPeerNetwork"
}
func (f dropPeerNetworkFailpoint) Available(config e2e.EtcdProcessClusterConfig, clus e2e.EtcdProcess) bool {
return config.ClusterSize > 1 && clus.PeerProxy() != nil
}