From 7b19ee6396de1c1ff6dda4b9ee444cb42010fa2f Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Wed, 23 Nov 2022 13:42:45 +0800 Subject: [PATCH] test: add integration test to cover the multiple member corruption case Signed-off-by: Benjamin Wang --- tests/integration/corrupt_test.go | 55 +++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/integration/corrupt_test.go b/tests/integration/corrupt_test.go index ee98a24fb..cd695c578 100644 --- a/tests/integration/corrupt_test.go +++ b/tests/integration/corrupt_test.go @@ -175,3 +175,58 @@ func TestCompactHashCheckDetectCorruption(t *testing.T) { assert.NoError(t, err, "error on alarm list") assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: uint64(clus.Members[0].ID())}}, alarmResponse.Alarms) } + +func TestCompactHashCheckDetectMultipleCorruption(t *testing.T) { + integration.BeforeTest(t) + + clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 5}) + defer clus.Terminate(t) + + cc, err := clus.ClusterClient(t) + require.NoError(t, err) + + ctx := context.Background() + + for i := 0; i < 10; i++ { + _, err := cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i)) + assert.NoError(t, err, "error on put") + } + + clus.Members[0].Server.CorruptionChecker().CompactHashCheck() + clus.Members[0].Stop(t) + clus.Members[1].Server.CorruptionChecker().CompactHashCheck() + clus.Members[1].Stop(t) + clus.WaitLeader(t) + + err = testutil.CorruptBBolt(clus.Members[0].BackendPath()) + require.NoError(t, err) + err = testutil.CorruptBBolt(clus.Members[1].BackendPath()) + require.NoError(t, err) + + err = clus.Members[0].Restart(t) + require.NoError(t, err) + err = clus.Members[1].Restart(t) + require.NoError(t, err) + + _, err = cc.Compact(ctx, 5) + require.NoError(t, err) + time.Sleep(50 * time.Millisecond) + leader := clus.WaitLeader(t) + + clus.Members[leader].Server.CorruptionChecker().CompactHashCheck() + time.Sleep(50 * time.Millisecond) + alarmResponse, err := cc.AlarmList(ctx) + assert.NoError(t, err, "error on alarm list") + + expectedAlarmMap := map[uint64]etcdserverpb.AlarmType{ + uint64(clus.Members[0].ID()): etcdserverpb.AlarmType_CORRUPT, + uint64(clus.Members[1].ID()): etcdserverpb.AlarmType_CORRUPT, + } + + actualAlarmMap := make(map[uint64]etcdserverpb.AlarmType) + for _, alarm := range alarmResponse.Alarms { + actualAlarmMap[alarm.MemberID] = alarm.Alarm + } + + require.Equal(t, expectedAlarmMap, actualAlarmMap) +}