server: Implement compaction hash checking

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz
2022-05-20 12:11:18 +02:00
parent a56ec0be4b
commit 21fb173f76
6 changed files with 335 additions and 6 deletions

View File

@@ -99,3 +99,76 @@ func TestPeriodicCheckDetectsCorruption(t *testing.T) {
// TODO: Investigate why MemberID is 0?
assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: 0}}, alarmResponse.Alarms)
}
func TestCompactHashCheck(t *testing.T) {
BeforeTest(t)
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
cc, err := clus.ClusterClient()
require.NoError(t, err)
ctx := context.Background()
var totalRevisions int64 = 1210
var rev int64
for ; rev < totalRevisions; rev += testutil.CompactionCycle {
testCompactionHash(ctx, t, cc, clus, rev, rev+testutil.CompactionCycle)
}
testCompactionHash(ctx, t, cc, clus, rev, rev+totalRevisions)
}
func testCompactionHash(ctx context.Context, t *testing.T, cc *clientv3.Client, clus *ClusterV3, start, stop int64) {
for i := start; i <= stop; i++ {
if i%67 == 0 {
_, err := cc.Delete(ctx, testutil.PickKey(i+83))
assert.NoError(t, err, "error on delete")
} else {
_, err := cc.Put(ctx, testutil.PickKey(i), fmt.Sprint(i))
assert.NoError(t, err, "error on put")
}
}
_, err := cc.Compact(ctx, stop)
assert.NoError(t, err, "error on compact (rev %v)", stop)
// Wait for compaction to be compacted
time.Sleep(50 * time.Millisecond)
clus.Members[0].CorruptionChecker().CompactHashCheck()
}
func TestCompactHashCheckDetectCorruption(t *testing.T) {
BeforeTest(t)
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
cc, err := clus.ClusterClient()
require.NoError(t, err)
ctx := context.Background()
for i := 0; i < 10; i++ {
_, err := cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i))
assert.NoError(t, err, "error on put")
}
clus.Members[0].CorruptionChecker().CompactHashCheck()
clus.Members[0].Stop(t)
clus.WaitLeader(t)
err = testutil.CorruptBBolt(clus.Members[0].BackendPath())
assert.NoError(t, err)
err = clus.Members[0].Restart(t)
assert.NoError(t, err)
_, err = cc.Compact(ctx, 5)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
leader := clus.WaitLeader(t)
clus.Members[leader].CorruptionChecker().CompactHashCheck()
time.Sleep(50 * time.Millisecond)
alarmResponse, err := cc.AlarmList(ctx)
assert.NoError(t, err, "error on alarm list")
assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: uint64(clus.Members[0].ID())}}, alarmResponse.Alarms)
}