server: Return revision range that hash was calcualted for

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz
2022-05-19 16:07:35 +02:00
parent 1ff59923d6
commit a3f609d742
9 changed files with 62 additions and 67 deletions

View File

@@ -22,8 +22,8 @@ import (
"go.etcd.io/etcd/server/v3/mvcc/buckets"
)
func unsafeHashByRev(tx backend.ReadTx, lower, upper int64, keep map[revision]struct{}) (uint32, error) {
h := newKVHasher(lower, upper, keep)
func unsafeHashByRev(tx backend.ReadTx, compactRevision, revision int64, keep map[revision]struct{}) (KeyValueHash, error) {
h := newKVHasher(compactRevision, revision, keep)
err := tx.UnsafeForEach(buckets.Key, func(k, v []byte) error {
h.WriteKeyValue(k, v)
return nil
@@ -32,29 +32,30 @@ func unsafeHashByRev(tx backend.ReadTx, lower, upper int64, keep map[revision]st
}
type kvHasher struct {
hash hash.Hash32
lower, upper int64
keep map[revision]struct{}
hash hash.Hash32
compactRevision int64
revision int64
keep map[revision]struct{}
}
func newKVHasher(lower, upper int64, keep map[revision]struct{}) kvHasher {
func newKVHasher(compactRev, rev int64, keep map[revision]struct{}) kvHasher {
h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
h.Write(buckets.Key.Name())
return kvHasher{
hash: h,
lower: lower,
upper: upper,
keep: keep,
hash: h,
compactRevision: compactRev,
revision: rev,
keep: keep,
}
}
func (h *kvHasher) WriteKeyValue(k, v []byte) {
kr := bytesToRev(k)
upper := revision{main: h.upper + 1}
upper := revision{main: h.revision + 1}
if !upper.GreaterThan(kr) {
return
}
lower := revision{main: h.lower + 1}
lower := revision{main: h.compactRevision + 1}
// skip revisions that are scheduled for deletion
// due to compacting; don't skip if there isn't one.
if lower.GreaterThan(kr) && len(h.keep) > 0 {
@@ -66,6 +67,12 @@ func (h *kvHasher) WriteKeyValue(k, v []byte) {
h.hash.Write(v)
}
func (h *kvHasher) Hash() uint32 {
return h.hash.Sum32()
func (h *kvHasher) Hash() KeyValueHash {
return KeyValueHash{Hash: h.hash.Sum32(), CompactRevision: h.compactRevision, Revision: h.revision}
}
type KeyValueHash struct {
Hash uint32
CompactRevision int64
Revision int64
}

View File

@@ -41,7 +41,7 @@ func TestHashByRevValue(t *testing.T) {
assert.Less(t, int64(s.cfg.CompactionBatchLimit), totalRevisions)
assert.Less(t, int64(compactionCycle*10), totalRevisions)
var rev int64
var got []kvHash
var got []KeyValueHash
for ; rev < totalRevisions; rev += compactionCycle {
putKVs(s, rev, compactionCycle)
hash := testHashByRev(t, s, rev+compactionCycle/2)
@@ -50,7 +50,7 @@ func TestHashByRevValue(t *testing.T) {
putKVs(s, rev, totalRevisions)
hash := testHashByRev(t, s, rev+totalRevisions/2)
got = append(got, hash)
assert.Equal(t, []kvHash{
assert.Equal(t, []KeyValueHash{
{4082599214, -1, 35},
{2279933401, 35, 106},
{3284231217, 106, 177},
@@ -81,7 +81,7 @@ func TestHashByRevValueZero(t *testing.T) {
assert.Less(t, int64(s.cfg.CompactionBatchLimit), totalRevisions)
assert.Less(t, int64(compactionCycle*10), totalRevisions)
var rev int64
var got []kvHash
var got []KeyValueHash
for ; rev < totalRevisions; rev += compactionCycle {
putKVs(s, rev, compactionCycle)
hash := testHashByRev(t, s, 0)
@@ -90,7 +90,7 @@ func TestHashByRevValueZero(t *testing.T) {
putKVs(s, rev, totalRevisions)
hash := testHashByRev(t, s, 0)
got = append(got, hash)
assert.Equal(t, []kvHash{
assert.Equal(t, []KeyValueHash{
{1913897190, -1, 73},
{224860069, 73, 145},
{1565167519, 145, 217},
@@ -119,8 +119,8 @@ func putKVs(s *store, rev, count int64) {
}
}
func testHashByRev(t *testing.T, s *store, rev int64) kvHash {
hash, currentRev, compactRev, err := s.HashByRev(rev)
func testHashByRev(t *testing.T, s *store, rev int64) KeyValueHash {
hash, currentRev, _, err := s.HashByRev(rev)
assert.NoError(t, err, "error on rev %v", rev)
if rev == 0 {
@@ -128,13 +128,7 @@ func testHashByRev(t *testing.T, s *store, rev int64) kvHash {
}
_, err = s.Compact(traceutil.TODO(), rev)
assert.NoError(t, err, "error on compact %v", rev)
return kvHash{hash: hash, compactRevision: compactRev, revision: rev}
}
type kvHash struct {
hash uint32
compactRevision int64
revision int64
return hash
}
// TODO: Change this to fuzz test

View File

@@ -136,7 +136,7 @@ type Hasher interface {
Hash() (hash uint32, revision int64, err error)
// HashByRev computes the hash of all MVCC revisions up to a given revision.
HashByRev(rev int64) (hash uint32, revision int64, compactRev int64, err error)
HashByRev(rev int64) (hash KeyValueHash, revision int64, compactRev int64, err error)
}
// WatchableKV is a KV that can be watched.

View File

@@ -164,7 +164,7 @@ func (s *store) Hash() (hash uint32, revision int64, err error) {
return h, s.currentRev, err
}
func (s *store) HashByRev(rev int64) (hash uint32, currentRev int64, compactRev int64, err error) {
func (s *store) HashByRev(rev int64) (hash KeyValueHash, currentRev int64, compactRev int64, err error) {
start := time.Now()
s.mu.RLock()
@@ -174,10 +174,10 @@ func (s *store) HashByRev(rev int64) (hash uint32, currentRev int64, compactRev
if rev > 0 && rev <= compactRev {
s.mu.RUnlock()
return 0, 0, compactRev, ErrCompacted
return KeyValueHash{}, 0, compactRev, ErrCompacted
} else if rev > 0 && rev > currentRev {
s.mu.RUnlock()
return 0, currentRev, 0, ErrFutureRev
return KeyValueHash{}, currentRev, 0, ErrFutureRev
}
if rev == 0 {

View File

@@ -23,7 +23,7 @@ import (
"go.uber.org/zap"
)
func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (uint32, error) {
func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyValueHash, error) {
totalStart := time.Now()
keep := s.kvindex.Compact(compactMainRev)
indexCompactionPauseMs.Observe(float64(time.Since(totalStart) / time.Millisecond))
@@ -67,7 +67,7 @@ func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (uint32
"finished scheduled compaction",
zap.Int64("compact-revision", compactMainRev),
zap.Duration("took", time.Since(totalStart)),
zap.Uint32("hash", hash),
zap.Uint32("hash", hash.Hash),
)
return hash, nil
}
@@ -82,7 +82,7 @@ func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (uint32
select {
case <-time.After(10 * time.Millisecond):
case <-s.stopc:
return 0, fmt.Errorf("interrupted due to stop signal")
return KeyValueHash{}, fmt.Errorf("interrupted due to stop signal")
}
}
}

View File

@@ -559,7 +559,7 @@ func TestHashKVWhenCompacting(t *testing.T) {
select {
case <-donec:
return
case hashCompactc <- hashKVResult{hash, compactRev}:
case hashCompactc <- hashKVResult{hash.Hash, compactRev}:
}
}
}()
@@ -618,7 +618,7 @@ func TestHashKVZeroRevision(t *testing.T) {
if err != nil {
t.Fatal(err)
}
var hash2 uint32
var hash2 KeyValueHash
hash2, _, _, err = s.HashByRev(0)
if err != nil {
t.Fatal(err)