mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
0263597ba8
This is a backporting cherry-pick of the following commits: - add flagsline Signed-off-by: Jalin Wang <JalinWang@outlook.com> - etcdserver: rename defaultCompactionSleepInterval var (#18495) etcdserver: rename `minimumBatchInterval` to `defaultCompactionSleepInterval` and `defaultCompactBatchLimit` to `defaultCompactionBatchLimit` Signed-off-by: Jalin Wang <JalinWang@outlook.com> (cherry picked from commit 2c53be7c5d91e9b618342d63d2d06c3b265abee4) - test: add CompactionSleepInterval in FakeStore's config After setting the ComparionSleepInterval, we can use time.Ticker instead of time.After to optimize the scheduleComparison(), otherwise it will fail in the 'TestStoreCompact(t)' test. Signed-off-by: guozhao <guozhao@360.cn> (cherry picked from commit fab8474ef8370a089fe0174e28c197b5b93898df) - add sleep interval (cherry picked from commit 184b0e5d4964f1115590acec50fa5e584a2f7770) Signed-off-by: Jalin Wang <JalinWang@outlook.com>
99 lines
3.1 KiB
Go
99 lines
3.1 KiB
Go
// Copyright 2015 The etcd Authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package mvcc
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
"time"
|
|
|
|
humanize "github.com/dustin/go-humanize"
|
|
"go.etcd.io/etcd/server/v3/mvcc/buckets"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyValueHash, error) {
|
|
totalStart := time.Now()
|
|
keep := s.kvindex.Compact(compactMainRev)
|
|
indexCompactionPauseMs.Observe(float64(time.Since(totalStart) / time.Millisecond))
|
|
|
|
totalStart = time.Now()
|
|
defer func() { dbCompactionTotalMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) }()
|
|
keyCompactions := 0
|
|
defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }()
|
|
defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }()
|
|
|
|
end := make([]byte, 8)
|
|
binary.BigEndian.PutUint64(end, uint64(compactMainRev+1))
|
|
|
|
batchNum := s.cfg.CompactionBatchLimit
|
|
batchTicker := time.NewTicker(s.cfg.CompactionSleepInterval)
|
|
defer batchTicker.Stop()
|
|
h := newKVHasher(prevCompactRev, compactMainRev, keep)
|
|
last := make([]byte, 8+1+8)
|
|
|
|
for {
|
|
var rev revision
|
|
|
|
start := time.Now()
|
|
|
|
tx := s.b.BatchTx()
|
|
tx.LockOutsideApply()
|
|
keys, values := tx.UnsafeRange(buckets.Key, last, end, int64(batchNum))
|
|
for i := range keys {
|
|
rev = bytesToRev(keys[i])
|
|
if _, ok := keep[rev]; !ok {
|
|
tx.UnsafeDelete(buckets.Key, keys[i])
|
|
keyCompactions++
|
|
}
|
|
h.WriteKeyValue(keys[i], values[i])
|
|
}
|
|
|
|
if len(keys) < batchNum {
|
|
// gofail: var compactBeforeSetFinishedCompact struct{}
|
|
rbytes := make([]byte, 8+1+8)
|
|
revToBytes(revision{main: compactMainRev}, rbytes)
|
|
tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes)
|
|
tx.Unlock()
|
|
hash := h.Hash()
|
|
size, sizeInUse := s.b.Size(), s.b.SizeInUse()
|
|
s.lg.Info(
|
|
"finished scheduled compaction",
|
|
zap.Int64("compact-revision", compactMainRev),
|
|
zap.Duration("took", time.Since(totalStart)),
|
|
zap.Uint32("hash", hash.Hash),
|
|
zap.Int64("current-db-size-bytes", size),
|
|
zap.String("current-db-size", humanize.Bytes(uint64(size))),
|
|
zap.Int64("current-db-size-in-use-bytes", sizeInUse),
|
|
zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse))),
|
|
)
|
|
return hash, nil
|
|
}
|
|
|
|
// update last
|
|
revToBytes(revision{main: rev.main, sub: rev.sub + 1}, last)
|
|
tx.Unlock()
|
|
// Immediately commit the compaction deletes instead of letting them accumulate in the write buffer
|
|
s.b.ForceCommit()
|
|
dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond))
|
|
|
|
select {
|
|
case <-batchTicker.C:
|
|
case <-s.stopc:
|
|
return KeyValueHash{}, fmt.Errorf("interrupted due to stop signal")
|
|
}
|
|
}
|
|
}
|