Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix ETCDCTL_API=2 etcdctl backup --with-v3 consistent index consistency

Browse Source 
Prior to this CL, `ETCDCTL_API=2 etcdctl backup --with-v3` was readacting WAL log
(by removal of some entries), but was NOT updating consistent_index in the backend.
Also the WAL editing logic was buggy, as it didn't took in consideration the fact
that when TERM changes, there can be entries with duplicated indexes in
the log. So its NOT sufficient to subtract number of removed entries to
get accurate log indexes.

The PR replaces removing and shifting of WAL entries with replacing them with an no-op entries.
Thanks to this consistent-index references are staying up to date.

The PR also:
  - updates 'verification' logic to check whether consistent_index does not lag befor last snapshot
  - env-gated execution of verification framework in `etcdctl backup`.

Tested with:
```
(./build.sh && cd tests && EXPECT_DEBUG=TRUE 'env' 'go' 'test' '-timeout=300m' 'go.etcd.io/etcd/tests/v3/e2e' -run=TestCtlV2Backup --count=1000 2>&1 | tee TestCtlV2BackupV3.log)
```
This commit is contained in:
Piotr Tabor
2021-04-28 22:46:42 +02:00
parent adc365e14f
commit 911204cd76
5 changed files with 84 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
server/verify/verify.go
View File

@@ -76,7 +76,7 @@ func Verify(cfg Config) error {
be := backend.New(beConfig)
defer be.Close()
_, hardstate, err := validateWal(cfg)
snapshot, hardstate, err := validateWal(cfg)
if err != nil {
return err
}
@@ -84,7 +84,7 @@ func Verify(cfg Config) error {
// TODO: Perform validation of consistency of membership between
// backend/members & WAL confstate (and maybe storev2 if still exists).
return validateConsistentIndex(cfg, hardstate, be)
return validateConsistentIndex(cfg, hardstate, snapshot, be)
}
// VerifyIfEnabled performs verification according to ETCD_VERIFY env settings.
@@ -101,22 +101,25 @@ func VerifyIfEnabled(cfg Config) error {
// See Verify for more information.
func MustVerifyIfEnabled(cfg Config) {
if err := VerifyIfEnabled(cfg); err != nil {
cfg.Logger.Panic("Verification failed",
cfg.Logger.Fatal("Verification failed",
zap.String("data-dir", cfg.DataDir),
zap.Error(err))
}
}
func validateConsistentIndex(cfg Config, hardstate *raftpb.HardState, be backend.Backend) error {
func validateConsistentIndex(cfg Config, hardstate *raftpb.HardState, snapshot *walpb.Snapshot, be backend.Backend) error {
tx := be.BatchTx()
ci := cindex.NewConsistentIndex(tx)
index := ci.ConsistentIndex()
index := cindex.ReadConsistentIndex(tx)
if cfg.ExactIndex && index != hardstate.Commit {
return fmt.Errorf("backend.ConsistentIndex (%v) expected == WAL.HardState.commit (%v)", index, hardstate.Commit)
}
if index > hardstate.Commit {
return fmt.Errorf("backend.ConsistentIndex (%v) must be <= WAL.HardState.commit (%v)", index, hardstate.Commit)
}
if index < snapshot.Index {
return fmt.Errorf("backend.ConsistentIndex (%v) must be >= last snapshot index (%v)", index, snapshot.Index)
}
cfg.Logger.Info("verification: consistentIndex OK", zap.Uint64("backend-consistent-index", index), zap.Uint64("hardstate-commit", hardstate.Commit))
return nil
}