Verification package: Verified given data-dir.

For now verifies whete Backend.cindex is consistent with WAL log,
but should get expanded to cover memberships & revisions.

server/verify/verify.go

@@ -0,0 +1,138 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package verify
+
+import (
+	"fmt"
+	"os"
+
+	"go.etcd.io/etcd/raft/v3/raftpb"
+	"go.etcd.io/etcd/server/v3/datadir"
+	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
+	"go.etcd.io/etcd/server/v3/mvcc/backend"
+	wal2 "go.etcd.io/etcd/server/v3/wal"
+	"go.etcd.io/etcd/server/v3/wal/walpb"
+	"go.uber.org/zap"
+)
+
+const ENV_VERIFY = "ETCD_VERIFY"
+const ENV_VERIFY_ALL_VALUE = "all"
+
+type Config struct {
+	// DataDir is a root directory where the data being verified are stored.
+	DataDir string
+
+	// ExactIndex requires consistent_index in backend exactly match the last committed WAL entry.
+	// Usually backend's consistent_index needs to be <= WAL.commit, but for backups the match
+	// is expected to be exact.
+	ExactIndex bool
+
+	Logger *zap.Logger
+}
+
+// Verify performs consistency checks of given etcd data-directory.
+// The errors are reported as the returned error, but for some situations
+// the function can also panic.
+// The function is expected to work on not-in-use data model, i.e.
+// no file-locks should be taken. Verify does not modified the data.
+func Verify(cfg Config) error {
+	lg := cfg.Logger
+	if lg == nil {
+		lg = zap.NewNop()
+	}
+
+	var err error
+	lg.Info("verification of persisted state", zap.String("data-dir", cfg.DataDir))
+	defer func() {
+		if err != nil {
+			lg.Error("verification of persisted state failed",
+				zap.String("data-dir", cfg.DataDir),
+				zap.Error(err))
+		} else if r := recover(); r != nil {
+			lg.Error("verification of persisted state failed",
+				zap.String("data-dir", cfg.DataDir))
+			panic(r)
+		} else {
+			lg.Info("verification of persisted state successful", zap.String("data-dir", cfg.DataDir))
+		}
+	}()
+
+	beConfig := backend.DefaultBackendConfig()
+	beConfig.Path = datadir.ToBackendFileName(cfg.DataDir)
+	beConfig.Logger = cfg.Logger
+
+	be := backend.New(beConfig)
+	defer be.Close()
+
+	_, hardstate, err := validateWal(cfg)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Perform validation of consistency of membership between
+	// backend/members & WAL confstate (and maybe storev2 if still exists).
+
+	return validateConsistentIndex(cfg, hardstate, be)
+}
+
+// VerifyIfEnabled performs verification according to ETCD_VERIFY env settings.
+// See Verify for more information.
+func VerifyIfEnabled(cfg Config) error {
+	if os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE {
+		return Verify(cfg)
+	}
+	return nil
+}
+
+// MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings
+// and exits in case of found problems.
+// See Verify for more information.
+func MustVerifyIfEnabled(cfg Config) {
+	if err := VerifyIfEnabled(cfg); err != nil {
+		cfg.Logger.Panic("Verification failed",
+			zap.String("data-dir", cfg.DataDir),
+			zap.Error(err))
+	}
+}
+
+func validateConsistentIndex(cfg Config, hardstate *raftpb.HardState, be backend.Backend) error {
+	tx := be.BatchTx()
+	ci := cindex.NewConsistentIndex(tx)
+	index := ci.ConsistentIndex()
+	if cfg.ExactIndex && index != hardstate.Commit {
+		return fmt.Errorf("backend.ConsistentIndex (%v) expected == WAL.HardState.commit (%v)", index, hardstate.Commit)
+	}
+	if index > hardstate.Commit {
+		return fmt.Errorf("backend.ConsistentIndex (%v) must be <= WAL.HardState.commit (%v)", index, hardstate.Commit)
+	}
+	cfg.Logger.Info("verification: consistentIndex OK", zap.Uint64("backend-consistent-index", index), zap.Uint64("hardstate-commit", hardstate.Commit))
+	return nil
+}
+
+func validateWal(cfg Config) (*walpb.Snapshot, *raftpb.HardState, error) {
+	walDir := datadir.ToWalDir(cfg.DataDir)
+
+	walSnaps, err := wal2.ValidSnapshotEntries(cfg.Logger, walDir)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	snapshot := walSnaps[len(walSnaps)-1]
+	hardstate, err := wal2.Verify(cfg.Logger, walDir, snapshot)
+	if err != nil {
+		return nil, nil, err
+	}
+	return &snapshot, hardstate, nil
+}