Unite reachability stores (#1963)

* Unite all reachability stores

* Upgrade script

* Fix tests

* Add UpdateReindexRoot to RebuildReachability

* Use dbTx when deleting reachability stores

* Use ghostdagDataWithoutPrunedBlocks when rebuilding reachability

* Use next tree ancestor wherever possible and avoid finality point search if the block is too close to pruning point

* Address the boundary case where the pruning point becomes the finality point

* some minor fixes

* Remove RebuildReachability and use manual syncing between old and new consensus for migration

* Remove sanity test (it failed when tips where not in the same order)

Co-authored-by: msutton <mikisiton2@gmail.com>

domain/consensus/consensus.go

@@ -35,7 +35,7 @@ type consensus struct {
 	headerTipsManager     model.HeadersSelectedTipManager
 	mergeDepthManager     model.MergeDepthManager
 	pruningManager        model.PruningManager
-	reachabilityManagers  []model.ReachabilityManager
+	reachabilityManager   model.ReachabilityManager
 	finalityManager       model.FinalityManager
 	pruningProofManager   model.PruningProofManager
 
@@ -49,7 +49,7 @@ type consensus struct {
 	consensusStateStore                 model.ConsensusStateStore
 	headersSelectedTipStore             model.HeaderSelectedTipStore
 	multisetStore                       model.MultisetStore
-	reachabilityDataStores              []model.ReachabilityDataStore
+	reachabilityDataStore               model.ReachabilityDataStore
 	utxoDiffStore                       model.UTXODiffStore
 	finalityStore                       model.FinalityStore
 	headersSelectedChainStore           model.HeadersSelectedChainStore
@@ -83,12 +83,10 @@ func (s *consensus) Init(skipAddingGenesis bool) error {
 	// on a node with pruned header all blocks without known parents points to it.
 	if !exists {
 		s.blockStatusStore.Stage(stagingArea, model.VirtualGenesisBlockHash, externalapi.StatusUTXOValid)
-		for _, reachabilityManager := range s.reachabilityManagers {
+		err = s.reachabilityManager.Init(stagingArea)
-			err = reachabilityManager.Init(stagingArea)
 		if err != nil {
 			return err
 		}
-		}
 
 		for _, dagTopologyManager := range s.dagTopologyManagers {
 			err = dagTopologyManager.SetParents(stagingArea, model.VirtualGenesisBlockHash, nil)

domain/consensus/datastructures/blockrelationstore/block_relation_store.go

@@ -75,6 +75,11 @@ func (brs *blockRelationStore) Has(dbContext model.DBReader, stagingArea *model.
 	return dbContext.Has(brs.hashAsKey(blockHash))
 }
 
+func (brs *blockRelationStore) UnstageAll(stagingArea *model.StagingArea) {
+	stagingShard := brs.stagingShard(stagingArea)
+	stagingShard.toAdd = make(map[externalapi.DomainHash]*model.BlockRelations)
+}
+
 func (brs *blockRelationStore) hashAsKey(hash *externalapi.DomainHash) model.DBKey {
 	return brs.bucket.Key(hash.ByteSlice())
 }

domain/consensus/datastructures/ghostdagdatastore/ghostdag_data_store.go

@@ -69,6 +69,12 @@ func (gds *ghostdagDataStore) Get(dbContext model.DBReader, stagingArea *model.S
 	return blockGHOSTDAGData, nil
 }
 
+func (gds *ghostdagDataStore) UnstageAll(stagingArea *model.StagingArea) {
+	stagingShard := gds.stagingShard(stagingArea)
+
+	stagingShard.toAdd = make(map[key]*externalapi.BlockGHOSTDAGData)
+}
+
 func (gds *ghostdagDataStore) serializeKey(k key) model.DBKey {
 	if k.isTrustedData {
 		return gds.trustedDataBucket.Key(k.hash.ByteSlice())

domain/consensus/datastructures/reachabilitydatastore/reachability_data_store.go

@@ -41,6 +41,27 @@ func (rds *reachabilityDataStore) StageReachabilityData(stagingArea *model.Stagi
 	stagingShard.reachabilityData[*blockHash] = reachabilityData
 }
 
+func (rds *reachabilityDataStore) Delete(dbContext model.DBWriter) error {
+	cursor, err := dbContext.Cursor(rds.reachabilityDataBucket)
+	if err != nil {
+		return err
+	}
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		key, err := cursor.Key()
+		if err != nil {
+			return err
+		}
+
+		err = dbContext.Delete(key)
+		if err != nil {
+			return err
+		}
+	}
+
+	return dbContext.Delete(rds.reachabilityReindexRootKey)
+}
+
 // StageReachabilityReindexRoot stages the given reachabilityReindexRoot
 func (rds *reachabilityDataStore) StageReachabilityReindexRoot(stagingArea *model.StagingArea, reachabilityReindexRoot *externalapi.DomainHash) {
 	stagingShard := rds.stagingShard(stagingArea)

domain/consensus/factory.go

@@ -1,16 +1,17 @@
 package consensus
 
 import (
-	"io/ioutil"
-	"os"
-	"sync"
-
 	"github.com/kaspanet/kaspad/domain/consensus/datastructures/blockwindowheapslicestore"
 	"github.com/kaspanet/kaspad/domain/consensus/datastructures/daawindowstore"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/processes/blockparentbuilder"
 	parentssanager "github.com/kaspanet/kaspad/domain/consensus/processes/parentsmanager"
 	"github.com/kaspanet/kaspad/domain/consensus/processes/pruningproofmanager"
+	"github.com/pkg/errors"
+	"io/ioutil"
+	"os"
+	"sync"
+
 	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
 	"github.com/kaspanet/kaspad/util/txmass"
 
@@ -74,7 +75,7 @@ type Config struct {
 // Factory instantiates new Consensuses
 type Factory interface {
 	NewConsensus(config *Config, db infrastructuredatabase.Database, dbPrefix *prefix.Prefix) (
-		externalapi.Consensus, error)
+		externalapi.Consensus, bool, error)
 	NewTestConsensus(config *Config, testName string) (
 		tc testapi.TestConsensus, teardown func(keepDataDir bool), err error)
 
@@ -106,7 +107,7 @@ func NewFactory() Factory {
 
 // NewConsensus instantiates a new Consensus
 func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Database, dbPrefix *prefix.Prefix) (
-	externalapi.Consensus, error) {
+	consensusInstance externalapi.Consensus, shouldMigrate bool, err error) {
 
 	dbManager := consensusdatabase.New(db)
 	prefixBucket := consensusdatabase.MakeBucket(dbPrefix.Serialize())
@@ -129,11 +130,11 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 	acceptanceDataStore := acceptancedatastore.New(prefixBucket, 200, preallocateCaches)
 	blockStore, err := blockstore.New(dbManager, prefixBucket, 200, preallocateCaches)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	blockHeaderStore, err := blockheaderstore.New(dbManager, prefixBucket, 10_000, preallocateCaches)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	blockStatusStore := blockstatusstore.New(prefixBucket, pruningWindowSizePlusFinalityDepthForCache, preallocateCaches)
@@ -148,11 +149,35 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 	daaBlocksStore := daablocksstore.New(prefixBucket, pruningWindowSizeForCaches, int(config.FinalityDepth()), preallocateCaches)
 	windowHeapSliceStore := blockwindowheapslicestore.New(2000, preallocateCaches)
 
+	newReachabilityDataStore := reachabilitydatastore.New(prefixBucket, pruningWindowSizePlusFinalityDepthForCache*2, preallocateCaches)
 	blockRelationStores, reachabilityDataStores, ghostdagDataStores := dagStores(config, prefixBucket, pruningWindowSizePlusFinalityDepthForCache, pruningWindowSizeForCaches, preallocateCaches)
-	reachabilityManagers, dagTopologyManagers, ghostdagManagers, dagTraversalManagers := f.dagProcesses(config, dbManager, blockHeaderStore, daaWindowStore, windowHeapSliceStore, blockRelationStores, reachabilityDataStores, ghostdagDataStores)
+	oldReachabilityManager := reachabilitymanager.New(
+		dbManager,
+		ghostdagDataStores[0],
+		reachabilityDataStores[0])
+	isOldReachabilityInitialized, err := reachabilityDataStores[0].HasReachabilityData(dbManager, model.NewStagingArea(), model.VirtualGenesisBlockHash)
+	if err != nil {
+		return nil, false, err
+	}
+
+	newReachabilityManager := reachabilitymanager.New(
+		dbManager,
+		ghostdagDataStores[0],
+		newReachabilityDataStore)
+	reachabilityManager := newReachabilityManager
+	if isOldReachabilityInitialized {
+		reachabilityManager = oldReachabilityManager
+	} else {
+		for i := range reachabilityDataStores {
+			reachabilityDataStores[i] = newReachabilityDataStore
+		}
+	}
+	reachabilityDataStore := reachabilityDataStores[0]
+
+	dagTopologyManagers, ghostdagManagers, dagTraversalManagers := f.dagProcesses(config, dbManager, blockHeaderStore, daaWindowStore, windowHeapSliceStore, blockRelationStores, reachabilityDataStores, ghostdagDataStores, isOldReachabilityInitialized)
 
 	blockRelationStore := blockRelationStores[0]
-	reachabilityDataStore := reachabilityDataStores[0]
+
 	ghostdagDataStore := ghostdagDataStores[0]
 
 	dagTopologyManager := dagTopologyManagers[0]
@@ -227,6 +252,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		dagTopologyManager,
 		finalityStore,
 		ghostdagDataStore,
+		pruningStore,
 		genesisHash,
 		config.FinalityDepth())
 	mergeDepthManager := mergedepthmanager.New(
@@ -264,7 +290,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		pruningStore,
 		daaBlocksStore)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	pruningManager := pruningmanager.New(
@@ -319,7 +345,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		dagTraversalManager,
 		coinbaseManager,
 		mergeDepthManager,
-		reachabilityManagers,
+		reachabilityManager,
 		finalityManager,
 		blockParentBuilder,
 		pruningManager,
@@ -383,7 +409,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		pruningManager,
 		blockValidator,
 		dagTopologyManager,
-		reachabilityManagers,
+		reachabilityManager,
 		difficultyManager,
 		pastMedianTimeManager,
 		coinbaseManager,
@@ -411,9 +437,10 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		dbManager,
 		dagTopologyManagers,
 		ghostdagManagers,
-		reachabilityManagers,
+		reachabilityManager,
 		dagTraversalManagers,
 		parentsManager,
+		pruningManager,
 
 		ghostdagDataStores,
 		pruningStore,
@@ -421,6 +448,8 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		blockStatusStore,
 		finalityStore,
 		consensusStateStore,
+		blockRelationStore,
+		reachabilityDataStore,
 
 		genesisHash,
 		config.K,
@@ -450,7 +479,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		headerTipsManager:     headerTipsManager,
 		mergeDepthManager:     mergeDepthManager,
 		pruningManager:        pruningManager,
-		reachabilityManagers:  reachabilityManagers,
+		reachabilityManager:   reachabilityManager,
 		finalityManager:       finalityManager,
 		pruningProofManager:   pruningProofManager,
 
@@ -464,7 +493,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		consensusStateStore:                 consensusStateStore,
 		headersSelectedTipStore:             headersSelectedTipStore,
 		multisetStore:                       multisetStore,
-		reachabilityDataStores:              reachabilityDataStores,
+		reachabilityDataStore:               reachabilityDataStore,
 		utxoDiffStore:                       utxoDiffStore,
 		finalityStore:                       finalityStore,
 		headersSelectedChainStore:           headersSelectedChainStore,
@@ -472,25 +501,29 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		blocksWithTrustedDataDAAWindowStore: daaWindowStore,
 	}
 
+	if isOldReachabilityInitialized {
+		return c, true, nil
+	}
+
 	err = c.Init(config.SkipAddingGenesis)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	err = consensusStateManager.RecoverUTXOIfRequired()
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	err = pruningManager.ClearImportedPruningPointData()
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	err = pruningManager.UpdatePruningPointIfRequired()
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
-	return c, nil
+	return c, false, nil
 }
 
 func (f *factory) NewTestConsensus(config *Config, testName string) (
@@ -517,11 +550,15 @@ func (f *factory) NewTestConsensus(config *Config, testName string) (
 	}
 
 	testConsensusDBPrefix := &prefix.Prefix{}
-	consensusAsInterface, err := f.NewConsensus(config, db, testConsensusDBPrefix)
+	consensusAsInterface, shouldMigrate, err := f.NewConsensus(config, db, testConsensusDBPrefix)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	if shouldMigrate {
+		return nil, nil, errors.Errorf("A fresh consensus should never return shouldMigrate=true")
+	}
+
 	consensusAsImplementation := consensusAsInterface.(*consensus)
 	testConsensusStateManager := consensusstatemanager.NewTestConsensusStateManager(consensusAsImplementation.consensusStateManager)
 	testTransactionValidator := transactionvalidator.NewTestTransactionValidator(consensusAsImplementation.transactionValidator)
@@ -532,7 +569,7 @@ func (f *factory) NewTestConsensus(config *Config, testName string) (
 		database:                  db,
 		testConsensusStateManager: testConsensusStateManager,
 		testReachabilityManager: reachabilitymanager.NewTestReachabilityManager(consensusAsImplementation.
-			reachabilityManagers[0]),
+			reachabilityManager),
 		testTransactionValidator: testTransactionValidator,
 	}
 	tstConsensus.testBlockBuilder = blockbuilder.NewTestBlockBuilder(consensusAsImplementation.blockBuilder, tstConsensus)
@@ -609,8 +646,8 @@ func (f *factory) dagProcesses(config *Config,
 	windowHeapSliceStore model.WindowHeapSliceStore,
 	blockRelationStores []model.BlockRelationStore,
 	reachabilityDataStores []model.ReachabilityDataStore,
-	ghostdagDataStores []model.GHOSTDAGDataStore) (
+	ghostdagDataStores []model.GHOSTDAGDataStore,
-	[]model.ReachabilityManager,
+	isOldReachabilityInitialized bool) (
 	[]model.DAGTopologyManager,
 	[]model.GHOSTDAGManager,
 	[]model.DAGTraversalManager,
@@ -621,11 +658,20 @@ func (f *factory) dagProcesses(config *Config,
 	ghostdagManagers := make([]model.GHOSTDAGManager, config.MaxBlockLevel+1)
 	dagTraversalManagers := make([]model.DAGTraversalManager, config.MaxBlockLevel+1)
 
+	newReachabilityManager := reachabilitymanager.New(
+		dbManager,
+		ghostdagDataStores[0],
+		reachabilityDataStores[0])
+
 	for i := 0; i <= config.MaxBlockLevel; i++ {
+		if isOldReachabilityInitialized {
 			reachabilityManagers[i] = reachabilitymanager.New(
 				dbManager,
 				ghostdagDataStores[i],
 				reachabilityDataStores[i])
+		} else {
+			reachabilityManagers[i] = newReachabilityManager
+		}
 
 		dagTopologyManagers[i] = dagtopologymanager.New(
 			dbManager,
@@ -645,7 +691,7 @@ func (f *factory) dagProcesses(config *Config,
 			dbManager,
 			dagTopologyManagers[i],
 			ghostdagDataStores[i],
-			reachabilityDataStores[i],
+			reachabilityManagers[i],
 			ghostdagManagers[i],
 			daaWindowStore,
 			windowHeapSliceStore,
@@ -653,5 +699,5 @@ func (f *factory) dagProcesses(config *Config,
 			config.DifficultyAdjustmentWindowSize)
 	}
 
-	return reachabilityManagers, dagTopologyManagers, ghostdagManagers, dagTraversalManagers
+	return dagTopologyManagers, ghostdagManagers, dagTraversalManagers
 }

domain/consensus/factory_test.go

@@ -24,8 +24,12 @@ func TestNewConsensus(t *testing.T) {
 		t.Fatalf("error in NewLevelDB: %s", err)
 	}
 
-	_, err = f.NewConsensus(config, db, &prefix.Prefix{})
+	_, shouldMigrate, err := f.NewConsensus(config, db, &prefix.Prefix{})
 	if err != nil {
 		t.Fatalf("error in NewConsensus: %+v", err)
 	}
+
+	if shouldMigrate {
+		t.Fatalf("A fresh consensus should never return shouldMigrate=true")
+	}
 }

domain/consensus/model/interface_datastructures_blockrelationstore.go

@@ -9,4 +9,5 @@ type BlockRelationStore interface {
 	IsStaged(stagingArea *StagingArea) bool
 	BlockRelation(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (*BlockRelations, error)
 	Has(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (bool, error)
+	UnstageAll(stagingArea *StagingArea)
 }

domain/consensus/model/interface_datastructures_ghostdagdatastore.go

@@ -8,4 +8,5 @@ type GHOSTDAGDataStore interface {
 	Stage(stagingArea *StagingArea, blockHash *externalapi.DomainHash, blockGHOSTDAGData *externalapi.BlockGHOSTDAGData, isTrustedData bool)
 	IsStaged(stagingArea *StagingArea) bool
 	Get(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash, isTrustedData bool) (*externalapi.BlockGHOSTDAGData, error)
+	UnstageAll(stagingArea *StagingArea)
 }

domain/consensus/model/interface_datastructures_reachabilitydatastore.go

@@ -11,4 +11,5 @@ type ReachabilityDataStore interface {
 	ReachabilityData(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (ReachabilityData, error)
 	HasReachabilityData(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (bool, error)
 	ReachabilityReindexRoot(dbContext DBReader, stagingArea *StagingArea) (*externalapi.DomainHash, error)
+	Delete(dbContext DBWriter) error
 }

domain/consensus/model/interface_processes_dagtopologymanager.go

@@ -13,7 +13,7 @@ type DAGTopologyManager interface {
 	IsAncestorOfAny(stagingArea *StagingArea, blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error)
 	IsAnyAncestorOf(stagingArea *StagingArea, potentialAncestors []*externalapi.DomainHash, blockHash *externalapi.DomainHash) (bool, error)
 	IsInSelectedParentChainOf(stagingArea *StagingArea, blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
-	ChildInSelectedParentChainOf(stagingArea *StagingArea, context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error)
+	ChildInSelectedParentChainOf(stagingArea *StagingArea, lowHash, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error)
 
 	SetParents(stagingArea *StagingArea, blockHash *externalapi.DomainHash, parentHashes []*externalapi.DomainHash) error
 }

domain/consensus/model/reachabilitydata.go

@@ -104,6 +104,7 @@ type FutureCoveringTreeNodeSet []*externalapi.DomainHash
 
 // Clone returns a clone of FutureCoveringTreeNodeSet
 func (fctns FutureCoveringTreeNodeSet) Clone() FutureCoveringTreeNodeSet {
+	//return fctns
 	return externalapi.CloneHashes(fctns)
 }
 

domain/consensus/processes/blockprocessor/blockprocessor.go

@@ -22,7 +22,7 @@ type blockProcessor struct {
 	pruningManager        model.PruningManager
 	blockValidator        model.BlockValidator
 	dagTopologyManager    model.DAGTopologyManager
-	reachabilityManagers  []model.ReachabilityManager
+	reachabilityManager   model.ReachabilityManager
 	difficultyManager     model.DifficultyManager
 	pastMedianTimeManager model.PastMedianTimeManager
 	coinbaseManager       model.CoinbaseManager
@@ -60,7 +60,7 @@ func New(
 	pruningManager model.PruningManager,
 	blockValidator model.BlockValidator,
 	dagTopologyManager model.DAGTopologyManager,
-	reachabilityManagers []model.ReachabilityManager,
+	reachabilityManager model.ReachabilityManager,
 	difficultyManager model.DifficultyManager,
 	pastMedianTimeManager model.PastMedianTimeManager,
 	coinbaseManager model.CoinbaseManager,
@@ -94,7 +94,7 @@ func New(
 		pruningManager:        pruningManager,
 		blockValidator:        blockValidator,
 		dagTopologyManager:    dagTopologyManager,
-		reachabilityManagers:  reachabilityManagers,
+		reachabilityManager:   reachabilityManager,
 		difficultyManager:     difficultyManager,
 		pastMedianTimeManager: pastMedianTimeManager,
 		coinbaseManager:       coinbaseManager,

domain/consensus/processes/blockprocessor/validate_and_insert_block.go

@@ -254,19 +254,7 @@ func (bp *blockProcessor) updateReachabilityReindexRoot(stagingArea *model.Stagi
 		return nil
 	}
 
-	headersSelectedTipHeader, err := bp.blockHeaderStore.BlockHeader(bp.databaseContext, stagingArea, headersSelectedTip)
+	return bp.reachabilityManager.UpdateReindexRoot(stagingArea, headersSelectedTip)
-	if err != nil {
-		return err
-	}
-
-	headersSelectedTipHeaderBlockLevel := headersSelectedTipHeader.BlockLevel(bp.maxBlockLevel)
-	for blockLevel := 0; blockLevel <= headersSelectedTipHeaderBlockLevel; blockLevel++ {
-		err := bp.reachabilityManagers[blockLevel].UpdateReindexRoot(stagingArea, headersSelectedTip)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
 }
 
 func (bp *blockProcessor) checkBlockStatus(stagingArea *model.StagingArea, block *externalapi.DomainBlock) error {

domain/consensus/processes/blockvalidator/block_header_in_context.go

@@ -62,14 +62,11 @@ func (v *blockValidator) ValidateHeaderInContext(stagingArea *model.StagingArea,
 		return err
 	}
 	if !hasReachabilityData {
-		blockLevel := header.BlockLevel(v.maxBlockLevel)
+		err = v.reachabilityManager.AddBlock(stagingArea, blockHash)
-		for i := 0; i <= blockLevel; i++ {
-			err = v.reachabilityManagers[i].AddBlock(stagingArea, blockHash)
 		if err != nil {
 			return err
 		}
 	}
-	}
 
 	if !isBlockWithTrustedData {
 		err = v.checkIndirectParents(stagingArea, header)

domain/consensus/processes/blockvalidator/blockvalidator.go

@@ -37,7 +37,7 @@ type blockValidator struct {
 	coinbaseManager       model.CoinbaseManager
 	mergeDepthManager     model.MergeDepthManager
 	pruningStore          model.PruningStore
-	reachabilityManagers  []model.ReachabilityManager
+	reachabilityManager   model.ReachabilityManager
 	finalityManager       model.FinalityManager
 	blockParentBuilder    model.BlockParentBuilder
 	pruningManager        model.PruningManager
@@ -77,7 +77,7 @@ func New(powMax *big.Int,
 	dagTraversalManager model.DAGTraversalManager,
 	coinbaseManager model.CoinbaseManager,
 	mergeDepthManager model.MergeDepthManager,
-	reachabilityManagers []model.ReachabilityManager,
+	reachabilityManager model.ReachabilityManager,
 	finalityManager model.FinalityManager,
 	blockParentBuilder model.BlockParentBuilder,
 	pruningManager model.PruningManager,
@@ -118,7 +118,7 @@ func New(powMax *big.Int,
 		dagTraversalManager:         dagTraversalManager,
 		coinbaseManager:             coinbaseManager,
 		mergeDepthManager:           mergeDepthManager,
-		reachabilityManagers:        reachabilityManagers,
+		reachabilityManager:         reachabilityManager,
 		finalityManager:             finalityManager,
 		blockParentBuilder:          blockParentBuilder,
 		pruningManager:              pruningManager,

domain/consensus/processes/dagtopologymanager/dagtopologymanager.go

@@ -105,6 +105,17 @@ func (dtm *dagTopologyManager) IsAnyAncestorOf(stagingArea *model.StagingArea, p
 
 // IsInSelectedParentChainOf returns true if blockHashA is in the selected parent chain of blockHashB
 func (dtm *dagTopologyManager) IsInSelectedParentChainOf(stagingArea *model.StagingArea, blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) {
+
+	// Virtual doesn't have reachability data, therefore, it should be treated as a special case -
+	// use its selected parent as blockHashB.
+	if blockHashB == model.VirtualBlockHash {
+		ghostdagData, err := dtm.ghostdagStore.Get(dtm.databaseContext, stagingArea, blockHashB, false)
+		if err != nil {
+			return false, err
+		}
+		blockHashB = ghostdagData.SelectedParent()
+	}
+
 	return dtm.reachabilityManager.IsReachabilityTreeAncestorOf(stagingArea, blockHashA, blockHashB)
 }
 
@@ -176,12 +187,11 @@ func (dtm *dagTopologyManager) SetParents(stagingArea *model.StagingArea, blockH
 	return nil
 }
 
-// ChildInSelectedParentChainOf returns the child of `context` that is in the selected-parent-chain of `highHash`
+// ChildInSelectedParentChainOf returns the child of `lowHash` that is in the selected-parent-chain of `highHash`
-func (dtm *dagTopologyManager) ChildInSelectedParentChainOf(stagingArea *model.StagingArea,
+func (dtm *dagTopologyManager) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, lowHash, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) {
-	context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) {
 
 	// Virtual doesn't have reachability data, therefore, it should be treated as a special case -
-	// use it's selected parent as highHash.
+	// use its selected parent as highHash.
 	specifiedHighHash := highHash
 	if highHash == model.VirtualBlockHash {
 		ghostdagData, err := dtm.ghostdagStore.Get(dtm.databaseContext, stagingArea, highHash, false)
@@ -191,20 +201,20 @@ func (dtm *dagTopologyManager) ChildInSelectedParentChainOf(stagingArea *model.S
 		selectedParent := ghostdagData.SelectedParent()
 
 		// In case where `context` is an immediate parent of `highHash`
-		if context.Equal(selectedParent) {
+		if lowHash.Equal(selectedParent) {
 			return highHash, nil
 		}
 		highHash = selectedParent
 	}
 
-	isInSelectedParentChain, err := dtm.IsInSelectedParentChainOf(stagingArea, context, highHash)
+	isInSelectedParentChain, err := dtm.IsInSelectedParentChainOf(stagingArea, lowHash, highHash)
 	if err != nil {
 		return nil, err
 	}
 	if !isInSelectedParentChain {
-		return nil, errors.Errorf("context(%s) is not in the selected-parent-chain of highHash(%s)",
+		return nil, errors.Errorf("Claimed chain ancestor (%s) is not in the selected-parent-chain of highHash (%s)",
-			context, specifiedHighHash)
+			lowHash, specifiedHighHash)
 	}
 
-	return dtm.reachabilityManager.FindNextAncestor(stagingArea, highHash, context)
+	return dtm.reachabilityManager.FindNextAncestor(stagingArea, highHash, lowHash)
 }

domain/consensus/processes/dagtraversalmanager/dagtraversalmanager.go

@@ -14,7 +14,7 @@ type dagTraversalManager struct {
 	dagTopologyManager             model.DAGTopologyManager
 	ghostdagManager                model.GHOSTDAGManager
 	ghostdagDataStore              model.GHOSTDAGDataStore
-	reachabilityDataStore          model.ReachabilityDataStore
+	reachabilityManager            model.ReachabilityManager
 	daaWindowStore                 model.BlocksWithTrustedDataDAAWindowStore
 	genesisHash                    *externalapi.DomainHash
 	difficultyAdjustmentWindowSize int
@@ -26,7 +26,7 @@ func New(
 	databaseContext model.DBReader,
 	dagTopologyManager model.DAGTopologyManager,
 	ghostdagDataStore model.GHOSTDAGDataStore,
-	reachabilityDataStore model.ReachabilityDataStore,
+	reachabilityManager model.ReachabilityManager,
 	ghostdagManager model.GHOSTDAGManager,
 	daaWindowStore model.BlocksWithTrustedDataDAAWindowStore,
 	windowHeapSliceStore model.WindowHeapSliceStore,
@@ -36,7 +36,7 @@ func New(
 		databaseContext:     databaseContext,
 		dagTopologyManager:  dagTopologyManager,
 		ghostdagDataStore:   ghostdagDataStore,
-		reachabilityDataStore: reachabilityDataStore,
+		reachabilityManager: reachabilityManager,
 		ghostdagManager:     ghostdagManager,
 		daaWindowStore:      daaWindowStore,
 

domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go

@@ -97,25 +97,13 @@ func (dtm *dagTraversalManager) SelectedChildIterator(stagingArea *model.Staging
 var errNoSelectedChild = errors.New("errNoSelectedChild")
 
 func (dtm *dagTraversalManager) SelectedChild(stagingArea *model.StagingArea,
-	context, blockHash *externalapi.DomainHash) (*externalapi.DomainHash, error) {
+	highHash, lowHash *externalapi.DomainHash) (*externalapi.DomainHash, error) {
 
-	data, err := dtm.reachabilityDataStore.ReachabilityData(dtm.databaseContext, stagingArea, blockHash)
+	// The selected child is in fact the next reachability tree nextAncestor
+	nextAncestor, err := dtm.reachabilityManager.FindNextAncestor(stagingArea, highHash, lowHash)
 	if err != nil {
-		return nil, err
-	}
-
-	for _, child := range data.Children() {
-		isChildInSelectedParentChainOfHighHash, err := dtm.dagTopologyManager.IsInSelectedParentChainOf(
-			stagingArea, child, context)
-		if err != nil {
-			return nil, err
-		}
-
-		if isChildInSelectedParentChainOfHighHash {
-			return child, nil
-		}
-	}
-
 		return nil, errors.Wrapf(errNoSelectedChild, "no selected child for %s from the point of view of %s",
-		blockHash, context)
+			lowHash, highHash)
+	}
+	return nextAncestor, nil
 }

domain/consensus/processes/finalitymanager/finality_manager.go

@@ -13,6 +13,7 @@ type finalityManager struct {
 	dagTopologyManager model.DAGTopologyManager
 	finalityStore      model.FinalityStore
 	ghostdagDataStore  model.GHOSTDAGDataStore
+	pruningStore       model.PruningStore
 	genesisHash        *externalapi.DomainHash
 	finalityDepth      uint64
 }
@@ -22,6 +23,7 @@ func New(databaseContext model.DBReader,
 	dagTopologyManager model.DAGTopologyManager,
 	finalityStore model.FinalityStore,
 	ghostdagDataStore model.GHOSTDAGDataStore,
+	pruningStore model.PruningStore,
 	genesisHash *externalapi.DomainHash,
 	finalityDepth uint64) model.FinalityManager {
 
@@ -31,6 +33,7 @@ func New(databaseContext model.DBReader,
 		dagTopologyManager: dagTopologyManager,
 		finalityStore:      finalityStore,
 		ghostdagDataStore:  ghostdagDataStore,
+		pruningStore:       pruningStore,
 		finalityDepth:      finalityDepth,
 	}
 }
@@ -96,6 +99,27 @@ func (fm *finalityManager) calculateFinalityPoint(stagingArea *model.StagingArea
 		return fm.genesisHash, nil
 	}
 
+	pruningPoint, err := fm.pruningStore.PruningPoint(fm.databaseContext, stagingArea)
+	if err != nil {
+		return nil, err
+	}
+	pruningPointGhostdagData, err := fm.ghostdagDataStore.Get(fm.databaseContext, stagingArea, pruningPoint, false)
+	if err != nil {
+		return nil, err
+	}
+	if ghostdagData.BlueScore() < pruningPointGhostdagData.BlueScore()+fm.finalityDepth {
+		log.Debugf("%s blue score less than finality distance over pruning point - returning virtual genesis as finality point", blockHash)
+		return model.VirtualGenesisBlockHash, nil
+	}
+	isPruningPointOnChain, err := fm.dagTopologyManager.IsInSelectedParentChainOf(stagingArea, pruningPoint, blockHash)
+	if err != nil {
+		return nil, err
+	}
+	if !isPruningPointOnChain {
+		log.Debugf("pruning point not in selected chain of %s - returning virtual genesis as finality point", blockHash)
+		return model.VirtualGenesisBlockHash, nil
+	}
+
 	selectedParent := ghostdagData.SelectedParent()
 	if selectedParent.Equal(fm.genesisHash) {
 		return fm.genesisHash, nil
@@ -105,6 +129,12 @@ func (fm *finalityManager) calculateFinalityPoint(stagingArea *model.StagingArea
 	if err != nil {
 		return nil, err
 	}
+	// In this case we expect the pruning point or a block above it to be the finality point.
+	// Note that above we already verified the chain and distance conditions for this
+	if current.Equal(model.VirtualGenesisBlockHash) {
+		current = pruningPoint
+	}
+
 	requiredBlueScore := ghostdagData.BlueScore() - fm.finalityDepth
 	log.Debugf("%s's finality point is the one having the highest blue score lower then %d", blockHash, requiredBlueScore)
 

domain/consensus/processes/ghostdagmanager/ghostdag_test.go

@@ -320,10 +320,6 @@ func (ds *GHOSTDAGDataStoreImpl) IsStaged(*model.StagingArea) bool {
 	panic("implement me")
 }
 
-func (ds *GHOSTDAGDataStoreImpl) Discard() {
-	panic("implement me")
-}
-
 func (ds *GHOSTDAGDataStoreImpl) Commit(dbTx model.DBTransaction) error {
 	panic("implement me")
 }
@@ -336,11 +332,15 @@ func (ds *GHOSTDAGDataStoreImpl) Get(dbContext model.DBReader, stagingArea *mode
 	return nil, nil
 }
 
+func (ds *GHOSTDAGDataStoreImpl) UnstageAll(stagingArea *model.StagingArea) {
+	panic("implement me")
+}
+
 type DAGTopologyManagerImpl struct {
 	parentsMap map[externalapi.DomainHash][]*externalapi.DomainHash
 }
 
-func (dt *DAGTopologyManagerImpl) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) {
+func (dt *DAGTopologyManagerImpl) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, lowHash, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) {
 	panic("implement me")
 }
 

domain/consensus/processes/pruningproofmanager/pruningproofmanager.go

@@ -9,6 +9,7 @@ import (
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/processes/dagtopologymanager"
+	"github.com/kaspanet/kaspad/domain/consensus/processes/dagtraversalmanager"
 	"github.com/kaspanet/kaspad/domain/consensus/processes/ghostdagmanager"
 	"github.com/kaspanet/kaspad/domain/consensus/processes/reachabilitymanager"
 	"github.com/kaspanet/kaspad/domain/consensus/ruleerrors"
@@ -26,9 +27,10 @@ type pruningProofManager struct {
 
 	dagTopologyManagers  []model.DAGTopologyManager
 	ghostdagManagers     []model.GHOSTDAGManager
-	reachabilityManagers []model.ReachabilityManager
+	reachabilityManager  model.ReachabilityManager
 	dagTraversalManagers []model.DAGTraversalManager
 	parentsManager       model.ParentsManager
+	pruningManager       model.PruningManager
 
 	ghostdagDataStores    []model.GHOSTDAGDataStore
 	pruningStore          model.PruningStore
@@ -36,6 +38,8 @@ type pruningProofManager struct {
 	blockStatusStore      model.BlockStatusStore
 	finalityStore         model.FinalityStore
 	consensusStateStore   model.ConsensusStateStore
+	blockRelationStore    model.BlockRelationStore
+	reachabilityDataStore model.ReachabilityDataStore
 
 	genesisHash   *externalapi.DomainHash
 	k             externalapi.KType
@@ -52,9 +56,10 @@ func New(
 
 	dagTopologyManagers []model.DAGTopologyManager,
 	ghostdagManagers []model.GHOSTDAGManager,
-	reachabilityManagers []model.ReachabilityManager,
+	reachabilityManager model.ReachabilityManager,
 	dagTraversalManagers []model.DAGTraversalManager,
 	parentsManager model.ParentsManager,
+	pruningManager model.PruningManager,
 
 	ghostdagDataStores []model.GHOSTDAGDataStore,
 	pruningStore model.PruningStore,
@@ -62,6 +67,8 @@ func New(
 	blockStatusStore model.BlockStatusStore,
 	finalityStore model.FinalityStore,
 	consensusStateStore model.ConsensusStateStore,
+	blockRelationStore model.BlockRelationStore,
+	reachabilityDataStore model.ReachabilityDataStore,
 
 	genesisHash *externalapi.DomainHash,
 	k externalapi.KType,
@@ -73,9 +80,10 @@ func New(
 		databaseContext:      databaseContext,
 		dagTopologyManagers:  dagTopologyManagers,
 		ghostdagManagers:     ghostdagManagers,
-		reachabilityManagers: reachabilityManagers,
+		reachabilityManager:  reachabilityManager,
 		dagTraversalManagers: dagTraversalManagers,
 		parentsManager:       parentsManager,
+		pruningManager:       pruningManager,
 
 		ghostdagDataStores:    ghostdagDataStores,
 		pruningStore:          pruningStore,
@@ -83,6 +91,8 @@ func New(
 		blockStatusStore:      blockStatusStore,
 		finalityStore:         finalityStore,
 		consensusStateStore:   consensusStateStore,
+		blockRelationStore:    blockRelationStore,
+		reachabilityDataStore: reachabilityDataStore,
 
 		genesisHash:   genesisHash,
 		k:             k,
@@ -611,6 +621,209 @@ func (ppm *pruningProofManager) dagProcesses(
 	return reachabilityManagers, dagTopologyManagers, ghostdagManagers
 }
 
+func (ppm *pruningProofManager) ghostdagDataWithoutPrunedBlocks(stagingArea *model.StagingArea, targetReachabilityDataStore model.ReachabilityDataStore,
+	data *externalapi.BlockGHOSTDAGData) (*externalapi.BlockGHOSTDAGData, bool, error) {
+
+	changed := false
+	mergeSetBlues := make([]*externalapi.DomainHash, 0, len(data.MergeSetBlues()))
+	for _, blockHash := range data.MergeSetBlues() {
+		hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, blockHash)
+		if err != nil {
+			return nil, false, err
+		}
+		if !hasReachabilityData {
+			changed = true
+			if data.SelectedParent().Equal(blockHash) {
+				mergeSetBlues = append(mergeSetBlues, model.VirtualGenesisBlockHash)
+			}
+			continue
+		}
+
+		mergeSetBlues = append(mergeSetBlues, blockHash)
+	}
+
+	mergeSetReds := make([]*externalapi.DomainHash, 0, len(data.MergeSetReds()))
+	for _, blockHash := range data.MergeSetReds() {
+		hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, blockHash)
+		if err != nil {
+			return nil, false, err
+		}
+		if !hasReachabilityData {
+			changed = true
+			continue
+		}
+
+		mergeSetReds = append(mergeSetReds, blockHash)
+	}
+
+	selectedParent := data.SelectedParent()
+	hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, data.SelectedParent())
+	if err != nil {
+		return nil, false, err
+	}
+
+	if !hasReachabilityData {
+		changed = true
+		selectedParent = model.VirtualGenesisBlockHash
+	}
+
+	return externalapi.NewBlockGHOSTDAGData(
+		data.BlueScore(),
+		data.BlueWork(),
+		selectedParent,
+		mergeSetBlues,
+		mergeSetReds,
+		data.BluesAnticoneSizes(),
+	), changed, nil
+}
+
+func (ppm *pruningProofManager) populateProofReachabilityAndHeaders(pruningPointProof *externalapi.PruningPointProof,
+	targetReachabilityDataStore model.ReachabilityDataStore) error {
+	// We build a DAG of all multi-level relations between blocks in the proof. We make a upHeap of all blocks, so we can iterate
+	// over them in a topological way, and then build a DAG where we use all multi-level parents of a block to create edges, except
+	// parents that are already in the past of another parent (This can happen between two levels). We run GHOSTDAG on each block of
+	// that DAG, because GHOSTDAG is a requirement to calculate reachability. We then dismiss the GHOSTDAG data because it's not related
+	// to the GHOSTDAG data of the real DAG, and was used only for reachability.
+
+	// We need two staging areas: stagingArea which is used to commit the reachability data, and tmpStagingArea for the GHOSTDAG data
+	// of allProofBlocksUpHeap. The reason we need two areas is that we use the real GHOSTDAG data in order to order the heap in a topological
+	// way, and fake GHOSTDAG data for calculating reachability.
+	stagingArea := model.NewStagingArea()
+	tmpStagingArea := model.NewStagingArea()
+
+	bucket := consensusDB.MakeBucket([]byte("TMP"))
+	ghostdagDataStoreForTargetReachabilityManager := ghostdagdatastore.New(bucket, 0, false)
+	ghostdagDataStoreForTargetReachabilityManager.Stage(stagingArea, model.VirtualGenesisBlockHash, externalapi.NewBlockGHOSTDAGData(
+		0,
+		big.NewInt(0),
+		nil,
+		nil,
+		nil,
+		nil,
+	), false)
+	targetReachabilityManager := reachabilitymanager.New(ppm.databaseContext, ghostdagDataStoreForTargetReachabilityManager, targetReachabilityDataStore)
+	blockRelationStoreForTargetReachabilityManager := blockrelationstore.New(bucket, 0, false)
+	dagTopologyManagerForTargetReachabilityManager := dagtopologymanager.New(ppm.databaseContext, targetReachabilityManager, blockRelationStoreForTargetReachabilityManager, nil)
+	ghostdagManagerForTargetReachabilityManager := ghostdagmanager.New(ppm.databaseContext, dagTopologyManagerForTargetReachabilityManager, ghostdagDataStoreForTargetReachabilityManager, ppm.blockHeaderStore, 0, nil)
+	err := dagTopologyManagerForTargetReachabilityManager.SetParents(stagingArea, model.VirtualGenesisBlockHash, nil)
+	if err != nil {
+		return err
+	}
+
+	dagTopologyManager := dagtopologymanager.New(ppm.databaseContext, targetReachabilityManager, nil, nil)
+	ghostdagDataStore := ghostdagdatastore.New(bucket, 0, false)
+	tmpGHOSTDAGManager := ghostdagmanager.New(ppm.databaseContext, nil, ghostdagDataStore, nil, 0, nil)
+	dagTraversalManager := dagtraversalmanager.New(ppm.databaseContext, nil, ghostdagDataStore, nil, tmpGHOSTDAGManager, nil, nil, nil, 0)
+	allProofBlocksUpHeap := dagTraversalManager.NewUpHeap(tmpStagingArea)
+	dag := make(map[externalapi.DomainHash]struct {
+		parents hashset.HashSet
+		header  externalapi.BlockHeader
+	})
+	for _, headers := range pruningPointProof.Headers {
+		for _, header := range headers {
+			blockHash := consensushashing.HeaderHash(header)
+			if _, ok := dag[*blockHash]; ok {
+				continue
+			}
+
+			dag[*blockHash] = struct {
+				parents hashset.HashSet
+				header  externalapi.BlockHeader
+			}{parents: hashset.New(), header: header}
+
+			for level := 0; level <= ppm.maxBlockLevel; level++ {
+				for _, parent := range ppm.parentsManager.ParentsAtLevel(header, level) {
+					parent := parent
+					dag[*blockHash].parents.Add(parent)
+				}
+			}
+
+			// We stage temporary GHOSTDAG data that is needed in order to sort allProofBlocksUpHeap.
+			ghostdagDataStore.Stage(tmpStagingArea, blockHash, externalapi.NewBlockGHOSTDAGData(header.BlueScore(), header.BlueWork(), nil, nil, nil, nil), false)
+			err := allProofBlocksUpHeap.Push(blockHash)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	var selectedTip *externalapi.DomainHash
+	for allProofBlocksUpHeap.Len() > 0 {
+		blockHash := allProofBlocksUpHeap.Pop()
+		block := dag[*blockHash]
+		parentsHeap := dagTraversalManager.NewDownHeap(tmpStagingArea)
+		for parent := range block.parents {
+			parent := parent
+			if _, ok := dag[parent]; !ok {
+				continue
+			}
+
+			err := parentsHeap.Push(&parent)
+			if err != nil {
+				return err
+			}
+		}
+
+		fakeParents := []*externalapi.DomainHash{}
+		for parentsHeap.Len() > 0 {
+			parent := parentsHeap.Pop()
+			isAncestorOfAny, err := dagTopologyManager.IsAncestorOfAny(stagingArea, parent, fakeParents)
+			if err != nil {
+				return err
+			}
+
+			if isAncestorOfAny {
+				continue
+			}
+
+			fakeParents = append(fakeParents, parent)
+		}
+
+		if len(fakeParents) == 0 {
+			fakeParents = append(fakeParents, model.VirtualGenesisBlockHash)
+		}
+
+		err := dagTopologyManagerForTargetReachabilityManager.SetParents(stagingArea, blockHash, fakeParents)
+		if err != nil {
+			return err
+		}
+
+		err = ghostdagManagerForTargetReachabilityManager.GHOSTDAG(stagingArea, blockHash)
+		if err != nil {
+			return err
+		}
+
+		err = targetReachabilityManager.AddBlock(stagingArea, blockHash)
+		if err != nil {
+			return err
+		}
+
+		if selectedTip == nil {
+			selectedTip = blockHash
+		} else {
+			selectedTip, err = ghostdagManagerForTargetReachabilityManager.ChooseSelectedParent(stagingArea, selectedTip, blockHash)
+			if err != nil {
+				return err
+			}
+		}
+
+		if selectedTip.Equal(blockHash) {
+			err := targetReachabilityManager.UpdateReindexRoot(stagingArea, selectedTip)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	ghostdagDataStoreForTargetReachabilityManager.UnstageAll(stagingArea)
+	blockRelationStoreForTargetReachabilityManager.UnstageAll(stagingArea)
+	err = staging.CommitAllChanges(ppm.databaseContext, stagingArea)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // ApplyPruningPointProof applies the given pruning proof to the current consensus. Specifically,
 // it's meant to be used against the StagingConsensus during headers-proof IBD. Note that for
 // performance reasons this operation is NOT atomic. If the process fails for whatever reason
@@ -619,9 +832,25 @@ func (ppm *pruningProofManager) ApplyPruningPointProof(pruningPointProof *extern
 	onEnd := logger.LogAndMeasureExecutionTime(log, "ApplyPruningPointProof")
 	defer onEnd()
 
+	stagingArea := model.NewStagingArea()
+	for _, headers := range pruningPointProof.Headers {
+		for _, header := range headers {
+			blockHash := consensushashing.HeaderHash(header)
+			ppm.blockHeaderStore.Stage(stagingArea, blockHash, header)
+		}
+	}
+	err := staging.CommitAllChanges(ppm.databaseContext, stagingArea)
+	if err != nil {
+		return err
+	}
+
+	err = ppm.populateProofReachabilityAndHeaders(pruningPointProof, ppm.reachabilityDataStore)
+	if err != nil {
+		return err
+	}
+
 	for blockLevel, headers := range pruningPointProof.Headers {
 		log.Infof("Applying level %d from the pruning point proof", blockLevel)
-		var selectedTip *externalapi.DomainHash
 		for i, header := range headers {
 			if i%1000 == 0 {
 				log.Infof("Applying level %d from the pruning point proof - applied %d headers out of %d", blockLevel, i, len(headers))
@@ -687,27 +916,6 @@ func (ppm *pruningProofManager) ApplyPruningPointProof(pruningPointProof *extern
 				ppm.blockStatusStore.Stage(stagingArea, blockHash, externalapi.StatusHeaderOnly)
 			}
 
-			if selectedTip == nil {
-				selectedTip = blockHash
-			} else {
-				selectedTip, err = ppm.ghostdagManagers[blockLevel].ChooseSelectedParent(stagingArea, selectedTip, blockHash)
-				if err != nil {
-					return err
-				}
-			}
-
-			err = ppm.reachabilityManagers[blockLevel].AddBlock(stagingArea, blockHash)
-			if err != nil {
-				return err
-			}
-
-			if selectedTip.Equal(blockHash) {
-				err := ppm.reachabilityManagers[blockLevel].UpdateReindexRoot(stagingArea, selectedTip)
-				if err != nil {
-					return err
-				}
-			}
-
 			err = staging.CommitAllChanges(ppm.databaseContext, stagingArea)
 			if err != nil {
 				return err
@@ -718,7 +926,7 @@ func (ppm *pruningProofManager) ApplyPruningPointProof(pruningPointProof *extern
 	pruningPointHeader := pruningPointProof.Headers[0][len(pruningPointProof.Headers[0])-1]
 	pruningPoint := consensushashing.HeaderHash(pruningPointHeader)
 
-	stagingArea := model.NewStagingArea()
+	stagingArea = model.NewStagingArea()
 	ppm.consensusStateStore.StageTips(stagingArea, []*externalapi.DomainHash{pruningPoint})
 	return staging.CommitAllChanges(ppm.databaseContext, stagingArea)
 }

domain/consensus/processes/reachabilitymanager/reachability_test.go

@@ -16,7 +16,7 @@ type reachabilityDataStoreMock struct {
 	reachabilityReindexRootStaging *externalapi.DomainHash
 }
 
-func (r *reachabilityDataStoreMock) Discard() {
+func (r *reachabilityDataStoreMock) Delete(_ model.DBWriter) error {
 	panic("implement me")
 }
 

domain/consensus/processes/reachabilitymanager/tree.go

@@ -152,6 +152,10 @@ func (rt *reachabilityManager) IsReachabilityTreeAncestorOf(stagingArea *model.S
 func (rt *reachabilityManager) FindNextAncestor(stagingArea *model.StagingArea,
 	descendant, ancestor *externalapi.DomainHash) (*externalapi.DomainHash, error) {
 
+	if ancestor.Equal(descendant) {
+		return nil, errors.Errorf("ancestor is equal to descendant")
+	}
+
 	childrenOfAncestor, err := rt.children(stagingArea, ancestor)
 	if err != nil {
 		return nil, err

domain/consensus/test_consensus_getters.go

@@ -60,7 +60,7 @@ func (tc *testConsensus) PruningStore() model.PruningStore {
 }
 
 func (tc *testConsensus) ReachabilityDataStore() model.ReachabilityDataStore {
-	return tc.reachabilityDataStores[0]
+	return tc.reachabilityDataStore
 }
 
 func (tc *testConsensus) UTXODiffStore() model.UTXODiffStore {

domain/consensus/utils/reachabilitydata/reachability_data.go

@@ -57,6 +57,7 @@ func (rd *reachabilityData) FutureCoveringSet() model.FutureCoveringTreeNodeSet
 }
 
 func (rd *reachabilityData) CloneMutable() model.MutableReachabilityData {
+	//return rd
 	return &reachabilityData{
 		children:          externalapi.CloneHashes(rd.children),
 		parent:            rd.parent,

domain/domain.go

@@ -82,11 +82,15 @@ func (d *domain) InitStagingConsensus() error {
 	cfg := *d.consensusConfig
 	cfg.SkipAddingGenesis = true
 
-	consensusInstance, err := consensusFactory.NewConsensus(&cfg, d.db, inactivePrefix)
+	consensusInstance, shouldMigrate, err := consensusFactory.NewConsensus(&cfg, d.db, inactivePrefix)
 	if err != nil {
 		return err
 	}
 
+	if shouldMigrate {
+		return errors.Errorf("A fresh consensus should never return shouldMigrate=true")
+	}
+
 	d.stagingConsensus = &consensusInstance
 	return nil
 }
@@ -183,7 +187,7 @@ func New(consensusConfig *consensus.Config, mempoolConfig *mempool.Config, db in
 	}
 
 	consensusFactory := consensus.NewFactory()
-	consensusInstance, err := consensusFactory.NewConsensus(consensusConfig, db, activePrefix)
+	consensusInstance, shouldMigrate, err := consensusFactory.NewConsensus(consensusConfig, db, activePrefix)
 	if err != nil {
 		return nil, err
 	}
@@ -194,6 +198,13 @@ func New(consensusConfig *consensus.Config, mempoolConfig *mempool.Config, db in
 		db:              db,
 	}
 
+	if shouldMigrate {
+		err := domainInstance.migrate()
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	miningManagerFactory := miningmanager.NewFactory()
 
 	// We create a consensus wrapper because the actual consensus might change

domain/log.go

@@ -0,0 +1,7 @@
+package domain
+
+import (
+	"github.com/kaspanet/kaspad/infrastructure/logger"
+)
+
+var log = logger.RegisterSubSystem("DOMN")

domain/migrate.go

@@ -0,0 +1,230 @@
+package domain
+
+import (
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+	"math"
+)
+
+func (d *domain) migrate() error {
+	log.Infof("Starting migration")
+	err := d.InitStagingConsensus()
+	if err != nil {
+		return err
+	}
+
+	err = syncConsensuses(d.Consensus(), d.StagingConsensus())
+	if err != nil {
+		return err
+	}
+
+	err = d.CommitStagingConsensus()
+	if err != nil {
+		return err
+	}
+
+	log.Info("Done migrating")
+	return nil
+}
+
+func syncConsensuses(syncer, syncee externalapi.Consensus) error {
+	pruningPointProof, err := syncer.BuildPruningPointProof()
+	if err != nil {
+		return err
+	}
+
+	err = syncee.ApplyPruningPointProof(pruningPointProof)
+	if err != nil {
+		return err
+	}
+
+	pruningPointHeaders, err := syncer.PruningPointHeaders()
+	if err != nil {
+		return err
+	}
+
+	err = syncee.ImportPruningPoints(pruningPointHeaders)
+	if err != nil {
+		return err
+	}
+
+	pruningPointAndItsAnticone, err := syncer.PruningPointAndItsAnticone()
+	if err != nil {
+		return err
+	}
+
+	for _, blockHash := range pruningPointAndItsAnticone {
+		block, err := syncer.GetBlock(blockHash)
+		if err != nil {
+			return err
+		}
+
+		blockDAAWindowHashes, err := syncer.BlockDAAWindowHashes(blockHash)
+		if err != nil {
+			return err
+		}
+
+		ghostdagDataBlockHashes, err := syncer.TrustedBlockAssociatedGHOSTDAGDataBlockHashes(blockHash)
+		if err != nil {
+			return err
+		}
+
+		blockWithTrustedData := &externalapi.BlockWithTrustedData{
+			Block:        block,
+			DAAWindow:    make([]*externalapi.TrustedDataDataDAAHeader, 0, len(blockDAAWindowHashes)),
+			GHOSTDAGData: make([]*externalapi.BlockGHOSTDAGDataHashPair, 0, len(ghostdagDataBlockHashes)),
+		}
+
+		for i, daaBlockHash := range blockDAAWindowHashes {
+			trustedDataDataDAAHeader, err := syncer.TrustedDataDataDAAHeader(blockHash, daaBlockHash, uint64(i))
+			if err != nil {
+				return err
+			}
+			blockWithTrustedData.DAAWindow = append(blockWithTrustedData.DAAWindow, trustedDataDataDAAHeader)
+		}
+
+		for _, ghostdagDataBlockHash := range ghostdagDataBlockHashes {
+			data, err := syncer.TrustedGHOSTDAGData(ghostdagDataBlockHash)
+			if err != nil {
+				return err
+			}
+			blockWithTrustedData.GHOSTDAGData = append(blockWithTrustedData.GHOSTDAGData, &externalapi.BlockGHOSTDAGDataHashPair{
+				Hash:         ghostdagDataBlockHash,
+				GHOSTDAGData: data,
+			})
+		}
+
+		_, err = syncee.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
+		if err != nil {
+			return err
+		}
+	}
+
+	syncerVirtualSelectedParent, err := syncer.GetVirtualSelectedParent()
+	if err != nil {
+		return err
+	}
+
+	pruningPoint, err := syncer.PruningPoint()
+	if err != nil {
+		return err
+	}
+
+	missingBlocks, _, err := syncer.GetHashesBetween(pruningPoint, syncerVirtualSelectedParent, math.MaxUint64)
+	if err != nil {
+		return err
+	}
+
+	syncerTips, err := syncer.Tips()
+	if err != nil {
+		return err
+	}
+
+	for _, tip := range syncerTips {
+		if tip.Equal(syncerVirtualSelectedParent) {
+			continue
+		}
+
+		anticone, err := syncer.GetAnticone(syncerVirtualSelectedParent, tip, 0)
+		if err != nil {
+			return err
+		}
+
+		missingBlocks = append(missingBlocks, anticone...)
+	}
+
+	percents := 0
+	for i, blocksHash := range missingBlocks {
+		blockInfo, err := syncee.GetBlockInfo(blocksHash)
+		if err != nil {
+			return err
+		}
+
+		if blockInfo.Exists {
+			continue
+		}
+
+		block, err := syncer.GetBlock(blocksHash)
+		if err != nil {
+			return err
+		}
+
+		_, err = syncee.ValidateAndInsertBlock(block, false)
+		if err != nil {
+			return err
+		}
+
+		newPercents := 100 * i / len(missingBlocks)
+		if newPercents > percents {
+			percents = newPercents
+			log.Infof("Processed %d%% of the blocks", 100*i/len(missingBlocks))
+		}
+	}
+
+	var fromOutpoint *externalapi.DomainOutpoint
+	const step = 100_000
+	for {
+		outpointAndUTXOEntryPairs, err := syncer.GetPruningPointUTXOs(pruningPoint, fromOutpoint, step)
+		if err != nil {
+			return err
+		}
+		fromOutpoint = outpointAndUTXOEntryPairs[len(outpointAndUTXOEntryPairs)-1].Outpoint
+		err = syncee.AppendImportedPruningPointUTXOs(outpointAndUTXOEntryPairs)
+		if err != nil {
+			return err
+		}
+		if len(outpointAndUTXOEntryPairs) < step {
+			break
+		}
+	}
+
+	// Check that ValidateAndInsertImportedPruningPoint works given the right arguments.
+	err = syncee.ValidateAndInsertImportedPruningPoint(pruningPoint)
+	if err != nil {
+		return err
+	}
+
+	emptyCoinbase := &externalapi.DomainCoinbaseData{
+		ScriptPublicKey: &externalapi.ScriptPublicKey{
+			Script:  nil,
+			Version: 0,
+		},
+	}
+
+	// Check that we can build a block just after importing the pruning point.
+	_, err = syncee.BuildBlock(emptyCoinbase, nil)
+	if err != nil {
+		return err
+	}
+
+	estimatedVirtualDAAScoreTarget, err := syncer.GetVirtualDAAScore()
+	if err != nil {
+		return err
+	}
+
+	virtualDAAScoreStart := uint64(0)
+	percents = 0
+	for i := 0; ; i++ {
+		if i%10 == 0 {
+			virtualDAAScore, err := syncee.GetVirtualDAAScore()
+			if err != nil {
+				return err
+			}
+			newPercents := int(float64(virtualDAAScore-virtualDAAScoreStart) / float64(estimatedVirtualDAAScoreTarget-virtualDAAScoreStart) * 100)
+			if newPercents > percents {
+				percents = newPercents
+				log.Infof("Resolving virtual. Estimated progress: %d%%", percents)
+			}
+		}
+		_, isCompletelyResolved, err := syncee.ResolveVirtual()
+		if err != nil {
+			return err
+		}
+
+		if isCompletelyResolved {
+			log.Infof("Resolved virtual")
+			break
+		}
+	}
+
+	return nil
+}