diff --git a/domain/consensus/consensus.go b/domain/consensus/consensus.go index df1777fcd..8d9c983c3 100644 --- a/domain/consensus/consensus.go +++ b/domain/consensus/consensus.go @@ -35,7 +35,7 @@ type consensus struct { headerTipsManager model.HeadersSelectedTipManager mergeDepthManager model.MergeDepthManager pruningManager model.PruningManager - reachabilityManagers []model.ReachabilityManager + reachabilityManager model.ReachabilityManager finalityManager model.FinalityManager pruningProofManager model.PruningProofManager @@ -49,7 +49,7 @@ type consensus struct { consensusStateStore model.ConsensusStateStore headersSelectedTipStore model.HeaderSelectedTipStore multisetStore model.MultisetStore - reachabilityDataStores []model.ReachabilityDataStore + reachabilityDataStore model.ReachabilityDataStore utxoDiffStore model.UTXODiffStore finalityStore model.FinalityStore headersSelectedChainStore model.HeadersSelectedChainStore @@ -83,11 +83,9 @@ func (s *consensus) Init(skipAddingGenesis bool) error { // on a node with pruned header all blocks without known parents points to it. if !exists { s.blockStatusStore.Stage(stagingArea, model.VirtualGenesisBlockHash, externalapi.StatusUTXOValid) - for _, reachabilityManager := range s.reachabilityManagers { - err = reachabilityManager.Init(stagingArea) - if err != nil { - return err - } + err = s.reachabilityManager.Init(stagingArea) + if err != nil { + return err } for _, dagTopologyManager := range s.dagTopologyManagers { diff --git a/domain/consensus/datastructures/blockrelationstore/block_relation_store.go b/domain/consensus/datastructures/blockrelationstore/block_relation_store.go index 74e11628d..e8638933c 100644 --- a/domain/consensus/datastructures/blockrelationstore/block_relation_store.go +++ b/domain/consensus/datastructures/blockrelationstore/block_relation_store.go @@ -75,6 +75,11 @@ func (brs *blockRelationStore) Has(dbContext model.DBReader, stagingArea *model. return dbContext.Has(brs.hashAsKey(blockHash)) } +func (brs *blockRelationStore) UnstageAll(stagingArea *model.StagingArea) { + stagingShard := brs.stagingShard(stagingArea) + stagingShard.toAdd = make(map[externalapi.DomainHash]*model.BlockRelations) +} + func (brs *blockRelationStore) hashAsKey(hash *externalapi.DomainHash) model.DBKey { return brs.bucket.Key(hash.ByteSlice()) } diff --git a/domain/consensus/datastructures/ghostdagdatastore/ghostdag_data_store.go b/domain/consensus/datastructures/ghostdagdatastore/ghostdag_data_store.go index c2d1c563a..614b38fe6 100644 --- a/domain/consensus/datastructures/ghostdagdatastore/ghostdag_data_store.go +++ b/domain/consensus/datastructures/ghostdagdatastore/ghostdag_data_store.go @@ -69,6 +69,12 @@ func (gds *ghostdagDataStore) Get(dbContext model.DBReader, stagingArea *model.S return blockGHOSTDAGData, nil } +func (gds *ghostdagDataStore) UnstageAll(stagingArea *model.StagingArea) { + stagingShard := gds.stagingShard(stagingArea) + + stagingShard.toAdd = make(map[key]*externalapi.BlockGHOSTDAGData) +} + func (gds *ghostdagDataStore) serializeKey(k key) model.DBKey { if k.isTrustedData { return gds.trustedDataBucket.Key(k.hash.ByteSlice()) diff --git a/domain/consensus/datastructures/reachabilitydatastore/reachability_data_store.go b/domain/consensus/datastructures/reachabilitydatastore/reachability_data_store.go index a1f5712fb..32af5fb92 100644 --- a/domain/consensus/datastructures/reachabilitydatastore/reachability_data_store.go +++ b/domain/consensus/datastructures/reachabilitydatastore/reachability_data_store.go @@ -41,6 +41,27 @@ func (rds *reachabilityDataStore) StageReachabilityData(stagingArea *model.Stagi stagingShard.reachabilityData[*blockHash] = reachabilityData } +func (rds *reachabilityDataStore) Delete(dbContext model.DBWriter) error { + cursor, err := dbContext.Cursor(rds.reachabilityDataBucket) + if err != nil { + return err + } + + for ok := cursor.First(); ok; ok = cursor.Next() { + key, err := cursor.Key() + if err != nil { + return err + } + + err = dbContext.Delete(key) + if err != nil { + return err + } + } + + return dbContext.Delete(rds.reachabilityReindexRootKey) +} + // StageReachabilityReindexRoot stages the given reachabilityReindexRoot func (rds *reachabilityDataStore) StageReachabilityReindexRoot(stagingArea *model.StagingArea, reachabilityReindexRoot *externalapi.DomainHash) { stagingShard := rds.stagingShard(stagingArea) diff --git a/domain/consensus/factory.go b/domain/consensus/factory.go index d427f50ab..7bb032e08 100644 --- a/domain/consensus/factory.go +++ b/domain/consensus/factory.go @@ -1,16 +1,17 @@ package consensus import ( - "io/ioutil" - "os" - "sync" - "github.com/kaspanet/kaspad/domain/consensus/datastructures/blockwindowheapslicestore" "github.com/kaspanet/kaspad/domain/consensus/datastructures/daawindowstore" "github.com/kaspanet/kaspad/domain/consensus/model" "github.com/kaspanet/kaspad/domain/consensus/processes/blockparentbuilder" parentssanager "github.com/kaspanet/kaspad/domain/consensus/processes/parentsmanager" "github.com/kaspanet/kaspad/domain/consensus/processes/pruningproofmanager" + "github.com/pkg/errors" + "io/ioutil" + "os" + "sync" + "github.com/kaspanet/kaspad/domain/prefixmanager/prefix" "github.com/kaspanet/kaspad/util/txmass" @@ -74,7 +75,7 @@ type Config struct { // Factory instantiates new Consensuses type Factory interface { NewConsensus(config *Config, db infrastructuredatabase.Database, dbPrefix *prefix.Prefix) ( - externalapi.Consensus, error) + externalapi.Consensus, bool, error) NewTestConsensus(config *Config, testName string) ( tc testapi.TestConsensus, teardown func(keepDataDir bool), err error) @@ -106,7 +107,7 @@ func NewFactory() Factory { // NewConsensus instantiates a new Consensus func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Database, dbPrefix *prefix.Prefix) ( - externalapi.Consensus, error) { + consensusInstance externalapi.Consensus, shouldMigrate bool, err error) { dbManager := consensusdatabase.New(db) prefixBucket := consensusdatabase.MakeBucket(dbPrefix.Serialize()) @@ -129,11 +130,11 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas acceptanceDataStore := acceptancedatastore.New(prefixBucket, 200, preallocateCaches) blockStore, err := blockstore.New(dbManager, prefixBucket, 200, preallocateCaches) if err != nil { - return nil, err + return nil, false, err } blockHeaderStore, err := blockheaderstore.New(dbManager, prefixBucket, 10_000, preallocateCaches) if err != nil { - return nil, err + return nil, false, err } blockStatusStore := blockstatusstore.New(prefixBucket, pruningWindowSizePlusFinalityDepthForCache, preallocateCaches) @@ -148,11 +149,35 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas daaBlocksStore := daablocksstore.New(prefixBucket, pruningWindowSizeForCaches, int(config.FinalityDepth()), preallocateCaches) windowHeapSliceStore := blockwindowheapslicestore.New(2000, preallocateCaches) + newReachabilityDataStore := reachabilitydatastore.New(prefixBucket, pruningWindowSizePlusFinalityDepthForCache*2, preallocateCaches) blockRelationStores, reachabilityDataStores, ghostdagDataStores := dagStores(config, prefixBucket, pruningWindowSizePlusFinalityDepthForCache, pruningWindowSizeForCaches, preallocateCaches) - reachabilityManagers, dagTopologyManagers, ghostdagManagers, dagTraversalManagers := f.dagProcesses(config, dbManager, blockHeaderStore, daaWindowStore, windowHeapSliceStore, blockRelationStores, reachabilityDataStores, ghostdagDataStores) + oldReachabilityManager := reachabilitymanager.New( + dbManager, + ghostdagDataStores[0], + reachabilityDataStores[0]) + isOldReachabilityInitialized, err := reachabilityDataStores[0].HasReachabilityData(dbManager, model.NewStagingArea(), model.VirtualGenesisBlockHash) + if err != nil { + return nil, false, err + } + + newReachabilityManager := reachabilitymanager.New( + dbManager, + ghostdagDataStores[0], + newReachabilityDataStore) + reachabilityManager := newReachabilityManager + if isOldReachabilityInitialized { + reachabilityManager = oldReachabilityManager + } else { + for i := range reachabilityDataStores { + reachabilityDataStores[i] = newReachabilityDataStore + } + } + reachabilityDataStore := reachabilityDataStores[0] + + dagTopologyManagers, ghostdagManagers, dagTraversalManagers := f.dagProcesses(config, dbManager, blockHeaderStore, daaWindowStore, windowHeapSliceStore, blockRelationStores, reachabilityDataStores, ghostdagDataStores, isOldReachabilityInitialized) blockRelationStore := blockRelationStores[0] - reachabilityDataStore := reachabilityDataStores[0] + ghostdagDataStore := ghostdagDataStores[0] dagTopologyManager := dagTopologyManagers[0] @@ -227,6 +252,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas dagTopologyManager, finalityStore, ghostdagDataStore, + pruningStore, genesisHash, config.FinalityDepth()) mergeDepthManager := mergedepthmanager.New( @@ -264,7 +290,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas pruningStore, daaBlocksStore) if err != nil { - return nil, err + return nil, false, err } pruningManager := pruningmanager.New( @@ -319,7 +345,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas dagTraversalManager, coinbaseManager, mergeDepthManager, - reachabilityManagers, + reachabilityManager, finalityManager, blockParentBuilder, pruningManager, @@ -383,7 +409,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas pruningManager, blockValidator, dagTopologyManager, - reachabilityManagers, + reachabilityManager, difficultyManager, pastMedianTimeManager, coinbaseManager, @@ -411,9 +437,10 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas dbManager, dagTopologyManagers, ghostdagManagers, - reachabilityManagers, + reachabilityManager, dagTraversalManagers, parentsManager, + pruningManager, ghostdagDataStores, pruningStore, @@ -421,6 +448,8 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas blockStatusStore, finalityStore, consensusStateStore, + blockRelationStore, + reachabilityDataStore, genesisHash, config.K, @@ -450,7 +479,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas headerTipsManager: headerTipsManager, mergeDepthManager: mergeDepthManager, pruningManager: pruningManager, - reachabilityManagers: reachabilityManagers, + reachabilityManager: reachabilityManager, finalityManager: finalityManager, pruningProofManager: pruningProofManager, @@ -464,7 +493,7 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas consensusStateStore: consensusStateStore, headersSelectedTipStore: headersSelectedTipStore, multisetStore: multisetStore, - reachabilityDataStores: reachabilityDataStores, + reachabilityDataStore: reachabilityDataStore, utxoDiffStore: utxoDiffStore, finalityStore: finalityStore, headersSelectedChainStore: headersSelectedChainStore, @@ -472,25 +501,29 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas blocksWithTrustedDataDAAWindowStore: daaWindowStore, } + if isOldReachabilityInitialized { + return c, true, nil + } + err = c.Init(config.SkipAddingGenesis) if err != nil { - return nil, err + return nil, false, err } err = consensusStateManager.RecoverUTXOIfRequired() if err != nil { - return nil, err + return nil, false, err } err = pruningManager.ClearImportedPruningPointData() if err != nil { - return nil, err + return nil, false, err } err = pruningManager.UpdatePruningPointIfRequired() if err != nil { - return nil, err + return nil, false, err } - return c, nil + return c, false, nil } func (f *factory) NewTestConsensus(config *Config, testName string) ( @@ -517,11 +550,15 @@ func (f *factory) NewTestConsensus(config *Config, testName string) ( } testConsensusDBPrefix := &prefix.Prefix{} - consensusAsInterface, err := f.NewConsensus(config, db, testConsensusDBPrefix) + consensusAsInterface, shouldMigrate, err := f.NewConsensus(config, db, testConsensusDBPrefix) if err != nil { return nil, nil, err } + if shouldMigrate { + return nil, nil, errors.Errorf("A fresh consensus should never return shouldMigrate=true") + } + consensusAsImplementation := consensusAsInterface.(*consensus) testConsensusStateManager := consensusstatemanager.NewTestConsensusStateManager(consensusAsImplementation.consensusStateManager) testTransactionValidator := transactionvalidator.NewTestTransactionValidator(consensusAsImplementation.transactionValidator) @@ -532,7 +569,7 @@ func (f *factory) NewTestConsensus(config *Config, testName string) ( database: db, testConsensusStateManager: testConsensusStateManager, testReachabilityManager: reachabilitymanager.NewTestReachabilityManager(consensusAsImplementation. - reachabilityManagers[0]), + reachabilityManager), testTransactionValidator: testTransactionValidator, } tstConsensus.testBlockBuilder = blockbuilder.NewTestBlockBuilder(consensusAsImplementation.blockBuilder, tstConsensus) @@ -609,8 +646,8 @@ func (f *factory) dagProcesses(config *Config, windowHeapSliceStore model.WindowHeapSliceStore, blockRelationStores []model.BlockRelationStore, reachabilityDataStores []model.ReachabilityDataStore, - ghostdagDataStores []model.GHOSTDAGDataStore) ( - []model.ReachabilityManager, + ghostdagDataStores []model.GHOSTDAGDataStore, + isOldReachabilityInitialized bool) ( []model.DAGTopologyManager, []model.GHOSTDAGManager, []model.DAGTraversalManager, @@ -621,11 +658,20 @@ func (f *factory) dagProcesses(config *Config, ghostdagManagers := make([]model.GHOSTDAGManager, config.MaxBlockLevel+1) dagTraversalManagers := make([]model.DAGTraversalManager, config.MaxBlockLevel+1) + newReachabilityManager := reachabilitymanager.New( + dbManager, + ghostdagDataStores[0], + reachabilityDataStores[0]) + for i := 0; i <= config.MaxBlockLevel; i++ { - reachabilityManagers[i] = reachabilitymanager.New( - dbManager, - ghostdagDataStores[i], - reachabilityDataStores[i]) + if isOldReachabilityInitialized { + reachabilityManagers[i] = reachabilitymanager.New( + dbManager, + ghostdagDataStores[i], + reachabilityDataStores[i]) + } else { + reachabilityManagers[i] = newReachabilityManager + } dagTopologyManagers[i] = dagtopologymanager.New( dbManager, @@ -645,7 +691,7 @@ func (f *factory) dagProcesses(config *Config, dbManager, dagTopologyManagers[i], ghostdagDataStores[i], - reachabilityDataStores[i], + reachabilityManagers[i], ghostdagManagers[i], daaWindowStore, windowHeapSliceStore, @@ -653,5 +699,5 @@ func (f *factory) dagProcesses(config *Config, config.DifficultyAdjustmentWindowSize) } - return reachabilityManagers, dagTopologyManagers, ghostdagManagers, dagTraversalManagers + return dagTopologyManagers, ghostdagManagers, dagTraversalManagers } diff --git a/domain/consensus/factory_test.go b/domain/consensus/factory_test.go index 9f7e2aa47..8e199c25d 100644 --- a/domain/consensus/factory_test.go +++ b/domain/consensus/factory_test.go @@ -24,8 +24,12 @@ func TestNewConsensus(t *testing.T) { t.Fatalf("error in NewLevelDB: %s", err) } - _, err = f.NewConsensus(config, db, &prefix.Prefix{}) + _, shouldMigrate, err := f.NewConsensus(config, db, &prefix.Prefix{}) if err != nil { t.Fatalf("error in NewConsensus: %+v", err) } + + if shouldMigrate { + t.Fatalf("A fresh consensus should never return shouldMigrate=true") + } } diff --git a/domain/consensus/model/interface_datastructures_blockrelationstore.go b/domain/consensus/model/interface_datastructures_blockrelationstore.go index 2e8a07f8b..008d93f2f 100644 --- a/domain/consensus/model/interface_datastructures_blockrelationstore.go +++ b/domain/consensus/model/interface_datastructures_blockrelationstore.go @@ -9,4 +9,5 @@ type BlockRelationStore interface { IsStaged(stagingArea *StagingArea) bool BlockRelation(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (*BlockRelations, error) Has(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (bool, error) + UnstageAll(stagingArea *StagingArea) } diff --git a/domain/consensus/model/interface_datastructures_ghostdagdatastore.go b/domain/consensus/model/interface_datastructures_ghostdagdatastore.go index c3f622f4c..17daadd24 100644 --- a/domain/consensus/model/interface_datastructures_ghostdagdatastore.go +++ b/domain/consensus/model/interface_datastructures_ghostdagdatastore.go @@ -8,4 +8,5 @@ type GHOSTDAGDataStore interface { Stage(stagingArea *StagingArea, blockHash *externalapi.DomainHash, blockGHOSTDAGData *externalapi.BlockGHOSTDAGData, isTrustedData bool) IsStaged(stagingArea *StagingArea) bool Get(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash, isTrustedData bool) (*externalapi.BlockGHOSTDAGData, error) + UnstageAll(stagingArea *StagingArea) } diff --git a/domain/consensus/model/interface_datastructures_reachabilitydatastore.go b/domain/consensus/model/interface_datastructures_reachabilitydatastore.go index 7eda6d945..fbbac1f4d 100644 --- a/domain/consensus/model/interface_datastructures_reachabilitydatastore.go +++ b/domain/consensus/model/interface_datastructures_reachabilitydatastore.go @@ -11,4 +11,5 @@ type ReachabilityDataStore interface { ReachabilityData(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (ReachabilityData, error) HasReachabilityData(dbContext DBReader, stagingArea *StagingArea, blockHash *externalapi.DomainHash) (bool, error) ReachabilityReindexRoot(dbContext DBReader, stagingArea *StagingArea) (*externalapi.DomainHash, error) + Delete(dbContext DBWriter) error } diff --git a/domain/consensus/model/interface_processes_dagtopologymanager.go b/domain/consensus/model/interface_processes_dagtopologymanager.go index d431e8cc0..9fceff43b 100644 --- a/domain/consensus/model/interface_processes_dagtopologymanager.go +++ b/domain/consensus/model/interface_processes_dagtopologymanager.go @@ -13,7 +13,7 @@ type DAGTopologyManager interface { IsAncestorOfAny(stagingArea *StagingArea, blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error) IsAnyAncestorOf(stagingArea *StagingArea, potentialAncestors []*externalapi.DomainHash, blockHash *externalapi.DomainHash) (bool, error) IsInSelectedParentChainOf(stagingArea *StagingArea, blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) - ChildInSelectedParentChainOf(stagingArea *StagingArea, context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) + ChildInSelectedParentChainOf(stagingArea *StagingArea, lowHash, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) SetParents(stagingArea *StagingArea, blockHash *externalapi.DomainHash, parentHashes []*externalapi.DomainHash) error } diff --git a/domain/consensus/model/reachabilitydata.go b/domain/consensus/model/reachabilitydata.go index 718d47c55..ab3643c3e 100644 --- a/domain/consensus/model/reachabilitydata.go +++ b/domain/consensus/model/reachabilitydata.go @@ -104,6 +104,7 @@ type FutureCoveringTreeNodeSet []*externalapi.DomainHash // Clone returns a clone of FutureCoveringTreeNodeSet func (fctns FutureCoveringTreeNodeSet) Clone() FutureCoveringTreeNodeSet { + //return fctns return externalapi.CloneHashes(fctns) } diff --git a/domain/consensus/processes/blockprocessor/blockprocessor.go b/domain/consensus/processes/blockprocessor/blockprocessor.go index 0d32d462b..d30733a62 100644 --- a/domain/consensus/processes/blockprocessor/blockprocessor.go +++ b/domain/consensus/processes/blockprocessor/blockprocessor.go @@ -22,7 +22,7 @@ type blockProcessor struct { pruningManager model.PruningManager blockValidator model.BlockValidator dagTopologyManager model.DAGTopologyManager - reachabilityManagers []model.ReachabilityManager + reachabilityManager model.ReachabilityManager difficultyManager model.DifficultyManager pastMedianTimeManager model.PastMedianTimeManager coinbaseManager model.CoinbaseManager @@ -60,7 +60,7 @@ func New( pruningManager model.PruningManager, blockValidator model.BlockValidator, dagTopologyManager model.DAGTopologyManager, - reachabilityManagers []model.ReachabilityManager, + reachabilityManager model.ReachabilityManager, difficultyManager model.DifficultyManager, pastMedianTimeManager model.PastMedianTimeManager, coinbaseManager model.CoinbaseManager, @@ -94,7 +94,7 @@ func New( pruningManager: pruningManager, blockValidator: blockValidator, dagTopologyManager: dagTopologyManager, - reachabilityManagers: reachabilityManagers, + reachabilityManager: reachabilityManager, difficultyManager: difficultyManager, pastMedianTimeManager: pastMedianTimeManager, coinbaseManager: coinbaseManager, diff --git a/domain/consensus/processes/blockprocessor/validate_and_insert_block.go b/domain/consensus/processes/blockprocessor/validate_and_insert_block.go index 2fdc61aed..b0659667b 100644 --- a/domain/consensus/processes/blockprocessor/validate_and_insert_block.go +++ b/domain/consensus/processes/blockprocessor/validate_and_insert_block.go @@ -254,19 +254,7 @@ func (bp *blockProcessor) updateReachabilityReindexRoot(stagingArea *model.Stagi return nil } - headersSelectedTipHeader, err := bp.blockHeaderStore.BlockHeader(bp.databaseContext, stagingArea, headersSelectedTip) - if err != nil { - return err - } - - headersSelectedTipHeaderBlockLevel := headersSelectedTipHeader.BlockLevel(bp.maxBlockLevel) - for blockLevel := 0; blockLevel <= headersSelectedTipHeaderBlockLevel; blockLevel++ { - err := bp.reachabilityManagers[blockLevel].UpdateReindexRoot(stagingArea, headersSelectedTip) - if err != nil { - return err - } - } - return nil + return bp.reachabilityManager.UpdateReindexRoot(stagingArea, headersSelectedTip) } func (bp *blockProcessor) checkBlockStatus(stagingArea *model.StagingArea, block *externalapi.DomainBlock) error { diff --git a/domain/consensus/processes/blockvalidator/block_header_in_context.go b/domain/consensus/processes/blockvalidator/block_header_in_context.go index 44efe6b59..b4d47fa05 100644 --- a/domain/consensus/processes/blockvalidator/block_header_in_context.go +++ b/domain/consensus/processes/blockvalidator/block_header_in_context.go @@ -62,12 +62,9 @@ func (v *blockValidator) ValidateHeaderInContext(stagingArea *model.StagingArea, return err } if !hasReachabilityData { - blockLevel := header.BlockLevel(v.maxBlockLevel) - for i := 0; i <= blockLevel; i++ { - err = v.reachabilityManagers[i].AddBlock(stagingArea, blockHash) - if err != nil { - return err - } + err = v.reachabilityManager.AddBlock(stagingArea, blockHash) + if err != nil { + return err } } diff --git a/domain/consensus/processes/blockvalidator/blockvalidator.go b/domain/consensus/processes/blockvalidator/blockvalidator.go index 870ce8c42..ae03f6825 100644 --- a/domain/consensus/processes/blockvalidator/blockvalidator.go +++ b/domain/consensus/processes/blockvalidator/blockvalidator.go @@ -37,7 +37,7 @@ type blockValidator struct { coinbaseManager model.CoinbaseManager mergeDepthManager model.MergeDepthManager pruningStore model.PruningStore - reachabilityManagers []model.ReachabilityManager + reachabilityManager model.ReachabilityManager finalityManager model.FinalityManager blockParentBuilder model.BlockParentBuilder pruningManager model.PruningManager @@ -77,7 +77,7 @@ func New(powMax *big.Int, dagTraversalManager model.DAGTraversalManager, coinbaseManager model.CoinbaseManager, mergeDepthManager model.MergeDepthManager, - reachabilityManagers []model.ReachabilityManager, + reachabilityManager model.ReachabilityManager, finalityManager model.FinalityManager, blockParentBuilder model.BlockParentBuilder, pruningManager model.PruningManager, @@ -118,7 +118,7 @@ func New(powMax *big.Int, dagTraversalManager: dagTraversalManager, coinbaseManager: coinbaseManager, mergeDepthManager: mergeDepthManager, - reachabilityManagers: reachabilityManagers, + reachabilityManager: reachabilityManager, finalityManager: finalityManager, blockParentBuilder: blockParentBuilder, pruningManager: pruningManager, diff --git a/domain/consensus/processes/dagtopologymanager/dagtopologymanager.go b/domain/consensus/processes/dagtopologymanager/dagtopologymanager.go index c7aac8422..ea8717aef 100644 --- a/domain/consensus/processes/dagtopologymanager/dagtopologymanager.go +++ b/domain/consensus/processes/dagtopologymanager/dagtopologymanager.go @@ -105,6 +105,17 @@ func (dtm *dagTopologyManager) IsAnyAncestorOf(stagingArea *model.StagingArea, p // IsInSelectedParentChainOf returns true if blockHashA is in the selected parent chain of blockHashB func (dtm *dagTopologyManager) IsInSelectedParentChainOf(stagingArea *model.StagingArea, blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) { + + // Virtual doesn't have reachability data, therefore, it should be treated as a special case - + // use its selected parent as blockHashB. + if blockHashB == model.VirtualBlockHash { + ghostdagData, err := dtm.ghostdagStore.Get(dtm.databaseContext, stagingArea, blockHashB, false) + if err != nil { + return false, err + } + blockHashB = ghostdagData.SelectedParent() + } + return dtm.reachabilityManager.IsReachabilityTreeAncestorOf(stagingArea, blockHashA, blockHashB) } @@ -176,12 +187,11 @@ func (dtm *dagTopologyManager) SetParents(stagingArea *model.StagingArea, blockH return nil } -// ChildInSelectedParentChainOf returns the child of `context` that is in the selected-parent-chain of `highHash` -func (dtm *dagTopologyManager) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, - context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) { +// ChildInSelectedParentChainOf returns the child of `lowHash` that is in the selected-parent-chain of `highHash` +func (dtm *dagTopologyManager) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, lowHash, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) { // Virtual doesn't have reachability data, therefore, it should be treated as a special case - - // use it's selected parent as highHash. + // use its selected parent as highHash. specifiedHighHash := highHash if highHash == model.VirtualBlockHash { ghostdagData, err := dtm.ghostdagStore.Get(dtm.databaseContext, stagingArea, highHash, false) @@ -191,20 +201,20 @@ func (dtm *dagTopologyManager) ChildInSelectedParentChainOf(stagingArea *model.S selectedParent := ghostdagData.SelectedParent() // In case where `context` is an immediate parent of `highHash` - if context.Equal(selectedParent) { + if lowHash.Equal(selectedParent) { return highHash, nil } highHash = selectedParent } - isInSelectedParentChain, err := dtm.IsInSelectedParentChainOf(stagingArea, context, highHash) + isInSelectedParentChain, err := dtm.IsInSelectedParentChainOf(stagingArea, lowHash, highHash) if err != nil { return nil, err } if !isInSelectedParentChain { - return nil, errors.Errorf("context(%s) is not in the selected-parent-chain of highHash(%s)", - context, specifiedHighHash) + return nil, errors.Errorf("Claimed chain ancestor (%s) is not in the selected-parent-chain of highHash (%s)", + lowHash, specifiedHighHash) } - return dtm.reachabilityManager.FindNextAncestor(stagingArea, highHash, context) + return dtm.reachabilityManager.FindNextAncestor(stagingArea, highHash, lowHash) } diff --git a/domain/consensus/processes/dagtraversalmanager/dagtraversalmanager.go b/domain/consensus/processes/dagtraversalmanager/dagtraversalmanager.go index 6263cd4f3..41302edb7 100644 --- a/domain/consensus/processes/dagtraversalmanager/dagtraversalmanager.go +++ b/domain/consensus/processes/dagtraversalmanager/dagtraversalmanager.go @@ -14,7 +14,7 @@ type dagTraversalManager struct { dagTopologyManager model.DAGTopologyManager ghostdagManager model.GHOSTDAGManager ghostdagDataStore model.GHOSTDAGDataStore - reachabilityDataStore model.ReachabilityDataStore + reachabilityManager model.ReachabilityManager daaWindowStore model.BlocksWithTrustedDataDAAWindowStore genesisHash *externalapi.DomainHash difficultyAdjustmentWindowSize int @@ -26,19 +26,19 @@ func New( databaseContext model.DBReader, dagTopologyManager model.DAGTopologyManager, ghostdagDataStore model.GHOSTDAGDataStore, - reachabilityDataStore model.ReachabilityDataStore, + reachabilityManager model.ReachabilityManager, ghostdagManager model.GHOSTDAGManager, daaWindowStore model.BlocksWithTrustedDataDAAWindowStore, windowHeapSliceStore model.WindowHeapSliceStore, genesisHash *externalapi.DomainHash, difficultyAdjustmentWindowSize int) model.DAGTraversalManager { return &dagTraversalManager{ - databaseContext: databaseContext, - dagTopologyManager: dagTopologyManager, - ghostdagDataStore: ghostdagDataStore, - reachabilityDataStore: reachabilityDataStore, - ghostdagManager: ghostdagManager, - daaWindowStore: daaWindowStore, + databaseContext: databaseContext, + dagTopologyManager: dagTopologyManager, + ghostdagDataStore: ghostdagDataStore, + reachabilityManager: reachabilityManager, + ghostdagManager: ghostdagManager, + daaWindowStore: daaWindowStore, genesisHash: genesisHash, difficultyAdjustmentWindowSize: difficultyAdjustmentWindowSize, diff --git a/domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go b/domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go index a20df548a..cb027ada9 100644 --- a/domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go +++ b/domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go @@ -97,25 +97,13 @@ func (dtm *dagTraversalManager) SelectedChildIterator(stagingArea *model.Staging var errNoSelectedChild = errors.New("errNoSelectedChild") func (dtm *dagTraversalManager) SelectedChild(stagingArea *model.StagingArea, - context, blockHash *externalapi.DomainHash) (*externalapi.DomainHash, error) { + highHash, lowHash *externalapi.DomainHash) (*externalapi.DomainHash, error) { - data, err := dtm.reachabilityDataStore.ReachabilityData(dtm.databaseContext, stagingArea, blockHash) + // The selected child is in fact the next reachability tree nextAncestor + nextAncestor, err := dtm.reachabilityManager.FindNextAncestor(stagingArea, highHash, lowHash) if err != nil { - return nil, err + return nil, errors.Wrapf(errNoSelectedChild, "no selected child for %s from the point of view of %s", + lowHash, highHash) } - - for _, child := range data.Children() { - isChildInSelectedParentChainOfHighHash, err := dtm.dagTopologyManager.IsInSelectedParentChainOf( - stagingArea, child, context) - if err != nil { - return nil, err - } - - if isChildInSelectedParentChainOfHighHash { - return child, nil - } - } - - return nil, errors.Wrapf(errNoSelectedChild, "no selected child for %s from the point of view of %s", - blockHash, context) + return nextAncestor, nil } diff --git a/domain/consensus/processes/finalitymanager/finality_manager.go b/domain/consensus/processes/finalitymanager/finality_manager.go index f16467a1e..d687ef65a 100644 --- a/domain/consensus/processes/finalitymanager/finality_manager.go +++ b/domain/consensus/processes/finalitymanager/finality_manager.go @@ -13,6 +13,7 @@ type finalityManager struct { dagTopologyManager model.DAGTopologyManager finalityStore model.FinalityStore ghostdagDataStore model.GHOSTDAGDataStore + pruningStore model.PruningStore genesisHash *externalapi.DomainHash finalityDepth uint64 } @@ -22,6 +23,7 @@ func New(databaseContext model.DBReader, dagTopologyManager model.DAGTopologyManager, finalityStore model.FinalityStore, ghostdagDataStore model.GHOSTDAGDataStore, + pruningStore model.PruningStore, genesisHash *externalapi.DomainHash, finalityDepth uint64) model.FinalityManager { @@ -31,6 +33,7 @@ func New(databaseContext model.DBReader, dagTopologyManager: dagTopologyManager, finalityStore: finalityStore, ghostdagDataStore: ghostdagDataStore, + pruningStore: pruningStore, finalityDepth: finalityDepth, } } @@ -96,6 +99,27 @@ func (fm *finalityManager) calculateFinalityPoint(stagingArea *model.StagingArea return fm.genesisHash, nil } + pruningPoint, err := fm.pruningStore.PruningPoint(fm.databaseContext, stagingArea) + if err != nil { + return nil, err + } + pruningPointGhostdagData, err := fm.ghostdagDataStore.Get(fm.databaseContext, stagingArea, pruningPoint, false) + if err != nil { + return nil, err + } + if ghostdagData.BlueScore() < pruningPointGhostdagData.BlueScore()+fm.finalityDepth { + log.Debugf("%s blue score less than finality distance over pruning point - returning virtual genesis as finality point", blockHash) + return model.VirtualGenesisBlockHash, nil + } + isPruningPointOnChain, err := fm.dagTopologyManager.IsInSelectedParentChainOf(stagingArea, pruningPoint, blockHash) + if err != nil { + return nil, err + } + if !isPruningPointOnChain { + log.Debugf("pruning point not in selected chain of %s - returning virtual genesis as finality point", blockHash) + return model.VirtualGenesisBlockHash, nil + } + selectedParent := ghostdagData.SelectedParent() if selectedParent.Equal(fm.genesisHash) { return fm.genesisHash, nil @@ -105,6 +129,12 @@ func (fm *finalityManager) calculateFinalityPoint(stagingArea *model.StagingArea if err != nil { return nil, err } + // In this case we expect the pruning point or a block above it to be the finality point. + // Note that above we already verified the chain and distance conditions for this + if current.Equal(model.VirtualGenesisBlockHash) { + current = pruningPoint + } + requiredBlueScore := ghostdagData.BlueScore() - fm.finalityDepth log.Debugf("%s's finality point is the one having the highest blue score lower then %d", blockHash, requiredBlueScore) diff --git a/domain/consensus/processes/ghostdagmanager/ghostdag_test.go b/domain/consensus/processes/ghostdagmanager/ghostdag_test.go index fc9c8a94e..11951d548 100644 --- a/domain/consensus/processes/ghostdagmanager/ghostdag_test.go +++ b/domain/consensus/processes/ghostdagmanager/ghostdag_test.go @@ -320,10 +320,6 @@ func (ds *GHOSTDAGDataStoreImpl) IsStaged(*model.StagingArea) bool { panic("implement me") } -func (ds *GHOSTDAGDataStoreImpl) Discard() { - panic("implement me") -} - func (ds *GHOSTDAGDataStoreImpl) Commit(dbTx model.DBTransaction) error { panic("implement me") } @@ -336,11 +332,15 @@ func (ds *GHOSTDAGDataStoreImpl) Get(dbContext model.DBReader, stagingArea *mode return nil, nil } +func (ds *GHOSTDAGDataStoreImpl) UnstageAll(stagingArea *model.StagingArea) { + panic("implement me") +} + type DAGTopologyManagerImpl struct { parentsMap map[externalapi.DomainHash][]*externalapi.DomainHash } -func (dt *DAGTopologyManagerImpl) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) { +func (dt *DAGTopologyManagerImpl) ChildInSelectedParentChainOf(stagingArea *model.StagingArea, lowHash, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error) { panic("implement me") } diff --git a/domain/consensus/processes/pruningproofmanager/pruningproofmanager.go b/domain/consensus/processes/pruningproofmanager/pruningproofmanager.go index a437a73ec..b735cedf1 100644 --- a/domain/consensus/processes/pruningproofmanager/pruningproofmanager.go +++ b/domain/consensus/processes/pruningproofmanager/pruningproofmanager.go @@ -9,6 +9,7 @@ import ( "github.com/kaspanet/kaspad/domain/consensus/model" "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" "github.com/kaspanet/kaspad/domain/consensus/processes/dagtopologymanager" + "github.com/kaspanet/kaspad/domain/consensus/processes/dagtraversalmanager" "github.com/kaspanet/kaspad/domain/consensus/processes/ghostdagmanager" "github.com/kaspanet/kaspad/domain/consensus/processes/reachabilitymanager" "github.com/kaspanet/kaspad/domain/consensus/ruleerrors" @@ -26,16 +27,19 @@ type pruningProofManager struct { dagTopologyManagers []model.DAGTopologyManager ghostdagManagers []model.GHOSTDAGManager - reachabilityManagers []model.ReachabilityManager + reachabilityManager model.ReachabilityManager dagTraversalManagers []model.DAGTraversalManager parentsManager model.ParentsManager + pruningManager model.PruningManager - ghostdagDataStores []model.GHOSTDAGDataStore - pruningStore model.PruningStore - blockHeaderStore model.BlockHeaderStore - blockStatusStore model.BlockStatusStore - finalityStore model.FinalityStore - consensusStateStore model.ConsensusStateStore + ghostdagDataStores []model.GHOSTDAGDataStore + pruningStore model.PruningStore + blockHeaderStore model.BlockHeaderStore + blockStatusStore model.BlockStatusStore + finalityStore model.FinalityStore + consensusStateStore model.ConsensusStateStore + blockRelationStore model.BlockRelationStore + reachabilityDataStore model.ReachabilityDataStore genesisHash *externalapi.DomainHash k externalapi.KType @@ -52,9 +56,10 @@ func New( dagTopologyManagers []model.DAGTopologyManager, ghostdagManagers []model.GHOSTDAGManager, - reachabilityManagers []model.ReachabilityManager, + reachabilityManager model.ReachabilityManager, dagTraversalManagers []model.DAGTraversalManager, parentsManager model.ParentsManager, + pruningManager model.PruningManager, ghostdagDataStores []model.GHOSTDAGDataStore, pruningStore model.PruningStore, @@ -62,6 +67,8 @@ func New( blockStatusStore model.BlockStatusStore, finalityStore model.FinalityStore, consensusStateStore model.ConsensusStateStore, + blockRelationStore model.BlockRelationStore, + reachabilityDataStore model.ReachabilityDataStore, genesisHash *externalapi.DomainHash, k externalapi.KType, @@ -73,16 +80,19 @@ func New( databaseContext: databaseContext, dagTopologyManagers: dagTopologyManagers, ghostdagManagers: ghostdagManagers, - reachabilityManagers: reachabilityManagers, + reachabilityManager: reachabilityManager, dagTraversalManagers: dagTraversalManagers, parentsManager: parentsManager, + pruningManager: pruningManager, - ghostdagDataStores: ghostdagDataStores, - pruningStore: pruningStore, - blockHeaderStore: blockHeaderStore, - blockStatusStore: blockStatusStore, - finalityStore: finalityStore, - consensusStateStore: consensusStateStore, + ghostdagDataStores: ghostdagDataStores, + pruningStore: pruningStore, + blockHeaderStore: blockHeaderStore, + blockStatusStore: blockStatusStore, + finalityStore: finalityStore, + consensusStateStore: consensusStateStore, + blockRelationStore: blockRelationStore, + reachabilityDataStore: reachabilityDataStore, genesisHash: genesisHash, k: k, @@ -611,6 +621,209 @@ func (ppm *pruningProofManager) dagProcesses( return reachabilityManagers, dagTopologyManagers, ghostdagManagers } +func (ppm *pruningProofManager) ghostdagDataWithoutPrunedBlocks(stagingArea *model.StagingArea, targetReachabilityDataStore model.ReachabilityDataStore, + data *externalapi.BlockGHOSTDAGData) (*externalapi.BlockGHOSTDAGData, bool, error) { + + changed := false + mergeSetBlues := make([]*externalapi.DomainHash, 0, len(data.MergeSetBlues())) + for _, blockHash := range data.MergeSetBlues() { + hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, blockHash) + if err != nil { + return nil, false, err + } + if !hasReachabilityData { + changed = true + if data.SelectedParent().Equal(blockHash) { + mergeSetBlues = append(mergeSetBlues, model.VirtualGenesisBlockHash) + } + continue + } + + mergeSetBlues = append(mergeSetBlues, blockHash) + } + + mergeSetReds := make([]*externalapi.DomainHash, 0, len(data.MergeSetReds())) + for _, blockHash := range data.MergeSetReds() { + hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, blockHash) + if err != nil { + return nil, false, err + } + if !hasReachabilityData { + changed = true + continue + } + + mergeSetReds = append(mergeSetReds, blockHash) + } + + selectedParent := data.SelectedParent() + hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, data.SelectedParent()) + if err != nil { + return nil, false, err + } + + if !hasReachabilityData { + changed = true + selectedParent = model.VirtualGenesisBlockHash + } + + return externalapi.NewBlockGHOSTDAGData( + data.BlueScore(), + data.BlueWork(), + selectedParent, + mergeSetBlues, + mergeSetReds, + data.BluesAnticoneSizes(), + ), changed, nil +} + +func (ppm *pruningProofManager) populateProofReachabilityAndHeaders(pruningPointProof *externalapi.PruningPointProof, + targetReachabilityDataStore model.ReachabilityDataStore) error { + // We build a DAG of all multi-level relations between blocks in the proof. We make a upHeap of all blocks, so we can iterate + // over them in a topological way, and then build a DAG where we use all multi-level parents of a block to create edges, except + // parents that are already in the past of another parent (This can happen between two levels). We run GHOSTDAG on each block of + // that DAG, because GHOSTDAG is a requirement to calculate reachability. We then dismiss the GHOSTDAG data because it's not related + // to the GHOSTDAG data of the real DAG, and was used only for reachability. + + // We need two staging areas: stagingArea which is used to commit the reachability data, and tmpStagingArea for the GHOSTDAG data + // of allProofBlocksUpHeap. The reason we need two areas is that we use the real GHOSTDAG data in order to order the heap in a topological + // way, and fake GHOSTDAG data for calculating reachability. + stagingArea := model.NewStagingArea() + tmpStagingArea := model.NewStagingArea() + + bucket := consensusDB.MakeBucket([]byte("TMP")) + ghostdagDataStoreForTargetReachabilityManager := ghostdagdatastore.New(bucket, 0, false) + ghostdagDataStoreForTargetReachabilityManager.Stage(stagingArea, model.VirtualGenesisBlockHash, externalapi.NewBlockGHOSTDAGData( + 0, + big.NewInt(0), + nil, + nil, + nil, + nil, + ), false) + targetReachabilityManager := reachabilitymanager.New(ppm.databaseContext, ghostdagDataStoreForTargetReachabilityManager, targetReachabilityDataStore) + blockRelationStoreForTargetReachabilityManager := blockrelationstore.New(bucket, 0, false) + dagTopologyManagerForTargetReachabilityManager := dagtopologymanager.New(ppm.databaseContext, targetReachabilityManager, blockRelationStoreForTargetReachabilityManager, nil) + ghostdagManagerForTargetReachabilityManager := ghostdagmanager.New(ppm.databaseContext, dagTopologyManagerForTargetReachabilityManager, ghostdagDataStoreForTargetReachabilityManager, ppm.blockHeaderStore, 0, nil) + err := dagTopologyManagerForTargetReachabilityManager.SetParents(stagingArea, model.VirtualGenesisBlockHash, nil) + if err != nil { + return err + } + + dagTopologyManager := dagtopologymanager.New(ppm.databaseContext, targetReachabilityManager, nil, nil) + ghostdagDataStore := ghostdagdatastore.New(bucket, 0, false) + tmpGHOSTDAGManager := ghostdagmanager.New(ppm.databaseContext, nil, ghostdagDataStore, nil, 0, nil) + dagTraversalManager := dagtraversalmanager.New(ppm.databaseContext, nil, ghostdagDataStore, nil, tmpGHOSTDAGManager, nil, nil, nil, 0) + allProofBlocksUpHeap := dagTraversalManager.NewUpHeap(tmpStagingArea) + dag := make(map[externalapi.DomainHash]struct { + parents hashset.HashSet + header externalapi.BlockHeader + }) + for _, headers := range pruningPointProof.Headers { + for _, header := range headers { + blockHash := consensushashing.HeaderHash(header) + if _, ok := dag[*blockHash]; ok { + continue + } + + dag[*blockHash] = struct { + parents hashset.HashSet + header externalapi.BlockHeader + }{parents: hashset.New(), header: header} + + for level := 0; level <= ppm.maxBlockLevel; level++ { + for _, parent := range ppm.parentsManager.ParentsAtLevel(header, level) { + parent := parent + dag[*blockHash].parents.Add(parent) + } + } + + // We stage temporary GHOSTDAG data that is needed in order to sort allProofBlocksUpHeap. + ghostdagDataStore.Stage(tmpStagingArea, blockHash, externalapi.NewBlockGHOSTDAGData(header.BlueScore(), header.BlueWork(), nil, nil, nil, nil), false) + err := allProofBlocksUpHeap.Push(blockHash) + if err != nil { + return err + } + } + } + + var selectedTip *externalapi.DomainHash + for allProofBlocksUpHeap.Len() > 0 { + blockHash := allProofBlocksUpHeap.Pop() + block := dag[*blockHash] + parentsHeap := dagTraversalManager.NewDownHeap(tmpStagingArea) + for parent := range block.parents { + parent := parent + if _, ok := dag[parent]; !ok { + continue + } + + err := parentsHeap.Push(&parent) + if err != nil { + return err + } + } + + fakeParents := []*externalapi.DomainHash{} + for parentsHeap.Len() > 0 { + parent := parentsHeap.Pop() + isAncestorOfAny, err := dagTopologyManager.IsAncestorOfAny(stagingArea, parent, fakeParents) + if err != nil { + return err + } + + if isAncestorOfAny { + continue + } + + fakeParents = append(fakeParents, parent) + } + + if len(fakeParents) == 0 { + fakeParents = append(fakeParents, model.VirtualGenesisBlockHash) + } + + err := dagTopologyManagerForTargetReachabilityManager.SetParents(stagingArea, blockHash, fakeParents) + if err != nil { + return err + } + + err = ghostdagManagerForTargetReachabilityManager.GHOSTDAG(stagingArea, blockHash) + if err != nil { + return err + } + + err = targetReachabilityManager.AddBlock(stagingArea, blockHash) + if err != nil { + return err + } + + if selectedTip == nil { + selectedTip = blockHash + } else { + selectedTip, err = ghostdagManagerForTargetReachabilityManager.ChooseSelectedParent(stagingArea, selectedTip, blockHash) + if err != nil { + return err + } + } + + if selectedTip.Equal(blockHash) { + err := targetReachabilityManager.UpdateReindexRoot(stagingArea, selectedTip) + if err != nil { + return err + } + } + } + + ghostdagDataStoreForTargetReachabilityManager.UnstageAll(stagingArea) + blockRelationStoreForTargetReachabilityManager.UnstageAll(stagingArea) + err = staging.CommitAllChanges(ppm.databaseContext, stagingArea) + if err != nil { + return err + } + return nil +} + // ApplyPruningPointProof applies the given pruning proof to the current consensus. Specifically, // it's meant to be used against the StagingConsensus during headers-proof IBD. Note that for // performance reasons this operation is NOT atomic. If the process fails for whatever reason @@ -619,9 +832,25 @@ func (ppm *pruningProofManager) ApplyPruningPointProof(pruningPointProof *extern onEnd := logger.LogAndMeasureExecutionTime(log, "ApplyPruningPointProof") defer onEnd() + stagingArea := model.NewStagingArea() + for _, headers := range pruningPointProof.Headers { + for _, header := range headers { + blockHash := consensushashing.HeaderHash(header) + ppm.blockHeaderStore.Stage(stagingArea, blockHash, header) + } + } + err := staging.CommitAllChanges(ppm.databaseContext, stagingArea) + if err != nil { + return err + } + + err = ppm.populateProofReachabilityAndHeaders(pruningPointProof, ppm.reachabilityDataStore) + if err != nil { + return err + } + for blockLevel, headers := range pruningPointProof.Headers { log.Infof("Applying level %d from the pruning point proof", blockLevel) - var selectedTip *externalapi.DomainHash for i, header := range headers { if i%1000 == 0 { log.Infof("Applying level %d from the pruning point proof - applied %d headers out of %d", blockLevel, i, len(headers)) @@ -687,27 +916,6 @@ func (ppm *pruningProofManager) ApplyPruningPointProof(pruningPointProof *extern ppm.blockStatusStore.Stage(stagingArea, blockHash, externalapi.StatusHeaderOnly) } - if selectedTip == nil { - selectedTip = blockHash - } else { - selectedTip, err = ppm.ghostdagManagers[blockLevel].ChooseSelectedParent(stagingArea, selectedTip, blockHash) - if err != nil { - return err - } - } - - err = ppm.reachabilityManagers[blockLevel].AddBlock(stagingArea, blockHash) - if err != nil { - return err - } - - if selectedTip.Equal(blockHash) { - err := ppm.reachabilityManagers[blockLevel].UpdateReindexRoot(stagingArea, selectedTip) - if err != nil { - return err - } - } - err = staging.CommitAllChanges(ppm.databaseContext, stagingArea) if err != nil { return err @@ -718,7 +926,7 @@ func (ppm *pruningProofManager) ApplyPruningPointProof(pruningPointProof *extern pruningPointHeader := pruningPointProof.Headers[0][len(pruningPointProof.Headers[0])-1] pruningPoint := consensushashing.HeaderHash(pruningPointHeader) - stagingArea := model.NewStagingArea() + stagingArea = model.NewStagingArea() ppm.consensusStateStore.StageTips(stagingArea, []*externalapi.DomainHash{pruningPoint}) return staging.CommitAllChanges(ppm.databaseContext, stagingArea) } diff --git a/domain/consensus/processes/reachabilitymanager/reachability_test.go b/domain/consensus/processes/reachabilitymanager/reachability_test.go index ed1e2e7cf..dc72936ac 100644 --- a/domain/consensus/processes/reachabilitymanager/reachability_test.go +++ b/domain/consensus/processes/reachabilitymanager/reachability_test.go @@ -16,7 +16,7 @@ type reachabilityDataStoreMock struct { reachabilityReindexRootStaging *externalapi.DomainHash } -func (r *reachabilityDataStoreMock) Discard() { +func (r *reachabilityDataStoreMock) Delete(_ model.DBWriter) error { panic("implement me") } diff --git a/domain/consensus/processes/reachabilitymanager/tree.go b/domain/consensus/processes/reachabilitymanager/tree.go index a5d2e848d..1c3159872 100644 --- a/domain/consensus/processes/reachabilitymanager/tree.go +++ b/domain/consensus/processes/reachabilitymanager/tree.go @@ -152,6 +152,10 @@ func (rt *reachabilityManager) IsReachabilityTreeAncestorOf(stagingArea *model.S func (rt *reachabilityManager) FindNextAncestor(stagingArea *model.StagingArea, descendant, ancestor *externalapi.DomainHash) (*externalapi.DomainHash, error) { + if ancestor.Equal(descendant) { + return nil, errors.Errorf("ancestor is equal to descendant") + } + childrenOfAncestor, err := rt.children(stagingArea, ancestor) if err != nil { return nil, err diff --git a/domain/consensus/test_consensus_getters.go b/domain/consensus/test_consensus_getters.go index 98749e1a4..877dec855 100644 --- a/domain/consensus/test_consensus_getters.go +++ b/domain/consensus/test_consensus_getters.go @@ -60,7 +60,7 @@ func (tc *testConsensus) PruningStore() model.PruningStore { } func (tc *testConsensus) ReachabilityDataStore() model.ReachabilityDataStore { - return tc.reachabilityDataStores[0] + return tc.reachabilityDataStore } func (tc *testConsensus) UTXODiffStore() model.UTXODiffStore { diff --git a/domain/consensus/utils/reachabilitydata/reachability_data.go b/domain/consensus/utils/reachabilitydata/reachability_data.go index 3d9ca8bb1..d4252aa24 100644 --- a/domain/consensus/utils/reachabilitydata/reachability_data.go +++ b/domain/consensus/utils/reachabilitydata/reachability_data.go @@ -57,6 +57,7 @@ func (rd *reachabilityData) FutureCoveringSet() model.FutureCoveringTreeNodeSet } func (rd *reachabilityData) CloneMutable() model.MutableReachabilityData { + //return rd return &reachabilityData{ children: externalapi.CloneHashes(rd.children), parent: rd.parent, diff --git a/domain/domain.go b/domain/domain.go index 5cc5c4db5..066e17197 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -82,11 +82,15 @@ func (d *domain) InitStagingConsensus() error { cfg := *d.consensusConfig cfg.SkipAddingGenesis = true - consensusInstance, err := consensusFactory.NewConsensus(&cfg, d.db, inactivePrefix) + consensusInstance, shouldMigrate, err := consensusFactory.NewConsensus(&cfg, d.db, inactivePrefix) if err != nil { return err } + if shouldMigrate { + return errors.Errorf("A fresh consensus should never return shouldMigrate=true") + } + d.stagingConsensus = &consensusInstance return nil } @@ -183,7 +187,7 @@ func New(consensusConfig *consensus.Config, mempoolConfig *mempool.Config, db in } consensusFactory := consensus.NewFactory() - consensusInstance, err := consensusFactory.NewConsensus(consensusConfig, db, activePrefix) + consensusInstance, shouldMigrate, err := consensusFactory.NewConsensus(consensusConfig, db, activePrefix) if err != nil { return nil, err } @@ -194,6 +198,13 @@ func New(consensusConfig *consensus.Config, mempoolConfig *mempool.Config, db in db: db, } + if shouldMigrate { + err := domainInstance.migrate() + if err != nil { + return nil, err + } + } + miningManagerFactory := miningmanager.NewFactory() // We create a consensus wrapper because the actual consensus might change diff --git a/domain/log.go b/domain/log.go new file mode 100644 index 000000000..359473161 --- /dev/null +++ b/domain/log.go @@ -0,0 +1,7 @@ +package domain + +import ( + "github.com/kaspanet/kaspad/infrastructure/logger" +) + +var log = logger.RegisterSubSystem("DOMN") diff --git a/domain/migrate.go b/domain/migrate.go new file mode 100644 index 000000000..45a68d7b2 --- /dev/null +++ b/domain/migrate.go @@ -0,0 +1,230 @@ +package domain + +import ( + "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" + "math" +) + +func (d *domain) migrate() error { + log.Infof("Starting migration") + err := d.InitStagingConsensus() + if err != nil { + return err + } + + err = syncConsensuses(d.Consensus(), d.StagingConsensus()) + if err != nil { + return err + } + + err = d.CommitStagingConsensus() + if err != nil { + return err + } + + log.Info("Done migrating") + return nil +} + +func syncConsensuses(syncer, syncee externalapi.Consensus) error { + pruningPointProof, err := syncer.BuildPruningPointProof() + if err != nil { + return err + } + + err = syncee.ApplyPruningPointProof(pruningPointProof) + if err != nil { + return err + } + + pruningPointHeaders, err := syncer.PruningPointHeaders() + if err != nil { + return err + } + + err = syncee.ImportPruningPoints(pruningPointHeaders) + if err != nil { + return err + } + + pruningPointAndItsAnticone, err := syncer.PruningPointAndItsAnticone() + if err != nil { + return err + } + + for _, blockHash := range pruningPointAndItsAnticone { + block, err := syncer.GetBlock(blockHash) + if err != nil { + return err + } + + blockDAAWindowHashes, err := syncer.BlockDAAWindowHashes(blockHash) + if err != nil { + return err + } + + ghostdagDataBlockHashes, err := syncer.TrustedBlockAssociatedGHOSTDAGDataBlockHashes(blockHash) + if err != nil { + return err + } + + blockWithTrustedData := &externalapi.BlockWithTrustedData{ + Block: block, + DAAWindow: make([]*externalapi.TrustedDataDataDAAHeader, 0, len(blockDAAWindowHashes)), + GHOSTDAGData: make([]*externalapi.BlockGHOSTDAGDataHashPair, 0, len(ghostdagDataBlockHashes)), + } + + for i, daaBlockHash := range blockDAAWindowHashes { + trustedDataDataDAAHeader, err := syncer.TrustedDataDataDAAHeader(blockHash, daaBlockHash, uint64(i)) + if err != nil { + return err + } + blockWithTrustedData.DAAWindow = append(blockWithTrustedData.DAAWindow, trustedDataDataDAAHeader) + } + + for _, ghostdagDataBlockHash := range ghostdagDataBlockHashes { + data, err := syncer.TrustedGHOSTDAGData(ghostdagDataBlockHash) + if err != nil { + return err + } + blockWithTrustedData.GHOSTDAGData = append(blockWithTrustedData.GHOSTDAGData, &externalapi.BlockGHOSTDAGDataHashPair{ + Hash: ghostdagDataBlockHash, + GHOSTDAGData: data, + }) + } + + _, err = syncee.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false) + if err != nil { + return err + } + } + + syncerVirtualSelectedParent, err := syncer.GetVirtualSelectedParent() + if err != nil { + return err + } + + pruningPoint, err := syncer.PruningPoint() + if err != nil { + return err + } + + missingBlocks, _, err := syncer.GetHashesBetween(pruningPoint, syncerVirtualSelectedParent, math.MaxUint64) + if err != nil { + return err + } + + syncerTips, err := syncer.Tips() + if err != nil { + return err + } + + for _, tip := range syncerTips { + if tip.Equal(syncerVirtualSelectedParent) { + continue + } + + anticone, err := syncer.GetAnticone(syncerVirtualSelectedParent, tip, 0) + if err != nil { + return err + } + + missingBlocks = append(missingBlocks, anticone...) + } + + percents := 0 + for i, blocksHash := range missingBlocks { + blockInfo, err := syncee.GetBlockInfo(blocksHash) + if err != nil { + return err + } + + if blockInfo.Exists { + continue + } + + block, err := syncer.GetBlock(blocksHash) + if err != nil { + return err + } + + _, err = syncee.ValidateAndInsertBlock(block, false) + if err != nil { + return err + } + + newPercents := 100 * i / len(missingBlocks) + if newPercents > percents { + percents = newPercents + log.Infof("Processed %d%% of the blocks", 100*i/len(missingBlocks)) + } + } + + var fromOutpoint *externalapi.DomainOutpoint + const step = 100_000 + for { + outpointAndUTXOEntryPairs, err := syncer.GetPruningPointUTXOs(pruningPoint, fromOutpoint, step) + if err != nil { + return err + } + fromOutpoint = outpointAndUTXOEntryPairs[len(outpointAndUTXOEntryPairs)-1].Outpoint + err = syncee.AppendImportedPruningPointUTXOs(outpointAndUTXOEntryPairs) + if err != nil { + return err + } + if len(outpointAndUTXOEntryPairs) < step { + break + } + } + + // Check that ValidateAndInsertImportedPruningPoint works given the right arguments. + err = syncee.ValidateAndInsertImportedPruningPoint(pruningPoint) + if err != nil { + return err + } + + emptyCoinbase := &externalapi.DomainCoinbaseData{ + ScriptPublicKey: &externalapi.ScriptPublicKey{ + Script: nil, + Version: 0, + }, + } + + // Check that we can build a block just after importing the pruning point. + _, err = syncee.BuildBlock(emptyCoinbase, nil) + if err != nil { + return err + } + + estimatedVirtualDAAScoreTarget, err := syncer.GetVirtualDAAScore() + if err != nil { + return err + } + + virtualDAAScoreStart := uint64(0) + percents = 0 + for i := 0; ; i++ { + if i%10 == 0 { + virtualDAAScore, err := syncee.GetVirtualDAAScore() + if err != nil { + return err + } + newPercents := int(float64(virtualDAAScore-virtualDAAScoreStart) / float64(estimatedVirtualDAAScoreTarget-virtualDAAScoreStart) * 100) + if newPercents > percents { + percents = newPercents + log.Infof("Resolving virtual. Estimated progress: %d%%", percents) + } + } + _, isCompletelyResolved, err := syncee.ResolveVirtual() + if err != nil { + return err + } + + if isCompletelyResolved { + log.Infof("Resolved virtual") + break + } + } + + return nil +}