stasatdaglabs 756f40c59a
Sync pruning point UTXO sets incrementally instead of all at once (#1431)
* Replaced the content of MsgIBDRootUTXOSetChunk with pairs of outpoint-utxo entry pairs.

* Rename utxoIter to utxoIterator.

* Add a big stinky TODO on an assert.

* Replace pruningStore staging with a UTXO set iterator.

* Reimplement receiveAndInsertIBDRootUTXOSet.

* Extract OutpointAndUTXOEntryPairsToDomainOutpointAndUTXOEntryPairs into domainconverters.go.

* Pass the outpoint and utxy entry pairs to the pruning store.

* Implement InsertCandidatePruningPointUTXOs.

* Implement ClearCandidatePruningPointUTXOs.

* Implement UpdateCandidatePruningPointMultiset.

* Use the candidate pruning point multiset in updatePruningPoint.

* Implement CandidatePruningPointUTXOIterator.

* Use the pruning point utxo set iterator for StageVirtualUTXOSet.

* Defer ClearCandidatePruningPointUTXOs.

* Implement OverwriteVirtualUTXOSet.

* Implement CommitCandidatePruningPointUTXOSet.

* Implement BeginOverwritingVirtualUTXOSet and FinishOverwritingVirtualUTXOSet.

* Implement overwriteVirtualUTXOSetAndCommitPruningPointUTXOSet.

* Rename ClearCandidatePruningPointUTXOs to ClearCandidatePruningPointData.

* Add missing methods to dbManager.

* Implement PruningPointUTXOs.

* Implement RecoverUTXOIfRequired.

* Delete the utxoserialization package.

* Fix compilation errors in TestValidateAndInsertPruningPoint.

* Switch order of operations in the if statements in PruningPointUTXOs so that Next() wouldn't be unnecessarily called.

* Fix missing pruning point utxo set staging and bad slice length.

* Fix no default multiset in InsertCandidatePruningPointUTXOs.

* Make go vet happy.

* Rename candidateXXX to importedXXX.

* Do some more renaming.

* Rename some more.

* Fix bad MsgIBDRootNotFound logic.

* Fix an error message.

* Simplify receiveIBDRootBlock.

* Fix error message in receiveAndInsertIBDRootUTXOSet.

* Do some more renaming.

* Fix merge errors.

* Fix a bug caused by calling iterator.First() unnecessarily.

* Remove databaseContext from stores and don't use a transaction in ClearXXX functions.

* Simplify receiveAndInsertIBDRootUTXOSet.

* Fix offset count in PruningPointUTXOs().

* Fix readOnlyUTXOIteratorWithDiff.First().

* Split handleRequestIBDRootUTXOSetAndBlockFlow into smaller methods.

* Rename IbdRootNotFound to UnexpectedPruningPoint.

* Rename requestIBDRootHash to requestPruningPointHash.

* Rename IBDRootHash to PruningPointHash.

* Rename RequestIBDRootUTXOSetAndBlock to RequestPruningPointUTXOSetAndBlock.

* Rename IBDRootUTXOSetChunk to PruningPointUTXOSetChunk.

* Rename RequestNextIBDRootUTXOSetChunk to RequestNextPruningPointUTXOSetChunk.

* Rename DoneIBDRootUTXOSetChunks to DonePruningPointUTXOSetChunks.

* Rename remaining references to IBD root.

* Fix an error message.

* Add a check for HadStartedImportingPruningPointUTXOSet in commitVirtualUTXODiff.

* Add a check for HadStartedImportingPruningPointUTXOSet in ImportPruningPointUTXOSetIntoVirtualUTXOSet.

* Move FinishImportingPruningPointUTXOSet closer to HadStartedImportingPruningPointUTXOSet.

* Remove reference to pruningStore in utxoSetIterator.

* Pointerify utxoSetIterator receivers.

* Fix bad insert in CommitImportedPruningPointUTXOSet.

* Rename commitImportedPruningPointUTXOSetAll to applyImportedPruningPointUTXOSet.

* Simplify PruningPointUTXOs.

* Add populateTransactionWithUTXOEntriesFromUTXOSet.

* Fix a TODO comment.

* Rename InsertImportedPruningPointUTXOs to AppendImportedPruningPointUTXOs.

* Extract handleRequestPruningPointUTXOSetAndBlockMessage to a separate method.

* Rename stuff in readOnlyUTXOIteratorWithDiff.First().

* Address toAddIterator in readOnlyUTXOIteratorWithDiff.First().

* Call First() before any full iteration on ReadOnlyUTXOSetIterator.

* Call First() before any full iteration on a database Cursor.

* Put StartImportingPruningPointUTXOSet inside the pruning point transaction.

* Make serializeOutpoint and serializeUTXOEntry free functions in pruningStore.

* Fix readOnlyUTXOIteratorWithDiff.First().

* Fix bad validations in importPruningPoint.

* Remove superfluous call to validateBlockTransactionsAgainstPastUTXO.
2021-01-21 17:24:52 +02:00

146 lines
5.2 KiB
Go

package blockprocessor
import (
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
"github.com/kaspanet/kaspad/infrastructure/logger"
"time"
)
// blockProcessor is responsible for processing incoming blocks
// and creating blocks from the current state
type blockProcessor struct {
genesisHash *externalapi.DomainHash
targetTimePerBlock time.Duration
databaseContext model.DBManager
consensusStateManager model.ConsensusStateManager
pruningManager model.PruningManager
blockValidator model.BlockValidator
dagTopologyManager model.DAGTopologyManager
reachabilityManager model.ReachabilityManager
difficultyManager model.DifficultyManager
ghostdagManager model.GHOSTDAGManager
pastMedianTimeManager model.PastMedianTimeManager
coinbaseManager model.CoinbaseManager
headerTipsManager model.HeadersSelectedTipManager
syncManager model.SyncManager
acceptanceDataStore model.AcceptanceDataStore
blockStore model.BlockStore
blockStatusStore model.BlockStatusStore
blockRelationStore model.BlockRelationStore
multisetStore model.MultisetStore
ghostdagDataStore model.GHOSTDAGDataStore
consensusStateStore model.ConsensusStateStore
pruningStore model.PruningStore
reachabilityDataStore model.ReachabilityDataStore
utxoDiffStore model.UTXODiffStore
blockHeaderStore model.BlockHeaderStore
headersSelectedTipStore model.HeaderSelectedTipStore
finalityStore model.FinalityStore
headersSelectedChainStore model.HeadersSelectedChainStore
stores []model.Store
}
// New instantiates a new BlockProcessor
func New(
genesisHash *externalapi.DomainHash,
targetTimePerBlock time.Duration,
databaseContext model.DBManager,
consensusStateManager model.ConsensusStateManager,
pruningManager model.PruningManager,
blockValidator model.BlockValidator,
dagTopologyManager model.DAGTopologyManager,
reachabilityManager model.ReachabilityManager,
difficultyManager model.DifficultyManager,
pastMedianTimeManager model.PastMedianTimeManager,
ghostdagManager model.GHOSTDAGManager,
coinbaseManager model.CoinbaseManager,
headerTipsManager model.HeadersSelectedTipManager,
syncManager model.SyncManager,
acceptanceDataStore model.AcceptanceDataStore,
blockStore model.BlockStore,
blockStatusStore model.BlockStatusStore,
blockRelationStore model.BlockRelationStore,
multisetStore model.MultisetStore,
ghostdagDataStore model.GHOSTDAGDataStore,
consensusStateStore model.ConsensusStateStore,
pruningStore model.PruningStore,
reachabilityDataStore model.ReachabilityDataStore,
utxoDiffStore model.UTXODiffStore,
blockHeaderStore model.BlockHeaderStore,
headersSelectedTipStore model.HeaderSelectedTipStore,
finalityStore model.FinalityStore,
headersSelectedChainStore model.HeadersSelectedChainStore,
) model.BlockProcessor {
return &blockProcessor{
genesisHash: genesisHash,
targetTimePerBlock: targetTimePerBlock,
databaseContext: databaseContext,
pruningManager: pruningManager,
blockValidator: blockValidator,
dagTopologyManager: dagTopologyManager,
reachabilityManager: reachabilityManager,
difficultyManager: difficultyManager,
pastMedianTimeManager: pastMedianTimeManager,
ghostdagManager: ghostdagManager,
coinbaseManager: coinbaseManager,
headerTipsManager: headerTipsManager,
syncManager: syncManager,
consensusStateManager: consensusStateManager,
acceptanceDataStore: acceptanceDataStore,
blockStore: blockStore,
blockStatusStore: blockStatusStore,
blockRelationStore: blockRelationStore,
multisetStore: multisetStore,
ghostdagDataStore: ghostdagDataStore,
consensusStateStore: consensusStateStore,
pruningStore: pruningStore,
reachabilityDataStore: reachabilityDataStore,
utxoDiffStore: utxoDiffStore,
blockHeaderStore: blockHeaderStore,
headersSelectedTipStore: headersSelectedTipStore,
finalityStore: finalityStore,
headersSelectedChainStore: headersSelectedChainStore,
stores: []model.Store{
consensusStateStore,
acceptanceDataStore,
blockStore,
blockStatusStore,
blockRelationStore,
multisetStore,
ghostdagDataStore,
consensusStateStore,
pruningStore,
reachabilityDataStore,
utxoDiffStore,
blockHeaderStore,
headersSelectedTipStore,
finalityStore,
headersSelectedChainStore,
},
}
}
// ValidateAndInsertBlock validates the given block and, if valid, applies it
// to the current state
func (bp *blockProcessor) ValidateAndInsertBlock(block *externalapi.DomainBlock) (*externalapi.BlockInsertionResult, error) {
onEnd := logger.LogAndMeasureExecutionTime(log, "ValidateAndInsertBlock")
defer onEnd()
return bp.validateAndInsertBlock(block, false)
}
func (bp *blockProcessor) ValidateAndInsertImportedPruningPoint(newPruningPoint *externalapi.DomainBlock) error {
onEnd := logger.LogAndMeasureExecutionTime(log, "ValidateAndInsertImportedPruningPoint")
defer onEnd()
return bp.validateAndInsertImportedPruningPoint(newPruningPoint)
}