stasatdaglabs 756f40c59a
Sync pruning point UTXO sets incrementally instead of all at once (#1431)
* Replaced the content of MsgIBDRootUTXOSetChunk with pairs of outpoint-utxo entry pairs.

* Rename utxoIter to utxoIterator.

* Add a big stinky TODO on an assert.

* Replace pruningStore staging with a UTXO set iterator.

* Reimplement receiveAndInsertIBDRootUTXOSet.

* Extract OutpointAndUTXOEntryPairsToDomainOutpointAndUTXOEntryPairs into domainconverters.go.

* Pass the outpoint and utxy entry pairs to the pruning store.

* Implement InsertCandidatePruningPointUTXOs.

* Implement ClearCandidatePruningPointUTXOs.

* Implement UpdateCandidatePruningPointMultiset.

* Use the candidate pruning point multiset in updatePruningPoint.

* Implement CandidatePruningPointUTXOIterator.

* Use the pruning point utxo set iterator for StageVirtualUTXOSet.

* Defer ClearCandidatePruningPointUTXOs.

* Implement OverwriteVirtualUTXOSet.

* Implement CommitCandidatePruningPointUTXOSet.

* Implement BeginOverwritingVirtualUTXOSet and FinishOverwritingVirtualUTXOSet.

* Implement overwriteVirtualUTXOSetAndCommitPruningPointUTXOSet.

* Rename ClearCandidatePruningPointUTXOs to ClearCandidatePruningPointData.

* Add missing methods to dbManager.

* Implement PruningPointUTXOs.

* Implement RecoverUTXOIfRequired.

* Delete the utxoserialization package.

* Fix compilation errors in TestValidateAndInsertPruningPoint.

* Switch order of operations in the if statements in PruningPointUTXOs so that Next() wouldn't be unnecessarily called.

* Fix missing pruning point utxo set staging and bad slice length.

* Fix no default multiset in InsertCandidatePruningPointUTXOs.

* Make go vet happy.

* Rename candidateXXX to importedXXX.

* Do some more renaming.

* Rename some more.

* Fix bad MsgIBDRootNotFound logic.

* Fix an error message.

* Simplify receiveIBDRootBlock.

* Fix error message in receiveAndInsertIBDRootUTXOSet.

* Do some more renaming.

* Fix merge errors.

* Fix a bug caused by calling iterator.First() unnecessarily.

* Remove databaseContext from stores and don't use a transaction in ClearXXX functions.

* Simplify receiveAndInsertIBDRootUTXOSet.

* Fix offset count in PruningPointUTXOs().

* Fix readOnlyUTXOIteratorWithDiff.First().

* Split handleRequestIBDRootUTXOSetAndBlockFlow into smaller methods.

* Rename IbdRootNotFound to UnexpectedPruningPoint.

* Rename requestIBDRootHash to requestPruningPointHash.

* Rename IBDRootHash to PruningPointHash.

* Rename RequestIBDRootUTXOSetAndBlock to RequestPruningPointUTXOSetAndBlock.

* Rename IBDRootUTXOSetChunk to PruningPointUTXOSetChunk.

* Rename RequestNextIBDRootUTXOSetChunk to RequestNextPruningPointUTXOSetChunk.

* Rename DoneIBDRootUTXOSetChunks to DonePruningPointUTXOSetChunks.

* Rename remaining references to IBD root.

* Fix an error message.

* Add a check for HadStartedImportingPruningPointUTXOSet in commitVirtualUTXODiff.

* Add a check for HadStartedImportingPruningPointUTXOSet in ImportPruningPointUTXOSetIntoVirtualUTXOSet.

* Move FinishImportingPruningPointUTXOSet closer to HadStartedImportingPruningPointUTXOSet.

* Remove reference to pruningStore in utxoSetIterator.

* Pointerify utxoSetIterator receivers.

* Fix bad insert in CommitImportedPruningPointUTXOSet.

* Rename commitImportedPruningPointUTXOSetAll to applyImportedPruningPointUTXOSet.

* Simplify PruningPointUTXOs.

* Add populateTransactionWithUTXOEntriesFromUTXOSet.

* Fix a TODO comment.

* Rename InsertImportedPruningPointUTXOs to AppendImportedPruningPointUTXOs.

* Extract handleRequestPruningPointUTXOSetAndBlockMessage to a separate method.

* Rename stuff in readOnlyUTXOIteratorWithDiff.First().

* Address toAddIterator in readOnlyUTXOIteratorWithDiff.First().

* Call First() before any full iteration on ReadOnlyUTXOSetIterator.

* Call First() before any full iteration on a database Cursor.

* Put StartImportingPruningPointUTXOSet inside the pruning point transaction.

* Make serializeOutpoint and serializeUTXOEntry free functions in pruningStore.

* Fix readOnlyUTXOIteratorWithDiff.First().

* Fix bad validations in importPruningPoint.

* Remove superfluous call to validateBlockTransactionsAgainstPastUTXO.
2021-01-21 17:24:52 +02:00

82 lines
2.2 KiB
Go

package consensusstatestore
import (
"github.com/kaspanet/kaspad/domain/consensus/database"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/pkg/errors"
)
var importingPruningPointUTXOSetKey = database.MakeBucket(nil).Key([]byte("importing-pruning-point-utxo-set"))
func (css *consensusStateStore) StartImportingPruningPointUTXOSet(dbContext model.DBWriter) error {
return dbContext.Put(importingPruningPointUTXOSetKey, []byte{0})
}
func (css *consensusStateStore) HadStartedImportingPruningPointUTXOSet(dbContext model.DBWriter) (bool, error) {
return dbContext.Has(importingPruningPointUTXOSetKey)
}
func (css *consensusStateStore) FinishImportingPruningPointUTXOSet(dbContext model.DBWriter) error {
return dbContext.Delete(importingPruningPointUTXOSetKey)
}
func (css *consensusStateStore) ImportPruningPointUTXOSetIntoVirtualUTXOSet(dbContext model.DBWriter,
pruningPointUTXOSetIterator model.ReadOnlyUTXOSetIterator) error {
if css.virtualUTXODiffStaging != nil {
return errors.New("cannot import virtual UTXO set while virtual UTXO diff is staged")
}
hadStartedImportingPruningPointUTXOSet, err := css.HadStartedImportingPruningPointUTXOSet(dbContext)
if err != nil {
return err
}
if !hadStartedImportingPruningPointUTXOSet {
return errors.New("cannot import pruning point UTXO set " +
"without calling StartImportingPruningPointUTXOSet first")
}
// Clear the cache
css.virtualUTXOSetCache.Clear()
// Delete all the old UTXOs from the database
deleteCursor, err := dbContext.Cursor(utxoSetBucket)
if err != nil {
return err
}
for ok := deleteCursor.First(); ok; ok = deleteCursor.Next() {
key, err := deleteCursor.Key()
if err != nil {
return err
}
err = dbContext.Delete(key)
if err != nil {
return err
}
}
// Insert all the new UTXOs into the database
for ok := pruningPointUTXOSetIterator.First(); ok; ok = pruningPointUTXOSetIterator.Next() {
outpoint, entry, err := pruningPointUTXOSetIterator.Get()
if err != nil {
return err
}
key, err := utxoKey(outpoint)
if err != nil {
return err
}
serializedUTXOEntry, err := serializeUTXOEntry(entry)
if err != nil {
return err
}
err = dbContext.Put(key, serializedUTXOEntry)
if err != nil {
return err
}
}
return nil
}