stasatdaglabs 756f40c59a
Sync pruning point UTXO sets incrementally instead of all at once (#1431)
* Replaced the content of MsgIBDRootUTXOSetChunk with pairs of outpoint-utxo entry pairs.

* Rename utxoIter to utxoIterator.

* Add a big stinky TODO on an assert.

* Replace pruningStore staging with a UTXO set iterator.

* Reimplement receiveAndInsertIBDRootUTXOSet.

* Extract OutpointAndUTXOEntryPairsToDomainOutpointAndUTXOEntryPairs into domainconverters.go.

* Pass the outpoint and utxy entry pairs to the pruning store.

* Implement InsertCandidatePruningPointUTXOs.

* Implement ClearCandidatePruningPointUTXOs.

* Implement UpdateCandidatePruningPointMultiset.

* Use the candidate pruning point multiset in updatePruningPoint.

* Implement CandidatePruningPointUTXOIterator.

* Use the pruning point utxo set iterator for StageVirtualUTXOSet.

* Defer ClearCandidatePruningPointUTXOs.

* Implement OverwriteVirtualUTXOSet.

* Implement CommitCandidatePruningPointUTXOSet.

* Implement BeginOverwritingVirtualUTXOSet and FinishOverwritingVirtualUTXOSet.

* Implement overwriteVirtualUTXOSetAndCommitPruningPointUTXOSet.

* Rename ClearCandidatePruningPointUTXOs to ClearCandidatePruningPointData.

* Add missing methods to dbManager.

* Implement PruningPointUTXOs.

* Implement RecoverUTXOIfRequired.

* Delete the utxoserialization package.

* Fix compilation errors in TestValidateAndInsertPruningPoint.

* Switch order of operations in the if statements in PruningPointUTXOs so that Next() wouldn't be unnecessarily called.

* Fix missing pruning point utxo set staging and bad slice length.

* Fix no default multiset in InsertCandidatePruningPointUTXOs.

* Make go vet happy.

* Rename candidateXXX to importedXXX.

* Do some more renaming.

* Rename some more.

* Fix bad MsgIBDRootNotFound logic.

* Fix an error message.

* Simplify receiveIBDRootBlock.

* Fix error message in receiveAndInsertIBDRootUTXOSet.

* Do some more renaming.

* Fix merge errors.

* Fix a bug caused by calling iterator.First() unnecessarily.

* Remove databaseContext from stores and don't use a transaction in ClearXXX functions.

* Simplify receiveAndInsertIBDRootUTXOSet.

* Fix offset count in PruningPointUTXOs().

* Fix readOnlyUTXOIteratorWithDiff.First().

* Split handleRequestIBDRootUTXOSetAndBlockFlow into smaller methods.

* Rename IbdRootNotFound to UnexpectedPruningPoint.

* Rename requestIBDRootHash to requestPruningPointHash.

* Rename IBDRootHash to PruningPointHash.

* Rename RequestIBDRootUTXOSetAndBlock to RequestPruningPointUTXOSetAndBlock.

* Rename IBDRootUTXOSetChunk to PruningPointUTXOSetChunk.

* Rename RequestNextIBDRootUTXOSetChunk to RequestNextPruningPointUTXOSetChunk.

* Rename DoneIBDRootUTXOSetChunks to DonePruningPointUTXOSetChunks.

* Rename remaining references to IBD root.

* Fix an error message.

* Add a check for HadStartedImportingPruningPointUTXOSet in commitVirtualUTXODiff.

* Add a check for HadStartedImportingPruningPointUTXOSet in ImportPruningPointUTXOSetIntoVirtualUTXOSet.

* Move FinishImportingPruningPointUTXOSet closer to HadStartedImportingPruningPointUTXOSet.

* Remove reference to pruningStore in utxoSetIterator.

* Pointerify utxoSetIterator receivers.

* Fix bad insert in CommitImportedPruningPointUTXOSet.

* Rename commitImportedPruningPointUTXOSetAll to applyImportedPruningPointUTXOSet.

* Simplify PruningPointUTXOs.

* Add populateTransactionWithUTXOEntriesFromUTXOSet.

* Fix a TODO comment.

* Rename InsertImportedPruningPointUTXOs to AppendImportedPruningPointUTXOs.

* Extract handleRequestPruningPointUTXOSetAndBlockMessage to a separate method.

* Rename stuff in readOnlyUTXOIteratorWithDiff.First().

* Address toAddIterator in readOnlyUTXOIteratorWithDiff.First().

* Call First() before any full iteration on ReadOnlyUTXOSetIterator.

* Call First() before any full iteration on a database Cursor.

* Put StartImportingPruningPointUTXOSet inside the pruning point transaction.

* Make serializeOutpoint and serializeUTXOEntry free functions in pruningStore.

* Fix readOnlyUTXOIteratorWithDiff.First().

* Fix bad validations in importPruningPoint.

* Remove superfluous call to validateBlockTransactionsAgainstPastUTXO.
2021-01-21 17:24:52 +02:00

205 lines
5.1 KiB
Go

package consensusstatestore
import (
"github.com/kaspanet/kaspad/domain/consensus/database"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
"github.com/kaspanet/kaspad/domain/consensus/utils/utxo"
"github.com/pkg/errors"
)
var utxoSetBucket = database.MakeBucket([]byte("virtual-utxo-set"))
func utxoKey(outpoint *externalapi.DomainOutpoint) (model.DBKey, error) {
serializedOutpoint, err := serializeOutpoint(outpoint)
if err != nil {
return nil, err
}
return utxoSetBucket.Key(serializedOutpoint), nil
}
func (css *consensusStateStore) StageVirtualUTXODiff(virtualUTXODiff model.UTXODiff) {
css.virtualUTXODiffStaging = virtualUTXODiff
}
func (css *consensusStateStore) commitVirtualUTXODiff(dbTx model.DBTransaction) error {
hadStartedImportingPruningPointUTXOSet, err := css.HadStartedImportingPruningPointUTXOSet(dbTx)
if err != nil {
return err
}
if hadStartedImportingPruningPointUTXOSet {
return errors.New("cannot commit virtual UTXO diff after starting to import the pruning point UTXO set")
}
if css.virtualUTXODiffStaging == nil {
return nil
}
toRemoveIterator := css.virtualUTXODiffStaging.ToRemove().Iterator()
for ok := toRemoveIterator.First(); ok; ok = toRemoveIterator.Next() {
toRemoveOutpoint, _, err := toRemoveIterator.Get()
if err != nil {
return err
}
css.virtualUTXOSetCache.Remove(toRemoveOutpoint)
dbKey, err := utxoKey(toRemoveOutpoint)
if err != nil {
return err
}
err = dbTx.Delete(dbKey)
if err != nil {
return err
}
}
toAddIterator := css.virtualUTXODiffStaging.ToAdd().Iterator()
for ok := toAddIterator.First(); ok; ok = toAddIterator.Next() {
toAddOutpoint, toAddEntry, err := toAddIterator.Get()
if err != nil {
return err
}
css.virtualUTXOSetCache.Add(toAddOutpoint, toAddEntry)
dbKey, err := utxoKey(toAddOutpoint)
if err != nil {
return err
}
serializedEntry, err := serializeUTXOEntry(toAddEntry)
if err != nil {
return err
}
err = dbTx.Put(dbKey, serializedEntry)
if err != nil {
return err
}
}
// Note: we don't discard the staging here since that's
// being done at the end of Commit()
return nil
}
func (css *consensusStateStore) UTXOByOutpoint(dbContext model.DBReader, outpoint *externalapi.DomainOutpoint) (
externalapi.UTXOEntry, error) {
return css.utxoByOutpointFromStagedVirtualUTXODiff(dbContext, outpoint)
}
func (css *consensusStateStore) utxoByOutpointFromStagedVirtualUTXODiff(dbContext model.DBReader,
outpoint *externalapi.DomainOutpoint) (
externalapi.UTXOEntry, error) {
if css.virtualUTXODiffStaging != nil {
if css.virtualUTXODiffStaging.ToRemove().Contains(outpoint) {
return nil, errors.Errorf("outpoint was not found")
}
if utxoEntry, ok := css.virtualUTXODiffStaging.ToAdd().Get(outpoint); ok {
return utxoEntry, nil
}
}
if entry, ok := css.virtualUTXOSetCache.Get(outpoint); ok {
return entry, nil
}
key, err := utxoKey(outpoint)
if err != nil {
return nil, err
}
serializedUTXOEntry, err := dbContext.Get(key)
if err != nil {
return nil, err
}
entry, err := deserializeUTXOEntry(serializedUTXOEntry)
if err != nil {
return nil, err
}
css.virtualUTXOSetCache.Add(outpoint, entry)
return entry, nil
}
func (css *consensusStateStore) HasUTXOByOutpoint(dbContext model.DBReader, outpoint *externalapi.DomainOutpoint) (bool, error) {
return css.hasUTXOByOutpointFromStagedVirtualUTXODiff(dbContext, outpoint)
}
func (css *consensusStateStore) hasUTXOByOutpointFromStagedVirtualUTXODiff(dbContext model.DBReader,
outpoint *externalapi.DomainOutpoint) (bool, error) {
if css.virtualUTXODiffStaging != nil {
if css.virtualUTXODiffStaging.ToRemove().Contains(outpoint) {
return false, nil
}
if _, ok := css.virtualUTXODiffStaging.ToAdd().Get(outpoint); ok {
return true, nil
}
}
key, err := utxoKey(outpoint)
if err != nil {
return false, err
}
return dbContext.Has(key)
}
func (css *consensusStateStore) VirtualUTXOSetIterator(dbContext model.DBReader) (model.ReadOnlyUTXOSetIterator, error) {
cursor, err := dbContext.Cursor(utxoSetBucket)
if err != nil {
return nil, err
}
mainIterator := newCursorUTXOSetIterator(cursor)
if css.virtualUTXODiffStaging != nil {
return utxo.IteratorWithDiff(mainIterator, css.virtualUTXODiffStaging)
}
return mainIterator, nil
}
type utxoSetIterator struct {
cursor model.DBCursor
}
func newCursorUTXOSetIterator(cursor model.DBCursor) model.ReadOnlyUTXOSetIterator {
return &utxoSetIterator{cursor: cursor}
}
func (u utxoSetIterator) First() bool {
return u.cursor.First()
}
func (u utxoSetIterator) Next() bool {
return u.cursor.Next()
}
func (u utxoSetIterator) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry externalapi.UTXOEntry, err error) {
key, err := u.cursor.Key()
if err != nil {
panic(err)
}
utxoEntryBytes, err := u.cursor.Value()
if err != nil {
return nil, nil, err
}
outpoint, err = deserializeOutpoint(key.Suffix())
if err != nil {
return nil, nil, err
}
utxoEntry, err = deserializeUTXOEntry(utxoEntryBytes)
if err != nil {
return nil, nil, err
}
return outpoint, utxoEntry, nil
}