stasatdaglabs 756f40c59a
Sync pruning point UTXO sets incrementally instead of all at once (#1431)
* Replaced the content of MsgIBDRootUTXOSetChunk with pairs of outpoint-utxo entry pairs.

* Rename utxoIter to utxoIterator.

* Add a big stinky TODO on an assert.

* Replace pruningStore staging with a UTXO set iterator.

* Reimplement receiveAndInsertIBDRootUTXOSet.

* Extract OutpointAndUTXOEntryPairsToDomainOutpointAndUTXOEntryPairs into domainconverters.go.

* Pass the outpoint and utxy entry pairs to the pruning store.

* Implement InsertCandidatePruningPointUTXOs.

* Implement ClearCandidatePruningPointUTXOs.

* Implement UpdateCandidatePruningPointMultiset.

* Use the candidate pruning point multiset in updatePruningPoint.

* Implement CandidatePruningPointUTXOIterator.

* Use the pruning point utxo set iterator for StageVirtualUTXOSet.

* Defer ClearCandidatePruningPointUTXOs.

* Implement OverwriteVirtualUTXOSet.

* Implement CommitCandidatePruningPointUTXOSet.

* Implement BeginOverwritingVirtualUTXOSet and FinishOverwritingVirtualUTXOSet.

* Implement overwriteVirtualUTXOSetAndCommitPruningPointUTXOSet.

* Rename ClearCandidatePruningPointUTXOs to ClearCandidatePruningPointData.

* Add missing methods to dbManager.

* Implement PruningPointUTXOs.

* Implement RecoverUTXOIfRequired.

* Delete the utxoserialization package.

* Fix compilation errors in TestValidateAndInsertPruningPoint.

* Switch order of operations in the if statements in PruningPointUTXOs so that Next() wouldn't be unnecessarily called.

* Fix missing pruning point utxo set staging and bad slice length.

* Fix no default multiset in InsertCandidatePruningPointUTXOs.

* Make go vet happy.

* Rename candidateXXX to importedXXX.

* Do some more renaming.

* Rename some more.

* Fix bad MsgIBDRootNotFound logic.

* Fix an error message.

* Simplify receiveIBDRootBlock.

* Fix error message in receiveAndInsertIBDRootUTXOSet.

* Do some more renaming.

* Fix merge errors.

* Fix a bug caused by calling iterator.First() unnecessarily.

* Remove databaseContext from stores and don't use a transaction in ClearXXX functions.

* Simplify receiveAndInsertIBDRootUTXOSet.

* Fix offset count in PruningPointUTXOs().

* Fix readOnlyUTXOIteratorWithDiff.First().

* Split handleRequestIBDRootUTXOSetAndBlockFlow into smaller methods.

* Rename IbdRootNotFound to UnexpectedPruningPoint.

* Rename requestIBDRootHash to requestPruningPointHash.

* Rename IBDRootHash to PruningPointHash.

* Rename RequestIBDRootUTXOSetAndBlock to RequestPruningPointUTXOSetAndBlock.

* Rename IBDRootUTXOSetChunk to PruningPointUTXOSetChunk.

* Rename RequestNextIBDRootUTXOSetChunk to RequestNextPruningPointUTXOSetChunk.

* Rename DoneIBDRootUTXOSetChunks to DonePruningPointUTXOSetChunks.

* Rename remaining references to IBD root.

* Fix an error message.

* Add a check for HadStartedImportingPruningPointUTXOSet in commitVirtualUTXODiff.

* Add a check for HadStartedImportingPruningPointUTXOSet in ImportPruningPointUTXOSetIntoVirtualUTXOSet.

* Move FinishImportingPruningPointUTXOSet closer to HadStartedImportingPruningPointUTXOSet.

* Remove reference to pruningStore in utxoSetIterator.

* Pointerify utxoSetIterator receivers.

* Fix bad insert in CommitImportedPruningPointUTXOSet.

* Rename commitImportedPruningPointUTXOSetAll to applyImportedPruningPointUTXOSet.

* Simplify PruningPointUTXOs.

* Add populateTransactionWithUTXOEntriesFromUTXOSet.

* Fix a TODO comment.

* Rename InsertImportedPruningPointUTXOs to AppendImportedPruningPointUTXOs.

* Extract handleRequestPruningPointUTXOSetAndBlockMessage to a separate method.

* Rename stuff in readOnlyUTXOIteratorWithDiff.First().

* Address toAddIterator in readOnlyUTXOIteratorWithDiff.First().

* Call First() before any full iteration on ReadOnlyUTXOSetIterator.

* Call First() before any full iteration on a database Cursor.

* Put StartImportingPruningPointUTXOSet inside the pruning point transaction.

* Make serializeOutpoint and serializeUTXOEntry free functions in pruningStore.

* Fix readOnlyUTXOIteratorWithDiff.First().

* Fix bad validations in importPruningPoint.

* Remove superfluous call to validateBlockTransactionsAgainstPastUTXO.
2021-01-21 17:24:52 +02:00

244 lines
6.8 KiB
Go

package pruningstore
import (
"github.com/golang/protobuf/proto"
"github.com/kaspanet/kaspad/domain/consensus/database"
"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
"github.com/kaspanet/kaspad/domain/consensus/model"
"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
)
var pruningBlockHashKey = database.MakeBucket(nil).Key([]byte("pruning-block-hash"))
var candidatePruningPointHashKey = database.MakeBucket(nil).Key([]byte("candidate-pruning-point-hash"))
var pruningPointUTXOSetBucket = database.MakeBucket([]byte("pruning-point-utxo-set"))
// pruningStore represents a store for the current pruning state
type pruningStore struct {
pruningPointStaging *externalapi.DomainHash
pruningPointCache *externalapi.DomainHash
pruningPointCandidateStaging *externalapi.DomainHash
pruningPointCandidateCache *externalapi.DomainHash
pruningPointUTXOSetStaging model.ReadOnlyUTXOSetIterator
}
// New instantiates a new PruningStore
func New() model.PruningStore {
return &pruningStore{}
}
func (ps *pruningStore) StagePruningPointCandidate(candidate *externalapi.DomainHash) {
ps.pruningPointCandidateStaging = candidate
}
func (ps *pruningStore) PruningPointCandidate(dbContext model.DBReader) (*externalapi.DomainHash, error) {
if ps.pruningPointCandidateStaging != nil {
return ps.pruningPointCandidateStaging, nil
}
if ps.pruningPointCandidateCache != nil {
return ps.pruningPointCandidateCache, nil
}
candidateBytes, err := dbContext.Get(pruningBlockHashKey)
if err != nil {
return nil, err
}
candidate, err := ps.deserializePruningPoint(candidateBytes)
if err != nil {
return nil, err
}
ps.pruningPointCandidateCache = candidate
return candidate, nil
}
func (ps *pruningStore) HasPruningPointCandidate(dbContext model.DBReader) (bool, error) {
if ps.pruningPointCandidateStaging != nil {
return true, nil
}
if ps.pruningPointCandidateCache != nil {
return true, nil
}
return dbContext.Has(candidatePruningPointHashKey)
}
// Stage stages the pruning state
func (ps *pruningStore) StagePruningPoint(pruningPointBlockHash *externalapi.DomainHash) {
ps.pruningPointStaging = pruningPointBlockHash
}
func (ps *pruningStore) StagePruningPointUTXOSet(pruningPointUTXOSetIterator model.ReadOnlyUTXOSetIterator) {
ps.pruningPointUTXOSetStaging = pruningPointUTXOSetIterator
}
func (ps *pruningStore) IsStaged() bool {
return ps.pruningPointStaging != nil || ps.pruningPointUTXOSetStaging != nil
}
func (ps *pruningStore) Discard() {
ps.pruningPointStaging = nil
ps.pruningPointUTXOSetStaging = nil
}
func (ps *pruningStore) Commit(dbTx model.DBTransaction) error {
if ps.pruningPointStaging != nil {
pruningPointBytes, err := ps.serializeHash(ps.pruningPointStaging)
if err != nil {
return err
}
err = dbTx.Put(pruningBlockHashKey, pruningPointBytes)
if err != nil {
return err
}
ps.pruningPointCache = ps.pruningPointStaging
}
if ps.pruningPointCandidateStaging != nil {
candidateBytes, err := ps.serializeHash(ps.pruningPointCandidateStaging)
if err != nil {
return err
}
err = dbTx.Put(candidatePruningPointHashKey, candidateBytes)
if err != nil {
return err
}
ps.pruningPointCandidateCache = ps.pruningPointCandidateStaging
}
if ps.pruningPointUTXOSetStaging != nil {
// Delete all the old UTXOs from the database
deleteCursor, err := dbTx.Cursor(pruningPointUTXOSetBucket)
if err != nil {
return err
}
for ok := deleteCursor.First(); ok; ok = deleteCursor.Next() {
key, err := deleteCursor.Key()
if err != nil {
return err
}
err = dbTx.Delete(key)
if err != nil {
return err
}
}
// Insert all the new UTXOs into the database
for ok := ps.pruningPointUTXOSetStaging.First(); ok; ok = ps.pruningPointUTXOSetStaging.Next() {
outpoint, entry, err := ps.pruningPointUTXOSetStaging.Get()
if err != nil {
return err
}
serializedOutpoint, err := serializeOutpoint(outpoint)
if err != nil {
return err
}
key := pruningPointUTXOSetBucket.Key(serializedOutpoint)
serializedUTXOEntry, err := serializeUTXOEntry(entry)
if err != nil {
return err
}
err = dbTx.Put(key, serializedUTXOEntry)
if err != nil {
return err
}
}
}
ps.Discard()
return nil
}
// PruningPoint gets the current pruning point
func (ps *pruningStore) PruningPoint(dbContext model.DBReader) (*externalapi.DomainHash, error) {
if ps.pruningPointStaging != nil {
return ps.pruningPointStaging, nil
}
if ps.pruningPointCache != nil {
return ps.pruningPointCache, nil
}
pruningPointBytes, err := dbContext.Get(pruningBlockHashKey)
if err != nil {
return nil, err
}
pruningPoint, err := ps.deserializePruningPoint(pruningPointBytes)
if err != nil {
return nil, err
}
ps.pruningPointCache = pruningPoint
return pruningPoint, nil
}
func (ps *pruningStore) serializeHash(hash *externalapi.DomainHash) ([]byte, error) {
return proto.Marshal(serialization.DomainHashToDbHash(hash))
}
func (ps *pruningStore) deserializePruningPoint(pruningPointBytes []byte) (*externalapi.DomainHash, error) {
dbHash := &serialization.DbHash{}
err := proto.Unmarshal(pruningPointBytes, dbHash)
if err != nil {
return nil, err
}
return serialization.DbHashToDomainHash(dbHash)
}
func (ps *pruningStore) serializeUTXOSetBytes(pruningPointUTXOSetBytes []byte) ([]byte, error) {
return proto.Marshal(&serialization.DbPruningPointUTXOSetBytes{Bytes: pruningPointUTXOSetBytes})
}
func (ps *pruningStore) deserializeUTXOSetBytes(dbPruningPointUTXOSetBytes []byte) ([]byte, error) {
dbPruningPointUTXOSet := &serialization.DbPruningPointUTXOSetBytes{}
err := proto.Unmarshal(dbPruningPointUTXOSetBytes, dbPruningPointUTXOSet)
if err != nil {
return nil, err
}
return dbPruningPointUTXOSet.Bytes, nil
}
func (ps *pruningStore) HasPruningPoint(dbContext model.DBReader) (bool, error) {
if ps.pruningPointStaging != nil {
return true, nil
}
if ps.pruningPointCache != nil {
return true, nil
}
return dbContext.Has(pruningBlockHashKey)
}
func (ps *pruningStore) PruningPointUTXOs(dbContext model.DBReader,
offset int, limit int) ([]*externalapi.OutpointAndUTXOEntryPair, error) {
cursor, err := dbContext.Cursor(pruningPointUTXOSetBucket)
if err != nil {
return nil, err
}
pruningPointUTXOIterator := ps.newCursorUTXOSetIterator(cursor)
offsetIndex := 0
for offsetIndex < offset && pruningPointUTXOIterator.Next() {
offsetIndex++
}
outpointAndUTXOEntryPairs := make([]*externalapi.OutpointAndUTXOEntryPair, 0, limit)
for len(outpointAndUTXOEntryPairs) < limit && pruningPointUTXOIterator.Next() {
outpoint, utxoEntry, err := pruningPointUTXOIterator.Get()
if err != nil {
return nil, err
}
outpointAndUTXOEntryPairs = append(outpointAndUTXOEntryPairs, &externalapi.OutpointAndUTXOEntryPair{
Outpoint: outpoint,
UTXOEntry: utxoEntry,
})
}
return outpointAndUTXOEntryPairs, nil
}