From aea3baf897466683545be821ff8db543b4a55aaa Mon Sep 17 00:00:00 2001 From: Kirill Date: Sun, 27 Sep 2020 13:16:11 +0300 Subject: [PATCH] [NOD-1320] Flush UTXOs to disk (#902) * [NOD-1320] Flush UTXOs to disk. * [NOD-1320] Minor improvements and fixes. * FullUTXOSet: change size type from int64 to uint64. * Rename FullUTXOSet.size to FullUTXOSet.estimatedSize * Fill NewFullUTXOSetFromContext with db context on virtual block creation. * Typo fixes. * [NOD-1320] Stylystic fixes. * [NOD-1320] Update tests. Improvements and fixes. * Update blockdag/dag tests: prepare DB for tests. * Update blockdag/utxoset tests: prepare DB for tests. * Update blockdag/test_utils utils. * Update blockdag/common tests. * FullUTXOSet: remove embedded utxoCollection type. Rename utxoCollection to utxoCache. * Fix blockdag/block_utxo genesisPastUTXO func. * Minor fixes and improvements. --- app/app.go | 15 ++-- domain/blockdag/block_utxo.go | 2 +- domain/blockdag/common_test.go | 2 +- domain/blockdag/config.go | 4 + domain/blockdag/dag.go | 4 +- domain/blockdag/dag_test.go | 8 +- domain/blockdag/dagio.go | 12 --- domain/blockdag/test_utils.go | 15 ++-- domain/blockdag/utxoset.go | 115 ++++++++++++++++++++---- domain/blockdag/utxoset_test.go | 138 +++++++++++++++++++++-------- domain/blockdag/virtualblock.go | 2 +- infrastructure/config/config.go | 11 ++- infrastructure/db/dbaccess/utxo.go | 12 +++ 13 files changed, 251 insertions(+), 89 deletions(-) diff --git a/app/app.go b/app/app.go index 3618224fc..ad2a590b8 100644 --- a/app/app.go +++ b/app/app.go @@ -201,13 +201,14 @@ func setupDAG(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, int sigCache *txscript.SigCache, indexManager blockdag.IndexManager) (*blockdag.BlockDAG, error) { dag, err := blockdag.New(&blockdag.Config{ - Interrupt: interrupt, - DatabaseContext: databaseContext, - DAGParams: cfg.NetParams(), - TimeSource: blockdag.NewTimeSource(), - SigCache: sigCache, - IndexManager: indexManager, - SubnetworkID: cfg.SubnetworkID, + Interrupt: interrupt, + DatabaseContext: databaseContext, + DAGParams: cfg.NetParams(), + TimeSource: blockdag.NewTimeSource(), + SigCache: sigCache, + IndexManager: indexManager, + SubnetworkID: cfg.SubnetworkID, + MaxUTXOCacheSize: cfg.MaxUTXOCacheSize, }) return dag, err } diff --git a/domain/blockdag/block_utxo.go b/domain/blockdag/block_utxo.go index 340417343..80aa52155 100644 --- a/domain/blockdag/block_utxo.go +++ b/domain/blockdag/block_utxo.go @@ -107,7 +107,7 @@ func genesisPastUTXO(virtual *virtualBlock) UTXOSet { // set by creating a diff UTXO set with the virtual UTXO // set, and adding all of its entries in toRemove diff := NewUTXODiff() - for outpoint, entry := range virtual.utxoSet.utxoCollection { + for outpoint, entry := range virtual.utxoSet.utxoCache { diff.toRemove[outpoint] = entry } genesisPastUTXO := UTXOSet(NewDiffUTXOSet(virtual.utxoSet, diff)) diff --git a/domain/blockdag/common_test.go b/domain/blockdag/common_test.go index 504891812..290284f8f 100644 --- a/domain/blockdag/common_test.go +++ b/domain/blockdag/common_test.go @@ -78,7 +78,7 @@ func loadUTXOSet(filename string) (UTXOSet, error) { if err != nil { return nil, err } - utxoSet.utxoCollection[appmessage.Outpoint{TxID: txID, Index: index}] = entry + utxoSet.utxoCache[appmessage.Outpoint{TxID: txID, Index: index}] = entry } return utxoSet, nil diff --git a/domain/blockdag/config.go b/domain/blockdag/config.go index 5685d3dab..9724b2987 100644 --- a/domain/blockdag/config.go +++ b/domain/blockdag/config.go @@ -51,4 +51,8 @@ type Config struct { // DatabaseContext is the context in which all database queries related to // this DAG are going to run. DatabaseContext *dbaccess.DatabaseContext + + // MaxUTXOCacheSize is the Max size of loaded UTXO into ram from the disk in bytes + // to support UTXO lazy-load + MaxUTXOCacheSize uint64 } diff --git a/domain/blockdag/dag.go b/domain/blockdag/dag.go index a184e77e3..9fd5def07 100644 --- a/domain/blockdag/dag.go +++ b/domain/blockdag/dag.go @@ -94,7 +94,8 @@ type BlockDAG struct { recentBlockProcessingTimestamps []mstime.Time startTime mstime.Time - tips blockSet + maxUTXOCacheSize uint64 + tips blockSet // validTips is a set of blocks with the status "valid", which have no valid descendants. // Note that some validTips might not be actual tips. @@ -122,6 +123,7 @@ func New(config *Config) (*BlockDAG, error) { blockCount: 0, subnetworkID: config.SubnetworkID, startTime: mstime.Now(), + maxUTXOCacheSize: config.MaxUTXOCacheSize, } dag.virtual = newVirtualBlock(dag, nil) diff --git a/domain/blockdag/dag_test.go b/domain/blockdag/dag_test.go index 3fd62497d..9d397d05c 100644 --- a/domain/blockdag/dag_test.go +++ b/domain/blockdag/dag_test.go @@ -17,6 +17,7 @@ import ( "github.com/kaspanet/kaspad/app/appmessage" "github.com/kaspanet/kaspad/domain/dagconfig" "github.com/kaspanet/kaspad/domain/txscript" + "github.com/kaspanet/kaspad/infrastructure/config" "github.com/kaspanet/kaspad/infrastructure/db/dbaccess" "github.com/kaspanet/kaspad/util" "github.com/kaspanet/kaspad/util/daghash" @@ -278,7 +279,10 @@ func TestCalcSequenceLock(t *testing.T) { // age of 4 blocks. msgTx := appmessage.NewNativeMsgTx(appmessage.TxVersion, nil, []*appmessage.TxOut{{ScriptPubKey: nil, Value: 10}}) targetTx := util.NewTx(msgTx) - utxoSet := NewFullUTXOSet() + fullUTXOCacheSize := config.DefaultConfig().MaxUTXOCacheSize + db, teardown := prepareDatabaseForTest(t, "TestCalcSequenceLock") + defer teardown() + utxoSet := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) blueScore := uint64(numBlocksToGenerate) - 4 if isAccepted, err := utxoSet.AddTx(targetTx.MsgTx(), blueScore); err != nil { t.Fatalf("AddTx unexpectedly failed. Error: %s", err) @@ -1272,7 +1276,7 @@ func TestUTXOCommitment(t *testing.T) { // Build a Multiset for block D multiset := secp256k1.NewMultiset() - for outpoint, entry := range blockDPastDiffUTXOSet.base.utxoCollection { + for outpoint, entry := range blockDPastDiffUTXOSet.base.utxoCache { var err error multiset, err = addUTXOToMultiset(multiset, entry, &outpoint) if err != nil { diff --git a/domain/blockdag/dagio.go b/domain/blockdag/dagio.go index 250e148b8..e6984b18c 100644 --- a/domain/blockdag/dagio.go +++ b/domain/blockdag/dagio.go @@ -211,12 +211,6 @@ func (dag *BlockDAG) initDAGState() error { return err } - log.Debugf("Loading UTXO set...") - fullUTXOCollection, err := dag.initUTXOSet() - if err != nil { - return err - } - log.Debugf("Loading reachability data...") err = dag.reachabilityTree.init(dag.databaseContext) if err != nil { @@ -229,12 +223,6 @@ func (dag *BlockDAG) initDAGState() error { return err } - log.Debugf("Applying the loaded utxoCollection to the virtual block...") - dag.virtual.utxoSet, err = newFullUTXOSetFromUTXOCollection(fullUTXOCollection) - if err != nil { - return errors.Wrap(err, "Error loading UTXOSet") - } - log.Debugf("Applying the stored tips to the virtual block...") err = dag.initTipsAndVirtualParents(dagState) if err != nil { diff --git a/domain/blockdag/test_utils.go b/domain/blockdag/test_utils.go index 7d94923cb..8a06f1d63 100644 --- a/domain/blockdag/test_utils.go +++ b/domain/blockdag/test_utils.go @@ -14,14 +14,14 @@ import ( "sync" "testing" + "github.com/kaspanet/kaspad/infrastructure/config" "github.com/kaspanet/kaspad/infrastructure/db/database/ffldb/ldb" "github.com/kaspanet/kaspad/infrastructure/db/dbaccess" "github.com/kaspanet/kaspad/util" + "github.com/kaspanet/kaspad/util/subnetworkid" "github.com/pkg/errors" "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/kaspanet/kaspad/util/subnetworkid" - "github.com/kaspanet/kaspad/app/appmessage" "github.com/kaspanet/kaspad/domain/txscript" "github.com/kaspanet/kaspad/util/daghash" @@ -43,7 +43,7 @@ func FileExists(name string) bool { // The openDB parameter instructs DAGSetup whether or not to also open the // database. Setting it to false is useful in tests that handle database // opening/closing by themselves. -func DAGSetup(dbName string, openDb bool, config Config) (*BlockDAG, func(), error) { +func DAGSetup(dbName string, openDb bool, dagConfig Config) (*BlockDAG, func(), error) { var teardown func() // To make sure that the teardown function is not called before any goroutines finished to run - @@ -81,7 +81,7 @@ func DAGSetup(dbName string, openDb bool, config Config) (*BlockDAG, func(), err return nil, nil, errors.Errorf("error creating db: %s", err) } - config.DatabaseContext = databaseContext + dagConfig.DatabaseContext = databaseContext // Setup a teardown function for cleaning up. This function is // returned to the caller to be invoked when it is done testing. @@ -99,11 +99,12 @@ func DAGSetup(dbName string, openDb bool, config Config) (*BlockDAG, func(), err } } - config.TimeSource = NewTimeSource() - config.SigCache = txscript.NewSigCache(1000) + dagConfig.TimeSource = NewTimeSource() + dagConfig.SigCache = txscript.NewSigCache(1000) + dagConfig.MaxUTXOCacheSize = config.DefaultConfig().MaxUTXOCacheSize // Create the DAG instance. - dag, err := New(&config) + dag, err := New(&dagConfig) if err != nil { teardown() err := errors.Wrapf(err, "failed to create dag instance") diff --git a/domain/blockdag/utxoset.go b/domain/blockdag/utxoset.go index 6e3a7d887..f35d0c1ec 100644 --- a/domain/blockdag/utxoset.go +++ b/domain/blockdag/utxoset.go @@ -1,14 +1,17 @@ package blockdag import ( + "bytes" "fmt" "math" "sort" "strings" + "unsafe" + + "github.com/kaspanet/kaspad/infrastructure/db/dbaccess" "github.com/pkg/errors" - "github.com/kaspanet/go-secp256k1" "github.com/kaspanet/kaspad/app/appmessage" ) @@ -484,29 +487,27 @@ type UTXOSet interface { // FullUTXOSet represents a full list of transaction outputs and their values type FullUTXOSet struct { - utxoCollection + utxoCache utxoCollection + dbContext dbaccess.Context + estimatedSize uint64 + maxUTXOCacheSize uint64 + outpointBuff *bytes.Buffer } // NewFullUTXOSet creates a new utxoSet with full list of transaction outputs and their values func NewFullUTXOSet() *FullUTXOSet { return &FullUTXOSet{ - utxoCollection: utxoCollection{}, + utxoCache: utxoCollection{}, } } -// newFullUTXOSetFromUTXOCollection converts a utxoCollection to a FullUTXOSet -func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet, error) { - var err error - multiset := secp256k1.NewMultiset() - for outpoint, utxoEntry := range collection { - multiset, err = addUTXOToMultiset(multiset, utxoEntry, &outpoint) - if err != nil { - return nil, err - } - } +// NewFullUTXOSetFromContext creates a new utxoSet and map the data context with caching +func NewFullUTXOSetFromContext(context dbaccess.Context, cacheSize uint64) *FullUTXOSet { return &FullUTXOSet{ - utxoCollection: collection, - }, nil + dbContext: context, + maxUTXOCacheSize: cacheSize, + utxoCache: make(utxoCollection), + } } // diffFrom returns the difference between this utxoSet and another @@ -564,15 +565,93 @@ func (fus *FullUTXOSet) containsInputs(tx *appmessage.MsgTx) bool { return true } +// contains returns a boolean value indicating whether a UTXO entry is in the set +func (fus *FullUTXOSet) contains(outpoint appmessage.Outpoint) bool { + _, ok := fus.Get(outpoint) + return ok +} + // clone returns a clone of this utxoSet func (fus *FullUTXOSet) clone() UTXOSet { - return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone()} + return &FullUTXOSet{ + utxoCache: fus.utxoCache.clone(), + dbContext: fus.dbContext, + estimatedSize: fus.estimatedSize, + maxUTXOCacheSize: fus.maxUTXOCacheSize, + } +} + +// get returns the UTXOEntry associated with the given Outpoint, and a boolean indicating if such entry was found +func (fus *FullUTXOSet) get(outpoint appmessage.Outpoint) (*UTXOEntry, bool) { + return fus.Get(outpoint) +} + +// getSizeOfUTXOEntryAndOutpoint returns estimated size of UTXOEntry & Outpoint in bytes +func getSizeOfUTXOEntryAndOutpoint(entry *UTXOEntry) uint64 { + const staticSize = uint64(unsafe.Sizeof(UTXOEntry{}) + unsafe.Sizeof(appmessage.Outpoint{})) + return staticSize + uint64(len(entry.scriptPubKey)) +} + +// checkAndCleanCachedData checks the FullUTXOSet estimated size and clean it if it reaches the limit +func (fus *FullUTXOSet) checkAndCleanCachedData() { + if fus.estimatedSize > fus.maxUTXOCacheSize { + fus.utxoCache = make(utxoCollection) + fus.estimatedSize = 0 + } +} + +// add adds a new UTXO entry to this FullUTXOSet +func (fus *FullUTXOSet) add(outpoint appmessage.Outpoint, entry *UTXOEntry) { + fus.utxoCache[outpoint] = entry + fus.estimatedSize += getSizeOfUTXOEntryAndOutpoint(entry) + fus.checkAndCleanCachedData() +} + +// remove removes a UTXO entry from this collection if it exists +func (fus *FullUTXOSet) remove(outpoint appmessage.Outpoint) { + entry, ok := fus.utxoCache.get(outpoint) + if ok { + delete(fus.utxoCache, outpoint) + fus.estimatedSize -= getSizeOfUTXOEntryAndOutpoint(entry) + } } // Get returns the UTXOEntry associated with the given Outpoint, and a boolean indicating if such entry was found +// If the UTXOEntry doesn't not exist in the memory then check in the database func (fus *FullUTXOSet) Get(outpoint appmessage.Outpoint) (*UTXOEntry, bool) { - utxoEntry, ok := fus.utxoCollection[outpoint] - return utxoEntry, ok + utxoEntry, ok := fus.utxoCache[outpoint] + if ok { + return utxoEntry, ok + } + + if fus.outpointBuff == nil { + fus.outpointBuff = bytes.NewBuffer(make([]byte, outpointSerializeSize)) + } + + fus.outpointBuff.Reset() + err := serializeOutpoint(fus.outpointBuff, &outpoint) + if err != nil { + return nil, false + } + + key := fus.outpointBuff.Bytes() + value, err := dbaccess.GetFromUTXOSet(fus.dbContext, key) + + if err != nil { + return nil, false + } + + entry, err := deserializeUTXOEntry(bytes.NewReader(value)) + if err != nil { + return nil, false + } + + fus.add(outpoint, entry) + return entry, true +} + +func (fus *FullUTXOSet) String() string { + return fus.utxoCache.String() } // DiffUTXOSet represents a utxoSet with a base fullUTXOSet and a UTXODiff diff --git a/domain/blockdag/utxoset_test.go b/domain/blockdag/utxoset_test.go index c290f8f0b..80f5eb32b 100644 --- a/domain/blockdag/utxoset_test.go +++ b/domain/blockdag/utxoset_test.go @@ -1,15 +1,47 @@ package blockdag import ( + "io/ioutil" + "os" + "path/filepath" "reflect" "testing" + "github.com/kaspanet/kaspad/infrastructure/config" + "github.com/kaspanet/kaspad/infrastructure/db/dbaccess" "github.com/kaspanet/kaspad/util/subnetworkid" "github.com/kaspanet/kaspad/app/appmessage" "github.com/kaspanet/kaspad/util/daghash" ) +func prepareDatabaseForTest(t *testing.T, testName string) (*dbaccess.DatabaseContext, func()) { + var err error + tmpDir, err := ioutil.TempDir("", "utxoset_test") + if err != nil { + t.Fatalf("error creating temp dir: %s", err) + return nil, nil + } + + dbPath := filepath.Join(tmpDir, testName) + _ = os.RemoveAll(dbPath) + databaseContext, err := dbaccess.New(dbPath) + if err != nil { + t.Fatalf("error creating db: %s", err) + return nil, nil + } + + // Setup a teardown function for cleaning up. This function is + // returned to the caller to be invoked when it is done testing. + teardown := func() { + databaseContext.Close() + os.RemoveAll(dbPath) + } + + return databaseContext, teardown + +} + // TestUTXOCollection makes sure that utxoCollection cloning and string representations work as expected. func TestUTXOCollection(t *testing.T) { txID0, _ := daghash.NewTxIDFromStr("0000000000000000000000000000000000000000000000000000000000000000") @@ -619,7 +651,7 @@ func (d *UTXODiff) equal(other *UTXODiff) bool { } func (fus *FullUTXOSet) equal(other *FullUTXOSet) bool { - return reflect.DeepEqual(fus.utxoCollection, other.utxoCollection) + return reflect.DeepEqual(fus.utxoCache, other.utxoCache) } func (dus *DiffUTXOSet) equal(other *DiffUTXOSet) bool { @@ -642,7 +674,10 @@ func TestFullUTXOSet(t *testing.T) { } // Test fullUTXOSet creation - emptySet := NewFullUTXOSet() + fullUTXOCacheSize := config.DefaultConfig().MaxUTXOCacheSize + db, teardown := prepareDatabaseForTest(t, "TestDiffUTXOSet") + defer teardown() + emptySet := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) if len(emptySet.collection()) != 0 { t.Errorf("new set is not empty") } @@ -668,7 +703,8 @@ func TestFullUTXOSet(t *testing.T) { } else if isAccepted { t.Errorf("addTx unexpectedly succeeded") } - emptySet = &FullUTXOSet{utxoCollection: utxoCollection{outpoint0: utxoEntry0}} + emptySet = NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + emptySet.add(outpoint0, utxoEntry0) if isAccepted, err := emptySet.AddTx(transaction0, 0); err != nil { t.Errorf("addTx unexpectedly failed. Error: %s", err) } else if !isAccepted { @@ -676,7 +712,7 @@ func TestFullUTXOSet(t *testing.T) { } // Test fullUTXOSet collection - if !reflect.DeepEqual(emptySet.collection(), emptySet.utxoCollection) { + if !reflect.DeepEqual(emptySet.collection(), emptySet.utxoCache) { t.Errorf("collection does not equal the set's utxoCollection") } @@ -704,9 +740,12 @@ func TestDiffUTXOSet(t *testing.T) { toAdd: utxoCollection{outpoint0: utxoEntry0}, toRemove: utxoCollection{outpoint1: utxoEntry1}, } + fullUTXOCacheSize := config.DefaultConfig().MaxUTXOCacheSize + db, teardown := prepareDatabaseForTest(t, "TestDiffUTXOSet") + defer teardown() // Test diffUTXOSet creation - emptySet := NewDiffUTXOSet(NewFullUTXOSet(), NewUTXODiff()) + emptySet := NewDiffUTXOSet(NewFullUTXOSetFromContext(db, fullUTXOCacheSize), NewUTXODiff()) if collection, err := emptySet.collection(); err != nil { t.Errorf("Error getting emptySet collection: %s", err) } else if len(collection) != 0 { @@ -726,7 +765,7 @@ func TestDiffUTXOSet(t *testing.T) { if !reflect.DeepEqual(withDiffUTXOSet.base, emptySet.base) || !reflect.DeepEqual(withDiffUTXOSet.UTXODiff, withDiff) { t.Errorf("WithDiff is of unexpected composition") } - _, err = NewDiffUTXOSet(NewFullUTXOSet(), diff).WithDiff(diff) + _, err = NewDiffUTXOSet(NewFullUTXOSetFromContext(db, fullUTXOCacheSize), diff).WithDiff(diff) if err == nil { t.Errorf("WithDiff unexpectedly succeeded") } @@ -748,14 +787,14 @@ func TestDiffUTXOSet(t *testing.T) { { name: "empty base, empty diff", diffSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, }, }, expectedMeldSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -767,14 +806,18 @@ func TestDiffUTXOSet(t *testing.T) { { name: "empty base, one member in diff toAdd", diffSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint0: utxoEntry0}, toRemove: utxoCollection{}, }, }, expectedMeldSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint0: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint0, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -786,7 +829,7 @@ func TestDiffUTXOSet(t *testing.T) { { name: "empty base, one member in diff toRemove", diffSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{outpoint0: utxoEntry0}, @@ -800,19 +843,23 @@ func TestDiffUTXOSet(t *testing.T) { { name: "one member in base toAdd, one member in diff toAdd", diffSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint0: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint0, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint1: utxoEntry1}, toRemove: utxoCollection{}, }, }, expectedMeldSet: &DiffUTXOSet{ - base: &FullUTXOSet{ - utxoCollection: utxoCollection{ - outpoint0: utxoEntry0, - outpoint1: utxoEntry1, - }, - }, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint0, utxoEntry0) + futxo.add(outpoint1, utxoEntry1) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -827,16 +874,18 @@ func TestDiffUTXOSet(t *testing.T) { { name: "one member in base toAdd, same one member in diff toRemove", diffSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint0: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint0, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{outpoint0: utxoEntry0}, }, }, expectedMeldSet: &DiffUTXOSet{ - base: &FullUTXOSet{ - utxoCollection: utxoCollection{}, - }, + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -949,6 +998,9 @@ func TestDiffUTXOSet_addTx(t *testing.T) { txOut0 := &appmessage.TxOut{ScriptPubKey: []byte{0}, Value: 10} utxoEntry0 := NewUTXOEntry(txOut0, true, 0) coinbaseTX := appmessage.NewSubnetworkMsgTx(1, []*appmessage.TxIn{}, []*appmessage.TxOut{txOut0}, subnetworkid.SubnetworkIDCoinbase, 0, nil) + fullUTXOCacheSize := config.DefaultConfig().MaxUTXOCacheSize + db, teardown := prepareDatabaseForTest(t, "TestDiffUTXOSet") + defer teardown() // transaction1 spends coinbaseTX id1 := coinbaseTX.TxID() @@ -982,11 +1034,11 @@ func TestDiffUTXOSet_addTx(t *testing.T) { }{ { name: "add coinbase transaction to empty set", - startSet: NewDiffUTXOSet(NewFullUTXOSet(), NewUTXODiff()), + startSet: NewDiffUTXOSet(NewFullUTXOSetFromContext(db, fullUTXOCacheSize), NewUTXODiff()), startHeight: 0, toAdd: []*appmessage.MsgTx{coinbaseTX}, expectedSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{}}, + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint1: utxoEntry0}, toRemove: utxoCollection{}, @@ -995,11 +1047,11 @@ func TestDiffUTXOSet_addTx(t *testing.T) { }, { name: "add regular transaction to empty set", - startSet: NewDiffUTXOSet(NewFullUTXOSet(), NewUTXODiff()), + startSet: NewDiffUTXOSet(NewFullUTXOSetFromContext(db, fullUTXOCacheSize), NewUTXODiff()), startHeight: 0, toAdd: []*appmessage.MsgTx{transaction1}, expectedSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{}}, + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -1009,7 +1061,11 @@ func TestDiffUTXOSet_addTx(t *testing.T) { { name: "add transaction to set with its input in base", startSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint1: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint1, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -1018,7 +1074,11 @@ func TestDiffUTXOSet_addTx(t *testing.T) { startHeight: 1, toAdd: []*appmessage.MsgTx{transaction1}, expectedSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint1: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint1, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint2: utxoEntry1}, toRemove: utxoCollection{outpoint1: utxoEntry0}, @@ -1028,7 +1088,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) { { name: "add transaction to set with its input in diff toAdd", startSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint1: utxoEntry0}, toRemove: utxoCollection{}, @@ -1037,7 +1097,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) { startHeight: 1, toAdd: []*appmessage.MsgTx{transaction1}, expectedSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint2: utxoEntry1}, toRemove: utxoCollection{}, @@ -1047,7 +1107,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) { { name: "add transaction to set with its input in diff toAdd and its output in diff toRemove", startSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint1: utxoEntry0}, toRemove: utxoCollection{outpoint2: utxoEntry1}, @@ -1056,7 +1116,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) { startHeight: 1, toAdd: []*appmessage.MsgTx{transaction1}, expectedSet: &DiffUTXOSet{ - base: NewFullUTXOSet(), + base: NewFullUTXOSetFromContext(db, fullUTXOCacheSize), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -1066,7 +1126,11 @@ func TestDiffUTXOSet_addTx(t *testing.T) { { name: "add two transactions, one spending the other, to set with the first input in base", startSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint1: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint1, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, @@ -1075,7 +1139,11 @@ func TestDiffUTXOSet_addTx(t *testing.T) { startHeight: 1, toAdd: []*appmessage.MsgTx{transaction1, transaction2}, expectedSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{outpoint1: utxoEntry0}}, + base: func() *FullUTXOSet { + futxo := NewFullUTXOSetFromContext(db, fullUTXOCacheSize) + futxo.add(outpoint1, utxoEntry0) + return futxo + }(), UTXODiff: &UTXODiff{ toAdd: utxoCollection{outpoint3: utxoEntry2}, toRemove: utxoCollection{outpoint1: utxoEntry0}, @@ -1108,7 +1176,7 @@ testLoop: // collection returns a collection of all UTXOs in this set func (fus *FullUTXOSet) collection() utxoCollection { - return fus.utxoCollection.clone() + return fus.utxoCache.clone() } // collection returns a collection of all UTXOs in this set diff --git a/domain/blockdag/virtualblock.go b/domain/blockdag/virtualblock.go index 73b8df140..bc978fb0f 100644 --- a/domain/blockdag/virtualblock.go +++ b/domain/blockdag/virtualblock.go @@ -32,7 +32,7 @@ func newVirtualBlock(dag *BlockDAG, parents blockSet) *virtualBlock { // The mutex is intentionally not held since this is a constructor. var virtual virtualBlock virtual.dag = dag - virtual.utxoSet = NewFullUTXOSet() + virtual.utxoSet = NewFullUTXOSetFromContext(dag.databaseContext, dag.maxUTXOCacheSize) virtual.selectedParentChainSet = newBlockSet() virtual.selectedParentChainSlice = nil virtual.blockNode, _ = dag.newBlockNode(nil, parents) diff --git a/infrastructure/config/config.go b/infrastructure/config/config.go index 352e7fe79..a9715b941 100644 --- a/infrastructure/config/config.go +++ b/infrastructure/config/config.go @@ -51,10 +51,11 @@ const ( defaultMinRelayTxFee = 1e-5 // 1 sompi per byte defaultMaxOrphanTransactions = 100 //DefaultMaxOrphanTxSize is the default maximum size for an orphan transaction - DefaultMaxOrphanTxSize = 100000 - defaultSigCacheMaxSize = 100000 - sampleConfigFilename = "sample-kaspad.conf" - defaultAcceptanceIndex = false + DefaultMaxOrphanTxSize = 100000 + defaultSigCacheMaxSize = 100000 + sampleConfigFilename = "sample-kaspad.conf" + defaultAcceptanceIndex = false + defaultMaxUTXOCacheSize = 5000000000 ) var ( @@ -121,6 +122,7 @@ type Flags struct { RelayNonStd bool `long:"relaynonstd" description:"Relay non-standard transactions regardless of the default settings for the active network."` RejectNonStd bool `long:"rejectnonstd" description:"Reject non-standard transactions regardless of the default settings for the active network."` ResetDatabase bool `long:"reset-db" description:"Reset database before starting node. It's needed when switching between subnetworks."` + MaxUTXOCacheSize uint64 `long:"maxutxocachesize" description:"Max size of loaded UTXO into ram from the disk in bytes"` NetworkFlags } @@ -186,6 +188,7 @@ func defaultFlags() *Flags { SigCacheMaxSize: defaultSigCacheMaxSize, MinRelayTxFee: defaultMinRelayTxFee, AcceptanceIndex: defaultAcceptanceIndex, + MaxUTXOCacheSize: defaultMaxUTXOCacheSize, } } diff --git a/infrastructure/db/dbaccess/utxo.go b/infrastructure/db/dbaccess/utxo.go index 3b945c278..d56338ade 100644 --- a/infrastructure/db/dbaccess/utxo.go +++ b/infrastructure/db/dbaccess/utxo.go @@ -36,6 +36,18 @@ func RemoveFromUTXOSet(context Context, outpointKey []byte) error { return accessor.Delete(key) } +// GetFromUTXOSet return the given outpoint from the +// database's UTXO set. +func GetFromUTXOSet(context Context, outpointKey []byte) ([]byte, error) { + accessor, err := context.accessor() + if err != nil { + return nil, err + } + + key := utxoKey(outpointKey) + return accessor.Get(key) +} + // UTXOSetCursor opens a cursor over all the UTXO entries // that have been previously added to the database. func UTXOSetCursor(context Context) (database.Cursor, error) {