From 3839767aede517622e15b4468189f84f3701cd01 Mon Sep 17 00:00:00 2001 From: oudeis Date: Tue, 1 Sep 2020 09:30:08 +0300 Subject: [PATCH] [NOD-815] Refactor all UTXO-diff algebra methods (#857) * UTXOSet: Refactoring. * Use set operations for utxo collection rules. * diffFrom Refactoring. * withDiffInPlace Refactoring. * [NOD-815] Refactor all UTXO-diff algebra methods * Use set operations for utxo collection rules. * diffFrom Refactoring. * withDiffInPlace Refactoring. * Stylistic fixes. * UTXOSet: Refactoring. Add benchmarks. Optimizations. * Add UTXOSet diffFrom, withDiff benchmarks. * Add performance optimizations. * Add both in-place and value-return methods for set operations. * Remove redundant blue score condition checking in withDiffInPlace second error rule. * Improve naming. * PR fixes. * After-merge build fixes. * Typo fixes. * Stylistic fixes. * After-merge build fixes. * Typo fixes. Co-authored-by: Septen --- domain/blockdag/bench_test.go | 137 ++++++++++++++++ domain/blockdag/utxoset.go | 288 +++++++++++++++++++++------------- 2 files changed, 320 insertions(+), 105 deletions(-) create mode 100644 domain/blockdag/bench_test.go diff --git a/domain/blockdag/bench_test.go b/domain/blockdag/bench_test.go new file mode 100644 index 000000000..7e002f7ff --- /dev/null +++ b/domain/blockdag/bench_test.go @@ -0,0 +1,137 @@ +package blockdag + +import ( + "strconv" + "testing" + + "github.com/kaspanet/kaspad/app/appmessage" + "github.com/kaspanet/kaspad/util/daghash" +) + +func generateNewUTXOEntry(index uint64) (appmessage.Outpoint, *UTXOEntry) { + txSuffix := strconv.FormatUint(index, 10) + txStr := "0000000000000000000000000000000000000000000000000000000000000000" + txID, _ := daghash.NewTxIDFromStr(txStr[0:len(txStr)-len(txSuffix)] + txSuffix) + outpoint := *appmessage.NewOutpoint(txID, 0) + utxoEntry := NewUTXOEntry(&appmessage.TxOut{ScriptPubKey: []byte{}, Value: index}, true, index) + + return outpoint, utxoEntry +} + +func generateUtxoCollection(startIndex uint64, numItems uint64) utxoCollection { + uc := make(utxoCollection) + + for i := uint64(0); i < numItems; i++ { + outpoint, utxoEntry := generateNewUTXOEntry(startIndex + i) + uc.add(outpoint, utxoEntry) + } + + return uc +} + +// BenchmarkDiffFrom performs a benchmark on how long it takes to calculate +// the difference between this utxoDiff and another one +func BenchmarkDiffFrom(b *testing.B) { + var numOfEntries uint64 = 100 + var startIndex uint64 = 0 + uc1 := generateUtxoCollection(startIndex, numOfEntries) + startIndex = startIndex + numOfEntries + uc2 := generateUtxoCollection(startIndex, numOfEntries) + startIndex = startIndex + numOfEntries + uc3 := generateUtxoCollection(startIndex, numOfEntries) + startIndex = startIndex + numOfEntries + uc4 := generateUtxoCollection(startIndex, numOfEntries) + + tests := []struct { + this *UTXODiff + other *UTXODiff + }{ + { + this: &UTXODiff{ + toAdd: uc1, + toRemove: uc2, + }, + other: &UTXODiff{ + toAdd: uc3, + toRemove: uc4, + }, + }, + { + this: &UTXODiff{ + toAdd: uc1, + toRemove: uc2, + }, + other: &UTXODiff{ + toAdd: uc3, + toRemove: uc1, + }, + }, + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, test := range tests { + test.this.diffFrom(test.other) + } + + } +} + +// BenchmarkWithDiff performs a benchmark on how long it takes to apply provided diff to this diff +func BenchmarkWithDiff(b *testing.B) { + var numOfEntries uint64 = 100 + var startIndex uint64 = 0 + uc1 := generateUtxoCollection(startIndex, numOfEntries) + startIndex = startIndex + numOfEntries + uc2 := generateUtxoCollection(startIndex, numOfEntries) + startIndex = startIndex + numOfEntries + uc3 := generateUtxoCollection(startIndex, numOfEntries) + startIndex = startIndex + numOfEntries + uc4 := generateUtxoCollection(startIndex, numOfEntries) + + tests := []struct { + this *UTXODiff + other *UTXODiff + }{ + { + this: &UTXODiff{ + toAdd: uc1, + toRemove: uc2, + }, + other: &UTXODiff{ + toAdd: uc3, + toRemove: uc4, + }, + }, + { + this: &UTXODiff{ + toAdd: uc1, + toRemove: uc2, + }, + other: &UTXODiff{ + toAdd: uc3, + toRemove: uc2, + }, + }, + { + this: &UTXODiff{ + toAdd: uc1, + toRemove: uc2, + }, + other: &UTXODiff{ + toAdd: uc1, + toRemove: uc3, + }, + }, + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for _, test := range tests { + test.this.WithDiff(test.other) + } + + } +} diff --git a/domain/blockdag/utxoset.go b/domain/blockdag/utxoset.go index 2dae4cff0..ec01d24a2 100644 --- a/domain/blockdag/utxoset.go +++ b/domain/blockdag/utxoset.go @@ -116,11 +116,25 @@ func (uc utxoCollection) add(outpoint appmessage.Outpoint, entry *UTXOEntry) { uc[outpoint] = entry } +// addMultiple adds multiple UTXO entries to this collection +func (uc utxoCollection) addMultiple(collectionToAdd utxoCollection) { + for outpoint, entry := range collectionToAdd { + uc[outpoint] = entry + } +} + // remove removes a UTXO entry from this collection if it exists func (uc utxoCollection) remove(outpoint appmessage.Outpoint) { delete(uc, outpoint) } +// removeMultiple removes multiple UTXO entries from this collection if it exists +func (uc utxoCollection) removeMultiple(collectionToRemove utxoCollection) { + for outpoint := range collectionToRemove { + delete(uc, outpoint) + } +} + // get returns the UTXOEntry represented by provided outpoint, // and a boolean value indicating if said UTXOEntry is in the set or not func (uc utxoCollection) get(outpoint appmessage.Outpoint) (*UTXOEntry, bool) { @@ -166,6 +180,100 @@ func NewUTXODiff() *UTXODiff { } } +// checkIntersection checks if there is an intersection between two utxoCollections +func checkIntersection(collection1 utxoCollection, collection2 utxoCollection) bool { + for outpoint := range collection1 { + if collection2.contains(outpoint) { + return true + } + } + + return false +} + +// checkIntersectionWithRule checks if there is an intersection between two utxoCollections satisfying arbitrary rule +func checkIntersectionWithRule(collection1 utxoCollection, collection2 utxoCollection, extraRule func(appmessage.Outpoint, *UTXOEntry, *UTXOEntry) bool) bool { + for outpoint, utxoEntry := range collection1 { + if diffEntry, ok := collection2.get(outpoint); ok { + if extraRule(outpoint, utxoEntry, diffEntry) { + return true + } + } + } + + return false +} + +// minInt returns the smaller of x or y integer values +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +// intersectionWithRemainderHavingBlueScore calculates an intersection between two utxoCollections +// having same blue score, returns the result and the remainder from collection1 +func intersectionWithRemainderHavingBlueScore(collection1, collection2 utxoCollection) (result, remainder utxoCollection) { + result = make(utxoCollection, minInt(len(collection1), len(collection2))) + remainder = make(utxoCollection, len(collection1)) + intersectionWithRemainderHavingBlueScoreInPlace(collection1, collection2, result, remainder) + return +} + +// intersectionWithRemainderHavingBlueScoreInPlace calculates an intersection between two utxoCollections +// having same blue score, puts it into result and into remainder from collection1 +func intersectionWithRemainderHavingBlueScoreInPlace(collection1, collection2, result, remainder utxoCollection) { + for outpoint, utxoEntry := range collection1 { + if collection2.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { + result.add(outpoint, utxoEntry) + } else { + remainder.add(outpoint, utxoEntry) + } + } +} + +// subtractionHavingBlueScore calculates a subtraction between collection1 and collection2 +// having same blue score, returns the result +func subtractionHavingBlueScore(collection1, collection2 utxoCollection) (result utxoCollection) { + result = make(utxoCollection, len(collection1)) + + subtractionHavingBlueScoreInPlace(collection1, collection2, result) + return +} + +// subtractionHavingBlueScoreInPlace calculates a subtraction between collection1 and collection2 +// having same blue score, puts it into result +func subtractionHavingBlueScoreInPlace(collection1, collection2, result utxoCollection) { + for outpoint, utxoEntry := range collection1 { + if !collection2.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { + result.add(outpoint, utxoEntry) + } + } +} + +// subtractionWithRemainderHavingBlueScore calculates a subtraction between collection1 and collection2 +// having same blue score, returns the result and the remainder from collection1 +func subtractionWithRemainderHavingBlueScore(collection1, collection2 utxoCollection) (result, remainder utxoCollection) { + result = make(utxoCollection, len(collection1)) + remainder = make(utxoCollection, len(collection1)) + + subtractionWithRemainderHavingBlueScoreInPlace(collection1, collection2, result, remainder) + return +} + +// subtractionWithRemainderHavingBlueScoreInPlace calculates a subtraction between collection1 and collection2 +// having same blue score, puts it into result and into remainder from collection1 +func subtractionWithRemainderHavingBlueScoreInPlace(collection1, collection2, result, remainder utxoCollection) { + for outpoint, utxoEntry := range collection1 { + if !collection2.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { + result.add(outpoint, utxoEntry) + } else { + remainder.add(outpoint, utxoEntry) + } + } +} + // diffFrom returns a new utxoDiff with the difference between this utxoDiff and another // Assumes that: // Both utxoDiffs are from the same base @@ -195,89 +303,69 @@ func NewUTXODiff() *UTXODiff { // 2. This diff contains a UTXO in toRemove, and the other diff does not contain it // diffFrom results in the UTXO being added to toAdd func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) { - result := UTXODiff{ - toAdd: make(utxoCollection, len(d.toRemove)+len(other.toAdd)), - toRemove: make(utxoCollection, len(d.toAdd)+len(other.toRemove)), - } - // Note that the following cases are not accounted for, as they are impossible // as long as the base utxoSet is the same: // - if utxoEntry is in d.toAdd and other.toRemove // - if utxoEntry is in d.toRemove and other.toAdd + // check that NOT (entries with unequal blue scores AND utxoEntry is in d.toAdd and/or other.toRemove) -> Error + isNotAddedOutputRemovedWithBlueScore := func(outpoint appmessage.Outpoint, utxoEntry, diffEntry *UTXOEntry) bool { + return !(diffEntry.blockBlueScore != utxoEntry.blockBlueScore && + (d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) || + other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore))) + } + + if checkIntersectionWithRule(d.toRemove, other.toAdd, isNotAddedOutputRemovedWithBlueScore) { + return nil, errors.New("diffFrom: outpoint both in d.toAdd and in other.toRemove") + } + + //check that NOT (entries with unequal blue score AND utxoEntry is in d.toRemove and/or other.toAdd) -> Error + isNotRemovedOutputAddedWithBlueScore := func(outpoint appmessage.Outpoint, utxoEntry, diffEntry *UTXOEntry) bool { + return !(diffEntry.blockBlueScore != utxoEntry.blockBlueScore && + (d.toRemove.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) || + other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore))) + } + + if checkIntersectionWithRule(d.toAdd, other.toRemove, isNotRemovedOutputAddedWithBlueScore) { + return nil, errors.New("diffFrom: outpoint both in d.toRemove and in other.toAdd") + } + + // if have the same entry in d.toRemove and other.toRemove + // and existing entry is with different blue score, in this case - this is an error + if checkIntersectionWithRule(d.toRemove, other.toRemove, + func(outpoint appmessage.Outpoint, utxoEntry, diffEntry *UTXOEntry) bool { + return utxoEntry.blockBlueScore != diffEntry.blockBlueScore + }) { + return nil, errors.New("diffFrom: outpoint both in d.toRemove and other.toRemove with different " + + "blue scores, with no corresponding entry in d.toAdd") + } + + result := UTXODiff{ + toAdd: make(utxoCollection, len(d.toRemove)+len(other.toAdd)), + toRemove: make(utxoCollection, len(d.toAdd)+len(other.toRemove)), + } + // All transactions in d.toAdd: // If they are not in other.toAdd - should be added in result.toRemove + inBothToAdd := make(utxoCollection, len(d.toAdd)) + subtractionWithRemainderHavingBlueScoreInPlace(d.toAdd, other.toAdd, result.toRemove, inBothToAdd) // If they are in other.toRemove - base utxoSet is not the same - for outpoint, utxoEntry := range d.toAdd { - if !other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { - result.toRemove.add(outpoint, utxoEntry) - } else if (d.toRemove.contains(outpoint) && !other.toRemove.contains(outpoint)) || - (!d.toRemove.contains(outpoint) && other.toRemove.contains(outpoint)) { - return nil, errors.New( - "diffFrom: outpoint both in d.toAdd, other.toAdd, and only one of d.toRemove and other.toRemove") - } - if diffEntry, ok := other.toRemove.get(outpoint); ok { - // An exception is made for entries with unequal blue scores - // as long as the appropriate entry exists in either d.toRemove - // or other.toAdd. - // These are just "updates" to accepted blue score - if diffEntry.blockBlueScore != utxoEntry.blockBlueScore && - (d.toRemove.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) || - other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) { - continue - } - return nil, errors.Errorf("diffFrom: outpoint %s both in d.toAdd and in other.toRemove", outpoint) - } - } - - // All transactions in d.toRemove: - // If they are not in other.toRemove - should be added in result.toAdd - // If they are in other.toAdd - base utxoSet is not the same - for outpoint, utxoEntry := range d.toRemove { - diffEntry, ok := other.toRemove.get(outpoint) - if ok { - // if have the same entry in d.toRemove - simply don't copy. - // unless existing entry is with different blue score, in this case - this is an error - if utxoEntry.blockBlueScore != diffEntry.blockBlueScore { - return nil, errors.New("diffFrom: outpoint both in d.toRemove and other.toRemove with different " + - "blue scores, with no corresponding entry in d.toAdd") - } - } else { // if no existing entry - add to result.toAdd - result.toAdd.add(outpoint, utxoEntry) - } - - if !other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { - result.toAdd.add(outpoint, utxoEntry) - } - if diffEntry, ok := other.toAdd.get(outpoint); ok { - // An exception is made for entries with unequal blue scores - // as long as the appropriate entry exists in either d.toAdd - // or other.toRemove. - // These are just "updates" to accepted blue score - if diffEntry.blockBlueScore != utxoEntry.blockBlueScore && - (d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) || - other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) { - continue - } - return nil, errors.New("diffFrom: outpoint both in d.toRemove and in other.toAdd") - } - } - - // All transactions in other.toAdd: - // If they are not in d.toAdd - should be added in result.toAdd - for outpoint, utxoEntry := range other.toAdd { - if !d.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { - result.toAdd.add(outpoint, utxoEntry) - } + if checkIntersection(inBothToAdd, d.toRemove) != checkIntersection(inBothToAdd, other.toRemove) { + return nil, errors.New( + "diffFrom: outpoint both in d.toAdd, other.toAdd, and only one of d.toRemove and other.toRemove") } // All transactions in other.toRemove: // If they are not in d.toRemove - should be added in result.toRemove - for outpoint, utxoEntry := range other.toRemove { - if !d.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) { - result.toRemove.add(outpoint, utxoEntry) - } - } + subtractionHavingBlueScoreInPlace(other.toRemove, d.toRemove, result.toRemove) + + // All transactions in d.toRemove: + // If they are not in other.toRemove - should be added in result.toAdd + subtractionHavingBlueScoreInPlace(d.toRemove, other.toRemove, result.toAdd) + + // All transactions in other.toAdd: + // If they are not in d.toAdd - should be added in result.toAdd + subtractionHavingBlueScoreInPlace(other.toAdd, d.toAdd, result.toAdd) return &result, nil } @@ -285,45 +373,35 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) { // withDiffInPlace applies provided diff to this diff in-place, that would be the result if // first d, and than diff were applied to the same base func (d *UTXODiff) withDiffInPlace(diff *UTXODiff) error { - for outpoint, entryToRemove := range diff.toRemove { - if d.toAdd.containsWithBlueScore(outpoint, entryToRemove.blockBlueScore) { - // If already exists in toAdd with the same blueScore - remove from toAdd - d.toAdd.remove(outpoint) - continue - } - if d.toRemove.contains(outpoint) { - // If already exists - this is an error - return errors.Errorf( - "withDiffInPlace: outpoint %s both in d.toRemove and in diff.toRemove", outpoint) - } + if checkIntersectionWithRule(diff.toRemove, d.toRemove, + func(outpoint appmessage.Outpoint, entryToAdd, existingEntry *UTXOEntry) bool { + return !d.toAdd.containsWithBlueScore(outpoint, entryToAdd.blockBlueScore) - // If not exists neither in toAdd nor in toRemove - add to toRemove - d.toRemove.add(outpoint, entryToRemove) + }) { + return errors.New( + "withDiffInPlace: outpoint both in d.toRemove and in diff.toRemove") } - for outpoint, entryToAdd := range diff.toAdd { - if d.toRemove.containsWithBlueScore(outpoint, entryToAdd.blockBlueScore) { - // If already exists in toRemove with the same blueScore - remove from toRemove - if d.toAdd.contains(outpoint) && !diff.toRemove.contains(outpoint) { - return errors.Errorf( - "withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd with no "+ - "corresponding entry in diff.toRemove", outpoint) - } - d.toRemove.remove(outpoint) - continue - } - if existingEntry, ok := d.toAdd.get(outpoint); ok && - (existingEntry.blockBlueScore == entryToAdd.blockBlueScore || - !diff.toRemove.containsWithBlueScore(outpoint, existingEntry.blockBlueScore)) { - // If already exists - this is an error - return errors.Errorf( - "withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd", outpoint) - } - - // If not exists neither in toAdd nor in toRemove, or exists in toRemove with different blueScore - add to toAdd - d.toAdd.add(outpoint, entryToAdd) + if checkIntersectionWithRule(diff.toAdd, d.toAdd, + func(outpoint appmessage.Outpoint, entryToAdd, existingEntry *UTXOEntry) bool { + return !diff.toRemove.containsWithBlueScore(outpoint, existingEntry.blockBlueScore) + }) { + return errors.New( + "withDiffInPlace: outpoint both in d.toAdd and in diff.toAdd") } + intersection := make(utxoCollection, minInt(len(diff.toRemove), len(d.toAdd))) + // If not exists neither in toAdd nor in toRemove - add to toRemove + intersectionWithRemainderHavingBlueScoreInPlace(diff.toRemove, d.toAdd, intersection, d.toRemove) + // If already exists in toAdd with the same blueScore - remove from toAdd + d.toAdd.removeMultiple(intersection) + + intersection = make(utxoCollection, minInt(len(diff.toAdd), len(d.toRemove))) + // If not exists neither in toAdd nor in toRemove, or exists in toRemove with different blueScore - add to toAdd + intersectionWithRemainderHavingBlueScoreInPlace(diff.toAdd, d.toRemove, intersection, d.toAdd) + // If already exists in toRemove with the same blueScore - remove from toRemove + d.toRemove.removeMultiple(intersection) + return nil }