Use SerializeUTXODiff and DeserializeUTXODiff in utxoDiffStore and implement TestUTXODiffSerializationAndDeserialization.

domain/consensus/database/binaryserialization/utxo_collection.go

@@ -0,0 +1,52 @@
+package binaryserialization
+
+import (
+	"github.com/kaspanet/kaspad/domain/consensus/model"
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/utxo"
+	"github.com/kaspanet/kaspad/util/binaryserializer"
+	"io"
+)
+
+// SerializeUTXOCollection serializes the given utxoCollection into the given writer
+func SerializeUTXOCollection(writer io.Writer, utxoCollection model.UTXOCollection) error {
+	length := uint64(utxoCollection.Len())
+	err := binaryserializer.PutUint64(writer, length)
+	if err != nil {
+		return err
+	}
+
+	utxoIterator := utxoCollection.Iterator()
+	for ok := utxoIterator.First(); ok; ok = utxoIterator.Next() {
+		outpoint, utxoEntry, err := utxoIterator.Get()
+		if err != nil {
+			return err
+		}
+		err = utxo.SerializeUTXOIntoWriter(writer, utxoEntry, outpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// DeserializeUTXOCollection deserializes a utxoCollection out of the given reader
+func DeserializeUTXOCollection(reader io.Reader) (model.UTXOCollection, error) {
+	length, err := binaryserializer.Uint64(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	utxoMap := make(map[externalapi.DomainOutpoint]externalapi.UTXOEntry, length)
+	for i := uint64(0); i < length; i++ {
+		utxoEntry, outpoint, err := utxo.DeserializeUTXOOutOfReader(reader)
+		if err != nil {
+			return nil, err
+		}
+		utxoMap[*outpoint] = utxoEntry
+	}
+
+	utxoCollection := utxo.NewUTXOCollection(utxoMap)
+	return utxoCollection, nil
+}

domain/consensus/database/binaryserialization/utxo_diff.go

@@ -0,0 +1,29 @@
+package binaryserialization
+
+import (
+	"github.com/kaspanet/kaspad/domain/consensus/model"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/utxo"
+	"io"
+)
+
+// SerializeUTXODiff serializes the given utxoDiff into the given writer
+func SerializeUTXODiff(writer io.Writer, utxoDiff model.UTXODiff) error {
+	err := SerializeUTXOCollection(writer, utxoDiff.ToAdd())
+	if err != nil {
+		return err
+	}
+	return SerializeUTXOCollection(writer, utxoDiff.ToRemove())
+}
+
+// DeserializeUTXODiff deserializes a utxoDiff out of the given reader
+func DeserializeUTXODiff(reader io.Reader) (model.UTXODiff, error) {
+	toAdd, err := DeserializeUTXOCollection(reader)
+	if err != nil {
+		return nil, err
+	}
+	toRemove, err := DeserializeUTXOCollection(reader)
+	if err != nil {
+		return nil, err
+	}
+	return utxo.NewUTXODiffFromCollections(toAdd, toRemove)
+}

domain/consensus/datastructures/utxodiffstore/utxodiffstore.go

@@ -1,8 +1,10 @@
 package utxodiffstore
 
 import (
+	"bytes"
 	"github.com/golang/protobuf/proto"
 	"github.com/kaspanet/kaspad/domain/consensus/database"
+	"github.com/kaspanet/kaspad/domain/consensus/database/binaryserialization"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
@@ -188,27 +190,17 @@ func (uds *utxoDiffStore) utxoDiffChildHashAsKey(hash *externalapi.DomainHash) m
 }
 
 func (uds *utxoDiffStore) serializeUTXODiff(utxoDiff model.UTXODiff) ([]byte, error) {
-	dbUtxoDiff, err := serialization.UTXODiffToDBUTXODiff(utxoDiff)
+	writer := &bytes.Buffer{}
+	err := binaryserialization.SerializeUTXODiff(writer, utxoDiff)
 	if err != nil {
 		return nil, err
 	}
-
-	bytes, err := proto.Marshal(dbUtxoDiff)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	return bytes, nil
+	return writer.Bytes(), nil
 }
 
 func (uds *utxoDiffStore) deserializeUTXODiff(utxoDiffBytes []byte) (model.UTXODiff, error) {
-	dbUTXODiff := &serialization.DbUtxoDiff{}
-	err := proto.Unmarshal(utxoDiffBytes, dbUTXODiff)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	return serialization.DBUTXODiffToUTXODiff(dbUTXODiff)
+	reader := bytes.NewReader(utxoDiffBytes)
+	return binaryserialization.DeserializeUTXODiff(reader)
 }
 
 func (uds *utxoDiffStore) serializeUTXODiffChild(utxoDiffChild *externalapi.DomainHash) ([]byte, error) {

domain/consensus/datastructures/utxodiffstore/utxodiffstore_test.go

@@ -0,0 +1,168 @@
+package utxodiffstore
+
+import (
+	"github.com/kaspanet/kaspad/domain/consensus/model"
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/utxo"
+	"math/rand"
+	"testing"
+)
+
+func TestUTXODiffSerializationAndDeserialization(t *testing.T) {
+	utxoDiffStore := New(0).(*utxoDiffStore)
+
+	testUTXODiff, err := buildTestUTXODiff()
+	if err != nil {
+		t.Fatalf("Could not create UTXODiff from toAdd and toRemove collections: %s", err)
+	}
+
+	serializedUTXODiff, err := utxoDiffStore.serializeUTXODiff(testUTXODiff)
+	if err != nil {
+		t.Fatalf("Could not serialize UTXO diff: %s", err)
+	}
+	deserializedUTXODiff, err := utxoDiffStore.deserializeUTXODiff(serializedUTXODiff)
+	if err != nil {
+		t.Fatalf("Could not deserialize UTXO diff: %s", err)
+	}
+
+	if testUTXODiff.ToAdd().Len() != deserializedUTXODiff.ToAdd().Len() {
+		t.Fatalf("Unexpected toAdd length in deserialized utxoDiff. Want: %d, got: %d",
+			testUTXODiff.ToAdd().Len(), deserializedUTXODiff.ToAdd().Len())
+	}
+	if testUTXODiff.ToRemove().Len() != deserializedUTXODiff.ToRemove().Len() {
+		t.Fatalf("Unexpected toRemove length in deserialized utxoDiff. Want: %d, got: %d",
+			testUTXODiff.ToRemove().Len(), deserializedUTXODiff.ToRemove().Len())
+	}
+
+	testToAddIterator := testUTXODiff.ToAdd().Iterator()
+	for ok := testToAddIterator.First(); ok; ok = testToAddIterator.Next() {
+		testOutpoint, testUTXOEntry, err := testToAddIterator.Get()
+		if err != nil {
+			t.Fatalf("Could not get an outpoint-utxoEntry pair out of the toAdd iterator: %s", err)
+		}
+		deserializedUTXOEntry, ok := deserializedUTXODiff.ToAdd().Get(testOutpoint)
+		if !ok {
+			t.Fatalf("Outpoint %s:%d not found in the deserialized toAdd collection",
+				testOutpoint.TransactionID, testOutpoint.Index)
+		}
+		if !testUTXOEntry.Equal(deserializedUTXOEntry) {
+			t.Fatalf("Deserialized UTXO entry is not equal to the original UTXO entry for outpoint %s:%d "+
+				"in the toAdd collection", testOutpoint.TransactionID, testOutpoint.Index)
+		}
+	}
+
+	testToRemoveIterator := testUTXODiff.ToRemove().Iterator()
+	for ok := testToRemoveIterator.First(); ok; ok = testToRemoveIterator.Next() {
+		testOutpoint, testUTXOEntry, err := testToRemoveIterator.Get()
+		if err != nil {
+			t.Fatalf("Could not get an outpoint-utxoEntry pair out of the toRemove iterator: %s", err)
+		}
+		deserializedUTXOEntry, ok := deserializedUTXODiff.ToRemove().Get(testOutpoint)
+		if !ok {
+			t.Fatalf("Outpoint %s:%d not found in the deserialized toRemove collection",
+				testOutpoint.TransactionID, testOutpoint.Index)
+		}
+		if !testUTXOEntry.Equal(deserializedUTXOEntry) {
+			t.Fatalf("Deserialized UTXO entry is not equal to the original UTXO entry for outpoint %s:%d "+
+				"in the toRemove collection", testOutpoint.TransactionID, testOutpoint.Index)
+		}
+	}
+}
+
+func BenchmarkUTXODiffSerialization(b *testing.B) {
+	utxoDiffStore := New(0).(*utxoDiffStore)
+
+	testUTXODiff, err := buildTestUTXODiff()
+	if err != nil {
+		b.Fatalf("Could not create UTXODiff from toAdd and toRemove collections: %s", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := utxoDiffStore.serializeUTXODiff(testUTXODiff)
+		if err != nil {
+			b.Fatalf("Could not serialize UTXO diff: %s", err)
+		}
+	}
+}
+
+func BenchmarkUTXODiffDeserialization(b *testing.B) {
+	utxoDiffStore := New(0).(*utxoDiffStore)
+
+	testUTXODiff, err := buildTestUTXODiff()
+	if err != nil {
+		b.Fatalf("Could not create UTXODiff from toAdd and toRemove collections: %s", err)
+	}
+	serializedUTXODiff, err := utxoDiffStore.serializeUTXODiff(testUTXODiff)
+	if err != nil {
+		b.Fatalf("Could not serialize UTXO diff: %s", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err = utxoDiffStore.deserializeUTXODiff(serializedUTXODiff)
+		if err != nil {
+			b.Fatalf("Could not deserialize UTXO diff: %s", err)
+		}
+	}
+}
+
+func BenchmarkUTXODiffSerializationAndDeserialization(b *testing.B) {
+	utxoDiffStore := New(0).(*utxoDiffStore)
+
+	testUTXODiff, err := buildTestUTXODiff()
+	if err != nil {
+		b.Fatalf("Could not create UTXODiff from toAdd and toRemove collections: %s", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		serializedUTXODiff, err := utxoDiffStore.serializeUTXODiff(testUTXODiff)
+		if err != nil {
+			b.Fatalf("Could not serialize UTXO diff: %s", err)
+		}
+		_, err = utxoDiffStore.deserializeUTXODiff(serializedUTXODiff)
+		if err != nil {
+			b.Fatalf("Could not deserialize UTXO diff: %s", err)
+		}
+	}
+}
+
+func buildTestUTXODiff() (model.UTXODiff, error) {
+	toAdd := buildTestUTXOCollection()
+	toRemove := buildTestUTXOCollection()
+
+	utxoDiff, err := utxo.NewUTXODiffFromCollections(toAdd, toRemove)
+	if err != nil {
+		return nil, err
+	}
+	return utxoDiff, nil
+}
+
+func buildTestUTXOCollection() model.UTXOCollection {
+	utxoMap := make(map[externalapi.DomainOutpoint]externalapi.UTXOEntry)
+
+	for i := 0; i < 100_000; i++ {
+		var outpointTransactionIDBytes [32]byte
+		rand.Read(outpointTransactionIDBytes[:])
+		outpointTransactionID := externalapi.NewDomainTransactionIDFromByteArray(&outpointTransactionIDBytes)
+		outpointIndex := rand.Uint32()
+		outpoint := externalapi.NewDomainOutpoint(outpointTransactionID, outpointIndex)
+
+		utxoEntryAmount := rand.Uint64()
+		var utxoEntryScriptPublicKeyScript [256]byte
+		rand.Read(utxoEntryScriptPublicKeyScript[:])
+		utxoEntryScriptPublicKeyVersion := uint16(rand.Uint32())
+		utxoEntryScriptPublicKey := &externalapi.ScriptPublicKey{
+			Script:  utxoEntryScriptPublicKeyScript[:],
+			Version: utxoEntryScriptPublicKeyVersion,
+		}
+		utxoEntryIsCoinbase := rand.Float32() > 0.5
+		utxoEntryBlockBlueScore := rand.Uint64()
+		utxoEntry := utxo.NewUTXOEntry(utxoEntryAmount, utxoEntryScriptPublicKey, utxoEntryIsCoinbase, utxoEntryBlockBlueScore)
+
+		utxoMap[*outpoint] = utxoEntry
+	}
+
+	return utxo.NewUTXOCollection(utxoMap)
+}

domain/consensus/utils/utxo/serialization.go

@@ -14,12 +14,7 @@ import (
 func SerializeUTXO(entry externalapi.UTXOEntry, outpoint *externalapi.DomainOutpoint) ([]byte, error) {
 	w := &bytes.Buffer{}
 
-	err := serializeOutpoint(w, outpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	err = serializeUTXOEntry(w, entry)
+	err := SerializeUTXOIntoWriter(w, entry, outpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -27,15 +22,30 @@ func SerializeUTXO(entry externalapi.UTXOEntry, outpoint *externalapi.DomainOutp
 	return w.Bytes(), nil
 }
 
+// SerializeUTXOIntoWriter serializes the byte-slice representation for given UTXOEntry-outpoint pair into the given writer
+func SerializeUTXOIntoWriter(writer io.Writer, entry externalapi.UTXOEntry, outpoint *externalapi.DomainOutpoint) error {
+	err := serializeOutpoint(writer, outpoint)
+	if err != nil {
+		return err
+	}
+
+	return serializeUTXOEntry(writer, entry)
+}
+
 // DeserializeUTXO deserializes the given byte slice to UTXOEntry-outpoint pair
 func DeserializeUTXO(utxoBytes []byte) (entry externalapi.UTXOEntry, outpoint *externalapi.DomainOutpoint, err error) {
 	r := bytes.NewReader(utxoBytes)
-	outpoint, err = deserializeOutpoint(r)
+	return DeserializeUTXOOutOfReader(r)
+}
+
+// DeserializeUTXOOutOfReader deserializes a UTXOEntry-outpoint pair out of the given reader
+func DeserializeUTXOOutOfReader(reader io.Reader) (entry externalapi.UTXOEntry, outpoint *externalapi.DomainOutpoint, err error) {
+	outpoint, err = deserializeOutpoint(reader)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	entry, err = deserializeUTXOEntry(r)
+	entry, err = deserializeUTXOEntry(reader)
 	if err != nil {
 		return nil, nil, err
 	}