From aa51b5f07196736cf65776d95dbf99a30ce88bf1 Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Thu, 23 May 2019 15:11:42 +0300 Subject: [PATCH] [NOD-179] Added ECMH-Multiset to all UTXO structs (#304) * [NOD-172] Port EMCH from bchd * [NOD-172] Fix hdkeychain.TestErrors and add btcec.TestRecoverCompact * [NOD-172] Make ECMH immutable * [NOD-172] Fix gofmt errors * [NOD-172] Add TestMultiset_NewMultisetFromDataSlice and fix Point to be immutable * [NOD-172] Fix gofmt errors * [NOD-172] Add test for checking that the Union of a multiset and its inverse is zero * [NOD-179] Add ECMH Point to all UTXO-structs * [NOD-179] Fix utxo set tests * [NOD-179] Fix mempool tests * [NOD-179] Remove RemoveTxOuts * [NOD-179] Move serializeBlockUTXODiffData to the top of the file * [NOD-179] Fix serializeBlockUTXODiffData comment format * [NOD-179] Fix AddTx comment and name return values --- blockdag/dag.go | 14 +- blockdag/dag_test.go | 13 +- blockdag/dagio.go | 94 +-------- blockdag/dagio_test.go | 7 +- blockdag/external_dag_test.go | 42 +++- blockdag/indexers/txindex_test.go | 14 +- blockdag/test_utils.go | 5 +- blockdag/utxodiffstore.go | 159 +-------------- blockdag/utxoio.go | 308 ++++++++++++++++++++++++++++ blockdag/utxoset.go | 204 +++++++++++++++---- blockdag/utxoset_test.go | 325 ++++++++++++++++++------------ btcec/ecmh.go | 15 +- mempool/mempool.go | 50 ++++- mempool/mempool_test.go | 238 ++++++++++++---------- mining/mining.go | 7 +- mining/mining_test.go | 20 +- mining/policy_test.go | 6 +- txscript/standard.go | 24 +++ util/testtools/testtools.go | 13 +- 19 files changed, 983 insertions(+), 575 deletions(-) create mode 100644 blockdag/utxoio.go diff --git a/blockdag/dag.go b/blockdag/dag.go index f7294cf44..e96734bc7 100644 --- a/blockdag/dag.go +++ b/blockdag/dag.go @@ -823,7 +823,10 @@ func (dag *BlockDAG) applyDAGChanges(node *blockNode, block *util.Block, newBloc // It is now safe to meld the UTXO set to base. diffSet := newVirtualUTXO.(*DiffUTXOSet) virtualUTXODiff = diffSet.UTXODiff - dag.meldVirtualUTXO(diffSet) + err = dag.meldVirtualUTXO(diffSet) + if err != nil { + return nil, fmt.Errorf("failed melding the virtual UTXO: %s", err) + } dag.index.SetStatusFlags(node, statusValid) @@ -833,10 +836,10 @@ func (dag *BlockDAG) applyDAGChanges(node *blockNode, block *util.Block, newBloc return virtualUTXODiff, nil } -func (dag *BlockDAG) meldVirtualUTXO(newVirtualUTXODiffSet *DiffUTXOSet) { +func (dag *BlockDAG) meldVirtualUTXO(newVirtualUTXODiffSet *DiffUTXOSet) error { dag.utxoLock.Lock() defer dag.utxoLock.Unlock() - newVirtualUTXODiffSet.meldToBase() + return newVirtualUTXODiffSet.meldToBase() } func (node *blockNode) diffFromTxs(pastUTXO UTXOSet, transactions []*util.Tx) (*UTXODiff, error) { @@ -966,7 +969,10 @@ func (node *blockNode) applyBlueBlocks(selectedParentUTXO UTXOSet, blueBlocks [] if isSelectedParent { isAccepted = true } else { - isAccepted = pastUTXO.AddTx(tx.MsgTx(), node.height) + isAccepted, err = pastUTXO.AddTx(tx.MsgTx(), node.height) + if err != nil { + return nil, nil, err + } } blockTxsAcceptanceData[i] = TxAcceptanceData{Tx: tx, IsAccepted: isAccepted} } diff --git a/blockdag/dag_test.go b/blockdag/dag_test.go index c17b3625f..b1eed3e4c 100644 --- a/blockdag/dag_test.go +++ b/blockdag/dag_test.go @@ -244,7 +244,11 @@ func TestCalcSequenceLock(t *testing.T) { msgTx := wire.NewNativeMsgTx(wire.TxVersion, nil, []*wire.TxOut{{PkScript: nil, Value: 10}}) targetTx := util.NewTx(msgTx) utxoSet := NewFullUTXOSet() - utxoSet.AddTx(targetTx.MsgTx(), uint64(numBlocksToGenerate)-4) + if isAccepted, err := utxoSet.AddTx(targetTx.MsgTx(), uint64(numBlocksToGenerate)-4); err != nil { + t.Fatalf("AddTx unexpectedly failed. Error: %s", err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", targetTx.ID()) + } // Create a utxo that spends the fake utxo created above for use in the // transactions created in the tests. It has an age of 4 blocks. Note @@ -276,8 +280,11 @@ func TestCalcSequenceLock(t *testing.T) { TxID: *unConfTx.TxID(), Index: 0, } - - utxoSet.AddTx(unConfTx, UnminedChainHeight) + if isAccepted, err := utxoSet.AddTx(unConfTx, UnminedChainHeight); err != nil { + t.Fatalf("AddTx unexpectedly failed. Error: %s", err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", unConfTx.TxID()) + } tests := []struct { name string diff --git a/blockdag/dagio.go b/blockdag/dagio.go index edf78eca9..aaef9420c 100644 --- a/blockdag/dagio.go +++ b/blockdag/dagio.go @@ -221,93 +221,6 @@ func recycleOutpointKey(key *[]byte) { outpointKeyPool.Put(key) } -// utxoEntryHeaderCode returns the calculated header code to be used when -// serializing the provided utxo entry. -func utxoEntryHeaderCode(entry *UTXOEntry) uint64 { - - // As described in the serialization format comments, the header code - // encodes the height shifted over one bit and the block reward flag in the - // lowest bit. - headerCode := uint64(entry.BlockChainHeight()) << 1 - if entry.IsBlockReward() { - headerCode |= 0x01 - } - - return headerCode -} - -// serializeUTXOEntry returns the entry serialized to a format that is suitable -// for long-term storage. The format is described in detail above. -func serializeUTXOEntry(entry *UTXOEntry) ([]byte, error) { - - // Encode the header code. - headerCode := utxoEntryHeaderCode(entry) - - // Calculate the size needed to serialize the entry. - size := serializeSizeVLQ(headerCode) + - compressedTxOutSize(uint64(entry.Amount()), entry.PkScript()) - - // Serialize the header code followed by the compressed unspent - // transaction output. - serialized := make([]byte, size) - offset := putVLQ(serialized, headerCode) - offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()), - entry.PkScript()) - - return serialized, nil -} - -// deserializeOutPoint decodes an outPoint from the passed serialized byte -// slice into a new wire.OutPoint using a format that is suitable for long- -// term storage. this format is described in detail above. -func deserializeOutPoint(serialized []byte) (*wire.OutPoint, error) { - if len(serialized) <= daghash.HashSize { - return nil, errDeserialize("unexpected end of data") - } - - txID := daghash.TxID{} - txID.SetBytes(serialized[:daghash.HashSize]) - index, _ := deserializeVLQ(serialized[daghash.HashSize:]) - return wire.NewOutPoint(&txID, uint32(index)), nil -} - -// deserializeUTXOEntry decodes a UTXO entry from the passed serialized byte -// slice into a new UTXOEntry using a format that is suitable for long-term -// storage. The format is described in detail above. -func deserializeUTXOEntry(serialized []byte) (*UTXOEntry, error) { - // Deserialize the header code. - code, offset := deserializeVLQ(serialized) - if offset >= len(serialized) { - return nil, errDeserialize("unexpected end of data after header") - } - - // Decode the header code. - // - // Bit 0 indicates whether the containing transaction is a block reward. - // Bits 1-x encode height of containing transaction. - isBlockReward := code&0x01 != 0 - blockChainHeight := code >> 1 - - // Decode the compressed unspent transaction output. - amount, pkScript, _, err := decodeCompressedTxOut(serialized[offset:]) - if err != nil { - return nil, errDeserialize(fmt.Sprintf("unable to decode "+ - "UTXO: %s", err)) - } - - entry := &UTXOEntry{ - amount: amount, - pkScript: pkScript, - blockChainHeight: blockChainHeight, - packedFlags: 0, - } - if isBlockReward { - entry.packedFlags |= tfBlockReward - } - - return entry, nil -} - // dbPutUTXODiff uses an existing database transaction to update the UTXO set // in the database based on the provided UTXO view contents and state. In // particular, only the entries that have been marked as modified are written @@ -325,13 +238,10 @@ func dbPutUTXODiff(dbTx database.Tx, diff *UTXODiff) error { for outPoint, entry := range diff.toAdd { // Serialize and store the UTXO entry. - serialized, err := serializeUTXOEntry(entry) - if err != nil { - return err - } + serialized := serializeUTXOEntry(entry) key := outpointKey(outPoint) - err = utxoBucket.Put(*key, serialized) + err := utxoBucket.Put(*key, serialized) // NOTE: The key is intentionally not recycled here since the // database interface contract prohibits modifications. It will // be garbage collected normally when the database is done with diff --git a/blockdag/dagio_test.go b/blockdag/dagio_test.go index 9c48ed0a7..5e523187d 100644 --- a/blockdag/dagio_test.go +++ b/blockdag/dagio_test.go @@ -74,12 +74,7 @@ func TestUtxoSerialization(t *testing.T) { for i, test := range tests { // Ensure the utxo entry serializes to the expected value. - gotBytes, err := serializeUTXOEntry(test.entry) - if err != nil { - t.Errorf("serializeUTXOEntry #%d (%s) unexpected "+ - "error: %v", i, test.name, err) - continue - } + gotBytes := serializeUTXOEntry(test.entry) if !bytes.Equal(gotBytes, test.serialized) { t.Errorf("serializeUTXOEntry #%d (%s): mismatched "+ "bytes - got %x, want %x", i, test.name, diff --git a/blockdag/external_dag_test.go b/blockdag/external_dag_test.go index a93565f9f..fb26f9c7d 100644 --- a/blockdag/external_dag_test.go +++ b/blockdag/external_dag_test.go @@ -13,6 +13,7 @@ import ( "github.com/daglabs/btcd/blockdag" "github.com/daglabs/btcd/dagconfig" "github.com/daglabs/btcd/mining" + "github.com/daglabs/btcd/txscript" "github.com/daglabs/btcd/util" "github.com/daglabs/btcd/wire" ) @@ -192,9 +193,13 @@ func TestChainedTransactions(t *testing.T) { } cbTx := block1.Transactions[0] + signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil) + if err != nil { + t.Fatalf("Failed to build signature script: %s", err) + } txIn := &wire.TxIn{ PreviousOutPoint: wire.OutPoint{TxID: *cbTx.TxID(), Index: 0}, - SignatureScript: nil, + SignatureScript: signatureScript, Sequence: wire.MaxTxInSequenceNum, } txOut := &wire.TxOut{ @@ -205,11 +210,16 @@ func TestChainedTransactions(t *testing.T) { chainedTxIn := &wire.TxIn{ PreviousOutPoint: wire.OutPoint{TxID: *tx.TxID(), Index: 0}, - SignatureScript: nil, + SignatureScript: signatureScript, Sequence: wire.MaxTxInSequenceNum, } + + pkScript, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript) + if err != nil { + t.Fatalf("Failed to build public key script: %s", err) + } chainedTxOut := &wire.TxOut{ - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, Value: uint64(1), } chainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{chainedTxIn}, []*wire.TxOut{chainedTxOut}) @@ -236,11 +246,11 @@ func TestChainedTransactions(t *testing.T) { nonChainedTxIn := &wire.TxIn{ PreviousOutPoint: wire.OutPoint{TxID: *cbTx.TxID(), Index: 0}, - SignatureScript: nil, + SignatureScript: signatureScript, Sequence: wire.MaxTxInSequenceNum, } nonChainedTxOut := &wire.TxOut{ - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, Value: uint64(1), } nonChainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{nonChainedTxIn}, []*wire.TxOut{nonChainedTxOut}) @@ -292,26 +302,38 @@ func TestGasLimit(t *testing.T) { t.Fatalf("ProcessBlock: funds block got unexpectedly orphan") } + signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil) + if err != nil { + t.Fatalf("Failed to build signature script: %s", err) + } + + pkScript, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript) + if err != nil { + t.Fatalf("Failed to build public key script: %s", err) + } + cbTxValue := fundsBlock.Transactions[0].TxOut[0].Value cbTxID := fundsBlock.Transactions[0].TxID() tx1In := &wire.TxIn{ PreviousOutPoint: *wire.NewOutPoint(cbTxID, 0), Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, } tx1Out := &wire.TxOut{ Value: cbTxValue, - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, } tx1 := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{tx1In}, []*wire.TxOut{tx1Out}, subnetworkID, 10000, []byte{}) tx2In := &wire.TxIn{ PreviousOutPoint: *wire.NewOutPoint(cbTxID, 1), Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, } tx2Out := &wire.TxOut{ Value: cbTxValue, - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, } tx2 := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{tx2In}, []*wire.TxOut{tx2Out}, subnetworkID, 10000, []byte{}) @@ -337,10 +359,11 @@ func TestGasLimit(t *testing.T) { overflowGasTxIn := &wire.TxIn{ PreviousOutPoint: *wire.NewOutPoint(cbTxID, 1), Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, } overflowGasTxOut := &wire.TxOut{ Value: cbTxValue, - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, } overflowGasTx := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{overflowGasTxIn}, []*wire.TxOut{overflowGasTxOut}, subnetworkID, math.MaxUint64, []byte{}) @@ -368,10 +391,11 @@ func TestGasLimit(t *testing.T) { nonExistentSubnetworkTxIn := &wire.TxIn{ PreviousOutPoint: *wire.NewOutPoint(cbTxID, 0), Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, } nonExistentSubnetworkTxOut := &wire.TxOut{ Value: cbTxValue, - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, } nonExistentSubnetworkTx := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{nonExistentSubnetworkTxIn}, []*wire.TxOut{nonExistentSubnetworkTxOut}, nonExistentSubnetwork, 1, []byte{}) diff --git a/blockdag/indexers/txindex_test.go b/blockdag/indexers/txindex_test.go index 1b5b76b2b..d9e11731d 100644 --- a/blockdag/indexers/txindex_test.go +++ b/blockdag/indexers/txindex_test.go @@ -8,18 +8,24 @@ import ( "github.com/daglabs/btcd/blockdag" "github.com/daglabs/btcd/dagconfig" "github.com/daglabs/btcd/mining" + "github.com/daglabs/btcd/txscript" "github.com/daglabs/btcd/util" "github.com/daglabs/btcd/util/daghash" "github.com/daglabs/btcd/wire" ) -func createTransaction(value uint64, originTx *wire.MsgTx, outputIndex uint32) *wire.MsgTx { +func createTransaction(t *testing.T, value uint64, originTx *wire.MsgTx, outputIndex uint32) *wire.MsgTx { + signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil) + if err != nil { + t.Fatalf("Error creating signature script: %s", err) + } txIn := &wire.TxIn{ PreviousOutPoint: wire.OutPoint{ TxID: *originTx.TxID(), Index: outputIndex, }, - Sequence: wire.MaxTxInSequenceNum, + Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, } txOut := wire.NewTxOut(value, blockdag.OpTrueScript) tx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut}) @@ -68,9 +74,9 @@ func TestTxIndexConnectBlock(t *testing.T) { } block1 := prepareAndProcessBlock([]*daghash.Hash{params.GenesisHash}, nil, "1") - block2Tx := createTransaction(block1.Transactions[0].TxOut[0].Value, block1.Transactions[0], 0) + block2Tx := createTransaction(t, block1.Transactions[0].TxOut[0].Value, block1.Transactions[0], 0) block2 := prepareAndProcessBlock([]*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{block2Tx}, "2") - block3Tx := createTransaction(block2.Transactions[0].TxOut[0].Value, block2.Transactions[0], 0) + block3Tx := createTransaction(t, block2.Transactions[0].TxOut[0].Value, block2.Transactions[0], 0) block3 := prepareAndProcessBlock([]*daghash.Hash{block2.BlockHash()}, []*wire.MsgTx{block3Tx}, "3") block3TxID := block3Tx.TxID() diff --git a/blockdag/test_utils.go b/blockdag/test_utils.go index 2e8203f01..3f1690a41 100644 --- a/blockdag/test_utils.go +++ b/blockdag/test_utils.go @@ -202,7 +202,10 @@ func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) ( return nil, err } diffPastUTXO := pastUTXO.clone().(*DiffUTXOSet) - diffPastUTXO.meldToBase() + err = diffPastUTXO.meldToBase() + if err != nil { + return nil, err + } virtual.utxoSet = diffPastUTXO.base return virtual, nil diff --git a/blockdag/utxodiffstore.go b/blockdag/utxodiffstore.go index c28d870c6..8dec97128 100644 --- a/blockdag/utxodiffstore.go +++ b/blockdag/utxodiffstore.go @@ -1,17 +1,15 @@ package blockdag import ( - "bytes" - "encoding/binary" "fmt" - "io" "sync" "github.com/daglabs/btcd/database" "github.com/daglabs/btcd/util/daghash" - "github.com/daglabs/btcd/wire" ) +var multisetPointSize = 32 + type blockUTXODiffData struct { diff *UTXODiff diffChild *blockNode @@ -133,159 +131,6 @@ func (diffStore *utxoDiffStore) diffDataFromDB(hash *daghash.Hash) (*blockUTXODi return diffData, nil } -func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataBytes []byte) (*blockUTXODiffData, error) { - diffData := &blockUTXODiffData{} - serializedDiffData := bytes.NewBuffer(serializedDiffDataBytes) - - var hasDiffChild bool - err := wire.ReadElement(serializedDiffData, &hasDiffChild) - if err != nil { - return nil, err - } - - if hasDiffChild { - hash := &daghash.Hash{} - err := wire.ReadElement(serializedDiffData, hash) - if err != nil { - return nil, err - } - diffData.diffChild = diffStore.dag.index.LookupNode(hash) - } - - diffData.diff = &UTXODiff{} - - diffData.diff.toAdd, err = deserializeDiffEntries(serializedDiffData) - if err != nil { - return nil, err - } - - diffData.diff.toRemove, err = deserializeDiffEntries(serializedDiffData) - if err != nil { - return nil, err - } - - return diffData, nil -} - -func deserializeDiffEntries(r io.Reader) (utxoCollection, error) { - count, err := wire.ReadVarInt(r) - if err != nil { - return nil, err - } - collection := utxoCollection{} - for i := uint64(0); i < count; i++ { - outPointSize, err := wire.ReadVarInt(r) - if err != nil { - return nil, err - } - - serializedOutPoint := make([]byte, outPointSize) - err = binary.Read(r, byteOrder, serializedOutPoint) - if err != nil { - return nil, err - } - outPoint, err := deserializeOutPoint(serializedOutPoint) - if err != nil { - return nil, err - } - - utxoEntrySize, err := wire.ReadVarInt(r) - if err != nil { - return nil, err - } - serializedEntry := make([]byte, utxoEntrySize) - err = binary.Read(r, byteOrder, serializedEntry) - if err != nil { - return nil, err - } - utxoEntry, err := deserializeUTXOEntry(serializedEntry) - if err != nil { - return nil, err - } - collection.add(*outPoint, utxoEntry) - } - return collection, nil -} - -// serializeBlockUTXODiffData serializes diff data in the following format: -// Name | Data type | Description -// ------------ | --------- | ----------- -// hasDiffChild | Boolean | Indicates if a diff child exist -// diffChild | Hash | The diffChild's hash. Empty if hasDiffChild is true. -// diff | UTXODiff | The diff data's diff -func serializeBlockUTXODiffData(diffData *blockUTXODiffData) ([]byte, error) { - w := &bytes.Buffer{} - hasDiffChild := diffData.diffChild != nil - err := wire.WriteElement(w, hasDiffChild) - if err != nil { - return nil, err - } - if hasDiffChild { - err := wire.WriteElement(w, diffData.diffChild.hash) - if err != nil { - return nil, err - } - } - - err = serializeUTXODiff(w, diffData.diff) - if err != nil { - return nil, err - } - - return w.Bytes(), nil -} - -// serializeUTXODiff serializes UTXODiff by serializing -// UTXODiff.toAdd and UTXODiff.toRemove one after the other. -func serializeUTXODiff(w io.Writer, diff *UTXODiff) error { - err := serializeUTXOCollection(w, diff.toAdd) - if err != nil { - return err - } - - err = serializeUTXOCollection(w, diff.toRemove) - if err != nil { - return err - } - return nil -} - -// serializeUTXOCollection serializes utxoCollection by iterating over -// the utxo entries and serializing them and their corresponding outpoint -// prefixed by a varint that indicates their size. -func serializeUTXOCollection(w io.Writer, collection utxoCollection) error { - err := wire.WriteVarInt(w, uint64(len(collection))) - if err != nil { - return err - } - for outPoint, utxoEntry := range collection { - serializedOutPoint := *outpointKey(outPoint) - err = wire.WriteVarInt(w, uint64(len(serializedOutPoint))) - if err != nil { - return err - } - - err := binary.Write(w, byteOrder, serializedOutPoint) - if err != nil { - return err - } - - serializedUTXOEntry, err := serializeUTXOEntry(utxoEntry) - if err != nil { - return err - } - err = wire.WriteVarInt(w, uint64(len(serializedUTXOEntry))) - if err != nil { - return err - } - err = binary.Write(w, byteOrder, serializedUTXOEntry) - if err != nil { - return err - } - } - return nil -} - // flushToDB writes all dirty diff data to the database. If all writes // succeed, this clears the dirty set. func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error { diff --git a/blockdag/utxoio.go b/blockdag/utxoio.go new file mode 100644 index 000000000..81ed0576e --- /dev/null +++ b/blockdag/utxoio.go @@ -0,0 +1,308 @@ +package blockdag + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math/big" + + "github.com/daglabs/btcd/btcec" + "github.com/daglabs/btcd/util/daghash" + "github.com/daglabs/btcd/wire" +) + +// serializeBlockUTXODiffData serializes diff data in the following format: +// Name | Data type | Description +// ------------ | --------- | ----------- +// hasDiffChild | Boolean | Indicates if a diff child exist +// diffChild | Hash | The diffChild's hash. Empty if hasDiffChild is true. +// diff | UTXODiff | The diff data's diff +func serializeBlockUTXODiffData(diffData *blockUTXODiffData) ([]byte, error) { + w := &bytes.Buffer{} + hasDiffChild := diffData.diffChild != nil + err := wire.WriteElement(w, hasDiffChild) + if err != nil { + return nil, err + } + if hasDiffChild { + err := wire.WriteElement(w, diffData.diffChild.hash) + if err != nil { + return nil, err + } + } + + err = serializeUTXODiff(w, diffData.diff) + if err != nil { + return nil, err + } + + return w.Bytes(), nil +} + +// utxoEntryHeaderCode returns the calculated header code to be used when +// serializing the provided utxo entry. +func utxoEntryHeaderCode(entry *UTXOEntry) uint64 { + // As described in the serialization format comments, the header code + // encodes the height shifted over one bit and the block reward flag in the + // lowest bit. + headerCode := uint64(entry.BlockChainHeight()) << 1 + if entry.IsBlockReward() { + headerCode |= 0x01 + } + + return headerCode +} + +func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataBytes []byte) (*blockUTXODiffData, error) { + diffData := &blockUTXODiffData{} + serializedDiffData := bytes.NewBuffer(serializedDiffDataBytes) + + var hasDiffChild bool + err := wire.ReadElement(serializedDiffData, &hasDiffChild) + if err != nil { + return nil, err + } + + if hasDiffChild { + hash := &daghash.Hash{} + err := wire.ReadElement(serializedDiffData, hash) + if err != nil { + return nil, err + } + diffData.diffChild = diffStore.dag.index.LookupNode(hash) + } + + diffData.diff = &UTXODiff{} + + diffData.diff.toAdd, err = deserializeDiffEntries(serializedDiffData) + if err != nil { + return nil, err + } + + diffData.diff.toRemove, err = deserializeDiffEntries(serializedDiffData) + if err != nil { + return nil, err + } + + diffData.diff.diffMultiset, err = deserializeMultiset(serializedDiffData) + if err != nil { + return nil, err + } + + return diffData, nil +} + +func deserializeDiffEntries(r io.Reader) (utxoCollection, error) { + count, err := wire.ReadVarInt(r) + if err != nil { + return nil, err + } + collection := utxoCollection{} + for i := uint64(0); i < count; i++ { + outPointSize, err := wire.ReadVarInt(r) + if err != nil { + return nil, err + } + + serializedOutPoint := make([]byte, outPointSize) + err = binary.Read(r, byteOrder, serializedOutPoint) + if err != nil { + return nil, err + } + outPoint, err := deserializeOutPoint(serializedOutPoint) + if err != nil { + return nil, err + } + + utxoEntrySize, err := wire.ReadVarInt(r) + if err != nil { + return nil, err + } + serializedEntry := make([]byte, utxoEntrySize) + err = binary.Read(r, byteOrder, serializedEntry) + if err != nil { + return nil, err + } + utxoEntry, err := deserializeUTXOEntry(serializedEntry) + if err != nil { + return nil, err + } + collection.add(*outPoint, utxoEntry) + } + return collection, nil +} + +// deserializeMultiset deserializes an EMCH multiset. +// See serializeMultiset for more details. +func deserializeMultiset(r io.Reader) (*btcec.Multiset, error) { + xBytes := make([]byte, multisetPointSize) + yBytes := make([]byte, multisetPointSize) + err := binary.Read(r, byteOrder, xBytes) + if err != nil { + return nil, err + } + err = binary.Read(r, byteOrder, yBytes) + if err != nil { + return nil, err + } + var x, y big.Int + x.SetBytes(xBytes) + y.SetBytes(yBytes) + return btcec.NewMultisetFromPoint(btcec.S256(), &x, &y), nil +} + +// serializeUTXODiff serializes UTXODiff by serializing +// UTXODiff.toAdd, UTXODiff.toRemove and UTXODiff.Multiset one after the other. +func serializeUTXODiff(w io.Writer, diff *UTXODiff) error { + err := serializeUTXOCollection(w, diff.toAdd) + if err != nil { + return err + } + + err = serializeUTXOCollection(w, diff.toRemove) + if err != nil { + return err + } + err = serializeMultiset(w, diff.diffMultiset) + if err != nil { + return err + } + return nil +} + +// serializeUTXOCollection serializes utxoCollection by iterating over +// the utxo entries and serializing them and their corresponding outpoint +// prefixed by a varint that indicates their size. +func serializeUTXOCollection(w io.Writer, collection utxoCollection) error { + err := wire.WriteVarInt(w, uint64(len(collection))) + if err != nil { + return err + } + for outPoint, utxoEntry := range collection { + err := serializeUTXO(w, utxoEntry, &outPoint) + if err != nil { + return err + } + } + return nil +} + +// serializeMultiset serializes an ECMH multiset. The serialization +// is done by taking the (x,y) coordinnates of the multiset point and +// padding each one of them with 32 byte (it'll be 32 byte in most +// cases anyway except one of the coordinates is zero) and writing +// them one after the other. +func serializeMultiset(w io.Writer, ms *btcec.Multiset) error { + x, y := ms.Point() + xBytes := make([]byte, multisetPointSize) + copy(xBytes, x.Bytes()) + yBytes := make([]byte, multisetPointSize) + copy(yBytes, y.Bytes()) + + err := binary.Write(w, byteOrder, xBytes) + if err != nil { + return err + } + err = binary.Write(w, byteOrder, yBytes) + if err != nil { + return err + } + return nil +} + +// serializeUTXO serializes a utxo entry-outpoint pair +func serializeUTXO(w io.Writer, entry *UTXOEntry, outPoint *wire.OutPoint) error { + serializedOutPoint := *outpointKey(*outPoint) + err := wire.WriteVarInt(w, uint64(len(serializedOutPoint))) + if err != nil { + return err + } + + err = binary.Write(w, byteOrder, serializedOutPoint) + if err != nil { + return err + } + + serializedUTXOEntry := serializeUTXOEntry(entry) + err = wire.WriteVarInt(w, uint64(len(serializedUTXOEntry))) + if err != nil { + return err + } + err = binary.Write(w, byteOrder, serializedUTXOEntry) + if err != nil { + return err + } + return nil +} + +// serializeUTXOEntry returns the entry serialized to a format that is suitable +// for long-term storage. The format is described in detail above. +func serializeUTXOEntry(entry *UTXOEntry) []byte { + // Encode the header code. + headerCode := utxoEntryHeaderCode(entry) + + // Calculate the size needed to serialize the entry. + size := serializeSizeVLQ(headerCode) + + compressedTxOutSize(uint64(entry.Amount()), entry.PkScript()) + + // Serialize the header code followed by the compressed unspent + // transaction output. + serialized := make([]byte, size) + offset := putVLQ(serialized, headerCode) + offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()), + entry.PkScript()) + + return serialized +} + +// deserializeOutPoint decodes an outPoint from the passed serialized byte +// slice into a new wire.OutPoint using a format that is suitable for long- +// term storage. this format is described in detail above. +func deserializeOutPoint(serialized []byte) (*wire.OutPoint, error) { + if len(serialized) <= daghash.HashSize { + return nil, errDeserialize("unexpected end of data") + } + + txID := daghash.TxID{} + txID.SetBytes(serialized[:daghash.HashSize]) + index, _ := deserializeVLQ(serialized[daghash.HashSize:]) + return wire.NewOutPoint(&txID, uint32(index)), nil +} + +// deserializeUTXOEntry decodes a UTXO entry from the passed serialized byte +// slice into a new UTXOEntry using a format that is suitable for long-term +// storage. The format is described in detail above. +func deserializeUTXOEntry(serialized []byte) (*UTXOEntry, error) { + // Deserialize the header code. + code, offset := deserializeVLQ(serialized) + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after header") + } + + // Decode the header code. + // + // Bit 0 indicates whether the containing transaction is a block reward. + // Bits 1-x encode height of containing transaction. + isBlockReward := code&0x01 != 0 + blockChainHeight := code >> 1 + + // Decode the compressed unspent transaction output. + amount, pkScript, _, err := decodeCompressedTxOut(serialized[offset:]) + if err != nil { + return nil, errDeserialize(fmt.Sprintf("unable to decode "+ + "UTXO: %s", err)) + } + + entry := &UTXOEntry{ + amount: amount, + pkScript: pkScript, + blockChainHeight: blockChainHeight, + packedFlags: 0, + } + if isBlockReward { + entry.packedFlags |= tfBlockReward + } + + return entry, nil +} diff --git a/blockdag/utxoset.go b/blockdag/utxoset.go index 05f6d9538..86e3fc33a 100644 --- a/blockdag/utxoset.go +++ b/blockdag/utxoset.go @@ -1,12 +1,14 @@ package blockdag import ( + "bytes" "errors" "fmt" "math" "sort" "strings" + "github.com/daglabs/btcd/btcec" "github.com/daglabs/btcd/wire" ) @@ -128,15 +130,17 @@ func (uc utxoCollection) clone() utxoCollection { // UTXODiff represents a diff between two UTXO Sets. type UTXODiff struct { - toAdd utxoCollection - toRemove utxoCollection + toAdd utxoCollection + toRemove utxoCollection + diffMultiset *btcec.Multiset } // NewUTXODiff creates a new, empty utxoDiff func NewUTXODiff() *UTXODiff { return &UTXODiff{ - toAdd: utxoCollection{}, - toRemove: utxoCollection{}, + toAdd: utxoCollection{}, + toRemove: utxoCollection{}, + diffMultiset: btcec.NewMultiset(btcec.S256()), } } @@ -216,6 +220,9 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) { } } + // Create a new diffMultiset as the subtraction of the two diffs. + result.diffMultiset = other.diffMultiset.Subtract(d.diffMultiset) + return result, nil } @@ -290,31 +297,61 @@ func (d *UTXODiff) WithDiff(diff *UTXODiff) (*UTXODiff, error) { } } + // Apply diff.diffMultiset to d.diffMultiset + result.diffMultiset = d.diffMultiset.Union(diff.diffMultiset) + return result, nil } // clone returns a clone of this utxoDiff func (d *UTXODiff) clone() *UTXODiff { return &UTXODiff{ - toAdd: d.toAdd.clone(), - toRemove: d.toRemove.clone(), + toAdd: d.toAdd.clone(), + toRemove: d.toRemove.clone(), + diffMultiset: d.diffMultiset.Clone(), } } -//RemoveTxOuts marks the transaction's outputs to removal -func (d *UTXODiff) RemoveTxOuts(tx *wire.MsgTx) { - for idx := range tx.TxOut { - d.toRemove.add(*wire.NewOutPoint(tx.TxID(), uint32(idx)), nil) +// AddEntry adds a UTXOEntry to the diff +func (d *UTXODiff) AddEntry(outPoint wire.OutPoint, entry *UTXOEntry) error { + if d.toRemove.contains(outPoint) { + d.toRemove.remove(outPoint) + } else if _, exists := d.toAdd[outPoint]; exists { + return fmt.Errorf("AddEntry: Cannot add outpoint %s twice", outPoint) + } else { + d.toAdd.add(outPoint, entry) } + + var err error + newMs, err := addUTXOToMultiset(d.diffMultiset, entry, &outPoint) + if err != nil { + return err + } + d.diffMultiset = newMs + return nil } -//AddEntry adds an UTXOEntry to the diff -func (d *UTXODiff) AddEntry(outpoint wire.OutPoint, entry *UTXOEntry) { - d.toAdd.add(outpoint, entry) +// RemoveEntry removes a UTXOEntry from the diff +func (d *UTXODiff) RemoveEntry(outPoint wire.OutPoint, entry *UTXOEntry) error { + if d.toAdd.contains(outPoint) { + d.toAdd.remove(outPoint) + } else if _, exists := d.toRemove[outPoint]; exists { + return fmt.Errorf("removeEntry: Cannot remove outpoint %s twice", outPoint) + } else { + d.toRemove.add(outPoint, entry) + } + + var err error + newMs, err := removeUTXOFromMultiset(d.diffMultiset, entry, &outPoint) + if err != nil { + return err + } + d.diffMultiset = newMs + return nil } func (d UTXODiff) String() string { - return fmt.Sprintf("toAdd: %s; toRemove: %s", d.toAdd, d.toRemove) + return fmt.Sprintf("toAdd: %s; toRemove: %s, Multiset-Hash: %s", d.toAdd, d.toRemove, d.diffMultiset.Hash()) } // NewUTXOEntry creates a new utxoEntry representing the given txOut @@ -349,9 +386,10 @@ type UTXOSet interface { diffFrom(other UTXOSet) (*UTXODiff, error) WithDiff(utxoDiff *UTXODiff) (UTXOSet, error) diffFromTx(tx *wire.MsgTx, node *blockNode) (*UTXODiff, error) - AddTx(tx *wire.MsgTx, blockHeight uint64) (ok bool) + AddTx(tx *wire.MsgTx, blockHeight uint64) (ok bool, err error) clone() UTXOSet Get(outPoint wire.OutPoint) (*UTXOEntry, bool) + Multiset() *btcec.Multiset } // diffFromTx is a common implementation for diffFromTx, that works @@ -364,7 +402,10 @@ func diffFromTx(u UTXOSet, tx *wire.MsgTx, containingNode *blockNode) (*UTXODiff if !isBlockReward { for _, txIn := range tx.TxIn { if entry, ok := u.Get(txIn.PreviousOutPoint); ok { - diff.toRemove.add(txIn.PreviousOutPoint, entry) + err := diff.RemoveEntry(txIn.PreviousOutPoint, entry) + if err != nil { + return nil, err + } } else { return nil, ruleError(ErrMissingTxOut, fmt.Sprintf( "Transaction %s is invalid because spends outpoint %s that is not in utxo set", @@ -375,7 +416,10 @@ func diffFromTx(u UTXOSet, tx *wire.MsgTx, containingNode *blockNode) (*UTXODiff for i, txOut := range tx.TxOut { entry := NewUTXOEntry(txOut, isBlockReward, containingNode.height) outPoint := *wire.NewOutPoint(tx.TxID(), uint32(i)) - diff.toAdd.add(outPoint, entry) + err := diff.AddEntry(outPoint, entry) + if err != nil { + return nil, err + } } return diff, nil } @@ -383,12 +427,14 @@ func diffFromTx(u UTXOSet, tx *wire.MsgTx, containingNode *blockNode) (*UTXODiff // FullUTXOSet represents a full list of transaction outputs and their values type FullUTXOSet struct { utxoCollection + UTXOMultiset *btcec.Multiset } // NewFullUTXOSet creates a new utxoSet with full list of transaction outputs and their values func NewFullUTXOSet() *FullUTXOSet { return &FullUTXOSet{ utxoCollection: utxoCollection{}, + UTXOMultiset: btcec.NewMultiset(btcec.S256()), } } @@ -412,17 +458,22 @@ func (fus *FullUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) { return NewDiffUTXOSet(fus, other.clone()), nil } -// AddTx adds a transaction to this utxoSet and returns true iff it's valid in this UTXO's context -func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blockHeight uint64) bool { +// AddTx adds a transaction to this utxoSet and returns isAccepted=true iff it's valid in this UTXO's context. +// It returns error if something unexpected happens, such as serialization error (isAccepted=false doesn't +// necessarily means there's an error). +func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blockHeight uint64) (isAccepted bool, err error) { isBlockReward := tx.IsBlockReward() if !isBlockReward { if !fus.containsInputs(tx) { - return false + return false, nil } for _, txIn := range tx.TxIn { outPoint := *wire.NewOutPoint(&txIn.PreviousOutPoint.TxID, txIn.PreviousOutPoint.Index) - fus.remove(outPoint) + err := fus.removeAndUpdateMultiset(outPoint) + if err != nil { + return false, err + } } } @@ -430,10 +481,13 @@ func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blockHeight uint64) bool { outPoint := *wire.NewOutPoint(tx.TxID(), uint32(i)) entry := NewUTXOEntry(txOut, isBlockReward, blockHeight) - fus.add(outPoint, entry) + err := fus.addAndUpdateMultiset(outPoint, entry) + if err != nil { + return false, err + } } - return true + return true, nil } // diffFromTx returns a diff that is equivalent to provided transaction, @@ -455,7 +509,7 @@ func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool { // clone returns a clone of this utxoSet func (fus *FullUTXOSet) clone() UTXOSet { - return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone()} + return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone(), UTXOMultiset: fus.UTXOMultiset.Clone()} } // Get returns the UTXOEntry associated with the given OutPoint, and a boolean indicating if such entry was found @@ -464,6 +518,38 @@ func (fus *FullUTXOSet) Get(outPoint wire.OutPoint) (*UTXOEntry, bool) { return utxoEntry, ok } +// Multiset returns the ecmh-Multiset of this utxoSet +func (fus *FullUTXOSet) Multiset() *btcec.Multiset { + return fus.UTXOMultiset +} + +// addAndUpdateMultiset adds a UTXOEntry to this utxoSet and updates its multiset accordingly +func (fus *FullUTXOSet) addAndUpdateMultiset(outPoint wire.OutPoint, entry *UTXOEntry) error { + fus.add(outPoint, entry) + newMs, err := addUTXOToMultiset(fus.UTXOMultiset, entry, &outPoint) + if err != nil { + return err + } + fus.UTXOMultiset = newMs + return nil +} + +// removeAndUpdateMultiset removes a UTXOEntry from this utxoSet and updates its multiset accordingly +func (fus *FullUTXOSet) removeAndUpdateMultiset(outPoint wire.OutPoint) error { + entry, ok := fus.Get(outPoint) + if !ok { + return fmt.Errorf("Couldn't find outpoint %s", outPoint) + } + fus.remove(outPoint) + var err error + newMs, err := removeUTXOFromMultiset(fus.UTXOMultiset, entry, &outPoint) + if err != nil { + return err + } + fus.UTXOMultiset = newMs + return nil +} + // DiffUTXOSet represents a utxoSet with a base fullUTXOSet and a UTXODiff type DiffUTXOSet struct { base *FullUTXOSet @@ -504,27 +590,31 @@ func (dus *DiffUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) { } // AddTx adds a transaction to this utxoSet and returns true iff it's valid in this UTXO's context -func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockHeight uint64) bool { +func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockHeight uint64) (bool, error) { isBlockReward := tx.IsBlockReward() if !isBlockReward && !dus.containsInputs(tx) { - return false + return false, nil } - dus.appendTx(tx, blockHeight, isBlockReward) + err := dus.appendTx(tx, blockHeight, isBlockReward) + if err != nil { + return false, err + } - return true + return true, nil } -func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockHeight uint64, isBlockReward bool) { +func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockHeight uint64, isBlockReward bool) error { if !isBlockReward { - for _, txIn := range tx.TxIn { outPoint := *wire.NewOutPoint(&txIn.PreviousOutPoint.TxID, txIn.PreviousOutPoint.Index) - if dus.UTXODiff.toAdd.contains(outPoint) { - dus.UTXODiff.toAdd.remove(outPoint) - } else { - prevUTXOEntry := dus.base.utxoCollection[outPoint] - dus.UTXODiff.toRemove.add(outPoint, prevUTXOEntry) + entry, ok := dus.Get(outPoint) + if !ok { + return fmt.Errorf("Couldn't find entry for outpoint %s", outPoint) + } + err := dus.UTXODiff.RemoveEntry(outPoint, entry) + if err != nil { + return err } } } @@ -533,12 +623,12 @@ func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockHeight uint64, isBlockRewa outPoint := *wire.NewOutPoint(tx.TxID(), uint32(i)) entry := NewUTXOEntry(txOut, isBlockReward, blockHeight) - if dus.UTXODiff.toRemove.contains(outPoint) { - dus.UTXODiff.toRemove.remove(outPoint) - } else { - dus.UTXODiff.toAdd.add(outPoint, entry) + err := dus.UTXODiff.AddEntry(outPoint, entry) + if err != nil { + return err } } + return nil } func (dus *DiffUTXOSet) containsInputs(tx *wire.MsgTx) bool { @@ -556,16 +646,23 @@ func (dus *DiffUTXOSet) containsInputs(tx *wire.MsgTx) bool { } // meldToBase updates the base fullUTXOSet with all changes in diff -func (dus *DiffUTXOSet) meldToBase() { +func (dus *DiffUTXOSet) meldToBase() error { for outPoint := range dus.UTXODiff.toRemove { - dus.base.remove(outPoint) + if _, ok := dus.base.Get(outPoint); ok { + dus.base.remove(outPoint) + } else { + return fmt.Errorf("Couldn't remove outpoint %s because it doesn't exist in the DiffUTXOSet base", outPoint) + } } for outPoint, utxoEntry := range dus.UTXODiff.toAdd { dus.base.add(outPoint, utxoEntry) } + dus.base.UTXOMultiset = dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset) + dus.UTXODiff = NewUTXODiff() + return nil } // diffFromTx returns a diff that is equivalent to provided transaction, @@ -575,7 +672,7 @@ func (dus *DiffUTXOSet) diffFromTx(tx *wire.MsgTx, node *blockNode) (*UTXODiff, } func (dus *DiffUTXOSet) String() string { - return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove) + return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s, Multiset-Hash:%s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove, dus.Multiset().Hash()) } // clone returns a clone of this UTXO Set @@ -595,3 +692,26 @@ func (dus *DiffUTXOSet) Get(outPoint wire.OutPoint) (*UTXOEntry, bool) { txOut, ok := dus.UTXODiff.toAdd.get(outPoint) return txOut, ok } + +// Multiset returns the ecmh-Multiset of this utxoSet +func (dus *DiffUTXOSet) Multiset() *btcec.Multiset { + return dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset) +} + +func addUTXOToMultiset(ms *btcec.Multiset, entry *UTXOEntry, outPoint *wire.OutPoint) (*btcec.Multiset, error) { + w := &bytes.Buffer{} + err := serializeUTXO(w, entry, outPoint) + if err != nil { + return nil, err + } + return ms.Add(w.Bytes()), nil +} + +func removeUTXOFromMultiset(ms *btcec.Multiset, entry *UTXOEntry, outPoint *wire.OutPoint) (*btcec.Multiset, error) { + w := &bytes.Buffer{} + err := serializeUTXO(w, entry, outPoint) + if err != nil { + return nil, err + } + return ms.Remove(w.Bytes()), nil +} diff --git a/blockdag/utxoset_test.go b/blockdag/utxoset_test.go index 3e4b0090f..70dc6167d 100644 --- a/blockdag/utxoset_test.go +++ b/blockdag/utxoset_test.go @@ -5,6 +5,7 @@ import ( "reflect" "testing" + "github.com/daglabs/btcd/btcec" "github.com/daglabs/btcd/dagconfig" "github.com/daglabs/btcd/util/daghash" "github.com/daglabs/btcd/wire" @@ -77,20 +78,26 @@ func TestUTXODiff(t *testing.T) { outPoint1 := *wire.NewOutPoint(txID1, 0) utxoEntry0 := NewUTXOEntry(&wire.TxOut{PkScript: []byte{}, Value: 10}, true, 0) utxoEntry1 := NewUTXOEntry(&wire.TxOut{PkScript: []byte{}, Value: 20}, false, 1) - diff := UTXODiff{ - toAdd: utxoCollection{outPoint0: utxoEntry0}, - toRemove: utxoCollection{outPoint1: utxoEntry1}, - } // Test utxoDiff creation - newDiff := NewUTXODiff() - if len(newDiff.toAdd) != 0 || len(newDiff.toRemove) != 0 { + diff := NewUTXODiff() + if len(diff.toAdd) != 0 || len(diff.toRemove) != 0 { t.Errorf("new diff is not empty") } + err := diff.AddEntry(outPoint0, utxoEntry0) + if err != nil { + t.Fatalf("Error adding entry to utxo diff: %s", err) + } + + err = diff.RemoveEntry(outPoint1, utxoEntry1) + if err != nil { + t.Fatalf("Error adding entry to utxo diff: %s", err) + } + // Test utxoDiff cloning - clonedDiff := *diff.clone() - if &clonedDiff == &diff { + clonedDiff := diff.clone() + if clonedDiff == diff { t.Errorf("cloned diff is reference-equal to the original") } if !reflect.DeepEqual(clonedDiff, diff) { @@ -99,7 +106,7 @@ func TestUTXODiff(t *testing.T) { } // Test utxoDiff string representation - expectedDiffString := "toAdd: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ]; toRemove: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20 ]" + expectedDiffString := "toAdd: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ]; toRemove: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20 ], Multiset-Hash: 7cb61e48005b0c817211d04589d719bff87d86a6a6ce2454515f57265382ded7" diffString := clonedDiff.String() if diffString != expectedDiffString { t.Errorf("unexpected diff string. "+ @@ -288,42 +295,111 @@ func TestUTXODiffRules(t *testing.T) { } for _, test := range tests { + this := addMultisetToDiff(t, test.this) + other := addMultisetToDiff(t, test.other) + expectedDiffFromResult := addMultisetToDiff(t, test.expectedDiffFromResult) + expectedWithDiffResult := addMultisetToDiff(t, test.expectedWithDiffResult) + // diffFrom from this to other - diffResult, err := test.this.diffFrom(test.other) + diffResult, err := this.diffFrom(other) // Test whether diffFrom returned an error isDiffFromOk := err == nil - expectedIsDiffFromOk := test.expectedDiffFromResult != nil + expectedIsDiffFromOk := expectedDiffFromResult != nil if isDiffFromOk != expectedIsDiffFromOk { t.Errorf("unexpected diffFrom error in test \"%s\". "+ "Expected: \"%t\", got: \"%t\".", test.name, expectedIsDiffFromOk, isDiffFromOk) } // If not error, test the diffFrom result - if isDiffFromOk && !reflect.DeepEqual(diffResult, test.expectedDiffFromResult) { + if isDiffFromOk && !expectedDiffFromResult.equal(diffResult) { t.Errorf("unexpected diffFrom result in test \"%s\". "+ - "Expected: \"%v\", got: \"%v\".", test.name, test.expectedDiffFromResult, diffResult) + "Expected: \"%v\", got: \"%v\".", test.name, expectedDiffFromResult, diffResult) } // WithDiff from this to other - withDiffResult, err := test.this.WithDiff(test.other) + withDiffResult, err := this.WithDiff(other) // Test whether WithDiff returned an error isWithDiffOk := err == nil - expectedIsWithDiffOk := test.expectedWithDiffResult != nil + expectedIsWithDiffOk := expectedWithDiffResult != nil if isWithDiffOk != expectedIsWithDiffOk { t.Errorf("unexpected WithDiff error in test \"%s\". "+ "Expected: \"%t\", got: \"%t\".", test.name, expectedIsWithDiffOk, isWithDiffOk) } - // Ig not error, test the WithDiff result - if isWithDiffOk && !reflect.DeepEqual(withDiffResult, test.expectedWithDiffResult) { + // If not error, test the WithDiff result + if isWithDiffOk && !withDiffResult.equal(expectedWithDiffResult) { t.Errorf("unexpected WithDiff result in test \"%s\". "+ - "Expected: \"%v\", got: \"%v\".", test.name, test.expectedWithDiffResult, withDiffResult) + "Expected: \"%v\", got: \"%v\".", test.name, expectedWithDiffResult, withDiffResult) } } } +func areMultisetsEqual(a *btcec.Multiset, b *btcec.Multiset) bool { + aX, aY := a.Point() + bX, bY := b.Point() + return aX.Cmp(bX) == 0 && aY.Cmp(bY) == 0 +} + +func (d *UTXODiff) equal(other *UTXODiff) bool { + return reflect.DeepEqual(d.toAdd, other.toAdd) && + reflect.DeepEqual(d.toRemove, other.toRemove) && + areMultisetsEqual(d.diffMultiset, other.diffMultiset) +} + +func (fus *FullUTXOSet) equal(other *FullUTXOSet) bool { + return reflect.DeepEqual(fus.utxoCollection, other.utxoCollection) && + areMultisetsEqual(fus.UTXOMultiset, other.UTXOMultiset) +} + +func (dus *DiffUTXOSet) equal(other *DiffUTXOSet) bool { + return dus.base.equal(other.base) && dus.UTXODiff.equal(other.UTXODiff) +} + +func addMultisetToDiff(t *testing.T, diff *UTXODiff) *UTXODiff { + if diff == nil { + return nil + } + diffWithMs := NewUTXODiff() + for outPoint, entry := range diff.toAdd { + err := diffWithMs.AddEntry(outPoint, entry) + if err != nil { + t.Fatalf("Error with diffWithMs.AddEntry: %s", err) + } + } + for outPoint, entry := range diff.toRemove { + err := diffWithMs.RemoveEntry(outPoint, entry) + if err != nil { + t.Fatalf("Error with diffWithMs.removeEntry: %s", err) + } + } + return diffWithMs +} + +func addMultisetToFullUTXOSet(t *testing.T, fus *FullUTXOSet) *FullUTXOSet { + if fus == nil { + return nil + } + fusWithMs := NewFullUTXOSet() + for outPoint, entry := range fus.utxoCollection { + err := fusWithMs.addAndUpdateMultiset(outPoint, entry) + if err != nil { + t.Fatalf("Error with diffWithMs.AddEntry: %s", err) + } + } + return fusWithMs +} + +func addMultisetToDiffUTXOSet(t *testing.T, diffSet *DiffUTXOSet) *DiffUTXOSet { + if diffSet == nil { + return nil + } + diffWithMs := addMultisetToDiff(t, diffSet.UTXODiff) + baseWithMs := addMultisetToFullUTXOSet(t, diffSet.base) + return NewDiffUTXOSet(baseWithMs, diffWithMs) +} + // TestFullUTXOSet makes sure that fullUTXOSet is working as expected. func TestFullUTXOSet(t *testing.T) { txID0, _ := daghash.NewTxIDFromStr("0000000000000000000000000000000000000000000000000000000000000000") @@ -334,10 +410,10 @@ func TestFullUTXOSet(t *testing.T) { txOut1 := &wire.TxOut{PkScript: []byte{}, Value: 20} utxoEntry0 := NewUTXOEntry(txOut0, true, 0) utxoEntry1 := NewUTXOEntry(txOut1, false, 1) - diff := &UTXODiff{ + diff := addMultisetToDiff(t, &UTXODiff{ toAdd: utxoCollection{outPoint0: utxoEntry0}, toRemove: utxoCollection{outPoint1: utxoEntry1}, - } + }) // Test fullUTXOSet creation emptySet := NewFullUTXOSet() @@ -361,12 +437,16 @@ func TestFullUTXOSet(t *testing.T) { // Test fullUTXOSet addTx txIn0 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutPoint: wire.OutPoint{TxID: *txID0, Index: 0}, Sequence: 0} transaction0 := wire.NewNativeMsgTx(1, []*wire.TxIn{txIn0}, []*wire.TxOut{txOut0}) - if ok = emptySet.AddTx(transaction0, 0); ok { + if isAccepted, err := emptySet.AddTx(transaction0, 0); err != nil { + t.Errorf("AddTx unexpectedly failed: %s", err) + } else if isAccepted { t.Errorf("addTx unexpectedly succeeded") } - emptySet = &FullUTXOSet{utxoCollection: utxoCollection{outPoint0: utxoEntry0}} - if ok = emptySet.AddTx(transaction0, 0); !ok { - t.Errorf("addTx unexpectedly failed") + emptySet = addMultisetToFullUTXOSet(t, &FullUTXOSet{utxoCollection: utxoCollection{outPoint0: utxoEntry0}}) + if isAccepted, err := emptySet.AddTx(transaction0, 0); err != nil { + t.Errorf("addTx unexpectedly failed. Error: %s", err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", transaction0.TxID()) } // Test fullUTXOSet collection @@ -394,14 +474,16 @@ func TestDiffUTXOSet(t *testing.T) { txOut1 := &wire.TxOut{PkScript: []byte{}, Value: 20} utxoEntry0 := NewUTXOEntry(txOut0, true, 0) utxoEntry1 := NewUTXOEntry(txOut1, false, 1) - diff := &UTXODiff{ + diff := addMultisetToDiff(t, &UTXODiff{ toAdd: utxoCollection{outPoint0: utxoEntry0}, toRemove: utxoCollection{outPoint1: utxoEntry1}, - } + }) // Test diffUTXOSet creation emptySet := NewDiffUTXOSet(NewFullUTXOSet(), NewUTXODiff()) - if len(emptySet.collection()) != 0 { + if collection, err := emptySet.collection(); err != nil { + t.Errorf("Error getting emptySet collection: %s", err) + } else if len(collection) != 0 { t.Errorf("new set is not empty") } @@ -430,11 +512,12 @@ func TestDiffUTXOSet(t *testing.T) { // .collection() the given diffSet and compare it to expectedCollection // .clone() the given diffSet and compare its value to itself (expected: equals) and its reference to itself (expected: not equal) tests := []struct { - name string - diffSet *DiffUTXOSet - expectedMeldSet *DiffUTXOSet - expectedString string - expectedCollection utxoCollection + name string + diffSet *DiffUTXOSet + expectedMeldSet *DiffUTXOSet + expectedString string + expectedCollection utxoCollection + expectedMeldToBaseError string }{ { name: "empty base, empty diff", @@ -452,7 +535,7 @@ func TestDiffUTXOSet(t *testing.T) { toRemove: utxoCollection{}, }, }, - expectedString: "{Base: [ ], To Add: [ ], To Remove: [ ]}", + expectedString: "{Base: [ ], To Add: [ ], To Remove: [ ], Multiset-Hash:0000000000000000000000000000000000000000000000000000000000000000}", expectedCollection: utxoCollection{}, }, { @@ -471,7 +554,7 @@ func TestDiffUTXOSet(t *testing.T) { toRemove: utxoCollection{}, }, }, - expectedString: "{Base: [ ], To Add: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], To Remove: [ ]}", + expectedString: "{Base: [ ], To Add: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], To Remove: [ ], Multiset-Hash:da4768bd0359c3426268d6707c1fc17a68c45ef1ea734331b07568418234487f}", expectedCollection: utxoCollection{outPoint0: utxoEntry0}, }, { @@ -483,15 +566,10 @@ func TestDiffUTXOSet(t *testing.T) { toRemove: utxoCollection{outPoint0: utxoEntry0}, }, }, - expectedMeldSet: &DiffUTXOSet{ - base: &FullUTXOSet{utxoCollection: utxoCollection{}}, - UTXODiff: &UTXODiff{ - toAdd: utxoCollection{}, - toRemove: utxoCollection{}, - }, - }, - expectedString: "{Base: [ ], To Add: [ ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ]}", - expectedCollection: utxoCollection{}, + expectedMeldSet: nil, + expectedString: "{Base: [ ], To Add: [ ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], Multiset-Hash:046242cb1bb1e6d3fd91d0f181e1b2d4a597ac57fa2584fc3c2eb0e0f46c9369}", + expectedCollection: utxoCollection{}, + expectedMeldToBaseError: "Couldn't remove outpoint 0000000000000000000000000000000000000000000000000000000000000000:0 because it doesn't exist in the DiffUTXOSet base", }, { name: "one member in base toAdd, one member in diff toAdd", @@ -514,7 +592,7 @@ func TestDiffUTXOSet(t *testing.T) { toRemove: utxoCollection{}, }, }, - expectedString: "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], To Add: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20 ], To Remove: [ ]}", + expectedString: "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], To Add: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20 ], To Remove: [ ], Multiset-Hash:556cc61fd4d7e74d7807ca2298c5320375a6a20310a18920e54667220924baff}", expectedCollection: utxoCollection{ outPoint0: utxoEntry0, outPoint1: utxoEntry1, @@ -538,41 +616,56 @@ func TestDiffUTXOSet(t *testing.T) { toRemove: utxoCollection{}, }, }, - expectedString: "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], To Add: [ ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ]}", + expectedString: "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], To Add: [ ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10 ], Multiset-Hash:0000000000000000000000000000000000000000000000000000000000000000}", expectedCollection: utxoCollection{}, }, } for _, test := range tests { - // Test meldToBase - meldSet := test.diffSet.clone().(*DiffUTXOSet) - meldSet.meldToBase() - if !reflect.DeepEqual(meldSet, test.expectedMeldSet) { - t.Errorf("unexpected melded set in test \"%s\". "+ - "Expected: \"%v\", got: \"%v\".", test.name, test.expectedMeldSet, meldSet) - } + diffSet := addMultisetToDiffUTXOSet(t, test.diffSet) + expectedMeldSet := addMultisetToDiffUTXOSet(t, test.expectedMeldSet) // Test string representation - setString := test.diffSet.String() + setString := diffSet.String() if setString != test.expectedString { t.Errorf("unexpected string in test \"%s\". "+ "Expected: \"%s\", got: \"%s\".", test.name, test.expectedString, setString) } + // Test meldToBase + meldSet := diffSet.clone().(*DiffUTXOSet) + err := meldSet.meldToBase() + errString := "" + if err != nil { + errString = err.Error() + } + if test.expectedMeldToBaseError != errString { + t.Errorf("meldToBase in test \"%s\" expected error \"%s\" but got: \"%s\"", test.name, test.expectedMeldToBaseError, errString) + } + if err != nil { + continue + } + if !meldSet.equal(expectedMeldSet) { + t.Errorf("unexpected melded set in test \"%s\". "+ + "Expected: \"%v\", got: \"%v\".", test.name, expectedMeldSet, meldSet) + } + // Test collection - setCollection := test.diffSet.collection() - if !reflect.DeepEqual(setCollection, test.expectedCollection) { + setCollection, err := diffSet.collection() + if err != nil { + t.Errorf("Error getting diffSet collection: %s", err) + } else if !reflect.DeepEqual(setCollection, test.expectedCollection) { t.Errorf("unexpected set collection in test \"%s\". "+ "Expected: \"%v\", got: \"%v\".", test.name, test.expectedCollection, setCollection) } // Test cloning - clonedSet := test.diffSet.clone().(*DiffUTXOSet) - if !reflect.DeepEqual(clonedSet, test.diffSet) { + clonedSet := diffSet.clone().(*DiffUTXOSet) + if !reflect.DeepEqual(clonedSet, diffSet) { t.Errorf("unexpected set clone in test \"%s\". "+ - "Expected: \"%v\", got: \"%v\".", test.name, test.diffSet, clonedSet) + "Expected: \"%v\", got: \"%v\".", test.name, diffSet, clonedSet) } - if clonedSet == test.diffSet { + if clonedSet == diffSet { t.Errorf("cloned set is reference-equal to the original") } } @@ -640,7 +733,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) { // transaction1 spends transaction0 id1 := transaction0.TxID() outPoint1 := *wire.NewOutPoint(id1, 0) - txIn1 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutPoint: wire.OutPoint{TxID: *id1, Index: 0}, Sequence: 0} + txIn1 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutPoint: outPoint1, Sequence: 0} txOut1 := &wire.TxOut{PkScript: []byte{1}, Value: 20} utxoEntry1 := NewUTXOEntry(txOut1, false, 1) transaction1 := wire.NewNativeMsgTx(1, []*wire.TxIn{txIn1}, []*wire.TxOut{txOut1}) @@ -648,7 +741,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) { // transaction2 spends transaction1 id2 := transaction1.TxID() outPoint2 := *wire.NewOutPoint(id2, 0) - txIn2 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutPoint: wire.OutPoint{TxID: *id2, Index: 0}, Sequence: 0} + txIn2 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutPoint: outPoint2, Sequence: 0} txOut2 := &wire.TxOut{PkScript: []byte{2}, Value: 30} utxoEntry2 := NewUTXOEntry(txOut2, false, 2) transaction2 := wire.NewNativeMsgTx(1, []*wire.TxIn{txIn2}, []*wire.TxOut{txOut2}) @@ -771,31 +864,43 @@ func TestDiffUTXOSet_addTx(t *testing.T) { }, } +testLoop: for _, test := range tests { - diffSet := test.startSet.clone() + startSet := addMultisetToDiffUTXOSet(t, test.startSet) + expectedSet := addMultisetToDiffUTXOSet(t, test.expectedSet) + + diffSet := startSet.clone() // Apply all transactions to diffSet, in order, with the initial block height startHeight for i, transaction := range test.toAdd { - diffSet.AddTx(transaction, test.startHeight+uint64(i)) + _, err := diffSet.AddTx(transaction, test.startHeight+uint64(i)) + if err != nil { + t.Errorf("Error adding tx %s in test \"%s\": %s", transaction.TxID(), test.name, err) + continue testLoop + } } // Make sure that the result diffSet equals to the expectedSet - if !reflect.DeepEqual(diffSet, test.expectedSet) { + if !diffSet.(*DiffUTXOSet).equal(expectedSet) { t.Errorf("unexpected diffSet in test \"%s\". "+ - "Expected: \"%v\", got: \"%v\".", test.name, test.expectedSet, diffSet) + "Expected: \"%v\", got: \"%v\".", test.name, expectedSet, diffSet) } } } func TestDiffFromTx(t *testing.T) { - fus := &FullUTXOSet{ + fus := addMultisetToFullUTXOSet(t, &FullUTXOSet{ utxoCollection: utxoCollection{}, - } + }) cbTx, err := createCoinbaseTxForTest(1, 1, 0, &dagconfig.SimNetParams) if err != nil { t.Errorf("createCoinbaseTxForTest: %v", err) } - fus.AddTx(cbTx, 1) + if isAccepted, err := fus.AddTx(cbTx, 1); err != nil { + t.Fatalf("AddTx unexpectedly failed. Error: %s", err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", cbTx.TxID()) + } node := &blockNode{height: 2} //Fake node cbOutpoint := wire.OutPoint{TxID: *cbTx.TxID(), Index: 0} txIns := []*wire.TxIn{&wire.TxIn{ @@ -841,11 +946,16 @@ func TestDiffFromTx(t *testing.T) { } //Test that we get an error if the outpoint is inside diffUTXOSet's toRemove - dus := NewDiffUTXOSet(fus, &UTXODiff{ + diff2 := addMultisetToDiff(t, &UTXODiff{ toAdd: utxoCollection{}, toRemove: utxoCollection{}, }) - dus.AddTx(tx, 2) + dus := NewDiffUTXOSet(fus, diff2) + if isAccepted, err := dus.AddTx(tx, 2); err != nil { + t.Fatalf("AddTx unexpectedly failed. Error: %s", err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", tx.TxID()) + } _, err = dus.diffFromTx(tx, node) if err == nil { t.Errorf("diffFromTx: expected an error but got ") @@ -858,11 +968,14 @@ func (fus *FullUTXOSet) collection() utxoCollection { } // collection returns a collection of all UTXOs in this set -func (dus *DiffUTXOSet) collection() utxoCollection { +func (dus *DiffUTXOSet) collection() (utxoCollection, error) { clone := dus.clone().(*DiffUTXOSet) - clone.meldToBase() + err := clone.meldToBase() + if err != nil { + return nil, err + } - return clone.base.collection() + return clone.base.collection(), nil } func TestUTXOSetAddEntry(t *testing.T) { @@ -880,6 +993,7 @@ func TestUTXOSetAddEntry(t *testing.T) { outPointToAdd *wire.OutPoint utxoEntryToAdd *UTXOEntry expectedUTXODiff *UTXODiff + expectedError string }{ { name: "add an entry", @@ -907,62 +1021,23 @@ func TestUTXOSetAddEntry(t *testing.T) { toAdd: utxoCollection{*outPoint0: utxoEntry0, *outPoint1: utxoEntry1}, toRemove: utxoCollection{}, }, + expectedError: "AddEntry: Cannot add outpoint 0000000000000000000000000000000000000000000000000000000000000000:0 twice", }, } for _, test := range tests { - utxoDiff.AddEntry(*test.outPointToAdd, test.utxoEntryToAdd) - if !reflect.DeepEqual(utxoDiff, test.expectedUTXODiff) { - t.Fatalf("utxoDiff.AddEntry: unexpected utxoDiff in test '%s'. "+ - "Expected: %v, got: %v", test.name, test.expectedUTXODiff, utxoDiff) - } - } -} - -func TestUTXOSetRemoveTxOuts(t *testing.T) { - tx0 := wire.NewNativeMsgTx(1, nil, []*wire.TxOut{{PkScript: []byte{1}, Value: 10}}) - tx1 := wire.NewNativeMsgTx(1, nil, []*wire.TxOut{{PkScript: []byte{2}, Value: 20}}) - outPoint0 := wire.NewOutPoint(tx0.TxID(), 0) - outPoint1 := wire.NewOutPoint(tx1.TxID(), 0) - - utxoDiff := NewUTXODiff() - - tests := []struct { - name string - txToRemove *wire.MsgTx - expectedUTXODiff *UTXODiff - }{ - { - name: "remove a transaction", - txToRemove: tx0, - expectedUTXODiff: &UTXODiff{ - toAdd: utxoCollection{}, - toRemove: utxoCollection{*outPoint0: nil}, - }, - }, - { - name: "remove another transaction", - txToRemove: tx1, - expectedUTXODiff: &UTXODiff{ - toAdd: utxoCollection{}, - toRemove: utxoCollection{*outPoint0: nil, *outPoint1: nil}, - }, - }, - { - name: "remove first entry again", - txToRemove: tx0, - expectedUTXODiff: &UTXODiff{ - toAdd: utxoCollection{}, - toRemove: utxoCollection{*outPoint0: nil, *outPoint1: nil}, - }, - }, - } - - for _, test := range tests { - utxoDiff.RemoveTxOuts(test.txToRemove) - if !reflect.DeepEqual(utxoDiff, test.expectedUTXODiff) { - t.Fatalf("utxoDiff.AddEntry: unexpected utxoDiff in test '%s'. "+ - "Expected: %v, got: %v", test.name, test.expectedUTXODiff, utxoDiff) + expectedUTXODiff := addMultisetToDiff(t, test.expectedUTXODiff) + err := utxoDiff.AddEntry(*test.outPointToAdd, test.utxoEntryToAdd) + errString := "" + if err != nil { + errString = err.Error() + } + if errString != test.expectedError { + t.Fatalf("utxoDiff.AddEntry: unexpected err in test \"%s\". Expected: %s but got: %s", test.name, test.expectedError, err) + } + if err == nil && !utxoDiff.equal(expectedUTXODiff) { + t.Fatalf("utxoDiff.AddEntry: unexpected utxoDiff in test \"%s\". "+ + "Expected: %v, got: %v", test.name, expectedUTXODiff, utxoDiff) } } } diff --git a/btcec/ecmh.go b/btcec/ecmh.go index a447f5066..9f2e237be 100644 --- a/btcec/ecmh.go +++ b/btcec/ecmh.go @@ -50,14 +50,15 @@ func NewMultisetFromDataSlice(curve *KoblitzCurve, datas [][]byte) *Multiset { return ms } -func (ms *Multiset) clone() *Multiset { +// Clone returns a clone of this multiset. +func (ms *Multiset) Clone() *Multiset { return NewMultisetFromPoint(ms.curve, ms.x, ms.y) } // Add hashes the data onto the curve and returns // a multiset with the new resulting point. func (ms *Multiset) Add(data []byte) *Multiset { - newMs := ms.clone() + newMs := ms.Clone() x, y := hashToPoint(ms.curve, data) newMs.addPoint(x, y) return newMs @@ -76,7 +77,7 @@ func (ms *Multiset) addPoint(x, y *big.Int) { // all the elements that were added, you will not get // back to the point at infinity (empty set). func (ms *Multiset) Remove(data []byte) *Multiset { - newMs := ms.clone() + newMs := ms.Clone() x, y := hashToPoint(ms.curve, data) newMs.removePoint(x, y) return newMs @@ -90,8 +91,8 @@ func (ms *Multiset) removePoint(x, y *big.Int) { // Union will add the point of the passed multiset instance to the point // of this multiset and will return a multiset with the resulting point. func (ms *Multiset) Union(otherMultiset *Multiset) *Multiset { - newMs := ms.clone() - otherMsCopy := otherMultiset.clone() + newMs := ms.Clone() + otherMsCopy := otherMultiset.Clone() newMs.addPoint(otherMsCopy.x, otherMsCopy.y) return newMs } @@ -99,8 +100,8 @@ func (ms *Multiset) Union(otherMultiset *Multiset) *Multiset { // Subtract will remove the point of the passed multiset instance from the point // of this multiset and will return a multiset with the resulting point. func (ms *Multiset) Subtract(otherMultiset *Multiset) *Multiset { - newMs := ms.clone() - otherMsCopy := otherMultiset.clone() + newMs := ms.Clone() + otherMsCopy := otherMultiset.Clone() newMs.removePoint(otherMsCopy.x, otherMsCopy.y) return newMs } diff --git a/mempool/mempool.go b/mempool/mempool.go index 37b0386e4..e4aa2c7a5 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -494,11 +494,16 @@ func (mp *TxPool) removeTransaction(tx *util.Tx, removeRedeemers bool, restoreIn for i := uint32(0); i < uint32(len(tx.MsgTx().TxOut)); i++ { prevOut := wire.OutPoint{TxID: *txID, Index: i} if txRedeemer, exists := mp.outpoints[prevOut]; exists { - mp.removeTransaction(txRedeemer, true, false) + err := mp.removeTransaction(txRedeemer, true, false) + if err != nil { + return err + } } } } + msgTx := tx.MsgTx() + // Remove the transaction if needed. if txDesc, exists := mp.fetchTransaction(txID); exists { // Remove unconfirmed address index entries associated with the @@ -508,20 +513,36 @@ func (mp *TxPool) removeTransaction(tx *util.Tx, removeRedeemers bool, restoreIn } diff := blockdag.NewUTXODiff() - diff.RemoveTxOuts(txDesc.Tx.MsgTx()) + + for idx := range msgTx.TxOut { + outPoint := *wire.NewOutPoint(txID, uint32(idx)) + entry, exists := mp.mpUTXOSet.Get(outPoint) + if exists { + err := diff.RemoveEntry(outPoint, entry) + if err != nil { + return err + } + } + } // Mark the referenced outpoints as unspent by the pool. - for _, txIn := range txDesc.Tx.MsgTx().TxIn { + for _, txIn := range msgTx.TxIn { if restoreInputs { if prevTxDesc, exists := mp.pool[txIn.PreviousOutPoint.TxID]; exists { prevOut := prevTxDesc.Tx.MsgTx().TxOut[txIn.PreviousOutPoint.Index] entry := blockdag.NewUTXOEntry(prevOut, false, blockdag.UnminedChainHeight) - diff.AddEntry(txIn.PreviousOutPoint, entry) + err := diff.AddEntry(txIn.PreviousOutPoint, entry) + if err != nil { + return err + } } if prevTxDesc, exists := mp.depends[txIn.PreviousOutPoint.TxID]; exists { prevOut := prevTxDesc.Tx.MsgTx().TxOut[txIn.PreviousOutPoint.Index] entry := blockdag.NewUTXOEntry(prevOut, false, blockdag.UnminedChainHeight) - diff.AddEntry(txIn.PreviousOutPoint, entry) + err := diff.AddEntry(txIn.PreviousOutPoint, entry) + if err != nil { + return err + } } } delete(mp.outpoints, txIn.PreviousOutPoint) @@ -535,7 +556,7 @@ func (mp *TxPool) removeTransaction(tx *util.Tx, removeRedeemers bool, restoreIn // Process dependent transactions prevOut := wire.OutPoint{TxID: *txID} - for txOutIdx := range tx.MsgTx().TxOut { + for txOutIdx := range msgTx.TxOut { // Skip to the next available output if there are none. prevOut.Index = uint32(txOutIdx) depends, exists := mp.dependsByPrev[prevOut] @@ -606,7 +627,7 @@ func (mp *TxPool) RemoveDoubleSpends(tx *util.Tx) { // helper for maybeAcceptTransaction. // // This function MUST be called with the mempool lock held (for writes). -func (mp *TxPool) addTransaction(tx *util.Tx, height uint64, fee uint64, parentsInPool []*wire.OutPoint) *TxDesc { +func (mp *TxPool) addTransaction(tx *util.Tx, height uint64, fee uint64, parentsInPool []*wire.OutPoint) (*TxDesc, error) { // Add the transaction to the pool and mark the referenced outpoints // as spent by the pool. txD := &TxDesc{ @@ -636,7 +657,11 @@ func (mp *TxPool) addTransaction(tx *util.Tx, height uint64, fee uint64, parents for _, txIn := range tx.MsgTx().TxIn { mp.outpoints[txIn.PreviousOutPoint] = tx } - mp.mpUTXOSet.AddTx(tx.MsgTx(), blockdag.UnminedChainHeight) + if isAccepted, err := mp.mpUTXOSet.AddTx(tx.MsgTx(), blockdag.UnminedChainHeight); err != nil { + return nil, err + } else if !isAccepted { + return nil, fmt.Errorf("unexpectedly failed to add tx %s to the mempool utxo set", tx.ID()) + } atomic.StoreInt64(&mp.lastUpdated, time.Now().Unix()) // Add unconfirmed address index entries associated with the transaction @@ -650,7 +675,7 @@ func (mp *TxPool) addTransaction(tx *util.Tx, height uint64, fee uint64, parents mp.cfg.FeeEstimator.ObserveTransaction(txD) } - return txD + return txD, nil } // checkPoolDoubleSpend checks whether or not the passed transaction is @@ -986,7 +1011,10 @@ func (mp *TxPool) maybeAcceptTransaction(tx *util.Tx, isNew, rateLimit, rejectDu } // Add to transaction pool. - txD := mp.addTransaction(tx, bestHeight, txFee, parentsInPool) + txD, err := mp.addTransaction(tx, bestHeight, txFee, parentsInPool) + if err != nil { + return nil, nil, err + } log.Debugf("Accepted transaction %s (pool size: %d)", txID, len(mp.pool)) @@ -1326,7 +1354,7 @@ func (mp *TxPool) HandleNewBlock(block *util.Block, txChan chan NewBlockMsg) err // no longer an orphan. Transactions which depend on a confirmed // transaction are NOT removed recursively because they are still // valid. - for _, tx := range block.Transactions()[1:] { + for _, tx := range block.Transactions()[util.FeeTransactionIndex:] { err := mp.RemoveTransaction(tx, false, false) if err != nil { mp.mpUTXOSet = oldUTXOSet diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 863f0819b..5e2ff9860 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -6,7 +6,6 @@ package mempool import ( "bytes" - "encoding/hex" "errors" "fmt" "math" @@ -22,8 +21,8 @@ import ( "bou.ke/monkey" "github.com/daglabs/btcd/blockdag" "github.com/daglabs/btcd/blockdag/indexers" - "github.com/daglabs/btcd/btcec" "github.com/daglabs/btcd/dagconfig" + "github.com/daglabs/btcd/mining" "github.com/daglabs/btcd/txscript" "github.com/daglabs/btcd/util" "github.com/daglabs/btcd/util/daghash" @@ -102,15 +101,9 @@ func txOutToSpendableOutpoint(tx *util.Tx, outputNum uint32) spendableOutpoint { // signing transactions as well as a fake chain that provides utxos for use in // generating valid transactions. type poolHarness struct { - // signKey is the signing key used for creating transactions throughout - // the tests. - // - // payAddr is the p2sh address for the signing key and is used for the - // payment address throughout the tests. - signKey *btcec.PrivateKey - payAddr util.Address - payScript []byte - chainParams *dagconfig.Params + signatureScript []byte + payScript []byte + chainParams *dagconfig.Params chain *fakeChain txPool *TxPool @@ -177,7 +170,7 @@ func (p *poolHarness) CreateSignedTxForSubnetwork(inputs []spendableOutpoint, nu for _, input := range inputs { txIns = append(txIns, &wire.TxIn{ PreviousOutPoint: input.outPoint, - SignatureScript: nil, + SignatureScript: p.signatureScript, Sequence: wire.MaxTxInSequenceNum, }) } @@ -200,12 +193,7 @@ func (p *poolHarness) CreateSignedTxForSubnetwork(inputs []spendableOutpoint, nu // Sign the new transaction. for i := range tx.TxIn { - sigScript, err := txscript.SignatureScript(tx, i, p.payScript, - txscript.SigHashAll, p.signKey, true) - if err != nil { - return nil, err - } - tx.TxIn[i].SignatureScript = sigScript + tx.TxIn[i].SignatureScript = p.signatureScript } return util.NewTx(tx), nil @@ -233,7 +221,7 @@ func (p *poolHarness) CreateTxChain(firstOutput spendableOutpoint, numTxns uint3 // with the harness. txIn := &wire.TxIn{ PreviousOutPoint: prevOutPoint, - SignatureScript: nil, + SignatureScript: p.signatureScript, Sequence: wire.MaxTxInSequenceNum, } txOut := &wire.TxOut{ @@ -242,14 +230,6 @@ func (p *poolHarness) CreateTxChain(firstOutput spendableOutpoint, numTxns uint3 } tx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut}) - // Sign the new transaction. - sigScript, err := txscript.SignatureScript(tx, 0, p.payScript, - txscript.SigHashAll, p.signKey, true) - if err != nil { - return nil, err - } - tx.TxIn[0].SignatureScript = sigScript - txChain = append(txChain, util.NewTx(tx)) // Next transaction uses outputs from this one. @@ -259,36 +239,58 @@ func (p *poolHarness) CreateTxChain(firstOutput spendableOutpoint, numTxns uint3 return txChain, nil } +func (tc *testContext) mineTransactions(transactions []*util.Tx, coinbaseOutputs uint64) *util.Block { + msgTxs := make([]*wire.MsgTx, len(transactions)) + for i, tx := range transactions { + msgTxs[i] = tx.MsgTx() + } + block, err := mining.PrepareBlockForTest(tc.harness.txPool.cfg.DAG, tc.harness.txPool.cfg.DAGParams, tc.harness.txPool.cfg.DAG.TipHashes(), msgTxs, true, coinbaseOutputs) + if err != nil { + tc.t.Fatalf("PrepareBlockForTest: %s", err) + } + utilBlock := util.NewBlock(block) + isOrphan, err := tc.harness.txPool.cfg.DAG.ProcessBlock(utilBlock, blockdag.BFNoPoWCheck) + if err != nil { + tc.t.Fatalf("ProcessBlock: %s", err) + } + if isOrphan { + tc.t.Fatalf("block %s was unexpectedly orphan", block.BlockHash()) + } + + // Handle new block by pool + ch := make(chan NewBlockMsg) + go func() { + err = tc.harness.txPool.HandleNewBlock(utilBlock, ch) + close(ch) + }() + + // process messages pushed by HandleNewBlock + for range ch { + } + // ensure that HandleNewBlock has not failed + if err != nil { + tc.t.Fatalf("HandleNewBlock failed to handle block %s", err) + } + return utilBlock +} + // newPoolHarness returns a new instance of a pool harness initialized with a // fake chain and a TxPool bound to it that is configured with a policy suitable // for testing. Also, the fake chain is populated with the returned spendable // outputs so the caller can easily create new valid transactions which build // off of it. -func newPoolHarness(dagParams *dagconfig.Params, numOutputs uint32, dbName string) (*poolHarness, []spendableOutpoint, func(), error) { - // Use a hard coded key pair for deterministic results. - keyBytes, err := hex.DecodeString("700868df1838811ffbdf918fb482c1f7e" + - "ad62db4b97bd7012c23e726485e577d") +func newPoolHarness(t *testing.T, dagParams *dagconfig.Params, numOutputs uint32, dbName string) (*testContext, []spendableOutpoint, func(), error) { + pkScript, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript) if err != nil { return nil, nil, nil, err } - signKey, signPub := btcec.PrivKeyFromBytes(btcec.S256(), keyBytes) - // Generate associated pay-to-script-hash address and resulting payment - // script. - pubKeyBytes := signPub.SerializeCompressed() - payPubKeyAddr, err := util.NewAddressPubKey(pubKeyBytes, dagParams.Prefix) - if err != nil { - return nil, nil, nil, err - } - payAddr := payPubKeyAddr.AddressPubKeyHash() - pkScript, err := txscript.PayToAddrScript(payAddr) - if err != nil { - return nil, nil, nil, err - } + params := *dagParams + params.BlockRewardMaturity = 0 // Create a new database and chain instance to run tests against. dag, teardownFunc, err := blockdag.DAGSetup(dbName, blockdag.Config{ - DAGParams: dagParams, + DAGParams: ¶ms, }) if err != nil { return nil, nil, nil, fmt.Errorf("Failed to setup DAG instance: %v", err) @@ -299,13 +301,17 @@ func newPoolHarness(dagParams *dagconfig.Params, numOutputs uint32, dbName strin } }() + signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil) + if err != nil { + return nil, nil, nil, fmt.Errorf("Failed to build harness signature script: %s", err) + } + // Create a new fake chain and harness bound to it. chain := &fakeChain{} - harness := poolHarness{ - signKey: signKey, - payAddr: payAddr, - payScript: pkScript, - chainParams: dagParams, + harness := &poolHarness{ + signatureScript: signatureScript, + payScript: pkScript, + chainParams: dagParams, chain: chain, txPool: New(&Config{ @@ -328,6 +334,10 @@ func newPoolHarness(dagParams *dagconfig.Params, numOutputs uint32, dbName strin }), } + tc := &testContext{harness: harness, t: t} + block := tc.mineTransactions(nil, uint64(numOutputs)) + coinbase := block.Transactions()[0] + // Create a single coinbase transaction and add it to the harness // chain's utxo set and set the harness chain height such that the // coinbase will mature in the next block. This ensures the txpool @@ -335,11 +345,6 @@ func newPoolHarness(dagParams *dagconfig.Params, numOutputs uint32, dbName strin // mature in the next block. outpoints := make([]spendableOutpoint, 0, numOutputs) curHeight := harness.chain.BestHeight() - coinbase, err := harness.CreateCoinbaseTx(curHeight+1, numOutputs) - if err != nil { - return nil, nil, nil, err - } - harness.txPool.mpUTXOSet.AddTx(coinbase.MsgTx(), curHeight+1) for i := uint32(0); i < numOutputs; i++ { outpoints = append(outpoints, txOutToSpendableOutpoint(coinbase, i)) } @@ -350,7 +355,7 @@ func newPoolHarness(dagParams *dagconfig.Params, numOutputs uint32, dbName strin } harness.chain.SetMedianTimePast(time.Now()) - return &harness, outpoints, teardownFunc, nil + return tc, outpoints, teardownFunc, nil } // testContext houses a test-related state that is useful to pass to helper @@ -457,24 +462,20 @@ func (p *poolHarness) createTx(outpoint spendableOutpoint, fee uint64, numOutput tx := wire.NewNativeMsgTx(wire.TxVersion, txIns, txOuts) // Sign the new transaction. - sigScript, err := txscript.SignatureScript(tx, 0, p.payScript, - txscript.SigHashAll, p.signKey, true) - if err != nil { - return nil, err - } - tx.TxIn[0].SignatureScript = sigScript + tx.TxIn[0].SignatureScript = p.signatureScript return util.NewTx(tx), nil } func TestProcessTransaction(t *testing.T) { params := dagconfig.SimNetParams params.BlockRewardMaturity = 0 - harness, spendableOuts, teardownFunc, err := newPoolHarness(¶ms, 6, "TestProcessTransaction") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, ¶ms, 6, "TestProcessTransaction") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + + harness := tc.harness //Checks that a transaction cannot be added to the transaction pool if it's already there tx, err := harness.createTx(spendableOuts[0], 0, 1) @@ -623,7 +624,11 @@ func TestProcessTransaction(t *testing.T) { t.Fatalf("PayToAddrScript: unexpected error: %v", err) } p2shTx := util.NewTx(wire.NewNativeMsgTx(1, nil, []*wire.TxOut{{Value: 5000000000, PkScript: p2shPKScript}})) - harness.txPool.mpUTXOSet.AddTx(p2shTx.MsgTx(), curHeight+1) + if isAccepted, err := harness.txPool.mpUTXOSet.AddTx(p2shTx.MsgTx(), curHeight+1); err != nil { + t.Fatalf("AddTx unexpectedly failed. Error: %s", err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", p2shTx.ID()) + } txIns := []*wire.TxIn{{ PreviousOutPoint: wire.OutPoint{TxID: *p2shTx.ID(), Index: 0}, @@ -754,11 +759,12 @@ func TestProcessTransaction(t *testing.T) { } func TestAddrIndex(t *testing.T) { - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 2, "TestAddrIndex") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 2, "TestAddrIndex") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness harness.txPool.cfg.AddrIndex = &indexers.AddrIndex{} enteredAddUnconfirmedTx := false guard := monkey.Patch((*indexers.AddrIndex).AddUnconfirmedTx, func(idx *indexers.AddrIndex, tx *util.Tx, utxoSet blockdag.UTXOSet) { @@ -795,11 +801,12 @@ func TestAddrIndex(t *testing.T) { } func TestFeeEstimatorCfg(t *testing.T) { - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 2, "TestFeeEstimatorCfg") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 2, "TestFeeEstimatorCfg") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness harness.txPool.cfg.FeeEstimator = &FeeEstimator{} enteredObserveTransaction := false guard := monkey.Patch((*FeeEstimator).ObserveTransaction, func(ef *FeeEstimator, t *TxDesc) { @@ -822,12 +829,12 @@ func TestFeeEstimatorCfg(t *testing.T) { } func TestDoubleSpends(t *testing.T) { - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 2, "TestDoubleSpends") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 2, "TestDoubleSpends") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness //Add two transactions to the mempool tx1, err := harness.createTx(spendableOuts[0], 0, 1) @@ -872,12 +879,12 @@ func TestDoubleSpends(t *testing.T) { //TestFetchTransaction checks that FetchTransaction //returns only transaction from the main pool and not from the orphan pool func TestFetchTransaction(t *testing.T) { - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestFetchTransaction") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestFetchTransaction") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness orphanedTx, err := harness.CreateSignedTx([]spendableOutpoint{{ amount: util.Amount(5000000000), @@ -918,12 +925,12 @@ func TestFetchTransaction(t *testing.T) { // they are all orphans. Finally, it adds the linking transaction and ensures // the entire orphan chain is moved to the transaction pool. func TestSimpleOrphanChain(t *testing.T) { - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestSimpleOrphanChain") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestSimpleOrphanChain") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness // Create a chain of transactions rooted with the first spendable output // provided by the harness. @@ -980,12 +987,12 @@ func TestSimpleOrphanChain(t *testing.T) { // TestOrphanReject ensures that orphans are properly rejected when the allow // orphans flag is not set on ProcessTransaction. func TestOrphanReject(t *testing.T) { - harness, outputs, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestOrphanReject") + tc, outputs, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestOrphanReject") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness // Create a chain of transactions rooted with the first spendable output // provided by the harness. @@ -1035,12 +1042,12 @@ func TestOrphanReject(t *testing.T) { // it will check if we are beyond nextExpireScan, and if so, it will remove // all expired orphan transactions func TestOrphanExpiration(t *testing.T) { - harness, _, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestOrphanExpiration") + tc, _, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestOrphanExpiration") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness expiredTx, err := harness.CreateSignedTx([]spendableOutpoint{{ amount: util.Amount(5000000000), @@ -1080,12 +1087,13 @@ func TestOrphanExpiration(t *testing.T) { //TestMaxOrphanTxSize ensures that a transaction that is //bigger than MaxOrphanTxSize will get rejected func TestMaxOrphanTxSize(t *testing.T) { - harness, _, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestMaxOrphanTxSize") + tc, _, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestMaxOrphanTxSize") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness + harness.txPool.cfg.Policy.MaxOrphanTxSize = 1 tx, err := harness.CreateSignedTx([]spendableOutpoint{{ @@ -1108,12 +1116,13 @@ func TestMaxOrphanTxSize(t *testing.T) { } func TestRemoveTransaction(t *testing.T) { - harness, outputs, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestRemoveTransaction") + tc, outputs, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 2, "TestRemoveTransaction") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness + chainedTxns, err := harness.CreateTxChain(outputs[0], 5) if err != nil { t.Fatalf("unable to create transaction chain: %v", err) @@ -1130,22 +1139,36 @@ func TestRemoveTransaction(t *testing.T) { } //Checks that when removeRedeemers is false, the specified transaction is the only transaction that gets removed - harness.txPool.RemoveTransaction(chainedTxns[3], false, true) - testPoolMembership(tc, chainedTxns[3], false, false, false) - testPoolMembership(tc, chainedTxns[4], false, true, false) + tc.mineTransactions(chainedTxns[:1], 1) + testPoolMembership(tc, chainedTxns[0], false, false, false) + testPoolMembership(tc, chainedTxns[1], false, true, false) + testPoolMembership(tc, chainedTxns[2], false, true, true) + testPoolMembership(tc, chainedTxns[3], false, true, true) + testPoolMembership(tc, chainedTxns[4], false, true, true) //Checks that when removeRedeemers is true, all of the transaction that are dependent on it get removed - harness.txPool.RemoveTransaction(chainedTxns[1], true, true) - testPoolMembership(tc, chainedTxns[0], false, true, false) + err = harness.txPool.RemoveTransaction(chainedTxns[1], true, true) + if err != nil { + t.Fatalf("RemoveTransaction: %v", err) + } testPoolMembership(tc, chainedTxns[1], false, false, false) testPoolMembership(tc, chainedTxns[2], false, false, false) + testPoolMembership(tc, chainedTxns[3], false, false, false) + testPoolMembership(tc, chainedTxns[4], false, false, false) fakeWithDiffErr := "error from WithDiff" guard := monkey.Patch((*blockdag.DiffUTXOSet).WithDiff, func(_ *blockdag.DiffUTXOSet, _ *blockdag.UTXODiff) (blockdag.UTXOSet, error) { return nil, errors.New(fakeWithDiffErr) }) defer guard.Unpatch() - err = harness.txPool.RemoveTransaction(chainedTxns[0], false, false) + + tx, err := harness.CreateSignedTx(outputs[1:], 1) + _, err = harness.txPool.ProcessTransaction(tx, true, + false, 0) + if err != nil { + t.Fatalf("ProcessTransaction: %v", err) + } + err = harness.txPool.RemoveTransaction(tx, false, false) if err == nil || err.Error() != fakeWithDiffErr { t.Errorf("RemoveTransaction: expected error %v but got %v", fakeWithDiffErr, err) } @@ -1154,12 +1177,12 @@ func TestRemoveTransaction(t *testing.T) { // TestOrphanEviction ensures that exceeding the maximum number of orphans // evicts entries to make room for the new ones. func TestOrphanEviction(t *testing.T) { - harness, outputs, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestOrphanEviction") + tc, outputs, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestOrphanEviction") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness // Create a chain of transactions rooted with the first spendable output // provided by the harness that is long enough to be able to force @@ -1216,12 +1239,12 @@ func TestOrphanEviction(t *testing.T) { // Attempt to remove orphans by tag, // and ensure the state of all other orphans are unaffected. func TestRemoveOrphansByTag(t *testing.T) { - harness, _, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestRemoveOrphansByTag") + tc, _, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestRemoveOrphansByTag") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness orphanedTx1, err := harness.CreateSignedTx([]spendableOutpoint{{ amount: util.Amount(5000000000), @@ -1273,13 +1296,14 @@ func TestRemoveOrphansByTag(t *testing.T) { // redeems it and when there is not. func TestBasicOrphanRemoval(t *testing.T) { const maxOrphans = 4 - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestBasicOrphanRemoval") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestBasicOrphanRemoval") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness + harness.txPool.cfg.Policy.MaxOrphanTxs = maxOrphans - tc := &testContext{t, harness} // Create a chain of transactions rooted with the first spendable output // provided by the harness. @@ -1347,13 +1371,13 @@ func TestBasicOrphanRemoval(t *testing.T) { // from other orphans) are removed as expected. func TestOrphanChainRemoval(t *testing.T) { const maxOrphans = 10 - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestOrphanChainRemoval") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestOrphanChainRemoval") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness harness.txPool.cfg.Policy.MaxOrphanTxs = maxOrphans - tc := &testContext{t, harness} // Create a chain of transactions rooted with the first spendable output // provided by the harness. @@ -1409,13 +1433,13 @@ func TestOrphanChainRemoval(t *testing.T) { // output that is spend by another transaction entering the pool are removed. func TestMultiInputOrphanDoubleSpend(t *testing.T) { const maxOrphans = 4 - harness, outputs, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestMultiInputOrphanDoubleSpend") + tc, outputs, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestMultiInputOrphanDoubleSpend") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness harness.txPool.cfg.Policy.MaxOrphanTxs = maxOrphans - tc := &testContext{t, harness} // Create a chain of transactions rooted with the first spendable output // provided by the harness. @@ -1496,11 +1520,12 @@ func TestMultiInputOrphanDoubleSpend(t *testing.T) { // TestCheckSpend tests that CheckSpend returns the expected spends found in // the mempool. func TestCheckSpend(t *testing.T) { - harness, outputs, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestCheckSpend") + tc, outputs, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestCheckSpend") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness // The mempool is empty, so none of the spendable outputs should have a // spend there. @@ -1562,11 +1587,12 @@ func TestCheckSpend(t *testing.T) { } func TestCount(t *testing.T) { - harness, outputs, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 1, "TestCount") + tc, outputs, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 1, "TestCount") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() + harness := tc.harness if harness.txPool.Count() != 0 { t.Errorf("TestCount: txPool should be initialized with 0 transactions") } @@ -1672,12 +1698,12 @@ func TestExtractRejectCode(t *testing.T) { // TestHandleNewBlock func TestHandleNewBlock(t *testing.T) { - harness, spendableOuts, teardownFunc, err := newPoolHarness(&dagconfig.MainNetParams, 2, "TestHandleNewBlock") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, &dagconfig.MainNetParams, 2, "TestHandleNewBlock") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - tc := &testContext{t, harness} + harness := tc.harness // Create parent transaction for orphan transaction below blockTx1, err := harness.CreateSignedTx(spendableOuts[:1], 1) @@ -1711,8 +1737,10 @@ func TestHandleNewBlock(t *testing.T) { // Create block and add its transactions to UTXO set block := util.NewBlock(&dummyBlock) for i, tx := range block.Transactions() { - if !harness.txPool.mpUTXOSet.AddTx(tx.MsgTx(), 1) { - t.Fatalf("Failed to add transaction %v to UTXO set: %v", i, tx.ID()) + if isAccepted, err := harness.txPool.mpUTXOSet.AddTx(tx.MsgTx(), 1); err != nil { + t.Fatalf("Failed to add transaction (%v,%v) to UTXO set: %v", i, tx.ID(), err) + } else if !isAccepted { + t.Fatalf("AddTx unexpectedly didn't add tx %s", tx.ID()) } } @@ -1831,12 +1859,12 @@ var dummyBlock = wire.MsgBlock{ func TestTransactionGas(t *testing.T) { params := dagconfig.SimNetParams params.BlockRewardMaturity = 1 - harness, spendableOuts, teardownFunc, err := newPoolHarness(¶ms, 6, "TestTransactionGas") + tc, spendableOuts, teardownFunc, err := newPoolHarness(t, ¶ms, 6, "TestTransactionGas") if err != nil { t.Fatalf("unable to create test pool: %v", err) } defer teardownFunc() - // tc := &testContext{t, harness} + harness := tc.harness const gasLimit = 10000 subnetworkID, err := testtools.RegisterSubnetworkForTest(harness.txPool.cfg.DAG, ¶ms, gasLimit) diff --git a/mining/mining.go b/mining/mining.go index 79cdd8854..483b3ac4c 100644 --- a/mining/mining.go +++ b/mining/mining.go @@ -236,9 +236,12 @@ func CreateCoinbaseTx(params *dagconfig.Params, coinbaseScript []byte, nextBlock return nil, err } } else { - var err error scriptBuilder := txscript.NewScriptBuilder() - pkScript, err = scriptBuilder.AddOp(txscript.OpTrue).Script() + opTrueScript, err := scriptBuilder.AddOp(txscript.OpTrue).Script() + if err != nil { + return nil, err + } + pkScript, err = txscript.PayToScriptHashScript(opTrueScript) if err != nil { return nil, err } diff --git a/mining/mining_test.go b/mining/mining_test.go index 52a2633ba..16efaee1d 100644 --- a/mining/mining_test.go +++ b/mining/mining_test.go @@ -197,13 +197,19 @@ func TestNewBlockTemplate(t *testing.T) { template1CbTx := template1.Block.Transactions[0] + signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil) + if err != nil { + t.Fatalf("Error creating signature script: %s", err) + } + // tx is a regular transaction, and should not be filtered by the miner txIn := &wire.TxIn{ PreviousOutPoint: wire.OutPoint{ TxID: *template1CbTx.TxID(), Index: 0, }, - Sequence: wire.MaxTxInSequenceNum, + Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, } txOut := &wire.TxOut{ PkScript: pkScript, @@ -217,7 +223,8 @@ func TestNewBlockTemplate(t *testing.T) { TxID: *template1CbTx.TxID(), Index: 1, }, - Sequence: 0, + Sequence: 0, + SignatureScript: signatureScript, } txOut = &wire.TxOut{ PkScript: pkScript, @@ -235,7 +242,8 @@ func TestNewBlockTemplate(t *testing.T) { TxID: *template1CbTx.TxID(), Index: 2, }, - Sequence: 0, + Sequence: 0, + SignatureScript: signatureScript, } txOut = &wire.TxOut{ PkScript: pkScript, @@ -250,7 +258,8 @@ func TestNewBlockTemplate(t *testing.T) { TxID: *template1CbTx.TxID(), Index: 3, }, - Sequence: 0, + Sequence: 0, + SignatureScript: signatureScript, } txOut = &wire.TxOut{ PkScript: pkScript, @@ -264,7 +273,8 @@ func TestNewBlockTemplate(t *testing.T) { TxID: *template1CbTx.TxID(), Index: 4, }, - Sequence: 0, + Sequence: 0, + SignatureScript: signatureScript, } txOut = &wire.TxOut{ PkScript: pkScript, diff --git a/mining/policy_test.go b/mining/policy_test.go index 34a2e2f4b..7aae66f9f 100644 --- a/mining/policy_test.go +++ b/mining/policy_test.go @@ -53,7 +53,11 @@ func newUTXOSet(sourceTxns []*wire.MsgTx, sourceTxHeights []uint64) blockdag.UTX utxoSet := blockdag.NewFullUTXOSet() for i, tx := range sourceTxns { - utxoSet.AddTx(tx, sourceTxHeights[i]) + if isAccepted, err := utxoSet.AddTx(tx, sourceTxHeights[i]); err != nil { + panic(fmt.Sprintf("AddTx unexpectedly failed. Error: %s", err)) + } else if !isAccepted { + panic(fmt.Sprintf("AddTx unexpectedly didn't add tx %s", tx.TxID())) + } } return utxoSet } diff --git a/txscript/standard.go b/txscript/standard.go index e69cebbdc..2e446fc98 100644 --- a/txscript/standard.go +++ b/txscript/standard.go @@ -352,6 +352,30 @@ func PayToAddrScript(addr util.Address) ([]byte, error) { return nil, scriptError(ErrUnsupportedAddress, str) } +// PayToScriptHashScript takes a script and returns an equivalent pay-to-script-hash script +func PayToScriptHashScript(redeemScript []byte) ([]byte, error) { + redeemScriptHash := util.Hash160(redeemScript) + script, err := NewScriptBuilder(). + AddOp(OpHash160).AddData(redeemScriptHash). + AddOp(OpEqual).Script() + if err != nil { + return nil, err + } + return script, nil +} + +// PayToScriptHashSignatureScript generates a signature script that fits a pay-to-script-hash script +func PayToScriptHashSignatureScript(redeemScript []byte, signature []byte) ([]byte, error) { + redeemScriptAsData, err := NewScriptBuilder().AddData(redeemScript).Script() + if err != nil { + return nil, err + } + signatureScript := make([]byte, len(signature)+len(redeemScriptAsData)) + copy(signatureScript, signature) + copy(signatureScript[len(signature):], redeemScriptAsData) + return signatureScript, nil +} + // NullDataScript creates a provably-prunable script containing OP_RETURN // followed by the passed data. An Error with the error code ErrTooMuchNullData // will be returned if the length of the passed data exceeds MaxDataCarrierSize. diff --git a/util/testtools/testtools.go b/util/testtools/testtools.go index 0a0997d89..18fc0ad99 100644 --- a/util/testtools/testtools.go +++ b/util/testtools/testtools.go @@ -10,6 +10,7 @@ import ( "github.com/daglabs/btcd/blockdag" + "github.com/daglabs/btcd/txscript" "github.com/daglabs/btcd/util" "github.com/daglabs/btcd/util/subnetworkid" "github.com/daglabs/btcd/wire" @@ -53,12 +54,22 @@ func RegisterSubnetworkForTest(dag *blockdag.BlockDAG, params *dagconfig.Params, fundsBlockCbTx := fundsBlock.Transactions()[0].MsgTx() // Create a block with a valid subnetwork registry transaction + signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil) + if err != nil { + return nil, fmt.Errorf("Failed to build signature script: %s", err) + } txIn := &wire.TxIn{ PreviousOutPoint: *wire.NewOutPoint(fundsBlockCbTx.TxID(), 0), Sequence: wire.MaxTxInSequenceNum, + SignatureScript: signatureScript, + } + + pkScript, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript) + if err != nil { + return nil, err } txOut := &wire.TxOut{ - PkScript: blockdag.OpTrueScript, + PkScript: pkScript, Value: fundsBlockCbTx.TxOut[0].Value, } registryTx := wire.NewRegistryMsgTx(1, []*wire.TxIn{txIn}, []*wire.TxOut{txOut}, gasLimit)