From 76f23d8a9b88096156110d9f0569e12db39144ac Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Tue, 15 Oct 2019 13:03:16 +0300 Subject: [PATCH] [NOD-359] Calculate Transaction mass from previous scriptPubKeys (#429) * [NOD-359] Calculate Transaction mass from previous scriptPubKeys * [NOD-359] Add missing block errors --- blockdag/dag.go | 17 ++- blockdag/scriptval.go | 2 +- blockdag/validate.go | 128 ++++++----------------- mining/txselection.go | 6 +- mining/txselection_test.go | 2 +- server/rpc/common.go | 9 +- server/rpc/handle_get_raw_transaction.go | 92 +++++++++++----- 7 files changed, 123 insertions(+), 133 deletions(-) diff --git a/blockdag/dag.go b/blockdag/dag.go index 55d8d30c5..e647416e3 100644 --- a/blockdag/dag.go +++ b/blockdag/dag.go @@ -373,7 +373,7 @@ func (dag *BlockDAG) calcSequenceLock(node *blockNode, utxoSet UTXOSet, tx *util entry, ok := utxoSet.Get(txIn.PreviousOutpoint) if !ok { str := fmt.Sprintf("output %s referenced from "+ - "transaction %s:%d either does not exist or "+ + "transaction %s input %d either does not exist or "+ "has already been spent", txIn.PreviousOutpoint, tx.ID(), txInIndex) return sequenceLock, ruleError(ErrMissingTxOut, str) @@ -850,10 +850,25 @@ func (dag *BlockDAG) TxsAcceptedByVirtual() (MultiBlockTxsAcceptanceData, error) // This function MUST be called with the DAG read-lock held func (dag *BlockDAG) TxsAcceptedByBlockHash(blockHash *daghash.Hash) (MultiBlockTxsAcceptanceData, error) { node := dag.index.LookupNode(blockHash) + if node == nil { + return nil, fmt.Errorf("Couldn't find block %s", blockHash) + } _, txsAcceptanceData, err := dag.pastUTXO(node) return txsAcceptanceData, err } +// BlockPastUTXO retrieves the past UTXO of this block +// +// This function MUST be called with the DAG read-lock held +func (dag *BlockDAG) BlockPastUTXO(blockHash *daghash.Hash) (UTXOSet, error) { + node := dag.index.LookupNode(blockHash) + if node == nil { + return nil, fmt.Errorf("Couldn't find block %s", blockHash) + } + pastUTXO, _, err := dag.pastUTXO(node) + return pastUTXO, err +} + // applyDAGChanges does the following: // 1. Connects each of the new block's parents to the block. // 2. Adds the new block to the DAG's tips. diff --git a/blockdag/scriptval.go b/blockdag/scriptval.go index e6d092104..91031e506 100644 --- a/blockdag/scriptval.go +++ b/blockdag/scriptval.go @@ -58,7 +58,7 @@ out: if !ok { str := fmt.Sprintf("unable to find unspent "+ "output %s referenced from "+ - "transaction %s:%d", + "transaction %s input %d", txIn.PreviousOutpoint, txVI.tx.ID(), txVI.txInIndex) err := ruleError(ErrMissingTxOut, str) diff --git a/blockdag/validate.go b/blockdag/validate.go index c61a32d9f..cec9df146 100644 --- a/blockdag/validate.go +++ b/blockdag/validate.go @@ -284,89 +284,11 @@ func (dag *BlockDAG) checkProofOfWork(header *wire.BlockHeader, flags BehaviorFl return nil } -// CountSigOps returns the number of signature operations for all transaction -// input and output scripts in the provided transaction. This uses the -// quicker, but imprecise, signature operation counting mechanism from -// txscript. -func CountSigOps(tx *util.Tx) int { - msgTx := tx.MsgTx() - - // Accumulate the number of signature operations in all transaction - // inputs. - totalSigOps := 0 - for _, txIn := range msgTx.TxIn { - numSigOps := txscript.GetSigOpCount(txIn.SignatureScript) - totalSigOps += numSigOps - } - - // Accumulate the number of signature operations in all transaction - // outputs. - for _, txOut := range msgTx.TxOut { - numSigOps := txscript.GetSigOpCount(txOut.ScriptPubKey) - totalSigOps += numSigOps - } - - return totalSigOps -} - -// CountP2SHSigOps returns the number of signature operations for all input -// transactions which are of the pay-to-script-hash type. This uses the -// precise, signature operation counting mechanism from the script engine which -// requires access to the input transaction scripts. -func CountP2SHSigOps(tx *util.Tx, isCoinbase bool, utxoSet UTXOSet) (int, error) { - // Coinbase transactions have no interesting inputs. - if isCoinbase { - return 0, nil - } - - // Accumulate the number of signature operations in all transaction - // inputs. - msgTx := tx.MsgTx() - totalSigOps := 0 - for txInIndex, txIn := range msgTx.TxIn { - // Ensure the referenced input transaction is available. - entry, ok := utxoSet.Get(txIn.PreviousOutpoint) - if !ok { - str := fmt.Sprintf("output %s referenced from "+ - "transaction %s:%d either does not exist or "+ - "has already been spent", txIn.PreviousOutpoint, - tx.ID(), txInIndex) - return 0, ruleError(ErrMissingTxOut, str) - } - - // We're only interested in pay-to-script-hash types, so skip - // this input if it's not one. - scriptPubKey := entry.ScriptPubKey() - if !txscript.IsPayToScriptHash(scriptPubKey) { - continue - } - - // Count the precise number of signature operations in the - // referenced public key script. - sigScript := txIn.SignatureScript - numSigOps := txscript.GetPreciseSigOpCount(sigScript, scriptPubKey, - true) - - // We could potentially overflow the accumulator so check for - // overflow. - lastSigOps := totalSigOps - totalSigOps += numSigOps - if totalSigOps < lastSigOps { - str := fmt.Sprintf("the public key script from output "+ - "%s contains too many signature operations - "+ - "overflow", txIn.PreviousOutpoint) - return 0, ruleError(ErrTooManySigOps, str) - } - } - - return totalSigOps, nil -} - // ValidateTxMass makes sure that the given transaction's mass does not exceed // the maximum allowed limit. Currently, it is equivalent to the block mass limit. // See CalcTxMass for further details. func ValidateTxMass(tx *util.Tx, utxoSet UTXOSet) error { - txMass, err := CalcTxMass(tx, utxoSet) + txMass, err := CalcTxMassFromUTXOSet(tx, utxoSet) if err != nil { return err } @@ -388,7 +310,7 @@ func validateBlockMass(pastUTXO UTXOSet, transactions []*util.Tx) error { func CalcBlockMass(pastUTXO UTXOSet, transactions []*util.Tx) (uint64, error) { totalMass := uint64(0) for _, tx := range transactions { - txMass, err := CalcTxMass(tx, pastUTXO) + txMass, err := CalcTxMassFromUTXOSet(tx, pastUTXO) if err != nil { return 0, err } @@ -405,6 +327,29 @@ func CalcBlockMass(pastUTXO UTXOSet, transactions []*util.Tx) (uint64, error) { return totalMass, nil } +// CalcTxMassFromUTXOSet calculates the transaction mass based on the +// UTXO set in its past. +// +// See CalcTxMass for more details. +func CalcTxMassFromUTXOSet(tx *util.Tx, utxoSet UTXOSet) (uint64, error) { + if tx.IsCoinBase() { + return CalcTxMass(tx, nil), nil + } + previousScriptPubKeys := make([][]byte, len(tx.MsgTx().TxIn)) + for txInIndex, txIn := range tx.MsgTx().TxIn { + entry, ok := utxoSet.Get(txIn.PreviousOutpoint) + if !ok { + str := fmt.Sprintf("output %s referenced from "+ + "transaction %s input %d either does not exist or "+ + "has already been spent", txIn.PreviousOutpoint, + tx.ID(), txInIndex) + return 0, ruleError(ErrMissingTxOut, str) + } + previousScriptPubKeys[txInIndex] = entry.ScriptPubKey() + } + return CalcTxMass(tx, previousScriptPubKeys), nil +} + // CalcTxMass sums up and returns the "mass" of a transaction. This number // is an approximation of how many resources (CPU, RAM, etc.) it would take // to process the transaction. @@ -412,11 +357,11 @@ func CalcBlockMass(pastUTXO UTXOSet, transactions []*util.Tx) (uint64, error) { // * The transaction length in bytes // * The length of all output scripts in bytes // * The count of all input sigOps -func CalcTxMass(tx *util.Tx, utxoSet UTXOSet) (uint64, error) { +func CalcTxMass(tx *util.Tx, previousScriptPubKeys [][]byte) uint64 { txSize := tx.MsgTx().SerializeSize() if tx.IsCoinBase() { - return uint64(txSize * massPerTxByte), nil + return uint64(txSize * massPerTxByte) } scriptPubKeySize := 0 @@ -426,27 +371,16 @@ func CalcTxMass(tx *util.Tx, utxoSet UTXOSet) (uint64, error) { sigOpsCount := 0 for txInIndex, txIn := range tx.MsgTx().TxIn { - // Ensure the referenced input transaction is available. - entry, ok := utxoSet.Get(txIn.PreviousOutpoint) - if !ok { - str := fmt.Sprintf("output %s referenced from "+ - "transaction %s:%d either does not exist or "+ - "has already been spent", txIn.PreviousOutpoint, - tx.ID(), txInIndex) - return 0, ruleError(ErrMissingTxOut, str) - } - // Count the precise number of signature operations in the // referenced public key script. - scriptPubKey := entry.ScriptPubKey() sigScript := txIn.SignatureScript - isP2SH := txscript.IsPayToScriptHash(scriptPubKey) - sigOpsCount += txscript.GetPreciseSigOpCount(sigScript, scriptPubKey, isP2SH) + isP2SH := txscript.IsPayToScriptHash(previousScriptPubKeys[txInIndex]) + sigOpsCount += txscript.GetPreciseSigOpCount(sigScript, previousScriptPubKeys[txInIndex], isP2SH) } return uint64(txSize*massPerTxByte + scriptPubKeySize*massPerScriptPubKeyByte + - sigOpsCount*massPerSigOp), nil + sigOpsCount*massPerSigOp) } // checkBlockHeaderSanity performs some preliminary checks on a block header to @@ -816,7 +750,7 @@ func CheckTransactionInputsAndCalulateFee(tx *util.Tx, txBlueScore uint64, utxoS entry, ok := utxoSet.Get(txIn.PreviousOutpoint) if !ok { str := fmt.Sprintf("output %s referenced from "+ - "transaction %s:%d either does not exist or "+ + "transaction %s input %d either does not exist or "+ "has already been spent", txIn.PreviousOutpoint, tx.ID(), txInIndex) return 0, ruleError(ErrMissingTxOut, str) diff --git a/mining/txselection.go b/mining/txselection.go index deee20d29..6be946427 100644 --- a/mining/txselection.go +++ b/mining/txselection.go @@ -120,7 +120,7 @@ func (g *BlkTmplGenerator) newTxsForBlockTemplate(payToAddress util.Address, sou if err != nil { return nil, err } - coinbaseTxMass, err := blockdag.CalcTxMass(coinbaseTx, g.dag.UTXOSet()) + coinbaseTxMass, err := blockdag.CalcTxMassFromUTXOSet(coinbaseTx, g.dag.UTXOSet()) if err != nil { return nil, err } @@ -157,7 +157,7 @@ func (g *BlkTmplGenerator) collectCandidatesTxs(sourceTxs []*TxDesc) []*candidat continue } - txMass, err := blockdag.CalcTxMass(tx, g.dag.UTXOSet()) + txMass, err := blockdag.CalcTxMassFromUTXOSet(tx, g.dag.UTXOSet()) if err != nil { log.Warnf("Skipping tx %s due to error in "+ "CalcTxMass: %s", tx.ID(), err) @@ -204,7 +204,7 @@ func (g *BlkTmplGenerator) collectCandidatesTxs(sourceTxs []*TxDesc) []*candidat // The higher the number the more likely it is that the transaction will be // included in the block. func (g *BlkTmplGenerator) calcTxValue(tx *util.Tx, fee uint64) (float64, error) { - mass, err := blockdag.CalcTxMass(tx, g.dag.UTXOSet()) + mass, err := blockdag.CalcTxMassFromUTXOSet(tx, g.dag.UTXOSet()) if err != nil { return 0, err } diff --git a/mining/txselection_test.go b/mining/txselection_test.go index 506e79c10..d7c4fd6cc 100644 --- a/mining/txselection_test.go +++ b/mining/txselection_test.go @@ -442,7 +442,7 @@ func TestSelectTxs(t *testing.T) { // Force the mass to be as defined in the definition. // We use the first payload byte to resolve which definition to use. - massPatch := monkey.Patch(blockdag.CalcTxMass, func(tx *util.Tx, _ blockdag.UTXOSet) (uint64, error) { + massPatch := monkey.Patch(blockdag.CalcTxMassFromUTXOSet, func(tx *util.Tx, _ blockdag.UTXOSet) (uint64, error) { if tx.IsCoinBase() { return 0, nil } diff --git a/server/rpc/common.go b/server/rpc/common.go index 0a463489b..c80b73998 100644 --- a/server/rpc/common.go +++ b/server/rpc/common.go @@ -240,7 +240,12 @@ func buildGetBlockVerboseResult(s *Server, block *util.Block, isVerboseTx bool) return nil, internalRPCError(err.Error(), context) } - blockMass, err := blockdag.CalcBlockMass(s.cfg.DAG.UTXOSet(), block.Transactions()) + pastUTXO, err := s.cfg.DAG.BlockPastUTXO(block.Hash()) + if err != nil { + context := "Could not get block past utxo" + return nil, internalRPCError(err.Error(), context) + } + blockMass, err := blockdag.CalcBlockMass(pastUTXO, block.Transactions()) if err != nil { context := "Could not get block mass" return nil, internalRPCError(err.Error(), context) @@ -294,7 +299,7 @@ func buildGetBlockVerboseResult(s *Server, block *util.Block, isVerboseTx bool) } confirmations = &txConfirmations } - txMass, err := blockdag.CalcTxMass(tx, s.cfg.DAG.UTXOSet()) + txMass, err := blockdag.CalcTxMassFromUTXOSet(tx, pastUTXO) if err != nil { return nil, err } diff --git a/server/rpc/handle_get_raw_transaction.go b/server/rpc/handle_get_raw_transaction.go index 4cddf3fb4..4ef885b1c 100644 --- a/server/rpc/handle_get_raw_transaction.go +++ b/server/rpc/handle_get_raw_transaction.go @@ -28,8 +28,9 @@ func handleGetRawTransaction(s *Server, cmd interface{}, closeChan <-chan struct // Try to fetch the transaction from the memory pool and if that fails, // try the block database. - var msgTx *wire.MsgTx + var tx *util.Tx var blkHash *daghash.Hash + var txMass uint64 isInMempool := false mempoolTx, err := s.cfg.TxMemPool.FetchTransaction(txID) if err != nil { @@ -42,25 +43,9 @@ func handleGetRawTransaction(s *Server, cmd interface{}, closeChan <-chan struct } } - // Look up the location of the transaction. - blockRegion, err := s.cfg.TxIndex.TxFirstBlockRegion(txID) + txBytes, txBlockHash, err := fetchTxBytesAndBlockHashFromTxIndex(s, txID) if err != nil { - context := "Failed to retrieve transaction location" - return nil, internalRPCError(err.Error(), context) - } - if blockRegion == nil { - return nil, rpcNoTxInfoError(txID) - } - - // Load the raw transaction bytes from the database. - var txBytes []byte - err = s.cfg.DB.View(func(dbTx database.Tx) error { - var err error - txBytes, err = dbTx.FetchBlockRegion(blockRegion) - return err - }) - if err != nil { - return nil, rpcNoTxInfoError(txID) + return nil, err } // When the verbose flag isn't set, simply return the serialized @@ -71,7 +56,7 @@ func handleGetRawTransaction(s *Server, cmd interface{}, closeChan <-chan struct } // Grab the block hash. - blkHash = blockRegion.Hash + blkHash = txBlockHash // Deserialize the transaction var mtx wire.MsgTx @@ -80,7 +65,12 @@ func handleGetRawTransaction(s *Server, cmd interface{}, closeChan <-chan struct context := "Failed to deserialize transaction" return nil, internalRPCError(err.Error(), context) } - msgTx = &mtx + + tx = util.NewTx(&mtx) + txMass, err = calcTxMassFromTxIndex(s, tx) + if err != nil { + return nil, err + } } else { // When the verbose flag isn't set, simply return the // network-serialized transaction as a hex-encoded string. @@ -97,7 +87,13 @@ func handleGetRawTransaction(s *Server, cmd interface{}, closeChan <-chan struct return mtxHex, nil } - msgTx = mempoolTx.MsgTx() + var err error + txMass, err = blockdag.CalcTxMassFromUTXOSet(mempoolTx, s.cfg.DAG.UTXOSet()) + if err != nil { + return nil, err + } + + tx = mempoolTx isInMempool = true } @@ -116,18 +112,58 @@ func handleGetRawTransaction(s *Server, cmd interface{}, closeChan <-chan struct blkHashStr = blkHash.String() } - confirmations, err := txConfirmations(s, msgTx.TxID()) + confirmations, err := txConfirmations(s, tx.ID()) if err != nil { return nil, err } - txMass, err := blockdag.CalcTxMass(util.NewTx(msgTx), s.cfg.DAG.UTXOSet()) - if err != nil { - return nil, err - } - rawTxn, err := createTxRawResult(s.cfg.DAGParams, msgTx, txID.String(), + rawTxn, err := createTxRawResult(s.cfg.DAGParams, tx.MsgTx(), txID.String(), blkHeader, blkHashStr, nil, &confirmations, isInMempool, txMass) if err != nil { return nil, err } return *rawTxn, nil } + +func calcTxMassFromTxIndex(s *Server, tx *util.Tx) (uint64, error) { + if tx.IsCoinBase() { + return blockdag.CalcTxMass(tx, nil), nil + } + previousScriptPubKeys := make([][]byte, len(tx.MsgTx().TxIn)) + for i, txIn := range tx.MsgTx().TxIn { + txBytes, _, err := fetchTxBytesAndBlockHashFromTxIndex(s, &txIn.PreviousOutpoint.TxID) + if err != nil { + return 0, err + } + var msgTx wire.MsgTx + err = msgTx.Deserialize(bytes.NewReader(txBytes)) + if err != nil { + context := "Failed to deserialize transaction" + return 0, internalRPCError(err.Error(), context) + } + previousScriptPubKeys[i] = msgTx.TxOut[txIn.PreviousOutpoint.Index].ScriptPubKey + } + return blockdag.CalcTxMass(tx, previousScriptPubKeys), nil +} + +func fetchTxBytesAndBlockHashFromTxIndex(s *Server, txID *daghash.TxID) ([]byte, *daghash.Hash, error) { + blockRegion, err := s.cfg.TxIndex.TxFirstBlockRegion(txID) + if err != nil { + context := "Failed to retrieve transaction location" + return nil, nil, internalRPCError(err.Error(), context) + } + if blockRegion == nil { + return nil, nil, rpcNoTxInfoError(txID) + } + + // Load the raw transaction bytes from the database. + var txBytes []byte + err = s.cfg.DB.View(func(dbTx database.Tx) error { + var err error + txBytes, err = dbTx.FetchBlockRegion(blockRegion) + return err + }) + if err != nil { + return nil, nil, rpcNoTxInfoError(txID) + } + return txBytes, blockRegion.Hash, nil +}