Block template cache (#1994)

* minor text fix

* Implement a block template cache with template modification/reuse mechanism

* Fix compilation error

* Address review comments

* Added a through TestModifyBlockTemplate test

* Update header timestamp if possible

* Avoid copying the transactions when only the header changed

* go fmt

app/rpc/manager.go

@@ -68,7 +68,7 @@ func (m *Manager) NotifyBlockAddedToDAG(block *externalapi.DomainBlock, virtualC
 
 // NotifyVirtualChange notifies the manager that the virtual block has been changed.
 func (m *Manager) NotifyVirtualChange(virtualChangeSet *externalapi.VirtualChangeSet) error {
-	onEnd := logger.LogAndMeasureExecutionTime(log, "RPCManager.NotifyBlockAddedToDAG")
+	onEnd := logger.LogAndMeasureExecutionTime(log, "RPCManager.NotifyVirtualChange")
 	defer onEnd()
 
 	if m.context.Config.UTXOIndex {

domain/consensus/consensus.go

@@ -150,6 +150,18 @@ func (s *consensus) BuildBlock(coinbaseData *externalapi.DomainCoinbaseData,
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
+	block, _, err := s.blockBuilder.BuildBlock(coinbaseData, transactions)
+	return block, err
+}
+
+// BuildBlockWithTemplateMetadata builds a block over the current state, with the transactions
+// selected by the given transactionSelector plus metadata information related to coinbase rewards
+func (s *consensus) BuildBlockWithTemplateMetadata(coinbaseData *externalapi.DomainCoinbaseData,
+	transactions []*externalapi.DomainTransaction) (block *externalapi.DomainBlock, coinbaseHasRedReward bool, err error) {
+
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
 	return s.blockBuilder.BuildBlock(coinbaseData, transactions)
 }
 

domain/consensus/model/externalapi/block.go

@@ -80,4 +80,5 @@ type MutableBlockHeader interface {
 	ToImmutable() BlockHeader
 	SetNonce(nonce uint64)
 	SetTimeInMilliseconds(timeInMilliseconds int64)
+	SetHashMerkleRoot(hashMerkleRoot *DomainHash)
 }

domain/consensus/model/externalapi/blocktemplate.go

@@ -0,0 +1,17 @@
+package externalapi
+
+// DomainBlockTemplate contains a Block plus metadata related to its generation
+type DomainBlockTemplate struct {
+	Block                *DomainBlock
+	CoinbaseData         *DomainCoinbaseData
+	CoinbaseHasRedReward bool
+}
+
+// Clone returns a clone of DomainBlockTemplate
+func (bt *DomainBlockTemplate) Clone() *DomainBlockTemplate {
+	return &DomainBlockTemplate{
+		Block:                bt.Block.Clone(),
+		CoinbaseData:         bt.CoinbaseData.Clone(),
+		CoinbaseHasRedReward: bt.CoinbaseHasRedReward,
+	}
+}

domain/consensus/model/externalapi/coinbase.go

@@ -1,5 +1,7 @@
 package externalapi
 
+import "bytes"
+
 // DomainCoinbaseData contains data by which a coinbase transaction
 // is built
 type DomainCoinbaseData struct {
@@ -21,3 +23,16 @@ func (dcd *DomainCoinbaseData) Clone() *DomainCoinbaseData {
 		ExtraData:       extraDataClone,
 	}
 }
+
+// Equal returns whether dcd equals to other
+func (dcd *DomainCoinbaseData) Equal(other *DomainCoinbaseData) bool {
+	if dcd == nil || other == nil {
+		return dcd == other
+	}
+
+	if !bytes.Equal(dcd.ExtraData, other.ExtraData) {
+		return false
+	}
+
+	return dcd.ScriptPublicKey.Equal(other.ScriptPublicKey)
+}

domain/consensus/model/externalapi/consensus.go

@@ -4,6 +4,7 @@ package externalapi
 type Consensus interface {
 	Init(skipAddingGenesis bool) error
 	BuildBlock(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (*DomainBlock, error)
+	BuildBlockWithTemplateMetadata(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (block *DomainBlock, coinbaseHasRedReward bool, err error)
 	ValidateAndInsertBlock(block *DomainBlock, shouldValidateAgainstUTXO bool) (*VirtualChangeSet, error)
 	ValidateAndInsertBlockWithTrustedData(block *BlockWithTrustedData, validateUTXO bool) (*VirtualChangeSet, error)
 	ValidateTransactionAndPopulateWithConsensusData(transaction *DomainTransaction) error

domain/consensus/model/externalapi/transaction.go

@@ -229,6 +229,19 @@ type ScriptPublicKey struct {
 	Version uint16
 }
 
+// Equal returns whether spk equals to other
+func (spk *ScriptPublicKey) Equal(other *ScriptPublicKey) bool {
+	if spk == nil || other == nil {
+		return spk == other
+	}
+
+	if spk.Version != other.Version {
+		return false
+	}
+
+	return bytes.Equal(spk.Script, other.Script)
+}
+
 // DomainTransactionOutput represents a Kaspad transaction output
 type DomainTransactionOutput struct {
 	Value           uint64
@@ -249,11 +262,7 @@ func (output *DomainTransactionOutput) Equal(other *DomainTransactionOutput) boo
 		return false
 	}
 
-	if !bytes.Equal(output.ScriptPublicKey.Script, other.ScriptPublicKey.Script) {
+	return output.ScriptPublicKey.Equal(other.ScriptPublicKey)
-		return false
-	}
-
-	return true
 }
 
 // Clone returns a clone of DomainTransactionOutput

domain/consensus/model/interface_processes_blockbuilder.go

@@ -4,5 +4,6 @@ import "github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 
 // BlockBuilder is responsible for creating blocks from the current state
 type BlockBuilder interface {
-	BuildBlock(coinbaseData *externalapi.DomainCoinbaseData, transactions []*externalapi.DomainTransaction) (*externalapi.DomainBlock, error)
+	BuildBlock(coinbaseData *externalapi.DomainCoinbaseData,
+		transactions []*externalapi.DomainTransaction) (block *externalapi.DomainBlock, coinbaseHasRedReward bool, err error)
 }

domain/consensus/model/interface_processes_coinbasemanager.go

@@ -6,7 +6,7 @@ import "github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 // coinbase transactions
 type CoinbaseManager interface {
 	ExpectedCoinbaseTransaction(stagingArea *StagingArea, blockHash *externalapi.DomainHash,
-		coinbaseData *externalapi.DomainCoinbaseData) (*externalapi.DomainTransaction, error)
+		coinbaseData *externalapi.DomainCoinbaseData) (expectedTransaction *externalapi.DomainTransaction, hasRedReward bool, err error)
 	CalcBlockSubsidy(stagingArea *StagingArea, blockHash *externalapi.DomainHash) (uint64, error)
 	ExtractCoinbaseDataBlueScoreAndSubsidy(coinbaseTx *externalapi.DomainTransaction) (blueScore uint64, coinbaseData *externalapi.DomainCoinbaseData, subsidy uint64, err error)
 }

domain/consensus/processes/blockbuilder/block_builder.go

@@ -85,7 +85,7 @@ func New(
 // BuildBlock builds a block over the current state, with the given
 // coinbaseData and the given transactions
 func (bb *blockBuilder) BuildBlock(coinbaseData *externalapi.DomainCoinbaseData,
-	transactions []*externalapi.DomainTransaction) (*externalapi.DomainBlock, error) {
+	transactions []*externalapi.DomainTransaction) (block *externalapi.DomainBlock, coinbaseHasRedReward bool, err error) {
 
 	onEnd := logger.LogAndMeasureExecutionTime(log, "BuildBlock")
 	defer onEnd()
@@ -96,32 +96,32 @@ func (bb *blockBuilder) BuildBlock(coinbaseData *externalapi.DomainCoinbaseData,
 }
 
 func (bb *blockBuilder) buildBlock(stagingArea *model.StagingArea, coinbaseData *externalapi.DomainCoinbaseData,
-	transactions []*externalapi.DomainTransaction) (*externalapi.DomainBlock, error) {
+	transactions []*externalapi.DomainTransaction) (block *externalapi.DomainBlock, coinbaseHasRedReward bool, err error) {
 
-	err := bb.validateTransactions(stagingArea, transactions)
+	err = bb.validateTransactions(stagingArea, transactions)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	newBlockPruningPoint, err := bb.newBlockPruningPoint(stagingArea, model.VirtualBlockHash)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
-	coinbase, err := bb.newBlockCoinbaseTransaction(stagingArea, coinbaseData)
+	coinbase, coinbaseHasRedReward, err := bb.newBlockCoinbaseTransaction(stagingArea, coinbaseData)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	transactionsWithCoinbase := append([]*externalapi.DomainTransaction{coinbase}, transactions...)
 
 	header, err := bb.buildHeader(stagingArea, transactionsWithCoinbase, newBlockPruningPoint)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	return &externalapi.DomainBlock{
 		Header:       header,
 		Transactions: transactionsWithCoinbase,
-	}, nil
+	}, coinbaseHasRedReward, nil
 }
 
 func (bb *blockBuilder) validateTransactions(stagingArea *model.StagingArea,
@@ -180,7 +180,7 @@ func (bb *blockBuilder) validateTransaction(
 }
 
 func (bb *blockBuilder) newBlockCoinbaseTransaction(stagingArea *model.StagingArea,
-	coinbaseData *externalapi.DomainCoinbaseData) (*externalapi.DomainTransaction, error) {
+	coinbaseData *externalapi.DomainCoinbaseData) (expectedTransaction *externalapi.DomainTransaction, hasRedReward bool, err error) {
 
 	return bb.coinbaseManager.ExpectedCoinbaseTransaction(stagingArea, model.VirtualBlockHash, coinbaseData)
 }

domain/consensus/processes/blockbuilder/block_builder_test.go

@@ -113,7 +113,7 @@ func TestBuildBlockErrorCases(t *testing.T) {
 		}
 
 		for _, test := range tests {
-			_, err = testConsensus.BlockBuilder().BuildBlock(test.coinbaseData, test.transactions)
+			_, _, err = testConsensus.BlockBuilder().BuildBlock(test.coinbaseData, test.transactions)
 			if err == nil {
 				t.Errorf("%s: No error from BuildBlock", test.name)
 				return

domain/consensus/processes/blockbuilder/test_block_builder.go

@@ -200,7 +200,7 @@ func (bb *testBlockBuilder) buildBlockWithParents(stagingArea *model.StagingArea
 
 	bb.acceptanceDataStore.Stage(stagingArea, tempBlockHash, acceptanceData)
 
-	coinbase, err := bb.coinbaseManager.ExpectedCoinbaseTransaction(stagingArea, tempBlockHash, coinbaseData)
+	coinbase, _, err := bb.coinbaseManager.ExpectedCoinbaseTransaction(stagingArea, tempBlockHash, coinbaseData)
 	if err != nil {
 		return nil, nil, err
 	}

domain/consensus/processes/coinbasemanager/coinbasemanager.go

@@ -31,29 +31,29 @@ type coinbaseManager struct {
 }
 
 func (c *coinbaseManager) ExpectedCoinbaseTransaction(stagingArea *model.StagingArea, blockHash *externalapi.DomainHash,
-	coinbaseData *externalapi.DomainCoinbaseData) (*externalapi.DomainTransaction, error) {
+	coinbaseData *externalapi.DomainCoinbaseData) (expectedTransaction *externalapi.DomainTransaction, hasRedReward bool, err error) {
 
 	ghostdagData, err := c.ghostdagDataStore.Get(c.databaseContext, stagingArea, blockHash, true)
 	if !database.IsNotFoundError(err) && err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	// If there's ghostdag data with trusted data we prefer it because we need the original merge set non-pruned merge set.
 	if database.IsNotFoundError(err) {
 		ghostdagData, err = c.ghostdagDataStore.Get(c.databaseContext, stagingArea, blockHash, false)
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 	}
 
 	acceptanceData, err := c.acceptanceDataStore.Get(c.databaseContext, stagingArea, blockHash)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	daaAddedBlocksSet, err := c.daaAddedBlocksSet(stagingArea, blockHash)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	txOuts := make([]*externalapi.DomainTransactionOutput, 0, len(ghostdagData.MergeSetBlues()))
@@ -61,7 +61,7 @@ func (c *coinbaseManager) ExpectedCoinbaseTransaction(stagingArea *model.Staging
 	for _, blue := range ghostdagData.MergeSetBlues() {
 		txOut, hasReward, err := c.coinbaseOutputForBlueBlock(stagingArea, blue, acceptanceDataMap[*blue], daaAddedBlocksSet)
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 
 		if hasReward {
@@ -69,24 +69,24 @@ func (c *coinbaseManager) ExpectedCoinbaseTransaction(stagingArea *model.Staging
 		}
 	}
 
-	txOut, hasReward, err := c.coinbaseOutputForRewardFromRedBlocks(
+	txOut, hasRedReward, err := c.coinbaseOutputForRewardFromRedBlocks(
 		stagingArea, ghostdagData, acceptanceData, daaAddedBlocksSet, coinbaseData)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
-	if hasReward {
+	if hasRedReward {
 		txOuts = append(txOuts, txOut)
 	}
 
 	subsidy, err := c.CalcBlockSubsidy(stagingArea, blockHash)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	payload, err := c.serializeCoinbasePayload(ghostdagData.BlueScore(), coinbaseData, subsidy)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	return &externalapi.DomainTransaction{
@@ -97,7 +97,7 @@ func (c *coinbaseManager) ExpectedCoinbaseTransaction(stagingArea *model.Staging
 		SubnetworkID: subnetworks.SubnetworkIDCoinbase,
 		Gas:          0,
 		Payload:      payload,
-	}, nil
+	}, hasRedReward, nil
 }
 
 func (c *coinbaseManager) daaAddedBlocksSet(stagingArea *model.StagingArea, blockHash *externalapi.DomainHash) (

domain/consensus/processes/coinbasemanager/payload.go

@@ -36,6 +36,30 @@ func (c *coinbaseManager) serializeCoinbasePayload(blueScore uint64,
 	return payload, nil
 }
 
+// ModifyCoinbasePayload modifies the coinbase payload based on the provided scriptPubKey and extra data.
+func ModifyCoinbasePayload(payload []byte, coinbaseData *externalapi.DomainCoinbaseData, coinbasePayloadScriptPublicKeyMaxLength uint8) ([]byte, error) {
+
+	scriptLengthOfScriptPubKey := len(coinbaseData.ScriptPublicKey.Script)
+	if scriptLengthOfScriptPubKey > int(coinbasePayloadScriptPublicKeyMaxLength) {
+		return nil, errors.Wrapf(ruleerrors.ErrBadCoinbasePayloadLen, "coinbase's payload script public key is "+
+			"longer than the max allowed length of %d", coinbasePayloadScriptPublicKeyMaxLength)
+	}
+
+	newPayloadLen := uint64Len + lengthOfVersionScriptPubKey + lengthOfScriptPubKeyLength + scriptLengthOfScriptPubKey + len(coinbaseData.ExtraData) + lengthOfSubsidy
+	if len(payload) != newPayloadLen {
+		newPayload := make([]byte, newPayloadLen)
+		copy(newPayload, payload[:uint64Len+lengthOfSubsidy])
+		payload = newPayload
+	}
+
+	payload[uint64Len+lengthOfSubsidy] = uint8(coinbaseData.ScriptPublicKey.Version)
+	payload[uint64Len+lengthOfSubsidy+lengthOfVersionScriptPubKey] = uint8(len(coinbaseData.ScriptPublicKey.Script))
+	copy(payload[uint64Len+lengthOfSubsidy+lengthOfVersionScriptPubKey+lengthOfScriptPubKeyLength:], coinbaseData.ScriptPublicKey.Script)
+	copy(payload[uint64Len+lengthOfSubsidy+lengthOfVersionScriptPubKey+lengthOfScriptPubKeyLength+scriptLengthOfScriptPubKey:], coinbaseData.ExtraData)
+
+	return payload, nil
+}
+
 // ExtractCoinbaseDataBlueScoreAndSubsidy deserializes the coinbase payload to its component (scriptPubKey, extra data, and subsidy).
 func (c *coinbaseManager) ExtractCoinbaseDataBlueScoreAndSubsidy(coinbaseTx *externalapi.DomainTransaction) (
 	blueScore uint64, coinbaseData *externalapi.DomainCoinbaseData, subsidy uint64, err error) {

domain/consensus/processes/consensusstatemanager/verify_and_build_utxo.go

@@ -166,7 +166,7 @@ func (csm *consensusStateManager) validateCoinbaseTransaction(stagingArea *model
 	}
 
 	log.Tracef("Calculating the expected coinbase transaction for the given coinbase data and block %s", blockHash)
-	expectedCoinbaseTransaction, err :=
+	expectedCoinbaseTransaction, _, err :=
 		csm.coinbaseManager.ExpectedCoinbaseTransaction(stagingArea, blockHash, coinbaseData)
 	if err != nil {
 		return err

domain/consensus/utils/blockheader/blockheader.go

@@ -54,6 +54,11 @@ func (bh *blockHeader) SetTimeInMilliseconds(timeInMilliseconds int64) {
 	bh.timeInMilliseconds = timeInMilliseconds
 }
 
+func (bh *blockHeader) SetHashMerkleRoot(hashMerkleRoot *externalapi.DomainHash) {
+	bh.isBlockLevelCached = false
+	bh.hashMerkleRoot = hashMerkleRoot
+}
+
 func (bh *blockHeader) Version() uint16 {
 	return bh.version
 }

domain/consensus/utils/utxo/utxo_entry.go

@@ -1,7 +1,6 @@
 package utxo
 
 import (
-	"bytes"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 )
 
@@ -60,11 +59,7 @@ func (u *utxoEntry) Equal(other externalapi.UTXOEntry) bool {
 		return false
 	}
 
-	if !bytes.Equal(u.ScriptPublicKey().Script, other.ScriptPublicKey().Script) {
+	if !u.ScriptPublicKey().Equal(other.ScriptPublicKey()) {
-		return false
-	}
-
-	if u.ScriptPublicKey().Version != other.ScriptPublicKey().Version {
 		return false
 	}
 

domain/miningmanager/blocktemplatebuilder/blocktemplatebuilder.go

@@ -1,7 +1,11 @@
 package blocktemplatebuilder
 
 import (
+	"github.com/kaspanet/kaspad/domain/consensus/processes/coinbasemanager"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/merkle"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/transactionhelper"
 	"github.com/kaspanet/kaspad/domain/consensusreference"
+	"github.com/kaspanet/kaspad/util/mstime"
 	"math"
 	"sort"
 
@@ -31,19 +35,24 @@ type blockTemplateBuilder struct {
 	consensusReference consensusreference.ConsensusReference
 	mempool            miningmanagerapi.Mempool
 	policy             policy
+
+	coinbasePayloadScriptPublicKeyMaxLength uint8
 }
 
 // New creates a new blockTemplateBuilder
-func New(consensusReference consensusreference.ConsensusReference, mempool miningmanagerapi.Mempool, blockMaxMass uint64) miningmanagerapi.BlockTemplateBuilder {
+func New(consensusReference consensusreference.ConsensusReference, mempool miningmanagerapi.Mempool,
+	blockMaxMass uint64, coinbasePayloadScriptPublicKeyMaxLength uint8) miningmanagerapi.BlockTemplateBuilder {
 	return &blockTemplateBuilder{
 		consensusReference: consensusReference,
 		mempool:            mempool,
 		policy:             policy{BlockMaxMass: blockMaxMass},
+
+		coinbasePayloadScriptPublicKeyMaxLength: coinbasePayloadScriptPublicKeyMaxLength,
 	}
 }
 
-// GetBlockTemplate creates a block template for a miner to consume
+// BuildBlockTemplate creates a block template for a miner to consume
-// GetBlockTemplate returns a new block template that is ready to be solved
+// BuildBlockTemplate returns a new block template that is ready to be solved
 // using the transactions from the passed transaction source pool and a coinbase
 // that either pays to the passed address if it is not nil, or a coinbase that
 // is redeemable by anyone if the passed address is nil. The nil address
@@ -106,7 +115,9 @@ func New(consensusReference consensusreference.ConsensusReference, mempool minin
 //  |  <= policy.BlockMinSize)          |   |
 //   -----------------------------------  --
 
-func (btb *blockTemplateBuilder) GetBlockTemplate(coinbaseData *consensusexternalapi.DomainCoinbaseData) (*consensusexternalapi.DomainBlock, error) {
+func (btb *blockTemplateBuilder) BuildBlockTemplate(
+	coinbaseData *consensusexternalapi.DomainCoinbaseData) (*consensusexternalapi.DomainBlockTemplate, error) {
+
 	mempoolTransactions := btb.mempool.BlockCandidateTransactions()
 	candidateTxs := make([]*candidateTx, 0, len(mempoolTransactions))
 	for _, tx := range mempoolTransactions {
@@ -131,11 +142,11 @@ func (btb *blockTemplateBuilder) GetBlockTemplate(coinbaseData *consensusexterna
 		len(candidateTxs))
 
 	blockTxs := btb.selectTransactions(candidateTxs)
-	blk, err := btb.consensusReference.Consensus().BuildBlock(coinbaseData, blockTxs.selectedTxs)
+	blk, coinbaseHasRedReward, err := btb.consensusReference.Consensus().BuildBlockWithTemplateMetadata(coinbaseData, blockTxs.selectedTxs)
 
 	invalidTxsErr := ruleerrors.ErrInvalidTransactionsInNewBlock{}
 	if errors.As(err, &invalidTxsErr) {
-		log.Criticalf("consensusReference.Consensus().BuildBlock returned invalid txs in GetBlockTemplate")
+		log.Criticalf("consensusReference.Consensus().BuildBlock returned invalid txs in BuildBlockTemplate")
 		invalidTxs := make([]*consensusexternalapi.DomainTransaction, 0, len(invalidTxsErr.InvalidTransactions))
 		for _, tx := range invalidTxsErr.InvalidTransactions {
 			invalidTxs = append(invalidTxs, tx.Transaction)
@@ -148,7 +159,7 @@ func (btb *blockTemplateBuilder) GetBlockTemplate(coinbaseData *consensusexterna
 			log.Criticalf("Error from mempool.RemoveTransactions: %+v", err)
 		}
 		// We can call this recursively without worry because this should almost never happen
-		return btb.GetBlockTemplate(coinbaseData)
+		return btb.BuildBlockTemplate(coinbaseData)
 	}
 
 	if err != nil {
@@ -158,7 +169,44 @@ func (btb *blockTemplateBuilder) GetBlockTemplate(coinbaseData *consensusexterna
 	log.Debugf("Created new block template (%d transactions, %d in fees, %d mass, target difficulty %064x)",
 		len(blk.Transactions), blockTxs.totalFees, blockTxs.totalMass, difficulty.CompactToBig(blk.Header.Bits()))
 
-	return blk, nil
+	return &consensusexternalapi.DomainBlockTemplate{
+		Block:                blk,
+		CoinbaseData:         coinbaseData,
+		CoinbaseHasRedReward: coinbaseHasRedReward,
+	}, nil
+}
+
+// ModifyBlockTemplate modifies an existing block template to the requested coinbase data and updates the timestamp
+func (btb *blockTemplateBuilder) ModifyBlockTemplate(newCoinbaseData *consensusexternalapi.DomainCoinbaseData,
+	blockTemplateToModify *consensusexternalapi.DomainBlockTemplate) (*consensusexternalapi.DomainBlockTemplate, error) {
+
+	// The first transaction is always the coinbase transaction
+	coinbaseTx := blockTemplateToModify.Block.Transactions[transactionhelper.CoinbaseTransactionIndex]
+	newPayload, err := coinbasemanager.ModifyCoinbasePayload(coinbaseTx.Payload, newCoinbaseData, btb.coinbasePayloadScriptPublicKeyMaxLength)
+	if err != nil {
+		return nil, err
+	}
+	coinbaseTx.Payload = newPayload
+	if blockTemplateToModify.CoinbaseHasRedReward {
+		// The last output is always the coinbase red blocks reward
+		coinbaseTx.Outputs[len(coinbaseTx.Outputs)-1].ScriptPublicKey = newCoinbaseData.ScriptPublicKey
+	}
+	// Update the hash merkle root according to the modified transactions
+	mutableHeader := blockTemplateToModify.Block.Header.ToMutable()
+	// TODO: can be optimized to O(log(#transactions)) by caching the whole merkle tree in BlockTemplate and changing only the relevant path
+	mutableHeader.SetHashMerkleRoot(merkle.CalculateHashMerkleRoot(blockTemplateToModify.Block.Transactions))
+
+	newTimestamp := mstime.Now().UnixMilliseconds()
+	if newTimestamp >= mutableHeader.TimeInMilliseconds() {
+		// Only if new time stamp is later than current, update the header. Otherwise,
+		// we keep the previous time as built by internal consensus median time logic
+		mutableHeader.SetTimeInMilliseconds(newTimestamp)
+	}
+
+	blockTemplateToModify.Block.Header = mutableHeader.ToImmutable()
+	blockTemplateToModify.CoinbaseData = newCoinbaseData
+
+	return blockTemplateToModify, nil
 }
 
 // calcTxValue calculates a value to be used in transaction selection.

domain/miningmanager/factory.go

@@ -5,6 +5,8 @@ import (
 	"github.com/kaspanet/kaspad/domain/dagconfig"
 	"github.com/kaspanet/kaspad/domain/miningmanager/blocktemplatebuilder"
 	mempoolpkg "github.com/kaspanet/kaspad/domain/miningmanager/mempool"
+	"sync"
+	"time"
 )
 
 // Factory instantiates new mining managers
@@ -19,11 +21,14 @@ func (f *factory) NewMiningManager(consensusReference consensusreference.Consens
 	mempoolConfig *mempoolpkg.Config) MiningManager {
 
 	mempool := mempoolpkg.New(mempoolConfig, consensusReference)
-	blockTemplateBuilder := blocktemplatebuilder.New(consensusReference, mempool, params.MaxBlockMass)
+	blockTemplateBuilder := blocktemplatebuilder.New(consensusReference, mempool, params.MaxBlockMass, params.CoinbasePayloadScriptPublicKeyMaxLength)
 
 	return &miningManager{
+		consensusReference:   consensusReference,
 		mempool:              mempool,
 		blockTemplateBuilder: blockTemplateBuilder,
+		cachingTime:          time.Now(),
+		cacheLock:            &sync.Mutex{},
 	}
 }
 

domain/miningmanager/miningmanager.go

@@ -2,13 +2,18 @@ package miningmanager
 
 import (
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+	"github.com/kaspanet/kaspad/domain/consensusreference"
 	miningmanagermodel "github.com/kaspanet/kaspad/domain/miningmanager/model"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"sync"
+	"time"
 )
 
 // MiningManager creates block templates for mining as well as maintaining
 // known transactions that have no yet been added to any block
 type MiningManager interface {
 	GetBlockTemplate(coinbaseData *externalapi.DomainCoinbaseData) (*externalapi.DomainBlock, error)
+	GetBlockTemplateBuilder() miningmanagermodel.BlockTemplateBuilder
 	GetTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool)
 	AllTransactions() []*externalapi.DomainTransaction
 	TransactionCount() int
@@ -19,16 +24,88 @@ type MiningManager interface {
 }
 
 type miningManager struct {
+	consensusReference   consensusreference.ConsensusReference
 	mempool              miningmanagermodel.Mempool
 	blockTemplateBuilder miningmanagermodel.BlockTemplateBuilder
+	cachedBlockTemplate  *externalapi.DomainBlockTemplate
+	cachingTime          time.Time
+	cacheLock            *sync.Mutex
 }
 
-// GetBlockTemplate creates a block template for a miner to consume
+// GetBlockTemplate obtains a block template for a miner to consume
 func (mm *miningManager) GetBlockTemplate(coinbaseData *externalapi.DomainCoinbaseData) (*externalapi.DomainBlock, error) {
-	return mm.blockTemplateBuilder.GetBlockTemplate(coinbaseData)
+	immutableCachedTemplate := mm.getImmutableCachedTemplate()
+	// We first try and use a cached template
+	if immutableCachedTemplate != nil {
+		virtualInfo, err := mm.consensusReference.Consensus().GetVirtualInfo()
+		if err != nil {
+			return nil, err
+		}
+		if externalapi.HashesEqual(virtualInfo.ParentHashes, immutableCachedTemplate.Block.Header.DirectParents()) {
+			if immutableCachedTemplate.CoinbaseData.Equal(coinbaseData) {
+				// Both, virtual parents and coinbase data are equal, simply return the cached block with updated time
+				newTimestamp := mstime.Now().UnixMilliseconds()
+				if newTimestamp < immutableCachedTemplate.Block.Header.TimeInMilliseconds() {
+					// Keep the previous time as built by internal consensus median time logic
+					return immutableCachedTemplate.Block, nil
+				}
+				// If new time stamp is later than current, update the header
+				mutableHeader := immutableCachedTemplate.Block.Header.ToMutable()
+				mutableHeader.SetTimeInMilliseconds(newTimestamp)
+
+				return &externalapi.DomainBlock{
+					Header:       mutableHeader.ToImmutable(),
+					Transactions: immutableCachedTemplate.Block.Transactions,
+				}, nil
+			}
+
+			// Virtual parents are equal, but coinbase data is new -- make the minimum changes required
+			// Note we first clone the block template since it is modified by the call
+			modifiedBlockTemplate, err := mm.blockTemplateBuilder.ModifyBlockTemplate(coinbaseData, immutableCachedTemplate.Clone())
+			if err != nil {
+				return nil, err
+			}
+
+			// No point in updating cache since we have no reason to believe this coinbase will be used more
+			// than the previous one, and we want to maintain the original template caching time
+			return modifiedBlockTemplate.Block, nil
+		}
+	}
+	// No relevant cache, build a template
+	blockTemplate, err := mm.blockTemplateBuilder.BuildBlockTemplate(coinbaseData)
+	if err != nil {
+		return nil, err
+	}
+	// Cache the built template
+	mm.setImmutableCachedTemplate(blockTemplate)
+	return blockTemplate.Block, nil
 }
 
-// HandleNewBlock handles the transactions for a new block that was just added to the DAG
+func (mm *miningManager) getImmutableCachedTemplate() *externalapi.DomainBlockTemplate {
+	mm.cacheLock.Lock()
+	defer mm.cacheLock.Unlock()
+	if time.Since(mm.cachingTime) > time.Second {
+		// No point in cache optimizations if queries are more than a second apart -- we prefer rechecking the mempool.
+		// Full explanation: On the one hand this is a sub-millisecond optimization, so there is no harm in doing the full block building
+		// every ~1 second. Additionally, we would like to refresh the mempool access even if virtual info was
+		// unmodified for a while. All in all, caching for max 1 second is a good compromise.
+		mm.cachedBlockTemplate = nil
+	}
+	return mm.cachedBlockTemplate
+}
+
+func (mm *miningManager) setImmutableCachedTemplate(blockTemplate *externalapi.DomainBlockTemplate) {
+	mm.cacheLock.Lock()
+	defer mm.cacheLock.Unlock()
+	mm.cachingTime = time.Now()
+	mm.cachedBlockTemplate = blockTemplate
+}
+
+func (mm *miningManager) GetBlockTemplateBuilder() miningmanagermodel.BlockTemplateBuilder {
+	return mm.blockTemplateBuilder
+}
+
+// HandleNewBlockTransactions handles the transactions for a new block that was just added to the DAG
 func (mm *miningManager) HandleNewBlockTransactions(txs []*externalapi.DomainTransaction) ([]*externalapi.DomainTransaction, error) {
 	return mm.mempool.HandleNewBlockTransactions(txs)
 }

domain/miningmanager/miningmanager_test.go

@@ -1,7 +1,11 @@
 package miningmanager_test
 
 import (
+	"github.com/kaspanet/kaspad/cmd/kaspawallet/libkaspawallet"
 	"github.com/kaspanet/kaspad/domain/consensusreference"
+	"github.com/kaspanet/kaspad/domain/miningmanager/model"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/version"
 	"reflect"
 	"strings"
 	"testing"
@@ -571,6 +575,279 @@ func TestRevalidateHighPriorityTransactions(t *testing.T) {
 	})
 }
 
+// TestModifyBlockTemplate verifies that modifying a block template changes coinbase data correctly.
+func TestModifyBlockTemplate(t *testing.T) {
+	testutils.ForAllNets(t, true, func(t *testing.T, consensusConfig *consensus.Config) {
+		consensusConfig.BlockCoinbaseMaturity = 0
+		factory := consensus.NewFactory()
+		tc, teardown, err := factory.NewTestConsensus(consensusConfig, "TestModifyBlockTemplate")
+		if err != nil {
+			t.Fatalf("Error setting up TestConsensus: %+v", err)
+		}
+		defer teardown(false)
+
+		miningFactory := miningmanager.NewFactory()
+		tcAsConsensus := tc.(externalapi.Consensus)
+		tcAsConsensusPointer := &tcAsConsensus
+		consensusReference := consensusreference.NewConsensusReference(&tcAsConsensusPointer)
+		miningManager := miningFactory.NewMiningManager(consensusReference, &consensusConfig.Params, mempool.DefaultConfig(&consensusConfig.Params))
+
+		// Create some complex transactions. Logic taken from TestOrphanTransactions
+
+		// Before each parent transaction, We will add two blocks by consensus in order to fund the parent transactions.
+		parentTransactions, childTransactions, err := createArraysOfParentAndChildrenTransactions(tc)
+		if err != nil {
+			t.Fatalf("Error in createArraysOfParentAndChildrenTransactions: %v", err)
+		}
+		for _, orphanTransaction := range childTransactions {
+			_, err = miningManager.ValidateAndInsertTransaction(orphanTransaction, false, true)
+			if err != nil {
+				t.Fatalf("ValidateAndInsertTransaction: %v", err)
+			}
+		}
+		transactionsMempool := miningManager.AllTransactions()
+		for _, transaction := range transactionsMempool {
+			if contains(transaction, childTransactions) {
+				t.Fatalf("Error: an orphan transaction is exist in the mempool")
+			}
+		}
+
+		emptyCoinbaseData := &externalapi.DomainCoinbaseData{
+			ScriptPublicKey: &externalapi.ScriptPublicKey{Script: nil, Version: 0},
+			ExtraData:       nil}
+		block, err := miningManager.GetBlockTemplate(emptyCoinbaseData)
+		if err != nil {
+			t.Fatalf("Failed get a block template: %v", err)
+		}
+
+		for _, transactionFromBlock := range block.Transactions[1:] {
+			for _, orphanTransaction := range childTransactions {
+				if consensushashing.TransactionID(transactionFromBlock) == consensushashing.TransactionID(orphanTransaction) {
+					t.Fatalf("Tranasaction with unknown parents is exist in a block that was built from GetTemplate option.")
+				}
+			}
+		}
+
+		// Run the purpose of this test, compare modified block templates
+		sweepCompareModifiedTemplateToBuilt(t, consensusConfig, miningManager.GetBlockTemplateBuilder())
+
+		// Create some more complex blocks and transactions. Logic taken from TestOrphanTransactions
+		tips, err := tc.Tips()
+		if err != nil {
+			t.Fatalf("Tips: %v.", err)
+		}
+		blockParentsTransactionsHash, _, err := tc.AddBlock(tips, nil, parentTransactions)
+		if err != nil {
+			t.Fatalf("AddBlock: %v", err)
+		}
+
+		_, _, err = tc.AddBlock([]*externalapi.DomainHash{blockParentsTransactionsHash}, nil, nil)
+		if err != nil {
+			t.Fatalf("AddBlock: %v", err)
+		}
+
+		blockParentsTransactions, err := tc.GetBlock(blockParentsTransactionsHash)
+		if err != nil {
+			t.Fatalf("GetBlock: %v", err)
+		}
+		_, err = miningManager.HandleNewBlockTransactions(blockParentsTransactions.Transactions)
+		if err != nil {
+			t.Fatalf("HandleNewBlockTransactions: %+v", err)
+		}
+		transactionsMempool = miningManager.AllTransactions()
+		if len(transactionsMempool) != len(childTransactions) {
+			t.Fatalf("Expected %d transactions in the mempool but got %d", len(childTransactions), len(transactionsMempool))
+		}
+
+		for _, transaction := range transactionsMempool {
+			if !contains(transaction, childTransactions) {
+				t.Fatalf("Error: the transaction %s, should be in the mempool since its not "+
+					"oprhan anymore.", consensushashing.TransactionID(transaction))
+			}
+		}
+		block, err = miningManager.GetBlockTemplate(emptyCoinbaseData)
+		if err != nil {
+			t.Fatalf("GetBlockTemplate: %v", err)
+		}
+
+		for _, transactionFromBlock := range block.Transactions[1:] {
+			isContained := false
+			for _, childTransaction := range childTransactions {
+				if *consensushashing.TransactionID(transactionFromBlock) == *consensushashing.TransactionID(childTransaction) {
+					isContained = true
+					break
+				}
+			}
+			if !isContained {
+				t.Fatalf("Error: Unknown Transaction %s in a block.", consensushashing.TransactionID(transactionFromBlock))
+			}
+		}
+
+		// Run the purpose of this test, compare modified block templates
+		sweepCompareModifiedTemplateToBuilt(t, consensusConfig, miningManager.GetBlockTemplateBuilder())
+
+		// Create a real coinbase to use
+		coinbaseUsual, err := generateNewCoinbase(consensusConfig.Prefix, opUsual)
+		if err != nil {
+			t.Fatalf("Generate coinbase: %v.", err)
+		}
+		var emptyTransactions []*externalapi.DomainTransaction
+
+		// Create interesting DAG structures and rerun the template comparisons
+		tips, err = tc.Tips()
+		if err != nil {
+			t.Fatalf("Tips: %v.", err)
+		}
+		// Create a fork
+		_, _, err = tc.AddBlock(tips[:1], coinbaseUsual, emptyTransactions)
+		if err != nil {
+			t.Fatalf("AddBlock: %v", err)
+		}
+		chainTip, _, err := tc.AddBlock(tips[:1], coinbaseUsual, emptyTransactions)
+		if err != nil {
+			t.Fatalf("AddBlock: %v", err)
+		}
+
+		sweepCompareModifiedTemplateToBuilt(t, consensusConfig, miningManager.GetBlockTemplateBuilder())
+
+		// Create some blue blocks
+		for i := externalapi.KType(0); i < consensusConfig.K-2; i++ {
+			chainTip, _, err = tc.AddBlock([]*externalapi.DomainHash{chainTip}, coinbaseUsual, emptyTransactions)
+			if err != nil {
+				t.Fatalf("AddBlock: %v", err)
+			}
+		}
+
+		sweepCompareModifiedTemplateToBuilt(t, consensusConfig, miningManager.GetBlockTemplateBuilder())
+
+		// Mine more such that we have a merged red
+		for i := externalapi.KType(0); i < consensusConfig.K; i++ {
+			chainTip, _, err = tc.AddBlock([]*externalapi.DomainHash{chainTip}, coinbaseUsual, emptyTransactions)
+			if err != nil {
+				t.Fatalf("AddBlock: %v", err)
+			}
+		}
+		blockTemplate, err := miningManager.GetBlockTemplateBuilder().BuildBlockTemplate(emptyCoinbaseData)
+		if err != nil {
+			t.Fatalf("BuildBlockTemplate: %v", err)
+		}
+		if !blockTemplate.CoinbaseHasRedReward {
+			t.Fatalf("Expected block template to have red reward")
+		}
+
+		sweepCompareModifiedTemplateToBuilt(t, consensusConfig, miningManager.GetBlockTemplateBuilder())
+	})
+}
+
+func sweepCompareModifiedTemplateToBuilt(
+	t *testing.T, consensusConfig *consensus.Config, builder model.BlockTemplateBuilder) {
+	for i := 0; i < 4; i++ {
+		// Run a few times to get more randomness
+		compareModifiedTemplateToBuilt(t, consensusConfig, builder, opUsual, opUsual)
+		compareModifiedTemplateToBuilt(t, consensusConfig, builder, opECDSA, opECDSA)
+	}
+	compareModifiedTemplateToBuilt(t, consensusConfig, builder, opTrue, opUsual)
+	compareModifiedTemplateToBuilt(t, consensusConfig, builder, opUsual, opTrue)
+	compareModifiedTemplateToBuilt(t, consensusConfig, builder, opECDSA, opUsual)
+	compareModifiedTemplateToBuilt(t, consensusConfig, builder, opUsual, opECDSA)
+	compareModifiedTemplateToBuilt(t, consensusConfig, builder, opEmpty, opUsual)
+	compareModifiedTemplateToBuilt(t, consensusConfig, builder, opUsual, opEmpty)
+}
+
+type opType uint8
+
+const (
+	opUsual opType = iota
+	opECDSA
+	opTrue
+	opEmpty
+)
+
+func compareModifiedTemplateToBuilt(
+	t *testing.T, consensusConfig *consensus.Config, builder model.BlockTemplateBuilder,
+	firstCoinbaseOp, secondCoinbaseOp opType) {
+	coinbase1, err := generateNewCoinbase(consensusConfig.Params.Prefix, firstCoinbaseOp)
+	if err != nil {
+		t.Fatalf("Failed to generate new coinbase: %v", err)
+	}
+	coinbase2, err := generateNewCoinbase(consensusConfig.Params.Prefix, secondCoinbaseOp)
+	if err != nil {
+		t.Fatalf("Failed to generate new coinbase: %v", err)
+	}
+
+	// Build a fresh template for coinbase2 as a reference
+	expectedTemplate, err := builder.BuildBlockTemplate(coinbase2)
+	if err != nil {
+		t.Fatalf("Failed to build block template: %v", err)
+	}
+	// Modify to coinbase1
+	modifiedTemplate, err := builder.ModifyBlockTemplate(coinbase1, expectedTemplate.Clone())
+	if err != nil {
+		t.Fatalf("Failed to modify block template: %v", err)
+	}
+	// And modify back to coinbase2
+	modifiedTemplate, err = builder.ModifyBlockTemplate(coinbase2, modifiedTemplate.Clone())
+	if err != nil {
+		t.Fatalf("Failed to modify block template: %v", err)
+	}
+
+	// Make sure timestamps are equal before comparing the hash
+	mutableHeader := modifiedTemplate.Block.Header.ToMutable()
+	mutableHeader.SetTimeInMilliseconds(expectedTemplate.Block.Header.TimeInMilliseconds())
+	modifiedTemplate.Block.Header = mutableHeader.ToImmutable()
+
+	// Assert hashes are equal
+	expectedTemplateHash := consensushashing.BlockHash(expectedTemplate.Block)
+	modifiedTemplateHash := consensushashing.BlockHash(modifiedTemplate.Block)
+	if !expectedTemplateHash.Equal(modifiedTemplateHash) {
+		t.Fatalf("Expected block hashes %s, %s to be equal", expectedTemplateHash, modifiedTemplateHash)
+	}
+}
+
+func generateNewCoinbase(addressPrefix util.Bech32Prefix, op opType) (*externalapi.DomainCoinbaseData, error) {
+	if op == opTrue {
+		scriptPublicKey, _ := testutils.OpTrueScript()
+		return &externalapi.DomainCoinbaseData{
+			ScriptPublicKey: scriptPublicKey, ExtraData: []byte(version.Version()),
+		}, nil
+	}
+	if op == opEmpty {
+		return &externalapi.DomainCoinbaseData{
+			ScriptPublicKey: &externalapi.ScriptPublicKey{Script: nil, Version: 0},
+			ExtraData:       nil,
+		}, nil
+	}
+	_, publicKey, err := libkaspawallet.CreateKeyPair(op == opECDSA)
+	if err != nil {
+		return nil, err
+	}
+	var address string
+	if op == opECDSA {
+		addressPublicKeyECDSA, err := util.NewAddressPublicKeyECDSA(publicKey, addressPrefix)
+		if err != nil {
+			return nil, err
+		}
+		address = addressPublicKeyECDSA.EncodeAddress()
+	} else {
+		addressPublicKey, err := util.NewAddressPublicKey(publicKey, addressPrefix)
+		if err != nil {
+			return nil, err
+		}
+		address = addressPublicKey.EncodeAddress()
+	}
+	payAddress, err := util.DecodeAddress(address, addressPrefix)
+	if err != nil {
+		return nil, err
+	}
+	scriptPublicKey, err := txscript.PayToAddrScript(payAddress)
+	if err != nil {
+		return nil, err
+	}
+	return &externalapi.DomainCoinbaseData{
+		ScriptPublicKey: scriptPublicKey, ExtraData: []byte(version.Version()),
+	}, nil
+}
+
 func createTransactionWithUTXOEntry(t *testing.T, i int, daaScore uint64) *externalapi.DomainTransaction {
 	prevOutTxID := externalapi.DomainTransactionID{}
 	prevOutPoint := externalapi.DomainOutpoint{TransactionID: prevOutTxID, Index: uint32(i)}

domain/miningmanager/model/interface_blocktemplatebuilder.go

@@ -6,5 +6,7 @@ import (
 
 // BlockTemplateBuilder builds block templates for miners to consume
 type BlockTemplateBuilder interface {
-	GetBlockTemplate(coinbaseData *consensusexternalapi.DomainCoinbaseData) (*consensusexternalapi.DomainBlock, error)
+	BuildBlockTemplate(coinbaseData *consensusexternalapi.DomainCoinbaseData) (*consensusexternalapi.DomainBlockTemplate, error)
+	ModifyBlockTemplate(newCoinbaseData *consensusexternalapi.DomainCoinbaseData,
+		blockTemplateToModify *consensusexternalapi.DomainBlockTemplate) (*consensusexternalapi.DomainBlockTemplate, error)
 }