Merge branch 'dev' into add_id_to_wallet_client_and_daemon_interaction

app/appmessage/rpc_get_mempool_entries.go

@@ -4,6 +4,8 @@ package appmessage
 // its respective RPC message
 type GetMempoolEntriesRequestMessage struct {
 	baseMessage
+	IncludeOrphanPool     bool
+	FilterTransactionPool bool
 }
 
 // Command returns the protocol command string for the message
@@ -12,8 +14,11 @@ func (msg *GetMempoolEntriesRequestMessage) Command() MessageCommand {
 }
 
 // NewGetMempoolEntriesRequestMessage returns a instance of the message
-func NewGetMempoolEntriesRequestMessage() *GetMempoolEntriesRequestMessage {
+func NewGetMempoolEntriesRequestMessage(includeOrphanPool bool, filterTransactionPool bool) *GetMempoolEntriesRequestMessage {
-	return &GetMempoolEntriesRequestMessage{}
+	return &GetMempoolEntriesRequestMessage{
+		IncludeOrphanPool:     includeOrphanPool,
+		FilterTransactionPool: filterTransactionPool,
+	}
 }
 
 // GetMempoolEntriesResponseMessage is an appmessage corresponding to

app/appmessage/rpc_get_mempool_entries_by_addresses.go

@@ -12,6 +12,8 @@ type MempoolEntryByAddress struct {
 type GetMempoolEntriesByAddressesRequestMessage struct {
 	baseMessage
 	Addresses             []string
+	IncludeOrphanPool     bool
+	FilterTransactionPool bool
 }
 
 // Command returns the protocol command string for the message
@@ -20,9 +22,11 @@ func (msg *GetMempoolEntriesByAddressesRequestMessage) Command() MessageCommand
 }
 
 // NewGetMempoolEntriesByAddressesRequestMessage returns a instance of the message
-func NewGetMempoolEntriesByAddressesRequestMessage(addresses []string) *GetMempoolEntriesByAddressesRequestMessage {
+func NewGetMempoolEntriesByAddressesRequestMessage(addresses []string, includeOrphanPool bool, filterTransactionPool bool) *GetMempoolEntriesByAddressesRequestMessage {
 	return &GetMempoolEntriesByAddressesRequestMessage{
 		Addresses:             addresses,
+		IncludeOrphanPool:     includeOrphanPool,
+		FilterTransactionPool: filterTransactionPool,
 	}
 }
 

app/appmessage/rpc_get_mempool_entry.go

@@ -5,6 +5,8 @@ package appmessage
 type GetMempoolEntryRequestMessage struct {
 	baseMessage
 	TxID                  string
+	IncludeOrphanPool     bool
+	FilterTransactionPool bool
 }
 
 // Command returns the protocol command string for the message
@@ -13,8 +15,12 @@ func (msg *GetMempoolEntryRequestMessage) Command() MessageCommand {
 }
 
 // NewGetMempoolEntryRequestMessage returns a instance of the message
-func NewGetMempoolEntryRequestMessage(txID string) *GetMempoolEntryRequestMessage {
+func NewGetMempoolEntryRequestMessage(txID string, includeOrphanPool bool, filterTransactionPool bool) *GetMempoolEntryRequestMessage {
-	return &GetMempoolEntryRequestMessage{TxID: txID}
+	return &GetMempoolEntryRequestMessage{
+		TxID:                  txID,
+		IncludeOrphanPool:     includeOrphanPool,
+		FilterTransactionPool: filterTransactionPool,
+	}
 }
 
 // GetMempoolEntryResponseMessage is an appmessage corresponding to
@@ -30,6 +36,7 @@ type GetMempoolEntryResponseMessage struct {
 type MempoolEntry struct {
 	Fee         uint64
 	Transaction *RPCTransaction
+	IsOrphan    bool
 }
 
 // Command returns the protocol command string for the message
@@ -38,11 +45,12 @@ func (msg *GetMempoolEntryResponseMessage) Command() MessageCommand {
 }
 
 // NewGetMempoolEntryResponseMessage returns a instance of the message
-func NewGetMempoolEntryResponseMessage(fee uint64, transaction *RPCTransaction) *GetMempoolEntryResponseMessage {
+func NewGetMempoolEntryResponseMessage(fee uint64, transaction *RPCTransaction, isOrphan bool) *GetMempoolEntryResponseMessage {
 	return &GetMempoolEntryResponseMessage{
 		Entry: &MempoolEntry{
 			Fee:         fee,
 			Transaction: transaction,
+			IsOrphan:    isOrphan,
 		},
 	}
 }

app/protocol/flowcontext/blocks.go

@@ -107,7 +107,7 @@ func (f *FlowContext) AddBlock(block *externalapi.DomainBlock) error {
 		return protocolerrors.Errorf(false, "cannot add header only block")
 	}
 
-	_, err := f.Domain().Consensus().ValidateAndInsertBlock(block, true)
+	err := f.Domain().Consensus().ValidateAndInsertBlock(block, true)
 	if err != nil {
 		if errors.As(err, &ruleerrors.RuleError{}) {
 			log.Warnf("Validation failed for block %s: %s", consensushashing.BlockHash(block), err)

app/protocol/flowcontext/orphans.go

@@ -141,7 +141,7 @@ func (f *FlowContext) unorphanBlock(orphanHash externalapi.DomainHash) (bool, er
 	}
 	delete(f.orphans, orphanHash)
 
-	_, err := f.domain.Consensus().ValidateAndInsertBlock(orphanBlock, true)
+	err := f.domain.Consensus().ValidateAndInsertBlock(orphanBlock, true)
 	if err != nil {
 		if errors.As(err, &ruleerrors.RuleError{}) {
 			log.Warnf("Validation failed for orphan block %s: %s", orphanHash, err)

app/protocol/flows/v5/blockrelay/handle_relay_invs.go

@@ -321,7 +321,7 @@ func (flow *handleRelayInvsFlow) readMsgBlock() (msgBlock *appmessage.MsgBlock,
 
 func (flow *handleRelayInvsFlow) processBlock(block *externalapi.DomainBlock) ([]*externalapi.DomainHash, error) {
 	blockHash := consensushashing.BlockHash(block)
-	_, err := flow.Domain().Consensus().ValidateAndInsertBlock(block, true)
+	err := flow.Domain().Consensus().ValidateAndInsertBlock(block, true)
 	if err != nil {
 		if !errors.As(err, &ruleerrors.RuleError{}) {
 			return nil, errors.Wrapf(err, "failed to process block %s", blockHash)

app/protocol/flows/v5/blockrelay/ibd.go

@@ -488,7 +488,7 @@ func (flow *handleIBDFlow) processHeader(consensus externalapi.Consensus, msgBlo
 		log.Debugf("Block header %s is already in the DAG. Skipping...", blockHash)
 		return nil
 	}
-	_, err = consensus.ValidateAndInsertBlock(block, false)
+	err = consensus.ValidateAndInsertBlock(block, false)
 	if err != nil {
 		if !errors.As(err, &ruleerrors.RuleError{}) {
 			return errors.Wrapf(err, "failed to process header %s during IBD", blockHash)
@@ -654,7 +654,7 @@ func (flow *handleIBDFlow) syncMissingBlockBodies(highHash *externalapi.DomainHa
 				return err
 			}
 
-			_, err = flow.Domain().Consensus().ValidateAndInsertBlock(block, false)
+			err = flow.Domain().Consensus().ValidateAndInsertBlock(block, false)
 			if err != nil {
 				if errors.Is(err, ruleerrors.ErrDuplicateBlock) {
 					log.Debugf("Skipping IBD Block %s as it has already been added to the DAG", blockHash)
@@ -705,7 +705,7 @@ func (flow *handleIBDFlow) resolveVirtual(estimatedVirtualDAAScoreTarget uint64)
 			}
 			log.Infof("Resolving virtual. Estimated progress: %d%%", percents)
 		}
-		_, isCompletelyResolved, err := flow.Domain().Consensus().ResolveVirtual()
+		isCompletelyResolved, err := flow.Domain().Consensus().ResolveVirtual()
 		if err != nil {
 			return err
 		}

app/protocol/flows/v5/blockrelay/ibd_with_headers_proof.go

@@ -280,7 +280,7 @@ func (flow *handleIBDFlow) processBlockWithTrustedData(
 		blockWithTrustedData.GHOSTDAGData = append(blockWithTrustedData.GHOSTDAGData, appmessage.GHOSTDAGHashPairToDomainGHOSTDAGHashPair(data.GHOSTDAGData[index]))
 	}
 
-	_, err := consensus.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
+	err := consensus.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
 	return err
 }
 

app/rpc/rpchandlers/get_mempool_entries.go

@@ -7,7 +7,32 @@ import (
 )
 
 // HandleGetMempoolEntries handles the respectively named RPC command
-func HandleGetMempoolEntries(context *rpccontext.Context, _ *router.Router, _ appmessage.Message) (appmessage.Message, error) {
+func HandleGetMempoolEntries(context *rpccontext.Context, _ *router.Router, request appmessage.Message) (appmessage.Message, error) {
+	getMempoolEntriesRequest := request.(*appmessage.GetMempoolEntriesRequestMessage)
+
+	entries := make([]*appmessage.MempoolEntry, 0)
+
+	if !getMempoolEntriesRequest.FilterTransactionPool {
+		transactionPoolEntries, err := getTransactionPoolMempoolEntries(context)
+		if err != nil {
+			return nil, err
+		}
+
+		entries = append(entries, transactionPoolEntries...)
+	}
+
+	if getMempoolEntriesRequest.IncludeOrphanPool {
+		orphanPoolEntries, err := getOrphanPoolMempoolEntries(context)
+		if err != nil {
+			return nil, err
+		}
+		entries = append(entries, orphanPoolEntries...)
+	}
+
+	return appmessage.NewGetMempoolEntriesResponseMessage(entries), nil
+}
+
+func getTransactionPoolMempoolEntries(context *rpccontext.Context) ([]*appmessage.MempoolEntry, error) {
 	transactions := context.Domain.MiningManager().AllTransactions()
 	entries := make([]*appmessage.MempoolEntry, 0, len(transactions))
 	for _, transaction := range transactions {
@@ -19,8 +44,26 @@ func HandleGetMempoolEntries(context *rpccontext.Context, _ *router.Router, _ ap
 		entries = append(entries, &appmessage.MempoolEntry{
 			Fee:         transaction.Fee,
 			Transaction: rpcTransaction,
+			IsOrphan:    false,
 		})
 	}
-
+	return entries, nil
-	return appmessage.NewGetMempoolEntriesResponseMessage(entries), nil
+}
+
+func getOrphanPoolMempoolEntries(context *rpccontext.Context) ([]*appmessage.MempoolEntry, error) {
+	orphanTransactions := context.Domain.MiningManager().AllOrphanTransactions()
+	entries := make([]*appmessage.MempoolEntry, 0, len(orphanTransactions))
+	for _, orphanTransaction := range orphanTransactions {
+		rpcTransaction := appmessage.DomainTransactionToRPCTransaction(orphanTransaction)
+		err := context.PopulateTransactionWithVerboseData(rpcTransaction, nil)
+		if err != nil {
+			return nil, err
+		}
+		entries = append(entries, &appmessage.MempoolEntry{
+			Fee:         orphanTransaction.Fee,
+			Transaction: rpcTransaction,
+			IsOrphan:    true,
+		})
+	}
+	return entries, nil
 }

app/rpc/rpchandlers/get_mempool_entries_by_addresses.go

@@ -1,8 +1,12 @@
 package rpchandlers
 
 import (
+	"errors"
+
 	"github.com/kaspanet/kaspad/app/appmessage"
 	"github.com/kaspanet/kaspad/app/rpc/rpccontext"
+
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/consensushashing"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/txscript"
 	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/router"
@@ -12,17 +16,62 @@ import (
 // HandleGetMempoolEntriesByAddresses handles the respectively named RPC command
 func HandleGetMempoolEntriesByAddresses(context *rpccontext.Context, _ *router.Router, request appmessage.Message) (appmessage.Message, error) {
 
-	transactions := context.Domain.MiningManager().AllTransactions()
 	getMempoolEntriesByAddressesRequest := request.(*appmessage.GetMempoolEntriesByAddressesRequestMessage)
+
 	mempoolEntriesByAddresses := make([]*appmessage.MempoolEntryByAddress, 0)
 
-	for _, addressString := range getMempoolEntriesByAddressesRequest.Addresses {
+	if !getMempoolEntriesByAddressesRequest.FilterTransactionPool {
+		transactionPoolTransactions := context.Domain.MiningManager().AllTransactions()
+		transactionPoolEntriesByAddresses, err := extractMempoolEntriesByAddressesFromTransactions(
+			context,
+			getMempoolEntriesByAddressesRequest.Addresses,
+			transactionPoolTransactions,
+			false,
+		)
+		if err != nil {
+			rpcError := &appmessage.RPCError{}
+			if !errors.As(err, &rpcError) {
+				return nil, err
+			}
+			errorMessage := &appmessage.GetUTXOsByAddressesResponseMessage{}
+			errorMessage.Error = rpcError
+			return errorMessage, nil
+		}
+		mempoolEntriesByAddresses = append(mempoolEntriesByAddresses, transactionPoolEntriesByAddresses...)
+	}
 
+	if getMempoolEntriesByAddressesRequest.IncludeOrphanPool {
+
+		orphanPoolTransactions := context.Domain.MiningManager().AllOrphanTransactions()
+		orphanPoolEntriesByAddresse, err := extractMempoolEntriesByAddressesFromTransactions(
+			context,
+			getMempoolEntriesByAddressesRequest.Addresses,
+			orphanPoolTransactions,
+			true,
+		)
+		if err != nil {
+			rpcError := &appmessage.RPCError{}
+			if !errors.As(err, &rpcError) {
+				return nil, err
+			}
+			errorMessage := &appmessage.GetUTXOsByAddressesResponseMessage{}
+			errorMessage.Error = rpcError
+			return errorMessage, nil
+		}
+
+		mempoolEntriesByAddresses = append(mempoolEntriesByAddresses, orphanPoolEntriesByAddresse...)
+	}
+
+	return appmessage.NewGetMempoolEntriesByAddressesResponseMessage(mempoolEntriesByAddresses), nil
+}
+
+//TO DO: optimize extractMempoolEntriesByAddressesFromTransactions
+func extractMempoolEntriesByAddressesFromTransactions(context *rpccontext.Context, addresses []string, transactions []*externalapi.DomainTransaction, areOrphans bool) ([]*appmessage.MempoolEntryByAddress, error) {
+	mempoolEntriesByAddresses := make([]*appmessage.MempoolEntryByAddress, 0)
+	for _, addressString := range addresses {
 		_, err := util.DecodeAddress(addressString, context.Config.ActiveNetParams.Prefix)
 		if err != nil {
-			errorMessage := &appmessage.GetUTXOsByAddressesResponseMessage{}
+			return nil, appmessage.RPCErrorf("Could not decode address '%s': %s", addressString, err)
-			errorMessage.Error = appmessage.RPCErrorf("Could not decode address '%s': %s", addressString, err)
-			return errorMessage, nil
 		}
 
 		sending := make([]*appmessage.MempoolEntry, 0)
@@ -50,6 +99,7 @@ func HandleGetMempoolEntriesByAddresses(context *rpccontext.Context, _ *router.R
 						&appmessage.MempoolEntry{
 							Fee:         transaction.Fee,
 							Transaction: rpcTransaction,
+							IsOrphan:    areOrphans,
 						},
 					)
 					break //one input is enough
@@ -71,6 +121,7 @@ func HandleGetMempoolEntriesByAddresses(context *rpccontext.Context, _ *router.R
 						&appmessage.MempoolEntry{
 							Fee:         transaction.Fee,
 							Transaction: rpcTransaction,
+							IsOrphan:    areOrphans,
 						},
 					)
 					break //one output is enough
@@ -89,9 +140,8 @@ func HandleGetMempoolEntriesByAddresses(context *rpccontext.Context, _ *router.R
 					},
 				)
 			}
-
-		}
 		}
 
-	return appmessage.NewGetMempoolEntriesByAddressesResponseMessage(mempoolEntriesByAddresses), nil
+	}
+	return mempoolEntriesByAddresses, nil
 }

app/rpc/rpchandlers/get_mempool_entry.go

@@ -3,12 +3,18 @@ package rpchandlers
 import (
 	"github.com/kaspanet/kaspad/app/appmessage"
 	"github.com/kaspanet/kaspad/app/rpc/rpccontext"
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/transactionid"
 	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/router"
 )
 
 // HandleGetMempoolEntry handles the respectively named RPC command
 func HandleGetMempoolEntry(context *rpccontext.Context, _ *router.Router, request appmessage.Message) (appmessage.Message, error) {
+
+	transaction := &externalapi.DomainTransaction{}
+	var found bool
+	var isOrphan bool
+
 	getMempoolEntryRequest := request.(*appmessage.GetMempoolEntryRequestMessage)
 
 	transactionID, err := transactionid.FromString(getMempoolEntryRequest.TxID)
@@ -18,17 +24,25 @@ func HandleGetMempoolEntry(context *rpccontext.Context, _ *router.Router, reques
 		return errorMessage, nil
 	}
 
-	transaction, ok := context.Domain.MiningManager().GetTransaction(transactionID)
+	if !getMempoolEntryRequest.FilterTransactionPool {
-	if !ok {
+		transaction, found = context.Domain.MiningManager().GetTransaction(transactionID)
+	}
+
+	if getMempoolEntryRequest.IncludeOrphanPool && !found {
+		transaction, found = context.Domain.MiningManager().GetOrphanTransaction(transactionID)
+		isOrphan = true
+	}
+
+	if !found {
 		errorMessage := &appmessage.GetMempoolEntryResponseMessage{}
 		errorMessage.Error = appmessage.RPCErrorf("Transaction %s was not found", transactionID)
 		return errorMessage, nil
 	}
+
 	rpcTransaction := appmessage.DomainTransactionToRPCTransaction(transaction)
 	err = context.PopulateTransactionWithVerboseData(rpcTransaction, nil)
 	if err != nil {
 		return nil, err
 	}
-
+	return appmessage.NewGetMempoolEntryResponseMessage(transaction.Fee, rpcTransaction, isOrphan), nil
-	return appmessage.NewGetMempoolEntryResponseMessage(transaction.Fee, rpcTransaction), nil
 }

changelog.txt

@@ -1,3 +1,23 @@
+Kaspad v0.12.2 - 2022-06-17
+===========================
+
+* Clarify wallet message concerning a wallet daemon sync state  (#2045)
+* Change the way the miner executable reports execution errors (closes issue #1677) (#2048)
+* Fix kaspawallet help messages, clarify sweep command help string (#2067)
+* Wallet parse/send/create commands improvement (#2024)
+* Use chunks for `GetBlocksAcceptanceData` calls in order to avoid blocking consensus for too long (#2075)
+* Unite multiple `GetBlockAcceptanceData` consensus calls to one (#2074)
+* Update many-small-chains-and-one-big-chain DAG to not fail merge depth limit (#2072)
+
+RPC API Changes:
+* RPC: include orphans into mempool entries (#2046)
+* RPC & UtxoIndex: keep track of, query and test circulating supply. (#2070)
+
+Bug Fixes:
+* Fix RPC connections counting (#2026)
+* Fix UTXO diff child error  (#2084)
+* Fix `not in selected chain` crash  (#2082)
+
 Kaspad v0.12.1 - 2022-05-31
 ===========================
 

cmd/kaspaminer/main.go

@@ -23,8 +23,7 @@ func main() {
 
 	cfg, err := parseConfig()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error parsing command-line arguments: %s\n", err)
+		printErrorAndExit(errors.Errorf("Error parsing command-line arguments: %s", err))
-		os.Exit(1)
 	}
 	defer backendLog.Close()
 
@@ -44,7 +43,7 @@ func main() {
 
 	miningAddr, err := util.DecodeAddress(cfg.MiningAddr, cfg.ActiveNetParams.Prefix)
 	if err != nil {
-		panic(errors.Wrap(err, "error decoding mining address"))
+		printErrorAndExit(errors.Errorf("Error decoding mining address: %s", err))
 	}
 
 	doneChan := make(chan struct{})
@@ -61,3 +60,8 @@ func main() {
 	case <-interrupt:
 	}
 }
+
+func printErrorAndExit(err error) {
+	fmt.Fprintf(os.Stderr, "%+v\n", err)
+	os.Exit(1)
+}

cmd/kaspawallet/daemon/server/address.go

@@ -39,10 +39,10 @@ func (s *server) ShowAddresses(_ context.Context, request *pb.ShowAddressesReque
 	defer s.lock.Unlock()
 
 	if !s.isSynced() {
-		return nil, errors.New("server is not synced")
+		return nil, errors.Errorf("wallet daemon is not synced yet, %s", s.formatSyncStateReport())
 	}
 
-	addresses := make([]string, 0)
+	addresses := make([]string, s.keysFile.LastUsedExternalIndex())
 	for i := uint32(1); i <= s.keysFile.LastUsedExternalIndex(); i++ {
 		walletAddr := &walletAddress{
 			index:         i,
@@ -54,7 +54,7 @@ func (s *server) ShowAddresses(_ context.Context, request *pb.ShowAddressesReque
 		if err != nil {
 			return nil, err
 		}
-		addresses = append(addresses, address.String())
+		addresses[i-1] = address.String()
 	}
 
 	return &pb.ShowAddressesResponse{Address: addresses}, nil
@@ -65,7 +65,7 @@ func (s *server) NewAddress(_ context.Context, request *pb.NewAddressRequest) (*
 	defer s.lock.Unlock()
 
 	if !s.isSynced() {
-		return nil, errors.New("server is not synced")
+		return nil, errors.Errorf("wallet daemon is not synced yet, %s", s.formatSyncStateReport())
 	}
 
 	err := s.keysFile.SetLastUsedExternalIndex(s.keysFile.LastUsedExternalIndex() + 1)

cmd/kaspawallet/daemon/server/create_unsigned_transaction.go

@@ -30,7 +30,7 @@ func (s *server) CreateUnsignedTransactions(_ context.Context, request *pb.Creat
 
 func (s *server) createUnsignedTransactions(address string, amount uint64, fromAddressesString []string) ([][]byte, error) {
 	if !s.isSynced() {
-		return nil, errors.New("server is not synced")
+		return nil, errors.Errorf("wallet daemon is not synced yet, %s", s.formatSyncStateReport())
 	}
 
 	err := s.refreshUTXOs()

cmd/kaspawallet/daemon/server/external_spendable_utxos.go

@@ -31,7 +31,6 @@ func (s *server) GetExternalSpendableUTXOs(_ context.Context, request *pb.GetExt
 }
 
 func (s *server) selectExternalSpendableUTXOs(externalUTXOs *appmessage.GetUTXOsByAddressesResponseMessage, address string) ([]*pb.UtxosByAddressesEntry, error) {
-
 	dagInfo, err := s.rpcClient.GetBlockDAGInfo()
 	if err != nil {
 		return nil, err

cmd/kaspawallet/daemon/server/sync.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"fmt"
 	"sort"
 	"time"
 
@@ -101,10 +102,14 @@ func (s *server) collectFarAddresses() error {
 	return nil
 }
 
-func (s *server) maxUsedIndex() uint32 {
+func (s *server) maxUsedIndexWithLock() uint32 {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
+	return s.maxUsedIndex()
+}
+
+func (s *server) maxUsedIndex() uint32 {
 	maxUsedIndex := s.keysFile.LastUsedExternalIndex()
 	if s.keysFile.LastUsedInternalIndex() > maxUsedIndex {
 		maxUsedIndex = s.keysFile.LastUsedInternalIndex()
@@ -122,10 +127,11 @@ func (s *server) collectRecentAddresses() error {
 	maxUsedIndex := uint32(0)
 	for ; index < maxUsedIndex+numIndexesToQueryForRecentAddresses; index += numIndexesToQueryForRecentAddresses {
 		err := s.collectAddressesWithLock(index, index+numIndexesToQueryForRecentAddresses)
+
 		if err != nil {
 			return err
 		}
-		maxUsedIndex = s.maxUsedIndex()
+		maxUsedIndex = s.maxUsedIndexWithLock()
 
 		s.updateSyncingProgressLog(index, maxUsedIndex)
 	}
@@ -261,7 +267,7 @@ func (s *server) refreshUTXOs() error {
 	// and not in consensus, and between the calls its spending transaction will be
 	// added to consensus and removed from the mempool, so `getUTXOsByAddressesResponse`
 	// will include an obsolete output.
-	mempoolEntriesByAddresses, err := s.rpcClient.GetMempoolEntriesByAddresses(s.addressSet.strings())
+	mempoolEntriesByAddresses, err := s.rpcClient.GetMempoolEntriesByAddresses(s.addressSet.strings(), true, true)
 	if err != nil {
 		return err
 	}
@@ -275,7 +281,18 @@ func (s *server) refreshUTXOs() error {
 }
 
 func (s *server) isSynced() bool {
-	return s.nextSyncStartIndex > s.keysFile.LastUsedInternalIndex() && s.nextSyncStartIndex > s.keysFile.LastUsedExternalIndex()
+	return s.nextSyncStartIndex > s.maxUsedIndex()
+}
+
+func (s *server) formatSyncStateReport() string {
+	maxUsedIndex := s.maxUsedIndex()
+
+	if s.nextSyncStartIndex > maxUsedIndex {
+		maxUsedIndex = s.nextSyncStartIndex
+	}
+
+	return fmt.Sprintf("scanned %d out of %d addresses (%.2f%%)",
+		s.nextSyncStartIndex, maxUsedIndex, float64(s.nextSyncStartIndex)*100.0/float64(maxUsedIndex))
 }
 
 func (s *server) updateSyncingProgressLog(currProcessedAddresses, currMaxUsedAddresses uint32) {

domain/consensus/consensus.go

@@ -61,17 +61,18 @@ type consensus struct {
 	blocksWithTrustedDataDAAWindowStore model.BlocksWithTrustedDataDAAWindowStore
 
 	consensusEventsChan chan externalapi.ConsensusEvent
+	virtualNotUpdated   bool
 }
 
-func (s *consensus) ValidateAndInsertBlockWithTrustedData(block *externalapi.BlockWithTrustedData, validateUTXO bool) (*externalapi.VirtualChangeSet, error) {
+func (s *consensus) ValidateAndInsertBlockWithTrustedData(block *externalapi.BlockWithTrustedData, validateUTXO bool) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	virtualChangeSet, _, err := s.blockProcessor.ValidateAndInsertBlockWithTrustedData(block, validateUTXO)
+	_, _, err := s.blockProcessor.ValidateAndInsertBlockWithTrustedData(block, validateUTXO)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	return virtualChangeSet, nil
+	return nil
 }
 
 // Init initializes consensus
@@ -193,21 +194,47 @@ func (s *consensus) BuildBlockTemplate(coinbaseData *externalapi.DomainCoinbaseD
 
 // ValidateAndInsertBlock validates the given block and, if valid, applies it
 // to the current state
-func (s *consensus) ValidateAndInsertBlock(block *externalapi.DomainBlock, shouldValidateAgainstUTXO bool) (*externalapi.VirtualChangeSet, error) {
+func (s *consensus) ValidateAndInsertBlock(block *externalapi.DomainBlock, shouldValidateAgainstUTXO bool) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	virtualChangeSet, blockStatus, err := s.blockProcessor.ValidateAndInsertBlock(block, shouldValidateAgainstUTXO)
+	_, err := s.validateAndInsertBlockNoLock(block, shouldValidateAgainstUTXO)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *consensus) validateAndInsertBlockNoLock(block *externalapi.DomainBlock, updateVirtual bool) (*externalapi.VirtualChangeSet, error) {
+	// If virtual is in non-updated state, and the caller requests updating virtual -- then we must first
+	// resolve virtual so that the new block can be fully processed properly
+	if updateVirtual && s.virtualNotUpdated {
+		for s.virtualNotUpdated {
+			// We use 10000 << finality interval. See comment in `ResolveVirtual`.
+			// We give up responsiveness of consensus in this rare case.
+			_, err := s.resolveVirtualNoLock(10000) // Note `s.virtualNotUpdated` is updated within the call
 			if err != nil {
 				return nil, err
 			}
+		}
+	}
+
+	virtualChangeSet, blockStatus, err := s.blockProcessor.ValidateAndInsertBlock(block, updateVirtual)
+	if err != nil {
+		return nil, err
+	}
+
+	// If block has a body, and yet virtual was not updated -- signify that virtual is in non-updated state
+	if !updateVirtual && blockStatus != externalapi.StatusHeaderOnly {
+		s.virtualNotUpdated = true
+	}
 
 	err = s.sendBlockAddedEvent(block, blockStatus)
 	if err != nil {
 		return nil, err
 	}
 
-	err = s.sendVirtualChangedEvent(virtualChangeSet, shouldValidateAgainstUTXO)
+	err = s.sendVirtualChangedEvent(virtualChangeSet, updateVirtual)
 	if err != nil {
 		return nil, err
 	}
@@ -861,7 +888,7 @@ func (s *consensus) PopulateMass(transaction *externalapi.DomainTransaction) {
 	s.transactionValidator.PopulateMass(transaction)
 }
 
-func (s *consensus) ResolveVirtual() (*externalapi.VirtualChangeSet, bool, error) {
+func (s *consensus) ResolveVirtual() (bool, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -869,28 +896,33 @@ func (s *consensus) ResolveVirtual() (*externalapi.VirtualChangeSet, bool, error
 	// release the lock each time resolve 100 blocks.
 	// Note: maxBlocksToResolve should be smaller than finality interval in order to avoid a situation
 	// where UpdatePruningPointByVirtual skips a pruning point.
-	virtualChangeSet, isCompletelyResolved, err := s.consensusStateManager.ResolveVirtual(100)
+	return s.resolveVirtualNoLock(100)
+}
+
+func (s *consensus) resolveVirtualNoLock(maxBlocksToResolve uint64) (bool, error) {
+	virtualChangeSet, isCompletelyResolved, err := s.consensusStateManager.ResolveVirtual(maxBlocksToResolve)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
+	s.virtualNotUpdated = !isCompletelyResolved
 
 	stagingArea := model.NewStagingArea()
 	err = s.pruningManager.UpdatePruningPointByVirtual(stagingArea)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
 	err = staging.CommitAllChanges(s.databaseContext, stagingArea)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
 	err = s.sendVirtualChangedEvent(virtualChangeSet, true)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
-	return virtualChangeSet, isCompletelyResolved, nil
+	return isCompletelyResolved, nil
 }
 
 func (s *consensus) BuildPruningPointProof() (*externalapi.PruningPointProof, error) {

domain/consensus/consensus_test.go

@@ -27,7 +27,7 @@ func TestConsensus_GetBlockInfo(t *testing.T) {
 		newHeader := invalidBlock.Header.ToMutable()
 		newHeader.SetTimeInMilliseconds(0)
 		invalidBlock.Header = newHeader.ToImmutable()
-		_, err = consensus.ValidateAndInsertBlock(invalidBlock, true)
+		err = consensus.ValidateAndInsertBlock(invalidBlock, true)
 		if !errors.Is(err, ruleerrors.ErrTimeTooOld) {
 			t.Fatalf("Expected block to be invalid with err: %v, instead found: %v", ruleerrors.ErrTimeTooOld, err)
 		}
@@ -55,7 +55,7 @@ func TestConsensus_GetBlockInfo(t *testing.T) {
 			t.Fatalf("consensus.BuildBlock with an empty coinbase shouldn't fail: %v", err)
 		}
 
-		_, err = consensus.ValidateAndInsertBlock(validBlock, true)
+		err = consensus.ValidateAndInsertBlock(validBlock, true)
 		if err != nil {
 			t.Fatalf("consensus.ValidateAndInsertBlock with a block straight from consensus.BuildBlock should not fail: %v", err)
 		}

domain/consensus/finality_test.go

@@ -33,7 +33,7 @@ func TestFinality(t *testing.T) {
 				return nil, err
 			}
 
-			_, err = consensus.ValidateAndInsertBlock(block, true)
+			err = consensus.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				return nil, err
 			}
@@ -201,7 +201,7 @@ func TestBoundedMergeDepth(t *testing.T) {
 				return nil, false // fo some reason go doesn't recognize that t.Fatalf never returns
 			}
 
-			_, err = consensus.ValidateAndInsertBlock(block, true)
+			err = consensus.ValidateAndInsertBlock(block, true)
 			if err == nil {
 				return block, false
 			} else if errors.Is(err, ruleerrors.ErrViolatingBoundedMergeDepth) {
@@ -213,7 +213,7 @@ func TestBoundedMergeDepth(t *testing.T) {
 		}
 
 		processBlock := func(consensus testapi.TestConsensus, block *externalapi.DomainBlock, name string) {
-			_, err := consensus.ValidateAndInsertBlock(block, true)
+			err := consensus.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("TestBoundedMergeDepth: %s got unexpected error from ProcessBlock: %+v", name, err)
 
@@ -225,7 +225,7 @@ func TestBoundedMergeDepth(t *testing.T) {
 			if err != nil {
 				t.Fatalf("TestBoundedMergeDepth: Failed building block: %+v", err)
 			}
-			_, err = consensus.ValidateAndInsertBlock(block, true)
+			err = consensus.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("TestBoundedMergeDepth: Failed Inserting block to consensus: %v", err)
 			}
@@ -268,7 +268,7 @@ func TestBoundedMergeDepth(t *testing.T) {
 					t.Fatalf("GetBlockHeader: %+v", err)
 				}
 
-				_, err = tcSyncee.ValidateAndInsertBlock(block, true)
+				err = tcSyncee.ValidateAndInsertBlock(block, true)
 				if err != nil {
 					t.Fatalf("ValidateAndInsertBlock %d: %+v", i, err)
 				}
@@ -556,7 +556,7 @@ func TestFinalityResolveVirtual(t *testing.T) {
 				block.Header = mutableHeader.ToImmutable()
 			}
 
-			_, err = tcAttacker.ValidateAndInsertBlock(block, true)
+			err = tcAttacker.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				panic(err)
 			}
@@ -583,14 +583,14 @@ func TestFinalityResolveVirtual(t *testing.T) {
 		t.Logf("Side chain tip (%s) blue score %d", sideChainTipHash, sideChainTipGHOSTDAGData.BlueScore())
 
 		for _, block := range sideChain {
-			_, err := tc.ValidateAndInsertBlock(block, false)
+			err := tc.ValidateAndInsertBlock(block, false)
 			if err != nil {
 				panic(err)
 			}
 		}
 
 		for i := 0; ; i++ {
-			_, isCompletelyResolved, err := tc.ResolveVirtual()
+			isCompletelyResolved, err := tc.ResolveVirtual()
 			if err != nil {
 				panic(err)
 			}

domain/consensus/model/externalapi/consensus.go

@@ -5,8 +5,8 @@ type Consensus interface {
 	Init(skipAddingGenesis bool) error
 	BuildBlock(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (*DomainBlock, error)
 	BuildBlockTemplate(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (*DomainBlockTemplate, error)
-	ValidateAndInsertBlock(block *DomainBlock, shouldValidateAgainstUTXO bool) (*VirtualChangeSet, error)
+	ValidateAndInsertBlock(block *DomainBlock, shouldValidateAgainstUTXO bool) error
-	ValidateAndInsertBlockWithTrustedData(block *BlockWithTrustedData, validateUTXO bool) (*VirtualChangeSet, error)
+	ValidateAndInsertBlockWithTrustedData(block *BlockWithTrustedData, validateUTXO bool) error
 	ValidateTransactionAndPopulateWithConsensusData(transaction *DomainTransaction) error
 	ImportPruningPoints(pruningPoints []BlockHeader) error
 	BuildPruningPointProof() (*PruningPointProof, error)
@@ -48,7 +48,7 @@ type Consensus interface {
 	Anticone(blockHash *DomainHash) ([]*DomainHash, error)
 	EstimateNetworkHashesPerSecond(startHash *DomainHash, windowSize int) (uint64, error)
 	PopulateMass(transaction *DomainTransaction)
-	ResolveVirtual() (*VirtualChangeSet, bool, error)
+	ResolveVirtual() (bool, error)
 	BlockDAAWindowHashes(blockHash *DomainHash) ([]*DomainHash, error)
 	TrustedDataDataDAAHeader(trustedBlockHash, daaBlockHash *DomainHash, daaBlockWindowIndex uint64) (*TrustedDataDataDAAHeader, error)
 	TrustedBlockAssociatedGHOSTDAGDataBlockHashes(blockHash *DomainHash) ([]*DomainHash, error)

domain/consensus/model/testapi/test_consensus.go

@@ -47,6 +47,9 @@ type TestConsensus interface {
 
 	AddUTXOInvalidBlock(parentHashes []*externalapi.DomainHash) (*externalapi.DomainHash,
 		*externalapi.VirtualChangeSet, error)
+	UpdatePruningPointByVirtual() error
+
+	ResolveVirtualWithMaxParam(maxBlocksToResolve uint64) (bool, error)
 
 	MineJSON(r io.Reader, blockType MineJSONBlockType) (tips []*externalapi.DomainHash, err error)
 	ToJSON(w io.Writer) error

domain/consensus/processes/blockprocessor/validate_and_insert_block.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
-	"github.com/kaspanet/kaspad/domain/consensus/processes/consensusstatemanager"
 	"github.com/kaspanet/kaspad/domain/consensus/ruleerrors"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/consensushashing"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/multiset"
@@ -165,12 +164,7 @@ func (bp *blockProcessor) validateAndInsertBlock(stagingArea *model.StagingArea,
 
 	if reversalData != nil {
 		err = bp.consensusStateManager.ReverseUTXODiffs(blockHash, reversalData)
-		// It's still not known what causes this error, but we can ignore it and not reverse the UTXO diffs
+		if err != nil {
-		// and harm performance in some cases.
-		// TODO: Investigate why this error happens in the first place, and remove the workaround.
-		if errors.Is(err, consensusstatemanager.ErrReverseUTXODiffsUTXODiffChildNotFound) {
-			log.Errorf("Could not reverse UTXO diffs while resolving virtual: %s", err)
-		} else if err != nil {
 			return nil, externalapi.StatusInvalid, err
 		}
 	}

domain/consensus/processes/blockprocessor/validate_and_insert_block_test.go

@@ -79,7 +79,7 @@ func TestBlockStatus(t *testing.T) {
 			disqualifiedBlock.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(disqualifiedBlock, true)
+		err = tc.ValidateAndInsertBlock(disqualifiedBlock, true)
 		if err != nil {
 			t.Fatalf("ValidateAndInsertBlock: %+v", err)
 		}
@@ -106,7 +106,7 @@ func TestBlockStatus(t *testing.T) {
 			disqualifiedBlock.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(invalidBlock, true)
+		err = tc.ValidateAndInsertBlock(invalidBlock, true)
 		if err == nil {
 			t.Fatalf("block is expected to be invalid")
 		}
@@ -139,11 +139,11 @@ func TestValidateAndInsertErrors(t *testing.T) {
 		if err != nil {
 			t.Fatalf("AddBlock: %+v", err)
 		}
-		_, err = tc.ValidateAndInsertBlock(blockWithStatusInvalid, true)
+		err = tc.ValidateAndInsertBlock(blockWithStatusInvalid, true)
 		if err == nil {
 			t.Fatalf("Test ValidateAndInsertBlock: Expected an error, because the block is invalid.")
 		}
-		_, err = tc.ValidateAndInsertBlock(blockWithStatusInvalid, true)
+		err = tc.ValidateAndInsertBlock(blockWithStatusInvalid, true)
 		if err == nil || !errors.Is(err, ruleerrors.ErrKnownInvalid) {
 			t.Fatalf("Expected block to be invalid with err: %v, instead found: %v", ruleerrors.ErrKnownInvalid, err)
 		}
@@ -155,12 +155,12 @@ func TestValidateAndInsertErrors(t *testing.T) {
 		if err != nil {
 			t.Fatalf("AddBlock: %+v", err)
 		}
-		_, err = tc.ValidateAndInsertBlock(block, true)
+		err = tc.ValidateAndInsertBlock(block, true)
 		if err != nil {
 			t.Fatalf("ValidateAndInsertBlock: %+v", err)
 		}
 		// resend the same block.
-		_, err = tc.ValidateAndInsertBlock(block, true)
+		err = tc.ValidateAndInsertBlock(block, true)
 		if err == nil || !errors.Is(err, ruleerrors.ErrDuplicateBlock) {
 			t.Fatalf("Expected block to be invalid with err: %v, instead found: %v", ruleerrors.ErrDuplicateBlock, err)
 		}
@@ -173,12 +173,12 @@ func TestValidateAndInsertErrors(t *testing.T) {
 			t.Fatalf("AddBlock: %+v", err)
 		}
 		onlyHeader.Transactions = []*externalapi.DomainTransaction{}
-		_, err = tc.ValidateAndInsertBlock(onlyHeader, true)
+		err = tc.ValidateAndInsertBlock(onlyHeader, true)
 		if err != nil {
 			t.Fatalf("AddBlock: %+v", err)
 		}
 		// resend the same header.
-		_, err = tc.ValidateAndInsertBlock(onlyHeader, true)
+		err = tc.ValidateAndInsertBlock(onlyHeader, true)
 		if err == nil || !errors.Is(err, ruleerrors.ErrDuplicateBlock) {
 			t.Fatalf("Expected block to be invalid with err: %v, instead found: %v", ruleerrors.ErrDuplicateBlock, err)
 		}

domain/consensus/processes/blockprocessor/validate_and_insert_imported_pruning_point_test.go

@@ -26,7 +26,7 @@ func addBlock(tc testapi.TestConsensus, parentHashes []*externalapi.DomainHash,
 	}
 
 	blockHash := consensushashing.BlockHash(block)
-	_, err = tc.ValidateAndInsertBlock(block, true)
+	err = tc.ValidateAndInsertBlock(block, true)
 	if err != nil {
 		t.Fatalf("ValidateAndInsertBlock: %+v", err)
 	}
@@ -44,7 +44,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 		consensusConfig.K = 0
 		consensusConfig.PruningProofM = 1
 
-		syncConsensuses := func(tcSyncerRef, tcSynceeRef *testapi.TestConsensus) {
+		syncConsensuses := func(tcSyncerRef, tcSynceeRef *testapi.TestConsensus, updatePruningPointJustAfterImportingPruningPoint bool) {
 			tcSyncer, tcSyncee := *tcSyncerRef, *tcSynceeRef
 			pruningPointProof, err := tcSyncer.BuildPruningPointProof()
 			if err != nil {
@@ -133,7 +133,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 					})
 				}
 
-				_, err = synceeStaging.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
+				err = synceeStaging.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
 				if err != nil {
 					t.Fatalf("ValidateAndInsertBlockWithTrustedData: %+v", err)
 				}
@@ -169,7 +169,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 					t.Fatalf("GetBlockHeader: %+v", err)
 				}
 
-				_, err = synceeStaging.ValidateAndInsertBlock(&externalapi.DomainBlock{Header: header}, false)
+				err = synceeStaging.ValidateAndInsertBlock(&externalapi.DomainBlock{Header: header}, false)
 				if err != nil {
 					t.Fatalf("ValidateAndInsertBlock %d: %+v", i, err)
 				}
@@ -236,6 +236,13 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 				t.Fatalf("ValidateAndInsertImportedPruningPoint: %+v", err)
 			}
 
+			if updatePruningPointJustAfterImportingPruningPoint {
+				err = synceeStaging.UpdatePruningPointByVirtual()
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+
 			emptyCoinbase := &externalapi.DomainCoinbaseData{
 				ScriptPublicKey: &externalapi.ScriptPublicKey{
 					Script:  nil,
@@ -266,7 +273,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 					t.Fatalf("GetBlock: %+v", err)
 				}
 
-				_, err = synceeStaging.ValidateAndInsertBlock(block, true)
+				err = synceeStaging.ValidateAndInsertBlock(block, true)
 				if err != nil {
 					t.Fatalf("ValidateAndInsertBlock: %+v", err)
 				}
@@ -292,7 +299,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 				t.Fatalf("GetBlock: %+v", err)
 			}
 
-			_, err = synceeStaging.ValidateAndInsertBlock(tip, true)
+			err = synceeStaging.ValidateAndInsertBlock(tip, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}
@@ -339,7 +346,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 				t.Fatalf("GetBlock: %+v", err)
 			}
 
-			_, err = tcSyncee1.ValidateAndInsertBlock(block, true)
+			err = tcSyncee1.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}
@@ -386,7 +393,7 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 		}
 
 		tcSyncee1Ref := &tcSyncee1
-		syncConsensuses(&tcSyncer, tcSyncee1Ref)
+		syncConsensuses(&tcSyncer, tcSyncee1Ref, false)
 
 		// Test a situation where a consensus with pruned headers syncs another fresh consensus.
 		tcSyncee2, teardownSyncee2, err := factory.NewTestConsensus(consensusConfig, "TestValidateAndInsertPruningPointSyncee2")
@@ -395,7 +402,17 @@ func TestValidateAndInsertImportedPruningPoint(t *testing.T) {
 		}
 		defer teardownSyncee2(false)
 
-		syncConsensuses(tcSyncee1Ref, &tcSyncee2)
+		syncConsensuses(tcSyncee1Ref, &tcSyncee2, false)
+
+		// Check the regular sync but try to update the pruning point after the pruning point was imported. It tests a situation where the node
+		// was restarted before the virtual was resolved and then it calls UpdatePruningPointByVirtual on init.
+		tcSyncee3, teardownSyncee3, err := factory.NewTestConsensus(consensusConfig, "TestValidateAndInsertPruningPointSyncee3")
+		if err != nil {
+			t.Fatalf("Error setting up tcSyncee1: %+v", err)
+		}
+		defer teardownSyncee3(false)
+
+		syncConsensuses(&tcSyncer, &tcSyncee3, true)
 	})
 }
 
@@ -461,7 +478,7 @@ func TestGetPruningPointUTXOs(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Error building block above genesis: %+v", err)
 		}
-		_, err = testConsensus.ValidateAndInsertBlock(blockAboveGenesis, true)
+		err = testConsensus.ValidateAndInsertBlock(blockAboveGenesis, true)
 		if err != nil {
 			t.Fatalf("Error validating and inserting block above genesis: %+v", err)
 		}
@@ -473,7 +490,7 @@ func TestGetPruningPointUTXOs(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Error building block with spendable coinbase: %+v", err)
 		}
-		_, err = testConsensus.ValidateAndInsertBlock(blockWithSpendableCoinbase, true)
+		err = testConsensus.ValidateAndInsertBlock(blockWithSpendableCoinbase, true)
 		if err != nil {
 			t.Fatalf("Error validating and inserting block with spendable coinbase: %+v", err)
 		}
@@ -512,7 +529,7 @@ func TestGetPruningPointUTXOs(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Error building including block: %+v", err)
 		}
-		_, err = testConsensus.ValidateAndInsertBlock(includingBlock, true)
+		err = testConsensus.ValidateAndInsertBlock(includingBlock, true)
 		if err != nil {
 			t.Fatalf("Error validating and inserting including block: %+v", err)
 		}
@@ -523,7 +540,7 @@ func TestGetPruningPointUTXOs(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Error building block: %+v", err)
 			}
-			_, err = testConsensus.ValidateAndInsertBlock(block, true)
+			err = testConsensus.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("Error validating and inserting block: %+v", err)
 			}
@@ -619,7 +636,7 @@ func BenchmarkGetPruningPointUTXOs(b *testing.B) {
 	if err != nil {
 		b.Fatalf("Error building block with spendable coinbase: %+v", err)
 	}
-	_, err = testConsensus.ValidateAndInsertBlock(blockWithSpendableCoinbase, true)
+	err = testConsensus.ValidateAndInsertBlock(blockWithSpendableCoinbase, true)
 	if err != nil {
 		b.Fatalf("Error validating and inserting block with spendable coinbase: %+v", err)
 	}
@@ -657,7 +674,7 @@ func BenchmarkGetPruningPointUTXOs(b *testing.B) {
 		if err != nil {
 			b.Fatalf("Error building block: %+v", err)
 		}
-		_, err = testConsensus.ValidateAndInsertBlock(block, true)
+		err = testConsensus.ValidateAndInsertBlock(block, true)
 		if err != nil {
 			b.Fatalf("Error validating and inserting block: %+v", err)
 		}
@@ -677,7 +694,7 @@ func BenchmarkGetPruningPointUTXOs(b *testing.B) {
 		if err != nil {
 			b.Fatalf("Error building block: %+v", err)
 		}
-		_, err = testConsensus.ValidateAndInsertBlock(block, true)
+		err = testConsensus.ValidateAndInsertBlock(block, true)
 		if err != nil {
 			b.Fatalf("Error validating and inserting block: %+v", err)
 		}

domain/consensus/processes/blockvalidator/block_body_in_context_test.go

@@ -62,7 +62,7 @@ func TestCheckBlockIsNotPruned(t *testing.T) {
 			}
 		}
 
-		_, err = tc.ValidateAndInsertBlock(beforePruningBlock, true)
+		err = tc.ValidateAndInsertBlock(beforePruningBlock, true)
 		if !errors.Is(err, ruleerrors.ErrPrunedBlock) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}
@@ -117,7 +117,7 @@ func TestCheckParentBlockBodiesExist(t *testing.T) {
 		}
 
 		// Add only the header of anticonePruningBlock
-		_, err = tc.ValidateAndInsertBlock(&externalapi.DomainBlock{
+		err = tc.ValidateAndInsertBlock(&externalapi.DomainBlock{
 			Header:       anticonePruningBlock.Header,
 			Transactions: nil,
 		}, true)
@@ -143,7 +143,7 @@ func TestCheckParentBlockBodiesExist(t *testing.T) {
 
 		// Add anticonePruningBlock's body and check that it's valid to point to
 		// a header only block in the past of the pruning point.
-		_, err = tc.ValidateAndInsertBlock(anticonePruningBlock, true)
+		err = tc.ValidateAndInsertBlock(anticonePruningBlock, true)
 		if err != nil {
 			t.Fatalf("ValidateAndInsertBlock: %+v", err)
 		}
@@ -189,7 +189,7 @@ func TestIsFinalizedTransaction(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Error getting block: %+v", err)
 		}
-		_, err = tc.ValidateAndInsertBlock(block, true)
+		err = tc.ValidateAndInsertBlock(block, true)
 		if err != nil {
 			t.Fatalf("Error Inserting block: %+v", err)
 		}

domain/consensus/processes/blockvalidator/block_body_in_isolation_test.go

@@ -1052,14 +1052,14 @@ func CheckBlockHashMerkleRoot(t *testing.T, tc testapi.TestConsensus, consensusC
 	blockWithInvalidMerkleRoot := block.Clone()
 	blockWithInvalidMerkleRoot.Transactions[0].Version += 1
 
-	_, err = tc.ValidateAndInsertBlock(blockWithInvalidMerkleRoot, true)
+	err = tc.ValidateAndInsertBlock(blockWithInvalidMerkleRoot, true)
 	if !errors.Is(err, ruleerrors.ErrBadMerkleRoot) {
 		t.Fatalf("Unexpected error: %+v", err)
 	}
 
 	// Check that a block with invalid merkle root is not marked as invalid
 	// and can be re-added with the right transactions.
-	_, err = tc.ValidateAndInsertBlock(block, true)
+	err = tc.ValidateAndInsertBlock(block, true)
 	if err != nil {
 		t.Fatalf("ValidateAndInsertBlock: %+v", err)
 	}

domain/consensus/processes/blockvalidator/block_header_in_context_test.go

@@ -34,7 +34,7 @@ func TestValidateMedianTime(t *testing.T) {
 			newHeader := block.Header.ToMutable()
 			newHeader.SetTimeInMilliseconds(blockTime)
 			block.Header = newHeader.ToImmutable()
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if !errors.Is(err, expectedErr) {
 				t.Fatalf("expected error %s but got %+v", expectedErr, err)
 			}
@@ -127,7 +127,7 @@ func TestCheckParentsIncest(t *testing.T) {
 			Transactions: nil,
 		}
 
-		_, err = tc.ValidateAndInsertBlock(directParentsRelationBlock, true)
+		err = tc.ValidateAndInsertBlock(directParentsRelationBlock, true)
 		if !errors.Is(err, ruleerrors.ErrInvalidParentsRelation) {
 			t.Fatalf("unexpected error %+v", err)
 		}
@@ -150,7 +150,7 @@ func TestCheckParentsIncest(t *testing.T) {
 			Transactions: nil,
 		}
 
-		_, err = tc.ValidateAndInsertBlock(indirectParentsRelationBlock, true)
+		err = tc.ValidateAndInsertBlock(indirectParentsRelationBlock, true)
 		if !errors.Is(err, ruleerrors.ErrInvalidParentsRelation) {
 			t.Fatalf("unexpected error %+v", err)
 		}

domain/consensus/processes/blockvalidator/block_header_in_isolation_test.go

@@ -78,7 +78,7 @@ func CheckBlockVersion(t *testing.T, tc testapi.TestConsensus, consensusConfig *
 		block.Header.PruningPoint(),
 	)
 
-	_, err = tc.ValidateAndInsertBlock(block, true)
+	err = tc.ValidateAndInsertBlock(block, true)
 	if !errors.Is(err, ruleerrors.ErrWrongBlockVersion) {
 		t.Fatalf("Unexpected error: %+v", err)
 	}
@@ -118,7 +118,7 @@ func CheckBlockTimestampInIsolation(t *testing.T, tc testapi.TestConsensus, cfg
 			block.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(block, true)
+		err = tc.ValidateAndInsertBlock(block, true)
 		if !errors.Is(err, ruleerrors.ErrTimeTooMuchInTheFuture) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}

domain/consensus/processes/blockvalidator/pruning_violation_proof_of_work_and_difficulty_test.go

@@ -39,7 +39,7 @@ func TestPOW(t *testing.T) {
 			t.Fatal(err)
 		}
 		invalidBlockWrongPOW = solveBlockWithWrongPOW(invalidBlockWrongPOW)
-		_, err = tc.ValidateAndInsertBlock(invalidBlockWrongPOW, true)
+		err = tc.ValidateAndInsertBlock(invalidBlockWrongPOW, true)
 		if !errors.Is(err, ruleerrors.ErrInvalidPoW) {
 			t.Fatalf("Expected block to be invalid with err: %v, instead found: %v", ruleerrors.ErrInvalidPoW, err)
 		}
@@ -65,7 +65,7 @@ func TestPOW(t *testing.T) {
 			abovePowMaxBlock.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(abovePowMaxBlock, true)
+		err = tc.ValidateAndInsertBlock(abovePowMaxBlock, true)
 		if !errors.Is(err, ruleerrors.ErrTargetTooHigh) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}
@@ -90,7 +90,7 @@ func TestPOW(t *testing.T) {
 			negativeTargetBlock.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(negativeTargetBlock, true)
+		err = tc.ValidateAndInsertBlock(negativeTargetBlock, true)
 		if !errors.Is(err, ruleerrors.ErrNegativeTarget) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}
@@ -104,7 +104,7 @@ func TestPOW(t *testing.T) {
 		// Difficulty is too high on mainnet to actually mine.
 		if consensusConfig.Name != "kaspa-mainnet" {
 			mining.SolveBlock(validBlock, random)
-			_, err = tc.ValidateAndInsertBlock(validBlock, true)
+			err = tc.ValidateAndInsertBlock(validBlock, true)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -161,7 +161,7 @@ func TestCheckParentHeadersExist(t *testing.T) {
 			orphanBlock.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(orphanBlock, true)
+		err = tc.ValidateAndInsertBlock(orphanBlock, true)
 		errMissingParents := &ruleerrors.ErrMissingParents{}
 		if !errors.As(err, errMissingParents) {
 			t.Fatalf("Unexpected error: %+v", err)
@@ -193,7 +193,7 @@ func TestCheckParentHeadersExist(t *testing.T) {
 			orphanBlock.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(invalidBlock, true)
+		err = tc.ValidateAndInsertBlock(invalidBlock, true)
 		if !errors.Is(err, ruleerrors.ErrTransactionVersionIsUnknown) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}
@@ -220,7 +220,7 @@ func TestCheckParentHeadersExist(t *testing.T) {
 			invalidBlockChild.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(invalidBlockChild, true)
+		err = tc.ValidateAndInsertBlock(invalidBlockChild, true)
 		if !errors.Is(err, ruleerrors.ErrInvalidAncestorBlock) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}
@@ -284,7 +284,7 @@ func TestCheckPruningPointViolation(t *testing.T) {
 			blockWithPruningViolation.Header.PruningPoint(),
 		)
 
-		_, err = tc.ValidateAndInsertBlock(blockWithPruningViolation, true)
+		err = tc.ValidateAndInsertBlock(blockWithPruningViolation, true)
 		if !errors.Is(err, ruleerrors.ErrPruningPointViolation) {
 			t.Fatalf("Unexpected error: %+v", err)
 		}

domain/consensus/processes/consensusstatemanager/resolve.go

@@ -5,7 +5,6 @@ import (
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"github.com/kaspanet/kaspad/util/staging"
-	"github.com/pkg/errors"
 	"sort"
 )
 
@@ -79,12 +78,7 @@ func (csm *consensusStateManager) ResolveVirtual(maxBlocksToResolve uint64) (*ex
 
 			if reversalData != nil {
 				err = csm.ReverseUTXODiffs(resolveTip, reversalData)
-				// It's still not known what causes this error, but we can ignore it and not reverse the UTXO diffs
+				if err != nil {
-				// and harm performance in some cases.
-				// TODO: Investigate why this error happens in the first place, and remove the workaround.
-				if errors.Is(err, ErrReverseUTXODiffsUTXODiffChildNotFound) {
-					log.Errorf("Could not reverse UTXO diffs while resolving virtual: %s", err)
-				} else if err != nil {
 					return nil, false, err
 				}
 			}

domain/consensus/processes/consensusstatemanager/resolve_virtual_test.go

@@ -0,0 +1,214 @@
+package consensusstatemanager_test
+
+import (
+	"github.com/kaspanet/kaspad/domain/consensus/utils/consensushashing"
+	"testing"
+
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+
+	"github.com/kaspanet/kaspad/domain/consensus"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/testutils"
+)
+
+func TestAddBlockBetweenResolveVirtualCalls(t *testing.T) {
+
+	testutils.ForAllNets(t, true, func(t *testing.T, consensusConfig *consensus.Config) {
+		factory := consensus.NewFactory()
+
+		tc, teardown, err := factory.NewTestConsensus(consensusConfig, "TestAddBlockBetweenResolveVirtualCalls")
+		if err != nil {
+			t.Fatalf("Error setting up consensus: %+v", err)
+		}
+		defer teardown(false)
+
+		// Create a chain of blocks
+		const initialChainLength = 10
+		previousBlockHash := consensusConfig.GenesisHash
+		for i := 0; i < initialChainLength; i++ {
+			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
+			}
+		}
+
+		// Mine a chain with more blocks, to re-organize the DAG
+		const reorgChainLength = initialChainLength + 1
+		previousBlockHash = consensusConfig.GenesisHash
+		for i := 0; i < reorgChainLength; i++ {
+			previousBlock, _, err := tc.BuildBlockWithParents([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
+			}
+			previousBlockHash = consensushashing.BlockHash(previousBlock)
+
+			// Do not UTXO validate in order to resolve virtual later
+			err = tc.ValidateAndInsertBlock(previousBlock, false)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
+			}
+		}
+
+		// Resolve one step
+		_, err = tc.ResolveVirtualWithMaxParam(2)
+		if err != nil {
+			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+		}
+
+		emptyCoinbase := &externalapi.DomainCoinbaseData{
+			ScriptPublicKey: &externalapi.ScriptPublicKey{
+				Script:  nil,
+				Version: 0,
+			},
+		}
+
+		// Get template based on current resolve state
+		blockTemplate, err := tc.BuildBlockTemplate(emptyCoinbase, nil)
+		if err != nil {
+			t.Fatalf("Error building block template during virtual resolution of reorg: %+v", err)
+		}
+
+		// Resolve one more step
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
+		if err != nil {
+			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+		}
+
+		// Add the mined block (now virtual was modified)
+		err = tc.ValidateAndInsertBlock(blockTemplate.Block, true)
+		if err != nil {
+			t.Fatalf("Error mining block during virtual resolution of reorg: %+v", err)
+		}
+
+		// Complete resolving virtual
+		for !isCompletelyResolved {
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
+			if err != nil {
+				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+			}
+		}
+	})
+}
+
+func TestAddGenesisChildAfterOneResolveVirtualCall(t *testing.T) {
+
+	testutils.ForAllNets(t, true, func(t *testing.T, consensusConfig *consensus.Config) {
+		factory := consensus.NewFactory()
+
+		tc, teardown, err := factory.NewTestConsensus(consensusConfig, "TestAddGenesisChildAfterOneResolveVirtualCall")
+		if err != nil {
+			t.Fatalf("Error setting up consensus: %+v", err)
+		}
+		defer teardown(false)
+
+		// Create a chain of blocks
+		const initialChainLength = 6
+		previousBlockHash := consensusConfig.GenesisHash
+		for i := 0; i < initialChainLength; i++ {
+			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
+			}
+		}
+
+		// Mine a chain with more blocks, to re-organize the DAG
+		const reorgChainLength = initialChainLength + 1
+		previousBlockHash = consensusConfig.GenesisHash
+		for i := 0; i < reorgChainLength; i++ {
+			previousBlock, _, err := tc.BuildBlockWithParents([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
+			}
+			previousBlockHash = consensushashing.BlockHash(previousBlock)
+
+			// Do not UTXO validate in order to resolve virtual later
+			err = tc.ValidateAndInsertBlock(previousBlock, false)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
+			}
+		}
+
+		// Resolve one step
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
+		if err != nil {
+			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+		}
+
+		_, _, err = tc.AddBlock([]*externalapi.DomainHash{consensusConfig.GenesisHash}, nil, nil)
+		if err != nil {
+			t.Fatalf("Error adding block during virtual resolution of reorg: %+v", err)
+		}
+
+		// Complete resolving virtual
+		for !isCompletelyResolved {
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
+			if err != nil {
+				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+			}
+		}
+	})
+}
+
+func TestAddGenesisChildAfterTwoResolveVirtualCalls(t *testing.T) {
+
+	testutils.ForAllNets(t, true, func(t *testing.T, consensusConfig *consensus.Config) {
+		factory := consensus.NewFactory()
+
+		tc, teardown, err := factory.NewTestConsensus(consensusConfig, "TestAddGenesisChildAfterTwoResolveVirtualCalls")
+		if err != nil {
+			t.Fatalf("Error setting up consensus: %+v", err)
+		}
+		defer teardown(false)
+
+		// Create a chain of blocks
+		const initialChainLength = 6
+		previousBlockHash := consensusConfig.GenesisHash
+		for i := 0; i < initialChainLength; i++ {
+			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
+			}
+		}
+
+		// Mine a chain with more blocks, to re-organize the DAG
+		const reorgChainLength = initialChainLength + 1
+		previousBlockHash = consensusConfig.GenesisHash
+		for i := 0; i < reorgChainLength; i++ {
+			previousBlock, _, err := tc.BuildBlockWithParents([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
+			}
+			previousBlockHash = consensushashing.BlockHash(previousBlock)
+
+			// Do not UTXO validate in order to resolve virtual later
+			err = tc.ValidateAndInsertBlock(previousBlock, false)
+			if err != nil {
+				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
+			}
+		}
+
+		// Resolve one step
+		_, err = tc.ResolveVirtualWithMaxParam(2)
+		if err != nil {
+			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+		}
+
+		// Resolve one more step
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
+		if err != nil {
+			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+		}
+
+		_, _, err = tc.AddBlock([]*externalapi.DomainHash{consensusConfig.GenesisHash}, nil, nil)
+		if err != nil {
+			t.Fatalf("Error adding block during virtual resolution of reorg: %+v", err)
+		}
+
+		// Complete resolving virtual
+		for !isCompletelyResolved {
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
+			if err != nil {
+				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
+			}
+		}
+	})
+}

domain/consensus/processes/consensusstatemanager/reverse_utxo_diffs.go

@@ -3,18 +3,10 @@ package consensusstatemanager
 import (
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
-	"github.com/kaspanet/kaspad/infrastructure/db/database"
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"github.com/kaspanet/kaspad/util/staging"
-	"github.com/pkg/errors"
 )
 
-// ErrReverseUTXODiffsUTXODiffChildNotFound indicates a UTXO diff child was not found while calling ReverseUTXODiffs.
-// It's still not known what causes this error, but we can ignore it and not reverse the UTXO diffs
-// and harm performance in some cases.
-// TODO: Investigate why this error happens in the first place, and remove the workaround.
-var ErrReverseUTXODiffsUTXODiffChildNotFound = errors.New("ErrReverseUTXODiffsUTXODiffChildNotFound")
-
 func (csm *consensusStateManager) ReverseUTXODiffs(tipHash *externalapi.DomainHash,
 	reversalData *model.UTXODiffReversalData) error {
 
@@ -57,9 +49,6 @@ func (csm *consensusStateManager) ReverseUTXODiffs(tipHash *externalapi.DomainHa
 
 		currentBlockUTXODiffChild, err := csm.utxoDiffStore.UTXODiffChild(csm.databaseContext, readStagingArea, currentBlock)
 		if err != nil {
-			if database.IsNotFoundError(err) {
-				return errors.Wrapf(ErrReverseUTXODiffsUTXODiffChildNotFound, "UTXO diff child was not found for block %s", currentBlock)
-			}
 			return err
 		}
 		currentBlockGHOSTDAGData, err := csm.ghostdagDataStore.Get(csm.databaseContext, readStagingArea, currentBlock, false)

domain/consensus/processes/consensusstatemanager/virtual_parents_test.go

@@ -84,7 +84,7 @@ func TestConsensusStateManager_pickVirtualParents(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Failed building a block: %v", err)
 			}
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("Failed Inserting block to tc: %v", err)
 			}

domain/consensus/processes/difficultymanager/difficultymanager_test.go

@@ -65,7 +65,7 @@ func TestDifficulty(t *testing.T) {
 			newHeader := block.Header.ToMutable()
 			newHeader.SetTimeInMilliseconds(blockTime)
 			block.Header = newHeader.ToImmutable()
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}

domain/consensus/processes/pastmediantimemanager/pastmediantimemanager_test.go

@@ -34,7 +34,7 @@ func TestPastMedianTime(t *testing.T) {
 			newHeader := block.Header.ToMutable()
 			newHeader.SetTimeInMilliseconds(blockTime)
 			block.Header = newHeader.ToImmutable()
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}

domain/consensus/processes/pruningmanager/pruningmanager.go

@@ -191,6 +191,46 @@ func (pm *pruningManager) UpdatePruningPointByVirtual(stagingArea *model.Staging
 	return nil
 }
 
+type blockIteratorFromOneBlock struct {
+	done, isClosed bool
+	hash           *externalapi.DomainHash
+}
+
+func (b *blockIteratorFromOneBlock) First() bool {
+	if b.isClosed {
+		panic("Tried using a closed blockIteratorFromOneBlock")
+	}
+
+	b.done = false
+	return true
+}
+
+func (b *blockIteratorFromOneBlock) Next() bool {
+	if b.isClosed {
+		panic("Tried using a closed blockIteratorFromOneBlock")
+	}
+
+	b.done = true
+	return false
+}
+
+func (b *blockIteratorFromOneBlock) Get() (*externalapi.DomainHash, error) {
+	if b.isClosed {
+		panic("Tried using a closed blockIteratorFromOneBlock")
+	}
+
+	return b.hash, nil
+}
+
+func (b *blockIteratorFromOneBlock) Close() error {
+	if b.isClosed {
+		panic("Tried using a closed blockIteratorFromOneBlock")
+	}
+
+	b.isClosed = true
+	return nil
+}
+
 func (pm *pruningManager) nextPruningPointAndCandidateByBlockHash(stagingArea *model.StagingArea,
 	blockHash, suggestedLowHash *externalapi.DomainHash) (*externalapi.DomainHash, *externalapi.DomainHash, error) {
 
@@ -222,12 +262,12 @@ func (pm *pruningManager) nextPruningPointAndCandidateByBlockHash(stagingArea *m
 		}
 	}
 
-	ghostdagData, err := pm.ghostdagDataStore.Get(pm.databaseContext, stagingArea, blockHash, false)
+	currentPruningPoint, err := pm.pruningStore.PruningPoint(pm.databaseContext, stagingArea)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	currentPruningPoint, err := pm.pruningStore.PruningPoint(pm.databaseContext, stagingArea)
+	ghostdagData, err := pm.ghostdagDataStore.Get(pm.databaseContext, stagingArea, blockHash, false)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -240,10 +280,15 @@ func (pm *pruningManager) nextPruningPointAndCandidateByBlockHash(stagingArea *m
 	// We iterate until the selected parent of the given block, in order to allow a situation where the given block hash
 	// belongs to the virtual. This shouldn't change anything since the max blue score difference between a block and its
 	// selected parent is K, and K << pm.pruningDepth.
-	iterator, err := pm.dagTraversalManager.SelectedChildIterator(stagingArea, ghostdagData.SelectedParent(), lowHash, true)
+	var iterator model.BlockIterator
+	if blockHash.Equal(lowHash) {
+		iterator = &blockIteratorFromOneBlock{hash: lowHash}
+	} else {
+		iterator, err = pm.dagTraversalManager.SelectedChildIterator(stagingArea, ghostdagData.SelectedParent(), lowHash, true)
 		if err != nil {
 			return nil, nil, err
 		}
+	}
 	defer iterator.Close()
 
 	// Finding the next pruning point candidate: look for the latest

domain/consensus/test_consensus.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/hashset"
+	"github.com/kaspanet/kaspad/util/staging"
 	"io"
 
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
@@ -60,7 +61,7 @@ func (tc *testConsensus) AddBlock(parentHashes []*externalapi.DomainHash, coinba
 		return nil, nil, err
 	}
 
-	virtualChangeSet, _, err := tc.blockProcessor.ValidateAndInsertBlock(block, true)
+	virtualChangeSet, err := tc.validateAndInsertBlockNoLock(block, true)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -80,7 +81,7 @@ func (tc *testConsensus) AddUTXOInvalidHeader(parentHashes []*externalapi.Domain
 		return nil, nil, err
 	}
 
-	virtualChangeSet, _, err := tc.blockProcessor.ValidateAndInsertBlock(&externalapi.DomainBlock{
+	virtualChangeSet, err := tc.validateAndInsertBlockNoLock(&externalapi.DomainBlock{
 		Header:       header,
 		Transactions: nil,
 	}, true)
@@ -103,7 +104,7 @@ func (tc *testConsensus) AddUTXOInvalidBlock(parentHashes []*externalapi.DomainH
 		return nil, nil, err
 	}
 
-	virtualChangeSet, _, err := tc.blockProcessor.ValidateAndInsertBlock(block, true)
+	virtualChangeSet, err := tc.validateAndInsertBlockNoLock(block, true)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -111,6 +112,13 @@ func (tc *testConsensus) AddUTXOInvalidBlock(parentHashes []*externalapi.DomainH
 	return consensushashing.BlockHash(block), virtualChangeSet, nil
 }
 
+func (tc *testConsensus) ResolveVirtualWithMaxParam(maxBlocksToResolve uint64) (bool, error) {
+	tc.lock.Lock()
+	defer tc.lock.Unlock()
+
+	return tc.resolveVirtualNoLock(maxBlocksToResolve)
+}
+
 // jsonBlock is a json representation of a block in mine format
 type jsonBlock struct {
 	ID      string   `json:"id"`
@@ -258,3 +266,16 @@ func (tc *testConsensus) BuildHeaderWithParents(parentHashes []*externalapi.Doma
 
 	return tc.testBlockBuilder.BuildUTXOInvalidHeader(parentHashes)
 }
+
+func (tc *testConsensus) UpdatePruningPointByVirtual() error {
+	tc.lock.Lock()
+	defer tc.lock.Unlock()
+
+	stagingArea := model.NewStagingArea()
+	err := tc.pruningManager.UpdatePruningPointByVirtual(stagingArea)
+	if err != nil {
+		return err
+	}
+
+	return staging.CommitAllChanges(tc.databaseContext, stagingArea)
+}

domain/consensus/timelock_CLTV_test.go

@@ -318,7 +318,7 @@ func TestCheckLockTimeVerifyConditionedByAbsoluteTime(t *testing.T) {
 			blockHeader := tipBlock.Header.ToMutable()
 			blockHeader.SetTimeInMilliseconds(timeStampBlockE + i*1000)
 			tipBlock.Header = blockHeader.ToImmutable()
-			_, err = testConsensus.ValidateAndInsertBlock(tipBlock, true)
+			err = testConsensus.ValidateAndInsertBlock(tipBlock, true)
 			if err != nil {
 				t.Fatalf("Error validating and inserting tip block: %v", err)
 			}
@@ -439,7 +439,7 @@ func TestCheckLockTimeVerifyConditionedByAbsoluteTimeWithWrongLockTime(t *testin
 			blockHeader := tipBlock.Header.ToMutable()
 			blockHeader.SetTimeInMilliseconds(timeStampBlockE + i*1000)
 			tipBlock.Header = blockHeader.ToImmutable()
-			_, err = testConsensus.ValidateAndInsertBlock(tipBlock, true)
+			err = testConsensus.ValidateAndInsertBlock(tipBlock, true)
 			if err != nil {
 				t.Fatalf("Error validating and inserting tip block: %v", err)
 			}

domain/domain_test.go

@@ -57,7 +57,7 @@ func TestCreateStagingConsensus(t *testing.T) {
 					},
 				},
 			}
-			_, err = domainInstance.StagingConsensus().ValidateAndInsertBlockWithTrustedData(genesisWithTrustedData, true)
+			err = domainInstance.StagingConsensus().ValidateAndInsertBlockWithTrustedData(genesisWithTrustedData, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlockWithTrustedData: %+v", err)
 			}
@@ -74,7 +74,7 @@ func TestCreateStagingConsensus(t *testing.T) {
 			t.Fatalf("BuildBlock: %+v", err)
 		}
 
-		_, err = domainInstance.StagingConsensus().ValidateAndInsertBlock(block, true)
+		err = domainInstance.StagingConsensus().ValidateAndInsertBlock(block, true)
 		if err != nil {
 			t.Fatalf("ValidateAndInsertBlock: %+v", err)
 		}
@@ -120,7 +120,7 @@ func TestCreateStagingConsensus(t *testing.T) {
 		}
 
 		addGenesisToStagingConsensus()
-		_, err = domainInstance.StagingConsensus().ValidateAndInsertBlock(block, true)
+		err = domainInstance.StagingConsensus().ValidateAndInsertBlock(block, true)
 		if err != nil {
 			t.Fatalf("ValidateAndInsertBlock: %+v", err)
 		}

domain/migrate.go

@@ -106,7 +106,7 @@ func syncConsensuses(syncer, syncee externalapi.Consensus) error {
 			})
 		}
 
-		_, err = syncee.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
+		err = syncee.ValidateAndInsertBlockWithTrustedData(blockWithTrustedData, false)
 		if err != nil {
 			return err
 		}
@@ -161,7 +161,7 @@ func syncConsensuses(syncer, syncee externalapi.Consensus) error {
 			return err
 		}
 
-		_, err = syncee.ValidateAndInsertBlock(block, false)
+		err = syncee.ValidateAndInsertBlock(block, false)
 		if err != nil {
 			return err
 		}
@@ -232,7 +232,7 @@ func syncConsensuses(syncer, syncee externalapi.Consensus) error {
 				log.Infof("Resolving virtual. Estimated progress: %d%%", percents)
 			}
 		}
-		_, isCompletelyResolved, err := syncee.ResolveVirtual()
+		isCompletelyResolved, err := syncee.ResolveVirtual()
 		if err != nil {
 			return err
 		}

domain/miningmanager/mempool/mempool.go

@@ -56,6 +56,20 @@ func (mp *mempool) AllTransactions() []*externalapi.DomainTransaction {
 	return mp.transactionsPool.getAllTransactions()
 }
 
+func (mp *mempool) GetOrphanTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool) {
+	mp.mtx.RLock()
+	defer mp.mtx.RUnlock()
+
+	return mp.orphansPool.getOrphanTransaction(transactionID)
+}
+
+func (mp *mempool) AllOrphanTransactions() []*externalapi.DomainTransaction {
+	mp.mtx.RLock()
+	defer mp.mtx.RUnlock()
+
+	return mp.orphansPool.getAllOrphanTransactions()
+}
+
 func (mp *mempool) TransactionCount() int {
 	mp.mtx.RLock()
 	defer mp.mtx.RUnlock()

domain/miningmanager/mempool/orphan_pool.go

@@ -328,3 +328,18 @@ func (op *orphansPool) randomNonHighPriorityOrphan() *model.OrphanTransaction {
 
 	return nil
 }
+
+func (op *orphansPool) getOrphanTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool) {
+	if orphanTransaction, ok := op.allOrphans[*transactionID]; ok {
+		return orphanTransaction.Transaction(), true
+	}
+	return nil, false
+}
+
+func (op *orphansPool) getAllOrphanTransactions() []*externalapi.DomainTransaction {
+	allOrphanTransactions := make([]*externalapi.DomainTransaction, 0, len(op.allOrphans))
+	for _, mempoolTransaction := range op.allOrphans {
+		allOrphanTransactions = append(allOrphanTransactions, mempoolTransaction.Transaction())
+	}
+	return allOrphanTransactions
+}

domain/miningmanager/miningmanager.go

@@ -1,11 +1,12 @@
 package miningmanager
 
 import (
+	"sync"
+	"time"
+
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensusreference"
 	miningmanagermodel "github.com/kaspanet/kaspad/domain/miningmanager/model"
-	"sync"
-	"time"
 )
 
 // MiningManager creates block templates for mining as well as maintaining
@@ -16,6 +17,8 @@ type MiningManager interface {
 	GetBlockTemplateBuilder() miningmanagermodel.BlockTemplateBuilder
 	GetTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool)
 	AllTransactions() []*externalapi.DomainTransaction
+	GetOrphanTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool)
+	AllOrphanTransactions() []*externalapi.DomainTransaction
 	TransactionCount() int
 	HandleNewBlockTransactions(txs []*externalapi.DomainTransaction) ([]*externalapi.DomainTransaction, error)
 	ValidateAndInsertTransaction(transaction *externalapi.DomainTransaction, isHighPriority bool, allowOrphan bool) (
@@ -115,6 +118,17 @@ func (mm *miningManager) AllTransactions() []*externalapi.DomainTransaction {
 	return mm.mempool.AllTransactions()
 }
 
+func (mm *miningManager) GetOrphanTransaction(
+	transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool) {
+
+	return mm.mempool.GetOrphanTransaction(transactionID)
+}
+
+func (mm *miningManager) AllOrphanTransactions() []*externalapi.DomainTransaction {
+
+	return mm.mempool.AllOrphanTransactions()
+}
+
 func (mm *miningManager) TransactionCount() int {
 	return mm.mempool.TransactionCount()
 }

domain/miningmanager/model/interface_mempool.go

@@ -14,6 +14,8 @@ type Mempool interface {
 	RemoveTransactions(txs []*externalapi.DomainTransaction, removeRedeemers bool) error
 	GetTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool)
 	AllTransactions() []*externalapi.DomainTransaction
+	GetOrphanTransaction(transactionID *externalapi.DomainTransactionID) (*externalapi.DomainTransaction, bool)
+	AllOrphanTransactions() []*externalapi.DomainTransaction
 	TransactionCount() int
 	RevalidateHighPriorityTransactions() (validTransactions []*externalapi.DomainTransaction, err error)
 	IsTransactionOutputDust(output *externalapi.DomainTransactionOutput) bool

infrastructure/config/config.go

@@ -39,7 +39,8 @@ const (
 	defaultBanThreshold        = 100
 	//DefaultConnectTimeout is the default connection timeout when dialing
 	DefaultConnectTimeout = time.Second * 30
-	defaultMaxRPCClients         = 10
+	//DefaultMaxRPCClients is the default max number of RPC clients
+	DefaultMaxRPCClients         = 128
 	defaultMaxRPCWebsockets      = 25
 	defaultMaxRPCConcurrentReqs  = 20
 	defaultBlockMaxMass          = 10_000_000
@@ -178,7 +179,7 @@ func defaultFlags() *Flags {
 		MaxInboundPeers:      defaultMaxInboundPeers,
 		BanDuration:          defaultBanDuration,
 		BanThreshold:         defaultBanThreshold,
-		RPCMaxClients:        defaultMaxRPCClients,
+		RPCMaxClients:        DefaultMaxRPCClients,
 		RPCMaxWebsockets:     defaultMaxRPCWebsockets,
 		RPCMaxConcurrentReqs: defaultMaxRPCConcurrentReqs,
 		AppDir:               defaultDataDir,

infrastructure/network/netadapter/netadapter.go

@@ -46,7 +46,7 @@ func NewNetAdapter(cfg *config.Config) (*NetAdapter, error) {
 	if err != nil {
 		return nil, err
 	}
-	rpcServer, err := grpcserver.NewRPCServer(cfg.RPCListeners)
+	rpcServer, err := grpcserver.NewRPCServer(cfg.RPCListeners, cfg.RPCMaxClients)
 	if err != nil {
 		return nil, err
 	}

infrastructure/network/netadapter/server/grpcserver/grpc_server.go

@@ -94,10 +94,11 @@ func (s *gRPCServer) SetOnConnectedHandler(onConnectedHandler server.OnConnected
 }
 
 func (s *gRPCServer) handleInboundConnection(ctx context.Context, stream grpcStream) error {
-	err := s.incrementInboundConnectionCountAndLimitIfRequired()
+	connectionCount, err := s.incrementInboundConnectionCountAndLimitIfRequired()
 	if err != nil {
 		return err
 	}
+	defer s.decrementInboundConnectionCount()
 
 	peerInfo, ok := peer.FromContext(ctx)
 	if !ok {
@@ -115,23 +116,23 @@ func (s *gRPCServer) handleInboundConnection(ctx context.Context, stream grpcStr
 		return err
 	}
 
-	log.Infof("%s Incoming connection from %s", s.name, peerInfo.Addr)
+	log.Infof("%s Incoming connection from %s #%d", s.name, peerInfo.Addr, connectionCount)
 
 	<-connection.stopChan
-	s.decrementInboundConnectionCount()
 	return nil
 }
 
-func (s *gRPCServer) incrementInboundConnectionCountAndLimitIfRequired() error {
+func (s *gRPCServer) incrementInboundConnectionCountAndLimitIfRequired() (int, error) {
 	s.inboundConnectionCountLock.Lock()
 	defer s.inboundConnectionCountLock.Unlock()
 
 	if s.maxInboundConnections > 0 && s.inboundConnectionCount == s.maxInboundConnections {
-		return errors.Errorf("limit of %d inbound connections has been exceeded", s.maxInboundConnections)
+		log.Warnf("Limit of %d %s inbound connections has been exceeded", s.maxInboundConnections, s.name)
+		return s.inboundConnectionCount, errors.Errorf("limit of %d %s inbound connections has been exceeded", s.maxInboundConnections, s.name)
 	}
 
 	s.inboundConnectionCount++
-	return nil
+	return s.inboundConnectionCount, nil
 }
 
 func (s *gRPCServer) decrementInboundConnectionCount() {

infrastructure/network/netadapter/server/grpcserver/protowire/rpc.md

@@ -609,6 +609,8 @@ in the mempool.
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
 | txId | [string](#string) |  | The transaction's TransactionID. |
+| includeOrphanPool | [bool](#bool) |  |  |
+| filterTransactionPool | [bool](#bool) |  |  |
 
 
 
@@ -638,6 +640,12 @@ GetMempoolEntriesRequestMessage requests information about all the transactions
 currently in the mempool.
 
 
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| includeOrphanPool | [bool](#bool) |  |  |
+| filterTransactionPool | [bool](#bool) |  |  |
+
+
 
 
 
@@ -668,6 +676,7 @@ currently in the mempool.
 | ----- | ---- | ----- | ----------- |
 | fee | [uint64](#uint64) |  |  |
 | transaction | [RpcTransaction](#protowire.RpcTransaction) |  |  |
+| isOrphan | [bool](#bool) |  |  |
 
 
 
@@ -1798,6 +1807,8 @@ See NotifyNewBlockTemplateRequestMessage
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
 | addresses | [string](#string) | repeated |  |
+| includeOrphanPool | [bool](#bool) |  |  |
+| filterTransactionPool | [bool](#bool) |  |  |
 
 
 

infrastructure/network/netadapter/server/grpcserver/protowire/rpc.proto

@@ -215,6 +215,8 @@ message GetSelectedTipHashResponseMessage{
 message GetMempoolEntryRequestMessage{
   // The transaction's TransactionID.
   string txId = 1;
+  bool includeOrphanPool = 2;
+  bool filterTransactionPool = 3;
 }
 
 message GetMempoolEntryResponseMessage{
@@ -226,6 +228,8 @@ message GetMempoolEntryResponseMessage{
 // GetMempoolEntriesRequestMessage requests information about all the transactions
 // currently in the mempool.
 message GetMempoolEntriesRequestMessage{
+        bool includeOrphanPool = 1;
+        bool filterTransactionPool = 2;
 }
 
 message GetMempoolEntriesResponseMessage{
@@ -237,6 +241,7 @@ message GetMempoolEntriesResponseMessage{
 message MempoolEntry{
   uint64 fee = 1;
   RpcTransaction transaction = 3;
+  bool isOrphan = 4;
 }
 
 // GetConnectedPeerInfoRequestMessage requests information about all the p2p peers
@@ -697,6 +702,8 @@ message MempoolEntryByAddress{
 
 message GetMempoolEntriesByAddressesRequestMessage{
   repeated string addresses = 1;
+  bool includeOrphanPool = 2;
+  bool filterTransactionPool = 3;
 }
 
 message  GetMempoolEntriesByAddressesResponseMessage{

infrastructure/network/netadapter/server/grpcserver/protowire/rpc_get_mempool_entries.go

@@ -9,14 +9,27 @@ func (x *KaspadMessage_GetMempoolEntriesRequest) toAppMessage() (appmessage.Mess
 	if x == nil {
 		return nil, errors.Wrapf(errorNil, "KaspadMessage_GetMempoolEntriesRequest is nil")
 	}
-	return &appmessage.GetMempoolEntriesRequestMessage{}, nil
+	return x.GetMempoolEntriesRequest.toAppMessage()
 }
 
-func (x *KaspadMessage_GetMempoolEntriesRequest) fromAppMessage(_ *appmessage.GetMempoolEntriesRequestMessage) error {
+func (x *KaspadMessage_GetMempoolEntriesRequest) fromAppMessage(message *appmessage.GetMempoolEntriesRequestMessage) error {
-	x.GetMempoolEntriesRequest = &GetMempoolEntriesRequestMessage{}
+	x.GetMempoolEntriesRequest = &GetMempoolEntriesRequestMessage{
+		IncludeOrphanPool:     message.IncludeOrphanPool,
+		FilterTransactionPool: message.FilterTransactionPool,
+	}
 	return nil
 }
 
+func (x *GetMempoolEntriesRequestMessage) toAppMessage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "GetMempoolEntryRequestMessage is nil")
+	}
+	return &appmessage.GetMempoolEntriesRequestMessage{
+		IncludeOrphanPool:     x.IncludeOrphanPool,
+		FilterTransactionPool: x.FilterTransactionPool,
+	}, nil
+}
+
 func (x *KaspadMessage_GetMempoolEntriesResponse) toAppMessage() (appmessage.Message, error) {
 	if x == nil {
 		return nil, errors.Wrapf(errorNil, "KaspadMessage_GetMempoolEntriesResponse is nil")

infrastructure/network/netadapter/server/grpcserver/protowire/rpc_get_mempool_entries_by_addresses.go

@@ -7,7 +7,7 @@ import (
 
 func (x *KaspadMessage_GetMempoolEntriesByAddressesRequest) toAppMessage() (appmessage.Message, error) {
 	if x == nil {
-		return nil, errors.Wrapf(errorNil, "KaspadMessage_GetMempoolEntriesRequest is nil")
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_KaspadMessage_GetMempoolEntriesByAddressesRequest is nil")
 	}
 	return x.GetMempoolEntriesByAddressesRequest.toAppMessage()
 }
@@ -15,6 +15,8 @@ func (x *KaspadMessage_GetMempoolEntriesByAddressesRequest) toAppMessage() (appm
 func (x *KaspadMessage_GetMempoolEntriesByAddressesRequest) fromAppMessage(message *appmessage.GetMempoolEntriesByAddressesRequestMessage) error {
 	x.GetMempoolEntriesByAddressesRequest = &GetMempoolEntriesByAddressesRequestMessage{
 		Addresses:             message.Addresses,
+		IncludeOrphanPool:     message.IncludeOrphanPool,
+		FilterTransactionPool: message.FilterTransactionPool,
 	}
 	return nil
 }
@@ -25,6 +27,8 @@ func (x *GetMempoolEntriesByAddressesRequestMessage) toAppMessage() (appmessage.
 	}
 	return &appmessage.GetMempoolEntriesByAddressesRequestMessage{
 		Addresses:             x.Addresses,
+		IncludeOrphanPool:     x.IncludeOrphanPool,
+		FilterTransactionPool: x.FilterTransactionPool,
 	}, nil
 }
 

infrastructure/network/netadapter/server/grpcserver/protowire/rpc_get_mempool_entry.go

@@ -15,6 +15,8 @@ func (x *KaspadMessage_GetMempoolEntryRequest) toAppMessage() (appmessage.Messag
 func (x *KaspadMessage_GetMempoolEntryRequest) fromAppMessage(message *appmessage.GetMempoolEntryRequestMessage) error {
 	x.GetMempoolEntryRequest = &GetMempoolEntryRequestMessage{
 		TxId:                  message.TxID,
+		IncludeOrphanPool:     message.IncludeOrphanPool,
+		FilterTransactionPool: message.FilterTransactionPool,
 	}
 	return nil
 }
@@ -25,6 +27,8 @@ func (x *GetMempoolEntryRequestMessage) toAppMessage() (appmessage.Message, erro
 	}
 	return &appmessage.GetMempoolEntryRequestMessage{
 		TxID:                  x.TxId,
+		IncludeOrphanPool:     x.IncludeOrphanPool,
+		FilterTransactionPool: x.FilterTransactionPool,
 	}, nil
 }
 
@@ -91,6 +95,7 @@ func (x *MempoolEntry) toAppMessage() (*appmessage.MempoolEntry, error) {
 	return &appmessage.MempoolEntry{
 		Fee:         x.Fee,
 		Transaction: transaction,
+		IsOrphan:    x.IsOrphan,
 	}, nil
 }
 
@@ -103,6 +108,7 @@ func (x *MempoolEntry) fromAppMessage(message *appmessage.MempoolEntry) error {
 	*x = MempoolEntry{
 		Fee:         message.Fee,
 		Transaction: transaction,
+		IsOrphan:    message.IsOrphan,
 	}
 	return nil
 }

infrastructure/network/netadapter/server/grpcserver/rpcserver.go

@@ -14,12 +14,9 @@ type rpcServer struct {
 // RPCMaxMessageSize is the max message size for the RPC server to send and receive
 const RPCMaxMessageSize = 1024 * 1024 * 1024 // 1 GB
 
-// RPCMaxInboundConnections is the max amount of inbound connections for the RPC server
-const RPCMaxInboundConnections = 128
-
 // NewRPCServer creates a new RPCServer
-func NewRPCServer(listeningAddresses []string) (server.Server, error) {
+func NewRPCServer(listeningAddresses []string, rpcMaxInboundConnections int) (server.Server, error) {
-	gRPCServer := newGRPCServer(listeningAddresses, RPCMaxMessageSize, RPCMaxInboundConnections, "RPC")
+	gRPCServer := newGRPCServer(listeningAddresses, RPCMaxMessageSize, rpcMaxInboundConnections, "RPC")
 	rpcServer := &rpcServer{gRPCServer: *gRPCServer}
 	protowire.RegisterRPCServer(gRPCServer.server, rpcServer)
 	return rpcServer, nil

infrastructure/network/rpcclient/rpc_get_mempool_entries.go

@@ -3,8 +3,8 @@ package rpcclient
 import "github.com/kaspanet/kaspad/app/appmessage"
 
 // GetMempoolEntries sends an RPC request respective to the function's name and returns the RPC server's response
-func (c *RPCClient) GetMempoolEntries() (*appmessage.GetMempoolEntriesResponseMessage, error) {
+func (c *RPCClient) GetMempoolEntries(includeOrphanPool bool, filterTransactionPool bool) (*appmessage.GetMempoolEntriesResponseMessage, error) {
-	err := c.rpcRouter.outgoingRoute().Enqueue(appmessage.NewGetMempoolEntriesRequestMessage())
+	err := c.rpcRouter.outgoingRoute().Enqueue(appmessage.NewGetMempoolEntriesRequestMessage(includeOrphanPool, filterTransactionPool))
 	if err != nil {
 		return nil, err
 	}

infrastructure/network/rpcclient/rpc_get_mempool_entries_by_address.go

@@ -3,8 +3,8 @@ package rpcclient
 import "github.com/kaspanet/kaspad/app/appmessage"
 
 // GetMempoolEntriesByAddresses sends an RPC request respective to the function's name and returns the RPC server's response
-func (c *RPCClient) GetMempoolEntriesByAddresses(addresses []string) (*appmessage.GetMempoolEntriesByAddressesResponseMessage, error) {
+func (c *RPCClient) GetMempoolEntriesByAddresses(addresses []string, includeOrphanPool bool, filterTransactionPool bool) (*appmessage.GetMempoolEntriesByAddressesResponseMessage, error) {
-	err := c.rpcRouter.outgoingRoute().Enqueue(appmessage.NewGetMempoolEntriesByAddressesRequestMessage(addresses))
+	err := c.rpcRouter.outgoingRoute().Enqueue(appmessage.NewGetMempoolEntriesByAddressesRequestMessage(addresses, includeOrphanPool, filterTransactionPool))
 	if err != nil {
 		return nil, err
 	}

infrastructure/network/rpcclient/rpc_get_mempool_entry.go

@@ -3,8 +3,8 @@ package rpcclient
 import "github.com/kaspanet/kaspad/app/appmessage"
 
 // GetMempoolEntry sends an RPC request respective to the function's name and returns the RPC server's response
-func (c *RPCClient) GetMempoolEntry(txID string) (*appmessage.GetMempoolEntryResponseMessage, error) {
+func (c *RPCClient) GetMempoolEntry(txID string, includeOrphanPool bool, filterTransactionPool bool) (*appmessage.GetMempoolEntryResponseMessage, error) {
-	err := c.rpcRouter.outgoingRoute().Enqueue(appmessage.NewGetMempoolEntryRequestMessage(txID))
+	err := c.rpcRouter.outgoingRoute().Enqueue(appmessage.NewGetMempoolEntryRequestMessage(txID, includeOrphanPool, filterTransactionPool))
 	if err != nil {
 		return nil, err
 	}

stability-tests/common/mine/mine.go

@@ -122,7 +122,7 @@ func mineOrFetchBlock(blockData JSONBlock, mdb *miningDB, testConsensus testapi.
 		SolveBlock(block)
 	}
 
-	_, err = testConsensus.ValidateAndInsertBlock(block, true)
+	err = testConsensus.ValidateAndInsertBlock(block, true)
 	if err != nil {
 		return nil, errors.Wrap(err, "error in ValidateAndInsertBlock")
 	}

stability-tests/netsync/fast-pruning-ibd-test/generate_test.go

@@ -57,7 +57,7 @@ func TestGenerateFastPruningIBDTest(t *testing.T) {
 			}
 			header.SetNonce(tip.Header.Nonce() + i)
 			block := &externalapi.DomainBlock{Header: header.ToImmutable(), Transactions: tip.Transactions}
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}
@@ -103,7 +103,7 @@ func TestGenerateFastPruningIBDTest(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}
@@ -124,7 +124,7 @@ func TestGenerateFastPruningIBDTest(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			_, err = tc.ValidateAndInsertBlock(block, true)
+			err = tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				t.Fatalf("ValidateAndInsertBlock: %+v", err)
 			}

stability-tests/orphans/prepare_blocks.go

@@ -49,7 +49,7 @@ func prepareBlocks() (blocks []*externalapi.DomainBlock, topBlock *externalapi.D
 		}
 
 		mine.SolveBlock(block)
-		_, err = testConsensus.ValidateAndInsertBlock(block, true)
+		err = testConsensus.ValidateAndInsertBlock(block, true)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "error in ValidateAndInsertBlock")
 		}

stability-tests/reorg/reorg.go

@@ -86,7 +86,7 @@ func testReorg(cfg *configFlags) {
 			block.Header = mutableHeader.ToImmutable()
 		}
 
-		_, err = tcAttacker.ValidateAndInsertBlock(block, true)
+		err = tcAttacker.ValidateAndInsertBlock(block, true)
 		if err != nil {
 			panic(err)
 		}
@@ -122,7 +122,7 @@ func testReorg(cfg *configFlags) {
 			if i%100 == 0 {
 				log.Infof("Validated %d blocks from the attacker chain", i)
 			}
-			_, err := tc.ValidateAndInsertBlock(block, true)
+			err := tc.ValidateAndInsertBlock(block, true)
 			if err != nil {
 				panic(err)
 			}

testing/integration/config_test.go

@@ -31,7 +31,7 @@ const (
 	miningAddress3           = "kaspasim:qqq754f2gdcjcnykwuwwr60c82rh5u6mxxe7yqxljnrxz9fu0h95kduq9ezng"
 	miningAddress3PrivateKey = "f6c8f31fd359cbb97007034780bc4021f6ad01c6bc10499b79849efd4cc7ca39"
 
-	defaultTimeout = 10 * time.Second
+	defaultTimeout = 30 * time.Second
 )
 
 func setConfig(t *testing.T, harness *appHarness, protocolVersion uint32) {

testing/integration/rpc_test.go

@@ -1,7 +1,7 @@
 package integration
 
 import (
-	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/server/grpcserver"
+	"github.com/kaspanet/kaspad/infrastructure/config"
 	"testing"
 	"time"
 
@@ -45,7 +45,7 @@ func TestRPCMaxInboundConnections(t *testing.T) {
 	rpcClients := []*testRPCClient{}
 	doneChan := make(chan error)
 	go func() {
-		for i := 0; i < grpcserver.RPCMaxInboundConnections; i++ {
+		for i := 0; i < config.DefaultMaxRPCClients; i++ {
 			rpcClient, err := newTestRPCClient(harness.rpcAddress)
 			if err != nil {
 				doneChan <- err
@@ -60,7 +60,7 @@ func TestRPCMaxInboundConnections(t *testing.T) {
 			t.Fatalf("newTestRPCClient: %s", err)
 		}
 	case <-time.After(time.Second * 5):
-		t.Fatalf("Timeout for connecting %d RPC connections elapsed", grpcserver.RPCMaxInboundConnections)
+		t.Fatalf("Timeout for connecting %d RPC connections elapsed", config.DefaultMaxRPCClients)
 	}
 
 	// Try to connect another client. We expect this to fail

testing/integration/tx_relay_test.go

@@ -69,7 +69,7 @@ func TestTxRelay(t *testing.T) {
 
 		for range ticker.C {
 
-			_, err := payee.rpcClient.GetMempoolEntry(txID)
+			getMempoolEntryResponse, err := payee.rpcClient.GetMempoolEntry(txID, true, false)
 			if err != nil {
 				if strings.Contains(err.Error(), "not found") {
 					continue
@@ -77,12 +77,16 @@ func TestTxRelay(t *testing.T) {
 
 				t.Fatalf("Error getting mempool entry: %+v", err)
 			}
+			mempoolEntry := getMempoolEntryResponse.Entry
+			if mempoolEntry.IsOrphan {
+				t.Fatalf("transaction %s is an orphan, although it shouldn't be", mempoolEntry.Transaction.VerboseData.TransactionID)
+			}
 
-			mempoolEntriesByAddresses, err := payee.rpcClient.GetMempoolEntriesByAddresses(mempoolAddressQuery)
+			getMempoolEntriesByAddressesResponse, err := payee.rpcClient.GetMempoolEntriesByAddresses(mempoolAddressQuery, true, false)
 			if err != nil {
 				t.Fatalf("Error getting mempool entry: %+v", err)
 			}
-			for _, mempoolEntryByAddress := range mempoolEntriesByAddresses.Entries {
+			for _, mempoolEntryByAddress := range getMempoolEntriesByAddressesResponse.Entries {
 				if payee.miningAddress == mempoolEntryByAddress.Address {
 					if len(mempoolEntryByAddress.Sending) > 1 {
 						t.Fatal("Error payee is sending")
@@ -99,7 +103,16 @@ func TestTxRelay(t *testing.T) {
 						t.Fatal("Error payer is reciving")
 					}
 				}
-				continue
+				for _, mempoolEntry := range mempoolEntryByAddress.Receiving {
+					if mempoolEntry.IsOrphan {
+						t.Fatalf("transaction %s is an orphan, although it shouldn't be", mempoolEntry.Transaction.VerboseData.TransactionID)
+					}
+				}
+				for _, mempoolEntry := range mempoolEntryByAddress.Sending {
+					if mempoolEntry.IsOrphan {
+						t.Fatalf("transaction %s is an orphan, although it shouldn't be", mempoolEntry.Transaction.VerboseData.TransactionID)
+					}
+				}
 			}
 
 			close(txAddedToMempoolChan)

util/difficulty/difficulty.go

@@ -136,11 +136,11 @@ func CalcWork(bits uint32) *big.Int {
 
 func getHashrate(target *big.Int, TargetTimePerBlock time.Duration) *big.Int {
 	// From: https://bitcoin.stackexchange.com/a/5557/40800
-	// difficulty = hashrate / (2^256 / max_target / block_rate_in_seconds)
+	// difficulty = hashrate / (2^256 / max_target / seconds_per_block)
-	// hashrate = difficulty * (2^256 / max_target / block_rate_in_seconds)
+	// hashrate = difficulty * (2^256 / max_target / seconds_per_block)
 	// difficulty = max_target / target
-	// hashrate = (max_target / target) * (2^256 / max_target / block_rate_in_seconds)
+	// hashrate = (max_target / target) * (2^256 / max_target / seconds_per_block)
-	// hashrate = 2^256 / (target * block_rate_in_seconds)
+	// hashrate = 2^256 / (target * seconds_per_block)
 
 	tmp := new(big.Int)
 	divisor := new(big.Int).Set(target)

version/version.go

@@ -11,7 +11,7 @@ const validCharacters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrs
 const (
 	appMajor uint = 0
 	appMinor uint = 12
-	appPatch uint = 1
+	appPatch uint = 2
 )
 
 // appBuild is defined as a variable so it can be overridden during the build