Update changelog for v0.9.0

* Refactor mergeSetIncrease to return the current BFS block to allow easier merging

* Remove unneeded Heap/HashSet usages

* Add new IsAnyAncestorOf to DagTopolyManager

* Check if the new candidate is in the future of any existing candidate

* Add comments and fix off-by-one in the mergeSetIncrease queue

* Fixed DAGToplogy test mock

* Fix review comments

app/app.go

@@ -7,19 +7,17 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/kaspanet/kaspad/infrastructure/config"
 	"github.com/kaspanet/kaspad/infrastructure/db/database"
 	"github.com/kaspanet/kaspad/infrastructure/db/database/ldb"
-
-	"github.com/kaspanet/kaspad/infrastructure/os/signal"
-	"github.com/kaspanet/kaspad/util/profiling"
-	"github.com/kaspanet/kaspad/version"
-
-	"github.com/kaspanet/kaspad/util/panics"
-
-	"github.com/kaspanet/kaspad/infrastructure/config"
+	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"github.com/kaspanet/kaspad/infrastructure/os/execenv"
 	"github.com/kaspanet/kaspad/infrastructure/os/limits"
+	"github.com/kaspanet/kaspad/infrastructure/os/signal"
 	"github.com/kaspanet/kaspad/infrastructure/os/winservice"
+	"github.com/kaspanet/kaspad/util/panics"
+	"github.com/kaspanet/kaspad/util/profiling"
+	"github.com/kaspanet/kaspad/version"
 )
 
 const leveldbCacheSizeMiB = 256
@@ -51,6 +49,7 @@ func StartApp() error {
 		fmt.Fprintln(os.Stderr, err)
 		return err
 	}
+	defer logger.BackendLog.Close()
 	defer panics.HandlePanic(log, "MAIN", nil)
 
 	app := &kaspadApp{cfg: cfg}

app/appmessage/domainconverters.go

@@ -164,11 +164,7 @@ func outpointToDomainOutpoint(outpoint *Outpoint) *externalapi.DomainOutpoint {
 func RPCTransactionToDomainTransaction(rpcTransaction *RPCTransaction) (*externalapi.DomainTransaction, error) {
 	inputs := make([]*externalapi.DomainTransactionInput, len(rpcTransaction.Inputs))
 	for i, input := range rpcTransaction.Inputs {
-		transactionIDBytes, err := hex.DecodeString(input.PreviousOutpoint.TransactionID)
-		if err != nil {
-			return nil, err
-		}
-		transactionID, err := transactionid.FromBytes(transactionIDBytes)
+		transactionID, err := transactionid.FromString(input.PreviousOutpoint.TransactionID)
 		if err != nil {
 			return nil, err
 		}
@@ -198,19 +194,11 @@ func RPCTransactionToDomainTransaction(rpcTransaction *RPCTransaction) (*externa
 		}
 	}
 
-	subnetworkIDBytes, err := hex.DecodeString(rpcTransaction.SubnetworkID)
+	subnetworkID, err := subnetworks.FromString(rpcTransaction.SubnetworkID)
 	if err != nil {
 		return nil, err
 	}
-	subnetworkID, err := subnetworks.FromBytes(subnetworkIDBytes)
-	if err != nil {
-		return nil, err
-	}
-	payloadHashBytes, err := hex.DecodeString(rpcTransaction.PayloadHash)
-	if err != nil {
-		return nil, err
-	}
-	payloadHash, err := externalapi.NewDomainHashFromByteSlice(payloadHashBytes)
+	payloadHash, err := externalapi.NewDomainHashFromString(rpcTransaction.PayloadHash)
 	if err != nil {
 		return nil, err
 	}
@@ -255,7 +243,7 @@ func DomainTransactionToRPCTransaction(transaction *externalapi.DomainTransactio
 			ScriptPublicKey: &RPCScriptPublicKey{Script: scriptPublicKey, Version: output.ScriptPublicKey.Version},
 		}
 	}
-	subnetworkID := hex.EncodeToString(transaction.SubnetworkID[:])
+	subnetworkID := transaction.SubnetworkID.String()
 	payloadHash := transaction.PayloadHash.String()
 	payload := hex.EncodeToString(transaction.Payload)
 	return &RPCTransaction{

app/appmessage/rpc_get_block.go

@@ -55,6 +55,7 @@ type BlockVerboseData struct {
 	Bits                   string
 	Difficulty             float64
 	ParentHashes           []string
+	ChildrenHashes         []string
 	SelectedParentHash     string
 	BlueScore              uint64
 	IsHeaderOnly           bool
@@ -104,4 +105,5 @@ type ScriptPubKeyResult struct {
 	Hex     string
 	Type    string
 	Address string
+	Version uint16
 }

app/appmessage/rpc_get_block_dag_info.go

@@ -27,6 +27,7 @@ type GetBlockDAGInfoResponseMessage struct {
 	VirtualParentHashes []string
 	Difficulty          float64
 	PastMedianTime      int64
+	PruningPointHash    string
 
 	Error *RPCError
 }

app/log.go

@@ -7,8 +7,6 @@ package app
 
 import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.KASD)
-var spawn = panics.GoroutineWrapperFunc(log)
+var log = logger.RegisterSubSystem("KASD")

app/protocol/flowcontext/log.go

@@ -2,8 +2,6 @@ package flowcontext
 
 import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.PROT)
-var spawn = panics.GoroutineWrapperFunc(log)
+var log = logger.RegisterSubSystem("PROT")

app/protocol/flows/blockrelay/handle_request_headers.go

@@ -17,7 +17,7 @@ type RequestIBDBlocksContext interface {
 	Domain() domain.Domain
 }
 
-type handleRequestBlocksFlow struct {
+type handleRequestHeadersFlow struct {
 	RequestIBDBlocksContext
 	incomingRoute, outgoingRoute *router.Route
 	peer                         *peer.Peer
@@ -27,7 +27,7 @@ type handleRequestBlocksFlow struct {
 func HandleRequestHeaders(context RequestIBDBlocksContext, incomingRoute *router.Route,
 	outgoingRoute *router.Route, peer *peer.Peer) error {
 
-	flow := &handleRequestBlocksFlow{
+	flow := &handleRequestHeadersFlow{
 		RequestIBDBlocksContext: context,
 		incomingRoute:           incomingRoute,
 		outgoingRoute:           outgoingRoute,
@@ -36,7 +36,7 @@ func HandleRequestHeaders(context RequestIBDBlocksContext, incomingRoute *router
 	return flow.start()
 }
 
-func (flow *handleRequestBlocksFlow) start() error {
+func (flow *handleRequestHeadersFlow) start() error {
 	for {
 		lowHash, highHash, err := receiveRequestHeaders(flow.incomingRoute)
 		if err != nil {

app/protocol/flows/blockrelay/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.PROT)
+var log = logger.RegisterSubSystem("PROT")
 var spawn = panics.GoroutineWrapperFunc(log)

app/protocol/flows/handshake/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.PROT)
+var log = logger.RegisterSubSystem("PROT")
 var spawn = panics.GoroutineWrapperFunc(log)

app/protocol/flows/testing/handle_relay_invs_test.go

@@ -129,6 +129,10 @@ type fakeRelayInvsContext struct {
 	rwLock                                        sync.RWMutex
 }
 
+func (f *fakeRelayInvsContext) GetBlockChildren(blockHash *externalapi.DomainHash) ([]*externalapi.DomainHash, error) {
+	panic(errors.Errorf("called unimplemented function from test '%s'", f.testName))
+}
+
 func (f *fakeRelayInvsContext) OnPruningPointUTXOSetOverride() error {
 	return nil
 }

app/protocol/flows/transactionrelay/handle_relayed_transactions.go

@@ -191,7 +191,7 @@ func (flow *handleRelayedTransactionsFlow) receiveTransactions(requestedTransact
 				continue
 			}
 
-			return protocolerrors.Errorf(true, "rejected transaction %s", txID)
+			return protocolerrors.Errorf(true, "rejected transaction %s: %s", txID, ruleErr)
 		}
 		err = flow.broadcastAcceptedTransactions([]*externalapi.DomainTransactionID{txID})
 		if err != nil {

app/protocol/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.PROT)
+var log = logger.RegisterSubSystem("PROT")
 var spawn = panics.GoroutineWrapperFunc(log)

app/protocol/peer/log.go

@@ -2,8 +2,6 @@ package peer
 
 import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.PROT)
-var spawn = panics.GoroutineWrapperFunc(log)
+var log = logger.RegisterSubSystem("PROT")

app/protocol/protocol.go

@@ -59,8 +59,20 @@ func (m *Manager) routerInitializer(router *routerpkg.Router, netConnection *net
 
 		peer, err := handshake.HandleHandshake(m.context, netConnection, receiveVersionRoute,
 			sendVersionRoute, router.OutgoingRoute())
+
 		if err != nil {
-			m.handleError(err, netConnection, router.OutgoingRoute())
+			// non-blocking read from channel
+			select {
+			case innerError := <-errChan:
+				if errors.Is(err, routerpkg.ErrRouteClosed) {
+					m.handleError(innerError, netConnection, router.OutgoingRoute())
+				} else {
+					log.Errorf("Peer %s sent invalid message: %s", netConnection, innerError)
+					m.handleError(err, netConnection, router.OutgoingRoute())
+				}
+			default:
+				m.handleError(err, netConnection, router.OutgoingRoute())
+			}
 			return
 		}
 		defer m.context.RemoveFromPeers(peer)

app/rpc/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.RPCS)
+var log = logger.RegisterSubSystem("RPCS")
 var spawn = panics.GoroutineWrapperFunc(log)

app/rpc/rpccontext/log.go

@@ -2,8 +2,6 @@ package rpccontext
 
 import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.RPCS)
-var spawn = panics.GoroutineWrapperFunc(log)
+var log = logger.RegisterSubSystem("RPCS")

app/rpc/rpccontext/verbosedata.go

@@ -3,13 +3,14 @@ package rpccontext
 import (
 	"encoding/hex"
 	"fmt"
+	"math"
+	"math/big"
+	"strconv"
+
 	"github.com/kaspanet/kaspad/domain/consensus/utils/constants"
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"github.com/kaspanet/kaspad/util/difficulty"
 	"github.com/pkg/errors"
-	"math"
-	"math/big"
-	"strconv"
 
 	"github.com/kaspanet/kaspad/domain/consensus/utils/hashes"
 
@@ -47,6 +48,11 @@ func (ctx *Context) BuildBlockVerboseData(blockHeader externalapi.BlockHeader, b
 			"invalid block")
 	}
 
+	childrenHashes, err := ctx.Domain.Consensus().GetBlockChildren(hash)
+	if err != nil {
+		return nil, err
+	}
+
 	result := &appmessage.BlockVerboseData{
 		Hash:                 hash.String(),
 		Version:              blockHeader.Version(),
@@ -55,6 +61,7 @@ func (ctx *Context) BuildBlockVerboseData(blockHeader externalapi.BlockHeader, b
 		AcceptedIDMerkleRoot: blockHeader.AcceptedIDMerkleRoot().String(),
 		UTXOCommitment:       blockHeader.UTXOCommitment().String(),
 		ParentHashes:         hashes.ToStrings(blockHeader.ParentHashes()),
+		ChildrenHashes:       hashes.ToStrings(childrenHashes),
 		Nonce:                blockHeader.Nonce(),
 		Time:                 blockHeader.TimeInMilliseconds(),
 		Bits:                 strconv.FormatInt(int64(blockHeader.Bits()), 16),
@@ -205,6 +212,7 @@ func (ctx *Context) buildTransactionVerboseOutputs(tx *externalapi.DomainTransac
 		output.Index = uint32(i)
 		output.Value = transactionOutput.Value
 		output.ScriptPubKey = &appmessage.ScriptPubKeyResult{
+			Version: transactionOutput.ScriptPublicKey.Version,
 			Address: encodedAddr,
 			Hex:     hex.EncodeToString(transactionOutput.ScriptPublicKey.Script),
 			Type:    scriptClass.String(),

app/rpc/rpchandlers/get_block_dag_info.go

@@ -36,5 +36,11 @@ func HandleGetBlockDAGInfo(context *rpccontext.Context, _ *router.Router, _ appm
 	response.Difficulty = context.GetDifficultyRatio(virtualInfo.Bits, context.Config.ActiveNetParams)
 	response.PastMedianTime = virtualInfo.PastMedianTime
 
+	pruningPoint, err := context.Domain.Consensus().PruningPoint()
+	if err != nil {
+		return nil, err
+	}
+	response.PruningPointHash = pruningPoint.String()
+
 	return response, nil
 }

app/rpc/rpchandlers/get_blocks.go

@@ -11,7 +11,7 @@ import (
 const (
 	// maxBlocksInGetBlocksResponse is the max amount of blocks that are
 	// allowed in a GetBlocksResult.
-	maxBlocksInGetBlocksResponse = 100
+	maxBlocksInGetBlocksResponse = 1000
 )
 
 // HandleGetBlocks handles the respectively named RPC command
@@ -37,6 +37,17 @@ func HandleGetBlocks(context *rpccontext.Context, _ *router.Router, request appm
 				Error: appmessage.RPCErrorf("Could not decode lowHash %s: %s", getBlocksRequest.LowHash, err),
 			}, nil
 		}
+
+		blockInfo, err := context.Domain.Consensus().GetBlockInfo(lowHash)
+		if err != nil {
+			return nil, err
+		}
+
+		if !blockInfo.Exists {
+			return &appmessage.GetBlocksResponseMessage{
+				Error: appmessage.RPCErrorf("Could not find lowHash %s", getBlocksRequest.LowHash),
+			}, nil
+		}
 	}
 
 	// Get hashes between lowHash and virtualSelectedParent

app/rpc/rpchandlers/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.RPCS)
+var log = logger.RegisterSubSystem("RPCS")
 var spawn = panics.GoroutineWrapperFunc(log)

changelog.txt

@@ -1,11 +1,31 @@
+
+Kaspad v0.9.0 - 2021-03-04
+===========================
+
+* Merge big subdags in pick virtual parents (#1574)
+* Write in the reject message the tx rejection reason (#1573)
+* Add nil checks for protowire (#1570)
+* Increase getBlocks limit to 1000 (#1572)
+* Return RPC error if getBlock's lowHash doesn't exist (#1569)
+* Add default dns-seeder to testnet (#1568)
+* Fix utxoindex deserialization (#1566)
+* Add pruning point hash to GetBlockDagInfo response (#1565)
+* Use EmitUnpopulated so that kaspactl prints all fields, even the default ones (#1561)
+* Stop logging an error whenever an RPC/P2P connection is canceled  (#1562)
+* Cleanup the logger and make it asynchronous (#1524)
+* Close all iterators (#1542)
+* Add childrenHashes to GetBlock/s RPC commands (#1560)
+* Add ScriptPublicKey.Version to RPC (#1559)
+* Fix the target block rate to create less bursty mining (#1554)
+
 Kaspad v0.8.10 - 2021-02-25
 ===========================
 
-[*] Fix bug where invalid mempool transactions were not removed (#1551)
-[*] Add RPC reconnection to the miner (#1552)
-[*] Remove virtual diff parents - only selectedTip is virtualDiffParent now (#1550)
-[*] Fix UTXO index (#1548)
-[*] Prevent fast failing (#1545)
-[*] Increase the sleep time in kaspaminer when the node is not synced (#1544)
-[*] Disallow header only blocks on RPC, relay and when requesting IBD full blocks (#1537)
-[*] Make templateManager hold a DomainBlock and isSynced bool instead of a GetBlockTemplateResponseMessage (#1538)
+* Fix bug where invalid mempool transactions were not removed (#1551)
+* Add RPC reconnection to the miner (#1552)
+* Remove virtual diff parents - only selectedTip is virtualDiffParent now (#1550)
+* Fix UTXO index (#1548)
+* Prevent fast failing (#1545)
+* Increase the sleep time in kaspaminer when the node is not synced (#1544)
+* Disallow header only blocks on RPC, relay and when requesting IBD full blocks (#1537)
+* Make templateManager hold a DomainBlock and isSynced bool instead of a GetBlockTemplateResponseMessage (#1538)

cmd/kaspactl/main.go

@@ -2,10 +2,11 @@ package main
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/server/grpcserver/protowire"
 	"os"
 	"time"
 
+	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/server/grpcserver/protowire"
+
 	"github.com/pkg/errors"
 	"google.golang.org/protobuf/encoding/protojson"
 
@@ -67,7 +68,7 @@ func postCommand(cfg *configFlags, client *grpcclient.GRPCClient, responseChan c
 	if err != nil {
 		printErrorAndExit(fmt.Sprintf("error posting the request to the RPC server: %s", err))
 	}
-	responseBytes, err := protojson.Marshal(response)
+	responseBytes, err := protojson.MarshalOptions{EmitUnpopulated: true}.Marshal(response)
 	if err != nil {
 		printErrorAndExit(errors.Wrapf(err, "error parsing the response from the RPC server").Error())
 	}
@@ -92,6 +93,7 @@ func prettifyResponse(response string) string {
 
 	marshalOptions := &protojson.MarshalOptions{}
 	marshalOptions.Indent = "    "
+	marshalOptions.EmitUnpopulated = true
 	return marshalOptions.Format(kaspadMessage)
 }
 

cmd/kaspaminer/log.go

@@ -14,6 +14,7 @@ var (
 )
 
 func initLog(logFile, errLogFile string) {
+	log.SetLevel(logger.LevelDebug)
 	err := backendLog.AddLogFile(logFile, logger.LevelTrace)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error adding log file %s as log rotator for level %s: %s", logFile, logger.LevelTrace, err)
@@ -24,4 +25,15 @@ func initLog(logFile, errLogFile string) {
 		fmt.Fprintf(os.Stderr, "Error adding log file %s as log rotator for level %s: %s", errLogFile, logger.LevelWarn, err)
 		os.Exit(1)
 	}
+	err = backendLog.AddLogWriter(os.Stdout, logger.LevelInfo)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error adding stdout to the loggerfor level %s: %s", logger.LevelWarn, err)
+		os.Exit(1)
+	}
+	err = backendLog.Run()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error starting the logger: %s ", err)
+		os.Exit(1)
+	}
+
 }

cmd/kaspaminer/main.go

@@ -26,6 +26,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Error parsing command-line arguments: %s\n", err)
 		os.Exit(1)
 	}
+	defer backendLog.Close()
 
 	// Show version at startup.
 	log.Infof("Version %s", version.Version())

cmd/kaspaminer/mineloop.go

@@ -53,6 +53,8 @@ func mineLoop(client *minerClient, numberOfBlocks uint64, targetBlocksPerSecond
 		}
 		blockInWindowIndex := 0
 
+		sleepTime := 0 * time.Second
+
 		for {
 			foundBlockChan <- mineNextBlock(mineWhenNotSynced)
 
@@ -61,15 +63,16 @@ func mineLoop(client *minerClient, numberOfBlocks uint64, targetBlocksPerSecond
 				if blockInWindowIndex == windowSize-1 {
 					deviation := windowExpectedEndTime.Sub(time.Now())
 					if deviation > 0 {
-						log.Infof("Finished to mine %d blocks %s earlier than expected. Sleeping %s to compensate",
-							windowSize, deviation, deviation)
-						time.Sleep(deviation)
+						sleepTime = deviation / windowSize
+						log.Infof("Finished to mine %d blocks %s earlier than expected. Setting the miner "+
+							"to sleep %s between blocks to compensate",
+							windowSize, deviation, sleepTime)
 					}
 					blockInWindowIndex = 0
 					windowExpectedEndTime = time.Now().Add(expectedDurationForWindow)
 				}
+				time.Sleep(sleepTime)
 			}
-
 		}
 	})
 

domain/consensus/consensus.go

@@ -157,6 +157,15 @@ func (s *consensus) GetBlockInfo(blockHash *externalapi.DomainHash) (*externalap
 	return blockInfo, nil
 }
 
+func (s *consensus) GetBlockChildren(blockHash *externalapi.DomainHash) ([]*externalapi.DomainHash, error) {
+	blockRelation, err := s.blockRelationStore.BlockRelation(s.databaseContext, blockHash)
+	if err != nil {
+		return nil, err
+	}
+
+	return blockRelation.Children, nil
+}
+
 func (s *consensus) GetBlockAcceptanceData(blockHash *externalapi.DomainHash) (externalapi.AcceptanceData, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()

domain/consensus/database/cursor.go

@@ -3,25 +3,40 @@ package database
 import (
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/infrastructure/db/database"
+	"github.com/pkg/errors"
 )
 
 type dbCursor struct {
-	cursor database.Cursor
+	cursor   database.Cursor
+	isClosed bool
 }
 
 func (d dbCursor) Next() bool {
+	if d.isClosed {
+		panic("Tried using a closed DBCursor")
+	}
+
 	return d.cursor.Next()
 }
 
 func (d dbCursor) First() bool {
+	if d.isClosed {
+		panic("Tried using a closed DBCursor")
+	}
 	return d.cursor.First()
 }
 
 func (d dbCursor) Seek(key model.DBKey) error {
+	if d.isClosed {
+		return errors.New("Tried using a closed DBCursor")
+	}
 	return d.cursor.Seek(dbKeyToDatabaseKey(key))
 }
 
 func (d dbCursor) Key() (model.DBKey, error) {
+	if d.isClosed {
+		return nil, errors.New("Tried using a closed DBCursor")
+	}
 	key, err := d.cursor.Key()
 	if err != nil {
 		return nil, err
@@ -31,11 +46,23 @@ func (d dbCursor) Key() (model.DBKey, error) {
 }
 
 func (d dbCursor) Value() ([]byte, error) {
+	if d.isClosed {
+		return nil, errors.New("Tried using a closed DBCursor")
+	}
 	return d.cursor.Value()
 }
 
 func (d dbCursor) Close() error {
-	return d.cursor.Close()
+	if d.isClosed {
+		return errors.New("Tried using a closed DBCursor")
+	}
+	d.isClosed = true
+	err := d.cursor.Close()
+	if err != nil {
+		return err
+	}
+	d.cursor = nil
+	return nil
 }
 
 func newDBCursor(cursor database.Cursor) model.DBCursor {

domain/consensus/database/serialization/utxo_collection.go

@@ -9,6 +9,7 @@ func utxoCollectionToDBUTXOCollection(utxoCollection externalapi.UTXOCollection)
 	items := make([]*DbUtxoCollectionItem, utxoCollection.Len())
 	i := 0
 	utxoIterator := utxoCollection.Iterator()
+	defer utxoIterator.Close()
 	for ok := utxoIterator.First(); ok; ok = utxoIterator.Next() {
 		outpoint, entry, err := utxoIterator.Get()
 		if err != nil {

domain/consensus/datastructures/blockstore/blockstore.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
+	"github.com/pkg/errors"
 )
 
 var bucket = database.MakeBucket([]byte("blocks"))
@@ -214,18 +215,28 @@ func (bs *blockStore) serializeBlockCount(count uint64) ([]byte, error) {
 }
 
 type allBlockHashesIterator struct {
-	cursor model.DBCursor
+	cursor   model.DBCursor
+	isClosed bool
 }
 
 func (a allBlockHashesIterator) First() bool {
+	if a.isClosed {
+		panic("Tried using a closed AllBlockHashesIterator")
+	}
 	return a.cursor.First()
 }
 
 func (a allBlockHashesIterator) Next() bool {
+	if a.isClosed {
+		panic("Tried using a closed AllBlockHashesIterator")
+	}
 	return a.cursor.Next()
 }
 
 func (a allBlockHashesIterator) Get() (*externalapi.DomainHash, error) {
+	if a.isClosed {
+		return nil, errors.New("Tried using a closed AllBlockHashesIterator")
+	}
 	key, err := a.cursor.Key()
 	if err != nil {
 		return nil, err
@@ -235,6 +246,19 @@ func (a allBlockHashesIterator) Get() (*externalapi.DomainHash, error) {
 	return externalapi.NewDomainHashFromByteSlice(blockHashBytes)
 }
 
+func (a allBlockHashesIterator) Close() error {
+	if a.isClosed {
+		return errors.New("Tried using a closed AllBlockHashesIterator")
+	}
+	a.isClosed = true
+	err := a.cursor.Close()
+	if err != nil {
+		return err
+	}
+	a.cursor = nil
+	return nil
+}
+
 func (bs *blockStore) AllBlockHashesIterator(dbContext model.DBReader) (model.BlockIterator, error) {
 	cursor, err := dbContext.Cursor(bucket)
 	if err != nil {

domain/consensus/datastructures/consensusstatestore/utxo.go

@@ -37,6 +37,7 @@ func (css *consensusStateStore) commitVirtualUTXODiff(dbTx model.DBTransaction)
 	}
 
 	toRemoveIterator := css.virtualUTXODiffStaging.ToRemove().Iterator()
+	defer toRemoveIterator.Close()
 	for ok := toRemoveIterator.First(); ok; ok = toRemoveIterator.Next() {
 		toRemoveOutpoint, _, err := toRemoveIterator.Get()
 		if err != nil {
@@ -56,6 +57,7 @@ func (css *consensusStateStore) commitVirtualUTXODiff(dbTx model.DBTransaction)
 	}
 
 	toAddIterator := css.virtualUTXODiffStaging.ToAdd().Iterator()
+	defer toAddIterator.Close()
 	for ok := toAddIterator.First(); ok; ok = toAddIterator.Next() {
 		toAddOutpoint, toAddEntry, err := toAddIterator.Get()
 		if err != nil {
@@ -156,6 +158,7 @@ func (css *consensusStateStore) VirtualUTXOs(dbContext model.DBReader,
 	if err != nil {
 		return nil, err
 	}
+	defer cursor.Close()
 
 	if fromOutpoint != nil {
 		serializedFromOutpoint, err := serializeOutpoint(fromOutpoint)
@@ -170,6 +173,7 @@ func (css *consensusStateStore) VirtualUTXOs(dbContext model.DBReader,
 	}
 
 	iterator := newCursorUTXOSetIterator(cursor)
+	defer iterator.Close()
 
 	outpointAndUTXOEntryPairs := make([]*externalapi.OutpointAndUTXOEntryPair, 0, limit)
 	for len(outpointAndUTXOEntryPairs) < limit && iterator.Next() {
@@ -200,7 +204,8 @@ func (css *consensusStateStore) VirtualUTXOSetIterator(dbContext model.DBReader)
 }
 
 type utxoSetIterator struct {
-	cursor model.DBCursor
+	cursor   model.DBCursor
+	isClosed bool
 }
 
 func newCursorUTXOSetIterator(cursor model.DBCursor) externalapi.ReadOnlyUTXOSetIterator {
@@ -208,14 +213,23 @@ func newCursorUTXOSetIterator(cursor model.DBCursor) externalapi.ReadOnlyUTXOSet
 }
 
 func (u utxoSetIterator) First() bool {
+	if u.isClosed {
+		panic("Tried using a closed utxoSetIterator")
+	}
 	return u.cursor.First()
 }
 
 func (u utxoSetIterator) Next() bool {
+	if u.isClosed {
+		panic("Tried using a closed utxoSetIterator")
+	}
 	return u.cursor.Next()
 }
 
 func (u utxoSetIterator) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry externalapi.UTXOEntry, err error) {
+	if u.isClosed {
+		return nil, nil, errors.New("Tried using a closed utxoSetIterator")
+	}
 	key, err := u.cursor.Key()
 	if err != nil {
 		panic(err)
@@ -238,3 +252,16 @@ func (u utxoSetIterator) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry
 
 	return outpoint, utxoEntry, nil
 }
+
+func (u utxoSetIterator) Close() error {
+	if u.isClosed {
+		return errors.New("Tried using a closed utxoSetIterator")
+	}
+	u.isClosed = true
+	err := u.cursor.Close()
+	if err != nil {
+		return err
+	}
+	u.cursor = nil
+	return nil
+}

domain/consensus/datastructures/consensusstatestore/virtual_utxo_set.go

@@ -45,6 +45,7 @@ func (css *consensusStateStore) ImportPruningPointUTXOSetIntoVirtualUTXOSet(dbCo
 	if err != nil {
 		return err
 	}
+	defer deleteCursor.Close()
 	for ok := deleteCursor.First(); ok; ok = deleteCursor.Next() {
 		key, err := deleteCursor.Key()
 		if err != nil {

domain/consensus/datastructures/pruningstore/imported_pruning_point.go

@@ -6,6 +6,7 @@ import (
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+	"github.com/pkg/errors"
 )
 
 var importedPruningPointUTXOsBucket = database.MakeBucket([]byte("imported-pruning-point-utxos"))
@@ -16,6 +17,7 @@ func (ps *pruningStore) ClearImportedPruningPointUTXOs(dbContext model.DBWriter)
 	if err != nil {
 		return err
 	}
+	defer cursor.Close()
 
 	for ok := cursor.First(); ok; ok = cursor.Next() {
 		key, err := cursor.Key()
@@ -60,7 +62,8 @@ func (ps *pruningStore) ImportedPruningPointUTXOIterator(dbContext model.DBReade
 }
 
 type utxoSetIterator struct {
-	cursor model.DBCursor
+	cursor   model.DBCursor
+	isClosed bool
 }
 
 func (ps *pruningStore) newCursorUTXOSetIterator(cursor model.DBCursor) externalapi.ReadOnlyUTXOSetIterator {
@@ -68,14 +71,23 @@ func (ps *pruningStore) newCursorUTXOSetIterator(cursor model.DBCursor) external
 }
 
 func (u *utxoSetIterator) First() bool {
+	if u.isClosed {
+		panic("Tried using a closed utxoSetIterator")
+	}
 	return u.cursor.First()
 }
 
 func (u *utxoSetIterator) Next() bool {
+	if u.isClosed {
+		panic("Tried using a closed utxoSetIterator")
+	}
 	return u.cursor.Next()
 }
 
 func (u *utxoSetIterator) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry externalapi.UTXOEntry, err error) {
+	if u.isClosed {
+		return nil, nil, errors.New("Tried using a closed utxoSetIterator")
+	}
 	key, err := u.cursor.Key()
 	if err != nil {
 		panic(err)
@@ -99,6 +111,19 @@ func (u *utxoSetIterator) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry
 	return outpoint, utxoEntry, nil
 }
 
+func (u *utxoSetIterator) Close() error {
+	if u.isClosed {
+		return errors.New("Tried using a closed utxoSetIterator")
+	}
+	u.isClosed = true
+	err := u.cursor.Close()
+	if err != nil {
+		return err
+	}
+	u.cursor = nil
+	return nil
+}
+
 func (ps *pruningStore) importedPruningPointUTXOKey(outpoint *externalapi.DomainOutpoint) (model.DBKey, error) {
 	serializedOutpoint, err := serializeOutpoint(outpoint)
 	if err != nil {
@@ -175,6 +200,7 @@ func (ps *pruningStore) CommitImportedPruningPointUTXOSet(dbContext model.DBWrit
 	if err != nil {
 		return err
 	}
+	defer deleteCursor.Close()
 	for ok := deleteCursor.First(); ok; ok = deleteCursor.Next() {
 		key, err := deleteCursor.Key()
 		if err != nil {
@@ -191,6 +217,7 @@ func (ps *pruningStore) CommitImportedPruningPointUTXOSet(dbContext model.DBWrit
 	if err != nil {
 		return err
 	}
+	defer insertCursor.Close()
 	for ok := insertCursor.First(); ok; ok = insertCursor.Next() {
 		importedPruningPointUTXOSetKey, err := insertCursor.Key()
 		if err != nil {

domain/consensus/datastructures/pruningstore/pruningstore.go

@@ -124,6 +124,7 @@ func (ps *pruningStore) UpdatePruningPointUTXOSet(dbContext model.DBWriter,
 	if err != nil {
 		return err
 	}
+	defer deleteCursor.Close()
 	for ok := deleteCursor.First(); ok; ok = deleteCursor.Next() {
 		key, err := deleteCursor.Key()
 		if err != nil {
@@ -215,6 +216,7 @@ func (ps *pruningStore) PruningPointUTXOs(dbContext model.DBReader,
 	if err != nil {
 		return nil, err
 	}
+	defer cursor.Close()
 
 	if fromOutpoint != nil {
 		serializedFromOutpoint, err := serializeOutpoint(fromOutpoint)
@@ -229,6 +231,7 @@ func (ps *pruningStore) PruningPointUTXOs(dbContext model.DBReader,
 	}
 
 	pruningPointUTXOIterator := ps.newCursorUTXOSetIterator(cursor)
+	defer pruningPointUTXOIterator.Close()
 
 	outpointAndUTXOEntryPairs := make([]*externalapi.OutpointAndUTXOEntryPair, 0, limit)
 	for len(outpointAndUTXOEntryPairs) < limit && pruningPointUTXOIterator.Next() {

domain/consensus/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var log = logger.RegisterSubSystem("BDAG")
 var spawn = panics.GoroutineWrapperFunc(log)

domain/consensus/model/blockiterator.go

@@ -7,4 +7,5 @@ type BlockIterator interface {
 	First() bool
 	Next() bool
 	Get() (*externalapi.DomainHash, error)
+	Close() error
 }

domain/consensus/model/externalapi/consensus.go

@@ -9,6 +9,7 @@ type Consensus interface {
 	GetBlock(blockHash *DomainHash) (*DomainBlock, error)
 	GetBlockHeader(blockHash *DomainHash) (BlockHeader, error)
 	GetBlockInfo(blockHash *DomainHash) (*BlockInfo, error)
+	GetBlockChildren(blockHash *DomainHash) ([]*DomainHash, error)
 	GetBlockAcceptanceData(blockHash *DomainHash) (AcceptanceData, error)
 
 	GetHashesBetween(lowHash, highHash *DomainHash, maxBlueScoreDifference uint64) ([]*DomainHash, error)

domain/consensus/model/externalapi/readonlyutxoset.go

@@ -6,4 +6,5 @@ type ReadOnlyUTXOSetIterator interface {
 	First() bool
 	Next() bool
 	Get() (outpoint *DomainOutpoint, utxoEntry UTXOEntry, err error)
+	Close() error
 }

domain/consensus/model/externalapi/subnetworkid.go

@@ -10,9 +10,6 @@ type DomainSubnetworkID [DomainSubnetworkIDSize]byte
 
 // String stringifies a subnetwork ID.
 func (id DomainSubnetworkID) String() string {
-	for i := 0; i < DomainSubnetworkIDSize/2; i++ {
-		id[i], id[DomainSubnetworkIDSize-1-i] = id[DomainSubnetworkIDSize-1-i], id[i]
-	}
 	return hex.EncodeToString(id[:])
 }
 

domain/consensus/model/interface_processes_dagtopologymanager.go

@@ -11,6 +11,7 @@ type DAGTopologyManager interface {
 	IsChildOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	IsAncestorOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	IsAncestorOfAny(blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error)
+	IsAnyAncestorOf(potentialAncestors []*externalapi.DomainHash, blockHash *externalapi.DomainHash) (bool, error)
 	IsInSelectedParentChainOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	ChildInSelectedParentChainOf(context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error)
 

domain/consensus/processes/blockbuilder/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var log = logger.RegisterSubSystem("BDAG")

domain/consensus/processes/blockprocessor/blocklogger/log.go

@@ -8,4 +8,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var log = logger.RegisterSubSystem("BDAG")

domain/consensus/processes/blockprocessor/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var log = logger.RegisterSubSystem("BDAG")

domain/consensus/processes/blockvalidator/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BLVL)
+var log = logger.RegisterSubSystem("BLVA")

domain/consensus/processes/consensusstatemanager/calculate_past_utxo_test.go

@@ -86,6 +86,7 @@ func checkBlockUTXOCommitment(t *testing.T, consensus testapi.TestConsensus, blo
 	if err != nil {
 		t.Fatalf("Error restoring past UTXO of block %s: %+v", blockName, err)
 	}
+	defer utxoSetIterator.Close()
 
 	// Build a Multiset
 	ms := multiset.New()

domain/consensus/processes/consensusstatemanager/import_pruning_utxo_set.go

@@ -88,6 +88,7 @@ func (csm *consensusStateManager) importPruningPoint(newPruningPoint *externalap
 	if err != nil {
 		return err
 	}
+	defer importedPruningPointUTXOIterator.Close()
 
 	// Clone the pruningPoint block here because validateBlockTransactionsAgainstPastUTXO
 	// assumes that the block UTXOEntries are pre-filled during further validations
@@ -176,6 +177,7 @@ func (csm *consensusStateManager) importVirtualUTXOSetAndPruningPointUTXOSet() e
 	if err != nil {
 		return err
 	}
+	defer pruningPointUTXOSetIterator.Close()
 
 	log.Debugf("Importing the virtual UTXO set")
 	err = csm.consensusStateStore.ImportPruningPointUTXOSetIntoVirtualUTXOSet(csm.databaseContext, pruningPointUTXOSetIterator)

domain/consensus/processes/consensusstatemanager/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var log = logger.RegisterSubSystem("BDAG")

domain/consensus/processes/consensusstatemanager/pick_virtual_parents.go

@@ -34,7 +34,6 @@ func (csm *consensusStateManager) pickVirtualParents(tips []*externalapi.DomainH
 	}
 	log.Debugf("The selected parent of the virtual is: %s", virtualSelectedParent)
 
-	selectedVirtualParents := hashset.NewFromSlice(virtualSelectedParent)
 	candidates := candidatesHeap.ToSlice()
 	// prioritize half the blocks with highest blueWork and half with lowest, so the network will merge splits faster.
 	if len(candidates) >= int(csm.maxBlockParents) {
@@ -46,7 +45,14 @@ func (csm *consensusStateManager) pickVirtualParents(tips []*externalapi.DomainH
 			end--
 		}
 	}
+	// Limit to maxBlockParents*3 candidates, that way we don't go over thousands of tips when the network isn't healthy.
+	// There's no specific reason for a factor of 3, and its not a consensus rule, just an estimation saying we probably
+	// don't want to consider and calculate 3 times the amount of candidates for the set of parents.
+	if len(candidates) > int(csm.maxBlockParents)*3 {
+		candidates = candidates[:int(csm.maxBlockParents)*3]
+	}
 
+	selectedVirtualParents := []*externalapi.DomainHash{virtualSelectedParent}
 	mergeSetSize := uint64(1) // starts counting from 1 because selectedParent is already in the mergeSet
 
 	for len(candidates) > 0 && uint64(len(selectedVirtualParents)) < uint64(csm.maxBlockParents) {
@@ -56,32 +62,68 @@ func (csm *consensusStateManager) pickVirtualParents(tips []*externalapi.DomainH
 		log.Debugf("Attempting to add %s to the virtual parents", candidate)
 		log.Debugf("The current merge set size is %d", mergeSetSize)
 
-		mergeSetIncrease, err := csm.mergeSetIncrease(candidate, selectedVirtualParents)
+		canBeParent, newCandidate, mergeSetIncrease, err := csm.mergeSetIncrease(candidate, selectedVirtualParents, mergeSetSize)
 		if err != nil {
 			return nil, err
 		}
-		log.Debugf("The merge set would increase by %d with block %s", mergeSetIncrease, candidate)
-
-		if mergeSetSize+mergeSetIncrease > csm.mergeSetSizeLimit {
-			log.Debugf("Cannot add block %s since that would violate the merge set size limit", candidate)
+		if canBeParent {
+			mergeSetSize += mergeSetIncrease
+			selectedVirtualParents = append(selectedVirtualParents, candidate)
+			log.Tracef("Added block %s to the virtual parents set", candidate)
 			continue
 		}
-
-		selectedVirtualParents.Add(candidate)
-		mergeSetSize += mergeSetIncrease
-		log.Tracef("Added block %s to the virtual parents set", candidate)
+		// If we already have a candidate in the past of newCandidate then skip.
+		isInFutureOfCandidates, err := csm.dagTopologyManager.IsAnyAncestorOf(candidates, newCandidate)
+		if err != nil {
+			return nil, err
+		}
+		if isInFutureOfCandidates {
+			continue
+		}
+		// Remove all candidates in the future of newCandidate
+		candidates, err = csm.removeHashesInFutureOf(candidates, newCandidate)
+		if err != nil {
+			return nil, err
+		}
+		candidates = append(candidates, newCandidate)
+		log.Debugf("Block %s increases merge set too much, instead adding its ancestor %s", candidate, newCandidate)
 	}
 
-	boundedMergeBreakingParents, err := csm.boundedMergeBreakingParents(selectedVirtualParents.ToSlice())
+	boundedMergeBreakingParents, err := csm.boundedMergeBreakingParents(selectedVirtualParents)
 	if err != nil {
 		return nil, err
 	}
 	log.Tracef("The following parents are omitted for "+
 		"breaking the bounded merge set: %s", boundedMergeBreakingParents)
 
-	virtualParents := selectedVirtualParents.Subtract(boundedMergeBreakingParents).ToSlice()
-	log.Tracef("The virtual parents resolved to be: %s", virtualParents)
-	return virtualParents, nil
+	// Remove all boundedMergeBreakingParents from selectedVirtualParents
+	for _, breakingParent := range boundedMergeBreakingParents {
+		for i, parent := range selectedVirtualParents {
+			if parent.Equal(breakingParent) {
+				selectedVirtualParents[i] = selectedVirtualParents[len(selectedVirtualParents)-1]
+				selectedVirtualParents = selectedVirtualParents[:len(selectedVirtualParents)-1]
+				break
+			}
+		}
+	}
+	log.Debugf("The virtual parents resolved to be: %s", selectedVirtualParents)
+	return selectedVirtualParents, nil
+}
+
+func (csm *consensusStateManager) removeHashesInFutureOf(hashes []*externalapi.DomainHash, ancestor *externalapi.DomainHash) ([]*externalapi.DomainHash, error) {
+	// Source: https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+	i := 0
+	for _, hash := range hashes {
+		isInFutureOfAncestor, err := csm.dagTopologyManager.IsAncestorOf(ancestor, hash)
+		if err != nil {
+			return nil, err
+		}
+		if !isInFutureOfAncestor {
+			hashes[i] = hash
+			i++
+		}
+	}
+	return hashes[:i], nil
 }
 
 func (csm *consensusStateManager) selectVirtualSelectedParent(
@@ -153,59 +195,66 @@ func (csm *consensusStateManager) selectVirtualSelectedParent(
 	}
 }
 
-func (csm *consensusStateManager) mergeSetIncrease(
-	candidate *externalapi.DomainHash, selectedVirtualParents hashset.HashSet) (uint64, error) {
-
+// mergeSetIncrease returns different things depending on the result:
+// If the candidate can be a virtual parent then canBeParent=true and mergeSetIncrease=The increase in merge set size
+// If the candidate can't be a virtual parent, then canBeParent=false and newCandidate is a new proposed candidate in the past of candidate.
+func (csm *consensusStateManager) mergeSetIncrease(candidate *externalapi.DomainHash, selectedVirtualParents []*externalapi.DomainHash, mergeSetSize uint64,
+) (canBeParent bool, newCandidate *externalapi.DomainHash, mergeSetIncrease uint64, err error) {
 	onEnd := logger.LogAndMeasureExecutionTime(log, "mergeSetIncrease")
 	defer onEnd()
 
 	visited := hashset.New()
-	queue := csm.dagTraversalManager.NewDownHeap()
-	err := queue.Push(candidate)
+	// Start with the candidate's parents in the queue as we already know the candidate isn't an ancestor of the selectedVirtualParents.
+	parents, err := csm.dagTopologyManager.Parents(candidate)
 	if err != nil {
-		return 0, err
+		return false, nil, 0, err
 	}
-	mergeSetIncrease := uint64(1) // starts with 1 for the candidate itself
+	for _, parent := range parents {
+		visited.Add(parent)
+	}
+	queue := parents
+	mergeSetIncrease = uint64(1) // starts with 1 for the candidate itself
 
-	for queue.Len() > 0 {
-		current := queue.Pop()
+	var current *externalapi.DomainHash
+	for len(queue) > 0 {
+		current, queue = queue[0], queue[1:]
 		log.Tracef("Attempting to increment the merge set size increase for block %s", current)
 
-		isInPastOfSelectedVirtualParents, err := csm.dagTopologyManager.IsAncestorOfAny(
-			current, selectedVirtualParents.ToSlice())
+		isInPastOfSelectedVirtualParents, err := csm.dagTopologyManager.IsAncestorOfAny(current, selectedVirtualParents)
 		if err != nil {
-			return 0, err
+			return false, nil, 0, err
 		}
 		if isInPastOfSelectedVirtualParents {
-			log.Tracef("Skipping block %s because it's in the past of one "+
-				"(or more) of the selected virtual parents", current)
+			log.Tracef("Skipping block %s because it's in the past of one (or more) of the selected virtual parents", current)
 			continue
 		}
 
 		log.Tracef("Incrementing the merge set size increase")
 		mergeSetIncrease++
 
+		if (mergeSetSize + mergeSetIncrease) > csm.mergeSetSizeLimit {
+			log.Debugf("The merge set would increase by more than the limit with block %s", candidate)
+			return false, current, mergeSetIncrease, nil
+		}
+
 		parents, err := csm.dagTopologyManager.Parents(current)
 		if err != nil {
-			return 0, err
+			return false, nil, 0, err
 		}
 		for _, parent := range parents {
 			if !visited.Contains(parent) {
 				visited.Add(parent)
-				err = queue.Push(parent)
-				if err != nil {
-					return 0, err
-				}
+				queue = append(queue, parent)
 			}
 		}
 	}
 	log.Debugf("The resolved merge set size increase is: %d", mergeSetIncrease)
 
-	return mergeSetIncrease, nil
+	return true, nil, mergeSetIncrease, nil
 }
 
 func (csm *consensusStateManager) boundedMergeBreakingParents(
-	parents []*externalapi.DomainHash) (hashset.HashSet, error) {
+	parents []*externalapi.DomainHash) ([]*externalapi.DomainHash, error) {
 
 	onEnd := logger.LogAndMeasureExecutionTime(log, "boundedMergeBreakingParents")
 	defer onEnd()
@@ -271,7 +320,7 @@ func (csm *consensusStateManager) boundedMergeBreakingParents(
 		}
 	}
 
-	boundedMergeBreakingParents := hashset.New()
+	var boundedMergeBreakingParents []*externalapi.DomainHash
 	for _, parent := range parents {
 		log.Debugf("Checking whether parent %s breaks the bounded merge set", parent)
 		isBadRedInPast := false
@@ -287,7 +336,7 @@ func (csm *consensusStateManager) boundedMergeBreakingParents(
 		}
 		if isBadRedInPast {
 			log.Debugf("Adding parent %s to the bounded merge breaking parents set", parent)
-			boundedMergeBreakingParents.Add(parent)
+			boundedMergeBreakingParents = append(boundedMergeBreakingParents, parent)
 		}
 	}
 

domain/consensus/processes/dagtopologymanager/dagtopologymanager.go

@@ -87,6 +87,22 @@ func (dtm *dagTopologyManager) IsAncestorOfAny(blockHash *externalapi.DomainHash
 	return false, nil
 }
 
+// IsAnyAncestorOf returns true if at least one of `potentialAncestors` is an ancestor of `blockHash`
+func (dtm *dagTopologyManager) IsAnyAncestorOf(potentialAncestors []*externalapi.DomainHash, blockHash *externalapi.DomainHash) (bool, error) {
+	for _, potentialAncestor := range potentialAncestors {
+		isAncestorOf, err := dtm.IsAncestorOf(potentialAncestor, blockHash)
+		if err != nil {
+			return false, err
+		}
+
+		if isAncestorOf {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
 // IsInSelectedParentChainOf returns true if blockHashA is in the selected parent chain of blockHashB
 func (dtm *dagTopologyManager) IsInSelectedParentChainOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) {
 	return dtm.reachabilityManager.IsReachabilityTreeAncestorOf(blockHashA, blockHashB)

domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go

@@ -14,14 +14,21 @@ type selectedChildIterator struct {
 	highHash, lowHash     *externalapi.DomainHash
 	current               *externalapi.DomainHash
 	err                   error
+	isClosed              bool
 }
 
 func (s *selectedChildIterator) First() bool {
+	if s.isClosed {
+		panic("Tried using a closed SelectedChildIterator")
+	}
 	s.current = s.lowHash
 	return s.Next()
 }
 
 func (s *selectedChildIterator) Next() bool {
+	if s.isClosed {
+		panic("Tried using a closed SelectedChildIterator")
+	}
 	if s.err != nil {
 		return true
 	}
@@ -50,9 +57,27 @@ func (s *selectedChildIterator) Next() bool {
 }
 
 func (s *selectedChildIterator) Get() (*externalapi.DomainHash, error) {
+	if s.isClosed {
+		return nil, errors.New("Tried using a closed SelectedChildIterator")
+	}
 	return s.current, s.err
 }
 
+func (s *selectedChildIterator) Close() error {
+	if s.isClosed {
+		return errors.New("Tried using a closed SelectedChildIterator")
+	}
+	s.isClosed = true
+	s.databaseContext = nil
+	s.dagTopologyManager = nil
+	s.reachabilityDataStore = nil
+	s.highHash = nil
+	s.lowHash = nil
+	s.current = nil
+	s.err = nil
+	return nil
+}
+
 // SelectedChildIterator returns a BlockIterator that iterates from lowHash (exclusive) to highHash (inclusive) over
 // highHash's selected parent chain
 func (dtm *dagTraversalManager) SelectedChildIterator(highHash, lowHash *externalapi.DomainHash) (model.BlockIterator, error) {

domain/consensus/processes/finalitymanager/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var log = logger.RegisterSubSystem("BDAG")

domain/consensus/processes/ghostdagmanager/ghostdag_test.go

@@ -390,6 +390,9 @@ func (dt *DAGTopologyManagerImpl) IsAncestorOf(hashBlockA *externalapi.DomainHas
 func (dt *DAGTopologyManagerImpl) IsAncestorOfAny(blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error) {
 	panic("unimplemented")
 }
+func (dt *DAGTopologyManagerImpl) IsAnyAncestorOf([]*externalapi.DomainHash, *externalapi.DomainHash) (bool, error) {
+	panic("unimplemented")
+}
 func (dt *DAGTopologyManagerImpl) IsInSelectedParentChainOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) {
 	panic("unimplemented")
 }

domain/consensus/processes/pruningmanager/log.go

@@ -2,4 +2,4 @@ package pruningmanager
 
 import "github.com/kaspanet/kaspad/infrastructure/logger"
 
-var log, _ = logger.Get(logger.SubsystemTags.PRNM)
+var log = logger.RegisterSubSystem("PRNM")

domain/consensus/processes/pruningmanager/pruningmanager.go

@@ -133,6 +133,7 @@ func (pm *pruningManager) UpdatePruningPointByVirtual() error {
 	if err != nil {
 		return err
 	}
+	defer iterator.Close()
 
 	// Finding the next pruning point candidate: look for the latest
 	// selected child of the current candidate that is in depth of at
@@ -425,6 +426,7 @@ func (pm *pruningManager) validateUTXOSetFitsCommitment(pruningPointHash *extern
 	if err != nil {
 		return err
 	}
+	defer utxoSetIterator.Close()
 
 	utxoSetMultiset := multiset.New()
 	for ok := utxoSetIterator.First(); ok; ok = utxoSetIterator.Next() {
@@ -544,6 +546,7 @@ func (pm *pruningManager) updatePruningPointUTXOSet() error {
 	if err != nil {
 		return err
 	}
+	defer utxoSetIterator.Close()
 
 	log.Debugf("Updating the pruning point UTXO set")
 	err = pm.pruningStore.UpdatePruningPointUTXOSet(pm.databaseContext, utxoSetIterator)
@@ -563,6 +566,7 @@ func (pm *pruningManager) PruneAllBlocksBelow(pruningPointHash *externalapi.Doma
 	if err != nil {
 		return err
 	}
+	defer iterator.Close()
 
 	for ok := iterator.First(); ok; ok = iterator.Next() {
 		blockHash, err := iterator.Get()

domain/consensus/processes/reachabilitymanager/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.REAC)
+var log = logger.RegisterSubSystem("REAC")

domain/consensus/processes/reachabilitymanager/main_test.go

@@ -0,0 +1,16 @@
+package reachabilitymanager_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/kaspanet/kaspad/infrastructure/logger"
+)
+
+const logLevel = logger.LevelWarn
+
+func TestMain(m *testing.M) {
+	logger.SetLogLevels(logLevel)
+	logger.InitLogStdout(logLevel)
+	os.Exit(m.Run())
+}

domain/consensus/processes/reachabilitymanager/reachability_stretch_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/model/testapi"
 	"github.com/kaspanet/kaspad/domain/dagconfig"
-	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"github.com/pkg/errors"
 	"math"
 	"math/rand"
@@ -16,14 +15,10 @@ import (
 )
 
 // Test configuration
-const (
-	numBlocksExponent = 12
-	logLevel          = "warn"
-)
+const numBlocksExponent = 12
 
 func initializeTest(t *testing.T, testName string) (tc testapi.TestConsensus, teardown func(keepDataDir bool)) {
 	t.Parallel()
-	logger.SetLogLevels(logLevel)
 	params := dagconfig.SimnetParams
 	params.SkipProofOfWork = true
 	tc, teardown, err := consensus.NewFactory().NewTestConsensus(&params, false, testName)

domain/consensus/processes/syncmanager/antipast.go

@@ -59,6 +59,7 @@ func (sm *syncManager) antiPastHashesBetween(lowHash, highHash *externalapi.Doma
 	if err != nil {
 		return nil, err
 	}
+	defer iterator.Close()
 	for ok := iterator.First(); ok; ok = iterator.Next() {
 		current, err := iterator.Get()
 		if err != nil {
@@ -145,6 +146,7 @@ func (sm *syncManager) findHighHashAccordingToMaxBlueScoreDifference(lowHash *ex
 	if err != nil {
 		return nil, err
 	}
+	defer iterator.Close()
 	for ok := iterator.First(); ok; ok = iterator.Next() {
 		highHashCandidate, err := iterator.Get()
 		if err != nil {
@@ -191,6 +193,7 @@ func (sm *syncManager) missingBlockBodyHashes(highHash *externalapi.DomainHash)
 	if err != nil {
 		return nil, err
 	}
+	defer selectedChildIterator.Close()
 
 	lowHash := pruningPoint
 	foundHeaderOnlyBlock := false

domain/consensus/processes/syncmanager/log.go

@@ -4,4 +4,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.SYNC)
+var log = logger.RegisterSubSystem("SYNC")

domain/consensus/test_consensus_render_to_dot.go

@@ -26,6 +26,7 @@ func (tc *testConsensus) convertToDot() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	defer blocksIterator.Close()
 
 	for ok := blocksIterator.First(); ok; ok = blocksIterator.Next() {
 		hash, err := blocksIterator.Get()

domain/consensus/utils/subnetworks/compare.go

@@ -1,23 +1,11 @@
 package subnetworks
 
 import (
+	"bytes"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 )
 
-func cmp(a, b externalapi.DomainSubnetworkID) int {
-	// We compare the hashes backwards because Hash is stored as a little endian byte array.
-	for i := externalapi.DomainSubnetworkIDSize - 1; i >= 0; i-- {
-		switch {
-		case a[i] < b[i]:
-			return -1
-		case a[i] > b[i]:
-			return 1
-		}
-	}
-	return 0
-}
-
 // Less returns true iff id a is less than id b
 func Less(a, b externalapi.DomainSubnetworkID) bool {
-	return cmp(a, b) < 0
+	return bytes.Compare(a[:], b[:]) < 0
 }

domain/consensus/utils/subnetworks/from_string.go

@@ -8,11 +8,7 @@ import (
 
 // FromString creates a DomainSubnetworkID from the given byte slice
 func FromString(str string) (*externalapi.DomainSubnetworkID, error) {
-	runes := []rune(str)
-	for i := 0; i < externalapi.DomainSubnetworkIDSize*2; i++ {
-		runes[i], runes[externalapi.DomainSubnetworkIDSize-1-i] = runes[externalapi.DomainSubnetworkIDSize-1-i], runes[i]
-	}
-	subnetworkIDBytes, err := hex.DecodeString(string(runes))
+	subnetworkIDBytes, err := hex.DecodeString(str)
 	if err != nil {
 		return nil, err
 	}

domain/consensus/utils/txscript/log.go

@@ -8,4 +8,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.SCRP)
+var log = logger.RegisterSubSystem("SCRP")

domain/consensus/utils/txscript/main_test.go

@@ -9,6 +9,7 @@ import (
 func TestMain(m *testing.M) {
 	// set log level to trace, so that logClosures passed to log.Tracef are covered
 	log.SetLevel(logger.LevelTrace)
+	logger.InitLogStdout(logger.LevelTrace)
 
 	os.Exit(m.Run())
 }

domain/consensus/utils/utxo/utxo_iterator.go

@@ -11,8 +11,9 @@ type utxoOutpointEntryPair struct {
 }
 
 type utxoCollectionIterator struct {
-	index int
-	pairs []utxoOutpointEntryPair
+	index    int
+	pairs    []utxoOutpointEntryPair
+	isClosed bool
 }
 
 func (uc utxoCollection) Iterator() externalapi.ReadOnlyUTXOSetIterator {
@@ -29,21 +30,33 @@ func (uc utxoCollection) Iterator() externalapi.ReadOnlyUTXOSetIterator {
 }
 
 func (uci *utxoCollectionIterator) First() bool {
+	if uci.isClosed {
+		panic("Tried using a closed utxoCollectionIterator")
+	}
 	uci.index = 0
 	return len(uci.pairs) > 0
 }
 
 func (uci *utxoCollectionIterator) Next() bool {
+	if uci.isClosed {
+		panic("Tried using a closed utxoCollectionIterator")
+	}
 	uci.index++
 	return uci.index < len(uci.pairs)
 }
 
 func (uci *utxoCollectionIterator) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry externalapi.UTXOEntry, err error) {
+	if uci.isClosed {
+		return nil, nil, errors.New("Tried using a closed utxoCollectionIterator")
+	}
 	pair := uci.pairs[uci.index]
 	return &pair.outpoint, pair.entry, nil
 }
 
 func (uci *utxoCollectionIterator) WithDiff(diff externalapi.UTXODiff) (externalapi.ReadOnlyUTXOSetIterator, error) {
+	if uci.isClosed {
+		return nil, errors.New("Tried using a closed utxoCollectionIterator")
+	}
 	d, ok := diff.(*immutableUTXODiff)
 	if !ok {
 		return nil, errors.New("diff is not of type *immutableUTXODiff")
@@ -55,3 +68,12 @@ func (uci *utxoCollectionIterator) WithDiff(diff externalapi.UTXODiff) (external
 		toAddIterator: diff.ToAdd().Iterator(),
 	}, nil
 }
+
+func (uci *utxoCollectionIterator) Close() error {
+	if uci.isClosed {
+		return errors.New("Tried using a closed utxoCollectionIterator")
+	}
+	uci.isClosed = true
+	uci.pairs = nil
+	return nil
+}

domain/consensus/utils/utxo/utxo_iterator_with_diff.go

@@ -14,6 +14,7 @@ type readOnlyUTXOIteratorWithDiff struct {
 	currentErr       error
 
 	toAddIterator externalapi.ReadOnlyUTXOSetIterator
+	isClosed      bool
 }
 
 // IteratorWithDiff applies a UTXODiff to given utxo iterator
@@ -40,9 +41,17 @@ func IteratorWithDiff(iterator externalapi.ReadOnlyUTXOSetIterator, diff externa
 }
 
 func (r *readOnlyUTXOIteratorWithDiff) First() bool {
+	if r.isClosed {
+		panic("Tried using a closed readOnlyUTXOIteratorWithDiff")
+	}
 	baseNotEmpty := r.baseIterator.First()
 	baseEmpty := !baseNotEmpty
 
+	err := r.toAddIterator.Close()
+	if err != nil {
+		r.currentErr = err
+		return true
+	}
 	r.toAddIterator = r.diff.ToAdd().Iterator()
 	toAddEmpty := r.diff.ToAdd().Len() == 0
 
@@ -61,6 +70,9 @@ func (r *readOnlyUTXOIteratorWithDiff) First() bool {
 }
 
 func (r *readOnlyUTXOIteratorWithDiff) Next() bool {
+	if r.isClosed {
+		panic("Tried using a closed readOnlyUTXOIteratorWithDiff")
+	}
 	for r.baseIterator.Next() { // keep looping until we reach an outpoint/entry pair that is not in r.diff.toRemove
 		r.currentOutpoint, r.currentUTXOEntry, r.currentErr = r.baseIterator.Get()
 		if !r.diff.mutableUTXODiff.toRemove.containsWithBlueScore(r.currentOutpoint, r.currentUTXOEntry.BlockBlueScore()) {
@@ -77,5 +89,30 @@ func (r *readOnlyUTXOIteratorWithDiff) Next() bool {
 }
 
 func (r *readOnlyUTXOIteratorWithDiff) Get() (outpoint *externalapi.DomainOutpoint, utxoEntry externalapi.UTXOEntry, err error) {
+	if r.isClosed {
+		return nil, nil, errors.New("Tried using a closed readOnlyUTXOIteratorWithDiff")
+	}
 	return r.currentOutpoint, r.currentUTXOEntry, r.currentErr
 }
+
+func (r *readOnlyUTXOIteratorWithDiff) Close() error {
+	if r.isClosed {
+		return errors.New("Tried using a closed readOnlyUTXOIteratorWithDiff")
+	}
+	r.isClosed = true
+	err := r.baseIterator.Close()
+	if err != nil {
+		return err
+	}
+	err = r.toAddIterator.Close()
+	if err != nil {
+		return err
+	}
+	r.baseIterator = nil
+	r.diff = nil
+	r.currentOutpoint = nil
+	r.currentUTXOEntry = nil
+	r.currentErr = nil
+	r.toAddIterator = nil
+	return nil
+}

domain/dagconfig/params.go

@@ -260,6 +260,7 @@ var TestnetParams = Params{
 	Net:         appmessage.Testnet,
 	RPCPort:     "16210",
 	DefaultPort: "16211",
+	DNSSeeds:    []string{"testnet-2-dnsseed.daglabs-dev.com"},
 
 	// DAG parameters
 	GenesisBlock:                   &testnetGenesisBlock,

domain/miningmanager/blocktemplatebuilder/log.go

@@ -8,4 +8,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.MINR)
+var log = logger.RegisterSubSystem("BLTB")

domain/miningmanager/mempool/log.go

@@ -8,4 +8,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.TXMP)
+var log = logger.RegisterSubSystem("TXMP")

domain/miningmanager/mempool/mempool.go

@@ -11,6 +11,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kaspanet/kaspad/infrastructure/logger"
+
 	"github.com/kaspanet/kaspad/domain/consensus/utils/constants"
 
 	"github.com/kaspanet/kaspad/domain/consensus/utils/transactionhelper"
@@ -20,7 +22,6 @@ import (
 	"github.com/kaspanet/kaspad/domain/consensus/utils/consensushashing"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/estimatedsize"
 	miningmanagermodel "github.com/kaspanet/kaspad/domain/miningmanager/model"
-	"github.com/kaspanet/kaspad/infrastructure/logger"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/mstime"
 	"github.com/pkg/errors"
@@ -229,9 +230,7 @@ func (mp *mempool) limitNumOrphans() error {
 
 		numOrphans := len(mp.orphans)
 		if numExpired := origNumOrphans - numOrphans; numExpired > 0 {
-			log.Debugf("Expired %d %s (remaining: %d)", numExpired,
-				logger.PickNoun(uint64(numExpired), "orphan", "orphans"),
-				numOrphans)
+			log.Debugf("Expired %d orphans (remaining: %d)", numExpired, numOrphans)
 		}
 	}
 
@@ -940,10 +939,14 @@ func (mp *mempool) ChainedCount() int {
 func (mp *mempool) BlockCandidateTransactions() []*consensusexternalapi.DomainTransaction {
 	mp.mtx.RLock()
 	defer mp.mtx.RUnlock()
+
+	onEnd := logger.LogAndMeasureExecutionTime(log, "BlockCandidateTransactions")
+	defer onEnd()
+
 	descs := make([]*consensusexternalapi.DomainTransaction, len(mp.pool))
 	i := 0
 	for _, desc := range mp.pool {
-		descs[i] = desc.DomainTransaction
+		descs[i] = desc.DomainTransaction.Clone() // Clone the transaction to prevent data races. A shallow-copy might do as well
 		i++
 	}
 

domain/utxoindex/log.go

@@ -8,4 +8,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.INDX)
+var log = logger.RegisterSubSystem("UTIN")

domain/utxoindex/serialization.go

@@ -45,8 +45,7 @@ func serializeHashes(hashes []*externalapi.DomainHash) []byte {
 	for i, hash := range hashes {
 		start := hashesLengthSize + externalapi.DomainHashSize*i
 		end := start + externalapi.DomainHashSize
-		copy(serializedHashes[start:end],
-			hash.ByteSlice())
+		copy(serializedHashes[start:end], hash.ByteSlice())
 	}
 	return serializedHashes
 }
@@ -63,12 +62,11 @@ func deserializeHashes(serializedHashes []byte) ([]*externalapi.DomainHash, erro
 		}
 
 		var err error
-		hashes[i], err = externalapi.
-			NewDomainHashFromByteSlice(serializedHashes[start:end])
+		hashes[i], err = externalapi.NewDomainHashFromByteSlice(serializedHashes[start:end])
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	return nil, nil
+	return hashes, nil
 }

domain/utxoindex/serialization_test.go

@@ -0,0 +1,44 @@
+package utxoindex
+
+import (
+	"encoding/binary"
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
+	"github.com/pkg/errors"
+	"io"
+	"math/rand"
+	"testing"
+)
+
+func Test_serializeHashes(t *testing.T) {
+	r := rand.New(rand.NewSource(0))
+
+	for length := 0; length < 32; length++ {
+		hashes := make([]*externalapi.DomainHash, length)
+		for i := range hashes {
+			var hashBytes [32]byte
+			r.Read(hashBytes[:])
+			hashes[i] = externalapi.NewDomainHashFromByteArray(&hashBytes)
+		}
+		result, err := deserializeHashes(serializeHashes(hashes))
+		if err != nil {
+			t.Fatalf("Failed deserializing hashes: %v", err)
+		}
+		if !externalapi.HashesEqual(hashes, result) {
+			t.Fatalf("Expected \n %s \n==\n %s\n", hashes, result)
+		}
+	}
+}
+
+func Test_deserializeHashesFailure(t *testing.T) {
+	hashes := []*externalapi.DomainHash{
+		externalapi.NewDomainHashFromByteArray(&[externalapi.DomainHashSize]byte{1}),
+		externalapi.NewDomainHashFromByteArray(&[externalapi.DomainHashSize]byte{2}),
+		externalapi.NewDomainHashFromByteArray(&[externalapi.DomainHashSize]byte{3}),
+	}
+	serialized := serializeHashes(hashes)
+	binary.LittleEndian.PutUint64(serialized[:8], uint64(len(hashes)+1))
+	_, err := deserializeHashes(serialized)
+	if !errors.Is(err, io.ErrUnexpectedEOF) {
+		t.Fatalf("Expected error to be EOF, instead got: %v", err)
+	}
+}

domain/utxoindex/store.go

@@ -249,6 +249,7 @@ func (uis *utxoIndexStore) getUTXOOutpointEntryPairs(scriptPublicKey *externalap
 	if err != nil {
 		return nil, err
 	}
+	defer cursor.Close()
 	utxoOutpointEntryPairs := make(UTXOOutpointEntryPairs)
 	for cursor.Next() {
 		key, err := cursor.Key()
@@ -297,6 +298,7 @@ func (uis *utxoIndexStore) deleteAll() error {
 	if err != nil {
 		return err
 	}
+	defer cursor.Close()
 	for cursor.Next() {
 		key, err := cursor.Key()
 		if err != nil {

domain/utxoindex/utxoindex.go

@@ -131,6 +131,7 @@ func (ui *UTXOIndex) Update(blockInsertionResult *externalapi.BlockInsertionResu
 
 func (ui *UTXOIndex) addUTXOs(toAdd externalapi.UTXOCollection) error {
 	iterator := toAdd.Iterator()
+	defer iterator.Close()
 	for ok := iterator.First(); ok; ok = iterator.Next() {
 		outpoint, entry, err := iterator.Get()
 		if err != nil {
@@ -148,6 +149,7 @@ func (ui *UTXOIndex) addUTXOs(toAdd externalapi.UTXOCollection) error {
 
 func (ui *UTXOIndex) removeUTXOs(toRemove externalapi.UTXOCollection) error {
 	iterator := toRemove.Iterator()
+	defer iterator.Close()
 	for ok := iterator.First(); ok; ok = iterator.Next() {
 		outpoint, entry, err := iterator.Get()
 		if err != nil {

infrastructure/config/log.go

@@ -8,4 +8,4 @@ import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.CNFG)
+var log = logger.RegisterSubSystem("CNFG")

infrastructure/db/database/ldb/cursor.go

@@ -106,7 +106,8 @@ func (c *LevelDBCursor) Close() error {
 		return errors.New("cannot close an already closed cursor")
 	}
 	c.isClosed = true
-
 	c.ldbIterator.Release()
+	c.ldbIterator = nil
+	c.bucket = nil
 	return nil
 }

infrastructure/db/database/ldb/log.go

@@ -1,5 +1,7 @@
 package ldb
 
-import "github.com/kaspanet/kaspad/infrastructure/logger"
+import (
+	"github.com/kaspanet/kaspad/infrastructure/logger"
+)
 
-var log, _ = logger.Get(logger.SubsystemTags.KSDB)
+var log = logger.RegisterSubSystem("KSDB")

infrastructure/logger/backend.go

@@ -0,0 +1,191 @@
+package logger
+
+import (
+	"fmt"
+	"github.com/jrick/logrotate/rotator"
+	"github.com/pkg/errors"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+const normalLogSize = 512
+
+// defaultFlags specifies changes to the default logger behavior. It is set
+// during package init and configured using the LOGFLAGS environment variable.
+// New logger backends can override these default flags using WithFlags.
+// We're using this instead of `init()` function because variables are initialized before init functions,
+// and this variable is used inside other variable intializations, so runs before them
+var defaultFlags = getDefaultFlags()
+
+// Flags to modify Backend's behavior.
+const (
+	// LogFlagLongFile modifies the logger output to include full path and line number
+	// of the logging callsite, e.g. /a/b/c/main.go:123.
+	LogFlagLongFile uint32 = 1 << iota
+
+	// LogFlagShortFile modifies the logger output to include filename and line number
+	// of the logging callsite, e.g. main.go:123. takes precedence over LogFlagLongFile.
+	LogFlagShortFile
+)
+
+// Read logger flags from the LOGFLAGS environment variable. Multiple flags can
+// be set at once, separated by commas.
+func getDefaultFlags() (flags uint32) {
+	for _, f := range strings.Split(os.Getenv("LOGFLAGS"), ",") {
+		switch f {
+		case "longfile":
+			flags |= LogFlagLongFile
+		case "shortfile":
+			flags |= LogFlagShortFile
+		}
+	}
+	return
+}
+
+const logsBuffer = 0
+
+// Backend is a logging backend. Subsystems created from the backend write to
+// the backend's Writer. Backend provides atomic writes to the Writer from all
+// subsystems.
+type Backend struct {
+	flag      uint32
+	isRunning uint32
+	writers   []logWriter
+	writeChan chan logEntry
+	syncClose sync.Mutex // used to sync that the logger finished writing everything
+}
+
+// NewBackendWithFlags configures a Backend to use the specified flags rather than using
+// the package's defaults as determined through the LOGFLAGS environment
+// variable.
+func NewBackendWithFlags(flags uint32) *Backend {
+	return &Backend{flag: flags, writeChan: make(chan logEntry, logsBuffer)}
+}
+
+// NewBackend creates a new logger backend.
+func NewBackend() *Backend {
+	return NewBackendWithFlags(defaultFlags)
+}
+
+const (
+	defaultThresholdKB = 100 * 1000 // 100 MB logs by default.
+	defaultMaxRolls    = 8          // keep 8 last logs by default.
+)
+
+type logWriter interface {
+	io.WriteCloser
+	LogLevel() Level
+}
+
+type logWriterWrap struct {
+	io.WriteCloser
+	logLevel Level
+}
+
+func (lw logWriterWrap) LogLevel() Level {
+	return lw.logLevel
+}
+
+// AddLogFile adds a file which the log will write into on a certain
+// log level with the default log rotation settings. It'll create the file if it doesn't exist.
+func (b *Backend) AddLogFile(logFile string, logLevel Level) error {
+	return b.AddLogFileWithCustomRotator(logFile, logLevel, defaultThresholdKB, defaultMaxRolls)
+}
+
+// AddLogWriter adds a type implementing io.WriteCloser which the log will write into on a certain
+// log level with the default log rotation settings. It'll create the file if it doesn't exist.
+func (b *Backend) AddLogWriter(logWriter io.WriteCloser, logLevel Level) error {
+	if b.IsRunning() {
+		return errors.New("The logger is already running")
+	}
+	b.writers = append(b.writers, logWriterWrap{
+		WriteCloser: logWriter,
+		logLevel:    logLevel,
+	})
+	return nil
+}
+
+// AddLogFileWithCustomRotator adds a file which the log will write into on a certain
+// log level, with the specified log rotation settings.
+// It'll create the file if it doesn't exist.
+func (b *Backend) AddLogFileWithCustomRotator(logFile string, logLevel Level, thresholdKB int64, maxRolls int) error {
+	if b.IsRunning() {
+		return errors.New("The logger is already running")
+	}
+	logDir, _ := filepath.Split(logFile)
+	// if the logDir is empty then `logFile` is in the cwd and there's no need to create any directory.
+	if logDir != "" {
+		err := os.MkdirAll(logDir, 0700)
+		if err != nil {
+			return errors.Errorf("failed to create log directory: %+v", err)
+		}
+	}
+	r, err := rotator.New(logFile, thresholdKB, false, maxRolls)
+	if err != nil {
+		return errors.Errorf("failed to create file rotator: %s", err)
+	}
+	b.writers = append(b.writers, logWriterWrap{
+		WriteCloser: r,
+		logLevel:    logLevel,
+	})
+	return nil
+}
+
+// Run launches the logger backend in a separate go-routine. should only be called once.
+func (b *Backend) Run() error {
+	if !atomic.CompareAndSwapUint32(&b.isRunning, 0, 1) {
+		return errors.New("The logger is already running")
+	}
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				_, _ = fmt.Fprintf(os.Stderr, "Fatal error in logger.Backend goroutine: %+v\n", err)
+				_, _ = fmt.Fprintf(os.Stderr, "Goroutine stacktrace: %s\n", debug.Stack())
+			}
+		}()
+		b.runBlocking()
+	}()
+	return nil
+}
+
+func (b *Backend) runBlocking() {
+	defer atomic.StoreUint32(&b.isRunning, 0)
+	b.syncClose.Lock()
+	defer b.syncClose.Unlock()
+
+	for log := range b.writeChan {
+		for _, writer := range b.writers {
+			if log.level >= writer.LogLevel() {
+				_, _ = writer.Write(log.log)
+			}
+		}
+	}
+}
+
+// IsRunning returns true if backend.Run() has been called and false if it hasn't.
+func (b *Backend) IsRunning() bool {
+	return atomic.LoadUint32(&b.isRunning) != 0
+}
+
+// Close finalizes all log rotators for this backend
+func (b *Backend) Close() {
+	close(b.writeChan)
+	// Wait for it to finish writing using the syncClose mutex.
+	b.syncClose.Lock()
+	defer b.syncClose.Unlock()
+	for _, writer := range b.writers {
+		_ = writer.Close()
+	}
+}
+
+// Logger returns a new logger for a particular subsystem that writes to the
+// Backend b. A tag describes the subsystem and is included in all log
+// messages. The logger uses the info verbosity level by default.
+func (b *Backend) Logger(subsystemTag string) *Logger {
+	return &Logger{LevelOff, subsystemTag, b, b.writeChan}
+}

infrastructure/logger/level.go

@@ -0,0 +1,54 @@
+package logger
+
+import "strings"
+
+// Level is the level at which a logger is configured. All messages sent
+// to a level which is below the current level are filtered.
+type Level uint32
+
+// Level constants.
+const (
+	LevelTrace Level = iota
+	LevelDebug
+	LevelInfo
+	LevelWarn
+	LevelError
+	LevelCritical
+	LevelOff
+)
+
+// levelStrs defines the human-readable names for each logging level.
+var levelStrs = [...]string{"TRC", "DBG", "INF", "WRN", "ERR", "CRT", "OFF"}
+
+// LevelFromString returns a level based on the input string s. If the input
+// can't be interpreted as a valid log level, the info level and false is
+// returned.
+func LevelFromString(s string) (l Level, ok bool) {
+	switch strings.ToLower(s) {
+	case "trace", "trc":
+		return LevelTrace, true
+	case "debug", "dbg":
+		return LevelDebug, true
+	case "info", "inf":
+		return LevelInfo, true
+	case "warn", "wrn":
+		return LevelWarn, true
+	case "error", "err":
+		return LevelError, true
+	case "critical", "crt":
+		return LevelCritical, true
+	case "off":
+		return LevelOff, true
+	default:
+		return LevelInfo, false
+	}
+}
+
+// String returns the tag of the logger used in log messages, or "OFF" if
+// the level will not produce any log output.
+func (l Level) String() string {
+	if l >= LevelOff {
+		return "OFF"
+	}
+	return levelStrs[l]
+}

infrastructure/logger/logger.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/pkg/errors"
 )
@@ -26,135 +27,37 @@ var (
 	// BackendLog is the logging backend used to create all subsystem loggers.
 	BackendLog = NewBackend()
 
-	adxrLog = BackendLog.Logger("ADXR")
-	amgrLog = BackendLog.Logger("AMGR")
-	cmgrLog = BackendLog.Logger("CMGR")
-	ksdbLog = BackendLog.Logger("KSDB")
-	kasdLog = BackendLog.Logger("KASD")
-	bdagLog = BackendLog.Logger("BDAG")
-	cnfgLog = BackendLog.Logger("CNFG")
-	discLog = BackendLog.Logger("DISC")
-	indxLog = BackendLog.Logger("INDX")
-	minrLog = BackendLog.Logger("MINR")
-	peerLog = BackendLog.Logger("PEER")
-	rpcsLog = BackendLog.Logger("RPCS")
-	rpccLog = BackendLog.Logger("RPCC")
-	scrpLog = BackendLog.Logger("SCRP")
-	srvrLog = BackendLog.Logger("SRVR")
-	syncLog = BackendLog.Logger("SYNC")
-	txmpLog = BackendLog.Logger("TXMP")
-	utilLog = BackendLog.Logger("UTIL")
-	profLog = BackendLog.Logger("PROF")
-	protLog = BackendLog.Logger("PROT")
-	muxxLog = BackendLog.Logger("MUXX")
-	grpcLog = BackendLog.Logger("GRPC")
-	p2psLog = BackendLog.Logger("P2PS")
-	ntarLog = BackendLog.Logger("NTAR")
-	dnssLog = BackendLog.Logger("DNSS")
-	snvrLog = BackendLog.Logger("SNVR")
-	wsvcLog = BackendLog.Logger("WSVC")
-	reacLog = BackendLog.Logger("REAC")
-	prnmLog = BackendLog.Logger("PRNM")
-	blvlLog = BackendLog.Logger("BLVL")
+	// subsystemLoggers maps each subsystem identifier to its associated logger.
+	subsystemLoggers      = make(map[string]*Logger)
+	subsystemLoggersMutex sync.Mutex
 )
 
-// SubsystemTags is an enum of all sub system tags
-var SubsystemTags = struct {
-	ADXR,
-	AMGR,
-	CMGR,
-	KSDB,
-	KASD,
-	BDAG,
-	CNFG,
-	DISC,
-	INDX,
-	MINR,
-	PEER,
-	RPCS,
-	RPCC,
-	SCRP,
-	SRVR,
-	SYNC,
-	TXMP,
-	UTIL,
-	PROF,
-	PROT,
-	MUXX,
-	GRPC,
-	P2PS,
-	NTAR,
-	DNSS,
-	SNVR,
-	WSVC,
-	REAC,
-	PRNM,
-	BLVL string
-}{
-	ADXR: "ADXR",
-	AMGR: "AMGR",
-	CMGR: "CMGR",
-	KSDB: "KSDB",
-	KASD: "KASD",
-	BDAG: "BDAG",
-	CNFG: "CNFG",
-	DISC: "DISC",
-	INDX: "INDX",
-	MINR: "MINR",
-	PEER: "PEER",
-	RPCS: "RPCS",
-	RPCC: "RPCC",
-	SCRP: "SCRP",
-	SRVR: "SRVR",
-	SYNC: "SYNC",
-	TXMP: "TXMP",
-	UTIL: "UTIL",
-	PROF: "PROF",
-	PROT: "PROT",
-	MUXX: "MUXX",
-	GRPC: "GRPC",
-	P2PS: "P2PS",
-	NTAR: "NTAR",
-	DNSS: "DNSS",
-	SNVR: "SNVR",
-	WSVC: "WSVC",
-	REAC: "REAC",
-	PRNM: "PRNM",
-	BLVL: "BLVL",
+// RegisterSubSystem Registers a new subsystem logger, should be called in a global variable,
+// returns the existing one if the subsystem is already registered
+func RegisterSubSystem(subsystem string) *Logger {
+	subsystemLoggersMutex.Lock()
+	defer subsystemLoggersMutex.Unlock()
+	logger, exists := subsystemLoggers[subsystem]
+	if !exists {
+		logger = BackendLog.Logger(subsystem)
+		subsystemLoggers[subsystem] = logger
+	}
+	return logger
 }
 
-// subsystemLoggers maps each subsystem identifier to its associated logger.
-var subsystemLoggers = map[string]*Logger{
-	SubsystemTags.ADXR: adxrLog,
-	SubsystemTags.AMGR: amgrLog,
-	SubsystemTags.CMGR: cmgrLog,
-	SubsystemTags.KSDB: ksdbLog,
-	SubsystemTags.KASD: kasdLog,
-	SubsystemTags.BDAG: bdagLog,
-	SubsystemTags.CNFG: cnfgLog,
-	SubsystemTags.DISC: discLog,
-	SubsystemTags.INDX: indxLog,
-	SubsystemTags.MINR: minrLog,
-	SubsystemTags.PEER: peerLog,
-	SubsystemTags.RPCS: rpcsLog,
-	SubsystemTags.RPCC: rpccLog,
-	SubsystemTags.SCRP: scrpLog,
-	SubsystemTags.SRVR: srvrLog,
-	SubsystemTags.SYNC: syncLog,
-	SubsystemTags.TXMP: txmpLog,
-	SubsystemTags.UTIL: utilLog,
-	SubsystemTags.PROF: profLog,
-	SubsystemTags.PROT: protLog,
-	SubsystemTags.MUXX: muxxLog,
-	SubsystemTags.GRPC: grpcLog,
-	SubsystemTags.P2PS: p2psLog,
-	SubsystemTags.NTAR: ntarLog,
-	SubsystemTags.DNSS: dnssLog,
-	SubsystemTags.SNVR: snvrLog,
-	SubsystemTags.WSVC: wsvcLog,
-	SubsystemTags.REAC: reacLog,
-	SubsystemTags.PRNM: prnmLog,
-	SubsystemTags.BLVL: blvlLog,
+// InitLogStdout attaches stdout to the backend log and starts the logger.
+func InitLogStdout(logLevel Level) {
+	err := BackendLog.AddLogWriter(os.Stdout, logLevel)
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "Error adding stdout to the loggerfor level %s: %s", LevelWarn, err)
+		os.Exit(1)
+	}
+
+	err = BackendLog.Run()
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "Error starting the logger: %s ", err)
+		os.Exit(1)
+	}
 }
 
 // InitLog attaches log file and error log file to the backend log.
@@ -162,63 +65,65 @@ func InitLog(logFile, errLogFile string) {
 	// 280 MB (MB=1000^2 bytes)
 	err := BackendLog.AddLogFileWithCustomRotator(logFile, LevelTrace, 1000*280, 64)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error adding log file %s as log rotator for level %s: %s", logFile, LevelTrace, err)
+		_, _ = fmt.Fprintf(os.Stderr, "Error adding log file %s as log rotator for level %s: %s", logFile, LevelTrace, err)
 		os.Exit(1)
 	}
 	err = BackendLog.AddLogFile(errLogFile, LevelWarn)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error adding log file %s as log rotator for level %s: %s", errLogFile, LevelWarn, err)
+		_, _ = fmt.Fprintf(os.Stderr, "Error adding log file %s as log rotator for level %s: %s", errLogFile, LevelWarn, err)
 		os.Exit(1)
 	}
+
+	InitLogStdout(LevelInfo)
 }
 
 // SetLogLevel sets the logging level for provided subsystem. Invalid
 // subsystems are ignored. Uninitialized subsystems are dynamically created as
 // needed.
-func SetLogLevel(subsystemID string, logLevel string) {
-	// Ignore invalid subsystems.
+func SetLogLevel(subsystemID string, logLevel string) error {
+	subsystemLoggersMutex.Lock()
+	defer subsystemLoggersMutex.Unlock()
 	logger, ok := subsystemLoggers[subsystemID]
 	if !ok {
-		return
+		return errors.Errorf("'%s' Isn't a valid subsystem", subsystemID)
+	}
+	level, ok := LevelFromString(logLevel)
+	if !ok {
+		return errors.Errorf("'%s' Isn't a valid log level", logLevel)
 	}
 
-	// Defaults to info if the log level is invalid.
-	level, _ := LevelFromString(logLevel)
 	logger.SetLevel(level)
+	return nil
+}
+
+// SetLogLevelsString the same as SetLogLevels but also parses the level from a string
+func SetLogLevelsString(logLevel string) error {
+	level, ok := LevelFromString(logLevel)
+	if !ok {
+		return errors.Errorf("'%s' Isn't a valid log level", logLevel)
+	}
+	SetLogLevels(level)
+	return nil
 }
 
 // SetLogLevels sets the log level for all subsystem loggers to the passed
 // level. It also dynamically creates the subsystem loggers as needed, so it
 // can be used to initialize the logging system.
-func SetLogLevels(logLevel string) {
+func SetLogLevels(logLevel Level) {
+	subsystemLoggersMutex.Lock()
+	defer subsystemLoggersMutex.Unlock()
 	// Configure all sub-systems with the new logging level. Dynamically
 	// create loggers as needed.
-	for subsystemID := range subsystemLoggers {
-		SetLogLevel(subsystemID, logLevel)
+	for _, logger := range subsystemLoggers {
+		logger.SetLevel(logLevel)
 	}
 }
 
-// DirectionString is a helper function that returns a string that represents
-// the direction of a connection (inbound or outbound).
-func DirectionString(inbound bool) string {
-	if inbound {
-		return "inbound"
-	}
-	return "outbound"
-}
-
-// PickNoun returns the singular or plural form of a noun depending
-// on the count n.
-func PickNoun(n uint64, singular, plural string) string {
-	if n == 1 {
-		return singular
-	}
-	return plural
-}
-
 // SupportedSubsystems returns a sorted slice of the supported subsystems for
 // logging purposes.
 func SupportedSubsystems() []string {
+	subsystemLoggersMutex.Lock()
+	defer subsystemLoggersMutex.Unlock()
 	// Convert the subsystemLoggers map keys to a slice.
 	subsystems := make([]string, 0, len(subsystemLoggers))
 	for subsysID := range subsystemLoggers {
@@ -230,8 +135,9 @@ func SupportedSubsystems() []string {
 	return subsystems
 }
 
-// Get returns a logger of a specific sub system
-func Get(tag string) (logger *Logger, ok bool) {
+func getSubsystem(tag string) (logger *Logger, ok bool) {
+	subsystemLoggersMutex.Lock()
+	defer subsystemLoggersMutex.Unlock()
 	logger, ok = subsystemLoggers[tag]
 	return
 }
@@ -243,16 +149,8 @@ func ParseAndSetLogLevels(logLevel string) error {
 	// When the specified string doesn't have any delimters, treat it as
 	// the log level for all subsystems.
 	if !strings.Contains(logLevel, ",") && !strings.Contains(logLevel, "=") {
-		// Validate debug log level.
-		if !validLogLevel(logLevel) {
-			str := "The specified debug level [%s] is invalid"
-			return errors.Errorf(str, logLevel)
-		}
-
-		// Change the logging level for all subsystems.
-		SetLogLevels(logLevel)
-
-		return nil
+		// Validate and change the logging level for all subsystems.
+		return SetLogLevelsString(logLevel)
 	}
 
 	// Split the specified string into subsystem/level pairs while detecting
@@ -269,54 +167,16 @@ func ParseAndSetLogLevels(logLevel string) error {
 		subsysID, logLevel := fields[0], fields[1]
 
 		// Validate subsystem.
-		if _, exists := Get(subsysID); !exists {
+		if _, exists := getSubsystem(subsysID); !exists {
 			str := "The specified subsystem [%s] is invalid -- " +
 				"supported subsytems %s"
 			return errors.Errorf(str, subsysID, strings.Join(SupportedSubsystems(), ", "))
 		}
 
-		// Validate log level.
-		if !validLogLevel(logLevel) {
-			str := "The specified debug level [%s] is invalid"
-			return errors.Errorf(str, logLevel)
+		err := SetLogLevel(subsysID, logLevel)
+		if err != nil {
+			return err
 		}
-
-		SetLogLevel(subsysID, logLevel)
 	}
-
 	return nil
 }
-
-// validLogLevel returns whether or not logLevel is a valid debug log level.
-func validLogLevel(logLevel string) bool {
-	switch logLevel {
-	case "trace":
-		fallthrough
-	case "debug":
-		fallthrough
-	case "info":
-		fallthrough
-	case "warn":
-		fallthrough
-	case "error":
-		fallthrough
-	case "critical":
-		return true
-	}
-	return false
-}
-
-// LogClosure is a closure that can be printed with %s to be used to
-// generate expensive-to-create data for a detailed log level and avoid doing
-// the work if the data isn't printed.
-type LogClosure func() string
-
-func (c LogClosure) String() string {
-	return c()
-}
-
-// NewLogClosure casts a function to a LogClosure.
-// See LogClosure for details.
-func NewLogClosure(c func() string) LogClosure {
-	return c
-}

infrastructure/logger/logs.go

@@ -36,362 +36,22 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/kaspanet/kaspad/util/mstime"
-	"github.com/pkg/errors"
 	"os"
-	"path/filepath"
 	"runtime"
-	"strings"
-	"sync"
 	"sync/atomic"
-
-	"github.com/jrick/logrotate/rotator"
 )
 
-// defaultFlags specifies changes to the default logger behavior. It is set
-// during package init and configured using the LOGFLAGS environment variable.
-// New logger backends can override these default flags using WithFlags.
-var defaultFlags uint32
-
-// Flags to modify Backend's behavior.
-const (
-	// Llongfile modifies the logger output to include full path and line number
-	// of the logging callsite, e.g. /a/b/c/main.go:123.
-	Llongfile uint32 = 1 << iota
-
-	// Lshortfile modifies the logger output to include filename and line number
-	// of the logging callsite, e.g. main.go:123. Overrides Llongfile.
-	Lshortfile
-)
-
-// Read logger flags from the LOGFLAGS environment variable. Multiple flags can
-// be set at once, separated by commas.
-func init() {
-	for _, f := range strings.Split(os.Getenv("LOGFLAGS"), ",") {
-		switch f {
-		case "longfile":
-			defaultFlags |= Llongfile
-		case "shortfile":
-			defaultFlags |= Lshortfile
-		}
-	}
-}
-
-// Level is the level at which a logger is configured. All messages sent
-// to a level which is below the current level are filtered.
-type Level uint32
-
-// Level constants.
-const (
-	LevelTrace Level = iota
-	LevelDebug
-	LevelInfo
-	LevelWarn
-	LevelError
-	LevelCritical
-	LevelOff
-)
-
-// levelStrs defines the human-readable names for each logging level.
-var levelStrs = [...]string{"TRC", "DBG", "INF", "WRN", "ERR", "CRT", "OFF"}
-
-// LevelFromString returns a level based on the input string s. If the input
-// can't be interpreted as a valid log level, the info level and false is
-// returned.
-func LevelFromString(s string) (l Level, ok bool) {
-	switch strings.ToLower(s) {
-	case "trace", "trc":
-		return LevelTrace, true
-	case "debug", "dbg":
-		return LevelDebug, true
-	case "info", "inf":
-		return LevelInfo, true
-	case "warn", "wrn":
-		return LevelWarn, true
-	case "error", "err":
-		return LevelError, true
-	case "critical", "crt":
-		return LevelCritical, true
-	case "off":
-		return LevelOff, true
-	default:
-		return LevelInfo, false
-	}
-}
-
-// String returns the tag of the logger used in log messages, or "OFF" if
-// the level will not produce any log output.
-func (l Level) String() string {
-	if l >= LevelOff {
-		return "OFF"
-	}
-	return levelStrs[l]
-}
-
-// NewBackend creates a new logger backend.
-func NewBackend(opts ...BackendOption) *Backend {
-	b := &Backend{flag: defaultFlags, stdoutLevel: LevelInfo}
-	for _, o := range opts {
-		o(b)
-	}
-	return b
-}
-
-type backendLogRotator struct {
-	*rotator.Rotator
-	logLevel Level
-}
-
-// Backend is a logging backend. Subsystems created from the backend write to
-// the backend's Writer. Backend provides atomic writes to the Writer from all
-// subsystems.
-type Backend struct {
-	rotators    []*backendLogRotator
-	mu          sync.Mutex // ensures atomic writes
-	flag        uint32
-	stdoutLevel Level
-}
-
-// BackendOption is a function used to modify the behavior of a Backend.
-type BackendOption func(b *Backend)
-
-// WithFlags configures a Backend to use the specified flags rather than using
-// the package's defaults as determined through the LOGFLAGS environment
-// variable.
-func WithFlags(flags uint32) BackendOption {
-	return func(b *Backend) {
-		b.flag = flags
-	}
-}
-
-// bufferPool defines a concurrent safe free list of byte slices used to provide
-// temporary buffers for formatting log messages prior to outputting them.
-var bufferPool = sync.Pool{
-	New: func() interface{} {
-		b := make([]byte, 0, 120)
-		return &b // pointer to slice to avoid boxing alloc
-	},
-}
-
-// buffer returns a byte slice from the free list. A new buffer is allocated if
-// there are not any available on the free list. The returned byte slice should
-// be returned to the fee list by using the recycleBuffer function when the
-// caller is done with it.
-func buffer() *[]byte {
-	return bufferPool.Get().(*[]byte)
-}
-
-// recycleBuffer puts the provided byte slice, which should have been obtain via
-// the buffer function, back on the free list.
-func recycleBuffer(b *[]byte) {
-	*b = (*b)[:0]
-	bufferPool.Put(b)
-}
-
-// From stdlib log package.
-// Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid
-// zero-padding.
-func itoa(buf *[]byte, i int, wid int) {
-	// Assemble decimal in reverse order.
-	var b [20]byte
-	bp := len(b) - 1
-	for i >= 10 || wid > 1 {
-		wid--
-		q := i / 10
-		b[bp] = byte('0' + i - q*10)
-		bp--
-		i = q
-	}
-	// i < 10
-	b[bp] = byte('0' + i)
-	*buf = append(*buf, b[bp:]...)
-}
-
-// Appends a header in the default format 'YYYY-MM-DD hh:mm:ss.sss [LVL] TAG: '.
-// If either of the Lshortfile or Llongfile flags are specified, the file named
-// and line number are included after the tag and before the final colon.
-func formatHeader(buf *[]byte, t mstime.Time, lvl, tag string, file string, line int) {
-	year, month, day := t.Date()
-	hour, min, sec := t.Clock()
-	ms := t.Millisecond()
-
-	itoa(buf, year, 4)
-	*buf = append(*buf, '-')
-	itoa(buf, int(month), 2)
-	*buf = append(*buf, '-')
-	itoa(buf, day, 2)
-	*buf = append(*buf, ' ')
-	itoa(buf, hour, 2)
-	*buf = append(*buf, ':')
-	itoa(buf, min, 2)
-	*buf = append(*buf, ':')
-	itoa(buf, sec, 2)
-	*buf = append(*buf, '.')
-	itoa(buf, ms, 3)
-	*buf = append(*buf, " ["...)
-	*buf = append(*buf, lvl...)
-	*buf = append(*buf, "] "...)
-	*buf = append(*buf, tag...)
-	if file != "" {
-		*buf = append(*buf, ' ')
-		*buf = append(*buf, file...)
-		*buf = append(*buf, ':')
-		itoa(buf, line, -1)
-	}
-	*buf = append(*buf, ": "...)
-}
-
-// calldepth is the call depth of the callsite function relative to the
-// caller of the subsystem logger. It is used to recover the filename and line
-// number of the logging call if either the short or long file flags are
-// specified.
-const calldepth = 3
-
-// callsite returns the file name and line number of the callsite to the
-// subsystem logger.
-func callsite(flag uint32) (string, int) {
-	_, file, line, ok := runtime.Caller(calldepth)
-	if !ok {
-		return "???", 0
-	}
-	if flag&Lshortfile != 0 {
-		short := file
-		for i := len(file) - 1; i > 0; i-- {
-			if os.IsPathSeparator(file[i]) {
-				short = file[i+1:]
-				break
-			}
-		}
-		file = short
-	}
-	return file, line
-}
-
-const (
-	defaultThresholdKB = 100 * 1000 // 100 MB logs by default.
-	defaultMaxRolls    = 8          // keep 8 last logs by default.
-)
-
-// AddLogFile adds a file which the log will write into on a certain
-// log level with the default log rotation settings. It'll create the file if it doesn't exist.
-func (b *Backend) AddLogFile(logFile string, logLevel Level) error {
-	return b.AddLogFileWithCustomRotator(logFile, logLevel, defaultThresholdKB, defaultMaxRolls)
-}
-
-// AddLogFileWithCustomRotator adds a file which the log will write into on a certain
-// log level, with the specified log rotation settings.
-// It'll create the file if it doesn't exist.
-func (b *Backend) AddLogFileWithCustomRotator(logFile string, logLevel Level, thresholdKB int64, maxRolls int) error {
-	logDir, _ := filepath.Split(logFile)
-	// if the logDir is empty then `logFile` is in the cwd and there's no need to create any directory.
-	if logDir != "" {
-		err := os.MkdirAll(logDir, 0700)
-		if err != nil {
-			return errors.Errorf("failed to create log directory: %+v", err)
-		}
-	}
-	r, err := rotator.New(logFile, thresholdKB, false, maxRolls)
-	if err != nil {
-		return errors.Errorf("failed to create file rotator: %s", err)
-	}
-	b.rotators = append(b.rotators, &backendLogRotator{
-		Rotator:  r,
-		logLevel: logLevel,
-	})
-	return nil
-}
-
-// print outputs a log message to the writer associated with the backend after
-// creating a prefix for the given level and tag according to the formatHeader
-// function and formatting the provided arguments using the default formatting
-// rules.
-func (b *Backend) print(lvl Level, tag string, args ...interface{}) {
-	t := mstime.Now() // get as early as possible
-
-	bytebuf := buffer()
-
-	var file string
-	var line int
-	if b.flag&(Lshortfile|Llongfile) != 0 {
-		file, line = callsite(b.flag)
-	}
-
-	formatHeader(bytebuf, t, lvl.String(), tag, file, line)
-	buf := bytes.NewBuffer(*bytebuf)
-	fmt.Fprintln(buf, args...)
-	*bytebuf = buf.Bytes()
-
-	b.write(lvl, *bytebuf)
-
-	recycleBuffer(bytebuf)
-}
-
-// printf outputs a log message to the writer associated with the backend after
-// creating a prefix for the given level and tag according to the formatHeader
-// function and formatting the provided arguments according to the given format
-// specifier.
-func (b *Backend) printf(lvl Level, tag string, format string, args ...interface{}) {
-	t := mstime.Now() // get as early as possible
-
-	bytebuf := buffer()
-
-	var file string
-	var line int
-	if b.flag&(Lshortfile|Llongfile) != 0 {
-		file, line = callsite(b.flag)
-	}
-
-	formatHeader(bytebuf, t, lvl.String(), tag, file, line)
-	buf := bytes.NewBuffer(*bytebuf)
-	fmt.Fprintf(buf, format, args...)
-	*bytebuf = append(buf.Bytes(), '\n')
-	b.write(lvl, *bytebuf)
-
-	recycleBuffer(bytebuf)
-}
-
-func (b *Backend) write(lvl Level, bytesToWrite []byte) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if lvl >= b.StdoutLevel() {
-		os.Stdout.Write(bytesToWrite)
-	}
-
-	for _, r := range b.rotators {
-		if lvl >= r.logLevel {
-			r.Write(bytesToWrite)
-		}
-	}
-}
-
-// StdoutLevel returns the current stdout logging level
-func (b *Backend) StdoutLevel() Level {
-	return Level(atomic.LoadUint32((*uint32)(&b.stdoutLevel)))
-}
-
-// SetStdoutLevel changes the logging level to the passed level.
-func (b *Backend) SetStdoutLevel(level Level) {
-	atomic.StoreUint32((*uint32)(&b.stdoutLevel), uint32(level))
-}
-
-// Close finalizes all log rotators for this backend
-func (b *Backend) Close() {
-	for _, r := range b.rotators {
-		r.Close()
-	}
-}
-
-// Logger returns a new logger for a particular subsystem that writes to the
-// Backend b. A tag describes the subsystem and is included in all log
-// messages. The logger uses the info verbosity level by default.
-func (b *Backend) Logger(subsystemTag string) *Logger {
-	return &Logger{LevelInfo, subsystemTag, b}
-}
-
 // Logger is a subsystem logger for a Backend.
 type Logger struct {
-	lvl Level // atomic
-	tag string
-	b   *Backend
+	lvl       Level // atomic
+	tag       string
+	b         *Backend
+	writeChan chan<- logEntry
+}
+
+type logEntry struct {
+	log   []byte
+	level Level
 }
 
 // Trace formats message using the default formats for its operands, prepends
@@ -471,7 +131,7 @@ func (l *Logger) Criticalf(format string, args ...interface{}) {
 func (l *Logger) Write(logLevel Level, args ...interface{}) {
 	lvl := l.Level()
 	if lvl <= logLevel {
-		l.b.print(logLevel, l.tag, args...)
+		l.print(logLevel, l.tag, args...)
 	}
 }
 
@@ -480,7 +140,7 @@ func (l *Logger) Write(logLevel Level, args ...interface{}) {
 func (l *Logger) Writef(logLevel Level, format string, args ...interface{}) {
 	lvl := l.Level()
 	if lvl <= logLevel {
-		l.b.printf(logLevel, l.tag, format, args...)
+		l.printf(logLevel, l.tag, format, args...)
 	}
 }
 
@@ -498,3 +158,135 @@ func (l *Logger) SetLevel(level Level) {
 func (l *Logger) Backend() *Backend {
 	return l.b
 }
+
+// printf outputs a log message to the writer associated with the backend after
+// creating a prefix for the given level and tag according to the formatHeader
+// function and formatting the provided arguments according to the given format
+// specifier.
+func (l *Logger) printf(lvl Level, tag string, format string, args ...interface{}) {
+	t := mstime.Now() // get as early as possible
+
+	var file string
+	var line int
+	if l.b.flag&(LogFlagShortFile|LogFlagLongFile) != 0 {
+		file, line = callsite(l.b.flag)
+	}
+
+	buf := make([]byte, 0, normalLogSize)
+
+	formatHeader(&buf, t, lvl.String(), tag, file, line)
+	bytesBuf := bytes.NewBuffer(buf)
+	_, _ = fmt.Fprintf(bytesBuf, format, args...)
+	bytesBuf.WriteByte('\n')
+
+	if !l.b.IsRunning() {
+		_, _ = fmt.Fprintf(os.Stderr, bytesBuf.String())
+		panic("Writing to the logger when it's not running")
+	}
+	l.writeChan <- logEntry{bytesBuf.Bytes(), lvl}
+}
+
+// print outputs a log message to the writer associated with the backend after
+// creating a prefix for the given level and tag according to the formatHeader
+// function and formatting the provided arguments using the default formatting
+// rules.
+func (l *Logger) print(lvl Level, tag string, args ...interface{}) {
+	if atomic.LoadUint32(&l.b.isRunning) == 0 {
+		panic("printing log without initializing")
+	}
+	t := mstime.Now() // get as early as possible
+
+	var file string
+	var line int
+	if l.b.flag&(LogFlagShortFile|LogFlagLongFile) != 0 {
+		file, line = callsite(l.b.flag)
+	}
+
+	buf := make([]byte, 0, normalLogSize)
+	formatHeader(&buf, t, lvl.String(), tag, file, line)
+	bytesBuf := bytes.NewBuffer(buf)
+	_, _ = fmt.Fprintln(bytesBuf, args...)
+
+	if !l.b.IsRunning() {
+		panic("Writing to the logger when it's not running")
+	}
+	l.writeChan <- logEntry{bytesBuf.Bytes(), lvl}
+}
+
+// From stdlib log package.
+// Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid zero-padding.
+func itoa(buf *[]byte, i int, wid int) {
+	// Assemble decimal in reverse order.
+	var b [20]byte
+	bp := len(b) - 1
+	for i >= 10 || wid > 1 {
+		wid--
+		q := i / 10
+		b[bp] = byte('0' + i - q*10)
+		bp--
+		i = q
+	}
+	// i < 10
+	b[bp] = byte('0' + i)
+	*buf = append(*buf, b[bp:]...)
+}
+
+// Appends a header in the default format 'YYYY-MM-DD hh:mm:ss.sss [LVL] TAG: '.
+// If either of the LogFlagShortFile or LogFlagLongFile flags are specified, the file named
+// and line number are included after the tag and before the final colon.
+func formatHeader(buf *[]byte, t mstime.Time, lvl, tag string, file string, line int) {
+	year, month, day := t.Date()
+	hour, min, sec := t.Clock()
+	ms := t.Millisecond()
+
+	itoa(buf, year, 4)
+	*buf = append(*buf, '-')
+	itoa(buf, int(month), 2)
+	*buf = append(*buf, '-')
+	itoa(buf, day, 2)
+	*buf = append(*buf, ' ')
+	itoa(buf, hour, 2)
+	*buf = append(*buf, ':')
+	itoa(buf, min, 2)
+	*buf = append(*buf, ':')
+	itoa(buf, sec, 2)
+	*buf = append(*buf, '.')
+	itoa(buf, ms, 3)
+	*buf = append(*buf, " ["...)
+	*buf = append(*buf, lvl...)
+	*buf = append(*buf, "] "...)
+	*buf = append(*buf, tag...)
+	if file != "" {
+		*buf = append(*buf, ' ')
+		*buf = append(*buf, file...)
+		*buf = append(*buf, ':')
+		itoa(buf, line, -1)
+	}
+	*buf = append(*buf, ": "...)
+}
+
+// calldepth is the call depth of the callsite function relative to the
+// caller of the subsystem logger. It is used to recover the filename and line
+// number of the logging call if either the short or long file flags are
+// specified.
+const calldepth = 4
+
+// callsite returns the file name and line number of the callsite to the
+// subsystem logger.
+func callsite(flag uint32) (string, int) {
+	_, file, line, ok := runtime.Caller(calldepth)
+	if !ok {
+		return "???", 0
+	}
+	if flag&LogFlagShortFile != 0 {
+		short := file
+		for i := len(file) - 1; i > 0; i-- {
+			if os.IsPathSeparator(file[i]) {
+				short = file[i+1:]
+				break
+			}
+		}
+		file = short
+	}
+	return file, line
+}

infrastructure/logger/utils.go

@@ -27,3 +27,18 @@ func LogMemoryStats(log *Logger, functionName string) {
 			stats.Alloc, stats.HeapIdle-stats.HeapReleased+stats.HeapInuse)
 	}))
 }
+
+// LogClosure is a closure that can be printed with %s to be used to
+// generate expensive-to-create data for a detailed log level and avoid doing
+// the work if the data isn't printed.
+type LogClosure func() string
+
+func (c LogClosure) String() string {
+	return c()
+}
+
+// NewLogClosure casts a function to a LogClosure.
+// See LogClosure for details.
+func NewLogClosure(c func() string) LogClosure {
+	return c
+}

infrastructure/network/addressmanager/log.go

@@ -6,8 +6,6 @@ package addressmanager
 
 import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.ADXR)
-var spawn = panics.GoroutineWrapperFunc(log)
+var log = logger.RegisterSubSystem("ADXR")

infrastructure/network/addressmanager/store.go

@@ -43,6 +43,7 @@ func (as *addressStore) restoreNotBannedAddresses() error {
 	if err != nil {
 		return err
 	}
+	defer cursor.Close()
 	for ok := cursor.First(); ok; ok = cursor.Next() {
 		databaseKey, err := cursor.Key()
 		if err != nil {
@@ -66,6 +67,7 @@ func (as *addressStore) restoreBannedAddresses() error {
 	if err != nil {
 		return err
 	}
+	defer cursor.Close()
 	for ok := cursor.First(); ok; ok = cursor.Next() {
 		databaseKey, err := cursor.Key()
 		if err != nil {

infrastructure/network/connmanager/log.go

@@ -5,5 +5,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.CMGR)
+var log = logger.RegisterSubSystem("CMGR")
 var spawn = panics.GoroutineWrapperFunc(log)

infrastructure/network/dnsseed/log.go

@@ -9,6 +9,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.CMGR)
+var log = logger.RegisterSubSystem("CMGR")
 var spawn = panics.GoroutineWrapperFunc(log)
-var spawnAfter = panics.AfterFuncWrapperFunc(log)

infrastructure/network/netadapter/log.go

@@ -2,8 +2,6 @@ package netadapter
 
 import (
 	"github.com/kaspanet/kaspad/infrastructure/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.NTAR)
-var spawn = panics.GoroutineWrapperFunc(log)
+var log = logger.RegisterSubSystem("NTAR")

infrastructure/network/netadapter/server/grpcserver/grpc_connection.go

@@ -1,13 +1,16 @@
 package grpcserver
 
 import (
-	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/router"
-	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/server/grpcserver/protowire"
-	"github.com/pkg/errors"
 	"net"
 	"sync"
 	"sync/atomic"
 
+	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/router"
+	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/server/grpcserver/protowire"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
 	"github.com/kaspanet/kaspad/infrastructure/network/netadapter/server"
 	"google.golang.org/grpc"
 )
@@ -62,7 +65,17 @@ func (c *gRPCConnection) Start(router *router.Router) {
 	spawn("gRPCConnection.Start-connectionLoops", func() {
 		err := c.connectionLoops()
 		if err != nil {
-			log.Errorf("error from connectionLoops for %s: %s", c.address, err)
+			status, isStatus := status.FromError(err)
+			if isStatus {
+				switch status.Code() {
+				case codes.Canceled:
+					log.Debugf("connectionLoop canceled connection for %s: %s", c.address, err)
+				default:
+					log.Errorf("status error from connectionLoops for %s: %s", c.address, err)
+				}
+			} else {
+				log.Errorf("unknown error from connectionLoops for %s: %s", c.address, err)
+			}
 		}
 	})
 }

infrastructure/network/netadapter/server/grpcserver/log.go

@@ -9,5 +9,5 @@ import (
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.TXMP)
+var log = logger.RegisterSubSystem("TXMP")
 var spawn = panics.GoroutineWrapperFunc(log)

infrastructure/network/netadapter/server/grpcserver/protowire/common.go

@@ -12,7 +12,12 @@ import (
 	"github.com/pkg/errors"
 )
 
+var errorNil = errors.New("a required field is nil")
+
 func (x *Hash) toDomain() (*externalapi.DomainHash, error) {
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "Hash is nil")
+	}
 	return externalapi.NewDomainHashFromByteSlice(x.Bytes)
 }
 
@@ -43,6 +48,9 @@ func domainHashesToProto(hashes []*externalapi.DomainHash) []*Hash {
 }
 
 func (x *TransactionId) toDomain() (*externalapi.DomainTransactionID, error) {
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "TransactionId is nil")
+	}
 	return transactionid.FromBytes(x.Bytes)
 }
 
@@ -74,7 +82,7 @@ func wireTransactionIDsToProto(ids []*externalapi.DomainTransactionID) []*Transa
 
 func (x *SubnetworkId) toDomain() (*externalapi.DomainSubnetworkID, error) {
 	if x == nil {
-		return nil, nil
+		return nil, errors.Wrap(errorNil, "SubnetworkId is nil")
 	}
 	return subnetworks.FromBytes(x.Bytes)
 }
@@ -89,6 +97,9 @@ func domainSubnetworkIDToProto(id *externalapi.DomainSubnetworkID) *SubnetworkId
 }
 
 func (x *NetAddress) toAppMessage() (*appmessage.NetAddress, error) {
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "NetAddress is nil")
+	}
 	if x.Port > math.MaxUint16 {
 		return nil, errors.Errorf("port number is larger than %d", math.MaxUint16)
 	}
@@ -108,3 +119,46 @@ func appMessageNetAddressToProto(address *appmessage.NetAddress) *NetAddress {
 		Port:      uint32(address.Port),
 	}
 }
+
+func (x *Outpoint) toAppMessage() (*appmessage.Outpoint, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "Outpoint is nil")
+	}
+	transactionID, err := x.TransactionId.toDomain()
+	if err != nil {
+		return nil, err
+	}
+	return &appmessage.Outpoint{
+		TxID:  *transactionID,
+		Index: x.Index,
+	}, nil
+}
+
+func (x *UtxoEntry) toAppMessage() (*appmessage.UTXOEntry, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "UtxoEntry is nil")
+	}
+	scriptPublicKey, err := x.ScriptPublicKey.toAppMessage()
+	if err != nil {
+		return nil, err
+	}
+	return &appmessage.UTXOEntry{
+		Amount:          x.Amount,
+		ScriptPublicKey: scriptPublicKey,
+		BlockBlueScore:  x.BlockBlueScore,
+		IsCoinbase:      x.IsCoinbase,
+	}, nil
+}
+
+func (x *ScriptPublicKey) toAppMessage() (*externalapi.ScriptPublicKey, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "ScriptPublicKey is nil")
+	}
+	if x.Version > math.MaxUint16 {
+		return nil, errors.Errorf("ScriptPublicKey version is bigger then uint16.")
+	}
+	return &externalapi.ScriptPublicKey{
+		Script:  x.Script,
+		Version: uint16(x.Version),
+	}, nil
+}

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_addresses.go

@@ -6,23 +6,36 @@ import (
 )
 
 func (x *KaspadMessage_Addresses) toAppMessage() (appmessage.Message, error) {
-	protoAddresses := x.Addresses
-	if len(x.Addresses.AddressList) > appmessage.MaxAddressesPerMsg {
-		return nil, errors.Errorf("too many addresses for message "+
-			"[count %d, max %d]", len(x.Addresses.AddressList), appmessage.MaxAddressesPerMsg)
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "KaspadMessage_Addresses is nil")
+	}
+	addressList, err := x.Addresses.toAppMessage()
+	if err != nil {
+		return nil, err
+	}
+	return &appmessage.MsgAddresses{
+		AddressList: addressList,
+	}, nil
+}
+
+func (x *AddressesMessage) toAppMessage() ([]*appmessage.NetAddress, error) {
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "AddressesMessage is nil")
 	}
 
-	addressList := make([]*appmessage.NetAddress, len(protoAddresses.AddressList))
-	for i, address := range protoAddresses.AddressList {
+	if len(x.AddressList) > appmessage.MaxAddressesPerMsg {
+		return nil, errors.Errorf("too many addresses for message "+
+			"[count %d, max %d]", len(x.AddressList), appmessage.MaxAddressesPerMsg)
+	}
+	addressList := make([]*appmessage.NetAddress, len(x.AddressList))
+	for i, address := range x.AddressList {
 		var err error
 		addressList[i], err = address.toAppMessage()
 		if err != nil {
 			return nil, err
 		}
 	}
-	return &appmessage.MsgAddresses{
-		AddressList: addressList,
-	}, nil
+	return addressList, nil
 }
 
 func (x *KaspadMessage_Addresses) fromAppMessage(msgAddresses *appmessage.MsgAddresses) error {

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_block.go

@@ -6,6 +6,9 @@ import (
 )
 
 func (x *KaspadMessage_Block) toAppMessage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "KaspadMessage_Block is nil")
+	}
 	return x.Block.toAppMessage()
 }
 
@@ -15,21 +18,14 @@ func (x *KaspadMessage_Block) fromAppMessage(msgBlock *appmessage.MsgBlock) erro
 }
 
 func (x *BlockMessage) toAppMessage() (*appmessage.MsgBlock, error) {
+	if x == nil {
+		return nil, errors.Wrap(errorNil, "BlockMessage is nil")
+	}
 	if len(x.Transactions) > appmessage.MaxTxPerBlock {
 		return nil, errors.Errorf("too many transactions to fit into a block "+
 			"[count %d, max %d]", len(x.Transactions), appmessage.MaxTxPerBlock)
 	}
 
-	protoBlockHeader := x.Header
-	if protoBlockHeader == nil {
-		return nil, errors.New("block header field cannot be nil")
-	}
-
-	if len(protoBlockHeader.ParentHashes) > appmessage.MaxBlockParents {
-		return nil, errors.Errorf("block header has %d parents, but the maximum allowed amount "+
-			"is %d", len(protoBlockHeader.ParentHashes), appmessage.MaxBlockParents)
-	}
-
 	header, err := x.Header.toAppMessage()
 	if err != nil {
 		return nil, err

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_block_headers.go

@@ -1,10 +1,29 @@
 package protowire
 
-import "github.com/kaspanet/kaspad/app/appmessage"
+import (
+	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/pkg/errors"
+)
 
 func (x *KaspadMessage_BlockHeaders) toAppMessage() (appmessage.Message, error) {
-	blockHeaders := make([]*appmessage.MsgBlockHeader, len(x.BlockHeaders.BlockHeaders))
-	for i, blockHeader := range x.BlockHeaders.BlockHeaders {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_BlockHeaders is nil")
+	}
+	blockHeaders, err := x.BlockHeaders.toAppMessage()
+	if err != nil {
+		return nil, err
+	}
+	return &appmessage.BlockHeadersMessage{
+		BlockHeaders: blockHeaders,
+	}, nil
+}
+
+func (x *BlockHeadersMessage) toAppMessage() ([]*appmessage.MsgBlockHeader, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "BlockHeadersMessage is nil")
+	}
+	blockHeaders := make([]*appmessage.MsgBlockHeader, len(x.BlockHeaders))
+	for i, blockHeader := range x.BlockHeaders {
 		var err error
 		blockHeaders[i], err = blockHeader.toAppMessage()
 		if err != nil {
@@ -12,9 +31,7 @@ func (x *KaspadMessage_BlockHeaders) toAppMessage() (appmessage.Message, error)
 		}
 	}
 
-	return &appmessage.BlockHeadersMessage{
-		BlockHeaders: blockHeaders,
-	}, nil
+	return blockHeaders, nil
 }
 
 func (x *KaspadMessage_BlockHeaders) fromAppMessage(blockHeadersMessage *appmessage.BlockHeadersMessage) error {

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_block_locator.go

@@ -2,21 +2,32 @@ package protowire
 
 import (
 	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/pkg/errors"
 )
 
 func (x *KaspadMessage_BlockLocator) toAppMessage() (appmessage.Message, error) {
-	if len(x.BlockLocator.Hashes) > appmessage.MaxBlockLocatorsPerMsg {
-		return nil, errors.Errorf("too many block locator hashes for message "+
-			"[count %d, max %d]", len(x.BlockLocator.Hashes), appmessage.MaxBlockLocatorsPerMsg)
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_BlockLocator is nil")
 	}
-	hashes, err := protoHashesToDomain(x.BlockLocator.Hashes)
+	hashes, err := x.BlockLocator.toAppMessage()
 	if err != nil {
 		return nil, err
 	}
 	return &appmessage.MsgBlockLocator{BlockLocatorHashes: hashes}, nil
 }
 
+func (x *BlockLocatorMessage) toAppMessage() ([]*externalapi.DomainHash, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "BlockLocatorMessage is nil")
+	}
+	if len(x.Hashes) > appmessage.MaxBlockLocatorsPerMsg {
+		return nil, errors.Errorf("too many block locator hashes for message "+
+			"[count %d, max %d]", len(x.Hashes), appmessage.MaxBlockLocatorsPerMsg)
+	}
+	return protoHashesToDomain(x.Hashes)
+}
+
 func (x *KaspadMessage_BlockLocator) fromAppMessage(msgBlockLocator *appmessage.MsgBlockLocator) error {
 	if len(msgBlockLocator.BlockLocatorHashes) > appmessage.MaxBlockLocatorsPerMsg {
 		return errors.Errorf("too many block locator hashes for message "+

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_done_ibd_blocks.go

@@ -1,8 +1,14 @@
 package protowire
 
-import "github.com/kaspanet/kaspad/app/appmessage"
+import (
+	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/pkg/errors"
+)
 
 func (x *KaspadMessage_DoneHeaders) toAppMessage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_DoneHeaders is nil")
+	}
 	return &appmessage.MsgDoneHeaders{}, nil
 }
 

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_done_pruning_point_utxo_set_chunks.go

@@ -1,8 +1,14 @@
 package protowire
 
-import "github.com/kaspanet/kaspad/app/appmessage"
+import (
+	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/pkg/errors"
+)
 
 func (x *KaspadMessage_DonePruningPointUtxoSetChunks) toAppMessage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_DonePruningPointUtxoSetChunks is nil")
+	}
 	return &appmessage.MsgDonePruningPointUTXOSetChunks{}, nil
 }
 

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_header.go

@@ -8,6 +8,9 @@ import (
 )
 
 func (x *BlockHeaderMessage) toAppMessage() (*appmessage.MsgBlockHeader, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "BlockHeaderMessage is nil")
+	}
 	if len(x.ParentHashes) > appmessage.MaxBlockParents {
 		return nil, errors.Errorf("block header has %d parents, but the maximum allowed amount "+
 			"is %d", len(x.ParentHashes), appmessage.MaxBlockParents)
@@ -17,17 +20,14 @@ func (x *BlockHeaderMessage) toAppMessage() (*appmessage.MsgBlockHeader, error)
 	if err != nil {
 		return nil, err
 	}
-
 	hashMerkleRoot, err := x.HashMerkleRoot.toDomain()
 	if err != nil {
 		return nil, err
 	}
-
 	acceptedIDMerkleRoot, err := x.AcceptedIdMerkleRoot.toDomain()
 	if err != nil {
 		return nil, err
 	}
-
 	utxoCommitment, err := x.UtxoCommitment.toDomain()
 	if err != nil {
 		return nil, err

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_ibd_block.go

@@ -1,8 +1,14 @@
 package protowire
 
-import "github.com/kaspanet/kaspad/app/appmessage"
+import (
+	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/pkg/errors"
+)
 
 func (x *KaspadMessage_IbdBlock) toAppMessage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_IbdBlock is nil")
+	}
 	msgBlock, err := x.IbdBlock.toAppMessage()
 	if err != nil {
 		return nil, err

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_ibd_block_locator.go

@@ -2,14 +2,25 @@ package protowire
 
 import (
 	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/pkg/errors"
 )
 
 func (x *KaspadMessage_IbdBlockLocator) toAppMessage() (appmessage.Message, error) {
-	targetHash, err := x.IbdBlockLocator.TargetHash.toDomain()
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_IbdBlockLocator is nil")
+	}
+	return x.IbdBlockLocator.toAppMessage()
+}
+
+func (x *IbdBlockLocatorMessage) toAppMessage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "IbdBlockLocatorMessage is nil")
+	}
+	targetHash, err := x.TargetHash.toDomain()
 	if err != nil {
 		return nil, err
 	}
-	blockLocatorHash, err := protoHashesToDomain(x.IbdBlockLocator.BlockLocatorHashes)
+	blockLocatorHash, err := protoHashesToDomain(x.BlockLocatorHashes)
 	if err != nil {
 		return nil, err
 	}

infrastructure/network/netadapter/server/grpcserver/protowire/p2p_ibd_block_locator_highest_hash.go

@@ -1,9 +1,22 @@
 package protowire
 
-import "github.com/kaspanet/kaspad/app/appmessage"
+import (
+	"github.com/kaspanet/kaspad/app/appmessage"
+	"github.com/pkg/errors"
+)
 
 func (x *KaspadMessage_IbdBlockLocatorHighestHash) toAppMessage() (appmessage.Message, error) {
-	highestHash, err := x.IbdBlockLocatorHighestHash.HighestHash.toDomain()
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "KaspadMessage_IbdBlockLocatorHighestHash is nil")
+	}
+	return x.IbdBlockLocatorHighestHash.toAppMessgage()
+}
+
+func (x *IbdBlockLocatorHighestHashMessage) toAppMessgage() (appmessage.Message, error) {
+	if x == nil {
+		return nil, errors.Wrapf(errorNil, "IbdBlockLocatorHighestHashMessage is nil")
+	}
+	highestHash, err := x.HighestHash.toDomain()
 	if err != nil {
 		return nil, err
 	}
@@ -11,6 +24,7 @@ func (x *KaspadMessage_IbdBlockLocatorHighestHash) toAppMessage() (appmessage.Me
 	return &appmessage.MsgIBDBlockLocatorHighestHash{
 		HighestHash: highestHash,
 	}, nil
+
 }
 
 func (x *KaspadMessage_IbdBlockLocatorHighestHash) fromAppMessage(message *appmessage.MsgIBDBlockLocatorHighestHash) error {