Merge branch 'utxo-child-bug' into bump_v0.12.4-rc1

changelog.txt

@@ -1,9 +1,3 @@
-Kaspad v0.12.4 - 2022-07-17
-===========================
-
-* Crucial fix for the UTXO difference mechanism (#2114)
-* Implement multi-layer auto-compound (#2115)
-
 Kaspad v0.12.3 - 2022-06-29
 ===========================
 

cmd/kaspawallet/daemon/server/split_transaction.go

@@ -27,10 +27,18 @@ func (s *server) maybeAutoCompoundTransaction(transactionBytes []byte, toAddress
 		return nil, err
 	}
 
-	splitTransactions, err := s.maybeSplitAndMergeTransaction(transaction, toAddress, changeAddress, changeWalletAddress)
+	splitTransactions, err := s.maybeSplitTransaction(transaction, changeAddress)
 	if err != nil {
 		return nil, err
 	}
+	if len(splitTransactions) > 1 {
+		mergeTransaction, err := s.mergeTransaction(splitTransactions, transaction, toAddress, changeAddress, changeWalletAddress)
+		if err != nil {
+			return nil, err
+		}
+		splitTransactions = append(splitTransactions, mergeTransaction)
+	}
+
 	splitTransactionsBytes := make([][]byte, len(splitTransactions))
 	for i, splitTransaction := range splitTransactions {
 		splitTransactionsBytes[i], err = serialization.SerializePartiallySignedTransaction(splitTransaction)
@@ -105,8 +113,8 @@ func (s *server) mergeTransaction(
 	return serialization.DeserializePartiallySignedTransaction(mergeTransactionBytes)
 }
 
-func (s *server) maybeSplitAndMergeTransaction(transaction *serialization.PartiallySignedTransaction, toAddress util.Address,
-	changeAddress util.Address, changeWalletAddress *walletAddress) ([]*serialization.PartiallySignedTransaction, error) {
+func (s *server) maybeSplitTransaction(transaction *serialization.PartiallySignedTransaction,
+	changeAddress util.Address) ([]*serialization.PartiallySignedTransaction, error) {
 
 	transactionMass, err := s.estimateMassAfterSignatures(transaction)
 	if err != nil {
@@ -133,20 +141,6 @@ func (s *server) maybeSplitAndMergeTransaction(transaction *serialization.Partia
 		}
 	}
 
-	if len(splitTransactions) > 1 {
-		mergeTransaction, err := s.mergeTransaction(splitTransactions, transaction, toAddress, changeAddress, changeWalletAddress)
-		if err != nil {
-			return nil, err
-		}
-		// Recursion will be 2-3 iterations deep even in the rarest` cases, so considered safe..
-		splitMergeTransaction, err := s.maybeSplitAndMergeTransaction(mergeTransaction, toAddress, changeAddress, changeWalletAddress)
-		if err != nil {
-			return nil, err
-		}
-		splitTransactions = append(splitTransactions, splitMergeTransaction...)
-
-	}
-
 	return splitTransactions, nil
 }
 

domain/consensus/consensus.go

@@ -913,10 +913,10 @@ func (s *consensus) ResolveVirtual(progressReportCallback func(uint64, uint64))
 		}
 
 		// In order to prevent a situation that the consensus lock is held for too much time, we
-		// release the lock each time we resolve 100 blocks.
-		// Note: maxBlocksToResolve should be smaller than `params.FinalityDuration` in order to avoid a situation
+		// release the lock each time resolve 100 blocks.
+		// Note: maxBlocksToResolve should be smaller than finality interval in order to avoid a situation
 		// where UpdatePruningPointByVirtual skips a pruning point.
-		_, isCompletelyResolved, err := s.resolveVirtualChunkWithLock(100)
+		isCompletelyResolved, err := s.resolveVirtualChunkWithLock(100)
 		if err != nil {
 			return err
 		}
@@ -928,37 +928,37 @@ func (s *consensus) ResolveVirtual(progressReportCallback func(uint64, uint64))
 	return nil
 }
 
-func (s *consensus) resolveVirtualChunkWithLock(maxBlocksToResolve uint64) (*externalapi.VirtualChangeSet, bool, error) {
+func (s *consensus) resolveVirtualChunkWithLock(maxBlocksToResolve uint64) (bool, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	return s.resolveVirtualChunkNoLock(maxBlocksToResolve)
+	return s.resolveVirtualNoLock(maxBlocksToResolve)
 }
 
-func (s *consensus) resolveVirtualChunkNoLock(maxBlocksToResolve uint64) (*externalapi.VirtualChangeSet, bool, error) {
+func (s *consensus) resolveVirtualNoLock(maxBlocksToResolve uint64) (bool, error) {
 	virtualChangeSet, isCompletelyResolved, err := s.consensusStateManager.ResolveVirtual(maxBlocksToResolve)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 	s.virtualNotUpdated = !isCompletelyResolved
 
 	stagingArea := model.NewStagingArea()
 	err = s.pruningManager.UpdatePruningPointByVirtual(stagingArea)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
 	err = staging.CommitAllChanges(s.databaseContext, stagingArea)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
 	err = s.sendVirtualChangedEvent(virtualChangeSet, true)
 	if err != nil {
-		return nil, false, err
+		return false, err
 	}
 
-	return virtualChangeSet, isCompletelyResolved, nil
+	return isCompletelyResolved, nil
 }
 
 func (s *consensus) BuildPruningPointProof() (*externalapi.PruningPointProof, error) {

domain/consensus/model/testapi/test_consensus.go

@@ -49,7 +49,7 @@ type TestConsensus interface {
 		*externalapi.VirtualChangeSet, error)
 	UpdatePruningPointByVirtual() error
 
-	ResolveVirtualWithMaxParam(maxBlocksToResolve uint64) (*externalapi.VirtualChangeSet, bool, error)
+	ResolveVirtualWithMaxParam(maxBlocksToResolve uint64) (bool, error)
 
 	MineJSON(r io.Reader, blockType MineJSONBlockType) (tips []*externalapi.DomainHash, err error)
 	ToJSON(w io.Writer) error

domain/consensus/processes/consensusstatemanager/resolve.go

@@ -9,13 +9,10 @@ import (
 	"sort"
 )
 
-// tipsInDecreasingGHOSTDAGParentSelectionOrder returns the current DAG tips in decreasing parent selection order.
-// This means that the first tip in the resulting list would be the GHOSTDAG selected parent, and if removed from the list,
-// the second tip would be the selected parent, and so on.
-func (csm *consensusStateManager) tipsInDecreasingGHOSTDAGParentSelectionOrder(stagingArea *model.StagingArea) ([]*externalapi.DomainHash, error) {
+func (csm *consensusStateManager) findNextPendingTip(stagingArea *model.StagingArea) (*externalapi.DomainHash, externalapi.BlockStatus, error) {
 	tips, err := csm.consensusStateStore.Tips(stagingArea, csm.databaseContext)
 	if err != nil {
-		return nil, err
+		return nil, externalapi.StatusInvalid, err
 	}
 
 	var sortErr error
@@ -29,18 +26,10 @@ func (csm *consensusStateManager) tipsInDecreasingGHOSTDAGParentSelectionOrder(s
 		return selectedParent.Equal(tips[i])
 	})
 	if sortErr != nil {
-		return nil, sortErr
-	}
-	return tips, nil
-}
-
-func (csm *consensusStateManager) findNextPendingTip(stagingArea *model.StagingArea) (*externalapi.DomainHash, externalapi.BlockStatus, error) {
-	orderedTips, err := csm.tipsInDecreasingGHOSTDAGParentSelectionOrder(stagingArea)
-	if err != nil {
-		return nil, externalapi.StatusInvalid, err
+		return nil, externalapi.StatusInvalid, sortErr
 	}
 
-	for _, tip := range orderedTips {
+	for _, tip := range tips {
 		log.Debugf("Resolving tip %s", tip)
 		isViolatingFinality, shouldNotify, err := csm.isViolatingFinality(stagingArea, tip)
 		if err != nil {
@@ -67,9 +56,7 @@ func (csm *consensusStateManager) findNextPendingTip(stagingArea *model.StagingA
 	return nil, externalapi.StatusInvalid, nil
 }
 
-// getGHOSTDAGLowerTips returns the set of tips which are lower in GHOSTDAG parent selection order than `pendingTip`. i.e.,
-// they can be added to virtual parents but `pendingTip` will remain the virtual selected parent
-func (csm *consensusStateManager) getGHOSTDAGLowerTips(stagingArea *model.StagingArea, pendingTip *externalapi.DomainHash) ([]*externalapi.DomainHash, error) {
+func (csm *consensusStateManager) getLowerTips(stagingArea *model.StagingArea, pendingTip *externalapi.DomainHash) ([]*externalapi.DomainHash, error) {
 	tips, err := csm.consensusStateStore.Tips(stagingArea, csm.databaseContext)
 	if err != nil {
 		return nil, err
@@ -95,8 +82,6 @@ func (csm *consensusStateManager) ResolveVirtual(maxBlocksToResolve uint64) (*ex
 	onEnd := logger.LogAndMeasureExecutionTime(log, "csm.ResolveVirtual")
 	defer onEnd()
 
-	// We use a read-only staging area for some read-only actions, to avoid
-	// confusion with the resolve/updateVirtual staging areas below
 	readStagingArea := model.NewStagingArea()
 
 	pendingTip, pendingTipStatus, err := csm.findNextPendingTip(readStagingArea)
@@ -105,16 +90,16 @@ func (csm *consensusStateManager) ResolveVirtual(maxBlocksToResolve uint64) (*ex
 	}
 
 	if pendingTip == nil {
-		log.Warnf("None of the DAG tips are valid")
+		log.Warnf("Non of the DAG tips are valid")
 		return nil, true, nil
 	}
 
-	previousVirtualSelectedParent, err := csm.virtualSelectedParent(readStagingArea)
+	prevVirtualSelectedParent, err := csm.virtualSelectedParent(readStagingArea)
 	if err != nil {
 		return nil, false, err
 	}
 
-	if pendingTipStatus == externalapi.StatusUTXOValid && previousVirtualSelectedParent.Equal(pendingTip) {
+	if pendingTipStatus == externalapi.StatusUTXOValid && prevVirtualSelectedParent.Equal(pendingTip) {
 		return nil, true, nil
 	}
 
@@ -125,77 +110,77 @@ func (csm *consensusStateManager) ResolveVirtual(maxBlocksToResolve uint64) (*ex
 		return nil, false, err
 	}
 
-	// Initially set the resolve processing point to the pending tip
-	processingPoint := pendingTip
+	intermediateTip := pendingTip
 
 	// Too many blocks to verify, so we only process a chunk and return
 	if maxBlocksToResolve != 0 && uint64(len(unverifiedBlocks)) > maxBlocksToResolve {
-		processingPointIndex := uint64(len(unverifiedBlocks)) - maxBlocksToResolve
-		processingPoint = unverifiedBlocks[processingPointIndex]
-		isNewVirtualSelectedParent, err := csm.isNewSelectedTip(readStagingArea, processingPoint, previousVirtualSelectedParent)
+
+		intermediateTipIndex := uint64(len(unverifiedBlocks)) - maxBlocksToResolve
+		intermediateTip = unverifiedBlocks[intermediateTipIndex]
+		isNewVirtualSelectedParent, err := csm.isNewSelectedTip(readStagingArea, intermediateTip, prevVirtualSelectedParent)
 		if err != nil {
 			return nil, false, err
 		}
 
-		// We must find a processing point which wins previous virtual selected parent
+		// We must find an intermediate tip which wins previous virtual selected parent
 		// even if we process more than `maxBlocksToResolve` for that.
 		// Otherwise, internal UTXO diff logic gets all messed up
 		for !isNewVirtualSelectedParent {
-			if processingPointIndex == 0 {
+			if intermediateTipIndex == 0 {
 				return nil, false, errors.Errorf(
-					"Expecting the pending tip %s to overcome the previous selected parent %s", pendingTip, previousVirtualSelectedParent)
+					"Expecting the pending tip %s to overcome the previous selected parent %s", pendingTip, prevVirtualSelectedParent)
 			}
-			processingPointIndex--
-			processingPoint = unverifiedBlocks[processingPointIndex]
-			isNewVirtualSelectedParent, err = csm.isNewSelectedTip(readStagingArea, processingPoint, previousVirtualSelectedParent)
+			intermediateTipIndex--
+			intermediateTip = unverifiedBlocks[intermediateTipIndex]
+			isNewVirtualSelectedParent, err = csm.isNewSelectedTip(readStagingArea, intermediateTip, prevVirtualSelectedParent)
 			if err != nil {
 				return nil, false, err
 			}
 		}
-		log.Debugf("Has more than %d blocks to resolve. Setting the resolve processing point to %s", maxBlocksToResolve, processingPoint)
+		log.Debugf("Has more than %d blocks to resolve. Changing the resolve tip to %s", maxBlocksToResolve, intermediateTip)
 	}
 
-	processingPointStatus, reversalData, err := csm.resolveBlockStatus(
-		resolveStagingArea, processingPoint, true)
+	intermediateTipStatus, reversalData, err := csm.resolveBlockStatus(
+		resolveStagingArea, intermediateTip, true)
 	if err != nil {
 		return nil, false, err
 	}
 
-	if processingPointStatus == externalapi.StatusUTXOValid {
+	if intermediateTipStatus == externalapi.StatusUTXOValid {
 		err = staging.CommitAllChanges(csm.databaseContext, resolveStagingArea)
 		if err != nil {
 			return nil, false, err
 		}
 
 		if reversalData != nil {
-			err = csm.ReverseUTXODiffs(processingPoint, reversalData)
+			err = csm.ReverseUTXODiffs(intermediateTip, reversalData)
 			if err != nil {
 				return nil, false, err
 			}
 		}
 	}
 
-	isActualTip := processingPoint.Equal(pendingTip)
-	isCompletelyResolved := isActualTip && processingPointStatus == externalapi.StatusUTXOValid
+	isActualTip := intermediateTip.Equal(pendingTip)
+	isCompletelyResolved := isActualTip && intermediateTipStatus == externalapi.StatusUTXOValid
 
 	updateVirtualStagingArea := model.NewStagingArea()
 
-	virtualParents := []*externalapi.DomainHash{processingPoint}
 	// If `isCompletelyResolved`, set virtual correctly with all tips which have less blue work than pending
+	virtualTipCandidates := []*externalapi.DomainHash{intermediateTip}
 	if isCompletelyResolved {
-		lowerTips, err := csm.getGHOSTDAGLowerTips(readStagingArea, pendingTip)
+		lowerTips, err := csm.getLowerTips(readStagingArea, pendingTip)
 		if err != nil {
 			return nil, false, err
 		}
-		log.Debugf("Picking virtual parents from relevant tips len: %d", len(lowerTips))
 
-		virtualParents, err = csm.pickVirtualParents(readStagingArea, lowerTips)
+		log.Debugf("Picking virtual parents from relevant tips len: %d", len(lowerTips))
+		virtualTipCandidates, err = csm.pickVirtualParents(readStagingArea, lowerTips)
 		if err != nil {
 			return nil, false, err
 		}
-		log.Debugf("Picked virtual parents: %s", virtualParents)
+		log.Debugf("Picked virtual parents: %s", virtualTipCandidates)
 	}
-	virtualUTXODiff, err := csm.updateVirtualWithParents(updateVirtualStagingArea, virtualParents)
+	virtualUTXODiff, err := csm.updateVirtualWithParents(updateVirtualStagingArea, virtualTipCandidates)
 	if err != nil {
 		return nil, false, err
 	}
@@ -206,12 +191,12 @@ func (csm *consensusStateManager) ResolveVirtual(maxBlocksToResolve uint64) (*ex
 	}
 
 	selectedParentChainChanges, err := csm.dagTraversalManager.
-		CalculateChainPath(updateVirtualStagingArea, previousVirtualSelectedParent, processingPoint)
+		CalculateChainPath(updateVirtualStagingArea, prevVirtualSelectedParent, pendingTip)
 	if err != nil {
 		return nil, false, err
 	}
 
-	virtualParentsOutcome, err := csm.dagTopologyManager.Parents(updateVirtualStagingArea, model.VirtualBlockHash)
+	virtualParents, err := csm.dagTopologyManager.Parents(updateVirtualStagingArea, model.VirtualBlockHash)
 	if err != nil {
 		return nil, false, err
 	}
@@ -219,6 +204,6 @@ func (csm *consensusStateManager) ResolveVirtual(maxBlocksToResolve uint64) (*ex
 	return &externalapi.VirtualChangeSet{
 		VirtualSelectedParentChainChanges: selectedParentChainChanges,
 		VirtualUTXODiff:                   virtualUTXODiff,
-		VirtualParents:                    virtualParentsOutcome,
+		VirtualParents:                    virtualParents,
 	}, isCompletelyResolved, nil
 }

domain/consensus/processes/consensusstatemanager/resolve_virtual_test.go

@@ -24,14 +24,11 @@ func TestAddBlockBetweenResolveVirtualCalls(t *testing.T) {
 		}
 		defer teardown(false)
 
-		hashes := []*externalapi.DomainHash{consensusConfig.GenesisHash}
-
 		// Create a chain of blocks
 		const initialChainLength = 10
 		previousBlockHash := consensusConfig.GenesisHash
 		for i := 0; i < initialChainLength; i++ {
 			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
-			hashes = append(hashes, previousBlockHash)
 			if err != nil {
 				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
 			}
@@ -46,7 +43,6 @@ func TestAddBlockBetweenResolveVirtualCalls(t *testing.T) {
 				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
 			}
 			previousBlockHash = consensushashing.BlockHash(previousBlock)
-			hashes = append(hashes, previousBlockHash)
 
 			// Do not UTXO validate in order to resolve virtual later
 			err = tc.ValidateAndInsertBlock(previousBlock, false)
@@ -56,7 +52,7 @@ func TestAddBlockBetweenResolveVirtualCalls(t *testing.T) {
 		}
 
 		// Resolve one step
-		_, _, err = tc.ResolveVirtualWithMaxParam(2)
+		_, err = tc.ResolveVirtualWithMaxParam(2)
 		if err != nil {
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
@@ -75,7 +71,7 @@ func TestAddBlockBetweenResolveVirtualCalls(t *testing.T) {
 		}
 
 		// Resolve one more step
-		_, isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
 		if err != nil {
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
@@ -85,17 +81,14 @@ func TestAddBlockBetweenResolveVirtualCalls(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Error mining block during virtual resolution of reorg: %+v", err)
 		}
-		hashes = append(hashes, consensushashing.BlockHash(blockTemplate.Block))
 
 		// Complete resolving virtual
 		for !isCompletelyResolved {
-			_, isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
 			if err != nil {
 				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 			}
 		}
-
-		verifyUtxoDiffPaths(t, tc, hashes)
 	})
 }
 
@@ -110,14 +103,11 @@ func TestAddGenesisChildAfterOneResolveVirtualCall(t *testing.T) {
 		}
 		defer teardown(false)
 
-		hashes := []*externalapi.DomainHash{consensusConfig.GenesisHash}
-
 		// Create a chain of blocks
 		const initialChainLength = 6
 		previousBlockHash := consensusConfig.GenesisHash
 		for i := 0; i < initialChainLength; i++ {
 			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
-			hashes = append(hashes, previousBlockHash)
 			if err != nil {
 				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
 			}
@@ -132,7 +122,6 @@ func TestAddGenesisChildAfterOneResolveVirtualCall(t *testing.T) {
 				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
 			}
 			previousBlockHash = consensushashing.BlockHash(previousBlock)
-			hashes = append(hashes, previousBlockHash)
 
 			// Do not UTXO validate in order to resolve virtual later
 			err = tc.ValidateAndInsertBlock(previousBlock, false)
@@ -142,7 +131,7 @@ func TestAddGenesisChildAfterOneResolveVirtualCall(t *testing.T) {
 		}
 
 		// Resolve one step
-		_, isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
 		if err != nil {
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
@@ -154,13 +143,11 @@ func TestAddGenesisChildAfterOneResolveVirtualCall(t *testing.T) {
 
 		// Complete resolving virtual
 		for !isCompletelyResolved {
-			_, isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
 			if err != nil {
 				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 			}
 		}
-
-		verifyUtxoDiffPaths(t, tc, hashes)
 	})
 }
 
@@ -175,14 +162,11 @@ func TestAddGenesisChildAfterTwoResolveVirtualCalls(t *testing.T) {
 		}
 		defer teardown(false)
 
-		hashes := []*externalapi.DomainHash{consensusConfig.GenesisHash}
-
 		// Create a chain of blocks
 		const initialChainLength = 6
 		previousBlockHash := consensusConfig.GenesisHash
 		for i := 0; i < initialChainLength; i++ {
 			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
-			hashes = append(hashes, previousBlockHash)
 			if err != nil {
 				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
 			}
@@ -197,7 +181,6 @@ func TestAddGenesisChildAfterTwoResolveVirtualCalls(t *testing.T) {
 				t.Fatalf("Error mining block no. %d in re-org chain: %+v", i, err)
 			}
 			previousBlockHash = consensushashing.BlockHash(previousBlock)
-			hashes = append(hashes, previousBlockHash)
 
 			// Do not UTXO validate in order to resolve virtual later
 			err = tc.ValidateAndInsertBlock(previousBlock, false)
@@ -207,13 +190,13 @@ func TestAddGenesisChildAfterTwoResolveVirtualCalls(t *testing.T) {
 		}
 
 		// Resolve one step
-		_, _, err = tc.ResolveVirtualWithMaxParam(2)
+		_, err = tc.ResolveVirtualWithMaxParam(2)
 		if err != nil {
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
 
 		// Resolve one more step
-		_, isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(2)
 		if err != nil {
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
@@ -225,17 +208,15 @@ func TestAddGenesisChildAfterTwoResolveVirtualCalls(t *testing.T) {
 
 		// Complete resolving virtual
 		for !isCompletelyResolved {
-			_, isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(2)
 			if err != nil {
 				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 			}
 		}
-
-		verifyUtxoDiffPaths(t, tc, hashes)
 	})
 }
 
-func TestResolveVirtualBackAndForthReorgs(t *testing.T) {
+func TestResolveVirtualMess(t *testing.T) {
 
 	testutils.ForAllNets(t, true, func(t *testing.T, consensusConfig *consensus.Config) {
 		factory := consensus.NewFactory()
@@ -250,7 +231,7 @@ func TestResolveVirtualBackAndForthReorgs(t *testing.T) {
 		blocks := make(map[externalapi.DomainHash]string)
 		blocks[*consensusConfig.GenesisHash] = "g"
 		blocks[*model.VirtualBlockHash] = "v"
-		printfDebug("%s\n\n", consensusConfig.GenesisHash)
+		//fmt.Printf("%s\n\n", consensusConfig.GenesisHash)
 
 		// Create a chain of blocks
 		const initialChainLength = 6
@@ -259,20 +240,17 @@ func TestResolveVirtualBackAndForthReorgs(t *testing.T) {
 			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
 			blocks[*previousBlockHash] = fmt.Sprintf("A_%d", i)
 			hashes = append(hashes, previousBlockHash)
-			printfDebug("A_%d: %s\n", i, previousBlockHash)
+			//fmt.Printf("A_%d: %s\n", i, previousBlockHash)
 
 			if err != nil {
 				t.Fatalf("Error mining block no. %d in initial chain: %+v", i, err)
 			}
 		}
 
-		printfDebug("\n")
-		verifyUtxoDiffPaths(t, tc, hashes)
-
-		firstChainTip := previousBlockHash
+		//fmt.Printf("\n")
 
 		// Mine a chain with more blocks, to re-organize the DAG
-		const reorgChainLength = 12 // initialChainLength + 1
+		const reorgChainLength = 20 // initialChainLength + 1
 		previousBlockHash = consensusConfig.GenesisHash
 		for i := 0; i < reorgChainLength; i++ {
 			previousBlock, _, err := tc.BuildBlockWithParents([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
@@ -282,7 +260,7 @@ func TestResolveVirtualBackAndForthReorgs(t *testing.T) {
 			previousBlockHash = consensushashing.BlockHash(previousBlock)
 			blocks[*previousBlockHash] = fmt.Sprintf("B_%d", i)
 			hashes = append(hashes, previousBlockHash)
-			printfDebug("B_%d: %s\n", i, previousBlockHash)
+			//fmt.Printf("B_%d: %s\n", i, previousBlockHash)
 
 			// Do not UTXO validate in order to resolve virtual later
 			err = tc.ValidateAndInsertBlock(previousBlock, false)
@@ -291,139 +269,37 @@ func TestResolveVirtualBackAndForthReorgs(t *testing.T) {
 			}
 		}
 
-		printfDebug("\n")
+		//fmt.Printf("\n")
 
-		printUtxoDiffChildren(t, tc, hashes, blocks)
-		verifyUtxoDiffPaths(t, tc, hashes)
-
-		previousVirtualSelectedParent, err := tc.GetVirtualSelectedParent()
-		if err != nil {
-			t.Fatal(err)
-		}
+		//printUtxoDiffChildren(t, hashes, tc, blocks)
 
 		// Resolve one step
-		virtualChangeSet, _, err := tc.ResolveVirtualWithMaxParam(3)
+		_, err = tc.ResolveVirtualWithMaxParam(3)
 		if err != nil {
-			printUtxoDiffChildren(t, tc, hashes, blocks)
+			printUtxoDiffChildren(t, hashes, tc, blocks)
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
 
-		newVirtualSelectedParent, err := tc.GetVirtualSelectedParent()
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Make sure the reported change-set is compatible with actual changes.
-		// Checking this for one call should suffice to avoid possible bugs.
-		reportedPreviousVirtualSelectedParent := virtualChangeSet.VirtualSelectedParentChainChanges.Removed[0]
-		reportedNewVirtualSelectedParent := virtualChangeSet.VirtualSelectedParentChainChanges.
-			Added[len(virtualChangeSet.VirtualSelectedParentChainChanges.Added)-1]
-
-		if !previousVirtualSelectedParent.Equal(reportedPreviousVirtualSelectedParent) {
-			t.Fatalf("The reported changeset is incompatible with actual changes")
-		}
-		if !newVirtualSelectedParent.Equal(reportedNewVirtualSelectedParent) {
-			t.Fatalf("The reported changeset is incompatible with actual changes")
-		}
-
 		// Resolve one more step
-		_, isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(3)
+		isCompletelyResolved, err := tc.ResolveVirtualWithMaxParam(3)
 		if err != nil {
 			t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 		}
 
 		// Complete resolving virtual
 		for !isCompletelyResolved {
-			_, isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(3)
+			isCompletelyResolved, err = tc.ResolveVirtualWithMaxParam(3)
 			if err != nil {
 				t.Fatalf("Error resolving virtual in re-org chain: %+v", err)
 			}
 		}
 
-		printUtxoDiffChildren(t, tc, hashes, blocks)
-		verifyUtxoDiffPaths(t, tc, hashes)
-
-		// Now get the first chain back to the wining position
-		previousBlockHash = firstChainTip
-		for i := 0; i < reorgChainLength; i++ {
-			previousBlockHash, _, err = tc.AddBlock([]*externalapi.DomainHash{previousBlockHash}, nil, nil)
-			blocks[*previousBlockHash] = fmt.Sprintf("A_%d", initialChainLength+i)
-			hashes = append(hashes, previousBlockHash)
-			printfDebug("A_%d: %s\n", initialChainLength+i, previousBlockHash)
-
-			if err != nil {
-				t.Fatalf("Error mining block no. %d in initial chain: %+v", initialChainLength+i, err)
-			}
-		}
-
-		printfDebug("\n")
-
-		printUtxoDiffChildren(t, tc, hashes, blocks)
-		verifyUtxoDiffPaths(t, tc, hashes)
+		//printUtxoDiffChildren(t, hashes, tc, blocks)
 	})
 }
 
-func verifyUtxoDiffPathToRoot(t *testing.T, tc testapi.TestConsensus, stagingArea *model.StagingArea, block, utxoDiffRoot *externalapi.DomainHash) {
-	current := block
-	for !current.Equal(utxoDiffRoot) {
-		hasUTXODiffChild, err := tc.UTXODiffStore().HasUTXODiffChild(tc.DatabaseContext(), stagingArea, current)
-		if err != nil {
-			t.Fatalf("Error while reading utxo diff store: %+v", err)
-		}
-		if !hasUTXODiffChild {
-			t.Fatalf("%s is expected to have a UTXO diff child", current)
-		}
-		current, err = tc.UTXODiffStore().UTXODiffChild(tc.DatabaseContext(), stagingArea, current)
-		if err != nil {
-			t.Fatalf("Error while reading utxo diff store: %+v", err)
-		}
-	}
-}
-
-func verifyUtxoDiffPaths(t *testing.T, tc testapi.TestConsensus, hashes []*externalapi.DomainHash) {
-	stagingArea := model.NewStagingArea()
-
-	virtualGHOSTDAGData, err := tc.GHOSTDAGDataStore().Get(tc.DatabaseContext(), stagingArea, model.VirtualBlockHash, false)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	utxoDiffRoot := virtualGHOSTDAGData.SelectedParent()
-	hasUTXODiffChild, err := tc.UTXODiffStore().HasUTXODiffChild(tc.DatabaseContext(), stagingArea, utxoDiffRoot)
-	if err != nil {
-		t.Fatalf("Error while reading utxo diff store: %+v", err)
-	}
-	if hasUTXODiffChild {
-		t.Fatalf("Virtual selected parent is not expected to have an explicit diff child")
-	}
-	_, err = tc.UTXODiffStore().UTXODiff(tc.DatabaseContext(), stagingArea, utxoDiffRoot)
-	if err != nil {
-		t.Fatalf("Virtual selected parent is expected to have a utxo diff: %+v", err)
-	}
-
-	for _, block := range hashes {
-		hasUTXODiffChild, err = tc.UTXODiffStore().HasUTXODiffChild(tc.DatabaseContext(), stagingArea, block)
-		if err != nil {
-			t.Fatalf("Error while reading utxo diff store: %+v", err)
-		}
-		isOnVirtualSelectedChain, err := tc.DAGTopologyManager().IsInSelectedParentChainOf(stagingArea, block, utxoDiffRoot)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// We expect a valid path to root in both cases: (i) block has a diff child, (ii) block is on the virtual selected chain
-		if hasUTXODiffChild || isOnVirtualSelectedChain {
-			verifyUtxoDiffPathToRoot(t, tc, stagingArea, block, utxoDiffRoot)
-		}
-	}
-}
-
-func printfDebug(format string, a ...any) {
-	// Uncomment below when debugging the test
-	//fmt.Printf(format, a...)
-}
-
-func printUtxoDiffChildren(t *testing.T, tc testapi.TestConsensus, hashes []*externalapi.DomainHash, blocks map[externalapi.DomainHash]string) {
-	printfDebug("\n===============================\nBlock\t\tDiff child\n")
+func printUtxoDiffChildren(t *testing.T, hashes []*externalapi.DomainHash, tc testapi.TestConsensus, blocks map[externalapi.DomainHash]string) {
+	fmt.Printf("\n===============================\nBlock\t\tDiff child\n")
 	stagingArea := model.NewStagingArea()
 	for _, block := range hashes {
 		hasUTXODiffChild, err := tc.UTXODiffStore().HasUTXODiffChild(tc.DatabaseContext(), stagingArea, block)
@@ -435,10 +311,10 @@ func printUtxoDiffChildren(t *testing.T, tc testapi.TestConsensus, hashes []*ext
 			if err != nil {
 				t.Fatalf("Error while reading utxo diff store: %+v", err)
 			}
-			printfDebug("%s\t\t\t%s\n", blocks[*block], blocks[*utxoDiffChild])
+			fmt.Printf("%s\t\t\t%s\n", blocks[*block], blocks[*utxoDiffChild])
 		} else {
-			printfDebug("%s\n", blocks[*block])
+			fmt.Printf("%s\n", blocks[*block])
 		}
 	}
-	printfDebug("\n===============================\n")
+	fmt.Printf("\n===============================\n")
 }

domain/consensus/processes/pruningmanager/pruningmanager.go

@@ -772,41 +772,78 @@ func (pm *pruningManager) calculateDiffBetweenPreviousAndCurrentPruningPoints(st
 	if err != nil {
 		return nil, err
 	}
-
-	utxoDiff := utxo.NewMutableUTXODiff()
-
-	iterator, err := pm.dagTraversalManager.SelectedChildIterator(stagingArea, currentPruningHash, previousPruningHash, false)
+	currentPruningGhostDAG, err := pm.ghostdagDataStore.Get(pm.databaseContext, stagingArea, currentPruningHash, false)
 	if err != nil {
 		return nil, err
 	}
-	defer iterator.Close()
-
-	for ok := iterator.First(); ok; ok = iterator.Next() {
-		child, err := iterator.Get()
-		if err != nil {
-			return nil, err
-		}
-		chainBlockAcceptanceData, err := pm.acceptanceDataStore.Get(pm.databaseContext, stagingArea, child)
-		if err != nil {
-			return nil, err
-		}
-		chainBlockHeader, err := pm.blockHeaderStore.BlockHeader(pm.databaseContext, stagingArea, child)
-		if err != nil {
-			return nil, err
-		}
-		for _, blockAcceptanceData := range chainBlockAcceptanceData {
-			for _, transactionAcceptanceData := range blockAcceptanceData.TransactionAcceptanceData {
-				if transactionAcceptanceData.IsAccepted {
-					err = utxoDiff.AddTransaction(transactionAcceptanceData.Transaction, chainBlockHeader.DAAScore())
-					if err != nil {
-						return nil, err
-					}
-				}
-			}
-		}
+	previousPruningGhostDAG, err := pm.ghostdagDataStore.Get(pm.databaseContext, stagingArea, previousPruningHash, false)
+	if err != nil {
+		return nil, err
 	}
 
-	return utxoDiff.ToImmutable(), err
+	currentPruningCurrentDiffChild := currentPruningHash
+	previousPruningCurrentDiffChild := previousPruningHash
+	// We need to use BlueWork because it's the only thing that's monotonic in the whole DAG
+	// We use the BlueWork to know which point is currently lower on the DAG so we can keep climbing its children,
+	// that way we keep climbing on the lowest point until they both reach the exact same descendant
+	currentPruningCurrentDiffChildBlueWork := currentPruningGhostDAG.BlueWork()
+	previousPruningCurrentDiffChildBlueWork := previousPruningGhostDAG.BlueWork()
+
+	var diffHashesFromPrevious []*externalapi.DomainHash
+	var diffHashesFromCurrent []*externalapi.DomainHash
+	for {
+		// if currentPruningCurrentDiffChildBlueWork > previousPruningCurrentDiffChildBlueWork
+		if currentPruningCurrentDiffChildBlueWork.Cmp(previousPruningCurrentDiffChildBlueWork) == 1 {
+			diffHashesFromPrevious = append(diffHashesFromPrevious, previousPruningCurrentDiffChild)
+			previousPruningCurrentDiffChild, err = pm.utxoDiffStore.UTXODiffChild(pm.databaseContext, stagingArea, previousPruningCurrentDiffChild)
+			if err != nil {
+				return nil, err
+			}
+			diffChildGhostDag, err := pm.ghostdagDataStore.Get(pm.databaseContext, stagingArea, previousPruningCurrentDiffChild, false)
+			if err != nil {
+				return nil, err
+			}
+			previousPruningCurrentDiffChildBlueWork = diffChildGhostDag.BlueWork()
+		} else if currentPruningCurrentDiffChild.Equal(previousPruningCurrentDiffChild) {
+			break
+		} else {
+			diffHashesFromCurrent = append(diffHashesFromCurrent, currentPruningCurrentDiffChild)
+			currentPruningCurrentDiffChild, err = pm.utxoDiffStore.UTXODiffChild(pm.databaseContext, stagingArea, currentPruningCurrentDiffChild)
+			if err != nil {
+				return nil, err
+			}
+			diffChildGhostDag, err := pm.ghostdagDataStore.Get(pm.databaseContext, stagingArea, currentPruningCurrentDiffChild, false)
+			if err != nil {
+				return nil, err
+			}
+			currentPruningCurrentDiffChildBlueWork = diffChildGhostDag.BlueWork()
+		}
+	}
+	// The order in which we apply the diffs should be from top to bottom, but we traversed from bottom to top
+	// so we apply the diffs in reverse order.
+	oldDiff := utxo.NewMutableUTXODiff()
+	for i := len(diffHashesFromPrevious) - 1; i >= 0; i-- {
+		utxoDiff, err := pm.utxoDiffStore.UTXODiff(pm.databaseContext, stagingArea, diffHashesFromPrevious[i])
+		if err != nil {
+			return nil, err
+		}
+		err = oldDiff.WithDiffInPlace(utxoDiff)
+		if err != nil {
+			return nil, err
+		}
+	}
+	newDiff := utxo.NewMutableUTXODiff()
+	for i := len(diffHashesFromCurrent) - 1; i >= 0; i-- {
+		utxoDiff, err := pm.utxoDiffStore.UTXODiff(pm.databaseContext, stagingArea, diffHashesFromCurrent[i])
+		if err != nil {
+			return nil, err
+		}
+		err = newDiff.WithDiffInPlace(utxoDiff)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return oldDiff.DiffFrom(newDiff.ToImmutable())
 }
 
 // finalityScore is the number of finality intervals passed since

domain/consensus/test_consensus.go

@@ -112,11 +112,11 @@ func (tc *testConsensus) AddUTXOInvalidBlock(parentHashes []*externalapi.DomainH
 	return consensushashing.BlockHash(block), virtualChangeSet, nil
 }
 
-func (tc *testConsensus) ResolveVirtualWithMaxParam(maxBlocksToResolve uint64) (*externalapi.VirtualChangeSet, bool, error) {
+func (tc *testConsensus) ResolveVirtualWithMaxParam(maxBlocksToResolve uint64) (bool, error) {
 	tc.lock.Lock()
 	defer tc.lock.Unlock()
 
-	return tc.resolveVirtualChunkNoLock(maxBlocksToResolve)
+	return tc.resolveVirtualNoLock(maxBlocksToResolve)
 }
 
 // jsonBlock is a json representation of a block in mine format