Prune blocks below pruning point when moving pruning point during IBD (#1513)

* Split deletePastBlocks into sub-routines

* Remove SelectedParentIterator, and refactor SelectedChildIterator to support First and Error

* Implement PruneAllBlocks

* Prune all blocks in the store

* Prune only blocks that are below the pruning point

* Defer call onEnd of LogAndMeasureExecutionTime

* Handle a forgotten error

* Minor style fixes

domain/consensus/datastructures/blockstore/blockstore.go

@@ -212,3 +212,34 @@ func (bs *blockStore) serializeBlockCount(count uint64) ([]byte, error) {
 	dbBlockCount := &serialization.DbBlockCount{Count: count}
 	return proto.Marshal(dbBlockCount)
 }
+
+type allBlockHashesIterator struct {
+	cursor model.DBCursor
+}
+
+func (a allBlockHashesIterator) First() bool {
+	return a.cursor.First()
+}
+
+func (a allBlockHashesIterator) Next() bool {
+	return a.cursor.Next()
+}
+
+func (a allBlockHashesIterator) Get() (*externalapi.DomainHash, error) {
+	key, err := a.cursor.Key()
+	if err != nil {
+		return nil, err
+	}
+
+	blockHashBytes := key.Suffix()
+	return externalapi.NewDomainHashFromByteSlice(blockHashBytes)
+}
+
+func (bs *blockStore) AllBlockHashesIterator(dbContext model.DBReader) (model.BlockIterator, error) {
+	cursor, err := dbContext.Cursor(bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	return &allBlockHashesIterator{cursor: cursor}, nil
+}

domain/consensus/model/block_heap.go

@@ -5,6 +5,7 @@ import "github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 // BlockHeap represents a heap of block hashes, providing a priority-queue functionality
 type BlockHeap interface {
 	Push(blockHash *externalapi.DomainHash) error
+	PushSlice(blockHash []*externalapi.DomainHash) error
 	Pop() *externalapi.DomainHash
 	Len() int
 	ToSlice() []*externalapi.DomainHash

domain/consensus/model/blockiterator.go

@@ -4,6 +4,7 @@ import "github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 
 // BlockIterator is an iterator over blocks according to some order.
 type BlockIterator interface {
+	First() bool
 	Next() bool
-	Get() *externalapi.DomainHash
+	Get() (*externalapi.DomainHash, error)
 }

domain/consensus/model/interface_datastructures_blockstore.go

@@ -12,4 +12,5 @@ type BlockStore interface {
 	Blocks(dbContext DBReader, blockHashes []*externalapi.DomainHash) ([]*externalapi.DomainBlock, error)
 	Delete(blockHash *externalapi.DomainHash)
 	Count() uint64
+	AllBlockHashesIterator(dbContext DBReader) (BlockIterator, error)
 }

domain/consensus/model/interface_processes_dagtopologymanager.go

@@ -10,7 +10,6 @@ type DAGTopologyManager interface {
 	IsParentOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	IsChildOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	IsAncestorOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
-	IsDescendantOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	IsAncestorOfAny(blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error)
 	IsInSelectedParentChainOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error)
 	ChildInSelectedParentChainOf(context, highHash *externalapi.DomainHash) (*externalapi.DomainHash, error)

domain/consensus/model/interface_processes_dagtraversalmanager.go

@@ -7,7 +7,8 @@ import "github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 type DAGTraversalManager interface {
 	BlockAtDepth(highHash *externalapi.DomainHash, depth uint64) (*externalapi.DomainHash, error)
 	LowestChainBlockAboveOrEqualToBlueScore(highHash *externalapi.DomainHash, blueScore uint64) (*externalapi.DomainHash, error)
-	SelectedParentIterator(highHash *externalapi.DomainHash) BlockIterator
+	// SelectedChildIterator should return a BlockIterator that iterates
+	// from lowHash (exclusive) to highHash (inclusive) over highHash's selected parent chain
 	SelectedChildIterator(highHash, lowHash *externalapi.DomainHash) (BlockIterator, error)
 	AnticoneFromContext(context, lowHash *externalapi.DomainHash) ([]*externalapi.DomainHash, error)
 	BlueWindow(highHash *externalapi.DomainHash, windowSize int) ([]*externalapi.DomainHash, error)

domain/consensus/model/interface_processes_pruningmanager.go

@@ -9,4 +9,5 @@ type PruningManager interface {
 	ClearImportedPruningPointData() error
 	AppendImportedPruningPointUTXOs(outpointAndUTXOEntryPairs []*externalapi.OutpointAndUTXOEntryPair) error
 	UpdatePruningPointUTXOSetIfRequired() error
+	PruneAllBlocksBelow(pruningPointHash *externalapi.DomainHash) error
 }

domain/consensus/processes/blockprocessor/validateandinsertimportedpruningpoint.go

@@ -28,6 +28,12 @@ func (bp *blockProcessor) validateAndInsertImportedPruningPoint(newPruningPoint
 		return err
 	}
 
+	log.Info("Deleting block data for all blocks in blockStore")
+	err = bp.pruningManager.PruneAllBlocksBelow(newPruningPointHash)
+	if err != nil {
+		return err
+	}
+
 	log.Infof("Updating consensus state manager according to the new pruning point %s", newPruningPointHash)
 	err = bp.consensusStateManager.ImportPruningPoint(newPruningPoint)
 	if err != nil {

domain/consensus/processes/consensusstatemanager/import_pruning_utxo_set.go

@@ -61,7 +61,7 @@ func (csm *consensusStateManager) importPruningPoint(newPruningPoint *externalap
 	}
 	log.Debugf("The new pruning point UTXO commitment validation passed")
 
-	log.Debugf("Staging the the pruning point as the only DAG tip")
+	log.Debugf("Staging the pruning point as the only DAG tip")
 	newTips := []*externalapi.DomainHash{newPruningPointHash}
 	csm.consensusStateStore.StageTips(newTips)
 

domain/consensus/processes/dagtopologymanager/dagtopologymanager.go

@@ -71,11 +71,6 @@ func (dtm *dagTopologyManager) IsAncestorOf(blockHashA *externalapi.DomainHash,
 	return dtm.reachabilityManager.IsDAGAncestorOf(blockHashA, blockHashB)
 }
 
-// IsDescendantOf returns true if blockHashA is a DAG descendant of blockHashB
-func (dtm *dagTopologyManager) IsDescendantOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) {
-	return dtm.reachabilityManager.IsDAGAncestorOf(blockHashB, blockHashA)
-}
-
 // IsAncestorOfAny returns true if `blockHash` is an ancestor of at least one of `potentialDescendants`
 func (dtm *dagTopologyManager) IsAncestorOfAny(blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error) {
 	for _, potentialDescendant := range potentialDescendants {

domain/consensus/processes/dagtraversalmanager/block_heap.go

@@ -109,6 +109,16 @@ func (bh *blockHeap) Push(blockHash *externalapi.DomainHash) error {
 	return nil
 }
 
+func (bh *blockHeap) PushSlice(blockHashes []*externalapi.DomainHash) error {
+	for _, blockHash := range blockHashes {
+		err := bh.Push(blockHash)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Len returns the length of this heap
 func (bh *blockHeap) Len() int {
 	return bh.impl.Len()

domain/consensus/processes/dagtraversalmanager/dagtraversalmanager.go

@@ -1,7 +1,6 @@
 package dagtraversalmanager
 
 import (
-	"fmt"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/pkg/errors"
@@ -18,29 +17,6 @@ type dagTraversalManager struct {
 	reachabilityDataStore model.ReachabilityDataStore
 }
 
-// selectedParentIterator implements the `model.BlockIterator` API
-type selectedParentIterator struct {
-	databaseContext   model.DBReader
-	ghostdagDataStore model.GHOSTDAGDataStore
-	current           *externalapi.DomainHash
-}
-
-func (spi *selectedParentIterator) Next() bool {
-	if spi.current == nil {
-		return false
-	}
-	ghostdagData, err := spi.ghostdagDataStore.Get(spi.databaseContext, spi.current)
-	if err != nil {
-		panic(fmt.Sprintf("ghostdagDataStore is missing ghostdagData for: %v. '%s' ", spi.current, err))
-	}
-	spi.current = ghostdagData.SelectedParent()
-	return spi.current != nil
-}
-
-func (spi *selectedParentIterator) Get() *externalapi.DomainHash {
-	return spi.current
-}
-
 // New instantiates a new DAGTraversalManager
 func New(
 	databaseContext model.DBReader,
@@ -57,16 +33,6 @@ func New(
 	}
 }
 
-// SelectedParentIterator creates an iterator over the selected
-// parent chain of the given highHash
-func (dtm *dagTraversalManager) SelectedParentIterator(highHash *externalapi.DomainHash) model.BlockIterator {
-	return &selectedParentIterator{
-		databaseContext:   dtm.databaseContext,
-		ghostdagDataStore: dtm.ghostdagDataStore,
-		current:           highHash,
-	}
-}
-
 // BlockAtDepth returns the hash of the highest block with a blue score
 // lower than (highHash.blueSore - depth) in the selected-parent-chain
 // of the block with the given highHash's selected parent chain.

domain/consensus/processes/dagtraversalmanager/selected_child_iterator.go

@@ -11,20 +11,34 @@ type selectedChildIterator struct {
 	dagTopologyManager model.DAGTopologyManager
 
 	reachabilityDataStore model.ReachabilityDataStore
-	highHash              *externalapi.DomainHash
+	highHash, lowHash     *externalapi.DomainHash
 	current               *externalapi.DomainHash
+	err                   error
+}
+
+func (s *selectedChildIterator) First() bool {
+	s.current = s.lowHash
+	return s.Next()
 }
 
 func (s *selectedChildIterator) Next() bool {
+	if s.err != nil {
+		return true
+	}
+
 	data, err := s.reachabilityDataStore.ReachabilityData(s.databaseContext, s.current)
 	if err != nil {
-		panic(err)
+		s.current = nil
+		s.err = err
+		return true
 	}
 
 	for _, child := range data.Children() {
 		isChildInSelectedParentChainOfHighHash, err := s.dagTopologyManager.IsInSelectedParentChainOf(child, s.highHash)
 		if err != nil {
-			panic(err)
+			s.current = nil
+			s.err = err
+			return true
 		}
 
 		if isChildInSelectedParentChainOfHighHash {
@@ -35,10 +49,12 @@ func (s *selectedChildIterator) Next() bool {
 	return false
 }
 
-func (s *selectedChildIterator) Get() *externalapi.DomainHash {
-	return s.current
+func (s *selectedChildIterator) Get() (*externalapi.DomainHash, error) {
+	return s.current, s.err
 }
 
+// SelectedChildIterator returns a BlockIterator that iterates from lowHash (exclusive) to highHash (inclusive) over
+// highHash's selected parent chain
 func (dtm *dagTraversalManager) SelectedChildIterator(highHash, lowHash *externalapi.DomainHash) (model.BlockIterator, error) {
 	isLowHashInSelectedParentChainOfHighHash, err := dtm.dagTopologyManager.IsInSelectedParentChainOf(lowHash, highHash)
 	if err != nil {
@@ -53,6 +69,7 @@ func (dtm *dagTraversalManager) SelectedChildIterator(highHash, lowHash *externa
 		dagTopologyManager:    dtm.dagTopologyManager,
 		reachabilityDataStore: dtm.reachabilityDataStore,
 		highHash:              highHash,
+		lowHash:               lowHash,
 		current:               lowHash,
 	}, nil
 }

domain/consensus/processes/ghostdagmanager/ghostdag_test.go

@@ -2,9 +2,6 @@ package ghostdagmanager_test
 
 import (
 	"encoding/json"
-	"github.com/kaspanet/kaspad/domain/consensus/utils/blockheader"
-	"github.com/kaspanet/kaspad/domain/consensus/utils/constants"
-	"github.com/kaspanet/kaspad/util/difficulty"
 	"math"
 	"math/big"
 	"os"
@@ -12,6 +9,10 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/kaspanet/kaspad/domain/consensus/utils/blockheader"
+	"github.com/kaspanet/kaspad/domain/consensus/utils/constants"
+	"github.com/kaspanet/kaspad/util/difficulty"
+
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/processes/ghostdag2"
@@ -386,10 +387,6 @@ func (dt *DAGTopologyManagerImpl) IsAncestorOf(hashBlockA *externalapi.DomainHas
 
 }
 
-func (dt *DAGTopologyManagerImpl) IsDescendantOf(blockHashA *externalapi.DomainHash, blockHashB *externalapi.DomainHash) (bool, error) {
-	panic("unimplemented")
-}
-
 func (dt *DAGTopologyManagerImpl) IsAncestorOfAny(blockHash *externalapi.DomainHash, potentialDescendants []*externalapi.DomainHash) (bool, error) {
 	panic("unimplemented")
 }

domain/consensus/processes/pruningmanager/pruningmanager.go

@@ -151,8 +151,11 @@ func (pm *pruningManager) UpdatePruningPointByVirtual() error {
 
 	newPruningPoint := currentPruningPoint
 	newPruningPointGHOSTDAGData := currentPruningPointGHOSTDAGData
-	for iterator.Next() {
-		selectedChild := iterator.Get()
+	for ok := iterator.First(); ok; ok = iterator.Next() {
+		selectedChild, err := iterator.Get()
+		if err != nil {
+			return err
+		}
 		selectedChildGHOSTDAGData, err := pm.ghostdagDataStore.Get(pm.databaseContext, selectedChild)
 		if err != nil {
 			return err
@@ -221,47 +224,47 @@ func (pm *pruningManager) deletePastBlocks(pruningPoint *externalapi.DomainHash)
 	onEnd := logger.LogAndMeasureExecutionTime(log, "pruningManager.deletePastBlocks")
 	defer onEnd()
 
-	// Go over all P.Past and P.AC that's not in V.Past
+	// Go over all pruningPoint.Past and pruningPoint.Anticone that's not in virtual.Past
 	queue := pm.dagTraversalManager.NewDownHeap()
-
-	// Find P.AC that's not in V.Past
-	dagTips, err := pm.consensusStateStore.Tips(pm.databaseContext)
-	if err != nil {
-		return err
-	}
-	newTips := make([]*externalapi.DomainHash, 0, len(dagTips))
 	virtualParents, err := pm.dagTopologyManager.Parents(model.VirtualBlockHash)
 	if err != nil {
 		return err
 	}
-	for _, tip := range dagTips {
-		isInPruningFutureOrInVirtualPast, err := pm.isInPruningFutureOrInVirtualPast(tip, pruningPoint, virtualParents)
-		if err != nil {
-			return err
-		}
-		if !isInPruningFutureOrInVirtualPast {
-			// Add them to the queue so they and their past will be pruned
-			err := queue.Push(tip)
-			if err != nil {
-				return err
-			}
-		} else {
-			newTips = append(newTips, tip)
-		}
+
+	// Start queue with all tips that are below the pruning point (and on the way remove them from list of tips)
+	prunedTips, err := pm.pruneTips(pruningPoint, virtualParents)
+	if err != nil {
+		return err
 	}
-	pm.consensusStateStore.StageTips(newTips)
-	// Add P.Parents
+	err = queue.PushSlice(prunedTips)
+	if err != nil {
+		return err
+	}
+
+	// Add pruningPoint.Parents to queue
 	parents, err := pm.dagTopologyManager.Parents(pruningPoint)
 	if err != nil {
 		return err
 	}
-	for _, parent := range parents {
-		err = queue.Push(parent)
-		if err != nil {
-			return err
-		}
+	err = queue.PushSlice(parents)
+	if err != nil {
+		return err
 	}
 
+	err = pm.deleteBlocksDownward(queue)
+	if err != nil {
+		return err
+	}
+
+	err = pm.pruneVirtualDiffParents(pruningPoint, virtualParents)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (pm *pruningManager) deleteBlocksDownward(queue model.BlockHeap) error {
 	visited := map[externalapi.DomainHash]struct{}{}
 	// Prune everything in the queue including its past
 	for queue.Len() > 0 {
@@ -280,16 +283,16 @@ func (pm *pruningManager) deletePastBlocks(pruningPoint *externalapi.DomainHash)
 			if err != nil {
 				return err
 			}
-			for _, parent := range parents {
-				err = queue.Push(parent)
-				if err != nil {
-					return err
-				}
+			err = queue.PushSlice(parents)
+			if err != nil {
+				return err
 			}
 		}
 	}
+	return nil
+}
 
-	// Delete virtual diff parents that are in PruningPoint's Anticone and not in Virtual's Past
+func (pm *pruningManager) pruneVirtualDiffParents(pruningPoint *externalapi.DomainHash, virtualParents []*externalapi.DomainHash) error {
 	virtualDiffParents, err := pm.consensusStateStore.VirtualDiffParents(pm.databaseContext)
 	if err != nil {
 		return err
@@ -309,6 +312,31 @@ func (pm *pruningManager) deletePastBlocks(pruningPoint *externalapi.DomainHash)
 	return nil
 }
 
+func (pm *pruningManager) pruneTips(pruningPoint *externalapi.DomainHash, virtualParents []*externalapi.DomainHash) (
+	prunedTips []*externalapi.DomainHash, err error) {
+
+	// Find P.AC that's not in V.Past
+	dagTips, err := pm.consensusStateStore.Tips(pm.databaseContext)
+	if err != nil {
+		return nil, err
+	}
+	newTips := make([]*externalapi.DomainHash, 0, len(dagTips))
+	for _, tip := range dagTips {
+		isInPruningFutureOrInVirtualPast, err := pm.isInPruningFutureOrInVirtualPast(tip, pruningPoint, virtualParents)
+		if err != nil {
+			return nil, err
+		}
+		if !isInPruningFutureOrInVirtualPast {
+			prunedTips = append(prunedTips, tip)
+		} else {
+			newTips = append(newTips, tip)
+		}
+	}
+	pm.consensusStateStore.StageTips(newTips)
+
+	return prunedTips, nil
+}
+
 func (pm *pruningManager) savePruningPoint(pruningPointHash *externalapi.DomainHash) error {
 	onEnd := logger.LogAndMeasureExecutionTime(log, "pruningManager.savePruningPoint")
 	defer onEnd()
@@ -551,3 +579,33 @@ func (pm *pruningManager) updatePruningPointUTXOSet() error {
 	log.Debugf("Finishing updating the pruning point UTXO set")
 	return pm.pruningStore.FinishUpdatingPruningPointUTXOSet(pm.databaseContext)
 }
+
+func (pm *pruningManager) PruneAllBlocksBelow(pruningPointHash *externalapi.DomainHash) error {
+	onEnd := logger.LogAndMeasureExecutionTime(log, "PruneAllBlocksBelow")
+	defer onEnd()
+
+	iterator, err := pm.blocksStore.AllBlockHashesIterator(pm.databaseContext)
+	if err != nil {
+		return err
+	}
+
+	for ok := iterator.First(); ok; ok = iterator.Next() {
+		blockHash, err := iterator.Get()
+		if err != nil {
+			return err
+		}
+		isInPastOfPruningPoint, err := pm.dagTopologyManager.IsAncestorOf(pruningPointHash, blockHash)
+		if err != nil {
+			return err
+		}
+		if !isInPastOfPruningPoint {
+			continue
+		}
+		_, err = pm.deleteBlock(blockHash)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

domain/consensus/processes/syncmanager/antipast.go

@@ -39,8 +39,11 @@ func (sm *syncManager) antiPastHashesBetween(lowHash, highHash *externalapi.Doma
 		if err != nil {
 			return nil, err
 		}
-		for iterator.Next() {
-			highHash = iterator.Get()
+		for ok := iterator.First(); ok; ok = iterator.Next() {
+			highHash, err = iterator.Get()
+			if err != nil {
+				return nil, err
+			}
 			highBlockGHOSTDAGData, err = sm.ghostdagDataStore.Get(sm.databaseContext, highHash)
 			if err != nil {
 				return nil, err
@@ -112,8 +115,11 @@ func (sm *syncManager) missingBlockBodyHashes(highHash *externalapi.DomainHash)
 
 	lowHash := pruningPoint
 	foundHeaderOnlyBlock := false
-	for selectedChildIterator.Next() {
-		selectedChild := selectedChildIterator.Get()
+	for ok := selectedChildIterator.First(); ok; ok = selectedChildIterator.Next() {
+		selectedChild, err := selectedChildIterator.Get()
+		if err != nil {
+			return nil, err
+		}
 		hasBlock, err := sm.blockStore.HasBlock(sm.databaseContext, selectedChild)
 		if err != nil {
 			return nil, err