package reachabilitymanager import ( "github.com/kaspanet/kaspad/domain/consensus/model" "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" "github.com/pkg/errors" ) var ( // defaultReindexWindow is the default target window size for reachability // reindexes. Note that this is not a constant for testing purposes. defaultReindexWindow uint64 = 200 // defaultReindexSlack is default the slack interval given to reachability // tree nodes not in the selected parent chain. Note that this is not // a constant for testing purposes. defaultReindexSlack uint64 = 1 << 12 ) // reindexContext is a struct used during reindex operations. It represents a temporary context // for caching subtree information during the *current* reindex operation only type reindexContext struct { manager *reachabilityManager subTreeSizesCache map[externalapi.DomainHash]uint64 } // newReindexContext creates a new empty reindex context func newReindexContext(rt *reachabilityManager) reindexContext { return reindexContext{ manager: rt, subTreeSizesCache: make(map[externalapi.DomainHash]uint64), } } /* Core (BFS) algorithms used during reindexing */ // countSubtrees counts the size of each subtree under this node, // and populates the provided subTreeSizeMap with the results. // It is equivalent to the following recursive implementation: // // func (rt *reachabilityManager) countSubtrees(node *model.ReachabilityTreeNode) uint64 { // subtreeSize := uint64(0) // for _, child := range node.children { // subtreeSize += child.countSubtrees() // } // return subtreeSize + 1 // } // // However, we are expecting (linearly) deep trees, and so a // recursive stack-based approach is inefficient and will hit // recursion limits. Instead, the same logic was implemented // using a (queue-based) BFS method. At a high level, the // algorithm uses BFS for reaching all leaves and pushes // intermediate updates from leaves via parent chains until all // size information is gathered at the root of the operation // (i.e. at node). func (rc *reindexContext) countSubtrees(node *externalapi.DomainHash) error { if _, ok := rc.subTreeSizesCache[*node]; ok { return nil } queue := []*externalapi.DomainHash{node} calculatedChildrenCount := make(map[externalapi.DomainHash]uint64) for len(queue) > 0 { var current *externalapi.DomainHash current, queue = queue[0], queue[1:] children, err := rc.manager.children(current) if err != nil { return err } if len(children) == 0 { // We reached a leaf rc.subTreeSizesCache[*current] = 1 } else if _, ok := rc.subTreeSizesCache[*current]; !ok { // We haven't yet calculated the subtree size of // the current node. Add all its children to the // queue queue = append(queue, children...) continue } // We reached a leaf or a pre-calculated subtree. // Push information up for !current.Equal(node) { current, err = rc.manager.parent(current) if err != nil { return err } // If the current is now nil, it means that the previous // `current` was the genesis block -- the only block that // does not have parents if current == nil { break } calculatedChildrenCount[*current]++ children, err := rc.manager.children(current) if err != nil { return err } if calculatedChildrenCount[*current] != uint64(len(children)) { // Not all subtrees of the current node are ready break } // All children of `current` have calculated their subtree size. // Sum them all together and add 1 to get the sub tree size of // `current`. childSubtreeSizeSum := uint64(0) for _, child := range children { childSubtreeSizeSum += rc.subTreeSizesCache[*child] } rc.subTreeSizesCache[*current] = childSubtreeSizeSum + 1 } } return nil } // propagateInterval propagates the new interval using a BFS traversal. // Subtree intervals are recursively allocated according to subtree sizes and // the allocation rule in splitWithExponentialBias. func (rc *reindexContext) propagateInterval(node *externalapi.DomainHash) error { // Make sure subtrees are counted before propagating err := rc.countSubtrees(node) if err != nil { return err } queue := []*externalapi.DomainHash{node} for len(queue) > 0 { var current *externalapi.DomainHash current, queue = queue[0], queue[1:] children, err := rc.manager.children(current) if err != nil { return err } if len(children) > 0 { sizes := make([]uint64, len(children)) for i, child := range children { sizes[i] = rc.subTreeSizesCache[*child] } interval, err := rc.manager.intervalRangeForChildAllocation(current) if err != nil { return err } intervals, err := intervalSplitWithExponentialBias(interval, sizes) if err != nil { return err } for i, child := range children { childInterval := intervals[i] err = rc.manager.stageInterval(child, childInterval) if err != nil { return err } queue = append(queue, child) } } } return nil } /* Functions for handling reindex triggered by adding child block */ // reindexIntervals traverses the reachability subtree that's // defined by the new child node and reallocates reachability interval space // such that another reindexing is unlikely to occur shortly // thereafter. It does this by traversing down the reachability // tree until it finds a node with a subtree size that's greater than // its interval size. See propagateInterval for further details. func (rc *reindexContext) reindexIntervals(newChild, reindexRoot *externalapi.DomainHash) error { current := newChild // Search for the first ancestor with sufficient interval space for { currentInterval, err := rc.manager.interval(current) if err != nil { return err } currentIntervalSize := intervalSize(currentInterval) err = rc.countSubtrees(current) if err != nil { return err } currentSubtreeSize := rc.subTreeSizesCache[*current] // Current has sufficient space, break and propagate if currentIntervalSize >= currentSubtreeSize { break } parent, err := rc.manager.parent(current) if err != nil { return err } if parent == nil { // If we ended up here it means that there are more // than 2^64 blocks, which shouldn't ever happen. return errors.Errorf("missing tree " + "parent during reindexing. Theoretically, this " + "should only ever happen if there are more " + "than 2^64 blocks in the DAG.") } if current.Equal(reindexRoot) { // Reindex root is expected to hold enough capacity as long as there are less // than ~2^52 blocks in the DAG, which should never happen in our lifetimes // even if block rate per second is above 100. The calculation follows from the allocation of // 2^12 (which equals 2^64/2^52) for slack per chain block below the reindex root. return errors.Errorf("unexpected behavior: reindex root %s is out of capacity"+ "during reindexing. Theoretically, this "+ "should only ever happen if there are more "+ "than ~2^52 blocks in the DAG.", reindexRoot.String()) } isParentStrictAncestorOfRoot, err := rc.manager.isStrictAncestorOf(parent, reindexRoot) if err != nil { return err } if isParentStrictAncestorOfRoot { // In this case parent is guaranteed to have sufficient interval space, // however we avoid reindexing the entire subtree above parent // (which includes root and thus majority of blocks mined since) // and use slacks along the chain up from parent to reindex root. // Notes: // 1. we set requiredAllocation=currentSubtreeSize in order to double the // current interval capacity // 2. it might be the case that current is the `newChild` itself return rc.reindexIntervalsEarlierThanRoot(current, reindexRoot, parent, currentSubtreeSize) } current = parent } // Propagate the interval to the subtree return rc.propagateInterval(current) } // reindexIntervalsEarlierThanRoot implements the reindex algorithm for the case where the // new child node is not in reindex root's subtree. The function is expected to allocate // `requiredAllocation` to be added to interval of `allocationNode`. `commonAncestor` is // expected to be a direct parent of `allocationNode` and an ancestor of `reindexRoot`. func (rc *reindexContext) reindexIntervalsEarlierThanRoot( allocationNode, reindexRoot, commonAncestor *externalapi.DomainHash, requiredAllocation uint64) error { // The chosen child is: // a. A reachability tree child of `commonAncestor` // b. A reachability tree ancestor of `reindexRoot` or `reindexRoot` itself chosenChild, err := rc.manager.FindNextAncestor(reindexRoot, commonAncestor) if err != nil { return err } nodeInterval, err := rc.manager.interval(allocationNode) if err != nil { return err } chosenInterval, err := rc.manager.interval(chosenChild) if err != nil { return err } if nodeInterval.Start < chosenInterval.Start { // allocationNode is in the subtree before the chosen child return rc.reclaimIntervalBefore(allocationNode, commonAncestor, chosenChild, reindexRoot, requiredAllocation) } // allocationNode is in the subtree after the chosen child return rc.reclaimIntervalAfter(allocationNode, commonAncestor, chosenChild, reindexRoot, requiredAllocation) } func (rc *reindexContext) reclaimIntervalBefore( allocationNode, commonAncestor, chosenChild, reindexRoot *externalapi.DomainHash, requiredAllocation uint64) error { var slackSum uint64 = 0 var pathLen uint64 = 0 var pathSlackAlloc uint64 = 0 var err error current := chosenChild // Walk up the chain from common ancestor's chosen child towards reindex root for { if current.Equal(reindexRoot) { // Reached reindex root. In this case, since we reached (the unlimited) root, // we also re-allocate new slack for the chain we just traversed previousInterval, err := rc.manager.interval(current) if err != nil { return err } offset := requiredAllocation + rc.manager.reindexSlack*pathLen - slackSum err = rc.manager.stageInterval(current, intervalIncreaseStart(previousInterval, offset)) if err != nil { return err } err = rc.propagateInterval(current) if err != nil { return err } err = rc.offsetSiblingsBefore(allocationNode, current, offset) if err != nil { return err } // Set the slack for each chain block to be reserved below during the chain walk-down pathSlackAlloc = rc.manager.reindexSlack break } slackBeforeCurrent, err := rc.manager.remainingSlackBefore(current) if err != nil { return err } slackSum += slackBeforeCurrent if slackSum >= requiredAllocation { previousInterval, err := rc.manager.interval(current) if err != nil { return err } // Set offset to be just enough to satisfy required allocation offset := slackBeforeCurrent - (slackSum - requiredAllocation) err = rc.manager.stageInterval(current, intervalIncreaseStart(previousInterval, offset)) if err != nil { return err } err = rc.offsetSiblingsBefore(allocationNode, current, offset) if err != nil { return err } break } current, err = rc.manager.FindNextAncestor(reindexRoot, current) if err != nil { return err } pathLen++ } // Go back down the reachability tree towards the common ancestor. // On every hop we reindex the reachability subtree before the // current node with an interval that is smaller. // This is to make room for the required allocation. for { current, err = rc.manager.parent(current) if err != nil { return err } if current.Equal(commonAncestor) { break } originalInterval, err := rc.manager.interval(current) if err != nil { return err } slackBeforeCurrent, err := rc.manager.remainingSlackBefore(current) if err != nil { return err } offset := slackBeforeCurrent - pathSlackAlloc err = rc.manager.stageInterval(current, intervalIncreaseStart(originalInterval, offset)) if err != nil { return err } err = rc.offsetSiblingsBefore(allocationNode, current, offset) if err != nil { return err } } return nil } func (rc *reindexContext) offsetSiblingsBefore(allocationNode, current *externalapi.DomainHash, offset uint64) error { parent, err := rc.manager.parent(current) if err != nil { return err } siblingsBefore, _, err := rc.manager.splitChildren(parent, current) if err != nil { return err } // Iterate over the slice in reverse order in order to break if reaching `allocationNode` for i := len(siblingsBefore) - 1; i >= 0; i-- { sibling := siblingsBefore[i] if sibling.Equal(allocationNode) { // We reached our final destination, allocate `offset` to `allocationNode` by increasing end and break previousInterval, err := rc.manager.interval(allocationNode) if err != nil { return err } err = rc.manager.stageInterval(allocationNode, intervalIncreaseEnd(previousInterval, offset)) if err != nil { return err } err = rc.propagateInterval(allocationNode) if err != nil { return err } break } previousInterval, err := rc.manager.interval(sibling) if err != nil { return err } err = rc.manager.stageInterval(sibling, intervalIncrease(previousInterval, offset)) if err != nil { return err } err = rc.propagateInterval(sibling) if err != nil { return err } } return nil } func (rc *reindexContext) reclaimIntervalAfter( allocationNode, commonAncestor, chosenChild, reindexRoot *externalapi.DomainHash, requiredAllocation uint64) error { var slackSum uint64 = 0 var pathLen uint64 = 0 var pathSlackAlloc uint64 = 0 var err error current := chosenChild // Walk up the chain from common ancestor's chosen child towards reindex root for { if current.Equal(reindexRoot) { // Reached reindex root. In this case, since we reached (the unlimited) root, // we also re-allocate new slack for the chain we just traversed previousInterval, err := rc.manager.interval(current) if err != nil { return err } offset := requiredAllocation + rc.manager.reindexSlack*pathLen - slackSum err = rc.manager.stageInterval(current, intervalDecreaseEnd(previousInterval, offset)) if err != nil { return err } err = rc.propagateInterval(current) if err != nil { return err } err = rc.offsetSiblingsAfter(allocationNode, current, offset) if err != nil { return err } // Set the slack for each chain block to be reserved below during the chain walk-down pathSlackAlloc = rc.manager.reindexSlack break } slackAfterCurrent, err := rc.manager.remainingSlackAfter(current) if err != nil { return err } slackSum += slackAfterCurrent if slackSum >= requiredAllocation { previousInterval, err := rc.manager.interval(current) if err != nil { return err } // Set offset to be just enough to satisfy required allocation offset := slackAfterCurrent - (slackSum - requiredAllocation) err = rc.manager.stageInterval(current, intervalDecreaseEnd(previousInterval, offset)) if err != nil { return err } err = rc.offsetSiblingsAfter(allocationNode, current, offset) if err != nil { return err } break } current, err = rc.manager.FindNextAncestor(reindexRoot, current) if err != nil { return err } pathLen++ } // Go back down the reachability tree towards the common ancestor. // On every hop we reindex the reachability subtree before the // current node with an interval that is smaller. // This is to make room for the required allocation. for { current, err = rc.manager.parent(current) if err != nil { return err } if current.Equal(commonAncestor) { break } originalInterval, err := rc.manager.interval(current) if err != nil { return err } slackAfterCurrent, err := rc.manager.remainingSlackAfter(current) if err != nil { return err } offset := slackAfterCurrent - pathSlackAlloc err = rc.manager.stageInterval(current, intervalDecreaseEnd(originalInterval, offset)) if err != nil { return err } err = rc.offsetSiblingsAfter(allocationNode, current, offset) if err != nil { return err } } return nil } func (rc *reindexContext) offsetSiblingsAfter(allocationNode, current *externalapi.DomainHash, offset uint64) error { parent, err := rc.manager.parent(current) if err != nil { return err } _, siblingsAfter, err := rc.manager.splitChildren(parent, current) if err != nil { return err } for _, sibling := range siblingsAfter { if sibling.Equal(allocationNode) { // We reached our final destination, allocate `offset` to `allocationNode` by decreasing start and break previousInterval, err := rc.manager.interval(allocationNode) if err != nil { return err } err = rc.manager.stageInterval(allocationNode, intervalDecreaseStart(previousInterval, offset)) if err != nil { return err } err = rc.propagateInterval(allocationNode) if err != nil { return err } break } previousInterval, err := rc.manager.interval(sibling) if err != nil { return err } err = rc.manager.stageInterval(sibling, intervalDecrease(previousInterval, offset)) if err != nil { return err } err = rc.propagateInterval(sibling) if err != nil { return err } } return nil } /* Functions for handling reindex triggered by moving reindex root */ func (rc *reindexContext) concentrateInterval(reindexRoot, chosenChild *externalapi.DomainHash, isFinalReindexRoot bool) error { siblingsBeforeChosen, siblingsAfterChosen, err := rc.manager.splitChildren(reindexRoot, chosenChild) if err != nil { return err } siblingsBeforeSizesSum, err := rc.tightenIntervalsBefore(reindexRoot, siblingsBeforeChosen) if err != nil { return err } siblingsAfterSizesSum, err := rc.tightenIntervalsAfter(reindexRoot, siblingsAfterChosen) if err != nil { return err } err = rc.expandIntervalToChosen( reindexRoot, chosenChild, siblingsBeforeSizesSum, siblingsAfterSizesSum, isFinalReindexRoot) if err != nil { return err } return nil } func (rc *reindexContext) tightenIntervalsBefore( reindexRoot *externalapi.DomainHash, siblingsBeforeChosen []*externalapi.DomainHash) (sizesSum uint64, err error) { siblingSubtreeSizes, sizesSum := rc.countChildrenSubtrees(siblingsBeforeChosen) rootInterval, err := rc.manager.interval(reindexRoot) if err != nil { return 0, err } intervalBeforeChosen := newReachabilityInterval( rootInterval.Start+rc.manager.reindexSlack, rootInterval.Start+rc.manager.reindexSlack+sizesSum-1, ) err = rc.propagateChildrenIntervals(intervalBeforeChosen, siblingsBeforeChosen, siblingSubtreeSizes) if err != nil { return 0, err } return sizesSum, nil } func (rc *reindexContext) tightenIntervalsAfter( reindexRoot *externalapi.DomainHash, siblingsAfterChosen []*externalapi.DomainHash) (sizesSum uint64, err error) { siblingSubtreeSizes, sizesSum := rc.countChildrenSubtrees(siblingsAfterChosen) rootInterval, err := rc.manager.interval(reindexRoot) if err != nil { return 0, err } intervalAfterChosen := newReachabilityInterval( rootInterval.End-rc.manager.reindexSlack-sizesSum, rootInterval.End-rc.manager.reindexSlack-1, ) err = rc.propagateChildrenIntervals(intervalAfterChosen, siblingsAfterChosen, siblingSubtreeSizes) if err != nil { return 0, err } return sizesSum, nil } func (rc *reindexContext) expandIntervalToChosen( reindexRoot, chosenChild *externalapi.DomainHash, sizesSumBefore, sizesSumAfter uint64, isFinalReindexRoot bool) error { rootInterval, err := rc.manager.interval(reindexRoot) if err != nil { return err } newChosenInterval := newReachabilityInterval( rootInterval.Start+sizesSumBefore+rc.manager.reindexSlack, rootInterval.End-sizesSumAfter-rc.manager.reindexSlack-1, ) currentChosenInterval, err := rc.manager.interval(chosenChild) if err != nil { return err } // Propagate interval only if chosenChild is the final reindex root if isFinalReindexRoot && !intervalContains(newChosenInterval, currentChosenInterval) { // New interval doesn't contain the previous one, propagation is required // We assign slack on both sides as an optimization. Were we to // assign a tight interval, the next time the reindex root moves we // would need to propagate intervals again. That is to say, when we // do allocate slack, next time // expandIntervalToChosen is called (next time the // reindex root moves), newChosenInterval is likely to // contain currentChosenInterval. err := rc.manager.stageInterval(chosenChild, newReachabilityInterval( newChosenInterval.Start+rc.manager.reindexSlack, newChosenInterval.End-rc.manager.reindexSlack, )) if err != nil { return err } err = rc.propagateInterval(chosenChild) if err != nil { return err } } err = rc.manager.stageInterval(chosenChild, newChosenInterval) if err != nil { return err } return nil } func (rc *reindexContext) countChildrenSubtrees(children []*externalapi.DomainHash) ( sizes []uint64, sum uint64) { sizes = make([]uint64, len(children)) sum = 0 for i, node := range children { err := rc.countSubtrees(node) if err != nil { return nil, 0 } subtreeSize := rc.subTreeSizesCache[*node] sizes[i] = subtreeSize sum += subtreeSize } return sizes, sum } func (rc *reindexContext) propagateChildrenIntervals( interval *model.ReachabilityInterval, children []*externalapi.DomainHash, sizes []uint64) error { childIntervals, err := intervalSplitExact(interval, sizes) if err != nil { return err } for i, child := range children { childInterval := childIntervals[i] err := rc.manager.stageInterval(child, childInterval) if err != nil { return err } err = rc.propagateInterval(child) if err != nil { return err } } return nil }