[NOD-1273] Order parents in PrepareBlockForTest (#872)

* [NOD-1271] Move version package to the top level

* [NOD-1271] Fix imports

addrmgr/internal_test.go

@@ -1,25 +0,0 @@
-// Copyright (c) 2013-2015 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package addrmgr
-
-import (
-	"time"
-
-	"github.com/kaspanet/kaspad/wire"
-)
-
-func TstKnownAddressIsBad(ka *KnownAddress) bool {
-	return ka.isBad()
-}
-
-func TstKnownAddressChance(ka *KnownAddress) float64 {
-	return ka.chance()
-}
-
-func TstNewKnownAddress(na *wire.NetAddress, attempts int,
-	lastattempt, lastsuccess time.Time, tried bool, refs int) *KnownAddress {
-	return &KnownAddress{na: na, attempts: attempts, lastattempt: lastattempt,
-		lastsuccess: lastsuccess, tried: tried, refs: refs}
-}

addrmgr/knownaddress_test.go

@@ -1,114 +0,0 @@
-// Copyright (c) 2013-2015 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package addrmgr_test
-
-import (
-	"math"
-	"testing"
-	"time"
-
-	"github.com/kaspanet/kaspad/addrmgr"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-func TestChance(t *testing.T) {
-	now := time.Unix(time.Now().Unix(), 0)
-	var tests = []struct {
-		addr     *addrmgr.KnownAddress
-		expected float64
-	}{
-		{
-			//Test normal case
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				0, time.Now().Add(-30*time.Minute), time.Now(), false, 0),
-			1.0,
-		}, {
-			//Test case in which lastseen < 0
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(20 * time.Second)},
-				0, time.Now().Add(-30*time.Minute), time.Now(), false, 0),
-			1.0,
-		}, {
-			//Test case in which lastattempt < 0
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				0, time.Now().Add(30*time.Minute), time.Now(), false, 0),
-			1.0 * .01,
-		}, {
-			//Test case in which lastattempt < ten minutes
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				0, time.Now().Add(-5*time.Minute), time.Now(), false, 0),
-			1.0 * .01,
-		}, {
-			//Test case with several failed attempts.
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				2, time.Now().Add(-30*time.Minute), time.Now(), false, 0),
-			1 / 1.5 / 1.5,
-		},
-	}
-
-	err := .0001
-	for i, test := range tests {
-		chance := addrmgr.TstKnownAddressChance(test.addr)
-		if math.Abs(test.expected-chance) >= err {
-			t.Errorf("case %d: got %f, expected %f", i, chance, test.expected)
-		}
-	}
-}
-
-func TestIsBad(t *testing.T) {
-	now := time.Unix(time.Now().Unix(), 0)
-	future := now.Add(35 * time.Minute)
-	monthOld := now.Add(-43 * time.Hour * 24)
-	secondsOld := now.Add(-2 * time.Second)
-	minutesOld := now.Add(-27 * time.Minute)
-	hoursOld := now.Add(-5 * time.Hour)
-	zeroTime := time.Time{}
-
-	futureNa := &wire.NetAddress{Timestamp: future}
-	minutesOldNa := &wire.NetAddress{Timestamp: minutesOld}
-	monthOldNa := &wire.NetAddress{Timestamp: monthOld}
-	currentNa := &wire.NetAddress{Timestamp: secondsOld}
-
-	//Test addresses that have been tried in the last minute.
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(futureNa, 3, secondsOld, zeroTime, false, 0)) {
-		t.Errorf("test case 1: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(monthOldNa, 3, secondsOld, zeroTime, false, 0)) {
-		t.Errorf("test case 2: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(currentNa, 3, secondsOld, zeroTime, false, 0)) {
-		t.Errorf("test case 3: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(currentNa, 3, secondsOld, monthOld, true, 0)) {
-		t.Errorf("test case 4: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(currentNa, 2, secondsOld, secondsOld, true, 0)) {
-		t.Errorf("test case 5: addresses that have been tried in the last minute are not bad.")
-	}
-
-	//Test address that claims to be from the future.
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(futureNa, 0, minutesOld, hoursOld, true, 0)) {
-		t.Errorf("test case 6: addresses that claim to be from the future are bad.")
-	}
-
-	//Test address that has not been seen in over a month.
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(monthOldNa, 0, minutesOld, hoursOld, true, 0)) {
-		t.Errorf("test case 7: addresses more than a month old are bad.")
-	}
-
-	//It has failed at least three times and never succeeded.
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(minutesOldNa, 3, minutesOld, zeroTime, true, 0)) {
-		t.Errorf("test case 8: addresses that have never succeeded are bad.")
-	}
-
-	//It has failed ten times in the last week
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(minutesOldNa, 10, minutesOld, monthOld, true, 0)) {
-		t.Errorf("test case 9: addresses that have not succeeded in too long are bad.")
-	}
-
-	//Test an address that should work.
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(minutesOldNa, 2, minutesOld, hoursOld, true, 0)) {
-		t.Errorf("test case 10: This should be a valid address.")
-	}
-}

app/app.go

@@ -0,0 +1,247 @@
+package app
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/kaspanet/kaspad/network/addressmanager"
+
+	"github.com/kaspanet/kaspad/network/netadapter/id"
+
+	"github.com/kaspanet/kaspad/domain/blockdag"
+	"github.com/kaspanet/kaspad/domain/blockdag/indexers"
+	"github.com/kaspanet/kaspad/domain/mempool"
+	"github.com/kaspanet/kaspad/domain/mining"
+	"github.com/kaspanet/kaspad/domain/txscript"
+	"github.com/kaspanet/kaspad/infrastructure/config"
+	"github.com/kaspanet/kaspad/infrastructure/dbaccess"
+	"github.com/kaspanet/kaspad/infrastructure/signal"
+	"github.com/kaspanet/kaspad/network/connmanager"
+	"github.com/kaspanet/kaspad/network/dnsseed"
+	"github.com/kaspanet/kaspad/network/domainmessage"
+	"github.com/kaspanet/kaspad/network/netadapter"
+	"github.com/kaspanet/kaspad/network/protocol"
+	"github.com/kaspanet/kaspad/network/rpc"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+// App is a wrapper for all the kaspad services
+type App struct {
+	cfg               *config.Config
+	rpcServer         *rpc.Server
+	addressManager    *addressmanager.AddressManager
+	protocolManager   *protocol.Manager
+	connectionManager *connmanager.ConnectionManager
+	netAdapter        *netadapter.NetAdapter
+
+	started, shutdown int32
+}
+
+// Start launches all the kaspad services.
+func (a *App) Start() {
+	// Already started?
+	if atomic.AddInt32(&a.started, 1) != 1 {
+		return
+	}
+
+	log.Trace("Starting kaspad")
+
+	err := a.protocolManager.Start()
+	if err != nil {
+		panics.Exit(log, fmt.Sprintf("Error starting the p2p protocol: %+v", err))
+	}
+
+	a.maybeSeedFromDNS()
+
+	a.connectionManager.Start()
+
+	if !a.cfg.DisableRPC {
+		a.rpcServer.Start()
+	}
+}
+
+// Stop gracefully shuts down all the kaspad services.
+func (a *App) Stop() {
+	// Make sure this only happens once.
+	if atomic.AddInt32(&a.shutdown, 1) != 1 {
+		log.Infof("Kaspad is already in the process of shutting down")
+		return
+	}
+
+	log.Warnf("Kaspad shutting down")
+
+	a.connectionManager.Stop()
+
+	err := a.protocolManager.Stop()
+	if err != nil {
+		log.Errorf("Error stopping the p2p protocol: %+v", err)
+	}
+
+	// Shutdown the RPC server if it's not disabled.
+	if !a.cfg.DisableRPC {
+		err := a.rpcServer.Stop()
+		if err != nil {
+			log.Errorf("Error stopping rpcServer: %+v", err)
+		}
+	}
+
+	err = a.addressManager.Stop()
+	if err != nil {
+		log.Errorf("Error stopping address manager: %s", err)
+	}
+
+	return
+}
+
+// New returns a new App instance configured to listen on addr for the
+// kaspa network type specified by dagParams. Use start to begin accepting
+// connections from peers.
+func New(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, interrupt <-chan struct{}) (*App, error) {
+	indexManager, acceptanceIndex := setupIndexes(cfg)
+
+	sigCache := txscript.NewSigCache(cfg.SigCacheMaxSize)
+
+	// Create a new block DAG instance with the appropriate configuration.
+	dag, err := setupDAG(cfg, databaseContext, interrupt, sigCache, indexManager)
+	if err != nil {
+		return nil, err
+	}
+
+	txMempool := setupMempool(cfg, dag, sigCache)
+
+	netAdapter, err := netadapter.NewNetAdapter(cfg)
+	if err != nil {
+		return nil, err
+	}
+	addressManager := addressmanager.New(cfg, databaseContext)
+
+	connectionManager, err := connmanager.New(cfg, netAdapter, addressManager)
+	if err != nil {
+		return nil, err
+	}
+
+	protocolManager, err := protocol.NewManager(cfg, dag, netAdapter, addressManager, txMempool, connectionManager)
+	if err != nil {
+		return nil, err
+	}
+	rpcServer, err := setupRPC(
+		cfg, dag, txMempool, sigCache, acceptanceIndex, connectionManager, addressManager, protocolManager)
+	if err != nil {
+		return nil, err
+	}
+
+	return &App{
+		cfg:               cfg,
+		rpcServer:         rpcServer,
+		protocolManager:   protocolManager,
+		connectionManager: connectionManager,
+		netAdapter:        netAdapter,
+		addressManager:    addressManager,
+	}, nil
+}
+
+func (a *App) maybeSeedFromDNS() {
+	if !a.cfg.DisableDNSSeed {
+		dnsseed.SeedFromDNS(a.cfg.NetParams(), a.cfg.DNSSeed, domainmessage.SFNodeNetwork, false, nil,
+			a.cfg.Lookup, func(addresses []*domainmessage.NetAddress) {
+				// Kaspad uses a lookup of the dns seeder here. Since seeder returns
+				// IPs of nodes and not its own IP, we can not know real IP of
+				// source. So we'll take first returned address as source.
+				a.addressManager.AddAddresses(addresses, addresses[0], nil)
+			})
+	}
+}
+func setupDAG(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, interrupt <-chan struct{},
+	sigCache *txscript.SigCache, indexManager blockdag.IndexManager) (*blockdag.BlockDAG, error) {
+
+	dag, err := blockdag.New(&blockdag.Config{
+		Interrupt:       interrupt,
+		DatabaseContext: databaseContext,
+		DAGParams:       cfg.NetParams(),
+		TimeSource:      blockdag.NewTimeSource(),
+		SigCache:        sigCache,
+		IndexManager:    indexManager,
+		SubnetworkID:    cfg.SubnetworkID,
+	})
+	return dag, err
+}
+
+func setupIndexes(cfg *config.Config) (blockdag.IndexManager, *indexers.AcceptanceIndex) {
+	// Create indexes if needed.
+	var indexes []indexers.Indexer
+	var acceptanceIndex *indexers.AcceptanceIndex
+	if cfg.AcceptanceIndex {
+		log.Info("acceptance index is enabled")
+		acceptanceIndex = indexers.NewAcceptanceIndex()
+		indexes = append(indexes, acceptanceIndex)
+	}
+
+	// Create an index manager if any of the optional indexes are enabled.
+	if len(indexes) < 0 {
+		return nil, nil
+	}
+	indexManager := indexers.NewManager(indexes)
+	return indexManager, acceptanceIndex
+}
+
+func setupMempool(cfg *config.Config, dag *blockdag.BlockDAG, sigCache *txscript.SigCache) *mempool.TxPool {
+	mempoolConfig := mempool.Config{
+		Policy: mempool.Policy{
+			AcceptNonStd:    cfg.RelayNonStd,
+			MaxOrphanTxs:    cfg.MaxOrphanTxs,
+			MaxOrphanTxSize: config.DefaultMaxOrphanTxSize,
+			MinRelayTxFee:   cfg.MinRelayTxFee,
+			MaxTxVersion:    1,
+		},
+		CalcSequenceLockNoLock: func(tx *util.Tx, utxoSet blockdag.UTXOSet) (*blockdag.SequenceLock, error) {
+			return dag.CalcSequenceLockNoLock(tx, utxoSet)
+		},
+		SigCache: sigCache,
+		DAG:      dag,
+	}
+
+	return mempool.New(&mempoolConfig)
+}
+
+func setupRPC(cfg *config.Config,
+	dag *blockdag.BlockDAG,
+	txMempool *mempool.TxPool,
+	sigCache *txscript.SigCache,
+	acceptanceIndex *indexers.AcceptanceIndex,
+	connectionManager *connmanager.ConnectionManager,
+	addressManager *addressmanager.AddressManager,
+	protocolManager *protocol.Manager) (*rpc.Server, error) {
+
+	if !cfg.DisableRPC {
+		policy := mining.Policy{
+			BlockMaxMass: cfg.BlockMaxMass,
+		}
+		blockTemplateGenerator := mining.NewBlkTmplGenerator(&policy, txMempool, dag, sigCache)
+
+		rpcServer, err := rpc.NewRPCServer(cfg, dag, txMempool, acceptanceIndex, blockTemplateGenerator,
+			connectionManager, addressManager, protocolManager)
+		if err != nil {
+			return nil, err
+		}
+
+		// Signal process shutdown when the RPC server requests it.
+		spawn("setupRPC-handleShutdownRequest", func() {
+			<-rpcServer.RequestedProcessShutdown()
+			signal.ShutdownRequestChannel <- struct{}{}
+		})
+
+		return rpcServer, nil
+	}
+	return nil, nil
+}
+
+// P2PNodeID returns the network ID associated with this App
+func (a *App) P2PNodeID() *id.ID {
+	return a.netAdapter.ID()
+}
+
+// AddressManager returns the AddressManager associated with this App
+func (a *App) AddressManager() *addressmanager.AddressManager {
+	return a.addressManager
+}

app/log.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Copyright (c) 2017 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package app
+
+import (
+	"github.com/kaspanet/kaspad/infrastructure/logger"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+var log, _ = logger.Get(logger.SubsystemTags.KASD)
+var spawn = panics.GoroutineWrapperFunc(log)

blockdag/accept.go

@@ -1,132 +0,0 @@
-// Copyright (c) 2013-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"fmt"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/pkg/errors"
-)
-
-func (dag *BlockDAG) addNodeToIndexWithInvalidAncestor(block *util.Block) error {
-	blockHeader := &block.MsgBlock().Header
-	newNode, _ := dag.newBlockNode(blockHeader, newBlockSet())
-	newNode.status = statusInvalidAncestor
-	dag.index.AddNode(newNode)
-	return dag.index.flushToDB()
-}
-
-// maybeAcceptBlock potentially accepts a block into the block DAG. It
-// performs several validation checks which depend on its position within
-// the block DAG before adding it. The block is expected to have already
-// gone through ProcessBlock before calling this function with it.
-//
-// The flags are also passed to checkBlockContext and connectToDAG. See
-// their documentation for how the flags modify their behavior.
-//
-// This function MUST be called with the dagLock held (for writes).
-func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) error {
-	parents, err := lookupParentNodes(block, dag)
-	if err != nil {
-		var ruleErr RuleError
-		if ok := errors.As(err, &ruleErr); ok && ruleErr.ErrorCode == ErrInvalidAncestorBlock {
-			err := dag.addNodeToIndexWithInvalidAncestor(block)
-			if err != nil {
-				return err
-			}
-		}
-		return err
-	}
-
-	// The block must pass all of the validation rules which depend on the
-	// position of the block within the block DAG.
-	err = dag.checkBlockContext(block, parents, flags)
-	if err != nil {
-		return err
-	}
-
-	// Create a new block node for the block and add it to the node index.
-	newNode, selectedParentAnticone := dag.newBlockNode(&block.MsgBlock().Header, parents)
-	newNode.status = statusDataStored
-	dag.index.AddNode(newNode)
-
-	// Insert the block into the database if it's not already there. Even
-	// though it is possible the block will ultimately fail to connect, it
-	// has already passed all proof-of-work and validity tests which means
-	// it would be prohibitively expensive for an attacker to fill up the
-	// disk with a bunch of blocks that fail to connect. This is necessary
-	// since it allows block download to be decoupled from the much more
-	// expensive connection logic. It also has some other nice properties
-	// such as making blocks that never become part of the DAG or
-	// blocks that fail to connect available for further analysis.
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		err := dbStoreBlock(dbTx, block)
-		if err != nil {
-			return err
-		}
-		return dag.index.flushToDBWithTx(dbTx)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Make sure that all the block's transactions are finalized
-	fastAdd := flags&BFFastAdd == BFFastAdd
-	bluestParent := parents.bluest()
-	if !fastAdd {
-		if err := dag.validateAllTxsFinalized(block, newNode, bluestParent); err != nil {
-			return err
-		}
-	}
-
-	block.SetBlueScore(newNode.blueScore)
-
-	// Connect the passed block to the DAG. This also handles validation of the
-	// transaction scripts.
-	chainUpdates, err := dag.addBlock(newNode, block, selectedParentAnticone, flags)
-	if err != nil {
-		return err
-	}
-
-	// Notify the caller that the new block was accepted into the block
-	// DAG. The caller would typically want to react by relaying the
-	// inventory to other peers.
-	dag.dagLock.Unlock()
-	dag.sendNotification(NTBlockAdded, &BlockAddedNotificationData{
-		Block:         block,
-		WasUnorphaned: flags&BFWasUnorphaned != 0,
-	})
-	if len(chainUpdates.addedChainBlockHashes) > 0 {
-		dag.sendNotification(NTChainChanged, &ChainChangedNotificationData{
-			RemovedChainBlockHashes: chainUpdates.removedChainBlockHashes,
-			AddedChainBlockHashes:   chainUpdates.addedChainBlockHashes,
-		})
-	}
-	dag.dagLock.Lock()
-
-	return nil
-}
-
-func lookupParentNodes(block *util.Block, blockDAG *BlockDAG) (blockSet, error) {
-	header := block.MsgBlock().Header
-	parentHashes := header.ParentHashes
-
-	nodes := newBlockSet()
-	for _, parentHash := range parentHashes {
-		node := blockDAG.index.LookupNode(parentHash)
-		if node == nil {
-			str := fmt.Sprintf("parent block %s is unknown", parentHash)
-			return nil, ruleError(ErrParentBlockUnknown, str)
-		} else if blockDAG.index.NodeStatus(node).KnownInvalid() {
-			str := fmt.Sprintf("parent block %s is known to be invalid", parentHash)
-			return nil, ruleError(ErrInvalidAncestorBlock, str)
-		}
-
-		nodes.add(node)
-	}
-
-	return nodes, nil
-}

blockdag/accept_test.go

@@ -1,104 +0,0 @@
-package blockdag
-
-import (
-	"errors"
-	"path/filepath"
-	"testing"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-)
-
-func TestMaybeAcceptBlockErrors(t *testing.T) {
-	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestMaybeAcceptBlockErrors", Config{
-		DAGParams: &dagconfig.SimnetParams,
-	})
-	if err != nil {
-		t.Fatalf("TestMaybeAcceptBlockErrors: Failed to setup DAG instance: %v", err)
-	}
-	defer teardownFunc()
-
-	dag.TestSetCoinbaseMaturity(0)
-
-	// Test rejecting the block if its parents are missing
-	orphanBlockFile := "blk_3B.dat"
-	loadedBlocks, err := LoadBlocks(filepath.Join("testdata/", orphanBlockFile))
-	if err != nil {
-		t.Fatalf("TestMaybeAcceptBlockErrors: "+
-			"Error loading file '%s': %s\n", orphanBlockFile, err)
-	}
-	block := loadedBlocks[0]
-
-	err = dag.maybeAcceptBlock(block, BFNone)
-	if err == nil {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
-			"Expected: %s, got: <nil>", ErrParentBlockUnknown)
-	}
-	var ruleErr RuleError
-	if ok := errors.As(err, &ruleErr); !ok {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
-			"Expected RuleError but got %s", err)
-	} else if ruleErr.ErrorCode != ErrParentBlockUnknown {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
-			"Unexpected error code. Want: %s, got: %s", ErrParentBlockUnknown, ruleErr.ErrorCode)
-	}
-
-	// Test rejecting the block if its parents are invalid
-	blocksFile := "blk_0_to_4.dat"
-	blocks, err := LoadBlocks(filepath.Join("testdata/", blocksFile))
-	if err != nil {
-		t.Fatalf("TestMaybeAcceptBlockErrors: "+
-			"Error loading file '%s': %s\n", blocksFile, err)
-	}
-
-	// Add a valid block and mark it as invalid
-	block1 := blocks[1]
-	isOrphan, isDelayed, err := dag.ProcessBlock(block1, BFNone)
-	if err != nil {
-		t.Fatalf("TestMaybeAcceptBlockErrors: Valid block unexpectedly returned an error: %s", err)
-	}
-	if isDelayed {
-		t.Fatalf("TestMaybeAcceptBlockErrors: block 1 is too far in the future")
-	}
-	if isOrphan {
-		t.Fatalf("TestMaybeAcceptBlockErrors: incorrectly returned block 1 is an orphan")
-	}
-	blockNode1 := dag.index.LookupNode(block1.Hash())
-	dag.index.SetStatusFlags(blockNode1, statusValidateFailed)
-
-	block2 := blocks[2]
-	err = dag.maybeAcceptBlock(block2, BFNone)
-	if err == nil {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
-			"Expected: %s, got: <nil>", ErrInvalidAncestorBlock)
-	}
-	if ok := errors.As(err, &ruleErr); !ok {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
-			"Expected RuleError but got %s", err)
-	} else if ruleErr.ErrorCode != ErrInvalidAncestorBlock {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
-			"Unexpected error. Want: %s, got: %s", ErrInvalidAncestorBlock, ruleErr.ErrorCode)
-	}
-
-	// Set block1's status back to valid for next tests
-	dag.index.UnsetStatusFlags(blockNode1, statusValidateFailed)
-
-	// Test rejecting the block due to bad context
-	originalBits := block2.MsgBlock().Header.Bits
-	block2.MsgBlock().Header.Bits = 0
-	err = dag.maybeAcceptBlock(block2, BFNone)
-	if err == nil {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
-			"Expected: %s, got: <nil>", ErrUnexpectedDifficulty)
-	}
-	if ok := errors.As(err, &ruleErr); !ok {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
-			"Expected RuleError but got %s", err)
-	} else if ruleErr.ErrorCode != ErrUnexpectedDifficulty {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
-			"Unexpected error. Want: %s, got: %s", ErrUnexpectedDifficulty, ruleErr.ErrorCode)
-	}
-
-	// Set block2's bits back to valid for next tests
-	block2.MsgBlock().Header.Bits = originalBits
-}

blockdag/blockidhash.go

@@ -1,136 +0,0 @@
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/pkg/errors"
-)
-
-var (
-	// idByHashIndexBucketName is the name of the db bucket used to house
-	// the block hash -> block id index.
-	idByHashIndexBucketName = []byte("idbyhashidx")
-
-	// hashByIDIndexBucketName is the name of the db bucket used to house
-	// the block id -> block hash index.
-	hashByIDIndexBucketName = []byte("hashbyididx")
-
-	currentBlockIDKey = []byte("currentblockid")
-)
-
-// -----------------------------------------------------------------------------
-// This is a mapping between block hashes and unique IDs. The ID
-// is simply a sequentially incremented uint64 that is used instead of block hash
-// for the indexers. This is useful because it is only 8 bytes versus 32 bytes
-// hashes and thus saves a ton of space when a block is referenced in an index.
-// It consists of three buckets: the first bucket maps the hash of each
-// block to the unique ID and the second maps that ID back to the block hash.
-// The third bucket contains the last received block ID, and is used
-// when starting the node to check that the enabled indexes are up to date
-// with the latest received block, and if not, initiate recovery process.
-//
-// The serialized format for keys and values in the block hash to ID bucket is:
-//   <hash> = <ID>
-//
-//   Field           Type              Size
-//   hash            daghash.Hash     32 bytes
-//   ID              uint64            8 bytes
-//   -----
-//   Total: 40 bytes
-//
-// The serialized format for keys and values in the ID to block hash bucket is:
-//   <ID> = <hash>
-//
-//   Field           Type              Size
-//   ID              uint64            8 bytes
-//   hash            daghash.Hash     32 bytes
-//   -----
-//   Total: 40 bytes
-//
-// -----------------------------------------------------------------------------
-
-const blockIDSize = 8 // 8 bytes for block ID
-
-// DBFetchBlockIDByHash uses an existing database transaction to retrieve the
-// block id for the provided hash from the index.
-func DBFetchBlockIDByHash(dbTx database.Tx, hash *daghash.Hash) (uint64, error) {
-	hashIndex := dbTx.Metadata().Bucket(idByHashIndexBucketName)
-	serializedID := hashIndex.Get(hash[:])
-	if serializedID == nil {
-		return 0, errors.Errorf("no entry in the block ID index for block with hash %s", hash)
-	}
-
-	return DeserializeBlockID(serializedID), nil
-}
-
-// DBFetchBlockHashBySerializedID uses an existing database transaction to
-// retrieve the hash for the provided serialized block id from the index.
-func DBFetchBlockHashBySerializedID(dbTx database.Tx, serializedID []byte) (*daghash.Hash, error) {
-	idIndex := dbTx.Metadata().Bucket(hashByIDIndexBucketName)
-	hashBytes := idIndex.Get(serializedID)
-	if hashBytes == nil {
-		return nil, errors.Errorf("no entry in the block ID index for block with id %d", byteOrder.Uint64(serializedID))
-	}
-
-	var hash daghash.Hash
-	copy(hash[:], hashBytes)
-	return &hash, nil
-}
-
-// dbPutBlockIDIndexEntry uses an existing database transaction to update or add
-// the index entries for the hash to id and id to hash mappings for the provided
-// values.
-func dbPutBlockIDIndexEntry(dbTx database.Tx, hash *daghash.Hash, serializedID []byte) error {
-	// Add the block hash to ID mapping to the index.
-	meta := dbTx.Metadata()
-	hashIndex := meta.Bucket(idByHashIndexBucketName)
-	if err := hashIndex.Put(hash[:], serializedID[:]); err != nil {
-		return err
-	}
-
-	// Add the block ID to hash mapping to the index.
-	idIndex := meta.Bucket(hashByIDIndexBucketName)
-	return idIndex.Put(serializedID[:], hash[:])
-}
-
-// DBFetchCurrentBlockID returns the last known block ID.
-func DBFetchCurrentBlockID(dbTx database.Tx) uint64 {
-	serializedID := dbTx.Metadata().Get(currentBlockIDKey)
-	if serializedID == nil {
-		return 0
-	}
-	return DeserializeBlockID(serializedID)
-}
-
-// DeserializeBlockID returns a deserialized block id
-func DeserializeBlockID(serializedID []byte) uint64 {
-	return byteOrder.Uint64(serializedID)
-}
-
-// SerializeBlockID returns a serialized block id
-func SerializeBlockID(blockID uint64) []byte {
-	serializedBlockID := make([]byte, blockIDSize)
-	byteOrder.PutUint64(serializedBlockID, blockID)
-	return serializedBlockID
-}
-
-// DBFetchBlockHashByID uses an existing database transaction to retrieve the
-// hash for the provided block id from the index.
-func DBFetchBlockHashByID(dbTx database.Tx, id uint64) (*daghash.Hash, error) {
-	return DBFetchBlockHashBySerializedID(dbTx, SerializeBlockID(id))
-}
-
-func createBlockID(dbTx database.Tx, blockHash *daghash.Hash) (uint64, error) {
-	currentBlockID := DBFetchCurrentBlockID(dbTx)
-	newBlockID := currentBlockID + 1
-	serializedNewBlockID := SerializeBlockID(newBlockID)
-	err := dbTx.Metadata().Put(currentBlockIDKey, serializedNewBlockID)
-	if err != nil {
-		return 0, err
-	}
-	err = dbPutBlockIDIndexEntry(dbTx, blockHash, serializedNewBlockID)
-	if err != nil {
-		return 0, err
-	}
-	return newBlockID, nil
-}

blockdag/blocklocator.go

@@ -1,102 +0,0 @@
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/pkg/errors"
-)
-
-// BlockLocator is used to help locate a specific block. The algorithm for
-// building the block locator is to add block hashes in reverse order on the
-// block's selected parent chain until the desired stop block is reached.
-// In order to keep the list of locator hashes to a reasonable number of entries,
-// the step between each entry is doubled each loop iteration to exponentially
-// decrease the number of hashes as a function of the distance from the block
-// being located.
-//
-// For example, assume a selected parent chain with IDs as depicted below, and the
-// stop block is genesis:
-// 	genesis -> 1 -> 2 -> ... -> 15 -> 16  -> 17  -> 18
-//
-// The block locator for block 17 would be the hashes of blocks:
-//  [17 16 14 11 7 2 genesis]
-type BlockLocator []*daghash.Hash
-
-// BlockLocatorFromHashes returns a block locator from high and low hash.
-// See BlockLocator for details on the algorithm used to create a block locator.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) BlockLocatorFromHashes(highHash, lowHash *daghash.Hash) (BlockLocator, error) {
-	dag.dagLock.RLock()
-	defer dag.dagLock.RUnlock()
-
-	highNode := dag.index.LookupNode(highHash)
-	lowNode := dag.index.LookupNode(lowHash)
-
-	return dag.blockLocator(highNode, lowNode)
-}
-
-// blockLocator returns a block locator for the passed high and low nodes.
-// See the BlockLocator type comments for more details.
-//
-// This function MUST be called with the DAG state lock held (for reads).
-func (dag *BlockDAG) blockLocator(highNode, lowNode *blockNode) (BlockLocator, error) {
-	// We use the selected parent of the high node, so the
-	// block locator won't contain the high node.
-	highNode = highNode.selectedParent
-
-	node := highNode
-	step := uint64(1)
-	locator := make(BlockLocator, 0)
-	for node != nil {
-		locator = append(locator, node.hash)
-
-		// Nothing more to add once the low node has been added.
-		if node.blueScore <= lowNode.blueScore {
-			if node != lowNode {
-				return nil, errors.Errorf("highNode and lowNode are " +
-					"not in the same selected parent chain.")
-			}
-			break
-		}
-
-		// Calculate blueScore of previous node to include ensuring the
-		// final node is lowNode.
-		nextBlueScore := node.blueScore - step
-		if nextBlueScore < lowNode.blueScore {
-			nextBlueScore = lowNode.blueScore
-		}
-
-		// walk backwards through the nodes to the correct ancestor.
-		node = node.SelectedAncestor(nextBlueScore)
-
-		// Double the distance between included hashes.
-		step *= 2
-	}
-
-	return locator, nil
-}
-
-// FindNextLocatorBoundaries returns the lowest unknown block locator, hash
-// and the highest known block locator hash. This is used to create the
-// next block locator to find the highest shared known chain block with the
-// sync peer.
-//
-// This function MUST be called with the DAG state lock held (for reads).
-func (dag *BlockDAG) FindNextLocatorBoundaries(locator BlockLocator) (highHash, lowHash *daghash.Hash) {
-	// Find the most recent locator block hash in the DAG. In the case none of
-	// the hashes in the locator are in the DAG, fall back to the genesis block.
-	lowNode := dag.genesis
-	nextBlockLocatorIndex := int64(len(locator) - 1)
-	for i, hash := range locator {
-		node := dag.index.LookupNode(hash)
-		if node != nil {
-			lowNode = node
-			nextBlockLocatorIndex = int64(i) - 1
-			break
-		}
-	}
-	if nextBlockLocatorIndex < 0 {
-		return nil, lowNode.hash
-	}
-	return locator[nextBlockLocatorIndex], lowNode.hash
-}

blockdag/dagio.go

@@ -1,856 +0,0 @@
-// Copyright (c) 2015-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/util/buffers"
-	"github.com/pkg/errors"
-	"io"
-	"sync"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/binaryserializer"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/util/subnetworkid"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-const (
-	// blockHdrSize is the size of a block header. This is simply the
-	// constant from wire and is only provided here for convenience since
-	// wire.MaxBlockHeaderPayload is quite long.
-	blockHdrSize = wire.MaxBlockHeaderPayload
-
-	// latestUTXOSetBucketVersion is the current version of the UTXO set
-	// bucket that is used to track all unspent outputs.
-	latestUTXOSetBucketVersion = 1
-)
-
-var (
-	// blockIndexBucketName is the name of the database bucket used to house the
-	// block headers and contextual information.
-	blockIndexBucketName = []byte("blockheaderidx")
-
-	// dagStateKeyName is the name of the db key used to store the DAG
-	// tip hashes.
-	dagStateKeyName = []byte("dagstate")
-
-	// utxoSetVersionKeyName is the name of the db key used to store the
-	// version of the utxo set currently in the database.
-	utxoSetVersionKeyName = []byte("utxosetversion")
-
-	// utxoSetBucketName is the name of the database bucket used to house the
-	// unspent transaction output set.
-	utxoSetBucketName = []byte("utxoset")
-
-	// utxoDiffsBucketName is the name of the database bucket used to house the
-	// diffs and diff children of blocks.
-	utxoDiffsBucketName = []byte("utxodiffs")
-
-	// reachabilityDataBucketName is the name of the database bucket used to house the
-	// reachability tree nodes and future covering sets of blocks.
-	reachabilityDataBucketName = []byte("reachability")
-
-	// multisetBucketName is the name of the database bucket used to house the
-	// ECMH multisets of blocks.
-	multisetBucketName = []byte("multiset")
-
-	// subnetworksBucketName is the name of the database bucket used to store the
-	// subnetwork registry.
-	subnetworksBucketName = []byte("subnetworks")
-
-	// localSubnetworkKeyName is the name of the db key used to store the
-	// node's local subnetwork ID.
-	localSubnetworkKeyName = []byte("localsubnetworkidkey")
-
-	// byteOrder is the preferred byte order used for serializing numeric
-	// fields for storage in the database.
-	byteOrder = binary.LittleEndian
-)
-
-// errNotInDAG signifies that a block hash or height that is not in the
-// DAG was requested.
-type errNotInDAG string
-
-// Error implements the error interface.
-func (e errNotInDAG) Error() string {
-	return string(e)
-}
-
-// isNotInDAGErr returns whether or not the passed error is an
-// errNotInDAG error.
-func isNotInDAGErr(err error) bool {
-	var notInDAGErr errNotInDAG
-	return errors.As(err, &notInDAGErr)
-}
-
-// dbPutVersion uses an existing database transaction to update the provided
-// key in the metadata bucket to the given version. It is primarily used to
-// track versions on entities such as buckets.
-func dbPutVersion(dbTx database.Tx, key []byte, version uint32) error {
-	var serialized [4]byte
-	byteOrder.PutUint32(serialized[:], version)
-	return dbTx.Metadata().Put(key, serialized[:])
-}
-
-// outpointKeyPool defines a concurrent safe free list of byte buffers used to
-// provide temporary buffers for outpoint database keys.
-var outpointKeyPool = sync.Pool{
-	New: func() interface{} {
-		return &bytes.Buffer{} // Pointer to a buffer to avoid boxing alloc.
-	},
-}
-
-// outpointIndexByteOrder is the byte order for serializing the outpoint index.
-// It uses big endian to ensure that when outpoint is used as database key, the
-// keys will be iterated in an ascending order by the outpoint index.
-var outpointIndexByteOrder = binary.BigEndian
-
-func serializeOutpoint(w io.Writer, outpoint *wire.Outpoint) error {
-	_, err := w.Write(outpoint.TxID[:])
-	if err != nil {
-		return err
-	}
-
-	return binaryserializer.PutUint32(w, outpointIndexByteOrder, outpoint.Index)
-}
-
-var outpointSerializeSize = daghash.TxIDSize + 4
-
-// deserializeOutpoint decodes an outpoint from the passed serialized byte
-// slice into a new wire.Outpoint using a format that is suitable for long-
-// term storage. This format is described in detail above.
-func deserializeOutpoint(r io.Reader) (*wire.Outpoint, error) {
-	outpoint := &wire.Outpoint{}
-	_, err := r.Read(outpoint.TxID[:])
-	if err != nil {
-		return nil, err
-	}
-
-	outpoint.Index, err = binaryserializer.Uint32(r, outpointIndexByteOrder)
-	if err != nil {
-		return nil, err
-	}
-
-	return outpoint, nil
-}
-
-// dbPutUTXODiff uses an existing database transaction to update the UTXO set
-// in the database based on the provided UTXO view contents and state. In
-// particular, only the entries that have been marked as modified are written
-// to the database.
-func dbPutUTXODiff(dbTx database.Tx, diff *UTXODiff) error {
-	utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName)
-	for outpoint := range diff.toRemove {
-		w := outpointKeyPool.Get().(*bytes.Buffer)
-		w.Reset()
-		err := serializeOutpoint(w, &outpoint)
-		if err != nil {
-			return err
-		}
-
-		key := w.Bytes()
-		err = utxoBucket.Delete(key)
-		if err != nil {
-			return err
-		}
-		outpointKeyPool.Put(w)
-	}
-
-	// We are preallocating for P2PKH entries because they are the most common ones.
-	// If we have entries with a compressed script bigger than P2PKH's, the buffer will grow.
-	bytesToPreallocate := (p2pkhUTXOEntrySerializeSize + outpointSerializeSize) * len(diff.toAdd)
-	buff := bytes.NewBuffer(make([]byte, bytesToPreallocate))
-	for outpoint, entry := range diff.toAdd {
-		// Serialize and store the UTXO entry.
-		sBuff := buffers.NewSubBuffer(buff)
-		err := serializeUTXOEntry(sBuff, entry)
-		if err != nil {
-			return err
-		}
-		serializedEntry := sBuff.Bytes()
-
-		sBuff = buffers.NewSubBuffer(buff)
-		err = serializeOutpoint(sBuff, &outpoint)
-		if err != nil {
-			return err
-		}
-
-		key := sBuff.Bytes()
-		err = utxoBucket.Put(key, serializedEntry)
-		// NOTE: The key is intentionally not recycled here since the
-		// database interface contract prohibits modifications. It will
-		// be garbage collected normally when the database is done with
-		// it.
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-type dagState struct {
-	TipHashes         []*daghash.Hash
-	LastFinalityPoint *daghash.Hash
-}
-
-// serializeDAGState returns the serialization of the DAG state.
-// This is data to be stored in the DAG state bucket.
-func serializeDAGState(state *dagState) ([]byte, error) {
-	return json.Marshal(state)
-}
-
-// deserializeDAGState deserializes the passed serialized DAG state.
-// This is data stored in the DAG state bucket and is updated after
-// every block is connected to the DAG.
-func deserializeDAGState(serializedData []byte) (*dagState, error) {
-	var state *dagState
-	err := json.Unmarshal(serializedData, &state)
-	if err != nil {
-		return nil, database.Error{
-			ErrorCode:   database.ErrCorruption,
-			Description: "corrupt DAG state",
-		}
-	}
-
-	return state, nil
-}
-
-// dbPutDAGState uses an existing database transaction to store the latest
-// tip hashes of the DAG.
-func dbPutDAGState(dbTx database.Tx, state *dagState) error {
-	serializedData, err := serializeDAGState(state)
-
-	if err != nil {
-		return err
-	}
-
-	return dbTx.Metadata().Put(dagStateKeyName, serializedData)
-}
-
-// createDAGState initializes both the database and the DAG state to the
-// genesis block. This includes creating the necessary buckets, so it
-// must only be called on an uninitialized database.
-func (dag *BlockDAG) createDAGState() error {
-	// Create the initial the database DAG state including creating the
-	// necessary index buckets and inserting the genesis block.
-	err := dag.db.Update(func(dbTx database.Tx) error {
-		meta := dbTx.Metadata()
-
-		// Create the bucket that houses the block index data.
-		_, err := meta.CreateBucket(blockIndexBucketName)
-		if err != nil {
-			return err
-		}
-
-		// Create the buckets that house the utxo set, the utxo diffs, and their
-		// version.
-		_, err = meta.CreateBucket(utxoSetBucketName)
-		if err != nil {
-			return err
-		}
-
-		_, err = meta.CreateBucket(utxoDiffsBucketName)
-		if err != nil {
-			return err
-		}
-
-		_, err = meta.CreateBucket(reachabilityDataBucketName)
-		if err != nil {
-			return err
-		}
-
-		_, err = meta.CreateBucket(multisetBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = dbPutVersion(dbTx, utxoSetVersionKeyName,
-			latestUTXOSetBucketVersion)
-		if err != nil {
-			return err
-		}
-
-		// Create the bucket that houses the registered subnetworks.
-		_, err = meta.CreateBucket(subnetworksBucketName)
-		if err != nil {
-			return err
-		}
-
-		if err := dbPutLocalSubnetworkID(dbTx, dag.subnetworkID); err != nil {
-			return err
-		}
-
-		if _, err := meta.CreateBucketIfNotExists(idByHashIndexBucketName); err != nil {
-			return err
-		}
-		if _, err := meta.CreateBucketIfNotExists(hashByIDIndexBucketName); err != nil {
-			return err
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (dag *BlockDAG) removeDAGState() error {
-	err := dag.db.Update(func(dbTx database.Tx) error {
-		meta := dbTx.Metadata()
-
-		err := meta.DeleteBucket(blockIndexBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = meta.DeleteBucket(utxoSetBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = meta.DeleteBucket(utxoDiffsBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = meta.DeleteBucket(reachabilityDataBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = meta.DeleteBucket(multisetBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = dbTx.Metadata().Delete(utxoSetVersionKeyName)
-		if err != nil {
-			return err
-		}
-
-		err = meta.DeleteBucket(subnetworksBucketName)
-		if err != nil {
-			return err
-		}
-
-		err = dbTx.Metadata().Delete(localSubnetworkKeyName)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func dbPutLocalSubnetworkID(dbTx database.Tx, subnetworkID *subnetworkid.SubnetworkID) error {
-	if subnetworkID == nil {
-		return dbTx.Metadata().Put(localSubnetworkKeyName, []byte{})
-	}
-	return dbTx.Metadata().Put(localSubnetworkKeyName, subnetworkID[:])
-}
-
-// initDAGState attempts to load and initialize the DAG state from the
-// database. When the db does not yet contain any DAG state, both it and the
-// DAG state are initialized to the genesis block.
-func (dag *BlockDAG) initDAGState() error {
-	// Determine the state of the DAG database. We may need to initialize
-	// everything from scratch or upgrade certain buckets.
-	var initialized bool
-	err := dag.db.View(func(dbTx database.Tx) error {
-		initialized = dbTx.Metadata().Get(dagStateKeyName) != nil
-		if initialized {
-			var localSubnetworkID *subnetworkid.SubnetworkID
-			localSubnetworkIDBytes := dbTx.Metadata().Get(localSubnetworkKeyName)
-			if len(localSubnetworkIDBytes) != 0 {
-				localSubnetworkID = &subnetworkid.SubnetworkID{}
-				localSubnetworkID.SetBytes(localSubnetworkIDBytes)
-			}
-			if !localSubnetworkID.IsEqual(dag.subnetworkID) {
-				return errors.Errorf("Cannot start kaspad with subnetwork ID %s because"+
-					" its database is already built with subnetwork ID %s. If you"+
-					" want to switch to a new database, please reset the"+
-					" database by starting kaspad with --reset-db flag", dag.subnetworkID, localSubnetworkID)
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
-	if !initialized {
-		// At this point the database has not already been initialized, so
-		// initialize both it and the DAG state to the genesis block.
-		return dag.createDAGState()
-	}
-
-	// Attempt to load the DAG state from the database.
-	return dag.db.View(func(dbTx database.Tx) error {
-		// Fetch the stored DAG tipHashes from the database metadata.
-		// When it doesn't exist, it means the database hasn't been
-		// initialized for use with the DAG yet, so break out now to allow
-		// that to happen under a writable database transaction.
-		serializedData := dbTx.Metadata().Get(dagStateKeyName)
-		log.Tracef("Serialized DAG tip hashes: %x", serializedData)
-		state, err := deserializeDAGState(serializedData)
-		if err != nil {
-			return err
-		}
-
-		// Load all of the headers from the data for the known DAG
-		// and construct the block index accordingly. Since the
-		// number of nodes are already known, perform a single alloc
-		// for them versus a whole bunch of little ones to reduce
-		// pressure on the GC.
-		log.Infof("Loading block index...")
-
-		blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName)
-
-		var unprocessedBlockNodes []*blockNode
-		cursor := blockIndexBucket.Cursor()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			node, err := dag.deserializeBlockNode(cursor.Value())
-			if err != nil {
-				return err
-			}
-
-			// Check to see if this node had been stored in the the block DB
-			// but not yet accepted. If so, add it to a slice to be processed later.
-			if node.status == statusDataStored {
-				unprocessedBlockNodes = append(unprocessedBlockNodes, node)
-				continue
-			}
-
-			// If the node is known to be invalid add it as-is to the block
-			// index and continue.
-			if node.status.KnownInvalid() {
-				dag.index.addNode(node)
-				continue
-			}
-
-			if dag.blockCount == 0 {
-				if !node.hash.IsEqual(dag.dagParams.GenesisHash) {
-					return AssertError(fmt.Sprintf("initDAGState: Expected "+
-						"first entry in block index to be genesis block, "+
-						"found %s", node.hash))
-				}
-			} else {
-				if len(node.parents) == 0 {
-					return AssertError(fmt.Sprintf("initDAGState: Could "+
-						"not find any parent for block %s", node.hash))
-				}
-			}
-
-			// Add the node to its parents children, connect it,
-			// and add it to the block index.
-			node.updateParentsChildren()
-			dag.index.addNode(node)
-
-			dag.blockCount++
-		}
-
-		// Load all of the known UTXO entries and construct the full
-		// UTXO set accordingly. Since the number of entries is already
-		// known, perform a single alloc for them versus a whole bunch
-		// of little ones to reduce pressure on the GC.
-		log.Infof("Loading UTXO set...")
-
-		utxoEntryBucket := dbTx.Metadata().Bucket(utxoSetBucketName)
-
-		// Determine how many UTXO entries will be loaded into the index so we can
-		// allocate the right amount.
-		var utxoEntryCount int32
-		cursor = utxoEntryBucket.Cursor()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			utxoEntryCount++
-		}
-
-		fullUTXOCollection := make(utxoCollection, utxoEntryCount)
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			// Deserialize the outpoint
-			outpoint, err := deserializeOutpoint(bytes.NewReader(cursor.Key()))
-			if err != nil {
-				return err
-			}
-
-			// Deserialize the utxo entry
-			entry, err := deserializeUTXOEntry(bytes.NewReader(cursor.Value()))
-			if err != nil {
-				return err
-			}
-
-			fullUTXOCollection[*outpoint] = entry
-		}
-
-		// Initialize the reachability store
-		log.Infof("Loading reachability data...")
-		err = dag.reachabilityStore.init(dbTx)
-		if err != nil {
-			return err
-		}
-
-		// Initialize the multiset store
-		log.Infof("Loading multiset data...")
-		err = dag.multisetStore.init(dbTx)
-		if err != nil {
-			return err
-		}
-
-		// Apply the loaded utxoCollection to the virtual block.
-		dag.virtual.utxoSet, err = newFullUTXOSetFromUTXOCollection(fullUTXOCollection)
-		if err != nil {
-			return AssertError(fmt.Sprintf("Error loading UTXOSet: %s", err))
-		}
-
-		// Apply the stored tips to the virtual block.
-		tips := newBlockSet()
-		for _, tipHash := range state.TipHashes {
-			tip := dag.index.LookupNode(tipHash)
-			if tip == nil {
-				return AssertError(fmt.Sprintf("initDAGState: cannot find "+
-					"DAG tip %s in block index", state.TipHashes))
-			}
-			tips.add(tip)
-		}
-		dag.virtual.SetTips(tips)
-
-		// Set the last finality point
-		dag.lastFinalityPoint = dag.index.LookupNode(state.LastFinalityPoint)
-		dag.finalizeNodesBelowFinalityPoint(false)
-
-		// Go over any unprocessed blockNodes and process them now.
-		for _, node := range unprocessedBlockNodes {
-			// Check to see if the block exists in the block DB. If it
-			// doesn't, the database has certainly been corrupted.
-			blockExists, err := dbTx.HasBlock(node.hash)
-			if err != nil {
-				return AssertError(fmt.Sprintf("initDAGState: HasBlock "+
-					"for block %s failed: %s", node.hash, err))
-			}
-			if !blockExists {
-				return AssertError(fmt.Sprintf("initDAGState: block %s "+
-					"exists in block index but not in block db", node.hash))
-			}
-
-			// Attempt to accept the block.
-			block, err := dbFetchBlockByNode(dbTx, node)
-			if err != nil {
-				return err
-			}
-			isOrphan, isDelayed, err := dag.ProcessBlock(block, BFWasStored)
-			if err != nil {
-				log.Warnf("Block %s, which was not previously processed, "+
-					"failed to be accepted to the DAG: %s", node.hash, err)
-				continue
-			}
-
-			// If the block is an orphan or is delayed then it couldn't have
-			// possibly been written to the block index in the first place.
-			if isOrphan {
-				return AssertError(fmt.Sprintf("Block %s, which was not "+
-					"previously processed, turned out to be an orphan, which is "+
-					"impossible.", node.hash))
-			}
-			if isDelayed {
-				return AssertError(fmt.Sprintf("Block %s, which was not "+
-					"previously processed, turned out to be delayed, which is "+
-					"impossible.", node.hash))
-			}
-		}
-
-		return nil
-	})
-}
-
-// deserializeBlockNode parses a value in the block index bucket and returns a block node.
-func (dag *BlockDAG) deserializeBlockNode(blockRow []byte) (*blockNode, error) {
-	buffer := bytes.NewReader(blockRow)
-
-	var header wire.BlockHeader
-	err := header.Deserialize(buffer)
-	if err != nil {
-		return nil, err
-	}
-
-	node := &blockNode{
-		hash:                 header.BlockHash(),
-		version:              header.Version,
-		bits:                 header.Bits,
-		nonce:                header.Nonce,
-		timestamp:            header.Timestamp.Unix(),
-		hashMerkleRoot:       header.HashMerkleRoot,
-		acceptedIDMerkleRoot: header.AcceptedIDMerkleRoot,
-		utxoCommitment:       header.UTXOCommitment,
-	}
-
-	node.children = newBlockSet()
-	node.parents = newBlockSet()
-
-	for _, hash := range header.ParentHashes {
-		parent := dag.index.LookupNode(hash)
-		if parent == nil {
-			return nil, AssertError(fmt.Sprintf("deserializeBlockNode: Could "+
-				"not find parent %s for block %s", hash, header.BlockHash()))
-		}
-		node.parents.add(parent)
-	}
-
-	statusByte, err := buffer.ReadByte()
-	if err != nil {
-		return nil, err
-	}
-	node.status = blockStatus(statusByte)
-
-	selectedParentHash := &daghash.Hash{}
-	if _, err := io.ReadFull(buffer, selectedParentHash[:]); err != nil {
-		return nil, err
-	}
-
-	// Because genesis doesn't have selected parent, it's serialized as zero hash
-	if !selectedParentHash.IsEqual(&daghash.ZeroHash) {
-		node.selectedParent = dag.index.LookupNode(selectedParentHash)
-	}
-
-	node.blueScore, err = binaryserializer.Uint64(buffer, byteOrder)
-	if err != nil {
-		return nil, err
-	}
-
-	bluesCount, err := wire.ReadVarInt(buffer)
-	if err != nil {
-		return nil, err
-	}
-
-	node.blues = make([]*blockNode, bluesCount)
-	for i := uint64(0); i < bluesCount; i++ {
-		hash := &daghash.Hash{}
-		if _, err := io.ReadFull(buffer, hash[:]); err != nil {
-			return nil, err
-		}
-		node.blues[i] = dag.index.LookupNode(hash)
-	}
-
-	bluesAnticoneSizesLen, err := wire.ReadVarInt(buffer)
-	if err != nil {
-		return nil, err
-	}
-
-	node.bluesAnticoneSizes = make(map[*blockNode]dagconfig.KType)
-	for i := uint64(0); i < bluesAnticoneSizesLen; i++ {
-		hash := &daghash.Hash{}
-		if _, err := io.ReadFull(buffer, hash[:]); err != nil {
-			return nil, err
-		}
-		bluesAnticoneSize, err := binaryserializer.Uint8(buffer)
-		if err != nil {
-			return nil, err
-		}
-		blue := dag.index.LookupNode(hash)
-		if blue == nil {
-			return nil, errors.Errorf("couldn't find block with hash %s", hash)
-		}
-		node.bluesAnticoneSizes[blue] = dagconfig.KType(bluesAnticoneSize)
-	}
-
-	return node, nil
-}
-
-// dbFetchBlockByNode uses an existing database transaction to retrieve the
-// raw block for the provided node, deserialize it, and return a util.Block
-// of it.
-func dbFetchBlockByNode(dbTx database.Tx, node *blockNode) (*util.Block, error) {
-	// Load the raw block bytes from the database.
-	blockBytes, err := dbTx.FetchBlock(node.hash)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the encapsulated block.
-	block, err := util.NewBlockFromBytes(blockBytes)
-	if err != nil {
-		return nil, err
-	}
-	return block, nil
-}
-
-func serializeBlockNode(node *blockNode) ([]byte, error) {
-	w := bytes.NewBuffer(make([]byte, 0, blockHdrSize+1))
-	header := node.Header()
-	err := header.Serialize(w)
-	if err != nil {
-		return nil, err
-	}
-
-	err = w.WriteByte(byte(node.status))
-	if err != nil {
-		return nil, err
-	}
-
-	// Because genesis doesn't have selected parent, it's serialized as zero hash
-	selectedParentHash := &daghash.ZeroHash
-	if node.selectedParent != nil {
-		selectedParentHash = node.selectedParent.hash
-	}
-	_, err = w.Write(selectedParentHash[:])
-	if err != nil {
-		return nil, err
-	}
-
-	err = binaryserializer.PutUint64(w, byteOrder, node.blueScore)
-	if err != nil {
-		return nil, err
-	}
-
-	err = wire.WriteVarInt(w, uint64(len(node.blues)))
-	if err != nil {
-		return nil, err
-	}
-
-	for _, blue := range node.blues {
-		_, err = w.Write(blue.hash[:])
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	err = wire.WriteVarInt(w, uint64(len(node.bluesAnticoneSizes)))
-	if err != nil {
-		return nil, err
-	}
-	for blue, blueAnticoneSize := range node.bluesAnticoneSizes {
-		_, err = w.Write(blue.hash[:])
-		if err != nil {
-			return nil, err
-		}
-
-		err = binaryserializer.PutUint8(w, uint8(blueAnticoneSize))
-		if err != nil {
-			return nil, err
-		}
-	}
-	return w.Bytes(), nil
-}
-
-// dbStoreBlockNode stores the block node data into the block
-// index bucket. This overwrites the current entry if there exists one.
-func dbStoreBlockNode(dbTx database.Tx, node *blockNode) error {
-	serializedNode, err := serializeBlockNode(node)
-	if err != nil {
-		return err
-	}
-	// Write block header data to block index bucket.
-	blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName)
-	key := BlockIndexKey(node.hash, node.blueScore)
-	return blockIndexBucket.Put(key, serializedNode)
-}
-
-// dbStoreBlock stores the provided block in the database if it is not already
-// there. The full block data is written to ffldb.
-func dbStoreBlock(dbTx database.Tx, block *util.Block) error {
-	hasBlock, err := dbTx.HasBlock(block.Hash())
-	if err != nil {
-		return err
-	}
-	if hasBlock {
-		return nil
-	}
-	return dbTx.StoreBlock(block)
-}
-
-// BlockIndexKey generates the binary key for an entry in the block index
-// bucket. The key is composed of the block blue score encoded as a big-endian
-// 64-bit unsigned int followed by the 32 byte block hash.
-// The blue score component is important for iteration order.
-func BlockIndexKey(blockHash *daghash.Hash, blueScore uint64) []byte {
-	indexKey := make([]byte, daghash.HashSize+8)
-	binary.BigEndian.PutUint64(indexKey[0:8], blueScore)
-	copy(indexKey[8:daghash.HashSize+8], blockHash[:])
-	return indexKey
-}
-
-func blockHashFromBlockIndexKey(BlockIndexKey []byte) (*daghash.Hash, error) {
-	return daghash.NewHash(BlockIndexKey[8 : daghash.HashSize+8])
-}
-
-// BlockByHash returns the block from the DAG with the given hash.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) BlockByHash(hash *daghash.Hash) (*util.Block, error) {
-	// Lookup the block hash in block index and ensure it is in the DAG
-	node := dag.index.LookupNode(hash)
-	if node == nil {
-		str := fmt.Sprintf("block %s is not in the DAG", hash)
-		return nil, errNotInDAG(str)
-	}
-
-	// Load the block from the database and return it.
-	var block *util.Block
-	err := dag.db.View(func(dbTx database.Tx) error {
-		var err error
-		block, err = dbFetchBlockByNode(dbTx, node)
-		return err
-	})
-	return block, err
-}
-
-// BlockHashesFrom returns a slice of blocks starting from lowHash
-// ordered by blueScore. If lowHash is nil then the genesis block is used.
-//
-// This method MUST be called with the DAG lock held
-func (dag *BlockDAG) BlockHashesFrom(lowHash *daghash.Hash, limit int) ([]*daghash.Hash, error) {
-	blockHashes := make([]*daghash.Hash, 0, limit)
-	if lowHash == nil {
-		lowHash = dag.genesis.hash
-
-		// If we're starting from the beginning we should include the
-		// genesis hash in the result
-		blockHashes = append(blockHashes, dag.genesis.hash)
-	}
-	if !dag.IsInDAG(lowHash) {
-		return nil, errors.Errorf("block %s not found", lowHash)
-	}
-	blueScore, err := dag.BlueScoreByBlockHash(lowHash)
-	if err != nil {
-		return nil, err
-	}
-
-	err = dag.index.db.View(func(dbTx database.Tx) error {
-		blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName)
-		lowKey := BlockIndexKey(lowHash, blueScore)
-
-		cursor := blockIndexBucket.Cursor()
-		cursor.Seek(lowKey)
-		for ok := cursor.Next(); ok; ok = cursor.Next() {
-			key := cursor.Key()
-			blockHash, err := blockHashFromBlockIndexKey(key)
-			if err != nil {
-				return err
-			}
-			blockHashes = append(blockHashes, blockHash)
-			if len(blockHashes) == limit {
-				break
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return blockHashes, nil
-}

blockdag/indexers/common.go

@@ -1,112 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-Package indexers implements optional block DAG indexes.
-*/
-package indexers
-
-import (
-	"encoding/binary"
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/pkg/errors"
-)
-
-var (
-	// byteOrder is the preferred byte order used for serializing numeric
-	// fields for storage in the database.
-	byteOrder = binary.LittleEndian
-
-	// errInterruptRequested indicates that an operation was cancelled due
-	// to a user-requested interrupt.
-	errInterruptRequested = errors.New("interrupt requested")
-)
-
-// NeedsInputser provides a generic interface for an indexer to specify the it
-// requires the ability to look up inputs for a transaction.
-type NeedsInputser interface {
-	NeedsInputs() bool
-}
-
-// Indexer provides a generic interface for an indexer that is managed by an
-// index manager such as the Manager type provided by this package.
-type Indexer interface {
-	// Key returns the key of the index as a byte slice.
-	Key() []byte
-
-	// Name returns the human-readable name of the index.
-	Name() string
-
-	// Create is invoked when the indexer manager determines the index needs
-	// to be created for the first time.
-	Create(dbTx database.Tx) error
-
-	// Init is invoked when the index manager is first initializing the
-	// index. This differs from the Create method in that it is called on
-	// every load, including the case the index was just created.
-	Init(db database.DB, dag *blockdag.BlockDAG) error
-
-	// ConnectBlock is invoked when the index manager is notified that a new
-	// block has been connected to the DAG.
-	ConnectBlock(dbTx database.Tx,
-		block *util.Block,
-		blockID uint64,
-		dag *blockdag.BlockDAG,
-		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData,
-		virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error
-
-	// Recover is invoked when the indexer wasn't turned on for several blocks
-	// and the indexer needs to close the gaps.
-	Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error
-}
-
-// AssertError identifies an error that indicates an internal code consistency
-// issue and should be treated as a critical and unrecoverable error.
-type AssertError string
-
-// Error returns the assertion error as a huma-readable string and satisfies
-// the error interface.
-func (e AssertError) Error() string {
-	return "assertion failed: " + string(e)
-}
-
-// errDeserialize signifies that a problem was encountered when deserializing
-// data.
-type errDeserialize string
-
-// Error implements the error interface.
-func (e errDeserialize) Error() string {
-	return string(e)
-}
-
-// isDeserializeErr returns whether or not the passed error is an errDeserialize
-// error.
-func isDeserializeErr(err error) bool {
-	var deserializeErr errDeserialize
-	return errors.As(err, &deserializeErr)
-}
-
-// internalBucket is an abstraction over a database bucket. It is used to make
-// the code easier to test since it allows mock objects in the tests to only
-// implement these functions instead of everything a database.Bucket supports.
-type internalBucket interface {
-	Get(key []byte) []byte
-	Put(key []byte, value []byte) error
-	Delete(key []byte) error
-}
-
-// interruptRequested returns true when the provided channel has been closed.
-// This simplifies early shutdown slightly since the caller can just use an if
-// statement instead of a select.
-func interruptRequested(interrupted <-chan struct{}) bool {
-	select {
-	case <-interrupted:
-		return true
-	default:
-	}
-
-	return false
-}

blockdag/indexers/manager.go

@@ -1,392 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package indexers
-
-import (
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-var (
-	// indexTipsBucketName is the name of the db bucket used to house the
-	// current tip of each index.
-	indexTipsBucketName = []byte("idxtips")
-
-	indexCurrentBlockIDBucketName = []byte("idxcurrentblockid")
-)
-
-// Manager defines an index manager that manages multiple optional indexes and
-// implements the blockdag.IndexManager interface so it can be seamlessly
-// plugged into normal DAG processing.
-type Manager struct {
-	db             database.DB
-	enabledIndexes []Indexer
-}
-
-// Ensure the Manager type implements the blockdag.IndexManager interface.
-var _ blockdag.IndexManager = (*Manager)(nil)
-
-// indexDropKey returns the key for an index which indicates it is in the
-// process of being dropped.
-func indexDropKey(idxKey []byte) []byte {
-	dropKey := make([]byte, len(idxKey)+1)
-	dropKey[0] = 'd'
-	copy(dropKey[1:], idxKey)
-	return dropKey
-}
-
-// maybeFinishDrops determines if each of the enabled indexes are in the middle
-// of being dropped and finishes dropping them when the are. This is necessary
-// because dropping and index has to be done in several atomic steps rather than
-// one big atomic step due to the massive number of entries.
-func (m *Manager) maybeFinishDrops(interrupt <-chan struct{}) error {
-	indexNeedsDrop := make([]bool, len(m.enabledIndexes))
-	err := m.db.View(func(dbTx database.Tx) error {
-		// None of the indexes needs to be dropped if the index tips
-		// bucket hasn't been created yet.
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		if indexesBucket == nil {
-			return nil
-		}
-
-		// Mark the indexer as requiring a drop if one is already in
-		// progress.
-		for i, indexer := range m.enabledIndexes {
-			dropKey := indexDropKey(indexer.Key())
-			if indexesBucket.Get(dropKey) != nil {
-				indexNeedsDrop[i] = true
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
-	if interruptRequested(interrupt) {
-		return errInterruptRequested
-	}
-
-	// Finish dropping any of the enabled indexes that are already in the
-	// middle of being dropped.
-	for i, indexer := range m.enabledIndexes {
-		if !indexNeedsDrop[i] {
-			continue
-		}
-
-		log.Infof("Resuming %s drop", indexer.Name())
-		err := dropIndex(m.db, indexer.Key(), indexer.Name(), interrupt)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// maybeCreateIndexes determines if each of the enabled indexes have already
-// been created and creates them if not.
-func (m *Manager) maybeCreateIndexes(dbTx database.Tx) error {
-	indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-	for _, indexer := range m.enabledIndexes {
-		// Nothing to do if the index tip already exists.
-		idxKey := indexer.Key()
-		if indexesBucket.Get(idxKey) != nil {
-			continue
-		}
-
-		// The tip for the index does not exist, so create it and
-		// invoke the create callback for the index so it can perform
-		// any one-time initialization it requires.
-		if err := indexer.Create(dbTx); err != nil {
-			return err
-		}
-
-		// TODO (Mike): this is temporary solution to prevent node from not starting
-		// because it thinks indexers are not initialized.
-		// Indexers, however, do not work properly, and a general solution to their work operation is required
-		indexesBucket.Put(idxKey, []byte{0})
-	}
-
-	return nil
-}
-
-// Init initializes the enabled indexes. This is called during DAG
-// initialization and primarily consists of catching up all indexes to the
-// current tips. This is necessary since each index can be disabled
-// and re-enabled at any time and attempting to catch-up indexes at the same
-// time new blocks are being downloaded would lead to an overall longer time to
-// catch up due to the I/O contention.
-//
-// This is part of the blockdag.IndexManager interface.
-func (m *Manager) Init(db database.DB, blockDAG *blockdag.BlockDAG, interrupt <-chan struct{}) error {
-	// Nothing to do when no indexes are enabled.
-	if len(m.enabledIndexes) == 0 {
-		return nil
-	}
-
-	if interruptRequested(interrupt) {
-		return errInterruptRequested
-	}
-
-	m.db = db
-
-	// Finish and drops that were previously interrupted.
-	if err := m.maybeFinishDrops(interrupt); err != nil {
-		return err
-	}
-
-	// Create the initial state for the indexes as needed.
-	err := m.db.Update(func(dbTx database.Tx) error {
-		// Create the bucket for the current tips as needed.
-		meta := dbTx.Metadata()
-		_, err := meta.CreateBucketIfNotExists(indexTipsBucketName)
-		if err != nil {
-			return err
-		}
-		if _, err := meta.CreateBucketIfNotExists(indexCurrentBlockIDBucketName); err != nil {
-			return err
-		}
-
-		return m.maybeCreateIndexes(dbTx)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Initialize each of the enabled indexes.
-	for _, indexer := range m.enabledIndexes {
-		if err := indexer.Init(db, blockDAG); err != nil {
-			return err
-		}
-	}
-
-	return m.recoverIfNeeded()
-}
-
-// recoverIfNeeded checks if the node worked for some time
-// without one of the current enabled indexes, and if it's
-// the case, recovers the missing blocks from the index.
-func (m *Manager) recoverIfNeeded() error {
-	return m.db.Update(func(dbTx database.Tx) error {
-		lastKnownBlockID := blockdag.DBFetchCurrentBlockID(dbTx)
-		for _, indexer := range m.enabledIndexes {
-			serializedCurrentIdxBlockID := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Get(indexer.Key())
-			currentIdxBlockID := uint64(0)
-			if serializedCurrentIdxBlockID != nil {
-				currentIdxBlockID = blockdag.DeserializeBlockID(serializedCurrentIdxBlockID)
-			}
-			if lastKnownBlockID > currentIdxBlockID {
-				err := indexer.Recover(dbTx, currentIdxBlockID, lastKnownBlockID)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
-}
-
-// ConnectBlock must be invoked when a block is added to the DAG. It
-// keeps track of the state of each index it is managing, performs some sanity
-// checks, and invokes each indexer.
-//
-// This is part of the blockdag.IndexManager interface.
-func (m *Manager) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
-	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
-
-	// Call each of the currently active optional indexes with the block
-	// being connected so they can update accordingly.
-	for _, index := range m.enabledIndexes {
-		// Notify the indexer with the connected block so it can index it.
-		if err := index.ConnectBlock(dbTx, block, blockID, dag, txsAcceptanceData, virtualTxsAcceptanceData); err != nil {
-			return err
-		}
-	}
-
-	// Add the new block ID index entry for the block being connected and
-	// update the current internal block ID accordingly.
-	err := m.updateIndexersWithCurrentBlockID(dbTx, block.Hash(), blockID)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (m *Manager) updateIndexersWithCurrentBlockID(dbTx database.Tx, blockHash *daghash.Hash, blockID uint64) error {
-	serializedBlockID := blockdag.SerializeBlockID(blockID)
-	for _, index := range m.enabledIndexes {
-		err := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Put(index.Key(), serializedBlockID)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// NewManager returns a new index manager with the provided indexes enabled.
-//
-// The manager returned satisfies the blockdag.IndexManager interface and thus
-// cleanly plugs into the normal blockdag processing path.
-func NewManager(enabledIndexes []Indexer) *Manager {
-	return &Manager{
-		enabledIndexes: enabledIndexes,
-	}
-}
-
-// dropIndex drops the passed index from the database. Since indexes can be
-// massive, it deletes the index in multiple database transactions in order to
-// keep memory usage to reasonable levels. It also marks the drop in progress
-// so the drop can be resumed if it is stopped before it is done before the
-// index can be used again.
-func dropIndex(db database.DB, idxKey []byte, idxName string, interrupt <-chan struct{}) error {
-	// Nothing to do if the index doesn't already exist.
-	var needsDelete bool
-	err := db.View(func(dbTx database.Tx) error {
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		if indexesBucket != nil && indexesBucket.Get(idxKey) != nil {
-			needsDelete = true
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	if !needsDelete {
-		log.Infof("Not dropping %s because it does not exist", idxName)
-		return nil
-	}
-
-	// Mark that the index is in the process of being dropped so that it
-	// can be resumed on the next start if interrupted before the process is
-	// complete.
-	log.Infof("Dropping all %s entries. This might take a while...",
-		idxName)
-	err = db.Update(func(dbTx database.Tx) error {
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		return indexesBucket.Put(indexDropKey(idxKey), idxKey)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Since the indexes can be so large, attempting to simply delete
-	// the bucket in a single database transaction would result in massive
-	// memory usage and likely crash many systems due to ulimits. In order
-	// to avoid this, use a cursor to delete a maximum number of entries out
-	// of the bucket at a time. Recurse buckets depth-first to delete any
-	// sub-buckets.
-	const maxDeletions = 2000000
-	var totalDeleted uint64
-
-	// Recurse through all buckets in the index, cataloging each for
-	// later deletion.
-	var subBuckets [][][]byte
-	var subBucketClosure func(database.Tx, []byte, [][]byte) error
-	subBucketClosure = func(dbTx database.Tx,
-		subBucket []byte, tlBucket [][]byte) error {
-		// Get full bucket name and append to subBuckets for later
-		// deletion.
-		var bucketName [][]byte
-		if (tlBucket == nil) || (len(tlBucket) == 0) {
-			bucketName = append(bucketName, subBucket)
-		} else {
-			bucketName = append(tlBucket, subBucket)
-		}
-		subBuckets = append(subBuckets, bucketName)
-		// Recurse sub-buckets to append to subBuckets slice.
-		bucket := dbTx.Metadata()
-		for _, subBucketName := range bucketName {
-			bucket = bucket.Bucket(subBucketName)
-		}
-		return bucket.ForEachBucket(func(k []byte) error {
-			return subBucketClosure(dbTx, k, bucketName)
-		})
-	}
-
-	// Call subBucketClosure with top-level bucket.
-	err = db.View(func(dbTx database.Tx) error {
-		return subBucketClosure(dbTx, idxKey, nil)
-	})
-	if err != nil {
-		return nil
-	}
-
-	// Iterate through each sub-bucket in reverse, deepest-first, deleting
-	// all keys inside them and then dropping the buckets themselves.
-	for i := range subBuckets {
-		bucketName := subBuckets[len(subBuckets)-1-i]
-		// Delete maxDeletions key/value pairs at a time.
-		for numDeleted := maxDeletions; numDeleted == maxDeletions; {
-			numDeleted = 0
-			err := db.Update(func(dbTx database.Tx) error {
-				subBucket := dbTx.Metadata()
-				for _, subBucketName := range bucketName {
-					subBucket = subBucket.Bucket(subBucketName)
-				}
-				cursor := subBucket.Cursor()
-				for ok := cursor.First(); ok; ok = cursor.Next() &&
-					numDeleted < maxDeletions {
-
-					if err := cursor.Delete(); err != nil {
-						return err
-					}
-					numDeleted++
-				}
-				return nil
-			})
-			if err != nil {
-				return err
-			}
-
-			if numDeleted > 0 {
-				totalDeleted += uint64(numDeleted)
-				log.Infof("Deleted %d keys (%d total) from %s",
-					numDeleted, totalDeleted, idxName)
-			}
-		}
-
-		if interruptRequested(interrupt) {
-			return errInterruptRequested
-		}
-
-		// Drop the bucket itself.
-		err = db.Update(func(dbTx database.Tx) error {
-			bucket := dbTx.Metadata()
-			for j := 0; j < len(bucketName)-1; j++ {
-				bucket = bucket.Bucket(bucketName[j])
-			}
-			return bucket.DeleteBucket(bucketName[len(bucketName)-1])
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	// Remove the index tip, index bucket, and in-progress drop flag now
-	// that all index entries have been removed.
-	err = db.Update(func(dbTx database.Tx) error {
-		meta := dbTx.Metadata()
-		indexesBucket := meta.Bucket(indexTipsBucketName)
-		if err := indexesBucket.Delete(idxKey); err != nil {
-			return err
-		}
-
-		if err := meta.Bucket(indexCurrentBlockIDBucketName).Delete(idxKey); err != nil {
-			return err
-		}
-
-		return indexesBucket.Delete(indexDropKey(idxKey))
-	})
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Dropped %s", idxName)
-	return nil
-}

blockdag/process.go

@@ -1,275 +0,0 @@
-// Copyright (c) 2013-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"fmt"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/pkg/errors"
-	"time"
-
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// BehaviorFlags is a bitmask defining tweaks to the normal behavior when
-// performing DAG processing and consensus rules checks.
-type BehaviorFlags uint32
-
-const (
-	// BFFastAdd may be set to indicate that several checks can be avoided
-	// for the block since it is already known to fit into the DAG due to
-	// already proving it correct links into the DAG.
-	BFFastAdd BehaviorFlags = 1 << iota
-
-	// BFNoPoWCheck may be set to indicate the proof of work check which
-	// ensures a block hashes to a value less than the required target will
-	// not be performed.
-	BFNoPoWCheck
-
-	// BFWasUnorphaned may be set to indicate that a block was just now
-	// unorphaned
-	BFWasUnorphaned
-
-	// BFAfterDelay may be set to indicate that a block had timestamp too far
-	// in the future, just finished the delay
-	BFAfterDelay
-
-	// BFIsSync may be set to indicate that the block was sent as part of the
-	// netsync process
-	BFIsSync
-
-	// BFWasStored is set to indicate that the block was previously stored
-	// in the block index but was never fully processed
-	BFWasStored
-
-	// BFDisallowDelay is set to indicate that a delayed block should be rejected.
-	// This is used for the case where a block is submitted through RPC.
-	BFDisallowDelay
-
-	// BFNone is a convenience value to specifically indicate no flags.
-	BFNone BehaviorFlags = 0
-)
-
-// IsInDAG determines whether a block with the given hash exists in
-// the DAG.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) IsInDAG(hash *daghash.Hash) bool {
-	return dag.index.HaveBlock(hash)
-}
-
-// processOrphans determines if there are any orphans which depend on the passed
-// block hash (they are no longer orphans if true) and potentially accepts them.
-// It repeats the process for the newly accepted blocks (to detect further
-// orphans which may no longer be orphans) until there are no more.
-//
-// The flags do not modify the behavior of this function directly, however they
-// are needed to pass along to maybeAcceptBlock.
-//
-// This function MUST be called with the DAG state lock held (for writes).
-func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) error {
-	// Start with processing at least the passed hash. Leave a little room
-	// for additional orphan blocks that need to be processed without
-	// needing to grow the array in the common case.
-	processHashes := make([]*daghash.Hash, 0, 10)
-	processHashes = append(processHashes, hash)
-	for len(processHashes) > 0 {
-		// Pop the first hash to process from the slice.
-		processHash := processHashes[0]
-		processHashes[0] = nil // Prevent GC leak.
-		processHashes = processHashes[1:]
-
-		// Look up all orphans that are parented by the block we just
-		// accepted.  An indexing for loop is
-		// intentionally used over a range here as range does not
-		// reevaluate the slice on each iteration nor does it adjust the
-		// index for the modified slice.
-		for i := 0; i < len(dag.prevOrphans[*processHash]); i++ {
-			orphan := dag.prevOrphans[*processHash][i]
-			if orphan == nil {
-				log.Warnf("Found a nil entry at index %d in the "+
-					"orphan dependency list for block %s", i,
-					processHash)
-				continue
-			}
-
-			// Skip this orphan if one or more of its parents are
-			// still missing.
-			_, err := lookupParentNodes(orphan.block, dag)
-			if err != nil {
-				var ruleErr RuleError
-				if ok := errors.As(err, &ruleErr); ok && ruleErr.ErrorCode == ErrParentBlockUnknown {
-					continue
-				}
-				return err
-			}
-
-			// Remove the orphan from the orphan pool.
-			orphanHash := orphan.block.Hash()
-			dag.removeOrphanBlock(orphan)
-			i--
-
-			// Potentially accept the block into the block DAG.
-			err = dag.maybeAcceptBlock(orphan.block, flags|BFWasUnorphaned)
-			if err != nil {
-				// Since we don't want to reject the original block because of
-				// a bad unorphaned child, only return an error if it's not a RuleError.
-				if !errors.As(err, &RuleError{}) {
-					return err
-				}
-				log.Warnf("Verification failed for orphan block %s: %s", orphanHash, err)
-			}
-
-			// Add this block to the list of blocks to process so
-			// any orphan blocks that depend on this block are
-			// handled too.
-			processHashes = append(processHashes, orphanHash)
-		}
-	}
-	return nil
-}
-
-// ProcessBlock is the main workhorse for handling insertion of new blocks into
-// the block DAG. It includes functionality such as rejecting duplicate
-// blocks, ensuring blocks follow all rules, orphan handling, and insertion into
-// the block DAG.
-//
-// When no errors occurred during processing, the first return value indicates
-// whether or not the block is an orphan.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrphan bool, isDelayed bool, err error) {
-	dag.dagLock.Lock()
-	defer dag.dagLock.Unlock()
-	return dag.processBlockNoLock(block, flags)
-}
-
-func (dag *BlockDAG) processBlockNoLock(block *util.Block, flags BehaviorFlags) (isOrphan bool, isDelayed bool, err error) {
-	isAfterDelay := flags&BFAfterDelay == BFAfterDelay
-	wasBlockStored := flags&BFWasStored == BFWasStored
-	disallowDelay := flags&BFDisallowDelay == BFDisallowDelay
-
-	blockHash := block.Hash()
-	log.Tracef("Processing block %s", blockHash)
-
-	// The block must not already exist in the DAG.
-	if dag.IsInDAG(blockHash) && !wasBlockStored {
-		str := fmt.Sprintf("already have block %s", blockHash)
-		return false, false, ruleError(ErrDuplicateBlock, str)
-	}
-
-	// The block must not already exist as an orphan.
-	if _, exists := dag.orphans[*blockHash]; exists {
-		str := fmt.Sprintf("already have block (orphan) %s", blockHash)
-		return false, false, ruleError(ErrDuplicateBlock, str)
-	}
-
-	if dag.isKnownDelayedBlock(blockHash) {
-		str := fmt.Sprintf("already have block (delayed) %s", blockHash)
-		return false, false, ruleError(ErrDuplicateBlock, str)
-	}
-
-	if !isAfterDelay {
-		// Perform preliminary sanity checks on the block and its transactions.
-		delay, err := dag.checkBlockSanity(block, flags)
-		if err != nil {
-			return false, false, err
-		}
-
-		if delay != 0 && disallowDelay {
-			str := fmt.Sprintf("Cannot process blocks beyond the allowed time offset while the BFDisallowDelay flag is raised %s", blockHash)
-			return false, true, ruleError(ErrDelayedBlockIsNotAllowed, str)
-		}
-
-		if delay != 0 {
-			err = dag.addDelayedBlock(block, delay)
-			if err != nil {
-				return false, false, err
-			}
-			return false, true, nil
-		}
-	}
-
-	var missingParents []*daghash.Hash
-	for _, parentHash := range block.MsgBlock().Header.ParentHashes {
-		if !dag.IsInDAG(parentHash) {
-			missingParents = append(missingParents, parentHash)
-		}
-	}
-
-	// Handle the case of a block with a valid timestamp(non-delayed) which points to a delayed block.
-	delay, isParentDelayed := dag.maxDelayOfParents(missingParents)
-	if isParentDelayed {
-		// Add Nanosecond to ensure that parent process time will be after its child.
-		delay += time.Nanosecond
-		err := dag.addDelayedBlock(block, delay)
-		if err != nil {
-			return false, false, err
-		}
-		return false, true, err
-	}
-
-	// Handle orphan blocks.
-	if len(missingParents) > 0 {
-		// Some orphans during netsync are a normal part of the process, since the anticone
-		// of the chain-split is never explicitly requested.
-		// Therefore, if we are during netsync - don't report orphans to default logs.
-		//
-		// The number K*2 was chosen since in peace times anticone is limited to K blocks,
-		// while some red block can make it a bit bigger, but much more than that indicates
-		// there might be some problem with the netsync process.
-		if flags&BFIsSync == BFIsSync && dagconfig.KType(len(dag.orphans)) < dag.dagParams.K*2 {
-			log.Debugf("Adding orphan block %s. This is normal part of netsync process", blockHash)
-		} else {
-			log.Infof("Adding orphan block %s", blockHash)
-		}
-		dag.addOrphanBlock(block)
-
-		return true, false, nil
-	}
-
-	// The block has passed all context independent checks and appears sane
-	// enough to potentially accept it into the block DAG.
-	err = dag.maybeAcceptBlock(block, flags)
-	if err != nil {
-		return false, false, err
-	}
-
-	// Accept any orphan blocks that depend on this block (they are
-	// no longer orphans) and repeat for those accepted blocks until
-	// there are no more.
-	err = dag.processOrphans(blockHash, flags)
-	if err != nil {
-		return false, false, err
-	}
-
-	if !isAfterDelay {
-		err = dag.processDelayedBlocks()
-		if err != nil {
-			return false, false, err
-		}
-	}
-
-	log.Debugf("Accepted block %s", blockHash)
-
-	return false, false, nil
-}
-
-// maxDelayOfParents returns the maximum delay of the given block hashes.
-// Note that delay could be 0, but isDelayed will return true. This is the case where the parent process time is due.
-func (dag *BlockDAG) maxDelayOfParents(parentHashes []*daghash.Hash) (delay time.Duration, isDelayed bool) {
-	for _, parentHash := range parentHashes {
-		if delayedParent, exists := dag.delayedBlocks[*parentHash]; exists {
-			isDelayed = true
-			parentDelay := delayedParent.processTime.Sub(dag.Now())
-			if parentDelay > delay {
-				delay = parentDelay
-			}
-		}
-	}
-
-	return delay, isDelayed
-}

blockdag/reachability.go

@@ -1,577 +0,0 @@
-package blockdag
-
-import (
-	"fmt"
-	"github.com/pkg/errors"
-	"math"
-	"strings"
-	"time"
-)
-
-// reachabilityInterval represents an interval to be used within the
-// tree reachability algorithm. See reachabilityTreeNode for further
-// details.
-type reachabilityInterval struct {
-	start uint64
-	end   uint64
-}
-
-func newReachabilityInterval(start uint64, end uint64) *reachabilityInterval {
-	return &reachabilityInterval{start: start, end: end}
-}
-
-// size returns the size of this interval. Note that intervals are
-// inclusive from both sides.
-func (ri *reachabilityInterval) size() uint64 {
-	return ri.end - ri.start + 1
-}
-
-// splitInHalf splits this interval by a fraction of 0.5.
-// See splitFraction for further details.
-func (ri *reachabilityInterval) splitInHalf() (
-	left *reachabilityInterval, right *reachabilityInterval, err error) {
-
-	return ri.splitFraction(0.5)
-}
-
-// splitFraction splits this interval to two parts such that their
-// union is equal to the original interval and the first (left) part
-// contains the given fraction of the original interval's size.
-// Note: if the split results in fractional parts, this method rounds
-// the first part up and the last part down.
-func (ri *reachabilityInterval) splitFraction(fraction float64) (
-	left *reachabilityInterval, right *reachabilityInterval, err error) {
-
-	if fraction < 0 || fraction > 1 {
-		return nil, nil, errors.Errorf("fraction must be between 0 and 1")
-	}
-	if ri.size() == 0 {
-		return nil, nil, errors.Errorf("cannot split an empty interval")
-	}
-
-	allocationSize := uint64(math.Ceil(float64(ri.size()) * fraction))
-	left = newReachabilityInterval(ri.start, ri.start+allocationSize-1)
-	right = newReachabilityInterval(ri.start+allocationSize, ri.end)
-	return left, right, nil
-}
-
-// splitExact splits this interval to exactly |sizes| parts where
-// |part_i| = sizes[i]. This method expects sum(sizes) to be exactly
-// equal to the interval's size.
-func (ri *reachabilityInterval) splitExact(sizes []uint64) ([]*reachabilityInterval, error) {
-	sizesSum := uint64(0)
-	for _, size := range sizes {
-		sizesSum += size
-	}
-	if sizesSum != ri.size() {
-		return nil, errors.Errorf("sum of sizes must be equal to the interval's size")
-	}
-
-	intervals := make([]*reachabilityInterval, len(sizes))
-	start := ri.start
-	for i, size := range sizes {
-		intervals[i] = newReachabilityInterval(start, start+size-1)
-		start += size
-	}
-	return intervals, nil
-}
-
-// splitWithExponentialBias splits this interval to |sizes| parts
-// by the allocation rule described below. This method expects sum(sizes)
-// to be smaller or equal to the interval's size. Every part_i is
-// allocated at least sizes[i] capacity. The remaining budget is
-// split by an exponentially biased rule described below.
-//
-// This rule follows the GHOSTDAG protocol behavior where the child
-// with the largest subtree is expected to dominate the competition
-// for new blocks and thus grow the most. However, we may need to
-// add slack for non-largest subtrees in order to make CPU reindexing
-// attacks unworthy.
-func (ri *reachabilityInterval) splitWithExponentialBias(sizes []uint64) ([]*reachabilityInterval, error) {
-	intervalSize := ri.size()
-	sizesSum := uint64(0)
-	for _, size := range sizes {
-		sizesSum += size
-	}
-	if sizesSum > intervalSize {
-		return nil, errors.Errorf("sum of sizes must be less than or equal to the interval's size")
-	}
-	if sizesSum == intervalSize {
-		return ri.splitExact(sizes)
-	}
-
-	// Add a fractional bias to every size in the given sizes
-	totalBias := intervalSize - sizesSum
-	remainingBias := totalBias
-	biasedSizes := make([]uint64, len(sizes))
-	fractions := exponentialFractions(sizes)
-	for i, fraction := range fractions {
-		var bias uint64
-		if i == len(fractions)-1 {
-			bias = remainingBias
-		} else {
-			bias = uint64(math.Round(float64(totalBias) * fraction))
-			if bias > remainingBias {
-				bias = remainingBias
-			}
-		}
-		biasedSizes[i] = sizes[i] + bias
-		remainingBias -= bias
-	}
-	return ri.splitExact(biasedSizes)
-}
-
-// exponentialFractions returns a fraction of each size in sizes
-// as follows:
-//   fraction[i] = 2^size[i] / sum_j(2^size[j])
-// In the code below the above equation is divided by 2^max(size)
-// to avoid exploding numbers. Note that in 1 / 2^(max(size)-size[i])
-// we divide 1 by potentially a very large number, which will
-// result in loss of float precision. This is not a problem - all
-// numbers close to 0 bear effectively the same weight.
-func exponentialFractions(sizes []uint64) []float64 {
-	maxSize := uint64(0)
-	for _, size := range sizes {
-		if size > maxSize {
-			maxSize = size
-		}
-	}
-	fractions := make([]float64, len(sizes))
-	for i, size := range sizes {
-		fractions[i] = 1 / math.Pow(2, float64(maxSize-size))
-	}
-	fractionsSum := float64(0)
-	for _, fraction := range fractions {
-		fractionsSum += fraction
-	}
-	for i, fraction := range fractions {
-		fractions[i] = fraction / fractionsSum
-	}
-	return fractions
-}
-
-// isAncestorOf checks if this interval's node is a reachability tree
-// ancestor of the other interval's node. The condition below is relying on the
-// property of reachability intervals that intervals are either completely disjoint,
-// or one strictly contains the other.
-func (ri *reachabilityInterval) isAncestorOf(other *reachabilityInterval) bool {
-	return ri.start <= other.end && other.end <= ri.end
-}
-
-// String returns a string representation of the interval.
-func (ri *reachabilityInterval) String() string {
-	return fmt.Sprintf("[%d,%d]", ri.start, ri.end)
-}
-
-// reachabilityTreeNode represents a node in the reachability tree
-// of some DAG block. It mainly provides the ability to query *tree*
-// reachability with O(1) query time. It does so by managing an
-// index interval for each node and making sure all nodes in its
-// subtree are indexed within the interval, so the query
-// B ∈ subtree(A) simply becomes B.interval ⊂ A.interval.
-//
-// The main challenge of maintaining such intervals is that our tree
-// is an ever-growing tree and as such pre-allocated intervals may
-// not suffice as per future events. This is where the reindexing
-// algorithm below comes into place.
-// We use the reasonable assumption that the initial root interval
-// (e.g., [0, 2^64-1]) should always suffice for any practical use-
-// case, and so reindexing should always succeed unless more than
-// 2^64 blocks are added to the DAG/tree.
-type reachabilityTreeNode struct {
-	blockNode *blockNode
-
-	children []*reachabilityTreeNode
-	parent   *reachabilityTreeNode
-
-	// interval is the index interval containing all intervals of
-	// blocks in this node's subtree
-	interval *reachabilityInterval
-
-	// remainingInterval is the not-yet allocated interval (within
-	// this node's interval) awaiting new children
-	remainingInterval *reachabilityInterval
-}
-
-func newReachabilityTreeNode(blockNode *blockNode) *reachabilityTreeNode {
-	// Please see the comment above reachabilityTreeNode to understand why
-	// we use these initial values.
-	interval := newReachabilityInterval(1, math.MaxUint64-1)
-	// We subtract 1 from the end of the remaining interval to prevent the node from allocating
-	// the entire interval to its child, so its interval would *strictly* contain the interval of its child.
-	remainingInterval := newReachabilityInterval(interval.start, interval.end-1)
-	return &reachabilityTreeNode{blockNode: blockNode, interval: interval, remainingInterval: remainingInterval}
-}
-
-// addChild adds child to this tree node. If this node has no
-// remaining interval to allocate, a reindexing is triggered.
-// This method returns a list of reachabilityTreeNodes modified
-// by it.
-func (rtn *reachabilityTreeNode) addChild(child *reachabilityTreeNode) ([]*reachabilityTreeNode, error) {
-	// Set the parent-child relationship
-	rtn.children = append(rtn.children, child)
-	child.parent = rtn
-
-	// No allocation space left -- reindex
-	if rtn.remainingInterval.size() == 0 {
-		reindexStartTime := time.Now()
-		modifiedNodes, err := rtn.reindexIntervals()
-		if err != nil {
-			return nil, err
-		}
-		reindexTimeElapsed := time.Since(reindexStartTime)
-		log.Debugf("Reachability reindex triggered for "+
-			"block %s. Modified %d tree nodes and took %dms.",
-			rtn.blockNode.hash, len(modifiedNodes), reindexTimeElapsed.Milliseconds())
-		return modifiedNodes, nil
-	}
-
-	// Allocate from the remaining space
-	allocated, remaining, err := rtn.remainingInterval.splitInHalf()
-	if err != nil {
-		return nil, err
-	}
-	child.setInterval(allocated)
-	rtn.remainingInterval = remaining
-	return []*reachabilityTreeNode{rtn, child}, nil
-}
-
-// setInterval sets the reachability interval for this node.
-func (rtn *reachabilityTreeNode) setInterval(interval *reachabilityInterval) {
-	rtn.interval = interval
-
-	// Reserve a single interval index for the current node. This
-	// is necessary to ensure that ancestor intervals are strictly
-	// supersets of any descendant intervals and not equal
-	rtn.remainingInterval = newReachabilityInterval(interval.start, interval.end-1)
-}
-
-// reindexIntervals traverses the reachability subtree that's
-// defined by this node and reallocates reachability interval space
-// such that another reindexing is unlikely to occur shortly
-// thereafter. It does this by traversing down the reachability
-// tree until it finds a node with a subreeSize that's greater than
-// its interval size. See propagateInterval for further details.
-// This method returns a list of reachabilityTreeNodes modified by it.
-func (rtn *reachabilityTreeNode) reindexIntervals() ([]*reachabilityTreeNode, error) {
-	current := rtn
-
-	// Initial interval and subtree sizes
-	intervalSize := current.interval.size()
-	subTreeSizeMap := make(map[*reachabilityTreeNode]uint64)
-	current.countSubtrees(subTreeSizeMap)
-	currentSubtreeSize := subTreeSizeMap[current]
-
-	// Find the first ancestor that has sufficient interval space
-	for intervalSize < currentSubtreeSize {
-		if current.parent == nil {
-			// If we ended up here it means that there are more
-			// than 2^64 blocks, which shouldn't ever happen.
-			return nil, errors.Errorf("missing tree " +
-				"parent during reindexing. Theoretically, this " +
-				"should only ever happen if there are more " +
-				"than 2^64 blocks in the DAG.")
-		}
-		current = current.parent
-		intervalSize = current.interval.size()
-		current.countSubtrees(subTreeSizeMap)
-		currentSubtreeSize = subTreeSizeMap[current]
-	}
-
-	// Propagate the interval to the subtree
-	return current.propagateInterval(subTreeSizeMap)
-}
-
-// countSubtrees counts the size of each subtree under this node,
-// and populates the provided subTreeSizeMap with the results.
-// It is equivalent to the following recursive implementation:
-//
-// func (rtn *reachabilityTreeNode) countSubtrees() uint64 {
-//     subtreeSize := uint64(0)
-//     for _, child := range rtn.children {
-//         subtreeSize += child.countSubtrees()
-//     }
-//     return subtreeSize + 1
-// }
-//
-// However, we are expecting (linearly) deep trees, and so a
-// recursive stack-based approach is inefficient and will hit
-// recursion limits. Instead, the same logic was implemented
-// using a (queue-based) BFS method. At a high level, the
-// algorithm uses BFS for reaching all leaves and pushes
-// intermediate updates from leaves via parent chains until all
-// size information is gathered at the root of the operation
-// (i.e. at rtn).
-func (rtn *reachabilityTreeNode) countSubtrees(subTreeSizeMap map[*reachabilityTreeNode]uint64) {
-	queue := []*reachabilityTreeNode{rtn}
-	calculatedChildrenCount := make(map[*reachabilityTreeNode]uint64)
-	for len(queue) > 0 {
-		var current *reachabilityTreeNode
-		current, queue = queue[0], queue[1:]
-		if len(current.children) == 0 {
-			// We reached a leaf
-			subTreeSizeMap[current] = 1
-		} else if calculatedChildrenCount[current] <= uint64(len(current.children)) {
-			// We haven't yet calculated the subtree size of
-			// the current node. Add all its children to the
-			// queue
-			queue = append(queue, current.children...)
-			continue
-		}
-
-		// We reached a leaf or a pre-calculated subtree.
-		// Push information up
-		for current != rtn {
-			current = current.parent
-			calculatedChildrenCount[current]++
-			if calculatedChildrenCount[current] != uint64(len(current.children)) {
-				// Not all subtrees of the current node are ready
-				break
-			}
-			// All children of `current` have calculated their subtree size.
-			// Sum them all together and add 1 to get the sub tree size of
-			// `current`.
-			childSubtreeSizeSum := uint64(0)
-			for _, child := range current.children {
-				childSubtreeSizeSum += subTreeSizeMap[child]
-			}
-			subTreeSizeMap[current] = childSubtreeSizeSum + 1
-		}
-	}
-}
-
-// propagateInterval propagates the new interval using a BFS traversal.
-// Subtree intervals are recursively allocated according to subtree sizes and
-// the allocation rule in splitWithExponentialBias. This method returns
-// a list of reachabilityTreeNodes modified by it.
-func (rtn *reachabilityTreeNode) propagateInterval(subTreeSizeMap map[*reachabilityTreeNode]uint64) ([]*reachabilityTreeNode, error) {
-	// We set the interval to reset its remainingInterval, so we could reallocate it while reindexing.
-	rtn.setInterval(rtn.interval)
-	queue := []*reachabilityTreeNode{rtn}
-	var modifiedNodes []*reachabilityTreeNode
-	for len(queue) > 0 {
-		var current *reachabilityTreeNode
-		current, queue = queue[0], queue[1:]
-		if len(current.children) > 0 {
-			sizes := make([]uint64, len(current.children))
-			for i, child := range current.children {
-				sizes[i] = subTreeSizeMap[child]
-			}
-			intervals, err := current.remainingInterval.splitWithExponentialBias(sizes)
-			if err != nil {
-				return nil, err
-			}
-			for i, child := range current.children {
-				childInterval := intervals[i]
-				child.setInterval(childInterval)
-				queue = append(queue, child)
-			}
-
-			// Empty up remaining interval
-			current.remainingInterval.start = current.remainingInterval.end + 1
-		}
-
-		modifiedNodes = append(modifiedNodes, current)
-	}
-	return modifiedNodes, nil
-}
-
-// isAncestorOf checks if this node is a reachability tree ancestor
-// of the other node.
-func (rtn *reachabilityTreeNode) isAncestorOf(other *reachabilityTreeNode) bool {
-	return rtn.interval.isAncestorOf(other.interval)
-}
-
-// String returns a string representation of a reachability tree node
-// and its children.
-func (rtn *reachabilityTreeNode) String() string {
-	queue := []*reachabilityTreeNode{rtn}
-	lines := []string{rtn.interval.String()}
-	for len(queue) > 0 {
-		var current *reachabilityTreeNode
-		current, queue = queue[0], queue[1:]
-		if len(current.children) == 0 {
-			continue
-		}
-
-		line := ""
-		for _, child := range current.children {
-			line += child.interval.String()
-			queue = append(queue, child)
-		}
-		lines = append([]string{line}, lines...)
-	}
-	return strings.Join(lines, "\n")
-}
-
-// futureCoveringBlockSet represents a collection of blocks in the future of
-// a certain block. Once a block B is added to the DAG, every block A_i in
-// B's selected parent anticone must register B in its futureCoveringBlockSet. This allows
-// to relatively quickly (O(log(|futureCoveringBlockSet|))) query whether B
-// is a descendent (is in the "future") of any block that previously
-// registered it.
-//
-// Note that futureCoveringBlockSet is meant to be queried only if B is not
-// a reachability tree descendant of the block in question, as reachability
-// tree queries are always O(1).
-//
-// See insertBlock, isInFuture, and dag.isAncestorOf for further details.
-type futureCoveringBlockSet []*futureCoveringBlock
-
-// futureCoveringBlock represents a block in the future of some other block.
-type futureCoveringBlock struct {
-	blockNode *blockNode
-	treeNode  *reachabilityTreeNode
-}
-
-// insertBlock inserts the given block into this futureCoveringBlockSet
-// while keeping futureCoveringBlockSet ordered by interval.
-// If a block B ∈ futureCoveringBlockSet exists such that its interval
-// contains block's interval, block need not be added. If block's
-// interval contains B's interval, it replaces it.
-//
-// Notes:
-// * Intervals never intersect unless one contains the other
-//   (this follows from the tree structure and the indexing rule).
-// * Since futureCoveringBlockSet is kept ordered, a binary search can be
-//   used for insertion/queries.
-// * Although reindexing may change a block's interval, the
-//   is-superset relation will by definition
-//   be always preserved.
-func (fb *futureCoveringBlockSet) insertBlock(block *futureCoveringBlock) {
-	blockInterval := block.treeNode.interval
-	i := fb.findIndex(block)
-	if i > 0 {
-		candidate := (*fb)[i-1]
-		candidateInterval := candidate.treeNode.interval
-		if candidateInterval.isAncestorOf(blockInterval) {
-			// candidate is an ancestor of block, no need to insert
-			return
-		}
-		if blockInterval.isAncestorOf(candidateInterval) {
-			// block is an ancestor of candidate, and can thus replace it
-			(*fb)[i-1] = block
-			return
-		}
-	}
-
-	// Insert block in the correct index to maintain futureCoveringBlockSet as
-	// a sorted-by-interval list.
-	// Note that i might be equal to len(futureCoveringBlockSet)
-	left := (*fb)[:i]
-	right := append([]*futureCoveringBlock{block}, (*fb)[i:]...)
-	*fb = append(left, right...)
-}
-
-// isInFuture resolves whether the given block is in the subtree of
-// any block in this futureCoveringBlockSet.
-// See insertBlock method for the complementary insertion behavior.
-//
-// Like the insert method, this method also relies on the fact that
-// futureCoveringBlockSet is kept ordered by interval to efficiently perform a
-// binary search over futureCoveringBlockSet and answer the query in
-// O(log(|futureCoveringBlockSet|)).
-func (fb futureCoveringBlockSet) isInFuture(block *futureCoveringBlock) bool {
-	i := fb.findIndex(block)
-	if i == 0 {
-		// No candidate to contain block
-		return false
-	}
-
-	candidate := fb[i-1]
-	return candidate.treeNode.isAncestorOf(block.treeNode)
-}
-
-// findIndex finds the index of the block with the maximum start that is below
-// the given block.
-func (fb futureCoveringBlockSet) findIndex(block *futureCoveringBlock) int {
-	blockInterval := block.treeNode.interval
-	end := blockInterval.end
-
-	low := 0
-	high := len(fb)
-	for low < high {
-		middle := (low + high) / 2
-		middleInterval := fb[middle].treeNode.interval
-		if end < middleInterval.start {
-			high = middle
-		} else {
-			low = middle + 1
-		}
-	}
-	return low
-}
-
-// String returns a string representation of the intervals in this futureCoveringBlockSet.
-func (fb futureCoveringBlockSet) String() string {
-	intervalsString := ""
-	for _, block := range fb {
-		intervalsString += block.treeNode.interval.String()
-	}
-	return intervalsString
-}
-
-func (dag *BlockDAG) updateReachability(node *blockNode, selectedParentAnticone []*blockNode) error {
-	// Allocate a new reachability tree node
-	newTreeNode := newReachabilityTreeNode(node)
-
-	// If this is the genesis node, simply initialize it and return
-	if node.isGenesis() {
-		dag.reachabilityStore.setTreeNode(newTreeNode)
-		return nil
-	}
-
-	// Insert the node into the selected parent's reachability tree
-	selectedParentTreeNode, err := dag.reachabilityStore.treeNodeByBlockNode(node.selectedParent)
-	if err != nil {
-		return err
-	}
-	modifiedTreeNodes, err := selectedParentTreeNode.addChild(newTreeNode)
-	if err != nil {
-		return err
-	}
-	for _, modifiedTreeNode := range modifiedTreeNodes {
-		dag.reachabilityStore.setTreeNode(modifiedTreeNode)
-	}
-
-	// Add the block to the futureCoveringSets of all the blocks
-	// in the selected parent's anticone
-	for _, current := range selectedParentAnticone {
-		currentFutureCoveringSet, err := dag.reachabilityStore.futureCoveringSetByBlockNode(current)
-		if err != nil {
-			return err
-		}
-		currentFutureCoveringSet.insertBlock(&futureCoveringBlock{blockNode: node, treeNode: newTreeNode})
-		err = dag.reachabilityStore.setFutureCoveringSet(current, currentFutureCoveringSet)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// isAncestorOf returns true if this node is in the past of the other node
-// in the DAG. The complexity of this method is O(log(|this.futureCoveringBlockSet|))
-func (dag *BlockDAG) isAncestorOf(this *blockNode, other *blockNode) (bool, error) {
-	// First, check if this node is a reachability tree ancestor of the
-	// other node
-	thisTreeNode, err := dag.reachabilityStore.treeNodeByBlockNode(this)
-	if err != nil {
-		return false, err
-	}
-	otherTreeNode, err := dag.reachabilityStore.treeNodeByBlockNode(other)
-	if err != nil {
-		return false, err
-	}
-	if thisTreeNode.isAncestorOf(otherTreeNode) {
-		return true, nil
-	}
-
-	// Otherwise, use previously registered future blocks to complete the
-	// reachability test
-	thisFutureCoveringSet, err := dag.reachabilityStore.futureCoveringSetByBlockNode(this)
-	if err != nil {
-		return false, err
-	}
-	return thisFutureCoveringSet.isInFuture(&futureCoveringBlock{blockNode: other, treeNode: otherTreeNode}), nil
-}

blockdag/reachability_test.go

@@ -1,648 +0,0 @@
-package blockdag
-
-import (
-	"reflect"
-	"strings"
-	"testing"
-)
-
-func TestAddChild(t *testing.T) {
-	// Scenario 1: test addChild in a chain
-	//             root -> a -> b -> c...
-	// Create the root node of a new reachability tree
-	root := newReachabilityTreeNode(&blockNode{})
-	root.setInterval(newReachabilityInterval(1, 100))
-
-	// Add a chain of child nodes just before a reindex occurs (2^6=64 < 100)
-	currentTip := root
-	for i := 0; i < 6; i++ {
-		node := newReachabilityTreeNode(&blockNode{})
-		modifiedNodes, err := currentTip.addChild(node)
-		if err != nil {
-			t.Fatalf("TestAddChild: addChild failed: %s", err)
-		}
-
-		// Expect only the node and its parent to be affected
-		expectedModifiedNodes := []*reachabilityTreeNode{currentTip, node}
-		if !reflect.DeepEqual(modifiedNodes, expectedModifiedNodes) {
-			t.Fatalf("TestAddChild: unexpected modifiedNodes. "+
-				"want: %s, got: %s", expectedModifiedNodes, modifiedNodes)
-		}
-
-		currentTip = node
-	}
-
-	// Add another node to the tip of the chain to trigger a reindex (100 < 2^7=128)
-	lastChild := newReachabilityTreeNode(&blockNode{})
-	modifiedNodes, err := currentTip.addChild(lastChild)
-	if err != nil {
-		t.Fatalf("TestAddChild: addChild failed: %s", err)
-	}
-
-	// Expect more than just the node and its parent to be modified but not
-	// all the nodes
-	if len(modifiedNodes) <= 2 && len(modifiedNodes) >= 7 {
-		t.Fatalf("TestAddChild: unexpected amount of modifiedNodes.")
-	}
-
-	// Expect the tip to have an interval of 1 and remaining interval of 0
-	tipInterval := lastChild.interval.size()
-	if tipInterval != 1 {
-		t.Fatalf("TestAddChild: unexpected tip interval size: want: 1, got: %d", tipInterval)
-	}
-	tipRemainingInterval := lastChild.remainingInterval.size()
-	if tipRemainingInterval != 0 {
-		t.Fatalf("TestAddChild: unexpected tip interval size: want: 0, got: %d", tipRemainingInterval)
-	}
-
-	// Expect all nodes to be descendant nodes of root
-	currentNode := currentTip
-	for currentNode != nil {
-		if !root.isAncestorOf(currentNode) {
-			t.Fatalf("TestAddChild: currentNode is not a descendant of root")
-		}
-		currentNode = currentNode.parent
-	}
-
-	// Scenario 2: test addChild where all nodes are direct descendants of root
-	//             root -> a, b, c...
-	// Create the root node of a new reachability tree
-	root = newReachabilityTreeNode(&blockNode{})
-	root.setInterval(newReachabilityInterval(1, 100))
-
-	// Add child nodes to root just before a reindex occurs (2^6=64 < 100)
-	childNodes := make([]*reachabilityTreeNode, 6)
-	for i := 0; i < len(childNodes); i++ {
-		childNodes[i] = newReachabilityTreeNode(&blockNode{})
-		modifiedNodes, err := root.addChild(childNodes[i])
-		if err != nil {
-			t.Fatalf("TestAddChild: addChild failed: %s", err)
-		}
-
-		// Expect only the node and the root to be affected
-		expectedModifiedNodes := []*reachabilityTreeNode{root, childNodes[i]}
-		if !reflect.DeepEqual(modifiedNodes, expectedModifiedNodes) {
-			t.Fatalf("TestAddChild: unexpected modifiedNodes. "+
-				"want: %s, got: %s", expectedModifiedNodes, modifiedNodes)
-		}
-	}
-
-	// Add another node to the root to trigger a reindex (100 < 2^7=128)
-	lastChild = newReachabilityTreeNode(&blockNode{})
-	modifiedNodes, err = root.addChild(lastChild)
-	if err != nil {
-		t.Fatalf("TestAddChild: addChild failed: %s", err)
-	}
-
-	// Expect more than just the node and the root to be modified but not
-	// all the nodes
-	if len(modifiedNodes) <= 2 && len(modifiedNodes) >= 7 {
-		t.Fatalf("TestAddChild: unexpected amount of modifiedNodes.")
-	}
-
-	// Expect the last-added child to have an interval of 1 and remaining interval of 0
-	lastChildInterval := lastChild.interval.size()
-	if lastChildInterval != 1 {
-		t.Fatalf("TestAddChild: unexpected lastChild interval size: want: 1, got: %d", lastChildInterval)
-	}
-	lastChildRemainingInterval := lastChild.remainingInterval.size()
-	if lastChildRemainingInterval != 0 {
-		t.Fatalf("TestAddChild: unexpected lastChild interval size: want: 0, got: %d", lastChildRemainingInterval)
-	}
-
-	// Expect all nodes to be descendant nodes of root
-	for _, childNode := range childNodes {
-		if !root.isAncestorOf(childNode) {
-			t.Fatalf("TestAddChild: childNode is not a descendant of root")
-		}
-	}
-}
-
-func TestSplitFraction(t *testing.T) {
-	tests := []struct {
-		interval      *reachabilityInterval
-		fraction      float64
-		expectedLeft  *reachabilityInterval
-		expectedRight *reachabilityInterval
-	}{
-		{
-			interval:      newReachabilityInterval(1, 100),
-			fraction:      0.5,
-			expectedLeft:  newReachabilityInterval(1, 50),
-			expectedRight: newReachabilityInterval(51, 100),
-		},
-		{
-			interval:      newReachabilityInterval(2, 100),
-			fraction:      0.5,
-			expectedLeft:  newReachabilityInterval(2, 51),
-			expectedRight: newReachabilityInterval(52, 100),
-		},
-		{
-			interval:      newReachabilityInterval(1, 99),
-			fraction:      0.5,
-			expectedLeft:  newReachabilityInterval(1, 50),
-			expectedRight: newReachabilityInterval(51, 99),
-		},
-		{
-			interval:      newReachabilityInterval(1, 100),
-			fraction:      0.2,
-			expectedLeft:  newReachabilityInterval(1, 20),
-			expectedRight: newReachabilityInterval(21, 100),
-		},
-		{
-			interval:      newReachabilityInterval(1, 100),
-			fraction:      0,
-			expectedLeft:  newReachabilityInterval(1, 0),
-			expectedRight: newReachabilityInterval(1, 100),
-		},
-		{
-			interval:      newReachabilityInterval(1, 100),
-			fraction:      1,
-			expectedLeft:  newReachabilityInterval(1, 100),
-			expectedRight: newReachabilityInterval(101, 100),
-		},
-	}
-
-	for i, test := range tests {
-		left, right, err := test.interval.splitFraction(test.fraction)
-		if err != nil {
-			t.Fatalf("TestSplitFraction: splitFraction unexpectedly failed in test #%d: %s", i, err)
-		}
-		if !reflect.DeepEqual(left, test.expectedLeft) {
-			t.Errorf("TestSplitFraction: unexpected left in test #%d. "+
-				"want: %s, got: %s", i, test.expectedLeft, left)
-		}
-		if !reflect.DeepEqual(right, test.expectedRight) {
-			t.Errorf("TestSplitFraction: unexpected right in test #%d. "+
-				"want: %s, got: %s", i, test.expectedRight, right)
-		}
-	}
-}
-
-func TestSplitExact(t *testing.T) {
-	tests := []struct {
-		interval          *reachabilityInterval
-		sizes             []uint64
-		expectedIntervals []*reachabilityInterval
-	}{
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{100},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{50, 50},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 50),
-				newReachabilityInterval(51, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{10, 20, 30, 40},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 10),
-				newReachabilityInterval(11, 30),
-				newReachabilityInterval(31, 60),
-				newReachabilityInterval(61, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{0, 100},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 0),
-				newReachabilityInterval(1, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{100, 0},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 100),
-				newReachabilityInterval(101, 100),
-			},
-		},
-	}
-
-	for i, test := range tests {
-		intervals, err := test.interval.splitExact(test.sizes)
-		if err != nil {
-			t.Fatalf("TestSplitExact: splitExact unexpectedly failed in test #%d: %s", i, err)
-		}
-		if !reflect.DeepEqual(intervals, test.expectedIntervals) {
-			t.Errorf("TestSplitExact: unexpected intervals in test #%d. "+
-				"want: %s, got: %s", i, test.expectedIntervals, intervals)
-		}
-	}
-}
-
-func TestSplitWithExponentialBias(t *testing.T) {
-	tests := []struct {
-		interval          *reachabilityInterval
-		sizes             []uint64
-		expectedIntervals []*reachabilityInterval
-	}{
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{100},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{50, 50},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 50),
-				newReachabilityInterval(51, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{10, 20, 30, 40},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 10),
-				newReachabilityInterval(11, 30),
-				newReachabilityInterval(31, 60),
-				newReachabilityInterval(61, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{25, 25},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 50),
-				newReachabilityInterval(51, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{1, 1},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 50),
-				newReachabilityInterval(51, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{33, 33, 33},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 33),
-				newReachabilityInterval(34, 66),
-				newReachabilityInterval(67, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{10, 15, 25},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 10),
-				newReachabilityInterval(11, 25),
-				newReachabilityInterval(26, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100),
-			sizes:    []uint64{25, 15, 10},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 75),
-				newReachabilityInterval(76, 90),
-				newReachabilityInterval(91, 100),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 10_000),
-			sizes:    []uint64{10, 10, 20},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 20),
-				newReachabilityInterval(21, 40),
-				newReachabilityInterval(41, 10_000),
-			},
-		},
-		{
-			interval: newReachabilityInterval(1, 100_000),
-			sizes:    []uint64{31_000, 31_000, 30_001},
-			expectedIntervals: []*reachabilityInterval{
-				newReachabilityInterval(1, 35_000),
-				newReachabilityInterval(35_001, 69_999),
-				newReachabilityInterval(70_000, 100_000),
-			},
-		},
-	}
-
-	for i, test := range tests {
-		intervals, err := test.interval.splitWithExponentialBias(test.sizes)
-		if err != nil {
-			t.Fatalf("TestSplitWithExponentialBias: splitWithExponentialBias unexpectedly failed in test #%d: %s", i, err)
-		}
-		if !reflect.DeepEqual(intervals, test.expectedIntervals) {
-			t.Errorf("TestSplitWithExponentialBias: unexpected intervals in test #%d. "+
-				"want: %s, got: %s", i, test.expectedIntervals, intervals)
-		}
-	}
-}
-
-func TestIsInFuture(t *testing.T) {
-	blocks := futureCoveringBlockSet{
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(2, 3)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-	}
-
-	tests := []struct {
-		block          *futureCoveringBlock
-		expectedResult bool
-	}{
-		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 1)}},
-			expectedResult: false,
-		},
-		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(5, 7)}},
-			expectedResult: true,
-		},
-		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 76)}},
-			expectedResult: true,
-		},
-		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(78, 100)}},
-			expectedResult: false,
-		},
-		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1980, 2000)}},
-			expectedResult: false,
-		},
-		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1920)}},
-			expectedResult: true,
-		},
-	}
-
-	for i, test := range tests {
-		result := blocks.isInFuture(test.block)
-		if result != test.expectedResult {
-			t.Errorf("TestIsInFuture: unexpected result in test #%d. Want: %t, got: %t",
-				i, test.expectedResult, result)
-		}
-	}
-}
-
-func TestInsertBlock(t *testing.T) {
-	blocks := futureCoveringBlockSet{
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-	}
-
-	tests := []struct {
-		toInsert       []*futureCoveringBlock
-		expectedResult futureCoveringBlockSet
-	}{
-		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(5, 7)}},
-			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-			},
-		},
-		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(65, 78)}},
-			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(65, 78)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-			},
-		},
-		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-			},
-		},
-		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(3000, 3010)}},
-			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(3000, 3010)}},
-			},
-		},
-	}
-
-	for i, test := range tests {
-		// Create a clone of blocks so that we have a clean start for every test
-		blocksClone := make(futureCoveringBlockSet, len(blocks))
-		for i, block := range blocks {
-			blocksClone[i] = block
-		}
-
-		for _, block := range test.toInsert {
-			blocksClone.insertBlock(block)
-		}
-		if !reflect.DeepEqual(blocksClone, test.expectedResult) {
-			t.Errorf("TestInsertBlock: unexpected result in test #%d. Want: %s, got: %s",
-				i, test.expectedResult, blocksClone)
-		}
-	}
-}
-
-func TestSplitFractionErrors(t *testing.T) {
-	interval := newReachabilityInterval(100, 200)
-
-	// Negative fraction
-	_, _, err := interval.splitFraction(-0.5)
-	if err == nil {
-		t.Fatalf("TestSplitFractionErrors: splitFraction unexpectedly " +
-			"didn't return an error for a negative fraction")
-	}
-	expectedErrSubstring := "fraction must be between 0 and 1"
-	if !strings.Contains(err.Error(), expectedErrSubstring) {
-		t.Fatalf("TestSplitFractionErrors: splitFraction returned wrong error "+
-			"for a negative fraction. "+
-			"Want: %s, got: %s", expectedErrSubstring, err)
-	}
-
-	// Fraction > 1
-	_, _, err = interval.splitFraction(1.5)
-	if err == nil {
-		t.Fatalf("TestSplitFractionErrors: splitFraction unexpectedly " +
-			"didn't return an error for a fraction greater than 1")
-	}
-	expectedErrSubstring = "fraction must be between 0 and 1"
-	if !strings.Contains(err.Error(), expectedErrSubstring) {
-		t.Fatalf("TestSplitFractionErrors: splitFraction returned wrong error "+
-			"for a fraction greater than 1. "+
-			"Want: %s, got: %s", expectedErrSubstring, err)
-	}
-
-	// Splitting an empty interval
-	emptyInterval := newReachabilityInterval(1, 0)
-	_, _, err = emptyInterval.splitFraction(0.5)
-	if err == nil {
-		t.Fatalf("TestSplitFractionErrors: splitFraction unexpectedly " +
-			"didn't return an error for an empty interval")
-	}
-	expectedErrSubstring = "cannot split an empty interval"
-	if !strings.Contains(err.Error(), expectedErrSubstring) {
-		t.Fatalf("TestSplitFractionErrors: splitFraction returned wrong error "+
-			"for an empty interval. "+
-			"Want: %s, got: %s", expectedErrSubstring, err)
-	}
-}
-
-func TestSplitExactErrors(t *testing.T) {
-	interval := newReachabilityInterval(100, 199)
-
-	// Sum of sizes greater than the size of the interval
-	sizes := []uint64{50, 51}
-	_, err := interval.splitExact(sizes)
-	if err == nil {
-		t.Fatalf("TestSplitExactErrors: splitExact unexpectedly " +
-			"didn't return an error for (sum of sizes) > (size of interval)")
-	}
-	expectedErrSubstring := "sum of sizes must be equal to the interval's size"
-	if !strings.Contains(err.Error(), expectedErrSubstring) {
-		t.Fatalf("TestSplitExactErrors: splitExact returned wrong error "+
-			"for (sum of sizes) > (size of interval). "+
-			"Want: %s, got: %s", expectedErrSubstring, err)
-	}
-
-	// Sum of sizes smaller than the size of the interval
-	sizes = []uint64{50, 49}
-	_, err = interval.splitExact(sizes)
-	if err == nil {
-		t.Fatalf("TestSplitExactErrors: splitExact unexpectedly " +
-			"didn't return an error for (sum of sizes) < (size of interval)")
-	}
-	expectedErrSubstring = "sum of sizes must be equal to the interval's size"
-	if !strings.Contains(err.Error(), expectedErrSubstring) {
-		t.Fatalf("TestSplitExactErrors: splitExact returned wrong error "+
-			"for (sum of sizes) < (size of interval). "+
-			"Want: %s, got: %s", expectedErrSubstring, err)
-	}
-}
-
-func TestSplitWithExponentialBiasErrors(t *testing.T) {
-	interval := newReachabilityInterval(100, 199)
-
-	// Sum of sizes greater than the size of the interval
-	sizes := []uint64{50, 51}
-	_, err := interval.splitWithExponentialBias(sizes)
-	if err == nil {
-		t.Fatalf("TestSplitWithExponentialBiasErrors: splitWithExponentialBias " +
-			"unexpectedly didn't return an error")
-	}
-	expectedErrSubstring := "sum of sizes must be less than or equal to the interval's size"
-	if !strings.Contains(err.Error(), expectedErrSubstring) {
-		t.Fatalf("TestSplitWithExponentialBiasErrors: splitWithExponentialBias "+
-			"returned wrong error. Want: %s, got: %s", expectedErrSubstring, err)
-	}
-}
-
-func TestReindexIntervalErrors(t *testing.T) {
-	// Create a treeNode and give it size = 100
-	treeNode := newReachabilityTreeNode(&blockNode{})
-	treeNode.setInterval(newReachabilityInterval(0, 99))
-
-	// Add a chain of 100 child treeNodes to treeNode
-	var err error
-	currentTreeNode := treeNode
-	for i := 0; i < 100; i++ {
-		childTreeNode := newReachabilityTreeNode(&blockNode{})
-		_, err = currentTreeNode.addChild(childTreeNode)
-		if err != nil {
-			break
-		}
-		currentTreeNode = childTreeNode
-	}
-
-	// At the 100th addChild we expect a reindex. This reindex should
-	// fail because our initial treeNode only has size = 100, and the
-	// reindex requires size > 100.
-	// This simulates the case when (somehow) there's more than 2^64
-	// blocks in the DAG, since the genesis block has size = 2^64.
-	if err == nil {
-		t.Fatalf("TestReindexIntervalErrors: reindexIntervals " +
-			"unexpectedly didn't return an error")
-	}
-	if !strings.Contains(err.Error(), "missing tree parent during reindexing") {
-		t.Fatalf("TestReindexIntervalErrors: reindexIntervals "+
-			"returned an expected error: %s", err)
-	}
-}
-
-func TestFutureCoveringBlockSetString(t *testing.T) {
-	treeNodeA := newReachabilityTreeNode(&blockNode{})
-	treeNodeA.setInterval(newReachabilityInterval(123, 456))
-	treeNodeB := newReachabilityTreeNode(&blockNode{})
-	treeNodeB.setInterval(newReachabilityInterval(457, 789))
-	futureCoveringSet := futureCoveringBlockSet{
-		&futureCoveringBlock{treeNode: treeNodeA},
-		&futureCoveringBlock{treeNode: treeNodeB},
-	}
-
-	str := futureCoveringSet.String()
-	expectedStr := "[123,456][457,789]"
-	if str != expectedStr {
-		t.Fatalf("TestFutureCoveringBlockSetString: unexpected "+
-			"string. Want: %s, got: %s", expectedStr, str)
-	}
-}
-
-func TestReachabilityTreeNodeString(t *testing.T) {
-	treeNodeA := newReachabilityTreeNode(&blockNode{})
-	treeNodeA.setInterval(newReachabilityInterval(100, 199))
-	treeNodeB1 := newReachabilityTreeNode(&blockNode{})
-	treeNodeB1.setInterval(newReachabilityInterval(100, 150))
-	treeNodeB2 := newReachabilityTreeNode(&blockNode{})
-	treeNodeB2.setInterval(newReachabilityInterval(150, 199))
-	treeNodeC := newReachabilityTreeNode(&blockNode{})
-	treeNodeC.setInterval(newReachabilityInterval(100, 149))
-	treeNodeA.children = []*reachabilityTreeNode{treeNodeB1, treeNodeB2}
-	treeNodeB2.children = []*reachabilityTreeNode{treeNodeC}
-
-	str := treeNodeA.String()
-	expectedStr := "[100,149]\n[100,150][150,199]\n[100,199]"
-	if str != expectedStr {
-		t.Fatalf("TestReachabilityTreeNodeString: unexpected "+
-			"string. Want: %s, got: %s", expectedStr, str)
-	}
-}

blockdag/test_utils_test.go

@@ -1,14 +0,0 @@
-package blockdag
-
-import (
-	"testing"
-)
-
-func TestIsSupportedDbType(t *testing.T) {
-	if !isSupportedDbType("ffldb") {
-		t.Errorf("ffldb should be a supported DB driver")
-	}
-	if isSupportedDbType("madeUpDb") {
-		t.Errorf("madeUpDb should not be a supported DB driver")
-	}
-}

blockdag/thresholdstate.go

@@ -1,355 +0,0 @@
-// Copyright (c) 2016-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"fmt"
-
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// ThresholdState define the various threshold states used when voting on
-// consensus changes.
-type ThresholdState byte
-
-// These constants are used to identify specific threshold states.
-const (
-	// ThresholdDefined is the first state for each deployment and is the
-	// state for the genesis block has by definition for all deployments.
-	ThresholdDefined ThresholdState = iota
-
-	// ThresholdStarted is the state for a deployment once its start time
-	// has been reached.
-	ThresholdStarted
-
-	// ThresholdLockedIn is the state for a deployment during the retarget
-	// period which is after the ThresholdStarted state period and the
-	// number of blocks that have voted for the deployment equal or exceed
-	// the required number of votes for the deployment.
-	ThresholdLockedIn
-
-	// ThresholdActive is the state for a deployment for all blocks after a
-	// retarget period in which the deployment was in the ThresholdLockedIn
-	// state.
-	ThresholdActive
-
-	// ThresholdFailed is the state for a deployment once its expiration
-	// time has been reached and it did not reach the ThresholdLockedIn
-	// state.
-	ThresholdFailed
-
-	// numThresholdsStates is the maximum number of threshold states used in
-	// tests.
-	numThresholdsStates
-)
-
-// thresholdStateStrings is a map of ThresholdState values back to their
-// constant names for pretty printing.
-var thresholdStateStrings = map[ThresholdState]string{
-	ThresholdDefined:  "ThresholdDefined",
-	ThresholdStarted:  "ThresholdStarted",
-	ThresholdLockedIn: "ThresholdLockedIn",
-	ThresholdActive:   "ThresholdActive",
-	ThresholdFailed:   "ThresholdFailed",
-}
-
-// String returns the ThresholdState as a human-readable name.
-func (t ThresholdState) String() string {
-	if s := thresholdStateStrings[t]; s != "" {
-		return s
-	}
-	return fmt.Sprintf("Unknown ThresholdState (%d)", int(t))
-}
-
-// thresholdConditionChecker provides a generic interface that is invoked to
-// determine when a consensus rule change threshold should be changed.
-type thresholdConditionChecker interface {
-	// BeginTime returns the unix timestamp for the median block time after
-	// which voting on a rule change starts (at the next window).
-	BeginTime() uint64
-
-	// EndTime returns the unix timestamp for the median block time after
-	// which an attempted rule change fails if it has not already been
-	// locked in or activated.
-	EndTime() uint64
-
-	// RuleChangeActivationThreshold is the number of blocks for which the
-	// condition must be true in order to lock in a rule change.
-	RuleChangeActivationThreshold() uint64
-
-	// MinerConfirmationWindow is the number of blocks in each threshold
-	// state retarget window.
-	MinerConfirmationWindow() uint64
-
-	// Condition returns whether or not the rule change activation condition
-	// has been met. This typically involves checking whether or not the
-	// bit associated with the condition is set, but can be more complex as
-	// needed.
-	Condition(*blockNode) (bool, error)
-}
-
-// thresholdStateCache provides a type to cache the threshold states of each
-// threshold window for a set of IDs.
-type thresholdStateCache struct {
-	entries map[daghash.Hash]ThresholdState
-}
-
-// Lookup returns the threshold state associated with the given hash along with
-// a boolean that indicates whether or not it is valid.
-func (c *thresholdStateCache) Lookup(hash *daghash.Hash) (ThresholdState, bool) {
-	state, ok := c.entries[*hash]
-	return state, ok
-}
-
-// Update updates the cache to contain the provided hash to threshold state
-// mapping.
-func (c *thresholdStateCache) Update(hash *daghash.Hash, state ThresholdState) {
-	c.entries[*hash] = state
-}
-
-// newThresholdCaches returns a new array of caches to be used when calculating
-// threshold states.
-func newThresholdCaches(numCaches uint32) []thresholdStateCache {
-	caches := make([]thresholdStateCache, numCaches)
-	for i := 0; i < len(caches); i++ {
-		caches[i] = thresholdStateCache{
-			entries: make(map[daghash.Hash]ThresholdState),
-		}
-	}
-	return caches
-}
-
-// thresholdState returns the current rule change threshold state for the block
-// AFTER the given node and deployment ID. The cache is used to ensure the
-// threshold states for previous windows are only calculated once.
-//
-// This function MUST be called with the DAG state lock held (for writes).
-func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdConditionChecker, cache *thresholdStateCache) (ThresholdState, error) {
-	// The threshold state for the window that contains the genesis block is
-	// defined by definition.
-	confirmationWindow := checker.MinerConfirmationWindow()
-	if prevNode == nil || (prevNode.blueScore+1) < confirmationWindow {
-		return ThresholdDefined, nil
-	}
-
-	// Get the ancestor that is the last block of the previous confirmation
-	// window in order to get its threshold state. This can be done because
-	// the state is the same for all blocks within a given window.
-	prevNode = prevNode.SelectedAncestor(prevNode.blueScore -
-		(prevNode.blueScore+1)%confirmationWindow)
-
-	// Iterate backwards through each of the previous confirmation windows
-	// to find the most recently cached threshold state.
-	var neededStates []*blockNode
-	for prevNode != nil {
-		// Nothing more to do if the state of the block is already
-		// cached.
-		if _, ok := cache.Lookup(prevNode.hash); ok {
-			break
-		}
-
-		// The start and expiration times are based on the median block
-		// time, so calculate it now.
-		medianTime := prevNode.PastMedianTime(dag)
-
-		// The state is simply defined if the start time hasn't been
-		// been reached yet.
-		if uint64(medianTime.Unix()) < checker.BeginTime() {
-			cache.Update(prevNode.hash, ThresholdDefined)
-			break
-		}
-
-		// Add this node to the list of nodes that need the state
-		// calculated and cached.
-		neededStates = append(neededStates, prevNode)
-
-		// Get the ancestor that is the last block of the previous
-		// confirmation window.
-		prevNode = prevNode.RelativeAncestor(confirmationWindow)
-	}
-
-	// Start with the threshold state for the most recent confirmation
-	// window that has a cached state.
-	state := ThresholdDefined
-	if prevNode != nil {
-		var ok bool
-		state, ok = cache.Lookup(prevNode.hash)
-		if !ok {
-			return ThresholdFailed, AssertError(fmt.Sprintf(
-				"thresholdState: cache lookup failed for %s",
-				prevNode.hash))
-		}
-	}
-
-	// Since each threshold state depends on the state of the previous
-	// window, iterate starting from the oldest unknown window.
-	for neededNum := len(neededStates) - 1; neededNum >= 0; neededNum-- {
-		prevNode := neededStates[neededNum]
-
-		switch state {
-		case ThresholdDefined:
-			// The deployment of the rule change fails if it expires
-			// before it is accepted and locked in.
-			medianTime := prevNode.PastMedianTime(dag)
-			medianTimeUnix := uint64(medianTime.Unix())
-			if medianTimeUnix >= checker.EndTime() {
-				state = ThresholdFailed
-				break
-			}
-
-			// The state for the rule moves to the started state
-			// once its start time has been reached (and it hasn't
-			// already expired per the above).
-			if medianTimeUnix >= checker.BeginTime() {
-				state = ThresholdStarted
-			}
-
-		case ThresholdStarted:
-			// The deployment of the rule change fails if it expires
-			// before it is accepted and locked in.
-			medianTime := prevNode.PastMedianTime(dag)
-			if uint64(medianTime.Unix()) >= checker.EndTime() {
-				state = ThresholdFailed
-				break
-			}
-
-			// At this point, the rule change is still being voted
-			// on by the miners, so iterate backwards through the
-			// confirmation window to count all of the votes in it.
-			var count uint64
-			windowNodes := make([]*blockNode, 0, confirmationWindow)
-			windowNodes = append(windowNodes, prevNode)
-			windowNodes = append(windowNodes, blueBlockWindow(prevNode, confirmationWindow-1)...)
-			for _, current := range windowNodes {
-				condition, err := checker.Condition(current)
-				if err != nil {
-					return ThresholdFailed, err
-				}
-				if condition {
-					count++
-				}
-			}
-
-			// The state is locked in if the number of blocks in the
-			// period that voted for the rule change meets the
-			// activation threshold.
-			if count >= checker.RuleChangeActivationThreshold() {
-				state = ThresholdLockedIn
-			}
-
-		case ThresholdLockedIn:
-			// The new rule becomes active when its previous state
-			// was locked in.
-			state = ThresholdActive
-
-			// Nothing to do if the previous state is active or failed since
-			// they are both terminal states.
-		case ThresholdActive:
-		case ThresholdFailed:
-		}
-
-		// Update the cache to avoid recalculating the state in the
-		// future.
-		cache.Update(prevNode.hash, state)
-	}
-
-	return state, nil
-}
-
-// ThresholdState returns the current rule change threshold state of the given
-// deployment ID for the block AFTER the blueScore of the current DAG.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) ThresholdState(deploymentID uint32) (ThresholdState, error) {
-	dag.dagLock.Lock()
-	defer dag.dagLock.Unlock()
-	state, err := dag.deploymentState(dag.selectedTip(), deploymentID)
-
-	return state, err
-}
-
-// IsDeploymentActive returns true if the target deploymentID is active, and
-// false otherwise.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) IsDeploymentActive(deploymentID uint32) (bool, error) {
-	dag.dagLock.Lock()
-	defer dag.dagLock.Unlock()
-	state, err := dag.deploymentState(dag.selectedTip(), deploymentID)
-	if err != nil {
-		return false, err
-	}
-
-	return state == ThresholdActive, nil
-}
-
-// deploymentState returns the current rule change threshold for a given
-// deploymentID. The threshold is evaluated from the point of view of the block
-// node passed in as the first argument to this method.
-//
-// It is important to note that, as the variable name indicates, this function
-// expects the block node prior to the block for which the deployment state is
-// desired. In other words, the returned deployment state is for the block
-// AFTER the passed node.
-//
-// This function MUST be called with the DAG state lock held (for writes).
-func (dag *BlockDAG) deploymentState(prevNode *blockNode, deploymentID uint32) (ThresholdState, error) {
-	if deploymentID > uint32(len(dag.dagParams.Deployments)) {
-		return ThresholdFailed, DeploymentError(deploymentID)
-	}
-
-	deployment := &dag.dagParams.Deployments[deploymentID]
-	checker := deploymentChecker{deployment: deployment, dag: dag}
-	cache := &dag.deploymentCaches[deploymentID]
-
-	return dag.thresholdState(prevNode, checker, cache)
-}
-
-// initThresholdCaches initializes the threshold state caches for each warning
-// bit and defined deployment and provides warnings if the DAG is current per
-// the warnUnknownVersions and warnUnknownRuleActivations functions.
-func (dag *BlockDAG) initThresholdCaches() error {
-	// Initialize the warning and deployment caches by calculating the
-	// threshold state for each of them. This will ensure the caches are
-	// populated and any states that needed to be recalculated due to
-	// definition changes is done now.
-	prevNode := dag.selectedTip().selectedParent
-	for bit := uint32(0); bit < vbNumBits; bit++ {
-		checker := bitConditionChecker{bit: bit, dag: dag}
-		cache := &dag.warningCaches[bit]
-		_, err := dag.thresholdState(prevNode, checker, cache)
-		if err != nil {
-			return err
-		}
-	}
-	for id := 0; id < len(dag.dagParams.Deployments); id++ {
-		deployment := &dag.dagParams.Deployments[id]
-		cache := &dag.deploymentCaches[id]
-		checker := deploymentChecker{deployment: deployment, dag: dag}
-		_, err := dag.thresholdState(prevNode, checker, cache)
-		if err != nil {
-			return err
-		}
-	}
-
-	// No warnings about unknown rules or versions until the DAG is
-	// current.
-	if dag.isCurrent() {
-		// Warn if a high enough percentage of the last blocks have
-		// unexpected versions.
-		bestNode := dag.selectedTip()
-		if err := dag.warnUnknownVersions(bestNode); err != nil {
-			return err
-		}
-
-		// Warn if any unknown new rules are either about to activate or
-		// have already been activated.
-		if err := dag.warnUnknownRuleActivations(bestNode); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

blockdag/thresholdstate_test.go

@@ -1,134 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"testing"
-
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// TestThresholdStateStringer tests the stringized output for the
-// ThresholdState type.
-func TestThresholdStateStringer(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		in   ThresholdState
-		want string
-	}{
-		{ThresholdDefined, "ThresholdDefined"},
-		{ThresholdStarted, "ThresholdStarted"},
-		{ThresholdLockedIn, "ThresholdLockedIn"},
-		{ThresholdActive, "ThresholdActive"},
-		{ThresholdFailed, "ThresholdFailed"},
-		{0xff, "Unknown ThresholdState (255)"},
-	}
-
-	// Detect additional threshold states that don't have the stringer added.
-	if len(tests)-1 != int(numThresholdsStates) {
-		t.Errorf("It appears a threshold statewas added without " +
-			"adding an associated stringer test")
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.String()
-		if result != test.want {
-			t.Errorf("String #%d\n got: %s want: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-// TestThresholdStateCache ensure the threshold state cache works as intended
-// including adding entries, updating existing entries, and flushing.
-func TestThresholdStateCache(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name       string
-		numEntries int
-		state      ThresholdState
-	}{
-		{name: "2 entries defined", numEntries: 2, state: ThresholdDefined},
-		{name: "7 entries started", numEntries: 7, state: ThresholdStarted},
-		{name: "10 entries active", numEntries: 10, state: ThresholdActive},
-		{name: "5 entries locked in", numEntries: 5, state: ThresholdLockedIn},
-		{name: "3 entries failed", numEntries: 3, state: ThresholdFailed},
-	}
-
-nextTest:
-	for _, test := range tests {
-		cache := &newThresholdCaches(1)[0]
-		for i := 0; i < test.numEntries; i++ {
-			var hash daghash.Hash
-			hash[0] = uint8(i + 1)
-
-			// Ensure the hash isn't available in the cache already.
-			_, ok := cache.Lookup(&hash)
-			if ok {
-				t.Errorf("Lookup (%s): has entry for hash %v",
-					test.name, hash)
-				continue nextTest
-			}
-
-			// Ensure hash that was added to the cache reports it's
-			// available and the state is the expected value.
-			cache.Update(&hash, test.state)
-			state, ok := cache.Lookup(&hash)
-			if !ok {
-				t.Errorf("Lookup (%s): missing entry for hash "+
-					"%v", test.name, hash)
-				continue nextTest
-			}
-			if state != test.state {
-				t.Errorf("Lookup (%s): state mismatch - got "+
-					"%v, want %v", test.name, state,
-					test.state)
-				continue nextTest
-			}
-
-			// Ensure adding an existing hash with the same state
-			// doesn't break the existing entry.
-			cache.Update(&hash, test.state)
-			state, ok = cache.Lookup(&hash)
-			if !ok {
-				t.Errorf("Lookup (%s): missing entry after "+
-					"second add for hash %v", test.name,
-					hash)
-				continue nextTest
-			}
-			if state != test.state {
-				t.Errorf("Lookup (%s): state mismatch after "+
-					"second add - got %v, want %v",
-					test.name, state, test.state)
-				continue nextTest
-			}
-
-			// Ensure adding an existing hash with a different state
-			// updates the existing entry.
-			newState := ThresholdFailed
-			if newState == test.state {
-				newState = ThresholdStarted
-			}
-			cache.Update(&hash, newState)
-			state, ok = cache.Lookup(&hash)
-			if !ok {
-				t.Errorf("Lookup (%s): missing entry after "+
-					"state change for hash %v", test.name,
-					hash)
-				continue nextTest
-			}
-			if state != newState {
-				t.Errorf("Lookup (%s): state mismatch after "+
-					"state change - got %v, want %v",
-					test.name, state, newState)
-				continue nextTest
-			}
-		}
-	}
-}

blockdag/utxodiffstore.go

@@ -1,206 +0,0 @@
-package blockdag
-
-import (
-	"bytes"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/util/locks"
-	"github.com/pkg/errors"
-)
-
-type blockUTXODiffData struct {
-	diff      *UTXODiff
-	diffChild *blockNode
-}
-
-type utxoDiffStore struct {
-	dag    *BlockDAG
-	dirty  map[daghash.Hash]struct{}
-	loaded map[daghash.Hash]*blockUTXODiffData
-	mtx    *locks.PriorityMutex
-}
-
-func newUTXODiffStore(dag *BlockDAG) *utxoDiffStore {
-	return &utxoDiffStore{
-		dag:    dag,
-		dirty:  make(map[daghash.Hash]struct{}),
-		loaded: make(map[daghash.Hash]*blockUTXODiffData),
-		mtx:    locks.NewPriorityMutex(),
-	}
-}
-
-func (diffStore *utxoDiffStore) setBlockDiff(node *blockNode, diff *UTXODiff) error {
-	diffStore.mtx.HighPriorityWriteLock()
-	defer diffStore.mtx.HighPriorityWriteUnlock()
-	// load the diff data from DB to diffStore.loaded
-	_, exists, err := diffStore.diffDataByHash(node.hash)
-	if err != nil {
-		return err
-	}
-	if !exists {
-		diffStore.loaded[*node.hash] = &blockUTXODiffData{}
-	}
-
-	diffStore.loaded[*node.hash].diff = diff
-	diffStore.setBlockAsDirty(node.hash)
-	return nil
-}
-
-func (diffStore *utxoDiffStore) setBlockDiffChild(node *blockNode, diffChild *blockNode) error {
-	diffStore.mtx.HighPriorityWriteLock()
-	defer diffStore.mtx.HighPriorityWriteUnlock()
-	// load the diff data from DB to diffStore.loaded
-	_, exists, err := diffStore.diffDataByHash(node.hash)
-	if err != nil {
-		return err
-	}
-	if !exists {
-		return diffNotFoundError(node)
-	}
-
-	diffStore.loaded[*node.hash].diffChild = diffChild
-	diffStore.setBlockAsDirty(node.hash)
-	return nil
-}
-
-func (diffStore *utxoDiffStore) removeBlocksDiffData(dbTx database.Tx, blockHashes []*daghash.Hash) error {
-	for _, hash := range blockHashes {
-		err := diffStore.removeBlockDiffData(dbTx, hash)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (diffStore *utxoDiffStore) removeBlockDiffData(dbTx database.Tx, blockHash *daghash.Hash) error {
-	diffStore.mtx.LowPriorityWriteLock()
-	defer diffStore.mtx.LowPriorityWriteUnlock()
-	delete(diffStore.loaded, *blockHash)
-	err := dbRemoveDiffData(dbTx, blockHash)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (diffStore *utxoDiffStore) setBlockAsDirty(blockHash *daghash.Hash) {
-	diffStore.dirty[*blockHash] = struct{}{}
-}
-
-func (diffStore *utxoDiffStore) diffDataByHash(hash *daghash.Hash) (*blockUTXODiffData, bool, error) {
-	if diffData, ok := diffStore.loaded[*hash]; ok {
-		return diffData, true, nil
-	}
-	diffData, err := diffStore.diffDataFromDB(hash)
-	if err != nil {
-		return nil, false, err
-	}
-	exists := diffData != nil
-	if exists {
-		diffStore.loaded[*hash] = diffData
-	}
-	return diffData, exists, nil
-}
-
-func diffNotFoundError(node *blockNode) error {
-	return errors.Errorf("Couldn't find diff data for block %s", node.hash)
-}
-
-func (diffStore *utxoDiffStore) diffByNode(node *blockNode) (*UTXODiff, error) {
-	diffStore.mtx.HighPriorityReadLock()
-	defer diffStore.mtx.HighPriorityReadUnlock()
-	diffData, exists, err := diffStore.diffDataByHash(node.hash)
-	if err != nil {
-		return nil, err
-	}
-	if !exists {
-		return nil, diffNotFoundError(node)
-	}
-	return diffData.diff, nil
-}
-
-func (diffStore *utxoDiffStore) diffChildByNode(node *blockNode) (*blockNode, error) {
-	diffStore.mtx.HighPriorityReadLock()
-	defer diffStore.mtx.HighPriorityReadUnlock()
-	diffData, exists, err := diffStore.diffDataByHash(node.hash)
-	if err != nil {
-		return nil, err
-	}
-	if !exists {
-		return nil, diffNotFoundError(node)
-	}
-	return diffData.diffChild, nil
-}
-
-func (diffStore *utxoDiffStore) diffDataFromDB(hash *daghash.Hash) (*blockUTXODiffData, error) {
-	var diffData *blockUTXODiffData
-	err := diffStore.dag.db.View(func(dbTx database.Tx) error {
-		bucket := dbTx.Metadata().Bucket(utxoDiffsBucketName)
-		serializedBlockDiffData := bucket.Get(hash[:])
-		if serializedBlockDiffData != nil {
-			var err error
-			diffData, err = diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return diffData, nil
-}
-
-// flushToDB writes all dirty diff data to the database. If all writes
-// succeed, this clears the dirty set.
-func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
-	diffStore.mtx.HighPriorityWriteLock()
-	defer diffStore.mtx.HighPriorityWriteUnlock()
-	if len(diffStore.dirty) == 0 {
-		return nil
-	}
-
-	// Allocate a buffer here to avoid needless allocations/grows
-	// while writing each entry.
-	buffer := &bytes.Buffer{}
-	for hash := range diffStore.dirty {
-		hash := hash // Copy hash to a new variable to avoid passing the same pointer
-		buffer.Reset()
-		diffData := diffStore.loaded[hash]
-		err := dbStoreDiffData(dbTx, buffer, &hash, diffData)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (diffStore *utxoDiffStore) clearDirtyEntries() {
-	diffStore.dirty = make(map[daghash.Hash]struct{})
-}
-
-// dbStoreDiffData stores the UTXO diff data to the database.
-// This overwrites the current entry if there exists one.
-func dbStoreDiffData(dbTx database.Tx, writeBuffer *bytes.Buffer, hash *daghash.Hash, diffData *blockUTXODiffData) error {
-	// To avoid a ton of allocs, use the given writeBuffer
-	// instead of allocating one. We expect the buffer to
-	// already be initalized and, in most cases, to already
-	// be large enough to accommodate the serialized data
-	// without growing.
-	err := serializeBlockUTXODiffData(writeBuffer, diffData)
-	if err != nil {
-		return err
-	}
-
-	// Bucket.Put doesn't copy on its own, so we manually
-	// copy here. We do so because we expect the buffer
-	// to be reused once we're done with it.
-	serializedDiffData := make([]byte, writeBuffer.Len())
-	copy(serializedDiffData, writeBuffer.Bytes())
-
-	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Put(hash[:], serializedDiffData)
-}
-
-func dbRemoveDiffData(dbTx database.Tx, hash *daghash.Hash) error {
-	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Delete(hash[:])
-}

blockdag/utxodiffstore_test.go

@@ -1,86 +0,0 @@
-package blockdag
-
-import (
-	"fmt"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-	"reflect"
-	"testing"
-)
-
-func TestUTXODiffStore(t *testing.T) {
-	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", Config{
-		DAGParams: &dagconfig.SimnetParams,
-	})
-	if err != nil {
-		t.Fatalf("TestUTXODiffStore: Failed to setup DAG instance: %v", err)
-	}
-	defer teardownFunc()
-
-	nodeCounter := byte(0)
-	createNode := func() *blockNode {
-		nodeCounter++
-		node := &blockNode{hash: &daghash.Hash{nodeCounter}}
-		dag.index.AddNode(node)
-		return node
-	}
-
-	// Check that an error is returned when asking for non existing node
-	nonExistingNode := createNode()
-	_, err = dag.utxoDiffStore.diffByNode(nonExistingNode)
-	expectedErrString := fmt.Sprintf("Couldn't find diff data for block %s", nonExistingNode.hash)
-	if err == nil || err.Error() != expectedErrString {
-		t.Errorf("diffByNode: expected error %s but got %s", expectedErrString, err)
-	}
-
-	// Add node's diff data to the utxoDiffStore and check if it's checked correctly.
-	node := createNode()
-	diff := NewUTXODiff()
-	diff.toAdd.add(wire.Outpoint{TxID: daghash.TxID{0x01}, Index: 0}, &UTXOEntry{amount: 1, scriptPubKey: []byte{0x01}})
-	diff.toRemove.add(wire.Outpoint{TxID: daghash.TxID{0x02}, Index: 0}, &UTXOEntry{amount: 2, scriptPubKey: []byte{0x02}})
-	if err := dag.utxoDiffStore.setBlockDiff(node, diff); err != nil {
-		t.Fatalf("setBlockDiff: unexpected error: %s", err)
-	}
-	diffChild := createNode()
-	if err := dag.utxoDiffStore.setBlockDiffChild(node, diffChild); err != nil {
-		t.Fatalf("setBlockDiffChild: unexpected error: %s", err)
-	}
-
-	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
-		t.Fatalf("diffByNode: unexpected error: %s", err)
-	} else if !reflect.DeepEqual(storeDiff, diff) {
-		t.Errorf("Expected diff and storeDiff to be equal")
-	}
-
-	if storeDiffChild, err := dag.utxoDiffStore.diffChildByNode(node); err != nil {
-		t.Fatalf("diffByNode: unexpected error: %s", err)
-	} else if !reflect.DeepEqual(storeDiffChild, diffChild) {
-		t.Errorf("Expected diff and storeDiff to be equal")
-	}
-
-	// Flush changes to db, delete them from the dag.utxoDiffStore.loaded
-	// map, and check if the diff data is re-fetched from the database.
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		return dag.utxoDiffStore.flushToDB(dbTx)
-	})
-	if err != nil {
-		t.Fatalf("Error flushing utxoDiffStore data to DB: %s", err)
-	}
-	delete(dag.utxoDiffStore.loaded, *node.hash)
-
-	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
-		t.Fatalf("diffByNode: unexpected error: %s", err)
-	} else if !reflect.DeepEqual(storeDiff, diff) {
-		t.Errorf("Expected diff and storeDiff to be equal")
-	}
-
-	// Check if getBlockDiff caches the result in dag.utxoDiffStore.loaded
-	if loadedDiffData, ok := dag.utxoDiffStore.loaded[*node.hash]; !ok {
-		t.Errorf("the diff data wasn't added to loaded map after requesting it")
-	} else if !reflect.DeepEqual(loadedDiffData.diff, diff) {
-		t.Errorf("Expected diff and loadedDiff to be equal")
-	}
-}

blockdag/versionbits.go

@@ -1,294 +0,0 @@
-// Copyright (c) 2016-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"math"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-)
-
-const (
-	// vbTopBits defines the bits to set in the version to signal that the
-	// version bits scheme is being used.
-	vbTopBits = 0x10000000
-
-	// vbTopMask is the bitmask to use to determine whether or not the
-	// version bits scheme is in use.
-	vbTopMask = 0xe0000000
-
-	// vbNumBits is the total number of bits available for use with the
-	// version bits scheme.
-	vbNumBits = 29
-
-	// unknownVerNumToCheck is the number of previous blocks to consider
-	// when checking for a threshold of unknown block versions for the
-	// purposes of warning the user.
-	unknownVerNumToCheck = 100
-
-	// unknownVerWarnNum is the threshold of previous blocks that have an
-	// unknown version to use for the purposes of warning the user.
-	unknownVerWarnNum = unknownVerNumToCheck / 2
-)
-
-// bitConditionChecker provides a thresholdConditionChecker which can be used to
-// test whether or not a specific bit is set when it's not supposed to be
-// according to the expected version based on the known deployments and the
-// current state of the DAG. This is useful for detecting and warning about
-// unknown rule activations.
-type bitConditionChecker struct {
-	bit uint32
-	dag *BlockDAG
-}
-
-// Ensure the bitConditionChecker type implements the thresholdConditionChecker
-// interface.
-var _ thresholdConditionChecker = bitConditionChecker{}
-
-// BeginTime returns the unix timestamp for the median block time after which
-// voting on a rule change starts (at the next window).
-//
-// Since this implementation checks for unknown rules, it returns 0 so the rule
-// is always treated as active.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) BeginTime() uint64 {
-	return 0
-}
-
-// EndTime returns the unix timestamp for the median block time after which an
-// attempted rule change fails if it has not already been locked in or
-// activated.
-//
-// Since this implementation checks for unknown rules, it returns the maximum
-// possible timestamp so the rule is always treated as active.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) EndTime() uint64 {
-	return math.MaxUint64
-}
-
-// RuleChangeActivationThreshold is the number of blocks for which the condition
-// must be true in order to lock in a rule change.
-//
-// This implementation returns the value defined by the DAG params the checker
-// is associated with.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) RuleChangeActivationThreshold() uint64 {
-	return c.dag.dagParams.RuleChangeActivationThreshold
-}
-
-// MinerConfirmationWindow is the number of blocks in each threshold state
-// retarget window.
-//
-// This implementation returns the value defined by the DAG params the checker
-// is associated with.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) MinerConfirmationWindow() uint64 {
-	return c.dag.dagParams.MinerConfirmationWindow
-}
-
-// Condition returns true when the specific bit associated with the checker is
-// set and it's not supposed to be according to the expected version based on
-// the known deployments and the current state of the DAG.
-//
-// This function MUST be called with the DAG state lock held (for writes).
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) Condition(node *blockNode) (bool, error) {
-	conditionMask := uint32(1) << c.bit
-	version := uint32(node.version)
-	if version&vbTopMask != vbTopBits {
-		return false, nil
-	}
-	if version&conditionMask == 0 {
-		return false, nil
-	}
-
-	expectedVersion, err := c.dag.calcNextBlockVersion(node.selectedParent)
-	if err != nil {
-		return false, err
-	}
-	return uint32(expectedVersion)&conditionMask == 0, nil
-}
-
-// deploymentChecker provides a thresholdConditionChecker which can be used to
-// test a specific deployment rule. This is required for properly detecting
-// and activating consensus rule changes.
-type deploymentChecker struct {
-	deployment *dagconfig.ConsensusDeployment
-	dag        *BlockDAG
-}
-
-// Ensure the deploymentChecker type implements the thresholdConditionChecker
-// interface.
-var _ thresholdConditionChecker = deploymentChecker{}
-
-// BeginTime returns the unix timestamp for the median block time after which
-// voting on a rule change starts (at the next window).
-//
-// This implementation returns the value defined by the specific deployment the
-// checker is associated with.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) BeginTime() uint64 {
-	return c.deployment.StartTime
-}
-
-// EndTime returns the unix timestamp for the median block time after which an
-// attempted rule change fails if it has not already been locked in or
-// activated.
-//
-// This implementation returns the value defined by the specific deployment the
-// checker is associated with.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) EndTime() uint64 {
-	return c.deployment.ExpireTime
-}
-
-// RuleChangeActivationThreshold is the number of blocks for which the condition
-// must be true in order to lock in a rule change.
-//
-// This implementation returns the value defined by the DAG params the checker
-// is associated with.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) RuleChangeActivationThreshold() uint64 {
-	return c.dag.dagParams.RuleChangeActivationThreshold
-}
-
-// MinerConfirmationWindow is the number of blocks in each threshold state
-// retarget window.
-//
-// This implementation returns the value defined by the DAG params the checker
-// is associated with.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) MinerConfirmationWindow() uint64 {
-	return c.dag.dagParams.MinerConfirmationWindow
-}
-
-// Condition returns true when the specific bit defined by the deployment
-// associated with the checker is set.
-//
-// This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) Condition(node *blockNode) (bool, error) {
-	conditionMask := uint32(1) << c.deployment.BitNumber
-	version := uint32(node.version)
-	return (version&vbTopMask == vbTopBits) && (version&conditionMask != 0),
-		nil
-}
-
-// calcNextBlockVersion calculates the expected version of the block after the
-// passed previous block node based on the state of started and locked in
-// rule change deployments.
-//
-// This function differs from the exported CalcNextBlockVersion in that the
-// exported version uses the selected tip as the previous block node
-// while this function accepts any block node.
-//
-// This function MUST be called with the DAG state lock held (for writes).
-func (dag *BlockDAG) calcNextBlockVersion(prevNode *blockNode) (int32, error) {
-	// Set the appropriate bits for each actively defined rule deployment
-	// that is either in the process of being voted on, or locked in for the
-	// activation at the next threshold window change.
-	expectedVersion := uint32(vbTopBits)
-	for id := 0; id < len(dag.dagParams.Deployments); id++ {
-		deployment := &dag.dagParams.Deployments[id]
-		cache := &dag.deploymentCaches[id]
-		checker := deploymentChecker{deployment: deployment, dag: dag}
-		state, err := dag.thresholdState(prevNode, checker, cache)
-		if err != nil {
-			return 0, err
-		}
-		if state == ThresholdStarted || state == ThresholdLockedIn {
-			expectedVersion |= uint32(1) << deployment.BitNumber
-		}
-	}
-	return int32(expectedVersion), nil
-}
-
-// CalcNextBlockVersion calculates the expected version of the block after the
-// end of the current selected tip based on the state of started and locked in
-// rule change deployments.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) CalcNextBlockVersion() (int32, error) {
-	version, err := dag.calcNextBlockVersion(dag.selectedTip())
-	return version, err
-}
-
-// warnUnknownRuleActivations displays a warning when any unknown new rules are
-// either about to activate or have been activated. This will only happen once
-// when new rules have been activated and every block for those about to be
-// activated.
-//
-// This function MUST be called with the DAG state lock held (for writes)
-func (dag *BlockDAG) warnUnknownRuleActivations(node *blockNode) error {
-	// Warn if any unknown new rules are either about to activate or have
-	// already been activated.
-	for bit := uint32(0); bit < vbNumBits; bit++ {
-		checker := bitConditionChecker{bit: bit, dag: dag}
-		cache := &dag.warningCaches[bit]
-		state, err := dag.thresholdState(node.selectedParent, checker, cache)
-		if err != nil {
-			return err
-		}
-
-		switch state {
-		case ThresholdActive:
-			if !dag.unknownRulesWarned {
-				log.Warnf("Unknown new rules activated (bit %d)",
-					bit)
-				dag.unknownRulesWarned = true
-			}
-
-		case ThresholdLockedIn:
-			window := checker.MinerConfirmationWindow()
-			activationBlueScore := window - (node.blueScore % window)
-			log.Warnf("Unknown new rules are about to activate in "+
-				"%d blueScore (bit %d)", activationBlueScore, bit)
-		}
-	}
-
-	return nil
-}
-
-// warnUnknownVersions logs a warning if a high enough percentage of the last
-// blocks have unexpected versions.
-//
-// This function MUST be called with the DAG state lock held (for writes)
-func (dag *BlockDAG) warnUnknownVersions(node *blockNode) error {
-	// Nothing to do if already warned.
-	if dag.unknownVersionsWarned {
-		return nil
-	}
-
-	// Warn if enough previous blocks have unexpected versions.
-	numUpgraded := uint32(0)
-	for i := uint32(0); i < unknownVerNumToCheck && node != nil; i++ {
-		expectedVersion, err := dag.calcNextBlockVersion(node.selectedParent)
-		if err != nil {
-			return err
-		}
-		if (node.version & ^expectedVersion) != 0 {
-
-			numUpgraded++
-		}
-
-		node = node.selectedParent
-	}
-	if numUpgraded > unknownVerWarnNum {
-		log.Warn("Unknown block versions are being mined, so new " +
-			"rules might be in effect. Are you running the " +
-			"latest version of the software?")
-		dag.unknownVersionsWarned = true
-	}
-
-	return nil
-}

cmd/addblock/addblock.go

@@ -5,14 +5,11 @@
 package main
 
 import (
-	"github.com/pkg/errors"
 	"os"
-	"path/filepath"
 	"runtime"
 
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/limits"
-	"github.com/kaspanet/kaspad/logs"
+	"github.com/kaspanet/kaspad/infrastructure/limits"
+	"github.com/kaspanet/kaspad/infrastructure/logs"
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
@@ -24,42 +21,9 @@ const (
 var (
 	cfg   *ConfigFlags
 	log   *logs.Logger
-	spawn func(func())
+	spawn func(string, func())
 )
 
-// loadBlockDB opens the block database and returns a handle to it.
-func loadBlockDB() (database.DB, error) {
-	// The database name is based on the database type.
-	dbName := blockDBNamePrefix + "_" + cfg.DBType
-	dbPath := filepath.Join(cfg.DataDir, dbName)
-
-	log.Infof("Loading block database from '%s'", dbPath)
-	db, err := database.Open(cfg.DBType, dbPath, ActiveConfig().NetParams().Net)
-	if err != nil {
-		// Return the error if it's not because the database doesn't
-		// exist.
-		var dbErr database.Error
-		if ok := errors.As(err, &dbErr); !ok || dbErr.ErrorCode !=
-			database.ErrDbDoesNotExist {
-
-			return nil, err
-		}
-
-		// Create the db if it does not exist.
-		err = os.MkdirAll(cfg.DataDir, 0700)
-		if err != nil {
-			return nil, err
-		}
-		db, err = database.Create(cfg.DBType, dbPath, ActiveConfig().NetParams().Net)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	log.Info("Block database loaded")
-	return db, nil
-}
-
 // realMain is the real main function for the utility. It is necessary to work
 // around the fact that deferred functions do not run when os.Exit() is called.
 func realMain() error {
@@ -76,14 +40,6 @@ func realMain() error {
 	log = backendLogger.Logger("MAIN")
 	spawn = panics.GoroutineWrapperFunc(log)
 
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		log.Errorf("Failed to load database: %s", err)
-		return err
-	}
-	defer db.Close()
-
 	fi, err := os.Open(cfg.InFile)
 	if err != nil {
 		log.Errorf("Failed to open file %s: %s", cfg.InFile, err)
@@ -94,7 +50,7 @@ func realMain() error {
 	// Create a block importer for the database and input file and start it.
 	// The done channel returned from start will contain an error if
 	// anything went wrong.
-	importer, err := newBlockImporter(db, fi)
+	importer, err := newBlockImporter(fi)
 	if err != nil {
 		log.Errorf("Failed create block importer: %s", err)
 		return err

cmd/addblock/config.go

@@ -6,20 +6,15 @@ package main
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/config"
+	flags "github.com/jessevdk/go-flags"
+	"github.com/kaspanet/kaspad/infrastructure/config"
+	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
 	"os"
 	"path/filepath"
-	"strings"
-
-	flags "github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
 )
 
 const (
-	defaultDBType   = "ffldb"
 	defaultDataFile = "bootstrap.dat"
 	defaultProgress = 10
 )
@@ -27,7 +22,6 @@ const (
 var (
 	kaspadHomeDir  = util.AppDataDir("kaspad", false)
 	defaultDataDir = filepath.Join(kaspadHomeDir, "data")
-	knownDbTypes   = database.SupportedDrivers()
 	activeConfig   *ConfigFlags
 )
 
@@ -41,7 +35,6 @@ func ActiveConfig() *ConfigFlags {
 // See loadConfig for details on the configuration load process.
 type ConfigFlags struct {
 	DataDir         string `short:"b" long:"datadir" description:"Location of the kaspad data directory"`
-	DBType          string `long:"dbtype" description:"Database backend to use for the Block DAG"`
 	InFile          string `short:"i" long:"infile" description:"File containing the block(s)"`
 	Progress        int    `short:"p" long:"progress" description:"Show a progress message each time this number of seconds have passed -- Use 0 to disable progress announcements"`
 	AcceptanceIndex bool   `long:"acceptanceindex" description:"Maintain a full hash-based acceptance index which makes the getChainFromBlock RPC available"`
@@ -58,23 +51,11 @@ func fileExists(name string) bool {
 	return true
 }
 
-// validDbType returns whether or not dbType is a supported database type.
-func validDbType(dbType string) bool {
-	for _, knownType := range knownDbTypes {
-		if dbType == knownType {
-			return true
-		}
-	}
-
-	return false
-}
-
 // loadConfig initializes and parses the config using command line options.
 func loadConfig() (*ConfigFlags, []string, error) {
 	// Default config.
 	activeConfig = &ConfigFlags{
 		DataDir:  defaultDataDir,
-		DBType:   defaultDBType,
 		InFile:   defaultDataFile,
 		Progress: defaultProgress,
 	}
@@ -95,16 +76,6 @@ func loadConfig() (*ConfigFlags, []string, error) {
 		return nil, nil, err
 	}
 
-	// Validate database type.
-	if !validDbType(activeConfig.DBType) {
-		str := "%s: The specified database type [%s] is invalid -- " +
-			"supported types %s"
-		err := errors.Errorf(str, "loadConfig", activeConfig.DBType, strings.Join(knownDbTypes, ", "))
-		fmt.Fprintln(os.Stderr, err)
-		parser.WriteHelp(os.Stderr)
-		return nil, nil, err
-	}
-
 	// Append the network type to the data directory so it is "namespaced"
 	// per network. In addition to the block database, there are other
 	// pieces of data that are saved to disk such as address manager state.

cmd/addblock/import.go

@@ -6,16 +6,16 @@ package main
 
 import (
 	"encoding/binary"
-	"github.com/kaspanet/kaspad/blockdag/indexers"
+	"github.com/kaspanet/kaspad/domain/blockdag/indexers"
+	"github.com/kaspanet/kaspad/util/mstime"
 	"github.com/pkg/errors"
 	"io"
 	"sync"
 	"time"
 
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/domain/blockdag"
+	"github.com/kaspanet/kaspad/network/domainmessage"
 	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
 )
 
 // importResults houses the stats and result as an import operation.
@@ -28,7 +28,6 @@ type importResults struct {
 // blockImporter houses information about an ongoing import from a block data
 // file to the block database.
 type blockImporter struct {
-	db                database.DB
 	dag               *blockdag.BlockDAG
 	r                 io.ReadSeeker
 	processQueue      chan []byte
@@ -41,8 +40,8 @@ type blockImporter struct {
 	receivedLogBlocks int64
 	receivedLogTx     int64
 	lastHeight        int64
-	lastBlockTime     time.Time
-	lastLogTime       time.Time
+	lastBlockTime     mstime.Time
+	lastLogTime       mstime.Time
 }
 
 // readBlock reads the next block from the input file.
@@ -69,10 +68,10 @@ func (bi *blockImporter) readBlock() ([]byte, error) {
 	if err := binary.Read(bi.r, binary.LittleEndian, &blockLen); err != nil {
 		return nil, err
 	}
-	if blockLen > wire.MaxMessagePayload {
+	if blockLen > domainmessage.MaxMessagePayload {
 		return nil, errors.Errorf("block payload of %d bytes is larger "+
 			"than the max allowed %d bytes", blockLen,
-			wire.MaxMessagePayload)
+			domainmessage.MaxMessagePayload)
 	}
 
 	serializedBlock := make([]byte, blockLen)
@@ -172,7 +171,7 @@ out:
 func (bi *blockImporter) logProgress() {
 	bi.receivedLogBlocks++
 
-	now := time.Now()
+	now := mstime.Now()
 	duration := now.Sub(bi.lastLogTime)
 	if duration < time.Second*time.Duration(cfg.Progress) {
 		return
@@ -266,12 +265,12 @@ func (bi *blockImporter) Import() chan *importResults {
 	// Start up the read and process handling goroutines. This setup allows
 	// blocks to be read from disk in parallel while being processed.
 	bi.wg.Add(2)
-	spawn(bi.readHandler)
-	spawn(bi.processHandler)
+	spawn("blockImporter.readHandler", bi.readHandler)
+	spawn("blockImporter.processHandler", bi.processHandler)
 
 	// Wait for the import to finish in a separate goroutine and signal
 	// the status handler when done.
-	spawn(func() {
+	spawn("blockImporter.sendToDoneChan", func() {
 		bi.wg.Wait()
 		bi.doneChan <- true
 	})
@@ -279,7 +278,7 @@ func (bi *blockImporter) Import() chan *importResults {
 	// Start the status handler and return the result channel that it will
 	// send the results on when the import is done.
 	resultChan := make(chan *importResults)
-	spawn(func() {
+	spawn("blockImporter.statusHandler", func() {
 		bi.statusHandler(resultChan)
 	})
 	return resultChan
@@ -287,7 +286,7 @@ func (bi *blockImporter) Import() chan *importResults {
 
 // newBlockImporter returns a new importer for the provided file reader seeker
 // and database.
-func newBlockImporter(db database.DB, r io.ReadSeeker) (*blockImporter, error) {
+func newBlockImporter(r io.ReadSeeker) (*blockImporter, error) {
 	// Create the acceptance index if needed.
 	var indexes []indexers.Indexer
 	if cfg.AcceptanceIndex {
@@ -302,7 +301,6 @@ func newBlockImporter(db database.DB, r io.ReadSeeker) (*blockImporter, error) {
 	}
 
 	dag, err := blockdag.New(&blockdag.Config{
-		DB:           db,
 		DAGParams:    ActiveConfig().NetParams(),
 		TimeSource:   blockdag.NewTimeSource(),
 		IndexManager: indexManager,
@@ -312,13 +310,12 @@ func newBlockImporter(db database.DB, r io.ReadSeeker) (*blockImporter, error) {
 	}
 
 	return &blockImporter{
-		db:           db,
 		r:            r,
 		processQueue: make(chan []byte, 2),
 		doneChan:     make(chan bool),
 		errChan:      make(chan error),
 		quit:         make(chan struct{}),
 		dag:          dag,
-		lastLogTime:  time.Now(),
+		lastLogTime:  mstime.Now(),
 	}, nil
 }

cmd/kaspactl/config.go

@@ -6,7 +6,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/infrastructure/config"
 	"github.com/kaspanet/kaspad/version"
 	"github.com/pkg/errors"
 	"io/ioutil"
@@ -16,7 +16,7 @@ import (
 	"strings"
 
 	"github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/rpcmodel"
+	"github.com/kaspanet/kaspad/network/rpc/model"
 	"github.com/kaspanet/kaspad/util"
 )
 
@@ -24,7 +24,7 @@ const (
 	// unusableFlags are the command usage flags which this utility are not
 	// able to use. In particular it doesn't support websockets and
 	// consequently notifications.
-	unusableFlags = rpcmodel.UFWebsocketOnly | rpcmodel.UFNotification
+	unusableFlags = model.UFWebsocketOnly | model.UFNotification
 )
 
 var (
@@ -45,10 +45,10 @@ func listCommands() {
 	)
 
 	// Get a list of registered commands and categorize and filter them.
-	cmdMethods := rpcmodel.RegisteredCmdMethods()
+	cmdMethods := model.RegisteredCmdMethods()
 	categorized := make([][]string, numCategories)
 	for _, method := range cmdMethods {
-		flags, err := rpcmodel.MethodUsageFlags(method)
+		flags, err := model.MethodUsageFlags(method)
 		if err != nil {
 			// This should never happen since the method was just
 			// returned from the package, but be safe.
@@ -60,7 +60,7 @@ func listCommands() {
 			continue
 		}
 
-		usage, err := rpcmodel.MethodUsageText(method)
+		usage, err := model.MethodUsageText(method)
 		if err != nil {
 			// This should never happen since the method was just
 			// returned from the package, but be safe.

cmd/kaspactl/httpclient.go

@@ -11,7 +11,7 @@ import (
 	"net/http"
 
 	"github.com/btcsuite/go-socks/socks"
-	"github.com/kaspanet/kaspad/rpcmodel"
+	"github.com/kaspanet/kaspad/network/rpc/model"
 )
 
 // newHTTPClient returns a new HTTP client that is configured according to the
@@ -117,7 +117,7 @@ func sendPostRequest(marshalledJSON []byte, cfg *ConfigFlags) ([]byte, error) {
 	}
 
 	// Unmarshal the response.
-	var resp rpcmodel.Response
+	var resp model.Response
 	if err := json.Unmarshal(respBytes, &resp); err != nil {
 		return nil, err
 	}

cmd/kaspactl/kaspactl.go

@@ -11,7 +11,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/kaspanet/kaspad/rpcmodel"
+	"github.com/kaspanet/kaspad/network/rpc/model"
 )
 
 const (
@@ -21,7 +21,7 @@ const (
 
 // commandUsage display the usage for a specific command.
 func commandUsage(method string) {
-	usage, err := rpcmodel.MethodUsageText(method)
+	usage, err := model.MethodUsageText(method)
 	if err != nil {
 		// This should never happen since the method was already checked
 		// before calling this function, but be safe.
@@ -60,7 +60,7 @@ func main() {
 	// Ensure the specified method identifies a valid registered command and
 	// is one of the usable types.
 	method := args[0]
-	usageFlags, err := rpcmodel.MethodUsageFlags(method)
+	usageFlags, err := model.MethodUsageFlags(method)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Unrecognized command '%s'\n", method)
 		fmt.Fprintln(os.Stderr, listCmdMessage)
@@ -105,13 +105,13 @@ func main() {
 
 	// Attempt to create the appropriate command using the arguments
 	// provided by the user.
-	cmd, err := rpcmodel.NewCommand(method, params...)
+	cmd, err := model.NewCommand(method, params...)
 	if err != nil {
 		// Show the error along with its error code when it's a
-		// rpcmodel.Error as it reallistcally will always be since the
+		// model.Error as it reallistcally will always be since the
 		// NewCommand function is only supposed to return errors of that
 		// type.
-		var rpcModelErr rpcmodel.Error
+		var rpcModelErr model.Error
 		if ok := errors.As(err, &rpcModelErr); ok {
 			fmt.Fprintf(os.Stderr, "%s error: %s (command code: %s)\n",
 				method, err, rpcModelErr.ErrorCode)
@@ -119,7 +119,7 @@ func main() {
 			os.Exit(1)
 		}
 
-		// The error is not a rpcmodel.Error and this really should not
+		// The error is not a model.Error and this really should not
 		// happen. Nevertheless, fallback to just showing the error
 		// if it should happen due to a bug in the package.
 		fmt.Fprintf(os.Stderr, "%s error: %s\n", method, err)
@@ -129,7 +129,7 @@ func main() {
 
 	// Marshal the command into a JSON-RPC byte slice in preparation for
 	// sending it to the RPC server.
-	marshalledJSON, err := rpcmodel.MarshalCommand(1, cmd)
+	marshalledJSON, err := model.MarshalCommand(1, cmd)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)

cmd/kaspaminer/client.go

@@ -1,39 +1,39 @@
 package main
 
 import (
-	"github.com/kaspanet/kaspad/rpcclient"
+	"github.com/kaspanet/kaspad/network/domainmessage"
+	"github.com/kaspanet/kaspad/network/rpc/client"
 	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
 	"github.com/pkg/errors"
 	"io/ioutil"
 	"time"
 )
 
 type minerClient struct {
-	*rpcclient.Client
+	*client.Client
 	onBlockAdded chan struct{}
 }
 
-func newMinerClient(connCfg *rpcclient.ConnConfig) (*minerClient, error) {
-	client := &minerClient{
+func newMinerClient(connCfg *client.ConnConfig) (*minerClient, error) {
+	minerClient := &minerClient{
 		onBlockAdded: make(chan struct{}, 1),
 	}
-	notificationHandlers := &rpcclient.NotificationHandlers{
-		OnFilteredBlockAdded: func(_ uint64, header *wire.BlockHeader,
+	notificationHandlers := &client.NotificationHandlers{
+		OnFilteredBlockAdded: func(_ uint64, header *domainmessage.BlockHeader,
 			txs []*util.Tx) {
-			client.onBlockAdded <- struct{}{}
+			minerClient.onBlockAdded <- struct{}{}
 		},
 	}
 	var err error
-	client.Client, err = rpcclient.New(connCfg, notificationHandlers)
+	minerClient.Client, err = client.New(connCfg, notificationHandlers)
 	if err != nil {
 		return nil, errors.Errorf("Error connecting to address %s: %s", connCfg.Host, err)
 	}
 
-	if err = client.NotifyBlocks(); err != nil {
-		return nil, errors.Errorf("Error while registering client %s for block notifications: %s", client.Host(), err)
+	if err = minerClient.NotifyBlocks(); err != nil {
+		return nil, errors.Wrapf(err, "error while registering minerClient %s for block notifications", minerClient.Host())
 	}
-	return client, nil
+	return minerClient, nil
 }
 
 func connectToServer(cfg *configFlags) (*minerClient, error) {
@@ -47,7 +47,7 @@ func connectToServer(cfg *configFlags) (*minerClient, error) {
 		return nil, err
 	}
 
-	connCfg := &rpcclient.ConnConfig{
+	connCfg := &client.ConnConfig{
 		Host:           rpcAddr,
 		Endpoint:       "ws",
 		User:           cfg.RPCUser,

cmd/kaspaminer/config.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/infrastructure/config"
 
 	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
@@ -30,16 +30,18 @@ var (
 )
 
 type configFlags struct {
-	ShowVersion    bool   `short:"V" long:"version" description:"Display version information and exit"`
-	RPCUser        string `short:"u" long:"rpcuser" description:"RPC username"`
-	RPCPassword    string `short:"P" long:"rpcpass" default-mask:"-" description:"RPC password"`
-	RPCServer      string `short:"s" long:"rpcserver" description:"RPC server to connect to"`
-	RPCCert        string `short:"c" long:"rpccert" description:"RPC server certificate chain for validation"`
-	DisableTLS     bool   `long:"notls" description:"Disable TLS"`
-	Verbose        bool   `long:"verbose" short:"v" description:"Enable logging of RPC requests"`
-	NumberOfBlocks uint64 `short:"n" long:"numblocks" description:"Number of blocks to mine. If omitted, will mine until the process is interrupted."`
-	BlockDelay     uint64 `long:"block-delay" description:"Delay for block submission (in milliseconds). This is used only for testing purposes."`
-	Profile        string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
+	ShowVersion       bool   `short:"V" long:"version" description:"Display version information and exit"`
+	RPCUser           string `short:"u" long:"rpcuser" description:"RPC username"`
+	RPCPassword       string `short:"P" long:"rpcpass" default-mask:"-" description:"RPC password"`
+	RPCServer         string `short:"s" long:"rpcserver" description:"RPC server to connect to"`
+	RPCCert           string `short:"c" long:"rpccert" description:"RPC server certificate chain for validation"`
+	DisableTLS        bool   `long:"notls" description:"Disable TLS"`
+	MiningAddr        string `long:"miningaddr" description:"Address to mine to"`
+	Verbose           bool   `long:"verbose" short:"v" description:"Enable logging of RPC requests"`
+	NumberOfBlocks    uint64 `short:"n" long:"numblocks" description:"Number of blocks to mine. If omitted, will mine until the process is interrupted."`
+	BlockDelay        uint64 `long:"block-delay" description:"Delay for block submission (in milliseconds). This is used only for testing purposes."`
+	MineWhenNotSynced bool   `long:"mine-when-not-synced" description:"Mine even if the node is not synced with the rest of the network."`
+	Profile           string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
 	config.NetworkFlags
 }
 
@@ -75,7 +77,7 @@ func parseConfig() (*configFlags, error) {
 	}
 
 	if cfg.RPCCert == "" && !cfg.DisableTLS {
-		return nil, errors.New("--notls has to be disabled if --cert is used")
+		return nil, errors.New("either --notls or --rpccert must be specified")
 	}
 	if cfg.RPCCert != "" && cfg.DisableTLS {
 		return nil, errors.New("--rpccert should be omitted if --notls is used")

cmd/kaspaminer/log.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/logs"
-	"github.com/kaspanet/kaspad/rpcclient"
+	"github.com/kaspanet/kaspad/infrastructure/logs"
+	"github.com/kaspanet/kaspad/network/rpc/client"
 	"github.com/kaspanet/kaspad/util/panics"
 	"os"
 )
@@ -28,5 +28,5 @@ func initLog(logFile, errLogFile string) {
 }
 
 func enableRPCLogging() {
-	rpcclient.UseLogger(backendLog, logs.LevelTrace)
+	client.UseLogger(backendLog, logs.LevelTrace)
 }

cmd/kaspaminer/main.go

@@ -2,8 +2,7 @@ package main
 
 import (
 	"fmt"
-	"net"
-	"net/http"
+	"github.com/kaspanet/kaspad/util"
 	"os"
 
 	"github.com/kaspanet/kaspad/version"
@@ -12,12 +11,13 @@ import (
 
 	_ "net/http/pprof"
 
-	"github.com/kaspanet/kaspad/signal"
+	"github.com/kaspanet/kaspad/infrastructure/signal"
 	"github.com/kaspanet/kaspad/util/panics"
+	"github.com/kaspanet/kaspad/util/profiling"
 )
 
 func main() {
-	defer panics.HandlePanic(log, nil)
+	defer panics.HandlePanic(log, "MAIN", nil)
 	interrupt := signal.InterruptListener()
 
 	cfg, err := parseConfig()
@@ -35,26 +35,25 @@ func main() {
 
 	// Enable http profiling server if requested.
 	if cfg.Profile != "" {
-		spawn(func() {
-			listenAddr := net.JoinHostPort("", cfg.Profile)
-			log.Infof("Profile server listening on %s", listenAddr)
-			profileRedirect := http.RedirectHandler("/debug/pprof", http.StatusSeeOther)
-			http.Handle("/", profileRedirect)
-			log.Errorf("%s", http.ListenAndServe(listenAddr, nil))
-		})
+		profiling.Start(cfg.Profile, log)
 	}
 
 	client, err := connectToServer(cfg)
 	if err != nil {
-		panic(errors.Wrap(err, "Error connecting to the RPC server"))
+		panic(errors.Wrap(err, "error connecting to the RPC server"))
 	}
 	defer client.Disconnect()
 
+	miningAddr, err := util.DecodeAddress(cfg.MiningAddr, cfg.ActiveNetParams.Prefix)
+	if err != nil {
+		panic(errors.Wrap(err, "error decoding mining address"))
+	}
+
 	doneChan := make(chan struct{})
-	spawn(func() {
-		err = mineLoop(client, cfg.NumberOfBlocks, cfg.BlockDelay)
+	spawn("mineLoop", func() {
+		err = mineLoop(client, cfg.NumberOfBlocks, cfg.BlockDelay, cfg.MineWhenNotSynced, miningAddr)
 		if err != nil {
-			panic(errors.Errorf("Error in mine loop: %s", err))
+			panic(errors.Wrap(err, "error in mine loop"))
 		}
 		doneChan <- struct{}{}
 	})

cmd/kaspaminer/mineloop.go

@@ -1,23 +1,17 @@
 package main
 
 import (
-	"encoding/hex"
 	nativeerrors "errors"
 	"math/rand"
-	"strconv"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/kaspanet/kaspad/rpcclient"
-	"github.com/pkg/errors"
-
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/rpcmodel"
+	clientpkg "github.com/kaspanet/kaspad/network/rpc/client"
+	"github.com/kaspanet/kaspad/network/rpc/model"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
 )
 
 var random = rand.New(rand.NewSource(time.Now().UnixNano()))
@@ -25,21 +19,23 @@ var hashesTried uint64
 
 const logHashRateInterval = 10 * time.Second
 
-func mineLoop(client *minerClient, numberOfBlocks uint64, blockDelay uint64) error {
+func mineLoop(client *minerClient, numberOfBlocks uint64, blockDelay uint64, mineWhenNotSynced bool,
+	miningAddr util.Address) error {
+
 	errChan := make(chan error)
 
 	templateStopChan := make(chan struct{})
 
 	doneChan := make(chan struct{})
-	spawn(func() {
+	spawn("mineLoop-internalLoop", func() {
 		wg := sync.WaitGroup{}
 		for i := uint64(0); numberOfBlocks == 0 || i < numberOfBlocks; i++ {
 			foundBlock := make(chan *util.Block)
-			mineNextBlock(client, foundBlock, templateStopChan, errChan)
+			mineNextBlock(client, miningAddr, foundBlock, mineWhenNotSynced, templateStopChan, errChan)
 			block := <-foundBlock
 			templateStopChan <- struct{}{}
 			wg.Add(1)
-			spawn(func() {
+			spawn("mineLoop-handleFoundBlock", func() {
 				if blockDelay != 0 {
 					time.Sleep(time.Duration(blockDelay) * time.Millisecond)
 				}
@@ -65,7 +61,7 @@ func mineLoop(client *minerClient, numberOfBlocks uint64, blockDelay uint64) err
 }
 
 func logHashRate() {
-	spawn(func() {
+	spawn("logHashRate", func() {
 		lastCheck := time.Now()
 		for range time.Tick(logHashRateInterval) {
 			currentHashesTried := hashesTried
@@ -80,77 +76,33 @@ func logHashRate() {
 	})
 }
 
-func mineNextBlock(client *minerClient, foundBlock chan *util.Block, templateStopChan chan struct{}, errChan chan error) {
-	newTemplateChan := make(chan *rpcmodel.GetBlockTemplateResult)
-	spawn(func() {
-		templatesLoop(client, newTemplateChan, errChan, templateStopChan)
+func mineNextBlock(client *minerClient, miningAddr util.Address, foundBlock chan *util.Block, mineWhenNotSynced bool,
+	templateStopChan chan struct{}, errChan chan error) {
+
+	newTemplateChan := make(chan *model.GetBlockTemplateResult)
+	spawn("templatesLoop", func() {
+		templatesLoop(client, miningAddr, newTemplateChan, errChan, templateStopChan)
 	})
-	spawn(func() {
-		solveLoop(newTemplateChan, foundBlock, errChan)
+	spawn("solveLoop", func() {
+		solveLoop(newTemplateChan, foundBlock, mineWhenNotSynced, errChan)
 	})
 }
 
 func handleFoundBlock(client *minerClient, block *util.Block) error {
 	log.Infof("Found block %s with parents %s. Submitting to %s", block.Hash(), block.MsgBlock().Header.ParentHashes, client.Host())
 
-	err := client.SubmitBlock(block, &rpcmodel.SubmitBlockOptions{})
+	err := client.SubmitBlock(block, &model.SubmitBlockOptions{})
 	if err != nil {
 		return errors.Errorf("Error submitting block %s to %s: %s", block.Hash(), client.Host(), err)
 	}
 	return nil
 }
 
-func parseBlock(template *rpcmodel.GetBlockTemplateResult) (*util.Block, error) {
-	// parse parent hashes
-	parentHashes := make([]*daghash.Hash, len(template.ParentHashes))
-	for i, parentHash := range template.ParentHashes {
-		hash, err := daghash.NewHashFromStr(parentHash)
-		if err != nil {
-			return nil, errors.Errorf("Error decoding hash %s: %s", parentHash, err)
-		}
-		parentHashes[i] = hash
-	}
-
-	// parse Bits
-	bitsUint64, err := strconv.ParseUint(template.Bits, 16, 32)
-	if err != nil {
-		return nil, errors.Errorf("Error decoding bits %s: %s", template.Bits, err)
-	}
-	bits := uint32(bitsUint64)
-
-	// parseAcceptedIDMerkleRoot
-	acceptedIDMerkleRoot, err := daghash.NewHashFromStr(template.AcceptedIDMerkleRoot)
-	if err != nil {
-		return nil, errors.Errorf("Error parsing acceptedIDMerkleRoot: %s", err)
-	}
-	utxoCommitment, err := daghash.NewHashFromStr(template.UTXOCommitment)
-	if err != nil {
-		return nil, errors.Errorf("Error parsing utxoCommitment: %s", err)
-	}
-	// parse rest of block
-	msgBlock := wire.NewMsgBlock(
-		wire.NewBlockHeader(template.Version, parentHashes, &daghash.Hash{},
-			acceptedIDMerkleRoot, utxoCommitment, bits, 0))
-
-	for i, txResult := range append([]rpcmodel.GetBlockTemplateResultTx{*template.CoinbaseTxn}, template.Transactions...) {
-		reader := hex.NewDecoder(strings.NewReader(txResult.Data))
-		tx := &wire.MsgTx{}
-		if err := tx.KaspaDecode(reader, 0); err != nil {
-			return nil, errors.Errorf("Error decoding tx #%d: %s", i, err)
-		}
-		msgBlock.AddTransaction(tx)
-	}
-
-	block := util.NewBlock(msgBlock)
-	msgBlock.Header.HashMerkleRoot = blockdag.BuildHashMerkleTreeStore(block.Transactions()).Root()
-	return block, nil
-}
-
 func solveBlock(block *util.Block, stopChan chan struct{}, foundBlock chan *util.Block) {
 	msgBlock := block.MsgBlock()
 	targetDifficulty := util.CompactToBig(msgBlock.Header.Bits)
 	initialNonce := random.Uint64()
-	for i := random.Uint64(); i != initialNonce-1; i++ {
+	for i := initialNonce; i != initialNonce-1; i++ {
 		select {
 		case <-stopChan:
 			return
@@ -167,7 +119,9 @@ func solveBlock(block *util.Block, stopChan chan struct{}, foundBlock chan *util
 
 }
 
-func templatesLoop(client *minerClient, newTemplateChan chan *rpcmodel.GetBlockTemplateResult, errChan chan error, stopChan chan struct{}) {
+func templatesLoop(client *minerClient, miningAddr util.Address,
+	newTemplateChan chan *model.GetBlockTemplateResult, errChan chan error, stopChan chan struct{}) {
+
 	longPollID := ""
 	getBlockTemplateLongPoll := func() {
 		if longPollID != "" {
@@ -175,8 +129,8 @@ func templatesLoop(client *minerClient, newTemplateChan chan *rpcmodel.GetBlockT
 		} else {
 			log.Infof("Requesting template without longPollID from %s", client.Host())
 		}
-		template, err := getBlockTemplate(client, longPollID)
-		if nativeerrors.Is(err, rpcclient.ErrResponseTimedOut) {
+		template, err := getBlockTemplate(client, miningAddr, longPollID)
+		if nativeerrors.Is(err, clientpkg.ErrResponseTimedOut) {
 			log.Infof("Got timeout while requesting template '%s' from %s", longPollID, client.Host())
 			return
 		} else if err != nil {
@@ -203,24 +157,35 @@ func templatesLoop(client *minerClient, newTemplateChan chan *rpcmodel.GetBlockT
 	}
 }
 
-func getBlockTemplate(client *minerClient, longPollID string) (*rpcmodel.GetBlockTemplateResult, error) {
-	return client.GetBlockTemplate([]string{"coinbasetxn"}, longPollID)
+func getBlockTemplate(client *minerClient, miningAddr util.Address, longPollID string) (*model.GetBlockTemplateResult, error) {
+	return client.GetBlockTemplate(miningAddr.String(), longPollID)
 }
 
-func solveLoop(newTemplateChan chan *rpcmodel.GetBlockTemplateResult, foundBlock chan *util.Block, errChan chan error) {
+func solveLoop(newTemplateChan chan *model.GetBlockTemplateResult, foundBlock chan *util.Block,
+	mineWhenNotSynced bool, errChan chan error) {
+
 	var stopOldTemplateSolving chan struct{}
 	for template := range newTemplateChan {
 		if stopOldTemplateSolving != nil {
 			close(stopOldTemplateSolving)
 		}
+
+		if !template.IsSynced {
+			if !mineWhenNotSynced {
+				errChan <- errors.Errorf("got template with isSynced=false")
+				return
+			}
+			log.Warnf("Got template with isSynced=false")
+		}
+
 		stopOldTemplateSolving = make(chan struct{})
-		block, err := parseBlock(template)
+		block, err := clientpkg.ConvertGetBlockTemplateResultToBlock(template)
 		if err != nil {
 			errChan <- errors.Errorf("Error parsing block: %s", err)
 			return
 		}
 
-		spawn(func() {
+		spawn("solveBlock", func() {
 			solveBlock(block, stopOldTemplateSolving, foundBlock)
 		})
 	}

cmd/txsigner/config.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/infrastructure/config"
 )
 
 var activeConfig *ConfigFlags

cmd/txsigner/txsigner.go

@@ -5,9 +5,9 @@ import (
 	"encoding/hex"
 	"fmt"
 	"github.com/kaspanet/go-secp256k1"
-	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/domain/txscript"
+	"github.com/kaspanet/kaspad/network/domainmessage"
 	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
 	"github.com/pkg/errors"
 	"os"
 )
@@ -58,12 +58,12 @@ func parsePrivateKey(privateKeyHex string) (*secp256k1.PrivateKey, error) {
 	return secp256k1.DeserializePrivateKeyFromSlice(privateKeyBytes)
 }
 
-func parseTransaction(transactionHex string) (*wire.MsgTx, error) {
+func parseTransaction(transactionHex string) (*domainmessage.MsgTx, error) {
 	serializedTx, err := hex.DecodeString(transactionHex)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't decode transaction hex")
 	}
-	var transaction wire.MsgTx
+	var transaction domainmessage.MsgTx
 	err = transaction.Deserialize(bytes.NewReader(serializedTx))
 	return &transaction, err
 }
@@ -81,7 +81,7 @@ func createScriptPubKey(publicKey *secp256k1.SchnorrPublicKey) ([]byte, error) {
 	return scriptPubKey, err
 }
 
-func signTransaction(transaction *wire.MsgTx, privateKey *secp256k1.PrivateKey, scriptPubKey []byte) error {
+func signTransaction(transaction *domainmessage.MsgTx, privateKey *secp256k1.PrivateKey, scriptPubKey []byte) error {
 	for i, transactionInput := range transaction.TxIn {
 		signatureScript, err := txscript.SignatureScript(transaction, i, scriptPubKey, txscript.SigHashAll, privateKey, true)
 		if err != nil {
@@ -92,7 +92,7 @@ func signTransaction(transaction *wire.MsgTx, privateKey *secp256k1.PrivateKey,
 	return nil
 }
 
-func serializeTransaction(transaction *wire.MsgTx) (string, error) {
+func serializeTransaction(transaction *domainmessage.MsgTx) (string, error) {
 	buf := bytes.NewBuffer(make([]byte, 0, transaction.SerializeSize()))
 	err := transaction.Serialize(buf)
 	serializedTransaction := hex.EncodeToString(buf.Bytes())

connmgr/README.md

@@ -1,27 +0,0 @@
-connmgr
-=======
-
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/connmgr)
-
-Package connmgr implements a generic Kaspa network connection manager.
-
-## Overview
-
-Connection Manager handles all the general connection concerns such as
-maintaining a set number of outbound connections, sourcing peers, banning,
-limiting max connections, etc.
-
-The package provides a generic connection manager which is able to accept
-connection requests from a source or a set of given addresses, dial them and
-notify the caller on connections. The main intended use is to initialize a pool
-of active connections and maintain them to remain connected to the P2P network.
-
-In addition the connection manager provides the following utilities:
-
-- Notifications on connections or disconnections
-- Handle failures and retry new addresses from the source
-- Connect only to specified addresses
-- Permanent connections with increasing backoff retry timers
-- Disconnect or Remove an established connection
-

connmgr/connmanager.go

@@ -1,659 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package connmgr
-
-import (
-	nativeerrors "errors"
-	"fmt"
-	"net"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/pkg/errors"
-)
-
-// maxFailedAttempts is the maximum number of successive failed connection
-// attempts after which network failure is assumed and new connections will
-// be delayed by the configured retry duration.
-const maxFailedAttempts = 25
-
-var (
-	// maxRetryDuration is the max duration of time retrying of a persistent
-	// connection is allowed to grow to. This is necessary since the retry
-	// logic uses a backoff mechanism which increases the interval base times
-	// the number of retries that have been done.
-	maxRetryDuration = time.Minute * 5
-
-	// defaultRetryDuration is the default duration of time for retrying
-	// persistent connections.
-	defaultRetryDuration = time.Second * 5
-
-	// defaultTargetOutbound is the default number of outbound connections to
-	// maintain.
-	defaultTargetOutbound = uint32(8)
-)
-
-var (
-	//ErrDialNil is used to indicate that Dial cannot be nil in the configuration.
-	ErrDialNil = errors.New("Config: Dial cannot be nil")
-
-	// ErrMaxOutboundPeers is an error that is thrown when the max amount of peers had
-	// been reached.
-	ErrMaxOutboundPeers = errors.New("max outbound peers reached")
-
-	// ErrAlreadyConnected is an error that is thrown if the peer is already
-	// connected.
-	ErrAlreadyConnected = errors.New("peer already connected")
-
-	// ErrAlreadyPermanent is an error that is thrown if the peer is already
-	// connected as a permanent peer.
-	ErrAlreadyPermanent = errors.New("peer exists as a permanent peer")
-
-	// ErrPeerNotFound is an error that is thrown if the peer was not found.
-	ErrPeerNotFound = errors.New("peer not found")
-)
-
-// ConnState represents the state of the requested connection.
-type ConnState uint8
-
-// ConnState can be either pending, established, disconnected or failed. When
-// a new connection is requested, it is attempted and categorized as
-// established or failed depending on the connection result. An established
-// connection which was disconnected is categorized as disconnected.
-const (
-	ConnPending ConnState = iota
-	ConnFailing
-	ConnCanceled
-	ConnEstablished
-	ConnDisconnected
-)
-
-// ConnReq is the connection request to a network address. If permanent, the
-// connection will be retried on disconnection.
-type ConnReq struct {
-	// The following variables must only be used atomically.
-	id uint64
-
-	Addr      net.Addr
-	Permanent bool
-
-	conn       net.Conn
-	state      ConnState
-	stateMtx   sync.RWMutex
-	retryCount uint32
-}
-
-// updateState updates the state of the connection request.
-func (c *ConnReq) updateState(state ConnState) {
-	c.stateMtx.Lock()
-	defer c.stateMtx.Unlock()
-	c.state = state
-}
-
-// ID returns a unique identifier for the connection request.
-func (c *ConnReq) ID() uint64 {
-	return atomic.LoadUint64(&c.id)
-}
-
-// State is the connection state of the requested connection.
-func (c *ConnReq) State() ConnState {
-	c.stateMtx.RLock()
-	defer c.stateMtx.RUnlock()
-	state := c.state
-	return state
-}
-
-// String returns a human-readable string for the connection request.
-func (c *ConnReq) String() string {
-	if c.Addr == nil || c.Addr.String() == "" {
-		return fmt.Sprintf("reqid %d", atomic.LoadUint64(&c.id))
-	}
-	return fmt.Sprintf("%s (reqid %d)", c.Addr, atomic.LoadUint64(&c.id))
-}
-
-// Config holds the configuration options related to the connection manager.
-type Config struct {
-	// Listeners defines a slice of listeners for which the connection
-	// manager will take ownership of and accept connections. When a
-	// connection is accepted, the OnAccept handler will be invoked with the
-	// connection. Since the connection manager takes ownership of these
-	// listeners, they will be closed when the connection manager is
-	// stopped.
-	//
-	// This field will not have any effect if the OnAccept field is not
-	// also specified. It may be nil if the caller does not wish to listen
-	// for incoming connections.
-	Listeners []net.Listener
-
-	// OnAccept is a callback that is fired when an inbound connection is
-	// accepted. It is the caller's responsibility to close the connection.
-	// Failure to close the connection will result in the connection manager
-	// believing the connection is still active and thus have undesirable
-	// side effects such as still counting toward maximum connection limits.
-	//
-	// This field will not have any effect if the Listeners field is not
-	// also specified since there couldn't possibly be any accepted
-	// connections in that case.
-	OnAccept func(net.Conn)
-
-	// TargetOutbound is the number of outbound network connections to
-	// maintain. Defaults to 8.
-	TargetOutbound uint32
-
-	// RetryDuration is the duration to wait before retrying connection
-	// requests. Defaults to 5s.
-	RetryDuration time.Duration
-
-	// OnConnection is a callback that is fired when a new outbound
-	// connection is established.
-	OnConnection func(*ConnReq, net.Conn)
-
-	// OnDisconnection is a callback that is fired when an outbound
-	// connection is disconnected.
-	OnDisconnection func(*ConnReq)
-
-	// GetNewAddress is a way to get an address to make a network connection
-	// to. If nil, no new connections will be made automatically.
-	GetNewAddress func() (net.Addr, error)
-
-	// Dial connects to the address on the named network. It cannot be nil.
-	Dial func(net.Addr) (net.Conn, error)
-}
-
-// registerPending is used to register a pending connection attempt. By
-// registering pending connection attempts we allow callers to cancel pending
-// connection attempts before their successful or in the case they're not
-// longer wanted.
-type registerPending struct {
-	c    *ConnReq
-	done chan struct{}
-}
-
-// handleConnected is used to queue a successful connection.
-type handleConnected struct {
-	c    *ConnReq
-	conn net.Conn
-}
-
-// handleDisconnected is used to remove a connection.
-type handleDisconnected struct {
-	id    uint64
-	retry bool
-}
-
-// handleFailed is used to remove a pending connection.
-type handleFailed struct {
-	c   *ConnReq
-	err error
-}
-
-// ConnManager provides a manager to handle network connections.
-type ConnManager struct {
-	// The following variables must only be used atomically.
-	connReqCount uint64
-	start        int32
-	stop         int32
-
-	newConnReqMtx sync.Mutex
-
-	cfg            Config
-	wg             sync.WaitGroup
-	failedAttempts uint64
-	requests       chan interface{}
-	quit           chan struct{}
-}
-
-// handleFailedConn handles a connection failed due to a disconnect or any
-// other failure. If permanent, it retries the connection after the configured
-// retry duration. Otherwise, if required, it makes a new connection request.
-// After maxFailedConnectionAttempts new connections will be retried after the
-// configured retry duration.
-func (cm *ConnManager) handleFailedConn(c *ConnReq, err error) {
-	if atomic.LoadInt32(&cm.stop) != 0 {
-		return
-	}
-
-	// Don't write throttled logs more than once every throttledConnFailedLogInterval
-	shouldWriteLog := shouldWriteConnFailedLog(err)
-	if shouldWriteLog {
-		// If we are to write a log, set its lastLogTime to now
-		setConnFailedLastLogTime(err, time.Now())
-	}
-
-	if c.Permanent {
-		c.retryCount++
-		d := time.Duration(c.retryCount) * cm.cfg.RetryDuration
-		if d > maxRetryDuration {
-			d = maxRetryDuration
-		}
-		if shouldWriteLog {
-			log.Debugf("Retrying further connections to %s every %s", c, d)
-		}
-		spawnAfter(d, func() {
-			cm.Connect(c)
-		})
-	} else if cm.cfg.GetNewAddress != nil {
-		cm.failedAttempts++
-		if cm.failedAttempts >= maxFailedAttempts {
-			if shouldWriteLog {
-				log.Debugf("Max failed connection attempts reached: [%d] "+
-					"-- retrying further connections every %s", maxFailedAttempts,
-					cm.cfg.RetryDuration)
-			}
-			spawnAfter(cm.cfg.RetryDuration, cm.NewConnReq)
-		} else {
-			spawn(cm.NewConnReq)
-		}
-	}
-}
-
-// throttledError defines an error type whose logs get throttled. This is to
-// prevent flooding the logs with identical errors.
-type throttledError error
-
-var (
-	// throttledConnFailedLogInterval is the minimum duration of time between
-	// the logs defined in throttledConnFailedLogs.
-	throttledConnFailedLogInterval = time.Minute * 10
-
-	// throttledConnFailedLogs are logs that get written at most every
-	// throttledConnFailedLogInterval. Each entry in this map defines a type
-	// of error that we want to throttle. The value of each entry is the last
-	// time that type of log had been written.
-	throttledConnFailedLogs = map[throttledError]time.Time{
-		ErrNoAddress: {},
-	}
-
-	// ErrNoAddress is an error that is thrown when there aren't any
-	// valid connection addresses.
-	ErrNoAddress throttledError = errors.New("no valid connect address")
-)
-
-// shouldWriteConnFailedLog resolves whether to write logs related to connection
-// failures. Errors that had not been previously registered in throttledConnFailedLogs
-// and non-error (nil values) must always be logged.
-func shouldWriteConnFailedLog(err error) bool {
-	if err == nil {
-		return true
-	}
-	lastLogTime, ok := throttledConnFailedLogs[err]
-	return !ok || lastLogTime.Add(throttledConnFailedLogInterval).Before(time.Now())
-}
-
-// setConnFailedLastLogTime sets the last log time of the specified error
-func setConnFailedLastLogTime(err error, lastLogTime time.Time) {
-	var throttledErr throttledError
-	nativeerrors.As(err, &throttledErr)
-	throttledConnFailedLogs[err] = lastLogTime
-}
-
-// connHandler handles all connection related requests. It must be run as a
-// goroutine.
-//
-// The connection handler makes sure that we maintain a pool of active outbound
-// connections so that we remain connected to the network. Connection requests
-// are processed and mapped by their assigned ids.
-func (cm *ConnManager) connHandler() {
-
-	var (
-		// pending holds all registered conn requests that have yet to
-		// succeed.
-		pending = make(map[uint64]*ConnReq)
-
-		// conns represents the set of all actively connected peers.
-		conns = make(map[uint64]*ConnReq, cm.cfg.TargetOutbound)
-	)
-
-out:
-	for {
-		select {
-		case req := <-cm.requests:
-			switch msg := req.(type) {
-
-			case registerPending:
-				connReq := msg.c
-				connReq.updateState(ConnPending)
-				pending[msg.c.id] = connReq
-				close(msg.done)
-
-			case handleConnected:
-				connReq := msg.c
-
-				if _, ok := pending[connReq.id]; !ok {
-					if msg.conn != nil {
-						msg.conn.Close()
-					}
-					log.Debugf("Ignoring connection for "+
-						"canceled connreq=%s", connReq)
-					continue
-				}
-
-				connReq.updateState(ConnEstablished)
-				connReq.conn = msg.conn
-				conns[connReq.id] = connReq
-				log.Debugf("Connected to %s", connReq)
-				connReq.retryCount = 0
-
-				delete(pending, connReq.id)
-
-				if cm.cfg.OnConnection != nil {
-					cm.cfg.OnConnection(connReq, msg.conn)
-				}
-
-			case handleDisconnected:
-				connReq, ok := conns[msg.id]
-				if !ok {
-					connReq, ok = pending[msg.id]
-					if !ok {
-						log.Errorf("Unknown connid=%d",
-							msg.id)
-						continue
-					}
-
-					// Pending connection was found, remove
-					// it from pending map if we should
-					// ignore a later, successful
-					// connection.
-					connReq.updateState(ConnCanceled)
-					log.Debugf("Canceling: %s", connReq)
-					delete(pending, msg.id)
-					continue
-
-				}
-
-				// An existing connection was located, mark as
-				// disconnected and execute disconnection
-				// callback.
-				log.Debugf("Disconnected from %s", connReq)
-				delete(conns, msg.id)
-
-				if connReq.conn != nil {
-					connReq.conn.Close()
-				}
-
-				if cm.cfg.OnDisconnection != nil {
-					spawn(func() {
-						cm.cfg.OnDisconnection(connReq)
-					})
-				}
-
-				// All internal state has been cleaned up, if
-				// this connection is being removed, we will
-				// make no further attempts with this request.
-				if !msg.retry {
-					connReq.updateState(ConnDisconnected)
-					continue
-				}
-
-				// Otherwise, we will attempt a reconnection if
-				// we do not have enough peers, or if this is a
-				// persistent peer. The connection request is
-				// re added to the pending map, so that
-				// subsequent processing of connections and
-				// failures do not ignore the request.
-				if uint32(len(conns)) < cm.cfg.TargetOutbound ||
-					connReq.Permanent {
-
-					connReq.updateState(ConnPending)
-					log.Debugf("Reconnecting to %s",
-						connReq)
-					pending[msg.id] = connReq
-					cm.handleFailedConn(connReq, nil)
-				}
-
-			case handleFailed:
-				connReq := msg.c
-
-				if _, ok := pending[connReq.id]; !ok {
-					log.Debugf("Ignoring connection for "+
-						"canceled conn req: %s", connReq)
-					continue
-				}
-
-				connReq.updateState(ConnFailing)
-				if shouldWriteConnFailedLog(msg.err) {
-					log.Debugf("Failed to connect to %s: %s",
-						connReq, msg.err)
-				}
-				cm.handleFailedConn(connReq, msg.err)
-			}
-
-		case <-cm.quit:
-			break out
-		}
-	}
-
-	cm.wg.Done()
-	log.Trace("Connection handler done")
-}
-
-// NotifyConnectionRequestComplete notifies the connection
-// manager that a peer had been successfully connected and
-// marked as good.
-func (cm *ConnManager) NotifyConnectionRequestComplete() {
-	cm.failedAttempts = 0
-}
-
-// NewConnReq creates a new connection request and connects to the
-// corresponding address.
-func (cm *ConnManager) NewConnReq() {
-	cm.newConnReqMtx.Lock()
-	defer cm.newConnReqMtx.Unlock()
-	if atomic.LoadInt32(&cm.stop) != 0 {
-		return
-	}
-	if cm.cfg.GetNewAddress == nil {
-		return
-	}
-
-	c := &ConnReq{}
-	atomic.StoreUint64(&c.id, atomic.AddUint64(&cm.connReqCount, 1))
-
-	// Submit a request of a pending connection attempt to the connection
-	// manager. By registering the id before the connection is even
-	// established, we'll be able to later cancel the connection via the
-	// Remove method.
-	done := make(chan struct{})
-	select {
-	case cm.requests <- registerPending{c, done}:
-	case <-cm.quit:
-		return
-	}
-
-	// Wait for the registration to successfully add the pending conn req to
-	// the conn manager's internal state.
-	select {
-	case <-done:
-	case <-cm.quit:
-		return
-	}
-
-	addr, err := cm.cfg.GetNewAddress()
-	if err != nil {
-		select {
-		case cm.requests <- handleFailed{c, err}:
-		case <-cm.quit:
-		}
-		return
-	}
-
-	c.Addr = addr
-
-	cm.Connect(c)
-}
-
-// Connect assigns an id and dials a connection to the address of the
-// connection request.
-func (cm *ConnManager) Connect(c *ConnReq) {
-	if atomic.LoadInt32(&cm.stop) != 0 {
-		return
-	}
-	if atomic.LoadUint64(&c.id) == 0 {
-		atomic.StoreUint64(&c.id, atomic.AddUint64(&cm.connReqCount, 1))
-
-		// Submit a request of a pending connection attempt to the
-		// connection manager. By registering the id before the
-		// connection is even established, we'll be able to later
-		// cancel the connection via the Remove method.
-		done := make(chan struct{})
-		select {
-		case cm.requests <- registerPending{c, done}:
-		case <-cm.quit:
-			return
-		}
-
-		// Wait for the registration to successfully add the pending
-		// conn req to the conn manager's internal state.
-		select {
-		case <-done:
-		case <-cm.quit:
-			return
-		}
-	}
-
-	log.Debugf("Attempting to connect to %s", c)
-
-	conn, err := cm.cfg.Dial(c.Addr)
-	if err != nil {
-		select {
-		case cm.requests <- handleFailed{c, err}:
-		case <-cm.quit:
-		}
-		return
-	}
-
-	select {
-	case cm.requests <- handleConnected{c, conn}:
-	case <-cm.quit:
-	}
-}
-
-// Disconnect disconnects the connection corresponding to the given connection
-// id. If permanent, the connection will be retried with an increasing backoff
-// duration.
-func (cm *ConnManager) Disconnect(id uint64) {
-	if atomic.LoadInt32(&cm.stop) != 0 {
-		return
-	}
-
-	select {
-	case cm.requests <- handleDisconnected{id, true}:
-	case <-cm.quit:
-	}
-}
-
-// Remove removes the connection corresponding to the given connection id from
-// known connections.
-//
-// NOTE: This method can also be used to cancel a lingering connection attempt
-// that hasn't yet succeeded.
-func (cm *ConnManager) Remove(id uint64) {
-	if atomic.LoadInt32(&cm.stop) != 0 {
-		return
-	}
-
-	select {
-	case cm.requests <- handleDisconnected{id, false}:
-	case <-cm.quit:
-	}
-}
-
-// listenHandler accepts incoming connections on a given listener. It must be
-// run as a goroutine.
-func (cm *ConnManager) listenHandler(listener net.Listener) {
-	log.Infof("Server listening on %s", listener.Addr())
-	for atomic.LoadInt32(&cm.stop) == 0 {
-		conn, err := listener.Accept()
-		if err != nil {
-			// Only log the error if not forcibly shutting down.
-			if atomic.LoadInt32(&cm.stop) == 0 {
-				log.Errorf("Can't accept connection: %s", err)
-			}
-			continue
-		}
-		spawn(func() {
-			cm.cfg.OnAccept(conn)
-		})
-	}
-
-	cm.wg.Done()
-	log.Tracef("Listener handler done for %s", listener.Addr())
-}
-
-// Start launches the connection manager and begins connecting to the network.
-func (cm *ConnManager) Start() {
-	// Already started?
-	if atomic.AddInt32(&cm.start, 1) != 1 {
-		return
-	}
-
-	log.Trace("Connection manager started")
-	cm.wg.Add(1)
-	spawn(cm.connHandler)
-
-	// Start all the listeners so long as the caller requested them and
-	// provided a callback to be invoked when connections are accepted.
-	if cm.cfg.OnAccept != nil {
-		for _, listener := range cm.cfg.Listeners {
-			// Declaring this variable is necessary as it needs be declared in the same
-			// scope of the anonymous function below it.
-			listenerCopy := listener
-			cm.wg.Add(1)
-			spawn(func() {
-				cm.listenHandler(listenerCopy)
-			})
-		}
-	}
-
-	for i := atomic.LoadUint64(&cm.connReqCount); i < uint64(cm.cfg.TargetOutbound); i++ {
-		spawn(cm.NewConnReq)
-	}
-}
-
-// Wait blocks until the connection manager halts gracefully.
-func (cm *ConnManager) Wait() {
-	cm.wg.Wait()
-}
-
-// Stop gracefully shuts down the connection manager.
-func (cm *ConnManager) Stop() {
-	if atomic.AddInt32(&cm.stop, 1) != 1 {
-		log.Warnf("Connection manager already stopped")
-		return
-	}
-
-	// Stop all the listeners. There will not be any listeners if
-	// listening is disabled.
-	for _, listener := range cm.cfg.Listeners {
-		// Ignore the error since this is shutdown and there is no way
-		// to recover anyways.
-		_ = listener.Close()
-	}
-
-	close(cm.quit)
-	log.Trace("Connection manager stopped")
-}
-
-// New returns a new connection manager.
-// Use Start to start connecting to the network.
-func New(cfg *Config) (*ConnManager, error) {
-	if cfg.Dial == nil {
-		return nil, ErrDialNil
-	}
-	// Default to sane values
-	if cfg.RetryDuration <= 0 {
-		cfg.RetryDuration = defaultRetryDuration
-	}
-	if cfg.TargetOutbound == 0 {
-		cfg.TargetOutbound = defaultTargetOutbound
-	}
-	cm := ConnManager{
-		cfg:      *cfg, // Copy so caller can't mutate
-		requests: make(chan interface{}),
-		quit:     make(chan struct{}),
-	}
-	return &cm, nil
-}

connmgr/connmanager_test.go

@@ -1,688 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package connmgr
-
-import (
-	"github.com/pkg/errors"
-	"io"
-	"net"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func init() {
-	// Override the max retry duration when running tests.
-	maxRetryDuration = 2 * time.Millisecond
-}
-
-// mockAddr mocks a network address
-type mockAddr struct {
-	net, address string
-}
-
-func (m mockAddr) Network() string { return m.net }
-func (m mockAddr) String() string  { return m.address }
-
-// mockConn mocks a network connection by implementing the net.Conn interface.
-type mockConn struct {
-	io.Reader
-	io.Writer
-	io.Closer
-
-	// local network, address for the connection.
-	lnet, laddr string
-
-	// remote network, address for the connection.
-	rAddr net.Addr
-}
-
-// LocalAddr returns the local address for the connection.
-func (c mockConn) LocalAddr() net.Addr {
-	return &mockAddr{c.lnet, c.laddr}
-}
-
-// RemoteAddr returns the remote address for the connection.
-func (c mockConn) RemoteAddr() net.Addr {
-	return &mockAddr{c.rAddr.Network(), c.rAddr.String()}
-}
-
-// Close handles closing the connection.
-func (c mockConn) Close() error {
-	return nil
-}
-
-func (c mockConn) SetDeadline(t time.Time) error      { return nil }
-func (c mockConn) SetReadDeadline(t time.Time) error  { return nil }
-func (c mockConn) SetWriteDeadline(t time.Time) error { return nil }
-
-// mockDialer mocks the net.Dial interface by returning a mock connection to
-// the given address.
-func mockDialer(addr net.Addr) (net.Conn, error) {
-	r, w := io.Pipe()
-	c := &mockConn{rAddr: addr}
-	c.Reader = r
-	c.Writer = w
-	return c, nil
-}
-
-// TestNewConfig tests that new ConnManager config is validated as expected.
-func TestNewConfig(t *testing.T) {
-	_, err := New(&Config{})
-	if err == nil {
-		t.Fatalf("New expected error: 'Dial can't be nil', got nil")
-	}
-	_, err = New(&Config{
-		Dial: mockDialer,
-	})
-	if err != nil {
-		t.Fatalf("New unexpected error: %v", err)
-	}
-}
-
-// TestStartStop tests that the connection manager starts and stops as
-// expected.
-func TestStartStop(t *testing.T) {
-	connected := make(chan *ConnReq)
-	disconnected := make(chan *ConnReq)
-	cmgr, err := New(&Config{
-		TargetOutbound: 1,
-		GetNewAddress: func() (net.Addr, error) {
-			return &net.TCPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: 18555,
-			}, nil
-		},
-		Dial: mockDialer,
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			connected <- c
-		},
-		OnDisconnection: func(c *ConnReq) {
-			disconnected <- c
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-	gotConnReq := <-connected
-	cmgr.Stop()
-	// already stopped
-	cmgr.Stop()
-	// ignored
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	cmgr.Connect(cr)
-	if cr.ID() != 0 {
-		t.Fatalf("start/stop: got id: %v, want: 0", cr.ID())
-	}
-	cmgr.Disconnect(gotConnReq.ID())
-	cmgr.Remove(gotConnReq.ID())
-	select {
-	case <-disconnected:
-		t.Fatalf("start/stop: unexpected disconnection")
-	case <-time.Tick(10 * time.Millisecond):
-		break
-	}
-}
-
-// TestConnectMode tests that the connection manager works in the connect mode.
-//
-// In connect mode, automatic connections are disabled, so we test that
-// requests using Connect are handled and that no other connections are made.
-func TestConnectMode(t *testing.T) {
-	connected := make(chan *ConnReq)
-	cmgr, err := New(&Config{
-		TargetOutbound: 2,
-		Dial:           mockDialer,
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			connected <- c
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	cmgr.Start()
-	cmgr.Connect(cr)
-	gotConnReq := <-connected
-	wantID := cr.ID()
-	gotID := gotConnReq.ID()
-	if gotID != wantID {
-		t.Fatalf("connect mode: %v - want ID %v, got ID %v", cr.Addr, wantID, gotID)
-	}
-	gotState := cr.State()
-	wantState := ConnEstablished
-	if gotState != wantState {
-		t.Fatalf("connect mode: %v - want state %v, got state %v", cr.Addr, wantState, gotState)
-	}
-	select {
-	case c := <-connected:
-		t.Fatalf("connect mode: got unexpected connection - %v", c.Addr)
-	case <-time.After(time.Millisecond):
-		break
-	}
-	cmgr.Stop()
-}
-
-// TestTargetOutbound tests the target number of outbound connections.
-//
-// We wait until all connections are established, then test they there are the
-// only connections made.
-func TestTargetOutbound(t *testing.T) {
-	targetOutbound := uint32(10)
-	connected := make(chan *ConnReq)
-	cmgr, err := New(&Config{
-		TargetOutbound: targetOutbound,
-		Dial:           mockDialer,
-		GetNewAddress: func() (net.Addr, error) {
-			return &net.TCPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: 18555,
-			}, nil
-		},
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			connected <- c
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-	for i := uint32(0); i < targetOutbound; i++ {
-		<-connected
-	}
-
-	select {
-	case c := <-connected:
-		t.Fatalf("target outbound: got unexpected connection - %v", c.Addr)
-	case <-time.After(time.Millisecond):
-		break
-	}
-	cmgr.Stop()
-}
-
-// TestRetryPermanent tests that permanent connection requests are retried.
-//
-// We make a permanent connection request using Connect, disconnect it using
-// Disconnect and we wait for it to be connected back.
-func TestRetryPermanent(t *testing.T) {
-	connected := make(chan *ConnReq)
-	disconnected := make(chan *ConnReq)
-	cmgr, err := New(&Config{
-		RetryDuration:  time.Millisecond,
-		TargetOutbound: 1,
-		Dial:           mockDialer,
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			connected <- c
-		},
-		OnDisconnection: func(c *ConnReq) {
-			disconnected <- c
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	go cmgr.Connect(cr)
-	cmgr.Start()
-	gotConnReq := <-connected
-	wantID := cr.ID()
-	gotID := gotConnReq.ID()
-	if gotID != wantID {
-		t.Fatalf("retry: %v - want ID %v, got ID %v", cr.Addr, wantID, gotID)
-	}
-	gotState := cr.State()
-	wantState := ConnEstablished
-	if gotState != wantState {
-		t.Fatalf("retry: %v - want state %v, got state %v", cr.Addr, wantState, gotState)
-	}
-
-	cmgr.Disconnect(cr.ID())
-	gotConnReq = <-disconnected
-	wantID = cr.ID()
-	gotID = gotConnReq.ID()
-	if gotID != wantID {
-		t.Fatalf("retry: %v - want ID %v, got ID %v", cr.Addr, wantID, gotID)
-	}
-	gotState = cr.State()
-	wantState = ConnPending
-	if gotState != wantState {
-		// There is a small chance that connection has already been established,
-		// so check for that as well
-		if gotState != ConnEstablished {
-			t.Fatalf("retry: %v - want state %v, got state %v", cr.Addr, wantState, gotState)
-		}
-	}
-
-	gotConnReq = <-connected
-	wantID = cr.ID()
-	gotID = gotConnReq.ID()
-	if gotID != wantID {
-		t.Fatalf("retry: %v - want ID %v, got ID %v", cr.Addr, wantID, gotID)
-	}
-	gotState = cr.State()
-	wantState = ConnEstablished
-	if gotState != wantState {
-		t.Fatalf("retry: %v - want state %v, got state %v", cr.Addr, wantState, gotState)
-	}
-
-	cmgr.Remove(cr.ID())
-	gotConnReq = <-disconnected
-	wantID = cr.ID()
-	gotID = gotConnReq.ID()
-	if gotID != wantID {
-		t.Fatalf("retry: %v - want ID %v, got ID %v", cr.Addr, wantID, gotID)
-	}
-	gotState = cr.State()
-	wantState = ConnDisconnected
-	if gotState != wantState {
-		t.Fatalf("retry: %v - want state %v, got state %v", cr.Addr, wantState, gotState)
-	}
-	cmgr.Stop()
-}
-
-// TestMaxRetryDuration tests the maximum retry duration.
-//
-// We have a timed dialer which initially returns err but after RetryDuration
-// hits maxRetryDuration returns a mock conn.
-func TestMaxRetryDuration(t *testing.T) {
-	networkUp := make(chan struct{})
-	time.AfterFunc(5*time.Millisecond, func() {
-		close(networkUp)
-	})
-	timedDialer := func(addr net.Addr) (net.Conn, error) {
-		select {
-		case <-networkUp:
-			return mockDialer(addr)
-		default:
-			return nil, errors.New("network down")
-		}
-	}
-
-	connected := make(chan *ConnReq)
-	cmgr, err := New(&Config{
-		RetryDuration:  time.Millisecond,
-		TargetOutbound: 1,
-		Dial:           timedDialer,
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			connected <- c
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	go cmgr.Connect(cr)
-	cmgr.Start()
-	// retry in 1ms
-	// retry in 2ms - max retry duration reached
-	// retry in 2ms - timedDialer returns mockDial
-	select {
-	case <-connected:
-	case <-time.Tick(100 * time.Millisecond):
-		t.Fatalf("max retry duration: connection timeout")
-	}
-}
-
-// TestNetworkFailure tests that the connection manager handles a network
-// failure gracefully.
-func TestNetworkFailure(t *testing.T) {
-	var dials uint32
-	errDialer := func(net net.Addr) (net.Conn, error) {
-		atomic.AddUint32(&dials, 1)
-		return nil, errors.New("network down")
-	}
-	cmgr, err := New(&Config{
-		TargetOutbound: 5,
-		RetryDuration:  5 * time.Millisecond,
-		Dial:           errDialer,
-		GetNewAddress: func() (net.Addr, error) {
-			return &net.TCPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: 18555,
-			}, nil
-		},
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			t.Fatalf("network failure: got unexpected connection - %v", c.Addr)
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-	time.AfterFunc(10*time.Millisecond, cmgr.Stop)
-	cmgr.Wait()
-	wantMaxDials := uint32(75)
-	if atomic.LoadUint32(&dials) > wantMaxDials {
-		t.Fatalf("network failure: unexpected number of dials - got %v, want < %v",
-			atomic.LoadUint32(&dials), wantMaxDials)
-	}
-}
-
-// TestStopFailed tests that failed connections are ignored after connmgr is
-// stopped.
-//
-// We have a dailer which sets the stop flag on the conn manager and returns an
-// err so that the handler assumes that the conn manager is stopped and ignores
-// the failure.
-func TestStopFailed(t *testing.T) {
-	done := make(chan struct{}, 1)
-	waitDialer := func(addr net.Addr) (net.Conn, error) {
-		done <- struct{}{}
-		time.Sleep(time.Millisecond)
-		return nil, errors.New("network down")
-	}
-	cmgr, err := New(&Config{
-		Dial: waitDialer,
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-	go func() {
-		<-done
-		atomic.StoreInt32(&cmgr.stop, 1)
-		time.Sleep(2 * time.Millisecond)
-		atomic.StoreInt32(&cmgr.stop, 0)
-		cmgr.Stop()
-	}()
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	go cmgr.Connect(cr)
-	cmgr.Wait()
-}
-
-// TestRemovePendingConnection tests that it's possible to cancel a pending
-// connection, removing its internal state from the ConnMgr.
-func TestRemovePendingConnection(t *testing.T) {
-	// Create a ConnMgr instance with an instance of a dialer that'll never
-	// succeed.
-	wait := make(chan struct{})
-	indefiniteDialer := func(addr net.Addr) (net.Conn, error) {
-		<-wait
-		return nil, errors.Errorf("error")
-	}
-	cmgr, err := New(&Config{
-		Dial: indefiniteDialer,
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-
-	// Establish a connection request to a random IP we've chosen.
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	go cmgr.Connect(cr)
-
-	time.Sleep(10 * time.Millisecond)
-
-	if cr.State() != ConnPending {
-		t.Fatalf("pending request hasn't been registered, status: %v",
-			cr.State())
-	}
-
-	// The request launched above will actually never be able to establish
-	// a connection. So we'll cancel it _before_ it's able to be completed.
-	cmgr.Remove(cr.ID())
-
-	time.Sleep(10 * time.Millisecond)
-
-	// Now examine the status of the connection request, it should read a
-	// status of failed.
-	if cr.State() != ConnCanceled {
-		t.Fatalf("request wasn't canceled, status is: %v", cr.State())
-	}
-
-	close(wait)
-	cmgr.Stop()
-}
-
-// TestCancelIgnoreDelayedConnection tests that a canceled connection request will
-// not execute the on connection callback, even if an outstanding retry
-// succeeds.
-func TestCancelIgnoreDelayedConnection(t *testing.T) {
-	retryTimeout := 10 * time.Millisecond
-
-	// Setup a dialer that will continue to return an error until the
-	// connect chan is signaled, the dial attempt immediately after will
-	// succeed in returning a connection.
-	connect := make(chan struct{})
-	failingDialer := func(addr net.Addr) (net.Conn, error) {
-		select {
-		case <-connect:
-			return mockDialer(addr)
-		default:
-		}
-
-		return nil, errors.Errorf("error")
-	}
-
-	connected := make(chan *ConnReq)
-	cmgr, err := New(&Config{
-		Dial:          failingDialer,
-		RetryDuration: retryTimeout,
-		OnConnection: func(c *ConnReq, conn net.Conn) {
-			connected <- c
-		},
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-	defer cmgr.Stop()
-
-	// Establish a connection request to a random IP we've chosen.
-	cr := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-	}
-	cmgr.Connect(cr)
-
-	// Allow for the first retry timeout to elapse.
-	time.Sleep(2 * retryTimeout)
-
-	// Connection be marked as failed, even after reattempting to
-	// connect.
-	if cr.State() != ConnFailing {
-		t.Fatalf("failing request should have status failed, status: %v",
-			cr.State())
-	}
-
-	// Remove the connection, and then immediately allow the next connection
-	// to succeed.
-	cmgr.Remove(cr.ID())
-	close(connect)
-
-	// Allow the connection manager to process the removal.
-	time.Sleep(5 * time.Millisecond)
-
-	// Now examine the status of the connection request, it should read a
-	// status of canceled.
-	if cr.State() != ConnCanceled {
-		t.Fatalf("request wasn't canceled, status is: %v", cr.State())
-	}
-
-	// Finally, the connection manager should not signal the on-connection
-	// callback, since we explicitly canceled this request. We give a
-	// generous window to ensure the connection manager's lienar backoff is
-	// allowed to properly elapse.
-	select {
-	case <-connected:
-		t.Fatalf("on-connect should not be called for canceled req")
-	case <-time.After(5 * retryTimeout):
-	}
-
-}
-
-// mockListener implements the net.Listener interface and is used to test
-// code that deals with net.Listeners without having to actually make any real
-// connections.
-type mockListener struct {
-	localAddr   string
-	provideConn chan net.Conn
-}
-
-// Accept returns a mock connection when it receives a signal via the Connect
-// function.
-//
-// This is part of the net.Listener interface.
-func (m *mockListener) Accept() (net.Conn, error) {
-	for conn := range m.provideConn {
-		return conn, nil
-	}
-	return nil, errors.New("network connection closed")
-}
-
-// Close closes the mock listener which will cause any blocked Accept
-// operations to be unblocked and return errors.
-//
-// This is part of the net.Listener interface.
-func (m *mockListener) Close() error {
-	close(m.provideConn)
-	return nil
-}
-
-// Addr returns the address the mock listener was configured with.
-//
-// This is part of the net.Listener interface.
-func (m *mockListener) Addr() net.Addr {
-	return &mockAddr{"tcp", m.localAddr}
-}
-
-// Connect fakes a connection to the mock listener from the provided remote
-// address. It will cause the Accept function to return a mock connection
-// configured with the provided remote address and the local address for the
-// mock listener.
-func (m *mockListener) Connect(ip string, port int) {
-	m.provideConn <- &mockConn{
-		laddr: m.localAddr,
-		lnet:  "tcp",
-		rAddr: &net.TCPAddr{
-			IP:   net.ParseIP(ip),
-			Port: port,
-		},
-	}
-}
-
-// newMockListener returns a new mock listener for the provided local address
-// and port. No ports are actually opened.
-func newMockListener(localAddr string) *mockListener {
-	return &mockListener{
-		localAddr:   localAddr,
-		provideConn: make(chan net.Conn),
-	}
-}
-
-// TestListeners ensures providing listeners to the connection manager along
-// with an accept callback works properly.
-func TestListeners(t *testing.T) {
-	// Setup a connection manager with a couple of mock listeners that
-	// notify a channel when they receive mock connections.
-	receivedConns := make(chan net.Conn)
-	listener1 := newMockListener("127.0.0.1:16111")
-	listener2 := newMockListener("127.0.0.1:9333")
-	listeners := []net.Listener{listener1, listener2}
-	cmgr, err := New(&Config{
-		Listeners: listeners,
-		OnAccept: func(conn net.Conn) {
-			receivedConns <- conn
-		},
-		Dial: mockDialer,
-	})
-	if err != nil {
-		t.Fatalf("New error: %v", err)
-	}
-	cmgr.Start()
-
-	// Fake a couple of mock connections to each of the listeners.
-	go func() {
-		for i, listener := range listeners {
-			l := listener.(*mockListener)
-			l.Connect("127.0.0.1", 10000+i*2)
-			l.Connect("127.0.0.1", 10000+i*2+1)
-		}
-	}()
-
-	// Tally the receive connections to ensure the expected number are
-	// received. Also, fail the test after a timeout so it will not hang
-	// forever should the test not work.
-	expectedNumConns := len(listeners) * 2
-	var numConns int
-out:
-	for {
-		select {
-		case <-receivedConns:
-			numConns++
-			if numConns == expectedNumConns {
-				break out
-			}
-
-		case <-time.After(time.Millisecond * 50):
-			t.Fatalf("Timeout waiting for %d expected connections",
-				expectedNumConns)
-		}
-	}
-
-	cmgr.Stop()
-	cmgr.Wait()
-}
-
-// TestConnReqString ensures that ConnReq.String() does not crash
-func TestConnReqString(t *testing.T) {
-	defer func() {
-		if r := recover(); r != nil {
-			t.Fatalf("ConnReq.String crashed %v", r)
-		}
-	}()
-	cr1 := &ConnReq{
-		Addr: &net.TCPAddr{
-			IP:   net.ParseIP("127.0.0.1"),
-			Port: 18555,
-		},
-		Permanent: true,
-	}
-	_ = cr1.String()
-	cr2 := &ConnReq{}
-	_ = cr2.String()
-}

connmgr/doc.go

@@ -1,23 +0,0 @@
-/*
-Package connmgr implements a generic Kaspa network connection manager.
-
-Connection Manager Overview
-
-Connection Manager handles all the general connection concerns such as
-maintaining a set number of outbound connections, sourcing peers, banning,
-limiting max connections, etc.
-
-The package provides a generic connection manager which is able to accept
-connection requests from a source or a set of given addresses, dial them and
-notify the caller on connections. The main intended use is to initialize a pool
-of active connections and maintain them to remain connected to the P2P network.
-
-In addition the connection manager provides the following utilities:
-
-- Notifications on connections or disconnections
-- Handle failures and retry new addresses from the source
-- Connect only to specified addresses
-- Permanent connections with increasing backoff retry timers
-- Disconnect or Remove an established connection
-*/
-package connmgr

connmgr/dynamicbanscore.go

@@ -1,144 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package connmgr
-
-import (
-	"fmt"
-	"math"
-	"sync"
-	"time"
-)
-
-const (
-	// Halflife defines the time (in seconds) by which the transient part
-	// of the ban score decays to one half of it's original value.
-	Halflife = 60
-
-	// lambda is the decaying constant.
-	lambda = math.Ln2 / Halflife
-
-	// Lifetime defines the maximum age of the transient part of the ban
-	// score to be considered a non-zero score (in seconds).
-	Lifetime = 1800
-
-	// precomputedLen defines the amount of decay factors (one per second) that
-	// should be precomputed at initialization.
-	precomputedLen = 64
-)
-
-// precomputedFactor stores precomputed exponential decay factors for the first
-// 'precomputedLen' seconds starting from t == 0.
-var precomputedFactor [precomputedLen]float64
-
-// init precomputes decay factors.
-func init() {
-	for i := range precomputedFactor {
-		precomputedFactor[i] = math.Exp(-1.0 * float64(i) * lambda)
-	}
-}
-
-// decayFactor returns the decay factor at t seconds, using precalculated values
-// if available, or calculating the factor if needed.
-func decayFactor(t int64) float64 {
-	if t < precomputedLen {
-		return precomputedFactor[t]
-	}
-	return math.Exp(-1.0 * float64(t) * lambda)
-}
-
-// DynamicBanScore provides dynamic ban scores consisting of a persistent and a
-// decaying component.
-//
-// The decaying score enables the creation of evasive logic which handles
-// misbehaving peers (especially application layer DoS attacks) gracefully
-// by disconnecting and banning peers attempting various kinds of flooding.
-// DynamicBanScore allows these two approaches to be used in tandem.
-//
-// Zero value: Values of type DynamicBanScore are immediately ready for use upon
-// declaration.
-type DynamicBanScore struct {
-	lastUnix   int64
-	transient  float64
-	persistent uint32
-	mtx        sync.Mutex
-}
-
-// String returns the ban score as a human-readable string.
-func (s *DynamicBanScore) String() string {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	r := fmt.Sprintf("persistent %d + transient %f at %d = %d as of now",
-		s.persistent, s.transient, s.lastUnix, s.Int())
-	return r
-}
-
-// Int returns the current ban score, the sum of the persistent and decaying
-// scores.
-//
-// This function is safe for concurrent access.
-func (s *DynamicBanScore) Int() uint32 {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	r := s.int(time.Now())
-	return r
-}
-
-// Increase increases both the persistent and decaying scores by the values
-// passed as parameters. The resulting score is returned.
-//
-// This function is safe for concurrent access.
-func (s *DynamicBanScore) Increase(persistent, transient uint32) uint32 {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	r := s.increase(persistent, transient, time.Now())
-	return r
-}
-
-// Reset set both persistent and decaying scores to zero.
-//
-// This function is safe for concurrent access.
-func (s *DynamicBanScore) Reset() {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	s.persistent = 0
-	s.transient = 0
-	s.lastUnix = 0
-}
-
-// int returns the ban score, the sum of the persistent and decaying scores at a
-// given point in time.
-//
-// This function is not safe for concurrent access. It is intended to be used
-// internally and during testing.
-func (s *DynamicBanScore) int(t time.Time) uint32 {
-	dt := t.Unix() - s.lastUnix
-	if s.transient < 1 || dt < 0 || Lifetime < dt {
-		return s.persistent
-	}
-	return s.persistent + uint32(s.transient*decayFactor(dt))
-}
-
-// increase increases the persistent, the decaying or both scores by the values
-// passed as parameters. The resulting score is calculated as if the action was
-// carried out at the point time represented by the third parameter. The
-// resulting score is returned.
-//
-// This function is not safe for concurrent access.
-func (s *DynamicBanScore) increase(persistent, transient uint32, t time.Time) uint32 {
-	s.persistent += persistent
-	tu := t.Unix()
-	dt := tu - s.lastUnix
-
-	if transient > 0 {
-		if Lifetime < dt {
-			s.transient = 0
-		} else if s.transient > 1 && dt > 0 {
-			s.transient *= decayFactor(dt)
-		}
-		s.transient += float64(transient)
-		s.lastUnix = tu
-	}
-	return s.persistent + uint32(s.transient)
-}

connmgr/dynamicbanscore_test.go

@@ -1,68 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package connmgr
-
-import (
-	"math"
-	"testing"
-	"time"
-)
-
-// TestDynamicBanScoreDecay tests the exponential decay implemented in
-// DynamicBanScore.
-func TestDynamicBanScoreDecay(t *testing.T) {
-	var bs DynamicBanScore
-	base := time.Now()
-
-	r := bs.increase(100, 50, base)
-	if r != 150 {
-		t.Errorf("Unexpected result %d after ban score increase.", r)
-	}
-
-	r = bs.int(base.Add(time.Minute))
-	if r != 125 {
-		t.Errorf("Halflife check failed - %d instead of 125", r)
-	}
-
-	r = bs.int(base.Add(7 * time.Minute))
-	if r != 100 {
-		t.Errorf("Decay after 7m - %d instead of 100", r)
-	}
-}
-
-// TestDynamicBanScoreLifetime tests that DynamicBanScore properly yields zero
-// once the maximum age is reached.
-func TestDynamicBanScoreLifetime(t *testing.T) {
-	var bs DynamicBanScore
-	base := time.Now()
-
-	bs.increase(0, math.MaxUint32, base)
-	r := bs.int(base.Add(Lifetime * time.Second))
-	if r != 3 { // 3, not 4 due to precision loss and truncating 3.999...
-		t.Errorf("Pre max age check with MaxUint32 failed - %d", r)
-	}
-	r = bs.int(base.Add((Lifetime + 1) * time.Second))
-	if r != 0 {
-		t.Errorf("Zero after max age check failed - %d instead of 0", r)
-	}
-}
-
-// TestDynamicBanScore tests exported functions of DynamicBanScore. Exponential
-// decay or other time based behavior is tested by other functions.
-func TestDynamicBanScoreReset(t *testing.T) {
-	var bs DynamicBanScore
-	if bs.Int() != 0 {
-		t.Errorf("Initial state is not zero.")
-	}
-	bs.Increase(100, 0)
-	r := bs.Int()
-	if r != 100 {
-		t.Errorf("Unexpected result %d after ban score increase.", r)
-	}
-	bs.Reset()
-	if bs.Int() != 0 {
-		t.Errorf("Failed to reset ban score.")
-	}
-}

dagconfig/genesis_test.go

@@ -1,334 +0,0 @@
-// Copyright (c) 2014-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package dagconfig
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/davecgh/go-spew/spew"
-)
-
-// TestGenesisBlock tests the genesis block of the main network for validity by
-// checking the encoded bytes and hashes.
-func TestGenesisBlock(t *testing.T) {
-	// Encode the genesis block to raw bytes.
-	var buf bytes.Buffer
-	err := MainnetParams.GenesisBlock.Serialize(&buf)
-	if err != nil {
-		t.Fatalf("TestGenesisBlock: %v", err)
-	}
-
-	// Ensure the encoded block matches the expected bytes.
-	if !bytes.Equal(buf.Bytes(), genesisBlockBytes) {
-		t.Fatalf("TestGenesisBlock: Genesis block does not appear valid - "+
-			"got %v, want %v", spew.Sdump(buf.Bytes()),
-			spew.Sdump(genesisBlockBytes))
-	}
-
-	// Check hash of the block against expected hash.
-	hash := MainnetParams.GenesisBlock.BlockHash()
-	if !MainnetParams.GenesisHash.IsEqual(hash) {
-		t.Fatalf("TestGenesisBlock: Genesis block hash does not "+
-			"appear valid - got %v, want %v", spew.Sdump(hash),
-			spew.Sdump(MainnetParams.GenesisHash))
-	}
-}
-
-// TestRegtestGenesisBlock tests the genesis block of the regression test
-// network for validity by checking the encoded bytes and hashes.
-func TestRegtestGenesisBlock(t *testing.T) {
-	// Encode the genesis block to raw bytes.
-	var buf bytes.Buffer
-	err := RegressionNetParams.GenesisBlock.Serialize(&buf)
-	if err != nil {
-		t.Fatalf("TestRegtestGenesisBlock: %v", err)
-	}
-
-	// Ensure the encoded block matches the expected bytes.
-	if !bytes.Equal(buf.Bytes(), regtestGenesisBlockBytes) {
-		t.Fatalf("TestRegtestGenesisBlock: Genesis block does not "+
-			"appear valid - got %v, want %v",
-			spew.Sdump(buf.Bytes()),
-			spew.Sdump(regtestGenesisBlockBytes))
-	}
-
-	// Check hash of the block against expected hash.
-	hash := RegressionNetParams.GenesisBlock.BlockHash()
-	if !RegressionNetParams.GenesisHash.IsEqual(hash) {
-		t.Fatalf("TestRegtestGenesisBlock: Genesis block hash does "+
-			"not appear valid - got %v, want %v", spew.Sdump(hash),
-			spew.Sdump(RegressionNetParams.GenesisHash))
-	}
-}
-
-// TestTestnetGenesisBlock tests the genesis block of the test network for
-// validity by checking the encoded bytes and hashes.
-func TestTestnetGenesisBlock(t *testing.T) {
-	// Encode the genesis block to raw bytes.
-	var buf bytes.Buffer
-	err := TestnetParams.GenesisBlock.Serialize(&buf)
-	if err != nil {
-		t.Fatalf("TestTestnetGenesisBlock: %v", err)
-	}
-
-	// Ensure the encoded block matches the expected bytes.
-	if !bytes.Equal(buf.Bytes(), testnetGenesisBlockBytes) {
-		t.Fatalf("TestTestnetGenesisBlock: Genesis block does not "+
-			"appear valid - got %v, want %v",
-			spew.Sdump(buf.Bytes()),
-			spew.Sdump(testnetGenesisBlockBytes))
-	}
-
-	// Check hash of the block against expected hash.
-	hash := TestnetParams.GenesisBlock.BlockHash()
-	if !TestnetParams.GenesisHash.IsEqual(hash) {
-		t.Fatalf("TestTestnetGenesisBlock: Genesis block hash does "+
-			"not appear valid - got %v, want %v", spew.Sdump(hash),
-			spew.Sdump(TestnetParams.GenesisHash))
-	}
-}
-
-// TestSimnetGenesisBlock tests the genesis block of the simulation test network
-// for validity by checking the encoded bytes and hashes.
-func TestSimnetGenesisBlock(t *testing.T) {
-	// Encode the genesis block to raw bytes.
-	var buf bytes.Buffer
-	err := SimnetParams.GenesisBlock.Serialize(&buf)
-	if err != nil {
-		t.Fatalf("TestSimnetGenesisBlock: %v", err)
-	}
-
-	// Ensure the encoded block matches the expected bytes.
-	if !bytes.Equal(buf.Bytes(), simnetGenesisBlockBytes) {
-		t.Fatalf("TestSimnetGenesisBlock: Genesis block does not "+
-			"appear valid - got %v, want %v",
-			spew.Sdump(buf.Bytes()),
-			spew.Sdump(simnetGenesisBlockBytes))
-	}
-
-	// Check hash of the block against expected hash.
-	hash := SimnetParams.GenesisBlock.BlockHash()
-	if !SimnetParams.GenesisHash.IsEqual(hash) {
-		t.Fatalf("TestSimnetGenesisBlock: Genesis block hash does "+
-			"not appear valid - got %v, want %v", spew.Sdump(hash),
-			spew.Sdump(SimnetParams.GenesisHash))
-	}
-}
-
-// TestDevnetGenesisBlock tests the genesis block of the development network
-// for validity by checking the encoded bytes and hashes.
-func TestDevnetGenesisBlock(t *testing.T) {
-	// Encode the genesis block to raw bytes.
-	var buf bytes.Buffer
-	err := DevnetParams.GenesisBlock.Serialize(&buf)
-	if err != nil {
-		t.Fatalf("TestDevnetGenesisBlock: %v", err)
-	}
-
-	// Ensure the encoded block matches the expected bytes.
-	if !bytes.Equal(buf.Bytes(), devnetGenesisBlockBytes) {
-		t.Fatalf("TestDevnetGenesisBlock: Genesis block does not "+
-			"appear valid - got %v, want %v",
-			spew.Sdump(buf.Bytes()),
-			spew.Sdump(simnetGenesisBlockBytes))
-	}
-
-	// Check hash of the block against expected hash.
-	hash := DevnetParams.GenesisBlock.BlockHash()
-	if !DevnetParams.GenesisHash.IsEqual(hash) {
-		t.Fatalf("TestDevnetGenesisBlock: Genesis block hash does "+
-			"not appear valid - got %v, want %v", spew.Sdump(hash),
-			spew.Sdump(DevnetParams.GenesisHash))
-	}
-}
-
-// genesisBlockBytes are the wire encoded bytes for the genesis block of the
-// main network as of protocol version 1.
-var genesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x72, 0x10, 0x35, 0x85, 0xdd, 0xac, 0x82, 0x5c, 0x49, 0x13, 0x9f,
-	0xc0, 0x0e, 0x37, 0xc0, 0x45, 0x71, 0xdf, 0xd9, 0xf6, 0x36, 0xdf, 0x4c, 0x42, 0x72, 0x7b, 0x9e,
-	0x86, 0xdd, 0x37, 0xd2, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xc4, 0xda, 0x5c, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2,
-	0xea, 0x82, 0x4e, 0xb8, 0x87, 0x42, 0xd0, 0x6d, 0x1f, 0x8d, 0xc3, 0xad, 0x9f, 0x43, 0x9e, 0xed,
-	0x6f, 0x43, 0x3c, 0x02, 0x71, 0x71, 0x69, 0xfb, 0xbc, 0x91, 0x44, 0xac, 0xf1, 0x93, 0xd3, 0x18,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-}
-
-// regtestGenesisBlockBytes are the wire encoded bytes for the genesis block of
-// the regression test network as of protocol version 1.
-var regtestGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x9f, 0x62,
-	0xc9, 0x2b, 0x16, 0x17, 0xb3, 0x41, 0x6d, 0x9e,
-	0x2d, 0x87, 0x93, 0xfd, 0x72, 0x77, 0x4d, 0x1d,
-	0x6f, 0x6d, 0x38, 0x5b, 0xf1, 0x24, 0x1b, 0xdc,
-	0x96, 0xce, 0xbf, 0xa1, 0x09, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0xe2, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x1e, 0xa6, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed,
-	0x32, 0xec, 0xb4, 0xf8, 0x3c, 0x7a, 0x32, 0x0f,
-	0xd2, 0xe5, 0x24, 0x77, 0x89, 0x43, 0x3a, 0x78,
-	0x0a, 0xda, 0x68, 0x2d, 0xf6, 0xaa, 0xb1, 0x19,
-	0xdd, 0xd8, 0x97, 0x15, 0x4b, 0xcb, 0x42, 0x25,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5,
-	0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x72, 0x65,
-	0x67, 0x74, 0x65, 0x73, 0x74,
-}
-
-// testnetGenesisBlockBytes are the wire encoded bytes for the genesis block of
-// the test network as of protocol version 1.
-var testnetGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x88, 0x05, 0xd0,
-	0xe7, 0x8f, 0x41, 0x77, 0x39, 0x2c, 0xb6, 0xbb,
-	0xb4, 0x19, 0xa8, 0x48, 0x4a, 0xdf, 0x77, 0xb0,
-	0x82, 0xd6, 0x70, 0xd8, 0x24, 0x6a, 0x36, 0x05,
-	0xaa, 0xbd, 0x7a, 0xd1, 0x62, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xad, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x1e, 0xa1, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc,
-	0x72, 0xe6, 0x7e, 0x37, 0xa1, 0x34, 0x89, 0x23,
-	0x24, 0xaf, 0xae, 0x99, 0x1f, 0x89, 0x09, 0x41,
-	0x1a, 0x4d, 0x58, 0xfe, 0x5a, 0x04, 0xb0, 0x3e,
-	0xeb, 0x1b, 0x5b, 0xb8, 0x65, 0xa8, 0x65, 0x0f,
-	0x01, 0x00, 0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d,
-	0x74, 0x65, 0x73, 0x74, 0x6e, 0x65, 0x74,
-}
-
-// simnetGenesisBlockBytes are the wire encoded bytes for the genesis block of
-// the simulation test network as of protocol version 1.
-var simnetGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x1c, 0x3b,
-	0x9e, 0x0d, 0x9a, 0xc0, 0x80, 0x0a, 0x08, 0x42,
-	0x50, 0x02, 0xa3, 0xea, 0xdb, 0xed, 0xc8, 0xd0,
-	0xad, 0x35, 0x03, 0xd8, 0x0e, 0x11, 0x3c, 0x7b,
-	0xb2, 0xb5, 0x20, 0xe5, 0x84, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xd3, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89,
-	0x48, 0xd3, 0x23, 0x9c, 0xf9, 0x88, 0x2b, 0x63,
-	0xc7, 0x33, 0x0f, 0xa3, 0x64, 0xf2, 0xdb, 0x39,
-	0x73, 0x5f, 0x2b, 0xa8, 0xd5, 0x7b, 0x5c, 0x31,
-	0x68, 0xc9, 0x63, 0x37, 0x5c, 0xe7, 0x41, 0x24,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5,
-	0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x73, 0x69,
-	0x6d, 0x6e, 0x65, 0x74,
-}
-
-// devnetGenesisBlockBytes are the wire encoded bytes for the genesis block of
-// the development network as of protocol version 1.
-var devnetGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x16, 0x0a, 0xc6,
-	0x8b, 0x77, 0x08, 0xf4, 0x96, 0xa3, 0x07, 0x05,
-	0xbc, 0x92, 0xda, 0xee, 0x73, 0x26, 0x5e, 0xd0,
-	0x85, 0x78, 0xa2, 0x5d, 0x02, 0x49, 0x8a, 0x2a,
-	0x22, 0xef, 0x41, 0xc9, 0xc3, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0xe7, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x1e, 0xac, 0x82, 0x02, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11,
-	0xc7, 0x0c, 0x02, 0x9e, 0xb2, 0x2e, 0xb3, 0xad,
-	0x24, 0x10, 0xfe, 0x2c, 0xdb, 0x8e, 0x1d, 0xde,
-	0x81, 0x5b, 0xbb, 0x42, 0xfe, 0xb4, 0x93, 0xd6,
-	0xe3, 0xbe, 0x86, 0x02, 0xe6, 0x3a, 0x65, 0x24,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5,
-	0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x64, 0x65,
-	0x76, 0x6e, 0x65, 0x74,
-}

dagconfig/register_test.go

@@ -1,209 +0,0 @@
-package dagconfig_test
-
-import (
-	"bytes"
-	"github.com/kaspanet/kaspad/util/hdkeychain"
-	"reflect"
-	"testing"
-
-	. "github.com/kaspanet/kaspad/dagconfig"
-)
-
-// Define some of the required parameters for a user-registered
-// network. This is necessary to test the registration of and
-// lookup of encoding magics from the network.
-var mockNetParams = Params{
-	Name: "mocknet",
-	Net:  1<<32 - 1,
-	HDKeyIDPair: hdkeychain.HDKeyIDPair{
-		PrivateKeyID: [4]byte{0x01, 0x02, 0x03, 0x04},
-		PublicKeyID:  [4]byte{0x05, 0x06, 0x07, 0x08},
-	},
-}
-
-func TestRegister(t *testing.T) {
-	type registerTest struct {
-		name   string
-		params *Params
-		err    error
-	}
-	type hdTest struct {
-		priv []byte
-		want []byte
-		err  error
-	}
-
-	tests := []struct {
-		name     string
-		register []registerTest
-		hdMagics []hdTest
-	}{
-		{
-			name: "default networks",
-			register: []registerTest{
-				{
-					name:   "duplicate mainnet",
-					params: &MainnetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate regtest",
-					params: &RegressionNetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate testnet",
-					params: &TestnetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate simnet",
-					params: &SimnetParams,
-					err:    ErrDuplicateNet,
-				},
-			},
-			hdMagics: []hdTest{
-				{
-					priv: MainnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: MainnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: TestnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: TestnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: RegressionNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: RegressionNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: SimnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: SimnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: mockNetParams.HDKeyIDPair.PrivateKeyID[:],
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-				{
-					priv: []byte{0xff, 0xff, 0xff, 0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-				{
-					priv: []byte{0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-			},
-		},
-		{
-			name: "register mocknet",
-			register: []registerTest{
-				{
-					name:   "mocknet",
-					params: &mockNetParams,
-					err:    nil,
-				},
-			},
-			hdMagics: []hdTest{
-				{
-					priv: mockNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: mockNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-			},
-		},
-		{
-			name: "more duplicates",
-			register: []registerTest{
-				{
-					name:   "duplicate mainnet",
-					params: &MainnetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate regtest",
-					params: &RegressionNetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate testnet",
-					params: &TestnetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate simnet",
-					params: &SimnetParams,
-					err:    ErrDuplicateNet,
-				},
-				{
-					name:   "duplicate mocknet",
-					params: &mockNetParams,
-					err:    ErrDuplicateNet,
-				},
-			},
-			hdMagics: []hdTest{
-				{
-					priv: MainnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: MainnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: TestnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: TestnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: RegressionNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: RegressionNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: SimnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: SimnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: mockNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: mockNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: []byte{0xff, 0xff, 0xff, 0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-				{
-					priv: []byte{0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-			},
-		},
-	}
-
-	for _, test := range tests {
-		for _, regtest := range test.register {
-			err := Register(regtest.params)
-
-			// HDKeyIDPairs must be registered separately
-			hdkeychain.RegisterHDKeyIDPair(regtest.params.HDKeyIDPair)
-
-			if err != regtest.err {
-				t.Errorf("%s:%s: Registered network with unexpected error: got %v expected %v",
-					test.name, regtest.name, err, regtest.err)
-			}
-		}
-		for i, magTest := range test.hdMagics {
-			pubKey, err := hdkeychain.HDPrivateKeyToPublicKeyID(magTest.priv[:])
-			if !reflect.DeepEqual(err, magTest.err) {
-				t.Errorf("%s: HD magic %d mismatched error: got %v expected %v ",
-					test.name, i, err, magTest.err)
-				continue
-			}
-			if magTest.err == nil && !bytes.Equal(pubKey, magTest.want[:]) {
-				t.Errorf("%s: HD magic %d private and public mismatch: got %v expected %v ",
-					test.name, i, pubKey, magTest.want[:])
-			}
-		}
-	}
-}

database/README.md

@@ -1,32 +0,0 @@
-database
-========
-
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/database)
-
-Package database provides a block and metadata storage database.
-
-Please note that this package is intended to enable kaspad to support different
-database backends and is not something that a client can directly access as only
-one entity can have the database open at a time (for most database backends),
-and that entity will be kaspad.
-
-When a client wants programmatic access to the data provided by kaspad, they'll
-likely want to use the [rpcclient](https://github.com/kaspanet/kaspad/tree/master/rpcclient)
-package which makes use of the [JSON-RPC API](https://github.com/kaspanet/kaspad/tree/master/docs/json_rpc_api.md).
-
-The default backend, ffldb, has a strong focus on speed, efficiency, and
-robustness. It makes use of leveldb for the metadata, flat files for block
-storage, and strict checksums in key areas to ensure data integrity.
-
-## Feature Overview
-
-- Key/value metadata store
-- Kaspa block storage
-- Efficient retrieval of block headers and regions (transactions, scripts, etc)
-- Read-only and read-write transactions with both manual and managed modes
-- Nested buckets
-- Iteration support including cursors with seek capability
-- Supports registration of backend databases
-- Comprehensive test coverage
-

database/cmd/dbtool/fetchblock.go

@@ -1,62 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"encoding/hex"
-	"github.com/pkg/errors"
-	"time"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// fetchBlockCmd defines the configuration options for the fetchblock command.
-type fetchBlockCmd struct{}
-
-var (
-	// fetchBlockCfg defines the configuration options for the command.
-	fetchBlockCfg = fetchBlockCmd{}
-)
-
-// Execute is the main entry point for the command. It's invoked by the parser.
-func (cmd *fetchBlockCmd) Execute(args []string) error {
-	// Setup the global config options and ensure they are valid.
-	if err := setupGlobalConfig(); err != nil {
-		return err
-	}
-
-	if len(args) < 1 {
-		return errors.New("required block hash parameter not specified")
-	}
-	blockHash, err := daghash.NewHashFromStr(args[0])
-	if err != nil {
-		return err
-	}
-
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	return db.View(func(dbTx database.Tx) error {
-		log.Infof("Fetching block %s", blockHash)
-		startTime := time.Now()
-		blockBytes, err := dbTx.FetchBlock(blockHash)
-		if err != nil {
-			return err
-		}
-		log.Infof("Loaded block in %s", time.Since(startTime))
-		log.Infof("Block Hex: %s", hex.EncodeToString(blockBytes))
-		return nil
-	})
-}
-
-// Usage overrides the usage display for the command.
-func (cmd *fetchBlockCmd) Usage() string {
-	return "<block-hash>"
-}

database/cmd/dbtool/fetchblockregion.go

@@ -1,90 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"encoding/hex"
-	"github.com/pkg/errors"
-	"strconv"
-	"time"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// blockRegionCmd defines the configuration options for the fetchblockregion
-// command.
-type blockRegionCmd struct{}
-
-var (
-	// blockRegionCfg defines the configuration options for the command.
-	blockRegionCfg = blockRegionCmd{}
-)
-
-// Execute is the main entry point for the command. It's invoked by the parser.
-func (cmd *blockRegionCmd) Execute(args []string) error {
-	// Setup the global config options and ensure they are valid.
-	if err := setupGlobalConfig(); err != nil {
-		return err
-	}
-
-	// Ensure expected arguments.
-	if len(args) < 1 {
-		return errors.New("required block hash parameter not specified")
-	}
-	if len(args) < 2 {
-		return errors.New("required start offset parameter not " +
-			"specified")
-	}
-	if len(args) < 3 {
-		return errors.New("required region length parameter not " +
-			"specified")
-	}
-
-	// Parse arguments.
-	blockHash, err := daghash.NewHashFromStr(args[0])
-	if err != nil {
-		return err
-	}
-	startOffset, err := strconv.ParseUint(args[1], 10, 32)
-	if err != nil {
-		return err
-	}
-	regionLen, err := strconv.ParseUint(args[2], 10, 32)
-	if err != nil {
-		return err
-	}
-
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	return db.View(func(dbTx database.Tx) error {
-		log.Infof("Fetching block region %s<%d:%d>", blockHash,
-			startOffset, startOffset+regionLen-1)
-		region := database.BlockRegion{
-			Hash:   blockHash,
-			Offset: uint32(startOffset),
-			Len:    uint32(regionLen),
-		}
-		startTime := time.Now()
-		regionBytes, err := dbTx.FetchBlockRegion(&region)
-		if err != nil {
-			return err
-		}
-		log.Infof("Loaded block region in %s", time.Since(startTime))
-		log.Infof("Double Hash: %s", daghash.DoubleHashH(regionBytes))
-		log.Infof("Region Hex: %s", hex.EncodeToString(regionBytes))
-		return nil
-	})
-}
-
-// Usage overrides the usage display for the command.
-func (cmd *blockRegionCmd) Usage() string {
-	return "<block-hash> <start-offset> <length-of-region>"
-}

database/cmd/dbtool/globalconfig.go

@@ -1,111 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
-)
-
-var (
-	kaspadHomeDir   = util.AppDataDir("kaspad", false)
-	knownDbTypes    = database.SupportedDrivers()
-	activeNetParams = &dagconfig.MainnetParams
-
-	// Default global config.
-	cfg = &config{
-		DataDir: filepath.Join(kaspadHomeDir, "data"),
-		DbType:  "ffldb",
-	}
-)
-
-// config defines the global configuration options.
-type config struct {
-	DataDir        string `short:"b" long:"datadir" description:"Location of the kaspad data directory"`
-	DbType         string `long:"dbtype" description:"Database backend to use for the Block DAG"`
-	Testnet        bool   `long:"testnet" description:"Use the test network"`
-	RegressionTest bool   `long:"regtest" description:"Use the regression test network"`
-	Simnet         bool   `long:"simnet" description:"Use the simulation test network"`
-	Devnet         bool   `long:"devnet" description:"Use the development test network"`
-}
-
-// fileExists reports whether the named file or directory exists.
-func fileExists(name string) bool {
-	if _, err := os.Stat(name); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
-
-// validDbType returns whether or not dbType is a supported database type.
-func validDbType(dbType string) bool {
-	for _, knownType := range knownDbTypes {
-		if dbType == knownType {
-			return true
-		}
-	}
-
-	return false
-}
-
-// setupGlobalConfig examine the global configuration options for any conditions
-// which are invalid as well as performs any addition setup necessary after the
-// initial parse.
-func setupGlobalConfig() error {
-	// Multiple networks can't be selected simultaneously.
-	// Count number of network flags passed; assign active network params
-	// while we're at it
-	numNets := 0
-	if cfg.Testnet {
-		numNets++
-		activeNetParams = &dagconfig.TestnetParams
-	}
-	if cfg.RegressionTest {
-		numNets++
-		activeNetParams = &dagconfig.RegressionNetParams
-	}
-	if cfg.Simnet {
-		numNets++
-		activeNetParams = &dagconfig.SimnetParams
-	}
-	if cfg.Devnet {
-		numNets++
-		activeNetParams = &dagconfig.DevnetParams
-	}
-	if numNets > 1 {
-		return errors.New("The testnet, regtest, simnet and devnet params " +
-			"can't be used together -- choose one of the four")
-	}
-
-	if numNets == 0 {
-		return errors.New("Mainnet has not launched yet, use --testnet to run in testnet mode")
-	}
-
-	// Validate database type.
-	if !validDbType(cfg.DbType) {
-		str := "The specified database type [%s] is invalid -- " +
-			"supported types: %s"
-		return errors.Errorf(str, cfg.DbType, strings.Join(knownDbTypes, ", "))
-	}
-
-	// Append the network type to the data directory so it is "namespaced"
-	// per network. In addition to the block database, there are other
-	// pieces of data that are saved to disk such as address manager state.
-	// All data is specific to a network, so namespacing the data directory
-	// means each individual piece of serialized data does not have to
-	// worry about changing names per network and such.
-	cfg.DataDir = filepath.Join(cfg.DataDir, activeNetParams.Name)
-
-	return nil
-}

database/cmd/dbtool/main.go

@@ -1,113 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"github.com/kaspanet/kaspad/util/panics"
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-
-	"github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/logger"
-	"github.com/kaspanet/kaspad/logs"
-)
-
-const (
-	// blockDbNamePrefix is the prefix for the kaspad block database.
-	blockDbNamePrefix = "blocks"
-)
-
-var (
-	log             *logs.Logger
-	spawn           func(func())
-	shutdownChannel = make(chan error)
-)
-
-// loadBlockDB opens the block database and returns a handle to it.
-func loadBlockDB() (database.DB, error) {
-	// The database name is based on the database type.
-	dbName := blockDbNamePrefix + "_" + cfg.DbType
-	dbPath := filepath.Join(cfg.DataDir, dbName)
-
-	log.Infof("Loading block database from '%s'", dbPath)
-	db, err := database.Open(cfg.DbType, dbPath, activeNetParams.Net)
-	if err != nil {
-		// Return the error if it's not because the database doesn't
-		// exist.
-		var dbErr database.Error
-		if ok := errors.As(err, &dbErr); !ok || dbErr.ErrorCode !=
-			database.ErrDbDoesNotExist {
-
-			return nil, err
-		}
-
-		// Create the db if it does not exist.
-		err = os.MkdirAll(cfg.DataDir, 0700)
-		if err != nil {
-			return nil, err
-		}
-		db, err = database.Create(cfg.DbType, dbPath, activeNetParams.Net)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	log.Info("Block database loaded")
-	return db, nil
-}
-
-// realMain is the real main function for the utility. It is necessary to work
-// around the fact that deferred functions do not run when os.Exit() is called.
-func realMain() error {
-	// Setup logging.
-	backendLogger := logs.NewBackend()
-	defer os.Stdout.Sync()
-	log = backendLogger.Logger("MAIN")
-	spawn = panics.GoroutineWrapperFunc(log)
-	dbLog, _ := logger.Get(logger.SubsystemTags.KSDB)
-	dbLog.SetLevel(logs.LevelDebug)
-
-	// Setup the parser options and commands.
-	appName := filepath.Base(os.Args[0])
-	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
-	parserFlags := flags.Options(flags.HelpFlag | flags.PassDoubleDash)
-	parser := flags.NewNamedParser(appName, parserFlags)
-	parser.AddGroup("Global Options", "", cfg)
-	parser.AddCommand("fetchblock",
-		"Fetch the specific block hash from the database", "",
-		&fetchBlockCfg)
-	parser.AddCommand("fetchblockregion",
-		"Fetch the specified block region from the database", "",
-		&blockRegionCfg)
-
-	// Parse command line and invoke the Execute function for the specified
-	// command.
-	if _, err := parser.Parse(); err != nil {
-		var flagsErr *flags.Error
-		if ok := errors.As(err, &flagsErr); ok && flagsErr.Type == flags.ErrHelp {
-			parser.WriteHelp(os.Stderr)
-		} else {
-			log.Error(err)
-		}
-
-		return err
-	}
-
-	return nil
-}
-
-func main() {
-	// Use all processor cores.
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	// Work around defer not working after os.Exit()
-	if err := realMain(); err != nil {
-		os.Exit(1)
-	}
-}

database/cmd/dbtool/signal.go

@@ -1,82 +0,0 @@
-// Copyright (c) 2013-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"os"
-	"os/signal"
-)
-
-// interruptChannel is used to receive SIGINT (Ctrl+C) signals.
-var interruptChannel chan os.Signal
-
-// addHandlerChannel is used to add an interrupt handler to the list of handlers
-// to be invoked on SIGINT (Ctrl+C) signals.
-var addHandlerChannel = make(chan func())
-
-// mainInterruptHandler listens for SIGINT (Ctrl+C) signals on the
-// interruptChannel and invokes the registered interruptCallbacks accordingly.
-// It also listens for callback registration. It must be run as a goroutine.
-func mainInterruptHandler() {
-	// interruptCallbacks is a list of callbacks to invoke when a
-	// SIGINT (Ctrl+C) is received.
-	var interruptCallbacks []func()
-
-	// isShutdown is a flag which is used to indicate whether or not
-	// the shutdown signal has already been received and hence any future
-	// attempts to add a new interrupt handler should invoke them
-	// immediately.
-	var isShutdown bool
-
-	for {
-		select {
-		case <-interruptChannel:
-			// Ignore more than one shutdown signal.
-			if isShutdown {
-				log.Infof("Received SIGINT (Ctrl+C). " +
-					"Already shutting down...")
-				continue
-			}
-
-			isShutdown = true
-			log.Infof("Received SIGINT (Ctrl+C). Shutting down...")
-
-			// Run handlers in LIFO order.
-			for i := range interruptCallbacks {
-				idx := len(interruptCallbacks) - 1 - i
-				callback := interruptCallbacks[idx]
-				callback()
-			}
-
-			// Signal the main goroutine to shutdown.
-			spawn(func() {
-				shutdownChannel <- nil
-			})
-
-		case handler := <-addHandlerChannel:
-			// The shutdown signal has already been received, so
-			// just invoke and new handlers immediately.
-			if isShutdown {
-				handler()
-			}
-
-			interruptCallbacks = append(interruptCallbacks, handler)
-		}
-	}
-}
-
-// addInterruptHandler adds a handler to call when a SIGINT (Ctrl+C) is
-// received.
-func addInterruptHandler(handler func()) {
-	// Create the channel and start the main interrupt handler which invokes
-	// all other callbacks and exits if not already done.
-	if interruptChannel == nil {
-		interruptChannel = make(chan os.Signal, 1)
-		signal.Notify(interruptChannel, os.Interrupt)
-		spawn(mainInterruptHandler)
-	}
-
-	addHandlerChannel <- handler
-}

database/doc.go

@@ -1,85 +0,0 @@
-/*
-Package database provides a block and metadata storage database.
-
-Overview
-
-This package provides a database layer to store and retrieve this data in a
-simple and efficient manner.
-
-The default backend, ffldb, has a strong focus on speed, efficiency, and
-robustness. It makes use leveldb for the metadata, flat files for block
-storage, and strict checksums in key areas to ensure data integrity.
-
-A quick overview of the features database provides are as follows:
-
- - Key/value metadata store
- - Kaspa block storage
- - Efficient retrieval of block headers and regions (transactions, scripts, etc)
- - Read-only and read-write transactions with both manual and managed modes
- - Nested buckets
- - Supports registration of backend databases
- - Comprehensive test coverage
-
-Database
-
-The main entry point is the DB interface. It exposes functionality for
-transactional-based access and storage of metadata and block data. It is
-obtained via the Create and Open functions which take a database type string
-that identifies the specific database driver (backend) to use as well as
-arguments specific to the specified driver.
-
-The interface provides facilities for obtaining transactions (the Tx interface)
-that are the basis of all database reads and writes. Unlike some database
-interfaces that support reading and writing without transactions, this interface
-requires transactions even when only reading or writing a single key.
-
-The Begin function provides an unmanaged transaction while the View and Update
-functions provide a managed transaction. These are described in more detail
-below.
-
-Transactions
-
-The Tx interface provides facilities for rolling back or committing changes that
-took place while the transaction was active. It also provides the root metadata
-bucket under which all keys, values, and nested buckets are stored. A
-transaction can either be read-only or read-write and managed or unmanaged.
-
-Managed versus Unmanaged Transactions
-
-A managed transaction is one where the caller provides a function to execute
-within the context of the transaction and the commit or rollback is handled
-automatically depending on whether or not the provided function returns an
-error. Attempting to manually call Rollback or Commit on the managed
-transaction will result in a panic.
-
-An unmanaged transaction, on the other hand, requires the caller to manually
-call Commit or Rollback when they are finished with it. Leaving transactions
-open for long periods of time can have several adverse effects, so it is
-recommended that managed transactions are used instead.
-
-Buckets
-
-The Bucket interface provides the ability to manipulate key/value pairs and
-nested buckets as well as iterate through them.
-
-The Get, Put, and Delete functions work with key/value pairs, while the Bucket,
-CreateBucket, CreateBucketIfNotExists, and DeleteBucket functions work with
-buckets. The ForEach function allows the caller to provide a function to be
-called with each key/value pair and nested bucket in the current bucket.
-
-Metadata Bucket
-
-As discussed above, all of the functions which are used to manipulate key/value
-pairs and nested buckets exist on the Bucket interface. The root metadata
-bucket is the upper-most bucket in which data is stored and is created at the
-same time as the database. Use the Metadata function on the Tx interface
-to retrieve it.
-
-Nested Buckets
-
-The CreateBucket and CreateBucketIfNotExists functions on the Bucket interface
-provide the ability to create an arbitrary number of nested buckets. It is
-a good idea to avoid a lot of buckets with little data in them as it could lead
-to poor page utilization depending on the specific driver in use.
-*/
-package database

database/driver.go

@@ -1,84 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"fmt"
-)
-
-// Driver defines a structure for backend drivers to use when they registered
-// themselves as a backend which implements the DB interface.
-type Driver struct {
-	// DbType is the identifier used to uniquely identify a specific
-	// database driver. There can be only one driver with the same name.
-	DbType string
-
-	// Create is the function that will be invoked with all user-specified
-	// arguments to create the database. This function must return
-	// ErrDbExists if the database already exists.
-	Create func(args ...interface{}) (DB, error)
-
-	// Open is the function that will be invoked with all user-specified
-	// arguments to open the database. This function must return
-	// ErrDbDoesNotExist if the database has not already been created.
-	Open func(args ...interface{}) (DB, error)
-}
-
-// driverList holds all of the registered database backends.
-var drivers = make(map[string]*Driver)
-
-// RegisterDriver adds a backend database driver to available interfaces.
-// ErrDbTypeRegistered will be returned if the database type for the driver has
-// already been registered.
-func RegisterDriver(driver Driver) error {
-	if _, exists := drivers[driver.DbType]; exists {
-		str := fmt.Sprintf("driver %q is already registered",
-			driver.DbType)
-		return makeError(ErrDbTypeRegistered, str, nil)
-	}
-
-	drivers[driver.DbType] = &driver
-	return nil
-}
-
-// SupportedDrivers returns a slice of strings that represent the database
-// drivers that have been registered and are therefore supported.
-func SupportedDrivers() []string {
-	supportedDBs := make([]string, 0, len(drivers))
-	for _, drv := range drivers {
-		supportedDBs = append(supportedDBs, drv.DbType)
-	}
-	return supportedDBs
-}
-
-// Create initializes and opens a database for the specified type. The
-// arguments are specific to the database type driver. See the documentation
-// for the database driver for further details.
-//
-// ErrDbUnknownType will be returned if the the database type is not registered.
-func Create(dbType string, args ...interface{}) (DB, error) {
-	drv, exists := drivers[dbType]
-	if !exists {
-		str := fmt.Sprintf("driver %q is not registered", dbType)
-		return nil, makeError(ErrDbUnknownType, str, nil)
-	}
-
-	return drv.Create(args...)
-}
-
-// Open opens an existing database for the specified type. The arguments are
-// specific to the database type driver. See the documentation for the database
-// driver for further details.
-//
-// ErrDbUnknownType will be returned if the the database type is not registered.
-func Open(dbType string, args ...interface{}) (DB, error) {
-	drv, exists := drivers[dbType]
-	if !exists {
-		str := fmt.Sprintf("driver %q is not registered", dbType)
-		return nil, makeError(ErrDbUnknownType, str, nil)
-	}
-
-	return drv.Open(args...)
-}

database/driver_test.go

@@ -1,128 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database_test
-
-import (
-	"github.com/pkg/errors"
-	"testing"
-
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-)
-
-// checkDbError ensures the passed error is a database.Error with an error code
-// that matches the passed  error code.
-func checkDbError(t *testing.T, testName string, gotErr error, wantErrCode database.ErrorCode) bool {
-	dbErr, ok := gotErr.(database.Error)
-	if !ok {
-		t.Errorf("%s: unexpected error type - got %T, want %T",
-			testName, gotErr, database.Error{})
-		return false
-	}
-	if dbErr.ErrorCode != wantErrCode {
-		t.Errorf("%s: unexpected error code - got %s (%s), want %s",
-			testName, dbErr.ErrorCode, dbErr.Description,
-			wantErrCode)
-		return false
-	}
-
-	return true
-}
-
-// TestAddDuplicateDriver ensures that adding a duplicate driver does not
-// overwrite an existing one.
-func TestAddDuplicateDriver(t *testing.T) {
-	supportedDrivers := database.SupportedDrivers()
-	if len(supportedDrivers) == 0 {
-		t.Errorf("no backends to test")
-		return
-	}
-	dbType := supportedDrivers[0]
-
-	// bogusCreateDB is a function which acts as a bogus create and open
-	// driver function and intentionally returns a failure that can be
-	// detected if the interface allows a duplicate driver to overwrite an
-	// existing one.
-	bogusCreateDB := func(args ...interface{}) (database.DB, error) {
-		return nil, errors.Errorf("duplicate driver allowed for database "+
-			"type [%v]", dbType)
-	}
-
-	// Create a driver that tries to replace an existing one. Set its
-	// create and open functions to a function that causes a test failure if
-	// they are invoked.
-	driver := database.Driver{
-		DbType: dbType,
-		Create: bogusCreateDB,
-		Open:   bogusCreateDB,
-	}
-	testName := "duplicate driver registration"
-	err := database.RegisterDriver(driver)
-	if !checkDbError(t, testName, err, database.ErrDbTypeRegistered) {
-		return
-	}
-}
-
-// TestCreateOpenFail ensures that errors which occur while opening or closing
-// a database are handled properly.
-func TestCreateOpenFail(t *testing.T) {
-	// bogusCreateDB is a function which acts as a bogus create and open
-	// driver function that intentionally returns a failure which can be
-	// detected.
-	dbType := "createopenfail"
-	openError := errors.Errorf("failed to create or open database for "+
-		"database type [%v]", dbType)
-	bogusCreateDB := func(args ...interface{}) (database.DB, error) {
-		return nil, openError
-	}
-
-	// Create and add driver that intentionally fails when created or opened
-	// to ensure errors on database open and create are handled properly.
-	driver := database.Driver{
-		DbType: dbType,
-		Create: bogusCreateDB,
-		Open:   bogusCreateDB,
-	}
-	database.RegisterDriver(driver)
-
-	// Ensure creating a database with the new type fails with the expected
-	// error.
-	_, err := database.Create(dbType)
-	if err != openError {
-		t.Errorf("expected error not received - got: %v, want %v", err,
-			openError)
-		return
-	}
-
-	// Ensure opening a database with the new type fails with the expected
-	// error.
-	_, err = database.Open(dbType)
-	if err != openError {
-		t.Errorf("expected error not received - got: %v, want %v", err,
-			openError)
-		return
-	}
-}
-
-// TestCreateOpenUnsupported ensures that attempting to create or open an
-// unsupported database type is handled properly.
-func TestCreateOpenUnsupported(t *testing.T) {
-	// Ensure creating a database with an unsupported type fails with the
-	// expected error.
-	testName := "create with unsupported database type"
-	dbType := "unsupported"
-	_, err := database.Create(dbType)
-	if !checkDbError(t, testName, err, database.ErrDbUnknownType) {
-		return
-	}
-
-	// Ensure opening a database with the an unsupported type fails with the
-	// expected error.
-	testName = "open with unsupported database type"
-	_, err = database.Open(dbType)
-	if !checkDbError(t, testName, err, database.ErrDbUnknownType) {
-		return
-	}
-}

database/error.go

@@ -1,211 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"fmt"
-	"github.com/pkg/errors"
-)
-
-// ErrorCode identifies a kind of error.
-type ErrorCode int
-
-// These constants are used to identify a specific database Error.
-const (
-	// **************************************
-	// Errors related to driver registration.
-	// **************************************
-
-	// ErrDbTypeRegistered indicates two different database drivers
-	// attempt to register with the name database type.
-	ErrDbTypeRegistered ErrorCode = iota
-
-	// *************************************
-	// Errors related to database functions.
-	// *************************************
-
-	// ErrDbUnknownType indicates there is no driver registered for
-	// the specified database type.
-	ErrDbUnknownType
-
-	// ErrDbDoesNotExist indicates open is called for a database that
-	// does not exist.
-	ErrDbDoesNotExist
-
-	// ErrDbExists indicates create is called for a database that
-	// already exists.
-	ErrDbExists
-
-	// ErrDbNotOpen indicates a database instance is accessed before
-	// it is opened or after it is closed.
-	ErrDbNotOpen
-
-	// ErrDbAlreadyOpen indicates open was called on a database that
-	// is already open.
-	ErrDbAlreadyOpen
-
-	// ErrInvalid indicates the specified database is not valid.
-	ErrInvalid
-
-	// ErrCorruption indicates a checksum failure occurred which invariably
-	// means the database is corrupt.
-	ErrCorruption
-
-	// ****************************************
-	// Errors related to database transactions.
-	// ****************************************
-
-	// ErrTxClosed indicates an attempt was made to commit or rollback a
-	// transaction that has already had one of those operations performed.
-	ErrTxClosed
-
-	// ErrTxNotWritable indicates an operation that requires write access to
-	// the database was attempted against a read-only transaction.
-	ErrTxNotWritable
-
-	// **************************************
-	// Errors related to metadata operations.
-	// **************************************
-
-	// ErrBucketNotFound indicates an attempt to access a bucket that has
-	// not been created yet.
-	ErrBucketNotFound
-
-	// ErrBucketExists indicates an attempt to create a bucket that already
-	// exists.
-	ErrBucketExists
-
-	// ErrBucketNameRequired indicates an attempt to create a bucket with a
-	// blank name.
-	ErrBucketNameRequired
-
-	// ErrKeyRequired indicates at attempt to insert a zero-length key.
-	ErrKeyRequired
-
-	// ErrKeyTooLarge indicates an attmempt to insert a key that is larger
-	// than the max allowed key size. The max key size depends on the
-	// specific backend driver being used. As a general rule, key sizes
-	// should be relatively, so this should rarely be an issue.
-	ErrKeyTooLarge
-
-	// ErrValueTooLarge indicates an attmpt to insert a value that is larger
-	// than max allowed value size. The max key size depends on the
-	// specific backend driver being used.
-	ErrValueTooLarge
-
-	// ErrIncompatibleValue indicates the value in question is invalid for
-	// the specific requested operation. For example, trying create or
-	// delete a bucket with an existing non-bucket key, attempting to create
-	// or delete a non-bucket key with an existing bucket key, or trying to
-	// delete a value via a cursor when it points to a nested bucket.
-	ErrIncompatibleValue
-
-	// ***************************************
-	// Errors related to block I/O operations.
-	// ***************************************
-
-	// ErrBlockNotFound indicates a block with the provided hash does not
-	// exist in the database.
-	ErrBlockNotFound
-
-	// ErrBlockExists indicates a block with the provided hash already
-	// exists in the database.
-	ErrBlockExists
-
-	// ErrBlockRegionInvalid indicates a region that exceeds the bounds of
-	// the specified block was requested. When the hash provided by the
-	// region does not correspond to an existing block, the error will be
-	// ErrBlockNotFound instead.
-	ErrBlockRegionInvalid
-
-	// ***********************************
-	// Support for driver-specific errors.
-	// ***********************************
-
-	// ErrDriverSpecific indicates the Err field is a driver-specific error.
-	// This provides a mechanism for drivers to plug-in their own custom
-	// errors for any situations which aren't already covered by the error
-	// codes provided by this package.
-	ErrDriverSpecific
-
-	// numErrorCodes is the maximum error code number used in tests.
-	numErrorCodes
-)
-
-// Map of ErrorCode values back to their constant names for pretty printing.
-var errorCodeStrings = map[ErrorCode]string{
-	ErrDbTypeRegistered:   "ErrDbTypeRegistered",
-	ErrDbUnknownType:      "ErrDbUnknownType",
-	ErrDbDoesNotExist:     "ErrDbDoesNotExist",
-	ErrDbExists:           "ErrDbExists",
-	ErrDbNotOpen:          "ErrDbNotOpen",
-	ErrDbAlreadyOpen:      "ErrDbAlreadyOpen",
-	ErrInvalid:            "ErrInvalid",
-	ErrCorruption:         "ErrCorruption",
-	ErrTxClosed:           "ErrTxClosed",
-	ErrTxNotWritable:      "ErrTxNotWritable",
-	ErrBucketNotFound:     "ErrBucketNotFound",
-	ErrBucketExists:       "ErrBucketExists",
-	ErrBucketNameRequired: "ErrBucketNameRequired",
-	ErrKeyRequired:        "ErrKeyRequired",
-	ErrKeyTooLarge:        "ErrKeyTooLarge",
-	ErrValueTooLarge:      "ErrValueTooLarge",
-	ErrIncompatibleValue:  "ErrIncompatibleValue",
-	ErrBlockNotFound:      "ErrBlockNotFound",
-	ErrBlockExists:        "ErrBlockExists",
-	ErrBlockRegionInvalid: "ErrBlockRegionInvalid",
-	ErrDriverSpecific:     "ErrDriverSpecific",
-}
-
-// String returns the ErrorCode as a human-readable name.
-func (e ErrorCode) String() string {
-	if s := errorCodeStrings[e]; s != "" {
-		return s
-	}
-	return fmt.Sprintf("Unknown ErrorCode (%d)", int(e))
-}
-
-// Error provides a single type for errors that can happen during database
-// operation. It is used to indicate several types of failures including errors
-// with caller requests such as specifying invalid block regions or attempting
-// to access data against closed database transactions, driver errors, errors
-// retrieving data, and errors communicating with database servers.
-//
-// The caller can use type assertions to determine if an error is an Error and
-// access the ErrorCode field to ascertain the specific reason for the failure.
-//
-// The ErrDriverSpecific error code will also have the Err field set with the
-// underlying error. Depending on the backend driver, the Err field might be
-// set to the underlying error for other error codes as well.
-type Error struct {
-	ErrorCode   ErrorCode // Describes the kind of error
-	Description string    // Human readable description of the issue
-	Err         error     // Underlying error
-}
-
-// Error satisfies the error interface and prints human-readable errors.
-func (e Error) Error() string {
-	if e.Err != nil {
-		return e.Description + ": " + e.Err.Error()
-	}
-	return e.Description
-}
-
-// makeError creates an Error given a set of arguments. The error code must
-// be one of the error codes provided by this package.
-func makeError(c ErrorCode, desc string, err error) Error {
-	return Error{ErrorCode: c, Description: desc, Err: err}
-}
-
-// IsErrorCode returns whether or not the provided error is a script error with
-// the provided error code.
-func IsErrorCode(err error, c ErrorCode) bool {
-	var errError Error
-	if ok := errors.As(err, &errError); ok {
-		return errError.ErrorCode == c
-	}
-
-	return false
-}

database/error_test.go

@@ -1,118 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"github.com/pkg/errors"
-	"testing"
-)
-
-// TestErrorCodeStringer tests the stringized output for the ErrorCode type.
-func TestErrorCodeStringer(t *testing.T) {
-	tests := []struct {
-		in   ErrorCode
-		want string
-	}{
-		{ErrDbTypeRegistered, "ErrDbTypeRegistered"},
-		{ErrDbUnknownType, "ErrDbUnknownType"},
-		{ErrDbDoesNotExist, "ErrDbDoesNotExist"},
-		{ErrDbExists, "ErrDbExists"},
-		{ErrDbNotOpen, "ErrDbNotOpen"},
-		{ErrDbAlreadyOpen, "ErrDbAlreadyOpen"},
-		{ErrInvalid, "ErrInvalid"},
-		{ErrCorruption, "ErrCorruption"},
-		{ErrTxClosed, "ErrTxClosed"},
-		{ErrTxNotWritable, "ErrTxNotWritable"},
-		{ErrBucketNotFound, "ErrBucketNotFound"},
-		{ErrBucketExists, "ErrBucketExists"},
-		{ErrBucketNameRequired, "ErrBucketNameRequired"},
-		{ErrKeyRequired, "ErrKeyRequired"},
-		{ErrKeyTooLarge, "ErrKeyTooLarge"},
-		{ErrValueTooLarge, "ErrValueTooLarge"},
-		{ErrIncompatibleValue, "ErrIncompatibleValue"},
-		{ErrBlockNotFound, "ErrBlockNotFound"},
-		{ErrBlockExists, "ErrBlockExists"},
-		{ErrBlockRegionInvalid, "ErrBlockRegionInvalid"},
-		{ErrDriverSpecific, "ErrDriverSpecific"},
-
-		{0xffff, "Unknown ErrorCode (65535)"},
-	}
-
-	// Detect additional error codes that don't have the stringer added.
-	if len(tests)-1 != int(TstNumErrorCodes) {
-		t.Errorf("It appears an error code was added without adding " +
-			"an associated stringer test")
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.String()
-		if result != test.want {
-			t.Errorf("String #%d\ngot: %s\nwant: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-// TestError tests the error output for the Error type.
-func TestError(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		in   Error
-		want string
-	}{
-		{
-			Error{Description: "some error"},
-			"some error",
-		},
-		{
-			Error{Description: "human-readable error"},
-			"human-readable error",
-		},
-		{
-			Error{
-				ErrorCode:   ErrDriverSpecific,
-				Description: "some error",
-				Err:         errors.New("driver-specific error"),
-			},
-			"some error: driver-specific error",
-		},
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.Error()
-		if result != test.want {
-			t.Errorf("Error #%d\n got: %s want: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-func TestIsErrorCode(t *testing.T) {
-	dummyError := errors.New("")
-
-	tests := []struct {
-		err            error
-		code           ErrorCode
-		expectedResult bool
-	}{
-		{makeError(ErrBucketExists, "", dummyError), ErrBucketExists, true},
-		{makeError(ErrBucketExists, "", dummyError), ErrBlockExists, false},
-		{dummyError, ErrBlockExists, false},
-		{nil, ErrBlockExists, false},
-	}
-
-	for i, test := range tests {
-		actualResult := IsErrorCode(test.err, test.code)
-		if test.expectedResult != actualResult {
-			t.Errorf("TestIsErrorCode: %d: Expected: %t, but got: %t",
-				i, test.expectedResult, actualResult)
-		}
-	}
-}

database/example_test.go

@@ -1,180 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database_test
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-// This example demonstrates creating a new database.
-func ExampleCreate() {
-	// This example assumes the ffldb driver is imported.
-	//
-	// import (
-	// 	"github.com/kaspanet/kaspad/database"
-	// 	_ "github.com/kaspanet/kaspad/database/ffldb"
-	// )
-
-	// Create a database and schedule it to be closed and removed on exit.
-	// Typically you wouldn't want to remove the database right away like
-	// this, nor put it in the temp directory, but it's done here to ensure
-	// the example cleans up after itself.
-	dbPath := filepath.Join(os.TempDir(), "examplecreate")
-	db, err := database.Create("ffldb", dbPath, wire.Mainnet)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Output:
-}
-
-// This example demonstrates creating a new database and using a managed
-// read-write transaction to store and retrieve metadata.
-func Example_basicUsage() {
-	// This example assumes the ffldb driver is imported.
-	//
-	// import (
-	// 	"github.com/kaspanet/kaspad/database"
-	// 	_ "github.com/kaspanet/kaspad/database/ffldb"
-	// )
-
-	// Create a database and schedule it to be closed and removed on exit.
-	// Typically you wouldn't want to remove the database right away like
-	// this, nor put it in the temp directory, but it's done here to ensure
-	// the example cleans up after itself.
-	dbPath := filepath.Join(os.TempDir(), "exampleusage")
-	// ensure that DB does not exist before test starts
-	os.RemoveAll(dbPath)
-	db, err := database.Create("ffldb", dbPath, wire.Mainnet)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Use the Update function of the database to perform a managed
-	// read-write transaction. The transaction will automatically be rolled
-	// back if the supplied inner function returns a non-nil error.
-	err = db.Update(func(dbTx database.Tx) error {
-		// Store a key/value pair directly in the metadata bucket.
-		// Typically a nested bucket would be used for a given feature,
-		// but this example is using the metadata bucket directly for
-		// simplicity.
-		key := []byte("mykey")
-		value := []byte("myvalue")
-		if err := dbTx.Metadata().Put(key, value); err != nil {
-			return err
-		}
-
-		// Read the key back and ensure it matches.
-		if !bytes.Equal(dbTx.Metadata().Get(key), value) {
-			return errors.Errorf("unexpected value for key '%s'", key)
-		}
-
-		// Create a new nested bucket under the metadata bucket.
-		nestedBucketKey := []byte("mybucket")
-		nestedBucket, err := dbTx.Metadata().CreateBucket(nestedBucketKey)
-		if err != nil {
-			return err
-		}
-
-		// The key from above that was set in the metadata bucket does
-		// not exist in this new nested bucket.
-		if nestedBucket.Get(key) != nil {
-			return errors.Errorf("key '%s' is not expected nil", key)
-		}
-
-		return nil
-	})
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	// Output:
-}
-
-// This example demonstrates creating a new database, using a managed read-write
-// transaction to store a block, and using a managed read-only transaction to
-// fetch the block.
-func Example_blockStorageAndRetrieval() {
-	// This example assumes the ffldb driver is imported.
-	//
-	// import (
-	// 	"github.com/kaspanet/kaspad/database"
-	// 	_ "github.com/kaspanet/kaspad/database/ffldb"
-	// )
-
-	// Create a database and schedule it to be closed and removed on exit.
-	// Typically you wouldn't want to remove the database right away like
-	// this, nor put it in the temp directory, but it's done here to ensure
-	// the example cleans up after itself.
-	dbPath := filepath.Join(os.TempDir(), "exampleblkstorage")
-	db, err := database.Create("ffldb", dbPath, wire.Mainnet)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Use the Update function of the database to perform a managed
-	// read-write transaction and store a genesis block in the database as
-	// and example.
-	err = db.Update(func(dbTx database.Tx) error {
-		genesisBlock := dagconfig.MainnetParams.GenesisBlock
-		return dbTx.StoreBlock(util.NewBlock(genesisBlock))
-	})
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	// Use the View function of the database to perform a managed read-only
-	// transaction and fetch the block stored above.
-	var loadedBlockBytes []byte
-	err = db.Update(func(dbTx database.Tx) error {
-		genesisHash := dagconfig.MainnetParams.GenesisHash
-		blockBytes, err := dbTx.FetchBlock(genesisHash)
-		if err != nil {
-			return err
-		}
-
-		// As documented, all data fetched from the database is only
-		// valid during a database transaction in order to support
-		// zero-copy backends. Thus, make a copy of the data so it
-		// can be used outside of the transaction.
-		loadedBlockBytes = make([]byte, len(blockBytes))
-		copy(loadedBlockBytes, blockBytes)
-		return nil
-	})
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	// Typically at this point, the block could be deserialized via the
-	// wire.MsgBlock.Deserialize function or used in its serialized form
-	// depending on need. However, for this example, just display the
-	// number of serialized bytes to show it was loaded as expected.
-	fmt.Printf("Serialized block size: %d bytes\n", len(loadedBlockBytes))
-
-	// Output:
-	// Serialized block size: 280 bytes
-}

database/export_test.go

@@ -1,17 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-This test file is part of the database package rather than than the
-database_test package so it can bridge access to the internals to properly test
-cases which are either not possible or can't reliably be tested via the public
-interface. The functions, constants, and variables are only exported while the
-tests are being run.
-*/
-
-package database
-
-// TstNumErrorCodes makes the internal numErrorCodes parameter available to the
-// test package.
-const TstNumErrorCodes = numErrorCodes

database/ffldb/README.md

@@ -1,34 +0,0 @@
-ffldb
-=====
-
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
-[![GoDoc](https://godoc.org/github.com/kaspanet/kaspad/database/ffldb?status.png)](http://godoc.org/github.com/kaspanet/kaspad/database/ffldb)
-=======
-
-Package ffldb implements a driver for the database package that uses leveldb for
-the backing metadata and flat files for block storage.
-
-This driver is the recommended driver for use with kaspad. It makes use of leveldb
-for the metadata, flat files for block storage, and checksums in key areas to
-ensure data integrity.
-
-## Usage
-
-This package is a driver to the database package and provides the database type
-of "ffldb". The parameters the Open and Create functions take are the
-database path as a string and the block network.
-
-```Go
-db, err := database.Open("ffldb", "path/to/database", wire.Mainnet)
-if err != nil {
-	// Handle error
-}
-```
-
-```Go
-db, err := database.Create("ffldb", "path/to/database", wire.Mainnet)
-if err != nil {
-	// Handle error
-}
-```
-

database/ffldb/bench_test.go

@@ -1,97 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-)
-
-// BenchmarkBlockHeader benchmarks how long it takes to load the mainnet genesis
-// block header.
-func BenchmarkBlockHeader(b *testing.B) {
-	// Start by creating a new database and populating it with the mainnet
-	// genesis block.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-benchblkhdr")
-	_ = os.RemoveAll(dbPath)
-	db, err := database.Create("ffldb", dbPath, blockDataNet)
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-	err = db.Update(func(dbTx database.Tx) error {
-		block := util.NewBlock(dagconfig.MainnetParams.GenesisBlock)
-		return dbTx.StoreBlock(block)
-	})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	b.ReportAllocs()
-	b.ResetTimer()
-	err = db.View(func(dbTx database.Tx) error {
-		blockHash := dagconfig.MainnetParams.GenesisHash
-		for i := 0; i < b.N; i++ {
-			_, err := dbTx.FetchBlockHeader(blockHash)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Don't benchmark teardown.
-	b.StopTimer()
-}
-
-// BenchmarkBlockHeader benchmarks how long it takes to load the mainnet genesis
-// block.
-func BenchmarkBlock(b *testing.B) {
-	// Start by creating a new database and populating it with the mainnet
-	// genesis block.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-benchblk")
-	_ = os.RemoveAll(dbPath)
-	db, err := database.Create("ffldb", dbPath, blockDataNet)
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-	err = db.Update(func(dbTx database.Tx) error {
-		block := util.NewBlock(dagconfig.MainnetParams.GenesisBlock)
-		return dbTx.StoreBlock(block)
-	})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	b.ReportAllocs()
-	b.ResetTimer()
-	err = db.View(func(dbTx database.Tx) error {
-		blockHash := dagconfig.MainnetParams.GenesisHash
-		for i := 0; i < b.N; i++ {
-			_, err := dbTx.FetchBlock(blockHash)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Don't benchmark teardown.
-	b.StopTimer()
-}

database/ffldb/blockio.go

@@ -1,765 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-// This file contains the implementation functions for reading, writing, and
-// otherwise working with the flat files that house the actual blocks.
-
-package ffldb
-
-import (
-	"container/list"
-	"encoding/binary"
-	"fmt"
-	"github.com/pkg/errors"
-	"hash/crc32"
-	"io"
-	"os"
-	"path/filepath"
-	"sync"
-	"syscall"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-const (
-	// maxOpenFiles is the max number of open files to maintain in the
-	// open blocks cache. Note that this does not include the current
-	// write file, so there will typically be one more than this value open.
-	maxOpenFiles = 25
-
-	// maxBlockFileSize is the maximum size for each file used to store
-	// blocks.
-	//
-	// NOTE: The current code uses uint32 for all offsets, so this value
-	// must be less than 2^32 (4 GiB). This is also why it's a typed
-	// constant.
-	maxBlockFileSize uint32 = 512 * 1024 * 1024 // 512 MiB
-)
-
-var (
-	// castagnoli houses the Catagnoli polynomial used for CRC-32 checksums.
-	castagnoli = crc32.MakeTable(crc32.Castagnoli)
-)
-
-// filer is an interface which acts very similar to a *os.File and is typically
-// implemented by it. It exists so the test code can provide mock files for
-// properly testing corruption and file system issues.
-type filer interface {
-	io.Closer
-	io.WriterAt
-	io.ReaderAt
-	Truncate(size int64) error
-	Sync() error
-}
-
-// lockableFile represents a block file on disk that has been opened for either
-// read or read/write access. It also contains a read-write mutex to support
-// multiple concurrent readers.
-type lockableFile struct {
-	sync.RWMutex
-	file filer
-}
-
-// writeCursor represents the current file and offset of the block file on disk
-// for performing all writes. It also contains a read-write mutex to support
-// multiple concurrent readers which can reuse the file handle.
-type writeCursor struct {
-	sync.RWMutex
-
-	// curFile is the current block file that will be appended to when
-	// writing new blocks.
-	curFile *lockableFile
-
-	// curFileNum is the current block file number and is used to allow
-	// readers to use the same open file handle.
-	curFileNum uint32
-
-	// curOffset is the offset in the current write block file where the
-	// next new block will be written.
-	curOffset uint32
-}
-
-// blockStore houses information used to handle reading and writing blocks (and
-// part of blocks) into flat files with support for multiple concurrent readers.
-type blockStore struct {
-	// network is the specific network to use in the flat files for each
-	// block.
-	network wire.KaspaNet
-
-	// basePath is the base path used for the flat block files and metadata.
-	basePath string
-
-	// maxBlockFileSize is the maximum size for each file used to store
-	// blocks. It is defined on the store so the whitebox tests can
-	// override the value.
-	maxBlockFileSize uint32
-
-	// maxOpenFiles is the max number of open files to maintain in the
-	// open blocks cache. Note that this does not include the current
-	// write file, so there will typically be one more than this value open.
-	// It is defined on the store so the whitebox tests can override the value.
-	maxOpenFiles int
-
-	// The following fields are related to the flat files which hold the
-	// actual blocks. The number of open files is limited by maxOpenFiles.
-	//
-	// obfMutex protects concurrent access to the openBlockFiles map. It is
-	// a RWMutex so multiple readers can simultaneously access open files.
-	//
-	// openBlockFiles houses the open file handles for existing block files
-	// which have been opened read-only along with an individual RWMutex.
-	// This scheme allows multiple concurrent readers to the same file while
-	// preventing the file from being closed out from under them.
-	//
-	// lruMutex protects concurrent access to the least recently used list
-	// and lookup map.
-	//
-	// openBlocksLRU tracks how the open files are refenced by pushing the
-	// most recently used files to the front of the list thereby trickling
-	// the least recently used files to end of the list. When a file needs
-	// to be closed due to exceeding the the max number of allowed open
-	// files, the one at the end of the list is closed.
-	//
-	// fileNumToLRUElem is a mapping between a specific block file number
-	// and the associated list element on the least recently used list.
-	//
-	// Thus, with the combination of these fields, the database supports
-	// concurrent non-blocking reads across multiple and individual files
-	// along with intelligently limiting the number of open file handles by
-	// closing the least recently used files as needed.
-	//
-	// NOTE: The locking order used throughout is well-defined and MUST be
-	// followed. Failure to do so could lead to deadlocks. In particular,
-	// the locking order is as follows:
-	//   1) obfMutex
-	//   2) lruMutex
-	//   3) writeCursor mutex
-	//   4) specific file mutexes
-	//
-	// None of the mutexes are required to be locked at the same time, and
-	// often aren't. However, if they are to be locked simultaneously, they
-	// MUST be locked in the order previously specified.
-	//
-	// Due to the high performance and multi-read concurrency requirements,
-	// write locks should only be held for the minimum time necessary.
-	obfMutex         sync.RWMutex
-	lruMutex         sync.Mutex
-	openBlocksLRU    *list.List // Contains uint32 block file numbers.
-	fileNumToLRUElem map[uint32]*list.Element
-	openBlockFiles   map[uint32]*lockableFile
-
-	// writeCursor houses the state for the current file and location that
-	// new blocks are written to.
-	writeCursor *writeCursor
-
-	// These functions are set to openFile, openWriteFile, and deleteFile by
-	// default, but are exposed here to allow the whitebox tests to replace
-	// them when working with mock files.
-	openFileFunc      func(fileNum uint32) (*lockableFile, error)
-	openWriteFileFunc func(fileNum uint32) (filer, error)
-	deleteFileFunc    func(fileNum uint32) error
-}
-
-// blockLocation identifies a particular block file and location.
-type blockLocation struct {
-	blockFileNum uint32
-	fileOffset   uint32
-	blockLen     uint32
-}
-
-// deserializeBlockLoc deserializes the passed serialized block location
-// information. This is data stored into the block index metadata for each
-// block. The serialized data passed to this function MUST be at least
-// blockLocSize bytes or it will panic. The error check is avoided here because
-// this information will always be coming from the block index which includes a
-// checksum to detect corruption. Thus it is safe to use this unchecked here.
-func deserializeBlockLoc(serializedLoc []byte) blockLocation {
-	// The serialized block location format is:
-	//
-	//  [0:4]  Block file (4 bytes)
-	//  [4:8]  File offset (4 bytes)
-	//  [8:12] Block length (4 bytes)
-	return blockLocation{
-		blockFileNum: byteOrder.Uint32(serializedLoc[0:4]),
-		fileOffset:   byteOrder.Uint32(serializedLoc[4:8]),
-		blockLen:     byteOrder.Uint32(serializedLoc[8:12]),
-	}
-}
-
-// serializeBlockLoc returns the serialization of the passed block location.
-// This is data to be stored into the block index metadata for each block.
-func serializeBlockLoc(loc blockLocation) []byte {
-	// The serialized block location format is:
-	//
-	//  [0:4]  Block file (4 bytes)
-	//  [4:8]  File offset (4 bytes)
-	//  [8:12] Block length (4 bytes)
-	var serializedData [12]byte
-	byteOrder.PutUint32(serializedData[0:4], loc.blockFileNum)
-	byteOrder.PutUint32(serializedData[4:8], loc.fileOffset)
-	byteOrder.PutUint32(serializedData[8:12], loc.blockLen)
-	return serializedData[:]
-}
-
-// blockFilePath return the file path for the provided block file number.
-func blockFilePath(dbPath string, fileNum uint32) string {
-	// Choose 9 digits of precision for the filenames. 9 digits provide
-	// 10^9 files @ 512MiB each a total of ~476.84PiB.
-
-	fileName := fmt.Sprintf("%09d.fdb", fileNum)
-	return filepath.Join(dbPath, fileName)
-}
-
-// openWriteFile returns a file handle for the passed flat file number in
-// read/write mode. The file will be created if needed. It is typically used
-// for the current file that will have all new data appended. Unlike openFile,
-// this function does not keep track of the open file and it is not subject to
-// the maxOpenFiles limit.
-func (s *blockStore) openWriteFile(fileNum uint32) (filer, error) {
-	// The current block file needs to be read-write so it is possible to
-	// append to it. Also, it shouldn't be part of the least recently used
-	// file.
-	filePath := blockFilePath(s.basePath, fileNum)
-	file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0666)
-	if err != nil {
-		str := fmt.Sprintf("failed to open file %q: %s", filePath, err)
-		return nil, makeDbErr(database.ErrDriverSpecific, str, err)
-	}
-
-	return file, nil
-}
-
-// openFile returns a read-only file handle for the passed flat file number.
-// The function also keeps track of the open files, performs least recently
-// used tracking, and limits the number of open files to maxOpenFiles by closing
-// the least recently used file as needed.
-//
-// This function MUST be called with the overall files mutex (s.obfMutex) locked
-// for WRITES.
-func (s *blockStore) openFile(fileNum uint32) (*lockableFile, error) {
-	// Open the appropriate file as read-only.
-	filePath := blockFilePath(s.basePath, fileNum)
-	file, err := os.Open(filePath)
-	if err != nil {
-		return nil, makeDbErr(database.ErrDriverSpecific, err.Error(),
-			err)
-	}
-	blockFile := &lockableFile{file: file}
-
-	// Close the least recently used file if the file exceeds the max
-	// allowed open files. This is not done until after the file open in
-	// case the file fails to open, there is no need to close any files.
-	//
-	// A write lock is required on the LRU list here to protect against
-	// modifications happening as already open files are read from and
-	// shuffled to the front of the list.
-	//
-	// Also, add the file that was just opened to the front of the least
-	// recently used list to indicate it is the most recently used file and
-	// therefore should be closed last.
-	s.lruMutex.Lock()
-	lruList := s.openBlocksLRU
-	if lruList.Len() >= s.maxOpenFiles {
-		lruFileNum := lruList.Remove(lruList.Back()).(uint32)
-		oldBlockFile := s.openBlockFiles[lruFileNum]
-
-		// Close the old file under the write lock for the file in case
-		// any readers are currently reading from it so it's not closed
-		// out from under them.
-		oldBlockFile.Lock()
-		_ = oldBlockFile.file.Close()
-		oldBlockFile.Unlock()
-
-		delete(s.openBlockFiles, lruFileNum)
-		delete(s.fileNumToLRUElem, lruFileNum)
-	}
-	s.fileNumToLRUElem[fileNum] = lruList.PushFront(fileNum)
-	s.lruMutex.Unlock()
-
-	// Store a reference to it in the open block files map.
-	s.openBlockFiles[fileNum] = blockFile
-
-	return blockFile, nil
-}
-
-// deleteFile removes the block file for the passed flat file number. The file
-// must already be closed and it is the responsibility of the caller to do any
-// other state cleanup necessary.
-func (s *blockStore) deleteFile(fileNum uint32) error {
-	filePath := blockFilePath(s.basePath, fileNum)
-	if err := os.Remove(filePath); err != nil {
-		return makeDbErr(database.ErrDriverSpecific, err.Error(), err)
-	}
-
-	return nil
-}
-
-// blockFile attempts to return an existing file handle for the passed flat file
-// number if it is already open as well as marking it as most recently used. It
-// will also open the file when it's not already open subject to the rules
-// described in openFile.
-//
-// NOTE: The returned block file will already have the read lock acquired and
-// the caller MUST call .RUnlock() to release it once it has finished all read
-// operations. This is necessary because otherwise it would be possible for a
-// separate goroutine to close the file after it is returned from here, but
-// before the caller has acquired a read lock.
-func (s *blockStore) blockFile(fileNum uint32) (*lockableFile, error) {
-	// When the requested block file is open for writes, return it.
-	wc := s.writeCursor
-	wc.RLock()
-	if fileNum == wc.curFileNum && wc.curFile.file != nil {
-		obf := wc.curFile
-		obf.RLock()
-		wc.RUnlock()
-		return obf, nil
-	}
-	wc.RUnlock()
-
-	// Try to return an open file under the overall files read lock.
-	s.obfMutex.RLock()
-	if obf, ok := s.openBlockFiles[fileNum]; ok {
-		s.lruMutex.Lock()
-		s.openBlocksLRU.MoveToFront(s.fileNumToLRUElem[fileNum])
-		s.lruMutex.Unlock()
-
-		obf.RLock()
-		s.obfMutex.RUnlock()
-		return obf, nil
-	}
-	s.obfMutex.RUnlock()
-
-	// Since the file isn't open already, need to check the open block files
-	// map again under write lock in case multiple readers got here and a
-	// separate one is already opening the file.
-	s.obfMutex.Lock()
-	if obf, ok := s.openBlockFiles[fileNum]; ok {
-		obf.RLock()
-		s.obfMutex.Unlock()
-		return obf, nil
-	}
-
-	// The file isn't open, so open it while potentially closing the least
-	// recently used one as needed.
-	obf, err := s.openFileFunc(fileNum)
-	if err != nil {
-		s.obfMutex.Unlock()
-		return nil, err
-	}
-	obf.RLock()
-	s.obfMutex.Unlock()
-	return obf, nil
-}
-
-// writeData is a helper function for writeBlock which writes the provided data
-// at the current write offset and updates the write cursor accordingly. The
-// field name parameter is only used when there is an error to provide a nicer
-// error message.
-//
-// The write cursor will be advanced the number of bytes actually written in the
-// event of failure.
-//
-// NOTE: This function MUST be called with the write cursor current file lock
-// held and must only be called during a write transaction so it is effectively
-// locked for writes. Also, the write cursor current file must NOT be nil.
-func (s *blockStore) writeData(data []byte, fieldName string) error {
-	wc := s.writeCursor
-	n, err := wc.curFile.file.WriteAt(data, int64(wc.curOffset))
-	wc.curOffset += uint32(n)
-	if err != nil {
-		var pathErr *os.PathError
-		if ok := errors.As(err, &pathErr); ok && pathErr.Err == syscall.ENOSPC {
-			log.Errorf("No space left on the hard disk, exiting...")
-			os.Exit(1)
-		}
-		str := fmt.Sprintf("failed to write %s to file %d at "+
-			"offset %d: %s", fieldName, wc.curFileNum,
-			wc.curOffset-uint32(n), err)
-		return makeDbErr(database.ErrDriverSpecific, str, err)
-	}
-
-	return nil
-}
-
-// writeBlock appends the specified raw block bytes to the store's write cursor
-// location and increments it accordingly. When the block would exceed the max
-// file size for the current flat file, this function will close the current
-// file, create the next file, update the write cursor, and write the block to
-// the new file.
-//
-// The write cursor will also be advanced the number of bytes actually written
-// in the event of failure.
-//
-// Format: <network><block length><serialized block><checksum>
-func (s *blockStore) writeBlock(rawBlock []byte) (blockLocation, error) {
-	// Compute how many bytes will be written.
-	// 4 bytes each for block network + 4 bytes for block length +
-	// length of raw block + 4 bytes for checksum.
-	blockLen := uint32(len(rawBlock))
-	fullLen := blockLen + 12
-
-	// Move to the next block file if adding the new block would exceed the
-	// max allowed size for the current block file. Also detect overflow
-	// to be paranoid, even though it isn't possible currently, numbers
-	// might change in the future to make it possible.
-	//
-	// NOTE: The writeCursor.offset field isn't protected by the mutex
-	// since it's only read/changed during this function which can only be
-	// called during a write transaction, of which there can be only one at
-	// a time.
-	wc := s.writeCursor
-	finalOffset := wc.curOffset + fullLen
-	if finalOffset < wc.curOffset || finalOffset > s.maxBlockFileSize {
-		// This is done under the write cursor lock since the curFileNum
-		// field is accessed elsewhere by readers.
-		//
-		// Close the current write file to force a read-only reopen
-		// with LRU tracking. The close is done under the write lock
-		// for the file to prevent it from being closed out from under
-		// any readers currently reading from it.
-		wc.Lock()
-		wc.curFile.Lock()
-		if wc.curFile.file != nil {
-			_ = wc.curFile.file.Close()
-			wc.curFile.file = nil
-		}
-		wc.curFile.Unlock()
-
-		// Start writes into next file.
-		wc.curFileNum++
-		wc.curOffset = 0
-		wc.Unlock()
-	}
-
-	// All writes are done under the write lock for the file to ensure any
-	// readers are finished and blocked first.
-	wc.curFile.Lock()
-	defer wc.curFile.Unlock()
-
-	// Open the current file if needed. This will typically only be the
-	// case when moving to the next file to write to or on initial database
-	// load. However, it might also be the case if rollbacks happened after
-	// file writes started during a transaction commit.
-	if wc.curFile.file == nil {
-		file, err := s.openWriteFileFunc(wc.curFileNum)
-		if err != nil {
-			return blockLocation{}, err
-		}
-		wc.curFile.file = file
-	}
-
-	// Kaspa network.
-	origOffset := wc.curOffset
-	hasher := crc32.New(castagnoli)
-	var scratch [4]byte
-	byteOrder.PutUint32(scratch[:], uint32(s.network))
-	if err := s.writeData(scratch[:], "network"); err != nil {
-		return blockLocation{}, err
-	}
-	_, _ = hasher.Write(scratch[:])
-
-	// Block length.
-	byteOrder.PutUint32(scratch[:], blockLen)
-	if err := s.writeData(scratch[:], "block length"); err != nil {
-		return blockLocation{}, err
-	}
-	_, _ = hasher.Write(scratch[:])
-
-	// Serialized block.
-	if err := s.writeData(rawBlock[:], "block"); err != nil {
-		return blockLocation{}, err
-	}
-	_, _ = hasher.Write(rawBlock)
-
-	// Castagnoli CRC-32 as a checksum of all the previous.
-	if err := s.writeData(hasher.Sum(nil), "checksum"); err != nil {
-		return blockLocation{}, err
-	}
-
-	loc := blockLocation{
-		blockFileNum: wc.curFileNum,
-		fileOffset:   origOffset,
-		blockLen:     fullLen,
-	}
-	return loc, nil
-}
-
-// readBlock reads the specified block record and returns the serialized block.
-// It ensures the integrity of the block data by checking that the serialized
-// network matches the current network associated with the block store and
-// comparing the calculated checksum against the one stored in the flat file.
-// This function also automatically handles all file management such as opening
-// and closing files as necessary to stay within the maximum allowed open files
-// limit.
-//
-// Returns ErrDriverSpecific if the data fails to read for any reason and
-// ErrCorruption if the checksum of the read data doesn't match the checksum
-// read from the file.
-//
-// Format: <network><block length><serialized block><checksum>
-func (s *blockStore) readBlock(hash *daghash.Hash, loc blockLocation) ([]byte, error) {
-	// Get the referenced block file handle opening the file as needed. The
-	// function also handles closing files as needed to avoid going over the
-	// max allowed open files.
-	blockFile, err := s.blockFile(loc.blockFileNum)
-	if err != nil {
-		return nil, err
-	}
-
-	serializedData := make([]byte, loc.blockLen)
-	n, err := blockFile.file.ReadAt(serializedData, int64(loc.fileOffset))
-	blockFile.RUnlock()
-	if err != nil {
-		str := fmt.Sprintf("failed to read block %s from file %d, "+
-			"offset %d: %s", hash, loc.blockFileNum, loc.fileOffset,
-			err)
-		return nil, makeDbErr(database.ErrDriverSpecific, str, err)
-	}
-
-	// Calculate the checksum of the read data and ensure it matches the
-	// serialized checksum. This will detect any data corruption in the
-	// flat file without having to do much more expensive merkle root
-	// calculations on the loaded block.
-	serializedChecksum := binary.BigEndian.Uint32(serializedData[n-4:])
-	calculatedChecksum := crc32.Checksum(serializedData[:n-4], castagnoli)
-	if serializedChecksum != calculatedChecksum {
-		str := fmt.Sprintf("block data for block %s checksum "+
-			"does not match - got %x, want %x", hash,
-			calculatedChecksum, serializedChecksum)
-		return nil, makeDbErr(database.ErrCorruption, str, nil)
-	}
-
-	// The network associated with the block must match the current active
-	// network, otherwise somebody probably put the block files for the
-	// wrong network in the directory.
-	serializedNet := byteOrder.Uint32(serializedData[:4])
-	if serializedNet != uint32(s.network) {
-		str := fmt.Sprintf("block data for block %s is for the "+
-			"wrong network - got %d, want %d", hash, serializedNet,
-			uint32(s.network))
-		return nil, makeDbErr(database.ErrDriverSpecific, str, nil)
-	}
-
-	// The raw block excludes the network, length of the block, and
-	// checksum.
-	return serializedData[8 : n-4], nil
-}
-
-// readBlockRegion reads the specified amount of data at the provided offset for
-// a given block location. The offset is relative to the start of the
-// serialized block (as opposed to the beginning of the block record). This
-// function automatically handles all file management such as opening and
-// closing files as necessary to stay within the maximum allowed open files
-// limit.
-//
-// Returns ErrDriverSpecific if the data fails to read for any reason.
-func (s *blockStore) readBlockRegion(loc blockLocation, offset, numBytes uint32) ([]byte, error) {
-	// Get the referenced block file handle opening the file as needed. The
-	// function also handles closing files as needed to avoid going over the
-	// max allowed open files.
-	blockFile, err := s.blockFile(loc.blockFileNum)
-	if err != nil {
-		return nil, err
-	}
-
-	// Regions are offsets into the actual block, however the serialized
-	// data for a block includes an initial 4 bytes for network + 4 bytes
-	// for block length. Thus, add 8 bytes to adjust.
-	readOffset := loc.fileOffset + 8 + offset
-	serializedData := make([]byte, numBytes)
-	_, err = blockFile.file.ReadAt(serializedData, int64(readOffset))
-	blockFile.RUnlock()
-	if err != nil {
-		str := fmt.Sprintf("failed to read region from block file %d, "+
-			"offset %d, len %d: %s", loc.blockFileNum, readOffset,
-			numBytes, err)
-		return nil, makeDbErr(database.ErrDriverSpecific, str, err)
-	}
-
-	return serializedData, nil
-}
-
-// syncBlocks performs a file system sync on the flat file associated with the
-// store's current write cursor. It is safe to call even when there is not a
-// current write file in which case it will have no effect.
-//
-// This is used when flushing cached metadata updates to disk to ensure all the
-// block data is fully written before updating the metadata. This ensures the
-// metadata and block data can be properly reconciled in failure scenarios.
-func (s *blockStore) syncBlocks() error {
-	wc := s.writeCursor
-	wc.RLock()
-	defer wc.RUnlock()
-
-	// Nothing to do if there is no current file associated with the write
-	// cursor.
-	wc.curFile.RLock()
-	defer wc.curFile.RUnlock()
-	if wc.curFile.file == nil {
-		return nil
-	}
-
-	// Sync the file to disk.
-	if err := wc.curFile.file.Sync(); err != nil {
-		str := fmt.Sprintf("failed to sync file %d: %s", wc.curFileNum,
-			err)
-		return makeDbErr(database.ErrDriverSpecific, str, err)
-	}
-
-	return nil
-}
-
-// handleRollback rolls the block files on disk back to the provided file number
-// and offset. This involves potentially deleting and truncating the files that
-// were partially written.
-//
-// There are effectively two scenarios to consider here:
-//   1) Transient write failures from which recovery is possible
-//   2) More permanent failures such as hard disk death and/or removal
-//
-// In either case, the write cursor will be repositioned to the old block file
-// offset regardless of any other errors that occur while attempting to undo
-// writes.
-//
-// For the first scenario, this will lead to any data which failed to be undone
-// being overwritten and thus behaves as desired as the system continues to run.
-//
-// For the second scenario, the metadata which stores the current write cursor
-// position within the block files will not have been updated yet and thus if
-// the system eventually recovers (perhaps the hard drive is reconnected), it
-// will also lead to any data which failed to be undone being overwritten and
-// thus behaves as desired.
-//
-// Therefore, any errors are simply logged at a warning level rather than being
-// returned since there is nothing more that could be done about it anyways.
-func (s *blockStore) handleRollback(oldBlockFileNum, oldBlockOffset uint32) {
-	// Grab the write cursor mutex since it is modified throughout this
-	// function.
-	wc := s.writeCursor
-	wc.Lock()
-	defer wc.Unlock()
-
-	// Nothing to do if the rollback point is the same as the current write
-	// cursor.
-	if wc.curFileNum == oldBlockFileNum && wc.curOffset == oldBlockOffset {
-		return
-	}
-
-	// Regardless of any failures that happen below, reposition the write
-	// cursor to the old block file and offset.
-	defer func() {
-		wc.curFileNum = oldBlockFileNum
-		wc.curOffset = oldBlockOffset
-	}()
-
-	log.Debugf("ROLLBACK: Rolling back to file %d, offset %d",
-		oldBlockFileNum, oldBlockOffset)
-
-	// Close the current write file if it needs to be deleted. Then delete
-	// all files that are newer than the provided rollback file while
-	// also moving the write cursor file backwards accordingly.
-	if wc.curFileNum > oldBlockFileNum {
-		wc.curFile.Lock()
-		if wc.curFile.file != nil {
-			_ = wc.curFile.file.Close()
-			wc.curFile.file = nil
-		}
-		wc.curFile.Unlock()
-	}
-	for ; wc.curFileNum > oldBlockFileNum; wc.curFileNum-- {
-		if err := s.deleteFileFunc(wc.curFileNum); err != nil {
-			log.Warnf("ROLLBACK: Failed to delete block file "+
-				"number %d: %s", wc.curFileNum, err)
-			return
-		}
-	}
-
-	// Open the file for the current write cursor if needed.
-	wc.curFile.Lock()
-	if wc.curFile.file == nil {
-		obf, err := s.openWriteFileFunc(wc.curFileNum)
-		if err != nil {
-			wc.curFile.Unlock()
-			log.Warnf("ROLLBACK: %s", err)
-			return
-		}
-		wc.curFile.file = obf
-	}
-
-	// Truncate the to the provided rollback offset.
-	if err := wc.curFile.file.Truncate(int64(oldBlockOffset)); err != nil {
-		wc.curFile.Unlock()
-		log.Warnf("ROLLBACK: Failed to truncate file %d: %s",
-			wc.curFileNum, err)
-		return
-	}
-
-	// Sync the file to disk.
-	err := wc.curFile.file.Sync()
-	wc.curFile.Unlock()
-	if err != nil {
-		log.Warnf("ROLLBACK: Failed to sync file %d: %s",
-			wc.curFileNum, err)
-		return
-	}
-}
-
-// scanBlockFiles searches the database directory for all flat block files to
-// find the end of the most recent file. This position is considered the
-// current write cursor which is also stored in the metadata. Thus, it is used
-// to detect unexpected shutdowns in the middle of writes so the block files
-// can be reconciled.
-func scanBlockFiles(dbPath string) (int, uint32) {
-	lastFile := -1
-	fileLen := uint32(0)
-	for i := 0; ; i++ {
-		filePath := blockFilePath(dbPath, uint32(i))
-		st, err := os.Stat(filePath)
-		if err != nil {
-			break
-		}
-		lastFile = i
-
-		fileLen = uint32(st.Size())
-	}
-
-	log.Tracef("Scan found latest block file #%d with length %d", lastFile,
-		fileLen)
-	return lastFile, fileLen
-}
-
-// newBlockStore returns a new block store with the current block file number
-// and offset set and all fields initialized.
-func newBlockStore(basePath string, network wire.KaspaNet) *blockStore {
-	// Look for the end of the latest block to file to determine what the
-	// write cursor position is from the viewpoing of the block files on
-	// disk.
-	fileNum, fileOff := scanBlockFiles(basePath)
-	if fileNum == -1 {
-		fileNum = 0
-		fileOff = 0
-	}
-
-	store := &blockStore{
-		network:          network,
-		basePath:         basePath,
-		maxBlockFileSize: maxBlockFileSize,
-		maxOpenFiles:     maxOpenFiles,
-		openBlockFiles:   make(map[uint32]*lockableFile),
-		openBlocksLRU:    list.New(),
-		fileNumToLRUElem: make(map[uint32]*list.Element),
-
-		writeCursor: &writeCursor{
-			curFile:    &lockableFile{},
-			curFileNum: uint32(fileNum),
-			curOffset:  fileOff,
-		},
-	}
-	store.openFileFunc = store.openFile
-	store.openWriteFileFunc = store.openWriteFile
-	store.deleteFileFunc = store.deleteFile
-	return store
-}

database/ffldb/blockio_test.go

@@ -1,108 +0,0 @@
-package ffldb
-
-import (
-	"os"
-	"testing"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-func TestDeleteFile(t *testing.T) {
-	testBlock := util.NewBlock(wire.NewMsgBlock(
-		wire.NewBlockHeader(1, []*daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, 0, 0)))
-
-	tests := []struct {
-		fileNum     uint32
-		expectedErr bool
-	}{
-		{0, false},
-		{1, true},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestDeleteFile", t)
-			defer func() {
-				if !pdb.closed {
-					pdb.Close()
-				}
-			}()
-
-			err := pdb.Update(func(dbTx database.Tx) error {
-				dbTx.StoreBlock(testBlock)
-				return nil
-			})
-			if err != nil {
-				t.Fatalf("TestDeleteFile: Error storing block: %s", err)
-			}
-
-			err = pdb.Close()
-			if err != nil {
-				t.Fatalf("TestDeleteFile: Error closing file before deletion: %s", err)
-			}
-
-			err = pdb.store.deleteFile(test.fileNum)
-			if (err != nil) != test.expectedErr {
-				t.Errorf("TestDeleteFile: %d: Expected error status: %t, but got: %t",
-					test.fileNum, test.expectedErr, (err != nil))
-			}
-			if err == nil {
-				filePath := blockFilePath(pdb.store.basePath, test.fileNum)
-				if _, err := os.Stat(filePath); !os.IsNotExist(err) {
-					t.Errorf("TestDeleteFile: %d: File %s still exists", test.fileNum, filePath)
-				}
-			}
-		}()
-	}
-}
-
-// TestHandleRollbackErrors tests all error-cases in *blockStore.handleRollback().
-// The non-error-cases are tested in the more general tests.
-// Since handleRollback just logs errors, this test simply causes all error-cases to be hit,
-// and makes sure no panic occurs, as well as ensures the writeCursor was updated correctly.
-func TestHandleRollbackErrors(t *testing.T) {
-	testBlock := util.NewBlock(wire.NewMsgBlock(
-		wire.NewBlockHeader(1, []*daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, 0, 0)))
-
-	testBlockSize := uint32(testBlock.MsgBlock().SerializeSize())
-	tests := []struct {
-		name    string
-		fileNum uint32
-		offset  uint32
-	}{
-		// offset should be size of block + 12 bytes for block network, size and checksum
-		{"Nothing to rollback", 1, testBlockSize + 12},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestHandleRollbackErrors", t)
-			defer pdb.Close()
-
-			// Set maxBlockFileSize to testBlockSize so that writeCursor.curFileNum increments
-			pdb.store.maxBlockFileSize = testBlockSize
-
-			err := pdb.Update(func(dbTx database.Tx) error {
-				return dbTx.StoreBlock(testBlock)
-			})
-			if err != nil {
-				t.Fatalf("TestHandleRollbackErrors: %s: Error adding test block to database: %s", test.name, err)
-			}
-
-			pdb.store.handleRollback(test.fileNum, test.offset)
-
-			if pdb.store.writeCursor.curFileNum != test.fileNum {
-				t.Errorf("TestHandleRollbackErrors: %s: Expected fileNum: %d, but got: %d",
-					test.name, test.fileNum, pdb.store.writeCursor.curFileNum)
-			}
-
-			if pdb.store.writeCursor.curOffset != test.offset {
-				t.Errorf("TestHandleRollbackErrors: %s: offset fileNum: %d, but got: %d",
-					test.name, test.offset, pdb.store.writeCursor.curOffset)
-			}
-		}()
-	}
-}

database/ffldb/common_test.go

@@ -1,43 +0,0 @@
-package ffldb
-
-import (
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/filter"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-)
-
-func newTestDb(testName string, t *testing.T) *db {
-	dbPath := path.Join(os.TempDir(), "db_test", testName)
-	err := os.RemoveAll(dbPath)
-	if err != nil && !os.IsNotExist(err) {
-		t.Fatalf("%s: Error deleting database folder before starting: %s", testName, err)
-	}
-
-	network := wire.Simnet
-
-	opts := opt.Options{
-		ErrorIfExist: true,
-		Strict:       opt.DefaultStrict,
-		Compression:  opt.NoCompression,
-		Filter:       filter.NewBloomFilter(10),
-	}
-	metadataDbPath := filepath.Join(dbPath, metadataDbName)
-	ldb, err := leveldb.OpenFile(metadataDbPath, &opts)
-	if err != nil {
-		t.Errorf("%s: Error opening metadataDbPath: %s", testName, err)
-	}
-	err = initDB(ldb)
-	if err != nil {
-		t.Errorf("%s: Error initializing metadata Db: %s", testName, err)
-	}
-
-	store := newBlockStore(dbPath, network)
-	cache := newDbCache(ldb, store, defaultCacheSize, defaultFlushSecs)
-	return &db{store: store, cache: cache}
-}

database/ffldb/db_test.go

@@ -1,658 +0,0 @@
-package ffldb
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/pkg/errors"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-// TestCursorDeleteErrors tests all error-cases in *cursor.Delete().
-// The non-error-cases are tested in the more general tests.
-func TestCursorDeleteErrors(t *testing.T) {
-	pdb := newTestDb("TestCursorDeleteErrors", t)
-
-	nestedBucket := []byte("nestedBucket")
-	key := []byte("key")
-	value := []byte("value")
-
-	err := pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		_, err := metadata.CreateBucket(nestedBucket)
-		if err != nil {
-			return err
-		}
-		metadata.Put(key, value)
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestCursorDeleteErrors: Error setting up test-database: %s", err)
-	}
-
-	// Check for error when attempted to delete a bucket
-	err = pdb.Update(func(dbTx database.Tx) error {
-		cursor := dbTx.Metadata().Cursor()
-		found := false
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			if bytes.Equal(cursor.Key(), nestedBucket) {
-				found = true
-				break
-			}
-		}
-		if !found {
-			t.Errorf("TestCursorDeleteErrors: Key '%s' not found", string(nestedBucket))
-		}
-
-		err := cursor.Delete()
-		if !database.IsErrorCode(err, database.ErrIncompatibleValue) {
-			t.Errorf("TestCursorDeleteErrors: Expected error of type ErrIncompatibleValue, "+
-				"when deleting bucket, but got %v", err)
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestCursorDeleteErrors: Unexpected error from pdb.Update "+
-			"when attempting to delete bucket: %s", err)
-	}
-
-	// Check for error when transaction is not writable
-	err = pdb.View(func(dbTx database.Tx) error {
-		cursor := dbTx.Metadata().Cursor()
-		if !cursor.First() {
-			t.Fatal("TestCursorDeleteErrors: Nothing in cursor when testing for delete in " +
-				"non-writable transaction")
-		}
-
-		err := cursor.Delete()
-		if !database.IsErrorCode(err, database.ErrTxNotWritable) {
-			t.Errorf("TestCursorDeleteErrors: Expected error of type ErrTxNotWritable "+
-				"when calling .Delete() on non-writable transaction, but got '%v' instead", err)
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestCursorDeleteErrors: Unexpected error from pdb.Update "+
-			"when attempting to delete on non-writable transaction: %s", err)
-	}
-
-	// Check for error when cursor was exhausted
-	err = pdb.Update(func(dbTx database.Tx) error {
-		cursor := dbTx.Metadata().Cursor()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-		}
-
-		err := cursor.Delete()
-		if !database.IsErrorCode(err, database.ErrIncompatibleValue) {
-			t.Errorf("TestCursorDeleteErrors: Expected error of type ErrIncompatibleValue "+
-				"when calling .Delete() on exhausted cursor, but got '%v' instead", err)
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestCursorDeleteErrors: Unexpected error from pdb.Update "+
-			"when attempting to delete on exhausted cursor: %s", err)
-	}
-
-	// Check for error when transaction is closed
-	tx, err := pdb.Begin(true)
-	if err != nil {
-		t.Fatalf("TestCursorDeleteErrors: Error in pdb.Begin(): %s", err)
-	}
-	cursor := tx.Metadata().Cursor()
-	err = tx.Commit()
-	if err != nil {
-		t.Fatalf("TestCursorDeleteErrors: Error in tx.Commit(): %s", err)
-	}
-
-	err = cursor.Delete()
-	if !database.IsErrorCode(err, database.ErrTxClosed) {
-		t.Errorf("TestCursorDeleteErrors: Expected error of type ErrTxClosed "+
-			"when calling .Delete() on with closed transaction, but got '%s' instead", err)
-	}
-}
-
-func TestSkipPendingUpdates(t *testing.T) {
-	pdb := newTestDb("TestSkipPendingUpdates", t)
-	defer pdb.Close()
-
-	value := []byte("value")
-	// Add numbered prefixes to keys so that they are in expected order, and before any other keys
-	firstKey := []byte("1 - first")
-	toDeleteKey := []byte("2 - toDelete")
-	toUpdateKey := []byte("3 - toUpdate")
-	secondKey := []byte("4 - second")
-
-	// create initial metadata for test
-	err := pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		if err := metadata.Put(firstKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(toDeleteKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(toUpdateKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(secondKey, value); err != nil {
-			return err
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestSkipPendingUpdates: Error adding to metadata: %s", err)
-	}
-
-	// test skips
-	err = pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		if err := metadata.Delete(toDeleteKey); err != nil {
-			return err
-		}
-		if err := metadata.Put(toUpdateKey, value); err != nil {
-			return err
-		}
-		cursor := metadata.Cursor().(*cursor)
-		dbIter := cursor.dbIter
-
-		// Check that first is ok
-		dbIter.First()
-		expectedKey := bucketizedKey(metadataBucketID, firstKey)
-		if !bytes.Equal(dbIter.Key(), expectedKey) {
-			t.Errorf("TestSkipPendingUpdates: 1: key expected to be %v but is %v", expectedKey, dbIter.Key())
-		}
-
-		// Go to the next key, which is toDelete
-		dbIter.Next()
-		expectedKey = bucketizedKey(metadataBucketID, toDeleteKey)
-		if !bytes.Equal(dbIter.Key(), expectedKey) {
-			t.Errorf("TestSkipPendingUpdates: 2: key expected to be %s but is %s", expectedKey, dbIter.Key())
-		}
-
-		// at this point toDeleteKey and toUpdateKey should be skipped
-		cursor.skipPendingUpdates(true)
-		expectedKey = bucketizedKey(metadataBucketID, secondKey)
-		if !bytes.Equal(dbIter.Key(), expectedKey) {
-			t.Errorf("TestSkipPendingUpdates: 3: key expected to be %s but is %s", expectedKey, dbIter.Key())
-		}
-
-		// now traverse backwards - should get toUpdate
-		dbIter.Prev()
-		expectedKey = bucketizedKey(metadataBucketID, toUpdateKey)
-		if !bytes.Equal(dbIter.Key(), expectedKey) {
-			t.Errorf("TestSkipPendingUpdates: 4: key expected to be %s but is %s", expectedKey, dbIter.Key())
-		}
-
-		// at this point toUpdateKey and toDeleteKey should be skipped
-		cursor.skipPendingUpdates(false)
-		expectedKey = bucketizedKey(metadataBucketID, firstKey)
-		if !bytes.Equal(dbIter.Key(), expectedKey) {
-			t.Errorf("TestSkipPendingUpdates: 5: key expected to be %s but is %s", expectedKey, dbIter.Key())
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestSkipPendingUpdates: Error running main part of test: %s", err)
-	}
-}
-
-// TestCursor tests various edge-cases in cursor that were not hit by the more general tests
-func TestCursor(t *testing.T) {
-	pdb := newTestDb("TestCursor", t)
-	defer pdb.Close()
-
-	value := []byte("value")
-	// Add numbered prefixes to keys so that they are in expected order, and before any other keys
-	firstKey := []byte("1 - first")
-	toDeleteKey := []byte("2 - toDelete")
-	toUpdateKey := []byte("3 - toUpdate")
-	secondKey := []byte("4 - second")
-
-	// create initial metadata for test
-	err := pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		if err := metadata.Put(firstKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(toDeleteKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(toUpdateKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(secondKey, value); err != nil {
-			return err
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("Error adding to metadata: %s", err)
-	}
-
-	// run the actual tests
-	err = pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		if err := metadata.Delete(toDeleteKey); err != nil {
-			return err
-		}
-		if err := metadata.Put(toUpdateKey, value); err != nil {
-			return err
-		}
-		cursor := metadata.Cursor().(*cursor)
-
-		// Check prev when currentIter == nil
-		if ok := cursor.Prev(); ok {
-			t.Error("1: .Prev() should have returned false, but have returned true")
-		}
-		// Same thing for .Next()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-		}
-		if ok := cursor.Next(); ok {
-			t.Error("2: .Next() should have returned false, but have returned true")
-		}
-
-		// Check that Key(), rawKey(), Value(), and rawValue() all return nil when currentIter == nil
-		if key := cursor.Key(); key != nil {
-			t.Errorf("3: .Key() should have returned nil, but have returned '%s' instead", key)
-		}
-		if key := cursor.rawKey(); key != nil {
-			t.Errorf("4: .rawKey() should have returned nil, but have returned '%s' instead", key)
-		}
-		if value := cursor.Value(); value != nil {
-			t.Errorf("5: .Value() should have returned nil, but have returned '%s' instead", value)
-		}
-		if value := cursor.rawValue(); value != nil {
-			t.Errorf("6: .rawValue() should have returned nil, but have returned '%s' instead", value)
-		}
-
-		// Check rawValue in normal operation
-		cursor.First()
-		if rawValue := cursor.rawValue(); !bytes.Equal(rawValue, value) {
-			t.Errorf("7: rawValue should have returned '%s' but have returned '%s' instead", value, rawValue)
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("Error running the actual tests: %s", err)
-	}
-}
-
-// TestCreateBucketErrors tests all error-cases in *bucket.CreateBucket().
-// The non-error-cases are tested in the more general tests.
-func TestCreateBucketErrors(t *testing.T) {
-	testKey := []byte("key")
-
-	tests := []struct {
-		name        string
-		key         []byte
-		isWritable  bool
-		isClosed    bool
-		expectedErr database.ErrorCode
-	}{
-		{"empty key", []byte{}, true, false, database.ErrBucketNameRequired},
-		{"transaction is closed", testKey, true, true, database.ErrTxClosed},
-		{"transaction is not writable", testKey, false, false, database.ErrTxNotWritable},
-		{"key already exists", blockIdxBucketName, true, false, database.ErrBucketExists},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestCreateBucketErrors", t)
-			defer pdb.Close()
-
-			tx, err := pdb.Begin(test.isWritable)
-			defer tx.Commit()
-			if err != nil {
-				t.Fatalf("TestCreateBucketErrors: %s: error from pdb.Begin: %s", test.name, err)
-			}
-			if test.isClosed {
-				err = tx.Commit()
-				if err != nil {
-					t.Fatalf("TestCreateBucketErrors: %s: error from tx.Commit: %s", test.name, err)
-				}
-			}
-
-			metadata := tx.Metadata()
-
-			_, err = metadata.CreateBucket(test.key)
-
-			if !database.IsErrorCode(err, test.expectedErr) {
-				t.Errorf("TestCreateBucketErrors: %s: Expected error of type %d "+
-					"but got '%v'", test.name, test.expectedErr, err)
-			}
-
-		}()
-	}
-}
-
-// TestPutErrors tests all error-cases in *bucket.Put().
-// The non-error-cases are tested in the more general tests.
-func TestPutErrors(t *testing.T) {
-	testKey := []byte("key")
-	testValue := []byte("value")
-
-	tests := []struct {
-		name        string
-		key         []byte
-		isWritable  bool
-		isClosed    bool
-		expectedErr database.ErrorCode
-	}{
-		{"empty key", []byte{}, true, false, database.ErrKeyRequired},
-		{"transaction is closed", testKey, true, true, database.ErrTxClosed},
-		{"transaction is not writable", testKey, false, false, database.ErrTxNotWritable},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestPutErrors", t)
-			defer pdb.Close()
-
-			tx, err := pdb.Begin(test.isWritable)
-			defer tx.Commit()
-			if err != nil {
-				t.Fatalf("TestPutErrors: %s: error from pdb.Begin: %s", test.name, err)
-			}
-			if test.isClosed {
-				err = tx.Commit()
-				if err != nil {
-					t.Fatalf("TestPutErrors: %s: error from tx.Commit: %s", test.name, err)
-				}
-			}
-
-			metadata := tx.Metadata()
-
-			err = metadata.Put(test.key, testValue)
-
-			if !database.IsErrorCode(err, test.expectedErr) {
-				t.Errorf("TestPutErrors: %s: Expected error of type %d "+
-					"but got '%v'", test.name, test.expectedErr, err)
-			}
-
-		}()
-	}
-}
-
-// TestGetErrors tests all error-cases in *bucket.Get().
-// The non-error-cases are tested in the more general tests.
-func TestGetErrors(t *testing.T) {
-	testKey := []byte("key")
-
-	tests := []struct {
-		name     string
-		key      []byte
-		isClosed bool
-	}{
-		{"empty key", []byte{}, false},
-		{"transaction is closed", testKey, true},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestGetErrors", t)
-			defer pdb.Close()
-
-			tx, err := pdb.Begin(false)
-			defer tx.Rollback()
-			if err != nil {
-				t.Fatalf("TestGetErrors: %s: error from pdb.Begin: %s", test.name, err)
-			}
-			if test.isClosed {
-				err = tx.Rollback()
-				if err != nil {
-					t.Fatalf("TestGetErrors: %s: error from tx.Commit: %s", test.name, err)
-				}
-			}
-
-			metadata := tx.Metadata()
-
-			if result := metadata.Get(test.key); result != nil {
-				t.Errorf("TestGetErrors: %s: Expected to return nil, but got %v", test.name, result)
-			}
-		}()
-	}
-}
-
-// TestDeleteErrors tests all error-cases in *bucket.Delete().
-// The non-error-cases are tested in the more general tests.
-func TestDeleteErrors(t *testing.T) {
-	testKey := []byte("key")
-
-	tests := []struct {
-		name        string
-		key         []byte
-		isWritable  bool
-		isClosed    bool
-		expectedErr database.ErrorCode
-	}{
-		{"empty key", []byte{}, true, false, database.ErrKeyRequired},
-		{"transaction is closed", testKey, true, true, database.ErrTxClosed},
-		{"transaction is not writable", testKey, false, false, database.ErrTxNotWritable},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestDeleteErrors", t)
-			defer pdb.Close()
-
-			tx, err := pdb.Begin(test.isWritable)
-			defer tx.Commit()
-			if err != nil {
-				t.Fatalf("TestDeleteErrors: %s: error from pdb.Begin: %s", test.name, err)
-			}
-			if test.isClosed {
-				err = tx.Commit()
-				if err != nil {
-					t.Fatalf("TestDeleteErrors: %s: error from tx.Commit: %s", test.name, err)
-				}
-			}
-
-			metadata := tx.Metadata()
-
-			err = metadata.Delete(test.key)
-
-			if !database.IsErrorCode(err, test.expectedErr) {
-				t.Errorf("TestDeleteErrors: %s: Expected error of type %d "+
-					"but got '%v'", test.name, test.expectedErr, err)
-			}
-		}()
-	}
-}
-
-func TestForEachBucket(t *testing.T) {
-	pdb := newTestDb("TestForEachBucket", t)
-
-	// set-up test
-	testKey := []byte("key")
-	testValue := []byte("value")
-	bucketKeys := [][]byte{{1}, {2}, {3}}
-
-	err := pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		for _, bucketKey := range bucketKeys {
-			bucket, err := metadata.CreateBucket(bucketKey)
-			if err != nil {
-				return err
-			}
-
-			err = bucket.Put(testKey, testValue)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestForEachBucket: Error setting up test-database: %s", err)
-	}
-
-	// actual test
-	err = pdb.View(func(dbTx database.Tx) error {
-		i := 0
-		metadata := dbTx.Metadata()
-
-		err := metadata.ForEachBucket(func(bucketKey []byte) error {
-			if i >= len(bucketKeys) { // in case there are any other buckets in metadata
-				return nil
-			}
-
-			expectedBucketKey := bucketKeys[i]
-			if !bytes.Equal(expectedBucketKey, bucketKey) {
-				t.Errorf("TestForEachBucket: %d: Expected bucket key: %v, but got: %v",
-					i, expectedBucketKey, bucketKey)
-				return nil
-			}
-			bucket := metadata.Bucket(bucketKey)
-			if bucket == nil {
-				t.Errorf("TestForEachBucket: %d: Bucket is nil", i)
-				return nil
-			}
-
-			value := bucket.Get(testKey)
-			if !bytes.Equal(testValue, value) {
-				t.Errorf("TestForEachBucket: %d: Expected value: %s, but got: %s",
-					i, testValue, value)
-				return nil
-			}
-
-			i++
-			return nil
-		})
-
-		return err
-	})
-	if err != nil {
-		t.Fatalf("TestForEachBucket: Error running actual tests: %s", err)
-	}
-}
-
-// TestStoreBlockErrors tests all error-cases in *tx.StoreBlock().
-// The non-error-cases are tested in the more general tests.
-func TestStoreBlockErrors(t *testing.T) {
-	testBlock := util.NewBlock(wire.NewMsgBlock(wire.NewBlockHeader(1, []*daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, 0, 0)))
-
-	tests := []struct {
-		name        string
-		isWritable  bool
-		isClosed    bool
-		expectedErr database.ErrorCode
-	}{
-		{"transaction is closed", true, true, database.ErrTxClosed},
-		{"transaction is not writable", false, false, database.ErrTxNotWritable},
-	}
-
-	for _, test := range tests {
-		func() {
-			pdb := newTestDb("TestStoreBlockErrors", t)
-			defer pdb.Close()
-
-			tx, err := pdb.Begin(test.isWritable)
-			defer tx.Commit()
-			if err != nil {
-				t.Fatalf("TestStoreBlockErrors: %s: error from pdb.Begin: %s", test.name, err)
-			}
-			if test.isClosed {
-				err = tx.Commit()
-				if err != nil {
-					t.Fatalf("TestStoreBlockErrors: %s: error from tx.Commit: %s", test.name, err)
-				}
-			}
-
-			err = tx.StoreBlock(testBlock)
-			if !database.IsErrorCode(err, test.expectedErr) {
-				t.Errorf("TestStoreBlockErrors: %s: Expected error of type %d "+
-					"but got '%v'", test.name, test.expectedErr, err)
-			}
-
-		}()
-	}
-}
-
-// TestDeleteDoubleNestedBucket tests what happens when bucket.DeleteBucket()
-// is invoked on a bucket that contains a nested bucket.
-func TestDeleteDoubleNestedBucket(t *testing.T) {
-	pdb := newTestDb("TestDeleteDoubleNestedBucket", t)
-	defer pdb.Close()
-
-	firstKey := []byte("first")
-	secondKey := []byte("second")
-	key := []byte("key")
-	value := []byte("value")
-	var rawKey, rawSecondKey []byte
-
-	// Test setup
-	err := pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		firstBucket, err := metadata.CreateBucket(firstKey)
-		if err != nil {
-			return errors.Errorf("Error creating first bucket: %s", err)
-		}
-		secondBucket, err := firstBucket.CreateBucket(secondKey)
-		if err != nil {
-			return errors.Errorf("Error creating second bucket: %s", err)
-		}
-		secondBucket.Put(key, value)
-
-		// extract rawKey from cursor and make sure it's in raw database
-		c := secondBucket.Cursor()
-		for ok := c.First(); ok && !bytes.Equal(c.Key(), key); ok = c.Next() {
-		}
-		if !bytes.Equal(c.Key(), key) {
-			return errors.Errorf("Couldn't find key to extract rawKey")
-		}
-		rawKey = c.(*cursor).rawKey()
-		if dbTx.(*transaction).fetchKey(rawKey) == nil {
-			return errors.Errorf("rawKey not found")
-		}
-
-		// extract rawSecondKey from cursor and make sure it's in raw database
-		c = firstBucket.Cursor()
-		for ok := c.First(); ok && !bytes.Equal(c.Key(), secondKey); ok = c.Next() {
-		}
-		if !bytes.Equal(c.Key(), secondKey) {
-			return errors.Errorf("Couldn't find secondKey to extract rawSecondKey")
-		}
-		rawSecondKey = c.(*cursor).rawKey()
-		if dbTx.(*transaction).fetchKey(rawSecondKey) == nil {
-			return errors.Errorf("rawSecondKey not found for some reason")
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestDeleteDoubleNestedBucket: Error in test setup pdb.Update: %s", err)
-	}
-
-	// Actual test
-	err = pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		err := metadata.DeleteBucket(firstKey)
-		if err != nil {
-			return err
-		}
-
-		if dbTx.(*transaction).fetchKey(rawSecondKey) != nil {
-			t.Error("TestDeleteDoubleNestedBucket: secondBucket was not deleted")
-		}
-
-		if dbTx.(*transaction).fetchKey(rawKey) != nil {
-			t.Error("TestDeleteDoubleNestedBucket: value inside secondBucket was not deleted")
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestDeleteDoubleNestedBucket: Error in actual test pdb.Update: %s", err)
-	}
-}

database/ffldb/dbcache.go

@@ -1,647 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb
-
-import (
-	"bytes"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/kaspanet/kaspad/database/internal/treap"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-const (
-	// defaultCacheSize is the default size for the database cache.
-	defaultCacheSize = 100 * 1024 * 1024 // 100 MB
-
-	// defaultFlushSecs is the default number of seconds to use as a
-	// threshold in between database cache flushes when the cache size has
-	// not been exceeded.
-	defaultFlushSecs = 300 // 5 minutes
-)
-
-// ldbCacheIter wraps a treap iterator to provide the additional functionality
-// needed to satisfy the leveldb iterator.Iterator interface.
-type ldbCacheIter struct {
-	*treap.Iterator
-}
-
-// Enforce ldbCacheIterator implements the leveldb iterator.Iterator interface.
-var _ iterator.Iterator = (*ldbCacheIter)(nil)
-
-// Error is only provided to satisfy the iterator interface as there are no
-// errors for this memory-only structure.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *ldbCacheIter) Error() error {
-	return nil
-}
-
-// SetReleaser is only provided to satisfy the iterator interface as there is no
-// need to override it.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *ldbCacheIter) SetReleaser(releaser util.Releaser) {
-}
-
-// Release is only provided to satisfy the iterator interface.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *ldbCacheIter) Release() {
-}
-
-// newLdbCacheIter creates a new treap iterator for the given slice against the
-// pending keys for the passed cache snapshot and returns it wrapped in an
-// ldbCacheIter so it can be used as a leveldb iterator.
-func newLdbCacheIter(snap *dbCacheSnapshot, slice *util.Range) *ldbCacheIter {
-	iter := snap.pendingKeys.Iterator(slice.Start, slice.Limit)
-	return &ldbCacheIter{Iterator: iter}
-}
-
-// dbCacheIterator defines an iterator over the key/value pairs in the database
-// cache and underlying database.
-type dbCacheIterator struct {
-	cacheSnapshot *dbCacheSnapshot
-	dbIter        iterator.Iterator
-	cacheIter     iterator.Iterator
-	currentIter   iterator.Iterator
-	released      bool
-}
-
-// Enforce dbCacheIterator implements the leveldb iterator.Iterator interface.
-var _ iterator.Iterator = (*dbCacheIterator)(nil)
-
-// skipPendingUpdates skips any keys at the current database iterator position
-// that are being updated by the cache. The forwards flag indicates the
-// direction the iterator is moving.
-func (iter *dbCacheIterator) skipPendingUpdates(forwards bool) {
-	for iter.dbIter.Valid() {
-		var skip bool
-		key := iter.dbIter.Key()
-		if iter.cacheSnapshot.pendingRemove.Has(key) {
-			skip = true
-		} else if iter.cacheSnapshot.pendingKeys.Has(key) {
-			skip = true
-		}
-		if !skip {
-			break
-		}
-
-		if forwards {
-			iter.dbIter.Next()
-		} else {
-			iter.dbIter.Prev()
-		}
-	}
-}
-
-// chooseIterator first skips any entries in the database iterator that are
-// being updated by the cache and sets the current iterator to the appropriate
-// iterator depending on their validity and the order they compare in while taking
-// into account the direction flag. When the iterator is being moved forwards
-// and both iterators are valid, the iterator with the smaller key is chosen and
-// vice versa when the iterator is being moved backwards.
-func (iter *dbCacheIterator) chooseIterator(forwards bool) bool {
-	// Skip any keys at the current database iterator position that are
-	// being updated by the cache.
-	iter.skipPendingUpdates(forwards)
-
-	// When both iterators are exhausted, the iterator is exhausted too.
-	if !iter.dbIter.Valid() && !iter.cacheIter.Valid() {
-		iter.currentIter = nil
-		return false
-	}
-
-	// Choose the database iterator when the cache iterator is exhausted.
-	if !iter.cacheIter.Valid() {
-		iter.currentIter = iter.dbIter
-		return true
-	}
-
-	// Choose the cache iterator when the database iterator is exhausted.
-	if !iter.dbIter.Valid() {
-		iter.currentIter = iter.cacheIter
-		return true
-	}
-
-	// Both iterators are valid, so choose the iterator with either the
-	// smaller or larger key depending on the forwards flag.
-	compare := bytes.Compare(iter.dbIter.Key(), iter.cacheIter.Key())
-	if (forwards && compare > 0) || (!forwards && compare < 0) {
-		iter.currentIter = iter.cacheIter
-	} else {
-		iter.currentIter = iter.dbIter
-	}
-	return true
-}
-
-// First positions the iterator at the first key/value pair and returns whether
-// or not the pair exists.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) First() bool {
-	// Seek to the first key in both the database and cache iterators and
-	// choose the iterator that is both valid and has the smaller key.
-	iter.dbIter.First()
-	iter.cacheIter.First()
-	return iter.chooseIterator(true)
-}
-
-// Last positions the iterator at the last key/value pair and returns whether or
-// not the pair exists.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Last() bool {
-	// Seek to the last key in both the database and cache iterators and
-	// choose the iterator that is both valid and has the larger key.
-	iter.dbIter.Last()
-	iter.cacheIter.Last()
-	return iter.chooseIterator(false)
-}
-
-// Next moves the iterator one key/value pair forward and returns whether or not
-// the pair exists.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Next() bool {
-	// Nothing to return if cursor is exhausted.
-	if iter.currentIter == nil {
-		return false
-	}
-
-	// Move the current iterator to the next entry and choose the iterator
-	// that is both valid and has the smaller key.
-	iter.currentIter.Next()
-	return iter.chooseIterator(true)
-}
-
-// Prev moves the iterator one key/value pair backward and returns whether or
-// not the pair exists.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Prev() bool {
-	// Nothing to return if cursor is exhausted.
-	if iter.currentIter == nil {
-		return false
-	}
-
-	// Move the current iterator to the previous entry and choose the
-	// iterator that is both valid and has the larger key.
-	iter.currentIter.Prev()
-	return iter.chooseIterator(false)
-}
-
-// Seek positions the iterator at the first key/value pair that is greater than
-// or equal to the passed seek key. Returns false if no suitable key was found.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Seek(key []byte) bool {
-	// Seek to the provided key in both the database and cache iterators
-	// then choose the iterator that is both valid and has the larger key.
-	iter.dbIter.Seek(key)
-	iter.cacheIter.Seek(key)
-	return iter.chooseIterator(true)
-}
-
-// Valid indicates whether the iterator is positioned at a valid key/value pair.
-// It will be considered invalid when the iterator is newly created or exhausted.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Valid() bool {
-	return iter.currentIter != nil
-}
-
-// Key returns the current key the iterator is pointing to.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Key() []byte {
-	// Nothing to return if iterator is exhausted.
-	if iter.currentIter == nil {
-		return nil
-	}
-
-	return iter.currentIter.Key()
-}
-
-// Value returns the current value the iterator is pointing to.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Value() []byte {
-	// Nothing to return if iterator is exhausted.
-	if iter.currentIter == nil {
-		return nil
-	}
-
-	return iter.currentIter.Value()
-}
-
-// SetReleaser is only provided to satisfy the iterator interface as there is no
-// need to override it.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) SetReleaser(releaser util.Releaser) {
-}
-
-// Release releases the iterator by removing the underlying treap iterator from
-// the list of active iterators against the pending keys treap.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Release() {
-	if !iter.released {
-		iter.dbIter.Release()
-		iter.cacheIter.Release()
-		iter.currentIter = nil
-		iter.released = true
-	}
-}
-
-// Error is only provided to satisfy the iterator interface as there are no
-// errors for this memory-only structure.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *dbCacheIterator) Error() error {
-	return nil
-}
-
-// dbCacheSnapshot defines a snapshot of the database cache and underlying
-// database at a particular point in time.
-type dbCacheSnapshot struct {
-	dbSnapshot    *leveldb.Snapshot
-	pendingKeys   *treap.Immutable
-	pendingRemove *treap.Immutable
-}
-
-// Has returns whether or not the passed key exists.
-func (snap *dbCacheSnapshot) Has(key []byte) bool {
-	// Check the cached entries first.
-	if snap.pendingRemove.Has(key) {
-		return false
-	}
-	if snap.pendingKeys.Has(key) {
-		return true
-	}
-
-	// Consult the database.
-	hasKey, _ := snap.dbSnapshot.Has(key, nil)
-	return hasKey
-}
-
-// Get returns the value for the passed key. The function will return nil when
-// the key does not exist.
-func (snap *dbCacheSnapshot) Get(key []byte) []byte {
-	// Check the cached entries first.
-	if snap.pendingRemove.Has(key) {
-		return nil
-	}
-	if value := snap.pendingKeys.Get(key); value != nil {
-		return value
-	}
-
-	// Consult the database.
-	value, err := snap.dbSnapshot.Get(key, nil)
-	if err != nil {
-		return nil
-	}
-	return value
-}
-
-// Release releases the snapshot.
-func (snap *dbCacheSnapshot) Release() {
-	snap.dbSnapshot.Release()
-	snap.pendingKeys = nil
-	snap.pendingRemove = nil
-}
-
-// NewIterator returns a new iterator for the snapshot. The newly returned
-// iterator is not pointing to a valid item until a call to one of the methods
-// to position it is made.
-//
-// The slice parameter allows the iterator to be limited to a range of keys.
-// The start key is inclusive and the limit key is exclusive. Either or both
-// can be nil if the functionality is not desired.
-func (snap *dbCacheSnapshot) NewIterator(slice *util.Range) *dbCacheIterator {
-	return &dbCacheIterator{
-		dbIter:        snap.dbSnapshot.NewIterator(slice, nil),
-		cacheIter:     newLdbCacheIter(snap, slice),
-		cacheSnapshot: snap,
-	}
-}
-
-// dbCache provides a database cache layer backed by an underlying database. It
-// allows a maximum cache size and flush interval to be specified such that the
-// cache is flushed to the database when the cache size exceeds the maximum
-// configured value or it has been longer than the configured interval since the
-// last flush. This effectively provides transaction batching so that callers
-// can commit transactions at will without incurring large performance hits due
-// to frequent disk syncs.
-type dbCache struct {
-	// ldb is the underlying leveldb DB for metadata.
-	ldb *leveldb.DB
-
-	// store is used to sync blocks to flat files.
-	store *blockStore
-
-	// The following fields are related to flushing the cache to persistent
-	// storage. Note that all flushing is performed in an opportunistic
-	// fashion. This means that it is only flushed during a transaction or
-	// when the database cache is closed.
-	//
-	// maxSize is the maximum size threshold the cache can grow to before
-	// it is flushed.
-	//
-	// flushInterval is the threshold interval of time that is allowed to
-	// pass before the cache is flushed.
-	//
-	// lastFlush is the time the cache was last flushed. It is used in
-	// conjunction with the current time and the flush interval.
-	//
-	// NOTE: These flush related fields are protected by the database write
-	// lock.
-	maxSize       uint64
-	flushInterval time.Duration
-	lastFlush     time.Time
-
-	// The following fields hold the keys that need to be stored or deleted
-	// from the underlying database once the cache is full, enough time has
-	// passed, or when the database is shutting down. Note that these are
-	// stored using immutable treaps to support O(1) MVCC snapshots against
-	// the cached data. The cacheLock is used to protect concurrent access
-	// for cache updates and snapshots.
-	cacheLock    sync.RWMutex
-	cachedKeys   *treap.Immutable
-	cachedRemove *treap.Immutable
-}
-
-// Snapshot returns a snapshot of the database cache and underlying database at
-// a particular point in time.
-//
-// The snapshot must be released after use by calling Release.
-func (c *dbCache) Snapshot() (*dbCacheSnapshot, error) {
-	dbSnapshot, err := c.ldb.GetSnapshot()
-	if err != nil {
-		str := "failed to open transaction"
-		return nil, convertErr(str, err)
-	}
-
-	// Since the cached keys to be added and removed use an immutable treap,
-	// a snapshot is simply obtaining the root of the tree under the lock
-	// which is used to atomically swap the root.
-	c.cacheLock.RLock()
-	cacheSnapshot := &dbCacheSnapshot{
-		dbSnapshot:    dbSnapshot,
-		pendingKeys:   c.cachedKeys,
-		pendingRemove: c.cachedRemove,
-	}
-	c.cacheLock.RUnlock()
-	return cacheSnapshot, nil
-}
-
-// updateDB invokes the passed function in the context of a managed leveldb
-// transaction. Any errors returned from the user-supplied function will cause
-// the transaction to be rolled back and are returned from this function.
-// Otherwise, the transaction is committed when the user-supplied function
-// returns a nil error.
-func (c *dbCache) updateDB(fn func(ldbTx *leveldb.Transaction) error) error {
-	// Start a leveldb transaction.
-	ldbTx, err := c.ldb.OpenTransaction()
-	if err != nil {
-		return convertErr("failed to open ldb transaction", err)
-	}
-
-	if err := fn(ldbTx); err != nil {
-		ldbTx.Discard()
-		return err
-	}
-
-	// Commit the leveldb transaction and convert any errors as needed.
-	if err := ldbTx.Commit(); err != nil {
-		return convertErr("failed to commit leveldb transaction", err)
-	}
-	return nil
-}
-
-// TreapForEacher is an interface which allows iteration of a treap in ascending
-// order using a user-supplied callback for each key/value pair. It mainly
-// exists so both mutable and immutable treaps can be atomically committed to
-// the database with the same function.
-type TreapForEacher interface {
-	ForEach(func(k, v []byte) bool)
-}
-
-// commitTreaps atomically commits all of the passed pending add/update/remove
-// updates to the underlying database.
-func (c *dbCache) commitTreaps(pendingKeys, pendingRemove TreapForEacher) error {
-	// Perform all leveldb updates using an atomic transaction.
-	return c.updateDB(func(ldbTx *leveldb.Transaction) error {
-		var innerErr error
-		pendingKeys.ForEach(func(k, v []byte) bool {
-			if dbErr := ldbTx.Put(k, v, nil); dbErr != nil {
-				str := fmt.Sprintf("failed to put key %q to "+
-					"ldb transaction", k)
-				innerErr = convertErr(str, dbErr)
-				return false
-			}
-			return true
-		})
-		if innerErr != nil {
-			return innerErr
-		}
-
-		pendingRemove.ForEach(func(k, v []byte) bool {
-			if dbErr := ldbTx.Delete(k, nil); dbErr != nil {
-				str := fmt.Sprintf("failed to delete "+
-					"key %q from ldb transaction",
-					k)
-				innerErr = convertErr(str, dbErr)
-				return false
-			}
-			return true
-		})
-		return innerErr
-	})
-}
-
-// flush flushes the database cache to persistent storage. This involes syncing
-// the block store and replaying all transactions that have been applied to the
-// cache to the underlying database.
-//
-// This function MUST be called with the database write lock held.
-func (c *dbCache) flush() error {
-	c.lastFlush = time.Now()
-
-	// Sync the current write file associated with the block store. This is
-	// necessary before writing the metadata to prevent the case where the
-	// metadata contains information about a block which actually hasn't
-	// been written yet in unexpected shutdown scenarios.
-	if err := c.store.syncBlocks(); err != nil {
-		return err
-	}
-
-	// Since the cached keys to be added and removed use an immutable treap,
-	// a snapshot is simply obtaining the root of the tree under the lock
-	// which is used to atomically swap the root.
-	c.cacheLock.RLock()
-	cachedKeys := c.cachedKeys
-	cachedRemove := c.cachedRemove
-	c.cacheLock.RUnlock()
-
-	// Nothing to do if there is no data to flush.
-	if cachedKeys.Len() == 0 && cachedRemove.Len() == 0 {
-		return nil
-	}
-
-	// Perform all leveldb updates using an atomic transaction.
-	if err := c.commitTreaps(cachedKeys, cachedRemove); err != nil {
-		return err
-	}
-
-	// Clear the cache since it has been flushed.
-	c.cacheLock.Lock()
-	c.cachedKeys = treap.NewImmutable()
-	c.cachedRemove = treap.NewImmutable()
-	c.cacheLock.Unlock()
-
-	return nil
-}
-
-// needsFlush returns whether or not the database cache needs to be flushed to
-// persistent storage based on its current size, whether or not adding all of
-// the entries in the passed database transaction would cause it to exceed the
-// configured limit, and how much time has elapsed since the last time the cache
-// was flushed.
-//
-// This function MUST be called with the database write lock held.
-func (c *dbCache) needsFlush(tx *transaction) bool {
-	// A flush is needed when more time has elapsed than the configured
-	// flush interval.
-	if time.Since(c.lastFlush) >= c.flushInterval {
-		return true
-	}
-
-	// A flush is needed when the size of the database cache exceeds the
-	// specified max cache size. The total calculated size is multiplied by
-	// 1.5 here to account for additional memory consumption that will be
-	// needed during the flush as well as old nodes in the cache that are
-	// referenced by the snapshot used by the transaction.
-	snap := tx.snapshot
-	totalSize := snap.pendingKeys.Size() + snap.pendingRemove.Size()
-	totalSize = uint64(float64(totalSize) * 1.5)
-	return totalSize > c.maxSize
-}
-
-// commitTx atomically adds all of the pending keys to add and remove into the
-// database cache. When adding the pending keys would cause the size of the
-// cache to exceed the max cache size, or the time since the last flush exceeds
-// the configured flush interval, the cache will be flushed to the underlying
-// persistent database.
-//
-// This is an atomic operation with respect to the cache in that either all of
-// the pending keys to add and remove in the transaction will be applied or none
-// of them will.
-//
-// The database cache itself might be flushed to the underlying persistent
-// database even if the transaction fails to apply, but it will only be the
-// state of the cache without the transaction applied.
-//
-// This function MUST be called during a database write transaction which in
-// turn implies the database write lock will be held.
-func (c *dbCache) commitTx(tx *transaction) error {
-	// Flush the cache and write the current transaction directly to the
-	// database if a flush is needed.
-	if c.needsFlush(tx) {
-		if err := c.flush(); err != nil {
-			return err
-		}
-
-		// Perform all leveldb updates using an atomic transaction.
-		err := c.commitTreaps(tx.pendingKeys, tx.pendingRemove)
-		if err != nil {
-			return err
-		}
-
-		// Clear the transaction entries since they have been committed.
-		tx.pendingKeys = nil
-		tx.pendingRemove = nil
-		return nil
-	}
-
-	// At this point a database flush is not needed, so atomically commit
-	// the transaction to the cache.
-
-	// Since the cached keys to be added and removed use an immutable treap,
-	// a snapshot is simply obtaining the root of the tree under the lock
-	// which is used to atomically swap the root.
-	c.cacheLock.RLock()
-	newCachedKeys := c.cachedKeys
-	newCachedRemove := c.cachedRemove
-	c.cacheLock.RUnlock()
-
-	// Apply every key to add in the database transaction to the cache.
-	tx.pendingKeys.ForEach(func(k, v []byte) bool {
-		newCachedRemove = newCachedRemove.Delete(k)
-		newCachedKeys = newCachedKeys.Put(k, v)
-		return true
-	})
-	tx.pendingKeys = nil
-
-	// Apply every key to remove in the database transaction to the cache.
-	tx.pendingRemove.ForEach(func(k, v []byte) bool {
-		newCachedKeys = newCachedKeys.Delete(k)
-		newCachedRemove = newCachedRemove.Put(k, nil)
-		return true
-	})
-	tx.pendingRemove = nil
-
-	// Atomically replace the immutable treaps which hold the cached keys to
-	// add and delete.
-	c.cacheLock.Lock()
-	c.cachedKeys = newCachedKeys
-	c.cachedRemove = newCachedRemove
-	c.cacheLock.Unlock()
-	return nil
-}
-
-// Close cleanly shuts down the database cache by syncing all data and closing
-// the underlying leveldb database.
-//
-// This function MUST be called with the database write lock held.
-func (c *dbCache) Close() error {
-	// Flush any outstanding cached entries to disk.
-	if err := c.flush(); err != nil {
-		// Even if there is an error while flushing, attempt to close
-		// the underlying database. The error is ignored since it would
-		// mask the flush error.
-		_ = c.ldb.Close()
-		return err
-	}
-
-	// Close the underlying leveldb database.
-	if err := c.ldb.Close(); err != nil {
-		str := "failed to close underlying leveldb database"
-		return convertErr(str, err)
-	}
-
-	return nil
-}
-
-// newDbCache returns a new database cache instance backed by the provided
-// leveldb instance. The cache will be flushed to leveldb when the max size
-// exceeds the provided value or it has been longer than the provided interval
-// since the last flush.
-func newDbCache(ldb *leveldb.DB, store *blockStore, maxSize uint64, flushIntervalSecs uint32) *dbCache {
-	return &dbCache{
-		ldb:           ldb,
-		store:         store,
-		maxSize:       maxSize,
-		flushInterval: time.Second * time.Duration(flushIntervalSecs),
-		lastFlush:     time.Now(),
-		cachedKeys:    treap.NewImmutable(),
-		cachedRemove:  treap.NewImmutable(),
-	}
-}

database/ffldb/dbcache_test.go

@@ -1,136 +0,0 @@
-package ffldb
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/kaspanet/kaspad/database"
-	ldbutil "github.com/syndtr/goleveldb/leveldb/util"
-)
-
-func TestExhaustedDbCacheIterator(t *testing.T) {
-	db := newTestDb("TestExhaustedDbCacheIterator", t)
-	defer db.Close()
-
-	snapshot, err := db.cache.Snapshot()
-	if err != nil {
-		t.Fatalf("TestExhaustedDbCacheIterator: Error creating cache snapshot: %s", err)
-	}
-	iterator := snapshot.NewIterator(&ldbutil.Range{})
-
-	if next := iterator.Next(); next != false {
-		t.Errorf("TestExhaustedDbCacheIterator: Expected .Next() = false, but got %v", next)
-	}
-
-	if prev := iterator.Prev(); prev != false {
-		t.Errorf("TestExhaustedDbCacheIterator: Expected .Prev() = false, but got %v", prev)
-	}
-
-	if key := iterator.Key(); key != nil {
-		t.Errorf("TestExhaustedDbCacheIterator: Expected .Key() = nil, but got %v", key)
-	}
-
-	if value := iterator.Value(); value != nil {
-		t.Errorf("TestExhaustedDbCacheIterator: Expected .Value() = nil, but got %v", value)
-	}
-}
-
-// TestLDBIteratorImplPlaceholders hits functions that are there to implement leveldb iterator.Iterator interface,
-// but surve no other purpose.
-func TestLDBIteratorImplPlaceholders(t *testing.T) {
-	db := newTestDb("TestIteratorImplPlaceholders", t)
-	defer db.Close()
-
-	snapshot, err := db.cache.Snapshot()
-	if err != nil {
-		t.Fatalf("TestLDBIteratorImplPlaceholders: Error creating cache snapshot: %s", err)
-	}
-	iterator := newLdbCacheIter(snapshot, &ldbutil.Range{})
-
-	if err = iterator.Error(); err != nil {
-		t.Errorf("TestLDBIteratorImplPlaceholders: Expected .Error() = nil, but got %v", err)
-	}
-
-	// Call SetReleaser to achieve coverage of it. Actually does nothing
-	iterator.SetReleaser(nil)
-}
-
-func TestSkipPendingUpdatesCache(t *testing.T) {
-	pdb := newTestDb("TestSkipPendingUpdatesCache", t)
-	defer pdb.Close()
-
-	value := []byte("value")
-	// Add numbered prefixes to keys so that they are in expected order, and before any other keys
-	firstKey := []byte("1 - first")
-	toDeleteKey := []byte("2 - toDelete")
-	toUpdateKey := []byte("3 - toUpdate")
-	secondKey := []byte("4 - second")
-
-	// create initial metadata for test
-	err := pdb.Update(func(dbTx database.Tx) error {
-		metadata := dbTx.Metadata()
-		if err := metadata.Put(firstKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(toDeleteKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(toUpdateKey, value); err != nil {
-			return err
-		}
-		if err := metadata.Put(secondKey, value); err != nil {
-			return err
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("Error adding to metadata: %s", err)
-	}
-
-	err = pdb.cache.flush()
-	if err != nil {
-		t.Fatalf("Error flushing cache: %s", err)
-	}
-
-	// test skips
-	err = pdb.Update(func(dbTx database.Tx) error {
-		snapshot, err := pdb.cache.Snapshot()
-		if err != nil {
-			t.Fatalf("TestSkipPendingUpdatesCache: Error getting snapshot: %s", err)
-		}
-
-		iterator := snapshot.NewIterator(&ldbutil.Range{})
-		snapshot.pendingRemove = snapshot.pendingRemove.Put(bucketizedKey(metadataBucketID, toDeleteKey), value)
-		snapshot.pendingKeys = snapshot.pendingKeys.Put(bucketizedKey(metadataBucketID, toUpdateKey), value)
-
-		// Check that first is ok
-		iterator.First()
-		expectedKey := bucketizedKey(metadataBucketID, firstKey)
-		actualKey := iterator.Key()
-		if !bytes.Equal(actualKey, expectedKey) {
-			t.Errorf("TestSkipPendingUpdatesCache: 1: key expected to be %v but is %v", expectedKey, actualKey)
-		}
-
-		// Go to the next key, which is second, toDelete and toUpdate will be skipped
-		iterator.Next()
-		expectedKey = bucketizedKey(metadataBucketID, secondKey)
-		actualKey = iterator.Key()
-		if !bytes.Equal(actualKey, expectedKey) {
-			t.Errorf("TestSkipPendingUpdatesCache: 2: key expected to be %s but is %s", expectedKey, actualKey)
-		}
-
-		// now traverse backwards - should get first, toUpdate and toDelete will be skipped
-		iterator.Prev()
-		expectedKey = bucketizedKey(metadataBucketID, firstKey)
-		actualKey = iterator.Key()
-		if !bytes.Equal(actualKey, expectedKey) {
-			t.Errorf("TestSkipPendingUpdatesCache: 4: key expected to be %s but is %s", expectedKey, actualKey)
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("TestSkipPendingUpdatesCache: Error running main part of test: %s", err)
-	}
-}

database/ffldb/doc.go

@@ -1,25 +0,0 @@
-/*
-Package ffldb implements a driver for the database package that uses leveldb
-for the backing metadata and flat files for block storage.
-
-This driver is the recommended driver for use with kaspad. It makes use leveldb
-for the metadata, flat files for block storage, and checksums in key areas to
-ensure data integrity.
-
-Usage
-
-This package is a driver to the database package and provides the database type
-of "ffldb". The parameters the Open and Create functions take are the
-database path as a string and the block network:
-
-	db, err := database.Open("ffldb", "path/to/database", wire.Mainnet)
-	if err != nil {
-		// Handle error
-	}
-
-	db, err := database.Create("ffldb", "path/to/database", wire.Mainnet)
-	if err != nil {
-		// Handle error
-	}
-*/
-package ffldb

database/ffldb/driver.go

@@ -1,60 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb
-
-import (
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/pkg/errors"
-)
-
-const (
-	dbType = "ffldb"
-)
-
-// parseArgs parses the arguments from the database Open/Create methods.
-func parseArgs(funcName string, args ...interface{}) (string, wire.KaspaNet, error) {
-	if len(args) != 2 {
-		return "", 0, errors.Errorf("invalid arguments to %s.%s -- "+
-			"expected database path and block network", dbType,
-			funcName)
-	}
-
-	dbPath, ok := args[0].(string)
-	if !ok {
-		return "", 0, errors.Errorf("first argument to %s.%s is invalid -- "+
-			"expected database path string", dbType, funcName)
-	}
-
-	network, ok := args[1].(wire.KaspaNet)
-	if !ok {
-		return "", 0, errors.Errorf("second argument to %s.%s is invalid -- "+
-			"expected block network", dbType, funcName)
-	}
-
-	return dbPath, network, nil
-}
-
-// openDBDriver is the callback provided during driver registration that opens
-// an existing database for use.
-func openDBDriver(args ...interface{}) (database.DB, error) {
-	dbPath, network, err := parseArgs("Open", args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return openDB(dbPath, network, false)
-}
-
-// createDBDriver is the callback provided during driver registration that
-// creates, initializes, and opens a database for use.
-func createDBDriver(args ...interface{}) (database.DB, error) {
-	dbPath, network, err := parseArgs("Create", args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return openDB(dbPath, network, true)
-}

database/ffldb/driver_test.go

@@ -1,290 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb_test
-
-import (
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-	"reflect"
-	"runtime"
-	"testing"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
-)
-
-// dbType is the database type name for this driver.
-const dbType = "ffldb"
-
-// TestCreateOpenFail ensures that errors related to creating and opening a
-// database are handled properly.
-func TestCreateOpenFail(t *testing.T) {
-	t.Parallel()
-
-	// Ensure that attempting to open a database that doesn't exist returns
-	// the expected error.
-	wantErrCode := database.ErrDbDoesNotExist
-	_, err := database.Open(dbType, "noexist", blockDataNet)
-	if !checkDbError(t, "Open", err, wantErrCode) {
-		return
-	}
-
-	// Ensure that attempting to open a database with the wrong number of
-	// parameters returns the expected error.
-	wantErr := errors.Errorf("invalid arguments to %s.Open -- expected "+
-		"database path and block network", dbType)
-	_, err = database.Open(dbType, 1, 2, 3)
-	if err.Error() != wantErr.Error() {
-		t.Errorf("Open: did not receive expected error - got %v, "+
-			"want %v", err, wantErr)
-		return
-	}
-
-	// Ensure that attempting to open a database with an invalid type for
-	// the first parameter returns the expected error.
-	wantErr = errors.Errorf("first argument to %s.Open is invalid -- "+
-		"expected database path string", dbType)
-	_, err = database.Open(dbType, 1, blockDataNet)
-	if err.Error() != wantErr.Error() {
-		t.Errorf("Open: did not receive expected error - got %v, "+
-			"want %v", err, wantErr)
-		return
-	}
-
-	// Ensure that attempting to open a database with an invalid type for
-	// the second parameter returns the expected error.
-	wantErr = errors.Errorf("second argument to %s.Open is invalid -- "+
-		"expected block network", dbType)
-	_, err = database.Open(dbType, "noexist", "invalid")
-	if err.Error() != wantErr.Error() {
-		t.Errorf("Open: did not receive expected error - got %v, "+
-			"want %v", err, wantErr)
-		return
-	}
-
-	// Ensure that attempting to create a database with the wrong number of
-	// parameters returns the expected error.
-	wantErr = errors.Errorf("invalid arguments to %s.Create -- expected "+
-		"database path and block network", dbType)
-	_, err = database.Create(dbType, 1, 2, 3)
-	if err.Error() != wantErr.Error() {
-		t.Errorf("Create: did not receive expected error - got %v, "+
-			"want %v", err, wantErr)
-		return
-	}
-
-	// Ensure that attempting to create a database with an invalid type for
-	// the first parameter returns the expected error.
-	wantErr = errors.Errorf("first argument to %s.Create is invalid -- "+
-		"expected database path string", dbType)
-	_, err = database.Create(dbType, 1, blockDataNet)
-	if err.Error() != wantErr.Error() {
-		t.Errorf("Create: did not receive expected error - got %v, "+
-			"want %v", err, wantErr)
-		return
-	}
-
-	// Ensure that attempting to create a database with an invalid type for
-	// the second parameter returns the expected error.
-	wantErr = errors.Errorf("second argument to %s.Create is invalid -- "+
-		"expected block network", dbType)
-	_, err = database.Create(dbType, "noexist", "invalid")
-	if err.Error() != wantErr.Error() {
-		t.Errorf("Create: did not receive expected error - got %v, "+
-			"want %v", err, wantErr)
-		return
-	}
-
-	// Ensure operations against a closed database return the expected
-	// error.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-createfail")
-	_ = os.RemoveAll(dbPath)
-	db, err := database.Create(dbType, dbPath, blockDataNet)
-	if err != nil {
-		t.Errorf("Create: unexpected error: %v", err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	db.Close()
-
-	wantErrCode = database.ErrDbNotOpen
-	err = db.View(func(dbTx database.Tx) error {
-		return nil
-	})
-	if !checkDbError(t, "View", err, wantErrCode) {
-		return
-	}
-
-	wantErrCode = database.ErrDbNotOpen
-	err = db.Update(func(dbTx database.Tx) error {
-		return nil
-	})
-	if !checkDbError(t, "Update", err, wantErrCode) {
-		return
-	}
-
-	wantErrCode = database.ErrDbNotOpen
-	_, err = db.Begin(false)
-	if !checkDbError(t, "Begin(false)", err, wantErrCode) {
-		return
-	}
-
-	wantErrCode = database.ErrDbNotOpen
-	_, err = db.Begin(true)
-	if !checkDbError(t, "Begin(true)", err, wantErrCode) {
-		return
-	}
-
-	wantErrCode = database.ErrDbNotOpen
-	err = db.Close()
-	if !checkDbError(t, "Close", err, wantErrCode) {
-		return
-	}
-}
-
-// TestPersistence ensures that values stored are still valid after closing and
-// reopening the database.
-func TestPersistence(t *testing.T) {
-	t.Parallel()
-
-	// Create a new database to run tests against.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-persistencetest")
-	_ = os.RemoveAll(dbPath)
-	db, err := database.Create(dbType, dbPath, blockDataNet)
-	if err != nil {
-		t.Errorf("Failed to create test database (%s) %v", dbType, err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Create a bucket, put some values into it, and store a block so they
-	// can be tested for existence on re-open.
-	bucket1Key := []byte("bucket1")
-	storeValues := map[string]string{
-		"b1key1": "foo1",
-		"b1key2": "foo2",
-		"b1key3": "foo3",
-	}
-	genesisBlock := util.NewBlock(dagconfig.MainnetParams.GenesisBlock)
-	genesisHash := dagconfig.MainnetParams.GenesisHash
-	err = db.Update(func(dbTx database.Tx) error {
-		metadataBucket := dbTx.Metadata()
-		if metadataBucket == nil {
-			return errors.Errorf("Metadata: unexpected nil bucket")
-		}
-
-		bucket1, err := metadataBucket.CreateBucket(bucket1Key)
-		if err != nil {
-			return errors.Errorf("CreateBucket: unexpected error: %v",
-				err)
-		}
-
-		for k, v := range storeValues {
-			err := bucket1.Put([]byte(k), []byte(v))
-			if err != nil {
-				return errors.Errorf("Put: unexpected error: %v",
-					err)
-			}
-		}
-
-		if err := dbTx.StoreBlock(genesisBlock); err != nil {
-			return errors.Errorf("StoreBlock: unexpected error: %v",
-				err)
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Errorf("Update: unexpected error: %v", err)
-		return
-	}
-
-	// Close and reopen the database to ensure the values persist.
-	db.Close()
-	db, err = database.Open(dbType, dbPath, blockDataNet)
-	if err != nil {
-		t.Errorf("Failed to open test database (%s) %v", dbType, err)
-		return
-	}
-	defer db.Close()
-
-	// Ensure the values previously stored in the 3rd namespace still exist
-	// and are correct.
-	err = db.View(func(dbTx database.Tx) error {
-		metadataBucket := dbTx.Metadata()
-		if metadataBucket == nil {
-			return errors.Errorf("Metadata: unexpected nil bucket")
-		}
-
-		bucket1 := metadataBucket.Bucket(bucket1Key)
-		if bucket1 == nil {
-			return errors.Errorf("Bucket1: unexpected nil bucket")
-		}
-
-		for k, v := range storeValues {
-			gotVal := bucket1.Get([]byte(k))
-			if !reflect.DeepEqual(gotVal, []byte(v)) {
-				return errors.Errorf("Get: key '%s' does not "+
-					"match expected value - got %s, want %s",
-					k, gotVal, v)
-			}
-		}
-
-		genesisBlockBytes, _ := genesisBlock.Bytes()
-		gotBytes, err := dbTx.FetchBlock(genesisHash)
-		if err != nil {
-			return errors.Errorf("FetchBlock: unexpected error: %v",
-				err)
-		}
-		if !reflect.DeepEqual(gotBytes, genesisBlockBytes) {
-			return errors.Errorf("FetchBlock: stored block mismatch")
-		}
-
-		return nil
-	})
-	if err != nil {
-		t.Errorf("View: unexpected error: %v", err)
-		return
-	}
-}
-
-// TestInterface performs all interfaces tests for this database driver.
-func TestInterface(t *testing.T) {
-	t.Parallel()
-
-	// Create a new database to run tests against.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-interfacetest")
-	_ = os.RemoveAll(dbPath)
-	db, err := database.Create(dbType, dbPath, blockDataNet)
-	if err != nil {
-		t.Errorf("Failed to create test database (%s) %v", dbType, err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Ensure the driver type is the expected value.
-	gotDbType := db.Type()
-	if gotDbType != dbType {
-		t.Errorf("Type: unepxected driver type - got %v, want %v",
-			gotDbType, dbType)
-		return
-	}
-
-	// Run all of the interface tests against the database.
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	// Change the maximum file size to a small value to force multiple flat
-	// files with the test data set.
-	// Change maximum open files to small value to force shifts in the LRU
-	// mechanism
-	ffldb.TstRunWithMaxBlockFileSizeAndMaxOpenFiles(db, 2048, 10, func() {
-		testInterface(t, db)
-	})
-}

database/ffldb/export_test.go

@@ -1,29 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-This test file is part of the ffldb package rather than than the ffldb_test
-package so it can bridge access to the internals to properly test cases which
-are either not possible or can't reliably be tested via the public interface.
-The functions are only exported while the tests are being run.
-*/
-
-package ffldb
-
-import "github.com/kaspanet/kaspad/database"
-
-// TstRunWithMaxBlockFileSize runs the passed function with the maximum allowed
-// file size for the database set to the provided value. The value will be set
-// back to the original value upon completion.
-func TstRunWithMaxBlockFileSizeAndMaxOpenFiles(idb database.DB, size uint32, maxOpenFiles int, fn func()) {
-	ffldb := idb.(*db)
-	origSize := ffldb.store.maxBlockFileSize
-	origMaxOpenFiles := ffldb.store.maxOpenFiles
-
-	ffldb.store.maxBlockFileSize = size
-	ffldb.store.maxOpenFiles = maxOpenFiles
-	fn()
-	ffldb.store.maxBlockFileSize = origSize
-	ffldb.store.maxOpenFiles = origMaxOpenFiles
-}

database/ffldb/init.go

@@ -1,23 +0,0 @@
-package ffldb
-
-import (
-	"fmt"
-
-	"github.com/kaspanet/kaspad/database"
-)
-
-func registerDriver() {
-	driver := database.Driver{
-		DbType: dbType,
-		Create: createDBDriver,
-		Open:   openDBDriver,
-	}
-	if err := database.RegisterDriver(driver); err != nil {
-		panic(fmt.Sprintf("Failed to regiser database driver '%s': %s",
-			dbType, err))
-	}
-}
-
-func init() {
-	registerDriver()
-}

database/ffldb/ldbtreapiter.go

@@ -1,58 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb
-
-import (
-	"github.com/kaspanet/kaspad/database/internal/treap"
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-// ldbTreapIter wraps a treap iterator to provide the additional functionality
-// needed to satisfy the leveldb iterator.Iterator interface.
-type ldbTreapIter struct {
-	*treap.Iterator
-	tx       *transaction
-	released bool
-}
-
-// Enforce ldbTreapIter implements the leveldb iterator.Iterator interface.
-var _ iterator.Iterator = (*ldbTreapIter)(nil)
-
-// Error is only provided to satisfy the iterator interface as there are no
-// errors for this memory-only structure.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *ldbTreapIter) Error() error {
-	return nil
-}
-
-// SetReleaser is only provided to satisfy the iterator interface as there is no
-// need to override it.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *ldbTreapIter) SetReleaser(releaser util.Releaser) {
-}
-
-// Release releases the iterator by removing the underlying treap iterator from
-// the list of active iterators against the pending keys treap.
-//
-// This is part of the leveldb iterator.Iterator interface implementation.
-func (iter *ldbTreapIter) Release() {
-	if !iter.released {
-		iter.tx.removeActiveIter(iter.Iterator)
-		iter.released = true
-	}
-}
-
-// newLdbTreapIter creates a new treap iterator for the given slice against the
-// pending keys for the passed transaction and returns it wrapped in an
-// ldbTreapIter so it can be used as a leveldb iterator. It also adds the new
-// iterator to the list of active iterators for the transaction.
-func newLdbTreapIter(tx *transaction, slice *util.Range) *ldbTreapIter {
-	iter := tx.pendingKeys.Iterator(slice.Start, slice.Limit)
-	tx.addActiveIter(iter)
-	return &ldbTreapIter{Iterator: iter, tx: tx}
-}

database/ffldb/log.go

@@ -1,11 +0,0 @@
-// Copyright (c) 2013-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb
-
-import (
-	"github.com/kaspanet/kaspad/logger"
-)
-
-var log, _ = logger.Get(logger.SubsystemTags.KSDB)

database/ffldb/mockfile_test.go

@@ -1,163 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-// This file is part of the ffldb package rather than the ffldb_test package as
-// it is part of the whitebox testing.
-
-package ffldb
-
-import (
-	"github.com/pkg/errors"
-	"io"
-	"sync"
-)
-
-// Errors used for the mock file.
-var (
-	// errMockFileClosed is used to indicate a mock file is closed.
-	errMockFileClosed = errors.New("file closed")
-
-	// errInvalidOffset is used to indicate an offset that is out of range
-	// for the file was provided.
-	errInvalidOffset = errors.New("invalid offset")
-
-	// errSyncFail is used to indicate simulated sync failure.
-	errSyncFail = errors.New("simulated sync failure")
-)
-
-// mockFile implements the filer interface and used in order to force failures
-// the database code related to reading and writing from the flat block files.
-// A maxSize of -1 is unlimited.
-type mockFile struct {
-	sync.RWMutex
-	maxSize      int64
-	data         []byte
-	forceSyncErr bool
-	closed       bool
-}
-
-// Close closes the mock file without releasing any data associated with it.
-// This allows it to be "reopened" without losing the data.
-//
-// This is part of the filer implementation.
-func (f *mockFile) Close() error {
-	f.Lock()
-	defer f.Unlock()
-
-	if f.closed {
-		return errMockFileClosed
-	}
-	f.closed = true
-	return nil
-}
-
-// ReadAt reads len(b) bytes from the mock file starting at byte offset off. It
-// returns the number of bytes read and the error, if any. ReadAt always
-// returns a non-nil error when n < len(b). At end of file, that error is
-// io.EOF.
-//
-// This is part of the filer implementation.
-func (f *mockFile) ReadAt(b []byte, off int64) (int, error) {
-	f.RLock()
-	defer f.RUnlock()
-
-	if f.closed {
-		return 0, errMockFileClosed
-	}
-	maxSize := int64(len(f.data))
-	if f.maxSize > -1 && maxSize > f.maxSize {
-		maxSize = f.maxSize
-	}
-	if off < 0 || off > maxSize {
-		return 0, errInvalidOffset
-	}
-
-	// Limit to the max size field, if set.
-	numToRead := int64(len(b))
-	endOffset := off + numToRead
-	if endOffset > maxSize {
-		numToRead = maxSize - off
-	}
-
-	copy(b, f.data[off:off+numToRead])
-	if numToRead < int64(len(b)) {
-		return int(numToRead), io.EOF
-	}
-	return int(numToRead), nil
-}
-
-// Truncate changes the size of the mock file.
-//
-// This is part of the filer implementation.
-func (f *mockFile) Truncate(size int64) error {
-	f.Lock()
-	defer f.Unlock()
-
-	if f.closed {
-		return errMockFileClosed
-	}
-	maxSize := int64(len(f.data))
-	if f.maxSize > -1 && maxSize > f.maxSize {
-		maxSize = f.maxSize
-	}
-	if size > maxSize {
-		return errInvalidOffset
-	}
-
-	f.data = f.data[:size]
-	return nil
-}
-
-// Write writes len(b) bytes to the mock file. It returns the number of bytes
-// written and an error, if any. Write returns a non-nil error any time
-// n != len(b).
-//
-// This is part of the filer implementation.
-func (f *mockFile) WriteAt(b []byte, off int64) (int, error) {
-	f.Lock()
-	defer f.Unlock()
-
-	if f.closed {
-		return 0, errMockFileClosed
-	}
-	maxSize := f.maxSize
-	if maxSize < 0 {
-		maxSize = 100 * 1024 // 100KiB
-	}
-	if off < 0 || off > maxSize {
-		return 0, errInvalidOffset
-	}
-
-	// Limit to the max size field, if set, and grow the slice if needed.
-	numToWrite := int64(len(b))
-	if off+numToWrite > maxSize {
-		numToWrite = maxSize - off
-	}
-	if off+numToWrite > int64(len(f.data)) {
-		newData := make([]byte, off+numToWrite)
-		copy(newData, f.data)
-		f.data = newData
-	}
-
-	copy(f.data[off:], b[:numToWrite])
-	if numToWrite < int64(len(b)) {
-		return int(numToWrite), io.EOF
-	}
-	return int(numToWrite), nil
-}
-
-// Sync doesn't do anything for mock files. However, it will return an error if
-// the mock file's forceSyncErr flag is set.
-//
-// This is part of the filer implementation.
-func (f *mockFile) Sync() error {
-	if f.forceSyncErr {
-		return errSyncFail
-	}
-
-	return nil
-}
-
-// Ensure the mockFile type implements the filer interface.
-var _ filer = (*mockFile)(nil)

database/ffldb/reconcile.go

@@ -1,117 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package ffldb
-
-import (
-	"fmt"
-	"hash/crc32"
-
-	"github.com/kaspanet/kaspad/database"
-)
-
-// The serialized write cursor location format is:
-//
-//  [0:4]  Block file (4 bytes)
-//  [4:8]  File offset (4 bytes)
-//  [8:12] Castagnoli CRC-32 checksum (4 bytes)
-
-// serializeWriteRow serialize the current block file and offset where new
-// will be written into a format suitable for storage into the metadata.
-func serializeWriteRow(curBlockFileNum, curFileOffset uint32) []byte {
-	var serializedRow [12]byte
-	byteOrder.PutUint32(serializedRow[0:4], curBlockFileNum)
-	byteOrder.PutUint32(serializedRow[4:8], curFileOffset)
-	checksum := crc32.Checksum(serializedRow[:8], castagnoli)
-	byteOrder.PutUint32(serializedRow[8:12], checksum)
-	return serializedRow[:]
-}
-
-// deserializeWriteRow deserializes the write cursor location stored in the
-// metadata. Returns ErrCorruption if the checksum of the entry doesn't match.
-func deserializeWriteRow(writeRow []byte) (uint32, uint32, error) {
-	// Ensure the checksum matches. The checksum is at the end.
-	gotChecksum := crc32.Checksum(writeRow[:8], castagnoli)
-	wantChecksumBytes := writeRow[8:12]
-	wantChecksum := byteOrder.Uint32(wantChecksumBytes)
-	if gotChecksum != wantChecksum {
-		str := fmt.Sprintf("metadata for write cursor does not match "+
-			"the expected checksum - got %d, want %d", gotChecksum,
-			wantChecksum)
-		return 0, 0, makeDbErr(database.ErrCorruption, str, nil)
-	}
-
-	fileNum := byteOrder.Uint32(writeRow[0:4])
-	fileOffset := byteOrder.Uint32(writeRow[4:8])
-	return fileNum, fileOffset, nil
-}
-
-// reconcileDB reconciles the metadata with the flat block files on disk. It
-// will also initialize the underlying database if the create flag is set.
-func reconcileDB(pdb *db, create bool) (database.DB, error) {
-	// Perform initial internal bucket and value creation during database
-	// creation.
-	if create {
-		if err := initDB(pdb.cache.ldb); err != nil {
-			return nil, err
-		}
-	}
-
-	// Load the current write cursor position from the metadata.
-	var curFileNum, curOffset uint32
-	err := pdb.View(func(dbTx database.Tx) error {
-		writeRow := dbTx.Metadata().Get(writeLocKeyName)
-		if writeRow == nil {
-			str := "write cursor does not exist"
-			return makeDbErr(database.ErrCorruption, str, nil)
-		}
-
-		var err error
-		curFileNum, curOffset, err = deserializeWriteRow(writeRow)
-		return err
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	// When the write cursor position found by scanning the block files on
-	// disk is AFTER the position the metadata believes to be true, truncate
-	// the files on disk to match the metadata. This can be a fairly common
-	// occurrence in unclean shutdown scenarios while the block files are in
-	// the middle of being written. Since the metadata isn't updated until
-	// after the block data is written, this is effectively just a rollback
-	// to the known good point before the unclean shutdown.
-	wc := pdb.store.writeCursor
-	if wc.curFileNum > curFileNum || (wc.curFileNum == curFileNum &&
-		wc.curOffset > curOffset) {
-
-		log.Info("Detected unclean shutdown - Repairing...")
-		log.Debugf("Metadata claims file %d, offset %d. Block data is "+
-			"at file %d, offset %d", curFileNum, curOffset,
-			wc.curFileNum, wc.curOffset)
-		pdb.store.handleRollback(curFileNum, curOffset)
-		log.Infof("Database sync complete")
-	}
-
-	// When the write cursor position found by scanning the block files on
-	// disk is BEFORE the position the metadata believes to be true, return
-	// a corruption error. Since sync is called after each block is written
-	// and before the metadata is updated, this should only happen in the
-	// case of missing, deleted, or truncated block files, which generally
-	// is not an easily recoverable scenario. In the future, it might be
-	// possible to rescan and rebuild the metadata from the block files,
-	// however, that would need to happen with coordination from a higher
-	// layer since it could invalidate other metadata.
-	if wc.curFileNum < curFileNum || (wc.curFileNum == curFileNum &&
-		wc.curOffset < curOffset) {
-
-		str := fmt.Sprintf("metadata claims file %d, offset %d, but "+
-			"block data is at file %d, offset %d", curFileNum,
-			curOffset, wc.curFileNum, wc.curOffset)
-		log.Warnf("***Database corruption detected***: %s", str)
-		return nil, makeDbErr(database.ErrCorruption, str, nil)
-	}
-
-	return pdb, nil
-}

database/ffldb/reconcile_test.go

@@ -1,144 +0,0 @@
-package ffldb
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestSerializeWriteRow(t *testing.T) {
-	tests := []struct {
-		curBlockFileNum  uint32
-		curFileOffset    uint32
-		expectedWriteRow []byte
-	}{
-		// WriteRow format:
-		// 		First 4 bytes: curBlockFileNum
-		//		Next  4 bytes: curFileOffset
-		//		Next  4 bytes: Castagnoli CRC-32 checksum
-		// One can easily calculate checksums using the following code:
-		// https://play.golang.org/p/zoMKT-ORyF9
-		{0, 0, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB2, 0x28, 0x8C}},
-		{10, 11, []byte{0x0A, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0xC1, 0xA6, 0x0D, 0xC8}},
-	}
-
-	for i, test := range tests {
-		actualWriteRow := serializeWriteRow(test.curBlockFileNum, test.curFileOffset)
-
-		if !reflect.DeepEqual(test.expectedWriteRow, actualWriteRow) {
-			t.Errorf("TestSerializeWriteRow: %d: Expected: %v, but got: %v",
-				i, test.expectedWriteRow, actualWriteRow)
-		}
-	}
-}
-
-func TestDeserializeWriteRow(t *testing.T) {
-	tests := []struct {
-		writeRow                []byte
-		expectedCurBlockFileNum uint32
-		expectedCurFileOffset   uint32
-		expectedError           bool
-	}{
-		// WriteRow format:
-		// 		First 4 bytes: curBlockFileNum
-		//		Next  4 bytes: curFileOffset
-		//		Next  4 bytes: Castagnoli CRC-32 checksum
-		// One can easily calculate checksums using the following code:
-		// https://play.golang.org/p/zoMKT-ORyF9
-		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB2, 0x28, 0x8C}, 0, 0, false},
-		{[]byte{0x0A, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0xC1, 0xA6, 0x0D, 0xC8}, 10, 11, false},
-		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB2, 0x28, 0x8D}, 0, 0, true},
-		{[]byte{0x0A, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, 0, 0, true},
-	}
-
-	for i, test := range tests {
-		actualCurBlockFileNum, actualCurFileOffset, err := deserializeWriteRow(test.writeRow)
-
-		if (err != nil) != test.expectedError {
-			t.Errorf("TestDeserializeWriteRow: %d: Expected error status: %t, but got: %t",
-				i, test.expectedError, err != nil)
-		}
-
-		if test.expectedCurBlockFileNum != actualCurBlockFileNum {
-			t.Errorf("TestDeserializeWriteRow: %d: Expected curBlockFileNum: %d, but got: %d",
-				i, test.expectedCurBlockFileNum, actualCurBlockFileNum)
-		}
-
-		if test.expectedCurFileOffset != actualCurFileOffset {
-			t.Errorf("TestDeserializeWriteRow: %d: Expected curFileOffset: %d, but got: %d",
-				i, test.expectedCurFileOffset, actualCurFileOffset)
-		}
-	}
-}
-
-// setWriteRow is a low-level helper method to update the write row in the
-// metadata bucket to enable certain test-cases in TestReconcileErrors
-// if writeRow = nil deletes the write row altogether
-func setWriteRow(pdb *db, writeRow []byte, t *testing.T) {
-	tx, err := pdb.begin(true)
-	if err != nil {
-		t.Fatalf("TestReconcileErrors: Error getting tx for setting "+
-			"writeLoc in metadata: %s", err)
-	}
-
-	if writeRow == nil {
-		tx.metaBucket.Delete(writeLocKeyName)
-		if err != nil {
-			t.Fatalf("TestReconcileErrors: Error deleting writeLoc from metadata: %s",
-				err)
-		}
-	} else {
-		tx.metaBucket.Put(writeLocKeyName, writeRow)
-		if err != nil {
-			t.Fatalf("TestReconcileErrors: Error updating writeLoc in metadata: %s",
-				err)
-		}
-	}
-
-	err = pdb.cache.commitTx(tx)
-	if err != nil {
-		t.Fatalf("TestReconcileErrors: Error commiting the update of "+
-			"writeLoc in metadata: %s", err)
-	}
-
-	pdb.writeLock.Unlock()
-	pdb.closeLock.RUnlock()
-}
-
-// TestReconcileErrors tests all error-cases in reconclieDB.
-// The non-error-cases are tested in the more general tests.
-func TestReconcileErrors(t *testing.T) {
-	// Set-up tests
-	pdb := newTestDb("TestReconcileErrors", t)
-
-	// Test without writeLoc
-	setWriteRow(pdb, nil, t)
-	_, err := reconcileDB(pdb, false)
-	if err == nil {
-		t.Errorf("TestReconcileErrors: ReconcileDB() didn't error out when " +
-			"running without a writeRowLoc")
-	}
-
-	// Test with writeLoc in metadata after the actual cursor position
-	setWriteRow(pdb, serializeWriteRow(1, 0), t)
-	_, err = reconcileDB(pdb, false)
-	if err == nil {
-		t.Errorf("TestReconcileErrors: ReconcileDB() didn't error out when " +
-			"curBlockFileNum after the actual cursor position")
-	}
-	setWriteRow(pdb, serializeWriteRow(0, 1), t)
-	_, err = reconcileDB(pdb, false)
-	if err == nil {
-		t.Errorf("TestReconcileErrors: ReconcileDB() didn't error out when " +
-			"curFileOffset after the actual cursor position")
-	}
-
-	// Restore previous writeRow
-	setWriteRow(pdb, serializeWriteRow(0, 0), t)
-
-	// Test create with closed DB to force initDB to fail
-	pdb.Close()
-	_, err = reconcileDB(pdb, true)
-	if err == nil {
-		t.Errorf("ReconcileDB didn't error out when running with closed db and create = true")
-	}
-}

database/ffldb/whitebox_test.go

@@ -1,707 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-// This file is part of the ffldb package rather than the ffldb_test package as
-// it provides whitebox testing.
-
-package ffldb
-
-import (
-	"compress/bzip2"
-	"encoding/binary"
-	"fmt"
-	"github.com/pkg/errors"
-	"hash/crc32"
-	"io"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/syndtr/goleveldb/leveldb"
-	ldberrors "github.com/syndtr/goleveldb/leveldb/errors"
-)
-
-var (
-	// blockDataNet is the expected network in the test block data.
-	blockDataNet = wire.Mainnet
-
-	// blockDataFile is the path to a file containing the first 256 blocks
-	// of the block DAG.
-	blockDataFile = filepath.Join("..", "testdata", "blocks1-256.bz2")
-
-	// errSubTestFail is used to signal that a sub test returned false.
-	errSubTestFail = errors.Errorf("sub test failure")
-)
-
-// loadBlocks loads the blocks contained in the testdata directory and returns
-// a slice of them.
-func loadBlocks(t *testing.T, dataFile string, network wire.KaspaNet) ([]*util.Block, error) {
-	// Open the file that contains the blocks for reading.
-	fi, err := os.Open(dataFile)
-	if err != nil {
-		t.Errorf("failed to open file %v, err %v", dataFile, err)
-		return nil, err
-	}
-	defer func() {
-		if err := fi.Close(); err != nil {
-			t.Errorf("failed to close file %v %v", dataFile,
-				err)
-		}
-	}()
-	dr := bzip2.NewReader(fi)
-
-	// Set the first block as the genesis block.
-	blocks := make([]*util.Block, 0, 256)
-	genesis := util.NewBlock(dagconfig.MainnetParams.GenesisBlock)
-	blocks = append(blocks, genesis)
-
-	// Load the remaining blocks.
-	for height := 1; ; height++ {
-		var net uint32
-		err := binary.Read(dr, binary.LittleEndian, &net)
-		if err == io.EOF {
-			// Hit end of file at the expected offset. No error.
-			break
-		}
-		if err != nil {
-			t.Errorf("Failed to load network type for block %d: %v",
-				height, err)
-			return nil, err
-		}
-		if net != uint32(network) {
-			t.Errorf("Block doesn't match network: %v expects %v",
-				net, network)
-			return nil, err
-		}
-
-		var blockLen uint32
-		err = binary.Read(dr, binary.LittleEndian, &blockLen)
-		if err != nil {
-			t.Errorf("Failed to load block size for block %d: %v",
-				height, err)
-			return nil, err
-		}
-
-		// Read the block.
-		blockBytes := make([]byte, blockLen)
-		_, err = io.ReadFull(dr, blockBytes)
-		if err != nil {
-			t.Errorf("Failed to load block %d: %v", height, err)
-			return nil, err
-		}
-
-		// Deserialize and store the block.
-		block, err := util.NewBlockFromBytes(blockBytes)
-		if err != nil {
-			t.Errorf("Failed to parse block %v: %v", height, err)
-			return nil, err
-		}
-		blocks = append(blocks, block)
-	}
-
-	return blocks, nil
-}
-
-// checkDbError ensures the passed error is a database.Error with an error code
-// that matches the passed  error code.
-func checkDbError(t *testing.T, testName string, gotErr error, wantErrCode database.ErrorCode) bool {
-	dbErr, ok := gotErr.(database.Error)
-	if !ok {
-		t.Errorf("%s: unexpected error type - got %T, want %T",
-			testName, gotErr, database.Error{})
-		return false
-	}
-	if dbErr.ErrorCode != wantErrCode {
-		t.Errorf("%s: unexpected error code - got %s (%s), want %s",
-			testName, dbErr.ErrorCode, dbErr.Description,
-			wantErrCode)
-		return false
-	}
-
-	return true
-}
-
-// testContext is used to store context information about a running test which
-// is passed into helper functions.
-type testContext struct {
-	t            *testing.T
-	db           database.DB
-	files        map[uint32]*lockableFile
-	maxFileSizes map[uint32]int64
-	blocks       []*util.Block
-}
-
-// TestConvertErr ensures the leveldb error to database error conversion works
-// as expected.
-func TestConvertErr(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		err         error
-		wantErrCode database.ErrorCode
-	}{
-		{&ldberrors.ErrCorrupted{}, database.ErrCorruption},
-		{leveldb.ErrClosed, database.ErrDbNotOpen},
-		{leveldb.ErrSnapshotReleased, database.ErrTxClosed},
-		{leveldb.ErrIterReleased, database.ErrTxClosed},
-	}
-
-	for i, test := range tests {
-		gotErr := convertErr("test", test.err)
-		if gotErr.ErrorCode != test.wantErrCode {
-			t.Errorf("convertErr #%d unexpected error - got %v, "+
-				"want %v", i, gotErr.ErrorCode, test.wantErrCode)
-			continue
-		}
-	}
-}
-
-// TestCornerCases ensures several corner cases which can happen when opening
-// a database and/or block files work as expected.
-func TestCornerCases(t *testing.T) {
-	t.Parallel()
-
-	// Create a file at the datapase path to force the open below to fail.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-errors")
-	_ = os.RemoveAll(dbPath)
-	fi, err := os.Create(dbPath)
-	if err != nil {
-		t.Errorf("os.Create: unexpected error: %v", err)
-		return
-	}
-	fi.Close()
-
-	// Ensure creating a new database fails when a file exists where a
-	// directory is needed.
-	testName := "openDB: fail due to file at target location"
-	wantErrCode := database.ErrDriverSpecific
-	idb, err := openDB(dbPath, blockDataNet, true)
-	if !checkDbError(t, testName, err, wantErrCode) {
-		if err == nil {
-			idb.Close()
-		}
-		_ = os.RemoveAll(dbPath)
-		return
-	}
-
-	// Remove the file and create the database to run tests against. It
-	// should be successful this time.
-	_ = os.RemoveAll(dbPath)
-	idb, err = openDB(dbPath, blockDataNet, true)
-	if err != nil {
-		t.Errorf("openDB: unexpected error: %v", err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer idb.Close()
-
-	// Ensure attempting to write to a file that can't be created returns
-	// the expected error.
-	testName = "writeBlock: open file failure"
-	filePath := blockFilePath(dbPath, 0)
-	if err := os.Mkdir(filePath, 0755); err != nil {
-		t.Errorf("os.Mkdir: unexpected error: %v", err)
-		return
-	}
-	store := idb.(*db).store
-	_, err = store.writeBlock([]byte{0x00})
-	if !checkDbError(t, testName, err, database.ErrDriverSpecific) {
-		return
-	}
-	_ = os.RemoveAll(filePath)
-
-	// Close the underlying leveldb database out from under the database.
-	ldb := idb.(*db).cache.ldb
-	ldb.Close()
-
-	// Ensure initilization errors in the underlying database work as
-	// expected.
-	testName = "initDB: reinitialization"
-	wantErrCode = database.ErrDbNotOpen
-	err = initDB(ldb)
-	if !checkDbError(t, testName, err, wantErrCode) {
-		return
-	}
-
-	// Ensure the View handles errors in the underlying leveldb database
-	// properly.
-	testName = "View: underlying leveldb error"
-	wantErrCode = database.ErrDbNotOpen
-	err = idb.View(func(dbTx database.Tx) error {
-		return nil
-	})
-	if !checkDbError(t, testName, err, wantErrCode) {
-		return
-	}
-
-	// Ensure the Update handles errors in the underlying leveldb database
-	// properly.
-	testName = "Update: underlying leveldb error"
-	err = idb.Update(func(dbTx database.Tx) error {
-		return nil
-	})
-	if !checkDbError(t, testName, err, wantErrCode) {
-		return
-	}
-}
-
-// resetDatabase removes everything from the opened database associated with the
-// test context including all metadata and the mock files.
-func resetDatabase(tc *testContext) bool {
-	// Reset the metadata.
-	err := tc.db.Update(func(dbTx database.Tx) error {
-		// Remove all the keys using a cursor while also generating a
-		// list of buckets. It's not safe to remove keys during ForEach
-		// iteration nor is it safe to remove buckets during cursor
-		// iteration, so this dual approach is needed.
-		var bucketNames [][]byte
-		cursor := dbTx.Metadata().Cursor()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			if cursor.Value() != nil {
-				if err := cursor.Delete(); err != nil {
-					return err
-				}
-			} else {
-				bucketNames = append(bucketNames, cursor.Key())
-			}
-		}
-
-		// Remove the buckets.
-		for _, k := range bucketNames {
-			if err := dbTx.Metadata().DeleteBucket(k); err != nil {
-				return err
-			}
-		}
-
-		_, err := dbTx.Metadata().CreateBucket(blockIdxBucketName)
-		return err
-	})
-	if err != nil {
-		tc.t.Errorf("Update: unexpected error: %v", err)
-		return false
-	}
-
-	// Reset the mock files.
-	store := tc.db.(*db).store
-	wc := store.writeCursor
-	wc.curFile.Lock()
-	if wc.curFile.file != nil {
-		wc.curFile.file.Close()
-		wc.curFile.file = nil
-	}
-	wc.curFile.Unlock()
-	wc.Lock()
-	wc.curFileNum = 0
-	wc.curOffset = 0
-	wc.Unlock()
-	tc.files = make(map[uint32]*lockableFile)
-	tc.maxFileSizes = make(map[uint32]int64)
-	return true
-}
-
-// testWriteFailures tests various failures paths when writing to the block
-// files.
-func testWriteFailures(tc *testContext) bool {
-	if !resetDatabase(tc) {
-		return false
-	}
-
-	// Ensure file sync errors during flush return the expected error.
-	store := tc.db.(*db).store
-	testName := "flush: file sync failure"
-	store.writeCursor.Lock()
-	oldFile := store.writeCursor.curFile
-	store.writeCursor.curFile = &lockableFile{
-		file: &mockFile{forceSyncErr: true, maxSize: -1},
-	}
-	store.writeCursor.Unlock()
-	err := tc.db.(*db).cache.flush()
-	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
-		return false
-	}
-	store.writeCursor.Lock()
-	store.writeCursor.curFile = oldFile
-	store.writeCursor.Unlock()
-
-	// Force errors in the various error paths when writing data by using
-	// mock files with a limited max size.
-	block0Bytes, _ := tc.blocks[0].Bytes()
-	tests := []struct {
-		fileNum uint32
-		maxSize int64
-	}{
-		// Force an error when writing the network bytes.
-		{fileNum: 0, maxSize: 2},
-
-		// Force an error when writing the block size.
-		{fileNum: 0, maxSize: 6},
-
-		// Force an error when writing the block.
-		{fileNum: 0, maxSize: 17},
-
-		// Force an error when writing the checksum.
-		{fileNum: 0, maxSize: int64(len(block0Bytes)) + 10},
-
-		// Force an error after writing enough blocks for force multiple
-		// files.
-		{fileNum: 15, maxSize: 1},
-	}
-
-	for i, test := range tests {
-		if !resetDatabase(tc) {
-			return false
-		}
-
-		// Ensure storing the specified number of blocks using a mock
-		// file that fails the write fails when the transaction is
-		// committed, not when the block is stored.
-		tc.maxFileSizes = map[uint32]int64{test.fileNum: test.maxSize}
-		err := tc.db.Update(func(dbTx database.Tx) error {
-			for i, block := range tc.blocks {
-				err := dbTx.StoreBlock(block)
-				if err != nil {
-					tc.t.Errorf("StoreBlock (%d): unexpected "+
-						"error: %v", i, err)
-					return errSubTestFail
-				}
-			}
-
-			return nil
-		})
-		testName := fmt.Sprintf("Force update commit failure - test "+
-			"%d, fileNum %d, maxsize %d", i, test.fileNum,
-			test.maxSize)
-		if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
-			tc.t.Errorf("%v", err)
-			return false
-		}
-
-		// Ensure the commit rollback removed all extra files and data.
-		if len(tc.files) != 1 {
-			tc.t.Errorf("Update rollback: new not removed - want "+
-				"1 file, got %d", len(tc.files))
-			return false
-		}
-		if _, ok := tc.files[0]; !ok {
-			tc.t.Error("Update rollback: file 0 does not exist")
-			return false
-		}
-		file := tc.files[0].file.(*mockFile)
-		if len(file.data) != 0 {
-			tc.t.Errorf("Update rollback: file did not truncate - "+
-				"want len 0, got len %d", len(file.data))
-			return false
-		}
-	}
-
-	return true
-}
-
-// testBlockFileErrors ensures the database returns expected errors with various
-// file-related issues such as closed and missing files.
-func testBlockFileErrors(tc *testContext) bool {
-	if !resetDatabase(tc) {
-		return false
-	}
-
-	// Ensure errors in blockFile and openFile when requesting invalid file
-	// numbers.
-	store := tc.db.(*db).store
-	testName := "blockFile invalid file open"
-	_, err := store.blockFile(^uint32(0))
-	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
-		return false
-	}
-	testName = "openFile invalid file open"
-	_, err = store.openFile(^uint32(0))
-	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
-		return false
-	}
-
-	// Insert the first block into the mock file.
-	err = tc.db.Update(func(dbTx database.Tx) error {
-		err := dbTx.StoreBlock(tc.blocks[0])
-		if err != nil {
-			tc.t.Errorf("StoreBlock: unexpected error: %v", err)
-			return errSubTestFail
-		}
-
-		return nil
-	})
-	if err != nil {
-		if err != errSubTestFail {
-			tc.t.Errorf("Update: unexpected error: %v", err)
-		}
-		return false
-	}
-
-	// Ensure errors in readBlock and readBlockRegion when requesting a file
-	// number that doesn't exist.
-	block0Hash := tc.blocks[0].Hash()
-	testName = "readBlock invalid file number"
-	invalidLoc := blockLocation{
-		blockFileNum: ^uint32(0),
-		blockLen:     80,
-	}
-	_, err = store.readBlock(block0Hash, invalidLoc)
-	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
-		return false
-	}
-	testName = "readBlockRegion invalid file number"
-	_, err = store.readBlockRegion(invalidLoc, 0, 80)
-	if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
-		return false
-	}
-
-	// Close the block file out from under the database.
-	store.writeCursor.curFile.Lock()
-	store.writeCursor.curFile.file.Close()
-	store.writeCursor.curFile.Unlock()
-
-	// Ensure failures in FetchBlock and FetchBlockRegion(s) since the
-	// underlying file they need to read from has been closed.
-	err = tc.db.View(func(dbTx database.Tx) error {
-		testName = "FetchBlock closed file"
-		wantErrCode := database.ErrDriverSpecific
-		_, err := dbTx.FetchBlock(block0Hash)
-		if !checkDbError(tc.t, testName, err, wantErrCode) {
-			return errSubTestFail
-		}
-
-		testName = "FetchBlockRegion closed file"
-		regions := []database.BlockRegion{
-			{
-				Hash:   block0Hash,
-				Len:    80,
-				Offset: 0,
-			},
-		}
-		_, err = dbTx.FetchBlockRegion(&regions[0])
-		if !checkDbError(tc.t, testName, err, wantErrCode) {
-			return errSubTestFail
-		}
-
-		testName = "FetchBlockRegions closed file"
-		_, err = dbTx.FetchBlockRegions(regions)
-		if !checkDbError(tc.t, testName, err, wantErrCode) {
-			return errSubTestFail
-		}
-
-		return nil
-	})
-	if err != nil {
-		if err != errSubTestFail {
-			tc.t.Errorf("View: unexpected error: %v", err)
-		}
-		return false
-	}
-
-	return true
-}
-
-// testCorruption ensures the database returns expected errors under various
-// corruption scenarios.
-func testCorruption(tc *testContext) bool {
-	if !resetDatabase(tc) {
-		return false
-	}
-
-	// Insert the first block into the mock file.
-	err := tc.db.Update(func(dbTx database.Tx) error {
-		err := dbTx.StoreBlock(tc.blocks[0])
-		if err != nil {
-			tc.t.Errorf("StoreBlock: unexpected error: %v", err)
-			return errSubTestFail
-		}
-
-		return nil
-	})
-	if err != nil {
-		if err != errSubTestFail {
-			tc.t.Errorf("Update: unexpected error: %v", err)
-		}
-		return false
-	}
-
-	// Ensure corruption is detected by intentionally modifying the bytes
-	// stored to the mock file and reading the block.
-	block0Bytes, _ := tc.blocks[0].Bytes()
-	block0Hash := tc.blocks[0].Hash()
-	tests := []struct {
-		offset      uint32
-		fixChecksum bool
-		wantErrCode database.ErrorCode
-	}{
-		// One of the network bytes. The checksum needs to be fixed so
-		// the invalid network is detected.
-		{2, true, database.ErrDriverSpecific},
-
-		// The same network byte, but this time don't fix the checksum
-		// to ensure the corruption is detected.
-		{2, false, database.ErrCorruption},
-
-		// One of the block length bytes.
-		{6, false, database.ErrCorruption},
-
-		// Random header byte.
-		{17, false, database.ErrCorruption},
-
-		// Random transaction byte.
-		{90, false, database.ErrCorruption},
-
-		// Random checksum byte.
-		{uint32(len(block0Bytes)) + 10, false, database.ErrCorruption},
-	}
-	err = tc.db.View(func(dbTx database.Tx) error {
-		data := tc.files[0].file.(*mockFile).data
-		for i, test := range tests {
-			// Corrupt the byte at the offset by a single bit.
-			data[test.offset] ^= 0x10
-
-			// Fix the checksum if requested to force other errors.
-			fileLen := len(data)
-			var oldChecksumBytes [4]byte
-			copy(oldChecksumBytes[:], data[fileLen-4:])
-			if test.fixChecksum {
-				toSum := data[:fileLen-4]
-				cksum := crc32.Checksum(toSum, castagnoli)
-				binary.BigEndian.PutUint32(data[fileLen-4:], cksum)
-			}
-
-			testName := fmt.Sprintf("FetchBlock (test #%d): "+
-				"corruption", i)
-			_, err := dbTx.FetchBlock(block0Hash)
-			if !checkDbError(tc.t, testName, err, test.wantErrCode) {
-				return errSubTestFail
-			}
-
-			// Reset the corrupted data back to the original.
-			data[test.offset] ^= 0x10
-			if test.fixChecksum {
-				copy(data[fileLen-4:], oldChecksumBytes[:])
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		if err != errSubTestFail {
-			tc.t.Errorf("View: unexpected error: %v", err)
-		}
-		return false
-	}
-
-	return true
-}
-
-// TestFailureScenarios ensures several failure scenarios such as database
-// corruption, block file write failures, and rollback failures are handled
-// correctly.
-func TestFailureScenarios(t *testing.T) {
-	// Create a new database to run tests against.
-	dbPath := filepath.Join(os.TempDir(), "ffldb-failurescenarios")
-	_ = os.RemoveAll(dbPath)
-	idb, err := database.Create(dbType, dbPath, blockDataNet)
-	if err != nil {
-		t.Errorf("Failed to create test database (%s) %v", dbType, err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer idb.Close()
-
-	// Create a test context to pass around.
-	tc := &testContext{
-		t:            t,
-		db:           idb,
-		files:        make(map[uint32]*lockableFile),
-		maxFileSizes: make(map[uint32]int64),
-	}
-
-	// Change the maximum file size to a small value to force multiple flat
-	// files with the test data set and replace the file-related functions
-	// to make use of mock files in memory. This allows injection of
-	// various file-related errors.
-	store := idb.(*db).store
-	store.maxBlockFileSize = 1024 // 1KiB
-	store.openWriteFileFunc = func(fileNum uint32) (filer, error) {
-		if file, ok := tc.files[fileNum]; ok {
-			// "Reopen" the file.
-			file.Lock()
-			mock := file.file.(*mockFile)
-			mock.Lock()
-			mock.closed = false
-			mock.Unlock()
-			file.Unlock()
-			return mock, nil
-		}
-
-		// Limit the max size of the mock file as specified in the test
-		// context.
-		maxSize := int64(-1)
-		if maxFileSize, ok := tc.maxFileSizes[fileNum]; ok {
-			maxSize = int64(maxFileSize)
-		}
-		file := &mockFile{maxSize: int64(maxSize)}
-		tc.files[fileNum] = &lockableFile{file: file}
-		return file, nil
-	}
-	store.openFileFunc = func(fileNum uint32) (*lockableFile, error) {
-		// Force error when trying to open max file num.
-		if fileNum == ^uint32(0) {
-			return nil, makeDbErr(database.ErrDriverSpecific,
-				"test", nil)
-		}
-		if file, ok := tc.files[fileNum]; ok {
-			// "Reopen" the file.
-			file.Lock()
-			mock := file.file.(*mockFile)
-			mock.Lock()
-			mock.closed = false
-			mock.Unlock()
-			file.Unlock()
-			return file, nil
-		}
-		file := &lockableFile{file: &mockFile{}}
-		tc.files[fileNum] = file
-		return file, nil
-	}
-	store.deleteFileFunc = func(fileNum uint32) error {
-		if file, ok := tc.files[fileNum]; ok {
-			file.Lock()
-			file.file.Close()
-			file.Unlock()
-			delete(tc.files, fileNum)
-			return nil
-		}
-
-		str := fmt.Sprintf("file %d does not exist", fileNum)
-		return makeDbErr(database.ErrDriverSpecific, str, nil)
-	}
-
-	// Load the test blocks and save in the test context for use throughout
-	// the tests.
-	blocks, err := loadBlocks(t, blockDataFile, blockDataNet)
-	if err != nil {
-		t.Errorf("loadBlocks: Unexpected error: %v", err)
-		return
-	}
-	tc.blocks = blocks
-
-	// Test various failures paths when writing to the block files.
-	if !testWriteFailures(tc) {
-		return
-	}
-
-	// Test various file-related issues such as closed and missing files.
-	if !testBlockFileErrors(tc) {
-		return
-	}
-
-	// Test various corruption scenarios.
-	testCorruption(tc)
-}

database/interface.go

@@ -1,469 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-// Parts of this interface were inspired heavily by the excellent boltdb project
-// at https://github.com/boltdb/bolt by Ben B. Johnson.
-
-package database
-
-import (
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// Cursor represents a cursor over key/value pairs and nested buckets of a
-// bucket.
-//
-// Note that open cursors are not tracked on bucket changes and any
-// modifications to the bucket, with the exception of Cursor.Delete, invalidates
-// the cursor. After invalidation, the cursor must be repositioned, or the keys
-// and values returned may be unpredictable.
-type Cursor interface {
-	// Bucket returns the bucket the cursor was created for.
-	Bucket() Bucket
-
-	// Delete removes the current key/value pair the cursor is at without
-	// invalidating the cursor.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrIncompatibleValue if attempted when the cursor points to a
-	//     nested bucket
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	Delete() error
-
-	// First positions the cursor at the first key/value pair and returns
-	// whether or not the pair exists.
-	First() bool
-
-	// Last positions the cursor at the last key/value pair and returns
-	// whether or not the pair exists.
-	Last() bool
-
-	// Next moves the cursor one key/value pair forward and returns whether
-	// or not the pair exists.
-	Next() bool
-
-	// Prev moves the cursor one key/value pair backward and returns whether
-	// or not the pair exists.
-	Prev() bool
-
-	// Seek positions the cursor at the first key/value pair that is greater
-	// than or equal to the passed seek key. Returns whether or not the
-	// pair exists.
-	Seek(seek []byte) bool
-
-	// Key returns the current key the cursor is pointing to.
-	Key() []byte
-
-	// Value returns the current value the cursor is pointing to. This will
-	// be nil for nested buckets.
-	Value() []byte
-}
-
-// Bucket represents a collection of key/value pairs.
-type Bucket interface {
-	// Bucket retrieves a nested bucket with the given key. Returns nil if
-	// the bucket does not exist.
-	Bucket(key []byte) Bucket
-
-	// CreateBucket creates and returns a new nested bucket with the given
-	// key.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBucketExists if the bucket already exists
-	//   - ErrBucketNameRequired if the key is empty
-	//   - ErrIncompatibleValue if the key is otherwise invalid for the
-	//     particular implementation
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	CreateBucket(key []byte) (Bucket, error)
-
-	// CreateBucketIfNotExists creates and returns a new nested bucket with
-	// the given key if it does not already exist.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBucketNameRequired if the key is empty
-	//   - ErrIncompatibleValue if the key is otherwise invalid for the
-	//     particular implementation
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	CreateBucketIfNotExists(key []byte) (Bucket, error)
-
-	// DeleteBucket removes a nested bucket with the given key. This also
-	// includes removing all nested buckets and keys under the bucket being
-	// deleted.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBucketNotFound if the specified bucket does not exist
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	DeleteBucket(key []byte) error
-
-	// ForEach invokes the passed function with every key/value pair in the
-	// bucket. This does not include nested buckets or the key/value pairs
-	// within those nested buckets.
-	//
-	// WARNING: It is not safe to mutate data while iterating with this
-	// method. Doing so may cause the underlying cursor to be invalidated
-	// and return unexpected keys and/or values.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrTxClosed if the transaction has already been closed
-	//
-	// NOTE: The slices returned by this function are only valid during a
-	// transaction. Attempting to access them after a transaction has ended
-	// results in undefined behavior. Additionally, the slices must NOT
-	// be modified by the caller. These constraints prevent additional data
-	// copies and allows support for memory-mapped database implementations.
-	ForEach(func(k, v []byte) error) error
-
-	// ForEachBucket invokes the passed function with the key of every
-	// nested bucket in the current bucket. This does not include any
-	// nested buckets within those nested buckets.
-	//
-	// WARNING: It is not safe to mutate data while iterating with this
-	// method. Doing so may cause the underlying cursor to be invalidated
-	// and return unexpected keys and/or values.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrTxClosed if the transaction has already been closed
-	//
-	// NOTE: The keys returned by this function are only valid during a
-	// transaction. Attempting to access them after a transaction has ended
-	// results in undefined behavior. This constraint prevents additional
-	// data copies and allows support for memory-mapped database
-	// implementations.
-	ForEachBucket(func(k []byte) error) error
-
-	// Cursor returns a new cursor, allowing for iteration over the bucket's
-	// key/value pairs and nested buckets in forward or backward order.
-	//
-	// You must seek to a position using the First, Last, or Seek functions
-	// before calling the Next, Prev, Key, or Value functions. Failure to
-	// do so will result in the same return values as an exhausted cursor,
-	// which is false for the Prev and Next functions and nil for Key and
-	// Value functions.
-	Cursor() Cursor
-
-	// Writable returns whether or not the bucket is writable.
-	Writable() bool
-
-	// Put saves the specified key/value pair to the bucket. Keys that do
-	// not already exist are added and keys that already exist are
-	// overwritten.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrKeyRequired if the key is empty
-	//   - ErrIncompatibleValue if the key is the same as an existing bucket
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	//
-	// NOTE: The slices passed to this function must NOT be modified by the
-	// caller. This constraint prevents the requirement for additional data
-	// copies and allows support for memory-mapped database implementations.
-	Put(key, value []byte) error
-
-	// Get returns the value for the given key. Returns nil if the key does
-	// not exist in this bucket. An empty slice is returned for keys that
-	// exist but have no value assigned.
-	//
-	// NOTE: The value returned by this function is only valid during a
-	// transaction. Attempting to access it after a transaction has ended
-	// results in undefined behavior. Additionally, the value must NOT
-	// be modified by the caller. These constraints prevent additional data
-	// copies and allows support for memory-mapped database implementations.
-	Get(key []byte) []byte
-
-	// Delete removes the specified key from the bucket. Deleting a key
-	// that does not exist does not return an error.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrKeyRequired if the key is empty
-	//   - ErrIncompatibleValue if the key is the same as an existing bucket
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	Delete(key []byte) error
-}
-
-// BlockRegion specifies a particular region of a block identified by the
-// specified hash, given an offset and length.
-type BlockRegion struct {
-	Hash   *daghash.Hash
-	Offset uint32
-	Len    uint32
-}
-
-// Tx represents a database transaction. It can either by read-only or
-// read-write. The transaction provides a metadata bucket against which all
-// read and writes occur.
-//
-// As would be expected with a transaction, no changes will be saved to the
-// database until it has been committed. The transaction will only provide a
-// view of the database at the time it was created. Transactions should not be
-// long running operations.
-type Tx interface {
-	// Metadata returns the top-most bucket for all metadata storage.
-	Metadata() Bucket
-
-	// StoreBlock stores the provided block into the database. There are no
-	// checks to ensure the block connects to a previous block, contains
-	// double spends, or any additional functionality such as transaction
-	// indexing. It simply stores the block in the database.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockExists when the block hash already exists
-	//   - ErrTxNotWritable if attempted against a read-only transaction
-	//   - ErrTxClosed if the transaction has already been closed
-	//
-	// Other errors are possible depending on the implementation.
-	StoreBlock(block *util.Block) error
-
-	// HasBlock returns whether or not a block with the given hash exists
-	// in the database.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrTxClosed if the transaction has already been closed
-	//
-	// Other errors are possible depending on the implementation.
-	HasBlock(hash *daghash.Hash) (bool, error)
-
-	// HasBlocks returns whether or not the blocks with the provided hashes
-	// exist in the database.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrTxClosed if the transaction has already been closed
-	//
-	// Other errors are possible depending on the implementation.
-	HasBlocks(hashes []*daghash.Hash) ([]bool, error)
-
-	// FetchBlockHeader returns the raw serialized bytes for the block
-	// header identified by the given hash. The raw bytes are in the format
-	// returned by Serialize on a wire.BlockHeader.
-	//
-	// It is highly recommended to use this function (or FetchBlockHeaders)
-	// to obtain block headers over the FetchBlockRegion(s) functions since
-	// it provides the backend drivers the freedom to perform very specific
-	// optimizations which can result in significant speed advantages when
-	// working with headers.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockNotFound if the requested block hash does not exist
-	//   - ErrTxClosed if the transaction has already been closed
-	//   - ErrCorruption if the database has somehow become corrupted
-	//
-	// NOTE: The data returned by this function is only valid during a
-	// database transaction. Attempting to access it after a transaction
-	// has ended results in undefined behavior. This constraint prevents
-	// additional data copies and allows support for memory-mapped database
-	// implementations.
-	FetchBlockHeader(hash *daghash.Hash) ([]byte, error)
-
-	// FetchBlockHeaders returns the raw serialized bytes for the block
-	// headers identified by the given hashes. The raw bytes are in the
-	// format returned by Serialize on a wire.BlockHeader.
-	//
-	// It is highly recommended to use this function (or FetchBlockHeader)
-	// to obtain block headers over the FetchBlockRegion(s) functions since
-	// it provides the backend drivers the freedom to perform very specific
-	// optimizations which can result in significant speed advantages when
-	// working with headers.
-	//
-	// Furthermore, depending on the specific implementation, this function
-	// can be more efficient for bulk loading multiple block headers than
-	// loading them one-by-one with FetchBlockHeader.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockNotFound if any of the request block hashes do not exist
-	//   - ErrTxClosed if the transaction has already been closed
-	//   - ErrCorruption if the database has somehow become corrupted
-	//
-	// NOTE: The data returned by this function is only valid during a
-	// database transaction. Attempting to access it after a transaction
-	// has ended results in undefined behavior. This constraint prevents
-	// additional data copies and allows support for memory-mapped database
-	// implementations.
-	FetchBlockHeaders(hashes []*daghash.Hash) ([][]byte, error)
-
-	// FetchBlock returns the raw serialized bytes for the block identified
-	// by the given hash. The raw bytes are in the format returned by
-	// Serialize on a wire.MsgBlock.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockNotFound if the requested block hash does not exist
-	//   - ErrTxClosed if the transaction has already been closed
-	//   - ErrCorruption if the database has somehow become corrupted
-	//
-	// NOTE: The data returned by this function is only valid during a
-	// database transaction. Attempting to access it after a transaction
-	// has ended results in undefined behavior. This constraint prevents
-	// additional data copies and allows support for memory-mapped database
-	// implementations.
-	FetchBlock(hash *daghash.Hash) ([]byte, error)
-
-	// FetchBlocks returns the raw serialized bytes for the blocks
-	// identified by the given hashes. The raw bytes are in the format
-	// returned by Serialize on a wire.MsgBlock.
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockNotFound if the any of the requested block hashes do not
-	//     exist
-	//   - ErrTxClosed if the transaction has already been closed
-	//   - ErrCorruption if the database has somehow become corrupted
-	//
-	// NOTE: The data returned by this function is only valid during a
-	// database transaction. Attempting to access it after a transaction
-	// has ended results in undefined behavior. This constraint prevents
-	// additional data copies and allows support for memory-mapped database
-	// implementations.
-	FetchBlocks(hashes []*daghash.Hash) ([][]byte, error)
-
-	// FetchBlockRegion returns the raw serialized bytes for the given
-	// block region.
-	//
-	// For example, it is possible to directly extract Kaspa transactions
-	// and/or scripts from a block with this function. Depending on the
-	// backend implementation, this can provide significant savings by
-	// avoiding the need to load entire blocks.
-	//
-	// The raw bytes are in the format returned by Serialize on a
-	// wire.MsgBlock and the Offset field in the provided BlockRegion is
-	// zero-based and relative to the start of the block (byte 0).
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockNotFound if the requested block hash does not exist
-	//   - ErrBlockRegionInvalid if the region exceeds the bounds of the
-	//     associated block
-	//   - ErrTxClosed if the transaction has already been closed
-	//   - ErrCorruption if the database has somehow become corrupted
-	//
-	// NOTE: The data returned by this function is only valid during a
-	// database transaction. Attempting to access it after a transaction
-	// has ended results in undefined behavior. This constraint prevents
-	// additional data copies and allows support for memory-mapped database
-	// implementations.
-	FetchBlockRegion(region *BlockRegion) ([]byte, error)
-
-	// FetchBlockRegions returns the raw serialized bytes for the given
-	// block regions.
-	//
-	// For example, it is possible to directly extract Kaspa transactions
-	// and/or scripts from various blocks with this function. Depending on
-	// the backend implementation, this can provide significant savings by
-	// avoiding the need to load entire blocks.
-	//
-	// The raw bytes are in the format returned by Serialize on a
-	// wire.MsgBlock and the Offset fields in the provided BlockRegions are
-	// zero-based and relative to the start of the block (byte 0).
-	//
-	// The interface contract guarantees at least the following errors will
-	// be returned (other implementation-specific errors are possible):
-	//   - ErrBlockNotFound if any of the requested block hashed do not
-	//     exist
-	//   - ErrBlockRegionInvalid if one or more region exceed the bounds of
-	//     the associated block
-	//   - ErrTxClosed if the transaction has already been closed
-	//   - ErrCorruption if the database has somehow become corrupted
-	//
-	// NOTE: The data returned by this function is only valid during a
-	// database transaction. Attempting to access it after a transaction
-	// has ended results in undefined behavior. This constraint prevents
-	// additional data copies and allows support for memory-mapped database
-	// implementations.
-	FetchBlockRegions(regions []BlockRegion) ([][]byte, error)
-
-	// ******************************************************************
-	// Methods related to both atomic metadata storage and block storage.
-	// ******************************************************************
-
-	// Commit commits all changes that have been made to the metadata or
-	// block storage. Depending on the backend implementation this could be
-	// to a cache that is periodically synced to persistent storage or
-	// directly to persistent storage. In any case, all transactions which
-	// are started after the commit finishes will include all changes made
-	// by this transaction. Calling this function on a managed transaction
-	// will result in a panic.
-	Commit() error
-
-	// Rollback undoes all changes that have been made to the metadata or
-	// block storage. Calling this function on a managed transaction will
-	// result in a panic.
-	Rollback() error
-}
-
-// DB provides a generic interface that is used to store kaspa blocks and
-// related metadata. This interface is intended to be agnostic to the actual
-// mechanism used for backend data storage. The RegisterDriver function can be
-// used to add a new backend data storage method.
-//
-// This interface is divided into two distinct categories of functionality.
-//
-// The first category is atomic metadata storage with bucket support. This is
-// accomplished through the use of database transactions.
-//
-// The second category is generic block storage. This functionality is
-// intentionally separate because the mechanism used for block storage may or
-// may not be the same mechanism used for metadata storage. For example, it is
-// often more efficient to store the block data as flat files while the metadata
-// is kept in a database. However, this interface aims to be generic enough to
-// support blocks in the database too, if needed by a particular backend.
-type DB interface {
-	// Type returns the database driver type the current database instance
-	// was created with.
-	Type() string
-
-	// Begin starts a transaction which is either read-only or read-write
-	// depending on the specified flag. Multiple read-only transactions
-	// can be started simultaneously while only a single read-write
-	// transaction can be started at a time. The call will block when
-	// starting a read-write transaction when one is already open.
-	//
-	// NOTE: The transaction must be closed by calling Rollback or Commit on
-	// it when it is no longer needed. Failure to do so can result in
-	// unclaimed memory and/or inablity to close the database due to locks
-	// depending on the specific database implementation.
-	Begin(writable bool) (Tx, error)
-
-	// View invokes the passed function in the context of a managed
-	// read-only transaction. Any errors returned from the user-supplied
-	// function are returned from this function.
-	//
-	// Calling Rollback or Commit on the transaction passed to the
-	// user-supplied function will result in a panic.
-	View(fn func(tx Tx) error) error
-
-	// Update invokes the passed function in the context of a managed
-	// read-write transaction. Any errors returned from the user-supplied
-	// function will cause the transaction to be rolled back and are
-	// returned from this function. Otherwise, the transaction is committed
-	// when the user-supplied function returns a nil error.
-	//
-	// Calling Rollback or Commit on the transaction passed to the
-	// user-supplied function will result in a panic.
-	Update(fn func(tx Tx) error) error
-
-	// Close cleanly shuts down the database and syncs all data. It will
-	// block until all database transactions have been finalized (rolled
-	// back or committed).
-	Close() error
-
-	// FlushCache flushes the db cache to the disk.
-	FlushCache() error
-}

database/internal/treap/README.md

@@ -1,27 +0,0 @@
-treap
-=====
-
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
-[![GoDoc](https://godoc.org/github.com/kaspanet/kaspad/database/internal/treap?status.png)](http://godoc.org/github.com/kaspanet/kaspad/database/internal/treap)
-
-Package treap implements a treap data structure that is used to hold ordered
-key/value pairs using a combination of binary search tree and heap semantics.
-It is a self-organizing and randomized data structure that doesn't require
-complex operations to to maintain balance. Search, insert, and delete
-operations are all O(log n). Both mutable and immutable variants are provided.
-
-The mutable variant is typically faster since it is able to simply update the
-treap when modifications are made. However, a mutable treap is not safe for
-concurrent access without careful use of locking by the caller and care must be
-taken when iterating since it can change out from under the iterator.
-
-The immutable variant works by creating a new version of the treap for all
-mutations by replacing modified nodes with new nodes that have updated values
-while sharing all unmodified nodes with the previous version. This is extremely
-useful in concurrent applications since the caller only has to atomically
-replace the treap pointer with the newly returned version after performing any
-mutations. All readers can simply use their existing pointer as a snapshot
-since the treap it points to is immutable. This effectively provides O(1)
-snapshot capability with efficient memory usage characteristics since the old
-nodes only remain allocated until there are no longer any references to them.
-

database/internal/treap/common.go

@@ -1,136 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"math/rand"
-	"time"
-)
-
-const (
-	// staticDepth is the size of the static array to use for keeping track
-	// of the parent stack during treap iteration. Since a treap has a very
-	// high probability that the tree height is logarithmic, it is
-	// exceedingly unlikely that the parent stack will ever exceed this size
-	// even for extremely large numbers of items.
-	staticDepth = 128
-
-	// nodeFieldsSize is the size the fields of each node takes excluding
-	// the contents of the key and value. It assumes 64-bit pointers so
-	// technically it is smaller on 32-bit platforms, but overestimating the
-	// size in that case is acceptable since it avoids the need to import
-	// unsafe. It consists of 24-bytes for each key and value + 8 bytes for
-	// each of the priority, left, and right fields (24*2 + 8*3).
-	nodeFieldsSize = 72
-)
-
-var (
-	// emptySlice is used for keys that have no value associated with them
-	// so callers can distinguish between a key that does not exist and one
-	// that has no value associated with it.
-	emptySlice = make([]byte, 0)
-)
-
-// treapNode represents a node in the treap.
-type treapNode struct {
-	key      []byte
-	value    []byte
-	priority int
-	left     *treapNode
-	right    *treapNode
-}
-
-// nodeSize returns the number of bytes the specified node occupies including
-// the struct fields and the contents of the key and value.
-func nodeSize(node *treapNode) uint64 {
-	return nodeFieldsSize + uint64(len(node.key)+len(node.value))
-}
-
-// newTreapNode returns a new node from the given key, value, and priority. The
-// node is not initially linked to any others.
-func newTreapNode(key, value []byte, priority int) *treapNode {
-	return &treapNode{key: key, value: value, priority: priority}
-}
-
-// parentStack represents a stack of parent treap nodes that are used during
-// iteration. It consists of a static array for holding the parents and a
-// dynamic overflow slice. It is extremely unlikely the overflow will ever be
-// hit during normal operation, however, since a treap's height is
-// probabilistic, the overflow case needs to be handled properly. This approach
-// is used because it is much more efficient for the majority case than
-// dynamically allocating heap space every time the treap is iterated.
-type parentStack struct {
-	index    int
-	items    [staticDepth]*treapNode
-	overflow []*treapNode
-}
-
-// Len returns the current number of items in the stack.
-func (s *parentStack) Len() int {
-	return s.index
-}
-
-// At returns the item n number of items from the top of the stack, where 0 is
-// the topmost item, without removing it. It returns nil if n exceeds the
-// number of items on the stack.
-func (s *parentStack) At(n int) *treapNode {
-	index := s.index - n - 1
-	if index < 0 {
-		return nil
-	}
-
-	if index < staticDepth {
-		return s.items[index]
-	}
-
-	return s.overflow[index-staticDepth]
-}
-
-// Pop removes the top item from the stack. It returns nil if the stack is
-// empty.
-func (s *parentStack) Pop() *treapNode {
-	if s.index == 0 {
-		return nil
-	}
-
-	s.index--
-	if s.index < staticDepth {
-		node := s.items[s.index]
-		s.items[s.index] = nil
-		return node
-	}
-
-	node := s.overflow[s.index-staticDepth]
-	s.overflow[s.index-staticDepth] = nil
-	return node
-}
-
-// Push pushes the passed item onto the top of the stack.
-func (s *parentStack) Push(node *treapNode) {
-	if s.index < staticDepth {
-		s.items[s.index] = node
-		s.index++
-		return
-	}
-
-	// This approach is used over append because reslicing the slice to pop
-	// the item causes the compiler to make unneeded allocations. Also,
-	// since the max number of items is related to the tree depth which
-	// requires expontentially more items to increase, only increase the cap
-	// one item at a time. This is more intelligent than the generic append
-	// expansion algorithm which often doubles the cap.
-	index := s.index - staticDepth
-	if index+1 > cap(s.overflow) {
-		overflow := make([]*treapNode, index+1)
-		copy(overflow, s.overflow)
-		s.overflow = overflow
-	}
-	s.overflow[index] = node
-	s.index++
-}
-
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}

database/internal/treap/common_test.go

@@ -1,108 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"encoding/binary"
-	"math/rand"
-	"reflect"
-	"testing"
-)
-
-// serializeUint32 returns the big-endian encoding of the passed uint32.
-func serializeUint32(ui uint32) []byte {
-	var ret [4]byte
-	binary.BigEndian.PutUint32(ret[:], ui)
-	return ret[:]
-}
-
-// TestParentStack ensures the treapParentStack functionality works as intended.
-func TestParentStack(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		numNodes int
-	}{
-		{numNodes: 1},
-		{numNodes: staticDepth},
-		{numNodes: staticDepth + 1}, // Test dynamic code paths
-	}
-
-testLoop:
-	for i, test := range tests {
-		nodes := make([]*treapNode, 0, test.numNodes)
-		for j := 0; j < test.numNodes; j++ {
-			var key [4]byte
-			binary.BigEndian.PutUint32(key[:], uint32(j))
-			node := newTreapNode(key[:], key[:], 0)
-			nodes = append(nodes, node)
-		}
-
-		// Push all of the nodes onto the parent stack while testing
-		// various stack properties.
-		stack := &parentStack{}
-		for j, node := range nodes {
-			stack.Push(node)
-
-			// Ensure the stack length is the expected value.
-			if stack.Len() != j+1 {
-				t.Errorf("Len #%d (%d): unexpected stack "+
-					"length - got %d, want %d", i, j,
-					stack.Len(), j+1)
-				continue testLoop
-			}
-
-			// Ensure the node at each index is the expected one.
-			for k := 0; k <= j; k++ {
-				atNode := stack.At(j - k)
-				if !reflect.DeepEqual(atNode, nodes[k]) {
-					t.Errorf("At #%d (%d): mismatched node "+
-						"- got %v, want %v", i, j-k,
-						atNode, nodes[k])
-					continue testLoop
-				}
-			}
-		}
-
-		// Ensure each popped node is the expected one.
-		for j := 0; j < len(nodes); j++ {
-			node := stack.Pop()
-			expected := nodes[len(nodes)-j-1]
-			if !reflect.DeepEqual(node, expected) {
-				t.Errorf("At #%d (%d): mismatched node - "+
-					"got %v, want %v", i, j, node, expected)
-				continue testLoop
-			}
-		}
-
-		// Ensure the stack is now empty.
-		if stack.Len() != 0 {
-			t.Errorf("Len #%d: stack is not empty - got %d", i,
-				stack.Len())
-			continue testLoop
-		}
-
-		// Ensure attempting to retrieve a node at an index beyond the
-		// stack's length returns nil.
-		if node := stack.At(2); node != nil {
-			t.Errorf("At #%d: did not give back nil - got %v", i,
-				node)
-			continue testLoop
-		}
-
-		// Ensure attempting to pop a node from an empty stack returns
-		// nil.
-		if node := stack.Pop(); node != nil {
-			t.Errorf("Pop #%d: did not give back nil - got %v", i,
-				node)
-			continue testLoop
-		}
-	}
-}
-
-func init() {
-	// Force the same pseudo random numbers for each test run.
-	rand.Seed(0)
-}

database/internal/treap/doc.go

@@ -1,23 +0,0 @@
-/*
-Package treap implements a treap data structure that is used to hold ordered
-key/value pairs using a combination of binary search tree and heap semantics.
-It is a self-organizing and randomized data structure that doesn't require
-complex operations to to maintain balance. Search, insert, and delete
-operations are all O(log n). Both mutable and immutable variants are provided.
-
-The mutable variant is typically faster since it is able to simply update the
-treap when modifications are made. However, a mutable treap is not safe for
-concurrent access without careful use of locking by the caller and care must be
-taken when iterating since it can change out from under the iterator.
-
-The immutable variant works by creating a new version of the treap for all
-mutations by replacing modified nodes with new nodes that have updated values
-while sharing all unmodified nodes with the previous version. This is extremely
-useful in concurrent applications since the caller only has to atomically
-replace the treap pointer with the newly returned version after performing any
-mutations. All readers can simply use their existing pointer as a snapshot
-since the treap it points to is immutable. This effectively provides O(1)
-snapshot capability with efficient memory usage characteristics since the old
-nodes only remain allocated until there are no longer any references to them.
-*/
-package treap

database/internal/treap/immutable.go

@@ -1,360 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"bytes"
-	"math/rand"
-)
-
-// cloneTreapNode returns a shallow copy of the passed node.
-func cloneTreapNode(node *treapNode) *treapNode {
-	return &treapNode{
-		key:      node.key,
-		value:    node.value,
-		priority: node.priority,
-		left:     node.left,
-		right:    node.right,
-	}
-}
-
-// Immutable represents a treap data structure which is used to hold ordered
-// key/value pairs using a combination of binary search tree and heap semantics.
-// It is a self-organizing and randomized data structure that doesn't require
-// complex operations to maintain balance. Search, insert, and delete
-// operations are all O(log n). In addition, it provides O(1) snapshots for
-// multi-version concurrency control (MVCC).
-//
-// All operations which result in modifying the treap return a new version of
-// the treap with only the modified nodes updated. All unmodified nodes are
-// shared with the previous version. This is extremely useful in concurrent
-// applications since the caller only has to atomically replace the treap
-// pointer with the newly returned version after performing any mutations. All
-// readers can simply use their existing pointer as a snapshot since the treap
-// it points to is immutable. This effectively provides O(1) snapshot
-// capability with efficient memory usage characteristics since the old nodes
-// only remain allocated until there are no longer any references to them.
-type Immutable struct {
-	root  *treapNode
-	count int
-
-	// totalSize is the best estimate of the total size of of all data in
-	// the treap including the keys, values, and node sizes.
-	totalSize uint64
-}
-
-// newImmutable returns a new immutable treap given the passed parameters.
-func newImmutable(root *treapNode, count int, totalSize uint64) *Immutable {
-	return &Immutable{root: root, count: count, totalSize: totalSize}
-}
-
-// Len returns the number of items stored in the treap.
-func (t *Immutable) Len() int {
-	return t.count
-}
-
-// Size returns a best estimate of the total number of bytes the treap is
-// consuming including all of the fields used to represent the nodes as well as
-// the size of the keys and values. Shared values are not detected, so the
-// returned size assumes each value is pointing to different memory.
-func (t *Immutable) Size() uint64 {
-	return t.totalSize
-}
-
-// get returns the treap node that contains the passed key. It will return nil
-// when the key does not exist.
-func (t *Immutable) get(key []byte) *treapNode {
-	for node := t.root; node != nil; {
-		// Traverse left or right depending on the result of the
-		// comparison.
-		compareResult := bytes.Compare(key, node.key)
-		if compareResult < 0 {
-			node = node.left
-			continue
-		}
-		if compareResult > 0 {
-			node = node.right
-			continue
-		}
-
-		// The key exists.
-		return node
-	}
-
-	// A nil node was reached which means the key does not exist.
-	return nil
-}
-
-// Has returns whether or not the passed key exists.
-func (t *Immutable) Has(key []byte) bool {
-	if node := t.get(key); node != nil {
-		return true
-	}
-	return false
-}
-
-// Get returns the value for the passed key. The function will return nil when
-// the key does not exist.
-func (t *Immutable) Get(key []byte) []byte {
-	if node := t.get(key); node != nil {
-		return node.value
-	}
-	return nil
-}
-
-// Put inserts the passed key/value pair.
-func (t *Immutable) Put(key, value []byte) *Immutable {
-	// Use an empty byte slice for the value when none was provided. This
-	// ultimately allows key existence to be determined from the value since
-	// an empty byte slice is distinguishable from nil.
-	if value == nil {
-		value = emptySlice
-	}
-
-	// The node is the root of the tree if there isn't already one.
-	if t.root == nil {
-		root := newTreapNode(key, value, rand.Int())
-		return newImmutable(root, 1, nodeSize(root))
-	}
-
-	// Find the binary tree insertion point and construct a replaced list of
-	// parents while doing so. This is done because this is an immutable
-	// data structure so regardless of where in the treap the new key/value
-	// pair ends up, all ancestors up to and including the root need to be
-	// replaced.
-	//
-	// When the key matches an entry already in the treap, replace the node
-	// with a new one that has the new value set and return.
-	var parents parentStack
-	var compareResult int
-	for node := t.root; node != nil; {
-		// Clone the node and link its parent to it if needed.
-		nodeCopy := cloneTreapNode(node)
-		if oldParent := parents.At(0); oldParent != nil {
-			if oldParent.left == node {
-				oldParent.left = nodeCopy
-			} else {
-				oldParent.right = nodeCopy
-			}
-		}
-		parents.Push(nodeCopy)
-
-		// Traverse left or right depending on the result of comparing
-		// the keys.
-		compareResult = bytes.Compare(key, node.key)
-		if compareResult < 0 {
-			node = node.left
-			continue
-		}
-		if compareResult > 0 {
-			node = node.right
-			continue
-		}
-
-		// The key already exists, so update its value.
-		nodeCopy.value = value
-
-		// Return new immutable treap with the replaced node and
-		// ancestors up to and including the root of the tree.
-		newRoot := parents.At(parents.Len() - 1)
-		newTotalSize := t.totalSize - uint64(len(node.value)) +
-			uint64(len(value))
-		return newImmutable(newRoot, t.count, newTotalSize)
-	}
-
-	// Link the new node into the binary tree in the correct position.
-	node := newTreapNode(key, value, rand.Int())
-	parent := parents.At(0)
-	if compareResult < 0 {
-		parent.left = node
-	} else {
-		parent.right = node
-	}
-
-	// Perform any rotations needed to maintain the min-heap and replace
-	// the ancestors up to and including the tree root.
-	newRoot := parents.At(parents.Len() - 1)
-	for parents.Len() > 0 {
-		// There is nothing left to do when the node's priority is
-		// greater than or equal to its parent's priority.
-		parent = parents.Pop()
-		if node.priority >= parent.priority {
-			break
-		}
-
-		// Perform a right rotation if the node is on the left side or
-		// a left rotation if the node is on the right side.
-		if parent.left == node {
-			node.right, parent.left = parent, node.right
-		} else {
-			node.left, parent.right = parent, node.left
-		}
-
-		// Either set the new root of the tree when there is no
-		// grandparent or relink the grandparent to the node based on
-		// which side the old parent the node is replacing was on.
-		grandparent := parents.At(0)
-		if grandparent == nil {
-			newRoot = node
-		} else if grandparent.left == parent {
-			grandparent.left = node
-		} else {
-			grandparent.right = node
-		}
-	}
-
-	return newImmutable(newRoot, t.count+1, t.totalSize+nodeSize(node))
-}
-
-// Delete removes the passed key from the treap and returns the resulting treap
-// if it exists. The original immutable treap is returned if the key does not
-// exist.
-func (t *Immutable) Delete(key []byte) *Immutable {
-	// Find the node for the key while constructing a list of parents while
-	// doing so.
-	var parents parentStack
-	var delNode *treapNode
-	for node := t.root; node != nil; {
-		parents.Push(node)
-
-		// Traverse left or right depending on the result of the
-		// comparison.
-		compareResult := bytes.Compare(key, node.key)
-		if compareResult < 0 {
-			node = node.left
-			continue
-		}
-		if compareResult > 0 {
-			node = node.right
-			continue
-		}
-
-		// The key exists.
-		delNode = node
-		break
-	}
-
-	// There is nothing to do if the key does not exist.
-	if delNode == nil {
-		return t
-	}
-
-	// When the only node in the tree is the root node and it is the one
-	// being deleted, there is nothing else to do besides removing it.
-	parent := parents.At(1)
-	if parent == nil && delNode.left == nil && delNode.right == nil {
-		return newImmutable(nil, 0, 0)
-	}
-
-	// Construct a replaced list of parents and the node to delete itself.
-	// This is done because this is an immutable data structure and
-	// therefore all ancestors of the node that will be deleted, up to and
-	// including the root, need to be replaced.
-	var newParents parentStack
-	for i := parents.Len(); i > 0; i-- {
-		node := parents.At(i - 1)
-		nodeCopy := cloneTreapNode(node)
-		if oldParent := newParents.At(0); oldParent != nil {
-			if oldParent.left == node {
-				oldParent.left = nodeCopy
-			} else {
-				oldParent.right = nodeCopy
-			}
-		}
-		newParents.Push(nodeCopy)
-	}
-	delNode = newParents.Pop()
-	parent = newParents.At(0)
-
-	// Perform rotations to move the node to delete to a leaf position while
-	// maintaining the min-heap while replacing the modified children.
-	var child *treapNode
-	newRoot := newParents.At(newParents.Len() - 1)
-	for delNode.left != nil || delNode.right != nil {
-		// Choose the child with the higher priority.
-		var isLeft bool
-		if delNode.left == nil {
-			child = delNode.right
-		} else if delNode.right == nil {
-			child = delNode.left
-			isLeft = true
-		} else if delNode.left.priority >= delNode.right.priority {
-			child = delNode.left
-			isLeft = true
-		} else {
-			child = delNode.right
-		}
-
-		// Rotate left or right depending on which side the child node
-		// is on. This has the effect of moving the node to delete
-		// towards the bottom of the tree while maintaining the
-		// min-heap.
-		child = cloneTreapNode(child)
-		if isLeft {
-			child.right, delNode.left = delNode, child.right
-		} else {
-			child.left, delNode.right = delNode, child.left
-		}
-
-		// Either set the new root of the tree when there is no
-		// grandparent or relink the grandparent to the node based on
-		// which side the old parent the node is replacing was on.
-		//
-		// Since the node to be deleted was just moved down a level, the
-		// new grandparent is now the current parent and the new parent
-		// is the current child.
-		if parent == nil {
-			newRoot = child
-		} else if parent.left == delNode {
-			parent.left = child
-		} else {
-			parent.right = child
-		}
-
-		// The parent for the node to delete is now what was previously
-		// its child.
-		parent = child
-	}
-
-	// Delete the node, which is now a leaf node, by disconnecting it from
-	// its parent.
-	if parent.right == delNode {
-		parent.right = nil
-	} else {
-		parent.left = nil
-	}
-
-	return newImmutable(newRoot, t.count-1, t.totalSize-nodeSize(delNode))
-}
-
-// ForEach invokes the passed function with every key/value pair in the treap
-// in ascending order.
-func (t *Immutable) ForEach(fn func(k, v []byte) bool) {
-	// Add the root node and all children to the left of it to the list of
-	// nodes to traverse and loop until they, and all of their child nodes,
-	// have been traversed.
-	var parents parentStack
-	for node := t.root; node != nil; node = node.left {
-		parents.Push(node)
-	}
-	for parents.Len() > 0 {
-		node := parents.Pop()
-		if !fn(node.key, node.value) {
-			return
-		}
-
-		// Extend the nodes to traverse by all children to the left of
-		// the current node's right child.
-		for node := node.right; node != nil; node = node.left {
-			parents.Push(node)
-		}
-	}
-}
-
-// NewImmutable returns a new empty immutable treap ready for use. See the
-// documentation for the Immutable structure for more details.
-func NewImmutable() *Immutable {
-	return &Immutable{}
-}

database/internal/treap/immutable_test.go

@@ -1,497 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"testing"
-)
-
-// TestImmutableEmpty ensures calling functions on an empty immutable treap
-// works as expected.
-func TestImmutableEmpty(t *testing.T) {
-	t.Parallel()
-
-	// Ensure the treap length is the expected value.
-	testTreap := NewImmutable()
-	if gotLen := testTreap.Len(); gotLen != 0 {
-		t.Fatalf("Len: unexpected length - got %d, want %d", gotLen, 0)
-	}
-
-	// Ensure the reported size is 0.
-	if gotSize := testTreap.Size(); gotSize != 0 {
-		t.Fatalf("Size: unexpected byte size - got %d, want 0",
-			gotSize)
-	}
-
-	// Ensure there are no errors with requesting keys from an empty treap.
-	key := serializeUint32(0)
-	if gotVal := testTreap.Has(key); gotVal {
-		t.Fatalf("Has: unexpected result - got %v, want false", gotVal)
-	}
-	if gotVal := testTreap.Get(key); gotVal != nil {
-		t.Fatalf("Get: unexpected result - got %x, want nil", gotVal)
-	}
-
-	// Ensure there are no panics when deleting keys from an empty treap.
-	testTreap.Delete(key)
-
-	// Ensure the number of keys iterated by ForEach on an empty treap is
-	// zero.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		numIterated++
-		return true
-	})
-	if numIterated != 0 {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want 0",
-			numIterated)
-	}
-}
-
-// TestImmutableSequential ensures that putting keys into an immutable treap in
-// sequential order works as expected.
-func TestImmutableSequential(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of sequential keys while checking several of the treap
-	// functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewImmutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Put(key, key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != i+1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i+1)
-		}
-
-		// Ensure the treap has the key.
-		if !testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap and ensure it is the expected
-		// value.
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize += (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-
-	// Ensure the all keys are iterated by ForEach in order.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		wantKey := serializeUint32(uint32(numIterated))
-
-		// Ensure the key is as expected.
-		if !bytes.Equal(k, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected key - got %x, want %x",
-				numIterated, k, wantKey)
-		}
-
-		// Ensure the value is as expected.
-		if !bytes.Equal(v, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected value - got %x, want %x",
-				numIterated, v, wantKey)
-		}
-
-		numIterated++
-		return true
-	})
-
-	// Ensure all items were iterated.
-	if numIterated != numItems {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want %d",
-			numIterated, numItems)
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Delete(key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != numItems-i-1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i-1)
-		}
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-}
-
-// TestImmutableReverseSequential ensures that putting keys into an immutable
-// treap in reverse sequential order works as expected.
-func TestImmutableReverseSequential(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of sequential keys while checking several of the treap
-	// functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewImmutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(numItems - i - 1))
-		testTreap = testTreap.Put(key, key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != i+1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i+1)
-		}
-
-		// Ensure the treap has the key.
-		if !testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap and ensure it is the expected
-		// value.
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize += (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-
-	// Ensure the all keys are iterated by ForEach in order.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		wantKey := serializeUint32(uint32(numIterated))
-
-		// Ensure the key is as expected.
-		if !bytes.Equal(k, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected key - got %x, want %x",
-				numIterated, k, wantKey)
-		}
-
-		// Ensure the value is as expected.
-		if !bytes.Equal(v, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected value - got %x, want %x",
-				numIterated, v, wantKey)
-		}
-
-		numIterated++
-		return true
-	})
-
-	// Ensure all items were iterated.
-	if numIterated != numItems {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want %d",
-			numIterated, numItems)
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		// Intentionally use the reverse order they were inserted here.
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Delete(key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != numItems-i-1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i-1)
-		}
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-}
-
-// TestImmutableUnordered ensures that putting keys into an immutable treap in
-// no paritcular order works as expected.
-func TestImmutableUnordered(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of out-of-order keys while checking several of the
-	// treap functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewImmutable()
-	for i := 0; i < numItems; i++ {
-		// Hash the serialized int to generate out-of-order keys.
-		hash := sha256.Sum256(serializeUint32(uint32(i)))
-		key := hash[:]
-		testTreap = testTreap.Put(key, key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != i+1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i+1)
-		}
-
-		// Ensure the treap has the key.
-		if !testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap and ensure it is the expected
-		// value.
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize += nodeFieldsSize + uint64(len(key)+len(key))
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		// Hash the serialized int to generate out-of-order keys.
-		hash := sha256.Sum256(serializeUint32(uint32(i)))
-		key := hash[:]
-		testTreap = testTreap.Delete(key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != numItems-i-1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i-1)
-		}
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= (nodeFieldsSize + 64)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-}
-
-// TestImmutableDuplicatePut ensures that putting a duplicate key into an
-// immutable treap works as expected.
-func TestImmutableDuplicatePut(t *testing.T) {
-	t.Parallel()
-
-	expectedVal := []byte("testval")
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewImmutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Put(key, key)
-		expectedSize += nodeFieldsSize + uint64(len(key)+len(key))
-
-		// Put a duplicate key with the the expected final value.
-		testTreap = testTreap.Put(key, expectedVal)
-
-		// Ensure the key still exists and is the new value.
-		if gotVal := testTreap.Has(key); !gotVal {
-			t.Fatalf("Has: unexpected result - got %v, want true",
-				gotVal)
-		}
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, expectedVal) {
-			t.Fatalf("Get: unexpected result - got %x, want %x",
-				gotVal, expectedVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= uint64(len(key))
-		expectedSize += uint64(len(expectedVal))
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size: unexpected byte size - got %d, want %d",
-				gotSize, expectedSize)
-		}
-	}
-}
-
-// TestImmutableNilValue ensures that putting a nil value into an immutable
-// treap results in a key being added with an empty byte slice.
-func TestImmutableNilValue(t *testing.T) {
-	t.Parallel()
-
-	key := serializeUint32(0)
-
-	// Put the key with a nil value.
-	testTreap := NewImmutable()
-	testTreap = testTreap.Put(key, nil)
-
-	// Ensure the key exists and is an empty byte slice.
-	if gotVal := testTreap.Has(key); !gotVal {
-		t.Fatalf("Has: unexpected result - got %v, want true", gotVal)
-	}
-	if gotVal := testTreap.Get(key); gotVal == nil {
-		t.Fatalf("Get: unexpected result - got nil, want empty slice")
-	}
-	if gotVal := testTreap.Get(key); len(gotVal) != 0 {
-		t.Fatalf("Get: unexpected result - got %x, want empty slice",
-			gotVal)
-	}
-}
-
-// TestImmutableForEachStopIterator ensures that returning false from the ForEach
-// callback on an immutable treap stops iteration early.
-func TestImmutableForEachStopIterator(t *testing.T) {
-	t.Parallel()
-
-	// Insert a few keys.
-	numItems := 10
-	testTreap := NewImmutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Put(key, key)
-	}
-
-	// Ensure ForEach exits early on false return by caller.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		numIterated++
-		return numIterated != numItems/2
-	})
-	if numIterated != numItems/2 {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want %d",
-			numIterated, numItems/2)
-	}
-}
-
-// TestImmutableSnapshot ensures that immutable treaps are actually immutable by
-// keeping a reference to the previous treap, performing a mutation, and then
-// ensuring the referenced treap does not have the mutation applied.
-func TestImmutableSnapshot(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of sequential keys while checking several of the treap
-	// functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewImmutable()
-	for i := 0; i < numItems; i++ {
-		treapSnap := testTreap
-
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Put(key, key)
-
-		// Ensure the length of the treap snapshot is the expected
-		// value.
-		if gotLen := treapSnap.Len(); gotLen != i {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i)
-		}
-
-		// Ensure the treap snapshot does not have the key.
-		if treapSnap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that doesn't exist in the treap snapshot and
-		// ensure it is nil.
-		if gotVal := treapSnap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		if gotSize := treapSnap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-		expectedSize += (nodeFieldsSize + 8)
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		treapSnap := testTreap
-
-		key := serializeUint32(uint32(i))
-		testTreap = testTreap.Delete(key)
-
-		// Ensure the length of the treap snapshot is the expected
-		// value.
-		if gotLen := treapSnap.Len(); gotLen != numItems-i {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i)
-		}
-
-		// Ensure the treap snapshot still has the key.
-		if !treapSnap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap snapshot and ensure it is still
-		// the expected value.
-		if gotVal := treapSnap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		if gotSize := treapSnap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-		expectedSize -= (nodeFieldsSize + 8)
-	}
-}

database/internal/treap/mutable.go

@@ -1,278 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"bytes"
-	"math/rand"
-)
-
-// Mutable represents a treap data structure which is used to hold ordered
-// key/value pairs using a combination of binary search tree and heap semantics.
-// It is a self-organizing and randomized data structure that doesn't require
-// complex operations to maintain balance. Search, insert, and delete
-// operations are all O(log n).
-type Mutable struct {
-	root  *treapNode
-	count int
-
-	// totalSize is the best estimate of the total size of of all data in
-	// the treap including the keys, values, and node sizes.
-	totalSize uint64
-}
-
-// Len returns the number of items stored in the treap.
-func (t *Mutable) Len() int {
-	return t.count
-}
-
-// Size returns a best estimate of the total number of bytes the treap is
-// consuming including all of the fields used to represent the nodes as well as
-// the size of the keys and values. Shared values are not detected, so the
-// returned size assumes each value is pointing to different memory.
-func (t *Mutable) Size() uint64 {
-	return t.totalSize
-}
-
-// get returns the treap node that contains the passed key and its parent. When
-// the found node is the root of the tree, the parent will be nil. When the key
-// does not exist, both the node and the parent will be nil.
-func (t *Mutable) get(key []byte) (*treapNode, *treapNode) {
-	var parent *treapNode
-	for node := t.root; node != nil; {
-		// Traverse left or right depending on the result of the
-		// comparison.
-		compareResult := bytes.Compare(key, node.key)
-		if compareResult < 0 {
-			parent = node
-			node = node.left
-			continue
-		}
-		if compareResult > 0 {
-			parent = node
-			node = node.right
-			continue
-		}
-
-		// The key exists.
-		return node, parent
-	}
-
-	// A nil node was reached which means the key does not exist.
-	return nil, nil
-}
-
-// Has returns whether or not the passed key exists.
-func (t *Mutable) Has(key []byte) bool {
-	if node, _ := t.get(key); node != nil {
-		return true
-	}
-	return false
-}
-
-// Get returns the value for the passed key. The function will return nil when
-// the key does not exist.
-func (t *Mutable) Get(key []byte) []byte {
-	if node, _ := t.get(key); node != nil {
-		return node.value
-	}
-	return nil
-}
-
-// relinkGrandparent relinks the node into the treap after it has been rotated
-// by changing the passed grandparent's left or right pointer, depending on
-// where the old parent was, to point at the passed node. Otherwise, when there
-// is no grandparent, it means the node is now the root of the tree, so update
-// it accordingly.
-func (t *Mutable) relinkGrandparent(node, parent, grandparent *treapNode) {
-	// The node is now the root of the tree when there is no grandparent.
-	if grandparent == nil {
-		t.root = node
-		return
-	}
-
-	// Relink the grandparent's left or right pointer based on which side
-	// the old parent was.
-	if grandparent.left == parent {
-		grandparent.left = node
-	} else {
-		grandparent.right = node
-	}
-}
-
-// Put inserts the passed key/value pair.
-func (t *Mutable) Put(key, value []byte) {
-	// Use an empty byte slice for the value when none was provided. This
-	// ultimately allows key existence to be determined from the value since
-	// an empty byte slice is distinguishable from nil.
-	if value == nil {
-		value = emptySlice
-	}
-
-	// The node is the root of the tree if there isn't already one.
-	if t.root == nil {
-		node := newTreapNode(key, value, rand.Int())
-		t.count = 1
-		t.totalSize = nodeSize(node)
-		t.root = node
-		return
-	}
-
-	// Find the binary tree insertion point and construct a list of parents
-	// while doing so. When the key matches an entry already in the treap,
-	// just update its value and return.
-	var parents parentStack
-	var compareResult int
-	for node := t.root; node != nil; {
-		parents.Push(node)
-		compareResult = bytes.Compare(key, node.key)
-		if compareResult < 0 {
-			node = node.left
-			continue
-		}
-		if compareResult > 0 {
-			node = node.right
-			continue
-		}
-
-		// The key already exists, so update its value.
-		t.totalSize -= uint64(len(node.value))
-		t.totalSize += uint64(len(value))
-		node.value = value
-		return
-	}
-
-	// Link the new node into the binary tree in the correct position.
-	node := newTreapNode(key, value, rand.Int())
-	t.count++
-	t.totalSize += nodeSize(node)
-	parent := parents.At(0)
-	if compareResult < 0 {
-		parent.left = node
-	} else {
-		parent.right = node
-	}
-
-	// Perform any rotations needed to maintain the min-heap.
-	for parents.Len() > 0 {
-		// There is nothing left to do when the node's priority is
-		// greater than or equal to its parent's priority.
-		parent = parents.Pop()
-		if node.priority >= parent.priority {
-			break
-		}
-
-		// Perform a right rotation if the node is on the left side or
-		// a left rotation if the node is on the right side.
-		if parent.left == node {
-			node.right, parent.left = parent, node.right
-		} else {
-			node.left, parent.right = parent, node.left
-		}
-		t.relinkGrandparent(node, parent, parents.At(0))
-	}
-}
-
-// Delete removes the passed key if it exists.
-func (t *Mutable) Delete(key []byte) {
-	// Find the node for the key along with its parent. There is nothing to
-	// do if the key does not exist.
-	node, parent := t.get(key)
-	if node == nil {
-		return
-	}
-
-	// When the only node in the tree is the root node and it is the one
-	// being deleted, there is nothing else to do besides removing it.
-	if parent == nil && node.left == nil && node.right == nil {
-		t.root = nil
-		t.count = 0
-		t.totalSize = 0
-		return
-	}
-
-	// Perform rotations to move the node to delete to a leaf position while
-	// maintaining the min-heap.
-	var isLeft bool
-	var child *treapNode
-	for node.left != nil || node.right != nil {
-		// Choose the child with the higher priority.
-		if node.left == nil {
-			child = node.right
-			isLeft = false
-		} else if node.right == nil {
-			child = node.left
-			isLeft = true
-		} else if node.left.priority >= node.right.priority {
-			child = node.left
-			isLeft = true
-		} else {
-			child = node.right
-			isLeft = false
-		}
-
-		// Rotate left or right depending on which side the child node
-		// is on. This has the effect of moving the node to delete
-		// towards the bottom of the tree while maintaining the
-		// min-heap.
-		if isLeft {
-			child.right, node.left = node, child.right
-		} else {
-			child.left, node.right = node, child.left
-		}
-		t.relinkGrandparent(child, node, parent)
-
-		// The parent for the node to delete is now what was previously
-		// its child.
-		parent = child
-	}
-
-	// Delete the node, which is now a leaf node, by disconnecting it from
-	// its parent.
-	if parent.right == node {
-		parent.right = nil
-	} else {
-		parent.left = nil
-	}
-	t.count--
-	t.totalSize -= nodeSize(node)
-}
-
-// ForEach invokes the passed function with every key/value pair in the treap
-// in ascending order.
-func (t *Mutable) ForEach(fn func(k, v []byte) bool) {
-	// Add the root node and all children to the left of it to the list of
-	// nodes to traverse and loop until they, and all of their child nodes,
-	// have been traversed.
-	var parents parentStack
-	for node := t.root; node != nil; node = node.left {
-		parents.Push(node)
-	}
-	for parents.Len() > 0 {
-		node := parents.Pop()
-		if !fn(node.key, node.value) {
-			return
-		}
-
-		// Extend the nodes to traverse by all children to the left of
-		// the current node's right child.
-		for node := node.right; node != nil; node = node.left {
-			parents.Push(node)
-		}
-	}
-}
-
-// Reset efficiently removes all items in the treap.
-func (t *Mutable) Reset() {
-	t.count = 0
-	t.totalSize = 0
-	t.root = nil
-}
-
-// NewMutable returns a new empty mutable treap ready for use. See the
-// documentation for the Mutable structure for more details.
-func NewMutable() *Mutable {
-	return &Mutable{}
-}

database/internal/treap/mutable_test.go

@@ -1,465 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"testing"
-)
-
-// TestMutableEmpty ensures calling functions on an empty mutable treap works as
-// expected.
-func TestMutableEmpty(t *testing.T) {
-	t.Parallel()
-
-	// Ensure the treap length is the expected value.
-	testTreap := NewMutable()
-	if gotLen := testTreap.Len(); gotLen != 0 {
-		t.Fatalf("Len: unexpected length - got %d, want %d", gotLen, 0)
-	}
-
-	// Ensure the reported size is 0.
-	if gotSize := testTreap.Size(); gotSize != 0 {
-		t.Fatalf("Size: unexpected byte size - got %d, want 0",
-			gotSize)
-	}
-
-	// Ensure there are no errors with requesting keys from an empty treap.
-	key := serializeUint32(0)
-	if gotVal := testTreap.Has(key); gotVal {
-		t.Fatalf("Has: unexpected result - got %v, want false", gotVal)
-	}
-	if gotVal := testTreap.Get(key); gotVal != nil {
-		t.Fatalf("Get: unexpected result - got %x, want nil", gotVal)
-	}
-
-	// Ensure there are no panics when deleting keys from an empty treap.
-	testTreap.Delete(key)
-
-	// Ensure the number of keys iterated by ForEach on an empty treap is
-	// zero.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		numIterated++
-		return true
-	})
-	if numIterated != 0 {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want 0",
-			numIterated)
-	}
-}
-
-// TestMutableReset ensures that resetting an existing mutable treap works as
-// expected.
-func TestMutableReset(t *testing.T) {
-	t.Parallel()
-
-	// Insert a few keys.
-	numItems := 10
-	testTreap := NewMutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap.Put(key, key)
-	}
-
-	// Reset it.
-	testTreap.Reset()
-
-	// Ensure the treap length is now 0.
-	if gotLen := testTreap.Len(); gotLen != 0 {
-		t.Fatalf("Len: unexpected length - got %d, want %d", gotLen, 0)
-	}
-
-	// Ensure the reported size is now 0.
-	if gotSize := testTreap.Size(); gotSize != 0 {
-		t.Fatalf("Size: unexpected byte size - got %d, want 0",
-			gotSize)
-	}
-
-	// Ensure the treap no longer has any of the keys.
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-	}
-
-	// Ensure the number of keys iterated by ForEach is zero.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		numIterated++
-		return true
-	})
-	if numIterated != 0 {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want 0",
-			numIterated)
-	}
-}
-
-// TestMutableSequential ensures that putting keys into a mutable treap in
-// sequential order works as expected.
-func TestMutableSequential(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of sequential keys while checking several of the treap
-	// functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewMutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap.Put(key, key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != i+1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i+1)
-		}
-
-		// Ensure the treap has the key.
-		if !testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap and ensure it is the expected
-		// value.
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize += (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-
-	// Ensure the all keys are iterated by ForEach in order.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		wantKey := serializeUint32(uint32(numIterated))
-
-		// Ensure the key is as expected.
-		if !bytes.Equal(k, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected key - got %x, want %x",
-				numIterated, k, wantKey)
-		}
-
-		// Ensure the value is as expected.
-		if !bytes.Equal(v, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected value - got %x, want %x",
-				numIterated, v, wantKey)
-		}
-
-		numIterated++
-		return true
-	})
-
-	// Ensure all items were iterated.
-	if numIterated != numItems {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want %d",
-			numIterated, numItems)
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap.Delete(key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != numItems-i-1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i-1)
-		}
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-}
-
-// TestMutableReverseSequential ensures that putting keys into a mutable treap
-// in reverse sequential order works as expected.
-func TestMutableReverseSequential(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of sequential keys while checking several of the treap
-	// functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewMutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(numItems - i - 1))
-		testTreap.Put(key, key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != i+1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i+1)
-		}
-
-		// Ensure the treap has the key.
-		if !testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap and ensure it is the expected
-		// value.
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize += (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-
-	// Ensure the all keys are iterated by ForEach in order.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		wantKey := serializeUint32(uint32(numIterated))
-
-		// Ensure the key is as expected.
-		if !bytes.Equal(k, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected key - got %x, want %x",
-				numIterated, k, wantKey)
-		}
-
-		// Ensure the value is as expected.
-		if !bytes.Equal(v, wantKey) {
-			t.Fatalf("ForEach #%d: unexpected value - got %x, want %x",
-				numIterated, v, wantKey)
-		}
-
-		numIterated++
-		return true
-	})
-
-	// Ensure all items were iterated.
-	if numIterated != numItems {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want %d",
-			numIterated, numItems)
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		// Intentionally use the reverse order they were inserted here.
-		key := serializeUint32(uint32(i))
-		testTreap.Delete(key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != numItems-i-1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i-1)
-		}
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= (nodeFieldsSize + 8)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-}
-
-// TestMutableUnordered ensures that putting keys into a mutable treap in no
-// paritcular order works as expected.
-func TestMutableUnordered(t *testing.T) {
-	t.Parallel()
-
-	// Insert a bunch of out-of-order keys while checking several of the
-	// treap functions work as expected.
-	expectedSize := uint64(0)
-	numItems := 1000
-	testTreap := NewMutable()
-	for i := 0; i < numItems; i++ {
-		// Hash the serialized int to generate out-of-order keys.
-		hash := sha256.Sum256(serializeUint32(uint32(i)))
-		key := hash[:]
-		testTreap.Put(key, key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != i+1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, i+1)
-		}
-
-		// Ensure the treap has the key.
-		if !testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is not in treap", i, key)
-		}
-
-		// Get the key from the treap and ensure it is the expected
-		// value.
-		if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, key) {
-			t.Fatalf("Get #%d: unexpected value - got %x, want %x",
-				i, gotVal, key)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize += nodeFieldsSize + uint64(len(key)+len(key))
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-
-	// Delete the keys one-by-one while checking several of the treap
-	// functions work as expected.
-	for i := 0; i < numItems; i++ {
-		// Hash the serialized int to generate out-of-order keys.
-		hash := sha256.Sum256(serializeUint32(uint32(i)))
-		key := hash[:]
-		testTreap.Delete(key)
-
-		// Ensure the treap length is the expected value.
-		if gotLen := testTreap.Len(); gotLen != numItems-i-1 {
-			t.Fatalf("Len #%d: unexpected length - got %d, want %d",
-				i, gotLen, numItems-i-1)
-		}
-
-		// Ensure the treap no longer has the key.
-		if testTreap.Has(key) {
-			t.Fatalf("Has #%d: key %q is in treap", i, key)
-		}
-
-		// Get the key that no longer exists from the treap and ensure
-		// it is nil.
-		if gotVal := testTreap.Get(key); gotVal != nil {
-			t.Fatalf("Get #%d: unexpected value - got %x, want nil",
-				i, gotVal)
-		}
-
-		// Ensure the expected size is reported.
-		expectedSize -= (nodeFieldsSize + 64)
-		if gotSize := testTreap.Size(); gotSize != expectedSize {
-			t.Fatalf("Size #%d: unexpected byte size - got %d, "+
-				"want %d", i, gotSize, expectedSize)
-		}
-	}
-}
-
-// TestMutableDuplicatePut ensures that putting a duplicate key into a mutable
-// treap updates the existing value.
-func TestMutableDuplicatePut(t *testing.T) {
-	t.Parallel()
-
-	key := serializeUint32(0)
-	val := []byte("testval")
-
-	// Put the key twice with the second put being the expected final value.
-	testTreap := NewMutable()
-	testTreap.Put(key, key)
-	testTreap.Put(key, val)
-
-	// Ensure the key still exists and is the new value.
-	if gotVal := testTreap.Has(key); !gotVal {
-		t.Fatalf("Has: unexpected result - got %v, want true", gotVal)
-	}
-	if gotVal := testTreap.Get(key); !bytes.Equal(gotVal, val) {
-		t.Fatalf("Get: unexpected result - got %x, want %x", gotVal, val)
-	}
-
-	// Ensure the expected size is reported.
-	expectedSize := uint64(nodeFieldsSize + len(key) + len(val))
-	if gotSize := testTreap.Size(); gotSize != expectedSize {
-		t.Fatalf("Size: unexpected byte size - got %d, want %d",
-			gotSize, expectedSize)
-	}
-}
-
-// TestMutableNilValue ensures that putting a nil value into a mutable treap
-// results in a key being added with an empty byte slice.
-func TestMutableNilValue(t *testing.T) {
-	t.Parallel()
-
-	key := serializeUint32(0)
-
-	// Put the key with a nil value.
-	testTreap := NewMutable()
-	testTreap.Put(key, nil)
-
-	// Ensure the key exists and is an empty byte slice.
-	if gotVal := testTreap.Has(key); !gotVal {
-		t.Fatalf("Has: unexpected result - got %v, want true", gotVal)
-	}
-	if gotVal := testTreap.Get(key); gotVal == nil {
-		t.Fatalf("Get: unexpected result - got nil, want empty slice")
-	}
-	if gotVal := testTreap.Get(key); len(gotVal) != 0 {
-		t.Fatalf("Get: unexpected result - got %x, want empty slice",
-			gotVal)
-	}
-}
-
-// TestMutableForEachStopIterator ensures that returning false from the ForEach
-// callback of a mutable treap stops iteration early.
-func TestMutableForEachStopIterator(t *testing.T) {
-	t.Parallel()
-
-	// Insert a few keys.
-	numItems := 10
-	testTreap := NewMutable()
-	for i := 0; i < numItems; i++ {
-		key := serializeUint32(uint32(i))
-		testTreap.Put(key, key)
-	}
-
-	// Ensure ForEach exits early on false return by caller.
-	var numIterated int
-	testTreap.ForEach(func(k, v []byte) bool {
-		numIterated++
-		return numIterated != numItems/2
-	})
-	if numIterated != numItems/2 {
-		t.Fatalf("ForEach: unexpected iterate count - got %d, want %d",
-			numIterated, numItems/2)
-	}
-}

database/internal/treap/treapiter.go

@@ -1,354 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import "bytes"
-
-// Iterator represents an iterator for forwards and backwards iteration over
-// the contents of a treap (mutable or immutable).
-type Iterator struct {
-	t        *Mutable    // Mutable treap iterator is associated with or nil
-	root     *treapNode  // Root node of treap iterator is associated with
-	node     *treapNode  // The node the iterator is positioned at
-	parents  parentStack // The stack of parents needed to iterate
-	isNew    bool        // Whether the iterator has been positioned
-	seekKey  []byte      // Used to handle dynamic updates for mutable treap
-	startKey []byte      // Used to limit the iterator to a range
-	limitKey []byte      // Used to limit the iterator to a range
-}
-
-// limitIterator clears the current iterator node if it is outside of the range
-// specified when the iterator was created. It returns whether the iterator is
-// valid.
-func (iter *Iterator) limitIterator() bool {
-	if iter.node == nil {
-		return false
-	}
-
-	node := iter.node
-	if iter.startKey != nil && bytes.Compare(node.key, iter.startKey) < 0 {
-		iter.node = nil
-		return false
-	}
-
-	if iter.limitKey != nil && bytes.Compare(node.key, iter.limitKey) >= 0 {
-		iter.node = nil
-		return false
-	}
-
-	return true
-}
-
-// seek moves the iterator based on the provided key and flags.
-//
-// When the exact match flag is set, the iterator will either be moved to first
-// key in the treap that exactly matches the provided key, or the one
-// before/after it depending on the greater flag.
-//
-// When the exact match flag is NOT set, the iterator will be moved to the first
-// key in the treap before/after the provided key depending on the greater flag.
-//
-// In all cases, the limits specified when the iterator was created are
-// respected.
-func (iter *Iterator) seek(key []byte, exactMatch bool, greater bool) bool {
-	iter.node = nil
-	iter.parents = parentStack{}
-	var selectedNodeDepth int
-	for node := iter.root; node != nil; {
-		iter.parents.Push(node)
-
-		// Traverse left or right depending on the result of the
-		// comparison. Also, set the iterator to the node depending on
-		// the flags so the iterator is positioned properly when an
-		// exact match isn't found.
-		compareResult := bytes.Compare(key, node.key)
-		if compareResult < 0 {
-			if greater {
-				iter.node = node
-				selectedNodeDepth = iter.parents.Len() - 1
-			}
-			node = node.left
-			continue
-		}
-		if compareResult > 0 {
-			if !greater {
-				iter.node = node
-				selectedNodeDepth = iter.parents.Len() - 1
-			}
-			node = node.right
-			continue
-		}
-
-		// The key is an exact match. Set the iterator and return now
-		// when the exact match flag is set.
-		if exactMatch {
-			iter.node = node
-			iter.parents.Pop()
-			return iter.limitIterator()
-		}
-
-		// The key is an exact match, but the exact match is not set, so
-		// choose which direction to go based on whether the larger or
-		// smaller key was requested.
-		if greater {
-			node = node.right
-		} else {
-			node = node.left
-		}
-	}
-
-	// There was either no exact match or there was an exact match but the
-	// exact match flag was not set. In any case, the parent stack might
-	// need to be adjusted to only include the parents up to the selected
-	// node. Also, ensure the selected node's key does not exceed the
-	// allowed range of the iterator.
-	for i := iter.parents.Len(); i > selectedNodeDepth; i-- {
-		iter.parents.Pop()
-	}
-	return iter.limitIterator()
-}
-
-// First moves the iterator to the first key/value pair. When there is only a
-// single key/value pair both First and Last will point to the same pair.
-// Returns false if there are no key/value pairs.
-func (iter *Iterator) First() bool {
-	// Seek the start key if the iterator was created with one. This will
-	// result in either an exact match, the first greater key, or an
-	// exhausted iterator if no such key exists.
-	iter.isNew = false
-	if iter.startKey != nil {
-		return iter.seek(iter.startKey, true, true)
-	}
-
-	// The smallest key is in the left-most node.
-	iter.parents = parentStack{}
-	for node := iter.root; node != nil; node = node.left {
-		if node.left == nil {
-			iter.node = node
-			return true
-		}
-		iter.parents.Push(node)
-	}
-	return false
-}
-
-// Last moves the iterator to the last key/value pair. When there is only a
-// single key/value pair both First and Last will point to the same pair.
-// Returns false if there are no key/value pairs.
-func (iter *Iterator) Last() bool {
-	// Seek the limit key if the iterator was created with one. This will
-	// result in the first key smaller than the limit key, or an exhausted
-	// iterator if no such key exists.
-	iter.isNew = false
-	if iter.limitKey != nil {
-		return iter.seek(iter.limitKey, false, false)
-	}
-
-	// The highest key is in the right-most node.
-	iter.parents = parentStack{}
-	for node := iter.root; node != nil; node = node.right {
-		if node.right == nil {
-			iter.node = node
-			return true
-		}
-		iter.parents.Push(node)
-	}
-	return false
-}
-
-// Next moves the iterator to the next key/value pair and returns false when the
-// iterator is exhausted. When invoked on a newly created iterator it will
-// position the iterator at the first item.
-func (iter *Iterator) Next() bool {
-	if iter.isNew {
-		return iter.First()
-	}
-
-	if iter.node == nil {
-		return false
-	}
-
-	// Reseek the previous key without allowing for an exact match if a
-	// force seek was requested. This results in the key greater than the
-	// previous one or an exhausted iterator if there is no such key.
-	if seekKey := iter.seekKey; seekKey != nil {
-		iter.seekKey = nil
-		return iter.seek(seekKey, false, true)
-	}
-
-	// When there is no right node walk the parents until the parent's right
-	// node is not equal to the previous child. This will be the next node.
-	if iter.node.right == nil {
-		parent := iter.parents.Pop()
-		for parent != nil && parent.right == iter.node {
-			iter.node = parent
-			parent = iter.parents.Pop()
-		}
-		iter.node = parent
-		return iter.limitIterator()
-	}
-
-	// There is a right node, so the next node is the left-most node down
-	// the right sub-tree.
-	iter.parents.Push(iter.node)
-	iter.node = iter.node.right
-	for node := iter.node.left; node != nil; node = node.left {
-		iter.parents.Push(iter.node)
-		iter.node = node
-	}
-	return iter.limitIterator()
-}
-
-// Prev moves the iterator to the previous key/value pair and returns false when
-// the iterator is exhausted. When invoked on a newly created iterator it will
-// position the iterator at the last item.
-func (iter *Iterator) Prev() bool {
-	if iter.isNew {
-		return iter.Last()
-	}
-
-	if iter.node == nil {
-		return false
-	}
-
-	// Reseek the previous key without allowing for an exact match if a
-	// force seek was requested. This results in the key smaller than the
-	// previous one or an exhausted iterator if there is no such key.
-	if seekKey := iter.seekKey; seekKey != nil {
-		iter.seekKey = nil
-		return iter.seek(seekKey, false, false)
-	}
-
-	// When there is no left node walk the parents until the parent's left
-	// node is not equal to the previous child. This will be the previous
-	// node.
-	for iter.node.left == nil {
-		parent := iter.parents.Pop()
-		for parent != nil && parent.left == iter.node {
-			iter.node = parent
-			parent = iter.parents.Pop()
-		}
-		iter.node = parent
-		return iter.limitIterator()
-	}
-
-	// There is a left node, so the previous node is the right-most node
-	// down the left sub-tree.
-	iter.parents.Push(iter.node)
-	iter.node = iter.node.left
-	for node := iter.node.right; node != nil; node = node.right {
-		iter.parents.Push(iter.node)
-		iter.node = node
-	}
-	return iter.limitIterator()
-}
-
-// Seek moves the iterator to the first key/value pair with a key that is
-// greater than or equal to the given key and returns true if successful.
-func (iter *Iterator) Seek(key []byte) bool {
-	iter.isNew = false
-	return iter.seek(key, true, true)
-}
-
-// Key returns the key of the current key/value pair or nil when the iterator
-// is exhausted. The caller should not modify the contents of the returned
-// slice.
-func (iter *Iterator) Key() []byte {
-	if iter.node == nil {
-		return nil
-	}
-	return iter.node.key
-}
-
-// Value returns the value of the current key/value pair or nil when the
-// iterator is exhausted. The caller should not modify the contents of the
-// returned slice.
-func (iter *Iterator) Value() []byte {
-	if iter.node == nil {
-		return nil
-	}
-	return iter.node.value
-}
-
-// Valid indicates whether the iterator is positioned at a valid key/value pair.
-// It will be considered invalid when the iterator is newly created or exhausted.
-func (iter *Iterator) Valid() bool {
-	return iter.node != nil
-}
-
-// ForceReseek notifies the iterator that the underlying mutable treap has been
-// updated, so the next call to Prev or Next needs to reseek in order to allow
-// the iterator to continue working properly.
-//
-// NOTE: Calling this function when the iterator is associated with an immutable
-// treap has no effect as you would expect.
-func (iter *Iterator) ForceReseek() {
-	// Nothing to do when the iterator is associated with an immutable
-	// treap.
-	if iter.t == nil {
-		return
-	}
-
-	// Update the iterator root to the mutable treap root in case it
-	// changed.
-	iter.root = iter.t.root
-
-	// Set the seek key to the current node. This will force the Next/Prev
-	// functions to reseek, and thus properly reconstruct the iterator, on
-	// their next call.
-	if iter.node == nil {
-		iter.seekKey = nil
-		return
-	}
-	iter.seekKey = iter.node.key
-}
-
-// Iterator returns a new iterator for the mutable treap. The newly returned
-// iterator is not pointing to a valid item until a call to one of the methods
-// to position it is made.
-//
-// The start key and limit key parameters cause the iterator to be limited to
-// a range of keys. The start key is inclusive and the limit key is exclusive.
-// Either or both can be nil if the functionality is not desired.
-//
-// WARNING: The ForceSeek method must be called on the returned iterator if
-// the treap is mutated. Failure to do so will cause the iterator to return
-// unexpected keys and/or values.
-//
-// For example:
-//   iter := t.Iterator(nil, nil)
-//   for iter.Next() {
-//   	if someCondition {
-//   		t.Delete(iter.Key())
-//   		iter.ForceReseek()
-//   	}
-//   }
-func (t *Mutable) Iterator(startKey, limitKey []byte) *Iterator {
-	iter := &Iterator{
-		t:        t,
-		root:     t.root,
-		isNew:    true,
-		startKey: startKey,
-		limitKey: limitKey,
-	}
-	return iter
-}
-
-// Iterator returns a new iterator for the immutable treap. The newly returned
-// iterator is not pointing to a valid item until a call to one of the methods
-// to position it is made.
-//
-// The start key and limit key parameters cause the iterator to be limited to
-// a range of keys. The start key is inclusive and the limit key is exclusive.
-// Either or both can be nil if the functionality is not desired.
-func (t *Immutable) Iterator(startKey, limitKey []byte) *Iterator {
-	iter := &Iterator{
-		root:     t.root,
-		isNew:    true,
-		startKey: startKey,
-		limitKey: limitKey,
-	}
-	return iter
-}

database/internal/treap/treapiter_test.go

@@ -1,719 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package treap
-
-import (
-	"bytes"
-	"encoding/binary"
-	"testing"
-)
-
-// TestMutableIterator ensures that the general behavior of mutable treap
-// iterators is as expected including tests for first, last, ordered and reverse
-// ordered iteration, limiting the range, seeking, and initially unpositioned.
-func TestMutableIterator(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		numKeys       int
-		step          int
-		startKey      []byte
-		limitKey      []byte
-		expectedFirst []byte
-		expectedLast  []byte
-		seekKey       []byte
-		expectedSeek  []byte
-	}{
-		// No range limits. Values are the set (0, 1, 2, ..., 49).
-		// Seek existing value.
-		{
-			numKeys:       50,
-			step:          1,
-			expectedFirst: serializeUint32(0),
-			expectedLast:  serializeUint32(49),
-			seekKey:       serializeUint32(12),
-			expectedSeek:  serializeUint32(12),
-		},
-
-		// Limited to range [24, end]. Values are the set
-		// (0, 2, 4, ..., 48). Seek value that doesn't exist and is
-		// greater than largest existing key.
-		{
-			numKeys:       50,
-			step:          2,
-			startKey:      serializeUint32(24),
-			expectedFirst: serializeUint32(24),
-			expectedLast:  serializeUint32(48),
-			seekKey:       serializeUint32(49),
-			expectedSeek:  nil,
-		},
-
-		// Limited to range [start, 25). Values are the set
-		// (0, 3, 6, ..., 48). Seek value that doesn't exist but is
-		// before an existing value within the range.
-		{
-			numKeys:       50,
-			step:          3,
-			limitKey:      serializeUint32(25),
-			expectedFirst: serializeUint32(0),
-			expectedLast:  serializeUint32(24),
-			seekKey:       serializeUint32(17),
-			expectedSeek:  serializeUint32(18),
-		},
-
-		// Limited to range [10, 21). Values are the set
-		// (0, 4, ..., 48). Seek value that exists, but is before the
-		// minimum allowed range.
-		{
-			numKeys:       50,
-			step:          4,
-			startKey:      serializeUint32(10),
-			limitKey:      serializeUint32(21),
-			expectedFirst: serializeUint32(12),
-			expectedLast:  serializeUint32(20),
-			seekKey:       serializeUint32(4),
-			expectedSeek:  nil,
-		},
-
-		// Limited by prefix {0,0,0}, range [{0,0,0}, {0,0,1}).
-		// Since it's a bytewise compare,  {0,0,0,...} < {0,0,1}.
-		// Seek existing value within the allowed range.
-		{
-			numKeys:       300,
-			step:          1,
-			startKey:      []byte{0x00, 0x00, 0x00},
-			limitKey:      []byte{0x00, 0x00, 0x01},
-			expectedFirst: serializeUint32(0),
-			expectedLast:  serializeUint32(255),
-			seekKey:       serializeUint32(100),
-			expectedSeek:  serializeUint32(100),
-		},
-	}
-
-testLoop:
-	for i, test := range tests {
-		// Insert a bunch of keys.
-		testTreap := NewMutable()
-		for i := 0; i < test.numKeys; i += test.step {
-			key := serializeUint32(uint32(i))
-			testTreap.Put(key, key)
-		}
-
-		// Create new iterator limited by the test params.
-		iter := testTreap.Iterator(test.startKey, test.limitKey)
-
-		// Ensure the first item is accurate.
-		hasFirst := iter.First()
-		if !hasFirst && test.expectedFirst != nil {
-			t.Errorf("First #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey := iter.Key()
-		if !bytes.Equal(gotKey, test.expectedFirst) {
-			t.Errorf("First.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedFirst)
-			continue
-		}
-		gotVal := iter.Value()
-		if !bytes.Equal(gotVal, test.expectedFirst) {
-			t.Errorf("First.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedFirst)
-			continue
-		}
-
-		// Ensure the iterator gives the expected items in order.
-		curNum := binary.BigEndian.Uint32(test.expectedFirst)
-		for iter.Next() {
-			curNum += uint32(test.step)
-
-			// Ensure key is as expected.
-			gotKey := iter.Key()
-			expectedKey := serializeUint32(curNum)
-			if !bytes.Equal(gotKey, expectedKey) {
-				t.Errorf("iter.Key #%d (%d): unexpected key - "+
-					"got %x, want %x", i, curNum, gotKey,
-					expectedKey)
-				continue testLoop
-			}
-
-			// Ensure value is as expected.
-			gotVal := iter.Value()
-			if !bytes.Equal(gotVal, expectedKey) {
-				t.Errorf("iter.Value #%d (%d): unexpected "+
-					"value - got %x, want %x", i, curNum,
-					gotVal, expectedKey)
-				continue testLoop
-			}
-		}
-
-		// Ensure iterator is exhausted.
-		if iter.Valid() {
-			t.Errorf("Valid #%d: iterator should be exhausted", i)
-			continue
-		}
-
-		// Ensure the last item is accurate.
-		hasLast := iter.Last()
-		if !hasLast && test.expectedLast != nil {
-			t.Errorf("Last #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedLast) {
-			t.Errorf("Last.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedLast)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedLast) {
-			t.Errorf("Last.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedLast)
-			continue
-		}
-
-		// Ensure the iterator gives the expected items in reverse
-		// order.
-		curNum = binary.BigEndian.Uint32(test.expectedLast)
-		for iter.Prev() {
-			curNum -= uint32(test.step)
-
-			// Ensure key is as expected.
-			gotKey := iter.Key()
-			expectedKey := serializeUint32(curNum)
-			if !bytes.Equal(gotKey, expectedKey) {
-				t.Errorf("iter.Key #%d (%d): unexpected key - "+
-					"got %x, want %x", i, curNum, gotKey,
-					expectedKey)
-				continue testLoop
-			}
-
-			// Ensure value is as expected.
-			gotVal := iter.Value()
-			if !bytes.Equal(gotVal, expectedKey) {
-				t.Errorf("iter.Value #%d (%d): unexpected "+
-					"value - got %x, want %x", i, curNum,
-					gotVal, expectedKey)
-				continue testLoop
-			}
-		}
-
-		// Ensure iterator is exhausted.
-		if iter.Valid() {
-			t.Errorf("Valid #%d: iterator should be exhausted", i)
-			continue
-		}
-
-		// Seek to the provided key.
-		seekValid := iter.Seek(test.seekKey)
-		if !seekValid && test.expectedSeek != nil {
-			t.Errorf("Seek #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedSeek) {
-			t.Errorf("Seek.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedSeek)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedSeek) {
-			t.Errorf("Seek.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedSeek)
-			continue
-		}
-
-		// Recreate the iterator and ensure calling Next on it before it
-		// has been positioned gives the first element.
-		iter = testTreap.Iterator(test.startKey, test.limitKey)
-		hasNext := iter.Next()
-		if !hasNext && test.expectedFirst != nil {
-			t.Errorf("Next #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedFirst) {
-			t.Errorf("Next.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedFirst)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedFirst) {
-			t.Errorf("Next.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedFirst)
-			continue
-		}
-
-		// Recreate the iterator and ensure calling Prev on it before it
-		// has been positioned gives the first element.
-		iter = testTreap.Iterator(test.startKey, test.limitKey)
-		hasPrev := iter.Prev()
-		if !hasPrev && test.expectedLast != nil {
-			t.Errorf("Prev #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedLast) {
-			t.Errorf("Prev.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedLast)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedLast) {
-			t.Errorf("Next.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedLast)
-			continue
-		}
-	}
-}
-
-// TestMutableEmptyIterator ensures that the various functions behave as
-// expected when a mutable treap is empty.
-func TestMutableEmptyIterator(t *testing.T) {
-	t.Parallel()
-
-	// Create iterator against empty treap.
-	testTreap := NewMutable()
-	iter := testTreap.Iterator(nil, nil)
-
-	// Ensure Valid on empty iterator reports it as exhausted.
-	if iter.Valid() {
-		t.Fatal("Valid: iterator should be exhausted")
-	}
-
-	// Ensure First and Last on empty iterator report it as exhausted.
-	if iter.First() {
-		t.Fatal("First: iterator should be exhausted")
-	}
-	if iter.Last() {
-		t.Fatal("Last: iterator should be exhausted")
-	}
-
-	// Ensure Next and Prev on empty iterator report it as exhausted.
-	if iter.Next() {
-		t.Fatal("Next: iterator should be exhausted")
-	}
-	if iter.Prev() {
-		t.Fatal("Prev: iterator should be exhausted")
-	}
-
-	// Ensure Key and Value on empty iterator are nil.
-	if gotKey := iter.Key(); gotKey != nil {
-		t.Fatalf("Key: should be nil - got %q", gotKey)
-	}
-	if gotVal := iter.Value(); gotVal != nil {
-		t.Fatalf("Value: should be nil - got %q", gotVal)
-	}
-
-	// Ensure Next and Prev report exhausted after forcing a reseek on an
-	// empty iterator.
-	iter.ForceReseek()
-	if iter.Next() {
-		t.Fatal("Next: iterator should be exhausted")
-	}
-	iter.ForceReseek()
-	if iter.Prev() {
-		t.Fatal("Prev: iterator should be exhausted")
-	}
-}
-
-// TestIteratorUpdates ensures that issuing a call to ForceReseek on an iterator
-// that had the underlying mutable treap updated works as expected.
-func TestIteratorUpdates(t *testing.T) {
-	t.Parallel()
-
-	// Create a new treap with various values inserted in no particular
-	// order. The resulting keys are the set (2, 4, 7, 11, 18, 25).
-	testTreap := NewMutable()
-	testTreap.Put(serializeUint32(7), nil)
-	testTreap.Put(serializeUint32(2), nil)
-	testTreap.Put(serializeUint32(18), nil)
-	testTreap.Put(serializeUint32(11), nil)
-	testTreap.Put(serializeUint32(25), nil)
-	testTreap.Put(serializeUint32(4), nil)
-
-	// Create an iterator against the treap with a range that excludes the
-	// lowest and highest entries. The limited set is then (4, 7, 11, 18)
-	iter := testTreap.Iterator(serializeUint32(3), serializeUint32(25))
-
-	// Delete a key from the middle of the range and notify the iterator to
-	// force a reseek.
-	testTreap.Delete(serializeUint32(11))
-	iter.ForceReseek()
-
-	// Ensure that calling Next on the iterator after the forced reseek
-	// gives the expected key. The limited set of keys at this point is
-	// (4, 7, 18) and the iterator has not yet been positioned.
-	if !iter.Next() {
-		t.Fatal("ForceReseek.Next: unexpected exhausted iterator")
-	}
-	wantKey := serializeUint32(4)
-	gotKey := iter.Key()
-	if !bytes.Equal(gotKey, wantKey) {
-		t.Fatalf("ForceReseek.Key: unexpected key - got %x, want %x",
-			gotKey, wantKey)
-	}
-
-	// Delete the key the iterator is currently position at and notify the
-	// iterator to force a reseek.
-	testTreap.Delete(serializeUint32(4))
-	iter.ForceReseek()
-
-	// Ensure that calling Next on the iterator after the forced reseek
-	// gives the expected key. The limited set of keys at this point is
-	// (7, 18) and the iterator is positioned at a deleted entry before 7.
-	if !iter.Next() {
-		t.Fatal("ForceReseek.Next: unexpected exhausted iterator")
-	}
-	wantKey = serializeUint32(7)
-	gotKey = iter.Key()
-	if !bytes.Equal(gotKey, wantKey) {
-		t.Fatalf("ForceReseek.Key: unexpected key - got %x, want %x",
-			gotKey, wantKey)
-	}
-
-	// Add a key before the current key the iterator is position at and
-	// notify the iterator to force a reseek.
-	testTreap.Put(serializeUint32(4), nil)
-	iter.ForceReseek()
-
-	// Ensure that calling Prev on the iterator after the forced reseek
-	// gives the expected key. The limited set of keys at this point is
-	// (4, 7, 18) and the iterator is positioned at 7.
-	if !iter.Prev() {
-		t.Fatal("ForceReseek.Prev: unexpected exhausted iterator")
-	}
-	wantKey = serializeUint32(4)
-	gotKey = iter.Key()
-	if !bytes.Equal(gotKey, wantKey) {
-		t.Fatalf("ForceReseek.Key: unexpected key - got %x, want %x",
-			gotKey, wantKey)
-	}
-
-	// Delete the next key the iterator would ordinarily move to then notify
-	// the iterator to force a reseek.
-	testTreap.Delete(serializeUint32(7))
-	iter.ForceReseek()
-
-	// Ensure that calling Next on the iterator after the forced reseek
-	// gives the expected key. The limited set of keys at this point is
-	// (4, 18) and the iterator is positioned at 4.
-	if !iter.Next() {
-		t.Fatal("ForceReseek.Next: unexpected exhausted iterator")
-	}
-	wantKey = serializeUint32(18)
-	gotKey = iter.Key()
-	if !bytes.Equal(gotKey, wantKey) {
-		t.Fatalf("ForceReseek.Key: unexpected key - got %x, want %x",
-			gotKey, wantKey)
-	}
-}
-
-// TestImmutableIterator ensures that the general behavior of immutable treap
-// iterators is as expected including tests for first, last, ordered and reverse
-// ordered iteration, limiting the range, seeking, and initially unpositioned.
-func TestImmutableIterator(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		numKeys       int
-		step          int
-		startKey      []byte
-		limitKey      []byte
-		expectedFirst []byte
-		expectedLast  []byte
-		seekKey       []byte
-		expectedSeek  []byte
-	}{
-		// No range limits. Values are the set (0, 1, 2, ..., 49).
-		// Seek existing value.
-		{
-			numKeys:       50,
-			step:          1,
-			expectedFirst: serializeUint32(0),
-			expectedLast:  serializeUint32(49),
-			seekKey:       serializeUint32(12),
-			expectedSeek:  serializeUint32(12),
-		},
-
-		// Limited to range [24, end]. Values are the set
-		// (0, 2, 4, ..., 48). Seek value that doesn't exist and is
-		// greater than largest existing key.
-		{
-			numKeys:       50,
-			step:          2,
-			startKey:      serializeUint32(24),
-			expectedFirst: serializeUint32(24),
-			expectedLast:  serializeUint32(48),
-			seekKey:       serializeUint32(49),
-			expectedSeek:  nil,
-		},
-
-		// Limited to range [start, 25). Values are the set
-		// (0, 3, 6, ..., 48). Seek value that doesn't exist but is
-		// before an existing value within the range.
-		{
-			numKeys:       50,
-			step:          3,
-			limitKey:      serializeUint32(25),
-			expectedFirst: serializeUint32(0),
-			expectedLast:  serializeUint32(24),
-			seekKey:       serializeUint32(17),
-			expectedSeek:  serializeUint32(18),
-		},
-
-		// Limited to range [10, 21). Values are the set
-		// (0, 4, ..., 48). Seek value that exists, but is before the
-		// minimum allowed range.
-		{
-			numKeys:       50,
-			step:          4,
-			startKey:      serializeUint32(10),
-			limitKey:      serializeUint32(21),
-			expectedFirst: serializeUint32(12),
-			expectedLast:  serializeUint32(20),
-			seekKey:       serializeUint32(4),
-			expectedSeek:  nil,
-		},
-
-		// Limited by prefix {0,0,0}, range [{0,0,0}, {0,0,1}).
-		// Since it's a bytewise compare,  {0,0,0,...} < {0,0,1}.
-		// Seek existing value within the allowed range.
-		{
-			numKeys:       300,
-			step:          1,
-			startKey:      []byte{0x00, 0x00, 0x00},
-			limitKey:      []byte{0x00, 0x00, 0x01},
-			expectedFirst: serializeUint32(0),
-			expectedLast:  serializeUint32(255),
-			seekKey:       serializeUint32(100),
-			expectedSeek:  serializeUint32(100),
-		},
-	}
-
-testLoop:
-	for i, test := range tests {
-		// Insert a bunch of keys.
-		testTreap := NewImmutable()
-		for i := 0; i < test.numKeys; i += test.step {
-			key := serializeUint32(uint32(i))
-			testTreap = testTreap.Put(key, key)
-		}
-
-		// Create new iterator limited by the test params.
-		iter := testTreap.Iterator(test.startKey, test.limitKey)
-
-		// Ensure the first item is accurate.
-		hasFirst := iter.First()
-		if !hasFirst && test.expectedFirst != nil {
-			t.Errorf("First #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey := iter.Key()
-		if !bytes.Equal(gotKey, test.expectedFirst) {
-			t.Errorf("First.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedFirst)
-			continue
-		}
-		gotVal := iter.Value()
-		if !bytes.Equal(gotVal, test.expectedFirst) {
-			t.Errorf("First.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedFirst)
-			continue
-		}
-
-		// Ensure the iterator gives the expected items in order.
-		curNum := binary.BigEndian.Uint32(test.expectedFirst)
-		for iter.Next() {
-			curNum += uint32(test.step)
-
-			// Ensure key is as expected.
-			gotKey := iter.Key()
-			expectedKey := serializeUint32(curNum)
-			if !bytes.Equal(gotKey, expectedKey) {
-				t.Errorf("iter.Key #%d (%d): unexpected key - "+
-					"got %x, want %x", i, curNum, gotKey,
-					expectedKey)
-				continue testLoop
-			}
-
-			// Ensure value is as expected.
-			gotVal := iter.Value()
-			if !bytes.Equal(gotVal, expectedKey) {
-				t.Errorf("iter.Value #%d (%d): unexpected "+
-					"value - got %x, want %x", i, curNum,
-					gotVal, expectedKey)
-				continue testLoop
-			}
-		}
-
-		// Ensure iterator is exhausted.
-		if iter.Valid() {
-			t.Errorf("Valid #%d: iterator should be exhausted", i)
-			continue
-		}
-
-		// Ensure the last item is accurate.
-		hasLast := iter.Last()
-		if !hasLast && test.expectedLast != nil {
-			t.Errorf("Last #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedLast) {
-			t.Errorf("Last.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedLast)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedLast) {
-			t.Errorf("Last.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedLast)
-			continue
-		}
-
-		// Ensure the iterator gives the expected items in reverse
-		// order.
-		curNum = binary.BigEndian.Uint32(test.expectedLast)
-		for iter.Prev() {
-			curNum -= uint32(test.step)
-
-			// Ensure key is as expected.
-			gotKey := iter.Key()
-			expectedKey := serializeUint32(curNum)
-			if !bytes.Equal(gotKey, expectedKey) {
-				t.Errorf("iter.Key #%d (%d): unexpected key - "+
-					"got %x, want %x", i, curNum, gotKey,
-					expectedKey)
-				continue testLoop
-			}
-
-			// Ensure value is as expected.
-			gotVal := iter.Value()
-			if !bytes.Equal(gotVal, expectedKey) {
-				t.Errorf("iter.Value #%d (%d): unexpected "+
-					"value - got %x, want %x", i, curNum,
-					gotVal, expectedKey)
-				continue testLoop
-			}
-		}
-
-		// Ensure iterator is exhausted.
-		if iter.Valid() {
-			t.Errorf("Valid #%d: iterator should be exhausted", i)
-			continue
-		}
-
-		// Seek to the provided key.
-		seekValid := iter.Seek(test.seekKey)
-		if !seekValid && test.expectedSeek != nil {
-			t.Errorf("Seek #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedSeek) {
-			t.Errorf("Seek.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedSeek)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedSeek) {
-			t.Errorf("Seek.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedSeek)
-			continue
-		}
-
-		// Recreate the iterator and ensure calling Next on it before it
-		// has been positioned gives the first element.
-		iter = testTreap.Iterator(test.startKey, test.limitKey)
-		hasNext := iter.Next()
-		if !hasNext && test.expectedFirst != nil {
-			t.Errorf("Next #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedFirst) {
-			t.Errorf("Next.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedFirst)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedFirst) {
-			t.Errorf("Next.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedFirst)
-			continue
-		}
-
-		// Recreate the iterator and ensure calling Prev on it before it
-		// has been positioned gives the first element.
-		iter = testTreap.Iterator(test.startKey, test.limitKey)
-		hasPrev := iter.Prev()
-		if !hasPrev && test.expectedLast != nil {
-			t.Errorf("Prev #%d: unexpected exhausted iterator", i)
-			continue
-		}
-		gotKey = iter.Key()
-		if !bytes.Equal(gotKey, test.expectedLast) {
-			t.Errorf("Prev.Key #%d: unexpected key - got %x, "+
-				"want %x", i, gotKey, test.expectedLast)
-			continue
-		}
-		gotVal = iter.Value()
-		if !bytes.Equal(gotVal, test.expectedLast) {
-			t.Errorf("Next.Value #%d: unexpected value - got %x, "+
-				"want %x", i, gotVal, test.expectedLast)
-			continue
-		}
-	}
-}
-
-// TestImmutableEmptyIterator ensures that the various functions behave as
-// expected when an immutable treap is empty.
-func TestImmutableEmptyIterator(t *testing.T) {
-	t.Parallel()
-
-	// Create iterator against empty treap.
-	testTreap := NewImmutable()
-	iter := testTreap.Iterator(nil, nil)
-
-	// Ensure Valid on empty iterator reports it as exhausted.
-	if iter.Valid() {
-		t.Fatal("Valid: iterator should be exhausted")
-	}
-
-	// Ensure First and Last on empty iterator report it as exhausted.
-	if iter.First() {
-		t.Fatal("First: iterator should be exhausted")
-	}
-	if iter.Last() {
-		t.Fatal("Last: iterator should be exhausted")
-	}
-
-	// Ensure Next and Prev on empty iterator report it as exhausted.
-	if iter.Next() {
-		t.Fatal("Next: iterator should be exhausted")
-	}
-	if iter.Prev() {
-		t.Fatal("Prev: iterator should be exhausted")
-	}
-
-	// Ensure Key and Value on empty iterator are nil.
-	if gotKey := iter.Key(); gotKey != nil {
-		t.Fatalf("Key: should be nil - got %q", gotKey)
-	}
-	if gotVal := iter.Value(); gotVal != nil {
-		t.Fatalf("Value: should be nil - got %q", gotVal)
-	}
-
-	// Ensure calling ForceReseek on an immutable treap iterator does not
-	// cause any issues since it only applies to mutable treap iterators.
-	iter.ForceReseek()
-	if iter.Next() {
-		t.Fatal("Next: iterator should be exhausted")
-	}
-	iter.ForceReseek()
-	if iter.Prev() {
-		t.Fatal("Prev: iterator should be exhausted")
-	}
-}

database/testdata/generator.go

@@ -1,153 +0,0 @@
-// This is a small tool to generate testdata blocks file
-
-package main
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"os"
-	"strconv"
-	"time"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/pkg/errors"
-)
-
-func main() {
-	targetFile, numBlocks := parseArgs()
-
-	out, err := os.Create(targetFile)
-	if err != nil {
-		panic(errors.Errorf("error reading target file: %s", err))
-	}
-	defer func() {
-		err := out.Close()
-		if err != nil {
-			panic(errors.Errorf("error closing target file: %s", err))
-		}
-	}()
-
-	generateBlocks(out, numBlocks)
-}
-
-func generateBlocks(out *os.File, numBlocks int) {
-	lastBlock := dagconfig.MainnetParams.GenesisBlock
-
-	for i := 0; i < numBlocks; i++ {
-		lastBlock = generateBlock(lastBlock)
-		writeBlock(out, lastBlock)
-	}
-}
-
-func generateBlock(parent *wire.MsgBlock) *wire.MsgBlock {
-	return &wire.MsgBlock{
-		Header: wire.BlockHeader{
-			Version:              1,
-			ParentHashes:         []*daghash.Hash{parent.BlockHash()},
-			HashMerkleRoot:       &genesisMerkleRoot,
-			AcceptedIDMerkleRoot: &daghash.ZeroHash,
-			UTXOCommitment:       &daghash.ZeroHash,
-			Timestamp:            time.Unix(0x5b28c4c8, 0), // 2018-06-19 08:54:32 +0000 UTC
-			Bits:                 0x2e00ffff,               // 503382015 [000000ffff000000000000000000000000000000000000000000000000000000]
-			Nonce:                0xc0192550,               // 2148484547
-		},
-		Transactions: []*wire.MsgTx{genesisCoinbaseTx},
-	}
-}
-
-func writeBlock(out *os.File, block *wire.MsgBlock) {
-	writeNet(out)
-
-	blockLen := uint32(block.SerializeSize())
-	buf := bytes.NewBuffer(make([]byte, 0, blockLen))
-
-	err := block.Serialize(buf)
-	if err != nil {
-		panic(errors.Errorf("error serializing block: %s", err))
-	}
-
-	err = binary.Write(out, binary.LittleEndian, blockLen)
-	if err != nil {
-		panic(errors.Errorf("error writing blockLen: %s", err))
-	}
-
-	_, err = out.Write(buf.Bytes())
-	if err != nil {
-		panic(errors.Errorf("error writing block: %s", err))
-	}
-}
-
-func writeNet(out *os.File) {
-	err := binary.Write(out, binary.LittleEndian, wire.Mainnet)
-	if err != nil {
-		panic(errors.Errorf("error writing net to file: %s", err))
-	}
-}
-
-func parseArgs() (targetFile string, numBlocks int) {
-	if len(os.Args) != 3 {
-		printUsage()
-	}
-
-	targetFile = os.Args[1]
-	numBlocks, err := strconv.Atoi(os.Args[2])
-	if err != nil {
-		printUsage()
-	}
-
-	return
-}
-
-func printUsage() {
-	fmt.Println("Usage: generator [targetFile] [numBlocks]")
-	os.Exit(1)
-}
-
-var genesisCoinbaseTxIns = []*wire.TxIn{
-	{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID{},
-			Index: 0xffffffff,
-		},
-		SignatureScript: []byte{
-			0x04, 0xff, 0xff, 0x00, 0x1d, 0x01, 0x04, 0x45, /* |.......E| */
-			0x54, 0x68, 0x65, 0x20, 0x54, 0x69, 0x6d, 0x65, /* |The Time| */
-			0x73, 0x20, 0x30, 0x33, 0x2f, 0x4a, 0x61, 0x6e, /* |s 03/Jan| */
-			0x2f, 0x32, 0x30, 0x30, 0x39, 0x20, 0x43, 0x68, /* |/2009 Ch| */
-			0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x6f, 0x72, /* |ancellor| */
-			0x20, 0x6f, 0x6e, 0x20, 0x62, 0x72, 0x69, 0x6e, /* | on brin| */
-			0x6b, 0x20, 0x6f, 0x66, 0x20, 0x73, 0x65, 0x63, /* |k of sec|*/
-			0x6f, 0x6e, 0x64, 0x20, 0x62, 0x61, 0x69, 0x6c, /* |ond bail| */
-			0x6f, 0x75, 0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, /* |out for |*/
-			0x62, 0x61, 0x6e, 0x6b, 0x73, /* |banks| */
-		},
-		Sequence: 0xffffffff,
-	},
-}
-var genesisCoinbaseTxOuts = []*wire.TxOut{
-	{
-		Value: 0x12a05f200,
-		ScriptPubKey: []byte{
-			0x41, 0x04, 0x67, 0x8a, 0xfd, 0xb0, 0xfe, 0x55, /* |A.g....U| */
-			0x48, 0x27, 0x19, 0x67, 0xf1, 0xa6, 0x71, 0x30, /* |H'.g..q0| */
-			0xb7, 0x10, 0x5c, 0xd6, 0xa8, 0x28, 0xe0, 0x39, /* |..\..(.9| */
-			0x09, 0xa6, 0x79, 0x62, 0xe0, 0xea, 0x1f, 0x61, /* |..yb...a| */
-			0xde, 0xb6, 0x49, 0xf6, 0xbc, 0x3f, 0x4c, 0xef, /* |..I..?L.| */
-			0x38, 0xc4, 0xf3, 0x55, 0x04, 0xe5, 0x1e, 0xc1, /* |8..U....| */
-			0x12, 0xde, 0x5c, 0x38, 0x4d, 0xf7, 0xba, 0x0b, /* |..\8M...| */
-			0x8d, 0x57, 0x8a, 0x4c, 0x70, 0x2b, 0x6b, 0xf1, /* |.W.Lp+k.| */
-			0x1d, 0x5f, 0xac, /* |._.| */
-		},
-	},
-}
-var genesisCoinbaseTx = wire.NewNativeMsgTx(1, genesisCoinbaseTxIns, genesisCoinbaseTxOuts)
-
-var genesisMerkleRoot = daghash.Hash([daghash.HashSize]byte{
-	0x3b, 0xa3, 0xed, 0xfd, 0x7a, 0x7b, 0x12, 0xb2,
-	0x7a, 0xc7, 0x2c, 0x3e, 0x67, 0x76, 0x8f, 0x61,
-	0x7f, 0xc8, 0x1b, 0xc3, 0x88, 0x8a, 0x51, 0x32,
-	0x3a, 0x9f, 0xb8, 0xaa, 0x4b, 0x1e, 0x5e, 0x4a,
-})

doc.go

@@ -6,7 +6,7 @@ Copyright (c) 2013-2014 Conformal Systems LLC.
 Use of this source code is governed by an ISC
 license that can be found in the LICENSE file.
 
-kaspad is a full-node kaspa implementation written in Go.
+Kaspad is a full-node kaspa implementation written in Go.
 
 The default options are sane for most users. This means kaspad will work 'out of
 the box' for most users. However, there are also a wide variety of flags that

domain/blockdag/behavior_flags.go

@@ -0,0 +1,44 @@
+package blockdag
+
+// BehaviorFlags is a bitmask defining tweaks to the normal behavior when
+// performing DAG processing and consensus rules checks.
+type BehaviorFlags uint32
+
+const (
+	// BFFastAdd may be set to indicate that several checks can be avoided
+	// for the block since it is already known to fit into the DAG due to
+	// already proving it correct links into the DAG.
+	BFFastAdd BehaviorFlags = 1 << iota
+
+	// BFNoPoWCheck may be set to indicate the proof of work check which
+	// ensures a block hashes to a value less than the required target will
+	// not be performed.
+	BFNoPoWCheck
+
+	// BFWasUnorphaned may be set to indicate that a block was just now
+	// unorphaned
+	BFWasUnorphaned
+
+	// BFAfterDelay may be set to indicate that a block had timestamp too far
+	// in the future, just finished the delay
+	BFAfterDelay
+
+	// BFWasStored is set to indicate that the block was previously stored
+	// in the block index but was never fully processed
+	BFWasStored
+
+	// BFDisallowDelay is set to indicate that a delayed block should be rejected.
+	// This is used for the case where a block is submitted through RPC.
+	BFDisallowDelay
+
+	// BFDisallowOrphans is set to indicate that an orphan block should be rejected.
+	// This is used for the case where a block is submitted through RPC.
+	BFDisallowOrphans
+
+	// BFNone is a convenience value to specifically indicate no flags.
+	BFNone BehaviorFlags = 0
+)
+
+func isBehaviorFlagRaised(flags BehaviorFlags, flag BehaviorFlags) bool {
+	return flags&flag == flag
+}

domain/blockdag/block_utxo.go

@@ -0,0 +1,302 @@
+package blockdag
+
+import (
+	"fmt"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+)
+
+// TxAcceptanceData stores a transaction together with an indication
+// if it was accepted or not by some block
+type TxAcceptanceData struct {
+	Tx         *util.Tx
+	IsAccepted bool
+}
+
+// BlockTxsAcceptanceData stores all transactions in a block with an indication
+// if they were accepted or not by some other block
+type BlockTxsAcceptanceData struct {
+	BlockHash        daghash.Hash
+	TxAcceptanceData []TxAcceptanceData
+}
+
+// MultiBlockTxsAcceptanceData stores data about which transactions were accepted by a block
+// It's a slice of the block's blues block IDs and their transaction acceptance data
+type MultiBlockTxsAcceptanceData []BlockTxsAcceptanceData
+
+// FindAcceptanceData finds the BlockTxsAcceptanceData that matches blockHash
+func (data MultiBlockTxsAcceptanceData) FindAcceptanceData(blockHash *daghash.Hash) (*BlockTxsAcceptanceData, bool) {
+	for _, acceptanceData := range data {
+		if acceptanceData.BlockHash.IsEqual(blockHash) {
+			return &acceptanceData, true
+		}
+	}
+	return nil, false
+}
+
+// TxsAcceptedByVirtual retrieves transactions accepted by the current virtual block
+//
+// This function MUST be called with the DAG read-lock held
+func (dag *BlockDAG) TxsAcceptedByVirtual() (MultiBlockTxsAcceptanceData, error) {
+	_, _, txsAcceptanceData, err := dag.pastUTXO(&dag.virtual.blockNode)
+	return txsAcceptanceData, err
+}
+
+// TxsAcceptedByBlockHash retrieves transactions accepted by the given block
+//
+// This function MUST be called with the DAG read-lock held
+func (dag *BlockDAG) TxsAcceptedByBlockHash(blockHash *daghash.Hash) (MultiBlockTxsAcceptanceData, error) {
+	node, ok := dag.index.LookupNode(blockHash)
+	if !ok {
+		return nil, errors.Errorf("Couldn't find block %s", blockHash)
+	}
+	_, _, txsAcceptanceData, err := dag.pastUTXO(node)
+	return txsAcceptanceData, err
+}
+
+func (dag *BlockDAG) meldVirtualUTXO(newVirtualUTXODiffSet *DiffUTXOSet) error {
+	return newVirtualUTXODiffSet.meldToBase()
+}
+
+// checkDoubleSpendsWithBlockPast checks that each block transaction
+// has a corresponding UTXO in the block pastUTXO.
+func checkDoubleSpendsWithBlockPast(pastUTXO UTXOSet, blockTransactions []*util.Tx) error {
+	for _, tx := range blockTransactions {
+		if tx.IsCoinBase() {
+			continue
+		}
+
+		for _, txIn := range tx.MsgTx().TxIn {
+			if _, ok := pastUTXO.Get(txIn.PreviousOutpoint); !ok {
+				return ruleError(ErrMissingTxOut, fmt.Sprintf("missing transaction "+
+					"output %s in the utxo set", txIn.PreviousOutpoint))
+			}
+		}
+	}
+
+	return nil
+}
+
+// verifyAndBuildUTXO verifies all transactions in the given block and builds its UTXO
+// to save extra traversals it returns the transactions acceptance data, the compactFeeData
+// for the new block and its multiset.
+func (node *blockNode) verifyAndBuildUTXO(dag *BlockDAG, transactions []*util.Tx, fastAdd bool) (
+	newBlockUTXO UTXOSet, txsAcceptanceData MultiBlockTxsAcceptanceData, newBlockFeeData compactFeeData, multiset *secp256k1.MultiSet, err error) {
+
+	pastUTXO, selectedParentPastUTXO, txsAcceptanceData, err := dag.pastUTXO(node)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	err = node.validateAcceptedIDMerkleRoot(dag, txsAcceptanceData)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	feeData, err := dag.checkConnectToPastUTXO(node, pastUTXO, transactions, fastAdd)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	multiset, err = node.calcMultiset(dag, txsAcceptanceData, selectedParentPastUTXO)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	err = node.validateUTXOCommitment(multiset)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	return pastUTXO, txsAcceptanceData, feeData, multiset, nil
+}
+
+func genesisPastUTXO(virtual *virtualBlock) UTXOSet {
+	// The genesis has no past UTXO, so we create an empty UTXO
+	// set by creating a diff UTXO set with the virtual UTXO
+	// set, and adding all of its entries in toRemove
+	diff := NewUTXODiff()
+	for outpoint, entry := range virtual.utxoSet.utxoCollection {
+		diff.toRemove[outpoint] = entry
+	}
+	genesisPastUTXO := UTXOSet(NewDiffUTXOSet(virtual.utxoSet, diff))
+	return genesisPastUTXO
+}
+
+// applyBlueBlocks adds all transactions in the blue blocks to the selectedParent's past UTXO set
+// Purposefully ignoring failures - these are just unaccepted transactions
+// Writing down which transactions were accepted or not in txsAcceptanceData
+func (node *blockNode) applyBlueBlocks(selectedParentPastUTXO UTXOSet, blueBlocks []*util.Block) (
+	pastUTXO UTXOSet, multiBlockTxsAcceptanceData MultiBlockTxsAcceptanceData, err error) {
+
+	pastUTXO = selectedParentPastUTXO.(*DiffUTXOSet).cloneWithoutBase()
+	multiBlockTxsAcceptanceData = make(MultiBlockTxsAcceptanceData, len(blueBlocks))
+
+	// Add blueBlocks to multiBlockTxsAcceptanceData in topological order. This
+	// is so that anyone who iterates over it would process blocks (and transactions)
+	// in their order of appearance in the DAG.
+	for i := 0; i < len(blueBlocks); i++ {
+		blueBlock := blueBlocks[i]
+		transactions := blueBlock.Transactions()
+		blockTxsAcceptanceData := BlockTxsAcceptanceData{
+			BlockHash:        *blueBlock.Hash(),
+			TxAcceptanceData: make([]TxAcceptanceData, len(transactions)),
+		}
+		isSelectedParent := i == 0
+
+		for j, tx := range blueBlock.Transactions() {
+			var isAccepted bool
+
+			// Coinbase transaction outputs are added to the UTXO
+			// only if they are in the selected parent chain.
+			if !isSelectedParent && tx.IsCoinBase() {
+				isAccepted = false
+			} else {
+				isAccepted, err = pastUTXO.AddTx(tx.MsgTx(), node.blueScore)
+				if err != nil {
+					return nil, nil, err
+				}
+			}
+			blockTxsAcceptanceData.TxAcceptanceData[j] = TxAcceptanceData{Tx: tx, IsAccepted: isAccepted}
+		}
+		multiBlockTxsAcceptanceData[i] = blockTxsAcceptanceData
+	}
+
+	return pastUTXO, multiBlockTxsAcceptanceData, nil
+}
+
+// pastUTXO returns the UTXO of a given block's past
+// To save traversals over the blue blocks, it also returns the transaction acceptance data for
+// all blue blocks
+func (dag *BlockDAG) pastUTXO(node *blockNode) (
+	pastUTXO, selectedParentPastUTXO UTXOSet, bluesTxsAcceptanceData MultiBlockTxsAcceptanceData, err error) {
+
+	if node.isGenesis() {
+		return genesisPastUTXO(dag.virtual), nil, MultiBlockTxsAcceptanceData{}, nil
+	}
+
+	selectedParentPastUTXO, err = dag.restorePastUTXO(node.selectedParent)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	blueBlocks, err := dag.fetchBlueBlocks(node)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	pastUTXO, bluesTxsAcceptanceData, err = node.applyBlueBlocks(selectedParentPastUTXO, blueBlocks)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return pastUTXO, selectedParentPastUTXO, bluesTxsAcceptanceData, nil
+}
+
+// restorePastUTXO restores the UTXO of a given block from its diff
+func (dag *BlockDAG) restorePastUTXO(node *blockNode) (UTXOSet, error) {
+	stack := []*blockNode{}
+
+	// Iterate over the chain of diff-childs from node till virtual and add them
+	// all into a stack
+	for current := node; current != nil; {
+		stack = append(stack, current)
+		var err error
+		current, err = dag.utxoDiffStore.diffChildByNode(current)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Start with the top item in the stack, going over it top-to-bottom,
+	// applying the UTXO-diff one-by-one.
+	topNode, stack := stack[len(stack)-1], stack[:len(stack)-1] // pop the top item in the stack
+	topNodeDiff, err := dag.utxoDiffStore.diffByNode(topNode)
+	if err != nil {
+		return nil, err
+	}
+	accumulatedDiff := topNodeDiff.clone()
+
+	for i := len(stack) - 1; i >= 0; i-- {
+		diff, err := dag.utxoDiffStore.diffByNode(stack[i])
+		if err != nil {
+			return nil, err
+		}
+		// Use withDiffInPlace, otherwise copying the diffs again and again create a polynomial overhead
+		err = accumulatedDiff.withDiffInPlace(diff)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewDiffUTXOSet(dag.virtual.utxoSet, accumulatedDiff), nil
+}
+
+// updateTipsUTXO builds and applies new diff UTXOs for all the DAG's tips
+func updateTipsUTXO(dag *BlockDAG, virtualUTXO UTXOSet) error {
+	for tip := range dag.virtual.parents {
+		tipPastUTXO, err := dag.restorePastUTXO(tip)
+		if err != nil {
+			return err
+		}
+		diff, err := virtualUTXO.diffFrom(tipPastUTXO)
+		if err != nil {
+			return err
+		}
+		err = dag.utxoDiffStore.setBlockDiff(tip, diff)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// updateParents adds this block to the children sets of its parents
+// and updates the diff of any parent whose DiffChild is this block
+func (node *blockNode) updateParents(dag *BlockDAG, newBlockUTXO UTXOSet) error {
+	node.updateParentsChildren()
+	return node.updateParentsDiffs(dag, newBlockUTXO)
+}
+
+// updateParentsDiffs updates the diff of any parent whose DiffChild is this block
+func (node *blockNode) updateParentsDiffs(dag *BlockDAG, newBlockUTXO UTXOSet) error {
+	virtualDiffFromNewBlock, err := dag.virtual.utxoSet.diffFrom(newBlockUTXO)
+	if err != nil {
+		return err
+	}
+
+	err = dag.utxoDiffStore.setBlockDiff(node, virtualDiffFromNewBlock)
+	if err != nil {
+		return err
+	}
+
+	for parent := range node.parents {
+		diffChild, err := dag.utxoDiffStore.diffChildByNode(parent)
+		if err != nil {
+			return err
+		}
+		if diffChild == nil {
+			parentPastUTXO, err := dag.restorePastUTXO(parent)
+			if err != nil {
+				return err
+			}
+			err = dag.utxoDiffStore.setBlockDiffChild(parent, node)
+			if err != nil {
+				return err
+			}
+			diff, err := newBlockUTXO.diffFrom(parentPastUTXO)
+			if err != nil {
+				return err
+			}
+			err = dag.utxoDiffStore.setBlockDiff(parent, diff)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}