[NOD-1238] Fix acceptance index never being initialized. (#859)

* [NOD-1225] Rename wire to domainmessage

* [NOD-1225] Get rid of references to package wire in the code, and get rid of InvType

CHANGES

@@ -1,955 +0,0 @@
-============================================================================
-User visible changes for btcd
-  A full-node bitcoin implementation written in Go
-============================================================================
-
-Changes in 0.12.0 (Fri Nov 20 2015)
-  - Protocol and network related changes:
-    - Add a new checkpoint at block height 382320 (#555)
-    - Implement BIP0065 which includes support for version 4 blocks, a new
-      consensus opcode (OP_CHECKLOCKTIMEVERIFY) that enforces transaction
-      lock times, and a double-threshold switchover mechanism (#535, #459,
-      #455)
-    - Implement BIP0111 which provides a new bloom filter service flag and
-      hence provides support for protocol version 70011 (#499)
-    - Add a new parameter --nopeerbloomfilters to allow disabling bloom
-      filter support (#499)
-    - Reject non-canonically encoded variable length integers (#507)
-    - Add mainnet peer discovery DNS seed (seed.bitcoin.jonasschnelli.ch)
-      (#496)
-    - Correct reconnect handling for persistent peers (#463, #464)
-    - Ignore requests for block headers if not fully synced (#444)
-    - Add CLI support for specifying the zone id on IPv6 addresses (#538)
-    - Fix a couple of issues where the initial block sync could stall (#518,
-      #229, #486)
-    - Fix an issue which prevented the --onion option from working as
-      intended (#446)
-  - Transaction relay (memory pool) changes:
-    - Require transactions to only include signatures encoded with the
-    canonical 'low-s' encoding (#512)
-    - Add a new parameter --minrelaytxfee to allow the minimum transaction
-      fee in BTC/kB to be overridden (#520)
-    - Retain memory pool transactions when they redeem another one that is
-      removed when a block is accepted (#539)
-    - Do not send reject messages for a transaction if it is valid but
-      causes an orphan transaction which depends on it to be determined
-      as invalid (#546)
-    - Refrain from attempting to add orphans to the memory pool multiple
-      times when the transaction they redeem is added (#551)
-    - Modify minimum transaction fee calculations to scale based on bytes
-      instead of full kilobyte boundaries (#521, #537)
-  - Implement signature cache:
-    - Provides a limited memory cache of validated signatures which is a
-      huge optimization when verifying blocks for transactions that are
-      already in the memory pool (#506)
-    - Add a new parameter '--sigcachemaxsize' which allows the size of the
-      new cache to be manually changed if desired (#506)
-  - Mining support changes:
-    - Notify getblocktemplate long polling clients when a block is pushed
-      via submitblock (#488)
-    - Speed up getblocktemplate by making use of the new signature cache
-      (#506)
-  - RPC changes:
-    - Implement getmempoolinfo command (#453)
-    - Implement getblockheader command (#461)
-    - Modify createrawtransaction command to accept a new optional parameter
-      'locktime' (#529)
-    - Modify listunspent result to include the 'spendable' field (#440)
-    - Modify getinfo command to include 'errors' field (#511)
-    - Add timestamps to blockconnected and blockdisconnected notifications
-      (#450)
-    - Several modifications to searchrawtranscations command:
-      - Accept a new optional parameter 'vinextra' which causes the results
-        to include information about the outputs referenced by a transaction's
-        inputs (#485, #487)
-      - Skip entries in the mempool too (#495)
-      - Accept a new optional parameter 'reverse' to return the results in
-        reverse order (most recent to oldest) (#497)
-      - Accept a new optional parameter 'filteraddrs' which causes the
-        results to only include inputs and outputs which involve the
-        provided addresses (#516)
-    - Change the notification order to notify clients about mined
-      transactions (recvtx, redeemingtx) before the blockconnected
-      notification (#449)
-    - Update verifymessage RPC to use the standard algorithm so it is
-      compatible with other implementations (#515)
-    - Improve ping statistics by pinging on an interval (#517)
-  - Websocket changes:
-    - Implement session command which returns a per-session unique id (#500,
-      #503)
-  - btcctl utility changes:
-    - Add getmempoolinfo command (#453)
-    - Add getblockheader command (#461)
-    - Add getwalletinfo command (#471)
-  - Notable developer-related package changes:
-    - Introduce a new peer package which acts a common base for creating and
-      concurrently managing bitcoin network peers (#445)
-    - Various cleanup of the new peer package (#528, #531, #524, #534,
-      #549)
-    - Blocks heights now consistently use int32 everywhere (#481)
-    - The BlockHeader type in the wire package now provides the BtcDecode
-      and BtcEncode methods (#467)
-    - Update wire package to recognize BIP0064 (getutxo) service bit (#489)
-    - Export LockTimeThreshold constant from txscript package (#454)
-    - Export MaxDataCarrierSize constant from txscript package (#466)
-    - Provide new IsUnspendable function from the txscript package (#478)
-    - Export variable length string functions from the wire package (#514)
-    - Export DNS Seeds for each network from the chaincfg package (#544)
-    - Preliminary work towards separating the memory pool into a separate
-      package (#525, #548)
-  - Misc changes:
-    - Various documentation updates (#442, #462, #465, #460, #470, #473,
-      #505, #530, #545)
-    - Add installation instructions for gentoo (#542)
-    - Ensure an error is shown if OS limits can't be set at startup (#498)
-    - Tighten the standardness checks for multisig scripts (#526)
-    - Test coverage improvement (#468, #494, #527, #543, #550)
-    - Several optimizations (#457, #474, #475, #476, #508, #509)
-    - Minor code cleanup and refactoring (#472, #479, #482, #519, #540)
-  - Contributors (alphabetical order):
-    - Ben Echols
-    - Bruno Clermont
-    - danda
-    - Daniel Krawisz
-    - Dario Nieuwenhuis
-    - Dave Collins
-    - David Hill
-    - Javed Khan
-    - Jonathan Gillham
-    - Joseph Becher
-    - Josh Rickmar
-    - Justus Ranvier
-    - Mawuli Adzoe
-    - Olaoluwa Osuntokun
-    - Rune T. Aune
-
-Changes in 0.11.1 (Wed May 27 2015)
-  - Protocol and network related changes:
-    - Use correct sub-command in reject message for rejected transactions
-      (#436, #437)
-    - Add a new parameter --torisolation which forces new circuits for each
-      connection when using tor (#430)
-  - Transaction relay (memory pool) changes:
-    - Reduce the default number max number of allowed orphan transactions
-      to 1000 (#419)
-    - Add a new parameter --maxorphantx which allows the maximum number of
-      orphan transactions stored in the mempool to be specified (#419)
-  - RPC changes:
-    - Modify listtransactions result to include the 'involveswatchonly' and
-      'vout' fields (#427)
-    - Update getrawtransaction result to omit the 'confirmations' field
-      when it is 0 (#420, #422)
-    - Update signrawtransaction result to include errors (#423)
-  - btcctl utility changes:
-    - Add gettxoutproof command (#428)
-    - Add verifytxoutproof command (#428)
-  - Notable developer-related package changes:
-    - The btcec package now provides the ability to perform ECDH
-      encryption and decryption (#375)
-    - The block and header validation in the blockchain package has been
-      split to help pave the way toward concurrent downloads (#386)
-  - Misc changes:
-    - Minor peer optimization (#433)
-  - Contributors (alphabetical order):
-    - Dave Collins
-    - David Hill
-    - Federico Bond
-    - Ishbir Singh
-    - Josh Rickmar
-
-Changes in 0.11.0 (Wed May 06 2015)
-  - Protocol and network related changes:
-    - **IMPORTANT: Update is required due to the following point**
-    - Correct a few corner cases in script handling which could result in
-      forking from the network on non-standard transactions (#425)
-    - Add a new checkpoint at block height 352940 (#418)
-    - Optimized script execution (#395, #400, #404, #409)
-    - Fix a case that could lead stalled syncs (#138, #296)
-  - Network address manager changes:
-    - Implement eclipse attack countermeasures as proposed in
-      http://cs-people.bu.edu/heilman/eclipse (#370, #373)
-  - Optional address indexing changes:
-    - Fix an issue where a reorg could cause an orderly shutdown when the
-      address index is active (#340, #357)
-  - Transaction relay (memory pool) changes:
-    - Increase maximum allowed space for nulldata transactions to 80 bytes
-      (#331)
-    - Implement support for the following rules specified by BIP0062:
-      - The S value in ECDSA signature must be at most half the curve order
-        (rule 5) (#349)
-      - Script execution must result in a single non-zero value on the stack
-        (rule 6) (#347)
-      - NOTE: All 7 rules of BIP0062 are now implemented
-    - Use network adjusted time in finalized transaction checks to improve
-      consistency across nodes (#332)
-    - Process orphan transactions on acceptance of new transactions (#345)
-  - RPC changes:
-    - Add support for a limited RPC user which is not allowed admin level
-      operations on the server (#363)
-    - Implement node command for more unified control over connected peers
-      (#79, #341)
-    - Implement generate command for regtest/simnet to support
-      deterministically mining a specified number of blocks (#362, #407)
-    - Update searchrawtransactions to return the matching transactions in
-      order (#354)
-    - Correct an issue with searchrawtransactions where it could return
-      duplicates (#346, #354)
-    - Increase precision of 'difficulty' field in getblock result to 8
-      (#414, #415)
-    - Omit 'nextblockhash' field from getblock result when it is empty
-      (#416, #417)
-    - Add 'id' and 'timeoffset' fields to getpeerinfo result (#335)
-  - Websocket changes:
-    - Implement new commands stopnotifyspent, stopnotifyreceived,
-      stopnotifyblocks, and stopnotifynewtransactions to allow clients to
-      cancel notification registrations (#122, #342)
-  - btcctl utility changes:
-    - A single dash can now be used as an argument to cause that argument to
-      be read from stdin (#348)
-    - Add generate command
-  - Notable developer-related package changes:
-    - The new version 2 btcjson package has now replaced the deprecated
-      version 1 package (#368)
-    - The btcec package now performs all signing using RFC6979 deterministic
-      signatures (#358, #360)
-    - The txscript package has been significantly cleaned up and had a few
-      API changes (#387, #388, #389, #390, #391, #392, #393, #395, #396,
-      #400, #403, #404, #405, #406, #408, #409, #410, #412)
-    - A new PkScriptLocs function has been added to the wire package MsgTx
-      type which provides callers that deal with scripts optimization
-      opportunities (#343)
-  - Misc changes:
-    - Minor wire hashing optimizations (#366, #367)
-    - Other minor internal optimizations
-  - Contributors (alphabetical order):
-    - Alex Akselrod
-    - Arne Brutschy
-    - Chris Jepson
-    - Daniel Krawisz
-    - Dave Collins
-    - David Hill
-    - Jimmy Song
-    - Jonas Nick
-    - Josh Rickmar
-    - Olaoluwa Osuntokun
-    - Oleg Andreev
-
-Changes in 0.10.0 (Sun Mar 01 2015)
-  - Protocol and network related changes:
-    - Add a new checkpoint at block height 343185
-    - Implement BIP066 which includes support for version 3 blocks, a new
-      consensus rule which prevents non-DER encoded signatures, and a
-      double-threshold switchover mechanism
-    - Rather than announcing all known addresses on getaddr requests which
-      can possibly result in multiple messages, randomize the results and
-      limit them to the max allowed by a single message (1000 addresses)
-    - Add more reserved IP spaces to the address manager
-  - Transaction relay (memory pool) changes:
-    - Make transactions which contain reserved opcodes nonstandard
-    - No longer accept or relay free and low-fee transactions that have
-      insufficient priority to be mined in the next block
-    - Implement support for the following rules specified by BIP0062:
-      - ECDSA signature must use strict DER encoding (rule 1)
-      - The signature script must only contain push operations (rule 2)
-      - All push operations must use the smallest possible encoding (rule 3)
-      - All stack values interpreted as a number must be encoding using the
-        shortest possible form (rule 4)
-      - NOTE: Rule 1 was already enforced, however the entire script now
-        evaluates to false rather than only the signature verification as
-        required by BIP0062
-    - Allow transactions with nulldata transaction outputs to be treated as
-      standard
-  - Mining support changes:
-    - Modify the getblocktemplate RPC to generate and return block templates
-      for version 3 blocks which are compatible with BIP0066
-    - Allow getblocktemplate to serve blocks when the current time is
-      less than the minimum allowed time for a generated block template
-      (https://github.com/btcsuite/btcd/issues/209)
-  - Crypto changes:
-    - Optimize scalar multiplication by the base point by using a
-      pre-computed table which results in approximately a 35% speedup
-     (https://github.com/btcsuite/btcec/issues/2)
-    - Optimize general scalar multiplication by using the secp256k1
-      endomorphism which results in approximately a 17-20% speedup
-     (https://github.com/btcsuite/btcec/issues/1)
-    - Optimize general scalar multiplication by using non-adjacent form
-      which results in approximately an additional 8% speedup
-     (https://github.com/btcsuite/btcec/issues/3)
-  - Implement optional address indexing:
-    - Add a new parameter --addrindex which will enable the creation of an
-      address index which can be queried to determine all transactions which
-      involve a given address
-      (https://github.com/btcsuite/btcd/issues/190)
-    - Add a new logging subsystem for address index related operations
-    - Support new searchrawtransactions RPC
-      (https://github.com/btcsuite/btcd/issues/185)
-  - RPC changes:
-    - Require TLS version 1.2 as the minimum version for all TLS connections
-    - Provide support for disabling TLS when only listening on localhost
-      (https://github.com/btcsuite/btcd/pull/192)
-    - Modify help output for all commands to provide much more consistent
-      and detailed information
-    - Correct case in getrawtransaction which would refuse to serve certain
-      transactions with invalid scripts
-      (https://github.com/btcsuite/btcd/issues/210)
-    - Correct error handling in the getrawtransaction RPC which could lead
-      to a crash in rare cases
-      (https://github.com/btcsuite/btcd/issues/196)
-    - Update getinfo RPC to include the appropriate 'timeoffset' calculated
-      from the median network time
-    - Modify listreceivedbyaddress result type to include txids field so it
-      is compatible
-    - Add 'iswatchonly' field to validateaddress result
-    - Add 'startingpriority' and 'currentpriority' fields to getrawmempool
-      (https://github.com/btcsuite/btcd/issues/178)
-    - Don't omit the 'confirmations' field from getrawtransaction when it is
-      zero
-  - Websocket changes:
-    - Modify the behavior of the rescan command to automatically register
-      for notifications about transactions paying to rescanned addresses
-      or spending outputs from the final rescan utxo set when the rescan
-      is through the best block in the chain
-  - btcctl utility changes:
-    - Make the list of commands available via the -l option rather than
-      dumping the entire list on usage errors
-    - Alphabetize and categorize the list of commands by chain and wallet
-    - Make the help option only show the help options instead of also
-      dumping all of the commands
-    - Make the usage syntax much more consistent and correct a few cases of
-      misnamed fields
-      (https://github.com/btcsuite/btcd/issues/305)
-    - Improve usage errors to show the specific parameter number, reason,
-      and error code
-    - Only show the usage for specific command is shown when a valid command
-      is provided with invalid parameters
-    - Add support for a SOCK5 proxy
-    - Modify output for integer fields (such as timestamps) to display
-      normally instead in scientific notation
-    - Add invalidateblock command
-    - Add reconsiderblock command
-    - Add createnewaccount command
-    - Add renameaccount command
-    - Add searchrawtransactions command
-    - Add importaddress command
-    - Add importpubkey command
-  - showblock utility changes:
-    - Remove utility in favor of the RPC getblock method
-  - Notable developer-related package changes:
-    - Many of the core packages have been relocated into the btcd repository
-      (https://github.com/btcsuite/btcd/issues/214)
-    - A new version of the btcjson package that has been completely
-      redesigned from the ground up based based upon how the project has
-      evolved and lessons learned while using it since it was first written
-      is now available in the btcjson/v2/btcjson directory
-      - This will ultimately replace the current version so anyone making
-        use of this package will need to update their code accordingly
-    - The btcec package now provides better facilities for working directly
-      with its public and private keys without having to mix elements from
-      the ecdsa package
-    - Update the script builder to ensure all rules specified by BIP0062 are
-      adhered to when creating scripts
-    - The blockchain package now provides a MedianTimeSource interface and
-      concrete implementation for providing time samples from remote peers
-      and using that data to calculate an offset against the local time
-  - Misc changes:
-    - Fix a slow memory leak due to tickers not being stopped
-      (https://github.com/btcsuite/btcd/issues/189)
-    - Fix an issue where a mix of orphans and SPV clients could trigger a
-      condition where peers would no longer be served
-      (https://github.com/btcsuite/btcd/issues/231)
-    - The RPC username and password can now contain symbols which previously
-      conflicted with special symbols used in URLs
-    - Improve handling of obtaining random nonces to prevent cases where it
-      could error when not enough entropy was available
-    - Improve handling of home directory creation errors such as in the case
-      of unmounted symlinks (https://github.com/btcsuite/btcd/issues/193)
-    - Improve the error reporting for rejected transactions to include the
-      inputs which are missing and/or being double spent
-    - Update sample config file with new options and correct a comment
-      regarding the fact the RPC server only listens on localhost by default
-      (https://github.com/btcsuite/btcd/issues/218)
-    - Update the continuous integration builds to run several tools which
-      help keep code quality high
-    - Significant amount of internal code cleanup and improvements
-    - Other minor internal optimizations
-  - Code Contributors (alphabetical order):
-    - Beldur
-    - Ben Holden-Crowther
-    - Dave Collins
-    - David Evans
-    - David Hill
-    - Guilherme Salgado
-    - Javed Khan
-    - Jimmy Song
-    - John C. Vernaleo
-    - Jonathan Gillham
-    - Josh Rickmar
-    - Michael Ford
-    - Michail Kargakis
-    - kac
-    - Olaoluwa Osuntokun
-
-Changes in 0.9.0 (Sat Sep 20 2014)
-  - Protocol and network related changes:
-    - Add a new checkpoint at block height 319400
-    - Add support for BIP0037 bloom filters
-      (https://github.com/conformal/btcd/issues/132)
-    - Implement BIP0061 reject handling and hence support for protocol
-      version 70002 (https://github.com/conformal/btcd/issues/133)
-    - Add testnet DNS seeds for peer discovery (testnet-seed.alexykot.me
-      and testnet-seed.bitcoin.schildbach.de)
-    - Add mainnet DNS seed for peer discovery (seeds.bitcoin.open-nodes.org)
-    - Make multisig transactions with non-null dummy data nonstandard
-      (https://github.com/conformal/btcd/issues/131)
-    - Make transactions with an excessive number of signature operations
-      nonstandard
-    - Perform initial DNS lookups concurrently which allows connections
-      more quickly
-    - Improve the address manager to significantly reduce memory usage and
-      add tests
-    - Remove orphan transactions when they appear in a mined block
-      (https://github.com/conformal/btcd/issues/166)
-    - Apply incremental back off on connection retries for persistent peers
-      that give invalid replies to mirror the logic used for failed
-      connections (https://github.com/conformal/btcd/issues/103)
-    - Correct rate-limiting of free and low-fee transactions
-  - Mining support changes:
-    - Implement getblocktemplate RPC with the following support:
-      (https://github.com/conformal/btcd/issues/124)
-      - BIP0022 Non-Optional Sections
-      - BIP0022 Long Polling
-      - BIP0023 Basic Pool Extensions
-      - BIP0023 Mutation coinbase/append
-      - BIP0023 Mutations time, time/increment, and time/decrement
-      - BIP0023 Mutation transactions/add
-      - BIP0023 Mutations prevblock, coinbase, and generation
-      - BIP0023 Block Proposals
-    - Implement built-in concurrent CPU miner
-      (https://github.com/conformal/btcd/issues/137)
-      NOTE: CPU mining on mainnet is pointless. This has been provided
-      for testing purposes such as for the new simulation test network
-    - Add --generate flag to enable CPU mining
-    - Deprecate the --getworkkey flag in favor of --miningaddr which
-      specifies which addresses generated blocks will choose from to pay
-      the subsidy to
-  - RPC changes:
-    - Implement gettxout command
-      (https://github.com/conformal/btcd/issues/141)
-    - Implement validateaddress command
-    - Implement verifymessage command
-    - Mark getunconfirmedbalance RPC as wallet-only
-    - Mark getwalletinfo RPC as wallet-only
-    - Update getgenerate, setgenerate, gethashespersec, and getmininginfo
-      to return the appropriate information about new CPU mining status
-    - Modify getpeerinfo pingtime and pingwait field types to float64 so
-      they are compatible
-    - Improve disconnect handling for normal HTTP clients
-    - Make error code returns for invalid hex more consistent
-  - Websocket changes:
-    - Switch to a new more efficient websocket package
-      (https://github.com/conformal/btcd/issues/134)
-    - Add rescanfinished notification
-    - Modify the rescanprogress notification to include block hash as well
-      as height (https://github.com/conformal/btcd/issues/151)
-  - btcctl utility changes:
-    - Accept --simnet flag which automatically selects the appropriate port
-      and TLS certificates needed to communicate with btcd and btcwallet on
-      the simulation test network
-    - Fix createrawtransaction command to send amounts denominated in BTC
-    - Add estimatefee command
-    - Add estimatepriority command
-    - Add getmininginfo command
-    - Add getnetworkinfo command
-    - Add gettxout command
-    - Add lockunspent command
-    - Add signrawtransaction command
-  - addblock utility changes:
-    - Accept --simnet flag which automatically selects the appropriate port
-      and TLS certificates needed to communicate with btcd and btcwallet on
-      the simulation test network
-  - Notable developer-related package changes:
-    - Provide a new bloom package in btcutil which allows creating and
-      working with BIP0037 bloom filters
-    - Provide a new hdkeychain package in btcutil which allows working with
-      BIP0032 hierarchical deterministic key chains
-    - Introduce a new btcnet package which houses network parameters
-    - Provide new simnet network (--simnet) which is useful for private
-      simulation testing
-    - Enforce low S values in serialized signatures as detailed in BIP0062
-    - Return errors from all methods on the btcdb.Db interface
-      (https://github.com/conformal/btcdb/issues/5)
-    - Allow behavior flags to alter btcchain.ProcessBlock
-      (https://github.com/conformal/btcchain/issues/5)
-    - Provide a new SerializeSize API for blocks
-      (https://github.com/conformal/btcwire/issues/19)
-    - Several of the core packages now work with Google App Engine
-  - Misc changes:
-    - Correct an issue where the database could corrupt under certain
-      circumstances which would require a new chain download
-    - Slightly optimize deserialization
-    - Use the correct IP block for he.net
-    - Fix an issue where it was possible the block manager could hang on
-      shutdown
-    - Update sample config file so the comments are on a separate line
-      rather than the end of a line so they are not interpreted as settings
-      (https://github.com/conformal/btcd/issues/135)
-    - Correct an issue where getdata requests were not being properly
-      throttled which could lead to larger than necessary memory usage
-    - Always show help when given the help flag even when the config file
-      contains invalid entries
-    - General code cleanup and minor optimizations
-
-Changes in 0.8.0-beta (Sun May 25 2014)
-  - Btcd is now Beta (https://github.com/conformal/btcd/issues/130)
-  - Add a new checkpoint at block height 300255
-  - Protocol and network related changes:
-    - Lower the minimum transaction relay fee to 1000 satoshi to match
-      recent reference client changes
-      (https://github.com/conformal/btcd/issues/100)
-    - Raise the maximum signature script size to support standard 15-of-15
-      multi-signature pay-to-sript-hash transactions with compressed pubkeys
-      to remain compatible with the reference client
-      (https://github.com/conformal/btcd/issues/128)
-    - Reduce max bytes allowed for a standard nulldata transaction to 40 for
-      compatibility with the reference client
-    - Introduce a new btcnet package which houses all of the network params
-      for each network (mainnet, testnet, regtest) to ultimately enable
-      easier addition and tweaking of networks without needing to change
-      several packages
-    - Fix several script discrepancies found by reference client test data
-    - Add new DNS seed for peer discovery (seed.bitnodes.io)
-    - Reduce the max known inventory cache from 20000 items to 1000 items
-    - Fix an issue where unknown inventory types could lead to a hung peer
-    - Implement inventory rebroadcast handler for sendrawtransaction
-      (https://github.com/conformal/btcd/issues/99)
-    - Update user agent to fully support BIP0014
-      (https://github.com/conformal/btcwire/issues/10)
-  - Implement initial mining support:
-    - Add a new logging subsystem for mining related operations
-    - Implement infrastructure for creating block templates
-    - Provide options to control block template creation settings
-    - Support the getwork RPC
-    - Allow address identifiers to apply to more than one network since both
-      testnet and the regression test network unfortunately use the same
-      identifier
-  - RPC changes:
-    - Set the content type for HTTP POST RPC connections to application/json
-      (https://github.com/conformal/btcd/issues/121)
-    - Modified the RPC server startup so it only requires at least one valid
-      listen interface
-    - Correct an error path where it was possible certain errors would not
-      be returned
-    - Implement getwork command
-      (https://github.com/conformal/btcd/issues/125)
-    - Update sendrawtransaction command to reject orphans
-    - Update sendrawtransaction command to include the reason a transaction
-      was rejected
-    - Update getinfo command to populate connection count field
-    - Update getinfo command to include relay fee field
-      (https://github.com/conformal/btcd/issues/107)
-    - Allow transactions submitted with sendrawtransaction to bypass the
-      rate limiter
-    - Allow the getcurrentnet and getbestblock extensions to be accessed via
-      HTTP POST in addition to Websockets
-      (https://github.com/conformal/btcd/issues/127)
-  - Websocket changes:
-    - Rework notifications to ensure they are delivered in the order they
-      occur
-    - Rename notifynewtxs command to notifyreceived (funds received)
-    - Rename notifyallnewtxs command to notifynewtransactions
-    - Rename alltx notification to txaccepted
-    - Rename allverbosetx notification to txacceptedverbose
-      (https://github.com/conformal/btcd/issues/98)
-    - Add rescan progress notification
-    - Add recvtx notification
-    - Add redeemingtx notification
-    - Modify notifyspent command to accept an array of outpoints
-      (https://github.com/conformal/btcd/issues/123)
-    - Significantly optimize the rescan command to yield up to a 60x speed
-      increase
-  - btcctl utility changes:
-    - Add createencryptedwallet command
-    - Add getblockchaininfo command
-    - Add importwallet command
-    - Add addmultisigaddress command
-    - Add setgenerate command
-    - Accept --testnet and --wallet flags which automatically select
-      the appropriate port and TLS certificates needed to communicate
-      with btcd and btcwallet (https://github.com/conformal/btcd/issues/112)
-    - Allow path expansion from config file entries
-      (https://github.com/conformal/btcd/issues/113)
-    - Minor refactor simplify handling of options
-  - addblock utility changes:
-    - Improve logging by making it consistent with the logging provided by
-      btcd (https://github.com/conformal/btcd/issues/90)
-  - Improve several package APIs for developers:
-    - Add new amount type for consistently handling monetary values
-    - Add new coin selector API
-    - Add new WIF (Wallet Import Format) API
-    - Add new crypto types for private keys and signatures
-    - Add new API to sign transactions including script merging and hash
-      types
-    - Expose function to extract all pushed data from a script
-      (https://github.com/conformal/btcscript/issues/8)
-  - Misc changes:
-    - Optimize address manager shuffling to do 67% less work on average
-    - Resolve a couple of benign data races found by the race detector
-      (https://github.com/conformal/btcd/issues/101)
-    - Add IP address to all peer related errors to clarify which peer is the
-      cause (https://github.com/conformal/btcd/issues/102)
-    - Fix a UPNP case issue that prevented the --upnp option from working
-      with some UPNP servers
-    - Update documentation in the sample config file regarding debug levels
-    - Adjust some logging levels to improve debug messages
-    - Improve the throughput of query messages to the block manager
-    - Several minor optimizations to reduce GC churn and enhance speed
-    - Other minor refactoring
-    - General code cleanup
-
-Changes in 0.7.0 (Thu Feb 20 2014)
-  - Fix an issue when parsing scripts which contain a multi-signature script
-    which require zero signatures such as testnet block
-    000000001881dccfeda317393c261f76d09e399e15e27d280e5368420f442632
-    (https://github.com/conformal/btcscript/issues/7)
-  - Add check to ensure all transactions accepted to mempool only contain
-    canonical data pushes (https://github.com/conformal/btcscript/issues/6)
-  - Fix an issue causing excessive memory consumption
-  - Significantly rework and improve the websocket notification system:
-    - Each client is now independent so slow clients no longer limit the
-      speed of other connected clients
-    - Potentially long-running operations such as rescans are now run in
-      their own handler and rate-limited to one operation at a time without
-      preventing simultaneous requests from the same client for the faster
-      requests or notifications
-    - A couple of scenarios which could cause shutdown to hang have been
-      resolved
-    - Update notifynewtx notifications to support all address types instead
-      of only pay-to-pubkey-hash
-    - Provide a --rpcmaxwebsockets option to allow limiting the number of
-      concurrent websocket clients
-    - Add a new websocket command notifyallnewtxs to request notifications
-      (https://github.com/conformal/btcd/issues/86) (thanks @flammit)
-  - Improve btcctl utility in the following ways:
-    - Add getnetworkhashps command
-    - Add gettransaction command (wallet-specific)
-    - Add signmessage command (wallet-specific)
-    - Update getwork command to accept
-  - Continue cleanup and work on implementing the RPC API:
-    - Implement getnettotals command
-      (https://github.com/conformal/btcd/issues/84)
-    - Implement networkhashps command
-      (https://github.com/conformal/btcd/issues/87)
-    - Update getpeerinfo to always include syncnode field even when false
-    - Remove help addenda for getpeerinfo now that it supports all fields
-  - Close standard RPC connections on auth failure
-  - Provide a --rpcmaxclients option to allow limiting the number of
-    concurrent RPC clients (https://github.com/conformal/btcd/issues/68)
-  - Include IP address in RPC auth failure log messages
-  - Resolve a rather harmless data races found by the race detector
-    (https://github.com/conformal/btcd/issues/94)
-  - Increase block priority size and max standard transaction size to 50k
-    and 100k, respectively (https://github.com/conformal/btcd/issues/71)
-  - Add rate limiting of free transactions to the memory pool to prevent
-    penny flooding (https://github.com/conformal/btcd/issues/40)
-  - Provide a --logdir option (https://github.com/conformal/btcd/issues/95)
-  - Change the default log file path to include the network
-  - Add a new ScriptBuilder interface to btcscript to support creation of
-    custom scripts (https://github.com/conformal/btcscript/issues/5)
-  - General code cleanup
-
-Changes in 0.6.0 (Tue Feb 04 2014)
-  - Fix an issue when parsing scripts which contain invalid signatures that
-    caused a chain fork on block
-    0000000000000001e4241fd0b3469a713f41c5682605451c05d3033288fb2244
-  - Correct an issue which could lead to an error in removeBlockNode
-    (https://github.com/conformal/btcchain/issues/4)
-  - Improve addblock utility as follows:
-    - Check imported blocks against all chain rules and checkpoints
-    - Skip blocks which are already known so you can stop and restart the
-      import or start the import after you have already downloaded a portion
-      of the chain
-    - Correct an issue where the utility did not shutdown cleanly after
-      processing all blocks
-    - Add error on attempt to import orphan blocks
-    - Improve error handling and reporting
-    - Display statistics after input file has been fully processed
-  - Rework, optimize, and improve headers-first mode:
-    - Resuming the chain sync from any point before the final checkpoint
-      will now use headers-first mode
-      (https://github.com/conformal/btcd/issues/69)
-    - Verify all checkpoints as opposed to only the final one
-    - Reduce and bound memory usage
-    - Rollback to the last known good point when a header does not match a
-      checkpoint
-    - Log information about what is happening with headers
-  - Improve btcctl utility in the following ways:
-    - Add getaddednodeinfo command
-    - Add getnettotals command
-    - Add getblocktemplate command (wallet-specific)
-    - Add getwork command (wallet-specific)
-    - Add getnewaddress command (wallet-specific)
-    - Add walletpassphrasechange command (wallet-specific)
-    - Add walletlock command (wallet-specific)
-    - Add sendfrom command (wallet-specific)
-    - Add sendmany command (wallet-specific)
-    - Add settxfee command (wallet-specific)
-    - Add listsinceblock command (wallet-specific)
-    - Add listaccounts command (wallet-specific)
-    - Add keypoolrefill command (wallet-specific)
-    - Add getreceivedbyaccount command (wallet-specific)
-    - Add getrawchangeaddress command (wallet-specific)
-    - Add gettxoutsetinfo command (wallet-specific)
-    - Add listaddressgroupings command (wallet-specific)
-    - Add listlockunspent command (wallet-specific)
-    - Add listlock command (wallet-specific)
-    - Add listreceivedbyaccount command (wallet-specific)
-    - Add validateaddress command (wallet-specific)
-    - Add verifymessage command (wallet-specific)
-    - Add sendtoaddress command (wallet-specific)
-  - Continue cleanup and work on implementing the RPC API:
-    - Implement submitblock command
-      (https://github.com/conformal/btcd/issues/61)
-    - Implement help command
-    - Implement ping command
-    - Implement getaddednodeinfo command
-      (https://github.com/conformal/btcd/issues/78)
-    - Implement getinfo command
-    - Update getpeerinfo to support bytesrecv and bytessent
-      (https://github.com/conformal/btcd/issues/83)
-  - Improve and correct several RPC server and websocket areas:
-    - Change the connection endpoint for websockets from /wallet to /ws
-      (https://github.com/conformal/btcd/issues/80)
-    - Implement an alternative authentication for websockets so clients
-      such as javascript from browsers that don't support setting HTTP
-      headers can authenticate (https://github.com/conformal/btcd/issues/77)
-    - Add an authentication deadline for RPC connections
-      (https://github.com/conformal/btcd/issues/68)
-    - Use standard authentication failure responses for RPC connections
-    - Make automatically generated certificate more standard so it works
-      from client such as node.js and Firefox
-    - Correct some minor issues which could prevent the RPC server from
-      shutting down in an orderly fashion
-    - Make all websocket notifications require registration
-    - Change the data sent over websockets to text since it is JSON-RPC
-    - Allow connections that do not have an Origin header set
-  - Expose and track the number of bytes read and written per peer
-    (https://github.com/conformal/btcwire/issues/6)
-  - Correct an issue with sendrawtransaction when invoked via websockets
-    which prevented a minedtx notification from being added
-  - Rescan operations issued from remote wallets are no stopped when
-    the wallet disconnects mid-operation
-    (https://github.com/conformal/btcd/issues/66)
-  - Several optimizations related to fetching block information from the
-    database
-  - General code cleanup
-
-Changes in 0.5.0 (Mon Jan 13 2014)
-  - Optimize initial block download by introducing a new mode which
-    downloads the block headers first (up to the final checkpoint)
-  - Improve peer handling to remove the potential for slow peers to cause
-    sluggishness amongst all peers
-    (https://github.com/conformal/btcd/issues/63)
-  - Fix an issue where the initial block sync could stall when the sync peer
-    disconnects (https://github.com/conformal/btcd/issues/62)
-  - Correct an issue where --externalip was doing a DNS lookup on the full
-    host:port instead of just the host portion
-    (https://github.com/conformal/btcd/issues/38)
-  - Fix an issue which could lead to a panic on chain switches
-    (https://github.com/conformal/btcd/issues/70)
-  - Improve btcctl utility in the following ways:
-    - Show getdifficulty output as floating point to 6 digits of precision
-    - Show all JSON object replies formatted as standard JSON
-    - Allow btcctl getblock to accept optional params
-    - Add getaccount command (wallet-specific)
-    - Add getaccountaddress command (wallet-specific)
-    - Add sendrawtransaction command
-  - Continue cleanup and work on implementing RPC API calls
-    - Update getrawmempool to support new optional verbose flag
-    - Update getrawtransaction to match the reference client
-    - Update getblock to support new optional verbose flag
-    - Update raw transactions to fully match the reference client including
-      support for all transaction types and address types
-    - Correct getrawmempool fee field to return BTC instead of Satoshi
-    - Correct getpeerinfo service flag to return 8 digit string so it
-      matches the reference client
-    - Correct verifychain to return a boolean
-    - Implement decoderawtransaction command
-    - Implement createrawtransaction command
-    - Implement decodescript command
-    - Implement gethashespersec command
-    - Allow RPC handler overrides when invoked via a websocket versus
-      legacy connection
-  - Add new DNS seed for peer discovery
-  - Display user agent on new valid peer log message
-    (https://github.com/conformal/btcd/issues/64)
-  - Notify wallet when new transactions that pay to registered addresses
-    show up in the mempool before being mined into a block
-  - Support a tor-specific proxy in addition to a normal proxy
-    (https://github.com/conformal/btcd/issues/47)
-  - Remove deprecated sqlite3 imports from utilities
-  - Remove leftover profile write from addblock utility
-  - Quite a bit of code cleanup and refactoring to improve maintainability
-
-Changes in 0.4.0 (Thu Dec 12 2013)
-  - Allow listen interfaces to be specified via --listen instead of only the
-    port (https://github.com/conformal/btcd/issues/33)
-  - Allow listen interfaces for the RPC server to be specified via
-    --rpclisten instead of only the port
-    (https://github.com/conformal/btcd/issues/34)
-  - Only disable listening when --connect or --proxy are used when no
-    --listen interface are specified
-    (https://github.com/conformal/btcd/issues/10)
-  - Add several new standard transaction checks to transaction memory pool:
-    - Support nulldata scripts as standard
-    - Only allow a max of one nulldata output per transaction
-    - Enforce a maximum of 3 public keys in multi-signature transactions
-    - The number of signatures in multi-signature transactions must not
-      exceed the number of public keys
-    - The number of inputs to a signature script must match the expected
-      number of inputs for the script type
-    - The number of inputs pushed onto the stack by a redeeming signature
-      script must match the number of inputs consumed by the referenced
-      public key script
-  - When a block is connected, remove any transactions from the memory pool
-    which are now double spends as a result of the newly connected
-    transactions
-  - Don't relay transactions resurrected during a chain switch since
-    other peers will also be switching chains and therefore already know
-    about them
-  - Cleanup a few cases where rejected transactions showed as an error
-    rather than as a rejected transaction
-  - Ignore the default configuration file when --regtest (regression test
-    mode) is specified
-  - Implement TLS support for RPC including automatic certificate generation
-  - Support HTTP authentication headers for web sockets
-  - Update address manager to recognize and properly work with Tor
-    addresses (https://github.com/conformal/btcd/issues/36) and
-    (https://github.com/conformal/btcd/issues/37)
-  - Improve btcctl utility in the following ways:
-    - Add the ability to specify a configuration file
-    - Add a default entry for the RPC cert to point to the location
-      it will likely be in the btcd home directory
-    - Implement --version flag
-    - Provide a --notls option to support non-TLS configurations
-  - Fix a couple of minor races found by the Go race detector
-  - Improve logging
-    - Allow logging level to be specified on a per subsystem basis
-      (https://github.com/conformal/btcd/issues/48)
-    - Allow logging levels to be dynamically changed via RPC
-      (https://github.com/conformal/btcd/issues/15)
-    - Implement a rolling log file with a max of 10MB per file and a
-      rotation size of 3 which results in a max logging size of 30 MB
-  - Correct a minor issue with the rescanning websocket call
-    (https://github.com/conformal/btcd/issues/54)
-  - Fix a race with pushing address messages that could lead to a panic
-    (https://github.com/conformal/btcd/issues/58)
-  - Improve which external IP address is reported to peers based on which
-    interface they are connected through
-    (https://github.com/conformal/btcd/issues/35)
-  - Add --externalip option to allow an external IP address to be specified
-    for cases such as tor hidden services or advanced network configurations
-    (https://github.com/conformal/btcd/issues/38)
-  - Add --upnp option to support automatic port mapping via UPnP
-    (https://github.com/conformal/btcd/issues/51)
-  - Update Ctrl+C interrupt handler to properly sync address manager and
-    remove the UPnP port mapping (if needed)
-  - Continue cleanup and work on implementing RPC API calls
-    - Add importprivkey (import private key) command to btcctl
-    - Update getrawtransaction to provide addresses properly, support
-      new verbose param, and match the reference implementation with the
-      exception of MULTISIG (thanks @flammit)
-    - Update getblock with new verbose flag (thanks @flammit)
-    - Add listtransactions command to btcctl
-    - Add getbalance command to btcctl
-  - Add basic support for btcd to run as a native Windows service
-    (https://github.com/conformal/btcd/issues/42)
-  - Package addblock utility with Windows MSIs
-  - Add support for TravisCI (continuous build integration)
-  - Cleanup some documentation and usage
-  - Several other minor bug fixes and general code cleanup
-
-Changes in 0.3.3 (Wed Nov 13 2013)
-  - Significantly improve initial block chain download speed
-    (https://github.com/conformal/btcd/issues/20)
-  - Add a new checkpoint at block height 267300
-  - Optimize most recently used inventory handling
-    (https://github.com/conformal/btcd/issues/21)
-  - Optimize duplicate transaction input check
-    (https://github.com/conformal/btcchain/issues/2)
-  - Optimize transaction hashing
-    (https://github.com/conformal/btcd/issues/25)
-  - Rework and optimize wallet listener notifications
-    (https://github.com/conformal/btcd/issues/22)
-  - Optimize serialization and deserialization
-    (https://github.com/conformal/btcd/issues/27)
-  - Add support for minimum transaction fee to memory pool acceptance
-    (https://github.com/conformal/btcd/issues/29)
-  - Improve leveldb database performance by removing explicit GC call
-  - Fix an issue where Ctrl+C was not always finishing orderly database
-    shutdown
-  - Fix an issue in the script handling for OP_CHECKSIG
-  - Impose max limits on all variable length protocol entries to prevent
-    abuse from malicious peers
-  - Enforce DER signatures for transactions allowed into the memory pool
-  - Separate the debug profile http server from the RPC server
-  - Rework of the RPC code to improve performance and make the code cleaner
-  - The getrawtransaction RPC call now properly checks the memory pool
-    before consulting the db (https://github.com/conformal/btcd/issues/26)
-  - Add support for the following RPC calls: getpeerinfo, getconnectedcount,
-    addnode, verifychain
-    (https://github.com/conformal/btcd/issues/13)
-    (https://github.com/conformal/btcd/issues/17)
-  - Implement rescan websocket extension to allow wallet rescans
-  - Use correct paths for application data storage for all supported
-    operating systems (https://github.com/conformal/btcd/issues/30)
-  - Add a default redirect to the http profiling page when accessing the
-    http profile server
-  - Add a new --cpuprofile option which can be used to generate CPU
-    profiling data on platforms that support it
-  - Several other minor performance optimizations
-  - Other minor bug fixes and general code cleanup
-
-Changes in 0.3.2 (Tue Oct 22 2013)
-  - Fix an issue that could cause the download of the block chain to stall
-    (https://github.com/conformal/btcd/issues/12)
-  - Remove deprecated sqlite as an available database backend
-  - Close sqlite compile issue as sqlite has now been removed
-    (https://github.com/conformal/btcd/issues/11)
-  - Change default RPC ports to 8334 (mainnet) and 18334 (testnet)
-  - Continue cleanup and work on implementing RPC API calls
-  - Add support for the following RPC calls: getrawmempool,
-    getbestblockhash, decoderawtransaction, getdifficulty,
-    getconnectioncount, getpeerinfo, and addnode
-  - Improve the btcctl utility that is used to issue JSON-RPC commands
-  - Fix an issue preventing btcd from cleanly shutting down with the RPC
-    stop command
-  - Add a number of database interface tests to ensure backends implement
-    the expected interface
-  - Expose some additional information from btcscript to be used for
-    identifying "standard"" transactions
-  - Add support for plan9 - thanks @mischief
-    (https://github.com/conformal/btcd/pull/19)
-  - Other minor bug fixes and general code cleanup
-
-Changes in 0.3.1-alpha (Tue Oct 15 2013)
-  - Change default database to leveldb
-    NOTE: This does mean you will have to redownload the block chain. Since we
-    are still in alpha, we didn't feel writing a converter was worth the time as
-    it would take away from more important issues at this stage
-  - Add a warning if there are multiple block chain databases of different types
-  - Fix issue with unexpected EOF in leveldb -- https://github.com/conformal/btcd/issues/18
-  - Fix issue preventing block 21066 on testnet -- https://github.com/conformal/btcchain/issues/1
-  - Fix issue preventing block 96464 on testnet -- https://github.com/conformal/btcscript/issues/1
-  - Optimize transaction lookups
-  - Correct a few cases of list removal that could result in improper cleanup
-    of no longer needed orphans
-  - Add functionality to increase ulimits on non-Windows platforms
-  - Add support for mempool command which allows remote peers to query the
-    transaction memory pool via the bitcoin protocol
-  - Clean up logging a bit
-  - Add a flag to disable checkpoints for developers
-  - Add a lot of useful debug logging such as message summaries
-  - Other minor bug fixes and general code cleanup
-
-Initial Release 0.3.0-alpha (Sat Oct 05 2013):
-  - Initial release

Jenkinsfile

@@ -1,10 +0,0 @@
-node {
-    stage 'Checkout'
-    checkout scm
-
-    stage 'Version'
-    sh './deploy.sh version'
-
-    stage 'Build'
-    sh "./deploy.sh build"
-}

LICENSE

@@ -3,6 +3,7 @@ ISC License
 Copyright (c) 2018-2019 The kaspanet developers
 Copyright (c) 2013-2018 The btcsuite developers
 Copyright (c) 2015-2016 The Decred developers
+Copyright (c) 2013-2014 Conformal Systems LLC.
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above

README.md

@@ -4,7 +4,7 @@ Kaspad
 Warning: This is pre-alpha software. There's no guarantee anything works.
 ====
 
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
 [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad)
 
 Kaspad is the reference full node Kaspa implementation written in Go (golang).
@@ -75,5 +75,5 @@ The documentation is a work-in-progress. It is located in the [docs](https://git
 
 ## License
 
-Kaspad is licensed under the [copyfree](http://copyfree.org) ISC License.
+Kaspad is licensed under the copyfree [ISC License](https://choosealicense.com/licenses/isc/).
 

addressmanager/addressmanager_test.go

@@ -2,29 +2,31 @@
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
-package addrmgr
+package addressmanager
 
 import (
-	"bou.ke/monkey"
 	"fmt"
-	"github.com/kaspanet/kaspad/config"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/pkg/errors"
+	"io/ioutil"
 	"net"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/mstime"
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/domainmessage"
 )
 
 // naTest is used to describe a test to be performed against the NetAddressKey
 // method.
 type naTest struct {
-	in   wire.NetAddress
-	want string
+	in   domainmessage.NetAddress
+	want AddressKey
 }
 
 // naTests houses all of the tests to be performed against the NetAddressKey
@@ -38,94 +40,114 @@ var someIP = "173.194.115.66"
 func addNaTests() {
 	// IPv4
 	// Localhost
-	addNaTest("127.0.0.1", 8333, "127.0.0.1:8333")
-	addNaTest("127.0.0.1", 8334, "127.0.0.1:8334")
+	addNaTest("127.0.0.1", 16111, "127.0.0.1:16111")
+	addNaTest("127.0.0.1", 16110, "127.0.0.1:16110")
 
 	// Class A
-	addNaTest("1.0.0.1", 8333, "1.0.0.1:8333")
-	addNaTest("2.2.2.2", 8334, "2.2.2.2:8334")
+	addNaTest("1.0.0.1", 16111, "1.0.0.1:16111")
+	addNaTest("2.2.2.2", 16110, "2.2.2.2:16110")
 	addNaTest("27.253.252.251", 8335, "27.253.252.251:8335")
 	addNaTest("123.3.2.1", 8336, "123.3.2.1:8336")
 
 	// Private Class A
-	addNaTest("10.0.0.1", 8333, "10.0.0.1:8333")
-	addNaTest("10.1.1.1", 8334, "10.1.1.1:8334")
+	addNaTest("10.0.0.1", 16111, "10.0.0.1:16111")
+	addNaTest("10.1.1.1", 16110, "10.1.1.1:16110")
 	addNaTest("10.2.2.2", 8335, "10.2.2.2:8335")
 	addNaTest("10.10.10.10", 8336, "10.10.10.10:8336")
 
 	// Class B
-	addNaTest("128.0.0.1", 8333, "128.0.0.1:8333")
-	addNaTest("129.1.1.1", 8334, "129.1.1.1:8334")
+	addNaTest("128.0.0.1", 16111, "128.0.0.1:16111")
+	addNaTest("129.1.1.1", 16110, "129.1.1.1:16110")
 	addNaTest("180.2.2.2", 8335, "180.2.2.2:8335")
 	addNaTest("191.10.10.10", 8336, "191.10.10.10:8336")
 
 	// Private Class B
-	addNaTest("172.16.0.1", 8333, "172.16.0.1:8333")
-	addNaTest("172.16.1.1", 8334, "172.16.1.1:8334")
+	addNaTest("172.16.0.1", 16111, "172.16.0.1:16111")
+	addNaTest("172.16.1.1", 16110, "172.16.1.1:16110")
 	addNaTest("172.16.2.2", 8335, "172.16.2.2:8335")
 	addNaTest("172.16.172.172", 8336, "172.16.172.172:8336")
 
 	// Class C
-	addNaTest("193.0.0.1", 8333, "193.0.0.1:8333")
-	addNaTest("200.1.1.1", 8334, "200.1.1.1:8334")
+	addNaTest("193.0.0.1", 16111, "193.0.0.1:16111")
+	addNaTest("200.1.1.1", 16110, "200.1.1.1:16110")
 	addNaTest("205.2.2.2", 8335, "205.2.2.2:8335")
 	addNaTest("223.10.10.10", 8336, "223.10.10.10:8336")
 
 	// Private Class C
-	addNaTest("192.168.0.1", 8333, "192.168.0.1:8333")
-	addNaTest("192.168.1.1", 8334, "192.168.1.1:8334")
+	addNaTest("192.168.0.1", 16111, "192.168.0.1:16111")
+	addNaTest("192.168.1.1", 16110, "192.168.1.1:16110")
 	addNaTest("192.168.2.2", 8335, "192.168.2.2:8335")
 	addNaTest("192.168.192.192", 8336, "192.168.192.192:8336")
 
 	// IPv6
 	// Localhost
-	addNaTest("::1", 8333, "[::1]:8333")
-	addNaTest("fe80::1", 8334, "[fe80::1]:8334")
+	addNaTest("::1", 16111, "[::1]:16111")
+	addNaTest("fe80::1", 16110, "[fe80::1]:16110")
 
 	// Link-local
-	addNaTest("fe80::1:1", 8333, "[fe80::1:1]:8333")
-	addNaTest("fe91::2:2", 8334, "[fe91::2:2]:8334")
+	addNaTest("fe80::1:1", 16111, "[fe80::1:1]:16111")
+	addNaTest("fe91::2:2", 16110, "[fe91::2:2]:16110")
 	addNaTest("fea2::3:3", 8335, "[fea2::3:3]:8335")
 	addNaTest("feb3::4:4", 8336, "[feb3::4:4]:8336")
 
 	// Site-local
-	addNaTest("fec0::1:1", 8333, "[fec0::1:1]:8333")
-	addNaTest("fed1::2:2", 8334, "[fed1::2:2]:8334")
+	addNaTest("fec0::1:1", 16111, "[fec0::1:1]:16111")
+	addNaTest("fed1::2:2", 16110, "[fed1::2:2]:16110")
 	addNaTest("fee2::3:3", 8335, "[fee2::3:3]:8335")
 	addNaTest("fef3::4:4", 8336, "[fef3::4:4]:8336")
 }
 
-func addNaTest(ip string, port uint16, want string) {
+func addNaTest(ip string, port uint16, want AddressKey) {
 	nip := net.ParseIP(ip)
-	na := *wire.NewNetAddressIPPort(nip, port, wire.SFNodeNetwork)
+	na := *domainmessage.NewNetAddressIPPort(nip, port, domainmessage.SFNodeNetwork)
 	test := naTest{na, want}
 	naTests = append(naTests, test)
 }
 
-func lookupFunc(host string) ([]net.IP, error) {
+func lookupFuncForTest(host string) ([]net.IP, error) {
 	return nil, errors.New("not implemented")
 }
 
+func newAddrManagerForTest(t *testing.T, testName string,
+	localSubnetworkID *subnetworkid.SubnetworkID) (addressManager *AddressManager, teardown func()) {
+
+	cfg := config.DefaultConfig()
+	cfg.SubnetworkID = localSubnetworkID
+
+	dbPath, err := ioutil.TempDir("", testName)
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+
+	databaseContext, err := dbaccess.New(dbPath)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	addressManager = New(cfg, databaseContext)
+
+	return addressManager, func() {
+		err := databaseContext.Close()
+		if err != nil {
+			t.Fatalf("error closing the database: %s", err)
+		}
+	}
+}
+
 func TestStartStop(t *testing.T) {
-	n := New("teststartstop", lookupFunc, nil)
-	n.Start()
-	err := n.Stop()
+	amgr, teardown := newAddrManagerForTest(t, "TestStartStop", nil)
+	defer teardown()
+	err := amgr.Start()
+	if err != nil {
+		t.Fatalf("Address Manager failed to start: %v", err)
+	}
+	err = amgr.Stop()
 	if err != nil {
 		t.Fatalf("Address Manager failed to stop: %v", err)
 	}
 }
 
 func TestAddAddressByIP(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
 	fmtErr := errors.Errorf("")
 	addrErr := &net.AddrError{}
 	var tests = []struct {
@@ -133,7 +155,7 @@ func TestAddAddressByIP(t *testing.T) {
 		err    error
 	}{
 		{
-			someIP + ":8333",
+			someIP + ":16111",
 			nil,
 		},
 		{
@@ -150,7 +172,8 @@ func TestAddAddressByIP(t *testing.T) {
 		},
 	}
 
-	amgr := New("testaddressbyip", nil, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
 	for i, test := range tests {
 		err := amgr.AddAddressByIP(test.addrIP, nil)
 		if test.err != nil && err == nil {
@@ -170,53 +193,44 @@ func TestAddAddressByIP(t *testing.T) {
 }
 
 func TestAddLocalAddress(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
 	var tests = []struct {
-		address  wire.NetAddress
+		address  domainmessage.NetAddress
 		priority AddressPriority
 		valid    bool
 	}{
 		{
-			wire.NetAddress{IP: net.ParseIP("192.168.0.100")},
+			domainmessage.NetAddress{IP: net.ParseIP("192.168.0.100")},
 			InterfacePrio,
 			false,
 		},
 		{
-			wire.NetAddress{IP: net.ParseIP("204.124.1.1")},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.1.1")},
 			InterfacePrio,
 			true,
 		},
 		{
-			wire.NetAddress{IP: net.ParseIP("204.124.1.1")},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.1.1")},
 			BoundPrio,
 			true,
 		},
 		{
-			wire.NetAddress{IP: net.ParseIP("::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("::1")},
 			InterfacePrio,
 			false,
 		},
 		{
-			wire.NetAddress{IP: net.ParseIP("fe80::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("fe80::1")},
 			InterfacePrio,
 			false,
 		},
 		{
-			wire.NetAddress{IP: net.ParseIP("2620:100::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("2620:100::1")},
 			InterfacePrio,
 			true,
 		},
 	}
-	amgr := New("testaddlocaladdress", nil, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestAddLocalAddress", nil)
+	defer teardown()
 	for x, test := range tests {
 		result := amgr.AddLocalAddress(&test.address, test.priority)
 		if result == nil && !test.valid {
@@ -233,31 +247,22 @@ func TestAddLocalAddress(t *testing.T) {
 }
 
 func TestAttempt(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
-	n := New("testattempt", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestAttempt", nil)
+	defer teardown()
 
 	// Add a new address and get it
-	err := n.AddAddressByIP(someIP+":8333", nil)
+	err := amgr.AddAddressByIP(someIP+":8333", nil)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka := n.GetAddress()
+	ka := amgr.GetAddress()
 
 	if !ka.LastAttempt().IsZero() {
 		t.Errorf("Address should not have attempts, but does")
 	}
 
 	na := ka.NetAddress()
-	n.Attempt(na)
+	amgr.Attempt(na)
 
 	if ka.LastAttempt().IsZero() {
 		t.Errorf("Address should have an attempt, but does not")
@@ -265,29 +270,20 @@ func TestAttempt(t *testing.T) {
 }
 
 func TestConnected(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
-	n := New("testconnected", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestConnected", nil)
+	defer teardown()
 
 	// Add a new address and get it
-	err := n.AddAddressByIP(someIP+":8333", nil)
+	err := amgr.AddAddressByIP(someIP+":8333", nil)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka := n.GetAddress()
+	ka := amgr.GetAddress()
 	na := ka.NetAddress()
 	// make it an hour ago
-	na.Timestamp = time.Unix(time.Now().Add(time.Hour*-1).Unix(), 0)
+	na.Timestamp = mstime.Now().Add(time.Hour * -1)
 
-	n.Connected(na)
+	amgr.Connected(na)
 
 	if !ka.NetAddress().Timestamp.After(na.Timestamp) {
 		t.Errorf("Address should have a new timestamp, but does not")
@@ -295,68 +291,50 @@ func TestConnected(t *testing.T) {
 }
 
 func TestNeedMoreAddresses(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
-	n := New("testneedmoreaddresses", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestNeedMoreAddresses", nil)
+	defer teardown()
 	addrsToAdd := 1500
-	b := n.NeedMoreAddresses()
+	b := amgr.NeedMoreAddresses()
 	if !b {
 		t.Errorf("Expected that we need more addresses")
 	}
-	addrs := make([]*wire.NetAddress, addrsToAdd)
+	addrs := make([]*domainmessage.NetAddress, addrsToAdd)
 
 	var err error
 	for i := 0; i < addrsToAdd; i++ {
-		s := fmt.Sprintf("%d.%d.173.147:8333", i/128+60, i%128+60)
-		addrs[i], err = n.DeserializeNetAddress(s)
+		s := AddressKey(fmt.Sprintf("%d.%d.173.147:8333", i/128+60, i%128+60))
+		addrs[i], err = amgr.DeserializeNetAddress(s)
 		if err != nil {
 			t.Errorf("Failed to turn %s into an address: %v", s, err)
 		}
 	}
 
-	srcAddr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+	srcAddr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 
-	n.AddAddresses(addrs, srcAddr, nil)
-	numAddrs := n.TotalNumAddresses()
+	amgr.AddAddresses(addrs, srcAddr, nil)
+	numAddrs := amgr.TotalNumAddresses()
 	if numAddrs > addrsToAdd {
 		t.Errorf("Number of addresses is too many %d vs %d", numAddrs, addrsToAdd)
 	}
 
-	b = n.NeedMoreAddresses()
+	b = amgr.NeedMoreAddresses()
 	if b {
 		t.Errorf("Expected that we don't need more addresses")
 	}
 }
 
 func TestGood(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
-	n := New("testgood", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestGood", nil)
+	defer teardown()
 	addrsToAdd := 64 * 64
-	addrs := make([]*wire.NetAddress, addrsToAdd)
+	addrs := make([]*domainmessage.NetAddress, addrsToAdd)
 	subnetworkCount := 32
 	subnetworkIDs := make([]*subnetworkid.SubnetworkID, subnetworkCount)
 
 	var err error
 	for i := 0; i < addrsToAdd; i++ {
-		s := fmt.Sprintf("%d.173.147.%d:8333", i/64+60, i%64+60)
-		addrs[i], err = n.DeserializeNetAddress(s)
+		s := AddressKey(fmt.Sprintf("%d.173.147.%d:8333", i/64+60, i%64+60))
+		addrs[i], err = amgr.DeserializeNetAddress(s)
 		if err != nil {
 			t.Errorf("Failed to turn %s into an address: %v", s, err)
 		}
@@ -366,26 +344,26 @@ func TestGood(t *testing.T) {
 		subnetworkIDs[i] = &subnetworkid.SubnetworkID{0xff - byte(i)}
 	}
 
-	srcAddr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+	srcAddr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 
-	n.AddAddresses(addrs, srcAddr, nil)
+	amgr.AddAddresses(addrs, srcAddr, nil)
 	for i, addr := range addrs {
-		n.Good(addr, subnetworkIDs[i%subnetworkCount])
+		amgr.Good(addr, subnetworkIDs[i%subnetworkCount])
 	}
 
-	numAddrs := n.TotalNumAddresses()
+	numAddrs := amgr.TotalNumAddresses()
 	if numAddrs >= addrsToAdd {
 		t.Errorf("Number of addresses is too many: %d vs %d", numAddrs, addrsToAdd)
 	}
 
-	numCache := len(n.AddressCache(true, nil))
+	numCache := len(amgr.AddressCache(true, nil))
 	if numCache == 0 || numCache >= numAddrs/4 {
 		t.Errorf("Number of addresses in cache: got %d, want positive and less than %d",
 			numCache, numAddrs/4)
 	}
 
 	for i := 0; i < subnetworkCount; i++ {
-		numCache = len(n.AddressCache(false, subnetworkIDs[i]))
+		numCache = len(amgr.AddressCache(false, subnetworkIDs[i]))
 		if numCache == 0 || numCache >= numAddrs/subnetworkCount {
 			t.Errorf("Number of addresses in subnetwork cache: got %d, want positive and less than %d",
 				numCache, numAddrs/4/subnetworkCount)
@@ -394,27 +372,18 @@ func TestGood(t *testing.T) {
 }
 
 func TestGoodChangeSubnetworkID(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
-	n := New("test_good_change_subnetwork_id", lookupFunc, nil)
-	addr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+	amgr, teardown := newAddrManagerForTest(t, "TestGoodChangeSubnetworkID", nil)
+	defer teardown()
+	addr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 	addrKey := NetAddressKey(addr)
-	srcAddr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+	srcAddr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 
 	oldSubnetwork := subnetworkid.SubnetworkIDNative
-	n.AddAddress(addr, srcAddr, oldSubnetwork)
-	n.Good(addr, oldSubnetwork)
+	amgr.AddAddress(addr, srcAddr, oldSubnetwork)
+	amgr.Good(addr, oldSubnetwork)
 
-	// make sure address was saved to addrIndex under oldSubnetwork
-	ka := n.find(addr)
+	// make sure address was saved to addressIndex under oldSubnetwork
+	ka := amgr.knownAddress(addr)
 	if ka == nil {
 		t.Fatalf("Address was not found after first time .Good called")
 	}
@@ -423,10 +392,10 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	}
 
 	// make sure address was added to correct bucket under oldSubnetwork
-	bucket := n.addrTried[*oldSubnetwork][n.getTriedBucket(addr)]
+	bucket := amgr.subnetworkTriedAddresBucketArrays[*oldSubnetwork][amgr.triedAddressBucketIndex(addr)]
 	wasFound := false
-	for e := bucket.Front(); e != nil; e = e.Next() {
-		if NetAddressKey(e.Value.(*KnownAddress).NetAddress()) == addrKey {
+	for _, ka := range bucket {
+		if NetAddressKey(ka.NetAddress()) == addrKey {
 			wasFound = true
 		}
 	}
@@ -436,10 +405,10 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 
 	// now call .Good again with a different subnetwork
 	newSubnetwork := subnetworkid.SubnetworkIDRegistry
-	n.Good(addr, newSubnetwork)
+	amgr.Good(addr, newSubnetwork)
 
-	// make sure address was updated in addrIndex under newSubnetwork
-	ka = n.find(addr)
+	// make sure address was updated in addressIndex under newSubnetwork
+	ka = amgr.knownAddress(addr)
 	if ka == nil {
 		t.Fatalf("Address was not found after second time .Good called")
 	}
@@ -448,10 +417,10 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	}
 
 	// make sure address was removed from bucket under oldSubnetwork
-	bucket = n.addrTried[*oldSubnetwork][n.getTriedBucket(addr)]
+	bucket = amgr.subnetworkTriedAddresBucketArrays[*oldSubnetwork][amgr.triedAddressBucketIndex(addr)]
 	wasFound = false
-	for e := bucket.Front(); e != nil; e = e.Next() {
-		if NetAddressKey(e.Value.(*KnownAddress).NetAddress()) == addrKey {
+	for _, ka := range bucket {
+		if NetAddressKey(ka.NetAddress()) == addrKey {
 			wasFound = true
 		}
 	}
@@ -460,10 +429,10 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	}
 
 	// make sure address was added to correct bucket under newSubnetwork
-	bucket = n.addrTried[*newSubnetwork][n.getTriedBucket(addr)]
+	bucket = amgr.subnetworkTriedAddresBucketArrays[*newSubnetwork][amgr.triedAddressBucketIndex(addr)]
 	wasFound = false
-	for e := bucket.Front(); e != nil; e = e.Next() {
-		if NetAddressKey(e.Value.(*KnownAddress).NetAddress()) == addrKey {
+	for _, ka := range bucket {
+		if NetAddressKey(ka.NetAddress()) == addrKey {
 			wasFound = true
 		}
 	}
@@ -473,45 +442,36 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 }
 
 func TestGetAddress(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
 	localSubnetworkID := &subnetworkid.SubnetworkID{0xff}
-	n := New("testgetaddress", lookupFunc, localSubnetworkID)
+	amgr, teardown := newAddrManagerForTest(t, "TestGetAddress", localSubnetworkID)
+	defer teardown()
 
 	// Get an address from an empty set (should error)
-	if rv := n.GetAddress(); rv != nil {
+	if rv := amgr.GetAddress(); rv != nil {
 		t.Errorf("GetAddress failed: got: %v want: %v\n", rv, nil)
 	}
 
 	// Add a new address and get it
-	err := n.AddAddressByIP(someIP+":8332", localSubnetworkID)
+	err := amgr.AddAddressByIP(someIP+":8332", localSubnetworkID)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka := n.GetAddress()
+	ka := amgr.GetAddress()
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
-	n.Attempt(ka.NetAddress())
+	amgr.Attempt(ka.NetAddress())
 
 	// Checks that we don't get it if we find that it has other subnetwork ID than expected.
 	actualSubnetworkID := &subnetworkid.SubnetworkID{0xfe}
-	n.Good(ka.NetAddress(), actualSubnetworkID)
-	ka = n.GetAddress()
+	amgr.Good(ka.NetAddress(), actualSubnetworkID)
+	ka = amgr.GetAddress()
 	if ka != nil {
 		t.Errorf("Didn't expect to get an address because there shouldn't be any address from subnetwork ID %s or nil", localSubnetworkID)
 	}
 
 	// Checks that the total number of addresses incremented although the new address is not full node or a partial node of the same subnetwork as the local node.
-	numAddrs := n.TotalNumAddresses()
+	numAddrs := amgr.TotalNumAddresses()
 	if numAddrs != 1 {
 		t.Errorf("Wrong number of addresses: got %d, want %d", numAddrs, 1)
 	}
@@ -519,11 +479,11 @@ func TestGetAddress(t *testing.T) {
 	// Now we repeat the same process, but now the address has the expected subnetwork ID.
 
 	// Add a new address and get it
-	err = n.AddAddressByIP(someIP+":8333", localSubnetworkID)
+	err = amgr.AddAddressByIP(someIP+":8333", localSubnetworkID)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka = n.GetAddress()
+	ka = amgr.GetAddress()
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
@@ -533,11 +493,11 @@ func TestGetAddress(t *testing.T) {
 	if !ka.SubnetworkID().IsEqual(localSubnetworkID) {
 		t.Errorf("Wrong Subnetwork ID: got %v, want %v", *ka.SubnetworkID(), localSubnetworkID)
 	}
-	n.Attempt(ka.NetAddress())
+	amgr.Attempt(ka.NetAddress())
 
 	// Mark this as a good address and get it
-	n.Good(ka.NetAddress(), localSubnetworkID)
-	ka = n.GetAddress()
+	amgr.Good(ka.NetAddress(), localSubnetworkID)
+	ka = amgr.GetAddress()
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
@@ -548,24 +508,14 @@ func TestGetAddress(t *testing.T) {
 		t.Errorf("Wrong Subnetwork ID: got %v, want %v", ka.SubnetworkID(), localSubnetworkID)
 	}
 
-	numAddrs = n.TotalNumAddresses()
+	numAddrs = amgr.TotalNumAddresses()
 	if numAddrs != 2 {
 		t.Errorf("Wrong number of addresses: got %d, want %d", numAddrs, 1)
 	}
 }
 
 func TestGetBestLocalAddress(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
-	localAddrs := []wire.NetAddress{
+	localAddrs := []domainmessage.NetAddress{
 		{IP: net.ParseIP("192.168.0.100")},
 		{IP: net.ParseIP("::1")},
 		{IP: net.ParseIP("fe80::1")},
@@ -573,48 +523,49 @@ func TestGetBestLocalAddress(t *testing.T) {
 	}
 
 	var tests = []struct {
-		remoteAddr wire.NetAddress
-		want0      wire.NetAddress
-		want1      wire.NetAddress
-		want2      wire.NetAddress
-		want3      wire.NetAddress
+		remoteAddr domainmessage.NetAddress
+		want0      domainmessage.NetAddress
+		want1      domainmessage.NetAddress
+		want2      domainmessage.NetAddress
+		want3      domainmessage.NetAddress
 	}{
 		{
 			// Remote connection from public IPv4
-			wire.NetAddress{IP: net.ParseIP("204.124.8.1")},
-			wire.NetAddress{IP: net.IPv4zero},
-			wire.NetAddress{IP: net.IPv4zero},
-			wire.NetAddress{IP: net.ParseIP("204.124.8.100")},
-			wire.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.8.1")},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.8.100")},
+			domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")},
 		},
 		{
 			// Remote connection from private IPv4
-			wire.NetAddress{IP: net.ParseIP("172.16.0.254")},
-			wire.NetAddress{IP: net.IPv4zero},
-			wire.NetAddress{IP: net.IPv4zero},
-			wire.NetAddress{IP: net.IPv4zero},
-			wire.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.ParseIP("172.16.0.254")},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
 		},
 		{
 			// Remote connection from public IPv6
-			wire.NetAddress{IP: net.ParseIP("2602:100:abcd::102")},
-			wire.NetAddress{IP: net.IPv6zero},
-			wire.NetAddress{IP: net.ParseIP("2001:470::1")},
-			wire.NetAddress{IP: net.ParseIP("2001:470::1")},
-			wire.NetAddress{IP: net.ParseIP("2001:470::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("2602:100:abcd::102")},
+			domainmessage.NetAddress{IP: net.IPv6zero},
+			domainmessage.NetAddress{IP: net.ParseIP("2001:470::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("2001:470::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("2001:470::1")},
 		},
 		/* XXX
 		{
 			// Remote connection from Tor
-			wire.NetAddress{IP: net.ParseIP("fd87:d87e:eb43::100")},
-			wire.NetAddress{IP: net.IPv4zero},
-			wire.NetAddress{IP: net.ParseIP("204.124.8.100")},
-			wire.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43::100")},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.8.100")},
+			domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")},
 		},
 		*/
 	}
 
-	amgr := New("testgetbestlocaladdress", nil, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestGetBestLocalAddress", nil)
+	defer teardown()
 
 	// Test against default when there's no address
 	for x, test := range tests {
@@ -641,7 +592,7 @@ func TestGetBestLocalAddress(t *testing.T) {
 	}
 
 	// Add a public IP to the list of local addresses.
-	localAddr := wire.NetAddress{IP: net.ParseIP("204.124.8.100")}
+	localAddr := domainmessage.NetAddress{IP: net.ParseIP("204.124.8.100")}
 	amgr.AddLocalAddress(&localAddr, InterfacePrio)
 
 	// Test against want2
@@ -655,9 +606,8 @@ func TestGetBestLocalAddress(t *testing.T) {
 	}
 	/*
 		// Add a Tor generated IP address
-		localAddr = wire.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")}
+		localAddr = domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")}
 		amgr.AddLocalAddress(&localAddr, ManualPrio)
-
 		// Test against want3
 		for x, test := range tests {
 			got := amgr.GetBestLocalAddress(&test.remoteAddr)

addressmanager/doc.go

@@ -1,9 +1,5 @@
-// Copyright (c) 2014 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
 /*
-Package addrmgr implements concurrency safe Kaspa address manager.
+Package addressmanager implements concurrency safe Kaspa address manager.
 
 Address Manager Overview
 
@@ -28,11 +24,11 @@ generally helps provide greater peer diversity, and perhaps more importantly,
 drastically reduces the chances an attacker is able to coerce your peer into
 only connecting to nodes they control.
 
-The address manager also understands routability and Tor addresses and tries
-hard to only return routable addresses. In addition, it uses the information
-provided by the caller about connected, known good, and attempted addresses to
-periodically purge peers which no longer appear to be good peers as well as
-bias the selection toward known good peers. The general idea is to make a best
-effort at only providing usable addresses.
+The address manager also understands routability and tries hard to only return
+routable addresses. In addition, it uses the information provided by the caller
+about connected, known good, and attempted addresses to periodically purge
+peers which no longer appear to be good peers as well as bias the selection
+toward known good peers. The general idea is to make a best effort at only
+providing usable addresses.
 */
-package addrmgr
+package addressmanager

addressmanager/internal_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2013-2015 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/mstime"
+)
+
+func TstKnownAddressIsBad(ka *KnownAddress) bool {
+	return ka.isBad()
+}
+
+func TstKnownAddressChance(ka *KnownAddress) float64 {
+	return ka.chance()
+}
+
+func TstNewKnownAddress(na *domainmessage.NetAddress, attempts int,
+	lastattempt, lastsuccess mstime.Time, tried bool, refs int) *KnownAddress {
+	return &KnownAddress{netAddress: na, attempts: attempts, lastAttempt: lastattempt,
+		lastSuccess: lastsuccess, tried: tried, referenceCount: refs}
+}

addressmanager/knownaddress.go

@@ -2,33 +2,36 @@
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
-package addrmgr
+package addressmanager
 
 import (
+	"github.com/kaspanet/kaspad/util/mstime"
 	"time"
 
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/kaspanet/kaspad/domainmessage"
 )
 
 // KnownAddress tracks information about a known network address that is used
 // to determine how viable an address is.
 type KnownAddress struct {
-	na           *wire.NetAddress
-	srcAddr      *wire.NetAddress
-	attempts     int
-	lastattempt  time.Time
-	lastsuccess  time.Time
-	tried        bool
-	refs         int // reference count of new buckets
-	subnetworkID *subnetworkid.SubnetworkID
+	netAddress     *domainmessage.NetAddress
+	sourceAddress  *domainmessage.NetAddress
+	attempts       int
+	lastAttempt    mstime.Time
+	lastSuccess    mstime.Time
+	tried          bool
+	referenceCount int // reference count of new buckets
+	subnetworkID   *subnetworkid.SubnetworkID
+	isBanned       bool
+	bannedTime     mstime.Time
 }
 
-// NetAddress returns the underlying wire.NetAddress associated with the
+// NetAddress returns the underlying domainmessage.NetAddress associated with the
 // known address.
-func (ka *KnownAddress) NetAddress() *wire.NetAddress {
-	return ka.na
+func (ka *KnownAddress) NetAddress() *domainmessage.NetAddress {
+	return ka.netAddress
 }
 
 // SubnetworkID returns the subnetwork ID of the known address.
@@ -37,16 +40,16 @@ func (ka *KnownAddress) SubnetworkID() *subnetworkid.SubnetworkID {
 }
 
 // LastAttempt returns the last time the known address was attempted.
-func (ka *KnownAddress) LastAttempt() time.Time {
-	return ka.lastattempt
+func (ka *KnownAddress) LastAttempt() mstime.Time {
+	return ka.lastAttempt
 }
 
 // chance returns the selection probability for a known address. The priority
 // depends upon how recently the address has been seen, how recently it was last
 // attempted and how often attempts to connect to it have failed.
 func (ka *KnownAddress) chance() float64 {
-	now := time.Now()
-	lastAttempt := now.Sub(ka.lastattempt)
+	now := mstime.Now()
+	lastAttempt := now.Sub(ka.lastAttempt)
 
 	if lastAttempt < 0 {
 		lastAttempt = 0
@@ -76,27 +79,27 @@ func (ka *KnownAddress) chance() float64 {
 // All addresses that meet these criteria are assumed to be worthless and not
 // worth keeping hold of.
 func (ka *KnownAddress) isBad() bool {
-	if ka.lastattempt.After(time.Now().Add(-1 * time.Minute)) {
+	if ka.lastAttempt.After(mstime.Now().Add(-1 * time.Minute)) {
 		return false
 	}
 
 	// From the future?
-	if ka.na.Timestamp.After(time.Now().Add(10 * time.Minute)) {
+	if ka.netAddress.Timestamp.After(mstime.Now().Add(10 * time.Minute)) {
 		return true
 	}
 
 	// Over a month old?
-	if ka.na.Timestamp.Before(time.Now().Add(-1 * numMissingDays * time.Hour * 24)) {
+	if ka.netAddress.Timestamp.Before(mstime.Now().Add(-1 * numMissingDays * time.Hour * 24)) {
 		return true
 	}
 
 	// Never succeeded?
-	if ka.lastsuccess.IsZero() && ka.attempts >= numRetries {
+	if ka.lastSuccess.IsZero() && ka.attempts >= numRetries {
 		return true
 	}
 
 	// Hasn't succeeded in too long?
-	if !ka.lastsuccess.After(time.Now().Add(-1*minBadDays*time.Hour*24)) &&
+	if !ka.lastSuccess.After(mstime.Now().Add(-1*minBadDays*time.Hour*24)) &&
 		ka.attempts >= maxFailures {
 		return true
 	}

addressmanager/knownaddress_test.go

@@ -0,0 +1,115 @@
+// Copyright (c) 2013-2015 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager_test
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/kaspanet/kaspad/addressmanager"
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+func TestChance(t *testing.T) {
+	now := mstime.Now()
+	var tests = []struct {
+		addr     *addressmanager.KnownAddress
+		expected float64
+	}{
+		{
+			//Test normal case
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				0, mstime.Now().Add(-30*time.Minute), mstime.Now(), false, 0),
+			1.0,
+		}, {
+			//Test case in which lastseen < 0
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(20 * time.Second)},
+				0, mstime.Now().Add(-30*time.Minute), mstime.Now(), false, 0),
+			1.0,
+		}, {
+			//Test case in which lastAttempt < 0
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				0, mstime.Now().Add(30*time.Minute), mstime.Now(), false, 0),
+			1.0 * .01,
+		}, {
+			//Test case in which lastAttempt < ten minutes
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				0, mstime.Now().Add(-5*time.Minute), mstime.Now(), false, 0),
+			1.0 * .01,
+		}, {
+			//Test case with several failed attempts.
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				2, mstime.Now().Add(-30*time.Minute), mstime.Now(), false, 0),
+			1 / 1.5 / 1.5,
+		},
+	}
+
+	err := .0001
+	for i, test := range tests {
+		chance := addressmanager.TstKnownAddressChance(test.addr)
+		if math.Abs(test.expected-chance) >= err {
+			t.Errorf("case %d: got %f, expected %f", i, chance, test.expected)
+		}
+	}
+}
+
+func TestIsBad(t *testing.T) {
+	now := mstime.Now()
+	future := now.Add(35 * time.Minute)
+	monthOld := now.Add(-43 * time.Hour * 24)
+	secondsOld := now.Add(-2 * time.Second)
+	minutesOld := now.Add(-27 * time.Minute)
+	hoursOld := now.Add(-5 * time.Hour)
+	zeroTime := mstime.Time{}
+
+	futureNa := &domainmessage.NetAddress{Timestamp: future}
+	minutesOldNa := &domainmessage.NetAddress{Timestamp: minutesOld}
+	monthOldNa := &domainmessage.NetAddress{Timestamp: monthOld}
+	currentNa := &domainmessage.NetAddress{Timestamp: secondsOld}
+
+	//Test addresses that have been tried in the last minute.
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(futureNa, 3, secondsOld, zeroTime, false, 0)) {
+		t.Errorf("test case 1: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(monthOldNa, 3, secondsOld, zeroTime, false, 0)) {
+		t.Errorf("test case 2: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(currentNa, 3, secondsOld, zeroTime, false, 0)) {
+		t.Errorf("test case 3: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(currentNa, 3, secondsOld, monthOld, true, 0)) {
+		t.Errorf("test case 4: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(currentNa, 2, secondsOld, secondsOld, true, 0)) {
+		t.Errorf("test case 5: addresses that have been tried in the last minute are not bad.")
+	}
+
+	//Test address that claims to be from the future.
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(futureNa, 0, minutesOld, hoursOld, true, 0)) {
+		t.Errorf("test case 6: addresses that claim to be from the future are bad.")
+	}
+
+	//Test address that has not been seen in over a month.
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(monthOldNa, 0, minutesOld, hoursOld, true, 0)) {
+		t.Errorf("test case 7: addresses more than a month old are bad.")
+	}
+
+	//It has failed at least three times and never succeeded.
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(minutesOldNa, 3, minutesOld, zeroTime, true, 0)) {
+		t.Errorf("test case 8: addresses that have never succeeded are bad.")
+	}
+
+	//It has failed ten times in the last week
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(minutesOldNa, 10, minutesOld, monthOld, true, 0)) {
+		t.Errorf("test case 9: addresses that have not succeeded in too long are bad.")
+	}
+
+	//Test an address that should work.
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(minutesOldNa, 2, minutesOld, hoursOld, true, 0)) {
+		t.Errorf("test case 10: This should be a valid address.")
+	}
+}

addressmanager/log.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
-package addrmgr
+package addressmanager
 
 import (
 	"github.com/kaspanet/kaspad/logger"
@@ -10,4 +10,4 @@ import (
 )
 
 var log, _ = logger.Get(logger.SubsystemTags.ADXR)
-var spawn = panics.GoroutineWrapperFuncWithPanicHandler(log)
+var spawn = panics.GoroutineWrapperFunc(log)

addressmanager/network.go

@@ -2,15 +2,12 @@
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
-package addrmgr
+package addressmanager
 
 import (
-	"fmt"
 	"net"
 
-	"github.com/kaspanet/kaspad/config"
-
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/kaspanet/kaspad/domainmessage"
 )
 
 var (
@@ -73,19 +70,6 @@ var (
 	// rfc6598Net specifies the IPv4 block as defined by RFC6598 (100.64.0.0/10)
 	rfc6598Net = ipNet("100.64.0.0", 10, 32)
 
-	// onionCatNet defines the IPv6 address block used to support Tor.
-	// We encode a .onion address as a 16 byte number by decoding the
-	// address prior to the .onion (i.e. the key hash) base32 into a ten
-	// byte number. It then stores the first 6 bytes of the address as
-	// 0xfd, 0x87, 0xd8, 0x7e, 0xeb, 0x43.
-	//
-	// This is the same range used by OnionCat, which is part part of the
-	// RFC4193 unique local IPv6 range.
-	//
-	// In summary the format is:
-	// { magic 6 bytes, 10 bytes base32 decode of key hash }
-	onionCatNet = ipNet("fd87:d87e:eb43::", 48, 128)
-
 	// zero4Net defines the IPv4 address block for address staring with 0
 	// (0.0.0.0/8).
 	zero4Net = ipNet("0.0.0.0", 8, 32)
@@ -102,27 +86,19 @@ func ipNet(ip string, ones, bits int) net.IPNet {
 }
 
 // IsIPv4 returns whether or not the given address is an IPv4 address.
-func IsIPv4(na *wire.NetAddress) bool {
+func IsIPv4(na *domainmessage.NetAddress) bool {
 	return na.IP.To4() != nil
 }
 
 // IsLocal returns whether or not the given address is a local address.
-func IsLocal(na *wire.NetAddress) bool {
+func IsLocal(na *domainmessage.NetAddress) bool {
 	return na.IP.IsLoopback() || zero4Net.Contains(na.IP)
 }
 
-// IsOnionCatTor returns whether or not the passed address is in the IPv6 range
-// used by Kaspa to support Tor (fd87:d87e:eb43::/48). Note that this range
-// is the same range used by OnionCat, which is part of the RFC4193 unique local
-// IPv6 range.
-func IsOnionCatTor(na *wire.NetAddress) bool {
-	return onionCatNet.Contains(na.IP)
-}
-
 // IsRFC1918 returns whether or not the passed address is part of the IPv4
 // private network address space as defined by RFC1918 (10.0.0.0/8,
 // 172.16.0.0/12, or 192.168.0.0/16).
-func IsRFC1918(na *wire.NetAddress) bool {
+func IsRFC1918(na *domainmessage.NetAddress) bool {
 	for _, rfc := range rfc1918Nets {
 		if rfc.Contains(na.IP) {
 			return true
@@ -133,56 +109,56 @@ func IsRFC1918(na *wire.NetAddress) bool {
 
 // IsRFC2544 returns whether or not the passed address is part of the IPv4
 // address space as defined by RFC2544 (198.18.0.0/15)
-func IsRFC2544(na *wire.NetAddress) bool {
+func IsRFC2544(na *domainmessage.NetAddress) bool {
 	return rfc2544Net.Contains(na.IP)
 }
 
 // IsRFC3849 returns whether or not the passed address is part of the IPv6
 // documentation range as defined by RFC3849 (2001:DB8::/32).
-func IsRFC3849(na *wire.NetAddress) bool {
+func IsRFC3849(na *domainmessage.NetAddress) bool {
 	return rfc3849Net.Contains(na.IP)
 }
 
 // IsRFC3927 returns whether or not the passed address is part of the IPv4
 // autoconfiguration range as defined by RFC3927 (169.254.0.0/16).
-func IsRFC3927(na *wire.NetAddress) bool {
+func IsRFC3927(na *domainmessage.NetAddress) bool {
 	return rfc3927Net.Contains(na.IP)
 }
 
 // IsRFC3964 returns whether or not the passed address is part of the IPv6 to
 // IPv4 encapsulation range as defined by RFC3964 (2002::/16).
-func IsRFC3964(na *wire.NetAddress) bool {
+func IsRFC3964(na *domainmessage.NetAddress) bool {
 	return rfc3964Net.Contains(na.IP)
 }
 
 // IsRFC4193 returns whether or not the passed address is part of the IPv6
 // unique local range as defined by RFC4193 (FC00::/7).
-func IsRFC4193(na *wire.NetAddress) bool {
+func IsRFC4193(na *domainmessage.NetAddress) bool {
 	return rfc4193Net.Contains(na.IP)
 }
 
 // IsRFC4380 returns whether or not the passed address is part of the IPv6
 // teredo tunneling over UDP range as defined by RFC4380 (2001::/32).
-func IsRFC4380(na *wire.NetAddress) bool {
+func IsRFC4380(na *domainmessage.NetAddress) bool {
 	return rfc4380Net.Contains(na.IP)
 }
 
 // IsRFC4843 returns whether or not the passed address is part of the IPv6
 // ORCHID range as defined by RFC4843 (2001:10::/28).
-func IsRFC4843(na *wire.NetAddress) bool {
+func IsRFC4843(na *domainmessage.NetAddress) bool {
 	return rfc4843Net.Contains(na.IP)
 }
 
 // IsRFC4862 returns whether or not the passed address is part of the IPv6
 // stateless address autoconfiguration range as defined by RFC4862 (FE80::/64).
-func IsRFC4862(na *wire.NetAddress) bool {
+func IsRFC4862(na *domainmessage.NetAddress) bool {
 	return rfc4862Net.Contains(na.IP)
 }
 
 // IsRFC5737 returns whether or not the passed address is part of the IPv4
 // documentation address space as defined by RFC5737 (192.0.2.0/24,
 // 198.51.100.0/24, 203.0.113.0/24)
-func IsRFC5737(na *wire.NetAddress) bool {
+func IsRFC5737(na *domainmessage.NetAddress) bool {
 	for _, rfc := range rfc5737Net {
 		if rfc.Contains(na.IP) {
 			return true
@@ -194,19 +170,19 @@ func IsRFC5737(na *wire.NetAddress) bool {
 
 // IsRFC6052 returns whether or not the passed address is part of the IPv6
 // well-known prefix range as defined by RFC6052 (64:FF9B::/96).
-func IsRFC6052(na *wire.NetAddress) bool {
+func IsRFC6052(na *domainmessage.NetAddress) bool {
 	return rfc6052Net.Contains(na.IP)
 }
 
 // IsRFC6145 returns whether or not the passed address is part of the IPv6 to
 // IPv4 translated address range as defined by RFC6145 (::FFFF:0:0:0/96).
-func IsRFC6145(na *wire.NetAddress) bool {
+func IsRFC6145(na *domainmessage.NetAddress) bool {
 	return rfc6145Net.Contains(na.IP)
 }
 
 // IsRFC6598 returns whether or not the passed address is part of the IPv4
 // shared address space specified by RFC6598 (100.64.0.0/10)
-func IsRFC6598(na *wire.NetAddress) bool {
+func IsRFC6598(na *domainmessage.NetAddress) bool {
 	return rfc6598Net.Contains(na.IP)
 }
 
@@ -214,7 +190,7 @@ func IsRFC6598(na *wire.NetAddress) bool {
 // considered invalid under the following circumstances:
 // IPv4: It is either a zero or all bits set address.
 // IPv6: It is either a zero or RFC3849 documentation address.
-func IsValid(na *wire.NetAddress) bool {
+func IsValid(na *domainmessage.NetAddress) bool {
 	// IsUnspecified returns if address is 0, so only all bits set, and
 	// RFC3849 need to be explicitly checked.
 	return na.IP != nil && !(na.IP.IsUnspecified() ||
@@ -224,27 +200,26 @@ func IsValid(na *wire.NetAddress) bool {
 // IsRoutable returns whether or not the passed address is routable over
 // the public internet. This is true as long as the address is valid and is not
 // in any reserved ranges.
-func IsRoutable(na *wire.NetAddress) bool {
-	if config.ActiveConfig().NetParams().AcceptUnroutable {
+func (am *AddressManager) IsRoutable(na *domainmessage.NetAddress) bool {
+	if am.cfg.NetParams().AcceptUnroutable {
 		return !IsLocal(na)
 	}
 
 	return IsValid(na) && !(IsRFC1918(na) || IsRFC2544(na) ||
 		IsRFC3927(na) || IsRFC4862(na) || IsRFC3849(na) ||
 		IsRFC4843(na) || IsRFC5737(na) || IsRFC6598(na) ||
-		IsLocal(na) || (IsRFC4193(na) && !IsOnionCatTor(na)))
+		IsLocal(na) || (IsRFC4193(na)))
 }
 
 // GroupKey returns a string representing the network group an address is part
 // of. This is the /16 for IPv4, the /32 (/36 for he.net) for IPv6, the string
-// "local" for a local address, the string "tor:key" where key is the /4 of the
-// onion address for Tor address, and the string "unroutable" for an unroutable
+// "local" for a local address, and the string "unroutable" for an unroutable
 // address.
-func GroupKey(na *wire.NetAddress) string {
+func (am *AddressManager) GroupKey(na *domainmessage.NetAddress) string {
 	if IsLocal(na) {
 		return "local"
 	}
-	if !IsRoutable(na) {
+	if !am.IsRoutable(na) {
 		return "unroutable"
 	}
 	if IsIPv4(na) {
@@ -270,10 +245,6 @@ func GroupKey(na *wire.NetAddress) string {
 		}
 		return ip.Mask(net.CIDRMask(16, 32)).String()
 	}
-	if IsOnionCatTor(na) {
-		// group is keyed off the first 4 bits of the actual onion key.
-		return fmt.Sprintf("tor:%d", na.IP[6]&((1<<4)-1))
-	}
 
 	// OK, so now we know ourselves to be a IPv6 address.
 	// We use /32 for everything, except for Hurricane Electric's

addressmanager/network_test.go

@@ -2,34 +2,22 @@
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
-package addrmgr_test
+package addressmanager
 
 import (
-	"bou.ke/monkey"
-	"github.com/kaspanet/kaspad/config"
-	"github.com/kaspanet/kaspad/dagconfig"
 	"net"
 	"testing"
 
-	"github.com/kaspanet/kaspad/addrmgr"
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/kaspanet/kaspad/domainmessage"
 )
 
 // TestIPTypes ensures the various functions which determine the type of an IP
 // address based on RFCs work as intended.
 func TestIPTypes(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
-
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
 	type ipTest struct {
-		in       wire.NetAddress
+		in       domainmessage.NetAddress
 		rfc1918  bool
 		rfc2544  bool
 		rfc3849  bool
@@ -52,7 +40,7 @@ func TestIPTypes(t *testing.T) {
 		rfc4193, rfc4380, rfc4843, rfc4862, rfc5737, rfc6052, rfc6145, rfc6598,
 		local, valid, routable bool) ipTest {
 		nip := net.ParseIP(ip)
-		na := *wire.NewNetAddressIPPort(nip, 8333, wire.SFNodeNetwork)
+		na := *domainmessage.NewNetAddressIPPort(nip, 16111, domainmessage.SFNodeNetwork)
 		test := ipTest{na, rfc1918, rfc2544, rfc3849, rfc3927, rfc3964, rfc4193, rfc4380,
 			rfc4843, rfc4862, rfc5737, rfc6052, rfc6145, rfc6598, local, valid, routable}
 		return test
@@ -101,55 +89,55 @@ func TestIPTypes(t *testing.T) {
 
 	t.Logf("Running %d tests", len(tests))
 	for _, test := range tests {
-		if rv := addrmgr.IsRFC1918(&test.in); rv != test.rfc1918 {
+		if rv := IsRFC1918(&test.in); rv != test.rfc1918 {
 			t.Errorf("IsRFC1918 %s\n got: %v want: %v", test.in.IP, rv, test.rfc1918)
 		}
 
-		if rv := addrmgr.IsRFC3849(&test.in); rv != test.rfc3849 {
+		if rv := IsRFC3849(&test.in); rv != test.rfc3849 {
 			t.Errorf("IsRFC3849 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3849)
 		}
 
-		if rv := addrmgr.IsRFC3927(&test.in); rv != test.rfc3927 {
+		if rv := IsRFC3927(&test.in); rv != test.rfc3927 {
 			t.Errorf("IsRFC3927 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3927)
 		}
 
-		if rv := addrmgr.IsRFC3964(&test.in); rv != test.rfc3964 {
+		if rv := IsRFC3964(&test.in); rv != test.rfc3964 {
 			t.Errorf("IsRFC3964 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3964)
 		}
 
-		if rv := addrmgr.IsRFC4193(&test.in); rv != test.rfc4193 {
+		if rv := IsRFC4193(&test.in); rv != test.rfc4193 {
 			t.Errorf("IsRFC4193 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4193)
 		}
 
-		if rv := addrmgr.IsRFC4380(&test.in); rv != test.rfc4380 {
+		if rv := IsRFC4380(&test.in); rv != test.rfc4380 {
 			t.Errorf("IsRFC4380 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4380)
 		}
 
-		if rv := addrmgr.IsRFC4843(&test.in); rv != test.rfc4843 {
+		if rv := IsRFC4843(&test.in); rv != test.rfc4843 {
 			t.Errorf("IsRFC4843 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4843)
 		}
 
-		if rv := addrmgr.IsRFC4862(&test.in); rv != test.rfc4862 {
+		if rv := IsRFC4862(&test.in); rv != test.rfc4862 {
 			t.Errorf("IsRFC4862 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4862)
 		}
 
-		if rv := addrmgr.IsRFC6052(&test.in); rv != test.rfc6052 {
+		if rv := IsRFC6052(&test.in); rv != test.rfc6052 {
 			t.Errorf("isRFC6052 %s\n got: %v want: %v", test.in.IP, rv, test.rfc6052)
 		}
 
-		if rv := addrmgr.IsRFC6145(&test.in); rv != test.rfc6145 {
+		if rv := IsRFC6145(&test.in); rv != test.rfc6145 {
 			t.Errorf("IsRFC1918 %s\n got: %v want: %v", test.in.IP, rv, test.rfc6145)
 		}
 
-		if rv := addrmgr.IsLocal(&test.in); rv != test.local {
+		if rv := IsLocal(&test.in); rv != test.local {
 			t.Errorf("IsLocal %s\n got: %v want: %v", test.in.IP, rv, test.local)
 		}
 
-		if rv := addrmgr.IsValid(&test.in); rv != test.valid {
+		if rv := IsValid(&test.in); rv != test.valid {
 			t.Errorf("IsValid %s\n got: %v want: %v", test.in.IP, rv, test.valid)
 		}
 
-		if rv := addrmgr.IsRoutable(&test.in); rv != test.routable {
+		if rv := amgr.IsRoutable(&test.in); rv != test.routable {
 			t.Errorf("IsRoutable %s\n got: %v want: %v", test.in.IP, rv, test.routable)
 		}
 	}
@@ -158,15 +146,8 @@ func TestIPTypes(t *testing.T) {
 // TestGroupKey tests the GroupKey function to ensure it properly groups various
 // IP addresses.
 func TestGroupKey(t *testing.T) {
-	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
-		return &config.Config{
-			Flags: &config.Flags{
-				NetworkFlags: config.NetworkFlags{
-					ActiveNetParams: &dagconfig.SimNetParams},
-			},
-		}
-	})
-	defer activeConfigPatch.Unpatch()
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
 
 	tests := []struct {
 		name     string
@@ -202,9 +183,9 @@ func TestGroupKey(t *testing.T) {
 		{name: "ipv6 rfc6145 translated ipv4", ip: "::ffff:0:0c01:0203", expected: "12.1.0.0"},
 
 		// Tor.
-		{name: "ipv6 tor onioncat", ip: "fd87:d87e:eb43:1234::5678", expected: "tor:2"},
-		{name: "ipv6 tor onioncat 2", ip: "fd87:d87e:eb43:1245::6789", expected: "tor:2"},
-		{name: "ipv6 tor onioncat 3", ip: "fd87:d87e:eb43:1345::6789", expected: "tor:3"},
+		{name: "ipv6 tor onioncat", ip: "fd87:d87e:eb43:1234::5678", expected: "unroutable"},
+		{name: "ipv6 tor onioncat 2", ip: "fd87:d87e:eb43:1245::6789", expected: "unroutable"},
+		{name: "ipv6 tor onioncat 3", ip: "fd87:d87e:eb43:1345::6789", expected: "unroutable"},
 
 		// IPv6 normal.
 		{name: "ipv6 normal", ip: "2602:100::1", expected: "2602:100::"},
@@ -215,8 +196,8 @@ func TestGroupKey(t *testing.T) {
 
 	for i, test := range tests {
 		nip := net.ParseIP(test.ip)
-		na := *wire.NewNetAddressIPPort(nip, 8333, wire.SFNodeNetwork)
-		if key := addrmgr.GroupKey(&na); key != test.expected {
+		na := *domainmessage.NewNetAddressIPPort(nip, 8333, domainmessage.SFNodeNetwork)
+		if key := amgr.GroupKey(&na); key != test.expected {
 			t.Errorf("TestGroupKey #%d (%s): unexpected group key "+
 				"- got '%s', want '%s'", i, test.name,
 				key, test.expected)

addrmgr/cov_report.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-# This script uses gocov to generate a test coverage report.
-# The gocov tool my be obtained with the following command:
-#   go get github.com/axw/gocov/gocov
-#
-# It will be installed to $GOPATH/bin, so ensure that location is in your $PATH.
-
-# Check for gocov.
-type gocov >/dev/null 2>&1
-if [ $? -ne 0 ]; then
-	echo >&2 "This script requires the gocov tool."
-	echo >&2 "You may obtain it with the following command:"
-	echo >&2 "go get github.com/axw/gocov/gocov"
-	exit 1
-fi
-gocov test | gocov report

addrmgr/internal_test.go

@@ -1,25 +0,0 @@
-// Copyright (c) 2013-2015 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package addrmgr
-
-import (
-	"time"
-
-	"github.com/kaspanet/kaspad/wire"
-)
-
-func TstKnownAddressIsBad(ka *KnownAddress) bool {
-	return ka.isBad()
-}
-
-func TstKnownAddressChance(ka *KnownAddress) float64 {
-	return ka.chance()
-}
-
-func TstNewKnownAddress(na *wire.NetAddress, attempts int,
-	lastattempt, lastsuccess time.Time, tried bool, refs int) *KnownAddress {
-	return &KnownAddress{na: na, attempts: attempts, lastattempt: lastattempt,
-		lastsuccess: lastsuccess, tried: tried, refs: refs}
-}

addrmgr/knownaddress_test.go

@@ -1,114 +0,0 @@
-// Copyright (c) 2013-2015 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package addrmgr_test
-
-import (
-	"math"
-	"testing"
-	"time"
-
-	"github.com/kaspanet/kaspad/addrmgr"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-func TestChance(t *testing.T) {
-	now := time.Unix(time.Now().Unix(), 0)
-	var tests = []struct {
-		addr     *addrmgr.KnownAddress
-		expected float64
-	}{
-		{
-			//Test normal case
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				0, time.Now().Add(-30*time.Minute), time.Now(), false, 0),
-			1.0,
-		}, {
-			//Test case in which lastseen < 0
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(20 * time.Second)},
-				0, time.Now().Add(-30*time.Minute), time.Now(), false, 0),
-			1.0,
-		}, {
-			//Test case in which lastattempt < 0
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				0, time.Now().Add(30*time.Minute), time.Now(), false, 0),
-			1.0 * .01,
-		}, {
-			//Test case in which lastattempt < ten minutes
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				0, time.Now().Add(-5*time.Minute), time.Now(), false, 0),
-			1.0 * .01,
-		}, {
-			//Test case with several failed attempts.
-			addrmgr.TstNewKnownAddress(&wire.NetAddress{Timestamp: now.Add(-35 * time.Second)},
-				2, time.Now().Add(-30*time.Minute), time.Now(), false, 0),
-			1 / 1.5 / 1.5,
-		},
-	}
-
-	err := .0001
-	for i, test := range tests {
-		chance := addrmgr.TstKnownAddressChance(test.addr)
-		if math.Abs(test.expected-chance) >= err {
-			t.Errorf("case %d: got %f, expected %f", i, chance, test.expected)
-		}
-	}
-}
-
-func TestIsBad(t *testing.T) {
-	now := time.Unix(time.Now().Unix(), 0)
-	future := now.Add(35 * time.Minute)
-	monthOld := now.Add(-43 * time.Hour * 24)
-	secondsOld := now.Add(-2 * time.Second)
-	minutesOld := now.Add(-27 * time.Minute)
-	hoursOld := now.Add(-5 * time.Hour)
-	zeroTime := time.Time{}
-
-	futureNa := &wire.NetAddress{Timestamp: future}
-	minutesOldNa := &wire.NetAddress{Timestamp: minutesOld}
-	monthOldNa := &wire.NetAddress{Timestamp: monthOld}
-	currentNa := &wire.NetAddress{Timestamp: secondsOld}
-
-	//Test addresses that have been tried in the last minute.
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(futureNa, 3, secondsOld, zeroTime, false, 0)) {
-		t.Errorf("test case 1: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(monthOldNa, 3, secondsOld, zeroTime, false, 0)) {
-		t.Errorf("test case 2: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(currentNa, 3, secondsOld, zeroTime, false, 0)) {
-		t.Errorf("test case 3: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(currentNa, 3, secondsOld, monthOld, true, 0)) {
-		t.Errorf("test case 4: addresses that have been tried in the last minute are not bad.")
-	}
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(currentNa, 2, secondsOld, secondsOld, true, 0)) {
-		t.Errorf("test case 5: addresses that have been tried in the last minute are not bad.")
-	}
-
-	//Test address that claims to be from the future.
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(futureNa, 0, minutesOld, hoursOld, true, 0)) {
-		t.Errorf("test case 6: addresses that claim to be from the future are bad.")
-	}
-
-	//Test address that has not been seen in over a month.
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(monthOldNa, 0, minutesOld, hoursOld, true, 0)) {
-		t.Errorf("test case 7: addresses more than a month old are bad.")
-	}
-
-	//It has failed at least three times and never succeeded.
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(minutesOldNa, 3, minutesOld, zeroTime, true, 0)) {
-		t.Errorf("test case 8: addresses that have never succeeded are bad.")
-	}
-
-	//It has failed ten times in the last week
-	if !addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(minutesOldNa, 10, minutesOld, monthOld, true, 0)) {
-		t.Errorf("test case 9: addresses that have not succeeded in too long are bad.")
-	}
-
-	//Test an address that should work.
-	if addrmgr.TstKnownAddressIsBad(addrmgr.TstNewKnownAddress(minutesOldNa, 2, minutesOld, hoursOld, true, 0)) {
-		t.Errorf("test case 10: This should be a valid address.")
-	}
-}

addrmgr/test_coverage.txt

@@ -1,62 +0,0 @@
-
-github.com/conformal/btcd/addrmgr/network.go		 GroupKey				 100.00% (23/23)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.reset			 100.00% (6/6)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC5737				 100.00% (4/4)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC1918				 100.00% (4/4)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 New					 100.00% (3/3)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 NetAddressKey				 100.00% (2/2)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC4862				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.numAddresses		 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/log.go		 init					 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/log.go		 DisableLog				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 ipNet					 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsIPv4					 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsLocal				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsOnionCatTor				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC2544				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC3849				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC3927				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC3964				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC4193				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC4380				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC4843				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC6052				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC6145				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRFC6598				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsValid				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/network.go		 IsRoutable				 100.00% (1/1)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.GetBestLocalAddress	 94.74% (18/19)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.AddLocalAddress		 90.91% (10/11)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 getReachabilityFrom			 51.52% (17/33)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 ipString				 50.00% (2/4)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.GetAddress			 9.30% (4/43)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.deserializePeers		 0.00% (0/50)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.Good			 0.00% (0/44)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.savePeers			 0.00% (0/39)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.updateAddress		 0.00% (0/30)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.expireNew			 0.00% (0/22)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.AddressCache		 0.00% (0/16)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.HostToNetAddress		 0.00% (0/15)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.getNewBucket		 0.00% (0/15)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.AddAddressByIP		 0.00% (0/14)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.getTriedBucket		 0.00% (0/14)
-github.com/conformal/btcd/addrmgr/knownaddress.go	 knownAddress.chance			 0.00% (0/13)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.loadPeers			 0.00% (0/11)
-github.com/conformal/btcd/addrmgr/knownaddress.go	 knownAddress.isBad			 0.00% (0/11)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.Connected			 0.00% (0/10)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.addressHandler		 0.00% (0/9)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.pickTried			 0.00% (0/8)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.DeserializeNetAddress	 0.00% (0/7)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.Stop			 0.00% (0/7)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.Attempt			 0.00% (0/7)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.Start			 0.00% (0/6)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.AddAddresses		 0.00% (0/4)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.NeedMoreAddresses		 0.00% (0/3)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.NumAddresses		 0.00% (0/3)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.AddAddress			 0.00% (0/3)
-github.com/conformal/btcd/addrmgr/knownaddress.go	 knownAddress.LastAttempt		 0.00% (0/1)
-github.com/conformal/btcd/addrmgr/knownaddress.go	 knownAddress.NetAddress		 0.00% (0/1)
-github.com/conformal/btcd/addrmgr/addrmanager.go	 AddrManager.find			 0.00% (0/1)
-github.com/conformal/btcd/addrmgr/log.go		 UseLogger				 0.00% (0/1)
-github.com/conformal/btcd/addrmgr			 ---------------------------------	 21.04% (113/537)
-

app/app.go

@@ -0,0 +1,249 @@
+package app
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/kaspanet/kaspad/addressmanager"
+
+	"github.com/kaspanet/kaspad/netadapter/id"
+
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/blockdag/indexers"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/connmanager"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/dnsseed"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/mempool"
+	"github.com/kaspanet/kaspad/mining"
+	"github.com/kaspanet/kaspad/netadapter"
+	"github.com/kaspanet/kaspad/protocol"
+	"github.com/kaspanet/kaspad/rpc"
+	"github.com/kaspanet/kaspad/signal"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+// App is a wrapper for all the kaspad services
+type App struct {
+	cfg               *config.Config
+	rpcServer         *rpc.Server
+	addressManager    *addressmanager.AddressManager
+	protocolManager   *protocol.Manager
+	connectionManager *connmanager.ConnectionManager
+	netAdapter        *netadapter.NetAdapter
+
+	started, shutdown int32
+}
+
+// Start launches all the kaspad services.
+func (a *App) Start() {
+	// Already started?
+	if atomic.AddInt32(&a.started, 1) != 1 {
+		return
+	}
+
+	log.Trace("Starting kaspad")
+
+	err := a.protocolManager.Start()
+	if err != nil {
+		panics.Exit(log, fmt.Sprintf("Error starting the p2p protocol: %+v", err))
+	}
+
+	a.maybeSeedFromDNS()
+
+	a.connectionManager.Start()
+
+	if !a.cfg.DisableRPC {
+		a.rpcServer.Start()
+	}
+}
+
+// Stop gracefully shuts down all the kaspad services.
+func (a *App) Stop() error {
+	// Make sure this only happens once.
+	if atomic.AddInt32(&a.shutdown, 1) != 1 {
+		log.Infof("Kaspad is already in the process of shutting down")
+		return nil
+	}
+
+	log.Warnf("Kaspad shutting down")
+
+	a.connectionManager.Stop()
+
+	err := a.protocolManager.Stop()
+	if err != nil {
+		log.Errorf("Error stopping the p2p protocol: %+v", err)
+	}
+
+	// Shutdown the RPC server if it's not disabled.
+	if !a.cfg.DisableRPC {
+		err := a.rpcServer.Stop()
+		if err != nil {
+			log.Errorf("Error stopping rpcServer: %+v", err)
+		}
+	}
+
+	return nil
+}
+
+// New returns a new App instance configured to listen on addr for the
+// kaspa network type specified by dagParams. Use start to begin accepting
+// connections from peers.
+func New(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, interrupt <-chan struct{}) (*App, error) {
+	indexManager, acceptanceIndex := setupIndexes(cfg)
+
+	sigCache := txscript.NewSigCache(cfg.SigCacheMaxSize)
+
+	// Create a new block DAG instance with the appropriate configuration.
+	dag, err := setupDAG(cfg, databaseContext, interrupt, sigCache, indexManager)
+	if err != nil {
+		return nil, err
+	}
+
+	txMempool := setupMempool(cfg, dag, sigCache)
+
+	netAdapter, err := netadapter.NewNetAdapter(cfg)
+	if err != nil {
+		return nil, err
+	}
+	addressManager := addressmanager.New(cfg, databaseContext)
+
+	connectionManager, err := connmanager.New(cfg, netAdapter, addressManager)
+	if err != nil {
+		return nil, err
+	}
+
+	protocolManager, err := protocol.NewManager(cfg, dag, netAdapter, addressManager, txMempool, connectionManager)
+	if err != nil {
+		return nil, err
+	}
+	rpcServer, err := setupRPC(
+		cfg, dag, txMempool, sigCache, acceptanceIndex, connectionManager, addressManager, protocolManager)
+	if err != nil {
+		return nil, err
+	}
+
+	return &App{
+		cfg:               cfg,
+		rpcServer:         rpcServer,
+		protocolManager:   protocolManager,
+		connectionManager: connectionManager,
+		netAdapter:        netAdapter,
+		addressManager:    addressManager,
+	}, nil
+}
+
+func (a *App) maybeSeedFromDNS() {
+	if !a.cfg.DisableDNSSeed {
+		dnsseed.SeedFromDNS(a.cfg.NetParams(), a.cfg.DNSSeed, domainmessage.SFNodeNetwork, false, nil,
+			a.cfg.Lookup, func(addresses []*domainmessage.NetAddress) {
+				// Kaspad uses a lookup of the dns seeder here. Since seeder returns
+				// IPs of nodes and not its own IP, we can not know real IP of
+				// source. So we'll take first returned address as source.
+				a.addressManager.AddAddresses(addresses, addresses[0], nil)
+			})
+	}
+}
+func setupDAG(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, interrupt <-chan struct{},
+	sigCache *txscript.SigCache, indexManager blockdag.IndexManager) (*blockdag.BlockDAG, error) {
+
+	dag, err := blockdag.New(&blockdag.Config{
+		Interrupt:       interrupt,
+		DatabaseContext: databaseContext,
+		DAGParams:       cfg.NetParams(),
+		TimeSource:      blockdag.NewTimeSource(),
+		SigCache:        sigCache,
+		IndexManager:    indexManager,
+		SubnetworkID:    cfg.SubnetworkID,
+	})
+	return dag, err
+}
+
+func setupIndexes(cfg *config.Config) (blockdag.IndexManager, *indexers.AcceptanceIndex) {
+	// Create indexes if needed.
+	var indexes []indexers.Indexer
+	var acceptanceIndex *indexers.AcceptanceIndex
+	if cfg.AcceptanceIndex {
+		log.Info("acceptance index is enabled")
+		acceptanceIndex = indexers.NewAcceptanceIndex()
+		indexes = append(indexes, acceptanceIndex)
+	}
+
+	// Create an index manager if any of the optional indexes are enabled.
+	if len(indexes) < 0 {
+		return nil, nil
+	}
+	indexManager := indexers.NewManager(indexes)
+	return indexManager, acceptanceIndex
+}
+
+func setupMempool(cfg *config.Config, dag *blockdag.BlockDAG, sigCache *txscript.SigCache) *mempool.TxPool {
+	mempoolConfig := mempool.Config{
+		Policy: mempool.Policy{
+			AcceptNonStd:    cfg.RelayNonStd,
+			MaxOrphanTxs:    cfg.MaxOrphanTxs,
+			MaxOrphanTxSize: config.DefaultMaxOrphanTxSize,
+			MinRelayTxFee:   cfg.MinRelayTxFee,
+			MaxTxVersion:    1,
+		},
+		CalcSequenceLockNoLock: func(tx *util.Tx, utxoSet blockdag.UTXOSet) (*blockdag.SequenceLock, error) {
+			return dag.CalcSequenceLockNoLock(tx, utxoSet, true)
+		},
+		IsDeploymentActive: dag.IsDeploymentActive,
+		SigCache:           sigCache,
+		DAG:                dag,
+	}
+
+	return mempool.New(&mempoolConfig)
+}
+
+func setupRPC(cfg *config.Config,
+	dag *blockdag.BlockDAG,
+	txMempool *mempool.TxPool,
+	sigCache *txscript.SigCache,
+	acceptanceIndex *indexers.AcceptanceIndex,
+	connectionManager *connmanager.ConnectionManager,
+	addressManager *addressmanager.AddressManager,
+	protocolManager *protocol.Manager) (*rpc.Server, error) {
+
+	if !cfg.DisableRPC {
+		policy := mining.Policy{
+			BlockMaxMass: cfg.BlockMaxMass,
+		}
+		blockTemplateGenerator := mining.NewBlkTmplGenerator(&policy, txMempool, dag, sigCache)
+
+		rpcServer, err := rpc.NewRPCServer(cfg, dag, txMempool, acceptanceIndex, blockTemplateGenerator,
+			connectionManager, addressManager, protocolManager)
+		if err != nil {
+			return nil, err
+		}
+
+		// Signal process shutdown when the RPC server requests it.
+		spawn("setupRPC-handleShutdownRequest", func() {
+			<-rpcServer.RequestedProcessShutdown()
+			signal.ShutdownRequestChannel <- struct{}{}
+		})
+
+		return rpcServer, nil
+	}
+	return nil, nil
+}
+
+// P2PNodeID returns the network ID associated with this App
+func (a *App) P2PNodeID() *id.ID {
+	return a.netAdapter.ID()
+}
+
+// AddressManager returns the AddressManager associated with this App
+func (a *App) AddressManager() *addressmanager.AddressManager {
+	return a.addressManager
+}
+
+// WaitForShutdown blocks until the main listener and peer handlers are stopped.
+func (a *App) WaitForShutdown() {
+	// TODO(libp2p)
+	// a.p2pServer.WaitForShutdown()
+}

app/log.go

@@ -1,13 +1,14 @@
-// Copyright (c) 2016 The btcsuite developers
+// Copyright (c) 2013-2017 The btcsuite developers
+// Copyright (c) 2017 The Decred developers
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
-package cpuminer
+package app
 
 import (
 	"github.com/kaspanet/kaspad/logger"
 	"github.com/kaspanet/kaspad/util/panics"
 )
 
-var log, _ = logger.Get(logger.SubsystemTags.MINR)
+var log, _ = logger.Get(logger.SubsystemTags.KASD)
 var spawn = panics.GoroutineWrapperFunc(log)

blockdag/README.md

@@ -1,7 +1,7 @@
 blockchain
 ==========
 
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
 [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/blockchain)
 
 Package blockdag implements Kaspa block handling, organization of the blockDAG, 

blockdag/accept.go

@@ -6,16 +6,28 @@ package blockdag
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/database"
+
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
 )
 
 func (dag *BlockDAG) addNodeToIndexWithInvalidAncestor(block *util.Block) error {
 	blockHeader := &block.MsgBlock().Header
-	newNode := newBlockNode(blockHeader, newSet(), dag.dagParams.K)
+	newNode, _ := dag.newBlockNode(blockHeader, newBlockSet())
 	newNode.status = statusInvalidAncestor
 	dag.index.AddNode(newNode)
-	return dag.index.flushToDB()
+
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = dag.index.flushToDB(dbTx)
+	if err != nil {
+		return err
+	}
+	return dbTx.Commit()
 }
 
 // maybeAcceptBlock potentially accepts a block into the block DAG. It
@@ -30,7 +42,8 @@ func (dag *BlockDAG) addNodeToIndexWithInvalidAncestor(block *util.Block) error
 func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) error {
 	parents, err := lookupParentNodes(block, dag)
 	if err != nil {
-		if rErr, ok := err.(RuleError); ok && rErr.ErrorCode == ErrInvalidAncestorBlock {
+		var ruleErr RuleError
+		if ok := errors.As(err, &ruleErr); ok && ruleErr.ErrorCode == ErrInvalidAncestorBlock {
 			err := dag.addNodeToIndexWithInvalidAncestor(block)
 			if err != nil {
 				return err
@@ -47,7 +60,7 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	}
 
 	// Create a new block node for the block and add it to the node index.
-	newNode := newBlockNode(&block.MsgBlock().Header, parents, dag.dagParams.K)
+	newNode, selectedParentAnticone := dag.newBlockNode(&block.MsgBlock().Header, parents)
 	newNode.status = statusDataStored
 	dag.index.AddNode(newNode)
 
@@ -60,13 +73,26 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	// expensive connection logic. It also has some other nice properties
 	// such as making blocks that never become part of the DAG or
 	// blocks that fail to connect available for further analysis.
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		err := dbStoreBlock(dbTx, block)
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+	blockExists, err := dbaccess.HasBlock(dbTx, block.Hash())
+	if err != nil {
+		return err
+	}
+	if !blockExists {
+		err := storeBlock(dbTx, block)
 		if err != nil {
 			return err
 		}
-		return dag.index.flushToDBWithTx(dbTx)
-	})
+	}
+	err = dag.index.flushToDB(dbTx)
+	if err != nil {
+		return err
+	}
+	err = dbTx.Commit()
 	if err != nil {
 		return err
 	}
@@ -80,11 +106,9 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 		}
 	}
 
-	block.SetChainHeight(newNode.chainHeight)
-
 	// Connect the passed block to the DAG. This also handles validation of the
 	// transaction scripts.
-	chainUpdates, err := dag.addBlock(newNode, parents, block, flags)
+	chainUpdates, err := dag.addBlock(newNode, block, selectedParentAnticone, flags)
 	if err != nil {
 		return err
 	}
@@ -108,18 +132,18 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	return nil
 }
 
-func lookupParentNodes(block *util.Block, blockDAG *BlockDAG) (blockSet, error) {
+func lookupParentNodes(block *util.Block, dag *BlockDAG) (blockSet, error) {
 	header := block.MsgBlock().Header
 	parentHashes := header.ParentHashes
 
-	nodes := newSet()
+	nodes := newBlockSet()
 	for _, parentHash := range parentHashes {
-		node := blockDAG.index.LookupNode(parentHash)
-		if node == nil {
-			str := fmt.Sprintf("parent block %s is unknown", parentHashes)
+		node, ok := dag.index.LookupNode(parentHash)
+		if !ok {
+			str := fmt.Sprintf("parent block %s is unknown", parentHash)
 			return nil, ruleError(ErrParentBlockUnknown, str)
-		} else if blockDAG.index.NodeStatus(node).KnownInvalid() {
-			str := fmt.Sprintf("parent block %s is known to be invalid", parentHashes)
+		} else if dag.index.NodeStatus(node).KnownInvalid() {
+			str := fmt.Sprintf("parent block %s is known to be invalid", parentHash)
 			return nil, ruleError(ErrInvalidAncestorBlock, str)
 		}
 

blockdag/accept_test.go

@@ -1,21 +1,17 @@
 package blockdag
 
 import (
-	"github.com/pkg/errors"
+	"errors"
 	"path/filepath"
-	"strings"
 	"testing"
 
-	"bou.ke/monkey"
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
 )
 
 func TestMaybeAcceptBlockErrors(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestMaybeAcceptBlockErrors", Config{
-		DAGParams: &dagconfig.SimNetParams,
+	dag, teardownFunc, err := DAGSetup("TestMaybeAcceptBlockErrors", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Fatalf("TestMaybeAcceptBlockErrors: Failed to setup DAG instance: %v", err)
@@ -38,8 +34,8 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
 			"Expected: %s, got: <nil>", ErrParentBlockUnknown)
 	}
-	ruleErr, ok := err.(RuleError)
-	if !ok {
+	var ruleErr RuleError
+	if ok := errors.As(err, &ruleErr); !ok {
 		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
 			"Expected RuleError but got %s", err)
 	} else if ruleErr.ErrorCode != ErrParentBlockUnknown {
@@ -57,17 +53,20 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 
 	// Add a valid block and mark it as invalid
 	block1 := blocks[1]
-	isOrphan, delay, err := dag.ProcessBlock(block1, BFNone)
+	isOrphan, isDelayed, err := dag.ProcessBlock(block1, BFNone)
 	if err != nil {
 		t.Fatalf("TestMaybeAcceptBlockErrors: Valid block unexpectedly returned an error: %s", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("TestMaybeAcceptBlockErrors: block 1 is too far in the future")
 	}
 	if isOrphan {
 		t.Fatalf("TestMaybeAcceptBlockErrors: incorrectly returned block 1 is an orphan")
 	}
-	blockNode1 := dag.index.LookupNode(block1.Hash())
+	blockNode1, ok := dag.index.LookupNode(block1.Hash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", block1.Hash())
+	}
 	dag.index.SetStatusFlags(blockNode1, statusValidateFailed)
 
 	block2 := blocks[2]
@@ -76,8 +75,7 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
 			"Expected: %s, got: <nil>", ErrInvalidAncestorBlock)
 	}
-	ruleErr, ok = err.(RuleError)
-	if !ok {
+	if ok := errors.As(err, &ruleErr); !ok {
 		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
 			"Expected RuleError but got %s", err)
 	} else if ruleErr.ErrorCode != ErrInvalidAncestorBlock {
@@ -96,8 +94,7 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
 			"Expected: %s, got: <nil>", ErrUnexpectedDifficulty)
 	}
-	ruleErr, ok = err.(RuleError)
-	if !ok {
+	if ok := errors.As(err, &ruleErr); !ok {
 		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
 			"Expected RuleError but got %s", err)
 	} else if ruleErr.ErrorCode != ErrUnexpectedDifficulty {
@@ -107,37 +104,4 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 
 	// Set block2's bits back to valid for next tests
 	block2.MsgBlock().Header.Bits = originalBits
-
-	// Test rejecting the node due to database error
-	databaseErrorMessage := "database error"
-	guard := monkey.Patch(dbStoreBlock, func(dbTx database.Tx, block *util.Block) error {
-		return errors.New(databaseErrorMessage)
-	})
-	defer guard.Unpatch()
-	err = dag.maybeAcceptBlock(block2, BFNone)
-	if err == nil {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the node due to database error: "+
-			"Expected: %s, got: <nil>", databaseErrorMessage)
-	}
-	if !strings.Contains(err.Error(), databaseErrorMessage) {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the node due to database error: "+
-			"Unexpected error. Want: %s, got: %s", databaseErrorMessage, err)
-	}
-	guard.Unpatch()
-
-	// Test rejecting the node due to index error
-	indexErrorMessage := "index error"
-	guard = monkey.Patch((*blockIndex).flushToDB, func(_ *blockIndex) error {
-		return errors.New(indexErrorMessage)
-	})
-	defer guard.Unpatch()
-	err = dag.maybeAcceptBlock(block2, BFNone)
-	if err == nil {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the node due to index error: "+
-			"Expected %s, got: <nil>", indexErrorMessage)
-	}
-	if !strings.Contains(err.Error(), indexErrorMessage) {
-		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the node due to index error: "+
-			"Unexpected error. Want: %s, got: %s", indexErrorMessage, err)
-	}
 }

blockdag/blockheap.go

@@ -2,8 +2,6 @@ package blockdag
 
 import (
 	"container/heap"
-
-	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // baseHeap is an implementation for heap.Interface that sorts blocks by their height
@@ -28,22 +26,14 @@ func (h *baseHeap) Pop() interface{} {
 type upHeap struct{ baseHeap }
 
 func (h upHeap) Less(i, j int) bool {
-	if h.baseHeap[i].blueScore == h.baseHeap[j].blueScore {
-		return daghash.HashToBig(h.baseHeap[i].hash).Cmp(daghash.HashToBig(h.baseHeap[j].hash)) < 0
-	}
-
-	return h.baseHeap[i].blueScore < h.baseHeap[j].blueScore
+	return h.baseHeap[i].less(h.baseHeap[j])
 }
 
 // downHeap extends baseHeap to include Less operation that traverses from top to bottom
 type downHeap struct{ baseHeap }
 
 func (h downHeap) Less(i, j int) bool {
-	if h.baseHeap[i].blueScore == h.baseHeap[j].blueScore {
-		return daghash.HashToBig(h.baseHeap[i].hash).Cmp(daghash.HashToBig(h.baseHeap[j].hash)) > 0
-	}
-
-	return h.baseHeap[i].blueScore > h.baseHeap[j].blueScore
+	return !h.baseHeap[i].less(h.baseHeap[j])
 }
 
 // blockHeap represents a mutable heap of Blocks, sorted by their height
@@ -77,7 +67,7 @@ func (bh blockHeap) Push(block *blockNode) {
 
 // pushSet pushes a blockset to the heap.
 func (bh blockHeap) pushSet(bs blockSet) {
-	for _, block := range bs {
+	for block := range bs {
 		heap.Push(bh.impl, block)
 	}
 }

blockdag/blockheap_test.go

@@ -9,13 +9,22 @@ import (
 
 // TestBlockHeap tests pushing, popping, and determining the length of the heap.
 func TestBlockHeap(t *testing.T) {
-	block0Header := dagconfig.MainNetParams.GenesisBlock.Header
-	block0 := newBlockNode(&block0Header, newSet(), dagconfig.MainNetParams.K)
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestBlockHeap", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestBlockHeap: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	block0Header := dagconfig.SimnetParams.GenesisBlock.Header
+	block0, _ := dag.newBlockNode(&block0Header, newBlockSet())
 
 	block100000Header := Block100000.Header
-	block100000 := newBlockNode(&block100000Header, setFromSlice(block0), dagconfig.MainNetParams.K)
+	block100000, _ := dag.newBlockNode(&block100000Header, blockSetFromSlice(block0))
 
-	block0smallHash := newBlockNode(&block0Header, newSet(), dagconfig.MainNetParams.K)
+	block0smallHash, _ := dag.newBlockNode(&block0Header, newBlockSet())
 	block0smallHash.hash = &daghash.Hash{}
 
 	tests := []struct {

blockdag/blockidhash.go

@@ -1,136 +0,0 @@
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/pkg/errors"
-)
-
-var (
-	// idByHashIndexBucketName is the name of the db bucket used to house
-	// the block hash -> block id index.
-	idByHashIndexBucketName = []byte("idbyhashidx")
-
-	// hashByIDIndexBucketName is the name of the db bucket used to house
-	// the block id -> block hash index.
-	hashByIDIndexBucketName = []byte("hashbyididx")
-
-	currentBlockIDKey = []byte("currentblockid")
-)
-
-// -----------------------------------------------------------------------------
-// This is a mapping between block hashes and unique IDs. The ID
-// is simply a sequentially incremented uint64 that is used instead of block hash
-// for the indexers. This is useful because it is only 8 bytes versus 32 bytes
-// hashes and thus saves a ton of space when a block is referenced in an index.
-// It consists of three buckets: the first bucket maps the hash of each
-// block to the unique ID and the second maps that ID back to the block hash.
-// The third bucket contains the last received block ID, and is used
-// when starting the node to check that the enabled indexes are up to date
-// with the latest received block, and if not, initiate recovery process.
-//
-// The serialized format for keys and values in the block hash to ID bucket is:
-//   <hash> = <ID>
-//
-//   Field           Type              Size
-//   hash            daghash.Hash     32 bytes
-//   ID              uint64            8 bytes
-//   -----
-//   Total: 40 bytes
-//
-// The serialized format for keys and values in the ID to block hash bucket is:
-//   <ID> = <hash>
-//
-//   Field           Type              Size
-//   ID              uint64            8 bytes
-//   hash            daghash.Hash     32 bytes
-//   -----
-//   Total: 40 bytes
-//
-// -----------------------------------------------------------------------------
-
-const blockIDSize = 8 // 8 bytes for block ID
-
-// DBFetchBlockIDByHash uses an existing database transaction to retrieve the
-// block id for the provided hash from the index.
-func DBFetchBlockIDByHash(dbTx database.Tx, hash *daghash.Hash) (uint64, error) {
-	hashIndex := dbTx.Metadata().Bucket(idByHashIndexBucketName)
-	serializedID := hashIndex.Get(hash[:])
-	if serializedID == nil {
-		return 0, errors.Errorf("no entry in the block ID index for block with hash %s", hash)
-	}
-
-	return DeserializeBlockID(serializedID), nil
-}
-
-// DBFetchBlockHashBySerializedID uses an existing database transaction to
-// retrieve the hash for the provided serialized block id from the index.
-func DBFetchBlockHashBySerializedID(dbTx database.Tx, serializedID []byte) (*daghash.Hash, error) {
-	idIndex := dbTx.Metadata().Bucket(hashByIDIndexBucketName)
-	hashBytes := idIndex.Get(serializedID)
-	if hashBytes == nil {
-		return nil, errors.Errorf("no entry in the block ID index for block with id %d", byteOrder.Uint64(serializedID))
-	}
-
-	var hash daghash.Hash
-	copy(hash[:], hashBytes)
-	return &hash, nil
-}
-
-// dbPutBlockIDIndexEntry uses an existing database transaction to update or add
-// the index entries for the hash to id and id to hash mappings for the provided
-// values.
-func dbPutBlockIDIndexEntry(dbTx database.Tx, hash *daghash.Hash, serializedID []byte) error {
-	// Add the block hash to ID mapping to the index.
-	meta := dbTx.Metadata()
-	hashIndex := meta.Bucket(idByHashIndexBucketName)
-	if err := hashIndex.Put(hash[:], serializedID[:]); err != nil {
-		return err
-	}
-
-	// Add the block ID to hash mapping to the index.
-	idIndex := meta.Bucket(hashByIDIndexBucketName)
-	return idIndex.Put(serializedID[:], hash[:])
-}
-
-// DBFetchCurrentBlockID returns the last known block ID.
-func DBFetchCurrentBlockID(dbTx database.Tx) uint64 {
-	serializedID := dbTx.Metadata().Get(currentBlockIDKey)
-	if serializedID == nil {
-		return 0
-	}
-	return DeserializeBlockID(serializedID)
-}
-
-// DeserializeBlockID returns a deserialized block id
-func DeserializeBlockID(serializedID []byte) uint64 {
-	return byteOrder.Uint64(serializedID)
-}
-
-// SerializeBlockID returns a serialized block id
-func SerializeBlockID(blockID uint64) []byte {
-	serializedBlockID := make([]byte, blockIDSize)
-	byteOrder.PutUint64(serializedBlockID, blockID)
-	return serializedBlockID
-}
-
-// DBFetchBlockHashByID uses an existing database transaction to retrieve the
-// hash for the provided block id from the index.
-func DBFetchBlockHashByID(dbTx database.Tx, id uint64) (*daghash.Hash, error) {
-	return DBFetchBlockHashBySerializedID(dbTx, SerializeBlockID(id))
-}
-
-func createBlockID(dbTx database.Tx, blockHash *daghash.Hash) (uint64, error) {
-	currentBlockID := DBFetchCurrentBlockID(dbTx)
-	newBlockID := currentBlockID + 1
-	serializedNewBlockID := SerializeBlockID(newBlockID)
-	err := dbTx.Metadata().Put(currentBlockIDKey, serializedNewBlockID)
-	if err != nil {
-		return 0, err
-	}
-	err = dbPutBlockIDIndexEntry(dbTx, blockHash, serializedNewBlockID)
-	if err != nil {
-		return 0, err
-	}
-	return newBlockID, nil
-}

blockdag/blockindex.go

@@ -5,10 +5,10 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/dbaccess"
 	"sync"
 
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
 
@@ -18,7 +18,6 @@ type blockIndex struct {
 	// The following fields are set when the instance is created and can't
 	// be changed afterwards, so there is no need to protect them with a
 	// separate mutex.
-	db        database.DB
 	dagParams *dagconfig.Params
 
 	sync.RWMutex
@@ -29,9 +28,8 @@ type blockIndex struct {
 // newBlockIndex returns a new empty instance of a block index. The index will
 // be dynamically populated as block nodes are loaded from the database and
 // manually added.
-func newBlockIndex(db database.DB, dagParams *dagconfig.Params) *blockIndex {
+func newBlockIndex(dagParams *dagconfig.Params) *blockIndex {
 	return &blockIndex{
-		db:        db,
 		dagParams: dagParams,
 		index:     make(map[daghash.Hash]*blockNode),
 		dirty:     make(map[*blockNode]struct{}),
@@ -43,8 +41,8 @@ func newBlockIndex(db database.DB, dagParams *dagconfig.Params) *blockIndex {
 // This function is safe for concurrent access.
 func (bi *blockIndex) HaveBlock(hash *daghash.Hash) bool {
 	bi.RLock()
+	defer bi.RUnlock()
 	_, hasBlock := bi.index[*hash]
-	bi.RUnlock()
 	return hasBlock
 }
 
@@ -52,11 +50,11 @@ func (bi *blockIndex) HaveBlock(hash *daghash.Hash) bool {
 // return nil if there is no entry for the hash.
 //
 // This function is safe for concurrent access.
-func (bi *blockIndex) LookupNode(hash *daghash.Hash) *blockNode {
+func (bi *blockIndex) LookupNode(hash *daghash.Hash) (*blockNode, bool) {
 	bi.RLock()
-	node := bi.index[*hash]
-	bi.RUnlock()
-	return node
+	defer bi.RUnlock()
+	node, ok := bi.index[*hash]
+	return node, ok
 }
 
 // AddNode adds the provided node to the block index and marks it as dirty.
@@ -65,9 +63,9 @@ func (bi *blockIndex) LookupNode(hash *daghash.Hash) *blockNode {
 // This function is safe for concurrent access.
 func (bi *blockIndex) AddNode(node *blockNode) {
 	bi.Lock()
+	defer bi.Unlock()
 	bi.addNode(node)
 	bi.dirty[node] = struct{}{}
-	bi.Unlock()
 }
 
 // addNode adds the provided node to the block index, but does not mark it as
@@ -83,8 +81,8 @@ func (bi *blockIndex) addNode(node *blockNode) {
 // This function is safe for concurrent access.
 func (bi *blockIndex) NodeStatus(node *blockNode) blockStatus {
 	bi.RLock()
+	defer bi.RUnlock()
 	status := node.status
-	bi.RUnlock()
 	return status
 }
 
@@ -95,9 +93,9 @@ func (bi *blockIndex) NodeStatus(node *blockNode) blockStatus {
 // This function is safe for concurrent access.
 func (bi *blockIndex) SetStatusFlags(node *blockNode, flags blockStatus) {
 	bi.Lock()
+	defer bi.Unlock()
 	node.status |= flags
 	bi.dirty[node] = struct{}{}
-	bi.Unlock()
 }
 
 // UnsetStatusFlags flips the provided status flags on the block node to off,
@@ -106,22 +104,13 @@ func (bi *blockIndex) SetStatusFlags(node *blockNode, flags blockStatus) {
 // This function is safe for concurrent access.
 func (bi *blockIndex) UnsetStatusFlags(node *blockNode, flags blockStatus) {
 	bi.Lock()
+	defer bi.Unlock()
 	node.status &^= flags
 	bi.dirty[node] = struct{}{}
-	bi.Unlock()
 }
 
-// flushToDB writes all dirty block nodes to the database. If all writes
-// succeed, this clears the dirty set.
-func (bi *blockIndex) flushToDB() error {
-	return bi.db.Update(func(dbTx database.Tx) error {
-		return bi.flushToDBWithTx(dbTx)
-	})
-}
-
-// flushToDBWithTx writes all dirty block nodes to the database. If all
-// writes succeed, this clears the dirty set.
-func (bi *blockIndex) flushToDBWithTx(dbTx database.Tx) error {
+// flushToDB writes all dirty block nodes to the database.
+func (bi *blockIndex) flushToDB(dbContext *dbaccess.TxContext) error {
 	bi.Lock()
 	defer bi.Unlock()
 	if len(bi.dirty) == 0 {
@@ -129,13 +118,19 @@ func (bi *blockIndex) flushToDBWithTx(dbTx database.Tx) error {
 	}
 
 	for node := range bi.dirty {
-		err := dbStoreBlockNode(dbTx, node)
+		serializedBlockNode, err := serializeBlockNode(node)
+		if err != nil {
+			return err
+		}
+		key := blockIndexKey(node.hash, node.blueScore)
+		err = dbaccess.StoreIndexBlock(dbContext, key, serializedBlockNode)
 		if err != nil {
 			return err
 		}
 	}
-
-	// If write was successful, clear the dirty set.
-	bi.dirty = make(map[*blockNode]struct{})
 	return nil
 }
+
+func (bi *blockIndex) clearDirtyEntries() {
+	bi.dirty = make(map[*blockNode]struct{})
+}

blockdag/blockindex_test.go

@@ -1,58 +1,26 @@
 package blockdag
 
 import (
-	"github.com/pkg/errors"
-	"strings"
-	"testing"
-	"time"
-
-	"bou.ke/monkey"
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"testing"
 )
 
 func TestAncestorErrors(t *testing.T) {
-	node := newTestNode(newSet(), int32(0x10000000), 0, time.Unix(0, 0), dagconfig.MainNetParams.K)
-	node.chainHeight = 2
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimnetParams
+	dag, teardownFunc, err := DAGSetup("TestAncestorErrors", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestAncestorErrors: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	node := newTestNode(dag, newBlockSet(), int32(0x10000000), 0, mstime.Now())
+	node.blueScore = 2
 	ancestor := node.SelectedAncestor(3)
 	if ancestor != nil {
 		t.Errorf("TestAncestorErrors: Ancestor() unexpectedly returned a node. Expected: <nil>")
 	}
 }
-
-func TestFlushToDBErrors(t *testing.T) {
-	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestFlushToDBErrors", Config{
-		DAGParams: &dagconfig.SimNetParams,
-	})
-	if err != nil {
-		t.Fatalf("TestFlushToDBErrors: Failed to setup DAG instance: %s", err)
-	}
-	defer teardownFunc()
-
-	// Call flushToDB without anything to flush. This should succeed
-	err = dag.index.flushToDB()
-	if err != nil {
-		t.Errorf("TestFlushToDBErrors: flushToDB without anything to flush: "+
-			"Unexpected flushToDB error: %s", err)
-	}
-
-	// Mark the genesis block as dirty
-	dag.index.SetStatusFlags(dag.genesis, statusValid)
-
-	// Test flushToDB failure due to database error
-	databaseErrorMessage := "database error"
-	guard := monkey.Patch(dbStoreBlockNode, func(_ database.Tx, _ *blockNode) error {
-		return errors.New(databaseErrorMessage)
-	})
-	defer guard.Unpatch()
-	err = dag.index.flushToDB()
-	if err == nil {
-		t.Errorf("TestFlushToDBErrors: flushToDB failure due to database error: "+
-			"Expected: %s, got: <nil>", databaseErrorMessage)
-	}
-	if !strings.Contains(err.Error(), databaseErrorMessage) {
-		t.Errorf("TestFlushToDBErrors: flushToDB failure due to database error: "+
-			"Unexpected flushToDB error. Expected: %s, got: %s", databaseErrorMessage, err)
-	}
-}

blockdag/blocklocator.go

@@ -1,8 +1,8 @@
 package blockdag
 
 import (
-	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
 )
 
 // BlockLocator is used to help locate a specific block. The algorithm for
@@ -21,100 +21,66 @@ import (
 //  [17 16 14 11 7 2 genesis]
 type BlockLocator []*daghash.Hash
 
-// BlockLocatorFromHashes returns a block locator from start and stop hash.
+// BlockLocatorFromHashes returns a block locator from high and low hash.
 // See BlockLocator for details on the algorithm used to create a block locator.
 //
-// In addition to the general algorithm referenced above, this function will
-// return the block locator for the selected tip if the passed hash is not currently
-// known.
-//
 // This function is safe for concurrent access.
-func (dag *BlockDAG) BlockLocatorFromHashes(startHash, stopHash *daghash.Hash) BlockLocator {
+func (dag *BlockDAG) BlockLocatorFromHashes(highHash, lowHash *daghash.Hash) (BlockLocator, error) {
 	dag.dagLock.RLock()
 	defer dag.dagLock.RUnlock()
-	startNode := dag.index.LookupNode(startHash)
-	var stopNode *blockNode
-	if !stopHash.IsEqual(&daghash.ZeroHash) {
-		stopNode = dag.index.LookupNode(stopHash)
+
+	highNode, ok := dag.index.LookupNode(highHash)
+	if !ok {
+		return nil, errors.Errorf("block %s is unknown", highHash)
 	}
-	return dag.blockLocator(startNode, stopNode)
+
+	lowNode, ok := dag.index.LookupNode(lowHash)
+	if !ok {
+		return nil, errors.Errorf("block %s is unknown", lowHash)
+	}
+
+	return dag.blockLocator(highNode, lowNode)
 }
 
-// LatestBlockLocator returns a block locator for the current tips of the DAG.
-//
-// This function is safe for concurrent access.
-func (dag *BlockDAG) LatestBlockLocator() BlockLocator {
-	dag.dagLock.RLock()
-	defer dag.dagLock.RUnlock()
-	return dag.blockLocator(nil, nil)
-}
-
-// blockLocator returns a block locator for the passed start and stop nodes.
-// The default value for the start node is the selected tip, and the default
-// values of the stop node is the genesis block.
-//
+// blockLocator returns a block locator for the passed high and low nodes.
 // See the BlockLocator type comments for more details.
 //
 // This function MUST be called with the DAG state lock held (for reads).
-func (dag *BlockDAG) blockLocator(startNode, stopNode *blockNode) BlockLocator {
-	// Use the selected tip if requested.
-	if startNode == nil {
-		startNode = dag.virtual.selectedParent
-	}
-
-	if stopNode == nil {
-		stopNode = dag.genesis
-	}
-
-	// We use the selected parent of the start node, so the
-	// block locator won't contain the start node.
-	startNode = startNode.selectedParent
-
-	// If the start node or the stop node are not in the
-	// virtual's selected parent chain, we replace them with their
-	// closest selected parent that is part of the virtual's
-	// selected parent chain.
-	for !dag.IsInSelectedParentChain(stopNode.hash) {
-		stopNode = stopNode.selectedParent
-	}
-
-	for !dag.IsInSelectedParentChain(startNode.hash) {
-		startNode = startNode.selectedParent
-	}
-
-	// Calculate the max number of entries that will ultimately be in the
-	// block locator. See the description of the algorithm for how these
-	// numbers are derived.
-
-	// startNode.hash + stopNode.hash.
-	// Then floor(log2(startNode.chainHeight-stopNode.chainHeight)) entries for the skip portion.
-	maxEntries := 2 + util.FastLog2Floor(startNode.chainHeight-stopNode.chainHeight)
-	locator := make(BlockLocator, 0, maxEntries)
+func (dag *BlockDAG) blockLocator(highNode, lowNode *blockNode) (BlockLocator, error) {
+	// We use the selected parent of the high node, so the
+	// block locator won't contain the high node.
+	highNode = highNode.selectedParent
 
+	node := highNode
 	step := uint64(1)
-	for node := startNode; node != nil; {
+	locator := make(BlockLocator, 0)
+	for node != nil {
 		locator = append(locator, node.hash)
 
-		// Nothing more to add once the stop node has been added.
-		if node.chainHeight == stopNode.chainHeight {
+		// Nothing more to add once the low node has been added.
+		if node.blueScore <= lowNode.blueScore {
+			if node != lowNode {
+				return nil, errors.Errorf("highNode and lowNode are " +
+					"not in the same selected parent chain.")
+			}
 			break
 		}
 
-		// Calculate chainHeight of previous node to include ensuring the
-		// final node is stopNode.
-		nextChainHeight := node.chainHeight - step
-		if nextChainHeight < stopNode.chainHeight {
-			nextChainHeight = stopNode.chainHeight
+		// Calculate blueScore of previous node to include ensuring the
+		// final node is lowNode.
+		nextBlueScore := node.blueScore - step
+		if nextBlueScore < lowNode.blueScore {
+			nextBlueScore = lowNode.blueScore
 		}
 
 		// walk backwards through the nodes to the correct ancestor.
-		node = node.SelectedAncestor(nextChainHeight)
+		node = node.SelectedAncestor(nextBlueScore)
 
 		// Double the distance between included hashes.
 		step *= 2
 	}
 
-	return locator
+	return locator, nil
 }
 
 // FindNextLocatorBoundaries returns the lowest unknown block locator, hash
@@ -123,21 +89,21 @@ func (dag *BlockDAG) blockLocator(startNode, stopNode *blockNode) BlockLocator {
 // sync peer.
 //
 // This function MUST be called with the DAG state lock held (for reads).
-func (dag *BlockDAG) FindNextLocatorBoundaries(locator BlockLocator) (startHash, stopHash *daghash.Hash) {
+func (dag *BlockDAG) FindNextLocatorBoundaries(locator BlockLocator) (highHash, lowHash *daghash.Hash) {
 	// Find the most recent locator block hash in the DAG. In the case none of
 	// the hashes in the locator are in the DAG, fall back to the genesis block.
-	stopNode := dag.genesis
+	lowNode := dag.genesis
 	nextBlockLocatorIndex := int64(len(locator) - 1)
 	for i, hash := range locator {
-		node := dag.index.LookupNode(hash)
-		if node != nil {
-			stopNode = node
+		node, ok := dag.index.LookupNode(hash)
+		if ok {
+			lowNode = node
 			nextBlockLocatorIndex = int64(i) - 1
 			break
 		}
 	}
 	if nextBlockLocatorIndex < 0 {
-		return nil, stopNode.hash
+		return nil, lowNode.hash
 	}
-	return locator[nextBlockLocatorIndex], stopNode.hash
+	return locator[nextBlockLocatorIndex], lowNode.hash
 }

blockdag/blocknode.go

@@ -6,10 +6,14 @@ package blockdag
 
 import (
 	"fmt"
-	"time"
+	"math"
 
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
 )
 
 // blockStatus is a bit field representing the validation state of the block.
@@ -28,11 +32,6 @@ const (
 	// statusInvalidAncestor indicates that one of the block's ancestors has
 	// has failed validation, thus the block is also invalid.
 	statusInvalidAncestor
-
-	// statusNone indicates that the block has no validation state flags set.
-	//
-	// NOTE: This must be defined last in order to avoid influencing iota.
-	statusNone blockStatus = 0
 )
 
 // KnownValid returns whether the block is known to be valid. This will return
@@ -75,12 +74,13 @@ type blockNode struct {
 	// blueScore is the count of all the blue blocks in this block's past
 	blueScore uint64
 
+	// bluesAnticoneSizes is a map holding the set of blues affected by this block and their
+	// modified blue anticone size.
+	bluesAnticoneSizes map[*blockNode]dagconfig.KType
+
 	// hash is the double sha 256 of the block.
 	hash *daghash.Hash
 
-	// chainHeight is the number of hops you need to go down the selected parent chain in order to get to the genesis block.
-	chainHeight uint64
-
 	// Some fields from block headers to aid in  reconstructing headers
 	// from memory. These must be treated as immutable and are intentionally
 	// ordered to avoid padding on 64-bit platforms.
@@ -102,14 +102,17 @@ type blockNode struct {
 	isFinalized bool
 }
 
-// initBlockNode initializes a block node from the given header and parent nodes.
-// This function is NOT safe for concurrent access. It must only be called when
-// initially creating a node.
-func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, parents blockSet, phantomK uint32) {
-	*node = blockNode{
-		parents:   parents,
-		children:  make(blockSet),
-		timestamp: time.Now().Unix(),
+// newBlockNode returns a new block node for the given block header and parents, and the
+// anticone of its selected parent (parent with highest blue score).
+// selectedParentAnticone is used to update reachability data we store for future reachability queries.
+// This function is NOT safe for concurrent access.
+func (dag *BlockDAG) newBlockNode(blockHeader *domainmessage.BlockHeader, parents blockSet) (node *blockNode, selectedParentAnticone []*blockNode) {
+	node = &blockNode{
+		parents:            parents,
+		children:           make(blockSet),
+		blueScore:          math.MaxUint64, // Initialized to the max value to avoid collisions with the genesis block
+		timestamp:          dag.Now().UnixMilliseconds(),
+		bluesAnticoneSizes: make(map[*blockNode]dagconfig.KType),
 	}
 
 	// blockHeader is nil only for the virtual block
@@ -118,7 +121,7 @@ func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, parents block
 		node.version = blockHeader.Version
 		node.bits = blockHeader.Bits
 		node.nonce = blockHeader.Nonce
-		node.timestamp = blockHeader.Timestamp.Unix()
+		node.timestamp = blockHeader.Timestamp.UnixMilliseconds()
 		node.hashMerkleRoot = blockHeader.HashMerkleRoot
 		node.acceptedIDMerkleRoot = blockHeader.AcceptedIDMerkleRoot
 		node.utxoCommitment = blockHeader.UTXOCommitment
@@ -126,89 +129,89 @@ func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, parents block
 		node.hash = &daghash.ZeroHash
 	}
 
-	if len(parents) > 0 {
-		node.blues, node.selectedParent, node.blueScore = phantom(node, phantomK)
-		node.chainHeight = calculateChainHeight(node)
+	if len(parents) == 0 {
+		// The genesis block is defined to have a blueScore of 0
+		node.blueScore = 0
+		return node, nil
 	}
-}
 
-func calculateChainHeight(node *blockNode) uint64 {
-	if node.isGenesis() {
-		return 0
+	selectedParentAnticone, err := dag.ghostdag(node)
+	if err != nil {
+		panic(errors.Wrap(err, "unexpected error in GHOSTDAG"))
 	}
-	return node.selectedParent.chainHeight + 1
-}
-
-// newBlockNode returns a new block node for the given block header and parent
-//nodes. This function is NOT safe for concurrent access.
-func newBlockNode(blockHeader *wire.BlockHeader, parents blockSet, phantomK uint32) *blockNode {
-	var node blockNode
-	initBlockNode(&node, blockHeader, parents, phantomK)
-	return &node
+	return node, selectedParentAnticone
 }
 
 // updateParentsChildren updates the node's parents to point to new node
 func (node *blockNode) updateParentsChildren() {
-	for _, parent := range node.parents {
+	for parent := range node.parents {
 		parent.children.add(node)
 	}
 }
 
+func (node *blockNode) less(other *blockNode) bool {
+	if node.blueScore == other.blueScore {
+		return daghash.Less(node.hash, other.hash)
+	}
+
+	return node.blueScore < other.blueScore
+}
+
 // Header constructs a block header from the node and returns it.
 //
 // This function is safe for concurrent access.
-func (node *blockNode) Header() *wire.BlockHeader {
+func (node *blockNode) Header() *domainmessage.BlockHeader {
 	// No lock is needed because all accessed fields are immutable.
-	return &wire.BlockHeader{
+	return &domainmessage.BlockHeader{
 		Version:              node.version,
 		ParentHashes:         node.ParentHashes(),
 		HashMerkleRoot:       node.hashMerkleRoot,
 		AcceptedIDMerkleRoot: node.acceptedIDMerkleRoot,
 		UTXOCommitment:       node.utxoCommitment,
-		Timestamp:            time.Unix(node.timestamp, 0),
+		Timestamp:            node.time(),
 		Bits:                 node.bits,
 		Nonce:                node.nonce,
 	}
 }
 
-// SelectedAncestor returns the ancestor block node at the provided chain-height by following
+// SelectedAncestor returns the ancestor block node at the provided blue score by following
 // the selected-parents chain backwards from this node. The returned block will be nil when a
-// height is requested that is after the height of the passed node.
+// blue score is requested that is higher than the blue score of the passed node.
 //
 // This function is safe for concurrent access.
-func (node *blockNode) SelectedAncestor(chainHeight uint64) *blockNode {
-	if chainHeight < 0 || chainHeight > node.chainHeight {
+func (node *blockNode) SelectedAncestor(blueScore uint64) *blockNode {
+	if blueScore > node.blueScore {
 		return nil
 	}
 
 	n := node
-	for ; n != nil && n.chainHeight != chainHeight; n = n.selectedParent {
-		// Intentionally left blank
+	for n != nil && n.blueScore > blueScore {
+		n = n.selectedParent
 	}
 
 	return n
 }
 
 // RelativeAncestor returns the ancestor block node a relative 'distance' of
-// chain-blocks before this node. This is equivalent to calling Ancestor with
-// the node's chain-height minus provided distance.
+// blue blocks before this node. This is equivalent to calling Ancestor with
+// the node's blue score minus provided distance.
 //
 // This function is safe for concurrent access.
 func (node *blockNode) RelativeAncestor(distance uint64) *blockNode {
-	return node.SelectedAncestor(node.chainHeight - distance)
+	return node.SelectedAncestor(node.blueScore - distance)
 }
 
 // CalcPastMedianTime returns the median time of the previous few blocks
 // prior to, and including, the block node.
 //
 // This function is safe for concurrent access.
-func (node *blockNode) PastMedianTime(dag *BlockDAG) time.Time {
+func (node *blockNode) PastMedianTime(dag *BlockDAG) mstime.Time {
 	window := blueBlockWindow(node, 2*dag.TimestampDeviationTolerance-1)
 	medianTimestamp, err := window.medianTimestamp()
 	if err != nil {
 		panic(fmt.Sprintf("blueBlockWindow: %s", err))
 	}
-	return time.Unix(medianTimestamp, 0)
+	return mstime.UnixMilliseconds(medianTimestamp)
 }
 
 func (node *blockNode) ParentHashes() []*daghash.Hash {
@@ -221,10 +224,14 @@ func (node *blockNode) isGenesis() bool {
 }
 
 func (node *blockNode) finalityScore(dag *BlockDAG) uint64 {
-	return node.blueScore / uint64(dag.dagParams.FinalityInterval)
+	return node.blueScore / uint64(dag.FinalityInterval())
 }
 
 // String returns a string that contains the block hash.
 func (node blockNode) String() string {
 	return node.hash.String()
 }
+
+func (node *blockNode) time() mstime.Time {
+	return mstime.UnixMilliseconds(node.timestamp)
+}

blockdag/blocknode_test.go

@@ -1,86 +1,41 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
 	"testing"
 )
 
-func TestChainHeight(t *testing.T) {
-	phantomK := uint32(2)
-	buildNode := buildNodeGenerator(phantomK, true)
+// This test is to ensure the size BlueAnticoneSizesSize is serialized to the size of KType.
+// We verify that by serializing and deserializing the block while making sure that we stay within the expected range.
+func TestBlueAnticoneSizesSize(t *testing.T) {
+	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizesSize", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestBlueAnticoneSizesSize: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
 
-	node0 := buildNode(setFromSlice())
-	node1 := buildNode(setFromSlice(node0))
-	node2 := buildNode(setFromSlice(node0))
-	node3 := buildNode(setFromSlice(node0))
-	node4 := buildNode(setFromSlice(node1, node2, node3))
-	node5 := buildNode(setFromSlice(node1, node2, node3))
-	node6 := buildNode(setFromSlice(node1, node2, node3))
-	node7 := buildNode(setFromSlice(node0))
-	node8 := buildNode(setFromSlice(node7))
-	node9 := buildNode(setFromSlice(node8))
-	node10 := buildNode(setFromSlice(node9, node6))
+	k := dagconfig.KType(0)
+	k--
 
-	// Because nodes 7 & 8 were mined secretly, node10's selected
-	// parent will be node6, although node9 is higher. So in this
-	// case, node10.height and node10.chainHeight will be different
-
-	tests := []struct {
-		node                *blockNode
-		expectedChainHeight uint64
-	}{
-		{
-			node:                node0,
-			expectedChainHeight: 0,
-		},
-		{
-			node:                node1,
-			expectedChainHeight: 1,
-		},
-		{
-			node:                node2,
-			expectedChainHeight: 1,
-		},
-		{
-			node:                node3,
-			expectedChainHeight: 1,
-		},
-		{
-			node:                node4,
-			expectedChainHeight: 2,
-		},
-		{
-			node:                node5,
-			expectedChainHeight: 2,
-		},
-		{
-			node:                node6,
-			expectedChainHeight: 2,
-		},
-		{
-			node:                node7,
-			expectedChainHeight: 1,
-		},
-		{
-			node:                node8,
-			expectedChainHeight: 2,
-		},
-		{
-			node:                node9,
-			expectedChainHeight: 3,
-		},
-		{
-			node:                node10,
-			expectedChainHeight: 3,
-		},
+	if k < dagconfig.KType(0) {
+		t.Fatalf("KType must be unsigned")
 	}
 
-	for _, test := range tests {
-		if test.node.chainHeight != test.expectedChainHeight {
-			t.Errorf("block %v expected chain height %v but got %v", test.node, test.expectedChainHeight, test.node.chainHeight)
-		}
-		if calculateChainHeight(test.node) != test.expectedChainHeight {
-			t.Errorf("block %v expected calculated chain height %v but got %v", test.node, test.expectedChainHeight, test.node.chainHeight)
-		}
+	blockHeader := dagconfig.SimnetParams.GenesisBlock.Header
+	node, _ := dag.newBlockNode(&blockHeader, newBlockSet())
+	fakeBlue := &blockNode{hash: &daghash.Hash{1}}
+	dag.index.AddNode(fakeBlue)
+	// Setting maxKType to maximum value of KType.
+	// As we verify above that KType is unsigned we can be sure that maxKType is indeed the maximum value of KType.
+	maxKType := ^dagconfig.KType(0)
+	node.bluesAnticoneSizes[fakeBlue] = maxKType
+	serializedNode, _ := serializeBlockNode(node)
+	deserializedNode, _ := dag.deserializeBlockNode(serializedNode)
+	if deserializedNode.bluesAnticoneSizes[fakeBlue] != maxKType {
+		t.Fatalf("TestBlueAnticoneSizesSize: BlueAnticoneSize should not change when deserializing. Expected: %v but got %v",
+			maxKType, deserializedNode.bluesAnticoneSizes[fakeBlue])
 	}
-
 }

blockdag/blockset.go

@@ -7,68 +7,68 @@ import (
 )
 
 // blockSet implements a basic unsorted set of blocks
-type blockSet map[daghash.Hash]*blockNode
+type blockSet map[*blockNode]struct{}
 
-// newSet creates a new, empty BlockSet
-func newSet() blockSet {
-	return map[daghash.Hash]*blockNode{}
+// newBlockSet creates a new, empty BlockSet
+func newBlockSet() blockSet {
+	return map[*blockNode]struct{}{}
 }
 
-// setFromSlice converts a slice of blocks into an unordered set represented as map
-func setFromSlice(blocks ...*blockNode) blockSet {
-	set := newSet()
-	for _, block := range blocks {
-		set.add(block)
+// blockSetFromSlice converts a slice of blockNodes into an unordered set represented as map
+func blockSetFromSlice(nodes ...*blockNode) blockSet {
+	set := newBlockSet()
+	for _, node := range nodes {
+		set.add(node)
 	}
 	return set
 }
 
-// add adds a block to this BlockSet
-func (bs blockSet) add(block *blockNode) {
-	bs[*block.hash] = block
+// add adds a blockNode to this BlockSet
+func (bs blockSet) add(node *blockNode) {
+	bs[node] = struct{}{}
 }
 
-// remove removes a block from this BlockSet, if exists
-// Does nothing if this set does not contain the block
-func (bs blockSet) remove(block *blockNode) {
-	delete(bs, *block.hash)
+// remove removes a blockNode from this BlockSet, if exists
+// Does nothing if this set does not contain the blockNode
+func (bs blockSet) remove(node *blockNode) {
+	delete(bs, node)
 }
 
 // clone clones thie block set
 func (bs blockSet) clone() blockSet {
-	clone := newSet()
-	for _, block := range bs {
-		clone.add(block)
+	clone := newBlockSet()
+	for node := range bs {
+		clone.add(node)
 	}
 	return clone
 }
 
 // subtract returns the difference between the BlockSet and another BlockSet
 func (bs blockSet) subtract(other blockSet) blockSet {
-	diff := newSet()
-	for _, block := range bs {
-		if !other.contains(block) {
-			diff.add(block)
+	diff := newBlockSet()
+	for node := range bs {
+		if !other.contains(node) {
+			diff.add(node)
 		}
 	}
 	return diff
 }
 
-// addSet adds all blocks in other set to this set
+// addSet adds all blockNodes in other set to this set
 func (bs blockSet) addSet(other blockSet) {
-	for _, block := range other {
-		bs.add(block)
+	for node := range other {
+		bs.add(node)
 	}
 }
 
 // addSlice adds provided slice to this set
 func (bs blockSet) addSlice(slice []*blockNode) {
-	for _, block := range slice {
-		bs.add(block)
+	for _, node := range slice {
+		bs.add(node)
 	}
 }
 
-// union returns a BlockSet that contains all blocks included in this set,
+// union returns a BlockSet that contains all blockNodes included in this set,
 // the other set, or both
 func (bs blockSet) union(other blockSet) blockSet {
 	union := bs.clone()
@@ -78,39 +78,16 @@ func (bs blockSet) union(other blockSet) blockSet {
 	return union
 }
 
-// contains returns true iff this set contains block
-func (bs blockSet) contains(block *blockNode) bool {
-	_, ok := bs[*block.hash]
+// contains returns true iff this set contains node
+func (bs blockSet) contains(node *blockNode) bool {
+	_, ok := bs[node]
 	return ok
 }
 
-// containsHash returns true iff this set contains a block hash
-func (bs blockSet) containsHash(hash *daghash.Hash) bool {
-	_, ok := bs[*hash]
-	return ok
-}
-
-// hashesEqual returns true if the given hashes are equal to the hashes
-// of the blocks in this set.
-// NOTE: The given hash slice must not contain duplicates.
-func (bs blockSet) hashesEqual(hashes []*daghash.Hash) bool {
-	if len(hashes) != len(bs) {
-		return false
-	}
-
-	for _, hash := range hashes {
-		if _, wasFound := bs[*hash]; !wasFound {
-			return false
-		}
-	}
-
-	return true
-}
-
-// hashes returns the hashes of the blocks in this set.
+// hashes returns the hashes of the blockNodes in this set.
 func (bs blockSet) hashes() []*daghash.Hash {
 	hashes := make([]*daghash.Hash, 0, len(bs))
-	for _, node := range bs {
+	for node := range bs {
 		hashes = append(hashes, node.hash)
 	}
 	daghash.Sort(hashes)
@@ -119,27 +96,16 @@ func (bs blockSet) hashes() []*daghash.Hash {
 
 func (bs blockSet) String() string {
 	nodeStrs := make([]string, 0, len(bs))
-	for _, node := range bs {
+	for node := range bs {
 		nodeStrs = append(nodeStrs, node.String())
 	}
 	return strings.Join(nodeStrs, ",")
 }
 
-// anyChildInSet returns true iff any child of block is contained within this set
-func (bs blockSet) anyChildInSet(block *blockNode) bool {
-	for _, child := range block.children {
-		if bs.contains(child) {
-			return true
-		}
-	}
-
-	return false
-}
-
 func (bs blockSet) bluest() *blockNode {
 	var bluestNode *blockNode
 	var maxScore uint64
-	for _, node := range bs {
+	for node := range bs {
 		if bluestNode == nil ||
 			node.blueScore > maxScore ||
 			(node.blueScore == maxScore && daghash.Less(node.hash, bluestNode.hash)) {

blockdag/blockset_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestHashes(t *testing.T) {
-	bs := setFromSlice(
+	bs := blockSetFromSlice(
 		&blockNode{
 			hash: &daghash.Hash{3},
 		},
@@ -49,33 +49,33 @@ func TestBlockSetSubtract(t *testing.T) {
 	}{
 		{
 			name:           "both sets empty",
-			setA:           setFromSlice(),
-			setB:           setFromSlice(),
-			expectedResult: setFromSlice(),
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(),
 		},
 		{
 			name:           "subtract an empty set",
-			setA:           setFromSlice(node1),
-			setB:           setFromSlice(),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "subtract from empty set",
-			setA:           setFromSlice(),
-			setB:           setFromSlice(node1),
-			expectedResult: setFromSlice(),
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(),
 		},
 		{
 			name:           "subtract unrelated set",
-			setA:           setFromSlice(node1),
-			setB:           setFromSlice(node2),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(node2),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "typical case",
-			setA:           setFromSlice(node1, node2),
-			setB:           setFromSlice(node2, node3),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node2, node3),
+			expectedResult: blockSetFromSlice(node1),
 		},
 	}
 
@@ -101,33 +101,33 @@ func TestBlockSetAddSet(t *testing.T) {
 	}{
 		{
 			name:           "both sets empty",
-			setA:           setFromSlice(),
-			setB:           setFromSlice(),
-			expectedResult: setFromSlice(),
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(),
 		},
 		{
 			name:           "add an empty set",
-			setA:           setFromSlice(node1),
-			setB:           setFromSlice(),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "add to empty set",
-			setA:           setFromSlice(),
-			setB:           setFromSlice(node1),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "add already added member",
-			setA:           setFromSlice(node1, node2),
-			setB:           setFromSlice(node1),
-			expectedResult: setFromSlice(node1, node2),
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1, node2),
 		},
 		{
 			name:           "typical case",
-			setA:           setFromSlice(node1, node2),
-			setB:           setFromSlice(node2, node3),
-			expectedResult: setFromSlice(node1, node2, node3),
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node2, node3),
+			expectedResult: blockSetFromSlice(node1, node2, node3),
 		},
 	}
 
@@ -153,33 +153,33 @@ func TestBlockSetAddSlice(t *testing.T) {
 	}{
 		{
 			name:           "add empty slice to empty set",
-			set:            setFromSlice(),
+			set:            blockSetFromSlice(),
 			slice:          []*blockNode{},
-			expectedResult: setFromSlice(),
+			expectedResult: blockSetFromSlice(),
 		},
 		{
 			name:           "add an empty slice",
-			set:            setFromSlice(node1),
+			set:            blockSetFromSlice(node1),
 			slice:          []*blockNode{},
-			expectedResult: setFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "add to empty set",
-			set:            setFromSlice(),
+			set:            blockSetFromSlice(),
 			slice:          []*blockNode{node1},
-			expectedResult: setFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "add already added member",
-			set:            setFromSlice(node1, node2),
+			set:            blockSetFromSlice(node1, node2),
 			slice:          []*blockNode{node1},
-			expectedResult: setFromSlice(node1, node2),
+			expectedResult: blockSetFromSlice(node1, node2),
 		},
 		{
 			name:           "typical case",
-			set:            setFromSlice(node1, node2),
+			set:            blockSetFromSlice(node1, node2),
 			slice:          []*blockNode{node2, node3},
-			expectedResult: setFromSlice(node1, node2, node3),
+			expectedResult: blockSetFromSlice(node1, node2, node3),
 		},
 	}
 
@@ -205,33 +205,33 @@ func TestBlockSetUnion(t *testing.T) {
 	}{
 		{
 			name:           "both sets empty",
-			setA:           setFromSlice(),
-			setB:           setFromSlice(),
-			expectedResult: setFromSlice(),
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(),
 		},
 		{
 			name:           "union against an empty set",
-			setA:           setFromSlice(node1),
-			setB:           setFromSlice(),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "union from an empty set",
-			setA:           setFromSlice(),
-			setB:           setFromSlice(node1),
-			expectedResult: setFromSlice(node1),
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1),
 		},
 		{
 			name:           "union with subset",
-			setA:           setFromSlice(node1, node2),
-			setB:           setFromSlice(node1),
-			expectedResult: setFromSlice(node1, node2),
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1, node2),
 		},
 		{
 			name:           "typical case",
-			setA:           setFromSlice(node1, node2),
-			setB:           setFromSlice(node2, node3),
-			expectedResult: setFromSlice(node1, node2, node3),
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node2, node3),
+			expectedResult: blockSetFromSlice(node1, node2, node3),
 		},
 	}
 
@@ -243,54 +243,3 @@ func TestBlockSetUnion(t *testing.T) {
 		}
 	}
 }
-
-func TestBlockSetHashesEqual(t *testing.T) {
-	node1 := &blockNode{hash: &daghash.Hash{10}}
-	node2 := &blockNode{hash: &daghash.Hash{20}}
-
-	tests := []struct {
-		name           string
-		set            blockSet
-		hashes         []*daghash.Hash
-		expectedResult bool
-	}{
-		{
-			name:           "empty set, no hashes",
-			set:            setFromSlice(),
-			hashes:         []*daghash.Hash{},
-			expectedResult: true,
-		},
-		{
-			name:           "empty set, one hash",
-			set:            setFromSlice(),
-			hashes:         []*daghash.Hash{node1.hash},
-			expectedResult: false,
-		},
-		{
-			name:           "set and hashes of different length",
-			set:            setFromSlice(node1, node2),
-			hashes:         []*daghash.Hash{node1.hash},
-			expectedResult: false,
-		},
-		{
-			name:           "set equal to hashes",
-			set:            setFromSlice(node1, node2),
-			hashes:         []*daghash.Hash{node1.hash, node2.hash},
-			expectedResult: true,
-		},
-		{
-			name:           "set equal to hashes, different order",
-			set:            setFromSlice(node1, node2),
-			hashes:         []*daghash.Hash{node2.hash, node1.hash},
-			expectedResult: true,
-		},
-	}
-
-	for _, test := range tests {
-		result := test.set.hashesEqual(test.hashes)
-		if result != test.expectedResult {
-			t.Errorf("blockSet.hashesEqual: unexpected result in test '%s'. "+
-				"Expected: %t, got: %t", test.name, test.expectedResult, result)
-		}
-	}
-}

blockdag/blockwindow.go

@@ -2,6 +2,7 @@ package blockdag
 
 import (
 	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/bigintpool"
 	"github.com/pkg/errors"
 	"math"
 	"math/big"
@@ -11,7 +12,7 @@ import (
 type blockWindow []*blockNode
 
 // blueBlockWindow returns a blockWindow of the given size that contains the
-// blues in the past of startindNode, sorted by phantom order.
+// blues in the past of startindNode, sorted by GHOSTDAG order.
 // If the number of blues in the past of startingNode is less then windowSize,
 // the window will be padded by genesis blocks to achieve a size of windowSize.
 func blueBlockWindow(startingNode *blockNode, windowSize uint64) blockWindow {
@@ -53,13 +54,19 @@ func (window blockWindow) minMaxTimestamps() (min, max int64) {
 	return
 }
 
-func (window blockWindow) averageTarget() *big.Int {
-	averageTarget := big.NewInt(0)
+func (window blockWindow) averageTarget(averageTarget *big.Int) {
+	averageTarget.SetInt64(0)
+
+	target := bigintpool.Acquire(0)
+	defer bigintpool.Release(target)
 	for _, node := range window {
-		target := util.CompactToBig(node.bits)
+		util.CompactToBigWithDestination(node.bits, target)
 		averageTarget.Add(averageTarget, target)
 	}
-	return averageTarget.Div(averageTarget, big.NewInt(int64(len(window))))
+
+	windowLen := bigintpool.Acquire(int64(len(window)))
+	defer bigintpool.Release(windowLen)
+	averageTarget.Div(averageTarget, windowLen)
 }
 
 func (window blockWindow) medianTimestamp() (int64, error) {

blockdag/blockwindow_test.go

@@ -2,7 +2,7 @@ package blockdag
 
 import (
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
 	"reflect"
 	"testing"
@@ -10,9 +10,17 @@ import (
 )
 
 func TestBlueBlockWindow(t *testing.T) {
-	params := dagconfig.SimNetParams
+	params := dagconfig.SimnetParams
 	params.K = 1
-	dag := newTestDAG(&params)
+	dag, teardownFunc, err := DAGSetup("TestBlueBlockWindow", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	resetExtraNonceForTest()
 
 	windowSize := uint64(10)
 	genesisNode := dag.genesis
@@ -21,7 +29,6 @@ func TestBlueBlockWindow(t *testing.T) {
 	idByBlockMap := make(map[*blockNode]string)
 	blockByIDMap["A"] = genesisNode
 	idByBlockMap[genesisNode] = "A"
-	blockVersion := int32(0x10000000)
 
 	blocksData := []*struct {
 		parents                          []string
@@ -44,12 +51,12 @@ func TestBlueBlockWindow(t *testing.T) {
 			expectedWindowWithGenesisPadding: []string{"B", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
 		},
 		{
-			parents:                          []string{"C", "D"},
+			parents:                          []string{"D", "C"},
 			id:                               "E",
 			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
 		},
 		{
-			parents:                          []string{"C", "D"},
+			parents:                          []string{"D", "C"},
 			id:                               "F",
 			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
 		},
@@ -107,14 +114,29 @@ func TestBlueBlockWindow(t *testing.T) {
 			parent := blockByIDMap[parentID]
 			parents.add(parent)
 		}
-		node := newTestNode(parents, blockVersion, 0, blockTime, dag.dagParams.K)
-		node.hash = &daghash.Hash{} // It helps to predict hash order
-		for i, char := range blockData.id {
-			node.hash[i] = byte(char)
+
+		block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+		if err != nil {
+			t.Fatalf("block %v got unexpected error from PrepareBlockForTest: %v", blockData.id, err)
 		}
 
-		dag.index.AddNode(node)
-		node.updateParentsChildren()
+		utilBlock := util.NewBlock(block)
+		isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+		if err != nil {
+			t.Fatalf("dag.ProcessBlock got unexpected error for block %v: %v", blockData.id, err)
+		}
+		if isDelayed {
+			t.Fatalf("block %s "+
+				"is too far in the future", blockData.id)
+		}
+		if isOrphan {
+			t.Fatalf("block %v was unexpectedly orphan", blockData.id)
+		}
+
+		node, ok := dag.index.LookupNode(utilBlock.Hash())
+		if !ok {
+			t.Fatalf("block %s does not exist in the DAG", utilBlock.Hash())
+		}
 
 		blockByIDMap[blockData.id] = node
 		idByBlockMap[node] = blockData.id

blockdag/coinbase.go

@@ -4,16 +4,16 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
-	"github.com/kaspanet/kaspad/util/subnetworkid"
-	"github.com/pkg/errors"
 	"io"
-	"math"
 
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/coinbasepayload"
 	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 	"github.com/kaspanet/kaspad/util/txsort"
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
 )
 
 // compactFeeData is a specialized data type to store a compact list of fees
@@ -73,55 +73,24 @@ func (cfr *compactFeeIterator) next() (uint64, error) {
 }
 
 // The following functions relate to storing and retrieving fee data from the database
-var feeBucket = []byte("fees")
 
 // getBluesFeeData returns the compactFeeData for all nodes's blues,
 // used to calculate the fees this blockNode needs to pay
-func (node *blockNode) getBluesFeeData(dag *BlockDAG) (map[daghash.Hash]compactFeeData, error) {
+func (dag *BlockDAG) getBluesFeeData(node *blockNode) (map[daghash.Hash]compactFeeData, error) {
 	bluesFeeData := make(map[daghash.Hash]compactFeeData)
 
-	err := dag.db.View(func(dbTx database.Tx) error {
-		for _, blueBlock := range node.blues {
-			feeData, err := dbFetchFeeData(dbTx, blueBlock.hash)
-			if err != nil {
-				return errors.Errorf("Error getting fee data for block %s: %s", blueBlock.hash, err)
-			}
-
-			bluesFeeData[*blueBlock.hash] = feeData
+	for _, blueBlock := range node.blues {
+		feeData, err := dbaccess.FetchFeeData(dag.databaseContext, blueBlock.hash)
+		if err != nil {
+			return nil, err
 		}
 
-		return nil
-	})
-	if err != nil {
-		return nil, err
+		bluesFeeData[*blueBlock.hash] = feeData
 	}
 
 	return bluesFeeData, nil
 }
 
-func dbStoreFeeData(dbTx database.Tx, blockHash *daghash.Hash, feeData compactFeeData) error {
-	feeBucket, err := dbTx.Metadata().CreateBucketIfNotExists(feeBucket)
-	if err != nil {
-		return errors.Errorf("Error creating or retrieving fee bucket: %s", err)
-	}
-
-	return feeBucket.Put(blockHash.CloneBytes(), feeData)
-}
-
-func dbFetchFeeData(dbTx database.Tx, blockHash *daghash.Hash) (compactFeeData, error) {
-	feeBucket := dbTx.Metadata().Bucket(feeBucket)
-	if feeBucket == nil {
-		return nil, errors.New("Fee bucket does not exist")
-	}
-
-	feeData := feeBucket.Get(blockHash.CloneBytes())
-	if feeData == nil {
-		return nil, errors.Errorf("No fee data found for block %s", blockHash)
-	}
-
-	return feeData, nil
-}
-
 // The following functions deal with building and validating the coinbase transaction
 
 func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Block, txsAcceptanceData MultiBlockTxsAcceptanceData) error {
@@ -129,7 +98,10 @@ func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Bl
 		return nil
 	}
 	blockCoinbaseTx := block.CoinbaseTransaction().MsgTx()
-	scriptPubKey, extraData, err := DeserializeCoinbasePayload(blockCoinbaseTx)
+	_, scriptPubKey, extraData, err := coinbasepayload.DeserializeCoinbasePayload(blockCoinbaseTx)
+	if errors.Is(err, coinbasepayload.ErrIncorrectScriptPubKeyLen) {
+		return ruleError(ErrBadCoinbaseTransaction, err.Error())
+	}
 	if err != nil {
 		return err
 	}
@@ -147,132 +119,81 @@ func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Bl
 
 // expectedCoinbaseTransaction returns the coinbase transaction for the current block
 func (node *blockNode) expectedCoinbaseTransaction(dag *BlockDAG, txsAcceptanceData MultiBlockTxsAcceptanceData, scriptPubKey []byte, extraData []byte) (*util.Tx, error) {
-	bluesFeeData, err := node.getBluesFeeData(dag)
+	bluesFeeData, err := dag.getBluesFeeData(node)
 	if err != nil {
 		return nil, err
 	}
 
-	txIns := []*wire.TxIn{}
-	txOuts := []*wire.TxOut{}
+	txIns := []*domainmessage.TxIn{}
+	txOuts := []*domainmessage.TxOut{}
 
 	for _, blue := range node.blues {
-		txIn, txOut, err := coinbaseInputAndOutputForBlueBlock(dag, blue, txsAcceptanceData, bluesFeeData)
+		txOut, err := coinbaseOutputForBlueBlock(dag, blue, txsAcceptanceData, bluesFeeData)
 		if err != nil {
 			return nil, err
 		}
-		txIns = append(txIns, txIn)
 		if txOut != nil {
 			txOuts = append(txOuts, txOut)
 		}
 	}
-	payload, err := SerializeCoinbasePayload(scriptPubKey, extraData)
+	payload, err := coinbasepayload.SerializeCoinbasePayload(node.blueScore, scriptPubKey, extraData)
 	if err != nil {
 		return nil, err
 	}
-	coinbaseTx := wire.NewSubnetworkMsgTx(wire.TxVersion, txIns, txOuts, subnetworkid.SubnetworkIDCoinbase, 0, payload)
+	coinbaseTx := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, txIns, txOuts, subnetworkid.SubnetworkIDCoinbase, 0, payload)
 	sortedCoinbaseTx := txsort.Sort(coinbaseTx)
 	return util.NewTx(sortedCoinbaseTx), nil
 }
 
-// SerializeCoinbasePayload builds the coinbase payload based on the provided scriptPubKey and extra data.
-func SerializeCoinbasePayload(scriptPubKey []byte, extraData []byte) ([]byte, error) {
-	w := &bytes.Buffer{}
-	err := wire.WriteVarInt(w, uint64(len(scriptPubKey)))
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(scriptPubKey)
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(extraData)
-	if err != nil {
-		return nil, err
-	}
-	return w.Bytes(), nil
-}
-
-// DeserializeCoinbasePayload deserialize the coinbase payload to its component (scriptPubKey and extra data).
-func DeserializeCoinbasePayload(tx *wire.MsgTx) (scriptPubKey []byte, extraData []byte, err error) {
-	r := bytes.NewReader(tx.Payload)
-	scriptPubKeyLen, err := wire.ReadVarInt(r)
-	if err != nil {
-		return nil, nil, err
-	}
-	scriptPubKey = make([]byte, scriptPubKeyLen)
-	_, err = r.Read(scriptPubKey)
-	if err != nil {
-		return nil, nil, err
-	}
-	extraData = make([]byte, r.Len())
-	if r.Len() != 0 {
-		_, err = r.Read(extraData)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-	return scriptPubKey, extraData, nil
-}
-
-// feeInputAndOutputForBlueBlock calculates the input and output that should go into the coinbase transaction of blueBlock
-// If blueBlock gets no fee - returns only txIn and nil for txOut
-func coinbaseInputAndOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
-	txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (
-	*wire.TxIn, *wire.TxOut, error) {
+// coinbaseOutputForBlueBlock calculates the output that should go into the coinbase transaction of blueBlock
+// If blueBlock gets no fee - returns nil for txOut
+func coinbaseOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
+	txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (*domainmessage.TxOut, error) {
 
 	blockTxsAcceptanceData, ok := txsAcceptanceData.FindAcceptanceData(blueBlock.hash)
 	if !ok {
-		return nil, nil, errors.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
+		return nil, errors.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
 	}
 	blockFeeData, ok := feeData[*blueBlock.hash]
 	if !ok {
-		return nil, nil, errors.Errorf("No feeData for block %s", blueBlock.hash)
+		return nil, errors.Errorf("No feeData for block %s", blueBlock.hash)
 	}
 
 	if len(blockTxsAcceptanceData.TxAcceptanceData) != blockFeeData.Len() {
-		return nil, nil, errors.Errorf(
+		return nil, errors.Errorf(
 			"length of accepted transaction data(%d) and fee data(%d) is not equal for block %s",
 			len(blockTxsAcceptanceData.TxAcceptanceData), blockFeeData.Len(), blueBlock.hash)
 	}
 
-	txIn := &wire.TxIn{
-		SignatureScript: []byte{},
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID(*blueBlock.hash),
-			Index: math.MaxUint32,
-		},
-		Sequence: wire.MaxTxInSequenceNum,
-	}
-
 	totalFees := uint64(0)
 	feeIterator := blockFeeData.iterator()
 
 	for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
 		fee, err := feeIterator.next()
 		if err != nil {
-			return nil, nil, errors.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
+			return nil, errors.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
 		}
 		if txAcceptanceData.IsAccepted {
 			totalFees += fee
 		}
 	}
 
-	totalReward := CalcBlockSubsidy(blueBlock.blueScore, dag.dagParams) + totalFees
+	totalReward := CalcBlockSubsidy(blueBlock.blueScore, dag.Params) + totalFees
 
 	if totalReward == 0 {
-		return txIn, nil, nil
+		return nil, nil
 	}
 
 	// the ScriptPubKey for the coinbase is parsed from the coinbase payload
-	scriptPubKey, _, err := DeserializeCoinbasePayload(blockTxsAcceptanceData.TxAcceptanceData[0].Tx.MsgTx())
+	_, scriptPubKey, _, err := coinbasepayload.DeserializeCoinbasePayload(blockTxsAcceptanceData.TxAcceptanceData[0].Tx.MsgTx())
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	txOut := &wire.TxOut{
+	txOut := &domainmessage.TxOut{
 		Value:        totalReward,
 		ScriptPubKey: scriptPubKey,
 	}
 
-	return txIn, txOut, nil
+	return txOut, nil
 }

blockdag/common_test.go

@@ -7,33 +7,22 @@ package blockdag
 import (
 	"compress/bzip2"
 	"encoding/binary"
-	"github.com/pkg/errors"
 	"io"
 	"os"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"testing"
-	"time"
+
+	"github.com/kaspanet/kaspad/util/mstime"
+
+	"github.com/pkg/errors"
 
 	"github.com/kaspanet/kaspad/dagconfig"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
 )
 
-func loadBlocksWithLog(t *testing.T, filename string) ([]*util.Block, error) {
-	blocks, err := LoadBlocks(filename)
-	if err == nil {
-		t.Logf("Loaded %d blocks from file %s", len(blocks), filename)
-		for i, b := range blocks {
-			t.Logf("Block #%d: %s", i, b.Hash())
-		}
-	}
-	return blocks, err
-}
-
 // loadUTXOSet returns a utxo view loaded from a file.
 func loadUTXOSet(filename string) (UTXOSet, error) {
 	// The utxostore file format is:
@@ -84,19 +73,12 @@ func loadUTXOSet(filename string) (UTXOSet, error) {
 			return nil, err
 		}
 
-		// Serialized utxo entry.
-		serialized := make([]byte, numBytes)
-		_, err = io.ReadAtLeast(r, serialized, int(numBytes))
+		// Deserialize the UTXO entry and add it to the UTXO set.
+		entry, err := deserializeUTXOEntry(r)
 		if err != nil {
 			return nil, err
 		}
-
-		// Deserialize it and add it to the view.
-		entry, err := deserializeUTXOEntry(serialized)
-		if err != nil {
-			return nil, err
-		}
-		utxoSet.utxoCollection[wire.Outpoint{TxID: txID, Index: index}] = entry
+		utxoSet.utxoCollection[domainmessage.Outpoint{TxID: txID, Index: index}] = entry
 	}
 
 	return utxoSet, nil
@@ -105,7 +87,7 @@ func loadUTXOSet(filename string) (UTXOSet, error) {
 // TestSetCoinbaseMaturity makes the ability to set the coinbase maturity
 // available when running tests.
 func (dag *BlockDAG) TestSetCoinbaseMaturity(maturity uint64) {
-	dag.dagParams.BlockCoinbaseMaturity = maturity
+	dag.Params.BlockCoinbaseMaturity = maturity
 }
 
 // newTestDAG returns a DAG that is usable for syntetic tests. It is
@@ -113,33 +95,32 @@ func (dag *BlockDAG) TestSetCoinbaseMaturity(maturity uint64) {
 // it is not usable with all functions and the tests must take care when making
 // use of it.
 func newTestDAG(params *dagconfig.Params) *BlockDAG {
-	// Create a genesis block node and block index index populated with it
-	// for use when creating the fake DAG below.
-	node := newBlockNode(&params.GenesisBlock.Header, newSet(), params.K)
-	index := newBlockIndex(nil, params)
-	index.AddNode(node)
-
-	targetTimePerBlock := int64(params.TargetTimePerBlock / time.Second)
-	return &BlockDAG{
-		dagParams:                      params,
-		timeSource:                     NewMedianTime(),
-		targetTimePerBlock:             targetTimePerBlock,
+	index := newBlockIndex(params)
+	dag := &BlockDAG{
+		Params:                         params,
+		timeSource:                     NewTimeSource(),
 		difficultyAdjustmentWindowSize: params.DifficultyAdjustmentWindowSize,
 		TimestampDeviationTolerance:    params.TimestampDeviationTolerance,
 		powMaxBits:                     util.BigToCompact(params.PowMax),
 		index:                          index,
-		virtual:                        newVirtualBlock(setFromSlice(node), params.K),
-		genesis:                        index.LookupNode(params.GenesisHash),
 		warningCaches:                  newThresholdCaches(vbNumBits),
 		deploymentCaches:               newThresholdCaches(dagconfig.DefinedDeployments),
 	}
+
+	// Create a genesis block node and block index index populated with it
+	// on the above fake DAG.
+	dag.genesis, _ = dag.newBlockNode(&params.GenesisBlock.Header, newBlockSet())
+	index.AddNode(dag.genesis)
+
+	dag.virtual = newVirtualBlock(dag, blockSetFromSlice(dag.genesis))
+	return dag
 }
 
 // newTestNode creates a block node connected to the passed parent with the
 // provided fields populated and fake values for the other fields.
-func newTestNode(parents blockSet, blockVersion int32, bits uint32, timestamp time.Time, phantomK uint32) *blockNode {
+func newTestNode(dag *BlockDAG, parents blockSet, blockVersion int32, bits uint32, timestamp mstime.Time) *blockNode {
 	// Make up a header and create a block node from it.
-	header := &wire.BlockHeader{
+	header := &domainmessage.BlockHeader{
 		Version:              blockVersion,
 		ParentHashes:         parents.hashes(),
 		Bits:                 bits,
@@ -148,64 +129,69 @@ func newTestNode(parents blockSet, blockVersion int32, bits uint32, timestamp ti
 		AcceptedIDMerkleRoot: &daghash.ZeroHash,
 		UTXOCommitment:       &daghash.ZeroHash,
 	}
-	return newBlockNode(header, parents, phantomK)
+	node, _ := dag.newBlockNode(header, parents)
+	return node
 }
 
 func addNodeAsChildToParents(node *blockNode) {
-	for _, parent := range node.parents {
+	for parent := range node.parents {
 		parent.children.add(node)
 	}
 }
 
-func buildNodeGenerator(phantomK uint32, withChildren bool) func(parents blockSet) *blockNode {
-	// For the purposes of these tests, we'll create blockNodes whose hashes are a
-	// series of numbers from 1 to 255.
-	hashCounter := byte(1)
-	buildNode := func(parents blockSet) *blockNode {
-		block := newBlockNode(nil, parents, phantomK)
-		block.hash = &daghash.Hash{hashCounter}
-		hashCounter++
-
-		return block
-	}
-	if withChildren {
-		return func(parents blockSet) *blockNode {
-			node := buildNode(parents)
-			addNodeAsChildToParents(node)
-			return node
-		}
-	}
-	return buildNode
-}
-
 // checkRuleError ensures the type of the two passed errors are of the
 // same type (either both nil or both of type RuleError) and their error codes
 // match when not nil.
 func checkRuleError(gotErr, wantErr error) error {
-	// Ensure the error code is of the expected type and the error
-	// code matches the value specified in the test instance.
-	if reflect.TypeOf(gotErr) != reflect.TypeOf(wantErr) {
-		return errors.Errorf("wrong error - got %T (%[1]v), want %T",
-			gotErr, wantErr)
-	}
-	if gotErr == nil {
+	if wantErr == nil && gotErr == nil {
 		return nil
 	}
 
-	// Ensure the want error type is a script error.
-	werr, ok := wantErr.(RuleError)
-	if !ok {
-		return errors.Errorf("unexpected test error type %T", wantErr)
+	var gotRuleErr RuleError
+	if ok := errors.As(gotErr, &gotRuleErr); !ok {
+		return errors.Errorf("gotErr expected to be RuleError, but got %+v instead", gotErr)
+	}
+
+	var wantRuleErr RuleError
+	if ok := errors.As(wantErr, &wantRuleErr); !ok {
+		return errors.Errorf("wantErr expected to be RuleError, but got %+v instead", wantErr)
 	}
 
 	// Ensure the error codes match. It's safe to use a raw type assert
 	// here since the code above already proved they are the same type and
 	// the want error is a script error.
-	gotErrorCode := gotErr.(RuleError).ErrorCode
-	if gotErrorCode != werr.ErrorCode {
+	if gotRuleErr.ErrorCode != wantRuleErr.ErrorCode {
 		return errors.Errorf("mismatched error code - got %v (%v), want %v",
-			gotErrorCode, gotErr, werr.ErrorCode)
+			gotRuleErr.ErrorCode, gotErr, wantRuleErr.ErrorCode)
 	}
 
 	return nil
 }
+
+func prepareAndProcessBlockByParentMsgBlocks(t *testing.T, dag *BlockDAG, parents ...*domainmessage.MsgBlock) *domainmessage.MsgBlock {
+	parentHashes := make([]*daghash.Hash, len(parents))
+	for i, parent := range parents {
+		parentHashes[i] = parent.BlockHash()
+	}
+	return PrepareAndProcessBlockForTest(t, dag, parentHashes, nil)
+}
+
+func nodeByMsgBlock(t *testing.T, dag *BlockDAG, block *domainmessage.MsgBlock) *blockNode {
+	node, ok := dag.index.LookupNode(block.BlockHash())
+	if !ok {
+		t.Fatalf("couldn't find block node with hash %s", block.BlockHash())
+	}
+	return node
+}
+
+type fakeTimeSource struct {
+	time mstime.Time
+}
+
+func (fts *fakeTimeSource) Now() mstime.Time {
+	return fts.time
+}
+
+func newFakeTimeSource(fakeTime mstime.Time) TimeSource {
+	return &fakeTimeSource{time: fakeTime}
+}

blockdag/compress.go

@@ -1,584 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/ecc"
-	"github.com/kaspanet/kaspad/txscript"
-)
-
-// -----------------------------------------------------------------------------
-// A variable length quantity (VLQ) is an encoding that uses an arbitrary number
-// of binary octets to represent an arbitrarily large integer. The scheme
-// employs a most significant byte (MSB) base-128 encoding where the high bit in
-// each byte indicates whether or not the byte is the final one. In addition,
-// to ensure there are no redundant encodings, an offset is subtracted every
-// time a group of 7 bits is shifted out. Therefore each integer can be
-// represented in exactly one way, and each representation stands for exactly
-// one integer.
-//
-// Another nice property of this encoding is that it provides a compact
-// representation of values that are typically used to indicate sizes. For
-// example, the values 0 - 127 are represented with a single byte, 128 - 16511
-// with two bytes, and 16512 - 2113663 with three bytes.
-//
-// While the encoding allows arbitrarily large integers, it is artificially
-// limited in this code to an unsigned 64-bit integer for efficiency purposes.
-//
-// Example encodings:
-//           0 -> [0x00]
-//         127 -> [0x7f]                 * Max 1-byte value
-//         128 -> [0x80 0x00]
-//         129 -> [0x80 0x01]
-//         255 -> [0x80 0x7f]
-//         256 -> [0x81 0x00]
-//       16511 -> [0xff 0x7f]            * Max 2-byte value
-//       16512 -> [0x80 0x80 0x00]
-//       32895 -> [0x80 0xff 0x7f]
-//     2113663 -> [0xff 0xff 0x7f]       * Max 3-byte value
-//   270549119 -> [0xff 0xff 0xff 0x7f]  * Max 4-byte value
-//      2^64-1 -> [0x80 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0x7f]
-//
-// References:
-//   https://en.wikipedia.org/wiki/Variable-length_quantity
-//   http://www.codecodex.com/wiki/Variable-Length_Integers
-// -----------------------------------------------------------------------------
-
-// serializeSizeVLQ returns the number of bytes it would take to serialize the
-// passed number as a variable-length quantity according to the format described
-// above.
-func serializeSizeVLQ(n uint64) int {
-	size := 1
-	for ; n > 0x7f; n = (n >> 7) - 1 {
-		size++
-	}
-
-	return size
-}
-
-// putVLQ serializes the provided number to a variable-length quantity according
-// to the format described above and returns the number of bytes of the encoded
-// value. The result is placed directly into the passed byte slice which must
-// be at least large enough to handle the number of bytes returned by the
-// serializeSizeVLQ function or it will panic.
-func putVLQ(target []byte, n uint64) int {
-	offset := 0
-	for ; ; offset++ {
-		// The high bit is set when another byte follows.
-		highBitMask := byte(0x80)
-		if offset == 0 {
-			highBitMask = 0x00
-		}
-
-		target[offset] = byte(n&0x7f) | highBitMask
-		if n <= 0x7f {
-			break
-		}
-		n = (n >> 7) - 1
-	}
-
-	// Reverse the bytes so it is MSB-encoded.
-	for i, j := 0, offset; i < j; i, j = i+1, j-1 {
-		target[i], target[j] = target[j], target[i]
-	}
-
-	return offset + 1
-}
-
-// deserializeVLQ deserializes the provided variable-length quantity according
-// to the format described above. It also returns the number of bytes
-// deserialized.
-func deserializeVLQ(serialized []byte) (uint64, int) {
-	var n uint64
-	var size int
-	for _, val := range serialized {
-		size++
-		n = (n << 7) | uint64(val&0x7f)
-		if val&0x80 != 0x80 {
-			break
-		}
-		n++
-	}
-
-	return n, size
-}
-
-// -----------------------------------------------------------------------------
-// In order to reduce the size of stored scripts, a domain specific compression
-// algorithm is used which recognizes standard scripts and stores them using
-// less bytes than the original script.
-//
-// The general serialized format is:
-//
-//   <script size or type><script data>
-//
-//   Field                 Type     Size
-//   script size or type   VLQ      variable
-//   script data           []byte   variable
-//
-// The specific serialized format for each recognized standard script is:
-//
-// - Pay-to-pubkey-hash: (21 bytes) - <0><20-byte pubkey hash>
-// - Pay-to-script-hash: (21 bytes) - <1><20-byte script hash>
-// - Pay-to-pubkey**:    (33 bytes) - <2, 3, 4, or 5><32-byte pubkey X value>
-//   2, 3 = compressed pubkey with bit 0 specifying the y coordinate to use
-//   4, 5 = uncompressed pubkey with bit 0 specifying the y coordinate to use
-//   ** Only valid public keys starting with 0x02, 0x03, and 0x04 are supported.
-//
-// Any scripts which are not recognized as one of the aforementioned standard
-// scripts are encoded using the general serialized format and encode the script
-// size as the sum of the actual size of the script and the number of special
-// cases.
-// -----------------------------------------------------------------------------
-
-// The following constants specify the special constants used to identify a
-// special script type in the domain-specific compressed script encoding.
-//
-// NOTE: This section specifically does not use iota since these values are
-// serialized and must be stable for long-term storage.
-const (
-	// cstPayToPubKeyHash identifies a compressed pay-to-pubkey-hash script.
-	cstPayToPubKeyHash = 0
-
-	// cstPayToScriptHash identifies a compressed pay-to-script-hash script.
-	cstPayToScriptHash = 1
-
-	// cstPayToPubKeyComp2 identifies a compressed pay-to-pubkey script to
-	// a compressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyComp2 = 2
-
-	// cstPayToPubKeyComp3 identifies a compressed pay-to-pubkey script to
-	// a compressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyComp3 = 3
-
-	// cstPayToPubKeyUncomp4 identifies a compressed pay-to-pubkey script to
-	// an uncompressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyUncomp4 = 4
-
-	// cstPayToPubKeyUncomp5 identifies a compressed pay-to-pubkey script to
-	// an uncompressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyUncomp5 = 5
-
-	// numSpecialScripts is the number of special scripts recognized by the
-	// domain-specific script compression algorithm.
-	numSpecialScripts = 6
-)
-
-// isPubKeyHash returns whether or not the passed public key script is a
-// standard pay-to-pubkey-hash script along with the pubkey hash it is paying to
-// if it is.
-func isPubKeyHash(script []byte) (bool, []byte) {
-	if len(script) == 25 && script[0] == txscript.OpDup &&
-		script[1] == txscript.OpHash160 &&
-		script[2] == txscript.OpData20 &&
-		script[23] == txscript.OpEqualVerify &&
-		script[24] == txscript.OpCheckSig {
-
-		return true, script[3:23]
-	}
-
-	return false, nil
-}
-
-// isScriptHash returns whether or not the passed public key script is a
-// standard pay-to-script-hash script along with the script hash it is paying to
-// if it is.
-func isScriptHash(script []byte) (bool, []byte) {
-	if len(script) == 23 && script[0] == txscript.OpHash160 &&
-		script[1] == txscript.OpData20 &&
-		script[22] == txscript.OpEqual {
-
-		return true, script[2:22]
-	}
-
-	return false, nil
-}
-
-// isPubKey returns whether or not the passed public key script is a standard
-// pay-to-pubkey script that pays to a valid compressed or uncompressed public
-// key along with the serialized pubkey it is paying to if it is.
-//
-// NOTE: This function ensures the public key is actually valid since the
-// compression algorithm requires valid pubkeys. It does not support hybrid
-// pubkeys. This means that even if the script has the correct form for a
-// pay-to-pubkey script, this function will only return true when it is paying
-// to a valid compressed or uncompressed pubkey.
-func isPubKey(script []byte) (bool, []byte) {
-	// Pay-to-compressed-pubkey script.
-	if len(script) == 35 && script[0] == txscript.OpData33 &&
-		script[34] == txscript.OpCheckSig && (script[1] == 0x02 ||
-		script[1] == 0x03) {
-
-		// Ensure the public key is valid.
-		serializedPubKey := script[1:34]
-		_, err := ecc.ParsePubKey(serializedPubKey, ecc.S256())
-		if err == nil {
-			return true, serializedPubKey
-		}
-	}
-
-	// Pay-to-uncompressed-pubkey script.
-	if len(script) == 67 && script[0] == txscript.OpData65 &&
-		script[66] == txscript.OpCheckSig && script[1] == 0x04 {
-
-		// Ensure the public key is valid.
-		serializedPubKey := script[1:66]
-		_, err := ecc.ParsePubKey(serializedPubKey, ecc.S256())
-		if err == nil {
-			return true, serializedPubKey
-		}
-	}
-
-	return false, nil
-}
-
-// compressedScriptSize returns the number of bytes the passed script would take
-// when encoded with the domain specific compression algorithm described above.
-func compressedScriptSize(scriptPubKey []byte) int {
-	// Pay-to-pubkey-hash script.
-	if valid, _ := isPubKeyHash(scriptPubKey); valid {
-		return 21
-	}
-
-	// Pay-to-script-hash script.
-	if valid, _ := isScriptHash(scriptPubKey); valid {
-		return 21
-	}
-
-	// Pay-to-pubkey (compressed or uncompressed) script.
-	if valid, _ := isPubKey(scriptPubKey); valid {
-		return 33
-	}
-
-	// When none of the above special cases apply, encode the script as is
-	// preceded by the sum of its size and the number of special cases
-	// encoded as a variable length quantity.
-	return serializeSizeVLQ(uint64(len(scriptPubKey)+numSpecialScripts)) +
-		len(scriptPubKey)
-}
-
-// decodeCompressedScriptSize treats the passed serialized bytes as a compressed
-// script, possibly followed by other data, and returns the number of bytes it
-// occupies taking into account the special encoding of the script size by the
-// domain specific compression algorithm described above.
-func decodeCompressedScriptSize(serialized []byte) int {
-	scriptSize, bytesRead := deserializeVLQ(serialized)
-	if bytesRead == 0 {
-		return 0
-	}
-
-	switch scriptSize {
-	case cstPayToPubKeyHash:
-		return 21
-
-	case cstPayToScriptHash:
-		return 21
-
-	case cstPayToPubKeyComp2, cstPayToPubKeyComp3, cstPayToPubKeyUncomp4,
-		cstPayToPubKeyUncomp5:
-		return 33
-	}
-
-	scriptSize -= numSpecialScripts
-	scriptSize += uint64(bytesRead)
-	return int(scriptSize)
-}
-
-// putCompressedScript compresses the passed script according to the domain
-// specific compression algorithm described above directly into the passed
-// target byte slice. The target byte slice must be at least large enough to
-// handle the number of bytes returned by the compressedScriptSize function or
-// it will panic.
-func putCompressedScript(target, scriptPubKey []byte) int {
-	// Pay-to-pubkey-hash script.
-	if valid, hash := isPubKeyHash(scriptPubKey); valid {
-		target[0] = cstPayToPubKeyHash
-		copy(target[1:21], hash)
-		return 21
-	}
-
-	// Pay-to-script-hash script.
-	if valid, hash := isScriptHash(scriptPubKey); valid {
-		target[0] = cstPayToScriptHash
-		copy(target[1:21], hash)
-		return 21
-	}
-
-	// Pay-to-pubkey (compressed or uncompressed) script.
-	if valid, serializedPubKey := isPubKey(scriptPubKey); valid {
-		pubKeyFormat := serializedPubKey[0]
-		switch pubKeyFormat {
-		case 0x02, 0x03:
-			target[0] = pubKeyFormat
-			copy(target[1:33], serializedPubKey[1:33])
-			return 33
-		case 0x04:
-			// Encode the oddness of the serialized pubkey into the
-			// compressed script type.
-			target[0] = pubKeyFormat | (serializedPubKey[64] & 0x01)
-			copy(target[1:33], serializedPubKey[1:33])
-			return 33
-		}
-	}
-
-	// When none of the above special cases apply, encode the unmodified
-	// script preceded by the sum of its size and the number of special
-	// cases encoded as a variable length quantity.
-	encodedSize := uint64(len(scriptPubKey) + numSpecialScripts)
-	vlqSizeLen := putVLQ(target, encodedSize)
-	copy(target[vlqSizeLen:], scriptPubKey)
-	return vlqSizeLen + len(scriptPubKey)
-}
-
-// decompressScript returns the original script obtained by decompressing the
-// passed compressed script according to the domain specific compression
-// algorithm described above.
-//
-// NOTE: The script parameter must already have been proven to be long enough
-// to contain the number of bytes returned by decodeCompressedScriptSize or it
-// will panic. This is acceptable since it is only an internal function.
-func decompressScript(compressedScriptPubKey []byte) []byte {
-	// In practice this function will not be called with a zero-length or
-	// nil script since the nil script encoding includes the length, however
-	// the code below assumes the length exists, so just return nil now if
-	// the function ever ends up being called with a nil script in the
-	// future.
-	if len(compressedScriptPubKey) == 0 {
-		return nil
-	}
-
-	// Decode the script size and examine it for the special cases.
-	encodedScriptSize, bytesRead := deserializeVLQ(compressedScriptPubKey)
-	switch encodedScriptSize {
-	// Pay-to-pubkey-hash script. The resulting script is:
-	// <OP_DUP><OP_HASH160><20 byte hash><OP_EQUALVERIFY><OP_CHECKSIG>
-	case cstPayToPubKeyHash:
-		scriptPubKey := make([]byte, 25)
-		scriptPubKey[0] = txscript.OpDup
-		scriptPubKey[1] = txscript.OpHash160
-		scriptPubKey[2] = txscript.OpData20
-		copy(scriptPubKey[3:], compressedScriptPubKey[bytesRead:bytesRead+20])
-		scriptPubKey[23] = txscript.OpEqualVerify
-		scriptPubKey[24] = txscript.OpCheckSig
-		return scriptPubKey
-
-	// Pay-to-script-hash script. The resulting script is:
-	// <OP_HASH160><20 byte script hash><OP_EQUAL>
-	case cstPayToScriptHash:
-		scriptPubKey := make([]byte, 23)
-		scriptPubKey[0] = txscript.OpHash160
-		scriptPubKey[1] = txscript.OpData20
-		copy(scriptPubKey[2:], compressedScriptPubKey[bytesRead:bytesRead+20])
-		scriptPubKey[22] = txscript.OpEqual
-		return scriptPubKey
-
-	// Pay-to-compressed-pubkey script. The resulting script is:
-	// <OP_DATA_33><33 byte compressed pubkey><OP_CHECKSIG>
-	case cstPayToPubKeyComp2, cstPayToPubKeyComp3:
-		scriptPubKey := make([]byte, 35)
-		scriptPubKey[0] = txscript.OpData33
-		scriptPubKey[1] = byte(encodedScriptSize)
-		copy(scriptPubKey[2:], compressedScriptPubKey[bytesRead:bytesRead+32])
-		scriptPubKey[34] = txscript.OpCheckSig
-		return scriptPubKey
-
-	// Pay-to-uncompressed-pubkey script. The resulting script is:
-	// <OP_DATA_65><65 byte uncompressed pubkey><OP_CHECKSIG>
-	case cstPayToPubKeyUncomp4, cstPayToPubKeyUncomp5:
-		// Change the leading byte to the appropriate compressed pubkey
-		// identifier (0x02 or 0x03) so it can be decoded as a
-		// compressed pubkey. This really should never fail since the
-		// encoding ensures it is valid before compressing to this type.
-		compressedKey := make([]byte, 33)
-		compressedKey[0] = byte(encodedScriptSize - 2)
-		copy(compressedKey[1:], compressedScriptPubKey[1:])
-		key, err := ecc.ParsePubKey(compressedKey, ecc.S256())
-		if err != nil {
-			return nil
-		}
-
-		scriptPubKey := make([]byte, 67)
-		scriptPubKey[0] = txscript.OpData65
-		copy(scriptPubKey[1:], key.SerializeUncompressed())
-		scriptPubKey[66] = txscript.OpCheckSig
-		return scriptPubKey
-	}
-
-	// When none of the special cases apply, the script was encoded using
-	// the general format, so reduce the script size by the number of
-	// special cases and return the unmodified script.
-	scriptSize := int(encodedScriptSize - numSpecialScripts)
-	scriptPubKey := make([]byte, scriptSize)
-	copy(scriptPubKey, compressedScriptPubKey[bytesRead:bytesRead+scriptSize])
-	return scriptPubKey
-}
-
-// -----------------------------------------------------------------------------
-// In order to reduce the size of stored amounts, a domain specific compression
-// algorithm is used which relies on there typically being a lot of zeroes at
-// end of the amounts.
-//
-// While this is simply exchanging one uint64 for another, the resulting value
-// for typical amounts has a much smaller magnitude which results in fewer bytes
-// when encoded as variable length quantity. For example, consider the amount
-// of 0.1 KAS which is 10000000 sompi. Encoding 10000000 as a VLQ would take
-// 4 bytes while encoding the compressed value of 8 as a VLQ only takes 1 byte.
-//
-// Essentially the compression is achieved by splitting the value into an
-// exponent in the range [0-9] and a digit in the range [1-9], when possible,
-// and encoding them in a way that can be decoded. More specifically, the
-// encoding is as follows:
-// - 0 is 0
-// - Find the exponent, e, as the largest power of 10 that evenly divides the
-//   value up to a maximum of 9
-// - When e < 9, the final digit can't be 0 so store it as d and remove it by
-//   dividing the value by 10 (call the result n). The encoded value is thus:
-//   1 + 10*(9*n + d-1) + e
-// - When e==9, the only thing known is the amount is not 0. The encoded value
-//   is thus:
-//   1 + 10*(n-1) + e   ==   10 + 10*(n-1)
-//
-// Example encodings:
-// (The numbers in parenthesis are the number of bytes when serialized as a VLQ)
-//            0 (1) -> 0        (1)           *  0.00000000 KAS
-//         1000 (2) -> 4        (1)           *  0.00001000 KAS
-//        10000 (2) -> 5        (1)           *  0.00010000 KAS
-//     12345678 (4) -> 111111101(4)           *  0.12345678 KAS
-//     50000000 (4) -> 47       (1)           *  0.50000000 KAS
-//    100000000 (4) -> 9        (1)           *  1.00000000 KAS
-//    500000000 (5) -> 49       (1)           *  5.00000000 KAS
-//   1000000000 (5) -> 10       (1)           * 10.00000000 KAS
-// -----------------------------------------------------------------------------
-
-// compressTxOutAmount compresses the passed amount according to the domain
-// specific compression algorithm described above.
-func compressTxOutAmount(amount uint64) uint64 {
-	// No need to do any work if it's zero.
-	if amount == 0 {
-		return 0
-	}
-
-	// Find the largest power of 10 (max of 9) that evenly divides the
-	// value.
-	exponent := uint64(0)
-	for amount%10 == 0 && exponent < 9 {
-		amount /= 10
-		exponent++
-	}
-
-	// The compressed result for exponents less than 9 is:
-	// 1 + 10*(9*n + d-1) + e
-	if exponent < 9 {
-		lastDigit := amount % 10
-		amount /= 10
-		return 1 + 10*(9*amount+lastDigit-1) + exponent
-	}
-
-	// The compressed result for an exponent of 9 is:
-	// 1 + 10*(n-1) + e   ==   10 + 10*(n-1)
-	return 10 + 10*(amount-1)
-}
-
-// decompressTxOutAmount returns the original amount the passed compressed
-// amount represents according to the domain specific compression algorithm
-// described above.
-func decompressTxOutAmount(amount uint64) uint64 {
-	// No need to do any work if it's zero.
-	if amount == 0 {
-		return 0
-	}
-
-	// The decompressed amount is either of the following two equations:
-	// x = 1 + 10*(9*n + d - 1) + e
-	// x = 1 + 10*(n - 1)       + 9
-	amount--
-
-	// The decompressed amount is now one of the following two equations:
-	// x = 10*(9*n + d - 1) + e
-	// x = 10*(n - 1)       + 9
-	exponent := amount % 10
-	amount /= 10
-
-	// The decompressed amount is now one of the following two equations:
-	// x = 9*n + d - 1  | where e < 9
-	// x = n - 1        | where e = 9
-	n := uint64(0)
-	if exponent < 9 {
-		lastDigit := amount%9 + 1
-		amount /= 9
-		n = amount*10 + lastDigit
-	} else {
-		n = amount + 1
-	}
-
-	// Apply the exponent.
-	for ; exponent > 0; exponent-- {
-		n *= 10
-	}
-
-	return n
-}
-
-// -----------------------------------------------------------------------------
-// Compressed transaction outputs consist of an amount and a public key script
-// both compressed using the domain specific compression algorithms previously
-// described.
-//
-// The serialized format is:
-//
-//   <compressed amount><compressed script>
-//
-//   Field                 Type     Size
-//     compressed amount   VLQ      variable
-//     compressed script   []byte   variable
-// -----------------------------------------------------------------------------
-
-// compressedTxOutSize returns the number of bytes the passed transaction output
-// fields would take when encoded with the format described above.
-func compressedTxOutSize(amount uint64, scriptPubKey []byte) int {
-	return serializeSizeVLQ(compressTxOutAmount(amount)) +
-		compressedScriptSize(scriptPubKey)
-}
-
-// putCompressedTxOut compresses the passed amount and script according to their
-// domain specific compression algorithms and encodes them directly into the
-// passed target byte slice with the format described above. The target byte
-// slice must be at least large enough to handle the number of bytes returned by
-// the compressedTxOutSize function or it will panic.
-func putCompressedTxOut(target []byte, amount uint64, scriptPubKey []byte) int {
-	offset := putVLQ(target, compressTxOutAmount(amount))
-	offset += putCompressedScript(target[offset:], scriptPubKey)
-	return offset
-}
-
-// decodeCompressedTxOut decodes the passed compressed txout, possibly followed
-// by other data, into its uncompressed amount and script and returns them along
-// with the number of bytes they occupied prior to decompression.
-func decodeCompressedTxOut(serialized []byte) (uint64, []byte, int, error) {
-	// Deserialize the compressed amount and ensure there are bytes
-	// remaining for the compressed script.
-	compressedAmount, bytesRead := deserializeVLQ(serialized)
-	if bytesRead >= len(serialized) {
-		return 0, nil, bytesRead, errDeserialize("unexpected end of " +
-			"data after compressed amount")
-	}
-
-	// Decode the compressed script size and ensure there are enough bytes
-	// left in the slice for it.
-	scriptSize := decodeCompressedScriptSize(serialized[bytesRead:])
-	if len(serialized[bytesRead:]) < scriptSize {
-		return 0, nil, bytesRead, errDeserialize("unexpected end of " +
-			"data after script size")
-	}
-
-	// Decompress and return the amount and script.
-	amount := decompressTxOutAmount(compressedAmount)
-	script := decompressScript(serialized[bytesRead : bytesRead+scriptSize])
-	return amount, script, bytesRead + scriptSize, nil
-}

blockdag/compress_test.go

@@ -1,436 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"bytes"
-	"encoding/hex"
-	"testing"
-)
-
-// hexToBytes converts the passed hex string into bytes and will panic if there
-// is an error. This is only provided for the hard-coded constants so errors in
-// the source code can be detected. It will only (and must only) be called with
-// hard-coded values.
-func hexToBytes(s string) []byte {
-	b, err := hex.DecodeString(s)
-	if err != nil {
-		panic("invalid hex in source file: " + s)
-	}
-	return b
-}
-
-// TestVLQ ensures the variable length quantity serialization, deserialization,
-// and size calculation works as expected.
-func TestVLQ(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		val        uint64
-		serialized []byte
-	}{
-		{0, hexToBytes("00")},
-		{1, hexToBytes("01")},
-		{127, hexToBytes("7f")},
-		{128, hexToBytes("8000")},
-		{129, hexToBytes("8001")},
-		{255, hexToBytes("807f")},
-		{256, hexToBytes("8100")},
-		{16383, hexToBytes("fe7f")},
-		{16384, hexToBytes("ff00")},
-		{16511, hexToBytes("ff7f")}, // Max 2-byte value
-		{16512, hexToBytes("808000")},
-		{16513, hexToBytes("808001")},
-		{16639, hexToBytes("80807f")},
-		{32895, hexToBytes("80ff7f")},
-		{2113663, hexToBytes("ffff7f")}, // Max 3-byte value
-		{2113664, hexToBytes("80808000")},
-		{270549119, hexToBytes("ffffff7f")}, // Max 4-byte value
-		{270549120, hexToBytes("8080808000")},
-		{2147483647, hexToBytes("86fefefe7f")},
-		{2147483648, hexToBytes("86fefeff00")},
-		{4294967295, hexToBytes("8efefefe7f")}, // Max uint32, 5 bytes
-		// Max uint64, 10 bytes
-		{18446744073709551615, hexToBytes("80fefefefefefefefe7f")},
-	}
-
-	for _, test := range tests {
-		// Ensure the function to calculate the serialized size without
-		// actually serializing the value is calculated properly.
-		gotSize := serializeSizeVLQ(test.val)
-		if gotSize != len(test.serialized) {
-			t.Errorf("serializeSizeVLQ: did not get expected size "+
-				"for %d - got %d, want %d", test.val, gotSize,
-				len(test.serialized))
-			continue
-		}
-
-		// Ensure the value serializes to the expected bytes.
-		gotBytes := make([]byte, gotSize)
-		gotBytesWritten := putVLQ(gotBytes, test.val)
-		if !bytes.Equal(gotBytes, test.serialized) {
-			t.Errorf("putVLQUnchecked: did not get expected bytes "+
-				"for %d - got %x, want %x", test.val, gotBytes,
-				test.serialized)
-			continue
-		}
-		if gotBytesWritten != len(test.serialized) {
-			t.Errorf("putVLQUnchecked: did not get expected number "+
-				"of bytes written for %d - got %d, want %d",
-				test.val, gotBytesWritten, len(test.serialized))
-			continue
-		}
-
-		// Ensure the serialized bytes deserialize to the expected
-		// value.
-		gotVal, gotBytesRead := deserializeVLQ(test.serialized)
-		if gotVal != test.val {
-			t.Errorf("deserializeVLQ: did not get expected value "+
-				"for %x - got %d, want %d", test.serialized,
-				gotVal, test.val)
-			continue
-		}
-		if gotBytesRead != len(test.serialized) {
-			t.Errorf("deserializeVLQ: did not get expected number "+
-				"of bytes read for %d - got %d, want %d",
-				test.serialized, gotBytesRead,
-				len(test.serialized))
-			continue
-		}
-	}
-}
-
-// TestScriptCompression ensures the domain-specific script compression and
-// decompression works as expected.
-func TestScriptCompression(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name         string
-		uncompressed []byte
-		compressed   []byte
-	}{
-		{
-			name:         "nil",
-			uncompressed: nil,
-			compressed:   hexToBytes("06"),
-		},
-		{
-			name:         "pay-to-pubkey-hash 1",
-			uncompressed: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
-			compressed:   hexToBytes("001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
-		},
-		{
-			name:         "pay-to-pubkey-hash 2",
-			uncompressed: hexToBytes("76a914e34cce70c86373273efcc54ce7d2a491bb4a0e8488ac"),
-			compressed:   hexToBytes("00e34cce70c86373273efcc54ce7d2a491bb4a0e84"),
-		},
-		{
-			name:         "pay-to-script-hash 1",
-			uncompressed: hexToBytes("a914da1745e9b549bd0bfa1a569971c77eba30cd5a4b87"),
-			compressed:   hexToBytes("01da1745e9b549bd0bfa1a569971c77eba30cd5a4b"),
-		},
-		{
-			name:         "pay-to-script-hash 2",
-			uncompressed: hexToBytes("a914f815b036d9bbbce5e9f2a00abd1bf3dc91e9551087"),
-			compressed:   hexToBytes("01f815b036d9bbbce5e9f2a00abd1bf3dc91e95510"),
-		},
-		{
-			name:         "pay-to-pubkey compressed 0x02",
-			uncompressed: hexToBytes("2102192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4ac"),
-			compressed:   hexToBytes("02192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
-		},
-		{
-			name:         "pay-to-pubkey compressed 0x03",
-			uncompressed: hexToBytes("2103b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e65ac"),
-			compressed:   hexToBytes("03b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e65"),
-		},
-		{
-			name:         "pay-to-pubkey uncompressed 0x04 even",
-			uncompressed: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
-			compressed:   hexToBytes("04192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
-		},
-		{
-			name:         "pay-to-pubkey uncompressed 0x04 odd",
-			uncompressed: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"),
-			compressed:   hexToBytes("0511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
-		},
-		{
-			name:         "pay-to-pubkey invalid pubkey",
-			uncompressed: hexToBytes("3302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
-			compressed:   hexToBytes("293302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
-		},
-		{
-			name:         "requires 2 size bytes - data push 200 bytes",
-			uncompressed: append(hexToBytes("4cc8"), bytes.Repeat([]byte{0x00}, 200)...),
-			// [0x80, 0x50] = 208 as a variable length quantity
-			// [0x4c, 0xc8] = OP_PUSHDATA1 200
-			compressed: append(hexToBytes("80504cc8"), bytes.Repeat([]byte{0x00}, 200)...),
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the function to calculate the serialized size without
-		// actually serializing the value is calculated properly.
-		gotSize := compressedScriptSize(test.uncompressed)
-		if gotSize != len(test.compressed) {
-			t.Errorf("compressedScriptSize (%s): did not get "+
-				"expected size - got %d, want %d", test.name,
-				gotSize, len(test.compressed))
-			continue
-		}
-
-		// Ensure the script compresses to the expected bytes.
-		gotCompressed := make([]byte, gotSize)
-		gotBytesWritten := putCompressedScript(gotCompressed,
-			test.uncompressed)
-		if !bytes.Equal(gotCompressed, test.compressed) {
-			t.Errorf("putCompressedScript (%s): did not get "+
-				"expected bytes - got %x, want %x", test.name,
-				gotCompressed, test.compressed)
-			continue
-		}
-		if gotBytesWritten != len(test.compressed) {
-			t.Errorf("putCompressedScript (%s): did not get "+
-				"expected number of bytes written - got %d, "+
-				"want %d", test.name, gotBytesWritten,
-				len(test.compressed))
-			continue
-		}
-
-		// Ensure the compressed script size is properly decoded from
-		// the compressed script.
-		gotDecodedSize := decodeCompressedScriptSize(test.compressed)
-		if gotDecodedSize != len(test.compressed) {
-			t.Errorf("decodeCompressedScriptSize (%s): did not get "+
-				"expected size - got %d, want %d", test.name,
-				gotDecodedSize, len(test.compressed))
-			continue
-		}
-
-		// Ensure the script decompresses to the expected bytes.
-		gotDecompressed := decompressScript(test.compressed)
-		if !bytes.Equal(gotDecompressed, test.uncompressed) {
-			t.Errorf("decompressScript (%s): did not get expected "+
-				"bytes - got %x, want %x", test.name,
-				gotDecompressed, test.uncompressed)
-			continue
-		}
-	}
-}
-
-// TestScriptCompressionErrors ensures calling various functions related to
-// script compression with incorrect data returns the expected results.
-func TestScriptCompressionErrors(t *testing.T) {
-	t.Parallel()
-
-	// A nil script must result in a decoded size of 0.
-	if gotSize := decodeCompressedScriptSize(nil); gotSize != 0 {
-		t.Fatalf("decodeCompressedScriptSize with nil script did not "+
-			"return 0 - got %d", gotSize)
-	}
-
-	// A nil script must result in a nil decompressed script.
-	if gotScript := decompressScript(nil); gotScript != nil {
-		t.Fatalf("decompressScript with nil script did not return nil "+
-			"decompressed script - got %x", gotScript)
-	}
-
-	// A compressed script for a pay-to-pubkey (uncompressed) that results
-	// in an invalid pubkey must result in a nil decompressed script.
-	compressedScript := hexToBytes("04012d74d0cb94344c9569c2e77901573d8d" +
-		"7903c3ebec3a957724895dca52c6b4")
-	if gotScript := decompressScript(compressedScript); gotScript != nil {
-		t.Fatalf("decompressScript with compressed pay-to-"+
-			"uncompressed-pubkey that is invalid did not return "+
-			"nil decompressed script - got %x", gotScript)
-	}
-}
-
-// TestAmountCompression ensures the domain-specific transaction output amount
-// compression and decompression works as expected.
-func TestAmountCompression(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name         string
-		uncompressed uint64
-		compressed   uint64
-	}{
-		{
-			name:         "0 KAS",
-			uncompressed: 0,
-			compressed:   0,
-		},
-		{
-			name:         "546 Sompi (current network dust value)",
-			uncompressed: 546,
-			compressed:   4911,
-		},
-		{
-			name:         "0.00001 KAS (typical transaction fee)",
-			uncompressed: 1000,
-			compressed:   4,
-		},
-		{
-			name:         "0.0001 KAS (typical transaction fee)",
-			uncompressed: 10000,
-			compressed:   5,
-		},
-		{
-			name:         "0.12345678 KAS",
-			uncompressed: 12345678,
-			compressed:   111111101,
-		},
-		{
-			name:         "0.5 KAS",
-			uncompressed: 50000000,
-			compressed:   48,
-		},
-		{
-			name:         "1 KAS",
-			uncompressed: 100000000,
-			compressed:   9,
-		},
-		{
-			name:         "5 KAS",
-			uncompressed: 500000000,
-			compressed:   49,
-		},
-		{
-			name:         "21000000 KAS (max minted coins)",
-			uncompressed: 2100000000000000,
-			compressed:   21000000,
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the amount compresses to the expected value.
-		gotCompressed := compressTxOutAmount(test.uncompressed)
-		if gotCompressed != test.compressed {
-			t.Errorf("compressTxOutAmount (%s): did not get "+
-				"expected value - got %d, want %d", test.name,
-				gotCompressed, test.compressed)
-			continue
-		}
-
-		// Ensure the value decompresses to the expected value.
-		gotDecompressed := decompressTxOutAmount(test.compressed)
-		if gotDecompressed != test.uncompressed {
-			t.Errorf("decompressTxOutAmount (%s): did not get "+
-				"expected value - got %d, want %d", test.name,
-				gotDecompressed, test.uncompressed)
-			continue
-		}
-	}
-}
-
-// TestCompressedTxOut ensures the transaction output serialization and
-// deserialization works as expected.
-func TestCompressedTxOut(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name         string
-		amount       uint64
-		scriptPubKey []byte
-		compressed   []byte
-	}{
-		{
-			name:         "pay-to-pubkey-hash dust",
-			amount:       546,
-			scriptPubKey: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
-			compressed:   hexToBytes("a52f001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
-		},
-		{
-			name:         "pay-to-pubkey uncompressed 1 KAS",
-			amount:       100000000,
-			scriptPubKey: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
-			compressed:   hexToBytes("0904192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the function to calculate the serialized size without
-		// actually serializing the txout is calculated properly.
-		gotSize := compressedTxOutSize(test.amount, test.scriptPubKey)
-		if gotSize != len(test.compressed) {
-			t.Errorf("compressedTxOutSize (%s): did not get "+
-				"expected size - got %d, want %d", test.name,
-				gotSize, len(test.compressed))
-			continue
-		}
-
-		// Ensure the txout compresses to the expected value.
-		gotCompressed := make([]byte, gotSize)
-		gotBytesWritten := putCompressedTxOut(gotCompressed,
-			test.amount, test.scriptPubKey)
-		if !bytes.Equal(gotCompressed, test.compressed) {
-			t.Errorf("compressTxOut (%s): did not get expected "+
-				"bytes - got %x, want %x", test.name,
-				gotCompressed, test.compressed)
-			continue
-		}
-		if gotBytesWritten != len(test.compressed) {
-			t.Errorf("compressTxOut (%s): did not get expected "+
-				"number of bytes written - got %d, want %d",
-				test.name, gotBytesWritten,
-				len(test.compressed))
-			continue
-		}
-
-		// Ensure the serialized bytes are decoded back to the expected
-		// uncompressed values.
-		gotAmount, gotScript, gotBytesRead, err := decodeCompressedTxOut(
-			test.compressed)
-		if err != nil {
-			t.Errorf("decodeCompressedTxOut (%s): unexpected "+
-				"error: %v", test.name, err)
-			continue
-		}
-		if gotAmount != test.amount {
-			t.Errorf("decodeCompressedTxOut (%s): did not get "+
-				"expected amount - got %d, want %d",
-				test.name, gotAmount, test.amount)
-			continue
-		}
-		if !bytes.Equal(gotScript, test.scriptPubKey) {
-			t.Errorf("decodeCompressedTxOut (%s): did not get "+
-				"expected script - got %x, want %x",
-				test.name, gotScript, test.scriptPubKey)
-			continue
-		}
-		if gotBytesRead != len(test.compressed) {
-			t.Errorf("decodeCompressedTxOut (%s): did not get "+
-				"expected number of bytes read - got %d, want %d",
-				test.name, gotBytesRead, len(test.compressed))
-			continue
-		}
-	}
-}
-
-// TestTxOutCompressionErrors ensures calling various functions related to
-// txout compression with incorrect data returns the expected results.
-func TestTxOutCompressionErrors(t *testing.T) {
-	t.Parallel()
-
-	// A compressed txout with missing compressed script must error.
-	compressedTxOut := hexToBytes("00")
-	_, _, _, err := decodeCompressedTxOut(compressedTxOut)
-	if !isDeserializeErr(err) {
-		t.Fatalf("decodeCompressedTxOut with missing compressed script "+
-			"did not return expected error type - got %T, want "+
-			"errDeserialize", err)
-	}
-
-	// A compressed txout with short compressed script must error.
-	compressedTxOut = hexToBytes("0010")
-	_, _, _, err = decodeCompressedTxOut(compressedTxOut)
-	if !isDeserializeErr(err) {
-		t.Fatalf("decodeCompressedTxOut with short compressed script "+
-			"did not return expected error type - got %T, want "+
-			"errDeserialize", err)
-	}
-}

blockdag/dagio_test.go

@@ -6,39 +6,51 @@ package blockdag
 
 import (
 	"bytes"
+	"encoding/hex"
 	"github.com/pkg/errors"
 	"reflect"
 	"testing"
 
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
 
-// TestErrNotInDAG ensures the functions related to errNotInDAG work
+// TestErrNotInDAG ensures the functions related to ErrNotInDAG work
 // as expected.
 func TestErrNotInDAG(t *testing.T) {
 	errStr := "no block at height 1 exists"
-	err := error(errNotInDAG(errStr))
+	err := error(ErrNotInDAG(errStr))
 
 	// Ensure the stringized output for the error is as expected.
 	if err.Error() != errStr {
-		t.Fatalf("errNotInDAG retuned unexpected error string - "+
+		t.Fatalf("ErrNotInDAG retuned unexpected error string - "+
 			"got %q, want %q", err.Error(), errStr)
 	}
 
 	// Ensure error is detected as the correct type.
-	if !isNotInDAGErr(err) {
-		t.Fatalf("isNotInDAGErr did not detect as expected type")
+	if !IsNotInDAGErr(err) {
+		t.Fatalf("IsNotInDAGErr did not detect as expected type")
 	}
 	err = errors.New("something else")
-	if isNotInDAGErr(err) {
-		t.Fatalf("isNotInDAGErr detected incorrect type")
+	if IsNotInDAGErr(err) {
+		t.Fatalf("IsNotInDAGErr detected incorrect type")
 	}
 }
 
-// TestUtxoSerialization ensures serializing and deserializing unspent
+// hexToBytes converts the passed hex string into bytes and will panic if there
+// is an error. This is only provided for the hard-coded constants so errors in
+// the source code can be detected. It will only (and must only) be called with
+// hard-coded values.
+func hexToBytes(s string) []byte {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		panic("invalid hex in source file: " + s)
+	}
+	return b
+}
+
+// TestUTXOSerialization ensures serializing and deserializing unspent
 // trasaction output entries works as expected.
-func TestUtxoSerialization(t *testing.T) {
+func TestUTXOSerialization(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
@@ -54,7 +66,7 @@ func TestUtxoSerialization(t *testing.T) {
 				blockBlueScore: 1,
 				packedFlags:    tfCoinbase,
 			},
-			serialized: hexToBytes("03320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"),
+			serialized: hexToBytes("01000000000000000100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
 		},
 		{
 			name: "blue score 100001, not coinbase",
@@ -64,13 +76,21 @@ func TestUtxoSerialization(t *testing.T) {
 				blockBlueScore: 100001,
 				packedFlags:    0,
 			},
-			serialized: hexToBytes("8b99420700ee8bd501094a7d5ca318da2506de35e1cb025ddc"),
+			serialized: hexToBytes("a1860100000000000040420f00000000001976a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
 		},
 	}
 
 	for i, test := range tests {
 		// Ensure the utxo entry serializes to the expected value.
-		gotBytes := serializeUTXOEntry(test.entry)
+		w := &bytes.Buffer{}
+		err := serializeUTXOEntry(w, test.entry)
+		if err != nil {
+			t.Errorf("serializeUTXOEntry #%d (%s) unexpected "+
+				"error: %v", i, test.name, err)
+			continue
+		}
+
+		gotBytes := w.Bytes()
 		if !bytes.Equal(gotBytes, test.serialized) {
 			t.Errorf("serializeUTXOEntry #%d (%s): mismatched "+
 				"bytes - got %x, want %x", i, test.name,
@@ -78,8 +98,8 @@ func TestUtxoSerialization(t *testing.T) {
 			continue
 		}
 
-		// Deserialize to a utxo entry.
-		utxoEntry, err := deserializeUTXOEntry(test.serialized)
+		// Deserialize to a utxo entry.gotBytes
+		utxoEntry, err := deserializeUTXOEntry(bytes.NewReader(test.serialized))
 		if err != nil {
 			t.Errorf("deserializeUTXOEntry #%d (%s) unexpected "+
 				"error: %v", i, test.name, err)
@@ -124,28 +144,24 @@ func TestUtxoEntryDeserializeErrors(t *testing.T) {
 	tests := []struct {
 		name       string
 		serialized []byte
-		errType    error
 	}{
 		{
 			name:       "no data after header code",
 			serialized: hexToBytes("02"),
-			errType:    errDeserialize(""),
 		},
 		{
 			name:       "incomplete compressed txout",
 			serialized: hexToBytes("0232"),
-			errType:    errDeserialize(""),
 		},
 	}
 
 	for _, test := range tests {
 		// Ensure the expected error type is returned and the returned
 		// entry is nil.
-		entry, err := deserializeUTXOEntry(test.serialized)
-		if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
-			t.Errorf("deserializeUTXOEntry (%s): expected error "+
-				"type does not match - got %T, want %T",
-				test.name, err, test.errType)
+		entry, err := deserializeUTXOEntry(bytes.NewReader(test.serialized))
+		if err == nil {
+			t.Errorf("deserializeUTXOEntry (%s): didn't return an error",
+				test.name)
 			continue
 		}
 		if entry != nil {
@@ -172,7 +188,7 @@ func TestDAGStateSerialization(t *testing.T) {
 				TipHashes:         []*daghash.Hash{newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f")},
 				LastFinalityPoint: newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"),
 			},
-			serialized: []byte("{\"TipHashes\":[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]}"),
+			serialized: []byte("{\"TipHashes\":[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0],\"LocalSubnetworkID\":null}"),
 		},
 		{
 			name: "block 1",
@@ -180,7 +196,7 @@ func TestDAGStateSerialization(t *testing.T) {
 				TipHashes:         []*daghash.Hash{newHashFromStr("00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048")},
 				LastFinalityPoint: newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"),
 			},
-			serialized: []byte("{\"TipHashes\":[[72,96,235,24,191,27,22,32,227,126,148,144,252,138,66,117,20,65,111,215,81,89,171,134,104,142,154,131,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]}"),
+			serialized: []byte("{\"TipHashes\":[[72,96,235,24,191,27,22,32,227,126,148,144,252,138,66,117,20,65,111,215,81,89,171,134,104,142,154,131,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0],\"LocalSubnetworkID\":null}"),
 		},
 	}
 
@@ -217,50 +233,6 @@ func TestDAGStateSerialization(t *testing.T) {
 	}
 }
 
-// TestDAGStateDeserializeErrors performs negative tests against
-// deserializing the DAG state to ensure error paths work as expected.
-func TestDAGStateDeserializeErrors(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name       string
-		serialized []byte
-		errType    error
-	}{
-		{
-			name:       "nothing serialized",
-			serialized: hexToBytes(""),
-			errType:    database.Error{ErrorCode: database.ErrCorruption},
-		},
-		{
-			name:       "corrupted data",
-			serialized: []byte("[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,7"),
-			errType:    database.Error{ErrorCode: database.ErrCorruption},
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the expected error type and code is returned.
-		_, err := deserializeDAGState(test.serialized)
-		if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
-			t.Errorf("deserializeDAGState (%s): expected "+
-				"error type does not match - got %T, want %T",
-				test.name, err, test.errType)
-			continue
-		}
-		if derr, ok := err.(database.Error); ok {
-			tderr := test.errType.(database.Error)
-			if derr.ErrorCode != tderr.ErrorCode {
-				t.Errorf("deserializeDAGState (%s): "+
-					"wrong error code got: %v, want: %v",
-					test.name, derr.ErrorCode,
-					tderr.ErrorCode)
-				continue
-			}
-		}
-	}
-}
-
 // newHashFromStr converts the passed big-endian hex string into a
 // daghash.Hash. It only differs from the one available in daghash in that
 // it panics in case of an error since it will only (and must only) be

blockdag/delayedblockheap.go

@@ -0,0 +1,73 @@
+package blockdag
+
+import (
+	"container/heap"
+)
+
+type baseDelayedBlocksHeap []*delayedBlock
+
+func (h baseDelayedBlocksHeap) Len() int {
+	return len(h)
+}
+func (h baseDelayedBlocksHeap) Swap(i, j int) {
+	h[i], h[j] = h[j], h[i]
+}
+
+func (h *baseDelayedBlocksHeap) Push(x interface{}) {
+	*h = append(*h, x.(*delayedBlock))
+}
+
+func (h *baseDelayedBlocksHeap) Pop() interface{} {
+	oldHeap := *h
+	oldLength := len(oldHeap)
+	popped := oldHeap[oldLength-1]
+	*h = oldHeap[0 : oldLength-1]
+	return popped
+}
+
+func (h baseDelayedBlocksHeap) peek() interface{} {
+	if h.Len() > 0 {
+		return h[h.Len()-1]
+	}
+	return nil
+}
+
+func (h baseDelayedBlocksHeap) Less(i, j int) bool {
+	return h[j].processTime.After(h[i].processTime)
+}
+
+type delayedBlocksHeap struct {
+	baseDelayedBlocksHeap *baseDelayedBlocksHeap
+	impl                  heap.Interface
+}
+
+// newDelayedBlocksHeap initializes and returns a new delayedBlocksHeap
+func newDelayedBlocksHeap() delayedBlocksHeap {
+	baseHeap := &baseDelayedBlocksHeap{}
+	h := delayedBlocksHeap{impl: baseHeap, baseDelayedBlocksHeap: baseHeap}
+	heap.Init(h.impl)
+	return h
+}
+
+// pop removes the block with lowest height from this heap and returns it
+func (dbh delayedBlocksHeap) pop() *delayedBlock {
+	return heap.Pop(dbh.impl).(*delayedBlock)
+}
+
+// Push pushes the block onto the heap
+func (dbh delayedBlocksHeap) Push(block *delayedBlock) {
+	heap.Push(dbh.impl, block)
+}
+
+// Len returns the length of this heap
+func (dbh delayedBlocksHeap) Len() int {
+	return dbh.impl.Len()
+}
+
+// peek returns the topmost element in the queue without poping it
+func (dbh delayedBlocksHeap) peek() *delayedBlock {
+	if dbh.baseDelayedBlocksHeap.peek() == nil {
+		return nil
+	}
+	return dbh.baseDelayedBlocksHeap.peek().(*delayedBlock)
+}

blockdag/difficulty.go

@@ -5,15 +5,15 @@
 package blockdag
 
 import (
-	"math/big"
-	"time"
+	"github.com/kaspanet/kaspad/util/bigintpool"
+	"github.com/kaspanet/kaspad/util/mstime"
 
 	"github.com/kaspanet/kaspad/util"
 )
 
 // requiredDifficulty calculates the required difficulty for a
 // block given its bluest parent.
-func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime time.Time) uint32 {
+func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime mstime.Time) uint32 {
 	// Genesis block.
 	if bluestParent == nil || bluestParent.blueScore < dag.difficultyAdjustmentWindowSize+1 {
 		return dag.powMaxBits
@@ -30,12 +30,21 @@ func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime ti
 	// averageWindowTarget * (windowMinTimestamp / (targetTimePerBlock * windowSize))
 	// The result uses integer division which means it will be slightly
 	// rounded down.
-	newTarget := targetsWindow.averageTarget()
+	newTarget := bigintpool.Acquire(0)
+	defer bigintpool.Release(newTarget)
+	windowTimeStampDifference := bigintpool.Acquire(windowMaxTimeStamp - windowMinTimestamp)
+	defer bigintpool.Release(windowTimeStampDifference)
+	targetTimePerBlock := bigintpool.Acquire(dag.Params.TargetTimePerBlock.Milliseconds())
+	defer bigintpool.Release(targetTimePerBlock)
+	difficultyAdjustmentWindowSize := bigintpool.Acquire(int64(dag.difficultyAdjustmentWindowSize))
+	defer bigintpool.Release(difficultyAdjustmentWindowSize)
+
+	targetsWindow.averageTarget(newTarget)
 	newTarget.
-		Mul(newTarget, big.NewInt(windowMaxTimeStamp-windowMinTimestamp)).
-		Div(newTarget, big.NewInt(dag.targetTimePerBlock)).
-		Div(newTarget, big.NewInt(int64(dag.difficultyAdjustmentWindowSize)))
-	if newTarget.Cmp(dag.dagParams.PowMax) > 0 {
+		Mul(newTarget, windowTimeStampDifference).
+		Div(newTarget, targetTimePerBlock).
+		Div(newTarget, difficultyAdjustmentWindowSize)
+	if newTarget.Cmp(dag.Params.PowMax) > 0 {
 		return dag.powMaxBits
 	}
 	newTargetBits := util.BigToCompact(newTarget)
@@ -46,7 +55,7 @@ func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime ti
 // be built on top of the current tips.
 //
 // This function is safe for concurrent access.
-func (dag *BlockDAG) NextRequiredDifficulty(timestamp time.Time) uint32 {
+func (dag *BlockDAG) NextRequiredDifficulty(timestamp mstime.Time) uint32 {
 	difficulty := dag.requiredDifficulty(dag.virtual.parents.bluest(), timestamp)
 	return difficulty
 }

blockdag/difficulty_test.go

@@ -6,8 +6,7 @@ package blockdag
 
 import (
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/kaspanet/kaspad/util/mstime"
 	"math/big"
 	"testing"
 	"time"
@@ -81,65 +80,84 @@ func TestCalcWork(t *testing.T) {
 }
 
 func TestDifficulty(t *testing.T) {
-	params := dagconfig.SimNetParams
+	params := dagconfig.SimnetParams
 	params.K = 1
-	dag := newTestDAG(&params)
-	nonce := uint64(0)
-	zeroTime := time.Unix(0, 0)
-	addNode := func(parents blockSet, blockTime time.Time) *blockNode {
+	params.DifficultyAdjustmentWindowSize = 264
+	dag, teardownFunc, err := DAGSetup("TestDifficulty", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	zeroTime := mstime.Time{}
+	addNode := func(parents blockSet, blockTime mstime.Time) *blockNode {
 		bluestParent := parents.bluest()
-		if blockTime == zeroTime {
-			blockTime = time.Unix(bluestParent.timestamp+1, 0)
+		if blockTime.IsZero() {
+			blockTime = bluestParent.time()
+			blockTime = blockTime.Add(params.TargetTimePerBlock)
 		}
-		header := &wire.BlockHeader{
-			ParentHashes:         parents.hashes(),
-			Bits:                 dag.requiredDifficulty(bluestParent, blockTime),
-			Nonce:                nonce,
-			Timestamp:            blockTime,
-			HashMerkleRoot:       &daghash.ZeroHash,
-			AcceptedIDMerkleRoot: &daghash.ZeroHash,
-			UTXOCommitment:       &daghash.ZeroHash,
+		block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+		if err != nil {
+			t.Fatalf("unexpected error in PrepareBlockForTest: %s", err)
+		}
+		block.Header.Timestamp = blockTime
+		block.Header.Bits = dag.requiredDifficulty(bluestParent, blockTime)
+
+		isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(block), BFNoPoWCheck)
+		if err != nil {
+			t.Fatalf("unexpected error in ProcessBlock: %s", err)
+		}
+		if isDelayed {
+			t.Fatalf("block is too far in the future")
+		}
+		if isOrphan {
+			t.Fatalf("block was unexpectedly orphan")
+		}
+		node, ok := dag.index.LookupNode(block.BlockHash())
+		if !ok {
+			t.Fatalf("block %s does not exist in the DAG", block.BlockHash())
 		}
-		node := newBlockNode(header, parents, dag.dagParams.K)
-		node.updateParentsChildren()
-		nonce++
 		return node
 	}
 	tip := dag.genesis
 	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
-		tip = addNode(setFromSlice(tip), zeroTime)
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
 		if tip.bits != dag.genesis.bits {
-			t.Fatalf("As long as the bluest parent's blue score is less then the difficulty adjustment window size, the difficulty should be the same as genesis'")
+			t.Fatalf("As long as the bluest parent's blue score is less then the difficulty adjustment " +
+				"window size, the difficulty should be the same as genesis'")
 		}
 	}
-	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize+1000; i++ {
-		tip = addNode(setFromSlice(tip), zeroTime)
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize+100; i++ {
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
 		if tip.bits != dag.genesis.bits {
 			t.Fatalf("As long as the block rate remains the same, the difficulty shouldn't change")
 		}
 	}
-	nodeInThePast := addNode(setFromSlice(tip), tip.PastMedianTime(dag))
+	nodeInThePast := addNode(blockSetFromSlice(tip), tip.PastMedianTime(dag))
 	if nodeInThePast.bits != tip.bits {
 		t.Fatalf("The difficulty should only change when nodeInThePast is in the past of a block bluest parent")
 	}
 	tip = nodeInThePast
 
-	tip = addNode(setFromSlice(tip), zeroTime)
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
 	if tip.bits != nodeInThePast.bits {
 		t.Fatalf("The difficulty should only change when nodeInThePast is in the past of a block bluest parent")
 	}
-	tip = addNode(setFromSlice(tip), zeroTime)
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
 	if compareBits(tip.bits, nodeInThePast.bits) >= 0 {
-		t.Fatalf("tip.bits should be smaller than nodeInThePast.bits because nodeInThePast increased the block rate, so the difficulty should increase as well")
+		t.Fatalf("tip.bits should be smaller than nodeInThePast.bits because nodeInThePast increased the " +
+			"block rate, so the difficulty should increase as well")
 	}
-	expectedBits := uint32(0x207ff395)
+	expectedBits := uint32(0x207f83df)
 	if tip.bits != expectedBits {
 		t.Errorf("tip.bits was expected to be %x but got %x", expectedBits, tip.bits)
 	}
 
 	// Increase block rate to increase difficulty
 	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
-		tip = addNode(setFromSlice(tip), tip.PastMedianTime(dag))
+		tip = addNode(blockSetFromSlice(tip), tip.PastMedianTime(dag))
 		if compareBits(tip.bits, tip.parents.bluest().bits) > 0 {
 			t.Fatalf("Because we're increasing the block rate, the difficulty can't decrease")
 		}
@@ -149,7 +167,7 @@ func TestDifficulty(t *testing.T) {
 	lastBits := tip.bits
 	sameBitsCount := uint64(0)
 	for sameBitsCount < dag.difficultyAdjustmentWindowSize+1 {
-		tip = addNode(setFromSlice(tip), zeroTime)
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
 		if tip.bits == lastBits {
 			sameBitsCount++
 		} else {
@@ -157,37 +175,41 @@ func TestDifficulty(t *testing.T) {
 			sameBitsCount = 0
 		}
 	}
-	slowNode := addNode(setFromSlice(tip), time.Unix(tip.timestamp+2, 0))
+	slowBlockTime := tip.time()
+	slowBlockTime = slowBlockTime.Add(params.TargetTimePerBlock + time.Second)
+	slowNode := addNode(blockSetFromSlice(tip), slowBlockTime)
 	if slowNode.bits != tip.bits {
 		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
 	}
 
 	tip = slowNode
 
-	tip = addNode(setFromSlice(tip), zeroTime)
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
 	if tip.bits != slowNode.bits {
 		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
 	}
-	tip = addNode(setFromSlice(tip), zeroTime)
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
 	if compareBits(tip.bits, slowNode.bits) <= 0 {
-		t.Fatalf("tip.bits should be smaller than slowNode.bits because slowNode decreased the block rate, so the difficulty should decrease as well")
+		t.Fatalf("tip.bits should be smaller than slowNode.bits because slowNode decreased the block" +
+			" rate, so the difficulty should decrease as well")
 	}
 
-	splitNode := addNode(setFromSlice(tip), zeroTime)
+	splitNode := addNode(blockSetFromSlice(tip), zeroTime)
 	tip = splitNode
 	for i := 0; i < 100; i++ {
-		tip = addNode(setFromSlice(tip), zeroTime)
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
 	}
 	blueTip := tip
 
 	redChainTip := splitNode
 	for i := 0; i < 10; i++ {
-		redChainTip = addNode(setFromSlice(redChainTip), redChainTip.PastMedianTime(dag))
+		redChainTip = addNode(blockSetFromSlice(redChainTip), redChainTip.PastMedianTime(dag))
 	}
-	tipWithRedPast := addNode(setFromSlice(redChainTip, blueTip), zeroTime)
-	tipWithoutRedPast := addNode(setFromSlice(blueTip), zeroTime)
+	tipWithRedPast := addNode(blockSetFromSlice(redChainTip, blueTip), zeroTime)
+	tipWithoutRedPast := addNode(blockSetFromSlice(blueTip), zeroTime)
 	if tipWithoutRedPast.bits != tipWithRedPast.bits {
-		t.Fatalf("tipWithoutRedPast.bits should be the same as tipWithRedPast.bits because red blocks shouldn't affect the difficulty")
+		t.Fatalf("tipWithoutRedPast.bits should be the same as tipWithRedPast.bits because red blocks" +
+			" shouldn't affect the difficulty")
 	}
 }
 

blockdag/doc.go

@@ -1,28 +1,21 @@
-// Copyright (c) 2013-2014 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
 /*
 Package blockdag implements kaspa block handling and DAG selection rules.
 
 The kaspa block handling and DAG selection rules are an integral, and quite
-likely the most important, part of kaspa. At its core, kaspa is a
-distributed consensus of which blocks are valid and which ones will comprise the
-DAG (public ledger) that ultimately determines accepted
-transactions, so it is extremely important that fully validating nodes agree on
-all rules.
+likely the most important, part of kaspa. At its core, kaspa is a distributed
+consensus of which blocks are valid and which ones will comprise the DAG
+(public ledger) that ultimately determines accepted transactions, so it is
+extremely important that fully validating nodes agree on all rules.
 
 At a high level, this package provides support for inserting new blocks into
-the block DAG according to the aforementioned rules. It includes
-functionality such as rejecting duplicate blocks, ensuring blocks and
-transactions follow all rules, orphan handling, and DAG order along
-with reorganization.
+the block DAG according to the aforementioned rules. It includes functionality
+such as rejecting duplicate blocks, ensuring blocks and transactions follow all
+rules, orphan handling, and DAG order along with reorganization.
 
 Since this package does not deal with other kaspa specifics such as network
-communication or wallets, it provides a notification system which gives the
-caller a high level of flexibility in how they want to react to certain events
-such as orphan blocks which need their parents requested and newly connected
-DAG blocks which might result in wallet updates.
+communication, it provides a notification system which gives the caller a high
+level of flexibility in how they want to react to certain events such as orphan
+blocks which need their parents requested and newly connected DAG blocks.
 
 Kaspa DAG Processing Overview
 

blockdag/error.go

@@ -6,28 +6,10 @@ package blockdag
 
 import (
 	"fmt"
+
+	"github.com/pkg/errors"
 )
 
-// DeploymentError identifies an error that indicates a deployment ID was
-// specified that does not exist.
-type DeploymentError uint32
-
-// Error returns the assertion error as a human-readable string and satisfies
-// the error interface.
-func (e DeploymentError) Error() string {
-	return fmt.Sprintf("deployment ID %d does not exist", uint32(e))
-}
-
-// AssertError identifies an error that indicates an internal code consistency
-// issue and should be treated as a critical and unrecoverable error.
-type AssertError string
-
-// Error returns the assertion error as a human-readable string and satisfies
-// the error interface.
-func (e AssertError) Error() string {
-	return "assertion failed: " + string(e)
-}
-
 // ErrorCode identifies a kind of error.
 type ErrorCode int
 
@@ -46,11 +28,6 @@ const (
 	// to a newer version.
 	ErrBlockVersionTooOld
 
-	// ErrInvalidTime indicates the time in the passed block has a precision
-	// that is more than one second. The DAG consensus rules require
-	// timestamps to have a maximum precision of one second.
-	ErrInvalidTime
-
 	// ErrTimeTooOld indicates the time is either before the median time of
 	// the last several blocks per the DAG consensus rules.
 	ErrTimeTooOld
@@ -87,6 +64,9 @@ const (
 	// the expected value.
 	ErrBadUTXOCommitment
 
+	// ErrInvalidSubnetwork indicates the subnetwork is now allowed.
+	ErrInvalidSubnetwork
+
 	// ErrFinalityPointTimeTooOld indicates a block has a timestamp before the
 	// last finality point.
 	ErrFinalityPointTimeTooOld
@@ -121,6 +101,11 @@ const (
 	// either does not exist or has already been spent.
 	ErrMissingTxOut
 
+	// ErrDoubleSpendInSameBlock indicates a transaction
+	// that spends an output that was already spent by another
+	// transaction in the same block.
+	ErrDoubleSpendInSameBlock
+
 	// ErrUnfinalizedTx indicates a transaction has not been finalized.
 	// A valid block may only contain finalized transactions.
 	ErrUnfinalizedTx
@@ -217,6 +202,14 @@ const (
 	// ErrInvalidParentsRelation indicates that one of the parents of a block
 	// is also an ancestor of another parent
 	ErrInvalidParentsRelation
+
+	// ErrDelayedBlockIsNotAllowed indicates that a block with a delayed timestamp was
+	// submitted with BFDisallowDelay flag raised.
+	ErrDelayedBlockIsNotAllowed
+
+	// ErrOrphanBlockIsNotAllowed indicates that an orphan block was submitted with
+	// BFDisallowOrphans flag raised.
+	ErrOrphanBlockIsNotAllowed
 )
 
 // Map of ErrorCode values back to their constant names for pretty printing.
@@ -224,7 +217,6 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrDuplicateBlock:            "ErrDuplicateBlock",
 	ErrBlockMassTooHigh:          "ErrBlockMassTooHigh",
 	ErrBlockVersionTooOld:        "ErrBlockVersionTooOld",
-	ErrInvalidTime:               "ErrInvalidTime",
 	ErrTimeTooOld:                "ErrTimeTooOld",
 	ErrTimeTooNew:                "ErrTimeTooNew",
 	ErrNoParents:                 "ErrNoParents",
@@ -241,6 +233,7 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrDuplicateTxInputs:         "ErrDuplicateTxInputs",
 	ErrBadTxInput:                "ErrBadTxInput",
 	ErrMissingTxOut:              "ErrMissingTxOut",
+	ErrDoubleSpendInSameBlock:    "ErrDoubleSpendInSameBlock",
 	ErrUnfinalizedTx:             "ErrUnfinalizedTx",
 	ErrDuplicateTx:               "ErrDuplicateTx",
 	ErrOverwriteTx:               "ErrOverwriteTx",
@@ -264,6 +257,8 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrInvalidPayload:            "ErrInvalidPayload",
 	ErrInvalidPayloadHash:        "ErrInvalidPayloadHash",
 	ErrInvalidParentsRelation:    "ErrInvalidParentsRelation",
+	ErrDelayedBlockIsNotAllowed:  "ErrDelayedBlockIsNotAllowed",
+	ErrOrphanBlockIsNotAllowed:   "ErrOrphanBlockIsNotAllowed",
 }
 
 // String returns the ErrorCode as a human-readable name.
@@ -289,7 +284,10 @@ func (e RuleError) Error() string {
 	return e.Description
 }
 
-// ruleError creates an RuleError given a set of arguments.
-func ruleError(c ErrorCode, desc string) RuleError {
-	return RuleError{ErrorCode: c, Description: desc}
+func ruleError(c ErrorCode, desc string) error {
+	return errors.WithStack(RuleError{ErrorCode: c, Description: desc})
 }
+
+// ErrInvalidParameter signifies that an invalid parameter has been
+// supplied to one of the BlockDAG functions.
+var ErrInvalidParameter = errors.New("invalid parameter")

blockdag/error_test.go

@@ -5,7 +5,6 @@
 package blockdag
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -18,7 +17,6 @@ func TestErrorCodeStringer(t *testing.T) {
 		{ErrDuplicateBlock, "ErrDuplicateBlock"},
 		{ErrBlockMassTooHigh, "ErrBlockMassTooHigh"},
 		{ErrBlockVersionTooOld, "ErrBlockVersionTooOld"},
-		{ErrInvalidTime, "ErrInvalidTime"},
 		{ErrTimeTooOld, "ErrTimeTooOld"},
 		{ErrTimeTooNew, "ErrTimeTooNew"},
 		{ErrNoParents, "ErrNoParents"},
@@ -58,6 +56,8 @@ func TestErrorCodeStringer(t *testing.T) {
 		{ErrInvalidPayload, "ErrInvalidPayload"},
 		{ErrInvalidPayloadHash, "ErrInvalidPayloadHash"},
 		{ErrInvalidParentsRelation, "ErrInvalidParentsRelation"},
+		{ErrDelayedBlockIsNotAllowed, "ErrDelayedBlockIsNotAllowed"},
+		{ErrOrphanBlockIsNotAllowed, "ErrOrphanBlockIsNotAllowed"},
 		{0xffff, "Unknown ErrorCode (65535)"},
 	}
 
@@ -98,46 +98,3 @@ func TestRuleError(t *testing.T) {
 		}
 	}
 }
-
-// TestDeploymentError tests the stringized output for the DeploymentError type.
-func TestDeploymentError(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		in   DeploymentError
-		want string
-	}{
-		{
-			DeploymentError(0),
-			"deployment ID 0 does not exist",
-		},
-		{
-			DeploymentError(10),
-			"deployment ID 10 does not exist",
-		},
-		{
-			DeploymentError(123),
-			"deployment ID 123 does not exist",
-		},
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.Error()
-		if result != test.want {
-			t.Errorf("Error #%d\n got: %s want: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-func TestAssertError(t *testing.T) {
-	message := "abc 123"
-	err := AssertError(message)
-	expectedMessage := fmt.Sprintf("assertion failed: %s", message)
-	if expectedMessage != err.Error() {
-		t.Errorf("Unexpected AssertError message. "+
-			"Got: %s, want: %s", err.Error(), expectedMessage)
-	}
-}

blockdag/external_dag_test.go

@@ -2,10 +2,12 @@ package blockdag_test
 
 import (
 	"fmt"
-	"github.com/pkg/errors"
 	"math"
+	"strings"
 	"testing"
 
+	"github.com/pkg/errors"
+
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 
 	"github.com/kaspanet/kaspad/util/daghash"
@@ -13,10 +15,10 @@ import (
 
 	"github.com/kaspanet/kaspad/blockdag"
 	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/mining"
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
 )
 
 // TestFinality checks that the finality mechanism works as expected.
@@ -37,10 +39,10 @@ import (
 // gets rejected because it doesn't have the last finality point in
 // its selected parent chain.
 func TestFinality(t *testing.T) {
-	params := dagconfig.SimNetParams
+	params := dagconfig.SimnetParams
 	params.K = 1
-	params.FinalityInterval = 100
-	dag, teardownFunc, err := blockdag.DAGSetup("TestFinality", blockdag.Config{
+	params.FinalityDuration = 100 * params.TargetTimePerBlock
+	dag, teardownFunc, err := blockdag.DAGSetup("TestFinality", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -48,17 +50,17 @@ func TestFinality(t *testing.T) {
 	}
 	defer teardownFunc()
 	buildNodeToDag := func(parentHashes []*daghash.Hash) (*util.Block, error) {
-		msgBlock, err := mining.PrepareBlockForTest(dag, &params, parentHashes, nil, false)
+		msgBlock, err := mining.PrepareBlockForTest(dag, parentHashes, nil, false)
 		if err != nil {
 			return nil, err
 		}
 		block := util.NewBlock(msgBlock)
 
-		isOrphan, delay, err := dag.ProcessBlock(block, blockdag.BFNoPoWCheck)
+		isOrphan, isDelayed, err := dag.ProcessBlock(block, blockdag.BFNoPoWCheck)
 		if err != nil {
 			return nil, err
 		}
-		if delay != 0 {
+		if isDelayed {
 			return nil, errors.Errorf("ProcessBlock: block " +
 				"is too far in the future")
 		}
@@ -73,7 +75,7 @@ func TestFinality(t *testing.T) {
 	currentNode := genesis
 
 	// First we build a chain of params.FinalityInterval blocks for future use
-	for i := 0; i < params.FinalityInterval; i++ {
+	for i := uint64(0); i < dag.FinalityInterval(); i++ {
 		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
 		if err != nil {
 			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
@@ -85,7 +87,7 @@ func TestFinality(t *testing.T) {
 	// Now we build a new chain of 2 * params.FinalityInterval blocks, pointed to genesis, and
 	// we expect the block with height 1 * params.FinalityInterval to be the last finality point
 	currentNode = genesis
-	for i := 0; i < params.FinalityInterval; i++ {
+	for i := uint64(0); i < dag.FinalityInterval(); i++ {
 		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
 		if err != nil {
 			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
@@ -94,7 +96,7 @@ func TestFinality(t *testing.T) {
 
 	expectedFinalityPoint := currentNode
 
-	for i := 0; i < params.FinalityInterval; i++ {
+	for i := uint64(0); i < dag.FinalityInterval(); i++ {
 		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
 		if err != nil {
 			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
@@ -123,7 +125,7 @@ func TestFinality(t *testing.T) {
 		t.Errorf("NextBlockCoinbaseTransaction: %s", err)
 	}
 	merkleRoot := blockdag.BuildHashMerkleTreeStore([]*util.Tx{fakeCoinbaseTx}).Root()
-	beforeFinalityBlock := wire.NewMsgBlock(&wire.BlockHeader{
+	beforeFinalityBlock := domainmessage.NewMsgBlock(&domainmessage.BlockHeader{
 		Version:              0x10000000,
 		ParentHashes:         []*daghash.Hash{genesis.Hash()},
 		HashMerkleRoot:       merkleRoot,
@@ -137,10 +139,10 @@ func TestFinality(t *testing.T) {
 	if err == nil {
 		t.Errorf("TestFinality: buildNodeToDag expected an error but got <nil>")
 	}
-	rErr, ok := err.(blockdag.RuleError)
-	if ok {
-		if rErr.ErrorCode != blockdag.ErrFinality {
-			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, rErr.ErrorCode)
+	var ruleErr blockdag.RuleError
+	if errors.As(err, &ruleErr) {
+		if ruleErr.ErrorCode != blockdag.ErrFinality {
+			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, ruleErr.ErrorCode)
 		}
 	} else {
 		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", err)
@@ -152,33 +154,51 @@ func TestFinality(t *testing.T) {
 	if err == nil {
 		t.Errorf("TestFinality: buildNodeToDag expected an error but got <nil>")
 	}
-	rErr, ok = err.(blockdag.RuleError)
-	if ok {
-		if rErr.ErrorCode != blockdag.ErrFinality {
-			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, rErr.ErrorCode)
+	if errors.As(err, &ruleErr) {
+		if ruleErr.ErrorCode != blockdag.ErrFinality {
+			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, ruleErr.ErrorCode)
 		}
 	} else {
-		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", rErr)
+		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", ruleErr)
 	}
 }
 
 // TestFinalityInterval tests that the finality interval is
-// smaller then wire.MaxInvPerMsg, so when a peer receives
+// smaller then domainmessage.MaxInvPerMsg, so when a peer receives
 // a getblocks message it should always be able to send
 // all the necessary invs.
 func TestFinalityInterval(t *testing.T) {
-	params := dagconfig.SimNetParams
-	if params.FinalityInterval > wire.MaxInvPerMsg {
-		t.Errorf("dagconfig.SimNetParams.FinalityInterval should be lower or equal to wire.MaxInvPerMsg")
+	netParams := []*dagconfig.Params{
+		&dagconfig.MainnetParams,
+		&dagconfig.TestnetParams,
+		&dagconfig.DevnetParams,
+		&dagconfig.RegressionNetParams,
+		&dagconfig.SimnetParams,
+	}
+	for _, params := range netParams {
+		func() {
+			dag, teardownFunc, err := blockdag.DAGSetup("TestFinalityInterval", true, blockdag.Config{
+				DAGParams: params,
+			})
+			if err != nil {
+				t.Fatalf("Failed to setup dag instance for %s: %v", params.Name, err)
+			}
+			defer teardownFunc()
+
+			if dag.FinalityInterval() > domainmessage.MaxInvPerMsg {
+				t.Errorf("FinalityInterval in %s should be lower or equal to domainmessage.MaxInvPerMsg", params.Name)
+			}
+		}()
 	}
 }
 
 // TestSubnetworkRegistry tests the full subnetwork registry flow
 func TestSubnetworkRegistry(t *testing.T) {
-	params := dagconfig.SimNetParams
+	params := dagconfig.SimnetParams
 	params.K = 1
 	params.BlockCoinbaseMaturity = 0
-	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", blockdag.Config{
+	params.EnableNonNativeSubnetworks = true
+	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -191,7 +211,7 @@ func TestSubnetworkRegistry(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not register network: %s", err)
 	}
-	limit, err := dag.SubnetworkStore.GasLimit(subnetworkID)
+	limit, err := dag.GasLimit(subnetworkID)
 	if err != nil {
 		t.Fatalf("could not retrieve gas limit: %s", err)
 	}
@@ -201,10 +221,10 @@ func TestSubnetworkRegistry(t *testing.T) {
 }
 
 func TestChainedTransactions(t *testing.T) {
-	params := dagconfig.SimNetParams
+	params := dagconfig.SimnetParams
 	params.BlockCoinbaseMaturity = 0
 	// Create a new database and dag instance to run tests against.
-	dag, teardownFunc, err := blockdag.DAGSetup("TestChainedTransactions", blockdag.Config{
+	dag, teardownFunc, err := blockdag.DAGSetup("TestChainedTransactions", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -212,15 +232,15 @@ func TestChainedTransactions(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	block1, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{params.GenesisHash}, nil, false)
+	block1, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{params.GenesisHash}, nil, false)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(block1), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(block1), blockdag.BFNoPoWCheck)
 	if err != nil {
 		t.Fatalf("ProcessBlock: %v", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("ProcessBlock: block1 " +
 			"is too far in the future")
 	}
@@ -233,50 +253,61 @@ func TestChainedTransactions(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to build signature script: %s", err)
 	}
-	txIn := &wire.TxIn{
-		PreviousOutpoint: wire.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+	txIn := &domainmessage.TxIn{
+		PreviousOutpoint: domainmessage.Outpoint{TxID: *cbTx.TxID(), Index: 0},
 		SignatureScript:  signatureScript,
-		Sequence:         wire.MaxTxInSequenceNum,
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 	}
-	txOut := &wire.TxOut{
+	txOut := &domainmessage.TxOut{
 		ScriptPubKey: blockdag.OpTrueScript,
 		Value:        uint64(1),
 	}
-	tx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
+	tx := domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn}, []*domainmessage.TxOut{txOut})
 
-	chainedTxIn := &wire.TxIn{
-		PreviousOutpoint: wire.Outpoint{TxID: *tx.TxID(), Index: 0},
+	chainedTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: domainmessage.Outpoint{TxID: *tx.TxID(), Index: 0},
 		SignatureScript:  signatureScript,
-		Sequence:         wire.MaxTxInSequenceNum,
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 	}
 
 	scriptPubKey, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript)
 	if err != nil {
 		t.Fatalf("Failed to build public key script: %s", err)
 	}
-	chainedTxOut := &wire.TxOut{
+	chainedTxOut := &domainmessage.TxOut{
 		ScriptPubKey: scriptPubKey,
 		Value:        uint64(1),
 	}
-	chainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{chainedTxIn}, []*wire.TxOut{chainedTxOut})
+	chainedTx := domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{chainedTxIn}, []*domainmessage.TxOut{chainedTxOut})
 
-	block2, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{tx, chainedTx}, true)
+	block2, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{block1.BlockHash()}, []*domainmessage.MsgTx{tx}, false)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
+	// Manually add a chained transaction to block2
+	block2.Transactions = append(block2.Transactions, chainedTx)
+	block2UtilTxs := make([]*util.Tx, len(block2.Transactions))
+	for i, tx := range block2.Transactions {
+		block2UtilTxs[i] = util.NewTx(tx)
+	}
+	block2.Header.HashMerkleRoot = blockdag.BuildHashMerkleTreeStore(block2UtilTxs).Root()
+
 	//Checks that dag.ProcessBlock fails because we don't allow a transaction to spend another transaction from the same block
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(block2), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(block2), blockdag.BFNoPoWCheck)
 	if err == nil {
 		t.Errorf("ProcessBlock expected an error")
-	} else if rErr, ok := err.(blockdag.RuleError); ok {
-		if rErr.ErrorCode != blockdag.ErrMissingTxOut {
-			t.Errorf("ProcessBlock expected an %v error code but got %v", blockdag.ErrMissingTxOut, rErr.ErrorCode)
-		}
 	} else {
-		t.Errorf("ProcessBlock expected a blockdag.RuleError but got %v", err)
+		var ruleErr blockdag.RuleError
+		if ok := errors.As(err, &ruleErr); ok {
+			if ruleErr.ErrorCode != blockdag.ErrMissingTxOut {
+				t.Errorf("ProcessBlock expected an %v error code but got %v", blockdag.ErrMissingTxOut, ruleErr.ErrorCode)
+			}
+		} else {
+			t.Errorf("ProcessBlock expected a blockdag.RuleError but got %v", err)
+		}
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("ProcessBlock: block2 " +
 			"is too far in the future")
 	}
@@ -284,28 +315,28 @@ func TestChainedTransactions(t *testing.T) {
 		t.Errorf("ProcessBlock: block2 got unexpectedly orphaned")
 	}
 
-	nonChainedTxIn := &wire.TxIn{
-		PreviousOutpoint: wire.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+	nonChainedTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: domainmessage.Outpoint{TxID: *cbTx.TxID(), Index: 0},
 		SignatureScript:  signatureScript,
-		Sequence:         wire.MaxTxInSequenceNum,
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 	}
-	nonChainedTxOut := &wire.TxOut{
+	nonChainedTxOut := &domainmessage.TxOut{
 		ScriptPubKey: scriptPubKey,
 		Value:        uint64(1),
 	}
-	nonChainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{nonChainedTxIn}, []*wire.TxOut{nonChainedTxOut})
+	nonChainedTx := domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{nonChainedTxIn}, []*domainmessage.TxOut{nonChainedTxOut})
 
-	block3, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{nonChainedTx}, false)
+	block3, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{block1.BlockHash()}, []*domainmessage.MsgTx{nonChainedTx}, false)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
 	//Checks that dag.ProcessBlock doesn't fail because all of its transaction are dependant on transactions from previous blocks
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(block3), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(block3), blockdag.BFNoPoWCheck)
 	if err != nil {
 		t.Errorf("ProcessBlock: %v", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("ProcessBlock: block3 " +
 			"is too far in the future")
 	}
@@ -319,9 +350,9 @@ func TestChainedTransactions(t *testing.T) {
 // before txB.
 func TestOrderInDiffFromAcceptanceData(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	params := dagconfig.SimNetParams
-	params.K = math.MaxUint32
-	dag, teardownFunc, err := blockdag.DAGSetup("TestOrderInDiffFromAcceptanceData", blockdag.Config{
+	params := dagconfig.SimnetParams
+	params.K = math.MaxUint8
+	dag, teardownFunc, err := blockdag.DAGSetup("TestOrderInDiffFromAcceptanceData", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -332,38 +363,38 @@ func TestOrderInDiffFromAcceptanceData(t *testing.T) {
 
 	createBlock := func(previousBlock *util.Block) *util.Block {
 		// Prepare a transaction that spends the previous block's coinbase transaction
-		var txs []*wire.MsgTx
+		var txs []*domainmessage.MsgTx
 		if !previousBlock.IsGenesis() {
 			previousCoinbaseTx := previousBlock.MsgBlock().Transactions[0]
 			signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
 			if err != nil {
 				t.Fatalf("TestOrderInDiffFromAcceptanceData: Failed to build signature script: %s", err)
 			}
-			txIn := &wire.TxIn{
-				PreviousOutpoint: wire.Outpoint{TxID: *previousCoinbaseTx.TxID(), Index: 0},
+			txIn := &domainmessage.TxIn{
+				PreviousOutpoint: domainmessage.Outpoint{TxID: *previousCoinbaseTx.TxID(), Index: 0},
 				SignatureScript:  signatureScript,
-				Sequence:         wire.MaxTxInSequenceNum,
+				Sequence:         domainmessage.MaxTxInSequenceNum,
 			}
-			txOut := &wire.TxOut{
+			txOut := &domainmessage.TxOut{
 				ScriptPubKey: blockdag.OpTrueScript,
 				Value:        uint64(1),
 			}
-			txs = append(txs, wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut}))
+			txs = append(txs, domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn}, []*domainmessage.TxOut{txOut}))
 		}
 
 		// Create the block
-		msgBlock, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{previousBlock.Hash()}, txs, false)
+		msgBlock, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{previousBlock.Hash()}, txs, false)
 		if err != nil {
 			t.Fatalf("TestOrderInDiffFromAcceptanceData: Failed to prepare block: %s", err)
 		}
 
 		// Add the block to the DAG
 		newBlock := util.NewBlock(msgBlock)
-		isOrphan, delay, err := dag.ProcessBlock(newBlock, blockdag.BFNoPoWCheck)
+		isOrphan, isDelayed, err := dag.ProcessBlock(newBlock, blockdag.BFNoPoWCheck)
 		if err != nil {
 			t.Errorf("TestOrderInDiffFromAcceptanceData: %s", err)
 		}
-		if delay != 0 {
+		if isDelayed {
 			t.Fatalf("TestOrderInDiffFromAcceptanceData: block is too far in the future")
 		}
 		if isOrphan {
@@ -388,10 +419,11 @@ func TestOrderInDiffFromAcceptanceData(t *testing.T) {
 
 // TestGasLimit tests the gas limit rules
 func TestGasLimit(t *testing.T) {
-	params := dagconfig.SimNetParams
+	params := dagconfig.SimnetParams
 	params.K = 1
 	params.BlockCoinbaseMaturity = 0
-	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", blockdag.Config{
+	params.EnableNonNativeSubnetworks = true
+	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -406,17 +438,17 @@ func TestGasLimit(t *testing.T) {
 		t.Fatalf("could not register network: %s", err)
 	}
 
-	cbTxs := []*wire.MsgTx{}
+	cbTxs := []*domainmessage.MsgTx{}
 	for i := 0; i < 4; i++ {
-		fundsBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), nil, false)
+		fundsBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), nil, false)
 		if err != nil {
 			t.Fatalf("PrepareBlockForTest: %v", err)
 		}
-		isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(fundsBlock), blockdag.BFNoPoWCheck)
+		isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(fundsBlock), blockdag.BFNoPoWCheck)
 		if err != nil {
 			t.Fatalf("ProcessBlock: %v", err)
 		}
-		if delay != 0 {
+		if isDelayed {
 			t.Fatalf("ProcessBlock: the funds block " +
 				"is too far in the future")
 		}
@@ -437,44 +469,44 @@ func TestGasLimit(t *testing.T) {
 		t.Fatalf("Failed to build public key script: %s", err)
 	}
 
-	tx1In := &wire.TxIn{
-		PreviousOutpoint: *wire.NewOutpoint(cbTxs[0].TxID(), 0),
-		Sequence:         wire.MaxTxInSequenceNum,
+	tx1In := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[0].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 		SignatureScript:  signatureScript,
 	}
-	tx1Out := &wire.TxOut{
+	tx1Out := &domainmessage.TxOut{
 		Value:        cbTxs[0].TxOut[0].Value,
 		ScriptPubKey: scriptPubKey,
 	}
-	tx1 := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{tx1In}, []*wire.TxOut{tx1Out}, subnetworkID, 10000, []byte{})
+	tx1 := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{tx1In}, []*domainmessage.TxOut{tx1Out}, subnetworkID, 10000, []byte{})
 
-	tx2In := &wire.TxIn{
-		PreviousOutpoint: *wire.NewOutpoint(cbTxs[1].TxID(), 0),
-		Sequence:         wire.MaxTxInSequenceNum,
+	tx2In := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[1].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 		SignatureScript:  signatureScript,
 	}
-	tx2Out := &wire.TxOut{
+	tx2Out := &domainmessage.TxOut{
 		Value:        cbTxs[1].TxOut[0].Value,
 		ScriptPubKey: scriptPubKey,
 	}
-	tx2 := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{tx2In}, []*wire.TxOut{tx2Out}, subnetworkID, 10000, []byte{})
+	tx2 := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{tx2In}, []*domainmessage.TxOut{tx2Out}, subnetworkID, 10000, []byte{})
 
 	// Here we check that we can't process a block that has transactions that exceed the gas limit
-	overLimitBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1, tx2}, true)
+	overLimitBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{tx1, tx2}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(overLimitBlock), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(overLimitBlock), blockdag.BFNoPoWCheck)
 	if err == nil {
 		t.Fatalf("ProcessBlock expected to have an error in block that exceeds gas limit")
 	}
-	rErr, ok := err.(blockdag.RuleError)
-	if !ok {
+	var ruleErr blockdag.RuleError
+	if !errors.As(err, &ruleErr) {
 		t.Fatalf("ProcessBlock expected a RuleError, but got %v", err)
-	} else if rErr.ErrorCode != blockdag.ErrInvalidGas {
-		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, rErr.ErrorCode)
+	} else if ruleErr.ErrorCode != blockdag.ErrInvalidGas {
+		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, ruleErr.ErrorCode)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("ProcessBlock: overLimitBlock " +
 			"is too far in the future")
 	}
@@ -482,73 +514,83 @@ func TestGasLimit(t *testing.T) {
 		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
 	}
 
-	overflowGasTxIn := &wire.TxIn{
-		PreviousOutpoint: *wire.NewOutpoint(cbTxs[2].TxID(), 0),
-		Sequence:         wire.MaxTxInSequenceNum,
+	overflowGasTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[2].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 		SignatureScript:  signatureScript,
 	}
-	overflowGasTxOut := &wire.TxOut{
+	overflowGasTxOut := &domainmessage.TxOut{
 		Value:        cbTxs[2].TxOut[0].Value,
 		ScriptPubKey: scriptPubKey,
 	}
-	overflowGasTx := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{overflowGasTxIn}, []*wire.TxOut{overflowGasTxOut},
+	overflowGasTx := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{overflowGasTxIn}, []*domainmessage.TxOut{overflowGasTxOut},
 		subnetworkID, math.MaxUint64, []byte{})
 
 	// Here we check that we can't process a block that its transactions' gas overflows uint64
-	overflowGasBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1, overflowGasTx}, true)
+	overflowGasBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{tx1, overflowGasTx}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(overflowGasBlock), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(overflowGasBlock), blockdag.BFNoPoWCheck)
 	if err == nil {
 		t.Fatalf("ProcessBlock expected to have an error")
 	}
-	rErr, ok = err.(blockdag.RuleError)
-	if !ok {
+	if !errors.As(err, &ruleErr) {
 		t.Fatalf("ProcessBlock expected a RuleError, but got %v", err)
-	} else if rErr.ErrorCode != blockdag.ErrInvalidGas {
-		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, rErr.ErrorCode)
+	} else if ruleErr.ErrorCode != blockdag.ErrInvalidGas {
+		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, ruleErr.ErrorCode)
 	}
 	if isOrphan {
 		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
 	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: overflowGasBlock " +
+			"is too far in the future")
+	}
 
 	nonExistentSubnetwork := &subnetworkid.SubnetworkID{123}
-	nonExistentSubnetworkTxIn := &wire.TxIn{
-		PreviousOutpoint: *wire.NewOutpoint(cbTxs[3].TxID(), 0),
-		Sequence:         wire.MaxTxInSequenceNum,
+	nonExistentSubnetworkTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[3].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
 		SignatureScript:  signatureScript,
 	}
-	nonExistentSubnetworkTxOut := &wire.TxOut{
+	nonExistentSubnetworkTxOut := &domainmessage.TxOut{
 		Value:        cbTxs[3].TxOut[0].Value,
 		ScriptPubKey: scriptPubKey,
 	}
-	nonExistentSubnetworkTx := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{nonExistentSubnetworkTxIn},
-		[]*wire.TxOut{nonExistentSubnetworkTxOut}, nonExistentSubnetwork, 1, []byte{})
+	nonExistentSubnetworkTx := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{nonExistentSubnetworkTxIn},
+		[]*domainmessage.TxOut{nonExistentSubnetworkTxOut}, nonExistentSubnetwork, 1, []byte{})
 
-	nonExistentSubnetworkBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{nonExistentSubnetworkTx, overflowGasTx}, true)
+	nonExistentSubnetworkBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{nonExistentSubnetworkTx, overflowGasTx}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
 	// Here we check that we can't process a block with a transaction from a non-existent subnetwork
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(nonExistentSubnetworkBlock), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(nonExistentSubnetworkBlock), blockdag.BFNoPoWCheck)
 	expectedErrStr := fmt.Sprintf("Error getting gas limit for subnetworkID '%s': subnetwork '%s' not found",
 		nonExistentSubnetwork, nonExistentSubnetwork)
-	if err.Error() != expectedErrStr {
+	if strings.Contains(err.Error(), expectedErrStr) {
 		t.Fatalf("ProcessBlock expected error \"%v\" but got \"%v\"", expectedErrStr, err)
 	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: nonExistentSubnetworkBlock " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: nonExistentSubnetworkBlock got unexpectedly orphan")
+	}
 
 	// Here we check that we can process a block with a transaction that doesn't exceed the gas limit
-	validBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1}, true)
+	validBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{tx1}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(validBlock), blockdag.BFNoPoWCheck)
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(validBlock), blockdag.BFNoPoWCheck)
 	if err != nil {
 		t.Fatalf("ProcessBlock: %v", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("ProcessBlock: overLimitBlock " +
 			"is too far in the future")
 	}

blockdag/fullblocks_test.go

@@ -1,317 +0,0 @@
-// Copyright (c) 2016 The Decred developers
-// Copyright (c) 2016-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag_test
-
-import (
-	"bytes"
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/blockdag/fullblocktests"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/txscript"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-const (
-	// testDbType is the database backend type to use for the tests.
-	testDbType = "ffldb"
-
-	// testDbRoot is the root directory used to create all test databases.
-	testDbRoot = "testdbs"
-
-	// blockDataNet is the expected network in the test block data.
-	blockDataNet = wire.MainNet
-)
-
-// filesExists returns whether or not the named file or directory exists.
-func fileExists(name string) bool {
-	if _, err := os.Stat(name); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
-
-// isSupportedDbType returns whether or not the passed database type is
-// currently supported.
-func isSupportedDbType(dbType string) bool {
-	supportedDrivers := database.SupportedDrivers()
-	for _, driver := range supportedDrivers {
-		if dbType == driver {
-			return true
-		}
-	}
-
-	return false
-}
-
-// DAGSetup is used to create a new db and DAG instance with the genesis
-// block already inserted. In addition to the new DAG instance, it returns
-// a teardown function the caller should invoke when done testing to clean up.
-func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func(), error) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, errors.Errorf("unsupported db type %v", testDbType)
-	}
-
-	// Handle memory database specially since it doesn't need the disk
-	// specific handling.
-	var db database.DB
-	var teardown func()
-	if testDbType == "memdb" {
-		ndb, err := database.Create(testDbType)
-		if err != nil {
-			return nil, nil, errors.Errorf("error creating db: %v", err)
-		}
-		db = ndb
-
-		// Setup a teardown function for cleaning up. This function is
-		// returned to the caller to be invoked when it is done testing.
-		teardown = func() {
-			db.Close()
-		}
-	} else {
-		// Create the root directory for test databases.
-		if !fileExists(testDbRoot) {
-			if err := os.MkdirAll(testDbRoot, 0700); err != nil {
-				err := errors.Errorf("unable to create test db "+
-					"root: %v", err)
-				return nil, nil, err
-			}
-		}
-
-		// Create a new database to store the accepted blocks into.
-		dbPath := filepath.Join(testDbRoot, dbName)
-		_ = os.RemoveAll(dbPath)
-		ndb, err := database.Create(testDbType, dbPath, blockDataNet)
-		if err != nil {
-			return nil, nil, errors.Errorf("error creating db: %v", err)
-		}
-		db = ndb
-
-		// Setup a teardown function for cleaning up. This function is
-		// returned to the caller to be invoked when it is done testing.
-		teardown = func() {
-			db.Close()
-			os.RemoveAll(dbPath)
-			os.RemoveAll(testDbRoot)
-		}
-	}
-
-	// Copy the DAG params to ensure any modifications the tests do to
-	// the DAG parameters do not affect the global instance.
-	paramsCopy := *params
-
-	// Create the DAG instance.
-	dag, err := blockdag.New(&blockdag.Config{
-		DB:         db,
-		DAGParams:  &paramsCopy,
-		TimeSource: blockdag.NewMedianTime(),
-		SigCache:   txscript.NewSigCache(1000),
-	})
-	if err != nil {
-		teardown()
-		err := errors.Errorf("failed to create DAG instance: %v", err)
-		return nil, nil, err
-	}
-	return dag, teardown, nil
-}
-
-// TestFullBlocks ensures all tests generated by the fullblocktests package
-// have the expected result when processed via ProcessBlock.
-func TestFullBlocks(t *testing.T) {
-	// TODO: (Stas) This test was disabled for until we have implemented Phantom
-	// Ticket: https://daglabs.atlassian.net/browse/DEV-60
-	t.SkipNow()
-
-	tests, err := fullblocktests.Generate(false)
-	if err != nil {
-		t.Fatalf("failed to generate tests: %v", err)
-	}
-
-	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("fullblocktest",
-		&dagconfig.RegressionNetParams)
-	if err != nil {
-		t.Errorf("Failed to setup chain instance: %v", err)
-		return
-	}
-	defer teardownFunc()
-
-	// testAcceptedBlock attempts to process the block in the provided test
-	// instance and ensures that it was accepted according to the flags
-	// specified in the test.
-	testAcceptedBlock := func(item fullblocktests.AcceptedBlock) {
-		blockHeight := item.Height
-		block := util.NewBlock(item.Block)
-		block.SetChainHeight(blockHeight)
-		t.Logf("Testing block %s (hash %s, height %d)",
-			item.Name, block.Hash(), blockHeight)
-
-		isOrphan, delay, err := dag.ProcessBlock(block,
-			blockdag.BFNone)
-		if err != nil {
-			t.Fatalf("block %q (hash %s, height %d) should "+
-				"have been accepted: %v", item.Name,
-				block.Hash(), blockHeight, err)
-		}
-
-		if delay != item.Delay {
-			t.Fatalf("block %q (hash %s, height %d) unexpected "+
-				"delay -- got %v, want %v", item.Name,
-				block.Hash(), blockHeight, delay,
-				item.Delay)
-		}
-
-		if isOrphan != item.IsOrphan {
-			t.Fatalf("block %q (hash %s, height %d) unexpected "+
-				"orphan flag -- got %v, want %v", item.Name,
-				block.Hash(), blockHeight, isOrphan,
-				item.IsOrphan)
-		}
-	}
-
-	// testRejectedBlock attempts to process the block in the provided test
-	// instance and ensures that it was rejected with the reject code
-	// specified in the test.
-	testRejectedBlock := func(item fullblocktests.RejectedBlock) {
-		blockHeight := item.Height
-		block := util.NewBlock(item.Block)
-		block.SetChainHeight(blockHeight)
-		t.Logf("Testing block %s (hash %s, height %d)",
-			item.Name, block.Hash(), blockHeight)
-
-		_, _, err := dag.ProcessBlock(block, blockdag.BFNone)
-		if err == nil {
-			t.Fatalf("block %q (hash %s, height %d) should not "+
-				"have been accepted", item.Name, block.Hash(),
-				blockHeight)
-		}
-
-		// Ensure the error code is of the expected type and the reject
-		// code matches the value specified in the test instance.
-		rerr, ok := err.(blockdag.RuleError)
-		if !ok {
-			t.Fatalf("block %q (hash %s, height %d) returned "+
-				"unexpected error type -- got %T, want "+
-				"blockchain.RuleError", item.Name, block.Hash(),
-				blockHeight, err)
-		}
-		if rerr.ErrorCode != item.RejectCode {
-			t.Fatalf("block %q (hash %s, height %d) does not have "+
-				"expected reject code -- got %v, want %v",
-				item.Name, block.Hash(), blockHeight,
-				rerr.ErrorCode, item.RejectCode)
-		}
-	}
-
-	// testRejectedNonCanonicalBlock attempts to decode the block in the
-	// provided test instance and ensures that it failed to decode with a
-	// message error.
-	testRejectedNonCanonicalBlock := func(item fullblocktests.RejectedNonCanonicalBlock) {
-		headerLen := len(item.RawBlock)
-		if headerLen > 80 {
-			headerLen = 80
-		}
-		blockHash := daghash.DoubleHashH(item.RawBlock[0:headerLen])
-		blockHeight := item.Height
-		t.Logf("Testing block %s (hash %s, height %d)", item.Name,
-			blockHash, blockHeight)
-
-		// Ensure there is an error due to deserializing the block.
-		var msgBlock wire.MsgBlock
-		err := msgBlock.KaspaDecode(bytes.NewReader(item.RawBlock), 0)
-		if _, ok := err.(*wire.MessageError); !ok {
-			t.Fatalf("block %q (hash %s, height %d) should have "+
-				"failed to decode", item.Name, blockHash,
-				blockHeight)
-		}
-	}
-
-	// testOrphanOrRejectedBlock attempts to process the block in the
-	// provided test instance and ensures that it was either accepted as an
-	// orphan or rejected with a rule violation.
-	testOrphanOrRejectedBlock := func(item fullblocktests.OrphanOrRejectedBlock) {
-		blockHeight := item.Height
-		block := util.NewBlock(item.Block)
-		block.SetChainHeight(blockHeight)
-		t.Logf("Testing block %s (hash %s, height %d)",
-			item.Name, block.Hash(), blockHeight)
-
-		isOrphan, delay, err := dag.ProcessBlock(block, blockdag.BFNone)
-		if err != nil {
-			// Ensure the error code is of the expected type.
-			if _, ok := err.(blockdag.RuleError); !ok {
-				t.Fatalf("block %q (hash %s, height %d) "+
-					"returned unexpected error type -- "+
-					"got %T, want blockdag.RuleError",
-					item.Name, block.Hash(), blockHeight,
-					err)
-			}
-		}
-
-		if delay != 0 {
-			t.Fatalf("block %q (hash %s, height %d) "+
-				"is too far in the future",
-				item.Name, block.Hash(), blockHeight)
-		}
-
-		if !isOrphan {
-			t.Fatalf("block %q (hash %s, height %d) was accepted, "+
-				"but is not considered an orphan", item.Name,
-				block.Hash(), blockHeight)
-		}
-	}
-
-	// testExpectedTip ensures the current tip of the blockDAG is the
-	// block specified in the provided test instance.
-	testExpectedTip := func(item fullblocktests.ExpectedTip) {
-		blockHeight := item.Height
-		block := util.NewBlock(item.Block)
-		block.SetChainHeight(blockHeight)
-		t.Logf("Testing tip for block %s (hash %s, height %d)",
-			item.Name, block.Hash(), blockHeight)
-
-		// Ensure hash and height match.
-		if dag.SelectedTipHash() != item.Block.BlockHash() ||
-			dag.ChainHeight() != blockHeight { //TODO: (Ori) the use of dag.ChainHeight() and virtualBlock.HighestTipHash() is wrong, and was done only for compilation
-
-			t.Fatalf("block %q (hash %s, height %d) should be "+
-				"the current tip -- got (hash %s, height %d)",
-				item.Name, block.Hash(), blockHeight, dag.SelectedTipHash(),
-				dag.ChainHeight()) //TODO: (Ori) the use of dag.ChainHeight() and virtualBlock.HighestTipHash() is wrong, and was done only for compilation
-		}
-	}
-
-	for testNum, test := range tests {
-		for itemNum, item := range test {
-			switch item := item.(type) {
-			case fullblocktests.AcceptedBlock:
-				testAcceptedBlock(item)
-			case fullblocktests.RejectedBlock:
-				testRejectedBlock(item)
-			case fullblocktests.RejectedNonCanonicalBlock:
-				testRejectedNonCanonicalBlock(item)
-			case fullblocktests.OrphanOrRejectedBlock:
-				testOrphanOrRejectedBlock(item)
-			case fullblocktests.ExpectedTip:
-				testExpectedTip(item)
-			default:
-				t.Fatalf("test #%d, item #%d is not one of "+
-					"the supported test instance types -- "+
-					"got type: %T", testNum, itemNum, item)
-			}
-		}
-	}
-}

blockdag/fullblocktests/README.md

@@ -1,29 +0,0 @@
-fullblocktests
-==============
-
-[![Build Status](http://img.shields.io/travis/btcsuite/btcd.svg)](https://travis-ci.org/btcsuite/btcd)
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/blockchain/fullblocktests)
-
-Package fullblocktests provides a set of full block tests to be used for testing
-the consensus validation rules. The tests are intended to be flexible enough to
-allow both unit-style tests directly against the blockchain code as well as
-integration style tests over the peer-to-peer network. To achieve that goal,
-each test contains additional information about the expected result, however
-that information can be ignored when doing comparison tests between two
-independent versions over the peer-to-peer network.
-
-This package has intentionally been designed so it can be used as a standalone
-package for any projects needing to test their implementation against a full set
-of blocks that exercise the consensus validation rules.
-
-## Installation and Updating
-
-```bash
-$ go get -u github.com/kaspanet/kaspad/blockchain/fullblocktests
-```
-
-## License
-
-Package fullblocktests is licensed under the [copyfree](http://copyfree.org) ISC
-License.

blockdag/fullblocktests/doc.go

@@ -1,20 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-Package fullblocktests provides a set of block consensus validation tests.
-
-All of the generated test instances involve full blocks that are to be used for
-testing the consensus validation rules. The tests are intended to be flexible
-enough to allow both unit-style tests directly against the blockchain code as
-well as integration style tests over the peer-to-peer network. To achieve that
-goal, each test contains additional information about the expected result,
-however that information can be ignored when doing comparison tests between two
-independent versions over the peer-to-peer network.
-
-This package has intentionally been designed so it can be used as a standalone
-package for any projects needing to test their implementation against a full set
-of blocks that exercise the consensus validation rules.
-*/
-package fullblocktests

blockdag/fullblocktests/params.go

@@ -1,140 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package fullblocktests
-
-import (
-	"encoding/hex"
-	"math"
-	"math/big"
-	"time"
-
-	"github.com/kaspanet/kaspad/util/hdkeychain"
-	"github.com/kaspanet/kaspad/util/subnetworkid"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-// newHashFromStr converts the passed big-endian hex string into a
-// wire.Hash. It only differs from the one available in daghash in that
-// it panics on an error since it will only (and must only) be called with
-// hard-coded, and therefore known good, hashes.
-func newHashFromStr(hexStr string) *daghash.Hash {
-	hash, err := daghash.NewHashFromStr(hexStr)
-	if err != nil {
-		panic(err)
-	}
-	return hash
-}
-
-// newTxIDFromStr converts the passed big-endian hex string into a
-// wire.TxID. It only differs from the one available in daghash in that
-// it panics on an error since it will only (and must only) be called with
-// hard-coded, and therefore known good, hashes.
-func newTxIDFromStr(hexStr string) *daghash.TxID {
-	txID, err := daghash.NewTxIDFromStr(hexStr)
-	if err != nil {
-		panic(err)
-	}
-	return txID
-}
-
-// fromHex converts the passed hex string into a byte slice and will panic if
-// there is an error. This is only provided for the hard-coded constants so
-// errors in the source code can be detected. It will only (and must only) be
-// called for initialization purposes.
-func fromHex(s string) []byte {
-	r, err := hex.DecodeString(s)
-	if err != nil {
-		panic("invalid hex in source file: " + s)
-	}
-	return r
-}
-
-var (
-	// bigOne is 1 represented as a big.Int. It is defined here to avoid
-	// the overhead of creating it multiple times.
-	bigOne = big.NewInt(1)
-
-	// regressionPowLimit is the highest proof of work value a Bitcoin block
-	// can have for the regression test network. It is the value 2^255 - 1.
-	regressionPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 255), bigOne)
-
-	// regTestGenesisBlock defines the genesis block of the block chain which serves
-	// as the public transaction ledger for the regression test network.
-	regTestGenesisBlock = wire.MsgBlock{
-		Header: wire.BlockHeader{
-			Version:        1,
-			ParentHashes:   []*daghash.Hash{},
-			HashMerkleRoot: newHashFromStr("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"),
-			Timestamp:      time.Unix(0x5b28c636, 0), // 2018-06-19 09:00:38 +0000 UTC
-			Bits:           0x207fffff,               // 545259519 [7fffff0000000000000000000000000000000000000000000000000000000000]
-			Nonce:          1,
-		},
-		Transactions: []*wire.MsgTx{{
-			Version: 1,
-			TxIn: []*wire.TxIn{{
-				PreviousOutpoint: wire.Outpoint{
-					TxID:  daghash.TxID{},
-					Index: 0xffffffff,
-				},
-				SignatureScript: fromHex("04ffff001d010445" +
-					"5468652054696d65732030332f4a616e2f" +
-					"32303039204368616e63656c6c6f72206f" +
-					"6e206272696e6b206f66207365636f6e64" +
-					"206261696c6f757420666f72206261686b73"),
-				Sequence: math.MaxUint64,
-			}},
-			TxOut: []*wire.TxOut{{
-				Value: 0,
-				ScriptPubKey: fromHex("4104678afdb0fe5548271967f1" +
-					"a67130b7105cd6a828e03909a67962e0ea1f" +
-					"61deb649f6bc3f4cef38c4f35504e51ec138" +
-					"c4f35504e51ec112de5c384df7ba0b8d578a" +
-					"4c702b6bf11d5fac"),
-			}},
-			LockTime:     0,
-			SubnetworkID: *subnetworkid.SubnetworkIDNative,
-		}},
-	}
-)
-
-// regressionNetParams defines the network parameters for the regression test
-// network.
-//
-// NOTE: The test generator intentionally does not use the existing definitions
-// in the dagconfig package since the intent is to be able to generate known
-// good tests which exercise that code. Using the dagconfig parameters would
-// allow them to change out from under the tests potentially invalidating them.
-var regressionNetParams = &dagconfig.Params{
-	Name:        "regtest",
-	Net:         wire.RegTest,
-	DefaultPort: "18444",
-
-	// DAG parameters
-	GenesisBlock:                   &regTestGenesisBlock,
-	GenesisHash:                    newHashFromStr("5bec7567af40504e0994db3b573c186fffcc4edefe096ff2e58d00523bd7e8a6"),
-	PowMax:                         regressionPowLimit,
-	BlockCoinbaseMaturity:          100,
-	SubsidyReductionInterval:       150,
-	TargetTimePerBlock:             time.Second * 10, // 10 seconds
-	DifficultyAdjustmentWindowSize: 2640,
-	TimestampDeviationTolerance:    132,
-	GenerateSupported:              true,
-
-	// Mempool parameters
-	RelayNonStdTxs: true,
-
-	// Address encoding magics
-	PrivateKeyID: 0xef, // starts with 9 (uncompressed) or c (compressed)
-
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair: hdkeychain.HDKeyPairRegressionNet,
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType: 1,
-}

blockdag/ghostdag.go

@@ -0,0 +1,176 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/pkg/errors"
+	"sort"
+)
+
+// ghostdag runs the GHOSTDAG protocol and updates newNode.blues,
+// newNode.selectedParent and newNode.bluesAnticoneSizes accordingly.
+// The function updates newNode.blues by iterating over the blocks in
+// the anticone of newNode.selectedParent (which is the parent with the
+// highest blue score) and adds any block to newNode.blues if by adding
+// it to newNode.blues these conditions will not be violated:
+//
+// 1) |anticone-of-candidate-block ∩ blue-set-of-newNode| ≤ K
+//
+// 2) For every blue block in blue-set-of-newNode:
+//    |(anticone-of-blue-block ∩ blue-set-newNode) ∪ {candidate-block}| ≤ K.
+//    We validate this condition by maintaining a map bluesAnticoneSizes for
+//    each block which holds all the blue anticone sizes that were affected by
+//    the new added blue blocks.
+//    So to find out what is |anticone-of-blue ∩ blue-set-of-newNode| we just iterate in
+//    the selected parent chain of newNode until we find an existing entry in
+//    bluesAnticoneSizes.
+//
+// For further details see the article https://eprint.iacr.org/2018/104.pdf
+func (dag *BlockDAG) ghostdag(newNode *blockNode) (selectedParentAnticone []*blockNode, err error) {
+	newNode.selectedParent = newNode.parents.bluest()
+	newNode.bluesAnticoneSizes[newNode.selectedParent] = 0
+	newNode.blues = []*blockNode{newNode.selectedParent}
+	selectedParentAnticone, err = dag.selectedParentAnticone(newNode)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Slice(selectedParentAnticone, func(i, j int) bool {
+		return selectedParentAnticone[i].less(selectedParentAnticone[j])
+	})
+
+	for _, blueCandidate := range selectedParentAnticone {
+		candidateBluesAnticoneSizes := make(map[*blockNode]dagconfig.KType)
+		var candidateAnticoneSize dagconfig.KType
+		possiblyBlue := true
+
+		// Iterate over all blocks in the blue set of newNode that are not in the past
+		// of blueCandidate, and check for each one of them if blueCandidate potentially
+		// enlarges their blue anticone to be over K, or that they enlarge the blue anticone
+		// of blueCandidate to be over K.
+		for chainBlock := newNode; possiblyBlue; chainBlock = chainBlock.selectedParent {
+			// If blueCandidate is in the future of chainBlock, it means
+			// that all remaining blues are in the past of chainBlock and thus
+			// in the past of blueCandidate. In this case we know for sure that
+			// the anticone of blueCandidate will not exceed K, and we can mark
+			// it as blue.
+			//
+			// newNode is always in the future of blueCandidate, so there's
+			// no point in checking it.
+			if chainBlock != newNode {
+				if isAncestorOfBlueCandidate, err := dag.isInPast(chainBlock, blueCandidate); err != nil {
+					return nil, err
+				} else if isAncestorOfBlueCandidate {
+					break
+				}
+			}
+
+			for _, block := range chainBlock.blues {
+				// Skip blocks that exist in the past of blueCandidate.
+				if isAncestorOfBlueCandidate, err := dag.isInPast(block, blueCandidate); err != nil {
+					return nil, err
+				} else if isAncestorOfBlueCandidate {
+					continue
+				}
+
+				candidateBluesAnticoneSizes[block], err = dag.blueAnticoneSize(block, newNode)
+				if err != nil {
+					return nil, err
+				}
+				candidateAnticoneSize++
+
+				if candidateAnticoneSize > dag.Params.K {
+					// k-cluster violation: The candidate's blue anticone exceeded k
+					possiblyBlue = false
+					break
+				}
+
+				if candidateBluesAnticoneSizes[block] == dag.Params.K {
+					// k-cluster violation: A block in candidate's blue anticone already
+					// has k blue blocks in its own anticone
+					possiblyBlue = false
+					break
+				}
+
+				// This is a sanity check that validates that a blue
+				// block's blue anticone is not already larger than K.
+				if candidateBluesAnticoneSizes[block] > dag.Params.K {
+					return nil, errors.New("found blue anticone size larger than k")
+				}
+			}
+		}
+
+		if possiblyBlue {
+			// No k-cluster violation found, we can now set the candidate block as blue
+			newNode.blues = append(newNode.blues, blueCandidate)
+			newNode.bluesAnticoneSizes[blueCandidate] = candidateAnticoneSize
+			for blue, blueAnticoneSize := range candidateBluesAnticoneSizes {
+				newNode.bluesAnticoneSizes[blue] = blueAnticoneSize + 1
+			}
+
+			// The maximum length of node.blues can be K+1 because
+			// it contains the selected parent.
+			if dagconfig.KType(len(newNode.blues)) == dag.Params.K+1 {
+				break
+			}
+		}
+	}
+
+	newNode.blueScore = newNode.selectedParent.blueScore + uint64(len(newNode.blues))
+	return selectedParentAnticone, nil
+}
+
+// selectedParentAnticone returns the blocks in the anticone of the selected parent of the given node.
+// The function work as follows.
+// We start by adding all parents of the node (other than the selected parent) to a process queue.
+// For each node in the queue:
+//   we check whether it is in the past of the selected parent.
+//   If not, we add the node to the resulting anticone-set and queue it for processing.
+func (dag *BlockDAG) selectedParentAnticone(node *blockNode) ([]*blockNode, error) {
+	anticoneSet := newBlockSet()
+	var anticoneSlice []*blockNode
+	selectedParentPast := newBlockSet()
+	var queue []*blockNode
+	// Queueing all parents (other than the selected parent itself) for processing.
+	for parent := range node.parents {
+		if parent == node.selectedParent {
+			continue
+		}
+		anticoneSet.add(parent)
+		anticoneSlice = append(anticoneSlice, parent)
+		queue = append(queue, parent)
+	}
+	for len(queue) > 0 {
+		var current *blockNode
+		current, queue = queue[0], queue[1:]
+		// For each parent of a the current node we check whether it is in the past of the selected parent. If not,
+		// we add the it to the resulting anticone-set and queue it for further processing.
+		for parent := range current.parents {
+			if anticoneSet.contains(parent) || selectedParentPast.contains(parent) {
+				continue
+			}
+			isAncestorOfSelectedParent, err := dag.isInPast(parent, node.selectedParent)
+			if err != nil {
+				return nil, err
+			}
+			if isAncestorOfSelectedParent {
+				selectedParentPast.add(parent)
+				continue
+			}
+			anticoneSet.add(parent)
+			anticoneSlice = append(anticoneSlice, parent)
+			queue = append(queue, parent)
+		}
+	}
+	return anticoneSlice, nil
+}
+
+// blueAnticoneSize returns the blue anticone size of 'block' from the worldview of 'context'.
+// Expects 'block' to be in the blue set of 'context'
+func (dag *BlockDAG) blueAnticoneSize(block, context *blockNode) (dagconfig.KType, error) {
+	for current := context; current != nil; current = current.selectedParent {
+		if blueAnticoneSize, ok := current.bluesAnticoneSizes[block]; ok {
+			return blueAnticoneSize, nil
+		}
+	}
+	return 0, errors.Errorf("block %s is not in blue set of %s", block.hash, context.hash)
+}

blockdag/ghostdag_test.go

@@ -0,0 +1,386 @@
+package blockdag
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+type testBlockData struct {
+	parents                []string
+	id                     string // id is a virtual entity that is used only for tests so we can define relations between blocks without knowing their hash
+	expectedScore          uint64
+	expectedSelectedParent string
+	expectedBlues          []string
+}
+
+// TestGHOSTDAG iterates over several dag simulations, and checks
+// that the blue score, blue set and selected parent of each
+// block are calculated as expected.
+func TestGHOSTDAG(t *testing.T) {
+	dagParams := dagconfig.SimnetParams
+
+	tests := []struct {
+		k            dagconfig.KType
+		expectedReds []string
+		dagData      []*testBlockData
+	}{
+		{
+			k:            3,
+			expectedReds: []string{"F", "G", "H", "I", "N", "P"},
+			dagData: []*testBlockData{
+				{
+					parents:                []string{"A"},
+					id:                     "B",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"B"},
+					id:                     "C",
+					expectedScore:          2,
+					expectedSelectedParent: "B",
+					expectedBlues:          []string{"B"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "D",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"C", "D"},
+					id:                     "E",
+					expectedScore:          4,
+					expectedSelectedParent: "C",
+					expectedBlues:          []string{"C", "D"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "F",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"F"},
+					id:                     "G",
+					expectedScore:          2,
+					expectedSelectedParent: "F",
+					expectedBlues:          []string{"F"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "H",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "I",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"E", "G"},
+					id:                     "J",
+					expectedScore:          5,
+					expectedSelectedParent: "E",
+					expectedBlues:          []string{"E"},
+				},
+				{
+					parents:                []string{"J"},
+					id:                     "K",
+					expectedScore:          6,
+					expectedSelectedParent: "J",
+					expectedBlues:          []string{"J"},
+				},
+				{
+					parents:                []string{"I", "K"},
+					id:                     "L",
+					expectedScore:          7,
+					expectedSelectedParent: "K",
+					expectedBlues:          []string{"K"},
+				},
+				{
+					parents:                []string{"L"},
+					id:                     "M",
+					expectedScore:          8,
+					expectedSelectedParent: "L",
+					expectedBlues:          []string{"L"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "N",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "O",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "P",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "Q",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "R",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"R"},
+					id:                     "S",
+					expectedScore:          10,
+					expectedSelectedParent: "R",
+					expectedBlues:          []string{"R"},
+				},
+				{
+					parents:                []string{"N", "O", "P", "Q", "S"},
+					id:                     "T",
+					expectedScore:          13,
+					expectedSelectedParent: "S",
+					expectedBlues:          []string{"S", "O", "Q"},
+				},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		func() {
+			resetExtraNonceForTest()
+			dagParams.K = test.k
+			dag, teardownFunc, err := DAGSetup(fmt.Sprintf("TestGHOSTDAG%d", i), true, Config{
+				DAGParams: &dagParams,
+			})
+			if err != nil {
+				t.Fatalf("Failed to setup dag instance: %v", err)
+			}
+			defer teardownFunc()
+
+			genesisNode := dag.genesis
+			blockByIDMap := make(map[string]*blockNode)
+			idByBlockMap := make(map[*blockNode]string)
+			blockByIDMap["A"] = genesisNode
+			idByBlockMap[genesisNode] = "A"
+
+			for _, blockData := range test.dagData {
+				parents := blockSet{}
+				for _, parentID := range blockData.parents {
+					parent := blockByIDMap[parentID]
+					parents.add(parent)
+				}
+
+				block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+				if err != nil {
+					t.Fatalf("TestGHOSTDAG: block %v got unexpected error from PrepareBlockForTest: %v", blockData.id, err)
+				}
+
+				utilBlock := util.NewBlock(block)
+				isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+				if err != nil {
+					t.Fatalf("TestGHOSTDAG: dag.ProcessBlock got unexpected error for block %v: %v", blockData.id, err)
+				}
+				if isDelayed {
+					t.Fatalf("TestGHOSTDAG: block %s "+
+						"is too far in the future", blockData.id)
+				}
+				if isOrphan {
+					t.Fatalf("TestGHOSTDAG: block %v was unexpectedly orphan", blockData.id)
+				}
+
+				node, ok := dag.index.LookupNode(utilBlock.Hash())
+				if !ok {
+					t.Fatalf("block %s does not exist in the DAG", utilBlock.Hash())
+				}
+
+				blockByIDMap[blockData.id] = node
+				idByBlockMap[node] = blockData.id
+
+				bluesIDs := make([]string, 0, len(node.blues))
+				for _, blue := range node.blues {
+					bluesIDs = append(bluesIDs, idByBlockMap[blue])
+				}
+				selectedParentID := idByBlockMap[node.selectedParent]
+				fullDataStr := fmt.Sprintf("blues: %v, selectedParent: %v, score: %v",
+					bluesIDs, selectedParentID, node.blueScore)
+				if blockData.expectedScore != node.blueScore {
+					t.Errorf("Test %d: Block %v expected to have score %v but got %v (fulldata: %v)",
+						i, blockData.id, blockData.expectedScore, node.blueScore, fullDataStr)
+				}
+				if blockData.expectedSelectedParent != selectedParentID {
+					t.Errorf("Test %d: Block %v expected to have selected parent %v but got %v (fulldata: %v)",
+						i, blockData.id, blockData.expectedSelectedParent, selectedParentID, fullDataStr)
+				}
+				if !reflect.DeepEqual(blockData.expectedBlues, bluesIDs) {
+					t.Errorf("Test %d: Block %v expected to have blues %v but got %v (fulldata: %v)",
+						i, blockData.id, blockData.expectedBlues, bluesIDs, fullDataStr)
+				}
+			}
+
+			reds := make(map[string]bool)
+
+			for id := range blockByIDMap {
+				reds[id] = true
+			}
+
+			for tip := &dag.virtual.blockNode; tip.selectedParent != nil; tip = tip.selectedParent {
+				tipID := idByBlockMap[tip]
+				delete(reds, tipID)
+				for _, blue := range tip.blues {
+					blueID := idByBlockMap[blue]
+					delete(reds, blueID)
+				}
+			}
+			if !checkReds(test.expectedReds, reds) {
+				redsIDs := make([]string, 0, len(reds))
+				for id := range reds {
+					redsIDs = append(redsIDs, id)
+				}
+				sort.Strings(redsIDs)
+				sort.Strings(test.expectedReds)
+				t.Errorf("Test %d: Expected reds %v but got %v", i, test.expectedReds, redsIDs)
+			}
+		}()
+	}
+}
+
+func checkReds(expectedReds []string, reds map[string]bool) bool {
+	if len(expectedReds) != len(reds) {
+		return false
+	}
+	for _, redID := range expectedReds {
+		if !reds[redID] {
+			return false
+		}
+	}
+	return true
+}
+
+func TestBlueAnticoneSizeErrors(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizeErrors", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestBlueAnticoneSizeErrors: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	// Prepare a block chain with size K beginning with the genesis block
+	currentBlockA := dag.Params.GenesisBlock
+	for i := dagconfig.KType(0); i < dag.Params.K; i++ {
+		newBlock := prepareAndProcessBlockByParentMsgBlocks(t, dag, currentBlockA)
+		currentBlockA = newBlock
+	}
+
+	// Prepare another block chain with size K beginning with the genesis block
+	currentBlockB := dag.Params.GenesisBlock
+	for i := dagconfig.KType(0); i < dag.Params.K; i++ {
+		newBlock := prepareAndProcessBlockByParentMsgBlocks(t, dag, currentBlockB)
+		currentBlockB = newBlock
+	}
+
+	// Get references to the tips of the two chains
+	blockNodeA, ok := dag.index.LookupNode(currentBlockA.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", currentBlockA.BlockHash())
+	}
+
+	blockNodeB, ok := dag.index.LookupNode(currentBlockB.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", currentBlockB.BlockHash())
+	}
+
+	// Try getting the blueAnticoneSize between them. Since the two
+	// blocks are not in the anticones of eachother, this should fail.
+	_, err = dag.blueAnticoneSize(blockNodeA, blockNodeB)
+	if err == nil {
+		t.Fatalf("TestBlueAnticoneSizeErrors: blueAnticoneSize unexpectedly succeeded")
+	}
+	expectedErrSubstring := "is not in blue set of"
+	if !strings.Contains(err.Error(), expectedErrSubstring) {
+		t.Fatalf("TestBlueAnticoneSizeErrors: blueAnticoneSize returned wrong error. "+
+			"Want: %s, got: %s", expectedErrSubstring, err)
+	}
+}
+
+func TestGHOSTDAGErrors(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestGHOSTDAGErrors", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestGHOSTDAGErrors: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	// Add two child blocks to the genesis
+	block1 := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.Params.GenesisBlock)
+	block2 := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.Params.GenesisBlock)
+
+	// Add a child block to the previous two blocks
+	block3 := prepareAndProcessBlockByParentMsgBlocks(t, dag, block1, block2)
+
+	// Clear the reachability store
+	dag.reachabilityTree.store.loaded = map[daghash.Hash]*reachabilityData{}
+
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		t.Fatalf("NewTx: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+
+	err = dbaccess.ClearReachabilityData(dbTx)
+	if err != nil {
+		t.Fatalf("ClearReachabilityData: %s", err)
+	}
+
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Commit: %s", err)
+	}
+
+	// Try to rerun GHOSTDAG on the last block. GHOSTDAG uses
+	// reachability data, so we expect it to fail.
+	blockNode3, ok := dag.index.LookupNode(block3.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", block3.BlockHash())
+	}
+	_, err = dag.ghostdag(blockNode3)
+	if err == nil {
+		t.Fatalf("TestGHOSTDAGErrors: ghostdag unexpectedly succeeded")
+	}
+	expectedErrSubstring := "couldn't find reachability data"
+	if !strings.Contains(err.Error(), expectedErrSubstring) {
+		t.Fatalf("TestGHOSTDAGErrors: ghostdag returned wrong error. "+
+			"Want: %s, got: %s", expectedErrSubstring, err)
+	}
+}

blockdag/indexers/README.md

@@ -1,7 +1,7 @@
 indexers
 ========
 
-[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
 [![GoDoc](https://godoc.org/github.com/kaspanet/kaspad/blockdag/indexers?status.png)](http://godoc.org/github.com/kaspanet/kaspad/blockdag/indexers)
 
 Package indexers implements optional block chain indexes.

blockdag/indexers/acceptanceindex.go

@@ -3,31 +3,20 @@ package indexers
 import (
 	"bytes"
 	"encoding/gob"
+
 	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/pkg/errors"
-)
-
-const (
-	// acceptanceIndexName is the human-readable name for the index.
-	acceptanceIndexName = "acceptance index"
-)
-
-var (
-	// acceptanceIndexKey is the key of the acceptance index and the db bucket used
-	// to house it.
-	acceptanceIndexKey = []byte("acceptanceidx")
 )
 
 // AcceptanceIndex implements a txAcceptanceData by block hash index. That is to say,
 // it stores a mapping between a block's hash and the set of transactions that the
 // block accepts among its blue blocks.
 type AcceptanceIndex struct {
-	db  database.DB
-	dag *blockdag.BlockDAG
+	dag             *blockdag.BlockDAG
+	databaseContext *dbaccess.DatabaseContext
 }
 
 // Ensure the AcceptanceIndex type implements the Indexer interface.
@@ -43,127 +32,88 @@ func NewAcceptanceIndex() *AcceptanceIndex {
 	return &AcceptanceIndex{}
 }
 
-// DropAcceptanceIndex drops the acceptance index from the provided database if it
-// exists.
-func DropAcceptanceIndex(db database.DB, interrupt <-chan struct{}) error {
-	return dropIndex(db, acceptanceIndexKey, acceptanceIndexName, interrupt)
-}
+// DropAcceptanceIndex drops the acceptance index.
+func DropAcceptanceIndex(databaseContext *dbaccess.DatabaseContext) error {
+	dbTx, err := databaseContext.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
 
-// Key returns the database key to use for the index as a byte slice.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Key() []byte {
-	return acceptanceIndexKey
-}
+	err = dbaccess.DropAcceptanceIndex(dbTx)
+	if err != nil {
+		return err
+	}
 
-// Name returns the human-readable name of the index.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Name() string {
-	return acceptanceIndexName
-}
-
-// Create is invoked when the indexer manager determines the index needs
-// to be created for the first time. It creates the bucket for the
-// acceptance index.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Create(dbTx database.Tx) error {
-	_, err := dbTx.Metadata().CreateBucket(acceptanceIndexKey)
-	return err
+	return dbTx.Commit()
 }
 
 // Init initializes the hash-based acceptance index.
 //
 // This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Init(db database.DB, dag *blockdag.BlockDAG) error {
-	idx.db = db
+func (idx *AcceptanceIndex) Init(dag *blockdag.BlockDAG, databaseContext *dbaccess.DatabaseContext) error {
 	idx.dag = dag
-	return nil
+	idx.databaseContext = databaseContext
+	return idx.recover()
+}
+
+// recover attempts to insert any data that's missing from the
+// acceptance index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) recover() error {
+	return idx.dag.ForEachHash(func(hash daghash.Hash) error {
+		dbTx, err := idx.databaseContext.NewTx()
+		if err != nil {
+			return err
+		}
+		defer dbTx.RollbackUnlessClosed()
+
+		exists, err := dbaccess.HasAcceptanceData(dbTx, &hash)
+		if err != nil {
+			return err
+		}
+		if exists {
+			return nil
+		}
+		txAcceptanceData, err := idx.dag.TxsAcceptedByBlockHash(&hash)
+		if err != nil {
+			return err
+		}
+		err = idx.ConnectBlock(dbTx, &hash, txAcceptanceData)
+		if err != nil {
+			return err
+		}
+
+		return dbTx.Commit()
+	})
 }
 
 // ConnectBlock is invoked by the index manager when a new block has been
 // connected to the DAG.
 //
 // This is part of the Indexer interface.
-func (idx *AcceptanceIndex) ConnectBlock(dbTx database.Tx, _ *util.Block, blockID uint64, _ *blockdag.BlockDAG,
-	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, _ blockdag.MultiBlockTxsAcceptanceData) error {
-	return dbPutTxsAcceptanceData(dbTx, blockID, txsAcceptanceData)
-}
-
-// TxsAcceptanceData returns the acceptance data of all the transactions that
-// were accepted by the block with hash blockHash.
-func (idx *AcceptanceIndex) TxsAcceptanceData(blockHash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
-	var txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData
-	err := idx.db.View(func(dbTx database.Tx) error {
-		var err error
-		txsAcceptanceData, err = dbFetchTxsAcceptanceDataByHash(dbTx, blockHash)
-		return err
-	})
-	if err != nil {
-		return nil, err
-	}
-	return txsAcceptanceData, nil
-}
-
-// Recover is invoked when the indexer wasn't turned on for several blocks
-// and the indexer needs to close the gaps.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
-	for blockID := currentBlockID + 1; blockID <= lastKnownBlockID; blockID++ {
-		hash, err := blockdag.DBFetchBlockHashByID(dbTx, currentBlockID)
-		if err != nil {
-			return err
-		}
-		txAcceptanceData, err := idx.dag.TxsAcceptedByBlockHash(hash)
-		if err != nil {
-			return err
-		}
-		err = idx.ConnectBlock(dbTx, nil, blockID, nil, txAcceptanceData, nil)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func dbPutTxsAcceptanceData(dbTx database.Tx, blockID uint64,
+func (idx *AcceptanceIndex) ConnectBlock(dbContext *dbaccess.TxContext, blockHash *daghash.Hash,
 	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
 	serializedTxsAcceptanceData, err := serializeMultiBlockTxsAcceptanceData(txsAcceptanceData)
 	if err != nil {
 		return err
 	}
-
-	bucket := dbTx.Metadata().Bucket(acceptanceIndexKey)
-	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedTxsAcceptanceData)
+	return dbaccess.StoreAcceptanceData(dbContext, blockHash, serializedTxsAcceptanceData)
 }
 
-func dbFetchTxsAcceptanceDataByHash(dbTx database.Tx,
-	hash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
-
-	blockID, err := blockdag.DBFetchBlockIDByHash(dbTx, hash)
+// TxsAcceptanceData returns the acceptance data of all the transactions that
+// were accepted by the block with hash blockHash.
+func (idx *AcceptanceIndex) TxsAcceptanceData(blockHash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	serializedTxsAcceptanceData, err := dbaccess.FetchAcceptanceData(idx.databaseContext, blockHash)
 	if err != nil {
 		return nil, err
 	}
-
-	return dbFetchTxsAcceptanceDataByID(dbTx, blockID)
-}
-
-func dbFetchTxsAcceptanceDataByID(dbTx database.Tx,
-	blockID uint64) (blockdag.MultiBlockTxsAcceptanceData, error) {
-	serializedBlockID := blockdag.SerializeBlockID(blockID)
-	bucket := dbTx.Metadata().Bucket(acceptanceIndexKey)
-	serializedTxsAcceptanceData := bucket.Get(serializedBlockID)
-	if serializedTxsAcceptanceData == nil {
-		return nil, errors.Errorf("no entry in the accpetance index for block id %d", blockID)
-	}
-
 	return deserializeMultiBlockTxsAcceptanceData(serializedTxsAcceptanceData)
 }
 
 type serializableTxAcceptanceData struct {
-	MsgTx      wire.MsgTx
+	MsgTx      domainmessage.MsgTx
 	IsAccepted bool
 }
 

blockdag/indexers/acceptanceindex_test.go

@@ -1,13 +1,6 @@
 package indexers
 
 import (
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
@@ -15,24 +8,32 @@ import (
 	"reflect"
 	"syscall"
 	"testing"
+
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
 )
 
 func TestAcceptanceIndexSerializationAndDeserialization(t *testing.T) {
 	// Create test data
 	hash, _ := daghash.NewHashFromStr("1111111111111111111111111111111111111111111111111111111111111111")
-	txIn1 := &wire.TxIn{SignatureScript: []byte{1}, PreviousOutpoint: wire.Outpoint{Index: 1}, Sequence: 0}
-	txIn2 := &wire.TxIn{SignatureScript: []byte{2}, PreviousOutpoint: wire.Outpoint{Index: 2}, Sequence: 0}
-	txOut1 := &wire.TxOut{ScriptPubKey: []byte{1}, Value: 10}
-	txOut2 := &wire.TxOut{ScriptPubKey: []byte{2}, Value: 20}
+	txIn1 := &domainmessage.TxIn{SignatureScript: []byte{1}, PreviousOutpoint: domainmessage.Outpoint{Index: 1}, Sequence: 0}
+	txIn2 := &domainmessage.TxIn{SignatureScript: []byte{2}, PreviousOutpoint: domainmessage.Outpoint{Index: 2}, Sequence: 0}
+	txOut1 := &domainmessage.TxOut{ScriptPubKey: []byte{1}, Value: 10}
+	txOut2 := &domainmessage.TxOut{ScriptPubKey: []byte{2}, Value: 20}
 	blockTxsAcceptanceData := blockdag.BlockTxsAcceptanceData{
 		BlockHash: *hash,
 		TxAcceptanceData: []blockdag.TxAcceptanceData{
 			{
-				Tx:         util.NewTx(wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn1}, []*wire.TxOut{txOut1})),
+				Tx:         util.NewTx(domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn1}, []*domainmessage.TxOut{txOut1})),
 				IsAccepted: true,
 			},
 			{
-				Tx:         util.NewTx(wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn2}, []*wire.TxOut{txOut2})),
+				Tx:         util.NewTx(domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn2}, []*domainmessage.TxOut{txOut2})),
 				IsAccepted: false,
 			},
 		},
@@ -71,7 +72,7 @@ func TestAcceptanceIndexSerializationAndDeserialization(t *testing.T) {
 //   comparing dag1's last block acceptance data and dag3's last block acceptance
 //   data.
 func TestAcceptanceIndexRecover(t *testing.T) {
-	params := &dagconfig.SimNetParams
+	params := &dagconfig.SimnetParams
 	params.BlockCoinbaseMaturity = 0
 
 	testFiles := []string{
@@ -96,18 +97,18 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	}
 	defer os.RemoveAll(db1Path)
 
-	db1, err := database.Create("ffldb", db1Path, params.Net)
+	databaseContext1, err := dbaccess.New(db1Path)
 	if err != nil {
 		t.Fatalf("error creating db: %s", err)
 	}
 
 	db1Config := blockdag.Config{
-		IndexManager: db1IndexManager,
-		DAGParams:    params,
-		DB:           db1,
+		IndexManager:    db1IndexManager,
+		DAGParams:       params,
+		DatabaseContext: databaseContext1,
 	}
 
-	db1DAG, teardown, err := blockdag.DAGSetup("", db1Config)
+	db1DAG, teardown, err := blockdag.DAGSetup("", false, db1Config)
 	if err != nil {
 		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
 	}
@@ -116,11 +117,11 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	}
 
 	for i := 1; i < len(blocks)-2; i++ {
-		isOrphan, delay, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		isOrphan, isDelayed, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
 		if err != nil {
 			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
 		}
-		if delay != 0 {
+		if isDelayed {
 			t.Fatalf("ProcessBlock: block %d "+
 				"is too far in the future", i)
 		}
@@ -130,11 +131,6 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		}
 	}
 
-	err = db1.FlushCache()
-	if err != nil {
-		t.Fatalf("Error flushing database to disk: %s", err)
-	}
-
 	db2Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover2")
 	if err != nil {
 		t.Fatalf("Error creating temporary directory: %s", err)
@@ -147,11 +143,11 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	}
 
 	for i := len(blocks) - 2; i < len(blocks); i++ {
-		isOrphan, delay, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		isOrphan, isDelayed, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
 		if err != nil {
 			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
 		}
-		if delay != 0 {
+		if isDelayed {
 			t.Fatalf("ProcessBlock: block %d "+
 				"is too far in the future", i)
 		}
@@ -166,17 +162,21 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		t.Fatalf("Error fetching acceptance data: %s", err)
 	}
 
-	db2, err := database.Open("ffldb", db2Path, params.Net)
+	err = databaseContext1.Close()
 	if err != nil {
-		t.Fatalf("Error opening database: %s", err)
+		t.Fatalf("Error closing the database: %s", err)
+	}
+	databaseContext2, err := dbaccess.New(db2Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
 	}
 
 	db2Config := blockdag.Config{
-		DAGParams: params,
-		DB:        db2,
+		DAGParams:       params,
+		DatabaseContext: databaseContext2,
 	}
 
-	db2DAG, teardown, err := blockdag.DAGSetup("", db2Config)
+	db2DAG, teardown, err := blockdag.DAGSetup("", false, db2Config)
 	if err != nil {
 		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
 	}
@@ -185,11 +185,11 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	}
 
 	for i := len(blocks) - 2; i < len(blocks); i++ {
-		isOrphan, delay, err := db2DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		isOrphan, isDelayed, err := db2DAG.ProcessBlock(blocks[i], blockdag.BFNone)
 		if err != nil {
 			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
 		}
-		if delay != 0 {
+		if isDelayed {
 			t.Fatalf("ProcessBlock: block %d "+
 				"is too far in the future", i)
 		}
@@ -199,10 +199,6 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		}
 	}
 
-	err = db2.FlushCache()
-	if err != nil {
-		t.Fatalf("Error flushing database to disk: %s", err)
-	}
 	db3Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover3")
 	if err != nil {
 		t.Fatalf("Error creating temporary directory: %s", err)
@@ -213,20 +209,24 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		t.Fatalf("copyDirectory: %s", err)
 	}
 
-	db3, err := database.Open("ffldb", db3Path, params.Net)
+	err = databaseContext2.Close()
 	if err != nil {
-		t.Fatalf("Error opening database: %s", err)
+		t.Fatalf("Error closing the database: %s", err)
+	}
+	databaseContext3, err := dbaccess.New(db3Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
 	}
 
 	db3AcceptanceIndex := NewAcceptanceIndex()
 	db3IndexManager := NewManager([]Indexer{db3AcceptanceIndex})
 	db3Config := blockdag.Config{
-		IndexManager: db3IndexManager,
-		DAGParams:    params,
-		DB:           db3,
+		IndexManager:    db3IndexManager,
+		DAGParams:       params,
+		DatabaseContext: databaseContext3,
 	}
 
-	_, teardown, err = blockdag.DAGSetup("", db3Config)
+	_, teardown, err = blockdag.DAGSetup("", false, db3Config)
 	if err != nil {
 		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
 	}
@@ -298,16 +298,16 @@ func copyDirectory(scrDir, dest string) error {
 // This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
 func copyFile(srcFile, dstFile string) error {
 	out, err := os.Create(dstFile)
-	defer out.Close()
 	if err != nil {
 		return err
 	}
+	defer out.Close()
 
 	in, err := os.Open(srcFile)
-	defer in.Close()
 	if err != nil {
 		return err
 	}
+	defer in.Close()
 
 	_, err = io.Copy(out, in)
 	if err != nil {

blockdag/indexers/addrindex.go

@@ -1,902 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package indexers
-
-import (
-	"fmt"
-	"github.com/pkg/errors"
-	"sync"
-
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/txscript"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-const (
-	// addrIndexName is the human-readable name for the index.
-	addrIndexName = "address index"
-
-	// level0MaxEntries is the maximum number of transactions that are
-	// stored in level 0 of an address index entry. Subsequent levels store
-	// 2^n * level0MaxEntries entries, or in words, double the maximum of
-	// the previous level.
-	level0MaxEntries = 8
-
-	// addrKeySize is the number of bytes an address key consumes in the
-	// index. It consists of 1 byte address type + 20 bytes hash160.
-	addrKeySize = 1 + 20
-
-	// levelKeySize is the number of bytes a level key in the address index
-	// consumes. It consists of the address key + 1 byte for the level.
-	levelKeySize = addrKeySize + 1
-
-	// levelOffset is the offset in the level key which identifes the level.
-	levelOffset = levelKeySize - 1
-
-	// addrKeyTypePubKeyHash is the address type in an address key which
-	// represents both a pay-to-pubkey-hash and a pay-to-pubkey address.
-	// This is done because both are identical for the purposes of the
-	// address index.
-	addrKeyTypePubKeyHash = 0
-
-	// addrKeyTypeScriptHash is the address type in an address key which
-	// represents a pay-to-script-hash address. This is necessary because
-	// the hash of a pubkey address might be the same as that of a script
-	// hash.
-	addrKeyTypeScriptHash = 1
-
-	// Size of a transaction entry. It consists of 8 bytes block id + 4
-	// bytes offset + 4 bytes length.
-	txEntrySize = 8 + 4 + 4
-)
-
-var (
-	// addrIndexKey is the key of the address index and the db bucket used
-	// to house it.
-	addrIndexKey = []byte("txbyaddridx")
-
-	// errUnsupportedAddressType is an error that is used to signal an
-	// unsupported address type has been used.
-	errUnsupportedAddressType = errors.New("address type is not supported " +
-		"by the address index")
-)
-
-// -----------------------------------------------------------------------------
-// The address index maps addresses referenced in the blockDAG to a list of
-// all the transactions involving that address. Transactions are stored
-// according to their order of appearance in the blockDAG. That is to say
-// first by block height and then by offset inside the block. It is also
-// important to note that this implementation requires the transaction index
-// since it is needed in order to catch up old blocks due to the fact the spent
-// outputs will already be pruned from the utxo set.
-//
-// The approach used to store the index is similar to a log-structured merge
-// tree (LSM tree) and is thus similar to how leveldb works internally.
-//
-// Every address consists of one or more entries identified by a level starting
-// from 0 where each level holds a maximum number of entries such that each
-// subsequent level holds double the maximum of the previous one. In equation
-// form, the number of entries each level holds is 2^n * firstLevelMaxSize.
-//
-// New transactions are appended to level 0 until it becomes full at which point
-// the entire level 0 entry is appended to the level 1 entry and level 0 is
-// cleared. This process continues until level 1 becomes full at which point it
-// will be appended to level 2 and cleared and so on.
-//
-// The result of this is the lower levels contain newer transactions and the
-// transactions within each level are ordered from oldest to newest.
-//
-// The intent of this approach is to provide a balance between space efficiency
-// and indexing cost. Storing one entry per transaction would have the lowest
-// indexing cost, but would waste a lot of space because the same address hash
-// would be duplicated for every transaction key. On the other hand, storing a
-// single entry with all transactions would be the most space efficient, but
-// would cause indexing cost to grow quadratically with the number of
-// transactions involving the same address. The approach used here provides
-// logarithmic insertion and retrieval.
-//
-// The serialized key format is:
-//
-//   <addr type><addr hash><level>
-//
-//   Field           Type      Size
-//   addr type       uint8     1 byte
-//   addr hash       hash160   20 bytes
-//   level           uint8     1 byte
-//   -----
-//   Total: 22 bytes
-//
-// The serialized value format is:
-//
-//   [<block id><start offset><tx length>,...]
-//
-//   Field           Type      Size
-//   block id        uint64    8 bytes
-//   start offset    uint32    4 bytes
-//   tx length       uint32    4 bytes
-//   -----
-//   Total: 16 bytes per indexed tx
-// -----------------------------------------------------------------------------
-
-// fetchBlockHashFunc defines a callback function to use in order to convert a
-// serialized block ID to an associated block hash.
-type fetchBlockHashFunc func(serializedID []byte) (*daghash.Hash, error)
-
-// serializeAddrIndexEntry serializes the provided block id and transaction
-// location according to the format described in detail above.
-func serializeAddrIndexEntry(blockID uint64, txLoc wire.TxLoc) []byte {
-	// Serialize the entry.
-	serialized := make([]byte, 16)
-	byteOrder.PutUint64(serialized, blockID)
-	byteOrder.PutUint32(serialized[8:], uint32(txLoc.TxStart))
-	byteOrder.PutUint32(serialized[12:], uint32(txLoc.TxLen))
-	return serialized
-}
-
-// deserializeAddrIndexEntry decodes the passed serialized byte slice into the
-// provided region struct according to the format described in detail above and
-// uses the passed block hash fetching function in order to conver the block ID
-// to the associated block hash.
-func deserializeAddrIndexEntry(serialized []byte, region *database.BlockRegion, fetchBlockHash fetchBlockHashFunc) error {
-	// Ensure there are enough bytes to decode.
-	if len(serialized) < txEntrySize {
-		return errDeserialize("unexpected end of data")
-	}
-
-	hash, err := fetchBlockHash(serialized[0:8])
-	if err != nil {
-		return err
-	}
-	region.Hash = hash
-	region.Offset = byteOrder.Uint32(serialized[8:12])
-	region.Len = byteOrder.Uint32(serialized[12:16])
-	return nil
-}
-
-// keyForLevel returns the key for a specific address and level in the address
-// index entry.
-func keyForLevel(addrKey [addrKeySize]byte, level uint8) [levelKeySize]byte {
-	var key [levelKeySize]byte
-	copy(key[:], addrKey[:])
-	key[levelOffset] = level
-	return key
-}
-
-// dbPutAddrIndexEntry updates the address index to include the provided entry
-// according to the level-based scheme described in detail above.
-func dbPutAddrIndexEntry(bucket internalBucket, addrKey [addrKeySize]byte, blockID uint64, txLoc wire.TxLoc) error {
-	// Start with level 0 and its initial max number of entries.
-	curLevel := uint8(0)
-	maxLevelBytes := level0MaxEntries * txEntrySize
-
-	// Simply append the new entry to level 0 and return now when it will
-	// fit. This is the most common path.
-	newData := serializeAddrIndexEntry(blockID, txLoc)
-	level0Key := keyForLevel(addrKey, 0)
-	level0Data := bucket.Get(level0Key[:])
-	if len(level0Data)+len(newData) <= maxLevelBytes {
-		mergedData := newData
-		if len(level0Data) > 0 {
-			mergedData = make([]byte, len(level0Data)+len(newData))
-			copy(mergedData, level0Data)
-			copy(mergedData[len(level0Data):], newData)
-		}
-		return bucket.Put(level0Key[:], mergedData)
-	}
-
-	// At this point, level 0 is full, so merge each level into higher
-	// levels as many times as needed to free up level 0.
-	prevLevelData := level0Data
-	for {
-		// Each new level holds twice as much as the previous one.
-		curLevel++
-		maxLevelBytes *= 2
-
-		// Move to the next level as long as the current level is full.
-		curLevelKey := keyForLevel(addrKey, curLevel)
-		curLevelData := bucket.Get(curLevelKey[:])
-		if len(curLevelData) == maxLevelBytes {
-			prevLevelData = curLevelData
-			continue
-		}
-
-		// The current level has room for the data in the previous one,
-		// so merge the data from previous level into it.
-		mergedData := prevLevelData
-		if len(curLevelData) > 0 {
-			mergedData = make([]byte, len(curLevelData)+
-				len(prevLevelData))
-			copy(mergedData, curLevelData)
-			copy(mergedData[len(curLevelData):], prevLevelData)
-		}
-		err := bucket.Put(curLevelKey[:], mergedData)
-		if err != nil {
-			return err
-		}
-
-		// Move all of the levels before the previous one up a level.
-		for mergeLevel := curLevel - 1; mergeLevel > 0; mergeLevel-- {
-			mergeLevelKey := keyForLevel(addrKey, mergeLevel)
-			prevLevelKey := keyForLevel(addrKey, mergeLevel-1)
-			prevData := bucket.Get(prevLevelKey[:])
-			err := bucket.Put(mergeLevelKey[:], prevData)
-			if err != nil {
-				return err
-			}
-		}
-		break
-	}
-
-	// Finally, insert the new entry into level 0 now that it is empty.
-	return bucket.Put(level0Key[:], newData)
-}
-
-// dbFetchAddrIndexEntries returns block regions for transactions referenced by
-// the given address key and the number of entries skipped since it could have
-// been less in the case where there are less total entries than the requested
-// number of entries to skip.
-func dbFetchAddrIndexEntries(bucket internalBucket, addrKey [addrKeySize]byte, numToSkip, numRequested uint32, reverse bool, fetchBlockHash fetchBlockHashFunc) ([]database.BlockRegion, uint32, error) {
-	// When the reverse flag is not set, all levels need to be fetched
-	// because numToSkip and numRequested are counted from the oldest
-	// transactions (highest level) and thus the total count is needed.
-	// However, when the reverse flag is set, only enough records to satisfy
-	// the requested amount are needed.
-	var level uint8
-	var serialized []byte
-	for !reverse || len(serialized) < int(numToSkip+numRequested)*txEntrySize {
-		curLevelKey := keyForLevel(addrKey, level)
-		levelData := bucket.Get(curLevelKey[:])
-		if levelData == nil {
-			// Stop when there are no more levels.
-			break
-		}
-
-		// Higher levels contain older transactions, so prepend them.
-		prepended := make([]byte, len(serialized)+len(levelData))
-		copy(prepended, levelData)
-		copy(prepended[len(levelData):], serialized)
-		serialized = prepended
-		level++
-	}
-
-	// When the requested number of entries to skip is larger than the
-	// number available, skip them all and return now with the actual number
-	// skipped.
-	numEntries := uint32(len(serialized) / txEntrySize)
-	if numToSkip >= numEntries {
-		return nil, numEntries, nil
-	}
-
-	// Nothing more to do when there are no requested entries.
-	if numRequested == 0 {
-		return nil, numToSkip, nil
-	}
-
-	// Limit the number to load based on the number of available entries,
-	// the number to skip, and the number requested.
-	numToLoad := numEntries - numToSkip
-	if numToLoad > numRequested {
-		numToLoad = numRequested
-	}
-
-	// Start the offset after all skipped entries and load the calculated
-	// number.
-	results := make([]database.BlockRegion, numToLoad)
-	for i := uint32(0); i < numToLoad; i++ {
-		// Calculate the read offset according to the reverse flag.
-		var offset uint32
-		if reverse {
-			offset = (numEntries - numToSkip - i - 1) * txEntrySize
-		} else {
-			offset = (numToSkip + i) * txEntrySize
-		}
-
-		// Deserialize and populate the result.
-		err := deserializeAddrIndexEntry(serialized[offset:],
-			&results[i], fetchBlockHash)
-		if err != nil {
-			// Ensure any deserialization errors are returned as
-			// database corruption errors.
-			if isDeserializeErr(err) {
-				err = database.Error{
-					ErrorCode: database.ErrCorruption,
-					Description: fmt.Sprintf("failed to "+
-						"deserialized address index "+
-						"for key %x: %s", addrKey, err),
-				}
-			}
-
-			return nil, 0, err
-		}
-	}
-
-	return results, numToSkip, nil
-}
-
-// minEntriesToReachLevel returns the minimum number of entries that are
-// required to reach the given address index level.
-func minEntriesToReachLevel(level uint8) int {
-	maxEntriesForLevel := level0MaxEntries
-	minRequired := 1
-	for l := uint8(1); l <= level; l++ {
-		minRequired += maxEntriesForLevel
-		maxEntriesForLevel *= 2
-	}
-	return minRequired
-}
-
-// maxEntriesForLevel returns the maximum number of entries allowed for the
-// given address index level.
-func maxEntriesForLevel(level uint8) int {
-	numEntries := level0MaxEntries
-	for l := level; l > 0; l-- {
-		numEntries *= 2
-	}
-	return numEntries
-}
-
-// dbRemoveAddrIndexEntries removes the specified number of entries from from
-// the address index for the provided key. An assertion error will be returned
-// if the count exceeds the total number of entries in the index.
-func dbRemoveAddrIndexEntries(bucket internalBucket, addrKey [addrKeySize]byte, count int) error {
-	// Nothing to do if no entries are being deleted.
-	if count <= 0 {
-		return nil
-	}
-
-	// Make use of a local map to track pending updates and define a closure
-	// to apply it to the database. This is done in order to reduce the
-	// number of database reads and because there is more than one exit
-	// path that needs to apply the updates.
-	pendingUpdates := make(map[uint8][]byte)
-	applyPending := func() error {
-		for level, data := range pendingUpdates {
-			curLevelKey := keyForLevel(addrKey, level)
-			if len(data) == 0 {
-				err := bucket.Delete(curLevelKey[:])
-				if err != nil {
-					return err
-				}
-				continue
-			}
-			err := bucket.Put(curLevelKey[:], data)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	// Loop forwards through the levels while removing entries until the
-	// specified number has been removed. This will potentially result in
-	// entirely empty lower levels which will be backfilled below.
-	var highestLoadedLevel uint8
-	numRemaining := count
-	for level := uint8(0); numRemaining > 0; level++ {
-		// Load the data for the level from the database.
-		curLevelKey := keyForLevel(addrKey, level)
-		curLevelData := bucket.Get(curLevelKey[:])
-		if len(curLevelData) == 0 && numRemaining > 0 {
-			return AssertError(fmt.Sprintf("dbRemoveAddrIndexEntries "+
-				"not enough entries for address key %x to "+
-				"delete %d entries", addrKey, count))
-		}
-		pendingUpdates[level] = curLevelData
-		highestLoadedLevel = level
-
-		// Delete the entire level as needed.
-		numEntries := len(curLevelData) / txEntrySize
-		if numRemaining >= numEntries {
-			pendingUpdates[level] = nil
-			numRemaining -= numEntries
-			continue
-		}
-
-		// Remove remaining entries to delete from the level.
-		offsetEnd := len(curLevelData) - (numRemaining * txEntrySize)
-		pendingUpdates[level] = curLevelData[:offsetEnd]
-		break
-	}
-
-	// When all elements in level 0 were not removed there is nothing left
-	// to do other than updating the database.
-	if len(pendingUpdates[0]) != 0 {
-		return applyPending()
-	}
-
-	// At this point there are one or more empty levels before the current
-	// level which need to be backfilled and the current level might have
-	// had some entries deleted from it as well. Since all levels after
-	// level 0 are required to either be empty, half full, or completely
-	// full, the current level must be adjusted accordingly by backfilling
-	// each previous levels in a way which satisfies the requirements. Any
-	// entries that are left are assigned to level 0 after the loop as they
-	// are guaranteed to fit by the logic in the loop. In other words, this
-	// effectively squashes all remaining entries in the current level into
-	// the lowest possible levels while following the level rules.
-	//
-	// Note that the level after the current level might also have entries
-	// and gaps are not allowed, so this also keeps track of the lowest
-	// empty level so the code below knows how far to backfill in case it is
-	// required.
-	lowestEmptyLevel := uint8(255)
-	curLevelData := pendingUpdates[highestLoadedLevel]
-	curLevelMaxEntries := maxEntriesForLevel(highestLoadedLevel)
-	for level := highestLoadedLevel; level > 0; level-- {
-		// When there are not enough entries left in the current level
-		// for the number that would be required to reach it, clear the
-		// the current level which effectively moves them all up to the
-		// previous level on the next iteration. Otherwise, there are
-		// are sufficient entries, so update the current level to
-		// contain as many entries as possible while still leaving
-		// enough remaining entries required to reach the level.
-		numEntries := len(curLevelData) / txEntrySize
-		prevLevelMaxEntries := curLevelMaxEntries / 2
-		minPrevRequired := minEntriesToReachLevel(level - 1)
-		if numEntries < prevLevelMaxEntries+minPrevRequired {
-			lowestEmptyLevel = level
-			pendingUpdates[level] = nil
-		} else {
-			// This level can only be completely full or half full,
-			// so choose the appropriate offset to ensure enough
-			// entries remain to reach the level.
-			var offset int
-			if numEntries-curLevelMaxEntries >= minPrevRequired {
-				offset = curLevelMaxEntries * txEntrySize
-			} else {
-				offset = prevLevelMaxEntries * txEntrySize
-			}
-			pendingUpdates[level] = curLevelData[:offset]
-			curLevelData = curLevelData[offset:]
-		}
-
-		curLevelMaxEntries = prevLevelMaxEntries
-	}
-	pendingUpdates[0] = curLevelData
-	if len(curLevelData) == 0 {
-		lowestEmptyLevel = 0
-	}
-
-	// When the highest loaded level is empty, it's possible the level after
-	// it still has data and thus that data needs to be backfilled as well.
-	for len(pendingUpdates[highestLoadedLevel]) == 0 {
-		// When the next level is empty too, the is no data left to
-		// continue backfilling, so there is nothing left to do.
-		// Otherwise, populate the pending updates map with the newly
-		// loaded data and update the highest loaded level accordingly.
-		level := highestLoadedLevel + 1
-		curLevelKey := keyForLevel(addrKey, level)
-		levelData := bucket.Get(curLevelKey[:])
-		if len(levelData) == 0 {
-			break
-		}
-		pendingUpdates[level] = levelData
-		highestLoadedLevel = level
-
-		// At this point the highest level is not empty, but it might
-		// be half full. When that is the case, move it up a level to
-		// simplify the code below which backfills all lower levels that
-		// are still empty. This also means the current level will be
-		// empty, so the loop will perform another another iteration to
-		// potentially backfill this level with data from the next one.
-		curLevelMaxEntries := maxEntriesForLevel(level)
-		if len(levelData)/txEntrySize != curLevelMaxEntries {
-			pendingUpdates[level] = nil
-			pendingUpdates[level-1] = levelData
-			level--
-			curLevelMaxEntries /= 2
-		}
-
-		// Backfill all lower levels that are still empty by iteratively
-		// halfing the data until the lowest empty level is filled.
-		for level > lowestEmptyLevel {
-			offset := (curLevelMaxEntries / 2) * txEntrySize
-			pendingUpdates[level] = levelData[:offset]
-			levelData = levelData[offset:]
-			pendingUpdates[level-1] = levelData
-			level--
-			curLevelMaxEntries /= 2
-		}
-
-		// The lowest possible empty level is now the highest loaded
-		// level.
-		lowestEmptyLevel = highestLoadedLevel
-	}
-
-	// Apply the pending updates.
-	return applyPending()
-}
-
-// addrToKey converts known address types to an addrindex key. An error is
-// returned for unsupported types.
-func addrToKey(addr util.Address) ([addrKeySize]byte, error) {
-	switch addr := addr.(type) {
-	case *util.AddressPubKeyHash:
-		var result [addrKeySize]byte
-		result[0] = addrKeyTypePubKeyHash
-		copy(result[1:], addr.Hash160()[:])
-		return result, nil
-
-	case *util.AddressScriptHash:
-		var result [addrKeySize]byte
-		result[0] = addrKeyTypeScriptHash
-		copy(result[1:], addr.Hash160()[:])
-		return result, nil
-	}
-
-	return [addrKeySize]byte{}, errUnsupportedAddressType
-}
-
-// AddrIndex implements a transaction by address index. That is to say, it
-// supports querying all transactions that reference a given address because
-// they are either crediting or debiting the address. The returned transactions
-// are ordered according to their order of appearance in the blockDAG. In
-// other words, first by block height and then by offset inside the block.
-//
-// In addition, support is provided for a memory-only index of unconfirmed
-// transactions such as those which are kept in the memory pool before inclusion
-// in a block.
-type AddrIndex struct {
-	// The following fields are set when the instance is created and can't
-	// be changed afterwards, so there is no need to protect them with a
-	// separate mutex.
-	db        database.DB
-	dagParams *dagconfig.Params
-
-	// The following fields are used to quickly link transactions and
-	// addresses that have not been included into a block yet when an
-	// address index is being maintained. The are protected by the
-	// unconfirmedLock field.
-	//
-	// The txnsByAddr field is used to keep an index of all transactions
-	// which either create an output to a given address or spend from a
-	// previous output to it keyed by the address.
-	//
-	// The addrsByTx field is essentially the reverse and is used to
-	// keep an index of all addresses which a given transaction involves.
-	// This allows fairly efficient updates when transactions are removed
-	// once they are included into a block.
-	unconfirmedLock sync.RWMutex
-	txnsByAddr      map[[addrKeySize]byte]map[daghash.TxID]*util.Tx
-	addrsByTx       map[daghash.TxID]map[[addrKeySize]byte]struct{}
-}
-
-// Ensure the AddrIndex type implements the Indexer interface.
-var _ Indexer = (*AddrIndex)(nil)
-
-// Ensure the AddrIndex type implements the NeedsInputser interface.
-var _ NeedsInputser = (*AddrIndex)(nil)
-
-// NeedsInputs signals that the index requires the referenced inputs in order
-// to properly create the index.
-//
-// This implements the NeedsInputser interface.
-func (idx *AddrIndex) NeedsInputs() bool {
-	return true
-}
-
-// Init is only provided to satisfy the Indexer interface as there is nothing to
-// initialize for this index.
-//
-// This is part of the Indexer interface.
-func (idx *AddrIndex) Init(db database.DB, _ *blockdag.BlockDAG) error {
-	idx.db = db
-	return nil
-}
-
-// Key returns the database key to use for the index as a byte slice.
-//
-// This is part of the Indexer interface.
-func (idx *AddrIndex) Key() []byte {
-	return addrIndexKey
-}
-
-// Name returns the human-readable name of the index.
-//
-// This is part of the Indexer interface.
-func (idx *AddrIndex) Name() string {
-	return addrIndexName
-}
-
-// Create is invoked when the indexer manager determines the index needs
-// to be created for the first time. It creates the bucket for the address
-// index.
-//
-// This is part of the Indexer interface.
-func (idx *AddrIndex) Create(dbTx database.Tx) error {
-	_, err := dbTx.Metadata().CreateBucket(addrIndexKey)
-	return err
-}
-
-// writeIndexData represents the address index data to be written for one block.
-// It consists of the address mapped to an ordered list of the transactions
-// that involve the address in block. It is ordered so the transactions can be
-// stored in the order they appear in the block.
-type writeIndexData map[[addrKeySize]byte][]int
-
-// indexScriptPubKey extracts all standard addresses from the passed public key
-// script and maps each of them to the associated transaction using the passed
-// map.
-func (idx *AddrIndex) indexScriptPubKey(data writeIndexData, scriptPubKey []byte, txIdx int) {
-	// Nothing to index if the script is non-standard or otherwise doesn't
-	// contain any addresses.
-	_, addr, err := txscript.ExtractScriptPubKeyAddress(scriptPubKey,
-		idx.dagParams)
-	if err != nil || addr == nil {
-		return
-	}
-
-	addrKey, err := addrToKey(addr)
-	if err != nil {
-		// Ignore unsupported address types.
-		return
-	}
-
-	// Avoid inserting the transaction more than once. Since the
-	// transactions are indexed serially any duplicates will be
-	// indexed in a row, so checking the most recent entry for the
-	// address is enough to detect duplicates.
-	indexedTxns := data[addrKey]
-	numTxns := len(indexedTxns)
-	if numTxns > 0 && indexedTxns[numTxns-1] == txIdx {
-		return
-	}
-	indexedTxns = append(indexedTxns, txIdx)
-	data[addrKey] = indexedTxns
-}
-
-// indexBlock extract all of the standard addresses from all of the transactions
-// in the passed block and maps each of them to the associated transaction using
-// the passed map.
-func (idx *AddrIndex) indexBlock(data writeIndexData, block *util.Block, dag *blockdag.BlockDAG) {
-	for txIdx, tx := range block.Transactions() {
-		// Coinbases do not reference any inputs. Since the block is
-		// required to have already gone through full validation, it has
-		// already been proven on the first transaction in the block is
-		// a coinbase.
-		if txIdx > util.CoinbaseTransactionIndex {
-			for _, txIn := range tx.MsgTx().TxIn {
-				// The UTXO should always have the input since
-				// the index contract requires it, however, be
-				// safe and simply ignore any missing entries.
-				entry, ok := dag.GetUTXOEntry(txIn.PreviousOutpoint)
-				if !ok {
-					continue
-				}
-
-				idx.indexScriptPubKey(data, entry.ScriptPubKey(), txIdx)
-			}
-		}
-
-		for _, txOut := range tx.MsgTx().TxOut {
-			idx.indexScriptPubKey(data, txOut.ScriptPubKey, txIdx)
-		}
-	}
-}
-
-// ConnectBlock is invoked by the index manager when a new block has been
-// connected to the DAG. This indexer adds a mapping for each address
-// the transactions in the block involve.
-//
-// This is part of the Indexer interface.
-func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
-	_ blockdag.MultiBlockTxsAcceptanceData, _ blockdag.MultiBlockTxsAcceptanceData) error {
-
-	// The offset and length of the transactions within the serialized
-	// block.
-	txLocs, err := block.TxLoc()
-	if err != nil {
-		return err
-	}
-
-	// Build all of the address to transaction mappings in a local map.
-	addrsToTxns := make(writeIndexData)
-	idx.indexBlock(addrsToTxns, block, dag)
-
-	// Add all of the index entries for each address.
-	addrIdxBucket := dbTx.Metadata().Bucket(addrIndexKey)
-	for addrKey, txIdxs := range addrsToTxns {
-		for _, txIdx := range txIdxs {
-			err := dbPutAddrIndexEntry(addrIdxBucket, addrKey,
-				blockID, txLocs[txIdx])
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// TxRegionsForAddress returns a slice of block regions which identify each
-// transaction that involves the passed address according to the specified
-// number to skip, number requested, and whether or not the results should be
-// reversed. It also returns the number actually skipped since it could be less
-// in the case where there are not enough entries.
-//
-// NOTE: These results only include transactions confirmed in blocks. See the
-// UnconfirmedTxnsForAddress method for obtaining unconfirmed transactions
-// that involve a given address.
-//
-// This function is safe for concurrent access.
-func (idx *AddrIndex) TxRegionsForAddress(dbTx database.Tx, addr util.Address, numToSkip, numRequested uint32, reverse bool) ([]database.BlockRegion, uint32, error) {
-	addrKey, err := addrToKey(addr)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	var regions []database.BlockRegion
-	var skipped uint32
-	err = idx.db.View(func(dbTx database.Tx) error {
-		// Create closure to lookup the block hash given the ID using
-		// the database transaction.
-		fetchBlockHash := func(id []byte) (*daghash.Hash, error) {
-			// Deserialize and populate the result.
-			return blockdag.DBFetchBlockHashBySerializedID(dbTx, id)
-		}
-
-		var err error
-		addrIdxBucket := dbTx.Metadata().Bucket(addrIndexKey)
-		regions, skipped, err = dbFetchAddrIndexEntries(addrIdxBucket,
-			addrKey, numToSkip, numRequested, reverse,
-			fetchBlockHash)
-		return err
-	})
-
-	return regions, skipped, err
-}
-
-// indexUnconfirmedAddresses modifies the unconfirmed (memory-only) address
-// index to include mappings for the addresses encoded by the passed public key
-// script to the transaction.
-//
-// This function is safe for concurrent access.
-func (idx *AddrIndex) indexUnconfirmedAddresses(scriptPubKey []byte, tx *util.Tx) {
-	// The error is ignored here since the only reason it can fail is if the
-	// script fails to parse and it was already validated before being
-	// admitted to the mempool.
-	_, addr, _ := txscript.ExtractScriptPubKeyAddress(scriptPubKey,
-		idx.dagParams)
-	// Ignore unsupported address types.
-	addrKey, err := addrToKey(addr)
-	if err != nil {
-		return
-	}
-
-	// Add a mapping from the address to the transaction.
-	idx.unconfirmedLock.Lock()
-	addrIndexEntry := idx.txnsByAddr[addrKey]
-	if addrIndexEntry == nil {
-		addrIndexEntry = make(map[daghash.TxID]*util.Tx)
-		idx.txnsByAddr[addrKey] = addrIndexEntry
-	}
-	addrIndexEntry[*tx.ID()] = tx
-
-	// Add a mapping from the transaction to the address.
-	addrsByTxEntry := idx.addrsByTx[*tx.ID()]
-	if addrsByTxEntry == nil {
-		addrsByTxEntry = make(map[[addrKeySize]byte]struct{})
-		idx.addrsByTx[*tx.ID()] = addrsByTxEntry
-	}
-	addrsByTxEntry[addrKey] = struct{}{}
-	idx.unconfirmedLock.Unlock()
-}
-
-// AddUnconfirmedTx adds all addresses related to the transaction to the
-// unconfirmed (memory-only) address index.
-//
-// NOTE: This transaction MUST have already been validated by the memory pool
-// before calling this function with it and have all of the inputs available in
-// the provided utxo view. Failure to do so could result in some or all
-// addresses not being indexed.
-//
-// This function is safe for concurrent access.
-func (idx *AddrIndex) AddUnconfirmedTx(tx *util.Tx, utxoSet blockdag.UTXOSet) {
-	// Index addresses of all referenced previous transaction outputs.
-	//
-	// The existence checks are elided since this is only called after the
-	// transaction has already been validated and thus all inputs are
-	// already known to exist.
-	for _, txIn := range tx.MsgTx().TxIn {
-		entry, ok := utxoSet.Get(txIn.PreviousOutpoint)
-		if !ok {
-			// Ignore missing entries. This should never happen
-			// in practice since the function comments specifically
-			// call out all inputs must be available.
-			continue
-		}
-		idx.indexUnconfirmedAddresses(entry.ScriptPubKey(), tx)
-	}
-
-	// Index addresses of all created outputs.
-	for _, txOut := range tx.MsgTx().TxOut {
-		idx.indexUnconfirmedAddresses(txOut.ScriptPubKey, tx)
-	}
-}
-
-// RemoveUnconfirmedTx removes the passed transaction from the unconfirmed
-// (memory-only) address index.
-//
-// This function is safe for concurrent access.
-func (idx *AddrIndex) RemoveUnconfirmedTx(txID *daghash.TxID) {
-	idx.unconfirmedLock.Lock()
-	defer idx.unconfirmedLock.Unlock()
-
-	// Remove all address references to the transaction from the address
-	// index and remove the entry for the address altogether if it no longer
-	// references any transactions.
-	for addrKey := range idx.addrsByTx[*txID] {
-		delete(idx.txnsByAddr[addrKey], *txID)
-		if len(idx.txnsByAddr[addrKey]) == 0 {
-			delete(idx.txnsByAddr, addrKey)
-		}
-	}
-
-	// Remove the entry from the transaction to address lookup map as well.
-	delete(idx.addrsByTx, *txID)
-}
-
-// UnconfirmedTxnsForAddress returns all transactions currently in the
-// unconfirmed (memory-only) address index that involve the passed address.
-// Unsupported address types are ignored and will result in no results.
-//
-// This function is safe for concurrent access.
-func (idx *AddrIndex) UnconfirmedTxnsForAddress(addr util.Address) []*util.Tx {
-	// Ignore unsupported address types.
-	addrKey, err := addrToKey(addr)
-	if err != nil {
-		return nil
-	}
-
-	// Protect concurrent access.
-	idx.unconfirmedLock.RLock()
-	defer idx.unconfirmedLock.RUnlock()
-
-	// Return a new slice with the results if there are any. This ensures
-	// safe concurrency.
-	if txns, exists := idx.txnsByAddr[addrKey]; exists {
-		addressTxns := make([]*util.Tx, 0, len(txns))
-		for _, tx := range txns {
-			addressTxns = append(addressTxns, tx)
-		}
-		return addressTxns
-	}
-
-	return nil
-}
-
-// Recover is invoked when the indexer wasn't turned on for several blocks
-// and the indexer needs to close the gaps.
-//
-// This is part of the Indexer interface.
-func (idx *AddrIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
-	return errors.Errorf("addrindex was turned off for %d blocks and can't be recovered."+
-		" To resume working drop the addrindex with --dropaddrindex", lastKnownBlockID-currentBlockID)
-}
-
-// NewAddrIndex returns a new instance of an indexer that is used to create a
-// mapping of all addresses in the blockDAG to the respective transactions
-// that involve them.
-//
-// It implements the Indexer interface which plugs into the IndexManager that in
-// turn is used by the blockDAG package. This allows the index to be
-// seamlessly maintained along with the DAG.
-func NewAddrIndex(dagParams *dagconfig.Params) *AddrIndex {
-	return &AddrIndex{
-		dagParams:  dagParams,
-		txnsByAddr: make(map[[addrKeySize]byte]map[daghash.TxID]*util.Tx),
-		addrsByTx:  make(map[daghash.TxID]map[[addrKeySize]byte]struct{}),
-	}
-}
-
-// DropAddrIndex drops the address index from the provided database if it
-// exists.
-func DropAddrIndex(db database.DB, interrupt <-chan struct{}) error {
-	return dropIndex(db, addrIndexKey, addrIndexName, interrupt)
-}

blockdag/indexers/addrindex_test.go

@@ -1,277 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package indexers
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/pkg/errors"
-	"testing"
-
-	"github.com/kaspanet/kaspad/wire"
-)
-
-// addrIndexBucket provides a mock address index database bucket by implementing
-// the internalBucket interface.
-type addrIndexBucket struct {
-	levels map[[levelKeySize]byte][]byte
-}
-
-// Clone returns a deep copy of the mock address index bucket.
-func (b *addrIndexBucket) Clone() *addrIndexBucket {
-	levels := make(map[[levelKeySize]byte][]byte)
-	for k, v := range b.levels {
-		vCopy := make([]byte, len(v))
-		copy(vCopy, v)
-		levels[k] = vCopy
-	}
-	return &addrIndexBucket{levels: levels}
-}
-
-// Get returns the value associated with the key from the mock address index
-// bucket.
-//
-// This is part of the internalBucket interface.
-func (b *addrIndexBucket) Get(key []byte) []byte {
-	var levelKey [levelKeySize]byte
-	copy(levelKey[:], key)
-	return b.levels[levelKey]
-}
-
-// Put stores the provided key/value pair to the mock address index bucket.
-//
-// This is part of the internalBucket interface.
-func (b *addrIndexBucket) Put(key []byte, value []byte) error {
-	var levelKey [levelKeySize]byte
-	copy(levelKey[:], key)
-	b.levels[levelKey] = value
-	return nil
-}
-
-// Delete removes the provided key from the mock address index bucket.
-//
-// This is part of the internalBucket interface.
-func (b *addrIndexBucket) Delete(key []byte) error {
-	var levelKey [levelKeySize]byte
-	copy(levelKey[:], key)
-	delete(b.levels, levelKey)
-	return nil
-}
-
-// printLevels returns a string with a visual representation of the provided
-// address key taking into account the max size of each level. It is useful
-// when creating and debugging test cases.
-func (b *addrIndexBucket) printLevels(addrKey [addrKeySize]byte) string {
-	highestLevel := uint8(0)
-	for k := range b.levels {
-		if !bytes.Equal(k[:levelOffset], addrKey[:]) {
-			continue
-		}
-		level := uint8(k[levelOffset])
-		if level > highestLevel {
-			highestLevel = level
-		}
-	}
-
-	var levelBuf bytes.Buffer
-	_, _ = levelBuf.WriteString("\n")
-	maxEntries := level0MaxEntries
-	for level := uint8(0); level <= highestLevel; level++ {
-		data := b.levels[keyForLevel(addrKey, level)]
-		numEntries := len(data) / txEntrySize
-		for i := 0; i < numEntries; i++ {
-			start := i * txEntrySize
-			num := byteOrder.Uint32(data[start:])
-			_, _ = levelBuf.WriteString(fmt.Sprintf("%02d ", num))
-		}
-		for i := numEntries; i < maxEntries; i++ {
-			_, _ = levelBuf.WriteString("_  ")
-		}
-		_, _ = levelBuf.WriteString("\n")
-		maxEntries *= 2
-	}
-
-	return levelBuf.String()
-}
-
-// sanityCheck ensures that all data stored in the bucket for the given address
-// adheres to the level-based rules described by the address index
-// documentation.
-func (b *addrIndexBucket) sanityCheck(addrKey [addrKeySize]byte, expectedTotal int) error {
-	// Find the highest level for the key.
-	highestLevel := uint8(0)
-	for k := range b.levels {
-		if !bytes.Equal(k[:levelOffset], addrKey[:]) {
-			continue
-		}
-		level := uint8(k[levelOffset])
-		if level > highestLevel {
-			highestLevel = level
-		}
-	}
-
-	// Ensure the expected total number of entries are present and that
-	// all levels adhere to the rules described in the address index
-	// documentation.
-	var totalEntries int
-	maxEntries := level0MaxEntries
-	for level := uint8(0); level <= highestLevel; level++ {
-		// Level 0 can'have more entries than the max allowed if the
-		// levels after it have data and it can't be empty. All other
-		// levels must either be half full or full.
-		data := b.levels[keyForLevel(addrKey, level)]
-		numEntries := len(data) / txEntrySize
-		totalEntries += numEntries
-		if level == 0 {
-			if (highestLevel != 0 && numEntries == 0) ||
-				numEntries > maxEntries {
-
-				return errors.Errorf("level %d has %d entries",
-					level, numEntries)
-			}
-		} else if numEntries != maxEntries && numEntries != maxEntries/2 {
-			return errors.Errorf("level %d has %d entries", level,
-				numEntries)
-		}
-		maxEntries *= 2
-	}
-	if totalEntries != expectedTotal {
-		return errors.Errorf("expected %d entries - got %d", expectedTotal,
-			totalEntries)
-	}
-
-	// Ensure all of the numbers are in order starting from the highest
-	// level moving to the lowest level.
-	expectedNum := uint32(0)
-	for level := highestLevel + 1; level > 0; level-- {
-		data := b.levels[keyForLevel(addrKey, level)]
-		numEntries := len(data) / txEntrySize
-		for i := 0; i < numEntries; i++ {
-			start := i * txEntrySize
-			num := byteOrder.Uint32(data[start:])
-			if num != expectedNum {
-				return errors.Errorf("level %d offset %d does "+
-					"not contain the expected number of "+
-					"%d - got %d", level, i, num,
-					expectedNum)
-			}
-			expectedNum++
-		}
-	}
-
-	return nil
-}
-
-// TestAddrIndexLevels ensures that adding and deleting entries to the address
-// index creates multiple levels as described by the address index
-// documentation.
-func TestAddrIndexLevels(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name        string
-		key         [addrKeySize]byte
-		numInsert   int
-		printLevels bool // Set to help debug a specific test.
-	}{
-		{
-			name:      "level 0 not full",
-			numInsert: level0MaxEntries - 1,
-		},
-		{
-			name:      "level 1 half",
-			numInsert: level0MaxEntries + 1,
-		},
-		{
-			name:      "level 1 full",
-			numInsert: level0MaxEntries*2 + 1,
-		},
-		{
-			name:      "level 2 half, level 1 half",
-			numInsert: level0MaxEntries*3 + 1,
-		},
-		{
-			name:      "level 2 half, level 1 full",
-			numInsert: level0MaxEntries*4 + 1,
-		},
-		{
-			name:      "level 2 full, level 1 half",
-			numInsert: level0MaxEntries*5 + 1,
-		},
-		{
-			name:      "level 2 full, level 1 full",
-			numInsert: level0MaxEntries*6 + 1,
-		},
-		{
-			name:      "level 3 half, level 2 half, level 1 half",
-			numInsert: level0MaxEntries*7 + 1,
-		},
-		{
-			name:      "level 3 full, level 2 half, level 1 full",
-			numInsert: level0MaxEntries*12 + 1,
-		},
-	}
-
-nextTest:
-	for testNum, test := range tests {
-		// Insert entries in order.
-		populatedBucket := &addrIndexBucket{
-			levels: make(map[[levelKeySize]byte][]byte),
-		}
-		for i := 0; i < test.numInsert; i++ {
-			txLoc := wire.TxLoc{TxStart: i * 2}
-			err := dbPutAddrIndexEntry(populatedBucket, test.key,
-				uint64(i), txLoc)
-			if err != nil {
-				t.Errorf("dbPutAddrIndexEntry #%d (%s) - "+
-					"unexpected error: %v", testNum,
-					test.name, err)
-				continue nextTest
-			}
-		}
-		if test.printLevels {
-			t.Log(populatedBucket.printLevels(test.key))
-		}
-
-		// Delete entries from the populated bucket until all entries
-		// have been deleted. The bucket is reset to the fully
-		// populated bucket on each iteration so every combination is
-		// tested. Notice the upper limit purposes exceeds the number
-		// of entries to ensure attempting to delete more entries than
-		// there are works correctly.
-		for numDelete := 0; numDelete <= test.numInsert+1; numDelete++ {
-			// Clone populated bucket to run each delete against.
-			bucket := populatedBucket.Clone()
-
-			// Remove the number of entries for this iteration.
-			err := dbRemoveAddrIndexEntries(bucket, test.key,
-				numDelete)
-			if err != nil {
-				if numDelete <= test.numInsert {
-					t.Errorf("dbRemoveAddrIndexEntries (%s) "+
-						" delete %d - unexpected error: "+
-						"%v", test.name, numDelete, err)
-					continue nextTest
-				}
-			}
-			if test.printLevels {
-				t.Log(bucket.printLevels(test.key))
-			}
-
-			// Sanity check the levels to ensure the adhere to all
-			// rules.
-			numExpected := test.numInsert
-			if numDelete <= test.numInsert {
-				numExpected -= numDelete
-			}
-			err = bucket.sanityCheck(test.key, numExpected)
-			if err != nil {
-				t.Errorf("sanity check fail (%s) delete %d: %v",
-					test.name, numDelete, err)
-				continue nextTest
-			}
-		}
-	}
-}

blockdag/indexers/common.go

@@ -1,112 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-Package indexers implements optional block DAG indexes.
-*/
-package indexers
-
-import (
-	"encoding/binary"
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/pkg/errors"
-)
-
-var (
-	// byteOrder is the preferred byte order used for serializing numeric
-	// fields for storage in the database.
-	byteOrder = binary.LittleEndian
-
-	// errInterruptRequested indicates that an operation was cancelled due
-	// to a user-requested interrupt.
-	errInterruptRequested = errors.New("interrupt requested")
-)
-
-// NeedsInputser provides a generic interface for an indexer to specify the it
-// requires the ability to look up inputs for a transaction.
-type NeedsInputser interface {
-	NeedsInputs() bool
-}
-
-// Indexer provides a generic interface for an indexer that is managed by an
-// index manager such as the Manager type provided by this package.
-type Indexer interface {
-	// Key returns the key of the index as a byte slice.
-	Key() []byte
-
-	// Name returns the human-readable name of the index.
-	Name() string
-
-	// Create is invoked when the indexer manager determines the index needs
-	// to be created for the first time.
-	Create(dbTx database.Tx) error
-
-	// Init is invoked when the index manager is first initializing the
-	// index. This differs from the Create method in that it is called on
-	// every load, including the case the index was just created.
-	Init(db database.DB, dag *blockdag.BlockDAG) error
-
-	// ConnectBlock is invoked when the index manager is notified that a new
-	// block has been connected to the DAG.
-	ConnectBlock(dbTx database.Tx,
-		block *util.Block,
-		blockID uint64,
-		dag *blockdag.BlockDAG,
-		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData,
-		virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error
-
-	// Recover is invoked when the indexer wasn't turned on for several blocks
-	// and the indexer needs to close the gaps.
-	Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error
-}
-
-// AssertError identifies an error that indicates an internal code consistency
-// issue and should be treated as a critical and unrecoverable error.
-type AssertError string
-
-// Error returns the assertion error as a huma-readable string and satisfies
-// the error interface.
-func (e AssertError) Error() string {
-	return "assertion failed: " + string(e)
-}
-
-// errDeserialize signifies that a problem was encountered when deserializing
-// data.
-type errDeserialize string
-
-// Error implements the error interface.
-func (e errDeserialize) Error() string {
-	return string(e)
-}
-
-// isDeserializeErr returns whether or not the passed error is an errDeserialize
-// error.
-func isDeserializeErr(err error) bool {
-	_, ok := err.(errDeserialize)
-	return ok
-}
-
-// internalBucket is an abstraction over a database bucket. It is used to make
-// the code easier to test since it allows mock objects in the tests to only
-// implement these functions instead of everything a database.Bucket supports.
-type internalBucket interface {
-	Get(key []byte) []byte
-	Put(key []byte, value []byte) error
-	Delete(key []byte) error
-}
-
-// interruptRequested returns true when the provided channel has been closed.
-// This simplifies early shutdown slightly since the caller can just use an if
-// statement instead of a select.
-func interruptRequested(interrupted <-chan struct{}) bool {
-	select {
-	case <-interrupted:
-		return true
-	default:
-	}
-
-	return false
-}

blockdag/indexers/indexer.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+/*
+Package indexers implements optional block DAG indexes.
+*/
+package indexers
+
+import (
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// Indexer provides a generic interface for an indexer that is managed by an
+// index manager such as the Manager type provided by this package.
+type Indexer interface {
+	// Init is invoked when the index manager is first initializing the
+	// index.
+	Init(dag *blockdag.BlockDAG, databaseContext *dbaccess.DatabaseContext) error
+
+	// ConnectBlock is invoked when the index manager is notified that a new
+	// block has been connected to the DAG.
+	ConnectBlock(dbContext *dbaccess.TxContext,
+		blockHash *daghash.Hash,
+		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData) error
+}

blockdag/indexers/log.go

@@ -6,8 +6,6 @@ package indexers
 
 import (
 	"github.com/kaspanet/kaspad/logger"
-	"github.com/kaspanet/kaspad/util/panics"
 )
 
 var log, _ = logger.Get(logger.SubsystemTags.INDX)
-var spawn = panics.GoroutineWrapperFunc(log)

blockdag/indexers/manager.go

@@ -6,190 +6,30 @@ package indexers
 
 import (
 	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
 
-var (
-	// indexTipsBucketName is the name of the db bucket used to house the
-	// current tip of each index.
-	indexTipsBucketName = []byte("idxtips")
-
-	indexCurrentBlockIDBucketName = []byte("idxcurrentblockid")
-)
-
 // Manager defines an index manager that manages multiple optional indexes and
 // implements the blockdag.IndexManager interface so it can be seamlessly
 // plugged into normal DAG processing.
 type Manager struct {
-	db             database.DB
 	enabledIndexes []Indexer
 }
 
 // Ensure the Manager type implements the blockdag.IndexManager interface.
 var _ blockdag.IndexManager = (*Manager)(nil)
 
-// indexDropKey returns the key for an index which indicates it is in the
-// process of being dropped.
-func indexDropKey(idxKey []byte) []byte {
-	dropKey := make([]byte, len(idxKey)+1)
-	dropKey[0] = 'd'
-	copy(dropKey[1:], idxKey)
-	return dropKey
-}
-
-// maybeFinishDrops determines if each of the enabled indexes are in the middle
-// of being dropped and finishes dropping them when the are. This is necessary
-// because dropping and index has to be done in several atomic steps rather than
-// one big atomic step due to the massive number of entries.
-func (m *Manager) maybeFinishDrops(interrupt <-chan struct{}) error {
-	indexNeedsDrop := make([]bool, len(m.enabledIndexes))
-	err := m.db.View(func(dbTx database.Tx) error {
-		// None of the indexes needs to be dropped if the index tips
-		// bucket hasn't been created yet.
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		if indexesBucket == nil {
-			return nil
-		}
-
-		// Mark the indexer as requiring a drop if one is already in
-		// progress.
-		for i, indexer := range m.enabledIndexes {
-			dropKey := indexDropKey(indexer.Key())
-			if indexesBucket.Get(dropKey) != nil {
-				indexNeedsDrop[i] = true
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
-	if interruptRequested(interrupt) {
-		return errInterruptRequested
-	}
-
-	// Finish dropping any of the enabled indexes that are already in the
-	// middle of being dropped.
-	for i, indexer := range m.enabledIndexes {
-		if !indexNeedsDrop[i] {
-			continue
-		}
-
-		log.Infof("Resuming %s drop", indexer.Name())
-		err := dropIndex(m.db, indexer.Key(), indexer.Name(), interrupt)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// maybeCreateIndexes determines if each of the enabled indexes have already
-// been created and creates them if not.
-func (m *Manager) maybeCreateIndexes(dbTx database.Tx) error {
-	indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-	for _, indexer := range m.enabledIndexes {
-		// Nothing to do if the index tip already exists.
-		idxKey := indexer.Key()
-		if indexesBucket.Get(idxKey) != nil {
-			continue
-		}
-
-		// The tip for the index does not exist, so create it and
-		// invoke the create callback for the index so it can perform
-		// any one-time initialization it requires.
-		if err := indexer.Create(dbTx); err != nil {
-			return err
-		}
-
-		// TODO (Mike): this is temporary solution to prevent node from not starting
-		// because it thinks indexers are not initialized.
-		// Indexers, however, do not work properly, and a general solution to their work operation is required
-		indexesBucket.Put(idxKey, []byte{0})
-	}
-
-	return nil
-}
-
-// Init initializes the enabled indexes. This is called during DAG
-// initialization and primarily consists of catching up all indexes to the
-// current tips. This is necessary since each index can be disabled
-// and re-enabled at any time and attempting to catch-up indexes at the same
-// time new blocks are being downloaded would lead to an overall longer time to
-// catch up due to the I/O contention.
-//
+// Init initializes the enabled indexes.
 // This is part of the blockdag.IndexManager interface.
-func (m *Manager) Init(db database.DB, blockDAG *blockdag.BlockDAG, interrupt <-chan struct{}) error {
-	// Nothing to do when no indexes are enabled.
-	if len(m.enabledIndexes) == 0 {
-		return nil
-	}
-
-	if interruptRequested(interrupt) {
-		return errInterruptRequested
-	}
-
-	m.db = db
-
-	// Finish and drops that were previously interrupted.
-	if err := m.maybeFinishDrops(interrupt); err != nil {
-		return err
-	}
-
-	// Create the initial state for the indexes as needed.
-	err := m.db.Update(func(dbTx database.Tx) error {
-		// Create the bucket for the current tips as needed.
-		meta := dbTx.Metadata()
-		_, err := meta.CreateBucketIfNotExists(indexTipsBucketName)
-		if err != nil {
-			return err
-		}
-		if _, err := meta.CreateBucketIfNotExists(indexCurrentBlockIDBucketName); err != nil {
-			return err
-		}
-
-		return m.maybeCreateIndexes(dbTx)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Initialize each of the enabled indexes.
+func (m *Manager) Init(dag *blockdag.BlockDAG, databaseContext *dbaccess.DatabaseContext) error {
 	for _, indexer := range m.enabledIndexes {
-		if err := indexer.Init(db, blockDAG); err != nil {
+		if err := indexer.Init(dag, databaseContext); err != nil {
 			return err
 		}
 	}
 
-	return m.recoverIfNeeded()
-}
-
-// recoverIfNeeded checks if the node worked for some time
-// without one of the current enabled indexes, and if it's
-// the case, recovers the missing blocks from the index.
-func (m *Manager) recoverIfNeeded() error {
-	return m.db.Update(func(dbTx database.Tx) error {
-		lastKnownBlockID := blockdag.DBFetchCurrentBlockID(dbTx)
-		for _, indexer := range m.enabledIndexes {
-			serializedCurrentIdxBlockID := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Get(indexer.Key())
-			currentIdxBlockID := uint64(0)
-			if serializedCurrentIdxBlockID != nil {
-				currentIdxBlockID = blockdag.DeserializeBlockID(serializedCurrentIdxBlockID)
-			}
-			if lastKnownBlockID > currentIdxBlockID {
-				err := indexer.Recover(dbTx, currentIdxBlockID, lastKnownBlockID)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
+	return nil
 }
 
 // ConnectBlock must be invoked when a block is added to the DAG. It
@@ -197,32 +37,13 @@ func (m *Manager) recoverIfNeeded() error {
 // checks, and invokes each indexer.
 //
 // This is part of the blockdag.IndexManager interface.
-func (m *Manager) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
-	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+func (m *Manager) ConnectBlock(dbContext *dbaccess.TxContext, blockHash *daghash.Hash, txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
 
 	// Call each of the currently active optional indexes with the block
 	// being connected so they can update accordingly.
 	for _, index := range m.enabledIndexes {
 		// Notify the indexer with the connected block so it can index it.
-		if err := index.ConnectBlock(dbTx, block, blockID, dag, txsAcceptanceData, virtualTxsAcceptanceData); err != nil {
-			return err
-		}
-	}
-
-	// Add the new block ID index entry for the block being connected and
-	// update the current internal block ID accordingly.
-	err := m.updateIndexersWithCurrentBlockID(dbTx, block.Hash(), blockID)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (m *Manager) updateIndexersWithCurrentBlockID(dbTx database.Tx, blockHash *daghash.Hash, blockID uint64) error {
-	serializedBlockID := blockdag.SerializeBlockID(blockID)
-	for _, index := range m.enabledIndexes {
-		err := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Put(index.Key(), serializedBlockID)
-		if err != nil {
+		if err := index.ConnectBlock(dbContext, blockHash, txsAcceptanceData); err != nil {
 			return err
 		}
 	}
@@ -238,152 +59,3 @@ func NewManager(enabledIndexes []Indexer) *Manager {
 		enabledIndexes: enabledIndexes,
 	}
 }
-
-// dropIndex drops the passed index from the database. Since indexes can be
-// massive, it deletes the index in multiple database transactions in order to
-// keep memory usage to reasonable levels. It also marks the drop in progress
-// so the drop can be resumed if it is stopped before it is done before the
-// index can be used again.
-func dropIndex(db database.DB, idxKey []byte, idxName string, interrupt <-chan struct{}) error {
-	// Nothing to do if the index doesn't already exist.
-	var needsDelete bool
-	err := db.View(func(dbTx database.Tx) error {
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		if indexesBucket != nil && indexesBucket.Get(idxKey) != nil {
-			needsDelete = true
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	if !needsDelete {
-		log.Infof("Not dropping %s because it does not exist", idxName)
-		return nil
-	}
-
-	// Mark that the index is in the process of being dropped so that it
-	// can be resumed on the next start if interrupted before the process is
-	// complete.
-	log.Infof("Dropping all %s entries. This might take a while...",
-		idxName)
-	err = db.Update(func(dbTx database.Tx) error {
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		return indexesBucket.Put(indexDropKey(idxKey), idxKey)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Since the indexes can be so large, attempting to simply delete
-	// the bucket in a single database transaction would result in massive
-	// memory usage and likely crash many systems due to ulimits. In order
-	// to avoid this, use a cursor to delete a maximum number of entries out
-	// of the bucket at a time. Recurse buckets depth-first to delete any
-	// sub-buckets.
-	const maxDeletions = 2000000
-	var totalDeleted uint64
-
-	// Recurse through all buckets in the index, cataloging each for
-	// later deletion.
-	var subBuckets [][][]byte
-	var subBucketClosure func(database.Tx, []byte, [][]byte) error
-	subBucketClosure = func(dbTx database.Tx,
-		subBucket []byte, tlBucket [][]byte) error {
-		// Get full bucket name and append to subBuckets for later
-		// deletion.
-		var bucketName [][]byte
-		if (tlBucket == nil) || (len(tlBucket) == 0) {
-			bucketName = append(bucketName, subBucket)
-		} else {
-			bucketName = append(tlBucket, subBucket)
-		}
-		subBuckets = append(subBuckets, bucketName)
-		// Recurse sub-buckets to append to subBuckets slice.
-		bucket := dbTx.Metadata()
-		for _, subBucketName := range bucketName {
-			bucket = bucket.Bucket(subBucketName)
-		}
-		return bucket.ForEachBucket(func(k []byte) error {
-			return subBucketClosure(dbTx, k, bucketName)
-		})
-	}
-
-	// Call subBucketClosure with top-level bucket.
-	err = db.View(func(dbTx database.Tx) error {
-		return subBucketClosure(dbTx, idxKey, nil)
-	})
-	if err != nil {
-		return nil
-	}
-
-	// Iterate through each sub-bucket in reverse, deepest-first, deleting
-	// all keys inside them and then dropping the buckets themselves.
-	for i := range subBuckets {
-		bucketName := subBuckets[len(subBuckets)-1-i]
-		// Delete maxDeletions key/value pairs at a time.
-		for numDeleted := maxDeletions; numDeleted == maxDeletions; {
-			numDeleted = 0
-			err := db.Update(func(dbTx database.Tx) error {
-				subBucket := dbTx.Metadata()
-				for _, subBucketName := range bucketName {
-					subBucket = subBucket.Bucket(subBucketName)
-				}
-				cursor := subBucket.Cursor()
-				for ok := cursor.First(); ok; ok = cursor.Next() &&
-					numDeleted < maxDeletions {
-
-					if err := cursor.Delete(); err != nil {
-						return err
-					}
-					numDeleted++
-				}
-				return nil
-			})
-			if err != nil {
-				return err
-			}
-
-			if numDeleted > 0 {
-				totalDeleted += uint64(numDeleted)
-				log.Infof("Deleted %d keys (%d total) from %s",
-					numDeleted, totalDeleted, idxName)
-			}
-		}
-
-		if interruptRequested(interrupt) {
-			return errInterruptRequested
-		}
-
-		// Drop the bucket itself.
-		err = db.Update(func(dbTx database.Tx) error {
-			bucket := dbTx.Metadata()
-			for j := 0; j < len(bucketName)-1; j++ {
-				bucket = bucket.Bucket(bucketName[j])
-			}
-			return bucket.DeleteBucket(bucketName[len(bucketName)-1])
-		})
-	}
-
-	// Remove the index tip, index bucket, and in-progress drop flag now
-	// that all index entries have been removed.
-	err = db.Update(func(dbTx database.Tx) error {
-		meta := dbTx.Metadata()
-		indexesBucket := meta.Bucket(indexTipsBucketName)
-		if err := indexesBucket.Delete(idxKey); err != nil {
-			return err
-		}
-
-		if err := meta.Bucket(indexCurrentBlockIDBucketName).Delete(idxKey); err != nil {
-			return err
-		}
-
-		return indexesBucket.Delete(indexDropKey(idxKey))
-	})
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Dropped %s", idxName)
-	return nil
-}

blockdag/indexers/txindex.go

@@ -1,431 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package indexers
-
-import (
-	"fmt"
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-	"github.com/pkg/errors"
-)
-
-const (
-	// txIndexName is the human-readable name for the index.
-	txIndexName = "transaction index"
-
-	includingBlocksIndexKeyEntrySize = 8 // 4 bytes for offset + 4 bytes for transaction length
-)
-
-var (
-	includingBlocksIndexKey = []byte("includingblocksidx")
-
-	acceptingBlocksIndexKey = []byte("acceptingblocksidx")
-)
-
-// txsAcceptedByVirtual is the in-memory index of txIDs that were accepted
-// by the current virtual
-var txsAcceptedByVirtual map[daghash.TxID]bool
-
-// -----------------------------------------------------------------------------
-// The transaction index consists of an entry for every transaction in the DAG.
-//
-// There are two buckets used in total. The first bucket maps the hash of
-// each transaction to its location in each block it's included in. The second bucket
-// contains all of the blocks that from their viewpoint the transaction has been
-// accepted (i.e. the transaction is found in their blue set without double spends),
-// and their blue block (or themselves) that included the transaction.
-//
-// NOTE: Although it is technically possible for multiple transactions to have
-// the same hash as long as the previous transaction with the same hash is fully
-// spent, this code only stores the most recent one because doing otherwise
-// would add a non-trivial amount of space and overhead for something that will
-// realistically never happen per the probability and even if it did, the old
-// one must be fully spent and so the most likely transaction a caller would
-// want for a given hash is the most recent one anyways.
-//
-// The including blocks index contains a sub bucket for each transaction hash (32 byte each), that its serialized format is:
-//
-//   <block id> = <start offset><tx length>
-//
-//   Field           Type              Size
-//   block id        uint64          8 bytes
-//   start offset    uint32          4 bytes
-//   tx length       uint32          4 bytes
-//   -----
-//   Total: 16 bytes
-//
-// The accepting blocks index contains a sub bucket for each transaction hash (32 byte each), that its serialized format is:
-//
-//   <accepting block id> = <including block id>
-//
-//   Field           Type              Size
-//   accepting block id        uint64          8 bytes
-//   including block id        uint64          8 bytes
-//   -----
-//   Total: 16 bytes
-//
-// -----------------------------------------------------------------------------
-
-func putIncludingBlocksEntry(target []byte, txLoc wire.TxLoc) {
-	byteOrder.PutUint32(target, uint32(txLoc.TxStart))
-	byteOrder.PutUint32(target[4:], uint32(txLoc.TxLen))
-}
-
-func dbPutIncludingBlocksEntry(dbTx database.Tx, txID *daghash.TxID, blockID uint64, serializedData []byte) error {
-	bucket, err := dbTx.Metadata().Bucket(includingBlocksIndexKey).CreateBucketIfNotExists(txID[:])
-	if err != nil {
-		return err
-	}
-	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedData)
-}
-
-func dbPutAcceptingBlocksEntry(dbTx database.Tx, txID *daghash.TxID, blockID uint64, serializedData []byte) error {
-	bucket, err := dbTx.Metadata().Bucket(acceptingBlocksIndexKey).CreateBucketIfNotExists(txID[:])
-	if err != nil {
-		return err
-	}
-	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedData)
-}
-
-// dbFetchFirstTxRegion uses an existing database transaction to fetch the block
-// region for the provided transaction hash from the transaction index. When
-// there is no entry for the provided hash, nil will be returned for the both
-// the region and the error.
-//
-// P.S Because the transaction can be found in multiple blocks, this function arbitarily
-// returns the first block region that is stored in the txindex.
-func dbFetchFirstTxRegion(dbTx database.Tx, txID *daghash.TxID) (*database.BlockRegion, error) {
-	// Load the record from the database and return now if it doesn't exist.
-	txBucket := dbTx.Metadata().Bucket(includingBlocksIndexKey).Bucket(txID[:])
-	if txBucket == nil {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No block region "+
-				"was found for %s", txID),
-		}
-	}
-	cursor := txBucket.Cursor()
-	if ok := cursor.First(); !ok {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No block region "+
-				"was found for %s", txID),
-		}
-	}
-	serializedBlockID := cursor.Key()
-	serializedData := cursor.Value()
-	if len(serializedData) == 0 {
-		return nil, nil
-	}
-
-	// Ensure the serialized data has enough bytes to properly deserialize.
-	if len(serializedData) < includingBlocksIndexKeyEntrySize {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("corrupt transaction index "+
-				"entry for %s", txID),
-		}
-	}
-
-	// Load the block hash associated with the block ID.
-	hash, err := blockdag.DBFetchBlockHashBySerializedID(dbTx, serializedBlockID)
-	if err != nil {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("corrupt transaction index "+
-				"entry for %s: %s", txID, err),
-		}
-	}
-
-	// Deserialize the final entry.
-	region := database.BlockRegion{Hash: &daghash.Hash{}}
-	copy(region.Hash[:], hash[:])
-	region.Offset = byteOrder.Uint32(serializedData[:4])
-	region.Len = byteOrder.Uint32(serializedData[4:])
-
-	return &region, nil
-}
-
-// dbAddTxIndexEntries uses an existing database transaction to add a
-// transaction index entry for every transaction in the passed block.
-func dbAddTxIndexEntries(dbTx database.Tx, block *util.Block, blockID uint64, multiBlockTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
-	// The offset and length of the transactions within the serialized
-	// block.
-	txLocs, err := block.TxLoc()
-	if err != nil {
-		return err
-	}
-
-	// As an optimization, allocate a single slice big enough to hold all
-	// of the serialized transaction index entries for the block and
-	// serialize them directly into the slice. Then, pass the appropriate
-	// subslice to the database to be written. This approach significantly
-	// cuts down on the number of required allocations.
-	includingBlocksOffset := 0
-	serializedIncludingBlocksValues := make([]byte, len(block.Transactions())*includingBlocksIndexKeyEntrySize)
-	for i, tx := range block.Transactions() {
-		putIncludingBlocksEntry(serializedIncludingBlocksValues[includingBlocksOffset:], txLocs[i])
-		endOffset := includingBlocksOffset + includingBlocksIndexKeyEntrySize
-		err := dbPutIncludingBlocksEntry(dbTx, tx.ID(), blockID,
-			serializedIncludingBlocksValues[includingBlocksOffset:endOffset:endOffset])
-		if err != nil {
-			return err
-		}
-		includingBlocksOffset += includingBlocksIndexKeyEntrySize
-	}
-
-	for _, blockTxsAcceptanceData := range multiBlockTxsAcceptanceData {
-		var includingBlockID uint64
-		if blockTxsAcceptanceData.BlockHash.IsEqual(block.Hash()) {
-			includingBlockID = blockID
-		} else {
-			includingBlockID, err = blockdag.DBFetchBlockIDByHash(dbTx, &blockTxsAcceptanceData.BlockHash)
-			if err != nil {
-				return err
-			}
-		}
-
-		serializedIncludingBlockID := blockdag.SerializeBlockID(includingBlockID)
-
-		for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
-			err = dbPutAcceptingBlocksEntry(dbTx, txAcceptanceData.Tx.ID(), blockID, serializedIncludingBlockID)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func updateTxsAcceptedByVirtual(virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
-	// Initialize a new txsAcceptedByVirtual
-	entries := 0
-	for _, blockTxsAcceptanceData := range virtualTxsAcceptanceData {
-		entries += len(blockTxsAcceptanceData.TxAcceptanceData)
-	}
-	txsAcceptedByVirtual = make(map[daghash.TxID]bool, entries)
-
-	// Copy virtualTxsAcceptanceData to txsAcceptedByVirtual
-	for _, blockTxsAcceptanceData := range virtualTxsAcceptanceData {
-		for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
-			txsAcceptedByVirtual[*txAcceptanceData.Tx.ID()] = true
-		}
-	}
-
-	return nil
-}
-
-// TxIndex implements a transaction by hash index. That is to say, it supports
-// querying all transactions by their hash.
-type TxIndex struct {
-	db database.DB
-}
-
-// Ensure the TxIndex type implements the Indexer interface.
-var _ Indexer = (*TxIndex)(nil)
-
-// Init initializes the hash-based transaction index. In particular, it finds
-// the highest used block ID and stores it for later use when connecting or
-// disconnecting blocks.
-//
-// This is part of the Indexer interface.
-func (idx *TxIndex) Init(db database.DB, dag *blockdag.BlockDAG) error {
-	idx.db = db
-
-	// Initialize the txsAcceptedByVirtual index
-	virtualTxsAcceptanceData, err := dag.TxsAcceptedByVirtual()
-	if err != nil {
-		return err
-	}
-	err = updateTxsAcceptedByVirtual(virtualTxsAcceptanceData)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// Key returns the database key to use for the index as a byte slice.
-//
-// This is part of the Indexer interface.
-func (idx *TxIndex) Key() []byte {
-	return includingBlocksIndexKey
-}
-
-// Name returns the human-readable name of the index.
-//
-// This is part of the Indexer interface.
-func (idx *TxIndex) Name() string {
-	return txIndexName
-}
-
-// Create is invoked when the indexer manager determines the index needs
-// to be created for the first time. It creates the buckets for the hash-based
-// transaction index and the internal block ID indexes.
-//
-// This is part of the Indexer interface.
-func (idx *TxIndex) Create(dbTx database.Tx) error {
-	meta := dbTx.Metadata()
-	if _, err := meta.CreateBucket(includingBlocksIndexKey); err != nil {
-		return err
-	}
-	_, err := meta.CreateBucket(acceptingBlocksIndexKey)
-	return err
-
-}
-
-// ConnectBlock is invoked by the index manager when a new block has been
-// connected to the DAG. This indexer adds a hash-to-transaction mapping
-// for every transaction in the passed block.
-//
-// This is part of the Indexer interface.
-func (idx *TxIndex) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
-	acceptedTxsData blockdag.MultiBlockTxsAcceptanceData, virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
-	if err := dbAddTxIndexEntries(dbTx, block, blockID, acceptedTxsData); err != nil {
-		return err
-	}
-
-	err := updateTxsAcceptedByVirtual(virtualTxsAcceptanceData)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// TxFirstBlockRegion returns the first block region for the provided transaction hash
-// from the transaction index. The block region can in turn be used to load the
-// raw transaction bytes. When there is no entry for the provided hash, nil
-// will be returned for the both the entry and the error.
-//
-// This function is safe for concurrent access.
-func (idx *TxIndex) TxFirstBlockRegion(txID *daghash.TxID) (*database.BlockRegion, error) {
-	var region *database.BlockRegion
-	err := idx.db.View(func(dbTx database.Tx) error {
-		var err error
-		region, err = dbFetchFirstTxRegion(dbTx, txID)
-		return err
-	})
-	return region, err
-}
-
-// TxBlocks returns the hashes of the blocks where the transaction exists
-func (idx *TxIndex) TxBlocks(txHash *daghash.Hash) ([]*daghash.Hash, error) {
-	blockHashes := make([]*daghash.Hash, 0)
-	err := idx.db.View(func(dbTx database.Tx) error {
-		var err error
-		blockHashes, err = dbFetchTxBlocks(dbTx, txHash)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	return blockHashes, err
-}
-
-func dbFetchTxBlocks(dbTx database.Tx, txHash *daghash.Hash) ([]*daghash.Hash, error) {
-	blockHashes := make([]*daghash.Hash, 0)
-	bucket := dbTx.Metadata().Bucket(includingBlocksIndexKey).Bucket(txHash[:])
-	if bucket == nil {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No including blocks "+
-				"were found for %s", txHash),
-		}
-	}
-	err := bucket.ForEach(func(serializedBlockID, _ []byte) error {
-		blockHash, err := blockdag.DBFetchBlockHashBySerializedID(dbTx, serializedBlockID)
-		if err != nil {
-			return err
-		}
-		blockHashes = append(blockHashes, blockHash)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return blockHashes, nil
-}
-
-// BlockThatAcceptedTx returns the hash of the block where the transaction got accepted (from the virtual block point of view)
-func (idx *TxIndex) BlockThatAcceptedTx(dag *blockdag.BlockDAG, txID *daghash.TxID) (*daghash.Hash, error) {
-	var acceptingBlock *daghash.Hash
-	err := idx.db.View(func(dbTx database.Tx) error {
-		var err error
-		acceptingBlock, err = dbFetchTxAcceptingBlock(dbTx, txID, dag)
-		return err
-	})
-	return acceptingBlock, err
-}
-
-func dbFetchTxAcceptingBlock(dbTx database.Tx, txID *daghash.TxID, dag *blockdag.BlockDAG) (*daghash.Hash, error) {
-	// If the transaction was accepted by the current virtual,
-	// return the zeroHash immediately
-	if _, ok := txsAcceptedByVirtual[*txID]; ok {
-		return &daghash.ZeroHash, nil
-	}
-
-	bucket := dbTx.Metadata().Bucket(acceptingBlocksIndexKey).Bucket(txID[:])
-	if bucket == nil {
-		return nil, nil
-	}
-	cursor := bucket.Cursor()
-	if !cursor.First() {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("Accepting blocks bucket is "+
-				"empty for %s", txID),
-		}
-	}
-	for ; cursor.Key() != nil; cursor.Next() {
-		blockHash, err := blockdag.DBFetchBlockHashBySerializedID(dbTx, cursor.Key())
-		if err != nil {
-			return nil, err
-		}
-		if dag.IsInSelectedParentChain(blockHash) {
-			return blockHash, nil
-		}
-	}
-	return nil, nil
-}
-
-// NewTxIndex returns a new instance of an indexer that is used to create a
-// mapping of the hashes of all transactions in the blockDAG to the respective
-// block, location within the block, and size of the transaction.
-//
-// It implements the Indexer interface which plugs into the IndexManager that in
-// turn is used by the blockdag package. This allows the index to be
-// seamlessly maintained along with the DAG.
-func NewTxIndex() *TxIndex {
-	return &TxIndex{}
-}
-
-// DropTxIndex drops the transaction index from the provided database if it
-// exists. Since the address index relies on it, the address index will also be
-// dropped when it exists.
-func DropTxIndex(db database.DB, interrupt <-chan struct{}) error {
-	err := dropIndex(db, addrIndexKey, addrIndexName, interrupt)
-	if err != nil {
-		return err
-	}
-
-	err = dropIndex(db, includingBlocksIndexKey, addrIndexName, interrupt)
-	if err != nil {
-		return err
-	}
-
-	return dropIndex(db, acceptingBlocksIndexKey, txIndexName, interrupt)
-}
-
-// Recover is invoked when the indexer wasn't turned on for several blocks
-// and the indexer needs to close the gaps.
-//
-// This is part of the Indexer interface.
-func (idx *TxIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
-	return errors.Errorf("txindex was turned off for %d blocks and can't be recovered."+
-		" To resume working drop the txindex with --droptxindex", lastKnownBlockID-currentBlockID)
-}

blockdag/indexers/txindex_test.go

@@ -1,144 +0,0 @@
-package indexers
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/mining"
-	"github.com/kaspanet/kaspad/txscript"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-func createTransaction(t *testing.T, value uint64, originTx *wire.MsgTx, outputIndex uint32) *wire.MsgTx {
-	signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
-	if err != nil {
-		t.Fatalf("Error creating signature script: %s", err)
-	}
-	txIn := &wire.TxIn{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  *originTx.TxID(),
-			Index: outputIndex,
-		},
-		Sequence:        wire.MaxTxInSequenceNum,
-		SignatureScript: signatureScript,
-	}
-	txOut := wire.NewTxOut(value, blockdag.OpTrueScript)
-	tx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
-
-	return tx
-}
-
-func TestTxIndexConnectBlock(t *testing.T) {
-	blocks := make(map[daghash.Hash]*util.Block)
-
-	txIndex := NewTxIndex()
-	indexManager := NewManager([]Indexer{txIndex})
-
-	params := dagconfig.SimNetParams
-	params.BlockCoinbaseMaturity = 0
-	params.K = 1
-
-	config := blockdag.Config{
-		IndexManager: indexManager,
-		DAGParams:    &params,
-	}
-
-	dag, teardown, err := blockdag.DAGSetup("TestTxIndexConnectBlock", config)
-	if err != nil {
-		t.Fatalf("TestTxIndexConnectBlock: Failed to setup DAG instance: %v", err)
-	}
-	if teardown != nil {
-		defer teardown()
-	}
-
-	prepareAndProcessBlock := func(parentHashes []*daghash.Hash, transactions []*wire.MsgTx, blockName string) *wire.MsgBlock {
-		block, err := mining.PrepareBlockForTest(dag, &params, parentHashes, transactions, false)
-		if err != nil {
-			t.Fatalf("TestTxIndexConnectBlock: block %v got unexpected error from PrepareBlockForTest: %v", blockName, err)
-		}
-		utilBlock := util.NewBlock(block)
-		blocks[*block.BlockHash()] = utilBlock
-		isOrphan, delay, err := dag.ProcessBlock(utilBlock, blockdag.BFNoPoWCheck)
-		if err != nil {
-			t.Fatalf("TestTxIndexConnectBlock: dag.ProcessBlock got unexpected error for block %v: %v", blockName, err)
-		}
-		if delay != 0 {
-			t.Fatalf("TestTxIndexConnectBlock: block %s "+
-				"is too far in the future", blockName)
-		}
-		if isOrphan {
-			t.Fatalf("TestTxIndexConnectBlock: block %v was unexpectedly orphan", blockName)
-		}
-		return block
-	}
-
-	block1 := prepareAndProcessBlock([]*daghash.Hash{params.GenesisHash}, nil, "1")
-	block2Tx := createTransaction(t, block1.Transactions[0].TxOut[0].Value, block1.Transactions[0], 0)
-	block2 := prepareAndProcessBlock([]*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{block2Tx}, "2")
-	block3Tx := createTransaction(t, block2.Transactions[0].TxOut[0].Value, block2.Transactions[0], 0)
-	block3 := prepareAndProcessBlock([]*daghash.Hash{block2.BlockHash()}, []*wire.MsgTx{block3Tx}, "3")
-
-	block2TxID := block2Tx.TxID()
-	block2TxNewAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, block2TxID)
-	if err != nil {
-		t.Errorf("TestTxIndexConnectBlock: TxAcceptedInBlock: %v", err)
-	}
-	block3Hash := block3.BlockHash()
-	if !block2TxNewAcceptedBlock.IsEqual(block3Hash) {
-		t.Errorf("TestTxIndexConnectBlock: block2Tx should've "+
-			"been accepted in block %v but instead got accepted in block %v", block3Hash, block2TxNewAcceptedBlock)
-	}
-
-	block3TxID := block3Tx.TxID()
-	block3TxNewAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, block3TxID)
-	if err != nil {
-		t.Errorf("TestTxIndexConnectBlock: TxAcceptedInBlock: %v", err)
-	}
-	if !block3TxNewAcceptedBlock.IsEqual(&daghash.ZeroHash) {
-		t.Errorf("TestTxIndexConnectBlock: block3Tx should've "+
-			"been accepted by the virtual block but instead got accepted in block %v", block3TxNewAcceptedBlock)
-	}
-
-	block3A := prepareAndProcessBlock([]*daghash.Hash{block2.BlockHash()}, []*wire.MsgTx{block3Tx}, "3A")
-	block4 := prepareAndProcessBlock([]*daghash.Hash{block3.BlockHash()}, nil, "4")
-	prepareAndProcessBlock([]*daghash.Hash{block3A.BlockHash(), block4.BlockHash()}, nil, "5")
-
-	block2TxAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, block2TxID)
-	if err != nil {
-		t.Errorf("TestTxIndexConnectBlock: TxAcceptedInBlock: %v", err)
-	}
-	block3AHash := block3A.BlockHash()
-	if !block2TxAcceptedBlock.IsEqual(block3AHash) {
-		t.Errorf("TestTxIndexConnectBlock: block2Tx should've "+
-			"been accepted in block %v but instead got accepted in block %v", block3AHash, block2TxAcceptedBlock)
-	}
-
-	region, err := txIndex.TxFirstBlockRegion(block3TxID)
-	if err != nil {
-		t.Fatalf("TestTxIndexConnectBlock: no block region was found for block3Tx")
-	}
-	regionBlock, ok := blocks[*region.Hash]
-	if !ok {
-		t.Fatalf("TestTxIndexConnectBlock: couldn't find block with hash %v", region.Hash)
-	}
-
-	regionBlockBytes, err := regionBlock.Bytes()
-	if err != nil {
-		t.Fatalf("TestTxIndexConnectBlock: Couldn't serialize block to bytes")
-	}
-	block3TxInBlock := regionBlockBytes[region.Offset : region.Offset+region.Len]
-
-	block3TxBuf := bytes.NewBuffer(make([]byte, 0, block3Tx.SerializeSize()))
-	block3Tx.KaspaEncode(block3TxBuf, 0)
-	blockTxBytes := block3TxBuf.Bytes()
-
-	if !reflect.DeepEqual(blockTxBytes, block3TxInBlock) {
-		t.Errorf("TestTxIndexConnectBlock: the block region that was in the bucket doesn't match block3Tx")
-	}
-
-}

blockdag/mediantime.go

@@ -1,218 +0,0 @@
-// Copyright (c) 2013-2014 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"math"
-	"sort"
-	"sync"
-	"time"
-)
-
-const (
-	// maxAllowedOffsetSeconds is the maximum number of seconds in either
-	// direction that local clock will be adjusted. When the median time
-	// of the network is outside of this range, no offset will be applied.
-	maxAllowedOffsetSecs = 70 * 60 // 1 hour 10 minutes
-
-	// similarTimeSecs is the number of seconds in either direction from the
-	// local clock that is used to determine that it is likley wrong and
-	// hence to show a warning.
-	similarTimeSecs = 5 * 60 // 5 minutes
-)
-
-var (
-	// maxMedianTimeEntries is the maximum number of entries allowed in the
-	// median time data. This is a variable as opposed to a constant so the
-	// test code can modify it.
-	maxMedianTimeEntries = 200
-)
-
-// MedianTimeSource provides a mechanism to add several time samples which are
-// used to determine a median time which is then used as an offset to the local
-// clock.
-type MedianTimeSource interface {
-	// AdjustedTime returns the current time adjusted by the median time
-	// offset as calculated from the time samples added by AddTimeSample.
-	AdjustedTime() time.Time
-
-	// AddTimeSample adds a time sample that is used when determining the
-	// median time of the added samples.
-	AddTimeSample(id string, timeVal time.Time)
-
-	// Offset returns the number of seconds to adjust the local clock based
-	// upon the median of the time samples added by AddTimeData.
-	Offset() time.Duration
-}
-
-// int64Sorter implements sort.Interface to allow a slice of 64-bit integers to
-// be sorted.
-type int64Sorter []int64
-
-// Len returns the number of 64-bit integers in the slice. It is part of the
-// sort.Interface implementation.
-func (s int64Sorter) Len() int {
-	return len(s)
-}
-
-// Swap swaps the 64-bit integers at the passed indices. It is part of the
-// sort.Interface implementation.
-func (s int64Sorter) Swap(i, j int) {
-	s[i], s[j] = s[j], s[i]
-}
-
-// Less returns whether the 64-bit integer with index i should sort before the
-// 64-bit integer with index j. It is part of the sort.Interface
-// implementation.
-func (s int64Sorter) Less(i, j int) bool {
-	return s[i] < s[j]
-}
-
-// medianTime provides an implementation of the MedianTimeSource interface.
-// It is limited to maxMedianTimeEntries includes the same buggy behavior as
-// the time offset mechanism in Bitcoin Core. This is necessary because it is
-// used in the consensus code.
-type medianTime struct {
-	mtx                sync.Mutex
-	knownIDs           map[string]struct{}
-	offsets            []int64
-	offsetSecs         int64
-	invalidTimeChecked bool
-}
-
-// Ensure the medianTime type implements the MedianTimeSource interface.
-var _ MedianTimeSource = (*medianTime)(nil)
-
-// AdjustedTime returns the current time adjusted by the median time offset as
-// calculated from the time samples added by AddTimeSample.
-//
-// This function is safe for concurrent access and is part of the
-// MedianTimeSource interface implementation.
-func (m *medianTime) AdjustedTime() time.Time {
-	m.mtx.Lock()
-	defer m.mtx.Unlock()
-
-	// Limit the adjusted time to 1 second precision.
-	now := time.Unix(time.Now().Unix(), 0)
-	return now.Add(time.Duration(m.offsetSecs) * time.Second)
-}
-
-// AddTimeSample adds a time sample that is used when determining the median
-// time of the added samples.
-//
-// This function is safe for concurrent access and is part of the
-// MedianTimeSource interface implementation.
-func (m *medianTime) AddTimeSample(sourceID string, timeVal time.Time) {
-	m.mtx.Lock()
-	defer m.mtx.Unlock()
-
-	// Don't add time data from the same source.
-	if _, exists := m.knownIDs[sourceID]; exists {
-		return
-	}
-	m.knownIDs[sourceID] = struct{}{}
-
-	// Truncate the provided offset to seconds and append it to the slice
-	// of offsets while respecting the maximum number of allowed entries by
-	// replacing the oldest entry with the new entry once the maximum number
-	// of entries is reached.
-	now := time.Unix(time.Now().Unix(), 0)
-	offsetSecs := int64(timeVal.Sub(now).Seconds())
-	numOffsets := len(m.offsets)
-	if numOffsets == maxMedianTimeEntries && maxMedianTimeEntries > 0 {
-		m.offsets = m.offsets[1:]
-		numOffsets--
-	}
-	m.offsets = append(m.offsets, offsetSecs)
-	numOffsets++
-
-	// Sort the offsets so the median can be obtained as needed later.
-	sortedOffsets := make([]int64, numOffsets)
-	copy(sortedOffsets, m.offsets)
-	sort.Sort(int64Sorter(sortedOffsets))
-
-	offsetDuration := time.Duration(offsetSecs) * time.Second
-	log.Debugf("Added time sample of %s (total: %d)", offsetDuration,
-		numOffsets)
-
-	// NOTE: The following code intentionally has a bug to mirror the
-	// buggy behavior in Bitcoin Core since the median time is used in the
-	// consensus rules.
-	//
-	// In particular, the offset is only updated when the number of entries
-	// is odd, but the max number of entries is 200, an even number. Thus,
-	// the offset will never be updated again once the max number of entries
-	// is reached.
-
-	// The median offset is only updated when there are enough offsets and
-	// the number of offsets is odd so the middle value is the true median.
-	// Thus, there is nothing to do when those conditions are not met.
-	if numOffsets < 5 || numOffsets&0x01 != 1 {
-		return
-	}
-
-	// At this point the number of offsets in the list is odd, so the
-	// middle value of the sorted offsets is the median.
-	median := sortedOffsets[numOffsets/2]
-
-	// Set the new offset when the median offset is within the allowed
-	// offset range.
-	if math.Abs(float64(median)) < maxAllowedOffsetSecs {
-		m.offsetSecs = median
-	} else {
-		// The median offset of all added time data is larger than the
-		// maximum allowed offset, so don't use an offset. This
-		// effectively limits how far the local clock can be skewed.
-		m.offsetSecs = 0
-
-		if !m.invalidTimeChecked {
-			m.invalidTimeChecked = true
-
-			// Find if any time samples have a time that is close
-			// to the local time.
-			var remoteHasCloseTime bool
-			for _, offset := range sortedOffsets {
-				if math.Abs(float64(offset)) < similarTimeSecs {
-					remoteHasCloseTime = true
-					break
-				}
-			}
-
-			// Warn if none of the time samples are close.
-			if !remoteHasCloseTime {
-				log.Warnf("Please check your date and time " +
-					"are correct!  kaspad will not work " +
-					"properly with an invalid time")
-			}
-		}
-	}
-
-	medianDuration := time.Duration(m.offsetSecs) * time.Second
-	log.Debugf("New time offset: %d", medianDuration)
-}
-
-// Offset returns the number of seconds to adjust the local clock based upon the
-// median of the time samples added by AddTimeData.
-//
-// This function is safe for concurrent access and is part of the
-// MedianTimeSource interface implementation.
-func (m *medianTime) Offset() time.Duration {
-	m.mtx.Lock()
-	defer m.mtx.Unlock()
-
-	return time.Duration(m.offsetSecs) * time.Second
-}
-
-// NewMedianTime returns a new instance of concurrency-safe implementation of
-// the MedianTimeSource interface. The returned implementation contains the
-// rules necessary for proper time handling in the DAG consensus rules and
-// expects the time samples to be added from the timestamp field of the version
-// message received from remote peers that successfully connect and negotiate.
-func NewMedianTime() MedianTimeSource {
-	return &medianTime{
-		knownIDs: make(map[string]struct{}),
-		offsets:  make([]int64, 0, maxMedianTimeEntries),
-	}
-}

blockdag/mediantime_test.go

@@ -1,104 +0,0 @@
-// Copyright (c) 2013-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"strconv"
-	"testing"
-	"time"
-)
-
-// TestMedianTime tests the medianTime implementation.
-func TestMedianTime(t *testing.T) {
-	tests := []struct {
-		in         []int64
-		wantOffset int64
-		useDupID   bool
-	}{
-		// Not enough samples must result in an offset of 0.
-		{in: []int64{1}, wantOffset: 0},
-		{in: []int64{1, 2}, wantOffset: 0},
-		{in: []int64{1, 2, 3}, wantOffset: 0},
-		{in: []int64{1, 2, 3, 4}, wantOffset: 0},
-
-		// Various number of entries. The expected offset is only
-		// updated on odd number of elements.
-		{in: []int64{-13, 57, -4, -23, -12}, wantOffset: -12},
-		{in: []int64{55, -13, 61, -52, 39, 55}, wantOffset: 39},
-		{in: []int64{-62, -58, -30, -62, 51, -30, 15}, wantOffset: -30},
-		{in: []int64{29, -47, 39, 54, 42, 41, 8, -33}, wantOffset: 39},
-		{in: []int64{37, 54, 9, -21, -56, -36, 5, -11, -39}, wantOffset: -11},
-		{in: []int64{57, -28, 25, -39, 9, 63, -16, 19, -60, 25}, wantOffset: 9},
-		{in: []int64{-5, -4, -3, -2, -1}, wantOffset: -3, useDupID: true},
-
-		// The offset stops being updated once the max number of entries
-		// has been reached. This is actually a bug from Bitcoin Core,
-		// but since the time is ultimately used as a part of the
-		// consensus rules, it must be mirrored.
-		{in: []int64{-67, 67, -50, 24, 63, 17, 58, -14, 5, -32, -52}, wantOffset: 17},
-		{in: []int64{-67, 67, -50, 24, 63, 17, 58, -14, 5, -32, -52, 45}, wantOffset: 17},
-		{in: []int64{-67, 67, -50, 24, 63, 17, 58, -14, 5, -32, -52, 45, 4}, wantOffset: 17},
-
-		// Offsets that are too far away from the local time should
-		// be ignored.
-		{in: []int64{-4201, 4202, -4203, 4204, -4205}, wantOffset: 0},
-
-		// Exercise the condition where the median offset is greater
-		// than the max allowed adjustment, but there is at least one
-		// sample that is close enough to the current time to avoid
-		// triggering a warning about an invalid local clock.
-		{in: []int64{4201, 4202, 4203, 4204, -299}, wantOffset: 0},
-	}
-
-	// Modify the max number of allowed median time entries for these tests.
-	maxMedianTimeEntries = 10
-	defer func() { maxMedianTimeEntries = 200 }()
-
-	for i, test := range tests {
-		filter := NewMedianTime()
-		for j, offset := range test.in {
-			id := strconv.Itoa(j)
-			now := time.Unix(time.Now().Unix(), 0)
-			tOffset := now.Add(time.Duration(offset) * time.Second)
-			filter.AddTimeSample(id, tOffset)
-
-			// Ensure the duplicate IDs are ignored.
-			if test.useDupID {
-				// Modify the offsets to ensure the final median
-				// would be different if the duplicate is added.
-				tOffset = tOffset.Add(time.Duration(offset) *
-					time.Second)
-				filter.AddTimeSample(id, tOffset)
-			}
-		}
-
-		// Since it is possible that the time.Now call in AddTimeSample
-		// and the time.Now calls here in the tests will be off by one
-		// second, allow a fudge factor to compensate.
-		gotOffset := filter.Offset()
-		wantOffset := time.Duration(test.wantOffset) * time.Second
-		wantOffset2 := time.Duration(test.wantOffset-1) * time.Second
-		if gotOffset != wantOffset && gotOffset != wantOffset2 {
-			t.Errorf("Offset #%d: unexpected offset -- got %v, "+
-				"want %v or %v", i, gotOffset, wantOffset,
-				wantOffset2)
-			continue
-		}
-
-		// Since it is possible that the time.Now call in AdjustedTime
-		// and the time.Now call here in the tests will be off by one
-		// second, allow a fudge factor to compensate.
-		adjustedTime := filter.AdjustedTime()
-		now := time.Unix(time.Now().Unix(), 0)
-		wantTime := now.Add(filter.Offset())
-		wantTime2 := now.Add(filter.Offset() - time.Second)
-		if !adjustedTime.Equal(wantTime) && !adjustedTime.Equal(wantTime2) {
-			t.Errorf("AdjustedTime #%d: unexpected result -- got %v, "+
-				"want %v or %v", i, adjustedTime, wantTime,
-				wantTime2)
-			continue
-		}
-	}
-}

blockdag/mining.go

@@ -0,0 +1,129 @@
+package blockdag
+
+import (
+	"bytes"
+	"encoding/binary"
+	"time"
+
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/mstime"
+)
+
+// BlockForMining returns a block with the given transactions
+// that points to the current DAG tips, that is valid from
+// all aspects except proof of work.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) BlockForMining(transactions []*util.Tx) (*domainmessage.MsgBlock, error) {
+	blockTimestamp := dag.NextBlockTime()
+	requiredDifficulty := dag.NextRequiredDifficulty(blockTimestamp)
+
+	// Calculate the next expected block version based on the state of the
+	// rule change deployments.
+	nextBlockVersion, err := dag.CalcNextBlockVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a new block ready to be solved.
+	hashMerkleTree := BuildHashMerkleTreeStore(transactions)
+	acceptedIDMerkleRoot, err := dag.NextAcceptedIDMerkleRootNoLock()
+	if err != nil {
+		return nil, err
+	}
+	var msgBlock domainmessage.MsgBlock
+	for _, tx := range transactions {
+		msgBlock.AddTransaction(tx.MsgTx())
+	}
+
+	multiset, err := dag.NextBlockMultiset()
+	if err != nil {
+		return nil, err
+	}
+
+	msgBlock.Header = domainmessage.BlockHeader{
+		Version:              nextBlockVersion,
+		ParentHashes:         dag.TipHashes(),
+		HashMerkleRoot:       hashMerkleTree.Root(),
+		AcceptedIDMerkleRoot: acceptedIDMerkleRoot,
+		UTXOCommitment:       (*daghash.Hash)(multiset.Finalize()),
+		Timestamp:            blockTimestamp,
+		Bits:                 requiredDifficulty,
+	}
+
+	return &msgBlock, nil
+}
+
+// NextBlockMultiset returns the multiset of an assumed next block
+// built on top of the current tips.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) NextBlockMultiset() (*secp256k1.MultiSet, error) {
+	_, selectedParentPastUTXO, txsAcceptanceData, err := dag.pastUTXO(&dag.virtual.blockNode)
+	if err != nil {
+		return nil, err
+	}
+
+	return dag.virtual.blockNode.calcMultiset(dag, txsAcceptanceData, selectedParentPastUTXO)
+}
+
+// CoinbasePayloadExtraData returns coinbase payload extra data parameter
+// which is built from extra nonce and coinbase flags.
+func CoinbasePayloadExtraData(extraNonce uint64, coinbaseFlags string) ([]byte, error) {
+	extraNonceBytes := make([]byte, 8)
+	binary.LittleEndian.PutUint64(extraNonceBytes, extraNonce)
+	w := &bytes.Buffer{}
+	_, err := w.Write(extraNonceBytes)
+	if err != nil {
+		return nil, err
+	}
+	_, err = w.Write([]byte(coinbaseFlags))
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+// NextCoinbaseFromAddress returns a coinbase transaction for the
+// next block with the given address and extra data in its payload.
+func (dag *BlockDAG) NextCoinbaseFromAddress(payToAddress util.Address, extraData []byte) (*util.Tx, error) {
+	coinbasePayloadScriptPubKey, err := txscript.PayToAddrScript(payToAddress)
+	if err != nil {
+		return nil, err
+	}
+	coinbaseTx, err := dag.NextBlockCoinbaseTransactionNoLock(coinbasePayloadScriptPubKey, extraData)
+	if err != nil {
+		return nil, err
+	}
+	return coinbaseTx, nil
+}
+
+// NextBlockMinimumTime returns the minimum allowed timestamp for a block building
+// on the end of the DAG. In particular, it is one second after
+// the median timestamp of the last several blocks per the DAG consensus
+// rules.
+func (dag *BlockDAG) NextBlockMinimumTime() mstime.Time {
+	return dag.CalcPastMedianTime().Add(time.Millisecond)
+}
+
+// NextBlockTime returns a valid block time for the
+// next block that will point to the existing DAG tips.
+func (dag *BlockDAG) NextBlockTime() mstime.Time {
+	// The timestamp for the block must not be before the median timestamp
+	// of the last several blocks. Thus, choose the maximum between the
+	// current time and one second after the past median time. The current
+	// timestamp is truncated to a millisecond boundary before comparison since a
+	// block timestamp does not supported a precision greater than one
+	// millisecond.
+	newTimestamp := dag.Now()
+	minTimestamp := dag.NextBlockMinimumTime()
+	if newTimestamp.Before(minTimestamp) {
+		newTimestamp = minTimestamp
+	}
+
+	return newTimestamp
+}

blockdag/multisetio.go

@@ -0,0 +1,29 @@
+package blockdag
+
+import (
+	"encoding/binary"
+	"github.com/kaspanet/go-secp256k1"
+	"io"
+)
+
+const multisetPointSize = 32
+
+// serializeMultiset serializes an ECMH multiset.
+func serializeMultiset(w io.Writer, ms *secp256k1.MultiSet) error {
+	serialized := ms.Serialize()
+	err := binary.Write(w, byteOrder, serialized)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// deserializeMultiset deserializes an EMCH multiset.
+func deserializeMultiset(r io.Reader) (*secp256k1.MultiSet, error) {
+	serialized := &secp256k1.SerializedMultiSet{}
+	err := binary.Read(r, byteOrder, serialized[:])
+	if err != nil {
+		return nil, err
+	}
+	return secp256k1.DeserializeMultiSet(serialized)
+}

blockdag/multisetstore.go

@@ -0,0 +1,131 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/locks"
+	"github.com/pkg/errors"
+)
+
+type multisetStore struct {
+	dag    *BlockDAG
+	new    map[daghash.Hash]struct{}
+	loaded map[daghash.Hash]secp256k1.MultiSet
+	mtx    *locks.PriorityMutex
+}
+
+func newMultisetStore(dag *BlockDAG) *multisetStore {
+	return &multisetStore{
+		dag:    dag,
+		new:    make(map[daghash.Hash]struct{}),
+		loaded: make(map[daghash.Hash]secp256k1.MultiSet),
+	}
+}
+
+func (store *multisetStore) setMultiset(node *blockNode, ms *secp256k1.MultiSet) {
+	store.loaded[*node.hash] = *ms
+	store.addToNewBlocks(node.hash)
+}
+
+func (store *multisetStore) addToNewBlocks(blockHash *daghash.Hash) {
+	store.new[*blockHash] = struct{}{}
+}
+
+func multisetNotFoundError(blockHash *daghash.Hash) error {
+	return errors.Errorf("Couldn't find multiset data for block %s", blockHash)
+}
+
+func (store *multisetStore) multisetByBlockNode(node *blockNode) (*secp256k1.MultiSet, error) {
+	ms, exists := store.multisetByBlockHash(node.hash)
+	if !exists {
+		return nil, multisetNotFoundError(node.hash)
+	}
+	return ms, nil
+}
+
+func (store *multisetStore) multisetByBlockHash(hash *daghash.Hash) (*secp256k1.MultiSet, bool) {
+	ms, ok := store.loaded[*hash]
+	return &ms, ok
+}
+
+// flushToDB writes all new multiset data to the database.
+func (store *multisetStore) flushToDB(dbContext *dbaccess.TxContext) error {
+	if len(store.new) == 0 {
+		return nil
+	}
+
+	w := &bytes.Buffer{}
+	for hash := range store.new {
+		hash := hash // Copy hash to a new variable to avoid passing the same pointer
+
+		w.Reset()
+		ms, exists := store.loaded[hash]
+		if !exists {
+			return multisetNotFoundError(&hash)
+		}
+
+		err := serializeMultiset(w, &ms)
+		if err != nil {
+			return err
+		}
+
+		err = store.storeMultiset(dbContext, &hash, w.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *multisetStore) clearNewEntries() {
+	store.new = make(map[daghash.Hash]struct{})
+}
+
+func (store *multisetStore) init(dbContext dbaccess.Context) error {
+	cursor, err := dbaccess.MultisetCursor(dbContext)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		key, err := cursor.Key()
+		if err != nil {
+			return err
+		}
+
+		hash, err := daghash.NewHash(key.Suffix())
+		if err != nil {
+			return err
+		}
+
+		serializedMS, err := cursor.Value()
+		if err != nil {
+			return err
+		}
+
+		ms, err := deserializeMultiset(bytes.NewReader(serializedMS))
+		if err != nil {
+			return err
+		}
+
+		store.loaded[*hash] = *ms
+	}
+	return nil
+}
+
+// storeMultiset stores the multiset data to the database.
+func (store *multisetStore) storeMultiset(dbContext dbaccess.Context, blockHash *daghash.Hash, serializedMS []byte) error {
+	exists, err := dbaccess.HasMultiset(dbContext, blockHash)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		return errors.Errorf("Can't override an existing multiset database entry for block %s", blockHash)
+	}
+
+	return dbaccess.StoreMultiset(dbContext, blockHash, serializedMS)
+}

blockdag/notifications.go

@@ -57,8 +57,8 @@ type Notification struct {
 // NotificationType for details on the types and contents of notifications.
 func (dag *BlockDAG) Subscribe(callback NotificationCallback) {
 	dag.notificationsLock.Lock()
+	defer dag.notificationsLock.Unlock()
 	dag.notifications = append(dag.notifications, callback)
-	dag.notificationsLock.Unlock()
 }
 
 // sendNotification sends a notification with the passed type and data if the
@@ -68,10 +68,10 @@ func (dag *BlockDAG) sendNotification(typ NotificationType, data interface{}) {
 	// Generate and send the notification.
 	n := Notification{Type: typ, Data: data}
 	dag.notificationsLock.RLock()
+	defer dag.notificationsLock.RUnlock()
 	for _, callback := range dag.notifications {
 		callback(&n)
 	}
-	dag.notificationsLock.RUnlock()
 }
 
 // BlockAddedNotificationData defines data to be sent along with a BlockAdded

blockdag/notifications_test.go

@@ -19,8 +19,8 @@ func TestNotifications(t *testing.T) {
 	}
 
 	// Create a new database and dag instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("notifications", Config{
-		DAGParams: &dagconfig.SimNetParams,
+	dag, teardownFunc, err := DAGSetup("notifications", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Fatalf("Failed to setup dag instance: %v", err)
@@ -41,11 +41,11 @@ func TestNotifications(t *testing.T) {
 		dag.Subscribe(callback)
 	}
 
-	isOrphan, delay, err := dag.ProcessBlock(blocks[1], BFNone)
+	isOrphan, isDelayed, err := dag.ProcessBlock(blocks[1], BFNone)
 	if err != nil {
 		t.Fatalf("ProcessBlock fail on block 1: %v\n", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("ProcessBlock: block 1 " +
 			"is too far in the future")
 	}

blockdag/phantom.go

@@ -1,105 +0,0 @@
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// phantom calculates and returns the block's blue set, selected parent and blue score.
-// Chain start is determined by going down the DAG through the selected path
-// (follow the selected parent of each block) k + 1 steps.
-// The blue set of a block are all blue blocks in its past.
-// To optimize memory usage, for each block we are storing only the blue blocks in
-// its selected parent's anticone that are in the future of the chain start
-// as well as the selected parent itself - the rest of the
-// blue set can be restored by traversing the selected parent chain and combining
-// the .blues of all blocks in it.
-// The blue score is the total number of blocks in this block's blue set
-// of the selected parent. (the blue score of the genesis block is defined as 0)
-// The selected parent is chosen by determining which block's parent will give this block the highest blue score.
-func phantom(block *blockNode, k uint32) (blues []*blockNode, selectedParent *blockNode, score uint64) {
-	bestScore := uint64(0)
-	var bestParent *blockNode
-	var bestBlues []*blockNode
-	var bestHash *daghash.Hash
-	for _, parent := range block.parents {
-		chainStart := digToChainStart(parent, k)
-		candidates := blueCandidates(chainStart)
-		blues := traverseCandidates(block, candidates, parent)
-		score := uint64(len(blues)) + parent.blueScore
-
-		if score > bestScore || (score == bestScore && (bestHash == nil || daghash.Less(parent.hash, bestHash))) {
-			bestScore = score
-			bestBlues = blues
-			bestParent = parent
-			bestHash = parent.hash
-		}
-	}
-
-	return bestBlues, bestParent, bestScore
-}
-
-// digToChainStart digs through the selected path and returns the block in depth k+1
-func digToChainStart(parent *blockNode, k uint32) *blockNode {
-	current := parent
-
-	for i := uint32(0); i < k; i++ {
-		if current.isGenesis() {
-			break
-		}
-		current = current.selectedParent
-	}
-
-	return current
-}
-
-func blueCandidates(chainStart *blockNode) blockSet {
-	candidates := newSet()
-	candidates.add(chainStart)
-
-	queue := []*blockNode{chainStart}
-	for len(queue) > 0 {
-		var current *blockNode
-		current, queue = queue[0], queue[1:]
-
-		children := current.children
-		for _, child := range children {
-			if !candidates.contains(child) {
-				candidates.add(child)
-				queue = append(queue, child)
-			}
-		}
-	}
-
-	return candidates
-}
-
-//traverseCandidates returns all the blocks that are in the future of the chain start and in the anticone of the selected parent
-func traverseCandidates(newBlock *blockNode, candidates blockSet, selectedParent *blockNode) []*blockNode {
-	blues := []*blockNode{}
-	selectedParentPast := newSet()
-	queue := newDownHeap()
-	visited := newSet()
-
-	for _, parent := range newBlock.parents {
-		queue.Push(parent)
-	}
-
-	for queue.Len() > 0 {
-		current := queue.pop()
-		if candidates.contains(current) {
-			if current == selectedParent || selectedParentPast.anyChildInSet(current) {
-				selectedParentPast.add(current)
-			} else {
-				blues = append(blues, current)
-			}
-			for _, parent := range current.parents {
-				if !visited.contains(parent) {
-					visited.add(parent)
-					queue.Push(parent)
-				}
-			}
-		}
-	}
-
-	return append(blues, selectedParent)
-}

blockdag/phantom_test.go

@@ -1,892 +0,0 @@
-package blockdag
-
-import (
-	"fmt"
-	"reflect"
-	"sort"
-	"testing"
-	"time"
-
-	"github.com/kaspanet/kaspad/util/daghash"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-)
-
-type testBlockData struct {
-	parents                []string
-	id                     string //id is a virtual entity that is used only for tests so we can define relations between blocks without knowing their hash
-	expectedScore          uint64
-	expectedSelectedParent string
-	expectedBlues          []string
-}
-
-//TestPhantom iterate over several dag simulations, and checks
-//that the blue score, blue set and selected parent of each
-//block calculated as expected
-func TestPhantom(t *testing.T) {
-	netParams := dagconfig.SimNetParams
-
-	blockVersion := int32(0x10000000)
-
-	tests := []struct {
-		k              uint32
-		dagData        []*testBlockData
-		virtualBlockID string
-		expectedReds   []string
-	}{
-		{
-			//Block hash order:AKJIHGFEDCB
-			k:              1,
-			virtualBlockID: "K",
-			expectedReds:   []string{"D"},
-			dagData: []*testBlockData{
-				{
-					parents:                []string{"A"},
-					id:                     "B",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "C",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "D",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "E",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "F",
-					expectedScore:          2,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"C", "D"},
-					id:                     "G",
-					expectedScore:          4,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"D", "B", "C"},
-				},
-				{
-					parents:                []string{"C", "E"},
-					id:                     "H",
-					expectedScore:          4,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"E", "B", "C"},
-				},
-				{
-					parents:                []string{"E", "G"},
-					id:                     "I",
-					expectedScore:          5,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"G", "D", "E"},
-				},
-				{
-					parents:                []string{"F"},
-					id:                     "J",
-					expectedScore:          3,
-					expectedSelectedParent: "F",
-					expectedBlues:          []string{"F"},
-				},
-				{
-					parents:                []string{"H", "I", "J"},
-					id:                     "K",
-					expectedScore:          9,
-					expectedSelectedParent: "H",
-					expectedBlues:          []string{"I", "G", "J", "F", "H"},
-				},
-			},
-		},
-		{
-			//block hash order:AVUTSRQPONMLKJIHGFEDCB
-			k:              2,
-			virtualBlockID: "V",
-			expectedReds:   []string{"D", "J", "P"},
-			dagData: []*testBlockData{
-				{
-					parents:                []string{"A"},
-					id:                     "B",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "C",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "D",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "E",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "F",
-					expectedScore:          2,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "G",
-					expectedScore:          2,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"G"},
-					id:                     "H",
-					expectedScore:          3,
-					expectedSelectedParent: "G",
-					expectedBlues:          []string{"G"},
-				},
-				{
-					parents:                []string{"E"},
-					id:                     "I",
-					expectedScore:          3,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"E"},
-				},
-				{
-					parents:                []string{"E"},
-					id:                     "J",
-					expectedScore:          3,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"E"},
-				},
-				{
-					parents:                []string{"I"},
-					id:                     "K",
-					expectedScore:          4,
-					expectedSelectedParent: "I",
-					expectedBlues:          []string{"I"},
-				},
-				{
-					parents:                []string{"K", "H"},
-					id:                     "L",
-					expectedScore:          5,
-					expectedSelectedParent: "K",
-					expectedBlues:          []string{"K"},
-				},
-				{
-					parents:                []string{"F", "L"},
-					id:                     "M",
-					expectedScore:          10,
-					expectedSelectedParent: "F",
-					expectedBlues:          []string{"L", "K", "I", "H", "G", "E", "B", "F"},
-				},
-				{
-					parents:                []string{"G", "K"},
-					id:                     "N",
-					expectedScore:          7,
-					expectedSelectedParent: "G",
-					expectedBlues:          []string{"K", "I", "E", "B", "G"},
-				},
-				{
-					parents:                []string{"J", "N"},
-					id:                     "O",
-					expectedScore:          8,
-					expectedSelectedParent: "N",
-					expectedBlues:          []string{"N"},
-				},
-				{
-					parents:                []string{"D"},
-					id:                     "P",
-					expectedScore:          3,
-					expectedSelectedParent: "D",
-					expectedBlues:          []string{"D"},
-				},
-				{
-					parents:                []string{"O", "P"},
-					id:                     "Q",
-					expectedScore:          10,
-					expectedSelectedParent: "P",
-					expectedBlues:          []string{"O", "N", "K", "J", "I", "E", "P"},
-				},
-				{
-					parents:                []string{"L", "Q"},
-					id:                     "R",
-					expectedScore:          11,
-					expectedSelectedParent: "Q",
-					expectedBlues:          []string{"Q"},
-				},
-				{
-					parents:                []string{"M", "R"},
-					id:                     "S",
-					expectedScore:          15,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"R", "Q", "O", "N", "M"},
-				},
-				{
-					parents:                []string{"H", "F"},
-					id:                     "T",
-					expectedScore:          5,
-					expectedSelectedParent: "F",
-					expectedBlues:          []string{"H", "G", "F"},
-				},
-				{
-					parents:                []string{"M", "T"},
-					id:                     "U",
-					expectedScore:          12,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"T", "M"},
-				},
-				{
-					parents:                []string{"S", "U"},
-					id:                     "V",
-					expectedScore:          18,
-					expectedSelectedParent: "S",
-					expectedBlues:          []string{"U", "T", "S"},
-				},
-			},
-		},
-		{
-			//Block hash order:AXWVUTSRQPONMLKJIHGFEDCB
-			k:              1,
-			virtualBlockID: "X",
-			expectedReds:   []string{"D", "F", "G", "H", "J", "K", "L", "N", "O", "Q", "R", "S", "U", "V"},
-			dagData: []*testBlockData{
-				{
-					parents:                []string{"A"},
-					id:                     "B",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "C",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "D",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "E",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "F",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "G",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "H",
-					expectedScore:          2,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "I",
-					expectedScore:          2,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "J",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"D"},
-					id:                     "K",
-					expectedScore:          2,
-					expectedSelectedParent: "D",
-					expectedBlues:          []string{"D"},
-				},
-				{
-					parents:                []string{"D"},
-					id:                     "L",
-					expectedScore:          2,
-					expectedSelectedParent: "D",
-					expectedBlues:          []string{"D"},
-				},
-				{
-					parents:                []string{"E"},
-					id:                     "M",
-					expectedScore:          2,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"E"},
-				},
-				{
-					parents:                []string{"E"},
-					id:                     "N",
-					expectedScore:          2,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"E"},
-				},
-				{
-					parents:                []string{"F", "G", "J"},
-					id:                     "O",
-					expectedScore:          5,
-					expectedSelectedParent: "F",
-					expectedBlues:          []string{"J", "G", "F"},
-				},
-				{
-					parents:                []string{"B", "M", "I"},
-					id:                     "P",
-					expectedScore:          6,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"M", "I", "E", "C", "B"},
-				},
-				{
-					parents:                []string{"K", "E"},
-					id:                     "Q",
-					expectedScore:          4,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"K", "D", "E"},
-				},
-				{
-					parents:                []string{"L", "N"},
-					id:                     "R",
-					expectedScore:          3,
-					expectedSelectedParent: "L",
-					expectedBlues:          []string{"L"},
-				},
-				{
-					parents:                []string{"I", "Q"},
-					id:                     "S",
-					expectedScore:          5,
-					expectedSelectedParent: "Q",
-					expectedBlues:          []string{"Q"},
-				},
-				{
-					parents:                []string{"K", "P"},
-					id:                     "T",
-					expectedScore:          7,
-					expectedSelectedParent: "P",
-					expectedBlues:          []string{"P"},
-				},
-				{
-					parents:                []string{"K", "L"},
-					id:                     "U",
-					expectedScore:          4,
-					expectedSelectedParent: "K",
-					expectedBlues:          []string{"L", "K"},
-				},
-				{
-					parents:                []string{"U", "R"},
-					id:                     "V",
-					expectedScore:          5,
-					expectedSelectedParent: "R",
-					expectedBlues:          []string{"U", "R"},
-				},
-				{
-					parents:                []string{"S", "U", "T"},
-					id:                     "W",
-					expectedScore:          8,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"T"},
-				},
-				{
-					parents:                []string{"V", "W", "H"},
-					id:                     "X",
-					expectedScore:          9,
-					expectedSelectedParent: "W",
-					expectedBlues:          []string{"W"},
-				},
-			},
-		},
-		{
-			//Secret mining attack: The attacker is mining
-			//blocks B,C,D,E,F,G,T in secret without propagating
-			//them, so all blocks except T should be red, because
-			//they don't follow the rules of PHANTOM that require
-			//you to point to all the parents that you know, and
-			//propagate your block as soon as it's mined
-
-			//Block hash order:AYXWVUTSRQPONMLKJIHGFEDCB
-			k:              1,
-			virtualBlockID: "Y",
-			expectedReds:   []string{"B", "C", "D", "E", "F", "G", "L"},
-			dagData: []*testBlockData{
-				{
-					parents:                []string{"A"},
-					id:                     "B",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "C",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "D",
-					expectedScore:          3,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"D"},
-					id:                     "E",
-					expectedScore:          4,
-					expectedSelectedParent: "D",
-					expectedBlues:          []string{"D"},
-				},
-				{
-					parents:                []string{"E"},
-					id:                     "F",
-					expectedScore:          5,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"E"},
-				},
-				{
-					parents:                []string{"F"},
-					id:                     "G",
-					expectedScore:          6,
-					expectedSelectedParent: "F",
-					expectedBlues:          []string{"F"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "H",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "I",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"H", "I"},
-					id:                     "J",
-					expectedScore:          3,
-					expectedSelectedParent: "H",
-					expectedBlues:          []string{"I", "H"},
-				},
-				{
-					parents:                []string{"H", "I"},
-					id:                     "K",
-					expectedScore:          3,
-					expectedSelectedParent: "H",
-					expectedBlues:          []string{"I", "H"},
-				},
-				{
-					parents:                []string{"I"},
-					id:                     "L",
-					expectedScore:          2,
-					expectedSelectedParent: "I",
-					expectedBlues:          []string{"I"},
-				},
-				{
-					parents:                []string{"J", "K", "L"},
-					id:                     "M",
-					expectedScore:          5,
-					expectedSelectedParent: "J",
-					expectedBlues:          []string{"K", "J"},
-				},
-				{
-					parents:                []string{"J", "K", "L"},
-					id:                     "N",
-					expectedScore:          5,
-					expectedSelectedParent: "J",
-					expectedBlues:          []string{"K", "J"},
-				},
-				{
-					parents:                []string{"N", "M"},
-					id:                     "O",
-					expectedScore:          7,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"N", "M"},
-				},
-				{
-					parents:                []string{"N", "M"},
-					id:                     "P",
-					expectedScore:          7,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"N", "M"},
-				},
-				{
-					parents:                []string{"N", "M"},
-					id:                     "Q",
-					expectedScore:          7,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"N", "M"},
-				},
-				{
-					parents:                []string{"O", "P", "Q"},
-					id:                     "R",
-					expectedScore:          10,
-					expectedSelectedParent: "O",
-					expectedBlues:          []string{"Q", "P", "O"},
-				},
-				{
-					parents:                []string{"O", "P", "Q"},
-					id:                     "S",
-					expectedScore:          10,
-					expectedSelectedParent: "O",
-					expectedBlues:          []string{"Q", "P", "O"},
-				},
-				{
-					parents:                []string{"G", "S", "R"},
-					id:                     "T",
-					expectedScore:          12,
-					expectedSelectedParent: "R",
-					expectedBlues:          []string{"S", "R"},
-				},
-				{
-					parents:                []string{"S", "R"},
-					id:                     "U",
-					expectedScore:          12,
-					expectedSelectedParent: "R",
-					expectedBlues:          []string{"S", "R"},
-				},
-				{
-					parents:                []string{"T", "U"},
-					id:                     "V",
-					expectedScore:          14,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"U", "T"},
-				},
-				{
-					parents:                []string{"T", "U"},
-					id:                     "W",
-					expectedScore:          14,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"U", "T"},
-				},
-				{
-					parents:                []string{"U", "T"},
-					id:                     "X",
-					expectedScore:          14,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"U", "T"},
-				},
-				{
-					parents:                []string{"V", "W", "X"},
-					id:                     "Y",
-					expectedScore:          17,
-					expectedSelectedParent: "V",
-					expectedBlues:          []string{"X", "W", "V"},
-				},
-			},
-		},
-		{
-			//Censorship mining attack: The attacker is mining blocks B,C,D,E,F,G in secret without propagating them,
-			//so all blocks except B and C should be red, because they don't follow the rules of
-			//PHANTOM that require you to point to all the parents that you know
-
-			//Block hash order:AYXWVUTSRQPONMLKJIHGFEDCB
-			k:              1,
-			virtualBlockID: "Y",
-			expectedReds:   []string{"D", "E", "F", "G", "L"},
-			dagData: []*testBlockData{
-				{
-					parents:                []string{"A"},
-					id:                     "B",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"B"},
-					id:                     "C",
-					expectedScore:          2,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"B"},
-				},
-				{
-					parents:                []string{"C"},
-					id:                     "D",
-					expectedScore:          3,
-					expectedSelectedParent: "C",
-					expectedBlues:          []string{"C"},
-				},
-				{
-					parents:                []string{"D"},
-					id:                     "E",
-					expectedScore:          4,
-					expectedSelectedParent: "D",
-					expectedBlues:          []string{"D"},
-				},
-				{
-					parents:                []string{"E"},
-					id:                     "F",
-					expectedScore:          5,
-					expectedSelectedParent: "E",
-					expectedBlues:          []string{"E"},
-				},
-				{
-					parents:                []string{"F"},
-					id:                     "G",
-					expectedScore:          6,
-					expectedSelectedParent: "F",
-					expectedBlues:          []string{"F"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "H",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"A"},
-					id:                     "I",
-					expectedScore:          1,
-					expectedSelectedParent: "A",
-					expectedBlues:          []string{"A"},
-				},
-				{
-					parents:                []string{"H", "I", "B"},
-					id:                     "J",
-					expectedScore:          4,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"I", "H", "B"},
-				},
-				{
-					parents:                []string{"H", "I", "B"},
-					id:                     "K",
-					expectedScore:          4,
-					expectedSelectedParent: "B",
-					expectedBlues:          []string{"I", "H", "B"},
-				},
-				{
-					parents:                []string{"I"},
-					id:                     "L",
-					expectedScore:          2,
-					expectedSelectedParent: "I",
-					expectedBlues:          []string{"I"},
-				},
-				{
-					parents:                []string{"J", "K", "L", "C"},
-					id:                     "M",
-					expectedScore:          7,
-					expectedSelectedParent: "J",
-					expectedBlues:          []string{"K", "C", "J"},
-				},
-				{
-					parents:                []string{"J", "K", "L", "C"},
-					id:                     "N",
-					expectedScore:          7,
-					expectedSelectedParent: "J",
-					expectedBlues:          []string{"K", "C", "J"},
-				},
-				{
-					parents:                []string{"N", "M", "D"},
-					id:                     "O",
-					expectedScore:          9,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"N", "M"},
-				},
-				{
-					parents:                []string{"N", "M", "D"},
-					id:                     "P",
-					expectedScore:          9,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"N", "M"},
-				},
-				{
-					parents:                []string{"N", "M", "D"},
-					id:                     "Q",
-					expectedScore:          9,
-					expectedSelectedParent: "M",
-					expectedBlues:          []string{"N", "M"},
-				},
-				{
-					parents:                []string{"O", "P", "Q", "E"},
-					id:                     "R",
-					expectedScore:          12,
-					expectedSelectedParent: "O",
-					expectedBlues:          []string{"Q", "P", "O"},
-				},
-				{
-					parents:                []string{"O", "P", "Q", "E"},
-					id:                     "S",
-					expectedScore:          12,
-					expectedSelectedParent: "O",
-					expectedBlues:          []string{"Q", "P", "O"},
-				},
-				{
-					parents:                []string{"G", "S", "R"},
-					id:                     "T",
-					expectedScore:          14,
-					expectedSelectedParent: "R",
-					expectedBlues:          []string{"S", "R"},
-				},
-				{
-					parents:                []string{"S", "R", "F"},
-					id:                     "U",
-					expectedScore:          14,
-					expectedSelectedParent: "R",
-					expectedBlues:          []string{"S", "R"},
-				},
-				{
-					parents:                []string{"T", "U"},
-					id:                     "V",
-					expectedScore:          16,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"U", "T"},
-				},
-				{
-					parents:                []string{"T", "U"},
-					id:                     "W",
-					expectedScore:          16,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"U", "T"},
-				},
-				{
-					parents:                []string{"T", "U"},
-					id:                     "X",
-					expectedScore:          16,
-					expectedSelectedParent: "T",
-					expectedBlues:          []string{"U", "T"},
-				},
-				{
-					parents:                []string{"V", "W", "X"},
-					id:                     "Y",
-					expectedScore:          19,
-					expectedSelectedParent: "V",
-					expectedBlues:          []string{"X", "W", "V"},
-				},
-			},
-		},
-	}
-
-	for i, test := range tests {
-		netParams.K = test.k
-		// Generate enough synthetic blocks for the rest of the test
-		blockDAG := newTestDAG(&netParams)
-		genesisNode := blockDAG.genesis
-		blockTime := genesisNode.Header().Timestamp
-		blockByIDMap := make(map[string]*blockNode)
-		idByBlockMap := make(map[*blockNode]string)
-		blockByIDMap["A"] = genesisNode
-		idByBlockMap[genesisNode] = "A"
-
-		for _, blockData := range test.dagData {
-			blockTime = blockTime.Add(time.Second)
-			parents := blockSet{}
-			for _, parentID := range blockData.parents {
-				parent := blockByIDMap[parentID]
-				parents.add(parent)
-			}
-			node := newTestNode(parents, blockVersion, 0, blockTime, test.k)
-			node.hash = &daghash.Hash{} //It helps to predict hash order
-			for i, char := range blockData.id {
-				node.hash[i] = byte(char)
-			}
-
-			blockDAG.index.AddNode(node)
-			addNodeAsChildToParents(node)
-
-			blockByIDMap[blockData.id] = node
-			idByBlockMap[node] = blockData.id
-
-			bluesIDs := make([]string, 0, len(node.blues))
-			for _, blue := range node.blues {
-				bluesIDs = append(bluesIDs, idByBlockMap[blue])
-			}
-			selectedParentID := idByBlockMap[node.selectedParent]
-			fullDataStr := fmt.Sprintf("blues: %v, selectedParent: %v, score: %v",
-				bluesIDs, selectedParentID, node.blueScore)
-			if blockData.expectedScore != node.blueScore {
-				t.Errorf("Test %d: Block %v expected to have score %v but got %v (fulldata: %v)",
-					i, blockData.id, blockData.expectedScore, node.blueScore, fullDataStr)
-			}
-			if blockData.expectedSelectedParent != selectedParentID {
-				t.Errorf("Test %d: Block %v expected to have selected parent %v but got %v (fulldata: %v)",
-					i, blockData.id, blockData.expectedSelectedParent, selectedParentID, fullDataStr)
-			}
-			if !reflect.DeepEqual(blockData.expectedBlues, bluesIDs) {
-				t.Errorf("Test %d: Block %v expected to have blues %v but got %v (fulldata: %v)",
-					i, blockData.id, blockData.expectedBlues, bluesIDs, fullDataStr)
-			}
-		}
-
-		reds := make(map[string]bool)
-
-		for id := range blockByIDMap {
-			reds[id] = true
-		}
-
-		for tip := blockByIDMap[test.virtualBlockID]; tip.selectedParent != nil; tip = tip.selectedParent {
-			tipID := idByBlockMap[tip]
-			delete(reds, tipID)
-			for _, blue := range tip.blues {
-				blueID := idByBlockMap[blue]
-				delete(reds, blueID)
-			}
-		}
-		if !checkReds(test.expectedReds, reds) {
-			redsIDs := make([]string, 0, len(reds))
-			for id := range reds {
-				redsIDs = append(redsIDs, id)
-			}
-			sort.Strings(redsIDs)
-			sort.Strings(test.expectedReds)
-			t.Errorf("Test %d: Expected reds %v but got %v", i, test.expectedReds, redsIDs)
-		}
-
-	}
-}
-
-func checkReds(expectedReds []string, reds map[string]bool) bool {
-	if len(expectedReds) != len(reds) {
-		return false
-	}
-	for _, redID := range expectedReds {
-		if !reds[redID] {
-			return false
-		}
-	}
-	return true
-}

blockdag/process.go

@@ -8,6 +8,9 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/pkg/errors"
+
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
@@ -43,15 +46,23 @@ const (
 	// in the block index but was never fully processed
 	BFWasStored
 
+	// BFDisallowDelay is set to indicate that a delayed block should be rejected.
+	// This is used for the case where a block is submitted through RPC.
+	BFDisallowDelay
+
+	// BFDisallowOrphans is set to indicate that an orphan block should be rejected.
+	// This is used for the case where a block is submitted through RPC.
+	BFDisallowOrphans
+
 	// BFNone is a convenience value to specifically indicate no flags.
 	BFNone BehaviorFlags = 0
 )
 
-// BlockExists determines whether a block with the given hash exists in
+// IsInDAG determines whether a block with the given hash exists in
 // the DAG.
 //
 // This function is safe for concurrent access.
-func (dag *BlockDAG) BlockExists(hash *daghash.Hash) bool {
+func (dag *BlockDAG) IsInDAG(hash *daghash.Hash) bool {
 	return dag.index.HaveBlock(hash)
 }
 
@@ -94,7 +105,8 @@ func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) err
 			// still missing.
 			_, err := lookupParentNodes(orphan.block, dag)
 			if err != nil {
-				if ruleErr, ok := err.(RuleError); ok && ruleErr.ErrorCode == ErrParentBlockUnknown {
+				var ruleErr RuleError
+				if ok := errors.As(err, &ruleErr); ok && ruleErr.ErrorCode == ErrParentBlockUnknown {
 					continue
 				}
 				return err
@@ -110,7 +122,7 @@ func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) err
 			if err != nil {
 				// Since we don't want to reject the original block because of
 				// a bad unorphaned child, only return an error if it's not a RuleError.
-				if _, ok := err.(RuleError); !ok {
+				if !errors.As(err, &RuleError{}) {
 					return err
 				}
 				log.Warnf("Verification failed for orphan block %s: %s", orphanHash, err)
@@ -134,49 +146,84 @@ func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) err
 // whether or not the block is an orphan.
 //
 // This function is safe for concurrent access.
-func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrphan bool, delay time.Duration, err error) {
+func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrphan bool, isDelayed bool, err error) {
 	dag.dagLock.Lock()
 	defer dag.dagLock.Unlock()
+	return dag.processBlockNoLock(block, flags)
+}
 
-	isDelayedBlock := flags&BFAfterDelay == BFAfterDelay
+func (dag *BlockDAG) processBlockNoLock(block *util.Block, flags BehaviorFlags) (isOrphan bool, isDelayed bool, err error) {
+	isAfterDelay := flags&BFAfterDelay == BFAfterDelay
 	wasBlockStored := flags&BFWasStored == BFWasStored
+	disallowDelay := flags&BFDisallowDelay == BFDisallowDelay
+	disallowOrphans := flags&BFDisallowOrphans == BFDisallowOrphans
 
 	blockHash := block.Hash()
 	log.Tracef("Processing block %s", blockHash)
 
 	// The block must not already exist in the DAG.
-	if dag.BlockExists(blockHash) && !wasBlockStored {
+	if dag.IsInDAG(blockHash) && !wasBlockStored {
 		str := fmt.Sprintf("already have block %s", blockHash)
-		return false, 0, ruleError(ErrDuplicateBlock, str)
+		return false, false, ruleError(ErrDuplicateBlock, str)
 	}
 
 	// The block must not already exist as an orphan.
 	if _, exists := dag.orphans[*blockHash]; exists {
 		str := fmt.Sprintf("already have block (orphan) %s", blockHash)
-		return false, 0, ruleError(ErrDuplicateBlock, str)
+		return false, false, ruleError(ErrDuplicateBlock, str)
 	}
 
-	if !isDelayedBlock {
+	if dag.isKnownDelayedBlock(blockHash) {
+		str := fmt.Sprintf("already have block (delayed) %s", blockHash)
+		return false, false, ruleError(ErrDuplicateBlock, str)
+	}
+
+	if !isAfterDelay {
 		// Perform preliminary sanity checks on the block and its transactions.
 		delay, err := dag.checkBlockSanity(block, flags)
 		if err != nil {
-			return false, 0, err
+			return false, false, err
+		}
+
+		if delay != 0 && disallowDelay {
+			str := fmt.Sprintf("Cannot process blocks beyond the allowed time offset while the BFDisallowDelay flag is raised %s", blockHash)
+			return false, true, ruleError(ErrDelayedBlockIsNotAllowed, str)
 		}
 
 		if delay != 0 {
-			return false, delay, err
+			err = dag.addDelayedBlock(block, delay)
+			if err != nil {
+				return false, false, err
+			}
+			return false, true, nil
 		}
 	}
 
+	var missingParents []*daghash.Hash
+	for _, parentHash := range block.MsgBlock().Header.ParentHashes {
+		if !dag.IsInDAG(parentHash) {
+			missingParents = append(missingParents, parentHash)
+		}
+	}
+	if len(missingParents) > 0 && disallowOrphans {
+		str := fmt.Sprintf("Cannot process orphan blocks while the BFDisallowOrphans flag is raised %s", blockHash)
+		return false, false, ruleError(ErrOrphanBlockIsNotAllowed, str)
+	}
+
+	// Handle the case of a block with a valid timestamp(non-delayed) which points to a delayed block.
+	delay, isParentDelayed := dag.maxDelayOfParents(missingParents)
+	if isParentDelayed {
+		// Add Millisecond to ensure that parent process time will be after its child.
+		delay += time.Millisecond
+		err := dag.addDelayedBlock(block, delay)
+		if err != nil {
+			return false, false, err
+		}
+		return false, true, err
+	}
+
 	// Handle orphan blocks.
-	allParentsExist := true
-	for _, parentHash := range block.MsgBlock().Header.ParentHashes {
-		if !dag.BlockExists(parentHash) {
-			allParentsExist = false
-		}
-	}
-
-	if !allParentsExist {
+	if len(missingParents) > 0 {
 		// Some orphans during netsync are a normal part of the process, since the anticone
 		// of the chain-split is never explicitly requested.
 		// Therefore, if we are during netsync - don't report orphans to default logs.
@@ -184,21 +231,21 @@ func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrp
 		// The number K*2 was chosen since in peace times anticone is limited to K blocks,
 		// while some red block can make it a bit bigger, but much more than that indicates
 		// there might be some problem with the netsync process.
-		if flags&BFIsSync == BFIsSync && uint32(len(dag.orphans)) < dag.dagParams.K*2 {
+		if flags&BFIsSync == BFIsSync && dagconfig.KType(len(dag.orphans)) < dag.Params.K*2 {
 			log.Debugf("Adding orphan block %s. This is normal part of netsync process", blockHash)
 		} else {
 			log.Infof("Adding orphan block %s", blockHash)
 		}
 		dag.addOrphanBlock(block)
 
-		return true, 0, nil
+		return true, false, nil
 	}
 
 	// The block has passed all context independent checks and appears sane
 	// enough to potentially accept it into the block DAG.
 	err = dag.maybeAcceptBlock(block, flags)
 	if err != nil {
-		return false, 0, err
+		return false, false, err
 	}
 
 	// Accept any orphan blocks that depend on this block (they are
@@ -206,10 +253,35 @@ func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrp
 	// there are no more.
 	err = dag.processOrphans(blockHash, flags)
 	if err != nil {
-		return false, 0, err
+		return false, false, err
 	}
 
+	if !isAfterDelay {
+		err = dag.processDelayedBlocks()
+		if err != nil {
+			return false, false, err
+		}
+	}
+
+	dag.addBlockProcessingTimestamp()
+
 	log.Debugf("Accepted block %s", blockHash)
 
-	return false, 0, nil
+	return false, false, nil
+}
+
+// maxDelayOfParents returns the maximum delay of the given block hashes.
+// Note that delay could be 0, but isDelayed will return true. This is the case where the parent process time is due.
+func (dag *BlockDAG) maxDelayOfParents(parentHashes []*daghash.Hash) (delay time.Duration, isDelayed bool) {
+	for _, parentHash := range parentHashes {
+		if delayedParent, exists := dag.delayedBlocks[*parentHash]; exists {
+			isDelayed = true
+			parentDelay := delayedParent.processTime.Sub(dag.Now())
+			if parentDelay > delay {
+				delay = parentDelay
+			}
+		}
+	}
+
+	return delay, isDelayed
 }

blockdag/process_test.go

@@ -1,76 +1,19 @@
 package blockdag
 
 import (
-	"bou.ke/monkey"
-	"fmt"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/daghash"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/kaspanet/kaspad/util"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
-func TestProcessBlock(t *testing.T) {
-	dag, teardownFunc, err := DAGSetup("TestProcessBlock", Config{
-		DAGParams: &dagconfig.SimNetParams,
-	})
-	if err != nil {
-		t.Errorf("Failed to setup dag instance: %v", err)
-		return
-	}
-	defer teardownFunc()
-
-	// Check that BFAfterDelay skip checkBlockSanity
-	called := false
-	guard := monkey.Patch((*BlockDAG).checkBlockSanity, func(_ *BlockDAG, _ *util.Block, _ BehaviorFlags) (time.Duration, error) {
-		called = true
-		return 0, nil
-	})
-	defer guard.Unpatch()
-
-	isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(&Block100000), BFNoPoWCheck)
-	if err != nil {
-		t.Errorf("ProcessBlock: %s", err)
-	}
-	if delay != 0 {
-		t.Errorf("ProcessBlock: block is too far in the future")
-	}
-	if !isOrphan {
-		t.Errorf("ProcessBlock: unexpected returned non orphan block")
-	}
-	if !called {
-		t.Errorf("ProcessBlock: expected checkBlockSanity to be called")
-	}
-
-	Block100000Copy := Block100000
-	// Change nonce to change block hash
-	Block100000Copy.Header.Nonce++
-	called = false
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(&Block100000Copy), BFAfterDelay|BFNoPoWCheck)
-	if err != nil {
-		t.Errorf("ProcessBlock: %s", err)
-	}
-	if delay != 0 {
-		t.Errorf("ProcessBlock: block is too far in the future")
-	}
-	if !isOrphan {
-		t.Errorf("ProcessBlock: unexpected returned non orphan block")
-	}
-	if called {
-		t.Errorf("ProcessBlock: Didn't expected checkBlockSanity to be called")
-	}
-
-	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(dagconfig.SimNetParams.GenesisBlock), BFNone)
-	expectedErrMsg := fmt.Sprintf("already have block %s", dagconfig.SimNetParams.GenesisHash)
-	if err == nil || err.Error() != expectedErrMsg {
-		t.Errorf("ProcessBlock: Expected error \"%s\" but got \"%s\"", expectedErrMsg, err)
-	}
-}
-
 func TestProcessOrphans(t *testing.T) {
-	dag, teardownFunc, err := DAGSetup("TestProcessOrphans", Config{
-		DAGParams: &dagconfig.SimNetParams,
+	dag, teardownFunc, err := DAGSetup("TestProcessOrphans", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Errorf("Failed to setup dag instance: %v", err)
@@ -97,11 +40,11 @@ func TestProcessOrphans(t *testing.T) {
 	childBlock.MsgBlock().Header.UTXOCommitment = &daghash.ZeroHash
 
 	// Process the child block so that it gets added to the orphan pool
-	isOrphan, delay, err := dag.ProcessBlock(childBlock, BFNoPoWCheck)
+	isOrphan, isDelayed, err := dag.ProcessBlock(childBlock, BFNoPoWCheck)
 	if err != nil {
 		t.Fatalf("TestProcessOrphans: child block unexpectedly returned an error: %s", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("TestProcessOrphans: child block is too far in the future")
 	}
 	if !isOrphan {
@@ -109,11 +52,11 @@ func TestProcessOrphans(t *testing.T) {
 	}
 
 	// Process the parent block. Note that this will attempt to unorphan the child block
-	isOrphan, delay, err = dag.ProcessBlock(parentBlock, BFNone)
+	isOrphan, isDelayed, err = dag.ProcessBlock(parentBlock, BFNone)
 	if err != nil {
 		t.Fatalf("TestProcessOrphans: parent block unexpectedly returned an error: %s", err)
 	}
-	if delay != 0 {
+	if isDelayed {
 		t.Fatalf("TestProcessOrphans: parent block is too far in the future")
 	}
 	if isOrphan {
@@ -121,11 +64,175 @@ func TestProcessOrphans(t *testing.T) {
 	}
 
 	// Make sure that the child block had been rejected
-	node := dag.index.LookupNode(childBlock.Hash())
-	if node == nil {
+	node, ok := dag.index.LookupNode(childBlock.Hash())
+	if !ok {
 		t.Fatalf("TestProcessOrphans: child block missing from block index")
 	}
 	if !dag.index.NodeStatus(node).KnownInvalid() {
 		t.Fatalf("TestProcessOrphans: child block erroneously not marked as invalid")
 	}
 }
+
+func TestProcessDelayedBlocks(t *testing.T) {
+	// We use dag1 so we can build the test blocks with the proper
+	// block header (UTXO commitment, acceptedIDMerkleroot, etc), and
+	// then we use dag2 for the actual test.
+	dag1, teardownFunc, err := DAGSetup("TestProcessDelayedBlocks1", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	isDAG1Open := true
+	defer func() {
+		if isDAG1Open {
+			teardownFunc()
+		}
+	}()
+
+	initialTime := dag1.Params.GenesisBlock.Header.Timestamp
+	// Here we use a fake time source that returns a timestamp
+	// one hour into the future to make delayedBlock artificially
+	// valid.
+	dag1.timeSource = newFakeTimeSource(initialTime.Add(time.Hour))
+
+	delayedBlock, err := PrepareBlockForTest(dag1, []*daghash.Hash{dag1.Params.GenesisBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+
+	blockDelay := time.Duration(dag1.Params.TimestampDeviationTolerance)*dag1.Params.TargetTimePerBlock + 5*time.Second
+	delayedBlock.Header.Timestamp = initialTime.Add(blockDelay)
+
+	isOrphan, isDelayed, err := dag1.ProcessBlock(util.NewBlock(delayedBlock), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is an orphan\n")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is delayed\n")
+	}
+
+	delayedBlockChild, err := PrepareBlockForTest(dag1, []*daghash.Hash{delayedBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+
+	teardownFunc()
+	isDAG1Open = false
+
+	// Here the actual test begins. We add a delayed block and
+	// its child and check that they are not added to the DAG,
+	// and check that they're added only if we add a new block
+	// after the delayed block timestamp is valid.
+	dag2, teardownFunc2, err := DAGSetup("TestProcessDelayedBlocks2", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc2()
+	dag2.timeSource = newFakeTimeSource(initialTime)
+
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(delayedBlock), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is an orphan\n")
+	}
+	if !isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is not delayed\n")
+	}
+
+	if dag2.IsInDAG(delayedBlock.BlockHash()) {
+		t.Errorf("dag.IsInDAG should return false for a delayed block")
+	}
+	if !dag2.IsKnownBlock(delayedBlock.BlockHash()) {
+		t.Errorf("dag.IsKnownBlock should return true for a a delayed block")
+	}
+
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(delayedBlockChild), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlockChild " +
+			"is an orphan\n")
+	}
+	if !isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlockChild " +
+			"is not delayed\n")
+	}
+
+	if dag2.IsInDAG(delayedBlockChild.BlockHash()) {
+		t.Errorf("dag.IsInDAG should return false for a child of a delayed block")
+	}
+	if !dag2.IsKnownBlock(delayedBlockChild.BlockHash()) {
+		t.Errorf("dag.IsKnownBlock should return true for a child of a delayed block")
+	}
+
+	blockBeforeDelay, err := PrepareBlockForTest(dag2, []*daghash.Hash{dag2.Params.GenesisBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(blockBeforeDelay), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is an orphan\n")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is delayed\n")
+	}
+
+	if dag2.IsInDAG(delayedBlock.BlockHash()) {
+		t.Errorf("delayedBlock shouldn't be added to the DAG because its time hasn't reached yet")
+	}
+	if dag2.IsInDAG(delayedBlockChild.BlockHash()) {
+		t.Errorf("delayedBlockChild shouldn't be added to the DAG because its parent is not in the DAG")
+	}
+
+	// We advance the clock to the point where delayedBlock timestamp is valid.
+	deviationTolerance := time.Duration(dag2.TimestampDeviationTolerance) * dag2.Params.TargetTimePerBlock
+	timeUntilDelayedBlockIsValid := delayedBlock.Header.Timestamp.
+		Add(-deviationTolerance).
+		Sub(dag2.Now()) +
+		time.Second
+	dag2.timeSource = newFakeTimeSource(initialTime.Add(timeUntilDelayedBlockIsValid))
+
+	blockAfterDelay, err := PrepareBlockForTest(dag2,
+		[]*daghash.Hash{dag2.Params.GenesisBlock.BlockHash()},
+		nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(blockAfterDelay), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is an orphan\n")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is not delayed\n")
+	}
+
+	if !dag2.IsInDAG(delayedBlock.BlockHash()) {
+		t.Fatalf("delayedBlock should be added to the DAG because its time has been reached")
+	}
+	if !dag2.IsInDAG(delayedBlockChild.BlockHash()) {
+		t.Errorf("delayedBlockChild shouldn't be added to the DAG because its parent has been added to the DAG")
+	}
+}

blockdag/reachabilitystore.go

@@ -0,0 +1,398 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+	"io"
+)
+
+type reachabilityData struct {
+	treeNode          *reachabilityTreeNode
+	futureCoveringSet futureCoveringTreeNodeSet
+}
+
+type reachabilityStore struct {
+	dag    *BlockDAG
+	dirty  map[daghash.Hash]struct{}
+	loaded map[daghash.Hash]*reachabilityData
+}
+
+func newReachabilityStore(dag *BlockDAG) *reachabilityStore {
+	return &reachabilityStore{
+		dag:    dag,
+		dirty:  make(map[daghash.Hash]struct{}),
+		loaded: make(map[daghash.Hash]*reachabilityData),
+	}
+}
+
+func (store *reachabilityStore) setTreeNode(treeNode *reachabilityTreeNode) {
+	// load the reachability data from DB to store.loaded
+	node := treeNode.blockNode
+	_, exists := store.reachabilityDataByHash(node.hash)
+	if !exists {
+		store.loaded[*node.hash] = &reachabilityData{}
+	}
+
+	store.loaded[*node.hash].treeNode = treeNode
+	store.setBlockAsDirty(node.hash)
+}
+
+func (store *reachabilityStore) setFutureCoveringSet(node *blockNode, futureCoveringSet futureCoveringTreeNodeSet) error {
+	// load the reachability data from DB to store.loaded
+	_, exists := store.reachabilityDataByHash(node.hash)
+	if !exists {
+		return reachabilityNotFoundError(node.hash)
+	}
+
+	store.loaded[*node.hash].futureCoveringSet = futureCoveringSet
+	store.setBlockAsDirty(node.hash)
+	return nil
+}
+
+func (store *reachabilityStore) setBlockAsDirty(blockHash *daghash.Hash) {
+	store.dirty[*blockHash] = struct{}{}
+}
+
+func reachabilityNotFoundError(hash *daghash.Hash) error {
+	return errors.Errorf("couldn't find reachability data for block %s", hash)
+}
+
+func (store *reachabilityStore) treeNodeByBlockHash(hash *daghash.Hash) (*reachabilityTreeNode, error) {
+	reachabilityData, exists := store.reachabilityDataByHash(hash)
+	if !exists {
+		return nil, reachabilityNotFoundError(hash)
+	}
+	return reachabilityData.treeNode, nil
+}
+
+func (store *reachabilityStore) treeNodeByBlockNode(node *blockNode) (*reachabilityTreeNode, error) {
+	return store.treeNodeByBlockHash(node.hash)
+}
+
+func (store *reachabilityStore) futureCoveringSetByBlockNode(node *blockNode) (futureCoveringTreeNodeSet, error) {
+	reachabilityData, exists := store.reachabilityDataByHash(node.hash)
+	if !exists {
+		return nil, reachabilityNotFoundError(node.hash)
+	}
+	return reachabilityData.futureCoveringSet, nil
+}
+
+func (store *reachabilityStore) reachabilityDataByHash(hash *daghash.Hash) (*reachabilityData, bool) {
+	reachabilityData, ok := store.loaded[*hash]
+	return reachabilityData, ok
+}
+
+// flushToDB writes all dirty reachability data to the database.
+func (store *reachabilityStore) flushToDB(dbContext *dbaccess.TxContext) error {
+	if len(store.dirty) == 0 {
+		return nil
+	}
+
+	for hash := range store.dirty {
+		hash := hash // Copy hash to a new variable to avoid passing the same pointer
+		reachabilityData := store.loaded[hash]
+		err := store.storeReachabilityData(dbContext, &hash, reachabilityData)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *reachabilityStore) clearDirtyEntries() {
+	store.dirty = make(map[daghash.Hash]struct{})
+}
+
+func (store *reachabilityStore) init(dbContext dbaccess.Context) error {
+	// TODO: (Stas) This is a quick and dirty hack.
+	// We iterate over the entire bucket twice:
+	// * First, populate the loaded set with all entries
+	// * Second, connect the parent/children pointers in each entry
+	//   with other nodes, which are now guaranteed to exist
+	cursor, err := dbaccess.ReachabilityDataCursor(dbContext)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		err := store.initReachabilityData(cursor)
+		if err != nil {
+			return err
+		}
+	}
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		err := store.loadReachabilityDataFromCursor(cursor)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *reachabilityStore) initReachabilityData(cursor database.Cursor) error {
+	key, err := cursor.Key()
+	if err != nil {
+		return err
+	}
+
+	hash, err := daghash.NewHash(key.Suffix())
+	if err != nil {
+		return err
+	}
+
+	store.loaded[*hash] = &reachabilityData{
+		treeNode:          &reachabilityTreeNode{},
+		futureCoveringSet: nil,
+	}
+	return nil
+}
+
+func (store *reachabilityStore) loadReachabilityDataFromCursor(cursor database.Cursor) error {
+	key, err := cursor.Key()
+	if err != nil {
+		return err
+	}
+
+	hash, err := daghash.NewHash(key.Suffix())
+	if err != nil {
+		return err
+	}
+
+	reachabilityData, ok := store.reachabilityDataByHash(hash)
+	if !ok {
+		return errors.Errorf("cannot find reachability data for block hash: %s", hash)
+	}
+
+	serializedReachabilityData, err := cursor.Value()
+	if err != nil {
+		return err
+	}
+
+	err = store.deserializeReachabilityData(serializedReachabilityData, reachabilityData)
+	if err != nil {
+		return err
+	}
+
+	// Connect the treeNode with its blockNode
+	reachabilityData.treeNode.blockNode, ok = store.dag.index.LookupNode(hash)
+	if !ok {
+		return errors.Errorf("block %s does not exist in the DAG", hash)
+	}
+
+	return nil
+}
+
+// storeReachabilityData stores the reachability data to the database.
+// This overwrites the current entry if there exists one.
+func (store *reachabilityStore) storeReachabilityData(dbContext dbaccess.Context, hash *daghash.Hash, reachabilityData *reachabilityData) error {
+	serializedReachabilyData, err := store.serializeReachabilityData(reachabilityData)
+	if err != nil {
+		return err
+	}
+
+	return dbaccess.StoreReachabilityData(dbContext, hash, serializedReachabilyData)
+}
+
+func (store *reachabilityStore) serializeReachabilityData(reachabilityData *reachabilityData) ([]byte, error) {
+	w := &bytes.Buffer{}
+	err := store.serializeTreeNode(w, reachabilityData.treeNode)
+	if err != nil {
+		return nil, err
+	}
+	err = store.serializeFutureCoveringSet(w, reachabilityData.futureCoveringSet)
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+func (store *reachabilityStore) serializeTreeNode(w io.Writer, treeNode *reachabilityTreeNode) error {
+	// Serialize the interval
+	err := store.serializeReachabilityInterval(w, treeNode.interval)
+	if err != nil {
+		return err
+	}
+
+	// Serialize the parent
+	// If this is the genesis block, write the zero hash instead
+	parentHash := &daghash.ZeroHash
+	if treeNode.parent != nil {
+		parentHash = treeNode.parent.blockNode.hash
+	}
+	err = domainmessage.WriteElement(w, parentHash)
+	if err != nil {
+		return err
+	}
+
+	// Serialize the amount of children
+	err = domainmessage.WriteVarInt(w, uint64(len(treeNode.children)))
+	if err != nil {
+		return err
+	}
+
+	// Serialize the children
+	for _, child := range treeNode.children {
+		err = domainmessage.WriteElement(w, child.blockNode.hash)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) serializeReachabilityInterval(w io.Writer, interval *reachabilityInterval) error {
+	// Serialize start
+	err := domainmessage.WriteElement(w, interval.start)
+	if err != nil {
+		return err
+	}
+
+	// Serialize end
+	err = domainmessage.WriteElement(w, interval.end)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) serializeFutureCoveringSet(w io.Writer, futureCoveringSet futureCoveringTreeNodeSet) error {
+	// Serialize the set size
+	err := domainmessage.WriteVarInt(w, uint64(len(futureCoveringSet)))
+	if err != nil {
+		return err
+	}
+
+	// Serialize each node in the set
+	for _, node := range futureCoveringSet {
+		err = domainmessage.WriteElement(w, node.blockNode.hash)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) deserializeReachabilityData(
+	serializedReachabilityDataBytes []byte, destination *reachabilityData) error {
+
+	r := bytes.NewBuffer(serializedReachabilityDataBytes)
+
+	// Deserialize the tree node
+	err := store.deserializeTreeNode(r, destination)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize the future covering set
+	err = store.deserializeFutureCoveringSet(r, destination)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) deserializeTreeNode(r io.Reader, destination *reachabilityData) error {
+	// Deserialize the interval
+	interval, err := store.deserializeReachabilityInterval(r)
+	if err != nil {
+		return err
+	}
+	destination.treeNode.interval = interval
+
+	// Deserialize the parent
+	// If this is the zero hash, this node is the genesis and as such doesn't have a parent
+	parentHash := &daghash.Hash{}
+	err = domainmessage.ReadElement(r, parentHash)
+	if err != nil {
+		return err
+	}
+	if !daghash.ZeroHash.IsEqual(parentHash) {
+		parentReachabilityData, ok := store.reachabilityDataByHash(parentHash)
+		if !ok {
+			return errors.Errorf("parent reachability data not found for hash: %s", parentHash)
+		}
+		destination.treeNode.parent = parentReachabilityData.treeNode
+	}
+
+	// Deserialize the amount of children
+	childCount, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize the children
+	children := make([]*reachabilityTreeNode, childCount)
+	for i := uint64(0); i < childCount; i++ {
+		childHash := &daghash.Hash{}
+		err = domainmessage.ReadElement(r, childHash)
+		if err != nil {
+			return err
+		}
+		childReachabilityData, ok := store.reachabilityDataByHash(childHash)
+		if !ok {
+			return errors.Errorf("child reachability data not found for hash: %s", parentHash)
+		}
+		children[i] = childReachabilityData.treeNode
+	}
+	destination.treeNode.children = children
+
+	return nil
+}
+
+func (store *reachabilityStore) deserializeReachabilityInterval(r io.Reader) (*reachabilityInterval, error) {
+	interval := &reachabilityInterval{}
+
+	// Deserialize start
+	start := uint64(0)
+	err := domainmessage.ReadElement(r, &start)
+	if err != nil {
+		return nil, err
+	}
+	interval.start = start
+
+	// Deserialize end
+	end := uint64(0)
+	err = domainmessage.ReadElement(r, &end)
+	if err != nil {
+		return nil, err
+	}
+	interval.end = end
+
+	return interval, nil
+}
+
+func (store *reachabilityStore) deserializeFutureCoveringSet(r io.Reader, destination *reachabilityData) error {
+	// Deserialize the set size
+	setSize, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize each block in the set
+	futureCoveringSet := make(futureCoveringTreeNodeSet, setSize)
+	for i := uint64(0); i < setSize; i++ {
+		blockHash := &daghash.Hash{}
+		err = domainmessage.ReadElement(r, blockHash)
+		if err != nil {
+			return err
+		}
+		blockReachabilityData, ok := store.reachabilityDataByHash(blockHash)
+		if !ok {
+			return errors.Errorf("block reachability data not found for hash: %s", blockHash)
+		}
+		futureCoveringSet[i] = blockReachabilityData.treeNode
+	}
+	destination.futureCoveringSet = futureCoveringSet
+
+	return nil
+}

blockdag/scriptval.go

@@ -9,15 +9,15 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
 )
 
 // txValidateItem holds a transaction along with which input to validate.
 type txValidateItem struct {
 	txInIndex int
-	txIn      *wire.TxIn
+	txIn      *domainmessage.TxIn
 	tx        *util.Tx
 }
 
@@ -126,7 +126,7 @@ func (v *txValidator) Validate(items []*txValidateItem) error {
 	// Start up validation handlers that are used to asynchronously
 	// validate each transaction input.
 	for i := 0; i < maxGoRoutines; i++ {
-		spawn(v.validateHandler)
+		spawn("txValidator.validateHandler", v.validateHandler)
 	}
 
 	// Validate each of the inputs. The quit channel is closed when any
@@ -179,11 +179,6 @@ func newTxValidator(utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscr
 // ValidateTransactionScripts validates the scripts for the passed transaction
 // using multiple goroutines.
 func ValidateTransactionScripts(tx *util.Tx, utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscript.SigCache) error {
-	// Don't validate coinbase transaction scripts.
-	if tx.IsCoinBase() {
-		return nil
-	}
-
 	// Collect all of the transaction inputs and required information for
 	// validation.
 	txIns := tx.MsgTx().TxIn
@@ -213,10 +208,6 @@ func checkBlockScripts(block *blockNode, utxoSet UTXOSet, transactions []*util.T
 	}
 	txValItems := make([]*txValidateItem, 0, numInputs)
 	for _, tx := range transactions {
-		// Skip coinbase transactions.
-		if tx.IsCoinBase() {
-			continue
-		}
 		for txInIdx, txIn := range tx.MsgTx().TxIn {
 			txVI := &txValidateItem{
 				txInIndex: txInIdx,

blockdag/subnetworks.go

@@ -4,32 +4,22 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/pkg/errors"
 
 	"github.com/kaspanet/kaspad/util"
 
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/util/subnetworkid"
-	"github.com/kaspanet/kaspad/wire"
 )
 
-// SubnetworkStore stores the subnetworks data
-type SubnetworkStore struct {
-	db database.DB
-}
-
-func newSubnetworkStore(db database.DB) *SubnetworkStore {
-	return &SubnetworkStore{
-		db: db,
-	}
-}
-
 // registerSubnetworks scans a list of transactions, singles out
 // subnetwork registry transactions, validates them, and registers a new
 // subnetwork based on it.
 // This function returns an error if one or more transactions are invalid
-func registerSubnetworks(dbTx database.Tx, txs []*util.Tx) error {
-	subnetworkRegistryTxs := make([]*wire.MsgTx, 0)
+func registerSubnetworks(dbContext dbaccess.Context, txs []*util.Tx) error {
+	subnetworkRegistryTxs := make([]*domainmessage.MsgTx, 0)
 	for _, tx := range txs {
 		msgTx := tx.MsgTx()
 
@@ -50,13 +40,13 @@ func registerSubnetworks(dbTx database.Tx, txs []*util.Tx) error {
 		if err != nil {
 			return err
 		}
-		sNet, err := dbGetSubnetwork(dbTx, subnetworkID)
+		exists, err := dbaccess.HasSubnetwork(dbContext, subnetworkID)
 		if err != nil {
 			return err
 		}
-		if sNet == nil {
+		if !exists {
 			createdSubnetwork := newSubnetwork(registryTx)
-			err := dbRegisterSubnetwork(dbTx, subnetworkID, createdSubnetwork)
+			err := registerSubnetwork(dbContext, subnetworkID, createdSubnetwork)
 			if err != nil {
 				return errors.Errorf("failed registering subnetwork"+
 					"for tx '%s': %s", registryTx.TxHash(), err)
@@ -70,7 +60,7 @@ func registerSubnetworks(dbTx database.Tx, txs []*util.Tx) error {
 // validateSubnetworkRegistryTransaction makes sure that a given subnetwork registry
 // transaction is valid. Such a transaction is valid iff:
 // - Its entire payload is a uint64 (8 bytes)
-func validateSubnetworkRegistryTransaction(tx *wire.MsgTx) error {
+func validateSubnetworkRegistryTransaction(tx *domainmessage.MsgTx) error {
 	if len(tx.Payload) != 8 {
 		return ruleError(ErrSubnetworkRegistry, fmt.Sprintf("validation failed: subnetwork registry"+
 			"tx '%s' has an invalid payload", tx.TxHash()))
@@ -80,85 +70,58 @@ func validateSubnetworkRegistryTransaction(tx *wire.MsgTx) error {
 }
 
 // TxToSubnetworkID creates a subnetwork ID from a subnetwork registry transaction
-func TxToSubnetworkID(tx *wire.MsgTx) (*subnetworkid.SubnetworkID, error) {
+func TxToSubnetworkID(tx *domainmessage.MsgTx) (*subnetworkid.SubnetworkID, error) {
 	txHash := tx.TxHash()
 	return subnetworkid.New(util.Hash160(txHash[:]))
 }
 
-// subnetwork returns a registered subnetwork. If the subnetwork does not exist
-// this method returns an error.
-func (s *SubnetworkStore) subnetwork(subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
-	var sNet *subnetwork
-	var err error
-	dbErr := s.db.View(func(dbTx database.Tx) error {
-		sNet, err = dbGetSubnetwork(dbTx, subnetworkID)
-		return nil
-	})
-	if dbErr != nil {
-		return nil, errors.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, dbErr)
-	}
+// fetchSubnetwork returns a registered subnetwork.
+func (dag *BlockDAG) fetchSubnetwork(subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
+	serializedSubnetwork, err := dbaccess.FetchSubnetworkData(dag.databaseContext, subnetworkID)
 	if err != nil {
-		return nil, errors.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, err)
+		return nil, err
 	}
 
-	return sNet, nil
+	subnet, err := deserializeSubnetwork(serializedSubnetwork)
+	if err != nil {
+		return nil, err
+	}
+
+	return subnet, nil
 }
 
 // GasLimit returns the gas limit of a registered subnetwork. If the subnetwork does not
 // exist this method returns an error.
-func (s *SubnetworkStore) GasLimit(subnetworkID *subnetworkid.SubnetworkID) (uint64, error) {
-	sNet, err := s.subnetwork(subnetworkID)
+func (dag *BlockDAG) GasLimit(subnetworkID *subnetworkid.SubnetworkID) (uint64, error) {
+	sNet, err := dag.fetchSubnetwork(subnetworkID)
 	if err != nil {
 		return 0, err
 	}
-	if sNet == nil {
-		return 0, errors.Errorf("subnetwork '%s' not found", subnetworkID)
-	}
 
 	return sNet.gasLimit, nil
 }
 
-// dbRegisterSubnetwork stores mappings from ID of the subnetwork to the subnetwork data.
-func dbRegisterSubnetwork(dbTx database.Tx, subnetworkID *subnetworkid.SubnetworkID, network *subnetwork) error {
-	// Serialize the subnetwork
+func registerSubnetwork(dbContext dbaccess.Context, subnetworkID *subnetworkid.SubnetworkID, network *subnetwork) error {
 	serializedSubnetwork, err := serializeSubnetwork(network)
 	if err != nil {
 		return errors.Errorf("failed to serialize sub-netowrk '%s': %s", subnetworkID, err)
 	}
 
-	// Store the subnetwork
-	subnetworksBucket := dbTx.Metadata().Bucket(subnetworksBucketName)
-	err = subnetworksBucket.Put(subnetworkID[:], serializedSubnetwork)
-	if err != nil {
-		return errors.Errorf("failed to write sub-netowrk '%s': %s", subnetworkID, err)
-	}
-
-	return nil
-}
-
-// dbGetSubnetwork returns the subnetwork associated with subnetworkID or nil if the subnetwork was not found.
-func dbGetSubnetwork(dbTx database.Tx, subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
-	bucket := dbTx.Metadata().Bucket(subnetworksBucketName)
-	serializedSubnetwork := bucket.Get(subnetworkID[:])
-	if serializedSubnetwork == nil {
-		return nil, nil
-	}
-
-	return deserializeSubnetwork(serializedSubnetwork)
+	return dbaccess.StoreSubnetwork(dbContext, subnetworkID, serializedSubnetwork)
 }
 
 type subnetwork struct {
 	gasLimit uint64
 }
 
-func newSubnetwork(tx *wire.MsgTx) *subnetwork {
+func newSubnetwork(tx *domainmessage.MsgTx) *subnetwork {
 	return &subnetwork{
 		gasLimit: ExtractGasLimit(tx),
 	}
 }
 
 // ExtractGasLimit extracts the gas limit from the transaction payload
-func ExtractGasLimit(tx *wire.MsgTx) uint64 {
+func ExtractGasLimit(tx *domainmessage.MsgTx) uint64 {
 	return binary.LittleEndian.Uint64(tx.Payload[:8])
 }
 

blockdag/sync_rate.go

@@ -0,0 +1,60 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+	"time"
+)
+
+const syncRateWindowDuration = 15 * time.Minute
+
+// addBlockProcessingTimestamp adds the last block processing timestamp in order to measure the recent sync rate.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) addBlockProcessingTimestamp() {
+	now := mstime.Now()
+	dag.recentBlockProcessingTimestamps = append(dag.recentBlockProcessingTimestamps, now)
+	dag.removeNonRecentTimestampsFromRecentBlockProcessingTimestamps()
+}
+
+// removeNonRecentTimestampsFromRecentBlockProcessingTimestamps removes timestamps older than syncRateWindowDuration
+// from dag.recentBlockProcessingTimestamps
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) removeNonRecentTimestampsFromRecentBlockProcessingTimestamps() {
+	dag.recentBlockProcessingTimestamps = dag.recentBlockProcessingTimestampsRelevantWindow()
+}
+
+func (dag *BlockDAG) recentBlockProcessingTimestampsRelevantWindow() []mstime.Time {
+	minTime := mstime.Now().Add(-syncRateWindowDuration)
+	windowStartIndex := len(dag.recentBlockProcessingTimestamps)
+	for i, processTime := range dag.recentBlockProcessingTimestamps {
+		if processTime.After(minTime) {
+			windowStartIndex = i
+			break
+		}
+	}
+	return dag.recentBlockProcessingTimestamps[windowStartIndex:]
+}
+
+// syncRate returns the rate of processed
+// blocks in the last syncRateWindowDuration
+// duration.
+func (dag *BlockDAG) syncRate() float64 {
+	dag.RLock()
+	defer dag.RUnlock()
+	return float64(len(dag.recentBlockProcessingTimestampsRelevantWindow())) / syncRateWindowDuration.Seconds()
+}
+
+// IsSyncRateBelowThreshold checks whether the sync rate
+// is below the expected threshold.
+func (dag *BlockDAG) IsSyncRateBelowThreshold(maxDeviation float64) bool {
+	if dag.uptime() < syncRateWindowDuration {
+		return false
+	}
+
+	return dag.syncRate() < 1/dag.Params.TargetTimePerBlock.Seconds()*maxDeviation
+}
+
+func (dag *BlockDAG) uptime() time.Duration {
+	return mstime.Now().Sub(dag.startTime)
+}

blockdag/test_utils.go

@@ -5,47 +5,28 @@ package blockdag
 import (
 	"compress/bzip2"
 	"encoding/binary"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/pkg/errors"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
+	"testing"
+
+	"github.com/kaspanet/kaspad/database/ffldb/ldb"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb" // blank import ffldb so that its init() function runs before tests
+	"github.com/kaspanet/kaspad/domainmessage"
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
 )
 
-const (
-	// testDbType is the database backend type to use for the tests.
-	testDbType = "ffldb"
-
-	// testDbRoot is the root directory used to create all test databases.
-	testDbRoot = "testdbs"
-
-	// blockDataNet is the expected network in the test block data.
-	blockDataNet = wire.MainNet
-)
-
-// isSupportedDbType returns whether or not the passed database type is
-// currently supported.
-func isSupportedDbType(dbType string) bool {
-	supportedDrivers := database.SupportedDrivers()
-	for _, driver := range supportedDrivers {
-		if dbType == driver {
-			return true
-		}
-	}
-
-	return false
-}
-
 // FileExists returns whether or not the named file or directory exists.
 func FileExists(name string) bool {
 	if _, err := os.Stat(name); err != nil {
@@ -59,61 +40,66 @@ func FileExists(name string) bool {
 // DAGSetup is used to create a new db and DAG instance with the genesis
 // block already inserted. In addition to the new DAG instance, it returns
 // a teardown function the caller should invoke when done testing to clean up.
-func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, errors.Errorf("unsupported db type %s", testDbType)
-	}
-
+// The openDB parameter instructs DAGSetup whether or not to also open the
+// database. Setting it to false is useful in tests that handle database
+// opening/closing by themselves.
+func DAGSetup(dbName string, openDb bool, config Config) (*BlockDAG, func(), error) {
 	var teardown func()
 
 	// To make sure that the teardown function is not called before any goroutines finished to run -
 	// overwrite `spawn` to count the number of running goroutines
 	spawnWaitGroup := sync.WaitGroup{}
 	realSpawn := spawn
-	spawn = func(f func()) {
+	spawn = func(name string, f func()) {
 		spawnWaitGroup.Add(1)
-		realSpawn(func() {
+		realSpawn(name, func() {
 			f()
 			spawnWaitGroup.Done()
 		})
 	}
 
-	if config.DB == nil {
-		// Create the root directory for test databases.
-		if !FileExists(testDbRoot) {
-			if err := os.MkdirAll(testDbRoot, 0700); err != nil {
-				err := errors.Errorf("unable to create test db "+
-					"root: %s", err)
-				return nil, nil, err
-			}
+	if openDb {
+		var err error
+		tmpDir, err := ioutil.TempDir("", "DAGSetup")
+		if err != nil {
+			return nil, nil, errors.Errorf("error creating temp dir: %s", err)
 		}
 
-		dbPath := filepath.Join(testDbRoot, dbName)
+		// We set ldb.Options here to return nil because normally
+		// the database is initialized with very large caches that
+		// can make opening/closing the database for every test
+		// quite heavy.
+		originalLDBOptions := ldb.Options
+		ldb.Options = func() *opt.Options {
+			return nil
+		}
+
+		dbPath := filepath.Join(tmpDir, dbName)
 		_ = os.RemoveAll(dbPath)
-		var err error
-		config.DB, err = database.Create(testDbType, dbPath, blockDataNet)
+		databaseContext, err := dbaccess.New(dbPath)
 		if err != nil {
 			return nil, nil, errors.Errorf("error creating db: %s", err)
 		}
 
+		config.DatabaseContext = databaseContext
+
 		// Setup a teardown function for cleaning up. This function is
 		// returned to the caller to be invoked when it is done testing.
 		teardown = func() {
 			spawnWaitGroup.Wait()
 			spawn = realSpawn
-			config.DB.Close()
+			databaseContext.Close()
+			ldb.Options = originalLDBOptions
 			os.RemoveAll(dbPath)
-			os.RemoveAll(testDbRoot)
 		}
 	} else {
 		teardown = func() {
 			spawnWaitGroup.Wait()
 			spawn = realSpawn
-			config.DB.Close()
 		}
 	}
 
-	config.TimeSource = NewMedianTime()
+	config.TimeSource = NewTimeSource()
 	config.SigCache = txscript.NewSigCache(1000)
 
 	// Create the DAG instance.
@@ -135,30 +121,30 @@ type txSubnetworkData struct {
 	Payload      []byte
 }
 
-func createTxForTest(numInputs uint32, numOutputs uint32, outputValue uint64, subnetworkData *txSubnetworkData) *wire.MsgTx {
-	txIns := []*wire.TxIn{}
-	txOuts := []*wire.TxOut{}
+func createTxForTest(numInputs uint32, numOutputs uint32, outputValue uint64, subnetworkData *txSubnetworkData) *domainmessage.MsgTx {
+	txIns := []*domainmessage.TxIn{}
+	txOuts := []*domainmessage.TxOut{}
 
 	for i := uint32(0); i < numInputs; i++ {
-		txIns = append(txIns, &wire.TxIn{
-			PreviousOutpoint: *wire.NewOutpoint(&daghash.TxID{}, i),
+		txIns = append(txIns, &domainmessage.TxIn{
+			PreviousOutpoint: *domainmessage.NewOutpoint(&daghash.TxID{}, i),
 			SignatureScript:  []byte{},
-			Sequence:         wire.MaxTxInSequenceNum,
+			Sequence:         domainmessage.MaxTxInSequenceNum,
 		})
 	}
 
 	for i := uint32(0); i < numOutputs; i++ {
-		txOuts = append(txOuts, &wire.TxOut{
+		txOuts = append(txOuts, &domainmessage.TxOut{
 			ScriptPubKey: OpTrueScript,
 			Value:        outputValue,
 		})
 	}
 
 	if subnetworkData != nil {
-		return wire.NewSubnetworkMsgTx(wire.TxVersion, txIns, txOuts, subnetworkData.subnetworkID, subnetworkData.Gas, subnetworkData.Payload)
+		return domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, txIns, txOuts, subnetworkData.subnetworkID, subnetworkData.Gas, subnetworkData.Payload)
 	}
 
-	return wire.NewNativeMsgTx(wire.TxVersion, txIns, txOuts)
+	return domainmessage.NewNativeMsgTx(domainmessage.TxVersion, txIns, txOuts)
 }
 
 // VirtualForTest is an exported version for virtualBlock, so that it can be returned by exported test_util methods
@@ -173,17 +159,17 @@ func SetVirtualForTest(dag *BlockDAG, virtual VirtualForTest) VirtualForTest {
 
 // GetVirtualFromParentsForTest generates a virtual block with the given parents.
 func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) (VirtualForTest, error) {
-	parents := newSet()
+	parents := newBlockSet()
 	for _, hash := range parentHashes {
-		parent := dag.index.LookupNode(hash)
-		if parent == nil {
+		parent, ok := dag.index.LookupNode(hash)
+		if !ok {
 			return nil, errors.Errorf("GetVirtualFromParentsForTest: didn't found node for hash %s", hash)
 		}
 		parents.add(parent)
 	}
-	virtual := newVirtualBlock(parents, dag.dagParams.K)
+	virtual := newVirtualBlock(dag, parents)
 
-	pastUTXO, _, err := dag.pastUTXO(&virtual.blockNode)
+	pastUTXO, _, _, err := dag.pastUTXO(&virtual.blockNode)
 	if err != nil {
 		return nil, err
 	}
@@ -200,7 +186,7 @@ func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) (
 // LoadBlocks reads files containing kaspa gzipped block data from disk
 // and returns them as an array of util.Block.
 func LoadBlocks(filename string) (blocks []*util.Block, err error) {
-	var network = wire.MainNet
+	var network = domainmessage.Mainnet
 	var dr io.Reader
 	var fi io.ReadCloser
 
@@ -251,3 +237,94 @@ func LoadBlocks(filename string) (blocks []*util.Block, err error) {
 
 	return
 }
+
+// opTrueAddress returns an address pointing to a P2SH anyone-can-spend script
+func opTrueAddress(prefix util.Bech32Prefix) (util.Address, error) {
+	return util.NewAddressScriptHash(OpTrueScript, prefix)
+}
+
+// PrepareBlockForTest generates a block with the proper merkle roots, coinbase transaction etc. This function is used for test purposes only
+func PrepareBlockForTest(dag *BlockDAG, parentHashes []*daghash.Hash, transactions []*domainmessage.MsgTx) (*domainmessage.MsgBlock, error) {
+	newVirtual, err := GetVirtualFromParentsForTest(dag, parentHashes)
+	if err != nil {
+		return nil, err
+	}
+	oldVirtual := SetVirtualForTest(dag, newVirtual)
+	defer SetVirtualForTest(dag, oldVirtual)
+
+	OpTrueAddr, err := opTrueAddress(dag.Params.Prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	blockTransactions := make([]*util.Tx, len(transactions)+1)
+
+	extraNonce := generateDeterministicExtraNonceForTest()
+	coinbasePayloadExtraData, err := CoinbasePayloadExtraData(extraNonce, "")
+	if err != nil {
+		return nil, err
+	}
+
+	blockTransactions[0], err = dag.NextCoinbaseFromAddress(OpTrueAddr, coinbasePayloadExtraData)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, tx := range transactions {
+		blockTransactions[i+1] = util.NewTx(tx)
+	}
+
+	// Sort transactions by subnetwork ID
+	sort.Slice(blockTransactions, func(i, j int) bool {
+		if blockTransactions[i].MsgTx().SubnetworkID.IsEqual(subnetworkid.SubnetworkIDCoinbase) {
+			return true
+		}
+		if blockTransactions[j].MsgTx().SubnetworkID.IsEqual(subnetworkid.SubnetworkIDCoinbase) {
+			return false
+		}
+		return subnetworkid.Less(&blockTransactions[i].MsgTx().SubnetworkID, &blockTransactions[j].MsgTx().SubnetworkID)
+	})
+
+	block, err := dag.BlockForMining(blockTransactions)
+	if err != nil {
+		return nil, err
+	}
+	block.Header.Timestamp = dag.NextBlockMinimumTime()
+	block.Header.Bits = dag.NextRequiredDifficulty(block.Header.Timestamp)
+
+	return block, nil
+}
+
+// PrepareAndProcessBlockForTest prepares a block that points to the given parent
+// hashes and process it.
+func PrepareAndProcessBlockForTest(t *testing.T, dag *BlockDAG, parentHashes []*daghash.Hash, transactions []*domainmessage.MsgTx) *domainmessage.MsgBlock {
+	daghash.Sort(parentHashes)
+	block, err := PrepareBlockForTest(dag, parentHashes, transactions)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	utilBlock := util.NewBlock(block)
+	isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("unexpected error in ProcessBlock: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("block is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("block was unexpectedly orphan")
+	}
+	return block
+}
+
+// generateDeterministicExtraNonceForTest returns a unique deterministic extra nonce for coinbase data, in order to create unique coinbase transactions.
+func generateDeterministicExtraNonceForTest() uint64 {
+	extraNonceForTest++
+	return extraNonceForTest
+}
+
+func resetExtraNonceForTest() {
+	extraNonceForTest = 0
+}
+
+var extraNonceForTest = uint64(0)

blockdag/test_utils_test.go

@@ -1,56 +0,0 @@
-package blockdag
-
-import (
-	"github.com/pkg/errors"
-	"os"
-	"strings"
-	"testing"
-
-	"bou.ke/monkey"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-)
-
-func TestIsSupportedDbType(t *testing.T) {
-	if !isSupportedDbType("ffldb") {
-		t.Errorf("ffldb should be a supported DB driver")
-	}
-	if isSupportedDbType("madeUpDb") {
-		t.Errorf("madeUpDb should not be a supported DB driver")
-	}
-}
-
-// TestDAGSetupErrors tests all error-cases in DAGSetup.
-// The non-error-cases are tested in the more general tests.
-func TestDAGSetupErrors(t *testing.T) {
-	os.RemoveAll(testDbRoot)
-	testDAGSetupErrorThroughPatching(t, "unable to create test db root: ", os.MkdirAll, func(path string, perm os.FileMode) error {
-		return errors.New("Made up error")
-	})
-
-	testDAGSetupErrorThroughPatching(t, "failed to create dag instance: ", New, func(config *Config) (*BlockDAG, error) {
-		return nil, errors.New("Made up error")
-	})
-
-	testDAGSetupErrorThroughPatching(t, "unsupported db type ", isSupportedDbType, func(dbType string) bool {
-		return false
-	})
-
-	testDAGSetupErrorThroughPatching(t, "error creating db: ", database.Create, func(dbType string, args ...interface{}) (database.DB, error) {
-		return nil, errors.New("Made up error")
-	})
-}
-
-func testDAGSetupErrorThroughPatching(t *testing.T, expectedErrorMessage string, targetFunction interface{}, replacementFunction interface{}) {
-	guard := monkey.Patch(targetFunction, replacementFunction)
-	defer guard.Unpatch()
-	_, tearDown, err := DAGSetup("TestDAGSetup", Config{
-		DAGParams: &dagconfig.MainNetParams,
-	})
-	if tearDown != nil {
-		defer tearDown()
-	}
-	if err == nil || !strings.HasPrefix(err.Error(), expectedErrorMessage) {
-		t.Errorf("DAGSetup: expected error to have prefix '%s' but got error '%v'", expectedErrorMessage, err)
-	}
-}

blockdag/thresholdstate.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 
 	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
 )
 
 // ThresholdState define the various threshold states used when voting on
@@ -130,15 +131,15 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 	// The threshold state for the window that contains the genesis block is
 	// defined by definition.
 	confirmationWindow := checker.MinerConfirmationWindow()
-	if prevNode == nil || (prevNode.chainHeight+1) < confirmationWindow {
+	if prevNode == nil || (prevNode.blueScore+1) < confirmationWindow {
 		return ThresholdDefined, nil
 	}
 
 	// Get the ancestor that is the last block of the previous confirmation
 	// window in order to get its threshold state. This can be done because
 	// the state is the same for all blocks within a given window.
-	prevNode = prevNode.SelectedAncestor(prevNode.chainHeight -
-		(prevNode.chainHeight+1)%confirmationWindow)
+	prevNode = prevNode.SelectedAncestor(prevNode.blueScore -
+		(prevNode.blueScore+1)%confirmationWindow)
 
 	// Iterate backwards through each of the previous confirmation windows
 	// to find the most recently cached threshold state.
@@ -156,7 +157,7 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 
 		// The state is simply defined if the start time hasn't been
 		// been reached yet.
-		if uint64(medianTime.Unix()) < checker.BeginTime() {
+		if uint64(medianTime.UnixMilliseconds()) < checker.BeginTime() {
 			cache.Update(prevNode.hash, ThresholdDefined)
 			break
 		}
@@ -177,9 +178,9 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 		var ok bool
 		state, ok = cache.Lookup(prevNode.hash)
 		if !ok {
-			return ThresholdFailed, AssertError(fmt.Sprintf(
+			return ThresholdFailed, errors.Errorf(
 				"thresholdState: cache lookup failed for %s",
-				prevNode.hash))
+				prevNode.hash)
 		}
 	}
 
@@ -193,7 +194,7 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 			// The deployment of the rule change fails if it expires
 			// before it is accepted and locked in.
 			medianTime := prevNode.PastMedianTime(dag)
-			medianTimeUnix := uint64(medianTime.Unix())
+			medianTimeUnix := uint64(medianTime.UnixMilliseconds())
 			if medianTimeUnix >= checker.EndTime() {
 				state = ThresholdFailed
 				break
@@ -210,7 +211,7 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 			// The deployment of the rule change fails if it expires
 			// before it is accepted and locked in.
 			medianTime := prevNode.PastMedianTime(dag)
-			if uint64(medianTime.Unix()) >= checker.EndTime() {
+			if uint64(medianTime.UnixMilliseconds()) >= checker.EndTime() {
 				state = ThresholdFailed
 				break
 			}
@@ -219,18 +220,17 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 			// on by the miners, so iterate backwards through the
 			// confirmation window to count all of the votes in it.
 			var count uint64
-			countNode := prevNode
-			for i := uint64(0); i < confirmationWindow; i++ {
-				condition, err := checker.Condition(countNode)
+			windowNodes := make([]*blockNode, 0, confirmationWindow)
+			windowNodes = append(windowNodes, prevNode)
+			windowNodes = append(windowNodes, blueBlockWindow(prevNode, confirmationWindow-1)...)
+			for _, current := range windowNodes {
+				condition, err := checker.Condition(current)
 				if err != nil {
 					return ThresholdFailed, err
 				}
 				if condition {
 					count++
 				}
-
-				// Get the previous block node.
-				countNode = countNode.selectedParent
 			}
 
 			// The state is locked in if the number of blocks in the
@@ -265,8 +265,8 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 // This function is safe for concurrent access.
 func (dag *BlockDAG) ThresholdState(deploymentID uint32) (ThresholdState, error) {
 	dag.dagLock.Lock()
+	defer dag.dagLock.Unlock()
 	state, err := dag.deploymentState(dag.selectedTip(), deploymentID)
-	dag.dagLock.Unlock()
 
 	return state, err
 }
@@ -277,8 +277,8 @@ func (dag *BlockDAG) ThresholdState(deploymentID uint32) (ThresholdState, error)
 // This function is safe for concurrent access.
 func (dag *BlockDAG) IsDeploymentActive(deploymentID uint32) (bool, error) {
 	dag.dagLock.Lock()
+	defer dag.dagLock.Unlock()
 	state, err := dag.deploymentState(dag.selectedTip(), deploymentID)
-	dag.dagLock.Unlock()
 	if err != nil {
 		return false, err
 	}
@@ -297,11 +297,11 @@ func (dag *BlockDAG) IsDeploymentActive(deploymentID uint32) (bool, error) {
 //
 // This function MUST be called with the DAG state lock held (for writes).
 func (dag *BlockDAG) deploymentState(prevNode *blockNode, deploymentID uint32) (ThresholdState, error) {
-	if deploymentID > uint32(len(dag.dagParams.Deployments)) {
-		return ThresholdFailed, DeploymentError(deploymentID)
+	if deploymentID > uint32(len(dag.Params.Deployments)) {
+		return ThresholdFailed, errors.Errorf("deployment ID %d does not exist", deploymentID)
 	}
 
-	deployment := &dag.dagParams.Deployments[deploymentID]
+	deployment := &dag.Params.Deployments[deploymentID]
 	checker := deploymentChecker{deployment: deployment, dag: dag}
 	cache := &dag.deploymentCaches[deploymentID]
 
@@ -325,8 +325,8 @@ func (dag *BlockDAG) initThresholdCaches() error {
 			return err
 		}
 	}
-	for id := 0; id < len(dag.dagParams.Deployments); id++ {
-		deployment := &dag.dagParams.Deployments[id]
+	for id := 0; id < len(dag.Params.Deployments); id++ {
+		deployment := &dag.Params.Deployments[id]
 		cache := &dag.deploymentCaches[id]
 		checker := deploymentChecker{deployment: deployment, dag: dag}
 		_, err := dag.thresholdState(prevNode, checker, cache)
@@ -336,8 +336,8 @@ func (dag *BlockDAG) initThresholdCaches() error {
 	}
 
 	// No warnings about unknown rules or versions until the DAG is
-	// current.
-	if dag.isCurrent() {
+	// synced.
+	if dag.isSynced() {
 		// Warn if a high enough percentage of the last blocks have
 		// unexpected versions.
 		bestNode := dag.selectedTip()

blockdag/timesource.go

@@ -0,0 +1,25 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+)
+
+// TimeSource is the interface to access time.
+type TimeSource interface {
+	// Now returns the current time.
+	Now() mstime.Time
+}
+
+// timeSource provides an implementation of the TimeSource interface
+// that simply returns the current local time.
+type timeSource struct{}
+
+// Now returns the current local time, with one millisecond precision.
+func (m *timeSource) Now() mstime.Time {
+	return mstime.Now()
+}
+
+// NewTimeSource returns a new instance of a TimeSource
+func NewTimeSource() TimeSource {
+	return &timeSource{}
+}

blockdag/utxo_ecmh.go

@@ -2,31 +2,26 @@ package blockdag
 
 import (
 	"bytes"
-	"github.com/golang/groupcache/lru"
-	"github.com/kaspanet/kaspad/ecc"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/domainmessage"
 )
 
-const ecmhCacheSize = 4_000_000
-
-var (
-	utxoToECMHCache = lru.New(ecmhCacheSize)
-)
-
-func utxoMultiset(entry *UTXOEntry, outpoint *wire.Outpoint) (*ecc.Multiset, error) {
+func addUTXOToMultiset(ms *secp256k1.MultiSet, entry *UTXOEntry, outpoint *domainmessage.Outpoint) (*secp256k1.MultiSet, error) {
 	w := &bytes.Buffer{}
 	err := serializeUTXO(w, entry, outpoint)
 	if err != nil {
 		return nil, err
 	}
-	serializedUTXO := w.Bytes()
-	utxoHash := daghash.DoubleHashH(serializedUTXO)
-
-	if cachedMSPoint, ok := utxoToECMHCache.Get(utxoHash); ok {
-		return cachedMSPoint.(*ecc.Multiset), nil
-	}
-	msPoint := ecc.NewMultiset(ecc.S256()).Add(serializedUTXO)
-	utxoToECMHCache.Add(utxoHash, msPoint)
-	return msPoint, nil
+	ms.Add(w.Bytes())
+	return ms, nil
+}
+
+func removeUTXOFromMultiset(ms *secp256k1.MultiSet, entry *UTXOEntry, outpoint *domainmessage.Outpoint) (*secp256k1.MultiSet, error) {
+	w := &bytes.Buffer{}
+	err := serializeUTXO(w, entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	ms.Remove(w.Bytes())
+	return ms, nil
 }

blockdag/utxodiffstore.go

@@ -1,14 +1,13 @@
 package blockdag
 
 import (
-	"github.com/kaspanet/kaspad/database"
+	"bytes"
+
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/util/locks"
-	"github.com/pkg/errors"
 )
 
-var multisetPointSize = 32
-
 type blockUTXODiffData struct {
 	diff      *UTXODiff
 	diffChild *blockNode
@@ -16,16 +15,16 @@ type blockUTXODiffData struct {
 
 type utxoDiffStore struct {
 	dag    *BlockDAG
-	dirty  map[daghash.Hash]struct{}
-	loaded map[daghash.Hash]*blockUTXODiffData
+	dirty  map[*blockNode]struct{}
+	loaded map[*blockNode]*blockUTXODiffData
 	mtx    *locks.PriorityMutex
 }
 
 func newUTXODiffStore(dag *BlockDAG) *utxoDiffStore {
 	return &utxoDiffStore{
 		dag:    dag,
-		dirty:  make(map[daghash.Hash]struct{}),
-		loaded: make(map[daghash.Hash]*blockUTXODiffData),
+		dirty:  make(map[*blockNode]struct{}),
+		loaded: make(map[*blockNode]*blockUTXODiffData),
 		mtx:    locks.NewPriorityMutex(),
 	}
 }
@@ -34,16 +33,15 @@ func (diffStore *utxoDiffStore) setBlockDiff(node *blockNode, diff *UTXODiff) er
 	diffStore.mtx.HighPriorityWriteLock()
 	defer diffStore.mtx.HighPriorityWriteUnlock()
 	// load the diff data from DB to diffStore.loaded
-	_, exists, err := diffStore.diffDataByHash(node.hash)
-	if err != nil {
+	_, err := diffStore.diffDataByBlockNode(node)
+	if dbaccess.IsNotFoundError(err) {
+		diffStore.loaded[node] = &blockUTXODiffData{}
+	} else if err != nil {
 		return err
 	}
-	if !exists {
-		diffStore.loaded[*node.hash] = &blockUTXODiffData{}
-	}
 
-	diffStore.loaded[*node.hash].diff = diff
-	diffStore.setBlockAsDirty(node.hash)
+	diffStore.loaded[node].diff = diff
+	diffStore.setBlockAsDirty(node)
 	return nil
 }
 
@@ -51,22 +49,19 @@ func (diffStore *utxoDiffStore) setBlockDiffChild(node *blockNode, diffChild *bl
 	diffStore.mtx.HighPriorityWriteLock()
 	defer diffStore.mtx.HighPriorityWriteUnlock()
 	// load the diff data from DB to diffStore.loaded
-	_, exists, err := diffStore.diffDataByHash(node.hash)
+	_, err := diffStore.diffDataByBlockNode(node)
 	if err != nil {
 		return err
 	}
-	if !exists {
-		return diffNotFoundError(node)
-	}
 
-	diffStore.loaded[*node.hash].diffChild = diffChild
-	diffStore.setBlockAsDirty(node.hash)
+	diffStore.loaded[node].diffChild = diffChild
+	diffStore.setBlockAsDirty(node)
 	return nil
 }
 
-func (diffStore *utxoDiffStore) removeBlocksDiffData(dbTx database.Tx, blockHashes []*daghash.Hash) error {
-	for _, hash := range blockHashes {
-		err := diffStore.removeBlockDiffData(dbTx, hash)
+func (diffStore *utxoDiffStore) removeBlocksDiffData(dbContext dbaccess.Context, nodes []*blockNode) error {
+	for _, node := range nodes {
+		err := diffStore.removeBlockDiffData(dbContext, node)
 		if err != nil {
 			return err
 		}
@@ -74,96 +69,77 @@ func (diffStore *utxoDiffStore) removeBlocksDiffData(dbTx database.Tx, blockHash
 	return nil
 }
 
-func (diffStore *utxoDiffStore) removeBlockDiffData(dbTx database.Tx, blockHash *daghash.Hash) error {
+func (diffStore *utxoDiffStore) removeBlockDiffData(dbContext dbaccess.Context, node *blockNode) error {
 	diffStore.mtx.LowPriorityWriteLock()
 	defer diffStore.mtx.LowPriorityWriteUnlock()
-	delete(diffStore.loaded, *blockHash)
-	err := dbRemoveDiffData(dbTx, blockHash)
+	delete(diffStore.loaded, node)
+	err := dbaccess.RemoveDiffData(dbContext, node.hash)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func (diffStore *utxoDiffStore) setBlockAsDirty(blockHash *daghash.Hash) {
-	diffStore.dirty[*blockHash] = struct{}{}
+func (diffStore *utxoDiffStore) setBlockAsDirty(node *blockNode) {
+	diffStore.dirty[node] = struct{}{}
 }
 
-func (diffStore *utxoDiffStore) diffDataByHash(hash *daghash.Hash) (*blockUTXODiffData, bool, error) {
-	if diffData, ok := diffStore.loaded[*hash]; ok {
-		return diffData, true, nil
+func (diffStore *utxoDiffStore) diffDataByBlockNode(node *blockNode) (*blockUTXODiffData, error) {
+	if diffData, ok := diffStore.loaded[node]; ok {
+		return diffData, nil
 	}
-	diffData, err := diffStore.diffDataFromDB(hash)
+	diffData, err := diffStore.diffDataFromDB(node.hash)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
-	exists := diffData != nil
-	if exists {
-		diffStore.loaded[*hash] = diffData
-	}
-	return diffData, exists, nil
-}
-
-func diffNotFoundError(node *blockNode) error {
-	return errors.Errorf("Couldn't find diff data for block %s", node.hash)
+	diffStore.loaded[node] = diffData
+	return diffData, nil
 }
 
 func (diffStore *utxoDiffStore) diffByNode(node *blockNode) (*UTXODiff, error) {
 	diffStore.mtx.HighPriorityReadLock()
 	defer diffStore.mtx.HighPriorityReadUnlock()
-	diffData, exists, err := diffStore.diffDataByHash(node.hash)
+	diffData, err := diffStore.diffDataByBlockNode(node)
 	if err != nil {
 		return nil, err
 	}
-	if !exists {
-		return nil, diffNotFoundError(node)
-	}
 	return diffData.diff, nil
 }
 
 func (diffStore *utxoDiffStore) diffChildByNode(node *blockNode) (*blockNode, error) {
 	diffStore.mtx.HighPriorityReadLock()
 	defer diffStore.mtx.HighPriorityReadUnlock()
-	diffData, exists, err := diffStore.diffDataByHash(node.hash)
+	diffData, err := diffStore.diffDataByBlockNode(node)
 	if err != nil {
 		return nil, err
 	}
-	if !exists {
-		return nil, diffNotFoundError(node)
-	}
 	return diffData.diffChild, nil
 }
 
 func (diffStore *utxoDiffStore) diffDataFromDB(hash *daghash.Hash) (*blockUTXODiffData, error) {
-	var diffData *blockUTXODiffData
-	err := diffStore.dag.db.View(func(dbTx database.Tx) error {
-		bucket := dbTx.Metadata().Bucket(utxoDiffsBucketName)
-		serializedBlockDiffData := bucket.Get(hash[:])
-		if serializedBlockDiffData != nil {
-			var err error
-			diffData, err = diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
-			return err
-		}
-		return nil
-	})
+	serializedBlockDiffData, err := dbaccess.FetchUTXODiffData(diffStore.dag.databaseContext, hash)
 	if err != nil {
 		return nil, err
 	}
-	return diffData, nil
+
+	return diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
 }
 
-// flushToDB writes all dirty diff data to the database. If all writes
-// succeed, this clears the dirty set.
-func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
+// flushToDB writes all dirty diff data to the database.
+func (diffStore *utxoDiffStore) flushToDB(dbContext *dbaccess.TxContext) error {
 	diffStore.mtx.HighPriorityWriteLock()
 	defer diffStore.mtx.HighPriorityWriteUnlock()
 	if len(diffStore.dirty) == 0 {
 		return nil
 	}
 
-	for hash := range diffStore.dirty {
-		diffData := diffStore.loaded[hash]
-		err := dbStoreDiffData(dbTx, &hash, diffData)
+	// Allocate a buffer here to avoid needless allocations/grows
+	// while writing each entry.
+	buffer := &bytes.Buffer{}
+	for node := range diffStore.dirty {
+		buffer.Reset()
+		diffData := diffStore.loaded[node]
+		err := storeDiffData(dbContext, buffer, node.hash, diffData)
 		if err != nil {
 			return err
 		}
@@ -172,20 +148,53 @@ func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
 }
 
 func (diffStore *utxoDiffStore) clearDirtyEntries() {
-	diffStore.dirty = make(map[daghash.Hash]struct{})
+	diffStore.dirty = make(map[*blockNode]struct{})
 }
 
-// dbStoreDiffData stores the UTXO diff data to the database.
+// maxBlueScoreDifferenceToKeepLoaded is the maximum difference
+// between the virtual's blueScore and a blockNode's blueScore
+// under which to keep diff data loaded in memory.
+var maxBlueScoreDifferenceToKeepLoaded uint64 = 100
+
+// clearOldEntries removes entries whose blue score is lower than
+// virtual.blueScore - maxBlueScoreDifferenceToKeepLoaded. Note
+// that tips are not removed either even if their blue score is
+// lower than the above.
+func (diffStore *utxoDiffStore) clearOldEntries() {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+
+	virtualBlueScore := diffStore.dag.VirtualBlueScore()
+	minBlueScore := virtualBlueScore - maxBlueScoreDifferenceToKeepLoaded
+	if maxBlueScoreDifferenceToKeepLoaded > virtualBlueScore {
+		minBlueScore = 0
+	}
+
+	tips := diffStore.dag.virtual.tips()
+
+	toRemove := make(map[*blockNode]struct{})
+	for node := range diffStore.loaded {
+		if node.blueScore < minBlueScore && !tips.contains(node) {
+			toRemove[node] = struct{}{}
+		}
+	}
+	for node := range toRemove {
+		delete(diffStore.loaded, node)
+	}
+}
+
+// storeDiffData stores the UTXO diff data to the database.
 // This overwrites the current entry if there exists one.
-func dbStoreDiffData(dbTx database.Tx, hash *daghash.Hash, diffData *blockUTXODiffData) error {
-	serializedDiffData, err := serializeBlockUTXODiffData(diffData)
+func storeDiffData(dbContext dbaccess.Context, w *bytes.Buffer, hash *daghash.Hash, diffData *blockUTXODiffData) error {
+	// To avoid a ton of allocs, use the io.Writer
+	// instead of allocating one. We expect the buffer to
+	// already be initialized and, in most cases, to already
+	// be large enough to accommodate the serialized data
+	// without growing.
+	err := serializeBlockUTXODiffData(w, diffData)
 	if err != nil {
 		return err
 	}
 
-	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Put(hash[:], serializedDiffData)
-}
-
-func dbRemoveDiffData(dbTx database.Tx, hash *daghash.Hash) error {
-	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Delete(hash[:])
+	return dbaccess.StoreUTXODiffData(dbContext, hash, w.Bytes())
 }

blockdag/utxodiffstore_test.go

@@ -1,19 +1,19 @@
 package blockdag
 
 import (
-	"fmt"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
 	"reflect"
 	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 func TestUTXODiffStore(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", Config{
-		DAGParams: &dagconfig.SimNetParams,
+	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Fatalf("TestUTXODiffStore: Failed to setup DAG instance: %v", err)
@@ -31,16 +31,19 @@ func TestUTXODiffStore(t *testing.T) {
 	// Check that an error is returned when asking for non existing node
 	nonExistingNode := createNode()
 	_, err = dag.utxoDiffStore.diffByNode(nonExistingNode)
-	expectedErrString := fmt.Sprintf("Couldn't find diff data for block %s", nonExistingNode.hash)
-	if err == nil || err.Error() != expectedErrString {
-		t.Errorf("diffByNode: expected error %s but got %s", expectedErrString, err)
+	if !dbaccess.IsNotFoundError(err) {
+		if err != nil {
+			t.Errorf("diffByNode: %s", err)
+		} else {
+			t.Errorf("diffByNode: unexpectedly found diff data")
+		}
 	}
 
 	// Add node's diff data to the utxoDiffStore and check if it's checked correctly.
 	node := createNode()
 	diff := NewUTXODiff()
-	diff.toAdd.add(wire.Outpoint{TxID: daghash.TxID{0x01}, Index: 0}, &UTXOEntry{amount: 1, scriptPubKey: []byte{0x01}})
-	diff.toRemove.add(wire.Outpoint{TxID: daghash.TxID{0x02}, Index: 0}, &UTXOEntry{amount: 2, scriptPubKey: []byte{0x02}})
+	diff.toAdd.add(domainmessage.Outpoint{TxID: daghash.TxID{0x01}, Index: 0}, &UTXOEntry{amount: 1, scriptPubKey: []byte{0x01}})
+	diff.toRemove.add(domainmessage.Outpoint{TxID: daghash.TxID{0x02}, Index: 0}, &UTXOEntry{amount: 2, scriptPubKey: []byte{0x02}})
 	if err := dag.utxoDiffStore.setBlockDiff(node, diff); err != nil {
 		t.Fatalf("setBlockDiff: unexpected error: %s", err)
 	}
@@ -63,13 +66,20 @@ func TestUTXODiffStore(t *testing.T) {
 
 	// Flush changes to db, delete them from the dag.utxoDiffStore.loaded
 	// map, and check if the diff data is re-fetched from the database.
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		return dag.utxoDiffStore.flushToDB(dbTx)
-	})
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		t.Fatalf("Failed to open database transaction: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = dag.utxoDiffStore.flushToDB(dbTx)
 	if err != nil {
 		t.Fatalf("Error flushing utxoDiffStore data to DB: %s", err)
 	}
-	delete(dag.utxoDiffStore.loaded, *node.hash)
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Failed to commit database transaction: %s", err)
+	}
+	delete(dag.utxoDiffStore.loaded, node)
 
 	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
 		t.Fatalf("diffByNode: unexpected error: %s", err)
@@ -78,9 +88,80 @@ func TestUTXODiffStore(t *testing.T) {
 	}
 
 	// Check if getBlockDiff caches the result in dag.utxoDiffStore.loaded
-	if loadedDiffData, ok := dag.utxoDiffStore.loaded[*node.hash]; !ok {
+	if loadedDiffData, ok := dag.utxoDiffStore.loaded[node]; !ok {
 		t.Errorf("the diff data wasn't added to loaded map after requesting it")
 	} else if !reflect.DeepEqual(loadedDiffData.diff, diff) {
 		t.Errorf("Expected diff and loadedDiff to be equal")
 	}
 }
+
+func TestClearOldEntries(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestClearOldEntries", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestClearOldEntries: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Set maxBlueScoreDifferenceToKeepLoaded to 10 to make this test fast to run
+	currentDifference := maxBlueScoreDifferenceToKeepLoaded
+	maxBlueScoreDifferenceToKeepLoaded = 10
+	defer func() { maxBlueScoreDifferenceToKeepLoaded = currentDifference }()
+
+	// Add 10 blocks
+	blockNodes := make([]*blockNode, 10)
+	for i := 0; i < 10; i++ {
+		processedBlock := PrepareAndProcessBlockForTest(t, dag, dag.TipHashes(), nil)
+
+		node, ok := dag.index.LookupNode(processedBlock.BlockHash())
+		if !ok {
+			t.Fatalf("TestClearOldEntries: missing blockNode for hash %s", processedBlock.BlockHash())
+		}
+		blockNodes[i] = node
+	}
+
+	// Make sure that all of them exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if !ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is not in the loaded set", node.hash)
+		}
+	}
+
+	// Add 10 more blocks on top of the others
+	for i := 0; i < 10; i++ {
+		PrepareAndProcessBlockForTest(t, dag, dag.TipHashes(), nil)
+	}
+
+	// Make sure that all the old nodes no longer exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is in the loaded set", node.hash)
+		}
+	}
+
+	// Add a block on top of the genesis to force the retrieval of all diffData
+	processedBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	node, ok := dag.index.LookupNode(processedBlock.BlockHash())
+	if !ok {
+		t.Fatalf("TestClearOldEntries: missing blockNode for hash %s", processedBlock.BlockHash())
+	}
+
+	// Make sure that the child-of-genesis node is in the loaded set, since it
+	// is a tip.
+	_, ok = dag.utxoDiffStore.loaded[node]
+	if !ok {
+		t.Fatalf("TestClearOldEntries: diffData for node %s is not in the loaded set", node.hash)
+	}
+
+	// Make sure that all the old nodes still do not exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is in the loaded set", node.hash)
+		}
+	}
+}

blockdag/utxoio.go

@@ -2,15 +2,11 @@ package blockdag
 
 import (
 	"bytes"
-	"encoding/binary"
-	"fmt"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/binaryserializer"
+	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/pkg/errors"
 	"io"
-	"math/big"
-
-	"github.com/kaspanet/kaspad/ecc"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
 )
 
 // serializeBlockUTXODiffData serializes diff data in the following format:
@@ -19,76 +15,52 @@ import (
 // 	hasDiffChild | Boolean   | Indicates if a diff child exist
 //  diffChild    | Hash      | The diffChild's hash. Empty if hasDiffChild is true.
 //  diff		 | UTXODiff  | The diff data's diff
-func serializeBlockUTXODiffData(diffData *blockUTXODiffData) ([]byte, error) {
-	w := &bytes.Buffer{}
+func serializeBlockUTXODiffData(w io.Writer, diffData *blockUTXODiffData) error {
 	hasDiffChild := diffData.diffChild != nil
-	err := wire.WriteElement(w, hasDiffChild)
+	err := domainmessage.WriteElement(w, hasDiffChild)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	if hasDiffChild {
-		err := wire.WriteElement(w, diffData.diffChild.hash)
+		err := domainmessage.WriteElement(w, diffData.diffChild.hash)
 		if err != nil {
-			return nil, err
+			return err
 		}
 	}
 
 	err = serializeUTXODiff(w, diffData.diff)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	return w.Bytes(), nil
+	return nil
 }
 
-// utxoEntryHeaderCode returns the calculated header code to be used when
-// serializing the provided utxo entry.
-func utxoEntryHeaderCode(entry *UTXOEntry) uint64 {
-	// As described in the serialization format comments, the header code
-	// encodes the blue score shifted over one bit and the block reward flag
-	// in the lowest bit.
-	headerCode := uint64(entry.BlockBlueScore()) << 1
-	if entry.IsCoinbase() {
-		headerCode |= 0x01
-	}
-
-	return headerCode
-}
-
-func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataBytes []byte) (*blockUTXODiffData, error) {
+func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffData []byte) (*blockUTXODiffData, error) {
 	diffData := &blockUTXODiffData{}
-	serializedDiffData := bytes.NewBuffer(serializedDiffDataBytes)
+	r := bytes.NewBuffer(serializedDiffData)
 
 	var hasDiffChild bool
-	err := wire.ReadElement(serializedDiffData, &hasDiffChild)
+	err := domainmessage.ReadElement(r, &hasDiffChild)
 	if err != nil {
 		return nil, err
 	}
 
 	if hasDiffChild {
 		hash := &daghash.Hash{}
-		err := wire.ReadElement(serializedDiffData, hash)
+		err := domainmessage.ReadElement(r, hash)
 		if err != nil {
 			return nil, err
 		}
-		diffData.diffChild = diffStore.dag.index.LookupNode(hash)
+
+		var ok bool
+		diffData.diffChild, ok = diffStore.dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("block %s does not exist in the DAG", hash)
+		}
 	}
 
-	diffData.diff = &UTXODiff{
-		useMultiset: true,
-	}
-
-	diffData.diff.toAdd, err = deserializeDiffEntries(serializedDiffData)
-	if err != nil {
-		return nil, err
-	}
-
-	diffData.diff.toRemove, err = deserializeDiffEntries(serializedDiffData)
-	if err != nil {
-		return nil, err
-	}
-
-	diffData.diff.diffMultiset, err = deserializeMultiset(serializedDiffData)
+	diffData.diff, err = deserializeUTXODiff(r)
 	if err != nil {
 		return nil, err
 	}
@@ -96,38 +68,31 @@ func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataB
 	return diffData, nil
 }
 
-func deserializeDiffEntries(r io.Reader) (utxoCollection, error) {
-	count, err := wire.ReadVarInt(r)
+func deserializeUTXODiff(r io.Reader) (*UTXODiff, error) {
+	diff := &UTXODiff{}
+
+	var err error
+	diff.toAdd, err = deserializeUTXOCollection(r)
+	if err != nil {
+		return nil, err
+	}
+
+	diff.toRemove, err = deserializeUTXOCollection(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return diff, nil
+}
+
+func deserializeUTXOCollection(r io.Reader) (utxoCollection, error) {
+	count, err := domainmessage.ReadVarInt(r)
 	if err != nil {
 		return nil, err
 	}
 	collection := utxoCollection{}
 	for i := uint64(0); i < count; i++ {
-		outpointSize, err := wire.ReadVarInt(r)
-		if err != nil {
-			return nil, err
-		}
-
-		serializedOutpoint := make([]byte, outpointSize)
-		err = binary.Read(r, byteOrder, serializedOutpoint)
-		if err != nil {
-			return nil, err
-		}
-		outpoint, err := deserializeOutpoint(serializedOutpoint)
-		if err != nil {
-			return nil, err
-		}
-
-		utxoEntrySize, err := wire.ReadVarInt(r)
-		if err != nil {
-			return nil, err
-		}
-		serializedEntry := make([]byte, utxoEntrySize)
-		err = binary.Read(r, byteOrder, serializedEntry)
-		if err != nil {
-			return nil, err
-		}
-		utxoEntry, err := deserializeUTXOEntry(serializedEntry)
+		utxoEntry, outpoint, err := deserializeUTXO(r)
 		if err != nil {
 			return nil, err
 		}
@@ -136,31 +101,22 @@ func deserializeDiffEntries(r io.Reader) (utxoCollection, error) {
 	return collection, nil
 }
 
-// deserializeMultiset deserializes an EMCH multiset.
-// See serializeMultiset for more details.
-func deserializeMultiset(r io.Reader) (*ecc.Multiset, error) {
-	xBytes := make([]byte, multisetPointSize)
-	yBytes := make([]byte, multisetPointSize)
-	err := binary.Read(r, byteOrder, xBytes)
+func deserializeUTXO(r io.Reader) (*UTXOEntry, *domainmessage.Outpoint, error) {
+	outpoint, err := deserializeOutpoint(r)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	err = binary.Read(r, byteOrder, yBytes)
+
+	utxoEntry, err := deserializeUTXOEntry(r)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	var x, y big.Int
-	x.SetBytes(xBytes)
-	y.SetBytes(yBytes)
-	return ecc.NewMultisetFromPoint(ecc.S256(), &x, &y), nil
+	return utxoEntry, outpoint, nil
 }
 
 // serializeUTXODiff serializes UTXODiff by serializing
 // UTXODiff.toAdd, UTXODiff.toRemove and UTXODiff.Multiset one after the other.
 func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
-	if !diff.useMultiset {
-		return errors.New("Cannot serialize a UTXO diff without a multiset")
-	}
 	err := serializeUTXOCollection(w, diff.toAdd)
 	if err != nil {
 		return err
@@ -170,10 +126,7 @@ func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
 	if err != nil {
 		return err
 	}
-	err = serializeMultiset(w, diff.diffMultiset)
-	if err != nil {
-		return err
-	}
+
 	return nil
 }
 
@@ -181,7 +134,7 @@ func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
 // the utxo entries and serializing them and their corresponding outpoint
 // prefixed by a varint that indicates their size.
 func serializeUTXOCollection(w io.Writer, collection utxoCollection) error {
-	err := wire.WriteVarInt(w, uint64(len(collection)))
+	err := domainmessage.WriteVarInt(w, uint64(len(collection)))
 	if err != nil {
 		return err
 	}
@@ -194,120 +147,93 @@ func serializeUTXOCollection(w io.Writer, collection utxoCollection) error {
 	return nil
 }
 
-// serializeMultiset serializes an ECMH multiset. The serialization
-// is done by taking the (x,y) coordinnates of the multiset point and
-// padding each one of them with 32 byte (it'll be 32 byte in most
-// cases anyway except one of the coordinates is zero) and writing
-// them one after the other.
-func serializeMultiset(w io.Writer, ms *ecc.Multiset) error {
-	x, y := ms.Point()
-	xBytes := make([]byte, multisetPointSize)
-	copy(xBytes, x.Bytes())
-	yBytes := make([]byte, multisetPointSize)
-	copy(yBytes, y.Bytes())
-
-	err := binary.Write(w, byteOrder, xBytes)
-	if err != nil {
-		return err
-	}
-	err = binary.Write(w, byteOrder, yBytes)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 // serializeUTXO serializes a utxo entry-outpoint pair
-func serializeUTXO(w io.Writer, entry *UTXOEntry, outpoint *wire.Outpoint) error {
-	serializedOutpoint := *outpointKey(*outpoint)
-	err := wire.WriteVarInt(w, uint64(len(serializedOutpoint)))
+func serializeUTXO(w io.Writer, entry *UTXOEntry, outpoint *domainmessage.Outpoint) error {
+	err := serializeOutpoint(w, outpoint)
 	if err != nil {
 		return err
 	}
 
-	err = binary.Write(w, byteOrder, serializedOutpoint)
-	if err != nil {
-		return err
-	}
-
-	serializedUTXOEntry := serializeUTXOEntry(entry)
-	err = wire.WriteVarInt(w, uint64(len(serializedUTXOEntry)))
-	if err != nil {
-		return err
-	}
-	err = binary.Write(w, byteOrder, serializedUTXOEntry)
+	err = serializeUTXOEntry(w, entry)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-// serializeUTXOEntry returns the entry serialized to a format that is suitable
-// for long-term storage. The format is described in detail above.
-func serializeUTXOEntry(entry *UTXOEntry) []byte {
-	// Encode the header code.
-	headerCode := utxoEntryHeaderCode(entry)
+// p2pkhUTXOEntrySerializeSize is the serialized size for a P2PKH UTXO entry.
+// 8 bytes (header code) + 8 bytes (amount) + varint for script pub key length of 25 (for P2PKH) + 25 bytes for P2PKH script.
+var p2pkhUTXOEntrySerializeSize = 8 + 8 + domainmessage.VarIntSerializeSize(25) + 25
 
-	// Calculate the size needed to serialize the entry.
-	size := serializeSizeVLQ(headerCode) +
-		compressedTxOutSize(uint64(entry.Amount()), entry.ScriptPubKey())
-
-	// Serialize the header code followed by the compressed unspent
-	// transaction output.
-	serialized := make([]byte, size)
-	offset := putVLQ(serialized, headerCode)
-	offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()),
-		entry.ScriptPubKey())
-
-	return serialized
-}
-
-// deserializeOutpoint decodes an outpoint from the passed serialized byte
-// slice into a new wire.Outpoint using a format that is suitable for long-
-// term storage. this format is described in detail above.
-func deserializeOutpoint(serialized []byte) (*wire.Outpoint, error) {
-	if len(serialized) <= daghash.HashSize {
-		return nil, errDeserialize("unexpected end of data")
-	}
-
-	txID := daghash.TxID{}
-	txID.SetBytes(serialized[:daghash.HashSize])
-	index, _ := deserializeVLQ(serialized[daghash.HashSize:])
-	return wire.NewOutpoint(&txID, uint32(index)), nil
-}
-
-// deserializeUTXOEntry decodes a UTXO entry from the passed serialized byte
-// slice into a new UTXOEntry using a format that is suitable for long-term
-// storage. The format is described in detail above.
-func deserializeUTXOEntry(serialized []byte) (*UTXOEntry, error) {
-	// Deserialize the header code.
-	code, offset := deserializeVLQ(serialized)
-	if offset >= len(serialized) {
-		return nil, errDeserialize("unexpected end of data after header")
-	}
-
-	// Decode the header code.
-	//
-	// Bit 0 indicates whether the containing transaction is a coinbase.
-	// Bits 1-x encode blue score of the containing transaction.
-	isCoinbase := code&0x01 != 0
-	blockBlueScore := code >> 1
-
-	// Decode the compressed unspent transaction output.
-	amount, scriptPubKey, _, err := decodeCompressedTxOut(serialized[offset:])
+// serializeUTXOEntry encodes the entry to the given io.Writer and use compression if useCompression is true.
+// The compression format is described in detail above.
+func serializeUTXOEntry(w io.Writer, entry *UTXOEntry) error {
+	// Encode the blueScore.
+	err := binaryserializer.PutUint64(w, byteOrder, entry.blockBlueScore)
 	if err != nil {
-		return nil, errDeserialize(fmt.Sprintf("unable to decode "+
-			"UTXO: %s", err))
+		return err
+	}
+
+	// Encode the packedFlags.
+	err = binaryserializer.PutUint8(w, uint8(entry.packedFlags))
+	if err != nil {
+		return err
+	}
+
+	err = binaryserializer.PutUint64(w, byteOrder, entry.Amount())
+	if err != nil {
+		return err
+	}
+
+	err = domainmessage.WriteVarInt(w, uint64(len(entry.ScriptPubKey())))
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(entry.ScriptPubKey())
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+// deserializeUTXOEntry decodes a UTXO entry from the passed reader
+// into a new UTXOEntry. If isCompressed is used it will decompress
+// the entry according to the format that is described in detail
+// above.
+func deserializeUTXOEntry(r io.Reader) (*UTXOEntry, error) {
+	// Deserialize the blueScore.
+	blockBlueScore, err := binaryserializer.Uint64(r, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decode the packedFlags.
+	packedFlags, err := binaryserializer.Uint8(r)
+	if err != nil {
+		return nil, err
 	}
 
 	entry := &UTXOEntry{
-		amount:         amount,
-		scriptPubKey:   scriptPubKey,
 		blockBlueScore: blockBlueScore,
-		packedFlags:    0,
+		packedFlags:    txoFlags(packedFlags),
 	}
-	if isCoinbase {
-		entry.packedFlags |= tfCoinbase
+
+	entry.amount, err = binaryserializer.Uint64(r, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	scriptPubKeyLen, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return nil, err
+	}
+
+	entry.scriptPubKey = make([]byte, scriptPubKeyLen)
+	_, err = r.Read(entry.scriptPubKey)
+	if err != nil {
+		return nil, errors.WithStack(err)
 	}
 
 	return entry, nil

blockdag/utxoset.go

@@ -2,13 +2,14 @@ package blockdag
 
 import (
 	"fmt"
-	"github.com/pkg/errors"
 	"math"
 	"sort"
 	"strings"
 
-	"github.com/kaspanet/kaspad/ecc"
-	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/domainmessage"
 )
 
 const (
@@ -77,7 +78,7 @@ const (
 )
 
 // NewUTXOEntry creates a new utxoEntry representing the given txOut
-func NewUTXOEntry(txOut *wire.TxOut, isCoinbase bool, blockBlueScore uint64) *UTXOEntry {
+func NewUTXOEntry(txOut *domainmessage.TxOut, isCoinbase bool, blockBlueScore uint64) *UTXOEntry {
 	entry := &UTXOEntry{
 		amount:         txOut.Value,
 		scriptPubKey:   txOut.ScriptPubKey,
@@ -92,7 +93,7 @@ func NewUTXOEntry(txOut *wire.TxOut, isCoinbase bool, blockBlueScore uint64) *UT
 }
 
 // utxoCollection represents a set of UTXOs indexed by their outpoints
-type utxoCollection map[wire.Outpoint]*UTXOEntry
+type utxoCollection map[domainmessage.Outpoint]*UTXOEntry
 
 func (uc utxoCollection) String() string {
 	utxoStrings := make([]string, len(uc))
@@ -111,31 +112,31 @@ func (uc utxoCollection) String() string {
 }
 
 // add adds a new UTXO entry to this collection
-func (uc utxoCollection) add(outpoint wire.Outpoint, entry *UTXOEntry) {
+func (uc utxoCollection) add(outpoint domainmessage.Outpoint, entry *UTXOEntry) {
 	uc[outpoint] = entry
 }
 
 // remove removes a UTXO entry from this collection if it exists
-func (uc utxoCollection) remove(outpoint wire.Outpoint) {
+func (uc utxoCollection) remove(outpoint domainmessage.Outpoint) {
 	delete(uc, outpoint)
 }
 
 // get returns the UTXOEntry represented by provided outpoint,
 // and a boolean value indicating if said UTXOEntry is in the set or not
-func (uc utxoCollection) get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
+func (uc utxoCollection) get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool) {
 	entry, ok := uc[outpoint]
 	return entry, ok
 }
 
 // contains returns a boolean value indicating whether a UTXO entry is in the set
-func (uc utxoCollection) contains(outpoint wire.Outpoint) bool {
+func (uc utxoCollection) contains(outpoint domainmessage.Outpoint) bool {
 	_, ok := uc[outpoint]
 	return ok
 }
 
 // containsWithBlueScore returns a boolean value indicating whether a UTXOEntry
 // is in the set and its blue score is equal to the given blue score.
-func (uc utxoCollection) containsWithBlueScore(outpoint wire.Outpoint, blueScore uint64) bool {
+func (uc utxoCollection) containsWithBlueScore(outpoint domainmessage.Outpoint, blueScore uint64) bool {
 	entry, ok := uc.get(outpoint)
 	return ok && entry.blockBlueScore == blueScore
 }
@@ -152,29 +153,16 @@ func (uc utxoCollection) clone() utxoCollection {
 
 // UTXODiff represents a diff between two UTXO Sets.
 type UTXODiff struct {
-	toAdd        utxoCollection
-	toRemove     utxoCollection
-	diffMultiset *ecc.Multiset
-	useMultiset  bool
+	toAdd    utxoCollection
+	toRemove utxoCollection
 }
 
-// NewUTXODiffWithoutMultiset creates a new, empty utxoDiff
+// NewUTXODiff creates a new, empty utxoDiff
 // without a multiset.
-func NewUTXODiffWithoutMultiset() *UTXODiff {
-	return &UTXODiff{
-		toAdd:       utxoCollection{},
-		toRemove:    utxoCollection{},
-		useMultiset: false,
-	}
-}
-
-// NewUTXODiff creates a new, empty utxoDiff.
 func NewUTXODiff() *UTXODiff {
 	return &UTXODiff{
-		toAdd:        utxoCollection{},
-		toRemove:     utxoCollection{},
-		useMultiset:  true,
-		diffMultiset: ecc.NewMultiset(ecc.S256()),
+		toAdd:    utxoCollection{},
+		toRemove: utxoCollection{},
 	}
 }
 
@@ -208,9 +196,8 @@ func NewUTXODiff() *UTXODiff {
 //    diffFrom results in the UTXO being added to toAdd
 func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 	result := UTXODiff{
-		toAdd:       make(utxoCollection, len(d.toRemove)+len(other.toAdd)),
-		toRemove:    make(utxoCollection, len(d.toAdd)+len(other.toRemove)),
-		useMultiset: d.useMultiset,
+		toAdd:    make(utxoCollection, len(d.toRemove)+len(other.toAdd)),
+		toRemove: make(utxoCollection, len(d.toAdd)+len(other.toRemove)),
 	}
 
 	// Note that the following cases are not accounted for, as they are impossible
@@ -224,6 +211,10 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 	for outpoint, utxoEntry := range d.toAdd {
 		if !other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
 			result.toRemove.add(outpoint, utxoEntry)
+		} else if (d.toRemove.contains(outpoint) && !other.toRemove.contains(outpoint)) ||
+			(!d.toRemove.contains(outpoint) && other.toRemove.contains(outpoint)) {
+			return nil, errors.New(
+				"diffFrom: outpoint both in d.toAdd, other.toAdd, and only one of d.toRemove and other.toRemove")
 		}
 		if diffEntry, ok := other.toRemove.get(outpoint); ok {
 			// An exception is made for entries with unequal blue scores
@@ -243,6 +234,18 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 	// If they are not in other.toRemove - should be added in result.toAdd
 	// If they are in other.toAdd - base utxoSet is not the same
 	for outpoint, utxoEntry := range d.toRemove {
+		diffEntry, ok := other.toRemove.get(outpoint)
+		if ok {
+			// if have the same entry in d.toRemove - simply don't copy.
+			// unless existing entry is with different blue score, in this case - this is an error
+			if utxoEntry.blockBlueScore != diffEntry.blockBlueScore {
+				return nil, errors.New("diffFrom: outpoint both in d.toRemove and other.toRemove with different " +
+					"blue scores, with no corresponding entry in d.toAdd")
+			}
+		} else { // if no existing entry - add to result.toAdd
+			result.toAdd.add(outpoint, utxoEntry)
+		}
+
 		if !other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
 			result.toAdd.add(outpoint, utxoEntry)
 		}
@@ -256,7 +259,7 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 					other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
 				continue
 			}
-			return nil, errors.New("diffFrom: transaction both in d.toRemove and in other.toAdd")
+			return nil, errors.New("diffFrom: outpoint both in d.toRemove and in other.toAdd")
 		}
 	}
 
@@ -276,124 +279,72 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 		}
 	}
 
-	if d.useMultiset {
-		// Create a new diffMultiset as the subtraction of the two diffs.
-		result.diffMultiset = other.diffMultiset.Subtract(d.diffMultiset)
-	}
-
 	return &result, nil
 }
 
-// WithDiff applies provided diff to this diff, creating a new utxoDiff, that would be the result if
+// withDiffInPlace applies provided diff to this diff in-place, that would be the result if
 // first d, and than diff were applied to the same base
-//
-// WithDiff follows a set of rules represented by the following 3 by 3 table:
-//
-//          |           | this      |           |
-// ---------+-----------+-----------+-----------+-----------
-//          |           | toAdd     | toRemove  | None
-// ---------+-----------+-----------+-----------+-----------
-// other    | toAdd     | X         | -         | toAdd
-// ---------+-----------+-----------+-----------+-----------
-//          | toRemove  | -         | X         | toRemove
-// ---------+-----------+-----------+-----------+-----------
-//          | None      | toAdd     | toRemove  | -
-//
-// Key:
-// -		Don't add anything to the result
-// X		Return an error
-// toAdd	Add the UTXO into the toAdd collection of the result
-// toRemove	Add the UTXO into the toRemove collection of the result
-//
-// Examples:
-// 1. This diff contains a UTXO in toAdd, and the other diff contains it in toRemove
-//    WithDiff results in nothing being added
-// 2. This diff contains a UTXO in toRemove, and the other diff does not contain it
-//    WithDiff results in the UTXO being added to toRemove
+func (d *UTXODiff) withDiffInPlace(diff *UTXODiff) error {
+	for outpoint, entryToRemove := range diff.toRemove {
+		if d.toAdd.containsWithBlueScore(outpoint, entryToRemove.blockBlueScore) {
+			// If already exists in toAdd with the same blueScore - remove from toAdd
+			d.toAdd.remove(outpoint)
+			continue
+		}
+		if d.toRemove.contains(outpoint) {
+			// If already exists - this is an error
+			return errors.Errorf(
+				"withDiffInPlace: outpoint %s both in d.toRemove and in diff.toRemove", outpoint)
+		}
+
+		// If not exists neither in toAdd nor in toRemove - add to toRemove
+		d.toRemove.add(outpoint, entryToRemove)
+	}
+
+	for outpoint, entryToAdd := range diff.toAdd {
+		if d.toRemove.containsWithBlueScore(outpoint, entryToAdd.blockBlueScore) {
+			// If already exists in toRemove with the same blueScore - remove from toRemove
+			if d.toAdd.contains(outpoint) && !diff.toRemove.contains(outpoint) {
+				return errors.Errorf(
+					"withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd with no "+
+						"corresponding entry in diff.toRemove", outpoint)
+			}
+			d.toRemove.remove(outpoint)
+			continue
+		}
+		if existingEntry, ok := d.toAdd.get(outpoint); ok &&
+			(existingEntry.blockBlueScore == entryToAdd.blockBlueScore ||
+				!diff.toRemove.containsWithBlueScore(outpoint, existingEntry.blockBlueScore)) {
+			// If already exists - this is an error
+			return errors.Errorf(
+				"withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd", outpoint)
+		}
+
+		// If not exists neither in toAdd nor in toRemove, or exists in toRemove with different blueScore - add to toAdd
+		d.toAdd.add(outpoint, entryToAdd)
+	}
+
+	return nil
+}
+
+// WithDiff applies provided diff to this diff, creating a new utxoDiff, that would be the result if
+// first d, and than diff were applied to some base
 func (d *UTXODiff) WithDiff(diff *UTXODiff) (*UTXODiff, error) {
-	result := UTXODiff{
-		toAdd:       make(utxoCollection, len(d.toAdd)+len(diff.toAdd)),
-		toRemove:    make(utxoCollection, len(d.toRemove)+len(diff.toRemove)),
-		useMultiset: d.useMultiset,
+	clone := d.clone()
+
+	err := clone.withDiffInPlace(diff)
+	if err != nil {
+		return nil, err
 	}
 
-	// All transactions in d.toAdd:
-	// If they are not in diff.toRemove - should be added in result.toAdd
-	// If they are in diff.toAdd - should throw an error
-	// Otherwise - should be ignored
-	for outpoint, utxoEntry := range d.toAdd {
-		if !diff.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toAdd.add(outpoint, utxoEntry)
-		}
-		if diffEntry, ok := diff.toAdd.get(outpoint); ok {
-			// An exception is made for entries with unequal blue scores
-			// as long as the appropriate entry exists in either d.toRemove
-			// or diff.toRemove.
-			// These are just "updates" to accepted blue score
-			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
-				(d.toRemove.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
-					diff.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
-				continue
-			}
-			return nil, ruleError(ErrWithDiff, fmt.Sprintf("WithDiff: outpoint %s both in d.toAdd and in other.toAdd", outpoint))
-		}
-	}
-
-	// All transactions in d.toRemove:
-	// If they are not in diff.toAdd - should be added in result.toRemove
-	// If they are in diff.toRemove - should throw an error
-	// Otherwise - should be ignored
-	for outpoint, utxoEntry := range d.toRemove {
-		if !diff.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toRemove.add(outpoint, utxoEntry)
-		}
-		if diffEntry, ok := diff.toRemove.get(outpoint); ok {
-			// An exception is made for entries with unequal blue scores
-			// as long as the appropriate entry exists in either d.toAdd
-			// or diff.toAdd.
-			// These are just "updates" to accepted blue score
-			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
-				(d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
-					diff.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
-				continue
-			}
-			return nil, ruleError(ErrWithDiff, "WithDiff: transaction both in d.toRemove and in other.toRemove")
-		}
-	}
-
-	// All transactions in diff.toAdd:
-	// If they are not in d.toRemove - should be added in result.toAdd
-	for outpoint, utxoEntry := range diff.toAdd {
-		if !d.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toAdd.add(outpoint, utxoEntry)
-		}
-	}
-
-	// All transactions in diff.toRemove:
-	// If they are not in d.toAdd - should be added in result.toRemove
-	for outpoint, utxoEntry := range diff.toRemove {
-		if !d.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toRemove.add(outpoint, utxoEntry)
-		}
-	}
-
-	// Apply diff.diffMultiset to d.diffMultiset
-	if d.useMultiset {
-		result.diffMultiset = d.diffMultiset.Union(diff.diffMultiset)
-	}
-
-	return &result, nil
+	return clone, nil
 }
 
 // clone returns a clone of this utxoDiff
 func (d *UTXODiff) clone() *UTXODiff {
 	clone := &UTXODiff{
-		toAdd:       d.toAdd.clone(),
-		toRemove:    d.toRemove.clone(),
-		useMultiset: d.useMultiset,
-	}
-	if d.useMultiset {
-		clone.diffMultiset = d.diffMultiset.Clone()
+		toAdd:    d.toAdd.clone(),
+		toRemove: d.toRemove.clone(),
 	}
 	return clone
 }
@@ -402,7 +353,7 @@ func (d *UTXODiff) clone() *UTXODiff {
 //
 // If d.useMultiset is true, this function MUST be
 // called with the DAG lock held.
-func (d *UTXODiff) AddEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
+func (d *UTXODiff) AddEntry(outpoint domainmessage.Outpoint, entry *UTXOEntry) error {
 	if d.toRemove.containsWithBlueScore(outpoint, entry.blockBlueScore) {
 		d.toRemove.remove(outpoint)
 	} else if _, exists := d.toAdd[outpoint]; exists {
@@ -410,14 +361,6 @@ func (d *UTXODiff) AddEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
 	} else {
 		d.toAdd.add(outpoint, entry)
 	}
-
-	if d.useMultiset {
-		newMs, err := addUTXOToMultiset(d.diffMultiset, entry, &outpoint)
-		if err != nil {
-			return err
-		}
-		d.diffMultiset = newMs
-	}
 	return nil
 }
 
@@ -425,7 +368,7 @@ func (d *UTXODiff) AddEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
 //
 // If d.useMultiset is true, this function MUST be
 // called with the DAG lock held.
-func (d *UTXODiff) RemoveEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
+func (d *UTXODiff) RemoveEntry(outpoint domainmessage.Outpoint, entry *UTXOEntry) error {
 	if d.toAdd.containsWithBlueScore(outpoint, entry.blockBlueScore) {
 		d.toAdd.remove(outpoint)
 	} else if _, exists := d.toRemove[outpoint]; exists {
@@ -433,21 +376,10 @@ func (d *UTXODiff) RemoveEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
 	} else {
 		d.toRemove.add(outpoint, entry)
 	}
-
-	if d.useMultiset {
-		newMs, err := removeUTXOFromMultiset(d.diffMultiset, entry, &outpoint)
-		if err != nil {
-			return err
-		}
-		d.diffMultiset = newMs
-	}
 	return nil
 }
 
 func (d UTXODiff) String() string {
-	if d.useMultiset {
-		return fmt.Sprintf("toAdd: %s; toRemove: %s, Multiset-Hash: %s", d.toAdd, d.toRemove, d.diffMultiset.Hash())
-	}
 	return fmt.Sprintf("toAdd: %s; toRemove: %s", d.toAdd, d.toRemove)
 }
 
@@ -467,97 +399,27 @@ type UTXOSet interface {
 	fmt.Stringer
 	diffFrom(other UTXOSet) (*UTXODiff, error)
 	WithDiff(utxoDiff *UTXODiff) (UTXOSet, error)
-	diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error)
-	diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error)
-	AddTx(tx *wire.MsgTx, blockBlueScore uint64) (ok bool, err error)
+	AddTx(tx *domainmessage.MsgTx, blockBlueScore uint64) (ok bool, err error)
 	clone() UTXOSet
-	Get(outpoint wire.Outpoint) (*UTXOEntry, bool)
-	Multiset() *ecc.Multiset
-	WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error)
-}
-
-// diffFromTx is a common implementation for diffFromTx, that works
-// for both diff-based and full UTXO sets
-// Returns a diff that is equivalent to provided transaction,
-// or an error if provided transaction is not valid in the context of this UTXOSet
-func diffFromTx(u UTXOSet, tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	diff := NewUTXODiff()
-	isCoinbase := tx.IsCoinBase()
-	if !isCoinbase {
-		for _, txIn := range tx.TxIn {
-			if entry, ok := u.Get(txIn.PreviousOutpoint); ok {
-				err := diff.RemoveEntry(txIn.PreviousOutpoint, entry)
-				if err != nil {
-					return nil, err
-				}
-			} else {
-				return nil, ruleError(ErrMissingTxOut, fmt.Sprintf(
-					"Transaction %s is invalid because spends outpoint %s that is not in utxo set",
-					tx.TxID(), txIn.PreviousOutpoint))
-			}
-		}
-	}
-	for i, txOut := range tx.TxOut {
-		entry := NewUTXOEntry(txOut, isCoinbase, acceptingBlueScore)
-		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
-		err := diff.AddEntry(outpoint, entry)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return diff, nil
-}
-
-// diffFromAcceptedTx is a common implementation for diffFromAcceptedTx, that works
-// for both diff-based and full UTXO sets.
-// Returns a diff that replaces an entry's blockBlueScore with the given acceptingBlueScore.
-// Returns an error if the provided transaction's entry is not valid in the context
-// of this UTXOSet.
-func diffFromAcceptedTx(u UTXOSet, tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	diff := NewUTXODiff()
-	isCoinbase := tx.IsCoinBase()
-	for i, txOut := range tx.TxOut {
-		// Fetch any unaccepted transaction
-		existingOutpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
-		existingEntry, ok := u.Get(existingOutpoint)
-		if !ok {
-			return nil, errors.Errorf("cannot accept outpoint %s because it doesn't exist in the given UTXO", existingOutpoint)
-		}
-
-		// Remove unaccepted entries
-		err := diff.RemoveEntry(existingOutpoint, existingEntry)
-		if err != nil {
-			return nil, err
-		}
-
-		// Add new entries with their accepting blue score
-		newEntry := NewUTXOEntry(txOut, isCoinbase, acceptingBlueScore)
-		err = diff.AddEntry(existingOutpoint, newEntry)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return diff, nil
+	Get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool)
 }
 
 // FullUTXOSet represents a full list of transaction outputs and their values
 type FullUTXOSet struct {
 	utxoCollection
-	UTXOMultiset *ecc.Multiset
 }
 
 // NewFullUTXOSet creates a new utxoSet with full list of transaction outputs and their values
 func NewFullUTXOSet() *FullUTXOSet {
 	return &FullUTXOSet{
 		utxoCollection: utxoCollection{},
-		UTXOMultiset:   ecc.NewMultiset(ecc.S256()),
 	}
 }
 
 // newFullUTXOSetFromUTXOCollection converts a utxoCollection to a FullUTXOSet
 func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet, error) {
 	var err error
-	multiset := ecc.NewMultiset(ecc.S256())
+	multiset := secp256k1.NewMultiset()
 	for outpoint, utxoEntry := range collection {
 		multiset, err = addUTXOToMultiset(multiset, utxoEntry, &outpoint)
 		if err != nil {
@@ -566,7 +428,6 @@ func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet,
 	}
 	return &FullUTXOSet{
 		utxoCollection: collection,
-		UTXOMultiset:   multiset,
 	}, nil
 }
 
@@ -595,44 +456,28 @@ func (fus *FullUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
 // necessarily means there's an error).
 //
 // This function MUST be called with the DAG lock held.
-func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blueScore uint64) (isAccepted bool, err error) {
-	isCoinbase := tx.IsCoinBase()
-	if !isCoinbase {
-		if !fus.containsInputs(tx) {
-			return false, nil
-		}
-
-		for _, txIn := range tx.TxIn {
-			outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
-			err := fus.removeAndUpdateMultiset(outpoint)
-			if err != nil {
-				return false, err
-			}
-		}
+func (fus *FullUTXOSet) AddTx(tx *domainmessage.MsgTx, blueScore uint64) (isAccepted bool, err error) {
+	if !fus.containsInputs(tx) {
+		return false, nil
 	}
 
-	for i, txOut := range tx.TxOut {
-		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
-		entry := NewUTXOEntry(txOut, isCoinbase, blueScore)
+	for _, txIn := range tx.TxIn {
+		fus.remove(txIn.PreviousOutpoint)
+	}
 
-		err := fus.addAndUpdateMultiset(outpoint, entry)
-		if err != nil {
-			return false, err
-		}
+	isCoinbase := tx.IsCoinBase()
+	for i, txOut := range tx.TxOut {
+		outpoint := *domainmessage.NewOutpoint(tx.TxID(), uint32(i))
+		entry := NewUTXOEntry(txOut, isCoinbase, blueScore)
+		fus.add(outpoint, entry)
 	}
 
 	return true, nil
 }
 
-// diffFromTx returns a diff that is equivalent to provided transaction,
-// or an error if provided transaction is not valid in the context of this UTXOSet
-func (fus *FullUTXOSet) diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromTx(fus, tx, acceptingBlueScore)
-}
-
-func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool {
+func (fus *FullUTXOSet) containsInputs(tx *domainmessage.MsgTx) bool {
 	for _, txIn := range tx.TxIn {
-		outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+		outpoint := *domainmessage.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
 		if !fus.contains(outpoint) {
 			return false
 		}
@@ -641,70 +486,17 @@ func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 	return true
 }
 
-func (fus *FullUTXOSet) diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromAcceptedTx(fus, tx, acceptingBlueScore)
-}
-
 // clone returns a clone of this utxoSet
 func (fus *FullUTXOSet) clone() UTXOSet {
-	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone(), UTXOMultiset: fus.UTXOMultiset.Clone()}
+	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone()}
 }
 
 // Get returns the UTXOEntry associated with the given Outpoint, and a boolean indicating if such entry was found
-func (fus *FullUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
+func (fus *FullUTXOSet) Get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool) {
 	utxoEntry, ok := fus.utxoCollection[outpoint]
 	return utxoEntry, ok
 }
 
-// Multiset returns the ecmh-Multiset of this utxoSet
-func (fus *FullUTXOSet) Multiset() *ecc.Multiset {
-	return fus.UTXOMultiset
-}
-
-// addAndUpdateMultiset adds a UTXOEntry to this utxoSet and updates its multiset accordingly
-func (fus *FullUTXOSet) addAndUpdateMultiset(outpoint wire.Outpoint, entry *UTXOEntry) error {
-	fus.add(outpoint, entry)
-	newMs, err := addUTXOToMultiset(fus.UTXOMultiset, entry, &outpoint)
-	if err != nil {
-		return err
-	}
-	fus.UTXOMultiset = newMs
-	return nil
-}
-
-// removeAndUpdateMultiset removes a UTXOEntry from this utxoSet and updates its multiset accordingly
-func (fus *FullUTXOSet) removeAndUpdateMultiset(outpoint wire.Outpoint) error {
-	entry, ok := fus.Get(outpoint)
-	if !ok {
-		return errors.Errorf("Couldn't find outpoint %s", outpoint)
-	}
-	fus.remove(outpoint)
-	var err error
-	newMs, err := removeUTXOFromMultiset(fus.UTXOMultiset, entry, &outpoint)
-	if err != nil {
-		return err
-	}
-	fus.UTXOMultiset = newMs
-	return nil
-}
-
-// WithTransactions returns a new UTXO Set with the added transactions.
-//
-// This function MUST be called with the DAG lock held.
-func (fus *FullUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error) {
-	diffSet := NewDiffUTXOSet(fus, NewUTXODiff())
-	for _, tx := range transactions {
-		isAccepted, err := diffSet.AddTx(tx, blockBlueScore)
-		if err != nil {
-			return nil, err
-		}
-		if !ignoreDoubleSpends && !isAccepted {
-			return nil, errors.Errorf("Transaction %s is not valid with the current UTXO set", tx.TxID())
-		}
-	}
-	return UTXOSet(diffSet), nil
-}
-
 // DiffUTXOSet represents a utxoSet with a base fullUTXOSet and a UTXODiff
 type DiffUTXOSet struct {
 	base     *FullUTXOSet
@@ -748,13 +540,12 @@ func (dus *DiffUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
 //
 // If dus.UTXODiff.useMultiset is true, this function MUST be
 // called with the DAG lock held.
-func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockBlueScore uint64) (bool, error) {
-	isCoinbase := tx.IsCoinBase()
-	if !isCoinbase && !dus.containsInputs(tx) {
+func (dus *DiffUTXOSet) AddTx(tx *domainmessage.MsgTx, blockBlueScore uint64) (bool, error) {
+	if !dus.containsInputs(tx) {
 		return false, nil
 	}
 
-	err := dus.appendTx(tx, blockBlueScore, isCoinbase)
+	err := dus.appendTx(tx, blockBlueScore)
 	if err != nil {
 		return false, err
 	}
@@ -762,23 +553,21 @@ func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockBlueScore uint64) (bool, erro
 	return true, nil
 }
 
-func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockBlueScore uint64, isCoinbase bool) error {
-	if !isCoinbase {
-		for _, txIn := range tx.TxIn {
-			outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
-			entry, ok := dus.Get(outpoint)
-			if !ok {
-				return errors.Errorf("Couldn't find entry for outpoint %s", outpoint)
-			}
-			err := dus.UTXODiff.RemoveEntry(outpoint, entry)
-			if err != nil {
-				return err
-			}
+func (dus *DiffUTXOSet) appendTx(tx *domainmessage.MsgTx, blockBlueScore uint64) error {
+	for _, txIn := range tx.TxIn {
+		entry, ok := dus.Get(txIn.PreviousOutpoint)
+		if !ok {
+			return errors.Errorf("couldn't find entry for outpoint %s", txIn.PreviousOutpoint)
+		}
+		err := dus.UTXODiff.RemoveEntry(txIn.PreviousOutpoint, entry)
+		if err != nil {
+			return err
 		}
 	}
 
+	isCoinbase := tx.IsCoinBase()
 	for i, txOut := range tx.TxOut {
-		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
+		outpoint := *domainmessage.NewOutpoint(tx.TxID(), uint32(i))
 		entry := NewUTXOEntry(txOut, isCoinbase, blockBlueScore)
 
 		err := dus.UTXODiff.AddEntry(outpoint, entry)
@@ -789,9 +578,9 @@ func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockBlueScore uint64, isCoinba
 	return nil
 }
 
-func (dus *DiffUTXOSet) containsInputs(tx *wire.MsgTx) bool {
+func (dus *DiffUTXOSet) containsInputs(tx *domainmessage.MsgTx) bool {
 	for _, txIn := range tx.TxIn {
-		outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+		outpoint := *domainmessage.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
 		isInBase := dus.base.contains(outpoint)
 		isInDiffToAdd := dus.UTXODiff.toAdd.contains(outpoint)
 		isInDiffToRemove := dus.UTXODiff.toRemove.contains(outpoint)
@@ -816,31 +605,12 @@ func (dus *DiffUTXOSet) meldToBase() error {
 	for outpoint, utxoEntry := range dus.UTXODiff.toAdd {
 		dus.base.add(outpoint, utxoEntry)
 	}
-
-	if dus.UTXODiff.useMultiset {
-		dus.base.UTXOMultiset = dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset)
-	}
-
-	if dus.UTXODiff.useMultiset {
-		dus.UTXODiff = NewUTXODiff()
-	} else {
-		dus.UTXODiff = NewUTXODiffWithoutMultiset()
-	}
+	dus.UTXODiff = NewUTXODiff()
 	return nil
 }
 
-// diffFromTx returns a diff that is equivalent to provided transaction,
-// or an error if provided transaction is not valid in the context of this UTXOSet
-func (dus *DiffUTXOSet) diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromTx(dus, tx, acceptingBlueScore)
-}
-
-func (dus *DiffUTXOSet) diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromAcceptedTx(dus, tx, acceptingBlueScore)
-}
-
 func (dus *DiffUTXOSet) String() string {
-	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s, Multiset-Hash:%s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove, dus.Multiset().Hash())
+	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove)
 }
 
 // clone returns a clone of this UTXO Set
@@ -848,9 +618,15 @@ func (dus *DiffUTXOSet) clone() UTXOSet {
 	return NewDiffUTXOSet(dus.base.clone().(*FullUTXOSet), dus.UTXODiff.clone())
 }
 
+// cloneWithoutBase returns a *DiffUTXOSet with same
+// base as this *DiffUTXOSet and a cloned diff.
+func (dus *DiffUTXOSet) cloneWithoutBase() UTXOSet {
+	return NewDiffUTXOSet(dus.base, dus.UTXODiff.clone())
+}
+
 // Get returns the UTXOEntry associated with provided outpoint in this UTXOSet.
 // Returns false in second output if this UTXOEntry was not found
-func (dus *DiffUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
+func (dus *DiffUTXOSet) Get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool) {
 	if toRemoveEntry, ok := dus.UTXODiff.toRemove.get(outpoint); ok {
 		// An exception is made for entries with unequal blue scores
 		// These are just "updates" to accepted blue score
@@ -865,42 +641,3 @@ func (dus *DiffUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
 	txOut, ok := dus.UTXODiff.toAdd.get(outpoint)
 	return txOut, ok
 }
-
-// Multiset returns the ecmh-Multiset of this utxoSet
-func (dus *DiffUTXOSet) Multiset() *ecc.Multiset {
-	return dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset)
-}
-
-// WithTransactions returns a new UTXO Set with the added transactions.
-//
-// If dus.UTXODiff.useMultiset is true, this function MUST be
-// called with the DAG lock held.
-func (dus *DiffUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error) {
-	diffSet := NewDiffUTXOSet(dus.base, dus.UTXODiff.clone())
-	for _, tx := range transactions {
-		isAccepted, err := diffSet.AddTx(tx, blockBlueScore)
-		if err != nil {
-			return nil, err
-		}
-		if !ignoreDoubleSpends && !isAccepted {
-			return nil, errors.Errorf("Transaction %s is not valid with the current UTXO set", tx.TxID())
-		}
-	}
-	return UTXOSet(diffSet), nil
-}
-
-func addUTXOToMultiset(ms *ecc.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*ecc.Multiset, error) {
-	utxoMS, err := utxoMultiset(entry, outpoint)
-	if err != nil {
-		return nil, err
-	}
-	return ms.Union(utxoMS), nil
-}
-
-func removeUTXOFromMultiset(ms *ecc.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*ecc.Multiset, error) {
-	utxoMS, err := utxoMultiset(entry, outpoint)
-	if err != nil {
-		return nil, err
-	}
-	return ms.Subtract(utxoMS), nil
-}