[NOD-1238] Fix acceptance index never being initialized. (#859)

* [NOD-1225] Rename wire to domainmessage

* [NOD-1225] Get rid of references to package wire in the code, and get rid of InvType

.dockerignore

@@ -0,0 +1,2 @@
+vendor/
+**/docker-compose.yml

.gitignore

@@ -2,7 +2,7 @@
 *~
 
 # Databases
-btcd.db
+kaspad.db
 *-shm
 *-wal
 
@@ -17,6 +17,7 @@ btcd.db
 # Folders
 _obj
 _test
+vendor
 
 # Architecture specific extensions/prefixes
 *.[568vq]
@@ -31,3 +32,17 @@ _cgo_export.*
 _testmain.go
 
 *.exe
+ 
+# IDE
+.idea
+.vscode
+debug
+debug.test
+__debug_bin
+
+# CI
+version.txt
+coverage.txt
+
+testdbs/
+coverage.tmp

CHANGES

@@ -1,49 +0,0 @@
-============================================================================
-User visible changes for btcd
-  A full-node bitcoin implementation written in Go
-============================================================================
-
-Changes in 0.3.2 (Tue Oct 22 2013)
-  - Fix an issue that could cause the download of the block chain to stall
-    (https://github.com/conformal/btcd/issues/12)
-  - Remove deprecated sqlite as an available database backend
-  - Close sqlite compile issue as sqlite has now been removed
-    (https://github.com/conformal/btcd/issues/11)
-  - Change default RPC ports to 8334 (mainnet) and 18334 (testnet)
-  - Continue cleanup and work on implementing RPC API calls
-  - Add support for the following RPC calls: getrawmempool,
-    getbestblockhash, decoderawtransaction, getdifficulty,
-    getconnectioncount, getpeerinfo, and addnode
-  - Improve the btcctl utility that is used to issue JSON-RPC commands
-  - Fix an issue preventing btcd from cleanly shutting down with the RPC
-    stop command
-  - Add a number of database interface tests to ensure backends implement
-    the expected interface
-  - Expose some additional information from btcscript to be used for
-    identifying "standard"" transactions
-  - Add support for plan9 - thanks @mischief
-    (https://github.com/conformal/btcd/pull/19)
-  - Other minor bug fixes and general code cleanup
-
-Changes in 0.3.1-alpha (Tue Oct 15 2013)
-  - Change default database to leveldb
-    NOTE: This does mean you will have to redownload the block chain.  Since we
-    are still in alpha, we didn't feel writing a converter was worth the time as
-    it would take away from more important issues at this stage
-  - Add a warning if there are multiple block chain databases of different types
-  - Fix issue with unexpected EOF in leveldb -- https://github.com/conformal/btcd/issues/18
-  - Fix issue preventing block 21066 on testnet -- https://github.com/conformal/btcchain/issues/1
-  - Fix issue preventing block 96464 on testnet -- https://github.com/conformal/btcscript/issues/1
-  - Optimize transaction lookups
-  - Correct a few cases of list removal that could result in improper cleanup
-    of no longer needed orphans
-  - Add functionality to increase ulimits on non-Windows platforms
-  - Add support for mempool command which allows remote peers to query the
-    transaction memory pool via the bitcoin protocol
-  - Clean up logging a bit
-  - Add a flag to disable checkpoints for developers
-  - Add a lot of useful debug logging such as message summaries
-  - Other minor bug fixes and general code cleanup
-
-Initial Release 0.3.0-alpha (Sat Oct 05 2013):
-  - Initial release

LICENSE

@@ -1,4 +1,9 @@
-Copyright (c) 2013 Conformal Systems LLC.
+ISC License
+
+Copyright (c) 2018-2019 The kaspanet developers
+Copyright (c) 2013-2018 The btcsuite developers
+Copyright (c) 2015-2016 The Decred developers
+Copyright (c) 2013-2014 Conformal Systems LLC.
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
@@ -10,4 +15,4 @@ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

README.md

@@ -1,115 +1,79 @@
-btcd
+
+Kaspad
+====
+Warning: This is pre-alpha software. There's no guarantee anything works.
 ====
 
-btcd is an alternative full node bitcoin implementation written in Go (golang).
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad)
 
-This project is currently under active development and is in an Alpha state.
+Kaspad is the reference full node Kaspa implementation written in Go (golang).
 
-It currently properly downloads, validates, and serves the block chain using the
-exact rules (including bugs) for block acceptance as the reference
-implementation (bitcoind).  We have taken great care to avoid btcd causing a
-fork to the block chain.  It passes all of the 'official' block acceptance tests
-(https://github.com/TheBlueMatt/test-scripts).
+This project is currently under active development and is in a pre-Alpha state. 
+Some things still don't work and APIs are far from finalized. The code is provided for reference only.
 
-It also properly relays newly mined blocks, maintains a transaction pool,
-and relays individual transactions that have not yet made it into a block.  It
-ensures all individual transactions admitted to the pool follow the rules
-required into the block chain and also includes the vast majority of the more
-strict checks which filter transactions based on miner requirements ("standard"
-transactions).
+## Requirements
 
-One key difference between btcd and bitcoind is that btcd does *NOT* include
-wallet functionality and this was a very intentional design decision. See the
-blog entry [here](https://blog.conformal.com/btcd-not-your-moms-bitcoin-daemon)
-for more details.  This means you can't actually make or receive payments
-directly with btcd.  That functionality will be provided by the forthcoming
-btcwallet and btcgui.
+Latest version of [Go](http://golang.org) (currently 1.13).
 
 ## Installation
 
-#### Windows - MSI Available
-
-https://github.com/conformal/btcd/releases
-
-#### Linux/BSD/POSIX - Build from Source
+#### Build from Source
 
 - Install Go according to the installation instructions here:
   http://golang.org/doc/install
 
-- Run the following command to obtain btcd, all dependencies, and install it:
-  ```$ go get github.com/conformal/btcd```
+- Ensure Go was installed properly and is a supported version:
 
-- btcd will now be installed in either ```$GOROOT/bin``` or ```$GOPATH/bin```
-  depending on your configuration.  If you did not already add to your system
-  path during the installation, we recommend you do so now.
+```bash
+$ go version
+$ go env GOROOT GOPATH
+```
 
-## Updating
+NOTE: The `GOROOT` and `GOPATH` above must not be the same path. It is
+recommended that `GOPATH` is set to a directory in your home directory such as
+`~/dev/go` to avoid write permission issues. It is also recommended to add
+`$GOPATH/bin` to your `PATH` at this point.
 
-#### Windows
+- Run the following commands to obtain and install kaspad including all dependencies:
 
-Install a newer MSI
+```bash
+$ git clone https://github.com/kaspanet/kaspad $GOPATH/src/github.com/kaspanet/kaspad
+$ cd $GOPATH/src/github.com/kaspanet/kaspad
+$ ./test.sh
+$ go install . ./cmd/...
+```
+`./test.sh` tests can be skipped, but some things might not run correctly on your system if tests fail.
 
-#### Linux/BSD/POSIX - Build from Source
+- Kaspad (and utilities) should now be installed in `$GOPATH/bin`. If you did
+  not already add the bin directory to your system path during Go installation,
+  you are encouraged to do so now.
 
-- Run the following command to update btcd, all dependencies, and install it:
-  ```$ go get -u -v github.com/conformal/btcd/...```
 
 ## Getting Started
 
-btcd has several configuration options avilable to tweak how it runs, but all
-of the basic operations described in the intro section work with zero
-configuration.
-
-#### Windows (Installed from MSI)
-
-Launch btcd from your Start menu.
+Kaspad has several configuration options available to tweak how it runs, but all
+of the basic operations work with zero configuration.
 
 #### Linux/BSD/POSIX/Source
 
 ```bash
-$ ./btcd
-````
-## Mailing lists
+$ ./kaspad
+```
 
-- btcd: discussion of btcd and its packages.
-- btcd-commits: readonly mail-out of source code changes.
+## Discord
+Join our discord server using the following link: https://discord.gg/WmGhhzk
 
-To subscribe to a given list, send email to list+subscribe@opensource.conformal.com
+## Issue Tracker
 
-## TODO
+The [integrated github issue tracker](https://github.com/kaspanet/kaspad/issues)
+is used for this project.
 
-The following is a brief overview of the next things we have planned to work on
-for btcd.  Note this does not include the separate btcwallet and btcgui which
-are currently under heavy development:
+## Documentation
 
-- Documentation
-- Code cleanup
-- Add remaining missing RPC calls
-- Add option to allow btcd run as a daemon/service
-- Complete several TODO items in the code
-- Offer 32-bit MSI as well as the 64-bit one
-- Offer cross-compiled binaries for popular OSes (Fedora, Ubuntu, FreeBSD, OpenBSD)
-
-## GPG Verification Key
-
-All official release tags are signed by Conformal so users can ensure the code
-has not been tampered with and is coming from Conformal.  To verify the
-signature perform the following:
-
-- Download the public key from the Conformal website at
-  https://opensource.conformal.com/GIT-GPG-KEY-conformal.txt
-
-- Import the public key into your GPG keyring:
-  ```bash
-  gpg --import GIT-GPG-KEY-conformal.txt
-  ```
-
-- Verify the release tag with the following command where `TAG_NAME` is a
-  placeholder for the specific tag:
-  ```bash
-  git tag -v TAG_NAME
-  ```
+The documentation is a work-in-progress. It is located in the [docs](https://github.com/kaspanet/kaspad/tree/master/docs) folder.
 
 ## License
 
-btcd is licensed under the liberal ISC License.
+Kaspad is licensed under the copyfree [ISC License](https://choosealicense.com/licenses/isc/).
+

addressmanager/addressmanager_test.go

@@ -0,0 +1,635 @@
+// Copyright (c) 2013-2014 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+// naTest is used to describe a test to be performed against the NetAddressKey
+// method.
+type naTest struct {
+	in   domainmessage.NetAddress
+	want AddressKey
+}
+
+// naTests houses all of the tests to be performed against the NetAddressKey
+// method.
+var naTests = make([]naTest, 0)
+
+// Put some IP in here for convenience. Points to google.
+var someIP = "173.194.115.66"
+
+// addNaTests
+func addNaTests() {
+	// IPv4
+	// Localhost
+	addNaTest("127.0.0.1", 16111, "127.0.0.1:16111")
+	addNaTest("127.0.0.1", 16110, "127.0.0.1:16110")
+
+	// Class A
+	addNaTest("1.0.0.1", 16111, "1.0.0.1:16111")
+	addNaTest("2.2.2.2", 16110, "2.2.2.2:16110")
+	addNaTest("27.253.252.251", 8335, "27.253.252.251:8335")
+	addNaTest("123.3.2.1", 8336, "123.3.2.1:8336")
+
+	// Private Class A
+	addNaTest("10.0.0.1", 16111, "10.0.0.1:16111")
+	addNaTest("10.1.1.1", 16110, "10.1.1.1:16110")
+	addNaTest("10.2.2.2", 8335, "10.2.2.2:8335")
+	addNaTest("10.10.10.10", 8336, "10.10.10.10:8336")
+
+	// Class B
+	addNaTest("128.0.0.1", 16111, "128.0.0.1:16111")
+	addNaTest("129.1.1.1", 16110, "129.1.1.1:16110")
+	addNaTest("180.2.2.2", 8335, "180.2.2.2:8335")
+	addNaTest("191.10.10.10", 8336, "191.10.10.10:8336")
+
+	// Private Class B
+	addNaTest("172.16.0.1", 16111, "172.16.0.1:16111")
+	addNaTest("172.16.1.1", 16110, "172.16.1.1:16110")
+	addNaTest("172.16.2.2", 8335, "172.16.2.2:8335")
+	addNaTest("172.16.172.172", 8336, "172.16.172.172:8336")
+
+	// Class C
+	addNaTest("193.0.0.1", 16111, "193.0.0.1:16111")
+	addNaTest("200.1.1.1", 16110, "200.1.1.1:16110")
+	addNaTest("205.2.2.2", 8335, "205.2.2.2:8335")
+	addNaTest("223.10.10.10", 8336, "223.10.10.10:8336")
+
+	// Private Class C
+	addNaTest("192.168.0.1", 16111, "192.168.0.1:16111")
+	addNaTest("192.168.1.1", 16110, "192.168.1.1:16110")
+	addNaTest("192.168.2.2", 8335, "192.168.2.2:8335")
+	addNaTest("192.168.192.192", 8336, "192.168.192.192:8336")
+
+	// IPv6
+	// Localhost
+	addNaTest("::1", 16111, "[::1]:16111")
+	addNaTest("fe80::1", 16110, "[fe80::1]:16110")
+
+	// Link-local
+	addNaTest("fe80::1:1", 16111, "[fe80::1:1]:16111")
+	addNaTest("fe91::2:2", 16110, "[fe91::2:2]:16110")
+	addNaTest("fea2::3:3", 8335, "[fea2::3:3]:8335")
+	addNaTest("feb3::4:4", 8336, "[feb3::4:4]:8336")
+
+	// Site-local
+	addNaTest("fec0::1:1", 16111, "[fec0::1:1]:16111")
+	addNaTest("fed1::2:2", 16110, "[fed1::2:2]:16110")
+	addNaTest("fee2::3:3", 8335, "[fee2::3:3]:8335")
+	addNaTest("fef3::4:4", 8336, "[fef3::4:4]:8336")
+}
+
+func addNaTest(ip string, port uint16, want AddressKey) {
+	nip := net.ParseIP(ip)
+	na := *domainmessage.NewNetAddressIPPort(nip, port, domainmessage.SFNodeNetwork)
+	test := naTest{na, want}
+	naTests = append(naTests, test)
+}
+
+func lookupFuncForTest(host string) ([]net.IP, error) {
+	return nil, errors.New("not implemented")
+}
+
+func newAddrManagerForTest(t *testing.T, testName string,
+	localSubnetworkID *subnetworkid.SubnetworkID) (addressManager *AddressManager, teardown func()) {
+
+	cfg := config.DefaultConfig()
+	cfg.SubnetworkID = localSubnetworkID
+
+	dbPath, err := ioutil.TempDir("", testName)
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+
+	databaseContext, err := dbaccess.New(dbPath)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	addressManager = New(cfg, databaseContext)
+
+	return addressManager, func() {
+		err := databaseContext.Close()
+		if err != nil {
+			t.Fatalf("error closing the database: %s", err)
+		}
+	}
+}
+
+func TestStartStop(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestStartStop", nil)
+	defer teardown()
+	err := amgr.Start()
+	if err != nil {
+		t.Fatalf("Address Manager failed to start: %v", err)
+	}
+	err = amgr.Stop()
+	if err != nil {
+		t.Fatalf("Address Manager failed to stop: %v", err)
+	}
+}
+
+func TestAddAddressByIP(t *testing.T) {
+	fmtErr := errors.Errorf("")
+	addrErr := &net.AddrError{}
+	var tests = []struct {
+		addrIP string
+		err    error
+	}{
+		{
+			someIP + ":16111",
+			nil,
+		},
+		{
+			someIP,
+			addrErr,
+		},
+		{
+			someIP[:12] + ":8333",
+			fmtErr,
+		},
+		{
+			someIP + ":abcd",
+			fmtErr,
+		},
+	}
+
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
+	for i, test := range tests {
+		err := amgr.AddAddressByIP(test.addrIP, nil)
+		if test.err != nil && err == nil {
+			t.Errorf("TestAddAddressByIP test %d failed expected an error and got none", i)
+			continue
+		}
+		if test.err == nil && err != nil {
+			t.Errorf("TestAddAddressByIP test %d failed expected no error and got one", i)
+			continue
+		}
+		if reflect.TypeOf(err) != reflect.TypeOf(test.err) {
+			t.Errorf("TestAddAddressByIP test %d failed got %v, want %v", i,
+				reflect.TypeOf(err), reflect.TypeOf(test.err))
+			continue
+		}
+	}
+}
+
+func TestAddLocalAddress(t *testing.T) {
+	var tests = []struct {
+		address  domainmessage.NetAddress
+		priority AddressPriority
+		valid    bool
+	}{
+		{
+			domainmessage.NetAddress{IP: net.ParseIP("192.168.0.100")},
+			InterfacePrio,
+			false,
+		},
+		{
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.1.1")},
+			InterfacePrio,
+			true,
+		},
+		{
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.1.1")},
+			BoundPrio,
+			true,
+		},
+		{
+			domainmessage.NetAddress{IP: net.ParseIP("::1")},
+			InterfacePrio,
+			false,
+		},
+		{
+			domainmessage.NetAddress{IP: net.ParseIP("fe80::1")},
+			InterfacePrio,
+			false,
+		},
+		{
+			domainmessage.NetAddress{IP: net.ParseIP("2620:100::1")},
+			InterfacePrio,
+			true,
+		},
+	}
+	amgr, teardown := newAddrManagerForTest(t, "TestAddLocalAddress", nil)
+	defer teardown()
+	for x, test := range tests {
+		result := amgr.AddLocalAddress(&test.address, test.priority)
+		if result == nil && !test.valid {
+			t.Errorf("TestAddLocalAddress test #%d failed: %s should have "+
+				"been accepted", x, test.address.IP)
+			continue
+		}
+		if result != nil && test.valid {
+			t.Errorf("TestAddLocalAddress test #%d failed: %s should not have "+
+				"been accepted", x, test.address.IP)
+			continue
+		}
+	}
+}
+
+func TestAttempt(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestAttempt", nil)
+	defer teardown()
+
+	// Add a new address and get it
+	err := amgr.AddAddressByIP(someIP+":8333", nil)
+	if err != nil {
+		t.Fatalf("Adding address failed: %v", err)
+	}
+	ka := amgr.GetAddress()
+
+	if !ka.LastAttempt().IsZero() {
+		t.Errorf("Address should not have attempts, but does")
+	}
+
+	na := ka.NetAddress()
+	amgr.Attempt(na)
+
+	if ka.LastAttempt().IsZero() {
+		t.Errorf("Address should have an attempt, but does not")
+	}
+}
+
+func TestConnected(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestConnected", nil)
+	defer teardown()
+
+	// Add a new address and get it
+	err := amgr.AddAddressByIP(someIP+":8333", nil)
+	if err != nil {
+		t.Fatalf("Adding address failed: %v", err)
+	}
+	ka := amgr.GetAddress()
+	na := ka.NetAddress()
+	// make it an hour ago
+	na.Timestamp = mstime.Now().Add(time.Hour * -1)
+
+	amgr.Connected(na)
+
+	if !ka.NetAddress().Timestamp.After(na.Timestamp) {
+		t.Errorf("Address should have a new timestamp, but does not")
+	}
+}
+
+func TestNeedMoreAddresses(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestNeedMoreAddresses", nil)
+	defer teardown()
+	addrsToAdd := 1500
+	b := amgr.NeedMoreAddresses()
+	if !b {
+		t.Errorf("Expected that we need more addresses")
+	}
+	addrs := make([]*domainmessage.NetAddress, addrsToAdd)
+
+	var err error
+	for i := 0; i < addrsToAdd; i++ {
+		s := AddressKey(fmt.Sprintf("%d.%d.173.147:8333", i/128+60, i%128+60))
+		addrs[i], err = amgr.DeserializeNetAddress(s)
+		if err != nil {
+			t.Errorf("Failed to turn %s into an address: %v", s, err)
+		}
+	}
+
+	srcAddr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+
+	amgr.AddAddresses(addrs, srcAddr, nil)
+	numAddrs := amgr.TotalNumAddresses()
+	if numAddrs > addrsToAdd {
+		t.Errorf("Number of addresses is too many %d vs %d", numAddrs, addrsToAdd)
+	}
+
+	b = amgr.NeedMoreAddresses()
+	if b {
+		t.Errorf("Expected that we don't need more addresses")
+	}
+}
+
+func TestGood(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestGood", nil)
+	defer teardown()
+	addrsToAdd := 64 * 64
+	addrs := make([]*domainmessage.NetAddress, addrsToAdd)
+	subnetworkCount := 32
+	subnetworkIDs := make([]*subnetworkid.SubnetworkID, subnetworkCount)
+
+	var err error
+	for i := 0; i < addrsToAdd; i++ {
+		s := AddressKey(fmt.Sprintf("%d.173.147.%d:8333", i/64+60, i%64+60))
+		addrs[i], err = amgr.DeserializeNetAddress(s)
+		if err != nil {
+			t.Errorf("Failed to turn %s into an address: %v", s, err)
+		}
+	}
+
+	for i := 0; i < subnetworkCount; i++ {
+		subnetworkIDs[i] = &subnetworkid.SubnetworkID{0xff - byte(i)}
+	}
+
+	srcAddr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+
+	amgr.AddAddresses(addrs, srcAddr, nil)
+	for i, addr := range addrs {
+		amgr.Good(addr, subnetworkIDs[i%subnetworkCount])
+	}
+
+	numAddrs := amgr.TotalNumAddresses()
+	if numAddrs >= addrsToAdd {
+		t.Errorf("Number of addresses is too many: %d vs %d", numAddrs, addrsToAdd)
+	}
+
+	numCache := len(amgr.AddressCache(true, nil))
+	if numCache == 0 || numCache >= numAddrs/4 {
+		t.Errorf("Number of addresses in cache: got %d, want positive and less than %d",
+			numCache, numAddrs/4)
+	}
+
+	for i := 0; i < subnetworkCount; i++ {
+		numCache = len(amgr.AddressCache(false, subnetworkIDs[i]))
+		if numCache == 0 || numCache >= numAddrs/subnetworkCount {
+			t.Errorf("Number of addresses in subnetwork cache: got %d, want positive and less than %d",
+				numCache, numAddrs/4/subnetworkCount)
+		}
+	}
+}
+
+func TestGoodChangeSubnetworkID(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestGoodChangeSubnetworkID", nil)
+	defer teardown()
+	addr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+	addrKey := NetAddressKey(addr)
+	srcAddr := domainmessage.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
+
+	oldSubnetwork := subnetworkid.SubnetworkIDNative
+	amgr.AddAddress(addr, srcAddr, oldSubnetwork)
+	amgr.Good(addr, oldSubnetwork)
+
+	// make sure address was saved to addressIndex under oldSubnetwork
+	ka := amgr.knownAddress(addr)
+	if ka == nil {
+		t.Fatalf("Address was not found after first time .Good called")
+	}
+	if !ka.SubnetworkID().IsEqual(oldSubnetwork) {
+		t.Fatalf("Address index did not point to oldSubnetwork")
+	}
+
+	// make sure address was added to correct bucket under oldSubnetwork
+	bucket := amgr.subnetworkTriedAddresBucketArrays[*oldSubnetwork][amgr.triedAddressBucketIndex(addr)]
+	wasFound := false
+	for _, ka := range bucket {
+		if NetAddressKey(ka.NetAddress()) == addrKey {
+			wasFound = true
+		}
+	}
+	if !wasFound {
+		t.Fatalf("Address was not found in the correct bucket in oldSubnetwork")
+	}
+
+	// now call .Good again with a different subnetwork
+	newSubnetwork := subnetworkid.SubnetworkIDRegistry
+	amgr.Good(addr, newSubnetwork)
+
+	// make sure address was updated in addressIndex under newSubnetwork
+	ka = amgr.knownAddress(addr)
+	if ka == nil {
+		t.Fatalf("Address was not found after second time .Good called")
+	}
+	if !ka.SubnetworkID().IsEqual(newSubnetwork) {
+		t.Fatalf("Address index did not point to newSubnetwork")
+	}
+
+	// make sure address was removed from bucket under oldSubnetwork
+	bucket = amgr.subnetworkTriedAddresBucketArrays[*oldSubnetwork][amgr.triedAddressBucketIndex(addr)]
+	wasFound = false
+	for _, ka := range bucket {
+		if NetAddressKey(ka.NetAddress()) == addrKey {
+			wasFound = true
+		}
+	}
+	if wasFound {
+		t.Fatalf("Address was not removed from bucket in oldSubnetwork")
+	}
+
+	// make sure address was added to correct bucket under newSubnetwork
+	bucket = amgr.subnetworkTriedAddresBucketArrays[*newSubnetwork][amgr.triedAddressBucketIndex(addr)]
+	wasFound = false
+	for _, ka := range bucket {
+		if NetAddressKey(ka.NetAddress()) == addrKey {
+			wasFound = true
+		}
+	}
+	if !wasFound {
+		t.Fatalf("Address was not found in the correct bucket in newSubnetwork")
+	}
+}
+
+func TestGetAddress(t *testing.T) {
+	localSubnetworkID := &subnetworkid.SubnetworkID{0xff}
+	amgr, teardown := newAddrManagerForTest(t, "TestGetAddress", localSubnetworkID)
+	defer teardown()
+
+	// Get an address from an empty set (should error)
+	if rv := amgr.GetAddress(); rv != nil {
+		t.Errorf("GetAddress failed: got: %v want: %v\n", rv, nil)
+	}
+
+	// Add a new address and get it
+	err := amgr.AddAddressByIP(someIP+":8332", localSubnetworkID)
+	if err != nil {
+		t.Fatalf("Adding address failed: %v", err)
+	}
+	ka := amgr.GetAddress()
+	if ka == nil {
+		t.Fatalf("Did not get an address where there is one in the pool")
+	}
+	amgr.Attempt(ka.NetAddress())
+
+	// Checks that we don't get it if we find that it has other subnetwork ID than expected.
+	actualSubnetworkID := &subnetworkid.SubnetworkID{0xfe}
+	amgr.Good(ka.NetAddress(), actualSubnetworkID)
+	ka = amgr.GetAddress()
+	if ka != nil {
+		t.Errorf("Didn't expect to get an address because there shouldn't be any address from subnetwork ID %s or nil", localSubnetworkID)
+	}
+
+	// Checks that the total number of addresses incremented although the new address is not full node or a partial node of the same subnetwork as the local node.
+	numAddrs := amgr.TotalNumAddresses()
+	if numAddrs != 1 {
+		t.Errorf("Wrong number of addresses: got %d, want %d", numAddrs, 1)
+	}
+
+	// Now we repeat the same process, but now the address has the expected subnetwork ID.
+
+	// Add a new address and get it
+	err = amgr.AddAddressByIP(someIP+":8333", localSubnetworkID)
+	if err != nil {
+		t.Fatalf("Adding address failed: %v", err)
+	}
+	ka = amgr.GetAddress()
+	if ka == nil {
+		t.Fatalf("Did not get an address where there is one in the pool")
+	}
+	if ka.NetAddress().IP.String() != someIP {
+		t.Errorf("Wrong IP: got %v, want %v", ka.NetAddress().IP.String(), someIP)
+	}
+	if !ka.SubnetworkID().IsEqual(localSubnetworkID) {
+		t.Errorf("Wrong Subnetwork ID: got %v, want %v", *ka.SubnetworkID(), localSubnetworkID)
+	}
+	amgr.Attempt(ka.NetAddress())
+
+	// Mark this as a good address and get it
+	amgr.Good(ka.NetAddress(), localSubnetworkID)
+	ka = amgr.GetAddress()
+	if ka == nil {
+		t.Fatalf("Did not get an address where there is one in the pool")
+	}
+	if ka.NetAddress().IP.String() != someIP {
+		t.Errorf("Wrong IP: got %v, want %v", ka.NetAddress().IP.String(), someIP)
+	}
+	if *ka.SubnetworkID() != *localSubnetworkID {
+		t.Errorf("Wrong Subnetwork ID: got %v, want %v", ka.SubnetworkID(), localSubnetworkID)
+	}
+
+	numAddrs = amgr.TotalNumAddresses()
+	if numAddrs != 2 {
+		t.Errorf("Wrong number of addresses: got %d, want %d", numAddrs, 1)
+	}
+}
+
+func TestGetBestLocalAddress(t *testing.T) {
+	localAddrs := []domainmessage.NetAddress{
+		{IP: net.ParseIP("192.168.0.100")},
+		{IP: net.ParseIP("::1")},
+		{IP: net.ParseIP("fe80::1")},
+		{IP: net.ParseIP("2001:470::1")},
+	}
+
+	var tests = []struct {
+		remoteAddr domainmessage.NetAddress
+		want0      domainmessage.NetAddress
+		want1      domainmessage.NetAddress
+		want2      domainmessage.NetAddress
+		want3      domainmessage.NetAddress
+	}{
+		{
+			// Remote connection from public IPv4
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.8.1")},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.8.100")},
+			domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")},
+		},
+		{
+			// Remote connection from private IPv4
+			domainmessage.NetAddress{IP: net.ParseIP("172.16.0.254")},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+		},
+		{
+			// Remote connection from public IPv6
+			domainmessage.NetAddress{IP: net.ParseIP("2602:100:abcd::102")},
+			domainmessage.NetAddress{IP: net.IPv6zero},
+			domainmessage.NetAddress{IP: net.ParseIP("2001:470::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("2001:470::1")},
+			domainmessage.NetAddress{IP: net.ParseIP("2001:470::1")},
+		},
+		/* XXX
+		{
+			// Remote connection from Tor
+			domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43::100")},
+			domainmessage.NetAddress{IP: net.IPv4zero},
+			domainmessage.NetAddress{IP: net.ParseIP("204.124.8.100")},
+			domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")},
+		},
+		*/
+	}
+
+	amgr, teardown := newAddrManagerForTest(t, "TestGetBestLocalAddress", nil)
+	defer teardown()
+
+	// Test against default when there's no address
+	for x, test := range tests {
+		got := amgr.GetBestLocalAddress(&test.remoteAddr)
+		if !test.want0.IP.Equal(got.IP) {
+			t.Errorf("TestGetBestLocalAddress test1 #%d failed for remote address %s: want %s got %s",
+				x, test.remoteAddr.IP, test.want1.IP, got.IP)
+			continue
+		}
+	}
+
+	for _, localAddr := range localAddrs {
+		amgr.AddLocalAddress(&localAddr, InterfacePrio)
+	}
+
+	// Test against want1
+	for x, test := range tests {
+		got := amgr.GetBestLocalAddress(&test.remoteAddr)
+		if !test.want1.IP.Equal(got.IP) {
+			t.Errorf("TestGetBestLocalAddress test1 #%d failed for remote address %s: want %s got %s",
+				x, test.remoteAddr.IP, test.want1.IP, got.IP)
+			continue
+		}
+	}
+
+	// Add a public IP to the list of local addresses.
+	localAddr := domainmessage.NetAddress{IP: net.ParseIP("204.124.8.100")}
+	amgr.AddLocalAddress(&localAddr, InterfacePrio)
+
+	// Test against want2
+	for x, test := range tests {
+		got := amgr.GetBestLocalAddress(&test.remoteAddr)
+		if !test.want2.IP.Equal(got.IP) {
+			t.Errorf("TestGetBestLocalAddress test2 #%d failed for remote address %s: want %s got %s",
+				x, test.remoteAddr.IP, test.want2.IP, got.IP)
+			continue
+		}
+	}
+	/*
+		// Add a Tor generated IP address
+		localAddr = domainmessage.NetAddress{IP: net.ParseIP("fd87:d87e:eb43:25::1")}
+		amgr.AddLocalAddress(&localAddr, ManualPrio)
+		// Test against want3
+		for x, test := range tests {
+			got := amgr.GetBestLocalAddress(&test.remoteAddr)
+			if !test.want3.IP.Equal(got.IP) {
+				t.Errorf("TestGetBestLocalAddress test3 #%d failed for remote address %s: want %s got %s",
+					x, test.remoteAddr.IP, test.want3.IP, got.IP)
+				continue
+			}
+		}
+	*/
+}
+
+func TestNetAddressKey(t *testing.T) {
+	addNaTests()
+
+	t.Logf("Running %d tests", len(naTests))
+	for i, test := range naTests {
+		key := NetAddressKey(&test.in)
+		if key != test.want {
+			t.Errorf("NetAddressKey #%d\n got: %s want: %s", i, key, test.want)
+			continue
+		}
+	}
+
+}

addressmanager/doc.go

@@ -0,0 +1,34 @@
+/*
+Package addressmanager implements concurrency safe Kaspa address manager.
+
+Address Manager Overview
+
+In order maintain the peer-to-peer Kaspa network, there needs to be a source
+of addresses to connect to as nodes come and go. The Kaspa protocol provides
+the getaddr and addr messages to allow peers to communicate known addresses with
+each other. However, there needs to a mechanism to store those results and
+select peers from them. It is also important to note that remote peers can't
+be trusted to send valid peers nor attempt to provide you with only peers they
+control with malicious intent.
+
+With that in mind, this package provides a concurrency safe address manager for
+caching and selecting peers in a non-deterministic manner. The general idea is
+the caller adds addresses to the address manager and notifies it when addresses
+are connected, known good, and attempted. The caller also requests addresses as
+it needs them.
+
+The address manager internally segregates the addresses into groups and
+non-deterministically selects groups in a cryptographically random manner. This
+reduce the chances multiple addresses from the same nets are selected which
+generally helps provide greater peer diversity, and perhaps more importantly,
+drastically reduces the chances an attacker is able to coerce your peer into
+only connecting to nodes they control.
+
+The address manager also understands routability and tries hard to only return
+routable addresses. In addition, it uses the information provided by the caller
+about connected, known good, and attempted addresses to periodically purge
+peers which no longer appear to be good peers as well as bias the selection
+toward known good peers. The general idea is to make a best effort at only
+providing usable addresses.
+*/
+package addressmanager

addressmanager/internal_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2013-2015 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/mstime"
+)
+
+func TstKnownAddressIsBad(ka *KnownAddress) bool {
+	return ka.isBad()
+}
+
+func TstKnownAddressChance(ka *KnownAddress) float64 {
+	return ka.chance()
+}
+
+func TstNewKnownAddress(na *domainmessage.NetAddress, attempts int,
+	lastattempt, lastsuccess mstime.Time, tried bool, refs int) *KnownAddress {
+	return &KnownAddress{netAddress: na, attempts: attempts, lastAttempt: lastattempt,
+		lastSuccess: lastsuccess, tried: tried, referenceCount: refs}
+}

addressmanager/knownaddress.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2013-2014 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+	"time"
+
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+// KnownAddress tracks information about a known network address that is used
+// to determine how viable an address is.
+type KnownAddress struct {
+	netAddress     *domainmessage.NetAddress
+	sourceAddress  *domainmessage.NetAddress
+	attempts       int
+	lastAttempt    mstime.Time
+	lastSuccess    mstime.Time
+	tried          bool
+	referenceCount int // reference count of new buckets
+	subnetworkID   *subnetworkid.SubnetworkID
+	isBanned       bool
+	bannedTime     mstime.Time
+}
+
+// NetAddress returns the underlying domainmessage.NetAddress associated with the
+// known address.
+func (ka *KnownAddress) NetAddress() *domainmessage.NetAddress {
+	return ka.netAddress
+}
+
+// SubnetworkID returns the subnetwork ID of the known address.
+func (ka *KnownAddress) SubnetworkID() *subnetworkid.SubnetworkID {
+	return ka.subnetworkID
+}
+
+// LastAttempt returns the last time the known address was attempted.
+func (ka *KnownAddress) LastAttempt() mstime.Time {
+	return ka.lastAttempt
+}
+
+// chance returns the selection probability for a known address. The priority
+// depends upon how recently the address has been seen, how recently it was last
+// attempted and how often attempts to connect to it have failed.
+func (ka *KnownAddress) chance() float64 {
+	now := mstime.Now()
+	lastAttempt := now.Sub(ka.lastAttempt)
+
+	if lastAttempt < 0 {
+		lastAttempt = 0
+	}
+
+	c := 1.0
+
+	// Very recent attempts are less likely to be retried.
+	if lastAttempt < 10*time.Minute {
+		c *= 0.01
+	}
+
+	// Failed attempts deprioritise.
+	for i := ka.attempts; i > 0; i-- {
+		c /= 1.5
+	}
+
+	return c
+}
+
+// isBad returns true if the address in question has not been tried in the last
+// minute and meets one of the following criteria:
+// 1) It claims to be from the future
+// 2) It hasn't been seen in over a month
+// 3) It has failed at least three times and never succeeded
+// 4) It has failed ten times in the last week
+// All addresses that meet these criteria are assumed to be worthless and not
+// worth keeping hold of.
+func (ka *KnownAddress) isBad() bool {
+	if ka.lastAttempt.After(mstime.Now().Add(-1 * time.Minute)) {
+		return false
+	}
+
+	// From the future?
+	if ka.netAddress.Timestamp.After(mstime.Now().Add(10 * time.Minute)) {
+		return true
+	}
+
+	// Over a month old?
+	if ka.netAddress.Timestamp.Before(mstime.Now().Add(-1 * numMissingDays * time.Hour * 24)) {
+		return true
+	}
+
+	// Never succeeded?
+	if ka.lastSuccess.IsZero() && ka.attempts >= numRetries {
+		return true
+	}
+
+	// Hasn't succeeded in too long?
+	if !ka.lastSuccess.After(mstime.Now().Add(-1*minBadDays*time.Hour*24)) &&
+		ka.attempts >= maxFailures {
+		return true
+	}
+
+	return false
+}

addressmanager/knownaddress_test.go

@@ -0,0 +1,115 @@
+// Copyright (c) 2013-2015 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager_test
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/kaspanet/kaspad/addressmanager"
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+func TestChance(t *testing.T) {
+	now := mstime.Now()
+	var tests = []struct {
+		addr     *addressmanager.KnownAddress
+		expected float64
+	}{
+		{
+			//Test normal case
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				0, mstime.Now().Add(-30*time.Minute), mstime.Now(), false, 0),
+			1.0,
+		}, {
+			//Test case in which lastseen < 0
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(20 * time.Second)},
+				0, mstime.Now().Add(-30*time.Minute), mstime.Now(), false, 0),
+			1.0,
+		}, {
+			//Test case in which lastAttempt < 0
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				0, mstime.Now().Add(30*time.Minute), mstime.Now(), false, 0),
+			1.0 * .01,
+		}, {
+			//Test case in which lastAttempt < ten minutes
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				0, mstime.Now().Add(-5*time.Minute), mstime.Now(), false, 0),
+			1.0 * .01,
+		}, {
+			//Test case with several failed attempts.
+			addressmanager.TstNewKnownAddress(&domainmessage.NetAddress{Timestamp: now.Add(-35 * time.Second)},
+				2, mstime.Now().Add(-30*time.Minute), mstime.Now(), false, 0),
+			1 / 1.5 / 1.5,
+		},
+	}
+
+	err := .0001
+	for i, test := range tests {
+		chance := addressmanager.TstKnownAddressChance(test.addr)
+		if math.Abs(test.expected-chance) >= err {
+			t.Errorf("case %d: got %f, expected %f", i, chance, test.expected)
+		}
+	}
+}
+
+func TestIsBad(t *testing.T) {
+	now := mstime.Now()
+	future := now.Add(35 * time.Minute)
+	monthOld := now.Add(-43 * time.Hour * 24)
+	secondsOld := now.Add(-2 * time.Second)
+	minutesOld := now.Add(-27 * time.Minute)
+	hoursOld := now.Add(-5 * time.Hour)
+	zeroTime := mstime.Time{}
+
+	futureNa := &domainmessage.NetAddress{Timestamp: future}
+	minutesOldNa := &domainmessage.NetAddress{Timestamp: minutesOld}
+	monthOldNa := &domainmessage.NetAddress{Timestamp: monthOld}
+	currentNa := &domainmessage.NetAddress{Timestamp: secondsOld}
+
+	//Test addresses that have been tried in the last minute.
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(futureNa, 3, secondsOld, zeroTime, false, 0)) {
+		t.Errorf("test case 1: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(monthOldNa, 3, secondsOld, zeroTime, false, 0)) {
+		t.Errorf("test case 2: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(currentNa, 3, secondsOld, zeroTime, false, 0)) {
+		t.Errorf("test case 3: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(currentNa, 3, secondsOld, monthOld, true, 0)) {
+		t.Errorf("test case 4: addresses that have been tried in the last minute are not bad.")
+	}
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(currentNa, 2, secondsOld, secondsOld, true, 0)) {
+		t.Errorf("test case 5: addresses that have been tried in the last minute are not bad.")
+	}
+
+	//Test address that claims to be from the future.
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(futureNa, 0, minutesOld, hoursOld, true, 0)) {
+		t.Errorf("test case 6: addresses that claim to be from the future are bad.")
+	}
+
+	//Test address that has not been seen in over a month.
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(monthOldNa, 0, minutesOld, hoursOld, true, 0)) {
+		t.Errorf("test case 7: addresses more than a month old are bad.")
+	}
+
+	//It has failed at least three times and never succeeded.
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(minutesOldNa, 3, minutesOld, zeroTime, true, 0)) {
+		t.Errorf("test case 8: addresses that have never succeeded are bad.")
+	}
+
+	//It has failed ten times in the last week
+	if !addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(minutesOldNa, 10, minutesOld, monthOld, true, 0)) {
+		t.Errorf("test case 9: addresses that have not succeeded in too long are bad.")
+	}
+
+	//Test an address that should work.
+	if addressmanager.TstKnownAddressIsBad(addressmanager.TstNewKnownAddress(minutesOldNa, 2, minutesOld, hoursOld, true, 0)) {
+		t.Errorf("test case 10: This should be a valid address.")
+	}
+}

addressmanager/log.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2013-2014 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"github.com/kaspanet/kaspad/logger"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+var log, _ = logger.Get(logger.SubsystemTags.ADXR)
+var spawn = panics.GoroutineWrapperFunc(log)

addressmanager/network.go

@@ -0,0 +1,258 @@
+// Copyright (c) 2013-2014 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"net"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+var (
+	// rfc1918Nets specifies the IPv4 private address blocks as defined by
+	// by RFC1918 (10.0.0.0/8, 172.16.0.0/12, and 192.168.0.0/16).
+	rfc1918Nets = []net.IPNet{
+		ipNet("10.0.0.0", 8, 32),
+		ipNet("172.16.0.0", 12, 32),
+		ipNet("192.168.0.0", 16, 32),
+	}
+
+	// rfc2544Net specifies the the IPv4 block as defined by RFC2544
+	// (198.18.0.0/15)
+	rfc2544Net = ipNet("198.18.0.0", 15, 32)
+
+	// rfc3849Net specifies the IPv6 documentation address block as defined
+	// by RFC3849 (2001:DB8::/32).
+	rfc3849Net = ipNet("2001:DB8::", 32, 128)
+
+	// rfc3927Net specifies the IPv4 auto configuration address block as
+	// defined by RFC3927 (169.254.0.0/16).
+	rfc3927Net = ipNet("169.254.0.0", 16, 32)
+
+	// rfc3964Net specifies the IPv6 to IPv4 encapsulation address block as
+	// defined by RFC3964 (2002::/16).
+	rfc3964Net = ipNet("2002::", 16, 128)
+
+	// rfc4193Net specifies the IPv6 unique local address block as defined
+	// by RFC4193 (FC00::/7).
+	rfc4193Net = ipNet("FC00::", 7, 128)
+
+	// rfc4380Net specifies the IPv6 teredo tunneling over UDP address block
+	// as defined by RFC4380 (2001::/32).
+	rfc4380Net = ipNet("2001::", 32, 128)
+
+	// rfc4843Net specifies the IPv6 ORCHID address block as defined by
+	// RFC4843 (2001:10::/28).
+	rfc4843Net = ipNet("2001:10::", 28, 128)
+
+	// rfc4862Net specifies the IPv6 stateless address autoconfiguration
+	// address block as defined by RFC4862 (FE80::/64).
+	rfc4862Net = ipNet("FE80::", 64, 128)
+
+	// rfc5737Net specifies the IPv4 documentation address blocks as defined
+	// by RFC5737 (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24)
+	rfc5737Net = []net.IPNet{
+		ipNet("192.0.2.0", 24, 32),
+		ipNet("198.51.100.0", 24, 32),
+		ipNet("203.0.113.0", 24, 32),
+	}
+
+	// rfc6052Net specifies the IPv6 well-known prefix address block as
+	// defined by RFC6052 (64:FF9B::/96).
+	rfc6052Net = ipNet("64:FF9B::", 96, 128)
+
+	// rfc6145Net specifies the IPv6 to IPv4 translated address range as
+	// defined by RFC6145 (::FFFF:0:0:0/96).
+	rfc6145Net = ipNet("::FFFF:0:0:0", 96, 128)
+
+	// rfc6598Net specifies the IPv4 block as defined by RFC6598 (100.64.0.0/10)
+	rfc6598Net = ipNet("100.64.0.0", 10, 32)
+
+	// zero4Net defines the IPv4 address block for address staring with 0
+	// (0.0.0.0/8).
+	zero4Net = ipNet("0.0.0.0", 8, 32)
+
+	// heNet defines the Hurricane Electric IPv6 address block.
+	heNet = ipNet("2001:470::", 32, 128)
+)
+
+// ipNet returns a net.IPNet struct given the passed IP address string, number
+// of one bits to include at the start of the mask, and the total number of bits
+// for the mask.
+func ipNet(ip string, ones, bits int) net.IPNet {
+	return net.IPNet{IP: net.ParseIP(ip), Mask: net.CIDRMask(ones, bits)}
+}
+
+// IsIPv4 returns whether or not the given address is an IPv4 address.
+func IsIPv4(na *domainmessage.NetAddress) bool {
+	return na.IP.To4() != nil
+}
+
+// IsLocal returns whether or not the given address is a local address.
+func IsLocal(na *domainmessage.NetAddress) bool {
+	return na.IP.IsLoopback() || zero4Net.Contains(na.IP)
+}
+
+// IsRFC1918 returns whether or not the passed address is part of the IPv4
+// private network address space as defined by RFC1918 (10.0.0.0/8,
+// 172.16.0.0/12, or 192.168.0.0/16).
+func IsRFC1918(na *domainmessage.NetAddress) bool {
+	for _, rfc := range rfc1918Nets {
+		if rfc.Contains(na.IP) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsRFC2544 returns whether or not the passed address is part of the IPv4
+// address space as defined by RFC2544 (198.18.0.0/15)
+func IsRFC2544(na *domainmessage.NetAddress) bool {
+	return rfc2544Net.Contains(na.IP)
+}
+
+// IsRFC3849 returns whether or not the passed address is part of the IPv6
+// documentation range as defined by RFC3849 (2001:DB8::/32).
+func IsRFC3849(na *domainmessage.NetAddress) bool {
+	return rfc3849Net.Contains(na.IP)
+}
+
+// IsRFC3927 returns whether or not the passed address is part of the IPv4
+// autoconfiguration range as defined by RFC3927 (169.254.0.0/16).
+func IsRFC3927(na *domainmessage.NetAddress) bool {
+	return rfc3927Net.Contains(na.IP)
+}
+
+// IsRFC3964 returns whether or not the passed address is part of the IPv6 to
+// IPv4 encapsulation range as defined by RFC3964 (2002::/16).
+func IsRFC3964(na *domainmessage.NetAddress) bool {
+	return rfc3964Net.Contains(na.IP)
+}
+
+// IsRFC4193 returns whether or not the passed address is part of the IPv6
+// unique local range as defined by RFC4193 (FC00::/7).
+func IsRFC4193(na *domainmessage.NetAddress) bool {
+	return rfc4193Net.Contains(na.IP)
+}
+
+// IsRFC4380 returns whether or not the passed address is part of the IPv6
+// teredo tunneling over UDP range as defined by RFC4380 (2001::/32).
+func IsRFC4380(na *domainmessage.NetAddress) bool {
+	return rfc4380Net.Contains(na.IP)
+}
+
+// IsRFC4843 returns whether or not the passed address is part of the IPv6
+// ORCHID range as defined by RFC4843 (2001:10::/28).
+func IsRFC4843(na *domainmessage.NetAddress) bool {
+	return rfc4843Net.Contains(na.IP)
+}
+
+// IsRFC4862 returns whether or not the passed address is part of the IPv6
+// stateless address autoconfiguration range as defined by RFC4862 (FE80::/64).
+func IsRFC4862(na *domainmessage.NetAddress) bool {
+	return rfc4862Net.Contains(na.IP)
+}
+
+// IsRFC5737 returns whether or not the passed address is part of the IPv4
+// documentation address space as defined by RFC5737 (192.0.2.0/24,
+// 198.51.100.0/24, 203.0.113.0/24)
+func IsRFC5737(na *domainmessage.NetAddress) bool {
+	for _, rfc := range rfc5737Net {
+		if rfc.Contains(na.IP) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// IsRFC6052 returns whether or not the passed address is part of the IPv6
+// well-known prefix range as defined by RFC6052 (64:FF9B::/96).
+func IsRFC6052(na *domainmessage.NetAddress) bool {
+	return rfc6052Net.Contains(na.IP)
+}
+
+// IsRFC6145 returns whether or not the passed address is part of the IPv6 to
+// IPv4 translated address range as defined by RFC6145 (::FFFF:0:0:0/96).
+func IsRFC6145(na *domainmessage.NetAddress) bool {
+	return rfc6145Net.Contains(na.IP)
+}
+
+// IsRFC6598 returns whether or not the passed address is part of the IPv4
+// shared address space specified by RFC6598 (100.64.0.0/10)
+func IsRFC6598(na *domainmessage.NetAddress) bool {
+	return rfc6598Net.Contains(na.IP)
+}
+
+// IsValid returns whether or not the passed address is valid. The address is
+// considered invalid under the following circumstances:
+// IPv4: It is either a zero or all bits set address.
+// IPv6: It is either a zero or RFC3849 documentation address.
+func IsValid(na *domainmessage.NetAddress) bool {
+	// IsUnspecified returns if address is 0, so only all bits set, and
+	// RFC3849 need to be explicitly checked.
+	return na.IP != nil && !(na.IP.IsUnspecified() ||
+		na.IP.Equal(net.IPv4bcast))
+}
+
+// IsRoutable returns whether or not the passed address is routable over
+// the public internet. This is true as long as the address is valid and is not
+// in any reserved ranges.
+func (am *AddressManager) IsRoutable(na *domainmessage.NetAddress) bool {
+	if am.cfg.NetParams().AcceptUnroutable {
+		return !IsLocal(na)
+	}
+
+	return IsValid(na) && !(IsRFC1918(na) || IsRFC2544(na) ||
+		IsRFC3927(na) || IsRFC4862(na) || IsRFC3849(na) ||
+		IsRFC4843(na) || IsRFC5737(na) || IsRFC6598(na) ||
+		IsLocal(na) || (IsRFC4193(na)))
+}
+
+// GroupKey returns a string representing the network group an address is part
+// of. This is the /16 for IPv4, the /32 (/36 for he.net) for IPv6, the string
+// "local" for a local address, and the string "unroutable" for an unroutable
+// address.
+func (am *AddressManager) GroupKey(na *domainmessage.NetAddress) string {
+	if IsLocal(na) {
+		return "local"
+	}
+	if !am.IsRoutable(na) {
+		return "unroutable"
+	}
+	if IsIPv4(na) {
+		return na.IP.Mask(net.CIDRMask(16, 32)).String()
+	}
+	if IsRFC6145(na) || IsRFC6052(na) {
+		// last four bytes are the ip address
+		ip := na.IP[12:16]
+		return ip.Mask(net.CIDRMask(16, 32)).String()
+	}
+
+	if IsRFC3964(na) {
+		ip := na.IP[2:6]
+		return ip.Mask(net.CIDRMask(16, 32)).String()
+
+	}
+	if IsRFC4380(na) {
+		// teredo tunnels have the last 4 bytes as the v4 address XOR
+		// 0xff.
+		ip := net.IP(make([]byte, 4))
+		for i, byte := range na.IP[12:16] {
+			ip[i] = byte ^ 0xff
+		}
+		return ip.Mask(net.CIDRMask(16, 32)).String()
+	}
+
+	// OK, so now we know ourselves to be a IPv6 address.
+	// We use /32 for everything, except for Hurricane Electric's
+	// (he.net) IP range, which we use /36 for.
+	bits := 32
+	if heNet.Contains(na.IP) {
+		bits = 36
+	}
+
+	return na.IP.Mask(net.CIDRMask(bits, 128)).String()
+}

addressmanager/network_test.go

@@ -0,0 +1,206 @@
+// Copyright (c) 2013-2014 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package addressmanager
+
+import (
+	"net"
+	"testing"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+// TestIPTypes ensures the various functions which determine the type of an IP
+// address based on RFCs work as intended.
+func TestIPTypes(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
+	type ipTest struct {
+		in       domainmessage.NetAddress
+		rfc1918  bool
+		rfc2544  bool
+		rfc3849  bool
+		rfc3927  bool
+		rfc3964  bool
+		rfc4193  bool
+		rfc4380  bool
+		rfc4843  bool
+		rfc4862  bool
+		rfc5737  bool
+		rfc6052  bool
+		rfc6145  bool
+		rfc6598  bool
+		local    bool
+		valid    bool
+		routable bool
+	}
+
+	newIPTest := func(ip string, rfc1918, rfc2544, rfc3849, rfc3927, rfc3964,
+		rfc4193, rfc4380, rfc4843, rfc4862, rfc5737, rfc6052, rfc6145, rfc6598,
+		local, valid, routable bool) ipTest {
+		nip := net.ParseIP(ip)
+		na := *domainmessage.NewNetAddressIPPort(nip, 16111, domainmessage.SFNodeNetwork)
+		test := ipTest{na, rfc1918, rfc2544, rfc3849, rfc3927, rfc3964, rfc4193, rfc4380,
+			rfc4843, rfc4862, rfc5737, rfc6052, rfc6145, rfc6598, local, valid, routable}
+		return test
+	}
+
+	tests := []ipTest{
+		newIPTest("10.255.255.255", true, false, false, false, false, false,
+			false, false, false, false, false, false, false, false, true, false),
+		newIPTest("192.168.0.1", true, false, false, false, false, false,
+			false, false, false, false, false, false, false, false, true, false),
+		newIPTest("172.31.255.1", true, false, false, false, false, false,
+			false, false, false, false, false, false, false, false, true, false),
+		newIPTest("172.32.1.1", false, false, false, false, false, false, false, false,
+			false, false, false, false, false, false, true, true),
+		newIPTest("169.254.250.120", false, false, false, true, false, false,
+			false, false, false, false, false, false, false, false, true, false),
+		newIPTest("0.0.0.0", false, false, false, false, false, false, false,
+			false, false, false, false, false, false, true, false, false),
+		newIPTest("255.255.255.255", false, false, false, false, false, false,
+			false, false, false, false, false, false, false, false, false, false),
+		newIPTest("127.0.0.1", false, false, false, false, false, false,
+			false, false, false, false, false, false, false, true, true, false),
+		newIPTest("fd00:dead::1", false, false, false, false, false, true,
+			false, false, false, false, false, false, false, false, true, false),
+		newIPTest("2001::1", false, false, false, false, false, false,
+			true, false, false, false, false, false, false, false, true, true),
+		newIPTest("2001:10:abcd::1:1", false, false, false, false, false, false,
+			false, true, false, false, false, false, false, false, true, false),
+		newIPTest("fe80::1", false, false, false, false, false, false,
+			false, false, true, false, false, false, false, false, true, false),
+		newIPTest("fe80:1::1", false, false, false, false, false, false,
+			false, false, false, false, false, false, false, false, true, true),
+		newIPTest("64:ff9b::1", false, false, false, false, false, false,
+			false, false, false, false, true, false, false, false, true, true),
+		newIPTest("::ffff:abcd:ef12:1", false, false, false, false, false, false,
+			false, false, false, false, false, false, false, false, true, true),
+		newIPTest("::1", false, false, false, false, false, false, false, false,
+			false, false, false, false, false, true, true, false),
+		newIPTest("198.18.0.1", false, true, false, false, false, false, false,
+			false, false, false, false, false, false, false, true, false),
+		newIPTest("100.127.255.1", false, false, false, false, false, false, false,
+			false, false, false, false, false, true, false, true, false),
+		newIPTest("203.0.113.1", false, false, false, false, false, false, false,
+			false, false, false, false, false, false, false, true, false),
+	}
+
+	t.Logf("Running %d tests", len(tests))
+	for _, test := range tests {
+		if rv := IsRFC1918(&test.in); rv != test.rfc1918 {
+			t.Errorf("IsRFC1918 %s\n got: %v want: %v", test.in.IP, rv, test.rfc1918)
+		}
+
+		if rv := IsRFC3849(&test.in); rv != test.rfc3849 {
+			t.Errorf("IsRFC3849 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3849)
+		}
+
+		if rv := IsRFC3927(&test.in); rv != test.rfc3927 {
+			t.Errorf("IsRFC3927 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3927)
+		}
+
+		if rv := IsRFC3964(&test.in); rv != test.rfc3964 {
+			t.Errorf("IsRFC3964 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3964)
+		}
+
+		if rv := IsRFC4193(&test.in); rv != test.rfc4193 {
+			t.Errorf("IsRFC4193 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4193)
+		}
+
+		if rv := IsRFC4380(&test.in); rv != test.rfc4380 {
+			t.Errorf("IsRFC4380 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4380)
+		}
+
+		if rv := IsRFC4843(&test.in); rv != test.rfc4843 {
+			t.Errorf("IsRFC4843 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4843)
+		}
+
+		if rv := IsRFC4862(&test.in); rv != test.rfc4862 {
+			t.Errorf("IsRFC4862 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4862)
+		}
+
+		if rv := IsRFC6052(&test.in); rv != test.rfc6052 {
+			t.Errorf("isRFC6052 %s\n got: %v want: %v", test.in.IP, rv, test.rfc6052)
+		}
+
+		if rv := IsRFC6145(&test.in); rv != test.rfc6145 {
+			t.Errorf("IsRFC1918 %s\n got: %v want: %v", test.in.IP, rv, test.rfc6145)
+		}
+
+		if rv := IsLocal(&test.in); rv != test.local {
+			t.Errorf("IsLocal %s\n got: %v want: %v", test.in.IP, rv, test.local)
+		}
+
+		if rv := IsValid(&test.in); rv != test.valid {
+			t.Errorf("IsValid %s\n got: %v want: %v", test.in.IP, rv, test.valid)
+		}
+
+		if rv := amgr.IsRoutable(&test.in); rv != test.routable {
+			t.Errorf("IsRoutable %s\n got: %v want: %v", test.in.IP, rv, test.routable)
+		}
+	}
+}
+
+// TestGroupKey tests the GroupKey function to ensure it properly groups various
+// IP addresses.
+func TestGroupKey(t *testing.T) {
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
+
+	tests := []struct {
+		name     string
+		ip       string
+		expected string
+	}{
+		// Local addresses.
+		{name: "ipv4 localhost", ip: "127.0.0.1", expected: "local"},
+		{name: "ipv6 localhost", ip: "::1", expected: "local"},
+		{name: "ipv4 zero", ip: "0.0.0.0", expected: "local"},
+		{name: "ipv4 first octet zero", ip: "0.1.2.3", expected: "local"},
+
+		// Unroutable addresses.
+		{name: "ipv4 invalid bcast", ip: "255.255.255.255", expected: "unroutable"},
+		{name: "ipv4 rfc1918 10/8", ip: "10.1.2.3", expected: "unroutable"},
+		{name: "ipv4 rfc1918 172.16/12", ip: "172.16.1.2", expected: "unroutable"},
+		{name: "ipv4 rfc1918 192.168/16", ip: "192.168.1.2", expected: "unroutable"},
+		{name: "ipv6 rfc3849 2001:db8::/32", ip: "2001:db8::1234", expected: "unroutable"},
+		{name: "ipv4 rfc3927 169.254/16", ip: "169.254.1.2", expected: "unroutable"},
+		{name: "ipv6 rfc4193 fc00::/7", ip: "fc00::1234", expected: "unroutable"},
+		{name: "ipv6 rfc4843 2001:10::/28", ip: "2001:10::1234", expected: "unroutable"},
+		{name: "ipv6 rfc4862 fe80::/64", ip: "fe80::1234", expected: "unroutable"},
+
+		// IPv4 normal.
+		{name: "ipv4 normal class a", ip: "12.1.2.3", expected: "12.1.0.0"},
+		{name: "ipv4 normal class b", ip: "173.1.2.3", expected: "173.1.0.0"},
+		{name: "ipv4 normal class c", ip: "196.1.2.3", expected: "196.1.0.0"},
+
+		// IPv6/IPv4 translations.
+		{name: "ipv6 rfc3964 with ipv4 encap", ip: "2002:0c01:0203::", expected: "12.1.0.0"},
+		{name: "ipv6 rfc4380 toredo ipv4", ip: "2001:0:1234::f3fe:fdfc", expected: "12.1.0.0"},
+		{name: "ipv6 rfc6052 well-known prefix with ipv4", ip: "64:ff9b::0c01:0203", expected: "12.1.0.0"},
+		{name: "ipv6 rfc6145 translated ipv4", ip: "::ffff:0:0c01:0203", expected: "12.1.0.0"},
+
+		// Tor.
+		{name: "ipv6 tor onioncat", ip: "fd87:d87e:eb43:1234::5678", expected: "unroutable"},
+		{name: "ipv6 tor onioncat 2", ip: "fd87:d87e:eb43:1245::6789", expected: "unroutable"},
+		{name: "ipv6 tor onioncat 3", ip: "fd87:d87e:eb43:1345::6789", expected: "unroutable"},
+
+		// IPv6 normal.
+		{name: "ipv6 normal", ip: "2602:100::1", expected: "2602:100::"},
+		{name: "ipv6 normal 2", ip: "2602:0100::1234", expected: "2602:100::"},
+		{name: "ipv6 hurricane electric", ip: "2001:470:1f10:a1::2", expected: "2001:470:1000::"},
+		{name: "ipv6 hurricane electric 2", ip: "2001:0470:1f10:a1::2", expected: "2001:470:1000::"},
+	}
+
+	for i, test := range tests {
+		nip := net.ParseIP(test.ip)
+		na := *domainmessage.NewNetAddressIPPort(nip, 8333, domainmessage.SFNodeNetwork)
+		if key := amgr.GroupKey(&na); key != test.expected {
+			t.Errorf("TestGroupKey #%d (%s): unexpected group key "+
+				"- got '%s', want '%s'", i, test.name,
+				key, test.expected)
+		}
+	}
+}

addrmanager_test.go

@@ -1,281 +0,0 @@
-// Copyright (c) 2013 Conformal Systems LLC.
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"github.com/conformal/btcwire"
-	"net"
-	"testing"
-	"time"
-)
-
-// naTest is used to describe a test to be perfomed against the NetAddressKey
-// method.
-type naTest struct {
-	in   btcwire.NetAddress
-	want string
-}
-
-type ipTest struct {
-	in       btcwire.NetAddress
-	rfc1918  bool
-	rfc3849  bool
-	rfc3927  bool
-	rfc3964  bool
-	rfc4193  bool
-	rfc4380  bool
-	rfc4843  bool
-	rfc4862  bool
-	rfc6052  bool
-	rfc6145  bool
-	local    bool
-	valid    bool
-	routable bool
-}
-
-// naTests houses all of the tests to be performed against the NetAddressKey
-// method.
-var naTests = make([]naTest, 0)
-var ipTests = make([]ipTest, 0)
-
-func addIpTest(ip string, rfc1918, rfc3849, rfc3927, rfc3964, rfc4193, rfc4380,
-	rfc4843, rfc4862, rfc6052, rfc6145, local, valid, routable bool) {
-	nip := net.ParseIP(ip)
-	na := btcwire.NetAddress{
-		Timestamp: time.Now(),
-		Services:  btcwire.SFNodeNetwork,
-		IP:        nip,
-		Port:      8333,
-	}
-	test := ipTest{na, rfc1918, rfc3849, rfc3927, rfc3964, rfc4193, rfc4380,
-		rfc4843, rfc4862, rfc6052, rfc6145, local, valid, routable}
-	ipTests = append(ipTests, test)
-}
-
-func addIpTests() {
-	addIpTest("10.255.255.255", true, false, false, false, false, false,
-		false, false, false, false, false, true, false)
-	addIpTest("192.168.0.1", true, false, false, false, false, false,
-		false, false, false, false, false, true, false)
-	addIpTest("172.31.255.1", true, false, false, false, false, false,
-		false, false, false, false, false, true, false)
-	addIpTest("172.32.1.1", false, false, false, false, false, false,
-		false, false, false, false, false, true, true)
-	addIpTest("169.254.250.120", false, false, true, false, false, false,
-		false, false, false, false, false, true, false)
-	addIpTest("0.0.0.0", false, false, false, false, false, false,
-		false, false, false, false, true, false, false)
-	addIpTest("255.255.255.255", false, false, false, false, false, false,
-		false, false, false, false, false, false, false)
-	addIpTest("127.0.0.1", false, false, false, false, false, false,
-		false, false, false, false, true, true, false)
-	addIpTest("fd00:dead::1", false, false, false, false, true, false,
-		false, false, false, false, false, true, false)
-	addIpTest("2001::1", false, false, false, false, false, true,
-		false, false, false, false, false, true, true)
-	addIpTest("2001:10:abcd::1:1", false, false, false, false, false, false,
-		true, false, false, false, false, true, false)
-	addIpTest("fe80::1", false, false, false, false, false, false,
-		false, true, false, false, false, true, false)
-	addIpTest("fe80:1::1", false, false, false, false, false, false,
-		false, false, false, false, false, true, true)
-	addIpTest("64:ff9b::1", false, false, false, false, false, false,
-		false, false, true, false, false, true, true)
-	addIpTest("::ffff:abcd:ef12:1", false, false, false, false, false, false,
-		false, false, false, false, false, true, true)
-	addIpTest("::1", false, false, false, false, false, false,
-		false, false, false, false, true, true, false)
-}
-
-func addNaTest(ip string, port uint16, want string) {
-	nip := net.ParseIP(ip)
-	na := btcwire.NetAddress{
-		Timestamp: time.Now(),
-		Services:  btcwire.SFNodeNetwork,
-		IP:        nip,
-		Port:      port,
-	}
-	test := naTest{na, want}
-	naTests = append(naTests, test)
-}
-
-// addNaTests
-func addNaTests() {
-	// IPv4
-	// Localhost
-	addNaTest("127.0.0.1", 8333, "127.0.0.1:8333")
-	addNaTest("127.0.0.1", 8334, "127.0.0.1:8334")
-
-	// Class A
-	addNaTest("1.0.0.1", 8333, "1.0.0.1:8333")
-	addNaTest("2.2.2.2", 8334, "2.2.2.2:8334")
-	addNaTest("27.253.252.251", 8335, "27.253.252.251:8335")
-	addNaTest("123.3.2.1", 8336, "123.3.2.1:8336")
-
-	// Private Class A
-	addNaTest("10.0.0.1", 8333, "10.0.0.1:8333")
-	addNaTest("10.1.1.1", 8334, "10.1.1.1:8334")
-	addNaTest("10.2.2.2", 8335, "10.2.2.2:8335")
-	addNaTest("10.10.10.10", 8336, "10.10.10.10:8336")
-
-	// Class B
-	addNaTest("128.0.0.1", 8333, "128.0.0.1:8333")
-	addNaTest("129.1.1.1", 8334, "129.1.1.1:8334")
-	addNaTest("180.2.2.2", 8335, "180.2.2.2:8335")
-	addNaTest("191.10.10.10", 8336, "191.10.10.10:8336")
-
-	// Private Class B
-	addNaTest("172.16.0.1", 8333, "172.16.0.1:8333")
-	addNaTest("172.16.1.1", 8334, "172.16.1.1:8334")
-	addNaTest("172.16.2.2", 8335, "172.16.2.2:8335")
-	addNaTest("172.16.172.172", 8336, "172.16.172.172:8336")
-
-	// Class C
-	addNaTest("193.0.0.1", 8333, "193.0.0.1:8333")
-	addNaTest("200.1.1.1", 8334, "200.1.1.1:8334")
-	addNaTest("205.2.2.2", 8335, "205.2.2.2:8335")
-	addNaTest("223.10.10.10", 8336, "223.10.10.10:8336")
-
-	// Private Class C
-	addNaTest("192.168.0.1", 8333, "192.168.0.1:8333")
-	addNaTest("192.168.1.1", 8334, "192.168.1.1:8334")
-	addNaTest("192.168.2.2", 8335, "192.168.2.2:8335")
-	addNaTest("192.168.192.192", 8336, "192.168.192.192:8336")
-
-	// IPv6
-	// Localhost
-	addNaTest("::1", 8333, "[::1]:8333")
-	addNaTest("fe80::1", 8334, "[fe80::1]:8334")
-
-	// Link-local
-	addNaTest("fe80::1:1", 8333, "[fe80::1:1]:8333")
-	addNaTest("fe91::2:2", 8334, "[fe91::2:2]:8334")
-	addNaTest("fea2::3:3", 8335, "[fea2::3:3]:8335")
-	addNaTest("feb3::4:4", 8336, "[feb3::4:4]:8336")
-
-	// Site-local
-	addNaTest("fec0::1:1", 8333, "[fec0::1:1]:8333")
-	addNaTest("fed1::2:2", 8334, "[fed1::2:2]:8334")
-	addNaTest("fee2::3:3", 8335, "[fee2::3:3]:8335")
-	addNaTest("fef3::4:4", 8336, "[fef3::4:4]:8336")
-}
-
-func TestGetAddress(t *testing.T) {
-	n := NewAddrManager()
-	if rv := n.GetAddress("any", 10); rv != nil {
-		t.Errorf("GetAddress failed: got: %v want: %v\n", rv, nil)
-	}
-}
-
-func TestIpTypes(t *testing.T) {
-	addIpTests()
-
-	t.Logf("Running %d tests", len(ipTests))
-	for _, test := range ipTests {
-		rv := RFC1918(&test.in)
-		if rv != test.rfc1918 {
-			t.Errorf("RFC1918 %s\n got: %v want: %v", test.in.IP, rv, test.rfc1918)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC3849(&test.in)
-		if rv != test.rfc3849 {
-			t.Errorf("RFC3849 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3849)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC3927(&test.in)
-		if rv != test.rfc3927 {
-			t.Errorf("RFC3927 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3927)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC3964(&test.in)
-		if rv != test.rfc3964 {
-			t.Errorf("RFC3964 %s\n got: %v want: %v", test.in.IP, rv, test.rfc3964)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC4193(&test.in)
-		if rv != test.rfc4193 {
-			t.Errorf("RFC4193 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4193)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC4380(&test.in)
-		if rv != test.rfc4380 {
-			t.Errorf("RFC4380 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4380)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC4843(&test.in)
-		if rv != test.rfc4843 {
-			t.Errorf("RFC4843 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4843)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC4862(&test.in)
-		if rv != test.rfc4862 {
-			t.Errorf("RFC4862 %s\n got: %v want: %v", test.in.IP, rv, test.rfc4862)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC6052(&test.in)
-		if rv != test.rfc6052 {
-			t.Errorf("RFC6052 %s\n got: %v want: %v", test.in.IP, rv, test.rfc6052)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := RFC6145(&test.in)
-		if rv != test.rfc6145 {
-			t.Errorf("RFC1918 %s\n got: %v want: %v", test.in.IP, rv, test.rfc6145)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := Local(&test.in)
-		if rv != test.local {
-			t.Errorf("Local %s\n got: %v want: %v", test.in.IP, rv, test.local)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := Valid(&test.in)
-		if rv != test.valid {
-			t.Errorf("Valid %s\n got: %v want: %v", test.in.IP, rv, test.valid)
-			continue
-		}
-	}
-	for _, test := range ipTests {
-		rv := Routable(&test.in)
-		if rv != test.routable {
-			t.Errorf("Routable %s\n got: %v want: %v", test.in.IP, rv, test.routable)
-			continue
-		}
-	}
-}
-
-func TestNetAddressKey(t *testing.T) {
-	addNaTests()
-
-	t.Logf("Running %d tests", len(naTests))
-	for i, test := range naTests {
-		key := NetAddressKey(&test.in)
-		if key != test.want {
-			t.Errorf("NetAddressKey #%d\n got: %s want: %s", i, key, test.want)
-			continue
-		}
-	}
-
-}

app/app.go

@@ -0,0 +1,249 @@
+package app
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/kaspanet/kaspad/addressmanager"
+
+	"github.com/kaspanet/kaspad/netadapter/id"
+
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/blockdag/indexers"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/connmanager"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/dnsseed"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/mempool"
+	"github.com/kaspanet/kaspad/mining"
+	"github.com/kaspanet/kaspad/netadapter"
+	"github.com/kaspanet/kaspad/protocol"
+	"github.com/kaspanet/kaspad/rpc"
+	"github.com/kaspanet/kaspad/signal"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+// App is a wrapper for all the kaspad services
+type App struct {
+	cfg               *config.Config
+	rpcServer         *rpc.Server
+	addressManager    *addressmanager.AddressManager
+	protocolManager   *protocol.Manager
+	connectionManager *connmanager.ConnectionManager
+	netAdapter        *netadapter.NetAdapter
+
+	started, shutdown int32
+}
+
+// Start launches all the kaspad services.
+func (a *App) Start() {
+	// Already started?
+	if atomic.AddInt32(&a.started, 1) != 1 {
+		return
+	}
+
+	log.Trace("Starting kaspad")
+
+	err := a.protocolManager.Start()
+	if err != nil {
+		panics.Exit(log, fmt.Sprintf("Error starting the p2p protocol: %+v", err))
+	}
+
+	a.maybeSeedFromDNS()
+
+	a.connectionManager.Start()
+
+	if !a.cfg.DisableRPC {
+		a.rpcServer.Start()
+	}
+}
+
+// Stop gracefully shuts down all the kaspad services.
+func (a *App) Stop() error {
+	// Make sure this only happens once.
+	if atomic.AddInt32(&a.shutdown, 1) != 1 {
+		log.Infof("Kaspad is already in the process of shutting down")
+		return nil
+	}
+
+	log.Warnf("Kaspad shutting down")
+
+	a.connectionManager.Stop()
+
+	err := a.protocolManager.Stop()
+	if err != nil {
+		log.Errorf("Error stopping the p2p protocol: %+v", err)
+	}
+
+	// Shutdown the RPC server if it's not disabled.
+	if !a.cfg.DisableRPC {
+		err := a.rpcServer.Stop()
+		if err != nil {
+			log.Errorf("Error stopping rpcServer: %+v", err)
+		}
+	}
+
+	return nil
+}
+
+// New returns a new App instance configured to listen on addr for the
+// kaspa network type specified by dagParams. Use start to begin accepting
+// connections from peers.
+func New(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, interrupt <-chan struct{}) (*App, error) {
+	indexManager, acceptanceIndex := setupIndexes(cfg)
+
+	sigCache := txscript.NewSigCache(cfg.SigCacheMaxSize)
+
+	// Create a new block DAG instance with the appropriate configuration.
+	dag, err := setupDAG(cfg, databaseContext, interrupt, sigCache, indexManager)
+	if err != nil {
+		return nil, err
+	}
+
+	txMempool := setupMempool(cfg, dag, sigCache)
+
+	netAdapter, err := netadapter.NewNetAdapter(cfg)
+	if err != nil {
+		return nil, err
+	}
+	addressManager := addressmanager.New(cfg, databaseContext)
+
+	connectionManager, err := connmanager.New(cfg, netAdapter, addressManager)
+	if err != nil {
+		return nil, err
+	}
+
+	protocolManager, err := protocol.NewManager(cfg, dag, netAdapter, addressManager, txMempool, connectionManager)
+	if err != nil {
+		return nil, err
+	}
+	rpcServer, err := setupRPC(
+		cfg, dag, txMempool, sigCache, acceptanceIndex, connectionManager, addressManager, protocolManager)
+	if err != nil {
+		return nil, err
+	}
+
+	return &App{
+		cfg:               cfg,
+		rpcServer:         rpcServer,
+		protocolManager:   protocolManager,
+		connectionManager: connectionManager,
+		netAdapter:        netAdapter,
+		addressManager:    addressManager,
+	}, nil
+}
+
+func (a *App) maybeSeedFromDNS() {
+	if !a.cfg.DisableDNSSeed {
+		dnsseed.SeedFromDNS(a.cfg.NetParams(), a.cfg.DNSSeed, domainmessage.SFNodeNetwork, false, nil,
+			a.cfg.Lookup, func(addresses []*domainmessage.NetAddress) {
+				// Kaspad uses a lookup of the dns seeder here. Since seeder returns
+				// IPs of nodes and not its own IP, we can not know real IP of
+				// source. So we'll take first returned address as source.
+				a.addressManager.AddAddresses(addresses, addresses[0], nil)
+			})
+	}
+}
+func setupDAG(cfg *config.Config, databaseContext *dbaccess.DatabaseContext, interrupt <-chan struct{},
+	sigCache *txscript.SigCache, indexManager blockdag.IndexManager) (*blockdag.BlockDAG, error) {
+
+	dag, err := blockdag.New(&blockdag.Config{
+		Interrupt:       interrupt,
+		DatabaseContext: databaseContext,
+		DAGParams:       cfg.NetParams(),
+		TimeSource:      blockdag.NewTimeSource(),
+		SigCache:        sigCache,
+		IndexManager:    indexManager,
+		SubnetworkID:    cfg.SubnetworkID,
+	})
+	return dag, err
+}
+
+func setupIndexes(cfg *config.Config) (blockdag.IndexManager, *indexers.AcceptanceIndex) {
+	// Create indexes if needed.
+	var indexes []indexers.Indexer
+	var acceptanceIndex *indexers.AcceptanceIndex
+	if cfg.AcceptanceIndex {
+		log.Info("acceptance index is enabled")
+		acceptanceIndex = indexers.NewAcceptanceIndex()
+		indexes = append(indexes, acceptanceIndex)
+	}
+
+	// Create an index manager if any of the optional indexes are enabled.
+	if len(indexes) < 0 {
+		return nil, nil
+	}
+	indexManager := indexers.NewManager(indexes)
+	return indexManager, acceptanceIndex
+}
+
+func setupMempool(cfg *config.Config, dag *blockdag.BlockDAG, sigCache *txscript.SigCache) *mempool.TxPool {
+	mempoolConfig := mempool.Config{
+		Policy: mempool.Policy{
+			AcceptNonStd:    cfg.RelayNonStd,
+			MaxOrphanTxs:    cfg.MaxOrphanTxs,
+			MaxOrphanTxSize: config.DefaultMaxOrphanTxSize,
+			MinRelayTxFee:   cfg.MinRelayTxFee,
+			MaxTxVersion:    1,
+		},
+		CalcSequenceLockNoLock: func(tx *util.Tx, utxoSet blockdag.UTXOSet) (*blockdag.SequenceLock, error) {
+			return dag.CalcSequenceLockNoLock(tx, utxoSet, true)
+		},
+		IsDeploymentActive: dag.IsDeploymentActive,
+		SigCache:           sigCache,
+		DAG:                dag,
+	}
+
+	return mempool.New(&mempoolConfig)
+}
+
+func setupRPC(cfg *config.Config,
+	dag *blockdag.BlockDAG,
+	txMempool *mempool.TxPool,
+	sigCache *txscript.SigCache,
+	acceptanceIndex *indexers.AcceptanceIndex,
+	connectionManager *connmanager.ConnectionManager,
+	addressManager *addressmanager.AddressManager,
+	protocolManager *protocol.Manager) (*rpc.Server, error) {
+
+	if !cfg.DisableRPC {
+		policy := mining.Policy{
+			BlockMaxMass: cfg.BlockMaxMass,
+		}
+		blockTemplateGenerator := mining.NewBlkTmplGenerator(&policy, txMempool, dag, sigCache)
+
+		rpcServer, err := rpc.NewRPCServer(cfg, dag, txMempool, acceptanceIndex, blockTemplateGenerator,
+			connectionManager, addressManager, protocolManager)
+		if err != nil {
+			return nil, err
+		}
+
+		// Signal process shutdown when the RPC server requests it.
+		spawn("setupRPC-handleShutdownRequest", func() {
+			<-rpcServer.RequestedProcessShutdown()
+			signal.ShutdownRequestChannel <- struct{}{}
+		})
+
+		return rpcServer, nil
+	}
+	return nil, nil
+}
+
+// P2PNodeID returns the network ID associated with this App
+func (a *App) P2PNodeID() *id.ID {
+	return a.netAdapter.ID()
+}
+
+// AddressManager returns the AddressManager associated with this App
+func (a *App) AddressManager() *addressmanager.AddressManager {
+	return a.addressManager
+}
+
+// WaitForShutdown blocks until the main listener and peer handlers are stopped.
+func (a *App) WaitForShutdown() {
+	// TODO(libp2p)
+	// a.p2pServer.WaitForShutdown()
+}

app/log.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Copyright (c) 2017 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package app
+
+import (
+	"github.com/kaspanet/kaspad/logger"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+var log, _ = logger.Get(logger.SubsystemTags.KASD)
+var spawn = panics.GoroutineWrapperFunc(log)

blockdag/README.md

@@ -0,0 +1,41 @@
+blockchain
+==========
+
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/blockchain)
+
+Package blockdag implements Kaspa block handling, organization of the blockDAG, 
+block sorting and UTXO-set maintenance.
+The test coverage is currently only around 75%, but will be increasing over
+time.
+
+## Kaspad BlockDAG Processing Overview
+
+Before a block is allowed into the block DAG, it must go through an intensive
+series of validation rules. The following list serves as a general outline of
+those rules to provide some intuition into what is going on under the hood, but
+is by no means exhaustive:
+
+ - Reject duplicate blocks
+ - Perform a series of sanity checks on the block and its transactions such as
+   verifying proof of work, timestamps, number and character of transactions,
+   transaction amounts, script complexity, and merkle root calculations
+ - Save the most recent orphan blocks for a limited time in case their parent
+   blocks become available.
+ - Save blocks from the future for delayed processing
+ - Stop processing if the block is an orphan or delayed as the rest of the 
+   processing depends on the block's position within the block chain
+ - Make sure the block does not violate finality rules
+ - Perform a series of more thorough checks that depend on the block's position
+   within the blockDAG such as verifying block difficulties adhere to
+   difficulty retarget rules, timestamps are after the median of the last
+   several blocks, all transactions are finalized, checkpoint blocks match, and
+   block versions are in line with the previous blocks
+ - Determine how the block fits into the DAG and perform different actions
+   accordingly 
+ - Run the transaction scripts to verify the spender is allowed to spend the
+   coins
+ - Run GhostDAG to fit the block in a canonical sorting
+ - Build the block's UTXO Set, as well as update the global UTXO Set accordingly
+ - Insert the block into the block database
+

blockdag/accept.go

@@ -0,0 +1,154 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
+)
+
+func (dag *BlockDAG) addNodeToIndexWithInvalidAncestor(block *util.Block) error {
+	blockHeader := &block.MsgBlock().Header
+	newNode, _ := dag.newBlockNode(blockHeader, newBlockSet())
+	newNode.status = statusInvalidAncestor
+	dag.index.AddNode(newNode)
+
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = dag.index.flushToDB(dbTx)
+	if err != nil {
+		return err
+	}
+	return dbTx.Commit()
+}
+
+// maybeAcceptBlock potentially accepts a block into the block DAG. It
+// performs several validation checks which depend on its position within
+// the block DAG before adding it. The block is expected to have already
+// gone through ProcessBlock before calling this function with it.
+//
+// The flags are also passed to checkBlockContext and connectToDAG. See
+// their documentation for how the flags modify their behavior.
+//
+// This function MUST be called with the dagLock held (for writes).
+func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) error {
+	parents, err := lookupParentNodes(block, dag)
+	if err != nil {
+		var ruleErr RuleError
+		if ok := errors.As(err, &ruleErr); ok && ruleErr.ErrorCode == ErrInvalidAncestorBlock {
+			err := dag.addNodeToIndexWithInvalidAncestor(block)
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	}
+
+	// The block must pass all of the validation rules which depend on the
+	// position of the block within the block DAG.
+	err = dag.checkBlockContext(block, parents, flags)
+	if err != nil {
+		return err
+	}
+
+	// Create a new block node for the block and add it to the node index.
+	newNode, selectedParentAnticone := dag.newBlockNode(&block.MsgBlock().Header, parents)
+	newNode.status = statusDataStored
+	dag.index.AddNode(newNode)
+
+	// Insert the block into the database if it's not already there. Even
+	// though it is possible the block will ultimately fail to connect, it
+	// has already passed all proof-of-work and validity tests which means
+	// it would be prohibitively expensive for an attacker to fill up the
+	// disk with a bunch of blocks that fail to connect. This is necessary
+	// since it allows block download to be decoupled from the much more
+	// expensive connection logic. It also has some other nice properties
+	// such as making blocks that never become part of the DAG or
+	// blocks that fail to connect available for further analysis.
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+	blockExists, err := dbaccess.HasBlock(dbTx, block.Hash())
+	if err != nil {
+		return err
+	}
+	if !blockExists {
+		err := storeBlock(dbTx, block)
+		if err != nil {
+			return err
+		}
+	}
+	err = dag.index.flushToDB(dbTx)
+	if err != nil {
+		return err
+	}
+	err = dbTx.Commit()
+	if err != nil {
+		return err
+	}
+
+	// Make sure that all the block's transactions are finalized
+	fastAdd := flags&BFFastAdd == BFFastAdd
+	bluestParent := parents.bluest()
+	if !fastAdd {
+		if err := dag.validateAllTxsFinalized(block, newNode, bluestParent); err != nil {
+			return err
+		}
+	}
+
+	// Connect the passed block to the DAG. This also handles validation of the
+	// transaction scripts.
+	chainUpdates, err := dag.addBlock(newNode, block, selectedParentAnticone, flags)
+	if err != nil {
+		return err
+	}
+
+	// Notify the caller that the new block was accepted into the block
+	// DAG. The caller would typically want to react by relaying the
+	// inventory to other peers.
+	dag.dagLock.Unlock()
+	dag.sendNotification(NTBlockAdded, &BlockAddedNotificationData{
+		Block:         block,
+		WasUnorphaned: flags&BFWasUnorphaned != 0,
+	})
+	if len(chainUpdates.addedChainBlockHashes) > 0 {
+		dag.sendNotification(NTChainChanged, &ChainChangedNotificationData{
+			RemovedChainBlockHashes: chainUpdates.removedChainBlockHashes,
+			AddedChainBlockHashes:   chainUpdates.addedChainBlockHashes,
+		})
+	}
+	dag.dagLock.Lock()
+
+	return nil
+}
+
+func lookupParentNodes(block *util.Block, dag *BlockDAG) (blockSet, error) {
+	header := block.MsgBlock().Header
+	parentHashes := header.ParentHashes
+
+	nodes := newBlockSet()
+	for _, parentHash := range parentHashes {
+		node, ok := dag.index.LookupNode(parentHash)
+		if !ok {
+			str := fmt.Sprintf("parent block %s is unknown", parentHash)
+			return nil, ruleError(ErrParentBlockUnknown, str)
+		} else if dag.index.NodeStatus(node).KnownInvalid() {
+			str := fmt.Sprintf("parent block %s is known to be invalid", parentHash)
+			return nil, ruleError(ErrInvalidAncestorBlock, str)
+		}
+
+		nodes.add(node)
+	}
+
+	return nodes, nil
+}

blockdag/accept_test.go

@@ -0,0 +1,107 @@
+package blockdag
+
+import (
+	"errors"
+	"path/filepath"
+	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+)
+
+func TestMaybeAcceptBlockErrors(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestMaybeAcceptBlockErrors", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestMaybeAcceptBlockErrors: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	dag.TestSetCoinbaseMaturity(0)
+
+	// Test rejecting the block if its parents are missing
+	orphanBlockFile := "blk_3B.dat"
+	loadedBlocks, err := LoadBlocks(filepath.Join("testdata/", orphanBlockFile))
+	if err != nil {
+		t.Fatalf("TestMaybeAcceptBlockErrors: "+
+			"Error loading file '%s': %s\n", orphanBlockFile, err)
+	}
+	block := loadedBlocks[0]
+
+	err = dag.maybeAcceptBlock(block, BFNone)
+	if err == nil {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
+			"Expected: %s, got: <nil>", ErrParentBlockUnknown)
+	}
+	var ruleErr RuleError
+	if ok := errors.As(err, &ruleErr); !ok {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
+			"Expected RuleError but got %s", err)
+	} else if ruleErr.ErrorCode != ErrParentBlockUnknown {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are missing: "+
+			"Unexpected error code. Want: %s, got: %s", ErrParentBlockUnknown, ruleErr.ErrorCode)
+	}
+
+	// Test rejecting the block if its parents are invalid
+	blocksFile := "blk_0_to_4.dat"
+	blocks, err := LoadBlocks(filepath.Join("testdata/", blocksFile))
+	if err != nil {
+		t.Fatalf("TestMaybeAcceptBlockErrors: "+
+			"Error loading file '%s': %s\n", blocksFile, err)
+	}
+
+	// Add a valid block and mark it as invalid
+	block1 := blocks[1]
+	isOrphan, isDelayed, err := dag.ProcessBlock(block1, BFNone)
+	if err != nil {
+		t.Fatalf("TestMaybeAcceptBlockErrors: Valid block unexpectedly returned an error: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("TestMaybeAcceptBlockErrors: block 1 is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("TestMaybeAcceptBlockErrors: incorrectly returned block 1 is an orphan")
+	}
+	blockNode1, ok := dag.index.LookupNode(block1.Hash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", block1.Hash())
+	}
+	dag.index.SetStatusFlags(blockNode1, statusValidateFailed)
+
+	block2 := blocks[2]
+	err = dag.maybeAcceptBlock(block2, BFNone)
+	if err == nil {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
+			"Expected: %s, got: <nil>", ErrInvalidAncestorBlock)
+	}
+	if ok := errors.As(err, &ruleErr); !ok {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
+			"Expected RuleError but got %s", err)
+	} else if ruleErr.ErrorCode != ErrInvalidAncestorBlock {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block if its parents are invalid: "+
+			"Unexpected error. Want: %s, got: %s", ErrInvalidAncestorBlock, ruleErr.ErrorCode)
+	}
+
+	// Set block1's status back to valid for next tests
+	dag.index.UnsetStatusFlags(blockNode1, statusValidateFailed)
+
+	// Test rejecting the block due to bad context
+	originalBits := block2.MsgBlock().Header.Bits
+	block2.MsgBlock().Header.Bits = 0
+	err = dag.maybeAcceptBlock(block2, BFNone)
+	if err == nil {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
+			"Expected: %s, got: <nil>", ErrUnexpectedDifficulty)
+	}
+	if ok := errors.As(err, &ruleErr); !ok {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
+			"Expected RuleError but got %s", err)
+	} else if ruleErr.ErrorCode != ErrUnexpectedDifficulty {
+		t.Errorf("TestMaybeAcceptBlockErrors: rejecting the block due to bad context: "+
+			"Unexpected error. Want: %s, got: %s", ErrUnexpectedDifficulty, ruleErr.ErrorCode)
+	}
+
+	// Set block2's bits back to valid for next tests
+	block2.MsgBlock().Header.Bits = originalBits
+}

blockdag/blockheap.go

@@ -0,0 +1,78 @@
+package blockdag
+
+import (
+	"container/heap"
+)
+
+// baseHeap is an implementation for heap.Interface that sorts blocks by their height
+type baseHeap []*blockNode
+
+func (h baseHeap) Len() int      { return len(h) }
+func (h baseHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+
+func (h *baseHeap) Push(x interface{}) {
+	*h = append(*h, x.(*blockNode))
+}
+
+func (h *baseHeap) Pop() interface{} {
+	oldHeap := *h
+	oldLength := len(oldHeap)
+	popped := oldHeap[oldLength-1]
+	*h = oldHeap[0 : oldLength-1]
+	return popped
+}
+
+// upHeap extends baseHeap to include Less operation that traverses from bottom to top
+type upHeap struct{ baseHeap }
+
+func (h upHeap) Less(i, j int) bool {
+	return h.baseHeap[i].less(h.baseHeap[j])
+}
+
+// downHeap extends baseHeap to include Less operation that traverses from top to bottom
+type downHeap struct{ baseHeap }
+
+func (h downHeap) Less(i, j int) bool {
+	return !h.baseHeap[i].less(h.baseHeap[j])
+}
+
+// blockHeap represents a mutable heap of Blocks, sorted by their height
+type blockHeap struct {
+	impl heap.Interface
+}
+
+// newDownHeap initializes and returns a new blockHeap
+func newDownHeap() blockHeap {
+	h := blockHeap{impl: &downHeap{}}
+	heap.Init(h.impl)
+	return h
+}
+
+// newUpHeap initializes and returns a new blockHeap
+func newUpHeap() blockHeap {
+	h := blockHeap{impl: &upHeap{}}
+	heap.Init(h.impl)
+	return h
+}
+
+// pop removes the block with lowest height from this heap and returns it
+func (bh blockHeap) pop() *blockNode {
+	return heap.Pop(bh.impl).(*blockNode)
+}
+
+// Push pushes the block onto the heap
+func (bh blockHeap) Push(block *blockNode) {
+	heap.Push(bh.impl, block)
+}
+
+// pushSet pushes a blockset to the heap.
+func (bh blockHeap) pushSet(bs blockSet) {
+	for block := range bs {
+		heap.Push(bh.impl, block)
+	}
+}
+
+// Len returns the length of this heap
+func (bh blockHeap) Len() int {
+	return bh.impl.Len()
+}

blockdag/blockheap_test.go

@@ -0,0 +1,129 @@
+package blockdag
+
+import (
+	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// TestBlockHeap tests pushing, popping, and determining the length of the heap.
+func TestBlockHeap(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestBlockHeap", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestBlockHeap: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	block0Header := dagconfig.SimnetParams.GenesisBlock.Header
+	block0, _ := dag.newBlockNode(&block0Header, newBlockSet())
+
+	block100000Header := Block100000.Header
+	block100000, _ := dag.newBlockNode(&block100000Header, blockSetFromSlice(block0))
+
+	block0smallHash, _ := dag.newBlockNode(&block0Header, newBlockSet())
+	block0smallHash.hash = &daghash.Hash{}
+
+	tests := []struct {
+		name            string
+		toPush          []*blockNode
+		expectedLength  int
+		expectedPopUp   *blockNode
+		expectedPopDown *blockNode
+	}{
+		{
+			name:            "empty heap must have length 0",
+			toPush:          []*blockNode{},
+			expectedLength:  0,
+			expectedPopDown: nil,
+			expectedPopUp:   nil,
+		},
+		{
+			name:            "heap with one push must have length 1",
+			toPush:          []*blockNode{block0},
+			expectedLength:  1,
+			expectedPopDown: nil,
+			expectedPopUp:   nil,
+		},
+		{
+			name:            "heap with one push and one pop",
+			toPush:          []*blockNode{block0},
+			expectedLength:  0,
+			expectedPopDown: block0,
+			expectedPopUp:   block0,
+		},
+		{
+			name: "push two blocks with different heights, heap shouldn't have to rebalance " +
+				"for down direction, but will have to rebalance for up direction",
+			toPush:          []*blockNode{block100000, block0},
+			expectedLength:  1,
+			expectedPopDown: block100000,
+			expectedPopUp:   block0,
+		},
+		{
+			name: "push two blocks with different heights, heap shouldn't have to rebalance " +
+				"for up direction, but will have to rebalance for down direction",
+			toPush:          []*blockNode{block0, block100000},
+			expectedLength:  1,
+			expectedPopDown: block100000,
+			expectedPopUp:   block0,
+		},
+		{
+			name: "push two blocks with equal heights but different hashes, heap shouldn't have to rebalance " +
+				"for down direction, but will have to rebalance for up direction",
+			toPush:          []*blockNode{block0, block0smallHash},
+			expectedLength:  1,
+			expectedPopDown: block0,
+			expectedPopUp:   block0smallHash,
+		},
+		{
+			name: "push two blocks with equal heights but different hashes, heap shouldn't have to rebalance " +
+				"for up direction, but will have to rebalance for down direction",
+			toPush:          []*blockNode{block0smallHash, block0},
+			expectedLength:  1,
+			expectedPopDown: block0,
+			expectedPopUp:   block0smallHash,
+		},
+	}
+
+	for _, test := range tests {
+		dHeap := newDownHeap()
+		for _, block := range test.toPush {
+			dHeap.Push(block)
+		}
+
+		var poppedBlock *blockNode
+		if test.expectedPopDown != nil {
+			poppedBlock = dHeap.pop()
+		}
+		if dHeap.Len() != test.expectedLength {
+			t.Errorf("unexpected down heap length in test \"%s\". "+
+				"Expected: %v, got: %v", test.name, test.expectedLength, dHeap.Len())
+		}
+		if poppedBlock != test.expectedPopDown {
+			t.Errorf("unexpected popped block for down heap in test \"%s\". "+
+				"Expected: %v, got: %v", test.name, test.expectedPopDown, poppedBlock)
+		}
+
+		uHeap := newUpHeap()
+		for _, block := range test.toPush {
+			uHeap.Push(block)
+		}
+
+		poppedBlock = nil
+		if test.expectedPopUp != nil {
+			poppedBlock = uHeap.pop()
+		}
+		if uHeap.Len() != test.expectedLength {
+			t.Errorf("unexpected up heap length in test \"%s\". "+
+				"Expected: %v, got: %v", test.name, test.expectedLength, uHeap.Len())
+		}
+		if poppedBlock != test.expectedPopUp {
+			t.Errorf("unexpected popped block for up heap in test \"%s\". "+
+				"Expected: %v, got: %v", test.name, test.expectedPopDown, poppedBlock)
+		}
+	}
+}

blockdag/blockindex.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2015-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dbaccess"
+	"sync"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// blockIndex provides facilities for keeping track of an in-memory index of the
+// block DAG.
+type blockIndex struct {
+	// The following fields are set when the instance is created and can't
+	// be changed afterwards, so there is no need to protect them with a
+	// separate mutex.
+	dagParams *dagconfig.Params
+
+	sync.RWMutex
+	index map[daghash.Hash]*blockNode
+	dirty map[*blockNode]struct{}
+}
+
+// newBlockIndex returns a new empty instance of a block index. The index will
+// be dynamically populated as block nodes are loaded from the database and
+// manually added.
+func newBlockIndex(dagParams *dagconfig.Params) *blockIndex {
+	return &blockIndex{
+		dagParams: dagParams,
+		index:     make(map[daghash.Hash]*blockNode),
+		dirty:     make(map[*blockNode]struct{}),
+	}
+}
+
+// HaveBlock returns whether or not the block index contains the provided hash.
+//
+// This function is safe for concurrent access.
+func (bi *blockIndex) HaveBlock(hash *daghash.Hash) bool {
+	bi.RLock()
+	defer bi.RUnlock()
+	_, hasBlock := bi.index[*hash]
+	return hasBlock
+}
+
+// LookupNode returns the block node identified by the provided hash. It will
+// return nil if there is no entry for the hash.
+//
+// This function is safe for concurrent access.
+func (bi *blockIndex) LookupNode(hash *daghash.Hash) (*blockNode, bool) {
+	bi.RLock()
+	defer bi.RUnlock()
+	node, ok := bi.index[*hash]
+	return node, ok
+}
+
+// AddNode adds the provided node to the block index and marks it as dirty.
+// Duplicate entries are not checked so it is up to caller to avoid adding them.
+//
+// This function is safe for concurrent access.
+func (bi *blockIndex) AddNode(node *blockNode) {
+	bi.Lock()
+	defer bi.Unlock()
+	bi.addNode(node)
+	bi.dirty[node] = struct{}{}
+}
+
+// addNode adds the provided node to the block index, but does not mark it as
+// dirty. This can be used while initializing the block index.
+//
+// This function is NOT safe for concurrent access.
+func (bi *blockIndex) addNode(node *blockNode) {
+	bi.index[*node.hash] = node
+}
+
+// NodeStatus provides concurrent-safe access to the status field of a node.
+//
+// This function is safe for concurrent access.
+func (bi *blockIndex) NodeStatus(node *blockNode) blockStatus {
+	bi.RLock()
+	defer bi.RUnlock()
+	status := node.status
+	return status
+}
+
+// SetStatusFlags flips the provided status flags on the block node to on,
+// regardless of whether they were on or off previously. This does not unset any
+// flags currently on.
+//
+// This function is safe for concurrent access.
+func (bi *blockIndex) SetStatusFlags(node *blockNode, flags blockStatus) {
+	bi.Lock()
+	defer bi.Unlock()
+	node.status |= flags
+	bi.dirty[node] = struct{}{}
+}
+
+// UnsetStatusFlags flips the provided status flags on the block node to off,
+// regardless of whether they were on or off previously.
+//
+// This function is safe for concurrent access.
+func (bi *blockIndex) UnsetStatusFlags(node *blockNode, flags blockStatus) {
+	bi.Lock()
+	defer bi.Unlock()
+	node.status &^= flags
+	bi.dirty[node] = struct{}{}
+}
+
+// flushToDB writes all dirty block nodes to the database.
+func (bi *blockIndex) flushToDB(dbContext *dbaccess.TxContext) error {
+	bi.Lock()
+	defer bi.Unlock()
+	if len(bi.dirty) == 0 {
+		return nil
+	}
+
+	for node := range bi.dirty {
+		serializedBlockNode, err := serializeBlockNode(node)
+		if err != nil {
+			return err
+		}
+		key := blockIndexKey(node.hash, node.blueScore)
+		err = dbaccess.StoreIndexBlock(dbContext, key, serializedBlockNode)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (bi *blockIndex) clearDirtyEntries() {
+	bi.dirty = make(map[*blockNode]struct{})
+}

blockdag/blockindex_test.go

@@ -0,0 +1,26 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"testing"
+)
+
+func TestAncestorErrors(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimnetParams
+	dag, teardownFunc, err := DAGSetup("TestAncestorErrors", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestAncestorErrors: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	node := newTestNode(dag, newBlockSet(), int32(0x10000000), 0, mstime.Now())
+	node.blueScore = 2
+	ancestor := node.SelectedAncestor(3)
+	if ancestor != nil {
+		t.Errorf("TestAncestorErrors: Ancestor() unexpectedly returned a node. Expected: <nil>")
+	}
+}

blockdag/blocklocator.go

@@ -0,0 +1,109 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+)
+
+// BlockLocator is used to help locate a specific block. The algorithm for
+// building the block locator is to add block hashes in reverse order on the
+// block's selected parent chain until the desired stop block is reached.
+// In order to keep the list of locator hashes to a reasonable number of entries,
+// the step between each entry is doubled each loop iteration to exponentially
+// decrease the number of hashes as a function of the distance from the block
+// being located.
+//
+// For example, assume a selected parent chain with IDs as depicted below, and the
+// stop block is genesis:
+// 	genesis -> 1 -> 2 -> ... -> 15 -> 16  -> 17  -> 18
+//
+// The block locator for block 17 would be the hashes of blocks:
+//  [17 16 14 11 7 2 genesis]
+type BlockLocator []*daghash.Hash
+
+// BlockLocatorFromHashes returns a block locator from high and low hash.
+// See BlockLocator for details on the algorithm used to create a block locator.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) BlockLocatorFromHashes(highHash, lowHash *daghash.Hash) (BlockLocator, error) {
+	dag.dagLock.RLock()
+	defer dag.dagLock.RUnlock()
+
+	highNode, ok := dag.index.LookupNode(highHash)
+	if !ok {
+		return nil, errors.Errorf("block %s is unknown", highHash)
+	}
+
+	lowNode, ok := dag.index.LookupNode(lowHash)
+	if !ok {
+		return nil, errors.Errorf("block %s is unknown", lowHash)
+	}
+
+	return dag.blockLocator(highNode, lowNode)
+}
+
+// blockLocator returns a block locator for the passed high and low nodes.
+// See the BlockLocator type comments for more details.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) blockLocator(highNode, lowNode *blockNode) (BlockLocator, error) {
+	// We use the selected parent of the high node, so the
+	// block locator won't contain the high node.
+	highNode = highNode.selectedParent
+
+	node := highNode
+	step := uint64(1)
+	locator := make(BlockLocator, 0)
+	for node != nil {
+		locator = append(locator, node.hash)
+
+		// Nothing more to add once the low node has been added.
+		if node.blueScore <= lowNode.blueScore {
+			if node != lowNode {
+				return nil, errors.Errorf("highNode and lowNode are " +
+					"not in the same selected parent chain.")
+			}
+			break
+		}
+
+		// Calculate blueScore of previous node to include ensuring the
+		// final node is lowNode.
+		nextBlueScore := node.blueScore - step
+		if nextBlueScore < lowNode.blueScore {
+			nextBlueScore = lowNode.blueScore
+		}
+
+		// walk backwards through the nodes to the correct ancestor.
+		node = node.SelectedAncestor(nextBlueScore)
+
+		// Double the distance between included hashes.
+		step *= 2
+	}
+
+	return locator, nil
+}
+
+// FindNextLocatorBoundaries returns the lowest unknown block locator, hash
+// and the highest known block locator hash. This is used to create the
+// next block locator to find the highest shared known chain block with the
+// sync peer.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) FindNextLocatorBoundaries(locator BlockLocator) (highHash, lowHash *daghash.Hash) {
+	// Find the most recent locator block hash in the DAG. In the case none of
+	// the hashes in the locator are in the DAG, fall back to the genesis block.
+	lowNode := dag.genesis
+	nextBlockLocatorIndex := int64(len(locator) - 1)
+	for i, hash := range locator {
+		node, ok := dag.index.LookupNode(hash)
+		if ok {
+			lowNode = node
+			nextBlockLocatorIndex = int64(i) - 1
+			break
+		}
+	}
+	if nextBlockLocatorIndex < 0 {
+		return nil, lowNode.hash
+	}
+	return locator[nextBlockLocatorIndex], lowNode.hash
+}

blockdag/blocknode.go

@@ -0,0 +1,237 @@
+// Copyright (c) 2015-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// blockStatus is a bit field representing the validation state of the block.
+type blockStatus byte
+
+const (
+	// statusDataStored indicates that the block's payload is stored on disk.
+	statusDataStored blockStatus = 1 << iota
+
+	// statusValid indicates that the block has been fully validated.
+	statusValid
+
+	// statusValidateFailed indicates that the block has failed validation.
+	statusValidateFailed
+
+	// statusInvalidAncestor indicates that one of the block's ancestors has
+	// has failed validation, thus the block is also invalid.
+	statusInvalidAncestor
+)
+
+// KnownValid returns whether the block is known to be valid. This will return
+// false for a valid block that has not been fully validated yet.
+func (status blockStatus) KnownValid() bool {
+	return status&statusValid != 0
+}
+
+// KnownInvalid returns whether the block is known to be invalid. This may be
+// because the block itself failed validation or any of its ancestors is
+// invalid. This will return false for invalid blocks that have not been proven
+// invalid yet.
+func (status blockStatus) KnownInvalid() bool {
+	return status&(statusValidateFailed|statusInvalidAncestor) != 0
+}
+
+// blockNode represents a block within the block DAG. The DAG is stored into
+// the block database.
+type blockNode struct {
+	// NOTE: Additions, deletions, or modifications to the order of the
+	// definitions in this struct should not be changed without considering
+	// how it affects alignment on 64-bit platforms. The current order is
+	// specifically crafted to result in minimal padding. There will be
+	// hundreds of thousands of these in memory, so a few extra bytes of
+	// padding adds up.
+
+	// parents is the parent blocks for this node.
+	parents blockSet
+
+	// selectedParent is the selected parent for this node.
+	// The selected parent is the parent that if chosen will maximize the blue score of this block
+	selectedParent *blockNode
+
+	// children are all the blocks that refer to this block as a parent
+	children blockSet
+
+	// blues are all blue blocks in this block's worldview that are in its selected parent anticone
+	blues []*blockNode
+
+	// blueScore is the count of all the blue blocks in this block's past
+	blueScore uint64
+
+	// bluesAnticoneSizes is a map holding the set of blues affected by this block and their
+	// modified blue anticone size.
+	bluesAnticoneSizes map[*blockNode]dagconfig.KType
+
+	// hash is the double sha 256 of the block.
+	hash *daghash.Hash
+
+	// Some fields from block headers to aid in  reconstructing headers
+	// from memory. These must be treated as immutable and are intentionally
+	// ordered to avoid padding on 64-bit platforms.
+	version              int32
+	bits                 uint32
+	nonce                uint64
+	timestamp            int64
+	hashMerkleRoot       *daghash.Hash
+	acceptedIDMerkleRoot *daghash.Hash
+	utxoCommitment       *daghash.Hash
+
+	// status is a bitfield representing the validation state of the block. The
+	// status field, unlike the other fields, may be written to and so should
+	// only be accessed using the concurrent-safe NodeStatus method on
+	// blockIndex once the node has been added to the global index.
+	status blockStatus
+
+	// isFinalized determines whether the node is below the finality point.
+	isFinalized bool
+}
+
+// newBlockNode returns a new block node for the given block header and parents, and the
+// anticone of its selected parent (parent with highest blue score).
+// selectedParentAnticone is used to update reachability data we store for future reachability queries.
+// This function is NOT safe for concurrent access.
+func (dag *BlockDAG) newBlockNode(blockHeader *domainmessage.BlockHeader, parents blockSet) (node *blockNode, selectedParentAnticone []*blockNode) {
+	node = &blockNode{
+		parents:            parents,
+		children:           make(blockSet),
+		blueScore:          math.MaxUint64, // Initialized to the max value to avoid collisions with the genesis block
+		timestamp:          dag.Now().UnixMilliseconds(),
+		bluesAnticoneSizes: make(map[*blockNode]dagconfig.KType),
+	}
+
+	// blockHeader is nil only for the virtual block
+	if blockHeader != nil {
+		node.hash = blockHeader.BlockHash()
+		node.version = blockHeader.Version
+		node.bits = blockHeader.Bits
+		node.nonce = blockHeader.Nonce
+		node.timestamp = blockHeader.Timestamp.UnixMilliseconds()
+		node.hashMerkleRoot = blockHeader.HashMerkleRoot
+		node.acceptedIDMerkleRoot = blockHeader.AcceptedIDMerkleRoot
+		node.utxoCommitment = blockHeader.UTXOCommitment
+	} else {
+		node.hash = &daghash.ZeroHash
+	}
+
+	if len(parents) == 0 {
+		// The genesis block is defined to have a blueScore of 0
+		node.blueScore = 0
+		return node, nil
+	}
+
+	selectedParentAnticone, err := dag.ghostdag(node)
+	if err != nil {
+		panic(errors.Wrap(err, "unexpected error in GHOSTDAG"))
+	}
+	return node, selectedParentAnticone
+}
+
+// updateParentsChildren updates the node's parents to point to new node
+func (node *blockNode) updateParentsChildren() {
+	for parent := range node.parents {
+		parent.children.add(node)
+	}
+}
+
+func (node *blockNode) less(other *blockNode) bool {
+	if node.blueScore == other.blueScore {
+		return daghash.Less(node.hash, other.hash)
+	}
+
+	return node.blueScore < other.blueScore
+}
+
+// Header constructs a block header from the node and returns it.
+//
+// This function is safe for concurrent access.
+func (node *blockNode) Header() *domainmessage.BlockHeader {
+	// No lock is needed because all accessed fields are immutable.
+	return &domainmessage.BlockHeader{
+		Version:              node.version,
+		ParentHashes:         node.ParentHashes(),
+		HashMerkleRoot:       node.hashMerkleRoot,
+		AcceptedIDMerkleRoot: node.acceptedIDMerkleRoot,
+		UTXOCommitment:       node.utxoCommitment,
+		Timestamp:            node.time(),
+		Bits:                 node.bits,
+		Nonce:                node.nonce,
+	}
+}
+
+// SelectedAncestor returns the ancestor block node at the provided blue score by following
+// the selected-parents chain backwards from this node. The returned block will be nil when a
+// blue score is requested that is higher than the blue score of the passed node.
+//
+// This function is safe for concurrent access.
+func (node *blockNode) SelectedAncestor(blueScore uint64) *blockNode {
+	if blueScore > node.blueScore {
+		return nil
+	}
+
+	n := node
+	for n != nil && n.blueScore > blueScore {
+		n = n.selectedParent
+	}
+
+	return n
+}
+
+// RelativeAncestor returns the ancestor block node a relative 'distance' of
+// blue blocks before this node. This is equivalent to calling Ancestor with
+// the node's blue score minus provided distance.
+//
+// This function is safe for concurrent access.
+func (node *blockNode) RelativeAncestor(distance uint64) *blockNode {
+	return node.SelectedAncestor(node.blueScore - distance)
+}
+
+// CalcPastMedianTime returns the median time of the previous few blocks
+// prior to, and including, the block node.
+//
+// This function is safe for concurrent access.
+func (node *blockNode) PastMedianTime(dag *BlockDAG) mstime.Time {
+	window := blueBlockWindow(node, 2*dag.TimestampDeviationTolerance-1)
+	medianTimestamp, err := window.medianTimestamp()
+	if err != nil {
+		panic(fmt.Sprintf("blueBlockWindow: %s", err))
+	}
+	return mstime.UnixMilliseconds(medianTimestamp)
+}
+
+func (node *blockNode) ParentHashes() []*daghash.Hash {
+	return node.parents.hashes()
+}
+
+// isGenesis returns if the current block is the genesis block
+func (node *blockNode) isGenesis() bool {
+	return len(node.parents) == 0
+}
+
+func (node *blockNode) finalityScore(dag *BlockDAG) uint64 {
+	return node.blueScore / uint64(dag.FinalityInterval())
+}
+
+// String returns a string that contains the block hash.
+func (node blockNode) String() string {
+	return node.hash.String()
+}
+
+func (node *blockNode) time() mstime.Time {
+	return mstime.UnixMilliseconds(node.timestamp)
+}

blockdag/blocknode_test.go

@@ -0,0 +1,41 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"testing"
+)
+
+// This test is to ensure the size BlueAnticoneSizesSize is serialized to the size of KType.
+// We verify that by serializing and deserializing the block while making sure that we stay within the expected range.
+func TestBlueAnticoneSizesSize(t *testing.T) {
+	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizesSize", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestBlueAnticoneSizesSize: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	k := dagconfig.KType(0)
+	k--
+
+	if k < dagconfig.KType(0) {
+		t.Fatalf("KType must be unsigned")
+	}
+
+	blockHeader := dagconfig.SimnetParams.GenesisBlock.Header
+	node, _ := dag.newBlockNode(&blockHeader, newBlockSet())
+	fakeBlue := &blockNode{hash: &daghash.Hash{1}}
+	dag.index.AddNode(fakeBlue)
+	// Setting maxKType to maximum value of KType.
+	// As we verify above that KType is unsigned we can be sure that maxKType is indeed the maximum value of KType.
+	maxKType := ^dagconfig.KType(0)
+	node.bluesAnticoneSizes[fakeBlue] = maxKType
+	serializedNode, _ := serializeBlockNode(node)
+	deserializedNode, _ := dag.deserializeBlockNode(serializedNode)
+	if deserializedNode.bluesAnticoneSizes[fakeBlue] != maxKType {
+		t.Fatalf("TestBlueAnticoneSizesSize: BlueAnticoneSize should not change when deserializing. Expected: %v but got %v",
+			maxKType, deserializedNode.bluesAnticoneSizes[fakeBlue])
+	}
+}

blockdag/blockset.go

@@ -0,0 +1,117 @@
+package blockdag
+
+import (
+	"strings"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// blockSet implements a basic unsorted set of blocks
+type blockSet map[*blockNode]struct{}
+
+// newBlockSet creates a new, empty BlockSet
+func newBlockSet() blockSet {
+	return map[*blockNode]struct{}{}
+}
+
+// blockSetFromSlice converts a slice of blockNodes into an unordered set represented as map
+func blockSetFromSlice(nodes ...*blockNode) blockSet {
+	set := newBlockSet()
+	for _, node := range nodes {
+		set.add(node)
+	}
+	return set
+}
+
+// add adds a blockNode to this BlockSet
+func (bs blockSet) add(node *blockNode) {
+	bs[node] = struct{}{}
+}
+
+// remove removes a blockNode from this BlockSet, if exists
+// Does nothing if this set does not contain the blockNode
+func (bs blockSet) remove(node *blockNode) {
+	delete(bs, node)
+}
+
+// clone clones thie block set
+func (bs blockSet) clone() blockSet {
+	clone := newBlockSet()
+	for node := range bs {
+		clone.add(node)
+	}
+	return clone
+}
+
+// subtract returns the difference between the BlockSet and another BlockSet
+func (bs blockSet) subtract(other blockSet) blockSet {
+	diff := newBlockSet()
+	for node := range bs {
+		if !other.contains(node) {
+			diff.add(node)
+		}
+	}
+	return diff
+}
+
+// addSet adds all blockNodes in other set to this set
+func (bs blockSet) addSet(other blockSet) {
+	for node := range other {
+		bs.add(node)
+	}
+}
+
+// addSlice adds provided slice to this set
+func (bs blockSet) addSlice(slice []*blockNode) {
+	for _, node := range slice {
+		bs.add(node)
+	}
+}
+
+// union returns a BlockSet that contains all blockNodes included in this set,
+// the other set, or both
+func (bs blockSet) union(other blockSet) blockSet {
+	union := bs.clone()
+
+	union.addSet(other)
+
+	return union
+}
+
+// contains returns true iff this set contains node
+func (bs blockSet) contains(node *blockNode) bool {
+	_, ok := bs[node]
+	return ok
+}
+
+// hashes returns the hashes of the blockNodes in this set.
+func (bs blockSet) hashes() []*daghash.Hash {
+	hashes := make([]*daghash.Hash, 0, len(bs))
+	for node := range bs {
+		hashes = append(hashes, node.hash)
+	}
+	daghash.Sort(hashes)
+	return hashes
+}
+
+func (bs blockSet) String() string {
+	nodeStrs := make([]string, 0, len(bs))
+	for node := range bs {
+		nodeStrs = append(nodeStrs, node.String())
+	}
+	return strings.Join(nodeStrs, ",")
+}
+
+func (bs blockSet) bluest() *blockNode {
+	var bluestNode *blockNode
+	var maxScore uint64
+	for node := range bs {
+		if bluestNode == nil ||
+			node.blueScore > maxScore ||
+			(node.blueScore == maxScore && daghash.Less(node.hash, bluestNode.hash)) {
+			bluestNode = node
+			maxScore = node.blueScore
+		}
+	}
+	return bluestNode
+}

blockdag/blockset_test.go

@@ -0,0 +1,245 @@
+package blockdag
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+func TestHashes(t *testing.T) {
+	bs := blockSetFromSlice(
+		&blockNode{
+			hash: &daghash.Hash{3},
+		},
+		&blockNode{
+			hash: &daghash.Hash{1},
+		},
+		&blockNode{
+			hash: &daghash.Hash{0},
+		},
+		&blockNode{
+			hash: &daghash.Hash{2},
+		},
+	)
+
+	expected := []*daghash.Hash{
+		{0},
+		{1},
+		{2},
+		{3},
+	}
+
+	hashes := bs.hashes()
+	if !daghash.AreEqual(hashes, expected) {
+		t.Errorf("TestHashes: hashes order is %s but expected %s", hashes, expected)
+	}
+}
+
+func TestBlockSetSubtract(t *testing.T) {
+	node1 := &blockNode{hash: &daghash.Hash{10}}
+	node2 := &blockNode{hash: &daghash.Hash{20}}
+	node3 := &blockNode{hash: &daghash.Hash{30}}
+
+	tests := []struct {
+		name           string
+		setA           blockSet
+		setB           blockSet
+		expectedResult blockSet
+	}{
+		{
+			name:           "both sets empty",
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(),
+		},
+		{
+			name:           "subtract an empty set",
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "subtract from empty set",
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(),
+		},
+		{
+			name:           "subtract unrelated set",
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(node2),
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "typical case",
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node2, node3),
+			expectedResult: blockSetFromSlice(node1),
+		},
+	}
+
+	for _, test := range tests {
+		result := test.setA.subtract(test.setB)
+		if !reflect.DeepEqual(result, test.expectedResult) {
+			t.Errorf("blockSet.subtract: unexpected result in test '%s'. "+
+				"Expected: %v, got: %v", test.name, test.expectedResult, result)
+		}
+	}
+}
+
+func TestBlockSetAddSet(t *testing.T) {
+	node1 := &blockNode{hash: &daghash.Hash{10}}
+	node2 := &blockNode{hash: &daghash.Hash{20}}
+	node3 := &blockNode{hash: &daghash.Hash{30}}
+
+	tests := []struct {
+		name           string
+		setA           blockSet
+		setB           blockSet
+		expectedResult blockSet
+	}{
+		{
+			name:           "both sets empty",
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(),
+		},
+		{
+			name:           "add an empty set",
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "add to empty set",
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "add already added member",
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1, node2),
+		},
+		{
+			name:           "typical case",
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node2, node3),
+			expectedResult: blockSetFromSlice(node1, node2, node3),
+		},
+	}
+
+	for _, test := range tests {
+		test.setA.addSet(test.setB)
+		if !reflect.DeepEqual(test.setA, test.expectedResult) {
+			t.Errorf("blockSet.addSet: unexpected result in test '%s'. "+
+				"Expected: %v, got: %v", test.name, test.expectedResult, test.setA)
+		}
+	}
+}
+
+func TestBlockSetAddSlice(t *testing.T) {
+	node1 := &blockNode{hash: &daghash.Hash{10}}
+	node2 := &blockNode{hash: &daghash.Hash{20}}
+	node3 := &blockNode{hash: &daghash.Hash{30}}
+
+	tests := []struct {
+		name           string
+		set            blockSet
+		slice          []*blockNode
+		expectedResult blockSet
+	}{
+		{
+			name:           "add empty slice to empty set",
+			set:            blockSetFromSlice(),
+			slice:          []*blockNode{},
+			expectedResult: blockSetFromSlice(),
+		},
+		{
+			name:           "add an empty slice",
+			set:            blockSetFromSlice(node1),
+			slice:          []*blockNode{},
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "add to empty set",
+			set:            blockSetFromSlice(),
+			slice:          []*blockNode{node1},
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "add already added member",
+			set:            blockSetFromSlice(node1, node2),
+			slice:          []*blockNode{node1},
+			expectedResult: blockSetFromSlice(node1, node2),
+		},
+		{
+			name:           "typical case",
+			set:            blockSetFromSlice(node1, node2),
+			slice:          []*blockNode{node2, node3},
+			expectedResult: blockSetFromSlice(node1, node2, node3),
+		},
+	}
+
+	for _, test := range tests {
+		test.set.addSlice(test.slice)
+		if !reflect.DeepEqual(test.set, test.expectedResult) {
+			t.Errorf("blockSet.addSlice: unexpected result in test '%s'. "+
+				"Expected: %v, got: %v", test.name, test.expectedResult, test.set)
+		}
+	}
+}
+
+func TestBlockSetUnion(t *testing.T) {
+	node1 := &blockNode{hash: &daghash.Hash{10}}
+	node2 := &blockNode{hash: &daghash.Hash{20}}
+	node3 := &blockNode{hash: &daghash.Hash{30}}
+
+	tests := []struct {
+		name           string
+		setA           blockSet
+		setB           blockSet
+		expectedResult blockSet
+	}{
+		{
+			name:           "both sets empty",
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(),
+		},
+		{
+			name:           "union against an empty set",
+			setA:           blockSetFromSlice(node1),
+			setB:           blockSetFromSlice(),
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "union from an empty set",
+			setA:           blockSetFromSlice(),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1),
+		},
+		{
+			name:           "union with subset",
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node1),
+			expectedResult: blockSetFromSlice(node1, node2),
+		},
+		{
+			name:           "typical case",
+			setA:           blockSetFromSlice(node1, node2),
+			setB:           blockSetFromSlice(node2, node3),
+			expectedResult: blockSetFromSlice(node1, node2, node3),
+		},
+	}
+
+	for _, test := range tests {
+		result := test.setA.union(test.setB)
+		if !reflect.DeepEqual(result, test.expectedResult) {
+			t.Errorf("blockSet.union: unexpected result in test '%s'. "+
+				"Expected: %v, got: %v", test.name, test.expectedResult, result)
+		}
+	}
+}

blockdag/blockwindow.go

@@ -0,0 +1,82 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/bigintpool"
+	"github.com/pkg/errors"
+	"math"
+	"math/big"
+	"sort"
+)
+
+type blockWindow []*blockNode
+
+// blueBlockWindow returns a blockWindow of the given size that contains the
+// blues in the past of startindNode, sorted by GHOSTDAG order.
+// If the number of blues in the past of startingNode is less then windowSize,
+// the window will be padded by genesis blocks to achieve a size of windowSize.
+func blueBlockWindow(startingNode *blockNode, windowSize uint64) blockWindow {
+	window := make(blockWindow, 0, windowSize)
+	currentNode := startingNode
+	for uint64(len(window)) < windowSize && currentNode.selectedParent != nil {
+		if currentNode.selectedParent != nil {
+			for _, blue := range currentNode.blues {
+				window = append(window, blue)
+				if uint64(len(window)) == windowSize {
+					break
+				}
+			}
+			currentNode = currentNode.selectedParent
+		}
+	}
+
+	if uint64(len(window)) < windowSize {
+		genesis := currentNode
+		for uint64(len(window)) < windowSize {
+			window = append(window, genesis)
+		}
+	}
+
+	return window
+}
+
+func (window blockWindow) minMaxTimestamps() (min, max int64) {
+	min = math.MaxInt64
+	max = 0
+	for _, node := range window {
+		if node.timestamp < min {
+			min = node.timestamp
+		}
+		if node.timestamp > max {
+			max = node.timestamp
+		}
+	}
+	return
+}
+
+func (window blockWindow) averageTarget(averageTarget *big.Int) {
+	averageTarget.SetInt64(0)
+
+	target := bigintpool.Acquire(0)
+	defer bigintpool.Release(target)
+	for _, node := range window {
+		util.CompactToBigWithDestination(node.bits, target)
+		averageTarget.Add(averageTarget, target)
+	}
+
+	windowLen := bigintpool.Acquire(int64(len(window)))
+	defer bigintpool.Release(windowLen)
+	averageTarget.Div(averageTarget, windowLen)
+}
+
+func (window blockWindow) medianTimestamp() (int64, error) {
+	if len(window) == 0 {
+		return 0, errors.New("Cannot calculate median timestamp for an empty block window")
+	}
+	timestamps := make([]int64, len(window))
+	for i, node := range window {
+		timestamps[i] = node.timestamp
+	}
+	sort.Sort(timeSorter(timestamps))
+	return timestamps[len(timestamps)/2], nil
+}

blockdag/blockwindow_test.go

@@ -0,0 +1,160 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestBlueBlockWindow(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.K = 1
+	dag, teardownFunc, err := DAGSetup("TestBlueBlockWindow", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	resetExtraNonceForTest()
+
+	windowSize := uint64(10)
+	genesisNode := dag.genesis
+	blockTime := genesisNode.Header().Timestamp
+	blockByIDMap := make(map[string]*blockNode)
+	idByBlockMap := make(map[*blockNode]string)
+	blockByIDMap["A"] = genesisNode
+	idByBlockMap[genesisNode] = "A"
+
+	blocksData := []*struct {
+		parents                          []string
+		id                               string //id is a virtual entity that is used only for tests so we can define relations between blocks without knowing their hash
+		expectedWindowWithGenesisPadding []string
+	}{
+		{
+			parents:                          []string{"A"},
+			id:                               "B",
+			expectedWindowWithGenesisPadding: []string{"A", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"B"},
+			id:                               "C",
+			expectedWindowWithGenesisPadding: []string{"B", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"B"},
+			id:                               "D",
+			expectedWindowWithGenesisPadding: []string{"B", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"D", "C"},
+			id:                               "E",
+			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"D", "C"},
+			id:                               "F",
+			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"A"},
+			id:                               "G",
+			expectedWindowWithGenesisPadding: []string{"A", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"G"},
+			id:                               "H",
+			expectedWindowWithGenesisPadding: []string{"G", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"H", "F"},
+			id:                               "I",
+			expectedWindowWithGenesisPadding: []string{"F", "D", "C", "B", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"I"},
+			id:                               "J",
+			expectedWindowWithGenesisPadding: []string{"I", "F", "D", "C", "B", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"J"},
+			id:                               "K",
+			expectedWindowWithGenesisPadding: []string{"J", "I", "F", "D", "C", "B", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"K"},
+			id:                               "L",
+			expectedWindowWithGenesisPadding: []string{"K", "J", "I", "F", "D", "C", "B", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"L"},
+			id:                               "M",
+			expectedWindowWithGenesisPadding: []string{"L", "K", "J", "I", "F", "D", "C", "B", "A", "A"},
+		},
+		{
+			parents:                          []string{"M"},
+			id:                               "N",
+			expectedWindowWithGenesisPadding: []string{"M", "L", "K", "J", "I", "F", "D", "C", "B", "A"},
+		},
+		{
+			parents:                          []string{"N"},
+			id:                               "O",
+			expectedWindowWithGenesisPadding: []string{"N", "M", "L", "K", "J", "I", "F", "D", "C", "B"},
+		},
+	}
+
+	for _, blockData := range blocksData {
+		blockTime = blockTime.Add(time.Second)
+		parents := blockSet{}
+		for _, parentID := range blockData.parents {
+			parent := blockByIDMap[parentID]
+			parents.add(parent)
+		}
+
+		block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+		if err != nil {
+			t.Fatalf("block %v got unexpected error from PrepareBlockForTest: %v", blockData.id, err)
+		}
+
+		utilBlock := util.NewBlock(block)
+		isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+		if err != nil {
+			t.Fatalf("dag.ProcessBlock got unexpected error for block %v: %v", blockData.id, err)
+		}
+		if isDelayed {
+			t.Fatalf("block %s "+
+				"is too far in the future", blockData.id)
+		}
+		if isOrphan {
+			t.Fatalf("block %v was unexpectedly orphan", blockData.id)
+		}
+
+		node, ok := dag.index.LookupNode(utilBlock.Hash())
+		if !ok {
+			t.Fatalf("block %s does not exist in the DAG", utilBlock.Hash())
+		}
+
+		blockByIDMap[blockData.id] = node
+		idByBlockMap[node] = blockData.id
+
+		window := blueBlockWindow(node, windowSize)
+		if err := checkWindowIDs(window, blockData.expectedWindowWithGenesisPadding, idByBlockMap); err != nil {
+			t.Errorf("Unexpected values for window for block %s: %s", blockData.id, err)
+		}
+	}
+}
+
+func checkWindowIDs(window []*blockNode, expectedIDs []string, idByBlockMap map[*blockNode]string) error {
+	ids := make([]string, len(window))
+	for i, node := range window {
+		ids[i] = idByBlockMap[node]
+	}
+	if !reflect.DeepEqual(ids, expectedIDs) {
+		return errors.Errorf("window expected to have blocks %s but got %s", expectedIDs, ids)
+	}
+	return nil
+}

blockdag/coinbase.go

@@ -0,0 +1,199 @@
+package blockdag
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"io"
+
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/coinbasepayload"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/txsort"
+	"github.com/pkg/errors"
+)
+
+// compactFeeData is a specialized data type to store a compact list of fees
+// inside a block.
+// Every transaction gets a single uint64 value, stored as a plain binary list.
+// The transactions are ordered the same way they are ordered inside the block, making it easy
+// to traverse every transaction in a block and extract its fee.
+//
+// compactFeeFactory is used to create such a list.
+// compactFeeIterator is used to iterate over such a list.
+
+type compactFeeData []byte
+
+func (cfd compactFeeData) Len() int {
+	return len(cfd) / 8
+}
+
+type compactFeeFactory struct {
+	buffer *bytes.Buffer
+	writer *bufio.Writer
+}
+
+func newCompactFeeFactory() *compactFeeFactory {
+	buffer := bytes.NewBuffer([]byte{})
+	return &compactFeeFactory{
+		buffer: buffer,
+		writer: bufio.NewWriter(buffer),
+	}
+}
+
+func (cfw *compactFeeFactory) add(txFee uint64) error {
+	return binary.Write(cfw.writer, binary.LittleEndian, txFee)
+}
+
+func (cfw *compactFeeFactory) data() (compactFeeData, error) {
+	err := cfw.writer.Flush()
+
+	return compactFeeData(cfw.buffer.Bytes()), err
+}
+
+type compactFeeIterator struct {
+	reader io.Reader
+}
+
+func (cfd compactFeeData) iterator() *compactFeeIterator {
+	return &compactFeeIterator{
+		reader: bufio.NewReader(bytes.NewBuffer(cfd)),
+	}
+}
+
+func (cfr *compactFeeIterator) next() (uint64, error) {
+	var txFee uint64
+
+	err := binary.Read(cfr.reader, binary.LittleEndian, &txFee)
+
+	return txFee, err
+}
+
+// The following functions relate to storing and retrieving fee data from the database
+
+// getBluesFeeData returns the compactFeeData for all nodes's blues,
+// used to calculate the fees this blockNode needs to pay
+func (dag *BlockDAG) getBluesFeeData(node *blockNode) (map[daghash.Hash]compactFeeData, error) {
+	bluesFeeData := make(map[daghash.Hash]compactFeeData)
+
+	for _, blueBlock := range node.blues {
+		feeData, err := dbaccess.FetchFeeData(dag.databaseContext, blueBlock.hash)
+		if err != nil {
+			return nil, err
+		}
+
+		bluesFeeData[*blueBlock.hash] = feeData
+	}
+
+	return bluesFeeData, nil
+}
+
+// The following functions deal with building and validating the coinbase transaction
+
+func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Block, txsAcceptanceData MultiBlockTxsAcceptanceData) error {
+	if node.isGenesis() {
+		return nil
+	}
+	blockCoinbaseTx := block.CoinbaseTransaction().MsgTx()
+	_, scriptPubKey, extraData, err := coinbasepayload.DeserializeCoinbasePayload(blockCoinbaseTx)
+	if errors.Is(err, coinbasepayload.ErrIncorrectScriptPubKeyLen) {
+		return ruleError(ErrBadCoinbaseTransaction, err.Error())
+	}
+	if err != nil {
+		return err
+	}
+	expectedCoinbaseTransaction, err := node.expectedCoinbaseTransaction(dag, txsAcceptanceData, scriptPubKey, extraData)
+	if err != nil {
+		return err
+	}
+
+	if !expectedCoinbaseTransaction.Hash().IsEqual(block.CoinbaseTransaction().Hash()) {
+		return ruleError(ErrBadCoinbaseTransaction, "Coinbase transaction is not built as expected")
+	}
+
+	return nil
+}
+
+// expectedCoinbaseTransaction returns the coinbase transaction for the current block
+func (node *blockNode) expectedCoinbaseTransaction(dag *BlockDAG, txsAcceptanceData MultiBlockTxsAcceptanceData, scriptPubKey []byte, extraData []byte) (*util.Tx, error) {
+	bluesFeeData, err := dag.getBluesFeeData(node)
+	if err != nil {
+		return nil, err
+	}
+
+	txIns := []*domainmessage.TxIn{}
+	txOuts := []*domainmessage.TxOut{}
+
+	for _, blue := range node.blues {
+		txOut, err := coinbaseOutputForBlueBlock(dag, blue, txsAcceptanceData, bluesFeeData)
+		if err != nil {
+			return nil, err
+		}
+		if txOut != nil {
+			txOuts = append(txOuts, txOut)
+		}
+	}
+	payload, err := coinbasepayload.SerializeCoinbasePayload(node.blueScore, scriptPubKey, extraData)
+	if err != nil {
+		return nil, err
+	}
+	coinbaseTx := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, txIns, txOuts, subnetworkid.SubnetworkIDCoinbase, 0, payload)
+	sortedCoinbaseTx := txsort.Sort(coinbaseTx)
+	return util.NewTx(sortedCoinbaseTx), nil
+}
+
+// coinbaseOutputForBlueBlock calculates the output that should go into the coinbase transaction of blueBlock
+// If blueBlock gets no fee - returns nil for txOut
+func coinbaseOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
+	txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (*domainmessage.TxOut, error) {
+
+	blockTxsAcceptanceData, ok := txsAcceptanceData.FindAcceptanceData(blueBlock.hash)
+	if !ok {
+		return nil, errors.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
+	}
+	blockFeeData, ok := feeData[*blueBlock.hash]
+	if !ok {
+		return nil, errors.Errorf("No feeData for block %s", blueBlock.hash)
+	}
+
+	if len(blockTxsAcceptanceData.TxAcceptanceData) != blockFeeData.Len() {
+		return nil, errors.Errorf(
+			"length of accepted transaction data(%d) and fee data(%d) is not equal for block %s",
+			len(blockTxsAcceptanceData.TxAcceptanceData), blockFeeData.Len(), blueBlock.hash)
+	}
+
+	totalFees := uint64(0)
+	feeIterator := blockFeeData.iterator()
+
+	for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
+		fee, err := feeIterator.next()
+		if err != nil {
+			return nil, errors.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
+		}
+		if txAcceptanceData.IsAccepted {
+			totalFees += fee
+		}
+	}
+
+	totalReward := CalcBlockSubsidy(blueBlock.blueScore, dag.Params) + totalFees
+
+	if totalReward == 0 {
+		return nil, nil
+	}
+
+	// the ScriptPubKey for the coinbase is parsed from the coinbase payload
+	_, scriptPubKey, _, err := coinbasepayload.DeserializeCoinbasePayload(blockTxsAcceptanceData.TxAcceptanceData[0].Tx.MsgTx())
+	if err != nil {
+		return nil, err
+	}
+
+	txOut := &domainmessage.TxOut{
+		Value:        totalReward,
+		ScriptPubKey: scriptPubKey,
+	}
+
+	return txOut, nil
+}

blockdag/coinbase_test.go

@@ -0,0 +1,60 @@
+package blockdag
+
+import (
+	"io"
+	"reflect"
+	"testing"
+)
+
+func TestFeeAccumulators(t *testing.T) {
+	fees := []uint64{1, 2, 3, 4, 5, 6, 7, 0xffffffffffffffff}
+
+	factory := newCompactFeeFactory()
+
+	for _, fee := range fees {
+		err := factory.add(fee)
+		if err != nil {
+			t.Fatalf("Error writing %d as tx fee: %s", fee, err)
+		}
+	}
+
+	expectedData := compactFeeData{
+		1, 0, 0, 0, 0, 0, 0, 0,
+		2, 0, 0, 0, 0, 0, 0, 0,
+		3, 0, 0, 0, 0, 0, 0, 0,
+		4, 0, 0, 0, 0, 0, 0, 0,
+		5, 0, 0, 0, 0, 0, 0, 0,
+		6, 0, 0, 0, 0, 0, 0, 0,
+		7, 0, 0, 0, 0, 0, 0, 0,
+		255, 255, 255, 255, 255, 255, 255, 255,
+	}
+	actualData, err := factory.data()
+
+	if err != nil {
+		t.Fatalf("Error getting bytes from writer: %s", err)
+	}
+	if !reflect.DeepEqual(expectedData, actualData) {
+		t.Errorf("Expected bytes: %v, but got: %v", expectedData, actualData)
+	}
+
+	iterator := actualData.iterator()
+
+	for i, expectedFee := range fees {
+		actualFee, err := iterator.next()
+		if err != nil {
+			t.Fatalf("Error getting fee for Tx#%d: %s", i, err)
+		}
+
+		if actualFee != expectedFee {
+			t.Errorf("Tx #%d: Expected fee: %d, but got %d", i, expectedFee, actualFee)
+		}
+	}
+
+	_, err = iterator.next()
+	if err == nil {
+		t.Fatal("No error from iterator.nextTxFee after done reading all transactions")
+	}
+	if err != io.EOF {
+		t.Fatalf("Error from iterator.nextTxFee after done reading all transactions is not io.EOF: %s", err)
+	}
+}

blockdag/common_test.go

@@ -0,0 +1,197 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"compress/bzip2"
+	"encoding/binary"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/kaspanet/kaspad/util/mstime"
+
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// loadUTXOSet returns a utxo view loaded from a file.
+func loadUTXOSet(filename string) (UTXOSet, error) {
+	// The utxostore file format is:
+	// <tx hash><output index><serialized utxo len><serialized utxo>
+	//
+	// The output index and serialized utxo len are little endian uint32s
+	// and the serialized utxo uses the format described in dagio.go.
+
+	filename = filepath.Join("testdata", filename)
+	fi, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	// Choose read based on whether the file is compressed or not.
+	var r io.Reader
+	if strings.HasSuffix(filename, ".bz2") {
+		r = bzip2.NewReader(fi)
+	} else {
+		r = fi
+	}
+	defer fi.Close()
+
+	utxoSet := NewFullUTXOSet()
+	for {
+		// Tx ID of the utxo entry.
+		var txID daghash.TxID
+		_, err := io.ReadAtLeast(r, txID[:], len(txID[:]))
+		if err != nil {
+			// Expected EOF at the right offset.
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+
+		// Output index of the utxo entry.
+		var index uint32
+		err = binary.Read(r, binary.LittleEndian, &index)
+		if err != nil {
+			return nil, err
+		}
+
+		// Num of serialized utxo entry bytes.
+		var numBytes uint32
+		err = binary.Read(r, binary.LittleEndian, &numBytes)
+		if err != nil {
+			return nil, err
+		}
+
+		// Deserialize the UTXO entry and add it to the UTXO set.
+		entry, err := deserializeUTXOEntry(r)
+		if err != nil {
+			return nil, err
+		}
+		utxoSet.utxoCollection[domainmessage.Outpoint{TxID: txID, Index: index}] = entry
+	}
+
+	return utxoSet, nil
+}
+
+// TestSetCoinbaseMaturity makes the ability to set the coinbase maturity
+// available when running tests.
+func (dag *BlockDAG) TestSetCoinbaseMaturity(maturity uint64) {
+	dag.Params.BlockCoinbaseMaturity = maturity
+}
+
+// newTestDAG returns a DAG that is usable for syntetic tests. It is
+// important to note that this DAG has no database associated with it, so
+// it is not usable with all functions and the tests must take care when making
+// use of it.
+func newTestDAG(params *dagconfig.Params) *BlockDAG {
+	index := newBlockIndex(params)
+	dag := &BlockDAG{
+		Params:                         params,
+		timeSource:                     NewTimeSource(),
+		difficultyAdjustmentWindowSize: params.DifficultyAdjustmentWindowSize,
+		TimestampDeviationTolerance:    params.TimestampDeviationTolerance,
+		powMaxBits:                     util.BigToCompact(params.PowMax),
+		index:                          index,
+		warningCaches:                  newThresholdCaches(vbNumBits),
+		deploymentCaches:               newThresholdCaches(dagconfig.DefinedDeployments),
+	}
+
+	// Create a genesis block node and block index index populated with it
+	// on the above fake DAG.
+	dag.genesis, _ = dag.newBlockNode(&params.GenesisBlock.Header, newBlockSet())
+	index.AddNode(dag.genesis)
+
+	dag.virtual = newVirtualBlock(dag, blockSetFromSlice(dag.genesis))
+	return dag
+}
+
+// newTestNode creates a block node connected to the passed parent with the
+// provided fields populated and fake values for the other fields.
+func newTestNode(dag *BlockDAG, parents blockSet, blockVersion int32, bits uint32, timestamp mstime.Time) *blockNode {
+	// Make up a header and create a block node from it.
+	header := &domainmessage.BlockHeader{
+		Version:              blockVersion,
+		ParentHashes:         parents.hashes(),
+		Bits:                 bits,
+		Timestamp:            timestamp,
+		HashMerkleRoot:       &daghash.ZeroHash,
+		AcceptedIDMerkleRoot: &daghash.ZeroHash,
+		UTXOCommitment:       &daghash.ZeroHash,
+	}
+	node, _ := dag.newBlockNode(header, parents)
+	return node
+}
+
+func addNodeAsChildToParents(node *blockNode) {
+	for parent := range node.parents {
+		parent.children.add(node)
+	}
+}
+
+// checkRuleError ensures the type of the two passed errors are of the
+// same type (either both nil or both of type RuleError) and their error codes
+// match when not nil.
+func checkRuleError(gotErr, wantErr error) error {
+	if wantErr == nil && gotErr == nil {
+		return nil
+	}
+
+	var gotRuleErr RuleError
+	if ok := errors.As(gotErr, &gotRuleErr); !ok {
+		return errors.Errorf("gotErr expected to be RuleError, but got %+v instead", gotErr)
+	}
+
+	var wantRuleErr RuleError
+	if ok := errors.As(wantErr, &wantRuleErr); !ok {
+		return errors.Errorf("wantErr expected to be RuleError, but got %+v instead", wantErr)
+	}
+
+	// Ensure the error codes match. It's safe to use a raw type assert
+	// here since the code above already proved they are the same type and
+	// the want error is a script error.
+	if gotRuleErr.ErrorCode != wantRuleErr.ErrorCode {
+		return errors.Errorf("mismatched error code - got %v (%v), want %v",
+			gotRuleErr.ErrorCode, gotErr, wantRuleErr.ErrorCode)
+	}
+
+	return nil
+}
+
+func prepareAndProcessBlockByParentMsgBlocks(t *testing.T, dag *BlockDAG, parents ...*domainmessage.MsgBlock) *domainmessage.MsgBlock {
+	parentHashes := make([]*daghash.Hash, len(parents))
+	for i, parent := range parents {
+		parentHashes[i] = parent.BlockHash()
+	}
+	return PrepareAndProcessBlockForTest(t, dag, parentHashes, nil)
+}
+
+func nodeByMsgBlock(t *testing.T, dag *BlockDAG, block *domainmessage.MsgBlock) *blockNode {
+	node, ok := dag.index.LookupNode(block.BlockHash())
+	if !ok {
+		t.Fatalf("couldn't find block node with hash %s", block.BlockHash())
+	}
+	return node
+}
+
+type fakeTimeSource struct {
+	time mstime.Time
+}
+
+func (fts *fakeTimeSource) Now() mstime.Time {
+	return fts.time
+}
+
+func newFakeTimeSource(fakeTime mstime.Time) TimeSource {
+	return &fakeTimeSource{time: fakeTime}
+}

blockdag/dagio.go

@@ -0,0 +1,664 @@
+// Copyright (c) 2015-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/binaryserializer"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+)
+
+var (
+	// byteOrder is the preferred byte order used for serializing numeric
+	// fields for storage in the database.
+	byteOrder = binary.LittleEndian
+)
+
+// ErrNotInDAG signifies that a block hash that is not in the
+// DAG was requested.
+type ErrNotInDAG string
+
+// Error implements the error interface.
+func (e ErrNotInDAG) Error() string {
+	return string(e)
+}
+
+// IsNotInDAGErr returns whether or not the passed error is an
+// ErrNotInDAG error.
+func IsNotInDAGErr(err error) bool {
+	var notInDAGErr ErrNotInDAG
+	return errors.As(err, &notInDAGErr)
+}
+
+// outpointIndexByteOrder is the byte order for serializing the outpoint index.
+// It uses big endian to ensure that when outpoint is used as database key, the
+// keys will be iterated in an ascending order by the outpoint index.
+var outpointIndexByteOrder = binary.BigEndian
+
+func serializeOutpoint(w io.Writer, outpoint *domainmessage.Outpoint) error {
+	_, err := w.Write(outpoint.TxID[:])
+	if err != nil {
+		return err
+	}
+
+	return binaryserializer.PutUint32(w, outpointIndexByteOrder, outpoint.Index)
+}
+
+var outpointSerializeSize = daghash.TxIDSize + 4
+
+// deserializeOutpoint decodes an outpoint from the passed serialized byte
+// slice into a new domainmessage.Outpoint using a format that is suitable for long-
+// term storage. This format is described in detail above.
+func deserializeOutpoint(r io.Reader) (*domainmessage.Outpoint, error) {
+	outpoint := &domainmessage.Outpoint{}
+	_, err := r.Read(outpoint.TxID[:])
+	if err != nil {
+		return nil, err
+	}
+
+	outpoint.Index, err = binaryserializer.Uint32(r, outpointIndexByteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	return outpoint, nil
+}
+
+// updateUTXOSet updates the UTXO set in the database based on the provided
+// UTXO diff.
+func updateUTXOSet(dbContext dbaccess.Context, virtualUTXODiff *UTXODiff) error {
+	outpointBuff := bytes.NewBuffer(make([]byte, outpointSerializeSize))
+	for outpoint := range virtualUTXODiff.toRemove {
+		outpointBuff.Reset()
+		err := serializeOutpoint(outpointBuff, &outpoint)
+		if err != nil {
+			return err
+		}
+
+		key := outpointBuff.Bytes()
+		err = dbaccess.RemoveFromUTXOSet(dbContext, key)
+		if err != nil {
+			return err
+		}
+	}
+
+	// We are preallocating for P2PKH entries because they are the most common ones.
+	// If we have entries with a compressed script bigger than P2PKH's, the buffer will grow.
+	utxoEntryBuff := bytes.NewBuffer(make([]byte, p2pkhUTXOEntrySerializeSize))
+
+	for outpoint, entry := range virtualUTXODiff.toAdd {
+		utxoEntryBuff.Reset()
+		outpointBuff.Reset()
+		// Serialize and store the UTXO entry.
+		err := serializeUTXOEntry(utxoEntryBuff, entry)
+		if err != nil {
+			return err
+		}
+		serializedEntry := utxoEntryBuff.Bytes()
+
+		err = serializeOutpoint(outpointBuff, &outpoint)
+		if err != nil {
+			return err
+		}
+
+		key := outpointBuff.Bytes()
+		err = dbaccess.AddToUTXOSet(dbContext, key, serializedEntry)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type dagState struct {
+	TipHashes         []*daghash.Hash
+	LastFinalityPoint *daghash.Hash
+	LocalSubnetworkID *subnetworkid.SubnetworkID
+}
+
+// serializeDAGState returns the serialization of the DAG state.
+// This is data to be stored in the DAG state bucket.
+func serializeDAGState(state *dagState) ([]byte, error) {
+	return json.Marshal(state)
+}
+
+// deserializeDAGState deserializes the passed serialized DAG state.
+// This is data stored in the DAG state bucket and is updated after
+// every block is connected to the DAG.
+func deserializeDAGState(serializedData []byte) (*dagState, error) {
+	var state *dagState
+	err := json.Unmarshal(serializedData, &state)
+	if err != nil {
+		return nil, err
+	}
+
+	return state, nil
+}
+
+// saveDAGState uses an existing database context to store the latest
+// tip hashes of the DAG.
+func saveDAGState(dbContext dbaccess.Context, state *dagState) error {
+	serializedDAGState, err := serializeDAGState(state)
+	if err != nil {
+		return err
+	}
+
+	return dbaccess.StoreDAGState(dbContext, serializedDAGState)
+}
+
+// createDAGState initializes the DAG state to the
+// genesis block and the node's local subnetwork id.
+func (dag *BlockDAG) createDAGState(localSubnetworkID *subnetworkid.SubnetworkID) error {
+	return saveDAGState(dag.databaseContext, &dagState{
+		TipHashes:         []*daghash.Hash{dag.Params.GenesisHash},
+		LastFinalityPoint: dag.Params.GenesisHash,
+		LocalSubnetworkID: localSubnetworkID,
+	})
+}
+
+// initDAGState attempts to load and initialize the DAG state from the
+// database. When the db does not yet contain any DAG state, both it and the
+// DAG state are initialized to the genesis block.
+func (dag *BlockDAG) initDAGState() error {
+	// Fetch the stored DAG state from the database. If it doesn't exist,
+	// it means that kaspad is running for the first time.
+	serializedDAGState, err := dbaccess.FetchDAGState(dag.databaseContext)
+	if dbaccess.IsNotFoundError(err) {
+		// Initialize the database and the DAG state to the genesis block.
+		return dag.createDAGState(dag.subnetworkID)
+	}
+	if err != nil {
+		return err
+	}
+
+	dagState, err := deserializeDAGState(serializedDAGState)
+	if err != nil {
+		return err
+	}
+
+	err = dag.validateLocalSubnetworkID(dagState)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Loading block index...")
+	unprocessedBlockNodes, err := dag.initBlockIndex()
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Loading UTXO set...")
+	fullUTXOCollection, err := dag.initUTXOSet()
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Loading reachability data...")
+	err = dag.reachabilityTree.init(dag.databaseContext)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Loading multiset data...")
+	err = dag.multisetStore.init(dag.databaseContext)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Applying the loaded utxoCollection to the virtual block...")
+	dag.virtual.utxoSet, err = newFullUTXOSetFromUTXOCollection(fullUTXOCollection)
+	if err != nil {
+		return errors.Wrap(err, "Error loading UTXOSet")
+	}
+
+	log.Debugf("Applying the stored tips to the virtual block...")
+	err = dag.initVirtualBlockTips(dagState)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Setting the last finality point...")
+	var ok bool
+	dag.lastFinalityPoint, ok = dag.index.LookupNode(dagState.LastFinalityPoint)
+	if !ok {
+		return errors.Errorf("finality point block %s "+
+			"does not exist in the DAG", dagState.LastFinalityPoint)
+	}
+	dag.finalizeNodesBelowFinalityPoint(false)
+
+	log.Debugf("Processing unprocessed blockNodes...")
+	err = dag.processUnprocessedBlockNodes(unprocessedBlockNodes)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("DAG state initialized.")
+
+	return nil
+}
+
+func (dag *BlockDAG) validateLocalSubnetworkID(state *dagState) error {
+	if !state.LocalSubnetworkID.IsEqual(dag.subnetworkID) {
+		return errors.Errorf("Cannot start kaspad with subnetwork ID %s because"+
+			" its database is already built with subnetwork ID %s. If you"+
+			" want to switch to a new database, please reset the"+
+			" database by starting kaspad with --reset-db flag", dag.subnetworkID, state.LocalSubnetworkID)
+	}
+	return nil
+}
+
+func (dag *BlockDAG) initBlockIndex() (unprocessedBlockNodes []*blockNode, err error) {
+	blockIndexCursor, err := dbaccess.BlockIndexCursor(dag.databaseContext)
+	if err != nil {
+		return nil, err
+	}
+	defer blockIndexCursor.Close()
+	for blockIndexCursor.Next() {
+		serializedDBNode, err := blockIndexCursor.Value()
+		if err != nil {
+			return nil, err
+		}
+		node, err := dag.deserializeBlockNode(serializedDBNode)
+		if err != nil {
+			return nil, err
+		}
+
+		// Check to see if this node had been stored in the the block DB
+		// but not yet accepted. If so, add it to a slice to be processed later.
+		if node.status == statusDataStored {
+			unprocessedBlockNodes = append(unprocessedBlockNodes, node)
+			continue
+		}
+
+		// If the node is known to be invalid add it as-is to the block
+		// index and continue.
+		if node.status.KnownInvalid() {
+			dag.index.addNode(node)
+			continue
+		}
+
+		if dag.blockCount == 0 {
+			if !node.hash.IsEqual(dag.Params.GenesisHash) {
+				return nil, errors.Errorf("Expected "+
+					"first entry in block index to be genesis block, "+
+					"found %s", node.hash)
+			}
+		} else {
+			if len(node.parents) == 0 {
+				return nil, errors.Errorf("block %s "+
+					"has no parents but it's not the genesis block", node.hash)
+			}
+		}
+
+		// Add the node to its parents children, connect it,
+		// and add it to the block index.
+		node.updateParentsChildren()
+		dag.index.addNode(node)
+
+		dag.blockCount++
+	}
+	return unprocessedBlockNodes, nil
+}
+
+func (dag *BlockDAG) initUTXOSet() (fullUTXOCollection utxoCollection, err error) {
+	fullUTXOCollection = make(utxoCollection)
+	cursor, err := dbaccess.UTXOSetCursor(dag.databaseContext)
+	if err != nil {
+		return nil, err
+	}
+	defer cursor.Close()
+
+	for cursor.Next() {
+		// Deserialize the outpoint
+		key, err := cursor.Key()
+		if err != nil {
+			return nil, err
+		}
+		outpoint, err := deserializeOutpoint(bytes.NewReader(key.Suffix()))
+		if err != nil {
+			return nil, err
+		}
+
+		// Deserialize the utxo entry
+		value, err := cursor.Value()
+		if err != nil {
+			return nil, err
+		}
+		entry, err := deserializeUTXOEntry(bytes.NewReader(value))
+		if err != nil {
+			return nil, err
+		}
+
+		fullUTXOCollection[*outpoint] = entry
+	}
+
+	return fullUTXOCollection, nil
+}
+
+func (dag *BlockDAG) initVirtualBlockTips(state *dagState) error {
+	tips := newBlockSet()
+	for _, tipHash := range state.TipHashes {
+		tip, ok := dag.index.LookupNode(tipHash)
+		if !ok {
+			return errors.Errorf("cannot find "+
+				"DAG tip %s in block index", state.TipHashes)
+		}
+		tips.add(tip)
+	}
+	dag.virtual.SetTips(tips)
+	return nil
+}
+
+func (dag *BlockDAG) processUnprocessedBlockNodes(unprocessedBlockNodes []*blockNode) error {
+	for _, node := range unprocessedBlockNodes {
+		// Check to see if the block exists in the block DB. If it
+		// doesn't, the database has certainly been corrupted.
+		blockExists, err := dbaccess.HasBlock(dag.databaseContext, node.hash)
+		if err != nil {
+			return errors.Wrapf(err, "HasBlock "+
+				"for block %s failed: %s", node.hash, err)
+		}
+		if !blockExists {
+			return errors.Errorf("block %s "+
+				"exists in block index but not in block db", node.hash)
+		}
+
+		// Attempt to accept the block.
+		block, err := dag.fetchBlockByHash(node.hash)
+		if err != nil {
+			return err
+		}
+		isOrphan, isDelayed, err := dag.ProcessBlock(block, BFWasStored)
+		if err != nil {
+			log.Warnf("Block %s, which was not previously processed, "+
+				"failed to be accepted to the DAG: %s", node.hash, err)
+			continue
+		}
+
+		// If the block is an orphan or is delayed then it couldn't have
+		// possibly been written to the block index in the first place.
+		if isOrphan {
+			return errors.Errorf("Block %s, which was not "+
+				"previously processed, turned out to be an orphan, which is "+
+				"impossible.", node.hash)
+		}
+		if isDelayed {
+			return errors.Errorf("Block %s, which was not "+
+				"previously processed, turned out to be delayed, which is "+
+				"impossible.", node.hash)
+		}
+	}
+	return nil
+}
+
+// deserializeBlockNode parses a value in the block index bucket and returns a block node.
+func (dag *BlockDAG) deserializeBlockNode(blockRow []byte) (*blockNode, error) {
+	buffer := bytes.NewReader(blockRow)
+
+	var header domainmessage.BlockHeader
+	err := header.Deserialize(buffer)
+	if err != nil {
+		return nil, err
+	}
+
+	node := &blockNode{
+		hash:                 header.BlockHash(),
+		version:              header.Version,
+		bits:                 header.Bits,
+		nonce:                header.Nonce,
+		timestamp:            header.Timestamp.UnixMilliseconds(),
+		hashMerkleRoot:       header.HashMerkleRoot,
+		acceptedIDMerkleRoot: header.AcceptedIDMerkleRoot,
+		utxoCommitment:       header.UTXOCommitment,
+	}
+
+	node.children = newBlockSet()
+	node.parents = newBlockSet()
+
+	for _, hash := range header.ParentHashes {
+		parent, ok := dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("deserializeBlockNode: Could "+
+				"not find parent %s for block %s", hash, header.BlockHash())
+		}
+		node.parents.add(parent)
+	}
+
+	statusByte, err := buffer.ReadByte()
+	if err != nil {
+		return nil, err
+	}
+	node.status = blockStatus(statusByte)
+
+	selectedParentHash := &daghash.Hash{}
+	if _, err := io.ReadFull(buffer, selectedParentHash[:]); err != nil {
+		return nil, err
+	}
+
+	// Because genesis doesn't have selected parent, it's serialized as zero hash
+	if !selectedParentHash.IsEqual(&daghash.ZeroHash) {
+		var ok bool
+		node.selectedParent, ok = dag.index.LookupNode(selectedParentHash)
+		if !ok {
+			return nil, errors.Errorf("block %s does not exist in the DAG", selectedParentHash)
+		}
+	}
+
+	node.blueScore, err = binaryserializer.Uint64(buffer, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	bluesCount, err := domainmessage.ReadVarInt(buffer)
+	if err != nil {
+		return nil, err
+	}
+
+	node.blues = make([]*blockNode, bluesCount)
+	for i := uint64(0); i < bluesCount; i++ {
+		hash := &daghash.Hash{}
+		if _, err := io.ReadFull(buffer, hash[:]); err != nil {
+			return nil, err
+		}
+
+		var ok bool
+		node.blues[i], ok = dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("block %s does not exist in the DAG", selectedParentHash)
+		}
+	}
+
+	bluesAnticoneSizesLen, err := domainmessage.ReadVarInt(buffer)
+	if err != nil {
+		return nil, err
+	}
+
+	node.bluesAnticoneSizes = make(map[*blockNode]dagconfig.KType)
+	for i := uint64(0); i < bluesAnticoneSizesLen; i++ {
+		hash := &daghash.Hash{}
+		if _, err := io.ReadFull(buffer, hash[:]); err != nil {
+			return nil, err
+		}
+		bluesAnticoneSize, err := binaryserializer.Uint8(buffer)
+		if err != nil {
+			return nil, err
+		}
+		blue, ok := dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("couldn't find block with hash %s", hash)
+		}
+		node.bluesAnticoneSizes[blue] = dagconfig.KType(bluesAnticoneSize)
+	}
+
+	return node, nil
+}
+
+// fetchBlockByHash retrieves the raw block for the provided hash,
+// deserializes it, and returns a util.Block of it.
+func (dag *BlockDAG) fetchBlockByHash(hash *daghash.Hash) (*util.Block, error) {
+	blockBytes, err := dbaccess.FetchBlock(dag.databaseContext, hash)
+	if err != nil {
+		return nil, err
+	}
+	return util.NewBlockFromBytes(blockBytes)
+}
+
+func storeBlock(dbContext *dbaccess.TxContext, block *util.Block) error {
+	blockBytes, err := block.Bytes()
+	if err != nil {
+		return err
+	}
+	return dbaccess.StoreBlock(dbContext, block.Hash(), blockBytes)
+}
+
+func serializeBlockNode(node *blockNode) ([]byte, error) {
+	w := bytes.NewBuffer(make([]byte, 0, domainmessage.MaxBlockHeaderPayload+1))
+	header := node.Header()
+	err := header.Serialize(w)
+	if err != nil {
+		return nil, err
+	}
+
+	err = w.WriteByte(byte(node.status))
+	if err != nil {
+		return nil, err
+	}
+
+	// Because genesis doesn't have selected parent, it's serialized as zero hash
+	selectedParentHash := &daghash.ZeroHash
+	if node.selectedParent != nil {
+		selectedParentHash = node.selectedParent.hash
+	}
+	_, err = w.Write(selectedParentHash[:])
+	if err != nil {
+		return nil, err
+	}
+
+	err = binaryserializer.PutUint64(w, byteOrder, node.blueScore)
+	if err != nil {
+		return nil, err
+	}
+
+	err = domainmessage.WriteVarInt(w, uint64(len(node.blues)))
+	if err != nil {
+		return nil, err
+	}
+
+	for _, blue := range node.blues {
+		_, err = w.Write(blue.hash[:])
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err = domainmessage.WriteVarInt(w, uint64(len(node.bluesAnticoneSizes)))
+	if err != nil {
+		return nil, err
+	}
+	for blue, blueAnticoneSize := range node.bluesAnticoneSizes {
+		_, err = w.Write(blue.hash[:])
+		if err != nil {
+			return nil, err
+		}
+
+		err = binaryserializer.PutUint8(w, uint8(blueAnticoneSize))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return w.Bytes(), nil
+}
+
+// blockIndexKey generates the binary key for an entry in the block index
+// bucket. The key is composed of the block blue score encoded as a big-endian
+// 64-bit unsigned int followed by the 32 byte block hash.
+// The blue score component is important for iteration order.
+func blockIndexKey(blockHash *daghash.Hash, blueScore uint64) []byte {
+	indexKey := make([]byte, daghash.HashSize+8)
+	binary.BigEndian.PutUint64(indexKey[0:8], blueScore)
+	copy(indexKey[8:daghash.HashSize+8], blockHash[:])
+	return indexKey
+}
+
+func blockHashFromBlockIndexKey(BlockIndexKey []byte) (*daghash.Hash, error) {
+	return daghash.NewHash(BlockIndexKey[8 : daghash.HashSize+8])
+}
+
+// BlockByHash returns the block from the DAG with the given hash.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) BlockByHash(hash *daghash.Hash) (*util.Block, error) {
+	// Lookup the block hash in block index and ensure it is in the DAG
+	node, ok := dag.index.LookupNode(hash)
+	if !ok {
+		str := fmt.Sprintf("block %s is not in the DAG", hash)
+		return nil, ErrNotInDAG(str)
+	}
+
+	block, err := dag.fetchBlockByHash(node.hash)
+	if err != nil {
+		return nil, err
+	}
+	return block, err
+}
+
+// BlockHashesFrom returns a slice of blocks starting from lowHash
+// ordered by blueScore. If lowHash is nil then the genesis block is used.
+//
+// This method MUST be called with the DAG lock held
+func (dag *BlockDAG) BlockHashesFrom(lowHash *daghash.Hash, limit int) ([]*daghash.Hash, error) {
+	blockHashes := make([]*daghash.Hash, 0, limit)
+	if lowHash == nil {
+		lowHash = dag.genesis.hash
+
+		// If we're starting from the beginning we should include the
+		// genesis hash in the result
+		blockHashes = append(blockHashes, dag.genesis.hash)
+	}
+	if !dag.IsInDAG(lowHash) {
+		return nil, errors.Errorf("block %s not found", lowHash)
+	}
+	blueScore, err := dag.BlueScoreByBlockHash(lowHash)
+	if err != nil {
+		return nil, err
+	}
+
+	key := blockIndexKey(lowHash, blueScore)
+	cursor, err := dbaccess.BlockIndexCursorFrom(dag.databaseContext, key)
+	if dbaccess.IsNotFoundError(err) {
+		return nil, errors.Wrapf(err, "block %s not in block index", lowHash)
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer cursor.Close()
+
+	for cursor.Next() && len(blockHashes) < limit {
+		key, err := cursor.Key()
+		if err != nil {
+			return nil, err
+		}
+		blockHash, err := blockHashFromBlockIndexKey(key.Suffix())
+		if err != nil {
+			return nil, err
+		}
+		blockHashes = append(blockHashes, blockHash)
+	}
+
+	return blockHashes, nil
+}

blockdag/dagio_test.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2015-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"bytes"
+	"encoding/hex"
+	"github.com/pkg/errors"
+	"reflect"
+	"testing"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// TestErrNotInDAG ensures the functions related to ErrNotInDAG work
+// as expected.
+func TestErrNotInDAG(t *testing.T) {
+	errStr := "no block at height 1 exists"
+	err := error(ErrNotInDAG(errStr))
+
+	// Ensure the stringized output for the error is as expected.
+	if err.Error() != errStr {
+		t.Fatalf("ErrNotInDAG retuned unexpected error string - "+
+			"got %q, want %q", err.Error(), errStr)
+	}
+
+	// Ensure error is detected as the correct type.
+	if !IsNotInDAGErr(err) {
+		t.Fatalf("IsNotInDAGErr did not detect as expected type")
+	}
+	err = errors.New("something else")
+	if IsNotInDAGErr(err) {
+		t.Fatalf("IsNotInDAGErr detected incorrect type")
+	}
+}
+
+// hexToBytes converts the passed hex string into bytes and will panic if there
+// is an error. This is only provided for the hard-coded constants so errors in
+// the source code can be detected. It will only (and must only) be called with
+// hard-coded values.
+func hexToBytes(s string) []byte {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		panic("invalid hex in source file: " + s)
+	}
+	return b
+}
+
+// TestUTXOSerialization ensures serializing and deserializing unspent
+// trasaction output entries works as expected.
+func TestUTXOSerialization(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		entry      *UTXOEntry
+		serialized []byte
+	}{
+		{
+			name: "blue score 1, coinbase",
+			entry: &UTXOEntry{
+				amount:         5000000000,
+				scriptPubKey:   hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
+				blockBlueScore: 1,
+				packedFlags:    tfCoinbase,
+			},
+			serialized: hexToBytes("01000000000000000100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
+		},
+		{
+			name: "blue score 100001, not coinbase",
+			entry: &UTXOEntry{
+				amount:         1000000,
+				scriptPubKey:   hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
+				blockBlueScore: 100001,
+				packedFlags:    0,
+			},
+			serialized: hexToBytes("a1860100000000000040420f00000000001976a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
+		},
+	}
+
+	for i, test := range tests {
+		// Ensure the utxo entry serializes to the expected value.
+		w := &bytes.Buffer{}
+		err := serializeUTXOEntry(w, test.entry)
+		if err != nil {
+			t.Errorf("serializeUTXOEntry #%d (%s) unexpected "+
+				"error: %v", i, test.name, err)
+			continue
+		}
+
+		gotBytes := w.Bytes()
+		if !bytes.Equal(gotBytes, test.serialized) {
+			t.Errorf("serializeUTXOEntry #%d (%s): mismatched "+
+				"bytes - got %x, want %x", i, test.name,
+				gotBytes, test.serialized)
+			continue
+		}
+
+		// Deserialize to a utxo entry.gotBytes
+		utxoEntry, err := deserializeUTXOEntry(bytes.NewReader(test.serialized))
+		if err != nil {
+			t.Errorf("deserializeUTXOEntry #%d (%s) unexpected "+
+				"error: %v", i, test.name, err)
+			continue
+		}
+
+		// Ensure the deserialized entry has the same properties as the
+		// ones in the test entry.
+		if utxoEntry.Amount() != test.entry.Amount() {
+			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
+				"amounts: got %d, want %d", i, test.name,
+				utxoEntry.Amount(), test.entry.Amount())
+			continue
+		}
+
+		if !bytes.Equal(utxoEntry.ScriptPubKey(), test.entry.ScriptPubKey()) {
+			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
+				"scripts: got %x, want %x", i, test.name,
+				utxoEntry.ScriptPubKey(), test.entry.ScriptPubKey())
+			continue
+		}
+		if utxoEntry.BlockBlueScore() != test.entry.BlockBlueScore() {
+			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
+				"block blue score: got %d, want %d", i, test.name,
+				utxoEntry.BlockBlueScore(), test.entry.BlockBlueScore())
+			continue
+		}
+		if utxoEntry.IsCoinbase() != test.entry.IsCoinbase() {
+			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
+				"coinbase flag: got %v, want %v", i, test.name,
+				utxoEntry.IsCoinbase(), test.entry.IsCoinbase())
+			continue
+		}
+	}
+}
+
+// TestUtxoEntryDeserializeErrors performs negative tests against deserializing
+// unspent transaction outputs to ensure error paths work as expected.
+func TestUtxoEntryDeserializeErrors(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		serialized []byte
+	}{
+		{
+			name:       "no data after header code",
+			serialized: hexToBytes("02"),
+		},
+		{
+			name:       "incomplete compressed txout",
+			serialized: hexToBytes("0232"),
+		},
+	}
+
+	for _, test := range tests {
+		// Ensure the expected error type is returned and the returned
+		// entry is nil.
+		entry, err := deserializeUTXOEntry(bytes.NewReader(test.serialized))
+		if err == nil {
+			t.Errorf("deserializeUTXOEntry (%s): didn't return an error",
+				test.name)
+			continue
+		}
+		if entry != nil {
+			t.Errorf("deserializeUTXOEntry (%s): returned entry "+
+				"is not nil", test.name)
+			continue
+		}
+	}
+}
+
+// TestDAGStateSerialization ensures serializing and deserializing the
+// DAG state works as expected.
+func TestDAGStateSerialization(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		state      *dagState
+		serialized []byte
+	}{
+		{
+			name: "genesis",
+			state: &dagState{
+				TipHashes:         []*daghash.Hash{newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f")},
+				LastFinalityPoint: newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"),
+			},
+			serialized: []byte("{\"TipHashes\":[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0],\"LocalSubnetworkID\":null}"),
+		},
+		{
+			name: "block 1",
+			state: &dagState{
+				TipHashes:         []*daghash.Hash{newHashFromStr("00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048")},
+				LastFinalityPoint: newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"),
+			},
+			serialized: []byte("{\"TipHashes\":[[72,96,235,24,191,27,22,32,227,126,148,144,252,138,66,117,20,65,111,215,81,89,171,134,104,142,154,131,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0],\"LocalSubnetworkID\":null}"),
+		},
+	}
+
+	for i, test := range tests {
+		gotBytes, err := serializeDAGState(test.state)
+		if err != nil {
+			t.Errorf("serializeDAGState #%d (%s) "+
+				"unexpected error: %v", i, test.name, err)
+			continue
+		}
+
+		// Ensure the dagState serializes to the expected value.
+		if !bytes.Equal(gotBytes, test.serialized) {
+			t.Errorf("serializeDAGState #%d (%s): mismatched "+
+				"bytes - got %s, want %s", i, test.name,
+				string(gotBytes), string(test.serialized))
+			continue
+		}
+
+		// Ensure the serialized bytes are decoded back to the expected
+		// dagState.
+		state, err := deserializeDAGState(test.serialized)
+		if err != nil {
+			t.Errorf("deserializeDAGState #%d (%s) "+
+				"unexpected error: %v", i, test.name, err)
+			continue
+		}
+		if !reflect.DeepEqual(state, test.state) {
+			t.Errorf("deserializeDAGState #%d (%s) "+
+				"mismatched state - got %v, want %v", i,
+				test.name, state, test.state)
+			continue
+		}
+	}
+}
+
+// newHashFromStr converts the passed big-endian hex string into a
+// daghash.Hash. It only differs from the one available in daghash in that
+// it panics in case of an error since it will only (and must only) be
+// called with hard-coded, and therefore known good, hashes.
+func newHashFromStr(hexStr string) *daghash.Hash {
+	hash, err := daghash.NewHashFromStr(hexStr)
+	if err != nil {
+		panic(err)
+	}
+	return hash
+}

blockdag/delayedblockheap.go

@@ -0,0 +1,73 @@
+package blockdag
+
+import (
+	"container/heap"
+)
+
+type baseDelayedBlocksHeap []*delayedBlock
+
+func (h baseDelayedBlocksHeap) Len() int {
+	return len(h)
+}
+func (h baseDelayedBlocksHeap) Swap(i, j int) {
+	h[i], h[j] = h[j], h[i]
+}
+
+func (h *baseDelayedBlocksHeap) Push(x interface{}) {
+	*h = append(*h, x.(*delayedBlock))
+}
+
+func (h *baseDelayedBlocksHeap) Pop() interface{} {
+	oldHeap := *h
+	oldLength := len(oldHeap)
+	popped := oldHeap[oldLength-1]
+	*h = oldHeap[0 : oldLength-1]
+	return popped
+}
+
+func (h baseDelayedBlocksHeap) peek() interface{} {
+	if h.Len() > 0 {
+		return h[h.Len()-1]
+	}
+	return nil
+}
+
+func (h baseDelayedBlocksHeap) Less(i, j int) bool {
+	return h[j].processTime.After(h[i].processTime)
+}
+
+type delayedBlocksHeap struct {
+	baseDelayedBlocksHeap *baseDelayedBlocksHeap
+	impl                  heap.Interface
+}
+
+// newDelayedBlocksHeap initializes and returns a new delayedBlocksHeap
+func newDelayedBlocksHeap() delayedBlocksHeap {
+	baseHeap := &baseDelayedBlocksHeap{}
+	h := delayedBlocksHeap{impl: baseHeap, baseDelayedBlocksHeap: baseHeap}
+	heap.Init(h.impl)
+	return h
+}
+
+// pop removes the block with lowest height from this heap and returns it
+func (dbh delayedBlocksHeap) pop() *delayedBlock {
+	return heap.Pop(dbh.impl).(*delayedBlock)
+}
+
+// Push pushes the block onto the heap
+func (dbh delayedBlocksHeap) Push(block *delayedBlock) {
+	heap.Push(dbh.impl, block)
+}
+
+// Len returns the length of this heap
+func (dbh delayedBlocksHeap) Len() int {
+	return dbh.impl.Len()
+}
+
+// peek returns the topmost element in the queue without poping it
+func (dbh delayedBlocksHeap) peek() *delayedBlock {
+	if dbh.baseDelayedBlocksHeap.peek() == nil {
+		return nil
+	}
+	return dbh.baseDelayedBlocksHeap.peek().(*delayedBlock)
+}

blockdag/difficulty.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/bigintpool"
+	"github.com/kaspanet/kaspad/util/mstime"
+
+	"github.com/kaspanet/kaspad/util"
+)
+
+// requiredDifficulty calculates the required difficulty for a
+// block given its bluest parent.
+func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime mstime.Time) uint32 {
+	// Genesis block.
+	if bluestParent == nil || bluestParent.blueScore < dag.difficultyAdjustmentWindowSize+1 {
+		return dag.powMaxBits
+	}
+
+	// Fetch window of dag.difficultyAdjustmentWindowSize + 1 so we can have dag.difficultyAdjustmentWindowSize block intervals
+	timestampsWindow := blueBlockWindow(bluestParent, dag.difficultyAdjustmentWindowSize+1)
+	windowMinTimestamp, windowMaxTimeStamp := timestampsWindow.minMaxTimestamps()
+
+	// Remove the last block from the window so to calculate the average target of dag.difficultyAdjustmentWindowSize blocks
+	targetsWindow := timestampsWindow[:dag.difficultyAdjustmentWindowSize]
+
+	// Calculate new target difficulty as:
+	// averageWindowTarget * (windowMinTimestamp / (targetTimePerBlock * windowSize))
+	// The result uses integer division which means it will be slightly
+	// rounded down.
+	newTarget := bigintpool.Acquire(0)
+	defer bigintpool.Release(newTarget)
+	windowTimeStampDifference := bigintpool.Acquire(windowMaxTimeStamp - windowMinTimestamp)
+	defer bigintpool.Release(windowTimeStampDifference)
+	targetTimePerBlock := bigintpool.Acquire(dag.Params.TargetTimePerBlock.Milliseconds())
+	defer bigintpool.Release(targetTimePerBlock)
+	difficultyAdjustmentWindowSize := bigintpool.Acquire(int64(dag.difficultyAdjustmentWindowSize))
+	defer bigintpool.Release(difficultyAdjustmentWindowSize)
+
+	targetsWindow.averageTarget(newTarget)
+	newTarget.
+		Mul(newTarget, windowTimeStampDifference).
+		Div(newTarget, targetTimePerBlock).
+		Div(newTarget, difficultyAdjustmentWindowSize)
+	if newTarget.Cmp(dag.Params.PowMax) > 0 {
+		return dag.powMaxBits
+	}
+	newTargetBits := util.BigToCompact(newTarget)
+	return newTargetBits
+}
+
+// NextRequiredDifficulty calculates the required difficulty for a block that will
+// be built on top of the current tips.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) NextRequiredDifficulty(timestamp mstime.Time) uint32 {
+	difficulty := dag.requiredDifficulty(dag.virtual.parents.bluest(), timestamp)
+	return difficulty
+}

blockdag/difficulty_test.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2014-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/mstime"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/kaspanet/kaspad/util"
+)
+
+// TestBigToCompact ensures BigToCompact converts big integers to the expected
+// compact representation.
+func TestBigToCompact(t *testing.T) {
+	tests := []struct {
+		in  int64
+		out uint32
+	}{
+		{0, 0},
+		{-1, 25231360},
+	}
+
+	for x, test := range tests {
+		n := big.NewInt(test.in)
+		r := util.BigToCompact(n)
+		if r != test.out {
+			t.Errorf("TestBigToCompact test #%d failed: got %d want %d\n",
+				x, r, test.out)
+			return
+		}
+	}
+}
+
+// TestCompactToBig ensures CompactToBig converts numbers using the compact
+// representation to the expected big intergers.
+func TestCompactToBig(t *testing.T) {
+	tests := []struct {
+		in  uint32
+		out int64
+	}{
+		{10000000, 0},
+	}
+
+	for x, test := range tests {
+		n := util.CompactToBig(test.in)
+		want := big.NewInt(test.out)
+		if n.Cmp(want) != 0 {
+			t.Errorf("TestCompactToBig test #%d failed: got %d want %d\n",
+				x, n.Int64(), want.Int64())
+			return
+		}
+	}
+}
+
+// TestCalcWork ensures CalcWork calculates the expected work value from values
+// in compact representation.
+func TestCalcWork(t *testing.T) {
+	tests := []struct {
+		in  uint32
+		out int64
+	}{
+		{10000000, 0},
+	}
+
+	for x, test := range tests {
+		bits := uint32(test.in)
+
+		r := util.CalcWork(bits)
+		if r.Int64() != test.out {
+			t.Errorf("TestCalcWork test #%d failed: got %v want %d\n",
+				x, r.Int64(), test.out)
+			return
+		}
+	}
+}
+
+func TestDifficulty(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.K = 1
+	params.DifficultyAdjustmentWindowSize = 264
+	dag, teardownFunc, err := DAGSetup("TestDifficulty", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	zeroTime := mstime.Time{}
+	addNode := func(parents blockSet, blockTime mstime.Time) *blockNode {
+		bluestParent := parents.bluest()
+		if blockTime.IsZero() {
+			blockTime = bluestParent.time()
+			blockTime = blockTime.Add(params.TargetTimePerBlock)
+		}
+		block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+		if err != nil {
+			t.Fatalf("unexpected error in PrepareBlockForTest: %s", err)
+		}
+		block.Header.Timestamp = blockTime
+		block.Header.Bits = dag.requiredDifficulty(bluestParent, blockTime)
+
+		isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(block), BFNoPoWCheck)
+		if err != nil {
+			t.Fatalf("unexpected error in ProcessBlock: %s", err)
+		}
+		if isDelayed {
+			t.Fatalf("block is too far in the future")
+		}
+		if isOrphan {
+			t.Fatalf("block was unexpectedly orphan")
+		}
+		node, ok := dag.index.LookupNode(block.BlockHash())
+		if !ok {
+			t.Fatalf("block %s does not exist in the DAG", block.BlockHash())
+		}
+		return node
+	}
+	tip := dag.genesis
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
+		if tip.bits != dag.genesis.bits {
+			t.Fatalf("As long as the bluest parent's blue score is less then the difficulty adjustment " +
+				"window size, the difficulty should be the same as genesis'")
+		}
+	}
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize+100; i++ {
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
+		if tip.bits != dag.genesis.bits {
+			t.Fatalf("As long as the block rate remains the same, the difficulty shouldn't change")
+		}
+	}
+	nodeInThePast := addNode(blockSetFromSlice(tip), tip.PastMedianTime(dag))
+	if nodeInThePast.bits != tip.bits {
+		t.Fatalf("The difficulty should only change when nodeInThePast is in the past of a block bluest parent")
+	}
+	tip = nodeInThePast
+
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
+	if tip.bits != nodeInThePast.bits {
+		t.Fatalf("The difficulty should only change when nodeInThePast is in the past of a block bluest parent")
+	}
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
+	if compareBits(tip.bits, nodeInThePast.bits) >= 0 {
+		t.Fatalf("tip.bits should be smaller than nodeInThePast.bits because nodeInThePast increased the " +
+			"block rate, so the difficulty should increase as well")
+	}
+	expectedBits := uint32(0x207f83df)
+	if tip.bits != expectedBits {
+		t.Errorf("tip.bits was expected to be %x but got %x", expectedBits, tip.bits)
+	}
+
+	// Increase block rate to increase difficulty
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
+		tip = addNode(blockSetFromSlice(tip), tip.PastMedianTime(dag))
+		if compareBits(tip.bits, tip.parents.bluest().bits) > 0 {
+			t.Fatalf("Because we're increasing the block rate, the difficulty can't decrease")
+		}
+	}
+
+	// Add blocks until difficulty stabilizes
+	lastBits := tip.bits
+	sameBitsCount := uint64(0)
+	for sameBitsCount < dag.difficultyAdjustmentWindowSize+1 {
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
+		if tip.bits == lastBits {
+			sameBitsCount++
+		} else {
+			lastBits = tip.bits
+			sameBitsCount = 0
+		}
+	}
+	slowBlockTime := tip.time()
+	slowBlockTime = slowBlockTime.Add(params.TargetTimePerBlock + time.Second)
+	slowNode := addNode(blockSetFromSlice(tip), slowBlockTime)
+	if slowNode.bits != tip.bits {
+		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
+	}
+
+	tip = slowNode
+
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
+	if tip.bits != slowNode.bits {
+		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
+	}
+	tip = addNode(blockSetFromSlice(tip), zeroTime)
+	if compareBits(tip.bits, slowNode.bits) <= 0 {
+		t.Fatalf("tip.bits should be smaller than slowNode.bits because slowNode decreased the block" +
+			" rate, so the difficulty should decrease as well")
+	}
+
+	splitNode := addNode(blockSetFromSlice(tip), zeroTime)
+	tip = splitNode
+	for i := 0; i < 100; i++ {
+		tip = addNode(blockSetFromSlice(tip), zeroTime)
+	}
+	blueTip := tip
+
+	redChainTip := splitNode
+	for i := 0; i < 10; i++ {
+		redChainTip = addNode(blockSetFromSlice(redChainTip), redChainTip.PastMedianTime(dag))
+	}
+	tipWithRedPast := addNode(blockSetFromSlice(redChainTip, blueTip), zeroTime)
+	tipWithoutRedPast := addNode(blockSetFromSlice(blueTip), zeroTime)
+	if tipWithoutRedPast.bits != tipWithRedPast.bits {
+		t.Fatalf("tipWithoutRedPast.bits should be the same as tipWithRedPast.bits because red blocks" +
+			" shouldn't affect the difficulty")
+	}
+}
+
+func compareBits(a uint32, b uint32) int {
+	aTarget := util.CompactToBig(a)
+	bTarget := util.CompactToBig(b)
+	return aTarget.Cmp(bTarget)
+}

blockdag/doc.go

@@ -0,0 +1,57 @@
+/*
+Package blockdag implements kaspa block handling and DAG selection rules.
+
+The kaspa block handling and DAG selection rules are an integral, and quite
+likely the most important, part of kaspa. At its core, kaspa is a distributed
+consensus of which blocks are valid and which ones will comprise the DAG
+(public ledger) that ultimately determines accepted transactions, so it is
+extremely important that fully validating nodes agree on all rules.
+
+At a high level, this package provides support for inserting new blocks into
+the block DAG according to the aforementioned rules. It includes functionality
+such as rejecting duplicate blocks, ensuring blocks and transactions follow all
+rules, orphan handling, and DAG order along with reorganization.
+
+Since this package does not deal with other kaspa specifics such as network
+communication, it provides a notification system which gives the caller a high
+level of flexibility in how they want to react to certain events such as orphan
+blocks which need their parents requested and newly connected DAG blocks.
+
+Kaspa DAG Processing Overview
+
+Before a block is allowed into the block DAG, it must go through an intensive
+series of validation rules. The following list serves as a general outline of
+those rules to provide some intuition into what is going on under the hood, but
+is by no means exhaustive:
+
+ - Reject duplicate blocks
+ - Perform a series of sanity checks on the block and its transactions such as
+   verifying proof of work, timestamps, number and character of transactions,
+   transaction amounts, script complexity, and merkle root calculations
+ - Save the most recent orphan blocks for a limited time in case their parent
+   blocks become available
+ - Stop processing if the block is an orphan as the rest of the processing
+   depends on the block's position within the block DAG
+ - Perform a series of more thorough checks that depend on the block's position
+   within the block DAG such as verifying block difficulties adhere to
+   difficulty retarget rules, timestamps are after the median of the last
+   several blocks, all transactions are finalized, and
+   block versions are in line with the previous blocks
+ - When a block is being connected to the DAG, perform further checks on the
+   block's transactions such as verifying transaction duplicates, script
+   complexity for the combination of connected scripts, coinbase maturity,
+   double spends, and connected transaction values
+ - Run the transaction scripts to verify the spender is allowed to spend the
+   coins
+ - Insert the block into the block database
+
+Errors
+
+Errors returned by this package are either the raw errors provided by underlying
+calls or of type blockdag.RuleError. This allows the caller to differentiate
+between unexpected errors, such as database errors, versus errors due to rule
+violations through type assertions. In addition, callers can programmatically
+determine the specific rule violation by examining the ErrorCode field of the
+type asserted blockdag.RuleError.
+*/
+package blockdag

blockdag/error.go

@@ -0,0 +1,293 @@
+// Copyright (c) 2014-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+// ErrorCode identifies a kind of error.
+type ErrorCode int
+
+// These constants are used to identify a specific RuleError.
+const (
+	// ErrDuplicateBlock indicates a block with the same hash already
+	// exists.
+	ErrDuplicateBlock ErrorCode = iota
+
+	// ErrBlockMassTooHigh indicates the mass of a block exceeds the maximum
+	// allowed limits.
+	ErrBlockMassTooHigh
+
+	// ErrBlockVersionTooOld indicates the block version is too old and is
+	// no longer accepted since the majority of the network has upgraded
+	// to a newer version.
+	ErrBlockVersionTooOld
+
+	// ErrTimeTooOld indicates the time is either before the median time of
+	// the last several blocks per the DAG consensus rules.
+	ErrTimeTooOld
+
+	// ErrTimeTooNew indicates the time is too far in the future as compared
+	// the current time.
+	ErrTimeTooNew
+
+	// ErrNoParents indicates that the block is missing parents
+	ErrNoParents
+
+	// ErrWrongParentsOrder indicates that the block's parents are not ordered by hash, as expected
+	ErrWrongParentsOrder
+
+	// ErrDifficultyTooLow indicates the difficulty for the block is lower
+	// than the difficulty required.
+	ErrDifficultyTooLow
+
+	// ErrUnexpectedDifficulty indicates specified bits do not align with
+	// the expected value either because it doesn't match the calculated
+	// valued based on difficulty regarted rules or it is out of the valid
+	// range.
+	ErrUnexpectedDifficulty
+
+	// ErrHighHash indicates the block does not hash to a value which is
+	// lower than the required target difficultly.
+	ErrHighHash
+
+	// ErrBadMerkleRoot indicates the calculated merkle root does not match
+	// the expected value.
+	ErrBadMerkleRoot
+
+	// ErrBadUTXOCommitment indicates the calculated UTXO commitment does not match
+	// the expected value.
+	ErrBadUTXOCommitment
+
+	// ErrInvalidSubnetwork indicates the subnetwork is now allowed.
+	ErrInvalidSubnetwork
+
+	// ErrFinalityPointTimeTooOld indicates a block has a timestamp before the
+	// last finality point.
+	ErrFinalityPointTimeTooOld
+
+	// ErrNoTransactions indicates the block does not have a least one
+	// transaction. A valid block must have at least the coinbase
+	// transaction.
+	ErrNoTransactions
+
+	// ErrNoTxInputs indicates a transaction does not have any inputs. A
+	// valid transaction must have at least one input.
+	ErrNoTxInputs
+
+	// ErrTxMassTooHigh indicates the mass of a transaction exceeds the maximum
+	// allowed limits.
+	ErrTxMassTooHigh
+
+	// ErrBadTxOutValue indicates an output value for a transaction is
+	// invalid in some way such as being out of range.
+	ErrBadTxOutValue
+
+	// ErrDuplicateTxInputs indicates a transaction references the same
+	// input more than once.
+	ErrDuplicateTxInputs
+
+	// ErrBadTxInput indicates a transaction input is invalid in some way
+	// such as referencing a previous transaction outpoint which is out of
+	// range or not referencing one at all.
+	ErrBadTxInput
+
+	// ErrMissingTxOut indicates a transaction output referenced by an input
+	// either does not exist or has already been spent.
+	ErrMissingTxOut
+
+	// ErrDoubleSpendInSameBlock indicates a transaction
+	// that spends an output that was already spent by another
+	// transaction in the same block.
+	ErrDoubleSpendInSameBlock
+
+	// ErrUnfinalizedTx indicates a transaction has not been finalized.
+	// A valid block may only contain finalized transactions.
+	ErrUnfinalizedTx
+
+	// ErrDuplicateTx indicates a block contains an identical transaction
+	// (or at least two transactions which hash to the same value). A
+	// valid block may only contain unique transactions.
+	ErrDuplicateTx
+
+	// ErrOverwriteTx indicates a block contains a transaction that has
+	// the same hash as a previous transaction which has not been fully
+	// spent.
+	ErrOverwriteTx
+
+	// ErrImmatureSpend indicates a transaction is attempting to spend a
+	// coinbase that has not yet reached the required maturity.
+	ErrImmatureSpend
+
+	// ErrSpendTooHigh indicates a transaction is attempting to spend more
+	// value than the sum of all of its inputs.
+	ErrSpendTooHigh
+
+	// ErrBadFees indicates the total fees for a block are invalid due to
+	// exceeding the maximum possible value.
+	ErrBadFees
+
+	// ErrTooManySigOps indicates the total number of signature operations
+	// for a transaction or block exceed the maximum allowed limits.
+	ErrTooManySigOps
+
+	// ErrFirstTxNotCoinbase indicates the first transaction in a block
+	// is not a coinbase transaction.
+	ErrFirstTxNotCoinbase
+
+	// ErrMultipleCoinbases indicates a block contains more than one
+	// coinbase transaction.
+	ErrMultipleCoinbases
+
+	// ErrBadCoinbasePayloadLen indicates the length of the payload
+	// for a coinbase transaction is too high.
+	ErrBadCoinbasePayloadLen
+
+	// ErrBadCoinbaseTransaction indicates that the block's coinbase transaction is not build as expected
+	ErrBadCoinbaseTransaction
+
+	// ErrScriptMalformed indicates a transaction script is malformed in
+	// some way. For example, it might be longer than the maximum allowed
+	// length or fail to parse.
+	ErrScriptMalformed
+
+	// ErrScriptValidation indicates the result of executing transaction
+	// script failed. The error covers any failure when executing scripts
+	// such signature verification failures and execution past the end of
+	// the stack.
+	ErrScriptValidation
+
+	// ErrParentBlockUnknown indicates that the parent block is not known.
+	ErrParentBlockUnknown
+
+	// ErrInvalidAncestorBlock indicates that an ancestor of this block has
+	// already failed validation.
+	ErrInvalidAncestorBlock
+
+	// ErrParentBlockNotCurrentTips indicates that the block's parents are not the
+	// current tips. This is not a block validation rule, but is required
+	// for block proposals submitted via getblocktemplate RPC.
+	ErrParentBlockNotCurrentTips
+
+	// ErrWithDiff indicates that there was an error with UTXOSet.WithDiff
+	ErrWithDiff
+
+	// ErrFinality indicates that a block doesn't adhere to the finality rules
+	ErrFinality
+
+	// ErrTransactionsNotSorted indicates that transactions in block are not
+	// sorted by subnetwork
+	ErrTransactionsNotSorted
+
+	// ErrInvalidGas transaction wants to use more GAS than allowed
+	// by subnetwork
+	ErrInvalidGas
+
+	// ErrInvalidPayload transaction includes a payload in a subnetwork that doesn't allow
+	// a Payload
+	ErrInvalidPayload
+
+	// ErrInvalidPayloadHash invalid hash of transaction's payload
+	ErrInvalidPayloadHash
+
+	// ErrSubnetwork indicates that a block doesn't adhere to the subnetwork
+	// registry rules
+	ErrSubnetworkRegistry
+
+	// ErrInvalidParentsRelation indicates that one of the parents of a block
+	// is also an ancestor of another parent
+	ErrInvalidParentsRelation
+
+	// ErrDelayedBlockIsNotAllowed indicates that a block with a delayed timestamp was
+	// submitted with BFDisallowDelay flag raised.
+	ErrDelayedBlockIsNotAllowed
+
+	// ErrOrphanBlockIsNotAllowed indicates that an orphan block was submitted with
+	// BFDisallowOrphans flag raised.
+	ErrOrphanBlockIsNotAllowed
+)
+
+// Map of ErrorCode values back to their constant names for pretty printing.
+var errorCodeStrings = map[ErrorCode]string{
+	ErrDuplicateBlock:            "ErrDuplicateBlock",
+	ErrBlockMassTooHigh:          "ErrBlockMassTooHigh",
+	ErrBlockVersionTooOld:        "ErrBlockVersionTooOld",
+	ErrTimeTooOld:                "ErrTimeTooOld",
+	ErrTimeTooNew:                "ErrTimeTooNew",
+	ErrNoParents:                 "ErrNoParents",
+	ErrWrongParentsOrder:         "ErrWrongParentsOrder",
+	ErrDifficultyTooLow:          "ErrDifficultyTooLow",
+	ErrUnexpectedDifficulty:      "ErrUnexpectedDifficulty",
+	ErrHighHash:                  "ErrHighHash",
+	ErrBadMerkleRoot:             "ErrBadMerkleRoot",
+	ErrFinalityPointTimeTooOld:   "ErrFinalityPointTimeTooOld",
+	ErrNoTransactions:            "ErrNoTransactions",
+	ErrNoTxInputs:                "ErrNoTxInputs",
+	ErrTxMassTooHigh:             "ErrTxMassTooHigh",
+	ErrBadTxOutValue:             "ErrBadTxOutValue",
+	ErrDuplicateTxInputs:         "ErrDuplicateTxInputs",
+	ErrBadTxInput:                "ErrBadTxInput",
+	ErrMissingTxOut:              "ErrMissingTxOut",
+	ErrDoubleSpendInSameBlock:    "ErrDoubleSpendInSameBlock",
+	ErrUnfinalizedTx:             "ErrUnfinalizedTx",
+	ErrDuplicateTx:               "ErrDuplicateTx",
+	ErrOverwriteTx:               "ErrOverwriteTx",
+	ErrImmatureSpend:             "ErrImmatureSpend",
+	ErrSpendTooHigh:              "ErrSpendTooHigh",
+	ErrBadFees:                   "ErrBadFees",
+	ErrTooManySigOps:             "ErrTooManySigOps",
+	ErrFirstTxNotCoinbase:        "ErrFirstTxNotCoinbase",
+	ErrMultipleCoinbases:         "ErrMultipleCoinbases",
+	ErrBadCoinbasePayloadLen:     "ErrBadCoinbasePayloadLen",
+	ErrBadCoinbaseTransaction:    "ErrBadCoinbaseTransaction",
+	ErrScriptMalformed:           "ErrScriptMalformed",
+	ErrScriptValidation:          "ErrScriptValidation",
+	ErrParentBlockUnknown:        "ErrParentBlockUnknown",
+	ErrInvalidAncestorBlock:      "ErrInvalidAncestorBlock",
+	ErrParentBlockNotCurrentTips: "ErrParentBlockNotCurrentTips",
+	ErrWithDiff:                  "ErrWithDiff",
+	ErrFinality:                  "ErrFinality",
+	ErrTransactionsNotSorted:     "ErrTransactionsNotSorted",
+	ErrInvalidGas:                "ErrInvalidGas",
+	ErrInvalidPayload:            "ErrInvalidPayload",
+	ErrInvalidPayloadHash:        "ErrInvalidPayloadHash",
+	ErrInvalidParentsRelation:    "ErrInvalidParentsRelation",
+	ErrDelayedBlockIsNotAllowed:  "ErrDelayedBlockIsNotAllowed",
+	ErrOrphanBlockIsNotAllowed:   "ErrOrphanBlockIsNotAllowed",
+}
+
+// String returns the ErrorCode as a human-readable name.
+func (e ErrorCode) String() string {
+	if s := errorCodeStrings[e]; s != "" {
+		return s
+	}
+	return fmt.Sprintf("Unknown ErrorCode (%d)", int(e))
+}
+
+// RuleError identifies a rule violation. It is used to indicate that
+// processing of a block or transaction failed due to one of the many validation
+// rules. The caller can use type assertions to determine if a failure was
+// specifically due to a rule violation and access the ErrorCode field to
+// ascertain the specific reason for the rule violation.
+type RuleError struct {
+	ErrorCode   ErrorCode // Describes the kind of error
+	Description string    // Human readable description of the issue
+}
+
+// Error satisfies the error interface and prints human-readable errors.
+func (e RuleError) Error() string {
+	return e.Description
+}
+
+func ruleError(c ErrorCode, desc string) error {
+	return errors.WithStack(RuleError{ErrorCode: c, Description: desc})
+}
+
+// ErrInvalidParameter signifies that an invalid parameter has been
+// supplied to one of the BlockDAG functions.
+var ErrInvalidParameter = errors.New("invalid parameter")

blockdag/error_test.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2014-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"testing"
+)
+
+// TestErrorCodeStringer tests the stringized output for the ErrorCode type.
+func TestErrorCodeStringer(t *testing.T) {
+	tests := []struct {
+		in   ErrorCode
+		want string
+	}{
+		{ErrDuplicateBlock, "ErrDuplicateBlock"},
+		{ErrBlockMassTooHigh, "ErrBlockMassTooHigh"},
+		{ErrBlockVersionTooOld, "ErrBlockVersionTooOld"},
+		{ErrTimeTooOld, "ErrTimeTooOld"},
+		{ErrTimeTooNew, "ErrTimeTooNew"},
+		{ErrNoParents, "ErrNoParents"},
+		{ErrWrongParentsOrder, "ErrWrongParentsOrder"},
+		{ErrDifficultyTooLow, "ErrDifficultyTooLow"},
+		{ErrUnexpectedDifficulty, "ErrUnexpectedDifficulty"},
+		{ErrHighHash, "ErrHighHash"},
+		{ErrBadMerkleRoot, "ErrBadMerkleRoot"},
+		{ErrFinalityPointTimeTooOld, "ErrFinalityPointTimeTooOld"},
+		{ErrNoTransactions, "ErrNoTransactions"},
+		{ErrNoTxInputs, "ErrNoTxInputs"},
+		{ErrTxMassTooHigh, "ErrTxMassTooHigh"},
+		{ErrBadTxOutValue, "ErrBadTxOutValue"},
+		{ErrDuplicateTxInputs, "ErrDuplicateTxInputs"},
+		{ErrBadTxInput, "ErrBadTxInput"},
+		{ErrMissingTxOut, "ErrMissingTxOut"},
+		{ErrUnfinalizedTx, "ErrUnfinalizedTx"},
+		{ErrDuplicateTx, "ErrDuplicateTx"},
+		{ErrOverwriteTx, "ErrOverwriteTx"},
+		{ErrImmatureSpend, "ErrImmatureSpend"},
+		{ErrSpendTooHigh, "ErrSpendTooHigh"},
+		{ErrBadFees, "ErrBadFees"},
+		{ErrTooManySigOps, "ErrTooManySigOps"},
+		{ErrFirstTxNotCoinbase, "ErrFirstTxNotCoinbase"},
+		{ErrMultipleCoinbases, "ErrMultipleCoinbases"},
+		{ErrBadCoinbasePayloadLen, "ErrBadCoinbasePayloadLen"},
+		{ErrBadCoinbaseTransaction, "ErrBadCoinbaseTransaction"},
+		{ErrScriptMalformed, "ErrScriptMalformed"},
+		{ErrScriptValidation, "ErrScriptValidation"},
+		{ErrParentBlockUnknown, "ErrParentBlockUnknown"},
+		{ErrInvalidAncestorBlock, "ErrInvalidAncestorBlock"},
+		{ErrParentBlockNotCurrentTips, "ErrParentBlockNotCurrentTips"},
+		{ErrWithDiff, "ErrWithDiff"},
+		{ErrFinality, "ErrFinality"},
+		{ErrTransactionsNotSorted, "ErrTransactionsNotSorted"},
+		{ErrInvalidGas, "ErrInvalidGas"},
+		{ErrInvalidPayload, "ErrInvalidPayload"},
+		{ErrInvalidPayloadHash, "ErrInvalidPayloadHash"},
+		{ErrInvalidParentsRelation, "ErrInvalidParentsRelation"},
+		{ErrDelayedBlockIsNotAllowed, "ErrDelayedBlockIsNotAllowed"},
+		{ErrOrphanBlockIsNotAllowed, "ErrOrphanBlockIsNotAllowed"},
+		{0xffff, "Unknown ErrorCode (65535)"},
+	}
+
+	t.Logf("Running %d tests", len(tests))
+	for i, test := range tests {
+		result := test.in.String()
+		if result != test.want {
+			t.Errorf("String #%d\n got: %s want: %s", i, result,
+				test.want)
+			continue
+		}
+	}
+}
+
+// TestRuleError tests the error output for the RuleError type.
+func TestRuleError(t *testing.T) {
+	tests := []struct {
+		in   RuleError
+		want string
+	}{
+		{
+			RuleError{Description: "duplicate block"},
+			"duplicate block",
+		},
+		{
+			RuleError{Description: "human-readable error"},
+			"human-readable error",
+		},
+	}
+
+	t.Logf("Running %d tests", len(tests))
+	for i, test := range tests {
+		result := test.in.Error()
+		if result != test.want {
+			t.Errorf("Error #%d\n got: %s want: %s", i, result,
+				test.want)
+			continue
+		}
+	}
+}

blockdag/external_dag_test.go

@@ -0,0 +1,600 @@
+package blockdag_test
+
+import (
+	"fmt"
+	"math"
+	"strings"
+	"testing"
+
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/testtools"
+
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/mining"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+)
+
+// TestFinality checks that the finality mechanism works as expected.
+// This is how the flow goes:
+// 1) We build a chain of params.FinalityInterval blocks and call its tip altChainTip.
+// 2) We build another chain (let's call it mainChain) of 2 * params.FinalityInterval
+// blocks, which points to genesis, and then we check that the block in that
+// chain with height of params.FinalityInterval is marked as finality point (This is
+// very predictable, because the blue score of each new block in a chain is the
+// parents plus one).
+// 3) We make a new child to block with height (2 * params.FinalityInterval - 1)
+// in mainChain, and we check that connecting it to the DAG
+// doesn't affect the last finality point.
+// 4) We make a block that points to genesis, and check that it
+// gets rejected because its blue score is lower then the last finality
+// point.
+// 5) We make a block that points to altChainTip, and check that it
+// gets rejected because it doesn't have the last finality point in
+// its selected parent chain.
+func TestFinality(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.K = 1
+	params.FinalityDuration = 100 * params.TargetTimePerBlock
+	dag, teardownFunc, err := blockdag.DAGSetup("TestFinality", true, blockdag.Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+	buildNodeToDag := func(parentHashes []*daghash.Hash) (*util.Block, error) {
+		msgBlock, err := mining.PrepareBlockForTest(dag, parentHashes, nil, false)
+		if err != nil {
+			return nil, err
+		}
+		block := util.NewBlock(msgBlock)
+
+		isOrphan, isDelayed, err := dag.ProcessBlock(block, blockdag.BFNoPoWCheck)
+		if err != nil {
+			return nil, err
+		}
+		if isDelayed {
+			return nil, errors.Errorf("ProcessBlock: block " +
+				"is too far in the future")
+		}
+		if isOrphan {
+			return nil, errors.Errorf("ProcessBlock: unexpected returned orphan block")
+		}
+
+		return block, nil
+	}
+
+	genesis := util.NewBlock(params.GenesisBlock)
+	currentNode := genesis
+
+	// First we build a chain of params.FinalityInterval blocks for future use
+	for i := uint64(0); i < dag.FinalityInterval(); i++ {
+		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
+		if err != nil {
+			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
+		}
+	}
+
+	altChainTip := currentNode
+
+	// Now we build a new chain of 2 * params.FinalityInterval blocks, pointed to genesis, and
+	// we expect the block with height 1 * params.FinalityInterval to be the last finality point
+	currentNode = genesis
+	for i := uint64(0); i < dag.FinalityInterval(); i++ {
+		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
+		if err != nil {
+			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
+		}
+	}
+
+	expectedFinalityPoint := currentNode
+
+	for i := uint64(0); i < dag.FinalityInterval(); i++ {
+		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
+		if err != nil {
+			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
+		}
+	}
+
+	if !dag.LastFinalityPointHash().IsEqual(expectedFinalityPoint.Hash()) {
+		t.Errorf("TestFinality: dag.lastFinalityPoint expected to be %v but got %v", expectedFinalityPoint, dag.LastFinalityPointHash())
+	}
+
+	// Here we check that even if we create a parallel tip (a new tip with
+	// the same parents as the current one) with the same blue score as the
+	// current tip, it still won't affect the last finality point.
+	_, err = buildNodeToDag(currentNode.MsgBlock().Header.ParentHashes)
+	if err != nil {
+		t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
+	}
+	if !dag.LastFinalityPointHash().IsEqual(expectedFinalityPoint.Hash()) {
+		t.Errorf("TestFinality: dag.lastFinalityPoint was unexpectly changed")
+	}
+
+	// Here we check that a block with lower blue score than the last finality
+	// point will get rejected
+	fakeCoinbaseTx, err := dag.NextBlockCoinbaseTransaction(nil, nil)
+	if err != nil {
+		t.Errorf("NextBlockCoinbaseTransaction: %s", err)
+	}
+	merkleRoot := blockdag.BuildHashMerkleTreeStore([]*util.Tx{fakeCoinbaseTx}).Root()
+	beforeFinalityBlock := domainmessage.NewMsgBlock(&domainmessage.BlockHeader{
+		Version:              0x10000000,
+		ParentHashes:         []*daghash.Hash{genesis.Hash()},
+		HashMerkleRoot:       merkleRoot,
+		AcceptedIDMerkleRoot: &daghash.ZeroHash,
+		UTXOCommitment:       &daghash.ZeroHash,
+		Timestamp:            dag.SelectedTipHeader().Timestamp,
+		Bits:                 genesis.MsgBlock().Header.Bits,
+	})
+	beforeFinalityBlock.AddTransaction(fakeCoinbaseTx.MsgTx())
+	_, _, err = dag.ProcessBlock(util.NewBlock(beforeFinalityBlock), blockdag.BFNoPoWCheck)
+	if err == nil {
+		t.Errorf("TestFinality: buildNodeToDag expected an error but got <nil>")
+	}
+	var ruleErr blockdag.RuleError
+	if errors.As(err, &ruleErr) {
+		if ruleErr.ErrorCode != blockdag.ErrFinality {
+			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, ruleErr.ErrorCode)
+		}
+	} else {
+		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", err)
+	}
+
+	// Here we check that a block that doesn't have the last finality point in
+	// its selected parent chain will get rejected
+	_, err = buildNodeToDag([]*daghash.Hash{altChainTip.Hash()})
+	if err == nil {
+		t.Errorf("TestFinality: buildNodeToDag expected an error but got <nil>")
+	}
+	if errors.As(err, &ruleErr) {
+		if ruleErr.ErrorCode != blockdag.ErrFinality {
+			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, ruleErr.ErrorCode)
+		}
+	} else {
+		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", ruleErr)
+	}
+}
+
+// TestFinalityInterval tests that the finality interval is
+// smaller then domainmessage.MaxInvPerMsg, so when a peer receives
+// a getblocks message it should always be able to send
+// all the necessary invs.
+func TestFinalityInterval(t *testing.T) {
+	netParams := []*dagconfig.Params{
+		&dagconfig.MainnetParams,
+		&dagconfig.TestnetParams,
+		&dagconfig.DevnetParams,
+		&dagconfig.RegressionNetParams,
+		&dagconfig.SimnetParams,
+	}
+	for _, params := range netParams {
+		func() {
+			dag, teardownFunc, err := blockdag.DAGSetup("TestFinalityInterval", true, blockdag.Config{
+				DAGParams: params,
+			})
+			if err != nil {
+				t.Fatalf("Failed to setup dag instance for %s: %v", params.Name, err)
+			}
+			defer teardownFunc()
+
+			if dag.FinalityInterval() > domainmessage.MaxInvPerMsg {
+				t.Errorf("FinalityInterval in %s should be lower or equal to domainmessage.MaxInvPerMsg", params.Name)
+			}
+		}()
+	}
+}
+
+// TestSubnetworkRegistry tests the full subnetwork registry flow
+func TestSubnetworkRegistry(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.K = 1
+	params.BlockCoinbaseMaturity = 0
+	params.EnableNonNativeSubnetworks = true
+	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", true, blockdag.Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	gasLimit := uint64(12345)
+	subnetworkID, err := testtools.RegisterSubnetworkForTest(dag, &params, gasLimit)
+	if err != nil {
+		t.Fatalf("could not register network: %s", err)
+	}
+	limit, err := dag.GasLimit(subnetworkID)
+	if err != nil {
+		t.Fatalf("could not retrieve gas limit: %s", err)
+	}
+	if limit != gasLimit {
+		t.Fatalf("unexpected gas limit. want: %d, got: %d", gasLimit, limit)
+	}
+}
+
+func TestChainedTransactions(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.BlockCoinbaseMaturity = 0
+	// Create a new database and dag instance to run tests against.
+	dag, teardownFunc, err := blockdag.DAGSetup("TestChainedTransactions", true, blockdag.Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	block1, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{params.GenesisHash}, nil, false)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+	isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(block1), blockdag.BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock: %v", err)
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: block1 " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: block1 got unexpectedly orphaned")
+	}
+	cbTx := block1.Transactions[0]
+
+	signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+	if err != nil {
+		t.Fatalf("Failed to build signature script: %s", err)
+	}
+	txIn := &domainmessage.TxIn{
+		PreviousOutpoint: domainmessage.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+	}
+	txOut := &domainmessage.TxOut{
+		ScriptPubKey: blockdag.OpTrueScript,
+		Value:        uint64(1),
+	}
+	tx := domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn}, []*domainmessage.TxOut{txOut})
+
+	chainedTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: domainmessage.Outpoint{TxID: *tx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+	}
+
+	scriptPubKey, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript)
+	if err != nil {
+		t.Fatalf("Failed to build public key script: %s", err)
+	}
+	chainedTxOut := &domainmessage.TxOut{
+		ScriptPubKey: scriptPubKey,
+		Value:        uint64(1),
+	}
+	chainedTx := domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{chainedTxIn}, []*domainmessage.TxOut{chainedTxOut})
+
+	block2, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{block1.BlockHash()}, []*domainmessage.MsgTx{tx}, false)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Manually add a chained transaction to block2
+	block2.Transactions = append(block2.Transactions, chainedTx)
+	block2UtilTxs := make([]*util.Tx, len(block2.Transactions))
+	for i, tx := range block2.Transactions {
+		block2UtilTxs[i] = util.NewTx(tx)
+	}
+	block2.Header.HashMerkleRoot = blockdag.BuildHashMerkleTreeStore(block2UtilTxs).Root()
+
+	//Checks that dag.ProcessBlock fails because we don't allow a transaction to spend another transaction from the same block
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(block2), blockdag.BFNoPoWCheck)
+	if err == nil {
+		t.Errorf("ProcessBlock expected an error")
+	} else {
+		var ruleErr blockdag.RuleError
+		if ok := errors.As(err, &ruleErr); ok {
+			if ruleErr.ErrorCode != blockdag.ErrMissingTxOut {
+				t.Errorf("ProcessBlock expected an %v error code but got %v", blockdag.ErrMissingTxOut, ruleErr.ErrorCode)
+			}
+		} else {
+			t.Errorf("ProcessBlock expected a blockdag.RuleError but got %v", err)
+		}
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: block2 " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Errorf("ProcessBlock: block2 got unexpectedly orphaned")
+	}
+
+	nonChainedTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: domainmessage.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+	}
+	nonChainedTxOut := &domainmessage.TxOut{
+		ScriptPubKey: scriptPubKey,
+		Value:        uint64(1),
+	}
+	nonChainedTx := domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{nonChainedTxIn}, []*domainmessage.TxOut{nonChainedTxOut})
+
+	block3, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{block1.BlockHash()}, []*domainmessage.MsgTx{nonChainedTx}, false)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	//Checks that dag.ProcessBlock doesn't fail because all of its transaction are dependant on transactions from previous blocks
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(block3), blockdag.BFNoPoWCheck)
+	if err != nil {
+		t.Errorf("ProcessBlock: %v", err)
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: block3 " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Errorf("ProcessBlock: block3 got unexpectedly orphaned")
+	}
+}
+
+// TestOrderInDiffFromAcceptanceData makes sure that the order of transactions in
+// dag.diffFromAcceptanceData is such that if txA is spent by txB then txA is processed
+// before txB.
+func TestOrderInDiffFromAcceptanceData(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimnetParams
+	params.K = math.MaxUint8
+	dag, teardownFunc, err := blockdag.DAGSetup("TestOrderInDiffFromAcceptanceData", true, blockdag.Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+	dag.TestSetCoinbaseMaturity(0)
+
+	createBlock := func(previousBlock *util.Block) *util.Block {
+		// Prepare a transaction that spends the previous block's coinbase transaction
+		var txs []*domainmessage.MsgTx
+		if !previousBlock.IsGenesis() {
+			previousCoinbaseTx := previousBlock.MsgBlock().Transactions[0]
+			signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+			if err != nil {
+				t.Fatalf("TestOrderInDiffFromAcceptanceData: Failed to build signature script: %s", err)
+			}
+			txIn := &domainmessage.TxIn{
+				PreviousOutpoint: domainmessage.Outpoint{TxID: *previousCoinbaseTx.TxID(), Index: 0},
+				SignatureScript:  signatureScript,
+				Sequence:         domainmessage.MaxTxInSequenceNum,
+			}
+			txOut := &domainmessage.TxOut{
+				ScriptPubKey: blockdag.OpTrueScript,
+				Value:        uint64(1),
+			}
+			txs = append(txs, domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn}, []*domainmessage.TxOut{txOut}))
+		}
+
+		// Create the block
+		msgBlock, err := mining.PrepareBlockForTest(dag, []*daghash.Hash{previousBlock.Hash()}, txs, false)
+		if err != nil {
+			t.Fatalf("TestOrderInDiffFromAcceptanceData: Failed to prepare block: %s", err)
+		}
+
+		// Add the block to the DAG
+		newBlock := util.NewBlock(msgBlock)
+		isOrphan, isDelayed, err := dag.ProcessBlock(newBlock, blockdag.BFNoPoWCheck)
+		if err != nil {
+			t.Errorf("TestOrderInDiffFromAcceptanceData: %s", err)
+		}
+		if isDelayed {
+			t.Fatalf("TestOrderInDiffFromAcceptanceData: block is too far in the future")
+		}
+		if isOrphan {
+			t.Fatalf("TestOrderInDiffFromAcceptanceData: block got unexpectedly orphaned")
+		}
+		return newBlock
+	}
+
+	// Create two block chains starting from the genesis block. Every time a block is added
+	// one of the chains is selected as the selected parent chain while all the blocks in
+	// the other chain (and their transactions) get accepted by the new virtual. If the
+	// transactions in the non-selected parent chain get processed in the wrong order then
+	// diffFromAcceptanceData panics.
+	blockAmountPerChain := 100
+	chainATip := util.NewBlock(params.GenesisBlock)
+	chainBTip := chainATip
+	for i := 0; i < blockAmountPerChain; i++ {
+		chainATip = createBlock(chainATip)
+		chainBTip = createBlock(chainBTip)
+	}
+}
+
+// TestGasLimit tests the gas limit rules
+func TestGasLimit(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.K = 1
+	params.BlockCoinbaseMaturity = 0
+	params.EnableNonNativeSubnetworks = true
+	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", true, blockdag.Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// First we prepare a subnetwork and a block with coinbase outputs to fund our tests
+	gasLimit := uint64(12345)
+	subnetworkID, err := testtools.RegisterSubnetworkForTest(dag, &params, gasLimit)
+	if err != nil {
+		t.Fatalf("could not register network: %s", err)
+	}
+
+	cbTxs := []*domainmessage.MsgTx{}
+	for i := 0; i < 4; i++ {
+		fundsBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), nil, false)
+		if err != nil {
+			t.Fatalf("PrepareBlockForTest: %v", err)
+		}
+		isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(fundsBlock), blockdag.BFNoPoWCheck)
+		if err != nil {
+			t.Fatalf("ProcessBlock: %v", err)
+		}
+		if isDelayed {
+			t.Fatalf("ProcessBlock: the funds block " +
+				"is too far in the future")
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock: fundsBlock got unexpectedly orphan")
+		}
+
+		cbTxs = append(cbTxs, fundsBlock.Transactions[util.CoinbaseTransactionIndex])
+	}
+
+	signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+	if err != nil {
+		t.Fatalf("Failed to build signature script: %s", err)
+	}
+
+	scriptPubKey, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript)
+	if err != nil {
+		t.Fatalf("Failed to build public key script: %s", err)
+	}
+
+	tx1In := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[0].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
+	}
+	tx1Out := &domainmessage.TxOut{
+		Value:        cbTxs[0].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
+	}
+	tx1 := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{tx1In}, []*domainmessage.TxOut{tx1Out}, subnetworkID, 10000, []byte{})
+
+	tx2In := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[1].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
+	}
+	tx2Out := &domainmessage.TxOut{
+		Value:        cbTxs[1].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
+	}
+	tx2 := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{tx2In}, []*domainmessage.TxOut{tx2Out}, subnetworkID, 10000, []byte{})
+
+	// Here we check that we can't process a block that has transactions that exceed the gas limit
+	overLimitBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{tx1, tx2}, true)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+	isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(overLimitBlock), blockdag.BFNoPoWCheck)
+	if err == nil {
+		t.Fatalf("ProcessBlock expected to have an error in block that exceeds gas limit")
+	}
+	var ruleErr blockdag.RuleError
+	if !errors.As(err, &ruleErr) {
+		t.Fatalf("ProcessBlock expected a RuleError, but got %v", err)
+	} else if ruleErr.ErrorCode != blockdag.ErrInvalidGas {
+		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, ruleErr.ErrorCode)
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: overLimitBlock " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
+	}
+
+	overflowGasTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[2].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
+	}
+	overflowGasTxOut := &domainmessage.TxOut{
+		Value:        cbTxs[2].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
+	}
+	overflowGasTx := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{overflowGasTxIn}, []*domainmessage.TxOut{overflowGasTxOut},
+		subnetworkID, math.MaxUint64, []byte{})
+
+	// Here we check that we can't process a block that its transactions' gas overflows uint64
+	overflowGasBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{tx1, overflowGasTx}, true)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(overflowGasBlock), blockdag.BFNoPoWCheck)
+	if err == nil {
+		t.Fatalf("ProcessBlock expected to have an error")
+	}
+	if !errors.As(err, &ruleErr) {
+		t.Fatalf("ProcessBlock expected a RuleError, but got %v", err)
+	} else if ruleErr.ErrorCode != blockdag.ErrInvalidGas {
+		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, ruleErr.ErrorCode)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: overflowGasBlock " +
+			"is too far in the future")
+	}
+
+	nonExistentSubnetwork := &subnetworkid.SubnetworkID{123}
+	nonExistentSubnetworkTxIn := &domainmessage.TxIn{
+		PreviousOutpoint: *domainmessage.NewOutpoint(cbTxs[3].TxID(), 0),
+		Sequence:         domainmessage.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
+	}
+	nonExistentSubnetworkTxOut := &domainmessage.TxOut{
+		Value:        cbTxs[3].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
+	}
+	nonExistentSubnetworkTx := domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{nonExistentSubnetworkTxIn},
+		[]*domainmessage.TxOut{nonExistentSubnetworkTxOut}, nonExistentSubnetwork, 1, []byte{})
+
+	nonExistentSubnetworkBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{nonExistentSubnetworkTx, overflowGasTx}, true)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Here we check that we can't process a block with a transaction from a non-existent subnetwork
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(nonExistentSubnetworkBlock), blockdag.BFNoPoWCheck)
+	expectedErrStr := fmt.Sprintf("Error getting gas limit for subnetworkID '%s': subnetwork '%s' not found",
+		nonExistentSubnetwork, nonExistentSubnetwork)
+	if strings.Contains(err.Error(), expectedErrStr) {
+		t.Fatalf("ProcessBlock expected error \"%v\" but got \"%v\"", expectedErrStr, err)
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: nonExistentSubnetworkBlock " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: nonExistentSubnetworkBlock got unexpectedly orphan")
+	}
+
+	// Here we check that we can process a block with a transaction that doesn't exceed the gas limit
+	validBlock, err := mining.PrepareBlockForTest(dag, dag.TipHashes(), []*domainmessage.MsgTx{tx1}, true)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(validBlock), blockdag.BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock: %v", err)
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: overLimitBlock " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
+	}
+}

blockdag/ghostdag.go

@@ -0,0 +1,176 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/pkg/errors"
+	"sort"
+)
+
+// ghostdag runs the GHOSTDAG protocol and updates newNode.blues,
+// newNode.selectedParent and newNode.bluesAnticoneSizes accordingly.
+// The function updates newNode.blues by iterating over the blocks in
+// the anticone of newNode.selectedParent (which is the parent with the
+// highest blue score) and adds any block to newNode.blues if by adding
+// it to newNode.blues these conditions will not be violated:
+//
+// 1) |anticone-of-candidate-block ∩ blue-set-of-newNode| ≤ K
+//
+// 2) For every blue block in blue-set-of-newNode:
+//    |(anticone-of-blue-block ∩ blue-set-newNode) ∪ {candidate-block}| ≤ K.
+//    We validate this condition by maintaining a map bluesAnticoneSizes for
+//    each block which holds all the blue anticone sizes that were affected by
+//    the new added blue blocks.
+//    So to find out what is |anticone-of-blue ∩ blue-set-of-newNode| we just iterate in
+//    the selected parent chain of newNode until we find an existing entry in
+//    bluesAnticoneSizes.
+//
+// For further details see the article https://eprint.iacr.org/2018/104.pdf
+func (dag *BlockDAG) ghostdag(newNode *blockNode) (selectedParentAnticone []*blockNode, err error) {
+	newNode.selectedParent = newNode.parents.bluest()
+	newNode.bluesAnticoneSizes[newNode.selectedParent] = 0
+	newNode.blues = []*blockNode{newNode.selectedParent}
+	selectedParentAnticone, err = dag.selectedParentAnticone(newNode)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Slice(selectedParentAnticone, func(i, j int) bool {
+		return selectedParentAnticone[i].less(selectedParentAnticone[j])
+	})
+
+	for _, blueCandidate := range selectedParentAnticone {
+		candidateBluesAnticoneSizes := make(map[*blockNode]dagconfig.KType)
+		var candidateAnticoneSize dagconfig.KType
+		possiblyBlue := true
+
+		// Iterate over all blocks in the blue set of newNode that are not in the past
+		// of blueCandidate, and check for each one of them if blueCandidate potentially
+		// enlarges their blue anticone to be over K, or that they enlarge the blue anticone
+		// of blueCandidate to be over K.
+		for chainBlock := newNode; possiblyBlue; chainBlock = chainBlock.selectedParent {
+			// If blueCandidate is in the future of chainBlock, it means
+			// that all remaining blues are in the past of chainBlock and thus
+			// in the past of blueCandidate. In this case we know for sure that
+			// the anticone of blueCandidate will not exceed K, and we can mark
+			// it as blue.
+			//
+			// newNode is always in the future of blueCandidate, so there's
+			// no point in checking it.
+			if chainBlock != newNode {
+				if isAncestorOfBlueCandidate, err := dag.isInPast(chainBlock, blueCandidate); err != nil {
+					return nil, err
+				} else if isAncestorOfBlueCandidate {
+					break
+				}
+			}
+
+			for _, block := range chainBlock.blues {
+				// Skip blocks that exist in the past of blueCandidate.
+				if isAncestorOfBlueCandidate, err := dag.isInPast(block, blueCandidate); err != nil {
+					return nil, err
+				} else if isAncestorOfBlueCandidate {
+					continue
+				}
+
+				candidateBluesAnticoneSizes[block], err = dag.blueAnticoneSize(block, newNode)
+				if err != nil {
+					return nil, err
+				}
+				candidateAnticoneSize++
+
+				if candidateAnticoneSize > dag.Params.K {
+					// k-cluster violation: The candidate's blue anticone exceeded k
+					possiblyBlue = false
+					break
+				}
+
+				if candidateBluesAnticoneSizes[block] == dag.Params.K {
+					// k-cluster violation: A block in candidate's blue anticone already
+					// has k blue blocks in its own anticone
+					possiblyBlue = false
+					break
+				}
+
+				// This is a sanity check that validates that a blue
+				// block's blue anticone is not already larger than K.
+				if candidateBluesAnticoneSizes[block] > dag.Params.K {
+					return nil, errors.New("found blue anticone size larger than k")
+				}
+			}
+		}
+
+		if possiblyBlue {
+			// No k-cluster violation found, we can now set the candidate block as blue
+			newNode.blues = append(newNode.blues, blueCandidate)
+			newNode.bluesAnticoneSizes[blueCandidate] = candidateAnticoneSize
+			for blue, blueAnticoneSize := range candidateBluesAnticoneSizes {
+				newNode.bluesAnticoneSizes[blue] = blueAnticoneSize + 1
+			}
+
+			// The maximum length of node.blues can be K+1 because
+			// it contains the selected parent.
+			if dagconfig.KType(len(newNode.blues)) == dag.Params.K+1 {
+				break
+			}
+		}
+	}
+
+	newNode.blueScore = newNode.selectedParent.blueScore + uint64(len(newNode.blues))
+	return selectedParentAnticone, nil
+}
+
+// selectedParentAnticone returns the blocks in the anticone of the selected parent of the given node.
+// The function work as follows.
+// We start by adding all parents of the node (other than the selected parent) to a process queue.
+// For each node in the queue:
+//   we check whether it is in the past of the selected parent.
+//   If not, we add the node to the resulting anticone-set and queue it for processing.
+func (dag *BlockDAG) selectedParentAnticone(node *blockNode) ([]*blockNode, error) {
+	anticoneSet := newBlockSet()
+	var anticoneSlice []*blockNode
+	selectedParentPast := newBlockSet()
+	var queue []*blockNode
+	// Queueing all parents (other than the selected parent itself) for processing.
+	for parent := range node.parents {
+		if parent == node.selectedParent {
+			continue
+		}
+		anticoneSet.add(parent)
+		anticoneSlice = append(anticoneSlice, parent)
+		queue = append(queue, parent)
+	}
+	for len(queue) > 0 {
+		var current *blockNode
+		current, queue = queue[0], queue[1:]
+		// For each parent of a the current node we check whether it is in the past of the selected parent. If not,
+		// we add the it to the resulting anticone-set and queue it for further processing.
+		for parent := range current.parents {
+			if anticoneSet.contains(parent) || selectedParentPast.contains(parent) {
+				continue
+			}
+			isAncestorOfSelectedParent, err := dag.isInPast(parent, node.selectedParent)
+			if err != nil {
+				return nil, err
+			}
+			if isAncestorOfSelectedParent {
+				selectedParentPast.add(parent)
+				continue
+			}
+			anticoneSet.add(parent)
+			anticoneSlice = append(anticoneSlice, parent)
+			queue = append(queue, parent)
+		}
+	}
+	return anticoneSlice, nil
+}
+
+// blueAnticoneSize returns the blue anticone size of 'block' from the worldview of 'context'.
+// Expects 'block' to be in the blue set of 'context'
+func (dag *BlockDAG) blueAnticoneSize(block, context *blockNode) (dagconfig.KType, error) {
+	for current := context; current != nil; current = current.selectedParent {
+		if blueAnticoneSize, ok := current.bluesAnticoneSizes[block]; ok {
+			return blueAnticoneSize, nil
+		}
+	}
+	return 0, errors.Errorf("block %s is not in blue set of %s", block.hash, context.hash)
+}

blockdag/ghostdag_test.go

@@ -0,0 +1,386 @@
+package blockdag
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+type testBlockData struct {
+	parents                []string
+	id                     string // id is a virtual entity that is used only for tests so we can define relations between blocks without knowing their hash
+	expectedScore          uint64
+	expectedSelectedParent string
+	expectedBlues          []string
+}
+
+// TestGHOSTDAG iterates over several dag simulations, and checks
+// that the blue score, blue set and selected parent of each
+// block are calculated as expected.
+func TestGHOSTDAG(t *testing.T) {
+	dagParams := dagconfig.SimnetParams
+
+	tests := []struct {
+		k            dagconfig.KType
+		expectedReds []string
+		dagData      []*testBlockData
+	}{
+		{
+			k:            3,
+			expectedReds: []string{"F", "G", "H", "I", "N", "P"},
+			dagData: []*testBlockData{
+				{
+					parents:                []string{"A"},
+					id:                     "B",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"B"},
+					id:                     "C",
+					expectedScore:          2,
+					expectedSelectedParent: "B",
+					expectedBlues:          []string{"B"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "D",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"C", "D"},
+					id:                     "E",
+					expectedScore:          4,
+					expectedSelectedParent: "C",
+					expectedBlues:          []string{"C", "D"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "F",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"F"},
+					id:                     "G",
+					expectedScore:          2,
+					expectedSelectedParent: "F",
+					expectedBlues:          []string{"F"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "H",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"A"},
+					id:                     "I",
+					expectedScore:          1,
+					expectedSelectedParent: "A",
+					expectedBlues:          []string{"A"},
+				},
+				{
+					parents:                []string{"E", "G"},
+					id:                     "J",
+					expectedScore:          5,
+					expectedSelectedParent: "E",
+					expectedBlues:          []string{"E"},
+				},
+				{
+					parents:                []string{"J"},
+					id:                     "K",
+					expectedScore:          6,
+					expectedSelectedParent: "J",
+					expectedBlues:          []string{"J"},
+				},
+				{
+					parents:                []string{"I", "K"},
+					id:                     "L",
+					expectedScore:          7,
+					expectedSelectedParent: "K",
+					expectedBlues:          []string{"K"},
+				},
+				{
+					parents:                []string{"L"},
+					id:                     "M",
+					expectedScore:          8,
+					expectedSelectedParent: "L",
+					expectedBlues:          []string{"L"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "N",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "O",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "P",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "Q",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"M"},
+					id:                     "R",
+					expectedScore:          9,
+					expectedSelectedParent: "M",
+					expectedBlues:          []string{"M"},
+				},
+				{
+					parents:                []string{"R"},
+					id:                     "S",
+					expectedScore:          10,
+					expectedSelectedParent: "R",
+					expectedBlues:          []string{"R"},
+				},
+				{
+					parents:                []string{"N", "O", "P", "Q", "S"},
+					id:                     "T",
+					expectedScore:          13,
+					expectedSelectedParent: "S",
+					expectedBlues:          []string{"S", "O", "Q"},
+				},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		func() {
+			resetExtraNonceForTest()
+			dagParams.K = test.k
+			dag, teardownFunc, err := DAGSetup(fmt.Sprintf("TestGHOSTDAG%d", i), true, Config{
+				DAGParams: &dagParams,
+			})
+			if err != nil {
+				t.Fatalf("Failed to setup dag instance: %v", err)
+			}
+			defer teardownFunc()
+
+			genesisNode := dag.genesis
+			blockByIDMap := make(map[string]*blockNode)
+			idByBlockMap := make(map[*blockNode]string)
+			blockByIDMap["A"] = genesisNode
+			idByBlockMap[genesisNode] = "A"
+
+			for _, blockData := range test.dagData {
+				parents := blockSet{}
+				for _, parentID := range blockData.parents {
+					parent := blockByIDMap[parentID]
+					parents.add(parent)
+				}
+
+				block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+				if err != nil {
+					t.Fatalf("TestGHOSTDAG: block %v got unexpected error from PrepareBlockForTest: %v", blockData.id, err)
+				}
+
+				utilBlock := util.NewBlock(block)
+				isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+				if err != nil {
+					t.Fatalf("TestGHOSTDAG: dag.ProcessBlock got unexpected error for block %v: %v", blockData.id, err)
+				}
+				if isDelayed {
+					t.Fatalf("TestGHOSTDAG: block %s "+
+						"is too far in the future", blockData.id)
+				}
+				if isOrphan {
+					t.Fatalf("TestGHOSTDAG: block %v was unexpectedly orphan", blockData.id)
+				}
+
+				node, ok := dag.index.LookupNode(utilBlock.Hash())
+				if !ok {
+					t.Fatalf("block %s does not exist in the DAG", utilBlock.Hash())
+				}
+
+				blockByIDMap[blockData.id] = node
+				idByBlockMap[node] = blockData.id
+
+				bluesIDs := make([]string, 0, len(node.blues))
+				for _, blue := range node.blues {
+					bluesIDs = append(bluesIDs, idByBlockMap[blue])
+				}
+				selectedParentID := idByBlockMap[node.selectedParent]
+				fullDataStr := fmt.Sprintf("blues: %v, selectedParent: %v, score: %v",
+					bluesIDs, selectedParentID, node.blueScore)
+				if blockData.expectedScore != node.blueScore {
+					t.Errorf("Test %d: Block %v expected to have score %v but got %v (fulldata: %v)",
+						i, blockData.id, blockData.expectedScore, node.blueScore, fullDataStr)
+				}
+				if blockData.expectedSelectedParent != selectedParentID {
+					t.Errorf("Test %d: Block %v expected to have selected parent %v but got %v (fulldata: %v)",
+						i, blockData.id, blockData.expectedSelectedParent, selectedParentID, fullDataStr)
+				}
+				if !reflect.DeepEqual(blockData.expectedBlues, bluesIDs) {
+					t.Errorf("Test %d: Block %v expected to have blues %v but got %v (fulldata: %v)",
+						i, blockData.id, blockData.expectedBlues, bluesIDs, fullDataStr)
+				}
+			}
+
+			reds := make(map[string]bool)
+
+			for id := range blockByIDMap {
+				reds[id] = true
+			}
+
+			for tip := &dag.virtual.blockNode; tip.selectedParent != nil; tip = tip.selectedParent {
+				tipID := idByBlockMap[tip]
+				delete(reds, tipID)
+				for _, blue := range tip.blues {
+					blueID := idByBlockMap[blue]
+					delete(reds, blueID)
+				}
+			}
+			if !checkReds(test.expectedReds, reds) {
+				redsIDs := make([]string, 0, len(reds))
+				for id := range reds {
+					redsIDs = append(redsIDs, id)
+				}
+				sort.Strings(redsIDs)
+				sort.Strings(test.expectedReds)
+				t.Errorf("Test %d: Expected reds %v but got %v", i, test.expectedReds, redsIDs)
+			}
+		}()
+	}
+}
+
+func checkReds(expectedReds []string, reds map[string]bool) bool {
+	if len(expectedReds) != len(reds) {
+		return false
+	}
+	for _, redID := range expectedReds {
+		if !reds[redID] {
+			return false
+		}
+	}
+	return true
+}
+
+func TestBlueAnticoneSizeErrors(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizeErrors", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestBlueAnticoneSizeErrors: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	// Prepare a block chain with size K beginning with the genesis block
+	currentBlockA := dag.Params.GenesisBlock
+	for i := dagconfig.KType(0); i < dag.Params.K; i++ {
+		newBlock := prepareAndProcessBlockByParentMsgBlocks(t, dag, currentBlockA)
+		currentBlockA = newBlock
+	}
+
+	// Prepare another block chain with size K beginning with the genesis block
+	currentBlockB := dag.Params.GenesisBlock
+	for i := dagconfig.KType(0); i < dag.Params.K; i++ {
+		newBlock := prepareAndProcessBlockByParentMsgBlocks(t, dag, currentBlockB)
+		currentBlockB = newBlock
+	}
+
+	// Get references to the tips of the two chains
+	blockNodeA, ok := dag.index.LookupNode(currentBlockA.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", currentBlockA.BlockHash())
+	}
+
+	blockNodeB, ok := dag.index.LookupNode(currentBlockB.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", currentBlockB.BlockHash())
+	}
+
+	// Try getting the blueAnticoneSize between them. Since the two
+	// blocks are not in the anticones of eachother, this should fail.
+	_, err = dag.blueAnticoneSize(blockNodeA, blockNodeB)
+	if err == nil {
+		t.Fatalf("TestBlueAnticoneSizeErrors: blueAnticoneSize unexpectedly succeeded")
+	}
+	expectedErrSubstring := "is not in blue set of"
+	if !strings.Contains(err.Error(), expectedErrSubstring) {
+		t.Fatalf("TestBlueAnticoneSizeErrors: blueAnticoneSize returned wrong error. "+
+			"Want: %s, got: %s", expectedErrSubstring, err)
+	}
+}
+
+func TestGHOSTDAGErrors(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestGHOSTDAGErrors", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestGHOSTDAGErrors: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	// Add two child blocks to the genesis
+	block1 := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.Params.GenesisBlock)
+	block2 := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.Params.GenesisBlock)
+
+	// Add a child block to the previous two blocks
+	block3 := prepareAndProcessBlockByParentMsgBlocks(t, dag, block1, block2)
+
+	// Clear the reachability store
+	dag.reachabilityTree.store.loaded = map[daghash.Hash]*reachabilityData{}
+
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		t.Fatalf("NewTx: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+
+	err = dbaccess.ClearReachabilityData(dbTx)
+	if err != nil {
+		t.Fatalf("ClearReachabilityData: %s", err)
+	}
+
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Commit: %s", err)
+	}
+
+	// Try to rerun GHOSTDAG on the last block. GHOSTDAG uses
+	// reachability data, so we expect it to fail.
+	blockNode3, ok := dag.index.LookupNode(block3.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", block3.BlockHash())
+	}
+	_, err = dag.ghostdag(blockNode3)
+	if err == nil {
+		t.Fatalf("TestGHOSTDAGErrors: ghostdag unexpectedly succeeded")
+	}
+	expectedErrSubstring := "couldn't find reachability data"
+	if !strings.Contains(err.Error(), expectedErrSubstring) {
+		t.Fatalf("TestGHOSTDAGErrors: ghostdag returned wrong error. "+
+			"Want: %s, got: %s", expectedErrSubstring, err)
+	}
+}

blockdag/indexers/README.md

@@ -0,0 +1,24 @@
+indexers
+========
+
+[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
+[![GoDoc](https://godoc.org/github.com/kaspanet/kaspad/blockdag/indexers?status.png)](http://godoc.org/github.com/kaspanet/kaspad/blockdag/indexers)
+
+Package indexers implements optional block chain indexes.
+
+These indexes are typically used to enhance the amount of information available
+via an RPC interface.
+
+## Supported Indexers
+
+- Transaction-by-hash (txindex) Index
+  - Creates a mapping from the hash of each transaction to the block that
+    contains it along with its offset and length within the serialized block
+- Transaction-by-address (addrindex) Index
+  - Creates a mapping from every address to all transactions which either credit
+    or debit the address
+  - Requires the transaction-by-hash index
+- AcceptanceData-by-block Index
+  - Creates a mapping from the hash of each block to the list of transaction this block
+    accepts from it's .Blues
+

blockdag/indexers/acceptanceindex.go

@@ -0,0 +1,184 @@
+package indexers
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// AcceptanceIndex implements a txAcceptanceData by block hash index. That is to say,
+// it stores a mapping between a block's hash and the set of transactions that the
+// block accepts among its blue blocks.
+type AcceptanceIndex struct {
+	dag             *blockdag.BlockDAG
+	databaseContext *dbaccess.DatabaseContext
+}
+
+// Ensure the AcceptanceIndex type implements the Indexer interface.
+var _ Indexer = (*AcceptanceIndex)(nil)
+
+// NewAcceptanceIndex returns a new instance of an indexer that is used to create a
+// mapping between block hashes and their txAcceptanceData.
+//
+// It implements the Indexer interface which plugs into the IndexManager that in
+// turn is used by the blockdag package. This allows the index to be
+// seamlessly maintained along with the DAG.
+func NewAcceptanceIndex() *AcceptanceIndex {
+	return &AcceptanceIndex{}
+}
+
+// DropAcceptanceIndex drops the acceptance index.
+func DropAcceptanceIndex(databaseContext *dbaccess.DatabaseContext) error {
+	dbTx, err := databaseContext.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+
+	err = dbaccess.DropAcceptanceIndex(dbTx)
+	if err != nil {
+		return err
+	}
+
+	return dbTx.Commit()
+}
+
+// Init initializes the hash-based acceptance index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) Init(dag *blockdag.BlockDAG, databaseContext *dbaccess.DatabaseContext) error {
+	idx.dag = dag
+	idx.databaseContext = databaseContext
+	return idx.recover()
+}
+
+// recover attempts to insert any data that's missing from the
+// acceptance index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) recover() error {
+	return idx.dag.ForEachHash(func(hash daghash.Hash) error {
+		dbTx, err := idx.databaseContext.NewTx()
+		if err != nil {
+			return err
+		}
+		defer dbTx.RollbackUnlessClosed()
+
+		exists, err := dbaccess.HasAcceptanceData(dbTx, &hash)
+		if err != nil {
+			return err
+		}
+		if exists {
+			return nil
+		}
+		txAcceptanceData, err := idx.dag.TxsAcceptedByBlockHash(&hash)
+		if err != nil {
+			return err
+		}
+		err = idx.ConnectBlock(dbTx, &hash, txAcceptanceData)
+		if err != nil {
+			return err
+		}
+
+		return dbTx.Commit()
+	})
+}
+
+// ConnectBlock is invoked by the index manager when a new block has been
+// connected to the DAG.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) ConnectBlock(dbContext *dbaccess.TxContext, blockHash *daghash.Hash,
+	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+	serializedTxsAcceptanceData, err := serializeMultiBlockTxsAcceptanceData(txsAcceptanceData)
+	if err != nil {
+		return err
+	}
+	return dbaccess.StoreAcceptanceData(dbContext, blockHash, serializedTxsAcceptanceData)
+}
+
+// TxsAcceptanceData returns the acceptance data of all the transactions that
+// were accepted by the block with hash blockHash.
+func (idx *AcceptanceIndex) TxsAcceptanceData(blockHash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	serializedTxsAcceptanceData, err := dbaccess.FetchAcceptanceData(idx.databaseContext, blockHash)
+	if err != nil {
+		return nil, err
+	}
+	return deserializeMultiBlockTxsAcceptanceData(serializedTxsAcceptanceData)
+}
+
+type serializableTxAcceptanceData struct {
+	MsgTx      domainmessage.MsgTx
+	IsAccepted bool
+}
+
+type serializableBlockTxsAcceptanceData struct {
+	BlockHash        daghash.Hash
+	TxAcceptanceData []serializableTxAcceptanceData
+}
+
+type serializableMultiBlockTxsAcceptanceData []serializableBlockTxsAcceptanceData
+
+func serializeMultiBlockTxsAcceptanceData(
+	multiBlockTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) ([]byte, error) {
+	// Convert MultiBlockTxsAcceptanceData to a serializable format
+	serializableData := make(serializableMultiBlockTxsAcceptanceData, len(multiBlockTxsAcceptanceData))
+	for i, blockTxsAcceptanceData := range multiBlockTxsAcceptanceData {
+		serializableBlockData := serializableBlockTxsAcceptanceData{
+			BlockHash:        blockTxsAcceptanceData.BlockHash,
+			TxAcceptanceData: make([]serializableTxAcceptanceData, len(blockTxsAcceptanceData.TxAcceptanceData)),
+		}
+		for i, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
+			serializableBlockData.TxAcceptanceData[i] = serializableTxAcceptanceData{
+				MsgTx:      *txAcceptanceData.Tx.MsgTx(),
+				IsAccepted: txAcceptanceData.IsAccepted,
+			}
+		}
+		serializableData[i] = serializableBlockData
+	}
+
+	// Serialize
+	var buffer bytes.Buffer
+	encoder := gob.NewEncoder(&buffer)
+	err := encoder.Encode(serializableData)
+	if err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+func deserializeMultiBlockTxsAcceptanceData(
+	serializedTxsAcceptanceData []byte) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	// Deserialize
+	buffer := bytes.NewBuffer(serializedTxsAcceptanceData)
+	decoder := gob.NewDecoder(buffer)
+	var serializedData serializableMultiBlockTxsAcceptanceData
+	err := decoder.Decode(&serializedData)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert serializable format to MultiBlockTxsAcceptanceData
+	multiBlockTxsAcceptanceData := make(blockdag.MultiBlockTxsAcceptanceData, len(serializedData))
+	for i, serializableBlockData := range serializedData {
+		blockTxsAcceptanceData := blockdag.BlockTxsAcceptanceData{
+			BlockHash:        serializableBlockData.BlockHash,
+			TxAcceptanceData: make([]blockdag.TxAcceptanceData, len(serializableBlockData.TxAcceptanceData)),
+		}
+		for i, txData := range serializableBlockData.TxAcceptanceData {
+			msgTx := txData.MsgTx
+			blockTxsAcceptanceData.TxAcceptanceData[i] = blockdag.TxAcceptanceData{
+				Tx:         util.NewTx(&msgTx),
+				IsAccepted: txData.IsAccepted,
+			}
+		}
+		multiBlockTxsAcceptanceData[i] = blockTxsAcceptanceData
+	}
+
+	return multiBlockTxsAcceptanceData, nil
+}

blockdag/indexers/acceptanceindex_test.go

@@ -0,0 +1,340 @@
+package indexers
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"syscall"
+	"testing"
+
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+)
+
+func TestAcceptanceIndexSerializationAndDeserialization(t *testing.T) {
+	// Create test data
+	hash, _ := daghash.NewHashFromStr("1111111111111111111111111111111111111111111111111111111111111111")
+	txIn1 := &domainmessage.TxIn{SignatureScript: []byte{1}, PreviousOutpoint: domainmessage.Outpoint{Index: 1}, Sequence: 0}
+	txIn2 := &domainmessage.TxIn{SignatureScript: []byte{2}, PreviousOutpoint: domainmessage.Outpoint{Index: 2}, Sequence: 0}
+	txOut1 := &domainmessage.TxOut{ScriptPubKey: []byte{1}, Value: 10}
+	txOut2 := &domainmessage.TxOut{ScriptPubKey: []byte{2}, Value: 20}
+	blockTxsAcceptanceData := blockdag.BlockTxsAcceptanceData{
+		BlockHash: *hash,
+		TxAcceptanceData: []blockdag.TxAcceptanceData{
+			{
+				Tx:         util.NewTx(domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn1}, []*domainmessage.TxOut{txOut1})),
+				IsAccepted: true,
+			},
+			{
+				Tx:         util.NewTx(domainmessage.NewNativeMsgTx(domainmessage.TxVersion, []*domainmessage.TxIn{txIn2}, []*domainmessage.TxOut{txOut2})),
+				IsAccepted: false,
+			},
+		},
+	}
+	multiBlockTxsAcceptanceData := blockdag.MultiBlockTxsAcceptanceData{blockTxsAcceptanceData}
+
+	// Serialize
+	serializedTxsAcceptanceData, err := serializeMultiBlockTxsAcceptanceData(multiBlockTxsAcceptanceData)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexSerializationAndDeserialization: serialization failed: %s", err)
+	}
+
+	// Deserialize
+	deserializedTxsAcceptanceData, err := deserializeMultiBlockTxsAcceptanceData(serializedTxsAcceptanceData)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexSerializationAndDeserialization: deserialization failed: %s", err)
+	}
+
+	// Check that they're the same
+	if !reflect.DeepEqual(multiBlockTxsAcceptanceData, deserializedTxsAcceptanceData) {
+		t.Fatalf("TestAcceptanceIndexSerializationAndDeserialization: original data and deseralize data aren't equal")
+	}
+}
+
+// TestAcceptanceIndexRecover tests the recoverability of the
+// acceptance index.
+// It does it by following these steps:
+// * It creates a DAG with enabled acceptance index (let's call it dag1) and
+//   make it process some blocks.
+// * It creates a copy of dag1 (let's call it dag2), and disables the acceptance
+//   index in it.
+// * It processes two more blocks in both dag1 and dag2.
+// * A copy of dag2 is created (let's call it dag3) with enabled
+//   acceptance index
+// * It checks that the two missing blocks are added to dag3 acceptance index by
+//   comparing dag1's last block acceptance data and dag3's last block acceptance
+//   data.
+func TestAcceptanceIndexRecover(t *testing.T) {
+	params := &dagconfig.SimnetParams
+	params.BlockCoinbaseMaturity = 0
+
+	testFiles := []string{
+		"blk_0_to_4.dat",
+		"blk_3B.dat",
+	}
+
+	var blocks []*util.Block
+	for _, file := range testFiles {
+		blockTmp, err := blockdag.LoadBlocks(filepath.Join("../testdata/", file))
+		if err != nil {
+			t.Fatalf("Error loading file: %v\n", err)
+		}
+		blocks = append(blocks, blockTmp...)
+	}
+
+	db1AcceptanceIndex := NewAcceptanceIndex()
+	db1IndexManager := NewManager([]Indexer{db1AcceptanceIndex})
+	db1Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover1")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+	defer os.RemoveAll(db1Path)
+
+	databaseContext1, err := dbaccess.New(db1Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	db1Config := blockdag.Config{
+		IndexManager:    db1IndexManager,
+		DAGParams:       params,
+		DatabaseContext: databaseContext1,
+	}
+
+	db1DAG, teardown, err := blockdag.DAGSetup("", false, db1Config)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
+	}
+	if teardown != nil {
+		defer teardown()
+	}
+
+	for i := 1; i < len(blocks)-2; i++ {
+		isOrphan, isDelayed, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		if err != nil {
+			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
+		}
+		if isDelayed {
+			t.Fatalf("ProcessBlock: block %d "+
+				"is too far in the future", i)
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock incorrectly returned block %v "+
+				"is an orphan\n", i)
+		}
+	}
+
+	db2Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover2")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+	defer os.RemoveAll(db2Path)
+
+	err = copyDirectory(db1Path, db2Path)
+	if err != nil {
+		t.Fatalf("copyDirectory: %s", err)
+	}
+
+	for i := len(blocks) - 2; i < len(blocks); i++ {
+		isOrphan, isDelayed, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		if err != nil {
+			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
+		}
+		if isDelayed {
+			t.Fatalf("ProcessBlock: block %d "+
+				"is too far in the future", i)
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock incorrectly returned block %v "+
+				"is an orphan\n", i)
+		}
+	}
+
+	db1LastBlockAcceptanceData, err := db1AcceptanceIndex.TxsAcceptanceData(blocks[len(blocks)-1].Hash())
+	if err != nil {
+		t.Fatalf("Error fetching acceptance data: %s", err)
+	}
+
+	err = databaseContext1.Close()
+	if err != nil {
+		t.Fatalf("Error closing the database: %s", err)
+	}
+	databaseContext2, err := dbaccess.New(db2Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	db2Config := blockdag.Config{
+		DAGParams:       params,
+		DatabaseContext: databaseContext2,
+	}
+
+	db2DAG, teardown, err := blockdag.DAGSetup("", false, db2Config)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
+	}
+	if teardown != nil {
+		defer teardown()
+	}
+
+	for i := len(blocks) - 2; i < len(blocks); i++ {
+		isOrphan, isDelayed, err := db2DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		if err != nil {
+			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
+		}
+		if isDelayed {
+			t.Fatalf("ProcessBlock: block %d "+
+				"is too far in the future", i)
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock incorrectly returned block %v "+
+				"is an orphan\n", i)
+		}
+	}
+
+	db3Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover3")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+	defer os.RemoveAll(db3Path)
+	err = copyDirectory(db2Path, db3Path)
+	if err != nil {
+		t.Fatalf("copyDirectory: %s", err)
+	}
+
+	err = databaseContext2.Close()
+	if err != nil {
+		t.Fatalf("Error closing the database: %s", err)
+	}
+	databaseContext3, err := dbaccess.New(db3Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	db3AcceptanceIndex := NewAcceptanceIndex()
+	db3IndexManager := NewManager([]Indexer{db3AcceptanceIndex})
+	db3Config := blockdag.Config{
+		IndexManager:    db3IndexManager,
+		DAGParams:       params,
+		DatabaseContext: databaseContext3,
+	}
+
+	_, teardown, err = blockdag.DAGSetup("", false, db3Config)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
+	}
+	if teardown != nil {
+		defer teardown()
+	}
+
+	db3LastBlockAcceptanceData, err := db3AcceptanceIndex.TxsAcceptanceData(blocks[len(blocks)-1].Hash())
+	if err != nil {
+		t.Fatalf("Error fetching acceptance data: %s", err)
+	}
+	if !reflect.DeepEqual(db1LastBlockAcceptanceData, db3LastBlockAcceptanceData) {
+		t.Fatalf("recovery failed")
+	}
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func copyDirectory(scrDir, dest string) error {
+	entries, err := ioutil.ReadDir(scrDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(scrDir, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+		if !ok {
+			return errors.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
+		}
+
+		switch fileInfo.Mode() & os.ModeType {
+		case os.ModeDir:
+			if err := createIfNotExists(destPath, 0755); err != nil {
+				return err
+			}
+			if err := copyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		case os.ModeSymlink:
+			if err := copySymLink(sourcePath, destPath); err != nil {
+				return err
+			}
+		default:
+			if err := copyFile(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+
+		if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
+			return err
+		}
+
+		isSymlink := entry.Mode()&os.ModeSymlink != 0
+		if !isSymlink {
+			if err := os.Chmod(destPath, entry.Mode()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func copyFile(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	in, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func createIfNotExists(dir string, perm os.FileMode) error {
+	if blockdag.FileExists(dir) {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return errors.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
+	}
+
+	return nil
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func copySymLink(source, dest string) error {
+	link, err := os.Readlink(source)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(link, dest)
+}

blockdag/indexers/indexer.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+/*
+Package indexers implements optional block DAG indexes.
+*/
+package indexers
+
+import (
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// Indexer provides a generic interface for an indexer that is managed by an
+// index manager such as the Manager type provided by this package.
+type Indexer interface {
+	// Init is invoked when the index manager is first initializing the
+	// index.
+	Init(dag *blockdag.BlockDAG, databaseContext *dbaccess.DatabaseContext) error
+
+	// ConnectBlock is invoked when the index manager is notified that a new
+	// block has been connected to the DAG.
+	ConnectBlock(dbContext *dbaccess.TxContext,
+		blockHash *daghash.Hash,
+		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData) error
+}

blockdag/indexers/log.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package indexers
+
+import (
+	"github.com/kaspanet/kaspad/logger"
+)
+
+var log, _ = logger.Get(logger.SubsystemTags.INDX)

blockdag/indexers/manager.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package indexers
+
+import (
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// Manager defines an index manager that manages multiple optional indexes and
+// implements the blockdag.IndexManager interface so it can be seamlessly
+// plugged into normal DAG processing.
+type Manager struct {
+	enabledIndexes []Indexer
+}
+
+// Ensure the Manager type implements the blockdag.IndexManager interface.
+var _ blockdag.IndexManager = (*Manager)(nil)
+
+// Init initializes the enabled indexes.
+// This is part of the blockdag.IndexManager interface.
+func (m *Manager) Init(dag *blockdag.BlockDAG, databaseContext *dbaccess.DatabaseContext) error {
+	for _, indexer := range m.enabledIndexes {
+		if err := indexer.Init(dag, databaseContext); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ConnectBlock must be invoked when a block is added to the DAG. It
+// keeps track of the state of each index it is managing, performs some sanity
+// checks, and invokes each indexer.
+//
+// This is part of the blockdag.IndexManager interface.
+func (m *Manager) ConnectBlock(dbContext *dbaccess.TxContext, blockHash *daghash.Hash, txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+
+	// Call each of the currently active optional indexes with the block
+	// being connected so they can update accordingly.
+	for _, index := range m.enabledIndexes {
+		// Notify the indexer with the connected block so it can index it.
+		if err := index.ConnectBlock(dbContext, blockHash, txsAcceptanceData); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// NewManager returns a new index manager with the provided indexes enabled.
+//
+// The manager returned satisfies the blockdag.IndexManager interface and thus
+// cleanly plugs into the normal blockdag processing path.
+func NewManager(enabledIndexes []Indexer) *Manager {
+	return &Manager{
+		enabledIndexes: enabledIndexes,
+	}
+}

blockdag/log.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2013-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/logger"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var spawn = panics.GoroutineWrapperFunc(log)

blockdag/merkle.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2013-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"math"
+
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// MerkleTree holds the hashes of a merkle tree
+type MerkleTree []*daghash.Hash
+
+// Root returns the root of the merkle tree
+func (mt MerkleTree) Root() *daghash.Hash {
+	return mt[len(mt)-1]
+}
+
+// nextPowerOfTwo returns the next highest power of two from a given number if
+// it is not already a power of two. This is a helper function used during the
+// calculation of a merkle tree.
+func nextPowerOfTwo(n int) int {
+	// Return the number if it's already a power of 2.
+	if n&(n-1) == 0 {
+		return n
+	}
+
+	// Figure out and return the next power of two.
+	exponent := uint(math.Log2(float64(n))) + 1
+	return 1 << exponent // 2^exponent
+}
+
+// HashMerkleBranches takes two hashes, treated as the left and right tree
+// nodes, and returns the hash of their concatenation. This is a helper
+// function used to aid in the generation of a merkle tree.
+func HashMerkleBranches(left *daghash.Hash, right *daghash.Hash) *daghash.Hash {
+	// Concatenate the left and right nodes.
+	var hash [daghash.HashSize * 2]byte
+	copy(hash[:daghash.HashSize], left[:])
+	copy(hash[daghash.HashSize:], right[:])
+
+	newHash := daghash.DoubleHashH(hash[:])
+	return &newHash
+}
+
+// BuildHashMerkleTreeStore creates a merkle tree from a slice of transactions, based
+// on their hash. See `buildMerkleTreeStore` for more info.
+func BuildHashMerkleTreeStore(transactions []*util.Tx) MerkleTree {
+	txHashes := make([]*daghash.Hash, len(transactions))
+	for i, tx := range transactions {
+		txHashes[i] = tx.Hash()
+	}
+	return buildMerkleTreeStore(txHashes)
+}
+
+// BuildIDMerkleTreeStore creates a merkle tree from a slice of transactions, based
+// on their ID. See `buildMerkleTreeStore` for more info.
+func BuildIDMerkleTreeStore(transactions []*util.Tx) MerkleTree {
+	txIDs := make([]*daghash.Hash, len(transactions))
+	for i, tx := range transactions {
+		txIDs[i] = (*daghash.Hash)(tx.ID())
+	}
+	return buildMerkleTreeStore(txIDs)
+}
+
+// buildMerkleTreeStore creates a merkle tree from a slice of hashes,
+// stores it using a linear array, and returns a slice of the backing array. A
+// linear array was chosen as opposed to an actual tree structure since it uses
+// about half as much memory. The following describes a merkle tree and how it
+// is stored in a linear array.
+//
+// A merkle tree is a tree in which every non-leaf node is the hash of its
+// children nodes. A diagram depicting how this works for kaspa transactions
+// where h(x) is a double sha256 follows:
+//
+//	         root = h1234 = h(h12 + h34)
+//	        /                           \
+//	  h12 = h(h1 + h2)            h34 = h(h3 + h4)
+//	   /            \              /            \
+//	h1 = h(tx1)  h2 = h(tx2)    h3 = h(tx3)  h4 = h(tx4)
+//
+// The above stored as a linear array is as follows:
+//
+// 	[h1 h2 h3 h4 h12 h34 root]
+//
+// As the above shows, the merkle root is always the last element in the array.
+//
+// The number of inputs is not always a power of two which results in a
+// balanced tree structure as above. In that case, parent nodes with no
+// children are also zero and parent nodes with only a single left node
+// are calculated by concatenating the left node with itself before hashing.
+// Since this function uses nodes that are pointers to the hashes, empty nodes
+// will be nil.
+func buildMerkleTreeStore(hashes []*daghash.Hash) MerkleTree {
+	// Calculate how many entries are required to hold the binary merkle
+	// tree as a linear array and create an array of that size.
+	nextPoT := nextPowerOfTwo(len(hashes))
+	arraySize := nextPoT*2 - 1
+	merkles := make(MerkleTree, arraySize)
+
+	// Create the base transaction hashes and populate the array with them.
+	for i, hash := range hashes {
+		merkles[i] = hash
+	}
+
+	// Start the array offset after the last transaction and adjusted to the
+	// next power of two.
+	offset := nextPoT
+	for i := 0; i < arraySize-1; i += 2 {
+		switch {
+		// When there is no left child node, the parent is nil too.
+		case merkles[i] == nil:
+			merkles[offset] = nil
+
+		// When there is no right child, the parent is generated by
+		// hashing the concatenation of the left child with itself.
+		case merkles[i+1] == nil:
+			newHash := HashMerkleBranches(merkles[i], merkles[i])
+			merkles[offset] = newHash
+
+		// The normal case sets the parent node to the double sha256
+		// of the concatentation of the left and right children.
+		default:
+			newHash := HashMerkleBranches(merkles[i], merkles[i+1])
+			merkles[offset] = newHash
+		}
+		offset++
+	}
+
+	return merkles
+}

blockdag/merkle_test.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/daghash"
+	"testing"
+
+	"github.com/kaspanet/kaspad/util"
+)
+
+// TestMerkle tests the BuildHashMerkleTreeStore API.
+func TestMerkle(t *testing.T) {
+	block := util.NewBlock(&Block100000)
+
+	hashMerkleTree := BuildHashMerkleTreeStore(block.Transactions())
+	calculatedHashMerkleRoot := hashMerkleTree.Root()
+	wantHashMerkleRoot := Block100000.Header.HashMerkleRoot
+	if !wantHashMerkleRoot.IsEqual(calculatedHashMerkleRoot) {
+		t.Errorf("BuildHashMerkleTreeStore: hash merkle root mismatch - "+
+			"got %v, want %v", calculatedHashMerkleRoot, wantHashMerkleRoot)
+	}
+
+	idMerkleTree := BuildIDMerkleTreeStore(block.Transactions())
+	calculatedIDMerkleRoot := idMerkleTree.Root()
+	wantIDMerkleRoot, err := daghash.NewHashFromStr("3f69feb7edf5d0d67930afc990c8ec931e3428d7c7a65d7af6b81079319eb110")
+	if err != nil {
+		t.Errorf("BuildIDMerkleTreeStore: unexpected error: %s", err)
+	}
+	if !calculatedIDMerkleRoot.IsEqual(wantIDMerkleRoot) {
+		t.Errorf("BuildIDMerkleTreeStore: ID merkle root mismatch - "+
+			"got %v, want %v", calculatedIDMerkleRoot, wantIDMerkleRoot)
+	}
+}

blockdag/mining.go

@@ -0,0 +1,129 @@
+package blockdag
+
+import (
+	"bytes"
+	"encoding/binary"
+	"time"
+
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/mstime"
+)
+
+// BlockForMining returns a block with the given transactions
+// that points to the current DAG tips, that is valid from
+// all aspects except proof of work.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) BlockForMining(transactions []*util.Tx) (*domainmessage.MsgBlock, error) {
+	blockTimestamp := dag.NextBlockTime()
+	requiredDifficulty := dag.NextRequiredDifficulty(blockTimestamp)
+
+	// Calculate the next expected block version based on the state of the
+	// rule change deployments.
+	nextBlockVersion, err := dag.CalcNextBlockVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a new block ready to be solved.
+	hashMerkleTree := BuildHashMerkleTreeStore(transactions)
+	acceptedIDMerkleRoot, err := dag.NextAcceptedIDMerkleRootNoLock()
+	if err != nil {
+		return nil, err
+	}
+	var msgBlock domainmessage.MsgBlock
+	for _, tx := range transactions {
+		msgBlock.AddTransaction(tx.MsgTx())
+	}
+
+	multiset, err := dag.NextBlockMultiset()
+	if err != nil {
+		return nil, err
+	}
+
+	msgBlock.Header = domainmessage.BlockHeader{
+		Version:              nextBlockVersion,
+		ParentHashes:         dag.TipHashes(),
+		HashMerkleRoot:       hashMerkleTree.Root(),
+		AcceptedIDMerkleRoot: acceptedIDMerkleRoot,
+		UTXOCommitment:       (*daghash.Hash)(multiset.Finalize()),
+		Timestamp:            blockTimestamp,
+		Bits:                 requiredDifficulty,
+	}
+
+	return &msgBlock, nil
+}
+
+// NextBlockMultiset returns the multiset of an assumed next block
+// built on top of the current tips.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) NextBlockMultiset() (*secp256k1.MultiSet, error) {
+	_, selectedParentPastUTXO, txsAcceptanceData, err := dag.pastUTXO(&dag.virtual.blockNode)
+	if err != nil {
+		return nil, err
+	}
+
+	return dag.virtual.blockNode.calcMultiset(dag, txsAcceptanceData, selectedParentPastUTXO)
+}
+
+// CoinbasePayloadExtraData returns coinbase payload extra data parameter
+// which is built from extra nonce and coinbase flags.
+func CoinbasePayloadExtraData(extraNonce uint64, coinbaseFlags string) ([]byte, error) {
+	extraNonceBytes := make([]byte, 8)
+	binary.LittleEndian.PutUint64(extraNonceBytes, extraNonce)
+	w := &bytes.Buffer{}
+	_, err := w.Write(extraNonceBytes)
+	if err != nil {
+		return nil, err
+	}
+	_, err = w.Write([]byte(coinbaseFlags))
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+// NextCoinbaseFromAddress returns a coinbase transaction for the
+// next block with the given address and extra data in its payload.
+func (dag *BlockDAG) NextCoinbaseFromAddress(payToAddress util.Address, extraData []byte) (*util.Tx, error) {
+	coinbasePayloadScriptPubKey, err := txscript.PayToAddrScript(payToAddress)
+	if err != nil {
+		return nil, err
+	}
+	coinbaseTx, err := dag.NextBlockCoinbaseTransactionNoLock(coinbasePayloadScriptPubKey, extraData)
+	if err != nil {
+		return nil, err
+	}
+	return coinbaseTx, nil
+}
+
+// NextBlockMinimumTime returns the minimum allowed timestamp for a block building
+// on the end of the DAG. In particular, it is one second after
+// the median timestamp of the last several blocks per the DAG consensus
+// rules.
+func (dag *BlockDAG) NextBlockMinimumTime() mstime.Time {
+	return dag.CalcPastMedianTime().Add(time.Millisecond)
+}
+
+// NextBlockTime returns a valid block time for the
+// next block that will point to the existing DAG tips.
+func (dag *BlockDAG) NextBlockTime() mstime.Time {
+	// The timestamp for the block must not be before the median timestamp
+	// of the last several blocks. Thus, choose the maximum between the
+	// current time and one second after the past median time. The current
+	// timestamp is truncated to a millisecond boundary before comparison since a
+	// block timestamp does not supported a precision greater than one
+	// millisecond.
+	newTimestamp := dag.Now()
+	minTimestamp := dag.NextBlockMinimumTime()
+	if newTimestamp.Before(minTimestamp) {
+		newTimestamp = minTimestamp
+	}
+
+	return newTimestamp
+}

blockdag/multisetio.go

@@ -0,0 +1,29 @@
+package blockdag
+
+import (
+	"encoding/binary"
+	"github.com/kaspanet/go-secp256k1"
+	"io"
+)
+
+const multisetPointSize = 32
+
+// serializeMultiset serializes an ECMH multiset.
+func serializeMultiset(w io.Writer, ms *secp256k1.MultiSet) error {
+	serialized := ms.Serialize()
+	err := binary.Write(w, byteOrder, serialized)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// deserializeMultiset deserializes an EMCH multiset.
+func deserializeMultiset(r io.Reader) (*secp256k1.MultiSet, error) {
+	serialized := &secp256k1.SerializedMultiSet{}
+	err := binary.Read(r, byteOrder, serialized[:])
+	if err != nil {
+		return nil, err
+	}
+	return secp256k1.DeserializeMultiSet(serialized)
+}

blockdag/multisetstore.go

@@ -0,0 +1,131 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/locks"
+	"github.com/pkg/errors"
+)
+
+type multisetStore struct {
+	dag    *BlockDAG
+	new    map[daghash.Hash]struct{}
+	loaded map[daghash.Hash]secp256k1.MultiSet
+	mtx    *locks.PriorityMutex
+}
+
+func newMultisetStore(dag *BlockDAG) *multisetStore {
+	return &multisetStore{
+		dag:    dag,
+		new:    make(map[daghash.Hash]struct{}),
+		loaded: make(map[daghash.Hash]secp256k1.MultiSet),
+	}
+}
+
+func (store *multisetStore) setMultiset(node *blockNode, ms *secp256k1.MultiSet) {
+	store.loaded[*node.hash] = *ms
+	store.addToNewBlocks(node.hash)
+}
+
+func (store *multisetStore) addToNewBlocks(blockHash *daghash.Hash) {
+	store.new[*blockHash] = struct{}{}
+}
+
+func multisetNotFoundError(blockHash *daghash.Hash) error {
+	return errors.Errorf("Couldn't find multiset data for block %s", blockHash)
+}
+
+func (store *multisetStore) multisetByBlockNode(node *blockNode) (*secp256k1.MultiSet, error) {
+	ms, exists := store.multisetByBlockHash(node.hash)
+	if !exists {
+		return nil, multisetNotFoundError(node.hash)
+	}
+	return ms, nil
+}
+
+func (store *multisetStore) multisetByBlockHash(hash *daghash.Hash) (*secp256k1.MultiSet, bool) {
+	ms, ok := store.loaded[*hash]
+	return &ms, ok
+}
+
+// flushToDB writes all new multiset data to the database.
+func (store *multisetStore) flushToDB(dbContext *dbaccess.TxContext) error {
+	if len(store.new) == 0 {
+		return nil
+	}
+
+	w := &bytes.Buffer{}
+	for hash := range store.new {
+		hash := hash // Copy hash to a new variable to avoid passing the same pointer
+
+		w.Reset()
+		ms, exists := store.loaded[hash]
+		if !exists {
+			return multisetNotFoundError(&hash)
+		}
+
+		err := serializeMultiset(w, &ms)
+		if err != nil {
+			return err
+		}
+
+		err = store.storeMultiset(dbContext, &hash, w.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *multisetStore) clearNewEntries() {
+	store.new = make(map[daghash.Hash]struct{})
+}
+
+func (store *multisetStore) init(dbContext dbaccess.Context) error {
+	cursor, err := dbaccess.MultisetCursor(dbContext)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		key, err := cursor.Key()
+		if err != nil {
+			return err
+		}
+
+		hash, err := daghash.NewHash(key.Suffix())
+		if err != nil {
+			return err
+		}
+
+		serializedMS, err := cursor.Value()
+		if err != nil {
+			return err
+		}
+
+		ms, err := deserializeMultiset(bytes.NewReader(serializedMS))
+		if err != nil {
+			return err
+		}
+
+		store.loaded[*hash] = *ms
+	}
+	return nil
+}
+
+// storeMultiset stores the multiset data to the database.
+func (store *multisetStore) storeMultiset(dbContext dbaccess.Context, blockHash *daghash.Hash, serializedMS []byte) error {
+	exists, err := dbaccess.HasMultiset(dbContext, blockHash)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		return errors.Errorf("Can't override an existing multiset database entry for block %s", blockHash)
+	}
+
+	return dbaccess.StoreMultiset(dbContext, blockHash, serializedMS)
+}

blockdag/notifications.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2013-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// NotificationType represents the type of a notification message.
+type NotificationType int
+
+// NotificationCallback is used for a caller to provide a callback for
+// notifications about various blockDAG events.
+type NotificationCallback func(*Notification)
+
+// Constants for the type of a notification message.
+const (
+	// NTBlockAdded indicates the associated block was added into
+	// the blockDAG.
+	NTBlockAdded NotificationType = iota
+
+	// NTChainChanged indicates that selected parent
+	// chain had changed.
+	NTChainChanged
+)
+
+// notificationTypeStrings is a map of notification types back to their constant
+// names for pretty printing.
+var notificationTypeStrings = map[NotificationType]string{
+	NTBlockAdded:   "NTBlockAdded",
+	NTChainChanged: "NTChainChanged",
+}
+
+// String returns the NotificationType in human-readable form.
+func (n NotificationType) String() string {
+	if s, ok := notificationTypeStrings[n]; ok {
+		return s
+	}
+	return fmt.Sprintf("Unknown Notification Type (%d)", int(n))
+}
+
+// Notification defines notification that is sent to the caller via the callback
+// function provided during the call to New and consists of a notification type
+// as well as associated data that depends on the type as follows:
+// 	- Added:     *util.Block
+type Notification struct {
+	Type NotificationType
+	Data interface{}
+}
+
+// Subscribe to block DAG notifications. Registers a callback to be executed
+// when various events take place. See the documentation on Notification and
+// NotificationType for details on the types and contents of notifications.
+func (dag *BlockDAG) Subscribe(callback NotificationCallback) {
+	dag.notificationsLock.Lock()
+	defer dag.notificationsLock.Unlock()
+	dag.notifications = append(dag.notifications, callback)
+}
+
+// sendNotification sends a notification with the passed type and data if the
+// caller requested notifications by providing a callback function in the call
+// to New.
+func (dag *BlockDAG) sendNotification(typ NotificationType, data interface{}) {
+	// Generate and send the notification.
+	n := Notification{Type: typ, Data: data}
+	dag.notificationsLock.RLock()
+	defer dag.notificationsLock.RUnlock()
+	for _, callback := range dag.notifications {
+		callback(&n)
+	}
+}
+
+// BlockAddedNotificationData defines data to be sent along with a BlockAdded
+// notification
+type BlockAddedNotificationData struct {
+	Block         *util.Block
+	WasUnorphaned bool
+}
+
+// ChainChangedNotificationData defines data to be sent along with a ChainChanged
+// notification
+type ChainChangedNotificationData struct {
+	RemovedChainBlockHashes []*daghash.Hash
+	AddedChainBlockHashes   []*daghash.Hash
+}

blockdag/notifications_test.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+)
+
+// TestNotifications ensures that notification callbacks are fired on events.
+func TestNotifications(t *testing.T) {
+	blocks, err := LoadBlocks(filepath.Join("testdata/blk_0_to_4.dat"))
+	if err != nil {
+		t.Fatalf("Error loading file: %v\n", err)
+	}
+
+	// Create a new database and dag instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("notifications", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	notificationCount := 0
+	callback := func(notification *Notification) {
+		if notification.Type == NTBlockAdded {
+			notificationCount++
+		}
+	}
+
+	// Register callback multiple times then assert it is called that many
+	// times.
+	const numSubscribers = 3
+	for i := 0; i < numSubscribers; i++ {
+		dag.Subscribe(callback)
+	}
+
+	isOrphan, isDelayed, err := dag.ProcessBlock(blocks[1], BFNone)
+	if err != nil {
+		t.Fatalf("ProcessBlock fail on block 1: %v\n", err)
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock: block 1 " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned block " +
+			"is an orphan\n")
+	}
+
+	if notificationCount != numSubscribers {
+		t.Fatalf("Expected notification callback to be executed %d "+
+			"times, found %d", numSubscribers, notificationCount)
+	}
+}

blockdag/process.go

@@ -0,0 +1,287 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// BehaviorFlags is a bitmask defining tweaks to the normal behavior when
+// performing DAG processing and consensus rules checks.
+type BehaviorFlags uint32
+
+const (
+	// BFFastAdd may be set to indicate that several checks can be avoided
+	// for the block since it is already known to fit into the DAG due to
+	// already proving it correct links into the DAG.
+	BFFastAdd BehaviorFlags = 1 << iota
+
+	// BFNoPoWCheck may be set to indicate the proof of work check which
+	// ensures a block hashes to a value less than the required target will
+	// not be performed.
+	BFNoPoWCheck
+
+	// BFWasUnorphaned may be set to indicate that a block was just now
+	// unorphaned
+	BFWasUnorphaned
+
+	// BFAfterDelay may be set to indicate that a block had timestamp too far
+	// in the future, just finished the delay
+	BFAfterDelay
+
+	// BFIsSync may be set to indicate that the block was sent as part of the
+	// netsync process
+	BFIsSync
+
+	// BFWasStored is set to indicate that the block was previously stored
+	// in the block index but was never fully processed
+	BFWasStored
+
+	// BFDisallowDelay is set to indicate that a delayed block should be rejected.
+	// This is used for the case where a block is submitted through RPC.
+	BFDisallowDelay
+
+	// BFDisallowOrphans is set to indicate that an orphan block should be rejected.
+	// This is used for the case where a block is submitted through RPC.
+	BFDisallowOrphans
+
+	// BFNone is a convenience value to specifically indicate no flags.
+	BFNone BehaviorFlags = 0
+)
+
+// IsInDAG determines whether a block with the given hash exists in
+// the DAG.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) IsInDAG(hash *daghash.Hash) bool {
+	return dag.index.HaveBlock(hash)
+}
+
+// processOrphans determines if there are any orphans which depend on the passed
+// block hash (they are no longer orphans if true) and potentially accepts them.
+// It repeats the process for the newly accepted blocks (to detect further
+// orphans which may no longer be orphans) until there are no more.
+//
+// The flags do not modify the behavior of this function directly, however they
+// are needed to pass along to maybeAcceptBlock.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) error {
+	// Start with processing at least the passed hash. Leave a little room
+	// for additional orphan blocks that need to be processed without
+	// needing to grow the array in the common case.
+	processHashes := make([]*daghash.Hash, 0, 10)
+	processHashes = append(processHashes, hash)
+	for len(processHashes) > 0 {
+		// Pop the first hash to process from the slice.
+		processHash := processHashes[0]
+		processHashes[0] = nil // Prevent GC leak.
+		processHashes = processHashes[1:]
+
+		// Look up all orphans that are parented by the block we just
+		// accepted.  An indexing for loop is
+		// intentionally used over a range here as range does not
+		// reevaluate the slice on each iteration nor does it adjust the
+		// index for the modified slice.
+		for i := 0; i < len(dag.prevOrphans[*processHash]); i++ {
+			orphan := dag.prevOrphans[*processHash][i]
+			if orphan == nil {
+				log.Warnf("Found a nil entry at index %d in the "+
+					"orphan dependency list for block %s", i,
+					processHash)
+				continue
+			}
+
+			// Skip this orphan if one or more of its parents are
+			// still missing.
+			_, err := lookupParentNodes(orphan.block, dag)
+			if err != nil {
+				var ruleErr RuleError
+				if ok := errors.As(err, &ruleErr); ok && ruleErr.ErrorCode == ErrParentBlockUnknown {
+					continue
+				}
+				return err
+			}
+
+			// Remove the orphan from the orphan pool.
+			orphanHash := orphan.block.Hash()
+			dag.removeOrphanBlock(orphan)
+			i--
+
+			// Potentially accept the block into the block DAG.
+			err = dag.maybeAcceptBlock(orphan.block, flags|BFWasUnorphaned)
+			if err != nil {
+				// Since we don't want to reject the original block because of
+				// a bad unorphaned child, only return an error if it's not a RuleError.
+				if !errors.As(err, &RuleError{}) {
+					return err
+				}
+				log.Warnf("Verification failed for orphan block %s: %s", orphanHash, err)
+			}
+
+			// Add this block to the list of blocks to process so
+			// any orphan blocks that depend on this block are
+			// handled too.
+			processHashes = append(processHashes, orphanHash)
+		}
+	}
+	return nil
+}
+
+// ProcessBlock is the main workhorse for handling insertion of new blocks into
+// the block DAG. It includes functionality such as rejecting duplicate
+// blocks, ensuring blocks follow all rules, orphan handling, and insertion into
+// the block DAG.
+//
+// When no errors occurred during processing, the first return value indicates
+// whether or not the block is an orphan.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrphan bool, isDelayed bool, err error) {
+	dag.dagLock.Lock()
+	defer dag.dagLock.Unlock()
+	return dag.processBlockNoLock(block, flags)
+}
+
+func (dag *BlockDAG) processBlockNoLock(block *util.Block, flags BehaviorFlags) (isOrphan bool, isDelayed bool, err error) {
+	isAfterDelay := flags&BFAfterDelay == BFAfterDelay
+	wasBlockStored := flags&BFWasStored == BFWasStored
+	disallowDelay := flags&BFDisallowDelay == BFDisallowDelay
+	disallowOrphans := flags&BFDisallowOrphans == BFDisallowOrphans
+
+	blockHash := block.Hash()
+	log.Tracef("Processing block %s", blockHash)
+
+	// The block must not already exist in the DAG.
+	if dag.IsInDAG(blockHash) && !wasBlockStored {
+		str := fmt.Sprintf("already have block %s", blockHash)
+		return false, false, ruleError(ErrDuplicateBlock, str)
+	}
+
+	// The block must not already exist as an orphan.
+	if _, exists := dag.orphans[*blockHash]; exists {
+		str := fmt.Sprintf("already have block (orphan) %s", blockHash)
+		return false, false, ruleError(ErrDuplicateBlock, str)
+	}
+
+	if dag.isKnownDelayedBlock(blockHash) {
+		str := fmt.Sprintf("already have block (delayed) %s", blockHash)
+		return false, false, ruleError(ErrDuplicateBlock, str)
+	}
+
+	if !isAfterDelay {
+		// Perform preliminary sanity checks on the block and its transactions.
+		delay, err := dag.checkBlockSanity(block, flags)
+		if err != nil {
+			return false, false, err
+		}
+
+		if delay != 0 && disallowDelay {
+			str := fmt.Sprintf("Cannot process blocks beyond the allowed time offset while the BFDisallowDelay flag is raised %s", blockHash)
+			return false, true, ruleError(ErrDelayedBlockIsNotAllowed, str)
+		}
+
+		if delay != 0 {
+			err = dag.addDelayedBlock(block, delay)
+			if err != nil {
+				return false, false, err
+			}
+			return false, true, nil
+		}
+	}
+
+	var missingParents []*daghash.Hash
+	for _, parentHash := range block.MsgBlock().Header.ParentHashes {
+		if !dag.IsInDAG(parentHash) {
+			missingParents = append(missingParents, parentHash)
+		}
+	}
+	if len(missingParents) > 0 && disallowOrphans {
+		str := fmt.Sprintf("Cannot process orphan blocks while the BFDisallowOrphans flag is raised %s", blockHash)
+		return false, false, ruleError(ErrOrphanBlockIsNotAllowed, str)
+	}
+
+	// Handle the case of a block with a valid timestamp(non-delayed) which points to a delayed block.
+	delay, isParentDelayed := dag.maxDelayOfParents(missingParents)
+	if isParentDelayed {
+		// Add Millisecond to ensure that parent process time will be after its child.
+		delay += time.Millisecond
+		err := dag.addDelayedBlock(block, delay)
+		if err != nil {
+			return false, false, err
+		}
+		return false, true, err
+	}
+
+	// Handle orphan blocks.
+	if len(missingParents) > 0 {
+		// Some orphans during netsync are a normal part of the process, since the anticone
+		// of the chain-split is never explicitly requested.
+		// Therefore, if we are during netsync - don't report orphans to default logs.
+		//
+		// The number K*2 was chosen since in peace times anticone is limited to K blocks,
+		// while some red block can make it a bit bigger, but much more than that indicates
+		// there might be some problem with the netsync process.
+		if flags&BFIsSync == BFIsSync && dagconfig.KType(len(dag.orphans)) < dag.Params.K*2 {
+			log.Debugf("Adding orphan block %s. This is normal part of netsync process", blockHash)
+		} else {
+			log.Infof("Adding orphan block %s", blockHash)
+		}
+		dag.addOrphanBlock(block)
+
+		return true, false, nil
+	}
+
+	// The block has passed all context independent checks and appears sane
+	// enough to potentially accept it into the block DAG.
+	err = dag.maybeAcceptBlock(block, flags)
+	if err != nil {
+		return false, false, err
+	}
+
+	// Accept any orphan blocks that depend on this block (they are
+	// no longer orphans) and repeat for those accepted blocks until
+	// there are no more.
+	err = dag.processOrphans(blockHash, flags)
+	if err != nil {
+		return false, false, err
+	}
+
+	if !isAfterDelay {
+		err = dag.processDelayedBlocks()
+		if err != nil {
+			return false, false, err
+		}
+	}
+
+	dag.addBlockProcessingTimestamp()
+
+	log.Debugf("Accepted block %s", blockHash)
+
+	return false, false, nil
+}
+
+// maxDelayOfParents returns the maximum delay of the given block hashes.
+// Note that delay could be 0, but isDelayed will return true. This is the case where the parent process time is due.
+func (dag *BlockDAG) maxDelayOfParents(parentHashes []*daghash.Hash) (delay time.Duration, isDelayed bool) {
+	for _, parentHash := range parentHashes {
+		if delayedParent, exists := dag.delayedBlocks[*parentHash]; exists {
+			isDelayed = true
+			parentDelay := delayedParent.processTime.Sub(dag.Now())
+			if parentDelay > delay {
+				delay = parentDelay
+			}
+		}
+	}
+
+	return delay, isDelayed
+}

blockdag/process_test.go

@@ -0,0 +1,238 @@
+package blockdag
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/kaspanet/kaspad/util"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+func TestProcessOrphans(t *testing.T) {
+	dag, teardownFunc, err := DAGSetup("TestProcessOrphans", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Errorf("Failed to setup dag instance: %v", err)
+		return
+	}
+	defer teardownFunc()
+
+	dag.TestSetCoinbaseMaturity(0)
+
+	blocksFile := "blk_0_to_4.dat"
+	blocks, err := LoadBlocks(filepath.Join("testdata/", blocksFile))
+	if err != nil {
+		t.Fatalf("TestProcessOrphans: "+
+			"Error loading file '%s': %s\n", blocksFile, err)
+	}
+
+	// Get a reference to a parent block
+	parentBlock := blocks[1]
+
+	// Get a reference to a child block and mess with it so that:
+	// a. It gets added to the orphan pool
+	// b. It gets rejected once it's unorphaned
+	childBlock := blocks[2]
+	childBlock.MsgBlock().Header.UTXOCommitment = &daghash.ZeroHash
+
+	// Process the child block so that it gets added to the orphan pool
+	isOrphan, isDelayed, err := dag.ProcessBlock(childBlock, BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("TestProcessOrphans: child block unexpectedly returned an error: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("TestProcessOrphans: child block is too far in the future")
+	}
+	if !isOrphan {
+		t.Fatalf("TestProcessOrphans: incorrectly returned that child block is not an orphan")
+	}
+
+	// Process the parent block. Note that this will attempt to unorphan the child block
+	isOrphan, isDelayed, err = dag.ProcessBlock(parentBlock, BFNone)
+	if err != nil {
+		t.Fatalf("TestProcessOrphans: parent block unexpectedly returned an error: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("TestProcessOrphans: parent block is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("TestProcessOrphans: incorrectly returned that parent block is an orphan")
+	}
+
+	// Make sure that the child block had been rejected
+	node, ok := dag.index.LookupNode(childBlock.Hash())
+	if !ok {
+		t.Fatalf("TestProcessOrphans: child block missing from block index")
+	}
+	if !dag.index.NodeStatus(node).KnownInvalid() {
+		t.Fatalf("TestProcessOrphans: child block erroneously not marked as invalid")
+	}
+}
+
+func TestProcessDelayedBlocks(t *testing.T) {
+	// We use dag1 so we can build the test blocks with the proper
+	// block header (UTXO commitment, acceptedIDMerkleroot, etc), and
+	// then we use dag2 for the actual test.
+	dag1, teardownFunc, err := DAGSetup("TestProcessDelayedBlocks1", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	isDAG1Open := true
+	defer func() {
+		if isDAG1Open {
+			teardownFunc()
+		}
+	}()
+
+	initialTime := dag1.Params.GenesisBlock.Header.Timestamp
+	// Here we use a fake time source that returns a timestamp
+	// one hour into the future to make delayedBlock artificially
+	// valid.
+	dag1.timeSource = newFakeTimeSource(initialTime.Add(time.Hour))
+
+	delayedBlock, err := PrepareBlockForTest(dag1, []*daghash.Hash{dag1.Params.GenesisBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+
+	blockDelay := time.Duration(dag1.Params.TimestampDeviationTolerance)*dag1.Params.TargetTimePerBlock + 5*time.Second
+	delayedBlock.Header.Timestamp = initialTime.Add(blockDelay)
+
+	isOrphan, isDelayed, err := dag1.ProcessBlock(util.NewBlock(delayedBlock), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is an orphan\n")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is delayed\n")
+	}
+
+	delayedBlockChild, err := PrepareBlockForTest(dag1, []*daghash.Hash{delayedBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+
+	teardownFunc()
+	isDAG1Open = false
+
+	// Here the actual test begins. We add a delayed block and
+	// its child and check that they are not added to the DAG,
+	// and check that they're added only if we add a new block
+	// after the delayed block timestamp is valid.
+	dag2, teardownFunc2, err := DAGSetup("TestProcessDelayedBlocks2", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc2()
+	dag2.timeSource = newFakeTimeSource(initialTime)
+
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(delayedBlock), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is an orphan\n")
+	}
+	if !isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlock " +
+			"is not delayed\n")
+	}
+
+	if dag2.IsInDAG(delayedBlock.BlockHash()) {
+		t.Errorf("dag.IsInDAG should return false for a delayed block")
+	}
+	if !dag2.IsKnownBlock(delayedBlock.BlockHash()) {
+		t.Errorf("dag.IsKnownBlock should return true for a a delayed block")
+	}
+
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(delayedBlockChild), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlockChild " +
+			"is an orphan\n")
+	}
+	if !isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned delayedBlockChild " +
+			"is not delayed\n")
+	}
+
+	if dag2.IsInDAG(delayedBlockChild.BlockHash()) {
+		t.Errorf("dag.IsInDAG should return false for a child of a delayed block")
+	}
+	if !dag2.IsKnownBlock(delayedBlockChild.BlockHash()) {
+		t.Errorf("dag.IsKnownBlock should return true for a child of a delayed block")
+	}
+
+	blockBeforeDelay, err := PrepareBlockForTest(dag2, []*daghash.Hash{dag2.Params.GenesisBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(blockBeforeDelay), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is an orphan\n")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is delayed\n")
+	}
+
+	if dag2.IsInDAG(delayedBlock.BlockHash()) {
+		t.Errorf("delayedBlock shouldn't be added to the DAG because its time hasn't reached yet")
+	}
+	if dag2.IsInDAG(delayedBlockChild.BlockHash()) {
+		t.Errorf("delayedBlockChild shouldn't be added to the DAG because its parent is not in the DAG")
+	}
+
+	// We advance the clock to the point where delayedBlock timestamp is valid.
+	deviationTolerance := time.Duration(dag2.TimestampDeviationTolerance) * dag2.Params.TargetTimePerBlock
+	timeUntilDelayedBlockIsValid := delayedBlock.Header.Timestamp.
+		Add(-deviationTolerance).
+		Sub(dag2.Now()) +
+		time.Second
+	dag2.timeSource = newFakeTimeSource(initialTime.Add(timeUntilDelayedBlockIsValid))
+
+	blockAfterDelay, err := PrepareBlockForTest(dag2,
+		[]*daghash.Hash{dag2.Params.GenesisBlock.BlockHash()},
+		nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(blockAfterDelay), BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("ProcessBlock returned unexpected error: %s\n", err)
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is an orphan\n")
+	}
+	if isDelayed {
+		t.Fatalf("ProcessBlock incorrectly returned blockBeforeDelay " +
+			"is not delayed\n")
+	}
+
+	if !dag2.IsInDAG(delayedBlock.BlockHash()) {
+		t.Fatalf("delayedBlock should be added to the DAG because its time has been reached")
+	}
+	if !dag2.IsInDAG(delayedBlockChild.BlockHash()) {
+		t.Errorf("delayedBlockChild shouldn't be added to the DAG because its parent has been added to the DAG")
+	}
+}

blockdag/reachabilitystore.go

@@ -0,0 +1,398 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+	"io"
+)
+
+type reachabilityData struct {
+	treeNode          *reachabilityTreeNode
+	futureCoveringSet futureCoveringTreeNodeSet
+}
+
+type reachabilityStore struct {
+	dag    *BlockDAG
+	dirty  map[daghash.Hash]struct{}
+	loaded map[daghash.Hash]*reachabilityData
+}
+
+func newReachabilityStore(dag *BlockDAG) *reachabilityStore {
+	return &reachabilityStore{
+		dag:    dag,
+		dirty:  make(map[daghash.Hash]struct{}),
+		loaded: make(map[daghash.Hash]*reachabilityData),
+	}
+}
+
+func (store *reachabilityStore) setTreeNode(treeNode *reachabilityTreeNode) {
+	// load the reachability data from DB to store.loaded
+	node := treeNode.blockNode
+	_, exists := store.reachabilityDataByHash(node.hash)
+	if !exists {
+		store.loaded[*node.hash] = &reachabilityData{}
+	}
+
+	store.loaded[*node.hash].treeNode = treeNode
+	store.setBlockAsDirty(node.hash)
+}
+
+func (store *reachabilityStore) setFutureCoveringSet(node *blockNode, futureCoveringSet futureCoveringTreeNodeSet) error {
+	// load the reachability data from DB to store.loaded
+	_, exists := store.reachabilityDataByHash(node.hash)
+	if !exists {
+		return reachabilityNotFoundError(node.hash)
+	}
+
+	store.loaded[*node.hash].futureCoveringSet = futureCoveringSet
+	store.setBlockAsDirty(node.hash)
+	return nil
+}
+
+func (store *reachabilityStore) setBlockAsDirty(blockHash *daghash.Hash) {
+	store.dirty[*blockHash] = struct{}{}
+}
+
+func reachabilityNotFoundError(hash *daghash.Hash) error {
+	return errors.Errorf("couldn't find reachability data for block %s", hash)
+}
+
+func (store *reachabilityStore) treeNodeByBlockHash(hash *daghash.Hash) (*reachabilityTreeNode, error) {
+	reachabilityData, exists := store.reachabilityDataByHash(hash)
+	if !exists {
+		return nil, reachabilityNotFoundError(hash)
+	}
+	return reachabilityData.treeNode, nil
+}
+
+func (store *reachabilityStore) treeNodeByBlockNode(node *blockNode) (*reachabilityTreeNode, error) {
+	return store.treeNodeByBlockHash(node.hash)
+}
+
+func (store *reachabilityStore) futureCoveringSetByBlockNode(node *blockNode) (futureCoveringTreeNodeSet, error) {
+	reachabilityData, exists := store.reachabilityDataByHash(node.hash)
+	if !exists {
+		return nil, reachabilityNotFoundError(node.hash)
+	}
+	return reachabilityData.futureCoveringSet, nil
+}
+
+func (store *reachabilityStore) reachabilityDataByHash(hash *daghash.Hash) (*reachabilityData, bool) {
+	reachabilityData, ok := store.loaded[*hash]
+	return reachabilityData, ok
+}
+
+// flushToDB writes all dirty reachability data to the database.
+func (store *reachabilityStore) flushToDB(dbContext *dbaccess.TxContext) error {
+	if len(store.dirty) == 0 {
+		return nil
+	}
+
+	for hash := range store.dirty {
+		hash := hash // Copy hash to a new variable to avoid passing the same pointer
+		reachabilityData := store.loaded[hash]
+		err := store.storeReachabilityData(dbContext, &hash, reachabilityData)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *reachabilityStore) clearDirtyEntries() {
+	store.dirty = make(map[daghash.Hash]struct{})
+}
+
+func (store *reachabilityStore) init(dbContext dbaccess.Context) error {
+	// TODO: (Stas) This is a quick and dirty hack.
+	// We iterate over the entire bucket twice:
+	// * First, populate the loaded set with all entries
+	// * Second, connect the parent/children pointers in each entry
+	//   with other nodes, which are now guaranteed to exist
+	cursor, err := dbaccess.ReachabilityDataCursor(dbContext)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		err := store.initReachabilityData(cursor)
+		if err != nil {
+			return err
+		}
+	}
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		err := store.loadReachabilityDataFromCursor(cursor)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *reachabilityStore) initReachabilityData(cursor database.Cursor) error {
+	key, err := cursor.Key()
+	if err != nil {
+		return err
+	}
+
+	hash, err := daghash.NewHash(key.Suffix())
+	if err != nil {
+		return err
+	}
+
+	store.loaded[*hash] = &reachabilityData{
+		treeNode:          &reachabilityTreeNode{},
+		futureCoveringSet: nil,
+	}
+	return nil
+}
+
+func (store *reachabilityStore) loadReachabilityDataFromCursor(cursor database.Cursor) error {
+	key, err := cursor.Key()
+	if err != nil {
+		return err
+	}
+
+	hash, err := daghash.NewHash(key.Suffix())
+	if err != nil {
+		return err
+	}
+
+	reachabilityData, ok := store.reachabilityDataByHash(hash)
+	if !ok {
+		return errors.Errorf("cannot find reachability data for block hash: %s", hash)
+	}
+
+	serializedReachabilityData, err := cursor.Value()
+	if err != nil {
+		return err
+	}
+
+	err = store.deserializeReachabilityData(serializedReachabilityData, reachabilityData)
+	if err != nil {
+		return err
+	}
+
+	// Connect the treeNode with its blockNode
+	reachabilityData.treeNode.blockNode, ok = store.dag.index.LookupNode(hash)
+	if !ok {
+		return errors.Errorf("block %s does not exist in the DAG", hash)
+	}
+
+	return nil
+}
+
+// storeReachabilityData stores the reachability data to the database.
+// This overwrites the current entry if there exists one.
+func (store *reachabilityStore) storeReachabilityData(dbContext dbaccess.Context, hash *daghash.Hash, reachabilityData *reachabilityData) error {
+	serializedReachabilyData, err := store.serializeReachabilityData(reachabilityData)
+	if err != nil {
+		return err
+	}
+
+	return dbaccess.StoreReachabilityData(dbContext, hash, serializedReachabilyData)
+}
+
+func (store *reachabilityStore) serializeReachabilityData(reachabilityData *reachabilityData) ([]byte, error) {
+	w := &bytes.Buffer{}
+	err := store.serializeTreeNode(w, reachabilityData.treeNode)
+	if err != nil {
+		return nil, err
+	}
+	err = store.serializeFutureCoveringSet(w, reachabilityData.futureCoveringSet)
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+func (store *reachabilityStore) serializeTreeNode(w io.Writer, treeNode *reachabilityTreeNode) error {
+	// Serialize the interval
+	err := store.serializeReachabilityInterval(w, treeNode.interval)
+	if err != nil {
+		return err
+	}
+
+	// Serialize the parent
+	// If this is the genesis block, write the zero hash instead
+	parentHash := &daghash.ZeroHash
+	if treeNode.parent != nil {
+		parentHash = treeNode.parent.blockNode.hash
+	}
+	err = domainmessage.WriteElement(w, parentHash)
+	if err != nil {
+		return err
+	}
+
+	// Serialize the amount of children
+	err = domainmessage.WriteVarInt(w, uint64(len(treeNode.children)))
+	if err != nil {
+		return err
+	}
+
+	// Serialize the children
+	for _, child := range treeNode.children {
+		err = domainmessage.WriteElement(w, child.blockNode.hash)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) serializeReachabilityInterval(w io.Writer, interval *reachabilityInterval) error {
+	// Serialize start
+	err := domainmessage.WriteElement(w, interval.start)
+	if err != nil {
+		return err
+	}
+
+	// Serialize end
+	err = domainmessage.WriteElement(w, interval.end)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) serializeFutureCoveringSet(w io.Writer, futureCoveringSet futureCoveringTreeNodeSet) error {
+	// Serialize the set size
+	err := domainmessage.WriteVarInt(w, uint64(len(futureCoveringSet)))
+	if err != nil {
+		return err
+	}
+
+	// Serialize each node in the set
+	for _, node := range futureCoveringSet {
+		err = domainmessage.WriteElement(w, node.blockNode.hash)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) deserializeReachabilityData(
+	serializedReachabilityDataBytes []byte, destination *reachabilityData) error {
+
+	r := bytes.NewBuffer(serializedReachabilityDataBytes)
+
+	// Deserialize the tree node
+	err := store.deserializeTreeNode(r, destination)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize the future covering set
+	err = store.deserializeFutureCoveringSet(r, destination)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *reachabilityStore) deserializeTreeNode(r io.Reader, destination *reachabilityData) error {
+	// Deserialize the interval
+	interval, err := store.deserializeReachabilityInterval(r)
+	if err != nil {
+		return err
+	}
+	destination.treeNode.interval = interval
+
+	// Deserialize the parent
+	// If this is the zero hash, this node is the genesis and as such doesn't have a parent
+	parentHash := &daghash.Hash{}
+	err = domainmessage.ReadElement(r, parentHash)
+	if err != nil {
+		return err
+	}
+	if !daghash.ZeroHash.IsEqual(parentHash) {
+		parentReachabilityData, ok := store.reachabilityDataByHash(parentHash)
+		if !ok {
+			return errors.Errorf("parent reachability data not found for hash: %s", parentHash)
+		}
+		destination.treeNode.parent = parentReachabilityData.treeNode
+	}
+
+	// Deserialize the amount of children
+	childCount, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize the children
+	children := make([]*reachabilityTreeNode, childCount)
+	for i := uint64(0); i < childCount; i++ {
+		childHash := &daghash.Hash{}
+		err = domainmessage.ReadElement(r, childHash)
+		if err != nil {
+			return err
+		}
+		childReachabilityData, ok := store.reachabilityDataByHash(childHash)
+		if !ok {
+			return errors.Errorf("child reachability data not found for hash: %s", parentHash)
+		}
+		children[i] = childReachabilityData.treeNode
+	}
+	destination.treeNode.children = children
+
+	return nil
+}
+
+func (store *reachabilityStore) deserializeReachabilityInterval(r io.Reader) (*reachabilityInterval, error) {
+	interval := &reachabilityInterval{}
+
+	// Deserialize start
+	start := uint64(0)
+	err := domainmessage.ReadElement(r, &start)
+	if err != nil {
+		return nil, err
+	}
+	interval.start = start
+
+	// Deserialize end
+	end := uint64(0)
+	err = domainmessage.ReadElement(r, &end)
+	if err != nil {
+		return nil, err
+	}
+	interval.end = end
+
+	return interval, nil
+}
+
+func (store *reachabilityStore) deserializeFutureCoveringSet(r io.Reader, destination *reachabilityData) error {
+	// Deserialize the set size
+	setSize, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize each block in the set
+	futureCoveringSet := make(futureCoveringTreeNodeSet, setSize)
+	for i := uint64(0); i < setSize; i++ {
+		blockHash := &daghash.Hash{}
+		err = domainmessage.ReadElement(r, blockHash)
+		if err != nil {
+			return err
+		}
+		blockReachabilityData, ok := store.reachabilityDataByHash(blockHash)
+		if !ok {
+			return errors.Errorf("block reachability data not found for hash: %s", blockHash)
+		}
+		futureCoveringSet[i] = blockReachabilityData.treeNode
+	}
+	destination.futureCoveringSet = futureCoveringSet
+
+	return nil
+}

blockdag/scriptval.go

@@ -0,0 +1,232 @@
+// Copyright (c) 2013-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+)
+
+// txValidateItem holds a transaction along with which input to validate.
+type txValidateItem struct {
+	txInIndex int
+	txIn      *domainmessage.TxIn
+	tx        *util.Tx
+}
+
+// txValidator provides a type which asynchronously validates transaction
+// inputs. It provides several channels for communication and a processing
+// function that is intended to be in run multiple goroutines.
+type txValidator struct {
+	validateChan chan *txValidateItem
+	quitChan     chan struct{}
+	resultChan   chan error
+	utxoSet      UTXOSet
+	flags        txscript.ScriptFlags
+	sigCache     *txscript.SigCache
+}
+
+// sendResult sends the result of a script pair validation on the internal
+// result channel while respecting the quit channel. This allows orderly
+// shutdown when the validation process is aborted early due to a validation
+// error in one of the other goroutines.
+func (v *txValidator) sendResult(result error) {
+	select {
+	case v.resultChan <- result:
+	case <-v.quitChan:
+	}
+}
+
+// validateHandler consumes items to validate from the internal validate channel
+// and returns the result of the validation on the internal result channel. It
+// must be run as a goroutine.
+func (v *txValidator) validateHandler() {
+out:
+	for {
+		select {
+		case txVI := <-v.validateChan:
+			// Ensure the referenced input utxo is available.
+			txIn := txVI.txIn
+			entry, ok := v.utxoSet.Get(txIn.PreviousOutpoint)
+			if !ok {
+				str := fmt.Sprintf("unable to find unspent "+
+					"output %s referenced from "+
+					"transaction %s input %d",
+					txIn.PreviousOutpoint, txVI.tx.ID(),
+					txVI.txInIndex)
+				err := ruleError(ErrMissingTxOut, str)
+				v.sendResult(err)
+				break out
+			}
+
+			// Create a new script engine for the script pair.
+			sigScript := txIn.SignatureScript
+			scriptPubKey := entry.ScriptPubKey()
+			vm, err := txscript.NewEngine(scriptPubKey, txVI.tx.MsgTx(),
+				txVI.txInIndex, v.flags, v.sigCache)
+			if err != nil {
+				str := fmt.Sprintf("failed to parse input "+
+					"%s:%d which references output %s - "+
+					"%s (input script bytes %x, prev "+
+					"output script bytes %x)",
+					txVI.tx.ID(), txVI.txInIndex,
+					txIn.PreviousOutpoint, err, sigScript, scriptPubKey)
+				err := ruleError(ErrScriptMalformed, str)
+				v.sendResult(err)
+				break out
+			}
+
+			// Execute the script pair.
+			if err := vm.Execute(); err != nil {
+				str := fmt.Sprintf("failed to validate input "+
+					"%s:%d which references output %s - "+
+					"%s (input script bytes %x, prev output "+
+					"script bytes %x)",
+					txVI.tx.ID(), txVI.txInIndex,
+					txIn.PreviousOutpoint, err, sigScript, scriptPubKey)
+				err := ruleError(ErrScriptValidation, str)
+				v.sendResult(err)
+				break out
+			}
+
+			// Validation succeeded.
+			v.sendResult(nil)
+
+		case <-v.quitChan:
+			break out
+		}
+	}
+}
+
+// Validate validates the scripts for all of the passed transaction inputs using
+// multiple goroutines.
+func (v *txValidator) Validate(items []*txValidateItem) error {
+	if len(items) == 0 {
+		return nil
+	}
+
+	// Limit the number of goroutines to do script validation based on the
+	// number of processor cores. This helps ensure the system stays
+	// reasonably responsive under heavy load.
+	maxGoRoutines := runtime.NumCPU() * 3
+	if maxGoRoutines <= 0 {
+		maxGoRoutines = 1
+	}
+	if maxGoRoutines > len(items) {
+		maxGoRoutines = len(items)
+	}
+
+	// Start up validation handlers that are used to asynchronously
+	// validate each transaction input.
+	for i := 0; i < maxGoRoutines; i++ {
+		spawn("txValidator.validateHandler", v.validateHandler)
+	}
+
+	// Validate each of the inputs. The quit channel is closed when any
+	// errors occur so all processing goroutines exit regardless of which
+	// input had the validation error.
+	numInputs := len(items)
+	currentItem := 0
+	processedItems := 0
+	for processedItems < numInputs {
+		// Only send items while there are still items that need to
+		// be processed. The select statement will never select a nil
+		// channel.
+		var validateChan chan *txValidateItem
+		var item *txValidateItem
+		if currentItem < numInputs {
+			validateChan = v.validateChan
+			item = items[currentItem]
+		}
+
+		select {
+		case validateChan <- item:
+			currentItem++
+
+		case err := <-v.resultChan:
+			processedItems++
+			if err != nil {
+				close(v.quitChan)
+				return err
+			}
+		}
+	}
+
+	close(v.quitChan)
+	return nil
+}
+
+// newTxValidator returns a new instance of txValidator to be used for
+// validating transaction scripts asynchronously.
+func newTxValidator(utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscript.SigCache) *txValidator {
+	return &txValidator{
+		validateChan: make(chan *txValidateItem),
+		quitChan:     make(chan struct{}),
+		resultChan:   make(chan error),
+		utxoSet:      utxoSet,
+		sigCache:     sigCache,
+		flags:        flags,
+	}
+}
+
+// ValidateTransactionScripts validates the scripts for the passed transaction
+// using multiple goroutines.
+func ValidateTransactionScripts(tx *util.Tx, utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscript.SigCache) error {
+	// Collect all of the transaction inputs and required information for
+	// validation.
+	txIns := tx.MsgTx().TxIn
+	txValItems := make([]*txValidateItem, 0, len(txIns))
+	for txInIdx, txIn := range txIns {
+		txVI := &txValidateItem{
+			txInIndex: txInIdx,
+			txIn:      txIn,
+			tx:        tx,
+		}
+		txValItems = append(txValItems, txVI)
+	}
+
+	// Validate all of the inputs.
+	validator := newTxValidator(utxoSet, flags, sigCache)
+	return validator.Validate(txValItems)
+}
+
+// checkBlockScripts executes and validates the scripts for all transactions in
+// the passed block using multiple goroutines.
+func checkBlockScripts(block *blockNode, utxoSet UTXOSet, transactions []*util.Tx, scriptFlags txscript.ScriptFlags, sigCache *txscript.SigCache) error {
+	// Collect all of the transaction inputs and required information for
+	// validation for all transactions in the block into a single slice.
+	numInputs := 0
+	for _, tx := range transactions {
+		numInputs += len(tx.MsgTx().TxIn)
+	}
+	txValItems := make([]*txValidateItem, 0, numInputs)
+	for _, tx := range transactions {
+		for txInIdx, txIn := range tx.MsgTx().TxIn {
+			txVI := &txValidateItem{
+				txInIndex: txInIdx,
+				txIn:      txIn,
+				tx:        tx,
+			}
+			txValItems = append(txValItems, txVI)
+		}
+	}
+
+	// Validate all of the inputs.
+	validator := newTxValidator(utxoSet, scriptFlags, sigCache)
+	start := time.Now()
+	if err := validator.Validate(txValItems); err != nil {
+		return err
+	}
+	elapsed := time.Since(start)
+
+	log.Tracef("block %s took %s to verify", block.hash, elapsed)
+
+	return nil
+}

blockdag/scriptval_test.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/kaspanet/kaspad/txscript"
+)
+
+// TestCheckBlockScripts ensures that validating the all of the scripts in a
+// known-good block doesn't return an error.
+func TestCheckBlockScripts(t *testing.T) {
+	t.Skip() // TODO: Reactivate this test once we have blocks from testnet.
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	testBlockNum := 277647
+	blockDataFile := fmt.Sprintf("%d.dat", testBlockNum)
+	blocks, err := LoadBlocks(filepath.Join("testdata/", blockDataFile))
+	if err != nil {
+		t.Errorf("Error loading file: %v\n", err)
+		return
+	}
+	if len(blocks) > 1 {
+		t.Errorf("The test block file must only have one block in it")
+		return
+	}
+	if len(blocks) == 0 {
+		t.Errorf("The test block file may not be empty")
+		return
+	}
+
+	storeDataFile := fmt.Sprintf("%d.utxostore", testBlockNum)
+	utxoSet, err := loadUTXOSet(storeDataFile)
+	if err != nil {
+		t.Errorf("Error loading txstore: %v\n", err)
+		return
+	}
+
+	node := &blockNode{
+		hash: blocks[0].Hash(),
+	}
+
+	scriptFlags := txscript.ScriptNoFlags
+	err = checkBlockScripts(node, utxoSet, blocks[0].Transactions(), scriptFlags, nil)
+	if err != nil {
+		t.Errorf("Transaction script validation failed: %v\n", err)
+		return
+	}
+}

blockdag/subnetworks.go

@@ -0,0 +1,157 @@
+package blockdag
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/kaspad/util"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+)
+
+// registerSubnetworks scans a list of transactions, singles out
+// subnetwork registry transactions, validates them, and registers a new
+// subnetwork based on it.
+// This function returns an error if one or more transactions are invalid
+func registerSubnetworks(dbContext dbaccess.Context, txs []*util.Tx) error {
+	subnetworkRegistryTxs := make([]*domainmessage.MsgTx, 0)
+	for _, tx := range txs {
+		msgTx := tx.MsgTx()
+
+		if msgTx.SubnetworkID.IsEqual(subnetworkid.SubnetworkIDRegistry) {
+			subnetworkRegistryTxs = append(subnetworkRegistryTxs, msgTx)
+		}
+
+		if subnetworkid.Less(subnetworkid.SubnetworkIDRegistry, &msgTx.SubnetworkID) {
+			// Transactions are ordered by subnetwork, so we can safely assume
+			// that the rest of the transactions will not be subnetwork registry
+			// transactions.
+			break
+		}
+	}
+
+	for _, registryTx := range subnetworkRegistryTxs {
+		subnetworkID, err := TxToSubnetworkID(registryTx)
+		if err != nil {
+			return err
+		}
+		exists, err := dbaccess.HasSubnetwork(dbContext, subnetworkID)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			createdSubnetwork := newSubnetwork(registryTx)
+			err := registerSubnetwork(dbContext, subnetworkID, createdSubnetwork)
+			if err != nil {
+				return errors.Errorf("failed registering subnetwork"+
+					"for tx '%s': %s", registryTx.TxHash(), err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// validateSubnetworkRegistryTransaction makes sure that a given subnetwork registry
+// transaction is valid. Such a transaction is valid iff:
+// - Its entire payload is a uint64 (8 bytes)
+func validateSubnetworkRegistryTransaction(tx *domainmessage.MsgTx) error {
+	if len(tx.Payload) != 8 {
+		return ruleError(ErrSubnetworkRegistry, fmt.Sprintf("validation failed: subnetwork registry"+
+			"tx '%s' has an invalid payload", tx.TxHash()))
+	}
+
+	return nil
+}
+
+// TxToSubnetworkID creates a subnetwork ID from a subnetwork registry transaction
+func TxToSubnetworkID(tx *domainmessage.MsgTx) (*subnetworkid.SubnetworkID, error) {
+	txHash := tx.TxHash()
+	return subnetworkid.New(util.Hash160(txHash[:]))
+}
+
+// fetchSubnetwork returns a registered subnetwork.
+func (dag *BlockDAG) fetchSubnetwork(subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
+	serializedSubnetwork, err := dbaccess.FetchSubnetworkData(dag.databaseContext, subnetworkID)
+	if err != nil {
+		return nil, err
+	}
+
+	subnet, err := deserializeSubnetwork(serializedSubnetwork)
+	if err != nil {
+		return nil, err
+	}
+
+	return subnet, nil
+}
+
+// GasLimit returns the gas limit of a registered subnetwork. If the subnetwork does not
+// exist this method returns an error.
+func (dag *BlockDAG) GasLimit(subnetworkID *subnetworkid.SubnetworkID) (uint64, error) {
+	sNet, err := dag.fetchSubnetwork(subnetworkID)
+	if err != nil {
+		return 0, err
+	}
+
+	return sNet.gasLimit, nil
+}
+
+func registerSubnetwork(dbContext dbaccess.Context, subnetworkID *subnetworkid.SubnetworkID, network *subnetwork) error {
+	serializedSubnetwork, err := serializeSubnetwork(network)
+	if err != nil {
+		return errors.Errorf("failed to serialize sub-netowrk '%s': %s", subnetworkID, err)
+	}
+
+	return dbaccess.StoreSubnetwork(dbContext, subnetworkID, serializedSubnetwork)
+}
+
+type subnetwork struct {
+	gasLimit uint64
+}
+
+func newSubnetwork(tx *domainmessage.MsgTx) *subnetwork {
+	return &subnetwork{
+		gasLimit: ExtractGasLimit(tx),
+	}
+}
+
+// ExtractGasLimit extracts the gas limit from the transaction payload
+func ExtractGasLimit(tx *domainmessage.MsgTx) uint64 {
+	return binary.LittleEndian.Uint64(tx.Payload[:8])
+}
+
+// serializeSubnetwork serializes a subnetwork into the following binary format:
+// | gasLimit (8 bytes) |
+func serializeSubnetwork(sNet *subnetwork) ([]byte, error) {
+	serializedSNet := bytes.NewBuffer(make([]byte, 0, 8))
+
+	// Write the gas limit
+	err := binary.Write(serializedSNet, byteOrder, sNet.gasLimit)
+	if err != nil {
+		return nil, errors.Errorf("failed to serialize subnetwork: %s", err)
+	}
+
+	return serializedSNet.Bytes(), nil
+}
+
+// deserializeSubnetwork deserializes a byte slice into a subnetwork.
+// See serializeSubnetwork for the binary format.
+func deserializeSubnetwork(serializedSNetBytes []byte) (*subnetwork, error) {
+	serializedSNet := bytes.NewBuffer(serializedSNetBytes)
+
+	// Read the gas limit
+	var gasLimit uint64
+	err := binary.Read(serializedSNet, byteOrder, &gasLimit)
+	if err != nil {
+		return nil, errors.Errorf("failed to deserialize subnetwork: %s", err)
+	}
+
+	return &subnetwork{
+		gasLimit: gasLimit,
+	}, nil
+}

blockdag/subnetworks_test.go

@@ -0,0 +1,26 @@
+package blockdag
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSerializeSubnetwork(t *testing.T) {
+	sNet := &subnetwork{
+		gasLimit: 1000,
+	}
+
+	serializedSNet, err := serializeSubnetwork(sNet)
+	if err != nil {
+		t.Fatalf("subnetwork serialization unexpectedly failed: %s", err)
+	}
+
+	deserializedSNet, err := deserializeSubnetwork(serializedSNet)
+	if err != nil {
+		t.Fatalf("subnetwork deserialization unexpectedly failed: %s", err)
+	}
+
+	if !reflect.DeepEqual(sNet, deserializedSNet) {
+		t.Errorf("original subnetwork and deserialized subnetwork are not equal")
+	}
+}

blockdag/sync_rate.go

@@ -0,0 +1,60 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+	"time"
+)
+
+const syncRateWindowDuration = 15 * time.Minute
+
+// addBlockProcessingTimestamp adds the last block processing timestamp in order to measure the recent sync rate.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) addBlockProcessingTimestamp() {
+	now := mstime.Now()
+	dag.recentBlockProcessingTimestamps = append(dag.recentBlockProcessingTimestamps, now)
+	dag.removeNonRecentTimestampsFromRecentBlockProcessingTimestamps()
+}
+
+// removeNonRecentTimestampsFromRecentBlockProcessingTimestamps removes timestamps older than syncRateWindowDuration
+// from dag.recentBlockProcessingTimestamps
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) removeNonRecentTimestampsFromRecentBlockProcessingTimestamps() {
+	dag.recentBlockProcessingTimestamps = dag.recentBlockProcessingTimestampsRelevantWindow()
+}
+
+func (dag *BlockDAG) recentBlockProcessingTimestampsRelevantWindow() []mstime.Time {
+	minTime := mstime.Now().Add(-syncRateWindowDuration)
+	windowStartIndex := len(dag.recentBlockProcessingTimestamps)
+	for i, processTime := range dag.recentBlockProcessingTimestamps {
+		if processTime.After(minTime) {
+			windowStartIndex = i
+			break
+		}
+	}
+	return dag.recentBlockProcessingTimestamps[windowStartIndex:]
+}
+
+// syncRate returns the rate of processed
+// blocks in the last syncRateWindowDuration
+// duration.
+func (dag *BlockDAG) syncRate() float64 {
+	dag.RLock()
+	defer dag.RUnlock()
+	return float64(len(dag.recentBlockProcessingTimestampsRelevantWindow())) / syncRateWindowDuration.Seconds()
+}
+
+// IsSyncRateBelowThreshold checks whether the sync rate
+// is below the expected threshold.
+func (dag *BlockDAG) IsSyncRateBelowThreshold(maxDeviation float64) bool {
+	if dag.uptime() < syncRateWindowDuration {
+		return false
+	}
+
+	return dag.syncRate() < 1/dag.Params.TargetTimePerBlock.Seconds()*maxDeviation
+}
+
+func (dag *BlockDAG) uptime() time.Duration {
+	return mstime.Now().Sub(dag.startTime)
+}

blockdag/test_utils.go

@@ -0,0 +1,330 @@
+package blockdag
+
+// This file functions are not considered safe for regular use, and should be used for test purposes only.
+
+import (
+	"compress/bzip2"
+	"encoding/binary"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/kaspanet/kaspad/database/ffldb/ldb"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// FileExists returns whether or not the named file or directory exists.
+func FileExists(name string) bool {
+	if _, err := os.Stat(name); err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}
+
+// DAGSetup is used to create a new db and DAG instance with the genesis
+// block already inserted. In addition to the new DAG instance, it returns
+// a teardown function the caller should invoke when done testing to clean up.
+// The openDB parameter instructs DAGSetup whether or not to also open the
+// database. Setting it to false is useful in tests that handle database
+// opening/closing by themselves.
+func DAGSetup(dbName string, openDb bool, config Config) (*BlockDAG, func(), error) {
+	var teardown func()
+
+	// To make sure that the teardown function is not called before any goroutines finished to run -
+	// overwrite `spawn` to count the number of running goroutines
+	spawnWaitGroup := sync.WaitGroup{}
+	realSpawn := spawn
+	spawn = func(name string, f func()) {
+		spawnWaitGroup.Add(1)
+		realSpawn(name, func() {
+			f()
+			spawnWaitGroup.Done()
+		})
+	}
+
+	if openDb {
+		var err error
+		tmpDir, err := ioutil.TempDir("", "DAGSetup")
+		if err != nil {
+			return nil, nil, errors.Errorf("error creating temp dir: %s", err)
+		}
+
+		// We set ldb.Options here to return nil because normally
+		// the database is initialized with very large caches that
+		// can make opening/closing the database for every test
+		// quite heavy.
+		originalLDBOptions := ldb.Options
+		ldb.Options = func() *opt.Options {
+			return nil
+		}
+
+		dbPath := filepath.Join(tmpDir, dbName)
+		_ = os.RemoveAll(dbPath)
+		databaseContext, err := dbaccess.New(dbPath)
+		if err != nil {
+			return nil, nil, errors.Errorf("error creating db: %s", err)
+		}
+
+		config.DatabaseContext = databaseContext
+
+		// Setup a teardown function for cleaning up. This function is
+		// returned to the caller to be invoked when it is done testing.
+		teardown = func() {
+			spawnWaitGroup.Wait()
+			spawn = realSpawn
+			databaseContext.Close()
+			ldb.Options = originalLDBOptions
+			os.RemoveAll(dbPath)
+		}
+	} else {
+		teardown = func() {
+			spawnWaitGroup.Wait()
+			spawn = realSpawn
+		}
+	}
+
+	config.TimeSource = NewTimeSource()
+	config.SigCache = txscript.NewSigCache(1000)
+
+	// Create the DAG instance.
+	dag, err := New(&config)
+	if err != nil {
+		teardown()
+		err := errors.Errorf("failed to create dag instance: %s", err)
+		return nil, nil, err
+	}
+	return dag, teardown, nil
+}
+
+// OpTrueScript is script returning TRUE
+var OpTrueScript = []byte{txscript.OpTrue}
+
+type txSubnetworkData struct {
+	subnetworkID *subnetworkid.SubnetworkID
+	Gas          uint64
+	Payload      []byte
+}
+
+func createTxForTest(numInputs uint32, numOutputs uint32, outputValue uint64, subnetworkData *txSubnetworkData) *domainmessage.MsgTx {
+	txIns := []*domainmessage.TxIn{}
+	txOuts := []*domainmessage.TxOut{}
+
+	for i := uint32(0); i < numInputs; i++ {
+		txIns = append(txIns, &domainmessage.TxIn{
+			PreviousOutpoint: *domainmessage.NewOutpoint(&daghash.TxID{}, i),
+			SignatureScript:  []byte{},
+			Sequence:         domainmessage.MaxTxInSequenceNum,
+		})
+	}
+
+	for i := uint32(0); i < numOutputs; i++ {
+		txOuts = append(txOuts, &domainmessage.TxOut{
+			ScriptPubKey: OpTrueScript,
+			Value:        outputValue,
+		})
+	}
+
+	if subnetworkData != nil {
+		return domainmessage.NewSubnetworkMsgTx(domainmessage.TxVersion, txIns, txOuts, subnetworkData.subnetworkID, subnetworkData.Gas, subnetworkData.Payload)
+	}
+
+	return domainmessage.NewNativeMsgTx(domainmessage.TxVersion, txIns, txOuts)
+}
+
+// VirtualForTest is an exported version for virtualBlock, so that it can be returned by exported test_util methods
+type VirtualForTest *virtualBlock
+
+// SetVirtualForTest replaces the dag's virtual block. This function is used for test purposes only
+func SetVirtualForTest(dag *BlockDAG, virtual VirtualForTest) VirtualForTest {
+	oldVirtual := dag.virtual
+	dag.virtual = virtual
+	return VirtualForTest(oldVirtual)
+}
+
+// GetVirtualFromParentsForTest generates a virtual block with the given parents.
+func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) (VirtualForTest, error) {
+	parents := newBlockSet()
+	for _, hash := range parentHashes {
+		parent, ok := dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("GetVirtualFromParentsForTest: didn't found node for hash %s", hash)
+		}
+		parents.add(parent)
+	}
+	virtual := newVirtualBlock(dag, parents)
+
+	pastUTXO, _, _, err := dag.pastUTXO(&virtual.blockNode)
+	if err != nil {
+		return nil, err
+	}
+	diffUTXO := pastUTXO.clone().(*DiffUTXOSet)
+	err = diffUTXO.meldToBase()
+	if err != nil {
+		return nil, err
+	}
+	virtual.utxoSet = diffUTXO.base
+
+	return VirtualForTest(virtual), nil
+}
+
+// LoadBlocks reads files containing kaspa gzipped block data from disk
+// and returns them as an array of util.Block.
+func LoadBlocks(filename string) (blocks []*util.Block, err error) {
+	var network = domainmessage.Mainnet
+	var dr io.Reader
+	var fi io.ReadCloser
+
+	fi, err = os.Open(filename)
+	if err != nil {
+		return
+	}
+
+	if strings.HasSuffix(filename, ".bz2") {
+		dr = bzip2.NewReader(fi)
+	} else {
+		dr = fi
+	}
+	defer fi.Close()
+
+	var block *util.Block
+
+	err = nil
+	for height := uint64(0); err == nil; height++ {
+		var rintbuf uint32
+		err = binary.Read(dr, binary.LittleEndian, &rintbuf)
+		if err == io.EOF {
+			// hit end of file at expected offset: no warning
+			height--
+			err = nil
+			break
+		}
+		if err != nil {
+			break
+		}
+		if rintbuf != uint32(network) {
+			break
+		}
+		err = binary.Read(dr, binary.LittleEndian, &rintbuf)
+		blocklen := rintbuf
+
+		rbytes := make([]byte, blocklen)
+
+		// read block
+		dr.Read(rbytes)
+
+		block, err = util.NewBlockFromBytes(rbytes)
+		if err != nil {
+			return
+		}
+		blocks = append(blocks, block)
+	}
+
+	return
+}
+
+// opTrueAddress returns an address pointing to a P2SH anyone-can-spend script
+func opTrueAddress(prefix util.Bech32Prefix) (util.Address, error) {
+	return util.NewAddressScriptHash(OpTrueScript, prefix)
+}
+
+// PrepareBlockForTest generates a block with the proper merkle roots, coinbase transaction etc. This function is used for test purposes only
+func PrepareBlockForTest(dag *BlockDAG, parentHashes []*daghash.Hash, transactions []*domainmessage.MsgTx) (*domainmessage.MsgBlock, error) {
+	newVirtual, err := GetVirtualFromParentsForTest(dag, parentHashes)
+	if err != nil {
+		return nil, err
+	}
+	oldVirtual := SetVirtualForTest(dag, newVirtual)
+	defer SetVirtualForTest(dag, oldVirtual)
+
+	OpTrueAddr, err := opTrueAddress(dag.Params.Prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	blockTransactions := make([]*util.Tx, len(transactions)+1)
+
+	extraNonce := generateDeterministicExtraNonceForTest()
+	coinbasePayloadExtraData, err := CoinbasePayloadExtraData(extraNonce, "")
+	if err != nil {
+		return nil, err
+	}
+
+	blockTransactions[0], err = dag.NextCoinbaseFromAddress(OpTrueAddr, coinbasePayloadExtraData)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, tx := range transactions {
+		blockTransactions[i+1] = util.NewTx(tx)
+	}
+
+	// Sort transactions by subnetwork ID
+	sort.Slice(blockTransactions, func(i, j int) bool {
+		if blockTransactions[i].MsgTx().SubnetworkID.IsEqual(subnetworkid.SubnetworkIDCoinbase) {
+			return true
+		}
+		if blockTransactions[j].MsgTx().SubnetworkID.IsEqual(subnetworkid.SubnetworkIDCoinbase) {
+			return false
+		}
+		return subnetworkid.Less(&blockTransactions[i].MsgTx().SubnetworkID, &blockTransactions[j].MsgTx().SubnetworkID)
+	})
+
+	block, err := dag.BlockForMining(blockTransactions)
+	if err != nil {
+		return nil, err
+	}
+	block.Header.Timestamp = dag.NextBlockMinimumTime()
+	block.Header.Bits = dag.NextRequiredDifficulty(block.Header.Timestamp)
+
+	return block, nil
+}
+
+// PrepareAndProcessBlockForTest prepares a block that points to the given parent
+// hashes and process it.
+func PrepareAndProcessBlockForTest(t *testing.T, dag *BlockDAG, parentHashes []*daghash.Hash, transactions []*domainmessage.MsgTx) *domainmessage.MsgBlock {
+	daghash.Sort(parentHashes)
+	block, err := PrepareBlockForTest(dag, parentHashes, transactions)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	utilBlock := util.NewBlock(block)
+	isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("unexpected error in ProcessBlock: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("block is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("block was unexpectedly orphan")
+	}
+	return block
+}
+
+// generateDeterministicExtraNonceForTest returns a unique deterministic extra nonce for coinbase data, in order to create unique coinbase transactions.
+func generateDeterministicExtraNonceForTest() uint64 {
+	extraNonceForTest++
+	return extraNonceForTest
+}
+
+func resetExtraNonceForTest() {
+	extraNonceForTest = 0
+}
+
+var extraNonceForTest = uint64(0)

blockdag/testdata/reorgtest.hex

@@ -0,0 +1,180 @@
+File path:  reorgTest/blk_0_to_4.dat
+
+Block 0:
+   f9beb4d9
+   1d010000
+   
+   01000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 3ba3edfd 7a7b12b2 7ac72c3e 67768f61 7fc81bc3 888a5132 3a9fb8aa 
+   4b1e5e4a 29ab5f49 ffff001d 1dac2b7c 
+   01
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff4d04ff ff001d01 04455468 65205469 6d657320 30332f4a 
+   616e2f32 30303920 4368616e 63656c6c 6f72206f 6e206272 696e6b20 6f662073 
+   65636f6e 64206261 696c6f75 7420666f 72206261 6e6b73ff ffffff01 00f2052a 
+   01000000 43410467 8afdb0fe 55482719 67f1a671 30b7105c d6a828e0 3909a679 
+   62e0ea1f 61deb649 f6bc3f4c ef38c4f3 5504e51e c112de5c 384df7ba 0b8d578a 
+   4c702b6b f11d5fac 00000000 
+Block 1:
+   f9beb4d9
+   d4000000
+   
+   01000000 6fe28c0a b6f1b372 c1a6a246 ae63f74f 931e8365 e15a089c 68d61900 
+   00000000 3bbd67ad e98fbbb7 0718cd80 f9e9acf9 3b5fae91 7bb2b41d 4c3bb82c 
+   77725ca5 81ad5f49 ffff001d 44e69904 
+   01
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff04722f 2e2bffff ffff0100 f2052a01 00000043 41046868 
+   0737c76d abb801cb 2204f57d be4e4579 e4f710cd 67dc1b42 27592c81 e9b5cf02 
+   b5ac9e8b 4c9f49be 5251056b 6a6d011e 4c37f6b6 d17ede6b 55faa235 19e2ac00 
+   000000 
+Block 2:
+   f9beb4d9
+   95010000
+   
+   01000000 13ca7940 4c11c63e ca906bbd f190b751 2872b857 1b5143ae e8cb5737 
+   00000000 fc07c983 d7391736 0aeda657 29d0d4d3 2533eb84 76ee9d64 aa27538f 
+   9b4fc00a d9af5f49 ffff001d 630bea22 
+   02
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff04eb96 14e5ffff ffff0100 f2052a01 00000043 41046868 
+   0737c76d abb801cb 2204f57d be4e4579 e4f710cd 67dc1b42 27592c81 e9b5cf02 
+   b5ac9e8b 4c9f49be 5251056b 6a6d011e 4c37f6b6 d17ede6b 55faa235 19e2ac00 
+   000000 
+   
+   01000000 0163451d 1002611c 1388d5ba 4ddfdf99 196a86b5 990fb5b0 dc786207 
+   4fdcb8ee d2000000 004a4930 46022100 3dde52c6 5e339f45 7fe1015e 70eed208 
+   872eb71e dd484c07 206b190e cb2ec3f8 02210011 c78dcfd0 3d43fa63 61242a33 
+   6291ba2a 8c1ef5bc d5472126 2468f2bf 8dee4d01 ffffffff 0200ca9a 3b000000 
+   001976a9 14cb2abd e8bccacc 32e893df 3a054b9e f7f227a4 ce88ac00 286bee00 
+   00000019 76a914ee 26c56fc1 d942be8d 7a24b2a1 001dd894 69398088 ac000000 
+   00 
+Block 3:
+   f9beb4d9
+   96020000
+   
+   01000000 7d338254 0506faab 0d4cf179 45dda023 49db51f9 6233f24c 28002258 
+   00000000 4806fe80 bf85931b 882ea645 77ca5a03 22bb8af2 3f277b20 55f160cd 
+   972c8e8b 31b25f49 ffff001d e8f0c653 
+   03
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff044abd 8159ffff ffff0100 f2052a01 00000043 4104b95c 
+   249d84f4 17e3e395 a1274254 28b54067 1cc15881 eb828c17 b722a53f c599e21c 
+   a5e56c90 f340988d 3933acc7 6beb832f d64cab07 8ddf3ce7 32923031 d1a8ac00 
+   000000 
+   
+   01000000 01f287b5 e067e1cf 80f7da8a f89917b5 505094db d82412d9 35b665eb 
+   bad253d3 77010000 008c4930 46022100 96ee0d02 b35fd61e 4960b44f f396f67e 
+   01fe17f9 de4e0c17 b6a963bd ab2b50a6 02210034 920d4daa 7e9f8abe 5675c931 
+   495809f9 0b9c1189 d05fbaf1 dd6696a5 b0d8f301 41046868 0737c76d abb801cb 
+   2204f57d be4e4579 e4f710cd 67dc1b42 27592c81 e9b5cf02 b5ac9e8b 4c9f49be 
+   5251056b 6a6d011e 4c37f6b6 d17ede6b 55faa235 19e2ffff ffff0100 286bee00 
+   00000019 76a914c5 22664fb0 e55cdc5c 0cea73b4 aad97ec8 34323288 ac000000 
+   00 
+   
+   01000000 01f287b5 e067e1cf 80f7da8a f89917b5 505094db d82412d9 35b665eb 
+   bad253d3 77000000 008c4930 46022100 b08b922a c4bde411 1c229f92 9fe6eb6a 
+   50161f98 1f4cf47e a9214d35 bf74d380 022100d2 f6640327 e677a1e1 cc474991 
+   b9a48ba5 bd1e0c94 d1c8df49 f7b0193b 7ea4fa01 4104b95c 249d84f4 17e3e395 
+   a1274254 28b54067 1cc15881 eb828c17 b722a53f c599e21c a5e56c90 f340988d 
+   3933acc7 6beb832f d64cab07 8ddf3ce7 32923031 d1a8ffff ffff0100 ca9a3b00 
+   00000019 76a914c5 22664fb0 e55cdc5c 0cea73b4 aad97ec8 34323288 ac000000 
+   00 
+
+Block 4:
+   f9beb4d9
+   73010000
+   
+   01000000 5da36499 06f35e09 9be42a1d 87b6dd42 11bc1400 6c220694 0807eaae 
+   00000000 48eeeaed 2d9d8522 e6201173 743823fd 4b87cd8a ca8e6408 ec75ca38 
+   302c2ff0 89b45f49 ffff001d 00530839 
+   02
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff04d41d 2213ffff ffff0100 f2052a01 00000043 4104678a 
+   fdb0fe55 48271967 f1a67130 b7105cd6 a828e039 09a67962 e0ea1f61 deb649f6 
+   bc3f4cef 38c4f355 04e51ec1 12de5c38 4df7ba0b 8d578a4c 702b6bf1 1d5fac00 
+   000000 
+   
+   01000000 0163451d 1002611c 1388d5ba 4ddfdf99 196a86b5 990fb5b0 dc786207 
+   4fdcb8ee d2000000 004a4930 46022100 8c8fd57b 48762135 8d8f3e69 19f33e08 
+   804736ff 83db47aa 248512e2 6df9b8ba 022100b0 c59e5ee7 bfcbfcd1 a4d83da9 
+   55fb260e fda7f42a 25522625 a3d6f2d9 1174a701 ffffffff 0100f205 2a010000 
+   001976a9 14c52266 4fb0e55c dc5c0cea 73b4aad9 7ec83432 3288ac00 000000 
+
+File path:  reorgTest/blk_3A.dat
+Block 3A:
+   f9beb4d9
+   96020000
+   
+   01000000 7d338254 0506faab 0d4cf179 45dda023 49db51f9 6233f24c 28002258 
+   00000000 5a15f573 1177a353 bdca7aab 20e16624 dfe90adc 70accadc 68016732 
+   302c20a7 31b25f49 ffff001d 6a901440 
+   03
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff04ad1b e7d5ffff ffff0100 f2052a01 00000043 4104ed83 
+   704c95d8 29046f1a c2780621 1132102c 34e9ac7f fa1b7111 0658e5b9 d1bdedc4 
+   16f5cefc 1db0625c d0c75de8 192d2b59 2d7e3b00 bcfb4a0e 860d880f d1fcac00 
+   000000 
+   
+   01000000 01f287b5 e067e1cf 80f7da8a f89917b5 505094db d82412d9 35b665eb 
+   bad253d3 77010000 008c4930 46022100 96ee0d02 b35fd61e 4960b44f f396f67e 
+   01fe17f9 de4e0c17 b6a963bd ab2b50a6 02210034 920d4daa 7e9f8abe 5675c931 
+   495809f9 0b9c1189 d05fbaf1 dd6696a5 b0d8f301 41046868 0737c76d abb801cb 
+   2204f57d be4e4579 e4f710cd 67dc1b42 27592c81 e9b5cf02 b5ac9e8b 4c9f49be 
+   5251056b 6a6d011e 4c37f6b6 d17ede6b 55faa235 19e2ffff ffff0100 286bee00 
+   00000019 76a914c5 22664fb0 e55cdc5c 0cea73b4 aad97ec8 34323288 ac000000 
+   00 
+   
+   01000000 01f287b5 e067e1cf 80f7da8a f89917b5 505094db d82412d9 35b665eb 
+   bad253d3 77000000 008c4930 46022100 9cc67ddd aa6f592a 6b2babd4 d6ff954f 
+   25a784cf 4fe4bb13 afb9f49b 08955119 022100a2 d99545b7 94080757 fcf2b563 
+   f2e91287 86332f46 0ec6b90f f085fb28 41a69701 4104b95c 249d84f4 17e3e395 
+   a1274254 28b54067 1cc15881 eb828c17 b722a53f c599e21c a5e56c90 f340988d 
+   3933acc7 6beb832f d64cab07 8ddf3ce7 32923031 d1a8ffff ffff0100 ca9a3b00 
+   00000019 76a914ee 26c56fc1 d942be8d 7a24b2a1 001dd894 69398088 ac000000 
+   00 
+
+File path:  reorgTest/blk_4A.dat
+Block 4A:
+   f9beb4d9
+   d4000000
+   
+   01000000 aae77468 2205667d 4f413a58 47cc8fe8 9795f1d5 645d5b24 1daf3c92 
+   00000000 361c9cde a09637a0 d0c05c3b 4e7a5d91 9edb184a 0a4c7633 d92e2ddd 
+   f04cb854 89b45f49 ffff001d 9e9aa1e8 
+   01
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff0401b8 f3eaffff ffff0100 f2052a01 00000043 4104678a 
+   fdb0fe55 48271967 f1a67130 b7105cd6 a828e039 09a67962 e0ea1f61 deb649f6 
+   bc3f4cef 38c4f355 04e51ec1 12de5c38 4df7ba0b 8d578a4c 702b6bf1 1d5fac00 
+   000000 
+
+File path:  reorgTest/blk_5A.dat
+Block 5A:
+   f9beb4d9
+   73010000
+   
+   01000000 ebc7d0de 9c31a71b 7f41d275 2c080ba4 11e1854b d45cb2cf 8c1e4624 
+   00000000 a607774b 79b8eb50 b52a5a32 c1754281 ec67f626 9561df28 57d1fe6a 
+   ea82c696 e1b65f49 ffff001d 4a263577 
+   02
+   
+   01000000 01000000 00000000 00000000 00000000 00000000 00000000 00000000 
+   00000000 00ffffff ff049971 0c7dffff ffff0100 f2052a01 00000043 4104678a 
+   fdb0fe55 48271967 f1a67130 b7105cd6 a828e039 09a67962 e0ea1f61 deb649f6 
+   bc3f4cef 38c4f355 04e51ec1 12de5c38 4df7ba0b 8d578a4c 702b6bf1 1d5fac00 
+   000000 
+   
+   01000000 0163451d 1002611c 1388d5ba 4ddfdf99 196a86b5 990fb5b0 dc786207 
+   4fdcb8ee d2000000 004a4930 46022100 8c8fd57b 48762135 8d8f3e69 19f33e08 
+   804736ff 83db47aa 248512e2 6df9b8ba 022100b0 c59e5ee7 bfcbfcd1 a4d83da9 
+   55fb260e fda7f42a 25522625 a3d6f2d9 1174a701 ffffffff 0100f205 2a010000 
+   001976a9 14c52266 4fb0e55c dc5c0cea 73b4aad9 7ec83432 3288ac00 000000 
+

blockdag/thresholdstate.go

@@ -0,0 +1,356 @@
+// Copyright (c) 2016-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"fmt"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+)
+
+// ThresholdState define the various threshold states used when voting on
+// consensus changes.
+type ThresholdState byte
+
+// These constants are used to identify specific threshold states.
+const (
+	// ThresholdDefined is the first state for each deployment and is the
+	// state for the genesis block has by definition for all deployments.
+	ThresholdDefined ThresholdState = iota
+
+	// ThresholdStarted is the state for a deployment once its start time
+	// has been reached.
+	ThresholdStarted
+
+	// ThresholdLockedIn is the state for a deployment during the retarget
+	// period which is after the ThresholdStarted state period and the
+	// number of blocks that have voted for the deployment equal or exceed
+	// the required number of votes for the deployment.
+	ThresholdLockedIn
+
+	// ThresholdActive is the state for a deployment for all blocks after a
+	// retarget period in which the deployment was in the ThresholdLockedIn
+	// state.
+	ThresholdActive
+
+	// ThresholdFailed is the state for a deployment once its expiration
+	// time has been reached and it did not reach the ThresholdLockedIn
+	// state.
+	ThresholdFailed
+
+	// numThresholdsStates is the maximum number of threshold states used in
+	// tests.
+	numThresholdsStates
+)
+
+// thresholdStateStrings is a map of ThresholdState values back to their
+// constant names for pretty printing.
+var thresholdStateStrings = map[ThresholdState]string{
+	ThresholdDefined:  "ThresholdDefined",
+	ThresholdStarted:  "ThresholdStarted",
+	ThresholdLockedIn: "ThresholdLockedIn",
+	ThresholdActive:   "ThresholdActive",
+	ThresholdFailed:   "ThresholdFailed",
+}
+
+// String returns the ThresholdState as a human-readable name.
+func (t ThresholdState) String() string {
+	if s := thresholdStateStrings[t]; s != "" {
+		return s
+	}
+	return fmt.Sprintf("Unknown ThresholdState (%d)", int(t))
+}
+
+// thresholdConditionChecker provides a generic interface that is invoked to
+// determine when a consensus rule change threshold should be changed.
+type thresholdConditionChecker interface {
+	// BeginTime returns the unix timestamp for the median block time after
+	// which voting on a rule change starts (at the next window).
+	BeginTime() uint64
+
+	// EndTime returns the unix timestamp for the median block time after
+	// which an attempted rule change fails if it has not already been
+	// locked in or activated.
+	EndTime() uint64
+
+	// RuleChangeActivationThreshold is the number of blocks for which the
+	// condition must be true in order to lock in a rule change.
+	RuleChangeActivationThreshold() uint64
+
+	// MinerConfirmationWindow is the number of blocks in each threshold
+	// state retarget window.
+	MinerConfirmationWindow() uint64
+
+	// Condition returns whether or not the rule change activation condition
+	// has been met. This typically involves checking whether or not the
+	// bit associated with the condition is set, but can be more complex as
+	// needed.
+	Condition(*blockNode) (bool, error)
+}
+
+// thresholdStateCache provides a type to cache the threshold states of each
+// threshold window for a set of IDs.
+type thresholdStateCache struct {
+	entries map[daghash.Hash]ThresholdState
+}
+
+// Lookup returns the threshold state associated with the given hash along with
+// a boolean that indicates whether or not it is valid.
+func (c *thresholdStateCache) Lookup(hash *daghash.Hash) (ThresholdState, bool) {
+	state, ok := c.entries[*hash]
+	return state, ok
+}
+
+// Update updates the cache to contain the provided hash to threshold state
+// mapping.
+func (c *thresholdStateCache) Update(hash *daghash.Hash, state ThresholdState) {
+	c.entries[*hash] = state
+}
+
+// newThresholdCaches returns a new array of caches to be used when calculating
+// threshold states.
+func newThresholdCaches(numCaches uint32) []thresholdStateCache {
+	caches := make([]thresholdStateCache, numCaches)
+	for i := 0; i < len(caches); i++ {
+		caches[i] = thresholdStateCache{
+			entries: make(map[daghash.Hash]ThresholdState),
+		}
+	}
+	return caches
+}
+
+// thresholdState returns the current rule change threshold state for the block
+// AFTER the given node and deployment ID. The cache is used to ensure the
+// threshold states for previous windows are only calculated once.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdConditionChecker, cache *thresholdStateCache) (ThresholdState, error) {
+	// The threshold state for the window that contains the genesis block is
+	// defined by definition.
+	confirmationWindow := checker.MinerConfirmationWindow()
+	if prevNode == nil || (prevNode.blueScore+1) < confirmationWindow {
+		return ThresholdDefined, nil
+	}
+
+	// Get the ancestor that is the last block of the previous confirmation
+	// window in order to get its threshold state. This can be done because
+	// the state is the same for all blocks within a given window.
+	prevNode = prevNode.SelectedAncestor(prevNode.blueScore -
+		(prevNode.blueScore+1)%confirmationWindow)
+
+	// Iterate backwards through each of the previous confirmation windows
+	// to find the most recently cached threshold state.
+	var neededStates []*blockNode
+	for prevNode != nil {
+		// Nothing more to do if the state of the block is already
+		// cached.
+		if _, ok := cache.Lookup(prevNode.hash); ok {
+			break
+		}
+
+		// The start and expiration times are based on the median block
+		// time, so calculate it now.
+		medianTime := prevNode.PastMedianTime(dag)
+
+		// The state is simply defined if the start time hasn't been
+		// been reached yet.
+		if uint64(medianTime.UnixMilliseconds()) < checker.BeginTime() {
+			cache.Update(prevNode.hash, ThresholdDefined)
+			break
+		}
+
+		// Add this node to the list of nodes that need the state
+		// calculated and cached.
+		neededStates = append(neededStates, prevNode)
+
+		// Get the ancestor that is the last block of the previous
+		// confirmation window.
+		prevNode = prevNode.RelativeAncestor(confirmationWindow)
+	}
+
+	// Start with the threshold state for the most recent confirmation
+	// window that has a cached state.
+	state := ThresholdDefined
+	if prevNode != nil {
+		var ok bool
+		state, ok = cache.Lookup(prevNode.hash)
+		if !ok {
+			return ThresholdFailed, errors.Errorf(
+				"thresholdState: cache lookup failed for %s",
+				prevNode.hash)
+		}
+	}
+
+	// Since each threshold state depends on the state of the previous
+	// window, iterate starting from the oldest unknown window.
+	for neededNum := len(neededStates) - 1; neededNum >= 0; neededNum-- {
+		prevNode := neededStates[neededNum]
+
+		switch state {
+		case ThresholdDefined:
+			// The deployment of the rule change fails if it expires
+			// before it is accepted and locked in.
+			medianTime := prevNode.PastMedianTime(dag)
+			medianTimeUnix := uint64(medianTime.UnixMilliseconds())
+			if medianTimeUnix >= checker.EndTime() {
+				state = ThresholdFailed
+				break
+			}
+
+			// The state for the rule moves to the started state
+			// once its start time has been reached (and it hasn't
+			// already expired per the above).
+			if medianTimeUnix >= checker.BeginTime() {
+				state = ThresholdStarted
+			}
+
+		case ThresholdStarted:
+			// The deployment of the rule change fails if it expires
+			// before it is accepted and locked in.
+			medianTime := prevNode.PastMedianTime(dag)
+			if uint64(medianTime.UnixMilliseconds()) >= checker.EndTime() {
+				state = ThresholdFailed
+				break
+			}
+
+			// At this point, the rule change is still being voted
+			// on by the miners, so iterate backwards through the
+			// confirmation window to count all of the votes in it.
+			var count uint64
+			windowNodes := make([]*blockNode, 0, confirmationWindow)
+			windowNodes = append(windowNodes, prevNode)
+			windowNodes = append(windowNodes, blueBlockWindow(prevNode, confirmationWindow-1)...)
+			for _, current := range windowNodes {
+				condition, err := checker.Condition(current)
+				if err != nil {
+					return ThresholdFailed, err
+				}
+				if condition {
+					count++
+				}
+			}
+
+			// The state is locked in if the number of blocks in the
+			// period that voted for the rule change meets the
+			// activation threshold.
+			if count >= checker.RuleChangeActivationThreshold() {
+				state = ThresholdLockedIn
+			}
+
+		case ThresholdLockedIn:
+			// The new rule becomes active when its previous state
+			// was locked in.
+			state = ThresholdActive
+
+			// Nothing to do if the previous state is active or failed since
+			// they are both terminal states.
+		case ThresholdActive:
+		case ThresholdFailed:
+		}
+
+		// Update the cache to avoid recalculating the state in the
+		// future.
+		cache.Update(prevNode.hash, state)
+	}
+
+	return state, nil
+}
+
+// ThresholdState returns the current rule change threshold state of the given
+// deployment ID for the block AFTER the blueScore of the current DAG.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) ThresholdState(deploymentID uint32) (ThresholdState, error) {
+	dag.dagLock.Lock()
+	defer dag.dagLock.Unlock()
+	state, err := dag.deploymentState(dag.selectedTip(), deploymentID)
+
+	return state, err
+}
+
+// IsDeploymentActive returns true if the target deploymentID is active, and
+// false otherwise.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) IsDeploymentActive(deploymentID uint32) (bool, error) {
+	dag.dagLock.Lock()
+	defer dag.dagLock.Unlock()
+	state, err := dag.deploymentState(dag.selectedTip(), deploymentID)
+	if err != nil {
+		return false, err
+	}
+
+	return state == ThresholdActive, nil
+}
+
+// deploymentState returns the current rule change threshold for a given
+// deploymentID. The threshold is evaluated from the point of view of the block
+// node passed in as the first argument to this method.
+//
+// It is important to note that, as the variable name indicates, this function
+// expects the block node prior to the block for which the deployment state is
+// desired. In other words, the returned deployment state is for the block
+// AFTER the passed node.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) deploymentState(prevNode *blockNode, deploymentID uint32) (ThresholdState, error) {
+	if deploymentID > uint32(len(dag.Params.Deployments)) {
+		return ThresholdFailed, errors.Errorf("deployment ID %d does not exist", deploymentID)
+	}
+
+	deployment := &dag.Params.Deployments[deploymentID]
+	checker := deploymentChecker{deployment: deployment, dag: dag}
+	cache := &dag.deploymentCaches[deploymentID]
+
+	return dag.thresholdState(prevNode, checker, cache)
+}
+
+// initThresholdCaches initializes the threshold state caches for each warning
+// bit and defined deployment and provides warnings if the DAG is current per
+// the warnUnknownVersions and warnUnknownRuleActivations functions.
+func (dag *BlockDAG) initThresholdCaches() error {
+	// Initialize the warning and deployment caches by calculating the
+	// threshold state for each of them. This will ensure the caches are
+	// populated and any states that needed to be recalculated due to
+	// definition changes is done now.
+	prevNode := dag.selectedTip().selectedParent
+	for bit := uint32(0); bit < vbNumBits; bit++ {
+		checker := bitConditionChecker{bit: bit, dag: dag}
+		cache := &dag.warningCaches[bit]
+		_, err := dag.thresholdState(prevNode, checker, cache)
+		if err != nil {
+			return err
+		}
+	}
+	for id := 0; id < len(dag.Params.Deployments); id++ {
+		deployment := &dag.Params.Deployments[id]
+		cache := &dag.deploymentCaches[id]
+		checker := deploymentChecker{deployment: deployment, dag: dag}
+		_, err := dag.thresholdState(prevNode, checker, cache)
+		if err != nil {
+			return err
+		}
+	}
+
+	// No warnings about unknown rules or versions until the DAG is
+	// synced.
+	if dag.isSynced() {
+		// Warn if a high enough percentage of the last blocks have
+		// unexpected versions.
+		bestNode := dag.selectedTip()
+		if err := dag.warnUnknownVersions(bestNode); err != nil {
+			return err
+		}
+
+		// Warn if any unknown new rules are either about to activate or
+		// have already been activated.
+		if err := dag.warnUnknownRuleActivations(bestNode); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

blockdag/thresholdstate_test.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"testing"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// TestThresholdStateStringer tests the stringized output for the
+// ThresholdState type.
+func TestThresholdStateStringer(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		in   ThresholdState
+		want string
+	}{
+		{ThresholdDefined, "ThresholdDefined"},
+		{ThresholdStarted, "ThresholdStarted"},
+		{ThresholdLockedIn, "ThresholdLockedIn"},
+		{ThresholdActive, "ThresholdActive"},
+		{ThresholdFailed, "ThresholdFailed"},
+		{0xff, "Unknown ThresholdState (255)"},
+	}
+
+	// Detect additional threshold states that don't have the stringer added.
+	if len(tests)-1 != int(numThresholdsStates) {
+		t.Errorf("It appears a threshold statewas added without " +
+			"adding an associated stringer test")
+	}
+
+	t.Logf("Running %d tests", len(tests))
+	for i, test := range tests {
+		result := test.in.String()
+		if result != test.want {
+			t.Errorf("String #%d\n got: %s want: %s", i, result,
+				test.want)
+			continue
+		}
+	}
+}
+
+// TestThresholdStateCache ensure the threshold state cache works as intended
+// including adding entries, updating existing entries, and flushing.
+func TestThresholdStateCache(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		numEntries int
+		state      ThresholdState
+	}{
+		{name: "2 entries defined", numEntries: 2, state: ThresholdDefined},
+		{name: "7 entries started", numEntries: 7, state: ThresholdStarted},
+		{name: "10 entries active", numEntries: 10, state: ThresholdActive},
+		{name: "5 entries locked in", numEntries: 5, state: ThresholdLockedIn},
+		{name: "3 entries failed", numEntries: 3, state: ThresholdFailed},
+	}
+
+nextTest:
+	for _, test := range tests {
+		cache := &newThresholdCaches(1)[0]
+		for i := 0; i < test.numEntries; i++ {
+			var hash daghash.Hash
+			hash[0] = uint8(i + 1)
+
+			// Ensure the hash isn't available in the cache already.
+			_, ok := cache.Lookup(&hash)
+			if ok {
+				t.Errorf("Lookup (%s): has entry for hash %v",
+					test.name, hash)
+				continue nextTest
+			}
+
+			// Ensure hash that was added to the cache reports it's
+			// available and the state is the expected value.
+			cache.Update(&hash, test.state)
+			state, ok := cache.Lookup(&hash)
+			if !ok {
+				t.Errorf("Lookup (%s): missing entry for hash "+
+					"%v", test.name, hash)
+				continue nextTest
+			}
+			if state != test.state {
+				t.Errorf("Lookup (%s): state mismatch - got "+
+					"%v, want %v", test.name, state,
+					test.state)
+				continue nextTest
+			}
+
+			// Ensure adding an existing hash with the same state
+			// doesn't break the existing entry.
+			cache.Update(&hash, test.state)
+			state, ok = cache.Lookup(&hash)
+			if !ok {
+				t.Errorf("Lookup (%s): missing entry after "+
+					"second add for hash %v", test.name,
+					hash)
+				continue nextTest
+			}
+			if state != test.state {
+				t.Errorf("Lookup (%s): state mismatch after "+
+					"second add - got %v, want %v",
+					test.name, state, test.state)
+				continue nextTest
+			}
+
+			// Ensure adding an existing hash with a different state
+			// updates the existing entry.
+			newState := ThresholdFailed
+			if newState == test.state {
+				newState = ThresholdStarted
+			}
+			cache.Update(&hash, newState)
+			state, ok = cache.Lookup(&hash)
+			if !ok {
+				t.Errorf("Lookup (%s): missing entry after "+
+					"state change for hash %v", test.name,
+					hash)
+				continue nextTest
+			}
+			if state != newState {
+				t.Errorf("Lookup (%s): state mismatch after "+
+					"state change - got %v, want %v",
+					test.name, state, newState)
+				continue nextTest
+			}
+		}
+	}
+}

blockdag/timesorter.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+// timeSorter implements sort.Interface to allow a slice of timestamps to
+// be sorted.
+type timeSorter []int64
+
+// Len returns the number of timestamps in the slice. It is part of the
+// sort.Interface implementation.
+func (s timeSorter) Len() int {
+	return len(s)
+}
+
+// Swap swaps the timestamps at the passed indices. It is part of the
+// sort.Interface implementation.
+func (s timeSorter) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+// Less returns whether the timstamp with index i should sort before the
+// timestamp with index j. It is part of the sort.Interface implementation.
+func (s timeSorter) Less(i, j int) bool {
+	return s[i] < s[j]
+}

blockdag/timesorter_test.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2013-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+// TestTimeSorter tests the timeSorter implementation.
+func TestTimeSorter(t *testing.T) {
+	tests := []struct {
+		in   []int64
+		want []int64
+	}{
+		{
+			in: []int64{
+				1351228575, // Fri Oct 26 05:16:15 UTC 2012 (Block #205000)
+				1348310759, // Sat Sep 22 10:45:59 UTC 2012 (Block #200000)
+				1305758502, // Wed May 18 22:41:42 UTC 2011 (Block #125000)
+				1347777156, // Sun Sep 16 06:32:36 UTC 2012 (Block #199000)
+				1349492104, // Sat Oct  6 02:55:04 UTC 2012 (Block #202000)
+			},
+			want: []int64{
+				1305758502, // Wed May 18 22:41:42 UTC 2011 (Block #125000)
+				1347777156, // Sun Sep 16 06:32:36 UTC 2012 (Block #199000)
+				1348310759, // Sat Sep 22 10:45:59 UTC 2012 (Block #200000)
+				1349492104, // Sat Oct  6 02:55:04 UTC 2012 (Block #202000)
+				1351228575, // Fri Oct 26 05:16:15 UTC 2012 (Block #205000)
+			},
+		},
+	}
+
+	for i, test := range tests {
+		result := make([]int64, len(test.in))
+		copy(result, test.in)
+		sort.Sort(timeSorter(result))
+		if !reflect.DeepEqual(result, test.want) {
+			t.Errorf("timeSorter #%d got %v want %v", i, result,
+				test.want)
+			continue
+		}
+	}
+}

blockdag/timesource.go

@@ -0,0 +1,25 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/mstime"
+)
+
+// TimeSource is the interface to access time.
+type TimeSource interface {
+	// Now returns the current time.
+	Now() mstime.Time
+}
+
+// timeSource provides an implementation of the TimeSource interface
+// that simply returns the current local time.
+type timeSource struct{}
+
+// Now returns the current local time, with one millisecond precision.
+func (m *timeSource) Now() mstime.Time {
+	return mstime.Now()
+}
+
+// NewTimeSource returns a new instance of a TimeSource
+func NewTimeSource() TimeSource {
+	return &timeSource{}
+}

blockdag/utxo_ecmh.go

@@ -0,0 +1,27 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+func addUTXOToMultiset(ms *secp256k1.MultiSet, entry *UTXOEntry, outpoint *domainmessage.Outpoint) (*secp256k1.MultiSet, error) {
+	w := &bytes.Buffer{}
+	err := serializeUTXO(w, entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	ms.Add(w.Bytes())
+	return ms, nil
+}
+
+func removeUTXOFromMultiset(ms *secp256k1.MultiSet, entry *UTXOEntry, outpoint *domainmessage.Outpoint) (*secp256k1.MultiSet, error) {
+	w := &bytes.Buffer{}
+	err := serializeUTXO(w, entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	ms.Remove(w.Bytes())
+	return ms, nil
+}

blockdag/utxodiffstore.go

@@ -0,0 +1,200 @@
+package blockdag
+
+import (
+	"bytes"
+
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/locks"
+)
+
+type blockUTXODiffData struct {
+	diff      *UTXODiff
+	diffChild *blockNode
+}
+
+type utxoDiffStore struct {
+	dag    *BlockDAG
+	dirty  map[*blockNode]struct{}
+	loaded map[*blockNode]*blockUTXODiffData
+	mtx    *locks.PriorityMutex
+}
+
+func newUTXODiffStore(dag *BlockDAG) *utxoDiffStore {
+	return &utxoDiffStore{
+		dag:    dag,
+		dirty:  make(map[*blockNode]struct{}),
+		loaded: make(map[*blockNode]*blockUTXODiffData),
+		mtx:    locks.NewPriorityMutex(),
+	}
+}
+
+func (diffStore *utxoDiffStore) setBlockDiff(node *blockNode, diff *UTXODiff) error {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+	// load the diff data from DB to diffStore.loaded
+	_, err := diffStore.diffDataByBlockNode(node)
+	if dbaccess.IsNotFoundError(err) {
+		diffStore.loaded[node] = &blockUTXODiffData{}
+	} else if err != nil {
+		return err
+	}
+
+	diffStore.loaded[node].diff = diff
+	diffStore.setBlockAsDirty(node)
+	return nil
+}
+
+func (diffStore *utxoDiffStore) setBlockDiffChild(node *blockNode, diffChild *blockNode) error {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+	// load the diff data from DB to diffStore.loaded
+	_, err := diffStore.diffDataByBlockNode(node)
+	if err != nil {
+		return err
+	}
+
+	diffStore.loaded[node].diffChild = diffChild
+	diffStore.setBlockAsDirty(node)
+	return nil
+}
+
+func (diffStore *utxoDiffStore) removeBlocksDiffData(dbContext dbaccess.Context, nodes []*blockNode) error {
+	for _, node := range nodes {
+		err := diffStore.removeBlockDiffData(dbContext, node)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (diffStore *utxoDiffStore) removeBlockDiffData(dbContext dbaccess.Context, node *blockNode) error {
+	diffStore.mtx.LowPriorityWriteLock()
+	defer diffStore.mtx.LowPriorityWriteUnlock()
+	delete(diffStore.loaded, node)
+	err := dbaccess.RemoveDiffData(dbContext, node.hash)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (diffStore *utxoDiffStore) setBlockAsDirty(node *blockNode) {
+	diffStore.dirty[node] = struct{}{}
+}
+
+func (diffStore *utxoDiffStore) diffDataByBlockNode(node *blockNode) (*blockUTXODiffData, error) {
+	if diffData, ok := diffStore.loaded[node]; ok {
+		return diffData, nil
+	}
+	diffData, err := diffStore.diffDataFromDB(node.hash)
+	if err != nil {
+		return nil, err
+	}
+	diffStore.loaded[node] = diffData
+	return diffData, nil
+}
+
+func (diffStore *utxoDiffStore) diffByNode(node *blockNode) (*UTXODiff, error) {
+	diffStore.mtx.HighPriorityReadLock()
+	defer diffStore.mtx.HighPriorityReadUnlock()
+	diffData, err := diffStore.diffDataByBlockNode(node)
+	if err != nil {
+		return nil, err
+	}
+	return diffData.diff, nil
+}
+
+func (diffStore *utxoDiffStore) diffChildByNode(node *blockNode) (*blockNode, error) {
+	diffStore.mtx.HighPriorityReadLock()
+	defer diffStore.mtx.HighPriorityReadUnlock()
+	diffData, err := diffStore.diffDataByBlockNode(node)
+	if err != nil {
+		return nil, err
+	}
+	return diffData.diffChild, nil
+}
+
+func (diffStore *utxoDiffStore) diffDataFromDB(hash *daghash.Hash) (*blockUTXODiffData, error) {
+	serializedBlockDiffData, err := dbaccess.FetchUTXODiffData(diffStore.dag.databaseContext, hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
+}
+
+// flushToDB writes all dirty diff data to the database.
+func (diffStore *utxoDiffStore) flushToDB(dbContext *dbaccess.TxContext) error {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+	if len(diffStore.dirty) == 0 {
+		return nil
+	}
+
+	// Allocate a buffer here to avoid needless allocations/grows
+	// while writing each entry.
+	buffer := &bytes.Buffer{}
+	for node := range diffStore.dirty {
+		buffer.Reset()
+		diffData := diffStore.loaded[node]
+		err := storeDiffData(dbContext, buffer, node.hash, diffData)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (diffStore *utxoDiffStore) clearDirtyEntries() {
+	diffStore.dirty = make(map[*blockNode]struct{})
+}
+
+// maxBlueScoreDifferenceToKeepLoaded is the maximum difference
+// between the virtual's blueScore and a blockNode's blueScore
+// under which to keep diff data loaded in memory.
+var maxBlueScoreDifferenceToKeepLoaded uint64 = 100
+
+// clearOldEntries removes entries whose blue score is lower than
+// virtual.blueScore - maxBlueScoreDifferenceToKeepLoaded. Note
+// that tips are not removed either even if their blue score is
+// lower than the above.
+func (diffStore *utxoDiffStore) clearOldEntries() {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+
+	virtualBlueScore := diffStore.dag.VirtualBlueScore()
+	minBlueScore := virtualBlueScore - maxBlueScoreDifferenceToKeepLoaded
+	if maxBlueScoreDifferenceToKeepLoaded > virtualBlueScore {
+		minBlueScore = 0
+	}
+
+	tips := diffStore.dag.virtual.tips()
+
+	toRemove := make(map[*blockNode]struct{})
+	for node := range diffStore.loaded {
+		if node.blueScore < minBlueScore && !tips.contains(node) {
+			toRemove[node] = struct{}{}
+		}
+	}
+	for node := range toRemove {
+		delete(diffStore.loaded, node)
+	}
+}
+
+// storeDiffData stores the UTXO diff data to the database.
+// This overwrites the current entry if there exists one.
+func storeDiffData(dbContext dbaccess.Context, w *bytes.Buffer, hash *daghash.Hash, diffData *blockUTXODiffData) error {
+	// To avoid a ton of allocs, use the io.Writer
+	// instead of allocating one. We expect the buffer to
+	// already be initialized and, in most cases, to already
+	// be large enough to accommodate the serialized data
+	// without growing.
+	err := serializeBlockUTXODiffData(w, diffData)
+	if err != nil {
+		return err
+	}
+
+	return dbaccess.StoreUTXODiffData(dbContext, hash, w.Bytes())
+}

blockdag/utxodiffstore_test.go

@@ -0,0 +1,167 @@
+package blockdag
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+func TestUTXODiffStore(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestUTXODiffStore: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	nodeCounter := byte(0)
+	createNode := func() *blockNode {
+		nodeCounter++
+		node := &blockNode{hash: &daghash.Hash{nodeCounter}}
+		dag.index.AddNode(node)
+		return node
+	}
+
+	// Check that an error is returned when asking for non existing node
+	nonExistingNode := createNode()
+	_, err = dag.utxoDiffStore.diffByNode(nonExistingNode)
+	if !dbaccess.IsNotFoundError(err) {
+		if err != nil {
+			t.Errorf("diffByNode: %s", err)
+		} else {
+			t.Errorf("diffByNode: unexpectedly found diff data")
+		}
+	}
+
+	// Add node's diff data to the utxoDiffStore and check if it's checked correctly.
+	node := createNode()
+	diff := NewUTXODiff()
+	diff.toAdd.add(domainmessage.Outpoint{TxID: daghash.TxID{0x01}, Index: 0}, &UTXOEntry{amount: 1, scriptPubKey: []byte{0x01}})
+	diff.toRemove.add(domainmessage.Outpoint{TxID: daghash.TxID{0x02}, Index: 0}, &UTXOEntry{amount: 2, scriptPubKey: []byte{0x02}})
+	if err := dag.utxoDiffStore.setBlockDiff(node, diff); err != nil {
+		t.Fatalf("setBlockDiff: unexpected error: %s", err)
+	}
+	diffChild := createNode()
+	if err := dag.utxoDiffStore.setBlockDiffChild(node, diffChild); err != nil {
+		t.Fatalf("setBlockDiffChild: unexpected error: %s", err)
+	}
+
+	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
+		t.Fatalf("diffByNode: unexpected error: %s", err)
+	} else if !reflect.DeepEqual(storeDiff, diff) {
+		t.Errorf("Expected diff and storeDiff to be equal")
+	}
+
+	if storeDiffChild, err := dag.utxoDiffStore.diffChildByNode(node); err != nil {
+		t.Fatalf("diffByNode: unexpected error: %s", err)
+	} else if !reflect.DeepEqual(storeDiffChild, diffChild) {
+		t.Errorf("Expected diff and storeDiff to be equal")
+	}
+
+	// Flush changes to db, delete them from the dag.utxoDiffStore.loaded
+	// map, and check if the diff data is re-fetched from the database.
+	dbTx, err := dag.databaseContext.NewTx()
+	if err != nil {
+		t.Fatalf("Failed to open database transaction: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = dag.utxoDiffStore.flushToDB(dbTx)
+	if err != nil {
+		t.Fatalf("Error flushing utxoDiffStore data to DB: %s", err)
+	}
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Failed to commit database transaction: %s", err)
+	}
+	delete(dag.utxoDiffStore.loaded, node)
+
+	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
+		t.Fatalf("diffByNode: unexpected error: %s", err)
+	} else if !reflect.DeepEqual(storeDiff, diff) {
+		t.Errorf("Expected diff and storeDiff to be equal")
+	}
+
+	// Check if getBlockDiff caches the result in dag.utxoDiffStore.loaded
+	if loadedDiffData, ok := dag.utxoDiffStore.loaded[node]; !ok {
+		t.Errorf("the diff data wasn't added to loaded map after requesting it")
+	} else if !reflect.DeepEqual(loadedDiffData.diff, diff) {
+		t.Errorf("Expected diff and loadedDiff to be equal")
+	}
+}
+
+func TestClearOldEntries(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestClearOldEntries", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestClearOldEntries: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Set maxBlueScoreDifferenceToKeepLoaded to 10 to make this test fast to run
+	currentDifference := maxBlueScoreDifferenceToKeepLoaded
+	maxBlueScoreDifferenceToKeepLoaded = 10
+	defer func() { maxBlueScoreDifferenceToKeepLoaded = currentDifference }()
+
+	// Add 10 blocks
+	blockNodes := make([]*blockNode, 10)
+	for i := 0; i < 10; i++ {
+		processedBlock := PrepareAndProcessBlockForTest(t, dag, dag.TipHashes(), nil)
+
+		node, ok := dag.index.LookupNode(processedBlock.BlockHash())
+		if !ok {
+			t.Fatalf("TestClearOldEntries: missing blockNode for hash %s", processedBlock.BlockHash())
+		}
+		blockNodes[i] = node
+	}
+
+	// Make sure that all of them exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if !ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is not in the loaded set", node.hash)
+		}
+	}
+
+	// Add 10 more blocks on top of the others
+	for i := 0; i < 10; i++ {
+		PrepareAndProcessBlockForTest(t, dag, dag.TipHashes(), nil)
+	}
+
+	// Make sure that all the old nodes no longer exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is in the loaded set", node.hash)
+		}
+	}
+
+	// Add a block on top of the genesis to force the retrieval of all diffData
+	processedBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	node, ok := dag.index.LookupNode(processedBlock.BlockHash())
+	if !ok {
+		t.Fatalf("TestClearOldEntries: missing blockNode for hash %s", processedBlock.BlockHash())
+	}
+
+	// Make sure that the child-of-genesis node is in the loaded set, since it
+	// is a tip.
+	_, ok = dag.utxoDiffStore.loaded[node]
+	if !ok {
+		t.Fatalf("TestClearOldEntries: diffData for node %s is not in the loaded set", node.hash)
+	}
+
+	// Make sure that all the old nodes still do not exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is in the loaded set", node.hash)
+		}
+	}
+}

blockdag/utxoio.go

@@ -0,0 +1,240 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/kaspad/domainmessage"
+	"github.com/kaspanet/kaspad/util/binaryserializer"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+	"io"
+)
+
+// serializeBlockUTXODiffData serializes diff data in the following format:
+// 	Name         | Data type | Description
+//  ------------ | --------- | -----------
+// 	hasDiffChild | Boolean   | Indicates if a diff child exist
+//  diffChild    | Hash      | The diffChild's hash. Empty if hasDiffChild is true.
+//  diff		 | UTXODiff  | The diff data's diff
+func serializeBlockUTXODiffData(w io.Writer, diffData *blockUTXODiffData) error {
+	hasDiffChild := diffData.diffChild != nil
+	err := domainmessage.WriteElement(w, hasDiffChild)
+	if err != nil {
+		return err
+	}
+	if hasDiffChild {
+		err := domainmessage.WriteElement(w, diffData.diffChild.hash)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = serializeUTXODiff(w, diffData.diff)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffData []byte) (*blockUTXODiffData, error) {
+	diffData := &blockUTXODiffData{}
+	r := bytes.NewBuffer(serializedDiffData)
+
+	var hasDiffChild bool
+	err := domainmessage.ReadElement(r, &hasDiffChild)
+	if err != nil {
+		return nil, err
+	}
+
+	if hasDiffChild {
+		hash := &daghash.Hash{}
+		err := domainmessage.ReadElement(r, hash)
+		if err != nil {
+			return nil, err
+		}
+
+		var ok bool
+		diffData.diffChild, ok = diffStore.dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("block %s does not exist in the DAG", hash)
+		}
+	}
+
+	diffData.diff, err = deserializeUTXODiff(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return diffData, nil
+}
+
+func deserializeUTXODiff(r io.Reader) (*UTXODiff, error) {
+	diff := &UTXODiff{}
+
+	var err error
+	diff.toAdd, err = deserializeUTXOCollection(r)
+	if err != nil {
+		return nil, err
+	}
+
+	diff.toRemove, err = deserializeUTXOCollection(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return diff, nil
+}
+
+func deserializeUTXOCollection(r io.Reader) (utxoCollection, error) {
+	count, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return nil, err
+	}
+	collection := utxoCollection{}
+	for i := uint64(0); i < count; i++ {
+		utxoEntry, outpoint, err := deserializeUTXO(r)
+		if err != nil {
+			return nil, err
+		}
+		collection.add(*outpoint, utxoEntry)
+	}
+	return collection, nil
+}
+
+func deserializeUTXO(r io.Reader) (*UTXOEntry, *domainmessage.Outpoint, error) {
+	outpoint, err := deserializeOutpoint(r)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	utxoEntry, err := deserializeUTXOEntry(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	return utxoEntry, outpoint, nil
+}
+
+// serializeUTXODiff serializes UTXODiff by serializing
+// UTXODiff.toAdd, UTXODiff.toRemove and UTXODiff.Multiset one after the other.
+func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
+	err := serializeUTXOCollection(w, diff.toAdd)
+	if err != nil {
+		return err
+	}
+
+	err = serializeUTXOCollection(w, diff.toRemove)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// serializeUTXOCollection serializes utxoCollection by iterating over
+// the utxo entries and serializing them and their corresponding outpoint
+// prefixed by a varint that indicates their size.
+func serializeUTXOCollection(w io.Writer, collection utxoCollection) error {
+	err := domainmessage.WriteVarInt(w, uint64(len(collection)))
+	if err != nil {
+		return err
+	}
+	for outpoint, utxoEntry := range collection {
+		err := serializeUTXO(w, utxoEntry, &outpoint)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// serializeUTXO serializes a utxo entry-outpoint pair
+func serializeUTXO(w io.Writer, entry *UTXOEntry, outpoint *domainmessage.Outpoint) error {
+	err := serializeOutpoint(w, outpoint)
+	if err != nil {
+		return err
+	}
+
+	err = serializeUTXOEntry(w, entry)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// p2pkhUTXOEntrySerializeSize is the serialized size for a P2PKH UTXO entry.
+// 8 bytes (header code) + 8 bytes (amount) + varint for script pub key length of 25 (for P2PKH) + 25 bytes for P2PKH script.
+var p2pkhUTXOEntrySerializeSize = 8 + 8 + domainmessage.VarIntSerializeSize(25) + 25
+
+// serializeUTXOEntry encodes the entry to the given io.Writer and use compression if useCompression is true.
+// The compression format is described in detail above.
+func serializeUTXOEntry(w io.Writer, entry *UTXOEntry) error {
+	// Encode the blueScore.
+	err := binaryserializer.PutUint64(w, byteOrder, entry.blockBlueScore)
+	if err != nil {
+		return err
+	}
+
+	// Encode the packedFlags.
+	err = binaryserializer.PutUint8(w, uint8(entry.packedFlags))
+	if err != nil {
+		return err
+	}
+
+	err = binaryserializer.PutUint64(w, byteOrder, entry.Amount())
+	if err != nil {
+		return err
+	}
+
+	err = domainmessage.WriteVarInt(w, uint64(len(entry.ScriptPubKey())))
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(entry.ScriptPubKey())
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+// deserializeUTXOEntry decodes a UTXO entry from the passed reader
+// into a new UTXOEntry. If isCompressed is used it will decompress
+// the entry according to the format that is described in detail
+// above.
+func deserializeUTXOEntry(r io.Reader) (*UTXOEntry, error) {
+	// Deserialize the blueScore.
+	blockBlueScore, err := binaryserializer.Uint64(r, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decode the packedFlags.
+	packedFlags, err := binaryserializer.Uint8(r)
+	if err != nil {
+		return nil, err
+	}
+
+	entry := &UTXOEntry{
+		blockBlueScore: blockBlueScore,
+		packedFlags:    txoFlags(packedFlags),
+	}
+
+	entry.amount, err = binaryserializer.Uint64(r, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	scriptPubKeyLen, err := domainmessage.ReadVarInt(r)
+	if err != nil {
+		return nil, err
+	}
+
+	entry.scriptPubKey = make([]byte, scriptPubKeyLen)
+	_, err = r.Read(entry.scriptPubKey)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return entry, nil
+}

blockdag/utxoset.go

@@ -0,0 +1,643 @@
+package blockdag
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/domainmessage"
+)
+
+const (
+	// UnacceptedBlueScore is the blue score used for the "block" blueScore
+	// field of the contextual transaction information provided in a
+	// transaction store when it has not yet been accepted by a block.
+	UnacceptedBlueScore uint64 = math.MaxUint64
+)
+
+// UTXOEntry houses details about an individual transaction output in a utxo
+// set such as whether or not it was contained in a coinbase tx, the blue
+// score of the block that accepts the tx, its public key script, and how
+// much it pays.
+type UTXOEntry struct {
+	// NOTE: Additions, deletions, or modifications to the order of the
+	// definitions in this struct should not be changed without considering
+	// how it affects alignment on 64-bit platforms. The current order is
+	// specifically crafted to result in minimal padding. There will be a
+	// lot of these in memory, so a few extra bytes of padding adds up.
+
+	amount         uint64
+	scriptPubKey   []byte // The public key script for the output.
+	blockBlueScore uint64 // Blue score of the block accepting the tx.
+
+	// packedFlags contains additional info about output such as whether it
+	// is a coinbase, and whether it has been modified
+	// since it was loaded. This approach is used in order to reduce memory
+	// usage since there will be a lot of these in memory.
+	packedFlags txoFlags
+}
+
+// IsCoinbase returns whether or not the output was contained in a block
+// reward transaction.
+func (entry *UTXOEntry) IsCoinbase() bool {
+	return entry.packedFlags&tfCoinbase == tfCoinbase
+}
+
+// BlockBlueScore returns the blue score of the block accepting the output.
+func (entry *UTXOEntry) BlockBlueScore() uint64 {
+	return entry.blockBlueScore
+}
+
+// Amount returns the amount of the output.
+func (entry *UTXOEntry) Amount() uint64 {
+	return entry.amount
+}
+
+// ScriptPubKey returns the public key script for the output.
+func (entry *UTXOEntry) ScriptPubKey() []byte {
+	return entry.scriptPubKey
+}
+
+// IsUnaccepted returns true iff this UTXOEntry has been included in a block
+// but has not yet been accepted by any block.
+func (entry *UTXOEntry) IsUnaccepted() bool {
+	return entry.blockBlueScore == UnacceptedBlueScore
+}
+
+// txoFlags is a bitmask defining additional information and state for a
+// transaction output in a UTXO set.
+type txoFlags uint8
+
+const (
+	// tfCoinbase indicates that a txout was contained in a coinbase tx.
+	tfCoinbase txoFlags = 1 << iota
+)
+
+// NewUTXOEntry creates a new utxoEntry representing the given txOut
+func NewUTXOEntry(txOut *domainmessage.TxOut, isCoinbase bool, blockBlueScore uint64) *UTXOEntry {
+	entry := &UTXOEntry{
+		amount:         txOut.Value,
+		scriptPubKey:   txOut.ScriptPubKey,
+		blockBlueScore: blockBlueScore,
+	}
+
+	if isCoinbase {
+		entry.packedFlags |= tfCoinbase
+	}
+
+	return entry
+}
+
+// utxoCollection represents a set of UTXOs indexed by their outpoints
+type utxoCollection map[domainmessage.Outpoint]*UTXOEntry
+
+func (uc utxoCollection) String() string {
+	utxoStrings := make([]string, len(uc))
+
+	i := 0
+	for outpoint, utxoEntry := range uc {
+		utxoStrings[i] = fmt.Sprintf("(%s, %d) => %d, blueScore: %d",
+			outpoint.TxID, outpoint.Index, utxoEntry.amount, utxoEntry.blockBlueScore)
+		i++
+	}
+
+	// Sort strings for determinism.
+	sort.Strings(utxoStrings)
+
+	return fmt.Sprintf("[ %s ]", strings.Join(utxoStrings, ", "))
+}
+
+// add adds a new UTXO entry to this collection
+func (uc utxoCollection) add(outpoint domainmessage.Outpoint, entry *UTXOEntry) {
+	uc[outpoint] = entry
+}
+
+// remove removes a UTXO entry from this collection if it exists
+func (uc utxoCollection) remove(outpoint domainmessage.Outpoint) {
+	delete(uc, outpoint)
+}
+
+// get returns the UTXOEntry represented by provided outpoint,
+// and a boolean value indicating if said UTXOEntry is in the set or not
+func (uc utxoCollection) get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool) {
+	entry, ok := uc[outpoint]
+	return entry, ok
+}
+
+// contains returns a boolean value indicating whether a UTXO entry is in the set
+func (uc utxoCollection) contains(outpoint domainmessage.Outpoint) bool {
+	_, ok := uc[outpoint]
+	return ok
+}
+
+// containsWithBlueScore returns a boolean value indicating whether a UTXOEntry
+// is in the set and its blue score is equal to the given blue score.
+func (uc utxoCollection) containsWithBlueScore(outpoint domainmessage.Outpoint, blueScore uint64) bool {
+	entry, ok := uc.get(outpoint)
+	return ok && entry.blockBlueScore == blueScore
+}
+
+// clone returns a clone of this collection
+func (uc utxoCollection) clone() utxoCollection {
+	clone := utxoCollection{}
+	for outpoint, entry := range uc {
+		clone.add(outpoint, entry)
+	}
+
+	return clone
+}
+
+// UTXODiff represents a diff between two UTXO Sets.
+type UTXODiff struct {
+	toAdd    utxoCollection
+	toRemove utxoCollection
+}
+
+// NewUTXODiff creates a new, empty utxoDiff
+// without a multiset.
+func NewUTXODiff() *UTXODiff {
+	return &UTXODiff{
+		toAdd:    utxoCollection{},
+		toRemove: utxoCollection{},
+	}
+}
+
+// diffFrom returns a new utxoDiff with the difference between this utxoDiff and another
+// Assumes that:
+// Both utxoDiffs are from the same base
+// If a txOut exists in both utxoDiffs, its underlying values would be the same
+//
+// diffFrom follows a set of rules represented by the following 3 by 3 table:
+//
+//          |           | this      |           |
+// ---------+-----------+-----------+-----------+-----------
+//          |           | toAdd     | toRemove  | None
+// ---------+-----------+-----------+-----------+-----------
+// other    | toAdd     | -         | X         | toAdd
+// ---------+-----------+-----------+-----------+-----------
+//          | toRemove  | X         | -         | toRemove
+// ---------+-----------+-----------+-----------+-----------
+//          | None      | toRemove  | toAdd     | -
+//
+// Key:
+// -		Don't add anything to the result
+// X		Return an error
+// toAdd	Add the UTXO into the toAdd collection of the result
+// toRemove	Add the UTXO into the toRemove collection of the result
+//
+// Examples:
+// 1. This diff contains a UTXO in toAdd, and the other diff contains it in toRemove
+//    diffFrom results in an error
+// 2. This diff contains a UTXO in toRemove, and the other diff does not contain it
+//    diffFrom results in the UTXO being added to toAdd
+func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
+	result := UTXODiff{
+		toAdd:    make(utxoCollection, len(d.toRemove)+len(other.toAdd)),
+		toRemove: make(utxoCollection, len(d.toAdd)+len(other.toRemove)),
+	}
+
+	// Note that the following cases are not accounted for, as they are impossible
+	// as long as the base utxoSet is the same:
+	// - if utxoEntry is in d.toAdd and other.toRemove
+	// - if utxoEntry is in d.toRemove and other.toAdd
+
+	// All transactions in d.toAdd:
+	// If they are not in other.toAdd - should be added in result.toRemove
+	// If they are in other.toRemove - base utxoSet is not the same
+	for outpoint, utxoEntry := range d.toAdd {
+		if !other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toRemove.add(outpoint, utxoEntry)
+		} else if (d.toRemove.contains(outpoint) && !other.toRemove.contains(outpoint)) ||
+			(!d.toRemove.contains(outpoint) && other.toRemove.contains(outpoint)) {
+			return nil, errors.New(
+				"diffFrom: outpoint both in d.toAdd, other.toAdd, and only one of d.toRemove and other.toRemove")
+		}
+		if diffEntry, ok := other.toRemove.get(outpoint); ok {
+			// An exception is made for entries with unequal blue scores
+			// as long as the appropriate entry exists in either d.toRemove
+			// or other.toAdd.
+			// These are just "updates" to accepted blue score
+			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
+				(d.toRemove.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
+					other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
+				continue
+			}
+			return nil, errors.Errorf("diffFrom: outpoint %s both in d.toAdd and in other.toRemove", outpoint)
+		}
+	}
+
+	// All transactions in d.toRemove:
+	// If they are not in other.toRemove - should be added in result.toAdd
+	// If they are in other.toAdd - base utxoSet is not the same
+	for outpoint, utxoEntry := range d.toRemove {
+		diffEntry, ok := other.toRemove.get(outpoint)
+		if ok {
+			// if have the same entry in d.toRemove - simply don't copy.
+			// unless existing entry is with different blue score, in this case - this is an error
+			if utxoEntry.blockBlueScore != diffEntry.blockBlueScore {
+				return nil, errors.New("diffFrom: outpoint both in d.toRemove and other.toRemove with different " +
+					"blue scores, with no corresponding entry in d.toAdd")
+			}
+		} else { // if no existing entry - add to result.toAdd
+			result.toAdd.add(outpoint, utxoEntry)
+		}
+
+		if !other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toAdd.add(outpoint, utxoEntry)
+		}
+		if diffEntry, ok := other.toAdd.get(outpoint); ok {
+			// An exception is made for entries with unequal blue scores
+			// as long as the appropriate entry exists in either d.toAdd
+			// or other.toRemove.
+			// These are just "updates" to accepted blue score
+			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
+				(d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
+					other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
+				continue
+			}
+			return nil, errors.New("diffFrom: outpoint both in d.toRemove and in other.toAdd")
+		}
+	}
+
+	// All transactions in other.toAdd:
+	// If they are not in d.toAdd - should be added in result.toAdd
+	for outpoint, utxoEntry := range other.toAdd {
+		if !d.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toAdd.add(outpoint, utxoEntry)
+		}
+	}
+
+	// All transactions in other.toRemove:
+	// If they are not in d.toRemove - should be added in result.toRemove
+	for outpoint, utxoEntry := range other.toRemove {
+		if !d.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toRemove.add(outpoint, utxoEntry)
+		}
+	}
+
+	return &result, nil
+}
+
+// withDiffInPlace applies provided diff to this diff in-place, that would be the result if
+// first d, and than diff were applied to the same base
+func (d *UTXODiff) withDiffInPlace(diff *UTXODiff) error {
+	for outpoint, entryToRemove := range diff.toRemove {
+		if d.toAdd.containsWithBlueScore(outpoint, entryToRemove.blockBlueScore) {
+			// If already exists in toAdd with the same blueScore - remove from toAdd
+			d.toAdd.remove(outpoint)
+			continue
+		}
+		if d.toRemove.contains(outpoint) {
+			// If already exists - this is an error
+			return errors.Errorf(
+				"withDiffInPlace: outpoint %s both in d.toRemove and in diff.toRemove", outpoint)
+		}
+
+		// If not exists neither in toAdd nor in toRemove - add to toRemove
+		d.toRemove.add(outpoint, entryToRemove)
+	}
+
+	for outpoint, entryToAdd := range diff.toAdd {
+		if d.toRemove.containsWithBlueScore(outpoint, entryToAdd.blockBlueScore) {
+			// If already exists in toRemove with the same blueScore - remove from toRemove
+			if d.toAdd.contains(outpoint) && !diff.toRemove.contains(outpoint) {
+				return errors.Errorf(
+					"withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd with no "+
+						"corresponding entry in diff.toRemove", outpoint)
+			}
+			d.toRemove.remove(outpoint)
+			continue
+		}
+		if existingEntry, ok := d.toAdd.get(outpoint); ok &&
+			(existingEntry.blockBlueScore == entryToAdd.blockBlueScore ||
+				!diff.toRemove.containsWithBlueScore(outpoint, existingEntry.blockBlueScore)) {
+			// If already exists - this is an error
+			return errors.Errorf(
+				"withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd", outpoint)
+		}
+
+		// If not exists neither in toAdd nor in toRemove, or exists in toRemove with different blueScore - add to toAdd
+		d.toAdd.add(outpoint, entryToAdd)
+	}
+
+	return nil
+}
+
+// WithDiff applies provided diff to this diff, creating a new utxoDiff, that would be the result if
+// first d, and than diff were applied to some base
+func (d *UTXODiff) WithDiff(diff *UTXODiff) (*UTXODiff, error) {
+	clone := d.clone()
+
+	err := clone.withDiffInPlace(diff)
+	if err != nil {
+		return nil, err
+	}
+
+	return clone, nil
+}
+
+// clone returns a clone of this utxoDiff
+func (d *UTXODiff) clone() *UTXODiff {
+	clone := &UTXODiff{
+		toAdd:    d.toAdd.clone(),
+		toRemove: d.toRemove.clone(),
+	}
+	return clone
+}
+
+// AddEntry adds a UTXOEntry to the diff
+//
+// If d.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (d *UTXODiff) AddEntry(outpoint domainmessage.Outpoint, entry *UTXOEntry) error {
+	if d.toRemove.containsWithBlueScore(outpoint, entry.blockBlueScore) {
+		d.toRemove.remove(outpoint)
+	} else if _, exists := d.toAdd[outpoint]; exists {
+		return errors.Errorf("AddEntry: Cannot add outpoint %s twice", outpoint)
+	} else {
+		d.toAdd.add(outpoint, entry)
+	}
+	return nil
+}
+
+// RemoveEntry removes a UTXOEntry from the diff.
+//
+// If d.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (d *UTXODiff) RemoveEntry(outpoint domainmessage.Outpoint, entry *UTXOEntry) error {
+	if d.toAdd.containsWithBlueScore(outpoint, entry.blockBlueScore) {
+		d.toAdd.remove(outpoint)
+	} else if _, exists := d.toRemove[outpoint]; exists {
+		return errors.Errorf("removeEntry: Cannot remove outpoint %s twice", outpoint)
+	} else {
+		d.toRemove.add(outpoint, entry)
+	}
+	return nil
+}
+
+func (d UTXODiff) String() string {
+	return fmt.Sprintf("toAdd: %s; toRemove: %s", d.toAdd, d.toRemove)
+}
+
+// UTXOSet represents a set of unspent transaction outputs
+// Every DAG has exactly one fullUTXOSet.
+// When a new block arrives, it is validated and applied to the fullUTXOSet in the following manner:
+// 1. Get the block's PastUTXO:
+// 2. Add all the block's transactions to the block's PastUTXO
+// 3. For each of the block's parents,
+// 3.1. Rebuild their utxoDiff
+// 3.2. Set the block as their diffChild
+// 4. Create and initialize a new virtual block
+// 5. Get the new virtual's PastUTXO
+// 6. Rebuild the utxoDiff for all the tips
+// 7. Convert (meld) the new virtual's diffUTXOSet into a fullUTXOSet. This updates the DAG's fullUTXOSet
+type UTXOSet interface {
+	fmt.Stringer
+	diffFrom(other UTXOSet) (*UTXODiff, error)
+	WithDiff(utxoDiff *UTXODiff) (UTXOSet, error)
+	AddTx(tx *domainmessage.MsgTx, blockBlueScore uint64) (ok bool, err error)
+	clone() UTXOSet
+	Get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool)
+}
+
+// FullUTXOSet represents a full list of transaction outputs and their values
+type FullUTXOSet struct {
+	utxoCollection
+}
+
+// NewFullUTXOSet creates a new utxoSet with full list of transaction outputs and their values
+func NewFullUTXOSet() *FullUTXOSet {
+	return &FullUTXOSet{
+		utxoCollection: utxoCollection{},
+	}
+}
+
+// newFullUTXOSetFromUTXOCollection converts a utxoCollection to a FullUTXOSet
+func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet, error) {
+	var err error
+	multiset := secp256k1.NewMultiset()
+	for outpoint, utxoEntry := range collection {
+		multiset, err = addUTXOToMultiset(multiset, utxoEntry, &outpoint)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &FullUTXOSet{
+		utxoCollection: collection,
+	}, nil
+}
+
+// diffFrom returns the difference between this utxoSet and another
+// diffFrom can only work when other is a diffUTXOSet, and its base utxoSet is this.
+func (fus *FullUTXOSet) diffFrom(other UTXOSet) (*UTXODiff, error) {
+	otherDiffSet, ok := other.(*DiffUTXOSet)
+	if !ok {
+		return nil, errors.New("can't diffFrom two fullUTXOSets")
+	}
+
+	if otherDiffSet.base != fus {
+		return nil, errors.New("can diffFrom only with diffUTXOSet where this fullUTXOSet is the base")
+	}
+
+	return otherDiffSet.UTXODiff, nil
+}
+
+// WithDiff returns a utxoSet which is a diff between this and another utxoSet
+func (fus *FullUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
+	return NewDiffUTXOSet(fus, other.clone()), nil
+}
+
+// AddTx adds a transaction to this utxoSet and returns isAccepted=true iff it's valid in this UTXO's context.
+// It returns error if something unexpected happens, such as serialization error (isAccepted=false doesn't
+// necessarily means there's an error).
+//
+// This function MUST be called with the DAG lock held.
+func (fus *FullUTXOSet) AddTx(tx *domainmessage.MsgTx, blueScore uint64) (isAccepted bool, err error) {
+	if !fus.containsInputs(tx) {
+		return false, nil
+	}
+
+	for _, txIn := range tx.TxIn {
+		fus.remove(txIn.PreviousOutpoint)
+	}
+
+	isCoinbase := tx.IsCoinBase()
+	for i, txOut := range tx.TxOut {
+		outpoint := *domainmessage.NewOutpoint(tx.TxID(), uint32(i))
+		entry := NewUTXOEntry(txOut, isCoinbase, blueScore)
+		fus.add(outpoint, entry)
+	}
+
+	return true, nil
+}
+
+func (fus *FullUTXOSet) containsInputs(tx *domainmessage.MsgTx) bool {
+	for _, txIn := range tx.TxIn {
+		outpoint := *domainmessage.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+		if !fus.contains(outpoint) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// clone returns a clone of this utxoSet
+func (fus *FullUTXOSet) clone() UTXOSet {
+	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone()}
+}
+
+// Get returns the UTXOEntry associated with the given Outpoint, and a boolean indicating if such entry was found
+func (fus *FullUTXOSet) Get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool) {
+	utxoEntry, ok := fus.utxoCollection[outpoint]
+	return utxoEntry, ok
+}
+
+// DiffUTXOSet represents a utxoSet with a base fullUTXOSet and a UTXODiff
+type DiffUTXOSet struct {
+	base     *FullUTXOSet
+	UTXODiff *UTXODiff
+}
+
+// NewDiffUTXOSet Creates a new utxoSet based on a base fullUTXOSet and a UTXODiff
+func NewDiffUTXOSet(base *FullUTXOSet, diff *UTXODiff) *DiffUTXOSet {
+	return &DiffUTXOSet{
+		base:     base,
+		UTXODiff: diff,
+	}
+}
+
+// diffFrom returns the difference between this utxoSet and another.
+// diffFrom can work if other is this's base fullUTXOSet, or a diffUTXOSet with the same base as this
+func (dus *DiffUTXOSet) diffFrom(other UTXOSet) (*UTXODiff, error) {
+	otherDiffSet, ok := other.(*DiffUTXOSet)
+	if !ok {
+		return nil, errors.New("can't diffFrom diffUTXOSet with fullUTXOSet")
+	}
+
+	if otherDiffSet.base != dus.base {
+		return nil, errors.New("can't diffFrom with another diffUTXOSet with a different base")
+	}
+
+	return dus.UTXODiff.diffFrom(otherDiffSet.UTXODiff)
+}
+
+// WithDiff return a new utxoSet which is a diffFrom between this and another utxoSet
+func (dus *DiffUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
+	diff, err := dus.UTXODiff.WithDiff(other)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewDiffUTXOSet(dus.base, diff), nil
+}
+
+// AddTx adds a transaction to this utxoSet and returns true iff it's valid in this UTXO's context.
+//
+// If dus.UTXODiff.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (dus *DiffUTXOSet) AddTx(tx *domainmessage.MsgTx, blockBlueScore uint64) (bool, error) {
+	if !dus.containsInputs(tx) {
+		return false, nil
+	}
+
+	err := dus.appendTx(tx, blockBlueScore)
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (dus *DiffUTXOSet) appendTx(tx *domainmessage.MsgTx, blockBlueScore uint64) error {
+	for _, txIn := range tx.TxIn {
+		entry, ok := dus.Get(txIn.PreviousOutpoint)
+		if !ok {
+			return errors.Errorf("couldn't find entry for outpoint %s", txIn.PreviousOutpoint)
+		}
+		err := dus.UTXODiff.RemoveEntry(txIn.PreviousOutpoint, entry)
+		if err != nil {
+			return err
+		}
+	}
+
+	isCoinbase := tx.IsCoinBase()
+	for i, txOut := range tx.TxOut {
+		outpoint := *domainmessage.NewOutpoint(tx.TxID(), uint32(i))
+		entry := NewUTXOEntry(txOut, isCoinbase, blockBlueScore)
+
+		err := dus.UTXODiff.AddEntry(outpoint, entry)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (dus *DiffUTXOSet) containsInputs(tx *domainmessage.MsgTx) bool {
+	for _, txIn := range tx.TxIn {
+		outpoint := *domainmessage.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+		isInBase := dus.base.contains(outpoint)
+		isInDiffToAdd := dus.UTXODiff.toAdd.contains(outpoint)
+		isInDiffToRemove := dus.UTXODiff.toRemove.contains(outpoint)
+		if (!isInBase && !isInDiffToAdd) || (isInDiffToRemove && !(isInBase && isInDiffToAdd)) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// meldToBase updates the base fullUTXOSet with all changes in diff
+func (dus *DiffUTXOSet) meldToBase() error {
+	for outpoint := range dus.UTXODiff.toRemove {
+		if _, ok := dus.base.Get(outpoint); ok {
+			dus.base.remove(outpoint)
+		} else {
+			return errors.Errorf("Couldn't remove outpoint %s because it doesn't exist in the DiffUTXOSet base", outpoint)
+		}
+	}
+
+	for outpoint, utxoEntry := range dus.UTXODiff.toAdd {
+		dus.base.add(outpoint, utxoEntry)
+	}
+	dus.UTXODiff = NewUTXODiff()
+	return nil
+}
+
+func (dus *DiffUTXOSet) String() string {
+	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove)
+}
+
+// clone returns a clone of this UTXO Set
+func (dus *DiffUTXOSet) clone() UTXOSet {
+	return NewDiffUTXOSet(dus.base.clone().(*FullUTXOSet), dus.UTXODiff.clone())
+}
+
+// cloneWithoutBase returns a *DiffUTXOSet with same
+// base as this *DiffUTXOSet and a cloned diff.
+func (dus *DiffUTXOSet) cloneWithoutBase() UTXOSet {
+	return NewDiffUTXOSet(dus.base, dus.UTXODiff.clone())
+}
+
+// Get returns the UTXOEntry associated with provided outpoint in this UTXOSet.
+// Returns false in second output if this UTXOEntry was not found
+func (dus *DiffUTXOSet) Get(outpoint domainmessage.Outpoint) (*UTXOEntry, bool) {
+	if toRemoveEntry, ok := dus.UTXODiff.toRemove.get(outpoint); ok {
+		// An exception is made for entries with unequal blue scores
+		// These are just "updates" to accepted blue score
+		if toAddEntry, ok := dus.UTXODiff.toAdd.get(outpoint); ok && toAddEntry.blockBlueScore != toRemoveEntry.blockBlueScore {
+			return toAddEntry, true
+		}
+		return nil, false
+	}
+	if txOut, ok := dus.base.get(outpoint); ok {
+		return txOut, true
+	}
+	txOut, ok := dus.UTXODiff.toAdd.get(outpoint)
+	return txOut, ok
+}

blockdag/versionbits.go

@@ -0,0 +1,294 @@
+// Copyright (c) 2016-2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"math"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+)
+
+const (
+	// vbTopBits defines the bits to set in the version to signal that the
+	// version bits scheme is being used.
+	vbTopBits = 0x10000000
+
+	// vbTopMask is the bitmask to use to determine whether or not the
+	// version bits scheme is in use.
+	vbTopMask = 0xe0000000
+
+	// vbNumBits is the total number of bits available for use with the
+	// version bits scheme.
+	vbNumBits = 29
+
+	// unknownVerNumToCheck is the number of previous blocks to consider
+	// when checking for a threshold of unknown block versions for the
+	// purposes of warning the user.
+	unknownVerNumToCheck = 100
+
+	// unknownVerWarnNum is the threshold of previous blocks that have an
+	// unknown version to use for the purposes of warning the user.
+	unknownVerWarnNum = unknownVerNumToCheck / 2
+)
+
+// bitConditionChecker provides a thresholdConditionChecker which can be used to
+// test whether or not a specific bit is set when it's not supposed to be
+// according to the expected version based on the known deployments and the
+// current state of the DAG. This is useful for detecting and warning about
+// unknown rule activations.
+type bitConditionChecker struct {
+	bit uint32
+	dag *BlockDAG
+}
+
+// Ensure the bitConditionChecker type implements the thresholdConditionChecker
+// interface.
+var _ thresholdConditionChecker = bitConditionChecker{}
+
+// BeginTime returns the unix timestamp for the median block time after which
+// voting on a rule change starts (at the next window).
+//
+// Since this implementation checks for unknown rules, it returns 0 so the rule
+// is always treated as active.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c bitConditionChecker) BeginTime() uint64 {
+	return 0
+}
+
+// EndTime returns the unix timestamp for the median block time after which an
+// attempted rule change fails if it has not already been locked in or
+// activated.
+//
+// Since this implementation checks for unknown rules, it returns the maximum
+// possible timestamp so the rule is always treated as active.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c bitConditionChecker) EndTime() uint64 {
+	return math.MaxUint64
+}
+
+// RuleChangeActivationThreshold is the number of blocks for which the condition
+// must be true in order to lock in a rule change.
+//
+// This implementation returns the value defined by the DAG params the checker
+// is associated with.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c bitConditionChecker) RuleChangeActivationThreshold() uint64 {
+	return c.dag.Params.RuleChangeActivationThreshold
+}
+
+// MinerConfirmationWindow is the number of blocks in each threshold state
+// retarget window.
+//
+// This implementation returns the value defined by the DAG params the checker
+// is associated with.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c bitConditionChecker) MinerConfirmationWindow() uint64 {
+	return c.dag.Params.MinerConfirmationWindow
+}
+
+// Condition returns true when the specific bit associated with the checker is
+// set and it's not supposed to be according to the expected version based on
+// the known deployments and the current state of the DAG.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c bitConditionChecker) Condition(node *blockNode) (bool, error) {
+	conditionMask := uint32(1) << c.bit
+	version := uint32(node.version)
+	if version&vbTopMask != vbTopBits {
+		return false, nil
+	}
+	if version&conditionMask == 0 {
+		return false, nil
+	}
+
+	expectedVersion, err := c.dag.calcNextBlockVersion(node.selectedParent)
+	if err != nil {
+		return false, err
+	}
+	return uint32(expectedVersion)&conditionMask == 0, nil
+}
+
+// deploymentChecker provides a thresholdConditionChecker which can be used to
+// test a specific deployment rule. This is required for properly detecting
+// and activating consensus rule changes.
+type deploymentChecker struct {
+	deployment *dagconfig.ConsensusDeployment
+	dag        *BlockDAG
+}
+
+// Ensure the deploymentChecker type implements the thresholdConditionChecker
+// interface.
+var _ thresholdConditionChecker = deploymentChecker{}
+
+// BeginTime returns the unix timestamp for the median block time after which
+// voting on a rule change starts (at the next window).
+//
+// This implementation returns the value defined by the specific deployment the
+// checker is associated with.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c deploymentChecker) BeginTime() uint64 {
+	return c.deployment.StartTime
+}
+
+// EndTime returns the unix timestamp for the median block time after which an
+// attempted rule change fails if it has not already been locked in or
+// activated.
+//
+// This implementation returns the value defined by the specific deployment the
+// checker is associated with.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c deploymentChecker) EndTime() uint64 {
+	return c.deployment.ExpireTime
+}
+
+// RuleChangeActivationThreshold is the number of blocks for which the condition
+// must be true in order to lock in a rule change.
+//
+// This implementation returns the value defined by the DAG params the checker
+// is associated with.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c deploymentChecker) RuleChangeActivationThreshold() uint64 {
+	return c.dag.Params.RuleChangeActivationThreshold
+}
+
+// MinerConfirmationWindow is the number of blocks in each threshold state
+// retarget window.
+//
+// This implementation returns the value defined by the DAG params the checker
+// is associated with.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c deploymentChecker) MinerConfirmationWindow() uint64 {
+	return c.dag.Params.MinerConfirmationWindow
+}
+
+// Condition returns true when the specific bit defined by the deployment
+// associated with the checker is set.
+//
+// This is part of the thresholdConditionChecker interface implementation.
+func (c deploymentChecker) Condition(node *blockNode) (bool, error) {
+	conditionMask := uint32(1) << c.deployment.BitNumber
+	version := uint32(node.version)
+	return (version&vbTopMask == vbTopBits) && (version&conditionMask != 0),
+		nil
+}
+
+// calcNextBlockVersion calculates the expected version of the block after the
+// passed previous block node based on the state of started and locked in
+// rule change deployments.
+//
+// This function differs from the exported CalcNextBlockVersion in that the
+// exported version uses the selected tip as the previous block node
+// while this function accepts any block node.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) calcNextBlockVersion(prevNode *blockNode) (int32, error) {
+	// Set the appropriate bits for each actively defined rule deployment
+	// that is either in the process of being voted on, or locked in for the
+	// activation at the next threshold window change.
+	expectedVersion := uint32(vbTopBits)
+	for id := 0; id < len(dag.Params.Deployments); id++ {
+		deployment := &dag.Params.Deployments[id]
+		cache := &dag.deploymentCaches[id]
+		checker := deploymentChecker{deployment: deployment, dag: dag}
+		state, err := dag.thresholdState(prevNode, checker, cache)
+		if err != nil {
+			return 0, err
+		}
+		if state == ThresholdStarted || state == ThresholdLockedIn {
+			expectedVersion |= uint32(1) << deployment.BitNumber
+		}
+	}
+	return int32(expectedVersion), nil
+}
+
+// CalcNextBlockVersion calculates the expected version of the block after the
+// end of the current selected tip based on the state of started and locked in
+// rule change deployments.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) CalcNextBlockVersion() (int32, error) {
+	version, err := dag.calcNextBlockVersion(dag.selectedTip())
+	return version, err
+}
+
+// warnUnknownRuleActivations displays a warning when any unknown new rules are
+// either about to activate or have been activated. This will only happen once
+// when new rules have been activated and every block for those about to be
+// activated.
+//
+// This function MUST be called with the DAG state lock held (for writes)
+func (dag *BlockDAG) warnUnknownRuleActivations(node *blockNode) error {
+	// Warn if any unknown new rules are either about to activate or have
+	// already been activated.
+	for bit := uint32(0); bit < vbNumBits; bit++ {
+		checker := bitConditionChecker{bit: bit, dag: dag}
+		cache := &dag.warningCaches[bit]
+		state, err := dag.thresholdState(node.selectedParent, checker, cache)
+		if err != nil {
+			return err
+		}
+
+		switch state {
+		case ThresholdActive:
+			if !dag.unknownRulesWarned {
+				log.Warnf("Unknown new rules activated (bit %d)",
+					bit)
+				dag.unknownRulesWarned = true
+			}
+
+		case ThresholdLockedIn:
+			window := checker.MinerConfirmationWindow()
+			activationBlueScore := window - (node.blueScore % window)
+			log.Warnf("Unknown new rules are about to activate in "+
+				"%d blueScore (bit %d)", activationBlueScore, bit)
+		}
+	}
+
+	return nil
+}
+
+// warnUnknownVersions logs a warning if a high enough percentage of the last
+// blocks have unexpected versions.
+//
+// This function MUST be called with the DAG state lock held (for writes)
+func (dag *BlockDAG) warnUnknownVersions(node *blockNode) error {
+	// Nothing to do if already warned.
+	if dag.unknownVersionsWarned {
+		return nil
+	}
+
+	// Warn if enough previous blocks have unexpected versions.
+	numUpgraded := uint32(0)
+	for i := uint32(0); i < unknownVerNumToCheck && node != nil; i++ {
+		expectedVersion, err := dag.calcNextBlockVersion(node.selectedParent)
+		if err != nil {
+			return err
+		}
+		if (node.version & ^expectedVersion) != 0 {
+
+			numUpgraded++
+		}
+
+		node = node.selectedParent
+	}
+	if numUpgraded > unknownVerWarnNum {
+		log.Warn("Unknown block versions are being mined, so new " +
+			"rules might be in effect. Are you running the " +
+			"latest version of the software?")
+		dag.unknownVersionsWarned = true
+	}
+
+	return nil
+}

blockdag/virtualblock.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util/daghash"
+	"sync"
+)
+
+// virtualBlock is a virtual block whose parents are the tips of the DAG.
+type virtualBlock struct {
+	mtx     sync.Mutex
+	dag     *BlockDAG
+	utxoSet *FullUTXOSet
+	blockNode
+
+	// selectedParentChainSet is a block set that includes all the blocks
+	// that belong to the chain of selected parents from the virtual block.
+	selectedParentChainSet blockSet
+
+	// selectedParentChainSlice is an ordered slice that includes all the
+	// blocks that belong the the chain of selected parents from the
+	// virtual block.
+	selectedParentChainSlice []*blockNode
+}
+
+// newVirtualBlock creates and returns a new VirtualBlock.
+func newVirtualBlock(dag *BlockDAG, tips blockSet) *virtualBlock {
+	// The mutex is intentionally not held since this is a constructor.
+	var virtual virtualBlock
+	virtual.dag = dag
+	virtual.utxoSet = NewFullUTXOSet()
+	virtual.selectedParentChainSet = newBlockSet()
+	virtual.selectedParentChainSlice = nil
+	virtual.setTips(tips)
+
+	return &virtual
+}
+
+// setTips replaces the tips of the virtual block with the blocks in the
+// given blockSet. This only differs from the exported version in that it
+// is up to the caller to ensure the lock is held.
+//
+// This function MUST be called with the view mutex locked (for writes).
+func (v *virtualBlock) setTips(tips blockSet) *chainUpdates {
+	oldSelectedParent := v.selectedParent
+	node, _ := v.dag.newBlockNode(nil, tips)
+	v.blockNode = *node
+	return v.updateSelectedParentSet(oldSelectedParent)
+}
+
+// updateSelectedParentSet updates the selectedParentSet to match the
+// new selected parent of the virtual block.
+// Every time the new selected parent is not a child of
+// the old one, it updates the selected path by removing from
+// the path blocks that are selected ancestors of the old selected
+// parent and are not selected ancestors of the new one, and adding
+// blocks that are selected ancestors of the new selected parent
+// and aren't selected ancestors of the old one.
+func (v *virtualBlock) updateSelectedParentSet(oldSelectedParent *blockNode) *chainUpdates {
+	var intersectionNode *blockNode
+	nodesToAdd := make([]*blockNode, 0)
+	for node := v.blockNode.selectedParent; intersectionNode == nil && node != nil; node = node.selectedParent {
+		if v.selectedParentChainSet.contains(node) {
+			intersectionNode = node
+		} else {
+			nodesToAdd = append(nodesToAdd, node)
+		}
+	}
+
+	if intersectionNode == nil && oldSelectedParent != nil {
+		panic("updateSelectedParentSet: Cannot find intersection node. The block index may be corrupted.")
+	}
+
+	// Remove the nodes in the set from the oldSelectedParent down to the intersectionNode
+	// Also, save the hashes of the removed blocks to removedChainBlockHashes
+	removeCount := 0
+	var removedChainBlockHashes []*daghash.Hash
+	if intersectionNode != nil {
+		for node := oldSelectedParent; !node.hash.IsEqual(intersectionNode.hash); node = node.selectedParent {
+			v.selectedParentChainSet.remove(node)
+			removedChainBlockHashes = append(removedChainBlockHashes, node.hash)
+			removeCount++
+		}
+	}
+	// Remove the last removeCount nodes from the slice
+	v.selectedParentChainSlice = v.selectedParentChainSlice[:len(v.selectedParentChainSlice)-removeCount]
+
+	// Reverse nodesToAdd, since we collected them in reverse order
+	for left, right := 0, len(nodesToAdd)-1; left < right; left, right = left+1, right-1 {
+		nodesToAdd[left], nodesToAdd[right] = nodesToAdd[right], nodesToAdd[left]
+	}
+	// Add the nodes to the set and to the slice
+	// Also, save the hashes of the added blocks to addedChainBlockHashes
+	var addedChainBlockHashes []*daghash.Hash
+	for _, node := range nodesToAdd {
+		v.selectedParentChainSet.add(node)
+		addedChainBlockHashes = append(addedChainBlockHashes, node.hash)
+	}
+	v.selectedParentChainSlice = append(v.selectedParentChainSlice, nodesToAdd...)
+
+	return &chainUpdates{
+		removedChainBlockHashes: removedChainBlockHashes,
+		addedChainBlockHashes:   addedChainBlockHashes,
+	}
+}
+
+// SetTips replaces the tips of the virtual block with the blocks in the
+// given blockSet.
+//
+// This function is safe for concurrent access.
+func (v *virtualBlock) SetTips(tips blockSet) {
+	v.mtx.Lock()
+	defer v.mtx.Unlock()
+	v.setTips(tips)
+}
+
+// addTip adds the given tip to the set of tips in the virtual block.
+// All former tips that happen to be the given tips parents are removed
+// from the set. This only differs from the exported version in that it
+// is up to the caller to ensure the lock is held.
+//
+// This function MUST be called with the view mutex locked (for writes).
+func (v *virtualBlock) addTip(newTip *blockNode) *chainUpdates {
+	updatedTips := v.tips().clone()
+	for parent := range newTip.parents {
+		updatedTips.remove(parent)
+	}
+
+	updatedTips.add(newTip)
+	return v.setTips(updatedTips)
+}
+
+// AddTip adds the given tip to the set of tips in the virtual block.
+// All former tips that happen to be the given tip's parents are removed
+// from the set.
+//
+// This function is safe for concurrent access.
+func (v *virtualBlock) AddTip(newTip *blockNode) *chainUpdates {
+	v.mtx.Lock()
+	defer v.mtx.Unlock()
+	return v.addTip(newTip)
+}
+
+// tips returns the current tip block nodes for the DAG. It will return
+// an empty blockSet if there is no tip.
+//
+// This function is safe for concurrent access.
+func (v *virtualBlock) tips() blockSet {
+	return v.parents
+}

blockdag/virtualblock_test.go

@@ -0,0 +1,282 @@
+// Copyright (c) 2017 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util"
+	"reflect"
+	"testing"
+)
+
+func buildNode(t *testing.T, dag *BlockDAG, parents blockSet) *blockNode {
+	block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	utilBlock := util.NewBlock(block)
+	isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("unexpected error in ProcessBlock: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("block is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("block was unexpectedly orphan")
+	}
+	return nodeByMsgBlock(t, dag, block)
+}
+
+// TestVirtualBlock ensures that VirtualBlock works as expected.
+func TestVirtualBlock(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimnetParams
+	params.K = 1
+	dag, teardownFunc, err := DAGSetup("TestVirtualBlock", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestVirtualBlock: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	resetExtraNonceForTest()
+
+	// Create a DAG as follows:
+	// 0 <- 1 <- 2
+	//  \
+	//   <- 3 <- 5
+	//  \    X
+	//   <- 4 <- 6
+	node0 := dag.genesis
+	node1 := buildNode(t, dag, blockSetFromSlice(node0))
+	node2 := buildNode(t, dag, blockSetFromSlice(node1))
+	node3 := buildNode(t, dag, blockSetFromSlice(node0))
+	node4 := buildNode(t, dag, blockSetFromSlice(node0))
+	node5 := buildNode(t, dag, blockSetFromSlice(node3, node4))
+	node6 := buildNode(t, dag, blockSetFromSlice(node3, node4))
+
+	// Given an empty VirtualBlock, each of the following test cases will:
+	// Set its tips to tipsToSet
+	// Add to it all the tips in tipsToAdd, one after the other
+	// Call .Tips() on it and compare the result to expectedTips
+	// Call .selectedTip() on it and compare the result to expectedSelectedParent
+	tests := []struct {
+		name                   string
+		tipsToSet              []*blockNode
+		tipsToAdd              []*blockNode
+		expectedTips           blockSet
+		expectedSelectedParent *blockNode
+	}{
+		{
+			name:                   "empty virtual",
+			tipsToSet:              []*blockNode{},
+			tipsToAdd:              []*blockNode{},
+			expectedTips:           newBlockSet(),
+			expectedSelectedParent: nil,
+		},
+		{
+			name:                   "virtual with genesis tip",
+			tipsToSet:              []*blockNode{node0},
+			tipsToAdd:              []*blockNode{},
+			expectedTips:           blockSetFromSlice(node0),
+			expectedSelectedParent: node0,
+		},
+		{
+			name:                   "virtual with genesis tip, add child of genesis",
+			tipsToSet:              []*blockNode{node0},
+			tipsToAdd:              []*blockNode{node1},
+			expectedTips:           blockSetFromSlice(node1),
+			expectedSelectedParent: node1,
+		},
+		{
+			name:                   "empty virtual, add a full DAG",
+			tipsToSet:              []*blockNode{},
+			tipsToAdd:              []*blockNode{node0, node1, node2, node3, node4, node5, node6},
+			expectedTips:           blockSetFromSlice(node2, node5, node6),
+			expectedSelectedParent: node5,
+		},
+	}
+
+	for _, test := range tests {
+		// Create an empty VirtualBlock
+		virtual := newVirtualBlock(dag, nil)
+
+		// Set the tips. This will be the initial state
+		virtual.SetTips(blockSetFromSlice(test.tipsToSet...))
+
+		// Add all blockNodes in tipsToAdd in order
+		for _, tipToAdd := range test.tipsToAdd {
+			addNodeAsChildToParents(tipToAdd)
+			virtual.AddTip(tipToAdd)
+		}
+
+		// Ensure that the virtual block's tips are now equal to expectedTips
+		resultTips := virtual.tips()
+		if !reflect.DeepEqual(resultTips, test.expectedTips) {
+			t.Errorf("unexpected tips in test \"%s\". "+
+				"Expected: %v, got: %v.", test.name, test.expectedTips, resultTips)
+		}
+
+		// Ensure that the virtual block's selectedParent is now equal to expectedSelectedParent
+		resultSelectedTip := virtual.selectedParent
+		if !reflect.DeepEqual(resultSelectedTip, test.expectedSelectedParent) {
+			t.Errorf("unexpected selected tip in test \"%s\". "+
+				"Expected: %v, got: %v.", test.name, test.expectedSelectedParent, resultSelectedTip)
+		}
+	}
+}
+
+func TestSelectedPath(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimnetParams
+	params.K = 1
+	dag, teardownFunc, err := DAGSetup("TestSelectedPath", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestSelectedPath: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	// Create an empty VirtualBlock
+	virtual := newVirtualBlock(dag, nil)
+
+	tip := dag.genesis
+	virtual.AddTip(tip)
+	initialPath := blockSetFromSlice(tip)
+	for i := 0; i < 5; i++ {
+		tip = buildNode(t, dag, blockSetFromSlice(tip))
+		initialPath.add(tip)
+		virtual.AddTip(tip)
+	}
+	initialTip := tip
+
+	firstPath := initialPath.clone()
+	for i := 0; i < 5; i++ {
+		tip = buildNode(t, dag, blockSetFromSlice(tip))
+		firstPath.add(tip)
+		virtual.AddTip(tip)
+	}
+	// For now we don't have any DAG, just chain, the selected path should include all the blocks on the chain.
+	if !reflect.DeepEqual(virtual.selectedParentChainSet, firstPath) {
+		t.Fatalf("TestSelectedPath: selectedPathSet doesn't include the expected values. got %v, want %v", virtual.selectedParent, firstPath)
+	}
+	// We expect that selectedParentChainSlice should have all the blocks we've added so far
+	wantLen := 11
+	gotLen := len(virtual.selectedParentChainSlice)
+	if wantLen != gotLen {
+		t.Fatalf("TestSelectedPath: selectedParentChainSlice doesn't have the expected length. got %d, want %d", gotLen, wantLen)
+	}
+
+	secondPath := initialPath.clone()
+	tip = initialTip
+	for i := 0; i < 100; i++ {
+		tip = buildNode(t, dag, blockSetFromSlice(tip))
+		secondPath.add(tip)
+		virtual.AddTip(tip)
+	}
+	// Because we added a chain that is much longer than the previous chain, the selected path should be re-organized.
+	if !reflect.DeepEqual(virtual.selectedParentChainSet, secondPath) {
+		t.Fatalf("TestSelectedPath: selectedPathSet didn't handle the re-org as expected. got %v, want %v", virtual.selectedParent, firstPath)
+	}
+	// We expect that selectedParentChainSlice should have all the blocks we've added so far except the old chain
+	wantLen = 106
+	gotLen = len(virtual.selectedParentChainSlice)
+	if wantLen != gotLen {
+		t.Fatalf("TestSelectedPath: selectedParentChainSlice doesn't have"+
+			"the expected length, possibly because it didn't handle the re-org as expected. got %d, want %d", gotLen, wantLen)
+	}
+
+	tip = initialTip
+	for i := 0; i < 3; i++ {
+		tip = buildNode(t, dag, blockSetFromSlice(tip))
+		virtual.AddTip(tip)
+	}
+	// Because we added a very short chain, the selected path should not be affected.
+	if !reflect.DeepEqual(virtual.selectedParentChainSet, secondPath) {
+		t.Fatalf("TestSelectedPath: selectedPathSet did an unexpected re-org. got %v, want %v", virtual.selectedParent, firstPath)
+	}
+	// We expect that selectedParentChainSlice not to change
+	wantLen = 106
+	gotLen = len(virtual.selectedParentChainSlice)
+	if wantLen != gotLen {
+		t.Fatalf("TestSelectedPath: selectedParentChainSlice doesn't"+
+			"have the expected length, possibly due to unexpected did an unexpected re-org. got %d, want %d", gotLen, wantLen)
+	}
+
+	// We call updateSelectedParentSet manually without updating the tips, to check if it panics
+	virtual2 := newVirtualBlock(dag, nil)
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatalf("updateSelectedParentSet didn't panic")
+		}
+	}()
+	virtual2.updateSelectedParentSet(buildNode(t, dag, blockSetFromSlice()))
+}
+
+func TestChainUpdates(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimnetParams
+	params.K = 1
+	dag, teardownFunc, err := DAGSetup("TestChainUpdates", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestChainUpdates: Failed to setup DAG instance: %s", err)
+	}
+	defer teardownFunc()
+
+	genesis := dag.genesis
+
+	// Create a chain to be removed
+	var toBeRemovedNodes []*blockNode
+	toBeRemovedTip := genesis
+	for i := 0; i < 5; i++ {
+		toBeRemovedTip = buildNode(t, dag, blockSetFromSlice(toBeRemovedTip))
+		toBeRemovedNodes = append(toBeRemovedNodes, toBeRemovedTip)
+	}
+
+	// Create a VirtualBlock with the toBeRemoved chain
+	virtual := newVirtualBlock(dag, blockSetFromSlice(toBeRemovedNodes...))
+
+	// Create a chain to be added
+	var toBeAddedNodes []*blockNode
+	toBeAddedTip := genesis
+	for i := 0; i < 8; i++ {
+		toBeAddedTip = buildNode(t, dag, blockSetFromSlice(toBeAddedTip))
+		toBeAddedNodes = append(toBeAddedNodes, toBeAddedTip)
+	}
+
+	// Set the virtual tip to be the tip of the toBeAdded chain
+	chainUpdates := virtual.setTips(blockSetFromSlice(toBeAddedTip))
+
+	// Make sure that the removed blocks are as expected (in reverse order)
+	if len(chainUpdates.removedChainBlockHashes) != len(toBeRemovedNodes) {
+		t.Fatalf("TestChainUpdates: wrong removed amount. "+
+			"Got: %d, want: %d", len(chainUpdates.removedChainBlockHashes), len(toBeRemovedNodes))
+	}
+	for i, removedHash := range chainUpdates.removedChainBlockHashes {
+		correspondingRemovedNode := toBeRemovedNodes[len(toBeRemovedNodes)-1-i]
+		if !removedHash.IsEqual(correspondingRemovedNode.hash) {
+			t.Fatalf("TestChainUpdates: wrong removed hash. "+
+				"Got: %s, want: %s", removedHash, correspondingRemovedNode.hash)
+		}
+	}
+
+	// Make sure that the added blocks are as expected (in forward order)
+	if len(chainUpdates.addedChainBlockHashes) != len(toBeAddedNodes) {
+		t.Fatalf("TestChainUpdates: wrong added amount. "+
+			"Got: %d, want: %d", len(chainUpdates.removedChainBlockHashes), len(toBeAddedNodes))
+	}
+	for i, addedHash := range chainUpdates.addedChainBlockHashes {
+		correspondingAddedNode := toBeAddedNodes[i]
+		if !addedHash.IsEqual(correspondingAddedNode.hash) {
+			t.Fatalf("TestChainUpdates: wrong added hash. "+
+				"Got: %s, want: %s", addedHash, correspondingAddedNode.hash)
+		}
+	}
+}

blockmanager.go

@@ -1,922 +0,0 @@
-// Copyright (c) 2013 Conformal Systems LLC.
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"container/list"
-	"fmt"
-	"github.com/conformal/btcchain"
-	"github.com/conformal/btcdb"
-	"github.com/conformal/btcutil"
-	"github.com/conformal/btcwire"
-	"net"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-const (
-	chanBufferSize = 50
-
-	// blockDbNamePrefix is the prefix for the block database name.  The
-	// database type is appended to this value to form the full block
-	// database name.
-	blockDbNamePrefix = "blocks"
-)
-
-// newPeerMsg signifies a newly connected peer to the block handler.
-type newPeerMsg struct {
-	peer *peer
-}
-
-// blockMsg packages a bitcoin block message and the peer it came from together
-// so the block handler has access to that information.
-type blockMsg struct {
-	block *btcutil.Block
-	peer  *peer
-}
-
-// invMsg packages a bitcoin inv message and the peer it came from together
-// so the block handler has access to that information.
-type invMsg struct {
-	inv  *btcwire.MsgInv
-	peer *peer
-}
-
-// donePeerMsg signifies a newly disconnected peer to the block handler.
-type donePeerMsg struct {
-	peer *peer
-}
-
-// txMsg packages a bitcoin tx message and the peer it came from together
-// so the block handler has access to that information.
-type txMsg struct {
-	tx   *btcwire.MsgTx
-	peer *peer
-}
-
-// blockManager provides a concurrency safe block manager for handling all
-// incoming blocks.
-type blockManager struct {
-	server            *server
-	started           int32
-	shutdown          int32
-	blockChain        *btcchain.BlockChain
-	blockPeer         map[btcwire.ShaHash]*peer
-	requestedTxns     map[btcwire.ShaHash]bool
-	requestedBlocks   map[btcwire.ShaHash]bool
-	receivedLogBlocks int64
-	receivedLogTx     int64
-	lastBlockLogTime  time.Time
-	processingReqs    bool
-	syncPeer          *peer
-	msgChan           chan interface{}
-	wg                sync.WaitGroup
-	quit              chan bool
-}
-
-// startSync will choose the best peer among the available candidate peers to
-// download/sync the blockchain from.  When syncing is already running, it
-// simply returns.  It also examines the candidates for any which are no longer
-// candidates and removes them as needed.
-func (b *blockManager) startSync(peers *list.List) {
-	// Return now if we're already syncing.
-	if b.syncPeer != nil {
-		return
-	}
-
-	// Find the height of the current known best block.
-	_, height, err := b.server.db.NewestSha()
-	if err != nil {
-		log.Errorf("BMGR: %v", err)
-		return
-	}
-
-	var bestPeer *peer
-	var enext *list.Element
-	for e := peers.Front(); e != nil; e = enext {
-		enext = e.Next()
-		p := e.Value.(*peer)
-
-		// Remove sync candidate peers that are no longer candidates due
-		// to passing their latest known block.  NOTE: The < is
-		// intentional as opposed to <=.  While techcnically the peer
-		// doesn't have a later block when it's equal, it will likely
-		// have one soon so it is a reasonable choice.  It also allows
-		// the case where both are at 0 such as during regression test.
-		if p.lastBlock < int32(height) {
-			peers.Remove(e)
-			continue
-		}
-
-		// TODO(davec): Use a better algorithm to choose the best peer.
-		// For now, just pick the first available candidate.
-		bestPeer = p
-	}
-
-	// Start syncing from the best peer if one was selected.
-	if bestPeer != nil {
-		locator, err := b.blockChain.LatestBlockLocator()
-		if err != nil {
-			log.Errorf("BMGR: Failed to get block locator for the "+
-				"latest block: %v", err)
-			return
-		}
-
-		log.Infof("BMGR: Syncing to block height %d from peer %v",
-			bestPeer.lastBlock, bestPeer.addr)
-		bestPeer.PushGetBlocksMsg(locator, &zeroHash)
-		b.syncPeer = bestPeer
-	} else {
-		log.Warnf("BMGR: No sync peer candidates available")
-	}
-}
-
-// isSyncCandidate returns whether or not the peer is a candidate to consider
-// syncing from.
-func (b *blockManager) isSyncCandidate(p *peer) bool {
-	// Typically a peer is not a candidate for sync if it's not a full node,
-	// however regression test is special in that the regression tool is
-	// not a full node and still needs to be considered a sync candidate.
-	if cfg.RegressionTest {
-		// The peer is not a candidate if it's not coming from localhost
-		// or the hostname can't be determined for some reason.
-		host, _, err := net.SplitHostPort(p.addr)
-		if err != nil {
-			return false
-		}
-
-		if host != "127.0.0.1" && host != "localhost" {
-			return false
-		}
-	} else {
-		// The peer is not a candidate for sync if it's not a full node.
-		if p.services&btcwire.SFNodeNetwork != btcwire.SFNodeNetwork {
-			return false
-		}
-	}
-
-	// Candidate if all checks passed.
-	return true
-}
-
-// handleNewPeerMsg deals with new peers that have signalled they may
-// be considered as a sync peer (they have already successfully negotiated).  It
-// also starts syncing if needed.  It is invoked from the syncHandler goroutine.
-func (b *blockManager) handleNewPeerMsg(peers *list.List, p *peer) {
-	// Ignore if in the process of shutting down.
-	if atomic.LoadInt32(&b.shutdown) != 0 {
-		return
-	}
-
-	log.Infof("BMGR: New valid peer %s", p)
-
-	// Ignore the peer if it's not a sync candidate.
-	if !b.isSyncCandidate(p) {
-		return
-	}
-
-	// Add the peer as a candidate to sync from.
-	peers.PushBack(p)
-
-	// Start syncing by choosing the best candidate if needed.
-	b.startSync(peers)
-}
-
-// handleDonePeerMsg deals with peers that have signalled they are done.  It
-// removes the peer as a candidate for syncing and in the case where it was
-// the current sync peer, attempts to select a new best peer to sync from.  It
-// is invoked from the syncHandler goroutine.
-func (b *blockManager) handleDonePeerMsg(peers *list.List, p *peer) {
-	// Remove the peer from the list of candidate peers.
-	for e := peers.Front(); e != nil; e = e.Next() {
-		if e.Value == p {
-			peers.Remove(e)
-			break
-		}
-	}
-
-	log.Infof("BMGR: Lost peer %s", p)
-
-	// Remove requested transactions from the global map so that they will
-	// be fetched from elsewhere next time we get an inv.
-	for k := range p.requestedTxns {
-		delete(b.requestedTxns, k)
-	}
-
-	// Remove requested blocks from the global map so that they will be
-	// fetched from elsewhere next time we get an inv.
-	// TODO(oga) we could possibly here check which peers have these blocks
-	// and request them now to speed things up a little.
-	for k := range p.requestedBlocks {
-		delete(b.requestedBlocks, k)
-	}
-
-	// Attempt to find a new peer to sync from if the quitting peer is the
-	// sync peer.
-	if b.syncPeer != nil && b.syncPeer == p {
-		b.syncPeer = nil
-		b.startSync(peers)
-	}
-}
-
-// logBlockHeight logs a new block height as an information message to show
-// progress to the user.  In order to prevent spam, it limits logging to one
-// message every 10 seconds with duration and totals included.
-func (b *blockManager) logBlockHeight(numTx, height int64, latestHash *btcwire.ShaHash) {
-	b.receivedLogBlocks++
-	b.receivedLogTx += numTx
-
-	now := time.Now()
-	duration := now.Sub(b.lastBlockLogTime)
-	if duration < time.Second*10 {
-		return
-	}
-
-	// Truncated the duration to 10s of milliseconds.
-	durationMillis := int64(duration / time.Millisecond)
-	tDuration := 10 * time.Millisecond * time.Duration(durationMillis/10)
-
-	// Attempt to get the timestamp of the latest block.
-	blockTimeStr := ""
-	block, err := b.server.db.FetchBlockBySha(latestHash)
-	if err == nil {
-		blockTimeStr = fmt.Sprintf(", %s", block.MsgBlock().Header.Timestamp)
-	}
-
-	// Log information about new block height.
-	blockStr := "blocks"
-	if b.receivedLogBlocks == 1 {
-		blockStr = "block"
-	}
-	txStr := "transactions"
-	if b.receivedLogTx == 1 {
-		txStr = "transaction"
-	}
-	log.Infof("BMGR: Processed %d %s in the last %s (%d %s, height %d%s)",
-		b.receivedLogBlocks, blockStr, tDuration, b.receivedLogTx,
-		txStr, height, blockTimeStr)
-
-	b.receivedLogBlocks = 0
-	b.receivedLogTx = 0
-	b.lastBlockLogTime = now
-}
-
-// handleTxMsg handles transaction messages from all peers.
-func (b *blockManager) handleTxMsg(tmsg *txMsg) {
-	// Keep track of which peer the tx was sent from.
-	txHash, _ := tmsg.tx.TxSha()
-
-	// If we didn't ask for this block then the peer is misbehaving.
-	if _, ok := tmsg.peer.requestedTxns[txHash]; !ok {
-		log.Warnf("BMGR: Got unrequested transaction %v from %s -- "+
-			"disconnecting", &txHash, tmsg.peer.addr)
-		tmsg.peer.Disconnect()
-		return
-	}
-
-	// Process the transaction to include validation, insertion in the
-	// memory pool, orphan handling, etc.
-	err := tmsg.peer.server.txMemPool.ProcessTransaction(tmsg.tx)
-
-	// Remove transaction from request maps. Either the mempool/chain
-	// already knows about it and as such we shouldn't have any more
-	// instances of trying to fetch it, or we failed to insert and thus
-	// we'll retry next time we get an inv.
-	delete(tmsg.peer.requestedTxns, txHash)
-	delete(b.requestedTxns, txHash)
-
-	if err != nil {
-		// When the error is a rule error, it means the transaction was
-		// simply rejected as opposed to something actually going wrong,
-		// so log it as such.  Otherwise, something really did go wrong,
-		// so log it as an actual error.
-		if _, ok := err.(TxRuleError); ok {
-			log.Debugf("Rejected transaction %v: %v", txHash, err)
-		} else {
-			log.Errorf("Failed to process transaction %v: %v", txHash, err)
-		}
-		return
-	}
-}
-
-// current returns true if we believe we are synced with our peers, false if we
-// still have blocks to check
-func (b *blockManager) current() bool {
-	if !b.blockChain.IsCurrent() {
-		return false
-	}
-
-	// if blockChain thinks we are current and we have no syncPeer it
-	// is probably right.
-	if b.syncPeer == nil {
-		return true
-	}
-
-	_, height, err := b.server.db.NewestSha()
-	// No matter what chain thinks, if we are below the block we are
-	// syncing to we are not current.
-	// TODO(oga) we can get chain to return the height of each block when we
-	// parse an orphan, which would allow us to update the height of peers
-	// from what it was at initial handshake.
-	if err != nil || height < int64(b.syncPeer.lastBlock) {
-		return false
-	}
-	return true
-}
-
-// handleBlockMsg handles block messages from all peers.
-func (b *blockManager) handleBlockMsg(bmsg *blockMsg) {
-	// Keep track of which peer the block was sent from so the notification
-	// handler can request the parent blocks from the appropriate peer.
-	blockSha, _ := bmsg.block.Sha()
-
-	// If we didn't ask for this block then the peer is misbehaving.
-	if _, ok := bmsg.peer.requestedBlocks[*blockSha]; !ok {
-		// The regression test intentionally sends some blocks twice
-		// to test duplicate block insertion fails.  Don't disconnect
-		// the peer or ignore the block when we're in regression test
-		// mode in this case so the chain code is actually fed the
-		// duplicate blocks.
-		if !cfg.RegressionTest {
-			log.Warnf("BMGR: Got unrequested block %v from %s -- "+
-				"disconnecting", blockSha, bmsg.peer.addr)
-			bmsg.peer.Disconnect()
-			return
-		}
-	}
-	b.blockPeer[*blockSha] = bmsg.peer
-
-	// Process the block to include validation, best chain selection, orphan
-	// handling, etc.
-	err := b.blockChain.ProcessBlock(bmsg.block)
-
-	// Remove block from request maps. Either chain knows about it and such
-	// we shouldn't have any more instances of trying to fetch it, or we
-	// failed to insert and thus we'll retry next time we get an inv.
-	delete(bmsg.peer.requestedBlocks, *blockSha)
-	delete(b.requestedBlocks, *blockSha)
-
-	if err != nil {
-		delete(b.blockPeer, *blockSha)
-
-		// When the error is a rule error, it means the block was simply
-		// rejected as opposed to something actually going wrong, so log
-		// it as such.  Otherwise, something really did go wrong, so log
-		// it as an actual error.
-		if _, ok := err.(btcchain.RuleError); ok {
-			log.Infof("BMGR: Rejected block %v: %v", blockSha, err)
-		} else {
-			log.Errorf("BMGR: Failed to process block %v: %v", blockSha, err)
-		}
-		return
-	}
-
-	// Don't keep track of the peer that sent the block any longer if it's
-	// not an orphan.
-	if !b.blockChain.IsKnownOrphan(blockSha) {
-		delete(b.blockPeer, *blockSha)
-	}
-
-	// Log info about the new block height.
-	latestHash, height, err := b.server.db.NewestSha()
-	if err != nil {
-		log.Warnf("BMGR: Failed to obtain latest sha - %v", err)
-		return
-	}
-	b.logBlockHeight(int64(len(bmsg.block.MsgBlock().Transactions)), height,
-		latestHash)
-
-	// Sync the db to disk.
-	b.server.db.Sync()
-}
-
-// haveInventory returns whether or not the inventory represented by the passed
-// inventory vector is known.  This includes checking all of the various places
-// inventory can be when it is in different states such as blocks that are part
-// of the main chain, on a side chain, in the orphan pool, and transactions that
-// in the memory pool (either the main pool or orphan pool).
-func (b *blockManager) haveInventory(invVect *btcwire.InvVect) bool {
-	switch invVect.Type {
-	case btcwire.InvVect_Block:
-		// Ask chain if the block is known to it in any form (main
-		// chain, side chain, or orphan).
-		return b.blockChain.HaveBlock(&invVect.Hash)
-
-	case btcwire.InvVect_Tx:
-		// Ask the transaction memory pool if the transaction is known
-		// to it in any form (main pool or orphan).
-		if b.server.txMemPool.HaveTransaction(&invVect.Hash) {
-			return true
-		}
-
-		// Check if the transaction exists from the point of view of the
-		// end of the main chain.
-		return b.server.db.ExistsTxSha(&invVect.Hash)
-	}
-
-	// The requested inventory is is an unsupported type, so just claim
-	// it is known to avoid requesting it.
-	return true
-}
-
-// handleInvMsg handles inv messages from all peers.
-// We examine the inventory advertised by the remote peer and act accordingly.
-func (b *blockManager) handleInvMsg(imsg *invMsg) {
-	// Ignore invs from peers that aren't the sync if we are not current.
-	// Helps prevent fetching a mass of orphans.
-	if imsg.peer != b.syncPeer && !b.current() {
-		return
-	}
-
-	// Attempt to find the final block in the inventory list.  There may
-	// not be one.
-	lastBlock := -1
-	invVects := imsg.inv.InvList
-	for i := len(invVects) - 1; i >= 0; i-- {
-		if invVects[i].Type == btcwire.InvTypeBlock {
-			lastBlock = i
-			break
-		}
-	}
-
-	// Request the advertised inventory if we don't already have it.  Also,
-	// request parent blocks of orphans if we receive one we already have.
-	// Finally, attempt to detect potential stalls due to long side chains
-	// we already have and request more blocks to prevent them.
-	chain := b.blockChain
-	for i, iv := range invVects {
-		// Ignore unsupported inventory types.
-		if iv.Type != btcwire.InvTypeBlock && iv.Type != btcwire.InvTypeTx {
-			continue
-		}
-
-		// Add the inventory to the cache of known inventory
-		// for the peer.
-		imsg.peer.addKnownInventory(iv)
-
-		// Request the inventory if we don't already have it.
-		if !b.haveInventory(iv) {
-			// Add it to the request queue.
-			imsg.peer.requestQueue.PushBack(iv)
-			continue
-		}
-
-		if iv.Type == btcwire.InvTypeBlock {
-			// The block is an orphan block that we already have.
-			// When the existing orphan was processed, it requested
-			// the missing parent blocks.  When this scenario
-			// happens, it means there were more blocks missing
-			// than are allowed into a single inventory message.  As
-			// a result, once this peer requested the final
-			// advertised block, the remote peer noticed and is now
-			// resending the orphan block as an available block
-			// to signal there are more missing blocks that need to
-			// be requested.
-			if chain.IsKnownOrphan(&iv.Hash) {
-				// Request blocks starting at the latest known
-				// up to the root of the orphan that just came
-				// in.
-				orphanRoot := chain.GetOrphanRoot(&iv.Hash)
-				locator, err := chain.LatestBlockLocator()
-				if err != nil {
-					log.Errorf("PEER: Failed to get block "+
-						"locator for the latest block: "+
-						"%v", err)
-					continue
-				}
-				imsg.peer.PushGetBlocksMsg(locator, orphanRoot)
-				continue
-			}
-
-			// We already have the final block advertised by this
-			// inventory message, so force a request for more.  This
-			// should only happen if we're on a really long side
-			// chain.
-			if i == lastBlock {
-				// Request blocks after this one up to the
-				// final one the remote peer knows about (zero
-				// stop hash).
-				locator := chain.BlockLocatorFromHash(&iv.Hash)
-				imsg.peer.PushGetBlocksMsg(locator, &zeroHash)
-			}
-		}
-	}
-
-	// Request as much as possible at once.  Anything that won't fit into
-	// the request will be requested on the next inv message.
-	numRequested := 0
-	gdmsg := btcwire.NewMsgGetData()
-	requestQueue := imsg.peer.requestQueue
-	for e := requestQueue.Front(); e != nil; e = requestQueue.Front() {
-		iv := e.Value.(*btcwire.InvVect)
-		imsg.peer.requestQueue.Remove(e)
-
-		switch iv.Type {
-		case btcwire.InvVect_Block:
-			// Request the block if there is not already a pending
-			// request.
-			if _, exists := b.requestedBlocks[iv.Hash]; !exists {
-				b.requestedBlocks[iv.Hash] = true
-				imsg.peer.requestedBlocks[iv.Hash] = true
-				gdmsg.AddInvVect(iv)
-				numRequested++
-			}
-
-		case btcwire.InvVect_Tx:
-			// Request the transaction if there is not already a
-			// pending request.
-			if _, exists := b.requestedTxns[iv.Hash]; !exists {
-				b.requestedTxns[iv.Hash] = true
-				imsg.peer.requestedTxns[iv.Hash] = true
-				gdmsg.AddInvVect(iv)
-				numRequested++
-			}
-		}
-
-		if numRequested >= btcwire.MaxInvPerMsg {
-			break
-		}
-	}
-	if len(gdmsg.InvList) > 0 {
-		imsg.peer.QueueMessage(gdmsg, nil)
-	}
-}
-
-// blockHandler is the main handler for the block manager.  It must be run
-// as a goroutine.  It processes block and inv messages in a separate goroutine
-// from the peer handlers so the block (MsgBlock) messages are handled by a
-// single thread without needing to lock memory data structures.  This is
-// important because the block manager controls which blocks are needed and how
-// the fetching should proceed.
-func (b *blockManager) blockHandler() {
-	candidatePeers := list.New()
-out:
-	for {
-		select {
-		case m := <-b.msgChan:
-			switch msg := m.(type) {
-			case *newPeerMsg:
-				b.handleNewPeerMsg(candidatePeers, msg.peer)
-
-			case *txMsg:
-				b.handleTxMsg(msg)
-				msg.peer.txProcessed <- true
-
-			case *blockMsg:
-				b.handleBlockMsg(msg)
-				msg.peer.blockProcessed <- true
-
-			case *invMsg:
-				b.handleInvMsg(msg)
-
-			case *donePeerMsg:
-				b.handleDonePeerMsg(candidatePeers, msg.peer)
-
-			default:
-				// bitch and whine.
-			}
-
-		case <-b.quit:
-			break out
-		}
-	}
-	b.wg.Done()
-	log.Trace("BMGR: Block handler done")
-}
-
-// handleNotifyMsg handles notifications from btcchain.  It does things such
-// as request orphan block parents and relay accepted blocks to connected peers.
-func (b *blockManager) handleNotifyMsg(notification *btcchain.Notification) {
-	switch notification.Type {
-	// An orphan block has been accepted by the block chain.  Request
-	// its parents from the peer that sent it.
-	case btcchain.NTOrphanBlock:
-		orphanHash := notification.Data.(*btcwire.ShaHash)
-		if peer, exists := b.blockPeer[*orphanHash]; exists {
-			orphanRoot := b.blockChain.GetOrphanRoot(orphanHash)
-			locator, err := b.blockChain.LatestBlockLocator()
-			if err != nil {
-				log.Errorf("BMGR: Failed to get block locator "+
-					"for the latest block: %v", err)
-				break
-			}
-			peer.PushGetBlocksMsg(locator, orphanRoot)
-			delete(b.blockPeer, *orphanRoot)
-		} else {
-			log.Warnf("Notification for orphan %v with no peer",
-				orphanHash)
-		}
-
-	// A block has been accepted into the block chain.  Relay it to other
-	// peers.
-	case btcchain.NTBlockAccepted:
-		// Don't relay if we are not current. Other peers that are
-		// current should already know about it.
-
-		if !b.current() {
-			return
-		}
-
-		block, ok := notification.Data.(*btcutil.Block)
-		if !ok {
-			log.Warnf("BMGR: Chain accepted notification is not a block.")
-			break
-		}
-
-		// It's ok to ignore the error here since the notification is
-		// coming from the chain code which has already cached the hash.
-		hash, _ := block.Sha()
-
-		// Generate the inventory vector and relay it.
-		iv := btcwire.NewInvVect(btcwire.InvTypeBlock, hash)
-		b.server.RelayInventory(iv)
-
-	// A block has been connected to the main block chain.
-	case btcchain.NTBlockConnected:
-		block, ok := notification.Data.(*btcutil.Block)
-		if !ok {
-			log.Warnf("BMGR: Chain connected notification is not a block.")
-			break
-		}
-
-		// Remove all of the transactions (except the coinbase) in the
-		// connected block from the transaction pool.
-		for _, tx := range block.MsgBlock().Transactions[1:] {
-			b.server.txMemPool.removeTransaction(tx)
-		}
-
-		// Notify frontends
-		if r := b.server.rpcServer; r != nil {
-			go r.NotifyBlockConnected(block)
-			go r.NotifyNewTxListeners(b.server.db, block)
-		}
-
-	// A block has been disconnected from the main block chain.
-	case btcchain.NTBlockDisconnected:
-		block, ok := notification.Data.(*btcutil.Block)
-		if !ok {
-			log.Warnf("BMGR: Chain disconnected notification is not a block.")
-			break
-		}
-
-		// Reinsert all of the transactions (except the coinbase) into
-		// the transaction pool.
-		for _, tx := range block.MsgBlock().Transactions[1:] {
-			err := b.server.txMemPool.ProcessTransaction(tx)
-			if err != nil {
-				// Remove the transaction and all transactions
-				// that depend on it if it wasn't accepted into
-				// the transaction pool.
-				b.server.txMemPool.removeTransaction(tx)
-			}
-		}
-
-		// Notify frontends
-		if r := b.server.rpcServer; r != nil {
-			go r.NotifyBlockDisconnected(block)
-		}
-	}
-}
-
-// NewPeer informs the block manager of a newly active peer.
-func (b *blockManager) NewPeer(p *peer) {
-	// Ignore if we are shutting down.
-	if atomic.LoadInt32(&b.shutdown) != 0 {
-		return
-	}
-
-	b.msgChan <- &newPeerMsg{peer: p}
-}
-
-// QueueTx adds the passed transaction message and peer to the block handling
-// queue.
-func (b *blockManager) QueueTx(tx *btcwire.MsgTx, p *peer) {
-	// Don't accept more transactions if we're shutting down.
-	if atomic.LoadInt32(&b.shutdown) != 0 {
-		p.txProcessed <- false
-		return
-	}
-
-	b.msgChan <- &txMsg{tx: tx, peer: p}
-}
-
-// QueueBlock adds the passed block message and peer to the block handling queue.
-func (b *blockManager) QueueBlock(block *btcutil.Block, p *peer) {
-	// Don't accept more blocks if we're shutting down.
-	if atomic.LoadInt32(&b.shutdown) != 0 {
-		p.blockProcessed <- false
-		return
-	}
-
-	b.msgChan <- &blockMsg{block: block, peer: p}
-}
-
-// QueueInv adds the passed inv message and peer to the block handling queue.
-func (b *blockManager) QueueInv(inv *btcwire.MsgInv, p *peer) {
-	// No channel handling here because peers do not need to block on inv
-	// messages.
-	if atomic.LoadInt32(&b.shutdown) != 0 {
-		return
-	}
-
-	b.msgChan <- &invMsg{inv: inv, peer: p}
-}
-
-// DonePeer informs the blockmanager that a peer has disconnected.
-func (b *blockManager) DonePeer(p *peer) {
-	// Ignore if we are shutting down.
-	if atomic.LoadInt32(&b.shutdown) != 0 {
-		return
-	}
-
-	b.msgChan <- &donePeerMsg{peer: p}
-}
-
-// Start begins the core block handler which processes block and inv messages.
-func (b *blockManager) Start() {
-	// Already started?
-	if atomic.AddInt32(&b.started, 1) != 1 {
-		return
-	}
-
-	log.Trace("BMGR: Starting block manager")
-	b.wg.Add(1)
-	go b.blockHandler()
-}
-
-// Stop gracefully shuts down the block manager by stopping all asynchronous
-// handlers and waiting for them to finish.
-func (b *blockManager) Stop() error {
-	if atomic.AddInt32(&b.shutdown, 1) != 1 {
-		log.Warnf("BMGR: Block manager is already in the process of " +
-			"shutting down")
-		return nil
-	}
-
-	log.Infof("BMGR: Block manager shutting down")
-	close(b.quit)
-	b.wg.Wait()
-	return nil
-}
-
-// newBlockManager returns a new bitcoin block manager.
-// Use Start to begin processing asynchronous block and inv updates.
-func newBlockManager(s *server) (*blockManager, error) {
-	bm := blockManager{
-		server:           s,
-		blockPeer:        make(map[btcwire.ShaHash]*peer),
-		requestedTxns:    make(map[btcwire.ShaHash]bool),
-		requestedBlocks:  make(map[btcwire.ShaHash]bool),
-		lastBlockLogTime: time.Now(),
-		msgChan:          make(chan interface{}, cfg.MaxPeers*3),
-		quit:             make(chan bool),
-	}
-	bm.blockChain = btcchain.New(s.db, s.btcnet, bm.handleNotifyMsg)
-	bm.blockChain.DisableCheckpoints(cfg.DisableCheckpoints)
-	if cfg.DisableCheckpoints {
-		log.Info("BMGR: Checkpoints are disabled")
-	}
-
-	log.Infof("BMGR: Generating initial block node index.  This may " +
-		"take a while...")
-	err := bm.blockChain.GenerateInitialIndex()
-	if err != nil {
-		return nil, err
-	}
-	log.Infof("BMGR: Block index generation complete")
-
-	return &bm, nil
-}
-
-// removeRegressionDB removes the existing regression test database if running
-// in regression test mode and it already exists.
-func removeRegressionDB(dbPath string) error {
-	// Dont do anything if not in regression test mode.
-	if !cfg.RegressionTest {
-		return nil
-	}
-
-	// Remove the old regression test database if it already exists.
-	fi, err := os.Stat(dbPath)
-	if err == nil {
-		log.Infof("BMGR: Removing regression test database from '%s'", dbPath)
-		if fi.IsDir() {
-			err := os.RemoveAll(dbPath)
-			if err != nil {
-				return err
-			}
-		} else {
-			err := os.Remove(dbPath)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// dbPath returns the path to the block database given a database type.
-func blockDbPath(dbType string) string {
-	// The database name is based on the database type.
-	dbName := blockDbNamePrefix + "_" + dbType
-	if dbType == "sqlite" {
-		dbName = dbName + ".db"
-	}
-	dbPath := filepath.Join(cfg.DataDir, dbName)
-	return dbPath
-}
-
-// warnMultipeDBs shows a warning if multiple block database types are detected.
-// This is not a situation most users want.  It is handy for development however
-// to support multiple side-by-side databases.
-func warnMultipeDBs() {
-	// This is intentionally not using the known db types which depend
-	// on the database types compiled into the binary since we want to
-	// detect legacy db types as well.
-	dbTypes := []string{"leveldb", "sqlite"}
-	duplicateDbPaths := make([]string, 0, len(dbTypes)-1)
-	for _, dbType := range dbTypes {
-		if dbType == cfg.DbType {
-			continue
-		}
-
-		// Store db path as a duplicate db if it exists.
-		dbPath := blockDbPath(dbType)
-		if fileExists(dbPath) {
-			duplicateDbPaths = append(duplicateDbPaths, dbPath)
-		}
-	}
-
-	// Warn if there are extra databases.
-	if len(duplicateDbPaths) > 0 {
-		selectedDbPath := blockDbPath(cfg.DbType)
-		log.Warnf("WARNING: There are multiple block chain databases "+
-			"using different database types.\nYou probably don't "+
-			"want to waste disk space by having more than one.\n"+
-			"Your current database is located at [%v].\nThe "+
-			"additional database is located at %v", selectedDbPath,
-			duplicateDbPaths)
-	}
-}
-
-// loadBlockDB opens the block database and returns a handle to it.
-func loadBlockDB() (btcdb.Db, error) {
-	warnMultipeDBs()
-
-	// The database name is based on the database type.
-	dbPath := blockDbPath(cfg.DbType)
-
-	// The regression test is special in that it needs a clean database for
-	// each run, so remove it now if it already exists.
-	removeRegressionDB(dbPath)
-
-	log.Infof("BMGR: Loading block database from '%s'", dbPath)
-	db, err := btcdb.OpenDB(cfg.DbType, dbPath)
-	if err != nil {
-		// Return the error if it's not because the database doesn't
-		// exist.
-		if err != btcdb.DbDoesNotExist {
-			return nil, err
-		}
-
-		// Create the db if it does not exist.
-		err = os.MkdirAll(cfg.DataDir, 0700)
-		if err != nil {
-			return nil, err
-		}
-		db, err = btcdb.CreateDB(cfg.DbType, dbPath)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Get the latest block height from the database.
-	_, height, err := db.NewestSha()
-	if err != nil {
-		db.Close()
-		return nil, err
-	}
-
-	// Insert the appropriate genesis block for the bitcoin network being
-	// connected to if needed.
-	if height == -1 {
-		genesis := btcutil.NewBlock(activeNetParams.genesisBlock)
-		_, err := db.InsertBlock(genesis)
-		if err != nil {
-			db.Close()
-			return nil, err
-		}
-		log.Infof("BMGR: Inserted genesis block %v",
-			activeNetParams.genesisHash)
-		height = 0
-	}
-
-	log.Infof("BMGR: Block database loaded with block height %d", height)
-	return db, nil
-}

btcd.go

@@ -1,100 +0,0 @@
-// Copyright (c) 2013 Conformal Systems LLC.
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"net"
-	"net/http"
-	_ "net/http/pprof"
-	"os"
-	"runtime"
-)
-
-var (
-	cfg *config
-)
-
-// btcdMain is the real main function for btcd.  It is necessary to work around
-// the fact that deferred functions do not run when os.Exit() is called.
-func btcdMain() error {
-	// Initialize logging and setup deferred flushing to ensure all
-	// outstanding messages are written on shutdown.
-	loggers := setLogLevel(defaultLogLevel)
-	defer func() {
-		for _, logger := range loggers {
-			logger.Flush()
-		}
-	}()
-
-	// Load configuration and parse command line.
-	tcfg, _, err := loadConfig()
-	if err != nil {
-		return err
-	}
-	cfg = tcfg
-
-	// Change the logging level if needed.
-	if cfg.DebugLevel != defaultLogLevel {
-		loggers = setLogLevel(cfg.DebugLevel)
-	}
-
-	// Show version at startup.
-	log.Infof("Version %s", version())
-
-	// See if we want to enable profiling.
-	if cfg.Profile != "" {
-		go func() {
-			listenAddr := net.JoinHostPort("", cfg.Profile)
-			log.Infof("Profile server listening on %s", listenAddr)
-			log.Errorf("%v", http.ListenAndServe(listenAddr, nil))
-		}()
-	}
-
-	// Perform upgrades to btcd as new versions require it.
-	if err := doUpgrades(); err != nil {
-		log.Errorf("%v", err)
-		return err
-	}
-
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		log.Errorf("%v", err)
-		return err
-	}
-	defer db.Close()
-
-	// Ensure the database is sync'd and closed on Ctrl+C.
-	addInterruptHandler(func() {
-		db.RollbackClose()
-	})
-
-	// Create server and start it.
-	listenAddr := net.JoinHostPort("", cfg.Port)
-	server, err := newServer(listenAddr, db, activeNetParams.btcnet)
-	if err != nil {
-		log.Errorf("Unable to start server on %v: %v", listenAddr, err)
-		return err
-	}
-	server.Start()
-
-	server.WaitForShutdown()
-	return nil
-}
-
-func main() {
-	// Use all processor cores.
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	// Up some limits.
-	if err := setLimits(); err != nil {
-		os.Exit(1)
-	}
-
-	// Work around defer not working after os.Exit()
-	if err := btcdMain(); err != nil {
-		os.Exit(1)
-	}
-}

cmd/addblock/addblock.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2013-2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/kaspanet/kaspad/limits"
+	"github.com/kaspanet/kaspad/logs"
+	"github.com/kaspanet/kaspad/util/panics"
+)
+
+const (
+	// blockDBNamePrefix is the prefix for the kaspad block database.
+	blockDBNamePrefix = "blocks"
+)
+
+var (
+	cfg   *ConfigFlags
+	log   *logs.Logger
+	spawn func(string, func())
+)
+
+// realMain is the real main function for the utility. It is necessary to work
+// around the fact that deferred functions do not run when os.Exit() is called.
+func realMain() error {
+	// Load configuration and parse command line.
+	tcfg, _, err := loadConfig()
+	if err != nil {
+		return err
+	}
+	cfg = tcfg
+
+	// Setup logging.
+	backendLogger := logs.NewBackend()
+	defer os.Stdout.Sync()
+	log = backendLogger.Logger("MAIN")
+	spawn = panics.GoroutineWrapperFunc(log)
+
+	fi, err := os.Open(cfg.InFile)
+	if err != nil {
+		log.Errorf("Failed to open file %s: %s", cfg.InFile, err)
+		return err
+	}
+	defer fi.Close()
+
+	// Create a block importer for the database and input file and start it.
+	// The done channel returned from start will contain an error if
+	// anything went wrong.
+	importer, err := newBlockImporter(fi)
+	if err != nil {
+		log.Errorf("Failed create block importer: %s", err)
+		return err
+	}
+
+	// Perform the import asynchronously. This allows blocks to be
+	// processed and read in parallel. The results channel returned from
+	// Import contains the statistics about the import including an error
+	// if something went wrong.
+	log.Info("Starting import")
+	resultsChan := importer.Import()
+	results := <-resultsChan
+	if results.err != nil {
+		log.Errorf("%s", results.err)
+		return results.err
+	}
+
+	log.Infof("Processed a total of %d blocks (%d imported, %d already "+
+		"known)", results.blocksProcessed, results.blocksImported,
+		results.blocksProcessed-results.blocksImported)
+	return nil
+}
+
+func main() {
+	// Use all processor cores and up some limits.
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	if err := limits.SetLimits(); err != nil {
+		os.Exit(1)
+	}
+
+	// Work around defer not working after os.Exit()
+	if err := realMain(); err != nil {
+		os.Exit(1)
+	}
+}