[NOD-339] Remove cfilters and cfindex (#527)

* [NOD-339] Remove cfilters and cfindex

* [NOD-339] Remove some leftovers

.gitignore

@@ -38,6 +38,7 @@ _testmain.go
 .vscode
 debug
 debug.test
+__debug_bin
 
 # CI
 version.txt

CHANGES

@@ -513,7 +513,7 @@ Changes in 0.8.0-beta (Sun May 25 2014)
     - Reduce max bytes allowed for a standard nulldata transaction to 40 for
       compatibility with the reference client
     - Introduce a new btcnet package which houses all of the network params
-      for each network (mainnet, testnet3, regtest) to ultimately enable
+      for each network (mainnet, testnet, regtest) to ultimately enable
       easier addition and tweaking of networks without needing to change
       several packages
     - Fix several script discrepancies found by reference client test data
@@ -530,7 +530,7 @@ Changes in 0.8.0-beta (Sun May 25 2014)
     - Provide options to control block template creation settings
     - Support the getwork RPC
     - Allow address identifiers to apply to more than one network since both
-      testnet3 and the regression test network unfortunately use the same
+      testnet and the regression test network unfortunately use the same
       identifier
   - RPC changes:
     - Set the content type for HTTP POST RPC connections to application/json

Gopkg.lock

@@ -1,105 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "bou.ke/monkey"
-  packages = ["."]
-  revision = "bdf6dea004c6fd1cdf4b25da8ad45a606c09409a"
-  version = "v1.0.1"
-
-[[projects]]
-  name = "github.com/aead/siphash"
-  packages = ["."]
-  revision = "83563a290f60225eb120d724600b9690c3fb536f"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/btcsuite/btclog"
-  packages = ["."]
-  revision = "84c8d2346e9fc8c7b947e243b9c24e6df9fd206a"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/btcsuite/go-socks"
-  packages = ["socks"]
-  revision = "4720035b7bfd2a9bb130b1c184f8bbe41b6f0d0f"
-
-[[projects]]
-  name = "github.com/btcsuite/goleveldb"
-  packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"]
-  revision = "3fd0373267b6461dbefe91cef614278064d05465"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/btcsuite/snappy-go"
-  packages = ["."]
-  revision = "b3db38edf0a9a11a115eb6b022d8c946024a9ac0"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/btcsuite/websocket"
-  packages = ["."]
-  revision = "31079b6807923eb23992c421b114992b95131b55"
-
-[[projects]]
-  name = "github.com/btcsuite/winsvc"
-  packages = ["eventlog","mgr","registry","svc","winapi"]
-  revision = "f8fb11f83f7e860e3769a08e6811d1b399a43722"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
-  version = "v1.1.1"
-
-[[projects]]
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  revision = "c6ca198ec95c841fdb89fc0de7496fed11ab854e"
-  version = "v1.4.0"
-
-[[projects]]
-  name = "github.com/jrick/logrotate"
-  packages = ["rotator"]
-  revision = "a93b200c26cbae3bb09dd0dc2c7c7fe1468a034a"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kkdai/bstream"
-  packages = ["."]
-  revision = "b3251f7901ec4dd4ec66b3210e8f4bd5c0f1c5a3"
-
-[[projects]]
-  name = "github.com/miekg/dns"
-  packages = ["."]
-  revision = "cc8cd02140663157ce797c6650488d6c8563f31f"
-  version = "v1.1.6"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = ["ed25519","ed25519/internal/edwards25519","ripemd160"]
-  revision = "c2843e01d9a2bc60bb26ad24e09734fdc2d9ec58"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = ["bpf","internal/iana","internal/socket","ipv4","ipv6"]
-  revision = "d8887717615a059821345a5c23649351b52a1c0b"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "fead79001313d15903fb4605b4a1b781532cd93e"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "00392a00928f96fc94e2c8c65ce3a98cc6f5e2f93dda64d3c4502f2f38026e96"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,78 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[constraint]]
-  name = "bou.ke/monkey"
-  version = "1.0.1"
-
-[[constraint]]
-  name = "github.com/aead/siphash"
-  version = "1.0.1"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/btcsuite/btclog"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/btcsuite/go-socks"
-
-[[constraint]]
-  name = "github.com/btcsuite/goleveldb"
-  version = "1.0.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/btcsuite/websocket"
-
-[[constraint]]
-  name = "github.com/btcsuite/winsvc"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/davecgh/go-spew"
-  version = "1.1.1"
-
-[[constraint]]
-  name = "github.com/jessevdk/go-flags"
-  version = "1.4.0"
-
-[[constraint]]
-  name = "github.com/jrick/logrotate"
-  version = "1.0.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/kkdai/bstream"
-
-[[constraint]]
-  name = "github.com/miekg/dns"
-  version = "1.1.6"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-
-[prune]
-  go-tests = true
-  unused-packages = true

README.md

@@ -3,7 +3,7 @@ btcd
 
 [![Build Status](https://travis-ci.org/btcsuite/btcd.png?branch=master)](https://travis-ci.org/btcsuite/btcd)
 [![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/daglabs/btcd)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad)
 
 btcd is an alternative full node bitcoin implementation written in Go (golang).
 
@@ -41,7 +41,7 @@ which are both under active development.
 
 #### Windows - MSI Available
 
-https://github.com/daglabs/btcd/releases
+https://github.com/kaspanet/kaspad/releases
 
 #### Linux/BSD/MacOSX/POSIX - Build from Source
 
@@ -64,8 +64,8 @@ recommended that `GOPATH` is set to a directory in your home directory such as
 
 ```bash
 $ # Install dep: https://golang.github.io/dep/docs/installation.html
-$ git clone https://github.com/daglabs/btcd $GOPATH/src/github.com/daglabs/btcd
-$ cd $GOPATH/src/github.com/daglabs/btcd
+$ git clone https://github.com/kaspanet/kaspad $GOPATH/src/github.com/kaspanet/kaspad
+$ cd $GOPATH/src/github.com/kaspanet/kaspad
 $ dep ensure
 $ go install . ./cmd/...
 ```
@@ -85,7 +85,7 @@ Install a newer MSI
 - Run the following commands to update btcd, all dependencies, and install it:
 
 ```bash
-$ cd $GOPATH/src/github.com/daglabs/btcd
+$ cd $GOPATH/src/github.com/kaspanet/kaspad
 $ git pull && dep ensure
 $ go install . ./cmd/...
 ```
@@ -114,12 +114,12 @@ $ ./btcd
 
 ## Issue Tracker
 
-The [integrated github issue tracker](https://github.com/daglabs/btcd/issues)
+The [integrated github issue tracker](https://github.com/kaspanet/kaspad/issues)
 is used for this project.
 
 ## Documentation
 
-The documentation is a work-in-progress.  It is located in the [docs](https://github.com/daglabs/btcd/tree/master/docs) folder.
+The documentation is a work-in-progress.  It is located in the [docs](https://github.com/kaspanet/kaspad/tree/master/docs) folder.
 
 ## GPG Verification Key
 

addrmgr/addrmanager.go

@@ -10,7 +10,7 @@ import (
 	"encoding/base32"
 	"encoding/binary"
 	"encoding/json"
-	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"math/rand"
 	"net"
@@ -22,10 +22,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/daglabs/btcd/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 type newBucket [newBucketCount]map[string]*KnownAddress
@@ -44,6 +44,7 @@ type AddrManager struct {
 	addrNewFullNodes   newBucket
 	addrTried          map[subnetworkid.SubnetworkID]*triedBucket
 	addrTriedFullNodes triedBucket
+	addrTrying         map[*KnownAddress]bool
 	started            int32
 	shutdown           int32
 	wg                 sync.WaitGroup
@@ -160,6 +161,10 @@ const (
 	// will consider evicting an address.
 	minBadDays = 7
 
+	// getAddrMin is the least addresses that we will send in response
+	// to a getAddr. If we have less than this amount, we send everything.
+	getAddrMin = 50
+
 	// getAddrMax is the most addresses that we will send in response
 	// to a getAddr (in practise the most addresses we will return from a
 	// call to AddressCache()).
@@ -560,7 +565,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 	}
 	r, err := os.Open(filePath)
 	if err != nil {
-		return fmt.Errorf("%s error opening file: %s", filePath, err)
+		return errors.Errorf("%s error opening file: %s", filePath, err)
 	}
 	defer r.Close()
 
@@ -568,11 +573,11 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 	dec := json.NewDecoder(r)
 	err = dec.Decode(&sam)
 	if err != nil {
-		return fmt.Errorf("error reading %s: %s", filePath, err)
+		return errors.Errorf("error reading %s: %s", filePath, err)
 	}
 
 	if sam.Version != serialisationVersion {
-		return fmt.Errorf("unknown version %d in serialized "+
+		return errors.Errorf("unknown version %d in serialized "+
 			"addrmanager", sam.Version)
 	}
 	copy(a.key[:], sam.Key[:])
@@ -581,18 +586,18 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		ka := new(KnownAddress)
 		ka.na, err = a.DeserializeNetAddress(v.Addr)
 		if err != nil {
-			return fmt.Errorf("failed to deserialize netaddress "+
+			return errors.Errorf("failed to deserialize netaddress "+
 				"%s: %s", v.Addr, err)
 		}
 		ka.srcAddr, err = a.DeserializeNetAddress(v.Src)
 		if err != nil {
-			return fmt.Errorf("failed to deserialize netaddress "+
+			return errors.Errorf("failed to deserialize netaddress "+
 				"%s: %s", v.Src, err)
 		}
 		if v.SubnetworkID != "" {
 			ka.subnetworkID, err = subnetworkid.NewFromStr(v.SubnetworkID)
 			if err != nil {
-				return fmt.Errorf("failed to deserialize subnetwork id "+
+				return errors.Errorf("failed to deserialize subnetwork id "+
 					"%s: %s", v.SubnetworkID, err)
 			}
 		}
@@ -611,7 +616,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 			for _, val := range subnetworkNewBucket {
 				ka, ok := a.addrIndex[val]
 				if !ok {
-					return fmt.Errorf("newbucket contains %s but "+
+					return errors.Errorf("newbucket contains %s but "+
 						"none in address list", val)
 				}
 
@@ -628,7 +633,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		for _, val := range newBucket {
 			ka, ok := a.addrIndex[val]
 			if !ok {
-				return fmt.Errorf("full nodes newbucket contains %s but "+
+				return errors.Errorf("full nodes newbucket contains %s but "+
 					"none in address list", val)
 			}
 
@@ -649,7 +654,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 			for _, val := range subnetworkTriedBucket {
 				ka, ok := a.addrIndex[val]
 				if !ok {
-					return fmt.Errorf("Tried bucket contains %s but "+
+					return errors.Errorf("Tried bucket contains %s but "+
 						"none in address list", val)
 				}
 
@@ -664,7 +669,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		for _, val := range triedBucket {
 			ka, ok := a.addrIndex[val]
 			if !ok {
-				return fmt.Errorf("Full nodes tried bucket contains %s but "+
+				return errors.Errorf("Full nodes tried bucket contains %s but "+
 					"none in address list", val)
 			}
 
@@ -677,12 +682,12 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 	// Sanity checking.
 	for k, v := range a.addrIndex {
 		if v.refs == 0 && !v.tried {
-			return fmt.Errorf("address %s after serialisation "+
+			return errors.Errorf("address %s after serialisation "+
 				"with no references", k)
 		}
 
 		if v.refs > 0 && v.tried {
-			return fmt.Errorf("address %s after serialisation "+
+			return errors.Errorf("address %s after serialisation "+
 				"which is both new and tried!", k)
 		}
 	}
@@ -719,7 +724,11 @@ func (a *AddrManager) Start() {
 
 	// Start the address ticker to save addresses periodically.
 	a.wg.Add(1)
-	go a.addressHandler()
+	spawn(a.addressHandler, a.handlePanic)
+}
+
+func (a *AddrManager) handlePanic() {
+	atomic.AddInt32(&a.shutdown, 1)
 }
 
 // Stop gracefully shuts down the address manager by stopping the main handler.
@@ -769,11 +778,11 @@ func (a *AddrManager) AddAddressByIP(addrIP string, subnetworkID *subnetworkid.S
 	// Put it in wire.Netaddress
 	ip := net.ParseIP(addr)
 	if ip == nil {
-		return fmt.Errorf("invalid ip address %s", addr)
+		return errors.Errorf("invalid ip address %s", addr)
 	}
 	port, err := strconv.ParseUint(portStr, 10, 0)
 	if err != nil {
-		return fmt.Errorf("invalid port %s: %s", portStr, err)
+		return errors.Errorf("invalid port %s: %s", portStr, err)
 	}
 	na := wire.NewNetAddressIPPort(ip, uint16(port), 0)
 	a.AddAddress(na, na, subnetworkID) // XXX use correct src address
@@ -844,6 +853,12 @@ func (a *AddrManager) AddressCache(includeAllSubnetworks bool, subnetworkID *sub
 	if numAddresses > getAddrMax {
 		numAddresses = getAddrMax
 	}
+	if len(allAddr) < getAddrMin {
+		numAddresses = len(allAddr)
+	}
+	if len(allAddr) > getAddrMin && numAddresses < getAddrMin {
+		numAddresses = getAddrMin
+	}
 
 	// Fisher-Yates shuffle the array. We only need to do the first
 	// `numAddresses' since we are throwing the rest.
@@ -879,6 +894,8 @@ func (a *AddrManager) reset() {
 	}
 	a.nNewFullNodes = 0
 	a.nTriedFullNodes = 0
+
+	a.addrTrying = make(map[*KnownAddress]bool)
 }
 
 // HostToNetAddress returns a netaddress given a host address.  If the address
@@ -904,7 +921,7 @@ func (a *AddrManager) HostToNetAddress(host string, port uint16, services wire.S
 			return nil, err
 		}
 		if len(ips) == 0 {
-			return nil, fmt.Errorf("no addresses found for %s", host)
+			return nil, errors.Errorf("no addresses found for %s", host)
 		}
 		ip = ips[0]
 	}
@@ -942,15 +959,25 @@ func (a *AddrManager) GetAddress() *KnownAddress {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 
+	var knownAddress *KnownAddress
 	if a.localSubnetworkID == nil {
-		return a.getAddress(&a.addrTriedFullNodes, a.nTriedFullNodes,
+		knownAddress = a.getAddress(&a.addrTriedFullNodes, a.nTriedFullNodes,
 			&a.addrNewFullNodes, a.nNewFullNodes)
+	} else {
+		subnetworkID := *a.localSubnetworkID
+		knownAddress = a.getAddress(a.addrTried[subnetworkID], a.nTried[subnetworkID],
+			a.addrNew[subnetworkID], a.nNew[subnetworkID])
 	}
 
-	subnetworkID := *a.localSubnetworkID
+	if knownAddress != nil {
+		if a.addrTrying[knownAddress] {
+			return nil
+		}
+		a.addrTrying[knownAddress] = true
+	}
+
+	return knownAddress
 
-	return a.getAddress(a.addrTried[subnetworkID], a.nTried[subnetworkID],
-		a.addrNew[subnetworkID], a.nNew[subnetworkID])
 }
 
 // see GetAddress for details
@@ -1033,6 +1060,8 @@ func (a *AddrManager) Attempt(addr *wire.NetAddress) {
 	// set last tried time to now
 	ka.attempts++
 	ka.lastattempt = time.Now()
+
+	delete(a.addrTrying, ka)
 }
 
 // Connected Marks the given address as currently connected and working at the
@@ -1224,7 +1253,7 @@ func (a *AddrManager) Good(addr *wire.NetAddress, subnetworkID *subnetworkid.Sub
 // with the given priority.
 func (a *AddrManager) AddLocalAddress(na *wire.NetAddress, priority AddressPriority) error {
 	if !IsRoutable(na) {
-		return fmt.Errorf("address %s is not routable", na.IP)
+		return errors.Errorf("address %s is not routable", na.IP)
 	}
 
 	a.lamtx.Lock()

addrmgr/addrmanager_test.go

@@ -5,16 +5,19 @@
 package addrmgr
 
 import (
-	"errors"
+	"bou.ke/monkey"
 	"fmt"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/pkg/errors"
 	"net"
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/daglabs/btcd/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // naTest is used to describe a test to be performed against the NetAddressKey
@@ -113,7 +116,17 @@ func TestStartStop(t *testing.T) {
 }
 
 func TestAddAddressByIP(t *testing.T) {
-	fmtErr := fmt.Errorf("")
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
+	fmtErr := errors.Errorf("")
 	addrErr := &net.AddrError{}
 	var tests = []struct {
 		addrIP string
@@ -157,6 +170,16 @@ func TestAddAddressByIP(t *testing.T) {
 }
 
 func TestAddLocalAddress(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	var tests = []struct {
 		address  wire.NetAddress
 		priority AddressPriority
@@ -210,6 +233,16 @@ func TestAddLocalAddress(t *testing.T) {
 }
 
 func TestAttempt(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	n := New("testattempt", lookupFunc, nil)
 
 	// Add a new address and get it
@@ -232,6 +265,16 @@ func TestAttempt(t *testing.T) {
 }
 
 func TestConnected(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	n := New("testconnected", lookupFunc, nil)
 
 	// Add a new address and get it
@@ -252,6 +295,16 @@ func TestConnected(t *testing.T) {
 }
 
 func TestNeedMoreAddresses(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	n := New("testneedmoreaddresses", lookupFunc, nil)
 	addrsToAdd := 1500
 	b := n.NeedMoreAddresses()
@@ -284,6 +337,16 @@ func TestNeedMoreAddresses(t *testing.T) {
 }
 
 func TestGood(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	n := New("testgood", lookupFunc, nil)
 	addrsToAdd := 64 * 64
 	addrs := make([]*wire.NetAddress, addrsToAdd)
@@ -331,6 +394,16 @@ func TestGood(t *testing.T) {
 }
 
 func TestGoodChangeSubnetworkID(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	n := New("test_good_change_subnetwork_id", lookupFunc, nil)
 	addr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 	addrKey := NetAddressKey(addr)
@@ -400,6 +473,16 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 }
 
 func TestGetAddress(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	localSubnetworkID := &subnetworkid.SubnetworkID{0xff}
 	n := New("testgetaddress", lookupFunc, localSubnetworkID)
 
@@ -417,6 +500,7 @@ func TestGetAddress(t *testing.T) {
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
+	n.Attempt(ka.NetAddress())
 
 	// Checks that we don't get it if we find that it has other subnetwork ID than expected.
 	actualSubnetworkID := &subnetworkid.SubnetworkID{0xfe}
@@ -449,6 +533,7 @@ func TestGetAddress(t *testing.T) {
 	if !ka.SubnetworkID().IsEqual(localSubnetworkID) {
 		t.Errorf("Wrong Subnetwork ID: got %v, want %v", *ka.SubnetworkID(), localSubnetworkID)
 	}
+	n.Attempt(ka.NetAddress())
 
 	// Mark this as a good address and get it
 	n.Good(ka.NetAddress(), localSubnetworkID)
@@ -470,6 +555,16 @@ func TestGetAddress(t *testing.T) {
 }
 
 func TestGetBestLocalAddress(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	localAddrs := []wire.NetAddress{
 		{IP: net.ParseIP("192.168.0.100")},
 		{IP: net.ParseIP("::1")},

addrmgr/internal_test.go

@@ -7,7 +7,7 @@ package addrmgr
 import (
 	"time"
 
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 func TstKnownAddressIsBad(ka *KnownAddress) bool {

addrmgr/knownaddress.go

@@ -7,9 +7,9 @@ package addrmgr
 import (
 	"time"
 
-	"github.com/daglabs/btcd/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // KnownAddress tracks information about a known network address that is used

addrmgr/knownaddress_test.go

@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/daglabs/btcd/addrmgr"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/addrmgr"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 func TestChance(t *testing.T) {

addrmgr/log.go

@@ -5,15 +5,9 @@
 package addrmgr
 
 import (
-	"github.com/btcsuite/btclog"
-	"github.com/daglabs/btcd/logger"
+	"github.com/kaspanet/kaspad/logger"
+	"github.com/kaspanet/kaspad/util/panics"
 )
 
-// log is a logger that is initialized with no output filters.  This
-// means the package will not perform any logging by default until the caller
-// requests it.
-var log btclog.Logger
-
-func init() {
-	log, _ = logger.Get(logger.SubsystemTags.ADXR)
-}
+var log, _ = logger.Get(logger.SubsystemTags.ADXR)
+var spawn = panics.GoroutineWrapperFuncWithPanicHandler(log)

addrmgr/network.go

@@ -6,10 +6,11 @@ package addrmgr
 
 import (
 	"fmt"
-	"github.com/daglabs/btcd/config"
 	"net"
 
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/config"
+
+	"github.com/kaspanet/kaspad/wire"
 )
 
 var (
@@ -224,8 +225,8 @@ func IsValid(na *wire.NetAddress) bool {
 // the public internet.  This is true as long as the address is valid and is not
 // in any reserved ranges.
 func IsRoutable(na *wire.NetAddress) bool {
-	if config.ActiveNetParams().AcceptUnroutable {
-		return true
+	if config.ActiveConfig().NetParams().AcceptUnroutable {
+		return !IsLocal(na)
 	}
 
 	return IsValid(na) && !(IsRFC1918(na) || IsRFC2544(na) ||

addrmgr/network_test.go

@@ -5,16 +5,29 @@
 package addrmgr_test
 
 import (
+	"bou.ke/monkey"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/dagconfig"
 	"net"
 	"testing"
 
-	"github.com/daglabs/btcd/addrmgr"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/addrmgr"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // TestIPTypes ensures the various functions which determine the type of an IP
 // address based on RFCs work as intended.
 func TestIPTypes(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	type ipTest struct {
 		in       wire.NetAddress
 		rfc1918  bool
@@ -145,6 +158,16 @@ func TestIPTypes(t *testing.T) {
 // TestGroupKey tests the GroupKey function to ensure it properly groups various
 // IP addresses.
 func TestGroupKey(t *testing.T) {
+	activeConfigPatch := monkey.Patch(config.ActiveConfig, func() *config.Config {
+		return &config.Config{
+			Flags: &config.Flags{
+				NetworkFlags: config.NetworkFlags{
+					ActiveNetParams: &dagconfig.SimNetParams},
+			},
+		}
+	})
+	defer activeConfigPatch.Unpatch()
+
 	tests := []struct {
 		name     string
 		ip       string

blockdag/README.md

@@ -3,7 +3,7 @@ blockchain
 
 [![Build Status](http://img.shields.io/travis/btcsuite/btcd.svg)](https://travis-ci.org/btcsuite/btcd)
 [![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/daglabs/btcd/blockchain)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/blockchain)
 
 Package blockchain implements bitcoin block handling and chain selection rules.
 The test coverage is currently only around 60%, but will be increasing over
@@ -21,7 +21,7 @@ block chain.
 ## Installation and Updating
 
 ```bash
-$ go get -u github.com/daglabs/btcd/blockchain
+$ go get -u github.com/kaspanet/kaspad/blockchain
 ```
 
 ## Bitcoin Chain Processing Overview
@@ -61,18 +61,18 @@ is by no means exhaustive:
 
 ## Examples
 
-* [ProcessBlock Example](http://godoc.org/github.com/daglabs/btcd/blockchain#example-BlockChain-ProcessBlock)  
+* [ProcessBlock Example](http://godoc.org/github.com/kaspanet/kaspad/blockchain#example-BlockChain-ProcessBlock)  
   Demonstrates how to create a new chain instance and use ProcessBlock to
   attempt to add a block to the chain.  This example intentionally
   attempts to insert a duplicate genesis block to illustrate how an invalid
   block is handled.
 
-* [CompactToBig Example](http://godoc.org/github.com/daglabs/btcd/blockchain#example-CompactToBig)  
+* [CompactToBig Example](http://godoc.org/github.com/kaspanet/kaspad/blockchain#example-CompactToBig)  
   Demonstrates how to convert the compact "bits" in a block header which
   represent the target difficulty to a big integer and display it using the
   typical hex notation.
 
-* [BigToCompact Example](http://godoc.org/github.com/daglabs/btcd/blockchain#example-BigToCompact)  
+* [BigToCompact Example](http://godoc.org/github.com/kaspanet/kaspad/blockchain#example-BigToCompact)  
   Demonstrates how to convert a target difficulty into the
   compact "bits" in a block header which represent that target difficulty.
 

blockdag/accept.go

@@ -6,11 +6,18 @@ package blockdag
 
 import (
 	"fmt"
-
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
 )
 
+func (dag *BlockDAG) addNodeToIndexWithInvalidAncestor(block *util.Block) error {
+	blockHeader := &block.MsgBlock().Header
+	newNode := newBlockNode(blockHeader, newSet(), dag.dagParams.K)
+	newNode.status = statusInvalidAncestor
+	dag.index.AddNode(newNode)
+	return dag.index.flushToDB()
+}
+
 // maybeAcceptBlock potentially accepts a block into the block DAG. It
 // performs several validation checks which depend on its position within
 // the block DAG before adding it. The block is expected to have already
@@ -21,27 +28,29 @@ import (
 //
 // This function MUST be called with the dagLock held (for writes).
 func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) error {
-	// The height of this block is one more than the referenced previous
-	// block.
 	parents, err := lookupParentNodes(block, dag)
 	if err != nil {
+		if rErr, ok := err.(RuleError); ok && rErr.ErrorCode == ErrInvalidAncestorBlock {
+			err := dag.addNodeToIndexWithInvalidAncestor(block)
+			if err != nil {
+				return err
+			}
+		}
 		return err
 	}
 
-	bluestParent := parents.bluest()
-	blockHeight := int32(0)
-	if !block.IsGenesis() {
-		blockHeight = parents.maxHeight() + 1
-	}
-	block.SetHeight(blockHeight)
-
 	// The block must pass all of the validation rules which depend on the
 	// position of the block within the block DAG.
-	err = dag.checkBlockContext(block, parents, bluestParent, flags)
+	err = dag.checkBlockContext(block, parents, flags)
 	if err != nil {
 		return err
 	}
 
+	// Create a new block node for the block and add it to the node index.
+	newNode := newBlockNode(&block.MsgBlock().Header, parents, dag.dagParams.K)
+	newNode.status = statusDataStored
+	dag.index.AddNode(newNode)
+
 	// Insert the block into the database if it's not already there.  Even
 	// though it is possible the block will ultimately fail to connect, it
 	// has already passed all proof-of-work and validity tests which means
@@ -52,26 +61,30 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	// such as making blocks that never become part of the DAG or
 	// blocks that fail to connect available for further analysis.
 	err = dag.db.Update(func(dbTx database.Tx) error {
-		return dbStoreBlock(dbTx, block)
+		err := dbStoreBlock(dbTx, block)
+		if err != nil {
+			return err
+		}
+		return dag.index.flushToDBWithTx(dbTx)
 	})
 	if err != nil {
 		return err
 	}
 
-	// Create a new block node for the block and add it to the node index.
-	blockHeader := &block.MsgBlock().Header
-	newNode := newBlockNode(blockHeader, parents, dag.dagParams.K)
-	newNode.status = statusDataStored
-
-	dag.index.AddNode(newNode)
-	err = dag.index.flushToDB()
-	if err != nil {
-		return err
+	// Make sure that all the block's transactions are finalized
+	fastAdd := flags&BFFastAdd == BFFastAdd
+	bluestParent := parents.bluest()
+	if !fastAdd {
+		if err := dag.validateAllTxsFinalized(block, newNode, bluestParent); err != nil {
+			return err
+		}
 	}
 
+	block.SetChainHeight(newNode.chainHeight)
+
 	// Connect the passed block to the DAG. This also handles validation of the
 	// transaction scripts.
-	err = dag.addBlock(newNode, parents, block, flags)
+	chainUpdates, err := dag.addBlock(newNode, parents, block, flags)
 	if err != nil {
 		return err
 	}
@@ -80,7 +93,16 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	// DAG.  The caller would typically want to react by relaying the
 	// inventory to other peers.
 	dag.dagLock.Unlock()
-	dag.sendNotification(NTBlockAdded, block)
+	dag.sendNotification(NTBlockAdded, &BlockAddedNotificationData{
+		Block:         block,
+		WasUnorphaned: flags&BFWasUnorphaned != 0,
+	})
+	if len(chainUpdates.addedChainBlockHashes) > 0 {
+		dag.sendNotification(NTChainChanged, &ChainChangedNotificationData{
+			RemovedChainBlockHashes: chainUpdates.removedChainBlockHashes,
+			AddedChainBlockHashes:   chainUpdates.addedChainBlockHashes,
+		})
+	}
 	dag.dagLock.Lock()
 
 	return nil

blockdag/accept_test.go

@@ -1,14 +1,15 @@
 package blockdag
 
 import (
-	"errors"
+	"github.com/pkg/errors"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"bou.ke/monkey"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
 )
 
 func TestMaybeAcceptBlockErrors(t *testing.T) {
@@ -21,11 +22,11 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	dag.TestSetBlockRewardMaturity(1)
+	dag.TestSetCoinbaseMaturity(0)
 
 	// Test rejecting the block if its parents are missing
 	orphanBlockFile := "blk_3B.dat"
-	loadedBlocks, err := loadBlocks(orphanBlockFile)
+	loadedBlocks, err := LoadBlocks(filepath.Join("testdata/", orphanBlockFile))
 	if err != nil {
 		t.Fatalf("TestMaybeAcceptBlockErrors: "+
 			"Error loading file '%s': %s\n", orphanBlockFile, err)
@@ -48,7 +49,7 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 
 	// Test rejecting the block if its parents are invalid
 	blocksFile := "blk_0_to_4.dat"
-	blocks, err := loadBlocks(blocksFile)
+	blocks, err := LoadBlocks(filepath.Join("testdata/", blocksFile))
 	if err != nil {
 		t.Fatalf("TestMaybeAcceptBlockErrors: "+
 			"Error loading file '%s': %s\n", blocksFile, err)
@@ -56,10 +57,13 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 
 	// Add a valid block and mark it as invalid
 	block1 := blocks[1]
-	isOrphan, err := dag.ProcessBlock(block1, BFNone)
+	isOrphan, delay, err := dag.ProcessBlock(block1, BFNone)
 	if err != nil {
 		t.Fatalf("TestMaybeAcceptBlockErrors: Valid block unexpectedly returned an error: %s", err)
 	}
+	if delay != 0 {
+		t.Fatalf("TestMaybeAcceptBlockErrors: block 1 is too far in the future")
+	}
 	if isOrphan {
 		t.Fatalf("TestMaybeAcceptBlockErrors: incorrectly returned block 1 is an orphan")
 	}

blockdag/blockheap.go

@@ -3,7 +3,7 @@ package blockdag
 import (
 	"container/heap"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // baseHeap is an implementation for heap.Interface that sorts blocks by their height
@@ -28,61 +28,61 @@ func (h *baseHeap) Pop() interface{} {
 type upHeap struct{ baseHeap }
 
 func (h upHeap) Less(i, j int) bool {
-	if h.baseHeap[i].height == h.baseHeap[j].height {
+	if h.baseHeap[i].blueScore == h.baseHeap[j].blueScore {
 		return daghash.HashToBig(h.baseHeap[i].hash).Cmp(daghash.HashToBig(h.baseHeap[j].hash)) < 0
 	}
 
-	return h.baseHeap[i].height < h.baseHeap[j].height
+	return h.baseHeap[i].blueScore < h.baseHeap[j].blueScore
 }
 
 // downHeap extends baseHeap to include Less operation that traverses from top to bottom
 type downHeap struct{ baseHeap }
 
 func (h downHeap) Less(i, j int) bool {
-	if h.baseHeap[i].height == h.baseHeap[j].height {
+	if h.baseHeap[i].blueScore == h.baseHeap[j].blueScore {
 		return daghash.HashToBig(h.baseHeap[i].hash).Cmp(daghash.HashToBig(h.baseHeap[j].hash)) > 0
 	}
 
-	return h.baseHeap[i].height > h.baseHeap[j].height
+	return h.baseHeap[i].blueScore > h.baseHeap[j].blueScore
 }
 
-// BlockHeap represents a mutable heap of Blocks, sorted by their height
-type BlockHeap struct {
+// blockHeap represents a mutable heap of Blocks, sorted by their height
+type blockHeap struct {
 	impl heap.Interface
 }
 
-// NewDownHeap initializes and returns a new BlockHeap
-func NewDownHeap() BlockHeap {
-	h := BlockHeap{impl: &downHeap{}}
+// newDownHeap initializes and returns a new blockHeap
+func newDownHeap() blockHeap {
+	h := blockHeap{impl: &downHeap{}}
 	heap.Init(h.impl)
 	return h
 }
 
-// NewUpHeap initializes and returns a new BlockHeap
-func NewUpHeap() BlockHeap {
-	h := BlockHeap{impl: &upHeap{}}
+// newUpHeap initializes and returns a new blockHeap
+func newUpHeap() blockHeap {
+	h := blockHeap{impl: &upHeap{}}
 	heap.Init(h.impl)
 	return h
 }
 
 // pop removes the block with lowest height from this heap and returns it
-func (bh BlockHeap) pop() *blockNode {
+func (bh blockHeap) pop() *blockNode {
 	return heap.Pop(bh.impl).(*blockNode)
 }
 
 // Push pushes the block onto the heap
-func (bh BlockHeap) Push(block *blockNode) {
+func (bh blockHeap) Push(block *blockNode) {
 	heap.Push(bh.impl, block)
 }
 
 // pushSet pushes a blockset to the heap.
-func (bh BlockHeap) pushSet(bs blockSet) {
+func (bh blockHeap) pushSet(bs blockSet) {
 	for _, block := range bs {
 		heap.Push(bh.impl, block)
 	}
 }
 
 // Len returns the length of this heap
-func (bh BlockHeap) Len() int {
+func (bh blockHeap) Len() int {
 	return bh.impl.Len()
 }

blockdag/blockheap_test.go

@@ -3,8 +3,8 @@ package blockdag
 import (
 	"testing"
 
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // TestBlockHeap tests pushing, popping, and determining the length of the heap.
@@ -81,7 +81,7 @@ func TestBlockHeap(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		dHeap := NewDownHeap()
+		dHeap := newDownHeap()
 		for _, block := range test.toPush {
 			dHeap.Push(block)
 		}
@@ -99,7 +99,7 @@ func TestBlockHeap(t *testing.T) {
 				"Expected: %v, got: %v", test.name, test.expectedPopDown, poppedBlock)
 		}
 
-		uHeap := NewUpHeap()
+		uHeap := newUpHeap()
 		for _, block := range test.toPush {
 			uHeap.Push(block)
 		}

blockdag/blockidhash.go

@@ -0,0 +1,136 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+)
+
+var (
+	// idByHashIndexBucketName is the name of the db bucket used to house
+	// the block hash -> block id index.
+	idByHashIndexBucketName = []byte("idbyhashidx")
+
+	// hashByIDIndexBucketName is the name of the db bucket used to house
+	// the block id -> block hash index.
+	hashByIDIndexBucketName = []byte("hashbyididx")
+
+	currentBlockIDKey = []byte("currentblockid")
+)
+
+// -----------------------------------------------------------------------------
+// This is a mapping between block hashes and unique IDs. The ID
+// is simply a sequentially incremented uint64 that is used instead of block hash
+// for the indexers. This is useful because it is only 8 bytes versus 32 bytes
+// hashes and thus saves a ton of space when a block is referenced in an index.
+// It consists of three buckets: the first bucket maps the hash of each
+// block to the unique ID and the second maps that ID back to the block hash.
+// The third bucket contains the last received block ID, and is used
+// when starting the node to check that the enabled indexes are up to date
+// with the latest received block, and if not, initiate recovery process.
+//
+// The serialized format for keys and values in the block hash to ID bucket is:
+//   <hash> = <ID>
+//
+//   Field           Type              Size
+//   hash            daghash.Hash     32 bytes
+//   ID              uint64            8 bytes
+//   -----
+//   Total: 40 bytes
+//
+// The serialized format for keys and values in the ID to block hash bucket is:
+//   <ID> = <hash>
+//
+//   Field           Type              Size
+//   ID              uint64            8 bytes
+//   hash            daghash.Hash     32 bytes
+//   -----
+//   Total: 40 bytes
+//
+// -----------------------------------------------------------------------------
+
+const blockIDSize = 8 // 8 bytes for block ID
+
+// DBFetchBlockIDByHash uses an existing database transaction to retrieve the
+// block id for the provided hash from the index.
+func DBFetchBlockIDByHash(dbTx database.Tx, hash *daghash.Hash) (uint64, error) {
+	hashIndex := dbTx.Metadata().Bucket(idByHashIndexBucketName)
+	serializedID := hashIndex.Get(hash[:])
+	if serializedID == nil {
+		return 0, errors.Errorf("no entry in the block ID index for block with hash %s", hash)
+	}
+
+	return DeserializeBlockID(serializedID), nil
+}
+
+// DBFetchBlockHashBySerializedID uses an existing database transaction to
+// retrieve the hash for the provided serialized block id from the index.
+func DBFetchBlockHashBySerializedID(dbTx database.Tx, serializedID []byte) (*daghash.Hash, error) {
+	idIndex := dbTx.Metadata().Bucket(hashByIDIndexBucketName)
+	hashBytes := idIndex.Get(serializedID)
+	if hashBytes == nil {
+		return nil, errors.Errorf("no entry in the block ID index for block with id %d", byteOrder.Uint64(serializedID))
+	}
+
+	var hash daghash.Hash
+	copy(hash[:], hashBytes)
+	return &hash, nil
+}
+
+// dbPutBlockIDIndexEntry uses an existing database transaction to update or add
+// the index entries for the hash to id and id to hash mappings for the provided
+// values.
+func dbPutBlockIDIndexEntry(dbTx database.Tx, hash *daghash.Hash, serializedID []byte) error {
+	// Add the block hash to ID mapping to the index.
+	meta := dbTx.Metadata()
+	hashIndex := meta.Bucket(idByHashIndexBucketName)
+	if err := hashIndex.Put(hash[:], serializedID[:]); err != nil {
+		return err
+	}
+
+	// Add the block ID to hash mapping to the index.
+	idIndex := meta.Bucket(hashByIDIndexBucketName)
+	return idIndex.Put(serializedID[:], hash[:])
+}
+
+// DBFetchCurrentBlockID returns the last known block ID.
+func DBFetchCurrentBlockID(dbTx database.Tx) uint64 {
+	serializedID := dbTx.Metadata().Get(currentBlockIDKey)
+	if serializedID == nil {
+		return 0
+	}
+	return DeserializeBlockID(serializedID)
+}
+
+// DeserializeBlockID returns a deserialized block id
+func DeserializeBlockID(serializedID []byte) uint64 {
+	return byteOrder.Uint64(serializedID)
+}
+
+// SerializeBlockID returns a serialized block id
+func SerializeBlockID(blockID uint64) []byte {
+	serializedBlockID := make([]byte, blockIDSize)
+	byteOrder.PutUint64(serializedBlockID, blockID)
+	return serializedBlockID
+}
+
+// DBFetchBlockHashByID uses an existing database transaction to retrieve the
+// hash for the provided block id from the index.
+func DBFetchBlockHashByID(dbTx database.Tx, id uint64) (*daghash.Hash, error) {
+	return DBFetchBlockHashBySerializedID(dbTx, SerializeBlockID(id))
+}
+
+func createBlockID(dbTx database.Tx, blockHash *daghash.Hash) (uint64, error) {
+	currentBlockID := DBFetchCurrentBlockID(dbTx)
+	newBlockID := currentBlockID + 1
+	serializedNewBlockID := SerializeBlockID(newBlockID)
+	err := dbTx.Metadata().Put(currentBlockIDKey, serializedNewBlockID)
+	if err != nil {
+		return 0, err
+	}
+	err = dbPutBlockIDIndexEntry(dbTx, blockHash, serializedNewBlockID)
+	if err != nil {
+		return 0, err
+	}
+	return newBlockID, nil
+}

blockdag/blockindex.go

@@ -7,9 +7,9 @@ package blockdag
 import (
 	"sync"
 
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // blockIndex provides facilities for keeping track of an in-memory index of the
@@ -117,27 +117,28 @@ func (bi *blockIndex) UnsetStatusFlags(node *blockNode, flags blockStatus) {
 // flushToDB writes all dirty block nodes to the database. If all writes
 // succeed, this clears the dirty set.
 func (bi *blockIndex) flushToDB() error {
+	return bi.db.Update(func(dbTx database.Tx) error {
+		return bi.flushToDBWithTx(dbTx)
+	})
+}
+
+// flushToDBWithTx writes all dirty block nodes to the database. If all
+// writes succeed, this clears the dirty set.
+func (bi *blockIndex) flushToDBWithTx(dbTx database.Tx) error {
 	bi.Lock()
+	defer bi.Unlock()
 	if len(bi.dirty) == 0 {
-		bi.Unlock()
 		return nil
 	}
 
-	err := bi.db.Update(func(dbTx database.Tx) error {
-		for node := range bi.dirty {
-			err := dbStoreBlockNode(dbTx, node)
-			if err != nil {
-				return err
-			}
+	for node := range bi.dirty {
+		err := dbStoreBlockNode(dbTx, node)
+		if err != nil {
+			return err
 		}
-		return nil
-	})
+	}
 
 	// If write was successful, clear the dirty set.
-	if err == nil {
-		bi.dirty = make(map[*blockNode]struct{})
-	}
-
-	bi.Unlock()
-	return err
+	bi.dirty = make(map[*blockNode]struct{})
+	return nil
 }

blockdag/blockindex_test.go

@@ -1,19 +1,19 @@
 package blockdag
 
 import (
-	"errors"
+	"github.com/pkg/errors"
 	"strings"
 	"testing"
 	"time"
 
 	"bou.ke/monkey"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/database"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
 )
 
 func TestAncestorErrors(t *testing.T) {
 	node := newTestNode(newSet(), int32(0x10000000), 0, time.Unix(0, 0), dagconfig.MainNetParams.K)
-	node.height = 2
+	node.chainHeight = 2
 	ancestor := node.SelectedAncestor(3)
 	if ancestor != nil {
 		t.Errorf("TestAncestorErrors: Ancestor() unexpectedly returned a node. Expected: <nil>")

blockdag/blocklocator.go

@@ -0,0 +1,143 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// BlockLocator is used to help locate a specific block.  The algorithm for
+// building the block locator is to add block hashes in reverse order on the
+// block's selected parent chain until the desired stop block is reached.
+// In order to keep the list of locator hashes to a reasonable number of entries,
+// the step between each entry is doubled each loop iteration to exponentially
+// decrease the number of hashes as a function of the distance from the block
+// being located.
+//
+// For example, assume a selected parent chain with IDs as depicted below, and the
+// stop block is genesis:
+// 	genesis -> 1 -> 2 -> ... -> 15 -> 16  -> 17  -> 18
+//
+// The block locator for block 17 would be the hashes of blocks:
+//  [17 16 14 11 7 2 genesis]
+type BlockLocator []*daghash.Hash
+
+// BlockLocatorFromHashes returns a block locator from start and stop hash.
+// See BlockLocator for details on the algorithm used to create a block locator.
+//
+// In addition to the general algorithm referenced above, this function will
+// return the block locator for the selected tip if the passed hash is not currently
+// known.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) BlockLocatorFromHashes(startHash, stopHash *daghash.Hash) BlockLocator {
+	dag.dagLock.RLock()
+	defer dag.dagLock.RUnlock()
+	startNode := dag.index.LookupNode(startHash)
+	var stopNode *blockNode
+	if !stopHash.IsEqual(&daghash.ZeroHash) {
+		stopNode = dag.index.LookupNode(stopHash)
+	}
+	return dag.blockLocator(startNode, stopNode)
+}
+
+// LatestBlockLocator returns a block locator for the current tips of the DAG.
+//
+// This function is safe for concurrent access.
+func (dag *BlockDAG) LatestBlockLocator() BlockLocator {
+	dag.dagLock.RLock()
+	defer dag.dagLock.RUnlock()
+	return dag.blockLocator(nil, nil)
+}
+
+// blockLocator returns a block locator for the passed start and stop nodes.
+// The default value for the start node is the selected tip, and the default
+// values of the stop node is the genesis block.
+//
+// See the BlockLocator type comments for more details.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) blockLocator(startNode, stopNode *blockNode) BlockLocator {
+	// Use the selected tip if requested.
+	if startNode == nil {
+		startNode = dag.virtual.selectedParent
+	}
+
+	if stopNode == nil {
+		stopNode = dag.genesis
+	}
+
+	// We use the selected parent of the start node, so the
+	// block locator won't contain the start node.
+	startNode = startNode.selectedParent
+
+	// If the start node or the stop node are not in the
+	// virtual's selected parent chain, we replace them with their
+	// closest selected parent that is part of the virtual's
+	// selected parent chain.
+	for !dag.IsInSelectedParentChain(stopNode.hash) {
+		stopNode = stopNode.selectedParent
+	}
+
+	for !dag.IsInSelectedParentChain(startNode.hash) {
+		startNode = startNode.selectedParent
+	}
+
+	// Calculate the max number of entries that will ultimately be in the
+	// block locator.  See the description of the algorithm for how these
+	// numbers are derived.
+
+	// startNode.hash + stopNode.hash.
+	// Then floor(log2(startNode.chainHeight-stopNode.chainHeight)) entries for the skip portion.
+	maxEntries := 2 + util.FastLog2Floor(startNode.chainHeight-stopNode.chainHeight)
+	locator := make(BlockLocator, 0, maxEntries)
+
+	step := uint64(1)
+	for node := startNode; node != nil; {
+		locator = append(locator, node.hash)
+
+		// Nothing more to add once the stop node has been added.
+		if node.chainHeight == stopNode.chainHeight {
+			break
+		}
+
+		// Calculate chainHeight of previous node to include ensuring the
+		// final node is stopNode.
+		nextChainHeight := node.chainHeight - step
+		if nextChainHeight < stopNode.chainHeight {
+			nextChainHeight = stopNode.chainHeight
+		}
+
+		// walk backwards through the nodes to the correct ancestor.
+		node = node.SelectedAncestor(nextChainHeight)
+
+		// Double the distance between included hashes.
+		step *= 2
+	}
+
+	return locator
+}
+
+// FindNextLocatorBoundaries returns the lowest unknown block locator, hash
+// and the highest known block locator hash. This is used to create the
+// next block locator to find the highest shared known chain block with the
+// sync peer.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) FindNextLocatorBoundaries(locator BlockLocator) (startHash, stopHash *daghash.Hash) {
+	// Find the most recent locator block hash in the DAG.  In the case none of
+	// the hashes in the locator are in the DAG, fall back to the genesis block.
+	stopNode := dag.genesis
+	nextBlockLocatorIndex := int64(len(locator) - 1)
+	for i, hash := range locator {
+		node := dag.index.LookupNode(hash)
+		if node != nil {
+			stopNode = node
+			nextBlockLocatorIndex = int64(i) - 1
+			break
+		}
+	}
+	if nextBlockLocatorIndex < 0 {
+		return nil, stopNode.hash
+	}
+	return locator[nextBlockLocatorIndex], stopNode.hash
+}

blockdag/blocknode.go

@@ -6,13 +6,10 @@ package blockdag
 
 import (
 	"fmt"
-	"math/big"
-	"sort"
 	"time"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // blockStatus is a bit field representing the validation state of the block.
@@ -78,84 +75,65 @@ type blockNode struct {
 	// blueScore is the count of all the blue blocks in this block's past
 	blueScore uint64
 
-	// diff is the UTXO representation of the block
-	// A block's UTXO is reconstituted by applying diffWith on every block in the chain of diffChildren
-	// from the virtual block down to the block. See diffChild
-	diff *UTXODiff
-
-	// diffChild is the child that diff will be built from. See diff
-	diffChild *blockNode
-
 	// hash is the double sha 256 of the block.
 	hash *daghash.Hash
 
-	// workSum is the total amount of work in the DAG up to and including
-	// this node.
-	workSum *big.Int
-
-	// height is the position in the block DAG.
-	height int32
-
 	// chainHeight is the number of hops you need to go down the selected parent chain in order to get to the genesis block.
-	chainHeight uint32
+	chainHeight uint64
 
 	// Some fields from block headers to aid in best chain selection and
 	// reconstructing headers from memory.  These must be treated as
 	// immutable and are intentionally ordered to avoid padding on 64-bit
 	// platforms.
-	version        int32
-	bits           uint32
-	nonce          uint64
-	timestamp      int64
-	hashMerkleRoot *daghash.Hash
-	idMerkleRoot   *daghash.Hash
+	version              int32
+	bits                 uint32
+	nonce                uint64
+	timestamp            int64
+	hashMerkleRoot       *daghash.Hash
+	acceptedIDMerkleRoot *daghash.Hash
+	utxoCommitment       *daghash.Hash
 
 	// status is a bitfield representing the validation state of the block. The
 	// status field, unlike the other fields, may be written to and so should
 	// only be accessed using the concurrent-safe NodeStatus method on
 	// blockIndex once the node has been added to the global index.
 	status blockStatus
+
+	// isFinalized determines whether the node is below the finality point.
+	isFinalized bool
 }
 
-// initBlockNode initializes a block node from the given header and parent nodes,
-// calculating the height and workSum from the respective fields on the first parent.
+// initBlockNode initializes a block node from the given header and parent nodes.
 // This function is NOT safe for concurrent access.  It must only be called when
 // initially creating a node.
 func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, parents blockSet, phantomK uint32) {
 	*node = blockNode{
 		parents:   parents,
 		children:  make(blockSet),
-		workSum:   big.NewInt(0),
 		timestamp: time.Now().Unix(),
 	}
 
 	// blockHeader is nil only for the virtual block
 	if blockHeader != nil {
 		node.hash = blockHeader.BlockHash()
-		node.workSum = util.CalcWork(blockHeader.Bits)
 		node.version = blockHeader.Version
 		node.bits = blockHeader.Bits
 		node.nonce = blockHeader.Nonce
 		node.timestamp = blockHeader.Timestamp.Unix()
 		node.hashMerkleRoot = blockHeader.HashMerkleRoot
-		node.idMerkleRoot = blockHeader.IDMerkleRoot
+		node.acceptedIDMerkleRoot = blockHeader.AcceptedIDMerkleRoot
+		node.utxoCommitment = blockHeader.UTXOCommitment
 	} else {
 		node.hash = &daghash.ZeroHash
 	}
 
 	if len(parents) > 0 {
 		node.blues, node.selectedParent, node.blueScore = phantom(node, phantomK)
-		node.height = calculateNodeHeight(node)
 		node.chainHeight = calculateChainHeight(node)
-		node.workSum = node.workSum.Add(node.selectedParent.workSum, node.workSum)
 	}
 }
 
-func calculateNodeHeight(node *blockNode) int32 {
-	return node.parents.maxHeight() + 1
-}
-
-func calculateChainHeight(node *blockNode) uint32 {
+func calculateChainHeight(node *blockNode) uint64 {
 	if node.isGenesis() {
 		return 0
 	}
@@ -163,8 +141,7 @@ func calculateChainHeight(node *blockNode) uint32 {
 }
 
 // newBlockNode returns a new block node for the given block header and parent
-// nodes, calculating the height and workSum from the respective fields on the
-// parent. This function is NOT safe for concurrent access.
+//nodes. This function is NOT safe for concurrent access.
 func newBlockNode(blockHeader *wire.BlockHeader, parents blockSet, phantomK uint32) *blockNode {
 	var node blockNode
 	initBlockNode(&node, blockHeader, parents, phantomK)
@@ -184,67 +161,54 @@ func (node *blockNode) updateParentsChildren() {
 func (node *blockNode) Header() *wire.BlockHeader {
 	// No lock is needed because all accessed fields are immutable.
 	return &wire.BlockHeader{
-		Version:        node.version,
-		ParentHashes:   node.ParentHashes(),
-		HashMerkleRoot: node.hashMerkleRoot,
-		IDMerkleRoot:   node.idMerkleRoot,
-		Timestamp:      time.Unix(node.timestamp, 0),
-		Bits:           node.bits,
-		Nonce:          node.nonce,
+		Version:              node.version,
+		ParentHashes:         node.ParentHashes(),
+		HashMerkleRoot:       node.hashMerkleRoot,
+		AcceptedIDMerkleRoot: node.acceptedIDMerkleRoot,
+		UTXOCommitment:       node.utxoCommitment,
+		Timestamp:            time.Unix(node.timestamp, 0),
+		Bits:                 node.bits,
+		Nonce:                node.nonce,
 	}
 }
 
-// SelectedAncestor returns the ancestor block node at the provided height by following
-// the selected chain backwards from this node.  The returned block will be nil when a
-// height is requested that is after the height of the passed node or is less than zero.
+// SelectedAncestor returns the ancestor block node at the provided chain-height by following
+// the selected-parents chain backwards from this node. The returned block will be nil when a
+// height is requested that is after the height of the passed node.
 //
 // This function is safe for concurrent access.
-func (node *blockNode) SelectedAncestor(height int32) *blockNode {
-	if height < 0 || height > node.height {
+func (node *blockNode) SelectedAncestor(chainHeight uint64) *blockNode {
+	if chainHeight < 0 || chainHeight > node.chainHeight {
 		return nil
 	}
 
 	n := node
-	for ; n != nil && n.height != height; n = n.selectedParent {
+	for ; n != nil && n.chainHeight != chainHeight; n = n.selectedParent {
 		// Intentionally left blank
 	}
 
 	return n
 }
 
-// RelativeAncestor returns the ancestor block node a relative 'distance' blocks
-// before this node.  This is equivalent to calling Ancestor with the node's
-// height minus provided distance.
+// RelativeAncestor returns the ancestor block node a relative 'distance' of
+// chain-blocks before this node. This is equivalent to calling Ancestor with
+// the node's chain-height minus provided distance.
 //
 // This function is safe for concurrent access.
-func (node *blockNode) RelativeAncestor(distance int32) *blockNode {
-	return node.SelectedAncestor(node.height - distance)
+func (node *blockNode) RelativeAncestor(distance uint64) *blockNode {
+	return node.SelectedAncestor(node.chainHeight - distance)
 }
 
-// PastMedianTime returns the median time of the previous few blocks
+// CalcPastMedianTime returns the median time of the previous few blocks
 // prior to, and including, the block node.
 //
 // This function is safe for concurrent access.
-func (node *blockNode) PastMedianTime() time.Time {
-	// Create a slice of the previous few block timestamps used to calculate
-	// the median per the number defined by the constant medianTimeBlocks.
-	// If there aren't enough blocks yet - pad remaining with genesis block's timestamp.
-	timestamps := make([]int64, medianTimeBlocks)
-	iterNode := node
-	for i := 0; i < medianTimeBlocks; i++ {
-		timestamps[i] = iterNode.timestamp
-
-		if !iterNode.isGenesis() {
-			iterNode = iterNode.selectedParent
-		}
+func (node *blockNode) PastMedianTime(dag *BlockDAG) time.Time {
+	window := blueBlockWindow(node, 2*dag.TimestampDeviationTolerance-1)
+	medianTimestamp, err := window.medianTimestamp()
+	if err != nil {
+		panic(fmt.Sprintf("blueBlockWindow: %s", err))
 	}
-
-	sort.Sort(timeSorter(timestamps))
-
-	// Note: This works when medianTimeBlockCount is an odd number.
-	// If it is to be changed to an even number - must take avarage of two middle values
-	// Since medianTimeBlockCount is a constant, we can skip the odd/even check
-	medianTimestamp := timestamps[medianTimeBlocks/2]
 	return time.Unix(medianTimestamp, 0)
 }
 
@@ -257,11 +221,11 @@ func (node *blockNode) isGenesis() bool {
 	return len(node.parents) == 0
 }
 
-func (node *blockNode) finalityScore() uint64 {
-	return node.blueScore / FinalityInterval
+func (node *blockNode) finalityScore(dag *BlockDAG) uint64 {
+	return node.blueScore / uint64(dag.dagParams.FinalityInterval)
 }
 
-// String returns a string that contains the block hash and height.
+// String returns a string that contains the block hash.
 func (node blockNode) String() string {
-	return fmt.Sprintf("%s (%d)", node.hash, node.height)
+	return node.hash.String()
 }

blockdag/blocknode_test.go

@@ -26,7 +26,7 @@ func TestChainHeight(t *testing.T) {
 
 	tests := []struct {
 		node                *blockNode
-		expectedChainHeight uint32
+		expectedChainHeight uint64
 	}{
 		{
 			node:                node0,

blockdag/blockset.go

@@ -3,7 +3,7 @@ package blockdag
 import (
 	"strings"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // blockSet implements a basic unsorted set of blocks
@@ -23,30 +23,6 @@ func setFromSlice(blocks ...*blockNode) blockSet {
 	return set
 }
 
-// maxHeight returns the height of the highest block in the block set
-func (bs blockSet) maxHeight() int32 {
-	var maxHeight int32
-	for _, node := range bs {
-		if maxHeight < node.height {
-			maxHeight = node.height
-		}
-	}
-	return maxHeight
-}
-
-func (bs blockSet) highest() *blockNode {
-	var highest *blockNode
-	for _, node := range bs {
-		if highest == nil ||
-			highest.height < node.height ||
-			(highest.height == node.height && daghash.Less(node.hash, highest.hash)) {
-
-			highest = node
-		}
-	}
-	return highest
-}
-
 // add adds a block to this BlockSet
 func (bs blockSet) add(block *blockNode) {
 	bs[*block.hash] = block

blockdag/blockset_test.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 func TestHashes(t *testing.T) {
@@ -35,47 +35,6 @@ func TestHashes(t *testing.T) {
 		t.Errorf("TestHashes: hashes order is %s but expected %s", hashes, expected)
 	}
 }
-func TestBlockSetHighest(t *testing.T) {
-	node1 := &blockNode{hash: &daghash.Hash{10}, height: 1}
-	node2a := &blockNode{hash: &daghash.Hash{20}, height: 2}
-	node2b := &blockNode{hash: &daghash.Hash{21}, height: 2}
-	node3 := &blockNode{hash: &daghash.Hash{30}, height: 3}
-
-	tests := []struct {
-		name            string
-		set             blockSet
-		expectedHighest *blockNode
-	}{
-		{
-			name:            "empty set",
-			set:             setFromSlice(),
-			expectedHighest: nil,
-		},
-		{
-			name:            "set with one member",
-			set:             setFromSlice(node1),
-			expectedHighest: node1,
-		},
-		{
-			name:            "same-height highest members in set",
-			set:             setFromSlice(node2b, node1, node2a),
-			expectedHighest: node2a,
-		},
-		{
-			name:            "typical set",
-			set:             setFromSlice(node2b, node3, node1, node2a),
-			expectedHighest: node3,
-		},
-	}
-
-	for _, test := range tests {
-		highest := test.set.highest()
-		if highest != test.expectedHighest {
-			t.Errorf("blockSet.highest: unexpected value in test '%s'. "+
-				"Expected: %v, got: %v", test.name, test.expectedHighest, highest)
-		}
-	}
-}
 
 func TestBlockSetSubtract(t *testing.T) {
 	node1 := &blockNode{hash: &daghash.Hash{10}}

blockdag/blockwindow.go

@@ -0,0 +1,75 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
+	"math"
+	"math/big"
+	"sort"
+)
+
+type blockWindow []*blockNode
+
+// blueBlockWindow returns a blockWindow of the given size that contains the
+// blues in the past of startindNode, sorted by phantom order.
+// If the number of blues in the past of startingNode is less then windowSize,
+// the window will be padded by genesis blocks to achieve a size of windowSize.
+func blueBlockWindow(startingNode *blockNode, windowSize uint64) blockWindow {
+	window := make(blockWindow, 0, windowSize)
+	currentNode := startingNode
+	for uint64(len(window)) < windowSize && currentNode.selectedParent != nil {
+		if currentNode.selectedParent != nil {
+			for _, blue := range currentNode.blues {
+				window = append(window, blue)
+				if uint64(len(window)) == windowSize {
+					break
+				}
+			}
+			currentNode = currentNode.selectedParent
+		}
+	}
+
+	if uint64(len(window)) < windowSize {
+		genesis := currentNode
+		for uint64(len(window)) < windowSize {
+			window = append(window, genesis)
+		}
+	}
+
+	return window
+}
+
+func (window blockWindow) minMaxTimestamps() (min, max int64) {
+	min = math.MaxInt64
+	max = 0
+	for _, node := range window {
+		if node.timestamp < min {
+			min = node.timestamp
+		}
+		if node.timestamp > max {
+			max = node.timestamp
+		}
+	}
+	return
+}
+
+func (window blockWindow) averageTarget() *big.Int {
+	averageTarget := big.NewInt(0)
+	for _, node := range window {
+		target := util.CompactToBig(node.bits)
+		averageTarget.Add(averageTarget, target)
+	}
+	return averageTarget.Div(averageTarget, big.NewInt(int64(len(window))))
+}
+
+func (window blockWindow) medianTimestamp() (int64, error) {
+	if len(window) == 0 {
+		return 0, errors.New("Cannot calculate median timestamp for an empty block window")
+	}
+	timestamps := make([]int64, len(window))
+	for i, node := range window {
+		timestamps[i] = node.timestamp
+	}
+	sort.Sort(timeSorter(timestamps))
+	return timestamps[len(timestamps)/2], nil
+}

blockdag/blockwindow_test.go

@@ -0,0 +1,138 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestBlueBlockWindow(t *testing.T) {
+	params := dagconfig.SimNetParams
+	params.K = 1
+	dag := newTestDAG(&params)
+
+	windowSize := uint64(10)
+	genesisNode := dag.genesis
+	blockTime := genesisNode.Header().Timestamp
+	blockByIDMap := make(map[string]*blockNode)
+	idByBlockMap := make(map[*blockNode]string)
+	blockByIDMap["A"] = genesisNode
+	idByBlockMap[genesisNode] = "A"
+	blockVersion := int32(0x10000000)
+
+	blocksData := []*struct {
+		parents                          []string
+		id                               string //id is a virtual entity that is used only for tests so we can define relations between blocks without knowing their hash
+		expectedWindowWithGenesisPadding []string
+	}{
+		{
+			parents:                          []string{"A"},
+			id:                               "B",
+			expectedWindowWithGenesisPadding: []string{"A", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"B"},
+			id:                               "C",
+			expectedWindowWithGenesisPadding: []string{"B", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"B"},
+			id:                               "D",
+			expectedWindowWithGenesisPadding: []string{"B", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"C", "D"},
+			id:                               "E",
+			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"C", "D"},
+			id:                               "F",
+			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"A"},
+			id:                               "G",
+			expectedWindowWithGenesisPadding: []string{"A", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"G"},
+			id:                               "H",
+			expectedWindowWithGenesisPadding: []string{"G", "A", "A", "A", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"H", "F"},
+			id:                               "I",
+			expectedWindowWithGenesisPadding: []string{"F", "D", "C", "B", "A", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"I"},
+			id:                               "J",
+			expectedWindowWithGenesisPadding: []string{"I", "F", "D", "C", "B", "A", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"J"},
+			id:                               "K",
+			expectedWindowWithGenesisPadding: []string{"J", "I", "F", "D", "C", "B", "A", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"K"},
+			id:                               "L",
+			expectedWindowWithGenesisPadding: []string{"K", "J", "I", "F", "D", "C", "B", "A", "A", "A"},
+		},
+		{
+			parents:                          []string{"L"},
+			id:                               "M",
+			expectedWindowWithGenesisPadding: []string{"L", "K", "J", "I", "F", "D", "C", "B", "A", "A"},
+		},
+		{
+			parents:                          []string{"M"},
+			id:                               "N",
+			expectedWindowWithGenesisPadding: []string{"M", "L", "K", "J", "I", "F", "D", "C", "B", "A"},
+		},
+		{
+			parents:                          []string{"N"},
+			id:                               "O",
+			expectedWindowWithGenesisPadding: []string{"N", "M", "L", "K", "J", "I", "F", "D", "C", "B"},
+		},
+	}
+
+	for _, blockData := range blocksData {
+		blockTime = blockTime.Add(time.Second)
+		parents := blockSet{}
+		for _, parentID := range blockData.parents {
+			parent := blockByIDMap[parentID]
+			parents.add(parent)
+		}
+		node := newTestNode(parents, blockVersion, 0, blockTime, dag.dagParams.K)
+		node.hash = &daghash.Hash{} // It helps to predict hash order
+		for i, char := range blockData.id {
+			node.hash[i] = byte(char)
+		}
+
+		dag.index.AddNode(node)
+		node.updateParentsChildren()
+
+		blockByIDMap[blockData.id] = node
+		idByBlockMap[node] = blockData.id
+
+		window := blueBlockWindow(node, windowSize)
+		if err := checkWindowIDs(window, blockData.expectedWindowWithGenesisPadding, idByBlockMap); err != nil {
+			t.Errorf("Unexpected values for window for block %s: %s", blockData.id, err)
+		}
+	}
+}
+
+func checkWindowIDs(window []*blockNode, expectedIDs []string, idByBlockMap map[*blockNode]string) error {
+	ids := make([]string, len(window))
+	for i, node := range window {
+		ids[i] = idByBlockMap[node]
+	}
+	if !reflect.DeepEqual(ids, expectedIDs) {
+		return errors.Errorf("window expected to have blocks %s but got %s", expectedIDs, ids)
+	}
+	return nil
+}

blockdag/checkpoints.go

@@ -6,12 +6,11 @@ package blockdag
 
 import (
 	"fmt"
-	"time"
-
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/txscript"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
 )
 
 // CheckpointConfirmations is the number of blocks before the end of the current
@@ -64,16 +63,16 @@ func (dag *BlockDAG) LatestCheckpoint() *dagconfig.Checkpoint {
 	return &dag.checkpoints[len(dag.checkpoints)-1]
 }
 
-// verifyCheckpoint returns whether the passed block height and hash combination
+// verifyCheckpoint returns whether the passed block chain height and hash combination
 // match the checkpoint data.  It also returns true if there is no checkpoint
-// data for the passed block height.
-func (dag *BlockDAG) verifyCheckpoint(height int32, hash *daghash.Hash) bool {
+// data for the passed block chain height.
+func (dag *BlockDAG) verifyCheckpoint(chainHeight uint64, hash *daghash.Hash) bool {
 	if !dag.HasCheckpoints() {
 		return true
 	}
 
-	// Nothing to check if there is no checkpoint data for the block height.
-	checkpoint, exists := dag.checkpointsByHeight[height]
+	// Nothing to check if there is no checkpoint data for the block chainHeight.
+	checkpoint, exists := dag.checkpointsByChainHeight[chainHeight]
 	if !exists {
 		return true
 	}
@@ -82,7 +81,7 @@ func (dag *BlockDAG) verifyCheckpoint(height int32, hash *daghash.Hash) bool {
 		return false
 	}
 
-	log.Infof("Verified checkpoint at height %d/block %s", checkpoint.Height,
+	log.Infof("Verified checkpoint at chainHeight %d/block %s", checkpoint.ChainHeight,
 		checkpoint.Hash)
 	return true
 }
@@ -136,10 +135,10 @@ func (dag *BlockDAG) findPreviousCheckpoint() (*blockNode, error) {
 		return dag.checkpointNode, nil
 	}
 
-	// When there is a next checkpoint and the height of the current best
-	// chain does not exceed it, the current checkpoint lockin is still
-	// the latest known checkpoint.
-	if dag.selectedTip().height < dag.nextCheckpoint.Height {
+	// When there is a next checkpoint and the chain height of the current
+	// selected tip of the DAG does not exceed it, the current checkpoint
+	// lockin is still the latest known checkpoint.
+	if dag.selectedTip().chainHeight < dag.nextCheckpoint.ChainHeight {
 		return dag.checkpointNode, nil
 	}
 
@@ -181,7 +180,7 @@ func (dag *BlockDAG) findPreviousCheckpoint() (*blockNode, error) {
 func isNonstandardTransaction(tx *util.Tx) bool {
 	// Check all of the output public key scripts for non-standard scripts.
 	for _, txOut := range tx.MsgTx().TxOut {
-		scriptClass := txscript.GetScriptClass(txOut.PkScript)
+		scriptClass := txscript.GetScriptClass(txOut.ScriptPubKey)
 		if scriptClass == txscript.NonStandardTy {
 			return true
 		}
@@ -216,19 +215,19 @@ func (dag *BlockDAG) IsCheckpointCandidate(block *util.Block) (bool, error) {
 		return false, nil
 	}
 
-	// Ensure the height of the passed block and the entry for the block in
-	// the main chain match.  This should always be the case unless the
+	// Ensure the chain height of the passed block and the entry for the block
+	// in the DAG match.  This should always be the case unless the
 	// caller provided an invalid block.
-	if node.height != block.Height() {
-		return false, fmt.Errorf("passed block height of %d does not "+
-			"match the main chain height of %d", block.Height(),
-			node.height)
+	if node.chainHeight != block.ChainHeight() {
+		return false, errors.Errorf("passed block chain height of %d does not "+
+			"match the its height in the DAG: %d", block.ChainHeight(),
+			node.chainHeight)
 	}
 
 	// A checkpoint must be at least CheckpointConfirmations blocks
 	// before the end of the main chain.
-	dagHeight := dag.selectedTip().height
-	if node.height > (dagHeight - CheckpointConfirmations) {
+	dagChainHeight := dag.selectedTip().chainHeight
+	if node.chainHeight > (dagChainHeight - CheckpointConfirmations) {
 		return false, nil
 	}
 
@@ -237,8 +236,7 @@ func (dag *BlockDAG) IsCheckpointCandidate(block *util.Block) (bool, error) {
 	// This should always succeed since the check above already made sure it
 	// is CheckpointConfirmations back, but be safe in case the constant
 	// changes.
-	nextNode := node.diffChild
-	if nextNode == nil {
+	if len(node.children) == 0 {
 		return false, nil
 	}
 
@@ -247,16 +245,6 @@ func (dag *BlockDAG) IsCheckpointCandidate(block *util.Block) (bool, error) {
 		return false, nil
 	}
 
-	// A checkpoint must have timestamps for the block and the blocks on
-	// either side of it in order (due to the median time allowance this is
-	// not always the case).
-	prevTime := time.Unix(node.selectedParent.timestamp, 0)
-	curTime := block.MsgBlock().Header.Timestamp
-	nextTime := time.Unix(nextNode.timestamp, 0)
-	if prevTime.After(curTime) || nextTime.Before(curTime) {
-		return false, nil
-	}
-
 	// A checkpoint must have transactions that only contain standard
 	// scripts.
 	for _, tx := range block.Transactions() {

blockdag/coinbase.go

@@ -0,0 +1,278 @@
+package blockdag
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+	"github.com/pkg/errors"
+	"io"
+	"math"
+
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/txsort"
+	"github.com/kaspanet/kaspad/wire"
+)
+
+// compactFeeData is a specialized data type to store a compact list of fees
+// inside a block.
+// Every transaction gets a single uint64 value, stored as a plain binary list.
+// The transactions are ordered the same way they are ordered inside the block, making it easy
+// to traverse every transaction in a block and extract its fee.
+//
+// compactFeeFactory is used to create such a list.
+// compactFeeIterator is used to iterate over such a list.
+
+type compactFeeData []byte
+
+func (cfd compactFeeData) Len() int {
+	return len(cfd) / 8
+}
+
+type compactFeeFactory struct {
+	buffer *bytes.Buffer
+	writer *bufio.Writer
+}
+
+func newCompactFeeFactory() *compactFeeFactory {
+	buffer := bytes.NewBuffer([]byte{})
+	return &compactFeeFactory{
+		buffer: buffer,
+		writer: bufio.NewWriter(buffer),
+	}
+}
+
+func (cfw *compactFeeFactory) add(txFee uint64) error {
+	return binary.Write(cfw.writer, binary.LittleEndian, txFee)
+}
+
+func (cfw *compactFeeFactory) data() (compactFeeData, error) {
+	err := cfw.writer.Flush()
+
+	return compactFeeData(cfw.buffer.Bytes()), err
+}
+
+type compactFeeIterator struct {
+	reader io.Reader
+}
+
+func (cfd compactFeeData) iterator() *compactFeeIterator {
+	return &compactFeeIterator{
+		reader: bufio.NewReader(bytes.NewBuffer(cfd)),
+	}
+}
+
+func (cfr *compactFeeIterator) next() (uint64, error) {
+	var txFee uint64
+
+	err := binary.Read(cfr.reader, binary.LittleEndian, &txFee)
+
+	return txFee, err
+}
+
+// The following functions relate to storing and retrieving fee data from the database
+var feeBucket = []byte("fees")
+
+// getBluesFeeData returns the compactFeeData for all nodes's blues,
+// used to calculate the fees this blockNode needs to pay
+func (node *blockNode) getBluesFeeData(dag *BlockDAG) (map[daghash.Hash]compactFeeData, error) {
+	bluesFeeData := make(map[daghash.Hash]compactFeeData)
+
+	err := dag.db.View(func(dbTx database.Tx) error {
+		for _, blueBlock := range node.blues {
+			feeData, err := dbFetchFeeData(dbTx, blueBlock.hash)
+			if err != nil {
+				return errors.Errorf("Error getting fee data for block %s: %s", blueBlock.hash, err)
+			}
+
+			bluesFeeData[*blueBlock.hash] = feeData
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return bluesFeeData, nil
+}
+
+func dbStoreFeeData(dbTx database.Tx, blockHash *daghash.Hash, feeData compactFeeData) error {
+	feeBucket, err := dbTx.Metadata().CreateBucketIfNotExists(feeBucket)
+	if err != nil {
+		return errors.Errorf("Error creating or retrieving fee bucket: %s", err)
+	}
+
+	return feeBucket.Put(blockHash.CloneBytes(), feeData)
+}
+
+func dbFetchFeeData(dbTx database.Tx, blockHash *daghash.Hash) (compactFeeData, error) {
+	feeBucket := dbTx.Metadata().Bucket(feeBucket)
+	if feeBucket == nil {
+		return nil, errors.New("Fee bucket does not exist")
+	}
+
+	feeData := feeBucket.Get(blockHash.CloneBytes())
+	if feeData == nil {
+		return nil, errors.Errorf("No fee data found for block %s", blockHash)
+	}
+
+	return feeData, nil
+}
+
+// The following functions deal with building and validating the coinbase transaction
+
+func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Block, txsAcceptanceData MultiBlockTxsAcceptanceData) error {
+	if node.isGenesis() {
+		return nil
+	}
+	blockCoinbaseTx := block.CoinbaseTransaction().MsgTx()
+	scriptPubKey, extraData, err := DeserializeCoinbasePayload(blockCoinbaseTx)
+	if err != nil {
+		return err
+	}
+	expectedCoinbaseTransaction, err := node.expectedCoinbaseTransaction(dag, txsAcceptanceData, scriptPubKey, extraData)
+	if err != nil {
+		return err
+	}
+
+	if !expectedCoinbaseTransaction.Hash().IsEqual(block.CoinbaseTransaction().Hash()) {
+		return ruleError(ErrBadCoinbaseTransaction, "Coinbase transaction is not built as expected")
+	}
+
+	return nil
+}
+
+// expectedCoinbaseTransaction returns the coinbase transaction for the current block
+func (node *blockNode) expectedCoinbaseTransaction(dag *BlockDAG, txsAcceptanceData MultiBlockTxsAcceptanceData, scriptPubKey []byte, extraData []byte) (*util.Tx, error) {
+	bluesFeeData, err := node.getBluesFeeData(dag)
+	if err != nil {
+		return nil, err
+	}
+
+	txIns := []*wire.TxIn{}
+	txOuts := []*wire.TxOut{}
+
+	for _, blue := range node.blues {
+		txIn, txOut, err := coinbaseInputAndOutputForBlueBlock(dag, blue, txsAcceptanceData, bluesFeeData)
+		if err != nil {
+			return nil, err
+		}
+		txIns = append(txIns, txIn)
+		if txOut != nil {
+			txOuts = append(txOuts, txOut)
+		}
+	}
+	payload, err := SerializeCoinbasePayload(scriptPubKey, extraData)
+	if err != nil {
+		return nil, err
+	}
+	coinbaseTx := wire.NewSubnetworkMsgTx(wire.TxVersion, txIns, txOuts, subnetworkid.SubnetworkIDCoinbase, 0, payload)
+	sortedCoinbaseTx := txsort.Sort(coinbaseTx)
+	return util.NewTx(sortedCoinbaseTx), nil
+}
+
+// SerializeCoinbasePayload builds the coinbase payload based on the provided scriptPubKey and extra data.
+func SerializeCoinbasePayload(scriptPubKey []byte, extraData []byte) ([]byte, error) {
+	w := &bytes.Buffer{}
+	err := wire.WriteVarInt(w, uint64(len(scriptPubKey)))
+	if err != nil {
+		return nil, err
+	}
+	_, err = w.Write(scriptPubKey)
+	if err != nil {
+		return nil, err
+	}
+	_, err = w.Write(extraData)
+	if err != nil {
+		return nil, err
+	}
+	return w.Bytes(), nil
+}
+
+// DeserializeCoinbasePayload deserialize the coinbase payload to its component (scriptPubKey and extra data).
+func DeserializeCoinbasePayload(tx *wire.MsgTx) (scriptPubKey []byte, extraData []byte, err error) {
+	r := bytes.NewReader(tx.Payload)
+	scriptPubKeyLen, err := wire.ReadVarInt(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	scriptPubKey = make([]byte, scriptPubKeyLen)
+	_, err = r.Read(scriptPubKey)
+	if err != nil {
+		return nil, nil, err
+	}
+	extraData = make([]byte, r.Len())
+	if r.Len() != 0 {
+		_, err = r.Read(extraData)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	return scriptPubKey, extraData, nil
+}
+
+// feeInputAndOutputForBlueBlock calculates the input and output that should go into the coinbase transaction of blueBlock
+// If blueBlock gets no fee - returns only txIn and nil for txOut
+func coinbaseInputAndOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
+	txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (
+	*wire.TxIn, *wire.TxOut, error) {
+
+	blockTxsAcceptanceData, ok := txsAcceptanceData.FindAcceptanceData(blueBlock.hash)
+	if !ok {
+		return nil, nil, errors.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
+	}
+	blockFeeData, ok := feeData[*blueBlock.hash]
+	if !ok {
+		return nil, nil, errors.Errorf("No feeData for block %s", blueBlock.hash)
+	}
+
+	if len(blockTxsAcceptanceData.TxAcceptanceData) != blockFeeData.Len() {
+		return nil, nil, errors.Errorf(
+			"length of accepted transaction data(%d) and fee data(%d) is not equal for block %s",
+			len(blockTxsAcceptanceData.TxAcceptanceData), blockFeeData.Len(), blueBlock.hash)
+	}
+
+	txIn := &wire.TxIn{
+		SignatureScript: []byte{},
+		PreviousOutpoint: wire.Outpoint{
+			TxID:  daghash.TxID(*blueBlock.hash),
+			Index: math.MaxUint32,
+		},
+		Sequence: wire.MaxTxInSequenceNum,
+	}
+
+	totalFees := uint64(0)
+	feeIterator := blockFeeData.iterator()
+
+	for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
+		fee, err := feeIterator.next()
+		if err != nil {
+			return nil, nil, errors.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
+		}
+		if txAcceptanceData.IsAccepted {
+			totalFees += fee
+		}
+	}
+
+	totalReward := CalcBlockSubsidy(blueBlock.blueScore, dag.dagParams) + totalFees
+
+	if totalReward == 0 {
+		return txIn, nil, nil
+	}
+
+	// the ScriptPubKey for the coinbase is parsed from the coinbase payload
+	scriptPubKey, _, err := DeserializeCoinbasePayload(blockTxsAcceptanceData.TxAcceptanceData[0].Tx.MsgTx())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	txOut := &wire.TxOut{
+		Value:        totalReward,
+		ScriptPubKey: scriptPubKey,
+	}
+
+	return txIn, txOut, nil
+}

blockdag/common_test.go

@@ -7,78 +7,31 @@ package blockdag
 import (
 	"compress/bzip2"
 	"encoding/binary"
-	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
+	"testing"
 	"time"
 
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	_ "github.com/daglabs/btcd/database/ffldb"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/dagconfig"
+	_ "github.com/kaspanet/kaspad/database/ffldb"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
-// loadBlocks reads files containing bitcoin block data (gzipped but otherwise
-// in the format bitcoind writes) from disk and returns them as an array of
-// util.Block.  This is largely borrowed from the test code in btcdb.
-func loadBlocks(filename string) (blocks []*util.Block, err error) {
-	filename = filepath.Join("testdata/", filename)
-
-	var network = wire.MainNet
-	var dr io.Reader
-	var fi io.ReadCloser
-
-	fi, err = os.Open(filename)
-	if err != nil {
-		return
+func loadBlocksWithLog(t *testing.T, filename string) ([]*util.Block, error) {
+	blocks, err := LoadBlocks(filename)
+	if err == nil {
+		t.Logf("Loaded %d blocks from file %s", len(blocks), filename)
+		for i, b := range blocks {
+			t.Logf("Block #%d: %s", i, b.Hash())
+		}
 	}
-
-	if strings.HasSuffix(filename, ".bz2") {
-		dr = bzip2.NewReader(fi)
-	} else {
-		dr = fi
-	}
-	defer fi.Close()
-
-	var block *util.Block
-
-	err = nil
-	for height := 0; err == nil; height++ {
-		var rintbuf uint32
-		err = binary.Read(dr, binary.LittleEndian, &rintbuf)
-		if err == io.EOF {
-			// hit end of file at expected offset: no warning
-			height--
-			err = nil
-			break
-		}
-		if err != nil {
-			break
-		}
-		if rintbuf != uint32(network) {
-			break
-		}
-		err = binary.Read(dr, binary.LittleEndian, &rintbuf)
-		blocklen := rintbuf
-
-		rbytes := make([]byte, blocklen)
-
-		// read block
-		dr.Read(rbytes)
-
-		block, err = util.NewBlockFromBytes(rbytes)
-		if err != nil {
-			return
-		}
-		block.SetHeight(int32(height))
-		blocks = append(blocks, block)
-	}
-
-	return
+	return blocks, err
 }
 
 // loadUTXOSet returns a utxo view loaded from a file.
@@ -143,16 +96,16 @@ func loadUTXOSet(filename string) (UTXOSet, error) {
 		if err != nil {
 			return nil, err
 		}
-		utxoSet.utxoCollection[wire.OutPoint{TxID: txID, Index: index}] = entry
+		utxoSet.utxoCollection[wire.Outpoint{TxID: txID, Index: index}] = entry
 	}
 
 	return utxoSet, nil
 }
 
-// TestSetBlockRewardMaturity makes the ability to set the block reward maturity
+// TestSetCoinbaseMaturity makes the ability to set the coinbase maturity
 // available when running tests.
-func (dag *BlockDAG) TestSetBlockRewardMaturity(maturity uint16) {
-	dag.dagParams.BlockRewardMaturity = maturity
+func (dag *BlockDAG) TestSetCoinbaseMaturity(maturity uint64) {
+	dag.dagParams.BlockCoinbaseMaturity = maturity
 }
 
 // newTestDAG returns a DAG that is usable for syntetic tests.  It is
@@ -166,20 +119,19 @@ func newTestDAG(params *dagconfig.Params) *BlockDAG {
 	index := newBlockIndex(nil, params)
 	index.AddNode(node)
 
-	targetTimespan := int64(params.TargetTimespan / time.Second)
 	targetTimePerBlock := int64(params.TargetTimePerBlock / time.Second)
-	adjustmentFactor := params.RetargetAdjustmentFactor
 	return &BlockDAG{
-		dagParams:           params,
-		timeSource:          NewMedianTime(),
-		minRetargetTimespan: targetTimespan / adjustmentFactor,
-		maxRetargetTimespan: targetTimespan * adjustmentFactor,
-		blocksPerRetarget:   int32(targetTimespan / targetTimePerBlock),
-		index:               index,
-		virtual:             newVirtualBlock(setFromSlice(node), params.K),
-		genesis:             index.LookupNode(params.GenesisHash),
-		warningCaches:       newThresholdCaches(vbNumBits),
-		deploymentCaches:    newThresholdCaches(dagconfig.DefinedDeployments),
+		dagParams:                      params,
+		timeSource:                     NewMedianTime(),
+		targetTimePerBlock:             targetTimePerBlock,
+		difficultyAdjustmentWindowSize: params.DifficultyAdjustmentWindowSize,
+		TimestampDeviationTolerance:    params.TimestampDeviationTolerance,
+		powMaxBits:                     util.BigToCompact(params.PowMax),
+		index:                          index,
+		virtual:                        newVirtualBlock(setFromSlice(node), params.K),
+		genesis:                        index.LookupNode(params.GenesisHash),
+		warningCaches:                  newThresholdCaches(vbNumBits),
+		deploymentCaches:               newThresholdCaches(dagconfig.DefinedDeployments),
 	}
 }
 
@@ -188,12 +140,13 @@ func newTestDAG(params *dagconfig.Params) *BlockDAG {
 func newTestNode(parents blockSet, blockVersion int32, bits uint32, timestamp time.Time, phantomK uint32) *blockNode {
 	// Make up a header and create a block node from it.
 	header := &wire.BlockHeader{
-		Version:        blockVersion,
-		ParentHashes:   parents.hashes(),
-		Bits:           bits,
-		Timestamp:      timestamp,
-		HashMerkleRoot: &daghash.ZeroHash,
-		IDMerkleRoot:   &daghash.ZeroHash,
+		Version:              blockVersion,
+		ParentHashes:         parents.hashes(),
+		Bits:                 bits,
+		Timestamp:            timestamp,
+		HashMerkleRoot:       &daghash.ZeroHash,
+		AcceptedIDMerkleRoot: &daghash.ZeroHash,
+		UTXOCommitment:       &daghash.ZeroHash,
 	}
 	return newBlockNode(header, parents, phantomK)
 }
@@ -232,7 +185,7 @@ func checkRuleError(gotErr, wantErr error) error {
 	// Ensure the error code is of the expected type and the error
 	// code matches the value specified in the test instance.
 	if reflect.TypeOf(gotErr) != reflect.TypeOf(wantErr) {
-		return fmt.Errorf("wrong error - got %T (%[1]v), want %T",
+		return errors.Errorf("wrong error - got %T (%[1]v), want %T",
 			gotErr, wantErr)
 	}
 	if gotErr == nil {
@@ -242,7 +195,7 @@ func checkRuleError(gotErr, wantErr error) error {
 	// Ensure the want error type is a script error.
 	werr, ok := wantErr.(RuleError)
 	if !ok {
-		return fmt.Errorf("unexpected test error type %T", wantErr)
+		return errors.Errorf("unexpected test error type %T", wantErr)
 	}
 
 	// Ensure the error codes match.  It's safe to use a raw type assert
@@ -250,7 +203,7 @@ func checkRuleError(gotErr, wantErr error) error {
 	// the want error is a script error.
 	gotErrorCode := gotErr.(RuleError).ErrorCode
 	if gotErrorCode != werr.ErrorCode {
-		return fmt.Errorf("mismatched error code - got %v (%v), want %v",
+		return errors.Errorf("mismatched error code - got %v (%v), want %v",
 			gotErrorCode, gotErr, werr.ErrorCode)
 	}
 

blockdag/compress.go

@@ -5,8 +5,8 @@
 package blockdag
 
 import (
-	"github.com/daglabs/btcd/btcec"
-	"github.com/daglabs/btcd/txscript"
+	"github.com/kaspanet/kaspad/btcec"
+	"github.com/kaspanet/kaspad/txscript"
 )
 
 // -----------------------------------------------------------------------------
@@ -241,27 +241,27 @@ func isPubKey(script []byte) (bool, []byte) {
 
 // compressedScriptSize returns the number of bytes the passed script would take
 // when encoded with the domain specific compression algorithm described above.
-func compressedScriptSize(pkScript []byte) int {
+func compressedScriptSize(scriptPubKey []byte) int {
 	// Pay-to-pubkey-hash script.
-	if valid, _ := isPubKeyHash(pkScript); valid {
+	if valid, _ := isPubKeyHash(scriptPubKey); valid {
 		return 21
 	}
 
 	// Pay-to-script-hash script.
-	if valid, _ := isScriptHash(pkScript); valid {
+	if valid, _ := isScriptHash(scriptPubKey); valid {
 		return 21
 	}
 
 	// Pay-to-pubkey (compressed or uncompressed) script.
-	if valid, _ := isPubKey(pkScript); valid {
+	if valid, _ := isPubKey(scriptPubKey); valid {
 		return 33
 	}
 
 	// When none of the above special cases apply, encode the script as is
 	// preceded by the sum of its size and the number of special cases
 	// encoded as a variable length quantity.
-	return serializeSizeVLQ(uint64(len(pkScript)+numSpecialScripts)) +
-		len(pkScript)
+	return serializeSizeVLQ(uint64(len(scriptPubKey)+numSpecialScripts)) +
+		len(scriptPubKey)
 }
 
 // decodeCompressedScriptSize treats the passed serialized bytes as a compressed
@@ -296,23 +296,23 @@ func decodeCompressedScriptSize(serialized []byte) int {
 // target byte slice.  The target byte slice must be at least large enough to
 // handle the number of bytes returned by the compressedScriptSize function or
 // it will panic.
-func putCompressedScript(target, pkScript []byte) int {
+func putCompressedScript(target, scriptPubKey []byte) int {
 	// Pay-to-pubkey-hash script.
-	if valid, hash := isPubKeyHash(pkScript); valid {
+	if valid, hash := isPubKeyHash(scriptPubKey); valid {
 		target[0] = cstPayToPubKeyHash
 		copy(target[1:21], hash)
 		return 21
 	}
 
 	// Pay-to-script-hash script.
-	if valid, hash := isScriptHash(pkScript); valid {
+	if valid, hash := isScriptHash(scriptPubKey); valid {
 		target[0] = cstPayToScriptHash
 		copy(target[1:21], hash)
 		return 21
 	}
 
 	// Pay-to-pubkey (compressed or uncompressed) script.
-	if valid, serializedPubKey := isPubKey(pkScript); valid {
+	if valid, serializedPubKey := isPubKey(scriptPubKey); valid {
 		pubKeyFormat := serializedPubKey[0]
 		switch pubKeyFormat {
 		case 0x02, 0x03:
@@ -331,10 +331,10 @@ func putCompressedScript(target, pkScript []byte) int {
 	// When none of the above special cases apply, encode the unmodified
 	// script preceded by the sum of its size and the number of special
 	// cases encoded as a variable length quantity.
-	encodedSize := uint64(len(pkScript) + numSpecialScripts)
+	encodedSize := uint64(len(scriptPubKey) + numSpecialScripts)
 	vlqSizeLen := putVLQ(target, encodedSize)
-	copy(target[vlqSizeLen:], pkScript)
-	return vlqSizeLen + len(pkScript)
+	copy(target[vlqSizeLen:], scriptPubKey)
+	return vlqSizeLen + len(scriptPubKey)
 }
 
 // decompressScript returns the original script obtained by decompressing the
@@ -344,50 +344,50 @@ func putCompressedScript(target, pkScript []byte) int {
 // NOTE: The script parameter must already have been proven to be long enough
 // to contain the number of bytes returned by decodeCompressedScriptSize or it
 // will panic.  This is acceptable since it is only an internal function.
-func decompressScript(compressedPkScript []byte) []byte {
+func decompressScript(compressedScriptPubKey []byte) []byte {
 	// In practice this function will not be called with a zero-length or
 	// nil script since the nil script encoding includes the length, however
 	// the code below assumes the length exists, so just return nil now if
 	// the function ever ends up being called with a nil script in the
 	// future.
-	if len(compressedPkScript) == 0 {
+	if len(compressedScriptPubKey) == 0 {
 		return nil
 	}
 
 	// Decode the script size and examine it for the special cases.
-	encodedScriptSize, bytesRead := deserializeVLQ(compressedPkScript)
+	encodedScriptSize, bytesRead := deserializeVLQ(compressedScriptPubKey)
 	switch encodedScriptSize {
 	// Pay-to-pubkey-hash script.  The resulting script is:
 	// <OP_DUP><OP_HASH160><20 byte hash><OP_EQUALVERIFY><OP_CHECKSIG>
 	case cstPayToPubKeyHash:
-		pkScript := make([]byte, 25)
-		pkScript[0] = txscript.OpDup
-		pkScript[1] = txscript.OpHash160
-		pkScript[2] = txscript.OpData20
-		copy(pkScript[3:], compressedPkScript[bytesRead:bytesRead+20])
-		pkScript[23] = txscript.OpEqualVerify
-		pkScript[24] = txscript.OpCheckSig
-		return pkScript
+		scriptPubKey := make([]byte, 25)
+		scriptPubKey[0] = txscript.OpDup
+		scriptPubKey[1] = txscript.OpHash160
+		scriptPubKey[2] = txscript.OpData20
+		copy(scriptPubKey[3:], compressedScriptPubKey[bytesRead:bytesRead+20])
+		scriptPubKey[23] = txscript.OpEqualVerify
+		scriptPubKey[24] = txscript.OpCheckSig
+		return scriptPubKey
 
 	// Pay-to-script-hash script.  The resulting script is:
 	// <OP_HASH160><20 byte script hash><OP_EQUAL>
 	case cstPayToScriptHash:
-		pkScript := make([]byte, 23)
-		pkScript[0] = txscript.OpHash160
-		pkScript[1] = txscript.OpData20
-		copy(pkScript[2:], compressedPkScript[bytesRead:bytesRead+20])
-		pkScript[22] = txscript.OpEqual
-		return pkScript
+		scriptPubKey := make([]byte, 23)
+		scriptPubKey[0] = txscript.OpHash160
+		scriptPubKey[1] = txscript.OpData20
+		copy(scriptPubKey[2:], compressedScriptPubKey[bytesRead:bytesRead+20])
+		scriptPubKey[22] = txscript.OpEqual
+		return scriptPubKey
 
 	// Pay-to-compressed-pubkey script.  The resulting script is:
 	// <OP_DATA_33><33 byte compressed pubkey><OP_CHECKSIG>
 	case cstPayToPubKeyComp2, cstPayToPubKeyComp3:
-		pkScript := make([]byte, 35)
-		pkScript[0] = txscript.OpData33
-		pkScript[1] = byte(encodedScriptSize)
-		copy(pkScript[2:], compressedPkScript[bytesRead:bytesRead+32])
-		pkScript[34] = txscript.OpCheckSig
-		return pkScript
+		scriptPubKey := make([]byte, 35)
+		scriptPubKey[0] = txscript.OpData33
+		scriptPubKey[1] = byte(encodedScriptSize)
+		copy(scriptPubKey[2:], compressedScriptPubKey[bytesRead:bytesRead+32])
+		scriptPubKey[34] = txscript.OpCheckSig
+		return scriptPubKey
 
 	// Pay-to-uncompressed-pubkey script.  The resulting script is:
 	// <OP_DATA_65><65 byte uncompressed pubkey><OP_CHECKSIG>
@@ -398,26 +398,26 @@ func decompressScript(compressedPkScript []byte) []byte {
 		// encoding ensures it is valid before compressing to this type.
 		compressedKey := make([]byte, 33)
 		compressedKey[0] = byte(encodedScriptSize - 2)
-		copy(compressedKey[1:], compressedPkScript[1:])
+		copy(compressedKey[1:], compressedScriptPubKey[1:])
 		key, err := btcec.ParsePubKey(compressedKey, btcec.S256())
 		if err != nil {
 			return nil
 		}
 
-		pkScript := make([]byte, 67)
-		pkScript[0] = txscript.OpData65
-		copy(pkScript[1:], key.SerializeUncompressed())
-		pkScript[66] = txscript.OpCheckSig
-		return pkScript
+		scriptPubKey := make([]byte, 67)
+		scriptPubKey[0] = txscript.OpData65
+		copy(scriptPubKey[1:], key.SerializeUncompressed())
+		scriptPubKey[66] = txscript.OpCheckSig
+		return scriptPubKey
 	}
 
 	// When none of the special cases apply, the script was encoded using
 	// the general format, so reduce the script size by the number of
 	// special cases and return the unmodified script.
 	scriptSize := int(encodedScriptSize - numSpecialScripts)
-	pkScript := make([]byte, scriptSize)
-	copy(pkScript, compressedPkScript[bytesRead:bytesRead+scriptSize])
-	return pkScript
+	scriptPubKey := make([]byte, scriptSize)
+	copy(scriptPubKey, compressedScriptPubKey[bytesRead:bytesRead+scriptSize])
+	return scriptPubKey
 }
 
 // -----------------------------------------------------------------------------
@@ -543,9 +543,9 @@ func decompressTxOutAmount(amount uint64) uint64 {
 
 // compressedTxOutSize returns the number of bytes the passed transaction output
 // fields would take when encoded with the format described above.
-func compressedTxOutSize(amount uint64, pkScript []byte) int {
+func compressedTxOutSize(amount uint64, scriptPubKey []byte) int {
 	return serializeSizeVLQ(compressTxOutAmount(amount)) +
-		compressedScriptSize(pkScript)
+		compressedScriptSize(scriptPubKey)
 }
 
 // putCompressedTxOut compresses the passed amount and script according to their
@@ -553,9 +553,9 @@ func compressedTxOutSize(amount uint64, pkScript []byte) int {
 // passed target byte slice with the format described above.  The target byte
 // slice must be at least large enough to handle the number of bytes returned by
 // the compressedTxOutSize function or it will panic.
-func putCompressedTxOut(target []byte, amount uint64, pkScript []byte) int {
+func putCompressedTxOut(target []byte, amount uint64, scriptPubKey []byte) int {
 	offset := putVLQ(target, compressTxOutAmount(amount))
-	offset += putCompressedScript(target[offset:], pkScript)
+	offset += putCompressedScript(target[offset:], scriptPubKey)
 	return offset
 }
 

blockdag/compress_test.go

@@ -162,11 +162,6 @@ func TestScriptCompression(t *testing.T) {
 			uncompressed: hexToBytes("3302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
 			compressed:   hexToBytes("293302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
 		},
-		{
-			name:         "null data",
-			uncompressed: hexToBytes("6a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
-			compressed:   hexToBytes("286a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
-		},
 		{
 			name:         "requires 2 size bytes - data push 200 bytes",
 			uncompressed: append(hexToBytes("4cc8"), bytes.Repeat([]byte{0x00}, 200)...),
@@ -265,7 +260,7 @@ func TestAmountCompression(t *testing.T) {
 		compressed   uint64
 	}{
 		{
-			name:         "0 BTC (sometimes used in nulldata)",
+			name:         "0 BTC",
 			uncompressed: 0,
 			compressed:   0,
 		},
@@ -338,35 +333,29 @@ func TestCompressedTxOut(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
-		name       string
-		amount     uint64
-		pkScript   []byte
-		compressed []byte
+		name         string
+		amount       uint64
+		scriptPubKey []byte
+		compressed   []byte
 	}{
 		{
-			name:       "nulldata with 0 BTC",
-			amount:     0,
-			pkScript:   hexToBytes("6a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
-			compressed: hexToBytes("00286a200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"),
+			name:         "pay-to-pubkey-hash dust",
+			amount:       546,
+			scriptPubKey: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
+			compressed:   hexToBytes("a52f001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
 		},
 		{
-			name:       "pay-to-pubkey-hash dust",
-			amount:     546,
-			pkScript:   hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
-			compressed: hexToBytes("a52f001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
-		},
-		{
-			name:       "pay-to-pubkey uncompressed 1 BTC",
-			amount:     100000000,
-			pkScript:   hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
-			compressed: hexToBytes("0904192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
+			name:         "pay-to-pubkey uncompressed 1 BTC",
+			amount:       100000000,
+			scriptPubKey: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
+			compressed:   hexToBytes("0904192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
 		},
 	}
 
 	for _, test := range tests {
 		// Ensure the function to calculate the serialized size without
 		// actually serializing the txout is calculated properly.
-		gotSize := compressedTxOutSize(test.amount, test.pkScript)
+		gotSize := compressedTxOutSize(test.amount, test.scriptPubKey)
 		if gotSize != len(test.compressed) {
 			t.Errorf("compressedTxOutSize (%s): did not get "+
 				"expected size - got %d, want %d", test.name,
@@ -377,7 +366,7 @@ func TestCompressedTxOut(t *testing.T) {
 		// Ensure the txout compresses to the expected value.
 		gotCompressed := make([]byte, gotSize)
 		gotBytesWritten := putCompressedTxOut(gotCompressed,
-			test.amount, test.pkScript)
+			test.amount, test.scriptPubKey)
 		if !bytes.Equal(gotCompressed, test.compressed) {
 			t.Errorf("compressTxOut (%s): did not get expected "+
 				"bytes - got %x, want %x", test.name,
@@ -407,10 +396,10 @@ func TestCompressedTxOut(t *testing.T) {
 				test.name, gotAmount, test.amount)
 			continue
 		}
-		if !bytes.Equal(gotScript, test.pkScript) {
+		if !bytes.Equal(gotScript, test.scriptPubKey) {
 			t.Errorf("decodeCompressedTxOut (%s): did not get "+
 				"expected script - got %x, want %x",
-				test.name, gotScript, test.pkScript)
+				test.name, gotScript, test.scriptPubKey)
 			continue
 		}
 		if gotBytesRead != len(test.compressed) {

blockdag/dagio.go

@@ -9,13 +9,16 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"github.com/pkg/errors"
+	"io"
 	"sync"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/util/subnetworkid"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/binaryserializer"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 const (
@@ -34,14 +37,6 @@ var (
 	// block headers and contextual information.
 	blockIndexBucketName = []byte("blockheaderidx")
 
-	// hashIndexBucketName is the name of the db bucket used to house to the
-	// block hash -> block height index.
-	hashIndexBucketName = []byte("hashidx")
-
-	// heightIndexBucketName is the name of the db bucket used to house to
-	// the block height -> block hash index.
-	heightIndexBucketName = []byte("heightidx")
-
 	// dagStateKeyName is the name of the db key used to store the DAG
 	// tip hashes.
 	dagStateKeyName = []byte("dagstate")
@@ -54,6 +49,10 @@ var (
 	// unspent transaction output set.
 	utxoSetBucketName = []byte("utxoset")
 
+	// utxoDiffsBucketName is the name of the db bucket used to house the
+	// diffs and diff children of blocks.
+	utxoDiffsBucketName = []byte("utxodiffs")
+
 	// subnetworksBucketName is the name of the db bucket used to store the
 	// subnetwork registry.
 	subnetworksBucketName = []byte("subnetworks")
@@ -137,7 +136,7 @@ func dbPutVersion(dbTx database.Tx, key []byte, version uint32) error {
 //     compressed script  []byte   variable
 //
 // The serialized header code format is:
-//   bit 0 - containing transaction is a block reward
+//   bit 0 - containing transaction is a coinbase
 //   bits 1-x - height of the block that contains the unspent txout
 //
 // Example 1:
@@ -205,7 +204,7 @@ var outpointKeyPool = sync.Pool{
 // returned to the free list by using the recycleOutpointKey function when the
 // caller is done with it _unless_ the slice will need to live for longer than
 // the caller can calculate such as when used to write to the database.
-func outpointKey(outpoint wire.OutPoint) *[]byte {
+func outpointKey(outpoint wire.Outpoint) *[]byte {
 	// A VLQ employs an MSB encoding, so they are useful not only to reduce
 	// the amount of storage space, but also so iteration of UTXOs when
 	// doing byte-wise comparisons will produce them in order.
@@ -223,101 +222,14 @@ func recycleOutpointKey(key *[]byte) {
 	outpointKeyPool.Put(key)
 }
 
-// utxoEntryHeaderCode returns the calculated header code to be used when
-// serializing the provided utxo entry.
-func utxoEntryHeaderCode(entry *UTXOEntry) uint64 {
-
-	// As described in the serialization format comments, the header code
-	// encodes the height shifted over one bit and the block reward flag in the
-	// lowest bit.
-	headerCode := uint64(entry.BlockHeight()) << 1
-	if entry.IsBlockReward() {
-		headerCode |= 0x01
-	}
-
-	return headerCode
-}
-
-// serializeUTXOEntry returns the entry serialized to a format that is suitable
-// for long-term storage.  The format is described in detail above.
-func serializeUTXOEntry(entry *UTXOEntry) ([]byte, error) {
-
-	// Encode the header code.
-	headerCode := utxoEntryHeaderCode(entry)
-
-	// Calculate the size needed to serialize the entry.
-	size := serializeSizeVLQ(headerCode) +
-		compressedTxOutSize(uint64(entry.Amount()), entry.PkScript())
-
-	// Serialize the header code followed by the compressed unspent
-	// transaction output.
-	serialized := make([]byte, size)
-	offset := putVLQ(serialized, headerCode)
-	offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()),
-		entry.PkScript())
-
-	return serialized, nil
-}
-
-// deserializeOutPoint decodes an outPoint from the passed serialized byte
-// slice into a new wire.OutPoint using a format that is suitable for long-
-// term storage. this format is described in detail above.
-func deserializeOutPoint(serialized []byte) (*wire.OutPoint, error) {
-	if len(serialized) <= daghash.HashSize {
-		return nil, errDeserialize("unexpected end of data")
-	}
-
-	txID := daghash.TxID{}
-	txID.SetBytes(serialized[:daghash.HashSize])
-	index, _ := deserializeVLQ(serialized[daghash.HashSize:])
-	return wire.NewOutPoint(&txID, uint32(index)), nil
-}
-
-// deserializeUTXOEntry decodes a UTXO entry from the passed serialized byte
-// slice into a new UTXOEntry using a format that is suitable for long-term
-// storage.  The format is described in detail above.
-func deserializeUTXOEntry(serialized []byte) (*UTXOEntry, error) {
-	// Deserialize the header code.
-	code, offset := deserializeVLQ(serialized)
-	if offset >= len(serialized) {
-		return nil, errDeserialize("unexpected end of data after header")
-	}
-
-	// Decode the header code.
-	//
-	// Bit 0 indicates whether the containing transaction is a block reward.
-	// Bits 1-x encode height of containing transaction.
-	isBlockReward := code&0x01 != 0
-	blockHeight := int32(code >> 1)
-
-	// Decode the compressed unspent transaction output.
-	amount, pkScript, _, err := decodeCompressedTxOut(serialized[offset:])
-	if err != nil {
-		return nil, errDeserialize(fmt.Sprintf("unable to decode "+
-			"UTXO: %s", err))
-	}
-
-	entry := &UTXOEntry{
-		amount:      amount,
-		pkScript:    pkScript,
-		blockHeight: blockHeight,
-		packedFlags: 0,
-	}
-	if isBlockReward {
-		entry.packedFlags |= tfBlockReward
-	}
-
-	return entry, nil
-}
-
 // dbPutUTXODiff uses an existing database transaction to update the UTXO set
 // in the database based on the provided UTXO view contents and state.  In
 // particular, only the entries that have been marked as modified are written
 // to the database.
 func dbPutUTXODiff(dbTx database.Tx, diff *UTXODiff) error {
 	utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName)
-	for outPoint := range diff.toRemove {
-		key := outpointKey(outPoint)
+	for outpoint := range diff.toRemove {
+		key := outpointKey(outpoint)
 		err := utxoBucket.Delete(*key)
 		recycleOutpointKey(key)
 		if err != nil {
@@ -325,15 +237,12 @@ func dbPutUTXODiff(dbTx database.Tx, diff *UTXODiff) error {
 		}
 	}
 
-	for outPoint, entry := range diff.toAdd {
+	for outpoint, entry := range diff.toAdd {
 		// Serialize and store the UTXO entry.
-		serialized, err := serializeUTXOEntry(entry)
-		if err != nil {
-			return err
-		}
+		serialized := serializeUTXOEntry(entry)
 
-		key := outpointKey(outPoint)
-		err = utxoBucket.Put(*key, serialized)
+		key := outpointKey(outpoint)
+		err := utxoBucket.Put(*key, serialized)
 		// NOTE: The key is intentionally not recycled here since the
 		// database interface contract prohibits modifications.  It will
 		// be garbage collected normally when the database is done with
@@ -346,58 +255,6 @@ func dbPutUTXODiff(dbTx database.Tx, diff *UTXODiff) error {
 	return nil
 }
 
-// -----------------------------------------------------------------------------
-// The block index consists of two buckets with an entry for every block in the
-// main chain.  One bucket is for the hash to height mapping and the other is
-// for the height to hash mapping.
-//
-// The serialized format for values in the hash to height bucket is:
-//   <height>
-//
-//   Field      Type     Size
-//   height     uint32   4 bytes
-//
-// The serialized format for values in the height to hash bucket is:
-//   <hash>
-//
-//   Field      Type             Size
-//   hash       daghash.Hash   daghash.HashSize
-// -----------------------------------------------------------------------------
-
-// dbPutBlockIndex uses an existing database transaction to update or add the
-// block index entries for the hash to height and height to hash mappings for
-// the provided values.
-func dbPutBlockIndex(dbTx database.Tx, hash *daghash.Hash, height int32) error {
-	// Serialize the height for use in the index entries.
-	var serializedHeight [4]byte
-	byteOrder.PutUint32(serializedHeight[:], uint32(height))
-
-	// Add the block hash to height mapping to the index.
-	meta := dbTx.Metadata()
-	hashIndex := meta.Bucket(hashIndexBucketName)
-	if err := hashIndex.Put(hash[:], serializedHeight[:]); err != nil {
-		return err
-	}
-
-	// Add the block height to hash mapping to the index.
-	heightIndex := meta.Bucket(heightIndexBucketName)
-	return heightIndex.Put(serializedHeight[:], hash[:])
-}
-
-// dbFetchHeightByHash uses an existing database transaction to retrieve the
-// height for the provided hash from the index.
-func dbFetchHeightByHash(dbTx database.Tx, hash *daghash.Hash) (int32, error) {
-	meta := dbTx.Metadata()
-	hashIndex := meta.Bucket(hashIndexBucketName)
-	serializedHeight := hashIndex.Get(hash[:])
-	if serializedHeight == nil {
-		str := fmt.Sprintf("block %s is not in the main chain", hash)
-		return 0, errNotInDAG(str)
-	}
-
-	return int32(byteOrder.Uint32(serializedHeight)), nil
-}
-
 type dagState struct {
 	TipHashes         []*daghash.Hash
 	LastFinalityPoint *daghash.Hash
@@ -452,28 +309,18 @@ func (dag *BlockDAG) createDAGState() error {
 			return err
 		}
 
-		// Create the bucket that houses the chain block hash to height
-		// index.
-		_, err = meta.CreateBucket(hashIndexBucketName)
-		if err != nil {
-			return err
-		}
-
-		// Create the bucket that houses the chain block height to hash
-		// index.
-		_, err = meta.CreateBucket(heightIndexBucketName)
-		if err != nil {
-			return err
-		}
-
-		// Create the bucket that houses the utxo set and store its
-		// version.  Note that the genesis block coinbase transaction is
-		// intentionally not inserted here since it is not spendable by
-		// consensus rules.
+		// Create the buckets that house the utxo set, the utxo diffs, and their
+		// version.
 		_, err = meta.CreateBucket(utxoSetBucketName)
 		if err != nil {
 			return err
 		}
+
+		_, err = meta.CreateBucket(utxoDiffsBucketName)
+		if err != nil {
+			return err
+		}
+
 		err = dbPutVersion(dbTx, utxoSetVersionKeyName,
 			latestUTXOSetBucketVersion)
 		if err != nil {
@@ -489,6 +336,56 @@ func (dag *BlockDAG) createDAGState() error {
 		if err := dbPutLocalSubnetworkID(dbTx, dag.subnetworkID); err != nil {
 			return err
 		}
+
+		if _, err := meta.CreateBucketIfNotExists(idByHashIndexBucketName); err != nil {
+			return err
+		}
+		if _, err := meta.CreateBucketIfNotExists(hashByIDIndexBucketName); err != nil {
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (dag *BlockDAG) removeDAGState() error {
+	err := dag.db.Update(func(dbTx database.Tx) error {
+		meta := dbTx.Metadata()
+
+		err := meta.DeleteBucket(blockIndexBucketName)
+		if err != nil {
+			return err
+		}
+
+		err = meta.DeleteBucket(utxoSetBucketName)
+		if err != nil {
+			return err
+		}
+
+		err = meta.DeleteBucket(utxoDiffsBucketName)
+		if err != nil {
+			return err
+		}
+
+		err = dbTx.Metadata().Delete(utxoSetVersionKeyName)
+		if err != nil {
+			return err
+		}
+
+		err = meta.DeleteBucket(subnetworksBucketName)
+		if err != nil {
+			return err
+		}
+
+		err = dbTx.Metadata().Delete(localSubnetworkKeyName)
+		if err != nil {
+			return err
+		}
+
 		return nil
 	})
 
@@ -522,7 +419,7 @@ func (dag *BlockDAG) initDAGState() error {
 				localSubnetworkID.SetBytes(localSubnetworkIDBytes)
 			}
 			if !localSubnetworkID.IsEqual(dag.subnetworkID) {
-				return fmt.Errorf("Cannot start btcd with subnetwork ID %s because"+
+				return errors.Errorf("Cannot start btcd with subnetwork ID %s because"+
 					" its database is already built with subnetwork ID %s. If you"+
 					" want to switch to a new database, please reset the"+
 					" database by starting btcd with --reset-db flag", dag.subnetworkID, localSubnetworkID)
@@ -562,55 +459,42 @@ func (dag *BlockDAG) initDAGState() error {
 
 		blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName)
 
-		// Determine how many blocks will be loaded into the index so we can
-		// allocate the right amount.
-		var blockCount int32
-		cursor := blockIndexBucket.Cursor()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			blockCount++
-		}
-		blockNodes := make([]blockNode, blockCount)
-
 		var i int32
 		var lastNode *blockNode
-		cursor = blockIndexBucket.Cursor()
+		var unprocessedBlockNodes []*blockNode
+		cursor := blockIndexBucket.Cursor()
 		for ok := cursor.First(); ok; ok = cursor.Next() {
-			header, status, err := deserializeBlockRow(cursor.Value())
+			node, err := dag.deserializeBlockNode(cursor.Value())
 			if err != nil {
 				return err
 			}
 
-			parents := newSet()
+			// Check to see if this node had been stored in the the block DB
+			// but not yet accepted. If so, add it to a slice to be processed later.
+			if node.status == statusDataStored {
+				unprocessedBlockNodes = append(unprocessedBlockNodes, node)
+				continue
+			}
+
 			if lastNode == nil {
-				blockHash := header.BlockHash()
-				if !blockHash.IsEqual(dag.dagParams.GenesisHash) {
+				if !node.hash.IsEqual(dag.dagParams.GenesisHash) {
 					return AssertError(fmt.Sprintf("initDAGState: Expected "+
 						"first entry in block index to be genesis block, "+
-						"found %s", blockHash))
+						"found %s", node.hash))
 				}
 			} else {
-				for _, hash := range header.ParentHashes {
-					parent := dag.index.LookupNode(hash)
-					if parent == nil {
-						return AssertError(fmt.Sprintf("initDAGState: Could "+
-							"not find parent %s for block %s", hash, header.BlockHash()))
-					}
-					parents.add(parent)
-				}
-				if len(parents) == 0 {
+				if len(node.parents) == 0 {
 					return AssertError(fmt.Sprintf("initDAGState: Could "+
-						"not find any parent for block %s", header.BlockHash()))
+						"not find any parent for block %s", node.hash))
 				}
 			}
 
-			// Initialize the block node for the block, connect it,
+			// Add the node to its parents children, connect it,
 			// and add it to the block index.
-			node := &blockNodes[i]
-			initBlockNode(node, header, parents, dag.dagParams.K)
-			node.status = status
+			node.updateParentsChildren()
 			dag.index.addNode(node)
 
-			if blockStatus(status).KnownValid() {
+			if node.status.KnownValid() {
 				dag.blockCount++
 			}
 
@@ -636,15 +520,15 @@ func (dag *BlockDAG) initDAGState() error {
 
 		fullUTXOCollection := make(utxoCollection, utxoEntryCount)
 		for ok := cursor.First(); ok; ok = cursor.Next() {
-			// Deserialize the outPoint
-			outPoint, err := deserializeOutPoint(cursor.Key())
+			// Deserialize the outpoint
+			outpoint, err := deserializeOutpoint(cursor.Key())
 			if err != nil {
 				// Ensure any deserialization errors are returned as database
 				// corruption errors.
 				if isDeserializeErr(err) {
 					return database.Error{
 						ErrorCode:   database.ErrCorruption,
-						Description: fmt.Sprintf("corrupt outPoint: %s", err),
+						Description: fmt.Sprintf("corrupt outpoint: %s", err),
 					}
 				}
 
@@ -666,11 +550,14 @@ func (dag *BlockDAG) initDAGState() error {
 				return err
 			}
 
-			fullUTXOCollection[*outPoint] = entry
+			fullUTXOCollection[*outpoint] = entry
 		}
 
 		// Apply the loaded utxoCollection to the virtual block.
-		dag.virtual.utxoSet.utxoCollection = fullUTXOCollection
+		dag.virtual.utxoSet, err = newFullUTXOSetFromUTXOCollection(fullUTXOCollection)
+		if err != nil {
+			return AssertError(fmt.Sprintf("Error loading UTXOSet: %s", err))
+		}
 
 		// Apply the stored tips to the virtual block.
 		tips := newSet()
@@ -686,33 +573,125 @@ func (dag *BlockDAG) initDAGState() error {
 
 		// Set the last finality point
 		dag.lastFinalityPoint = dag.index.LookupNode(state.LastFinalityPoint)
+		dag.finalizeNodesBelowFinalityPoint(false)
+
+		// Go over any unprocessed blockNodes and process them now.
+		for _, node := range unprocessedBlockNodes {
+			// Check to see if the block exists in the block DB. If it
+			// doesn't, the database has certainly been corrupted.
+			blockExists, err := dbTx.HasBlock(node.hash)
+			if err != nil {
+				return AssertError(fmt.Sprintf("initDAGState: HasBlock "+
+					"for block %s failed: %s", node.hash, err))
+			}
+			if !blockExists {
+				return AssertError(fmt.Sprintf("initDAGState: block %s "+
+					"exists in block index but not in block db", node.hash))
+			}
+
+			// Attempt to accept the block.
+			block, err := dbFetchBlockByNode(dbTx, node)
+			isOrphan, delay, err := dag.ProcessBlock(block, BFWasStored)
+			if err != nil {
+				log.Warnf("Block %s, which was not previously processed, "+
+					"failed to be accepted to the DAG: %s", node.hash, err)
+				continue
+			}
+
+			// If the block is an orphan or is delayed then it couldn't have
+			// possibly been written to the block index in the first place.
+			if isOrphan {
+				return AssertError(fmt.Sprintf("Block %s, which was not "+
+					"previously processed, turned out to be an orphan, which is "+
+					"impossible.", node.hash))
+			}
+			if delay != 0 {
+				return AssertError(fmt.Sprintf("Block %s, which was not "+
+					"previously processed, turned out to be delayed, which is "+
+					"impossible.", node.hash))
+			}
+		}
 
 		return nil
 	})
 }
 
-// deserializeBlockRow parses a value in the block index bucket into a block
-// header and block status bitfield.
-func deserializeBlockRow(blockRow []byte) (*wire.BlockHeader, blockStatus, error) {
+// deserializeBlockNode parses a value in the block index bucket and returns a block node.
+func (dag *BlockDAG) deserializeBlockNode(blockRow []byte) (*blockNode, error) {
 	buffer := bytes.NewReader(blockRow)
 
 	var header wire.BlockHeader
 	err := header.Deserialize(buffer)
 	if err != nil {
-		return nil, statusNone, err
+		return nil, err
+	}
+
+	node := &blockNode{
+		hash:                 header.BlockHash(),
+		version:              header.Version,
+		bits:                 header.Bits,
+		nonce:                header.Nonce,
+		timestamp:            header.Timestamp.Unix(),
+		hashMerkleRoot:       header.HashMerkleRoot,
+		acceptedIDMerkleRoot: header.AcceptedIDMerkleRoot,
+		utxoCommitment:       header.UTXOCommitment,
+	}
+
+	node.children = newSet()
+	node.parents = newSet()
+
+	for _, hash := range header.ParentHashes {
+		parent := dag.index.LookupNode(hash)
+		if parent == nil {
+			return nil, AssertError(fmt.Sprintf("deserializeBlockNode: Could "+
+				"not find parent %s for block %s", hash, header.BlockHash()))
+		}
+		node.parents.add(parent)
 	}
 
 	statusByte, err := buffer.ReadByte()
 	if err != nil {
-		return nil, statusNone, err
+		return nil, err
+	}
+	node.status = blockStatus(statusByte)
+
+	selectedParentHash := &daghash.Hash{}
+	if _, err := io.ReadFull(buffer, selectedParentHash[:]); err != nil {
+		return nil, err
 	}
 
-	return &header, blockStatus(statusByte), nil
+	// Because genesis doesn't have selected parent, it's serialized as zero hash
+	if !selectedParentHash.IsEqual(&daghash.ZeroHash) {
+		node.selectedParent = dag.index.LookupNode(selectedParentHash)
+	}
+
+	node.blueScore, err = binaryserializer.Uint64(buffer, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	bluesCount, err := wire.ReadVarInt(buffer)
+	if err != nil {
+		return nil, err
+	}
+
+	node.blues = make([]*blockNode, bluesCount)
+	for i := uint64(0); i < bluesCount; i++ {
+		hash := &daghash.Hash{}
+		if _, err := io.ReadFull(buffer, hash[:]); err != nil {
+			return nil, err
+		}
+		node.blues[i] = dag.index.LookupNode(hash)
+	}
+
+	node.chainHeight = calculateChainHeight(node)
+
+	return node, nil
 }
 
 // dbFetchBlockByNode uses an existing database transaction to retrieve the
 // raw block for the provided node, deserialize it, and return a util.Block
-// with the height set.
+// of it.
 func dbFetchBlockByNode(dbTx database.Tx, node *blockNode) (*util.Block, error) {
 	// Load the raw block bytes from the database.
 	blockBytes, err := dbTx.FetchBlock(node.hash)
@@ -720,17 +699,15 @@ func dbFetchBlockByNode(dbTx database.Tx, node *blockNode) (*util.Block, error)
 		return nil, err
 	}
 
-	// Create the encapsulated block and set the height appropriately.
+	// Create the encapsulated block.
 	block, err := util.NewBlockFromBytes(blockBytes)
 	if err != nil {
 		return nil, err
 	}
-	block.SetHeight(node.height)
-
 	return block, nil
 }
 
-// dbStoreBlockNode stores the block header and validation status to the block
+// dbStoreBlockNode stores the block node data into the block
 // index bucket. This overwrites the current entry if there exists one.
 func dbStoreBlockNode(dbTx database.Tx, node *blockNode) error {
 	// Serialize block data to be stored.
@@ -740,15 +717,44 @@ func dbStoreBlockNode(dbTx database.Tx, node *blockNode) error {
 	if err != nil {
 		return err
 	}
+
 	err = w.WriteByte(byte(node.status))
 	if err != nil {
 		return err
 	}
+
+	// Because genesis doesn't have selected parent, it's serialized as zero hash
+	selectedParentHash := &daghash.ZeroHash
+	if node.selectedParent != nil {
+		selectedParentHash = node.selectedParent.hash
+	}
+	_, err = w.Write(selectedParentHash[:])
+	if err != nil {
+		return err
+	}
+
+	err = binaryserializer.PutUint64(w, byteOrder, node.blueScore)
+	if err != nil {
+		return err
+	}
+
+	err = wire.WriteVarInt(w, uint64(len(node.blues)))
+	if err != nil {
+		return err
+	}
+
+	for _, blue := range node.blues {
+		_, err = w.Write(blue.hash[:])
+		if err != nil {
+			return err
+		}
+	}
+
 	value := w.Bytes()
 
 	// Write block header data to block index bucket.
 	blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName)
-	key := blockIndexKey(node.hash, uint32(node.height))
+	key := BlockIndexKey(node.hash, node.blueScore)
 	return blockIndexBucket.Put(key, value)
 }
 
@@ -765,23 +771,26 @@ func dbStoreBlock(dbTx database.Tx, block *util.Block) error {
 	return dbTx.StoreBlock(block)
 }
 
-// blockIndexKey generates the binary key for an entry in the block index
-// bucket. The key is composed of the block height encoded as a big-endian
-// 32-bit unsigned int followed by the 32 byte block hash.
-func blockIndexKey(blockHash *daghash.Hash, blockHeight uint32) []byte {
-	indexKey := make([]byte, daghash.HashSize+4)
-	binary.BigEndian.PutUint32(indexKey[0:4], blockHeight)
-	copy(indexKey[4:daghash.HashSize+4], blockHash[:])
+// BlockIndexKey generates the binary key for an entry in the block index
+// bucket. The key is composed of the block blue score encoded as a big-endian
+// 64-bit unsigned int followed by the 32 byte block hash.
+// The blue score component is important for iteration order.
+func BlockIndexKey(blockHash *daghash.Hash, blueScore uint64) []byte {
+	indexKey := make([]byte, daghash.HashSize+8)
+	binary.BigEndian.PutUint64(indexKey[0:8], blueScore)
+	copy(indexKey[8:daghash.HashSize+8], blockHash[:])
 	return indexKey
 }
 
-// BlockByHash returns the block from the main chain with the given hash with
-// the appropriate chain height set.
+func blockHashFromBlockIndexKey(BlockIndexKey []byte) (*daghash.Hash, error) {
+	return daghash.NewHash(BlockIndexKey[8 : daghash.HashSize+8])
+}
+
+// BlockByHash returns the block from the DAG with the given hash.
 //
 // This function is safe for concurrent access.
 func (dag *BlockDAG) BlockByHash(hash *daghash.Hash) (*util.Block, error) {
-	// Lookup the block hash in block index and ensure it is in the best
-	// chain.
+	// Lookup the block hash in block index and ensure it is in the DAG
 	node := dag.index.LookupNode(hash)
 	if node == nil {
 		str := fmt.Sprintf("block %s is not in the main chain", hash)
@@ -797,3 +806,49 @@ func (dag *BlockDAG) BlockByHash(hash *daghash.Hash) (*util.Block, error) {
 	})
 	return block, err
 }
+
+// BlockHashesFrom returns a slice of blocks starting from startHash
+// ordered by blueScore. If startHash is nil then the genesis block is used.
+//
+// This method MUST be called with the DAG lock held
+func (dag *BlockDAG) BlockHashesFrom(startHash *daghash.Hash, limit int) ([]*daghash.Hash, error) {
+	blockHashes := make([]*daghash.Hash, 0, limit)
+	if startHash == nil {
+		startHash = dag.genesis.hash
+
+		// If we're starting from the beginning we should include the
+		// genesis hash in the result
+		blockHashes = append(blockHashes, dag.genesis.hash)
+	}
+	if !dag.BlockExists(startHash) {
+		return nil, errors.Errorf("block %s not found", startHash)
+	}
+	blueScore, err := dag.BlueScoreByBlockHash(startHash)
+	if err != nil {
+		return nil, err
+	}
+
+	err = dag.index.db.View(func(dbTx database.Tx) error {
+		blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName)
+		startKey := BlockIndexKey(startHash, blueScore)
+
+		cursor := blockIndexBucket.Cursor()
+		cursor.Seek(startKey)
+		for ok := cursor.Next(); ok; ok = cursor.Next() {
+			key := cursor.Key()
+			blockHash, err := blockHashFromBlockIndexKey(key)
+			if err != nil {
+				return err
+			}
+			blockHashes = append(blockHashes, blockHash)
+			if len(blockHashes) == limit {
+				break
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return blockHashes, nil
+}

blockdag/dagio_test.go

@@ -6,12 +6,12 @@ package blockdag
 
 import (
 	"bytes"
-	"errors"
+	"github.com/pkg/errors"
 	"reflect"
 	"testing"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // TestErrNotInDAG ensures the functions related to errNotInDAG work
@@ -49,24 +49,24 @@ func TestUtxoSerialization(t *testing.T) {
 		// From tx in main blockchain:
 		// b7c3332bc138e2c9429818f5fed500bcc1746544218772389054dc8047d7cd3f:0
 		{
-			name: "height 1, coinbase",
+			name: "blue score 1, coinbase",
 			entry: &UTXOEntry{
-				amount:      5000000000,
-				pkScript:    hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
-				blockHeight: 1,
-				packedFlags: tfBlockReward,
+				amount:         5000000000,
+				scriptPubKey:   hexToBytes("410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
+				blockBlueScore: 1,
+				packedFlags:    tfCoinbase,
 			},
 			serialized: hexToBytes("03320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"),
 		},
 		// From tx in main blockchain:
 		// 8131ffb0a2c945ecaf9b9063e59558784f9c3a74741ce6ae2a18d0571dac15bb:1
 		{
-			name: "height 100001, not coinbase",
+			name: "blue score 100001, not coinbase",
 			entry: &UTXOEntry{
-				amount:      1000000,
-				pkScript:    hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
-				blockHeight: 100001,
-				packedFlags: 0,
+				amount:         1000000,
+				scriptPubKey:   hexToBytes("76a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
+				blockBlueScore: 100001,
+				packedFlags:    0,
 			},
 			serialized: hexToBytes("8b99420700ee8bd501094a7d5ca318da2506de35e1cb025ddc"),
 		},
@@ -74,12 +74,7 @@ func TestUtxoSerialization(t *testing.T) {
 
 	for i, test := range tests {
 		// Ensure the utxo entry serializes to the expected value.
-		gotBytes, err := serializeUTXOEntry(test.entry)
-		if err != nil {
-			t.Errorf("serializeUTXOEntry #%d (%s) unexpected "+
-				"error: %v", i, test.name, err)
-			continue
-		}
+		gotBytes := serializeUTXOEntry(test.entry)
 		if !bytes.Equal(gotBytes, test.serialized) {
 			t.Errorf("serializeUTXOEntry #%d (%s): mismatched "+
 				"bytes - got %x, want %x", i, test.name,
@@ -104,22 +99,22 @@ func TestUtxoSerialization(t *testing.T) {
 			continue
 		}
 
-		if !bytes.Equal(utxoEntry.PkScript(), test.entry.PkScript()) {
+		if !bytes.Equal(utxoEntry.ScriptPubKey(), test.entry.ScriptPubKey()) {
 			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
 				"scripts: got %x, want %x", i, test.name,
-				utxoEntry.PkScript(), test.entry.PkScript())
+				utxoEntry.ScriptPubKey(), test.entry.ScriptPubKey())
 			continue
 		}
-		if utxoEntry.BlockHeight() != test.entry.BlockHeight() {
+		if utxoEntry.BlockBlueScore() != test.entry.BlockBlueScore() {
 			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
-				"block height: got %d, want %d", i, test.name,
-				utxoEntry.BlockHeight(), test.entry.BlockHeight())
+				"block blue score: got %d, want %d", i, test.name,
+				utxoEntry.BlockBlueScore(), test.entry.BlockBlueScore())
 			continue
 		}
-		if utxoEntry.IsBlockReward() != test.entry.IsBlockReward() {
+		if utxoEntry.IsCoinbase() != test.entry.IsCoinbase() {
 			t.Errorf("deserializeUTXOEntry #%d (%s) mismatched "+
 				"coinbase flag: got %v, want %v", i, test.name,
-				utxoEntry.IsBlockReward(), test.entry.IsBlockReward())
+				utxoEntry.IsCoinbase(), test.entry.IsCoinbase())
 			continue
 		}
 	}

blockdag/difficulty.go

@@ -8,162 +8,45 @@ import (
 	"math/big"
 	"time"
 
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/util"
 )
 
-// calcEasiestDifficulty calculates the easiest possible difficulty that a block
-// can have given starting difficulty bits and a duration.  It is mainly used to
-// verify that claimed proof of work by a block is sane as compared to a
-// known good checkpoint.
-func (dag *BlockDAG) calcEasiestDifficulty(bits uint32, duration time.Duration) uint32 {
-	// Convert types used in the calculations below.
-	durationVal := int64(duration / time.Second)
-	adjustmentFactor := big.NewInt(dag.dagParams.RetargetAdjustmentFactor)
-
-	// The test network rules allow minimum difficulty blocks after more
-	// than twice the desired amount of time needed to generate a block has
-	// elapsed.
-	if dag.dagParams.ReduceMinDifficulty {
-		reductionTime := int64(dag.dagParams.MinDiffReductionTime /
-			time.Second)
-		if durationVal > reductionTime {
-			return dag.dagParams.PowLimitBits
-		}
-	}
-
-	// Since easier difficulty equates to higher numbers, the easiest
-	// difficulty for a given duration is the largest value possible given
-	// the number of retargets for the duration and starting difficulty
-	// multiplied by the max adjustment factor.
-	newTarget := util.CompactToBig(bits)
-	for durationVal > 0 && newTarget.Cmp(dag.dagParams.PowLimit) < 0 {
-		newTarget.Mul(newTarget, adjustmentFactor)
-		durationVal -= dag.maxRetargetTimespan
-	}
-
-	// Limit new value to the proof of work limit.
-	if newTarget.Cmp(dag.dagParams.PowLimit) > 0 {
-		newTarget.Set(dag.dagParams.PowLimit)
-	}
-
-	return util.BigToCompact(newTarget)
-}
-
-// findPrevTestNetDifficulty returns the difficulty of the previous block which
-// did not have the special testnet minimum difficulty rule applied.
-//
-// This function MUST be called with the chain state lock held (for writes).
-func (dag *BlockDAG) findPrevTestNetDifficulty(startNode *blockNode) uint32 {
-	// Search backwards through the chain for the last block without
-	// the special rule applied.
-	iterNode := startNode
-	for iterNode != nil && iterNode.height%dag.blocksPerRetarget != 0 &&
-		iterNode.bits == dag.dagParams.PowLimitBits {
-
-		iterNode = iterNode.selectedParent
-	}
-
-	// Return the found difficulty or the minimum difficulty if no
-	// appropriate block was found.
-	lastBits := dag.dagParams.PowLimitBits
-	if iterNode != nil {
-		lastBits = iterNode.bits
-	}
-	return lastBits
-}
-
-// calcNextRequiredDifficulty calculates the required difficulty for the block
-// after the passed previous block node based on the difficulty retarget rules.
-// This function differs from the exported CalcNextRequiredDifficulty in that
-// the exported version uses the current best chain as the previous block node
-// while this function accepts any block node.
-func (dag *BlockDAG) calcNextRequiredDifficulty(bluestParent *blockNode, newBlockTime time.Time) (uint32, error) {
+// requiredDifficulty calculates the required difficulty for a
+// block given its bluest parent.
+func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime time.Time) uint32 {
 	// Genesis block.
-	if bluestParent == nil {
-		return dag.dagParams.PowLimitBits, nil
+	if bluestParent == nil || bluestParent.blueScore < dag.difficultyAdjustmentWindowSize+1 {
+		return dag.powMaxBits
 	}
 
-	// Return the previous block's difficulty requirements if this block
-	// is not at a difficulty retarget interval.
-	if (bluestParent.height+1)%dag.blocksPerRetarget != 0 {
-		// For networks that support it, allow special reduction of the
-		// required difficulty once too much time has elapsed without
-		// mining a block.
-		if dag.dagParams.ReduceMinDifficulty {
-			// Return minimum difficulty when more than the desired
-			// amount of time has elapsed without mining a block.
-			reductionTime := int64(dag.dagParams.MinDiffReductionTime /
-				time.Second)
-			allowMinTime := bluestParent.timestamp + reductionTime
-			if newBlockTime.Unix() > allowMinTime {
-				return dag.dagParams.PowLimitBits, nil
-			}
+	// Fetch window of dag.difficultyAdjustmentWindowSize + 1 so we can have dag.difficultyAdjustmentWindowSize block intervals
+	timestampsWindow := blueBlockWindow(bluestParent, dag.difficultyAdjustmentWindowSize+1)
+	windowMinTimestamp, windowMaxTimeStamp := timestampsWindow.minMaxTimestamps()
 
-			// The block was mined within the desired timeframe, so
-			// return the difficulty for the last block which did
-			// not have the special minimum difficulty rule applied.
-			return dag.findPrevTestNetDifficulty(bluestParent), nil
-		}
-
-		// For the main network (or any unrecognized networks), simply
-		// return the previous block's difficulty requirements.
-		return bluestParent.bits, nil
-	}
-
-	// Get the block node at the previous retarget (targetTimespan days
-	// worth of blocks).
-	firstNode := bluestParent.RelativeAncestor(dag.blocksPerRetarget - 1)
-	if firstNode == nil {
-		return 0, AssertError("unable to obtain previous retarget block")
-	}
-
-	// Limit the amount of adjustment that can occur to the previous
-	// difficulty.
-	actualTimespan := bluestParent.timestamp - firstNode.timestamp
-	adjustedTimespan := actualTimespan
-	if actualTimespan < dag.minRetargetTimespan {
-		adjustedTimespan = dag.minRetargetTimespan
-	} else if actualTimespan > dag.maxRetargetTimespan {
-		adjustedTimespan = dag.maxRetargetTimespan
-	}
+	// Remove the last block from the window so to calculate the average target of dag.difficultyAdjustmentWindowSize blocks
+	targetsWindow := timestampsWindow[:dag.difficultyAdjustmentWindowSize]
 
 	// Calculate new target difficulty as:
-	//  currentDifficulty * (adjustedTimespan / targetTimespan)
+	// averageWindowTarget * (windowMinTimestamp / (targetTimePerBlock * windowSize))
 	// The result uses integer division which means it will be slightly
-	// rounded down.  Bitcoind also uses integer division to calculate this
-	// result.
-	oldTarget := util.CompactToBig(bluestParent.bits)
-	newTarget := new(big.Int).Mul(oldTarget, big.NewInt(adjustedTimespan))
-	targetTimeSpan := int64(dag.dagParams.TargetTimespan / time.Second)
-	newTarget.Div(newTarget, big.NewInt(targetTimeSpan))
-
-	// Limit new value to the proof of work limit.
-	if newTarget.Cmp(dag.dagParams.PowLimit) > 0 {
-		newTarget.Set(dag.dagParams.PowLimit)
+	// rounded down.
+	newTarget := targetsWindow.averageTarget()
+	newTarget.
+		Mul(newTarget, big.NewInt(windowMaxTimeStamp-windowMinTimestamp)).
+		Div(newTarget, big.NewInt(dag.targetTimePerBlock)).
+		Div(newTarget, big.NewInt(int64(dag.difficultyAdjustmentWindowSize)))
+	if newTarget.Cmp(dag.dagParams.PowMax) > 0 {
+		return dag.powMaxBits
 	}
-
-	// Log new target difficulty and return it.  The new target logging is
-	// intentionally converting the bits back to a number instead of using
-	// newTarget since conversion to the compact representation loses
-	// precision.
 	newTargetBits := util.BigToCompact(newTarget)
-	log.Debugf("Difficulty retarget at block height %d", bluestParent.height+1)
-	log.Debugf("Old target %08x (%064x)", bluestParent.bits, oldTarget)
-	log.Debugf("New target %08x (%064x)", newTargetBits, util.CompactToBig(newTargetBits))
-	log.Debugf("Actual timespan %s, adjusted timespan %s, target timespan %s",
-		time.Duration(actualTimespan)*time.Second,
-		time.Duration(adjustedTimespan)*time.Second,
-		dag.dagParams.TargetTimespan)
-
-	return newTargetBits, nil
+	return newTargetBits
 }
 
-// CalcNextRequiredDifficulty calculates the required difficulty for the block
-// after the end of the current best chain based on the difficulty retarget
-// rules.
+// NextRequiredDifficulty calculates the required difficulty for a block that will
+// be built on top of the current tips.
 //
 // This function is safe for concurrent access.
-func (dag *BlockDAG) CalcNextRequiredDifficulty(timestamp time.Time) (uint32, error) {
-	difficulty, err := dag.calcNextRequiredDifficulty(dag.selectedTip(), timestamp)
-	return difficulty, err
+func (dag *BlockDAG) NextRequiredDifficulty(timestamp time.Time) uint32 {
+	difficulty := dag.requiredDifficulty(dag.virtual.parents.bluest(), timestamp)
+	return difficulty
 }

blockdag/difficulty_test.go

@@ -5,10 +5,14 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 	"math/big"
 	"testing"
+	"time"
 
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/util"
 )
 
 // TestBigToCompact ensures BigToCompact converts big integers to the expected
@@ -75,3 +79,120 @@ func TestCalcWork(t *testing.T) {
 		}
 	}
 }
+
+func TestDifficulty(t *testing.T) {
+	params := dagconfig.SimNetParams
+	params.K = 1
+	dag := newTestDAG(&params)
+	nonce := uint64(0)
+	zeroTime := time.Unix(0, 0)
+	addNode := func(parents blockSet, blockTime time.Time) *blockNode {
+		bluestParent := parents.bluest()
+		if blockTime == zeroTime {
+			blockTime = time.Unix(bluestParent.timestamp+1, 0)
+		}
+		header := &wire.BlockHeader{
+			ParentHashes:         parents.hashes(),
+			Bits:                 dag.requiredDifficulty(bluestParent, blockTime),
+			Nonce:                nonce,
+			Timestamp:            blockTime,
+			HashMerkleRoot:       &daghash.ZeroHash,
+			AcceptedIDMerkleRoot: &daghash.ZeroHash,
+			UTXOCommitment:       &daghash.ZeroHash,
+		}
+		node := newBlockNode(header, parents, dag.dagParams.K)
+		node.updateParentsChildren()
+		nonce++
+		return node
+	}
+	tip := dag.genesis
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
+		tip = addNode(setFromSlice(tip), zeroTime)
+		if tip.bits != dag.genesis.bits {
+			t.Fatalf("As long as the bluest parent's blue score is less then the difficulty adjustment window size, the difficulty should be the same as genesis'")
+		}
+	}
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize+1000; i++ {
+		tip = addNode(setFromSlice(tip), zeroTime)
+		if tip.bits != dag.genesis.bits {
+			t.Fatalf("As long as the block rate remains the same, the difficulty shouldn't change")
+		}
+	}
+	nodeInThePast := addNode(setFromSlice(tip), tip.PastMedianTime(dag))
+	if nodeInThePast.bits != tip.bits {
+		t.Fatalf("The difficulty should only change when nodeInThePast is in the past of a block bluest parent")
+	}
+	tip = nodeInThePast
+
+	tip = addNode(setFromSlice(tip), zeroTime)
+	if tip.bits != nodeInThePast.bits {
+		t.Fatalf("The difficulty should only change when nodeInThePast is in the past of a block bluest parent")
+	}
+	tip = addNode(setFromSlice(tip), zeroTime)
+	if compareBits(tip.bits, nodeInThePast.bits) >= 0 {
+		t.Fatalf("tip.bits should be smaller than nodeInThePast.bits because nodeInThePast increased the block rate, so the difficulty should increase as well")
+	}
+	expectedBits := uint32(0x207ff395)
+	if tip.bits != expectedBits {
+		t.Errorf("tip.bits was expected to be %x but got %x", expectedBits, tip.bits)
+	}
+
+	// Increase block rate to increase difficulty
+	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
+		tip = addNode(setFromSlice(tip), tip.PastMedianTime(dag))
+		if compareBits(tip.bits, tip.parents.bluest().bits) > 0 {
+			t.Fatalf("Because we're increasing the block rate, the difficulty can't decrease")
+		}
+	}
+
+	// Add blocks until difficulty stabilizes
+	lastBits := tip.bits
+	sameBitsCount := uint64(0)
+	for sameBitsCount < dag.difficultyAdjustmentWindowSize+1 {
+		tip = addNode(setFromSlice(tip), zeroTime)
+		if tip.bits == lastBits {
+			sameBitsCount++
+		} else {
+			lastBits = tip.bits
+			sameBitsCount = 0
+		}
+	}
+	slowNode := addNode(setFromSlice(tip), time.Unix(tip.timestamp+2, 0))
+	if slowNode.bits != tip.bits {
+		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
+	}
+
+	tip = slowNode
+
+	tip = addNode(setFromSlice(tip), zeroTime)
+	if tip.bits != slowNode.bits {
+		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
+	}
+	tip = addNode(setFromSlice(tip), zeroTime)
+	if compareBits(tip.bits, slowNode.bits) <= 0 {
+		t.Fatalf("tip.bits should be smaller than slowNode.bits because slowNode decreased the block rate, so the difficulty should decrease as well")
+	}
+
+	splitNode := addNode(setFromSlice(tip), zeroTime)
+	tip = splitNode
+	for i := 0; i < 100; i++ {
+		tip = addNode(setFromSlice(tip), zeroTime)
+	}
+	blueTip := tip
+
+	redChainTip := splitNode
+	for i := 0; i < 10; i++ {
+		redChainTip = addNode(setFromSlice(redChainTip), redChainTip.PastMedianTime(dag))
+	}
+	tipWithRedPast := addNode(setFromSlice(redChainTip, blueTip), zeroTime)
+	tipWithoutRedPast := addNode(setFromSlice(blueTip), zeroTime)
+	if tipWithoutRedPast.bits != tipWithRedPast.bits {
+		t.Fatalf("tipWithoutRedPast.bits should be the same as tipWithRedPast.bits because red blocks shouldn't affect the difficulty")
+	}
+}
+
+func compareBits(a uint32, b uint32) int {
+	aTarget := util.CompactToBig(a)
+	bTarget := util.CompactToBig(b)
+	return aTarget.Cmp(bTarget)
+}

blockdag/error.go

@@ -37,9 +37,9 @@ const (
 	// exists.
 	ErrDuplicateBlock ErrorCode = iota
 
-	// ErrBlockTooBig indicates the serialized block size exceeds the
-	// maximum allowed size.
-	ErrBlockTooBig
+	// ErrBlockMassTooHigh indicates the mass of a block exceeds the maximum
+	// allowed limits.
+	ErrBlockMassTooHigh
 
 	// ErrBlockVersionTooOld indicates the block version is too old and is
 	// no longer accepted since the majority of the network has upgraded
@@ -84,17 +84,17 @@ const (
 	// the expected value.
 	ErrBadMerkleRoot
 
+	// ErrBadUTXOCommitment indicates the calculated UTXO commitment does not match
+	// the expected value.
+	ErrBadUTXOCommitment
+
 	// ErrBadCheckpoint indicates a block that is expected to be at a
 	// checkpoint height does not match the expected one.
 	ErrBadCheckpoint
 
-	// ErrForkTooOld indicates a block is attempting to fork the block chain
-	// before the most recent checkpoint.
-	ErrForkTooOld
-
-	// ErrCheckpointTimeTooOld indicates a block has a timestamp before the
-	// most recent checkpoint.
-	ErrCheckpointTimeTooOld
+	// ErrFinalityPointTimeTooOld indicates a block has a timestamp before the
+	// last finality point.
+	ErrFinalityPointTimeTooOld
 
 	// ErrNoTransactions indicates the block does not have a least one
 	// transaction.  A valid block must have at least the coinbase
@@ -105,9 +105,9 @@ const (
 	// valid transaction must have at least one input.
 	ErrNoTxInputs
 
-	// ErrTxTooBig indicates a transaction exceeds the maximum allowed size
-	// when serialized.
-	ErrTxTooBig
+	// ErrTxMassTooHigh indicates the mass of a transaction exceeds the maximum
+	// allowed limits.
+	ErrTxMassTooHigh
 
 	// ErrBadTxOutValue indicates an output value for a transaction is
 	// invalid in some way such as being out of range.
@@ -141,7 +141,7 @@ const (
 	ErrOverwriteTx
 
 	// ErrImmatureSpend indicates a transaction is attempting to spend a
-	// block reward that has not yet reached the required maturity.
+	// coinbase that has not yet reached the required maturity.
 	ErrImmatureSpend
 
 	// ErrSpendTooHigh indicates a transaction is attempting to spend more
@@ -164,34 +164,12 @@ const (
 	// coinbase transaction.
 	ErrMultipleCoinbases
 
-	// ErrBadCoinbaseScriptLen indicates the length of the signature script
-	// for a coinbase transaction is not within the valid range.
-	ErrBadCoinbaseScriptLen
+	// ErrBadCoinbasePayloadLen indicates the length of the payload
+	// for a coinbase transaction is too high.
+	ErrBadCoinbasePayloadLen
 
-	// ErrBadCoinbaseValue indicates the amount of a coinbase value does
-	// not match the expected value of the subsidy plus the sum of all fees.
-	ErrBadCoinbaseValue
-
-	// ErrMissingCoinbaseHeight indicates the coinbase transaction for a
-	// block does not start with the serialized block block height as
-	// required for version 2 and higher blocks.
-	ErrMissingCoinbaseHeight
-
-	// ErrBadCoinbaseHeight indicates the serialized block height in the
-	// coinbase transaction for version 2 and higher blocks does not match
-	// the expected value.
-	ErrBadCoinbaseHeight
-
-	// ErrSecondTxNotFeeTransaction indicates the second transaction in
-	// a block is not a fee transaction.
-	ErrSecondTxNotFeeTransaction
-
-	// ErrBadFeeTransaction indicates that the block's fee transaction is not build as expected
-	ErrBadFeeTransaction
-
-	// ErrMultipleFeeTransactions indicates a block contains more than one
-	// fee transaction.
-	ErrMultipleFeeTransactions
+	// ErrBadCoinbaseTransaction indicates that the block's coinbase transaction is not build as expected
+	ErrBadCoinbaseTransaction
 
 	// ErrScriptMalformed indicates a transaction script is malformed in
 	// some way.  For example, it might be longer than the maximum allowed
@@ -240,12 +218,16 @@ const (
 	// ErrSubnetwork indicates that a block doesn't adhere to the subnetwork
 	// registry rules
 	ErrSubnetworkRegistry
+
+	// ErrInvalidParentsRelation indicates that one of the parents of a block
+	// is also an ancestor of another parent
+	ErrInvalidParentsRelation
 )
 
 // Map of ErrorCode values back to their constant names for pretty printing.
 var errorCodeStrings = map[ErrorCode]string{
 	ErrDuplicateBlock:            "ErrDuplicateBlock",
-	ErrBlockTooBig:               "ErrBlockTooBig",
+	ErrBlockMassTooHigh:          "ErrBlockMassTooHigh",
 	ErrBlockVersionTooOld:        "ErrBlockVersionTooOld",
 	ErrInvalidTime:               "ErrInvalidTime",
 	ErrTimeTooOld:                "ErrTimeTooOld",
@@ -257,11 +239,10 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrHighHash:                  "ErrHighHash",
 	ErrBadMerkleRoot:             "ErrBadMerkleRoot",
 	ErrBadCheckpoint:             "ErrBadCheckpoint",
-	ErrForkTooOld:                "ErrForkTooOld",
-	ErrCheckpointTimeTooOld:      "ErrCheckpointTimeTooOld",
+	ErrFinalityPointTimeTooOld:   "ErrFinalityPointTimeTooOld",
 	ErrNoTransactions:            "ErrNoTransactions",
 	ErrNoTxInputs:                "ErrNoTxInputs",
-	ErrTxTooBig:                  "ErrTxTooBig",
+	ErrTxMassTooHigh:             "ErrTxMassTooHigh",
 	ErrBadTxOutValue:             "ErrBadTxOutValue",
 	ErrDuplicateTxInputs:         "ErrDuplicateTxInputs",
 	ErrBadTxInput:                "ErrBadTxInput",
@@ -275,13 +256,8 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrTooManySigOps:             "ErrTooManySigOps",
 	ErrFirstTxNotCoinbase:        "ErrFirstTxNotCoinbase",
 	ErrMultipleCoinbases:         "ErrMultipleCoinbases",
-	ErrBadCoinbaseScriptLen:      "ErrBadCoinbaseScriptLen",
-	ErrBadCoinbaseValue:          "ErrBadCoinbaseValue",
-	ErrMissingCoinbaseHeight:     "ErrMissingCoinbaseHeight",
-	ErrBadCoinbaseHeight:         "ErrBadCoinbaseHeight",
-	ErrSecondTxNotFeeTransaction: "ErrSecondTxNotFeeTransaction",
-	ErrBadFeeTransaction:         "ErrBadFeeTransaction",
-	ErrMultipleFeeTransactions:   "ErrMultipleFeeTransactions",
+	ErrBadCoinbasePayloadLen:     "ErrBadCoinbasePayloadLen",
+	ErrBadCoinbaseTransaction:    "ErrBadCoinbaseTransaction",
 	ErrScriptMalformed:           "ErrScriptMalformed",
 	ErrScriptValidation:          "ErrScriptValidation",
 	ErrParentBlockUnknown:        "ErrParentBlockUnknown",
@@ -293,6 +269,7 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrInvalidGas:                "ErrInvalidGas",
 	ErrInvalidPayload:            "ErrInvalidPayload",
 	ErrInvalidPayloadHash:        "ErrInvalidPayloadHash",
+	ErrInvalidParentsRelation:    "ErrInvalidParentsRelation",
 }
 
 // String returns the ErrorCode as a human-readable name.

blockdag/error_test.go

@@ -16,7 +16,7 @@ func TestErrorCodeStringer(t *testing.T) {
 		want string
 	}{
 		{ErrDuplicateBlock, "ErrDuplicateBlock"},
-		{ErrBlockTooBig, "ErrBlockTooBig"},
+		{ErrBlockMassTooHigh, "ErrBlockMassTooHigh"},
 		{ErrBlockVersionTooOld, "ErrBlockVersionTooOld"},
 		{ErrInvalidTime, "ErrInvalidTime"},
 		{ErrTimeTooOld, "ErrTimeTooOld"},
@@ -28,11 +28,10 @@ func TestErrorCodeStringer(t *testing.T) {
 		{ErrHighHash, "ErrHighHash"},
 		{ErrBadMerkleRoot, "ErrBadMerkleRoot"},
 		{ErrBadCheckpoint, "ErrBadCheckpoint"},
-		{ErrForkTooOld, "ErrForkTooOld"},
-		{ErrCheckpointTimeTooOld, "ErrCheckpointTimeTooOld"},
+		{ErrFinalityPointTimeTooOld, "ErrFinalityPointTimeTooOld"},
 		{ErrNoTransactions, "ErrNoTransactions"},
 		{ErrNoTxInputs, "ErrNoTxInputs"},
-		{ErrTxTooBig, "ErrTxTooBig"},
+		{ErrTxMassTooHigh, "ErrTxMassTooHigh"},
 		{ErrBadTxOutValue, "ErrBadTxOutValue"},
 		{ErrDuplicateTxInputs, "ErrDuplicateTxInputs"},
 		{ErrBadTxInput, "ErrBadTxInput"},
@@ -47,13 +46,8 @@ func TestErrorCodeStringer(t *testing.T) {
 		{ErrTooManySigOps, "ErrTooManySigOps"},
 		{ErrFirstTxNotCoinbase, "ErrFirstTxNotCoinbase"},
 		{ErrMultipleCoinbases, "ErrMultipleCoinbases"},
-		{ErrBadCoinbaseScriptLen, "ErrBadCoinbaseScriptLen"},
-		{ErrBadCoinbaseValue, "ErrBadCoinbaseValue"},
-		{ErrMissingCoinbaseHeight, "ErrMissingCoinbaseHeight"},
-		{ErrBadCoinbaseHeight, "ErrBadCoinbaseHeight"},
-		{ErrSecondTxNotFeeTransaction, "ErrSecondTxNotFeeTransaction"},
-		{ErrBadFeeTransaction, "ErrBadFeeTransaction"},
-		{ErrMultipleFeeTransactions, "ErrMultipleFeeTransactions"},
+		{ErrBadCoinbasePayloadLen, "ErrBadCoinbasePayloadLen"},
+		{ErrBadCoinbaseTransaction, "ErrBadCoinbaseTransaction"},
 		{ErrScriptMalformed, "ErrScriptMalformed"},
 		{ErrScriptValidation, "ErrScriptValidation"},
 		{ErrParentBlockUnknown, "ErrParentBlockUnknown"},
@@ -65,6 +59,7 @@ func TestErrorCodeStringer(t *testing.T) {
 		{ErrInvalidGas, "ErrInvalidGas"},
 		{ErrInvalidPayload, "ErrInvalidPayload"},
 		{ErrInvalidPayloadHash, "ErrInvalidPayloadHash"},
+		{ErrInvalidParentsRelation, "ErrInvalidParentsRelation"},
 		{0xffff, "Unknown ErrorCode (65535)"},
 	}
 

blockdag/example_test.go

@@ -1,71 +0,0 @@
-// Copyright (c) 2014-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag_test
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/database"
-	_ "github.com/daglabs/btcd/database/ffldb"
-	"github.com/daglabs/btcd/util"
-)
-
-// This example demonstrates how to create a new chain instance and use
-// ProcessBlock to attempt to add a block to the chain.  As the package
-// overview documentation describes, this includes all of the Bitcoin consensus
-// rules.  This example intentionally attempts to insert a duplicate genesis
-// block to illustrate how an invalid block is handled.
-func ExampleBlockDAG_ProcessBlock() {
-	// Create a new database to store the accepted blocks into.  Typically
-	// this would be opening an existing database and would not be deleting
-	// and creating a new database like this, but it is done here so this is
-	// a complete working example and does not leave temporary files laying
-	// around.
-	dbPath := filepath.Join(os.TempDir(), "exampleprocessblock")
-	_ = os.RemoveAll(dbPath)
-	db, err := database.Create("ffldb", dbPath, dagconfig.MainNetParams.Net)
-	if err != nil {
-		fmt.Printf("Failed to create database: %v\n", err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Create a new BlockDAG instance using the underlying database for
-	// the main bitcoin network.  This example does not demonstrate some
-	// of the other available configuration options such as specifying a
-	// notification callback and signature cache.  Also, the caller would
-	// ordinarily keep a reference to the median time source and add time
-	// values obtained from other peers on the network so the local time is
-	// adjusted to be in agreement with other peers.
-	chain, err := blockdag.New(&blockdag.Config{
-		DB:         db,
-		DAGParams:  &dagconfig.MainNetParams,
-		TimeSource: blockdag.NewMedianTime(),
-	})
-	if err != nil {
-		fmt.Printf("Failed to create chain instance: %v\n", err)
-		return
-	}
-
-	// Process a block.  For this example, we are going to intentionally
-	// cause an error by trying to process the genesis block which already
-	// exists.
-	genesisBlock := util.NewBlock(dagconfig.MainNetParams.GenesisBlock)
-	isOrphan, err := chain.ProcessBlock(genesisBlock,
-		blockdag.BFNone)
-	if err != nil {
-		fmt.Printf("Failed to process block: %v\n", err)
-		return
-	}
-	fmt.Printf("Block accepted. Is it an orphan?: %v", isOrphan)
-
-	// Output:
-	// Failed to process block: already have block 6477863f190fac902e556da4671c7537da4fe367022b1f00fa5270e0d073cc08
-}

blockdag/external_dag_test.go

@@ -2,30 +2,32 @@ package blockdag_test
 
 import (
 	"fmt"
+	"github.com/pkg/errors"
 	"math"
 	"testing"
 
-	"github.com/daglabs/btcd/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/util/testtools"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/testtools"
 
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/mining"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/mining"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // TestFinality checks that the finality mechanism works as expected.
 // This is how the flow goes:
-// 1) We build a chain of blockdag.FinalityInterval blocks and call its tip altChainTip.
-// 2) We build another chain (let's call it mainChain) of 2 * blockdag.FinalityInterval
+// 1) We build a chain of params.FinalityInterval blocks and call its tip altChainTip.
+// 2) We build another chain (let's call it mainChain) of 2 * params.FinalityInterval
 // blocks, which points to genesis, and then we check that the block in that
-// chain with height of blockdag.FinalityInterval is marked as finality point (This is
+// chain with height of params.FinalityInterval is marked as finality point (This is
 // very predictable, because the blue score of each new block in a chain is the
 // parents plus one).
-// 3) We make a new child to block with height (2 * blockdag.FinalityInterval - 1)
+// 3) We make a new child to block with height (2 * params.FinalityInterval - 1)
 // in mainChain, and we check that connecting it to the DAG
 // doesn't affect the last finality point.
 // 4) We make a block that points to genesis, and check that it
@@ -37,6 +39,7 @@ import (
 func TestFinality(t *testing.T) {
 	params := dagconfig.SimNetParams
 	params.K = 1
+	params.FinalityInterval = 100
 	dag, teardownFunc, err := blockdag.DAGSetup("TestFinality", blockdag.Config{
 		DAGParams: &params,
 	})
@@ -45,18 +48,22 @@ func TestFinality(t *testing.T) {
 	}
 	defer teardownFunc()
 	buildNodeToDag := func(parentHashes []*daghash.Hash) (*util.Block, error) {
-		msgBlock, err := mining.PrepareBlockForTest(dag, &params, parentHashes, nil, false, 1)
+		msgBlock, err := mining.PrepareBlockForTest(dag, &params, parentHashes, nil, false)
 		if err != nil {
 			return nil, err
 		}
 		block := util.NewBlock(msgBlock)
 
-		isOrphan, err := dag.ProcessBlock(block, blockdag.BFNoPoWCheck)
+		isOrphan, delay, err := dag.ProcessBlock(block, blockdag.BFNoPoWCheck)
 		if err != nil {
 			return nil, err
 		}
+		if delay != 0 {
+			return nil, errors.Errorf("ProcessBlock: block " +
+				"is too far in the future")
+		}
 		if isOrphan {
-			return nil, fmt.Errorf("ProcessBlock: unexpected returned orphan block")
+			return nil, errors.Errorf("ProcessBlock: unexpected returned orphan block")
 		}
 
 		return block, nil
@@ -65,8 +72,8 @@ func TestFinality(t *testing.T) {
 	genesis := util.NewBlock(params.GenesisBlock)
 	currentNode := genesis
 
-	// First we build a chain of blockdag.FinalityInterval blocks for future use
-	for i := 0; i < blockdag.FinalityInterval; i++ {
+	// First we build a chain of params.FinalityInterval blocks for future use
+	for i := 0; i < params.FinalityInterval; i++ {
 		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
 		if err != nil {
 			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
@@ -75,10 +82,10 @@ func TestFinality(t *testing.T) {
 
 	altChainTip := currentNode
 
-	// Now we build a new chain of 2 * blockdag.FinalityInterval blocks, pointed to genesis, and
-	// we expect the block with height 1 * blockdag.FinalityInterval to be the last finality point
+	// Now we build a new chain of 2 * params.FinalityInterval blocks, pointed to genesis, and
+	// we expect the block with height 1 * params.FinalityInterval to be the last finality point
 	currentNode = genesis
-	for i := 0; i < blockdag.FinalityInterval; i++ {
+	for i := 0; i < params.FinalityInterval; i++ {
 		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
 		if err != nil {
 			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
@@ -87,7 +94,7 @@ func TestFinality(t *testing.T) {
 
 	expectedFinalityPoint := currentNode
 
-	for i := 0; i < blockdag.FinalityInterval; i++ {
+	for i := 0; i < params.FinalityInterval; i++ {
 		currentNode, err = buildNodeToDag([]*daghash.Hash{currentNode.Hash()})
 		if err != nil {
 			t.Fatalf("TestFinality: buildNodeToDag unexpectedly returned an error: %v", err)
@@ -111,7 +118,22 @@ func TestFinality(t *testing.T) {
 
 	// Here we check that a block with lower blue score than the last finality
 	// point will get rejected
-	_, err = buildNodeToDag([]*daghash.Hash{genesis.Hash()})
+	fakeCoinbaseTx, err := dag.NextBlockCoinbaseTransaction(nil, nil)
+	if err != nil {
+		t.Errorf("NextBlockCoinbaseTransaction: %s", err)
+	}
+	merkleRoot := blockdag.BuildHashMerkleTreeStore([]*util.Tx{fakeCoinbaseTx}).Root()
+	beforeFinalityBlock := wire.NewMsgBlock(&wire.BlockHeader{
+		Version:              0x10000000,
+		ParentHashes:         []*daghash.Hash{genesis.Hash()},
+		HashMerkleRoot:       merkleRoot,
+		AcceptedIDMerkleRoot: &daghash.ZeroHash,
+		UTXOCommitment:       &daghash.ZeroHash,
+		Timestamp:            dag.SelectedTipHeader().Timestamp,
+		Bits:                 genesis.MsgBlock().Header.Bits,
+	})
+	beforeFinalityBlock.AddTransaction(fakeCoinbaseTx.MsgTx())
+	_, _, err = dag.ProcessBlock(util.NewBlock(beforeFinalityBlock), blockdag.BFNoPoWCheck)
 	if err == nil {
 		t.Errorf("TestFinality: buildNodeToDag expected an error but got <nil>")
 	}
@@ -121,7 +143,7 @@ func TestFinality(t *testing.T) {
 			t.Errorf("TestFinality: buildNodeToDag expected an error with code %v but instead got %v", blockdag.ErrFinality, rErr.ErrorCode)
 		}
 	} else {
-		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", rErr)
+		t.Errorf("TestFinality: buildNodeToDag got unexpected error: %v", err)
 	}
 
 	// Here we check that a block that doesn't have the last finality point in
@@ -140,11 +162,22 @@ func TestFinality(t *testing.T) {
 	}
 }
 
+// TestFinalityInterval tests that the finality interval is
+// smaller then wire.MaxInvPerMsg, so when a peer receives
+// a getblocks message it should always be able to send
+// all the necessary invs.
+func TestFinalityInterval(t *testing.T) {
+	params := dagconfig.SimNetParams
+	if params.FinalityInterval > wire.MaxInvPerMsg {
+		t.Errorf("dagconfig.SimNetParams.FinalityInterval should be lower or equal to wire.MaxInvPerMsg")
+	}
+}
+
 // TestSubnetworkRegistry tests the full subnetwork registry flow
 func TestSubnetworkRegistry(t *testing.T) {
 	params := dagconfig.SimNetParams
 	params.K = 1
-	params.BlockRewardMaturity = 1
+	params.BlockCoinbaseMaturity = 0
 	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", blockdag.Config{
 		DAGParams: &params,
 	})
@@ -169,7 +202,7 @@ func TestSubnetworkRegistry(t *testing.T) {
 
 func TestChainedTransactions(t *testing.T) {
 	params := dagconfig.SimNetParams
-	params.BlockRewardMaturity = 1
+	params.BlockCoinbaseMaturity = 0
 	// Create a new database and dag instance to run tests against.
 	dag, teardownFunc, err := blockdag.DAGSetup("TestChainedTransactions", blockdag.Config{
 		DAGParams: &params,
@@ -179,48 +212,61 @@ func TestChainedTransactions(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	block1, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{params.GenesisHash}, nil, false, 1)
+	block1, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{params.GenesisHash}, nil, false)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, err := dag.ProcessBlock(util.NewBlock(block1), blockdag.BFNoPoWCheck)
+	isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(block1), blockdag.BFNoPoWCheck)
 	if err != nil {
 		t.Fatalf("ProcessBlock: %v", err)
 	}
+	if delay != 0 {
+		t.Fatalf("ProcessBlock: block1 " +
+			"is too far in the future")
+	}
 	if isOrphan {
 		t.Fatalf("ProcessBlock: block1 got unexpectedly orphaned")
 	}
 	cbTx := block1.Transactions[0]
 
+	signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+	if err != nil {
+		t.Fatalf("Failed to build signature script: %s", err)
+	}
 	txIn := &wire.TxIn{
-		PreviousOutPoint: wire.OutPoint{TxID: cbTx.TxID(), Index: 0},
-		SignatureScript:  nil,
+		PreviousOutpoint: wire.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
 		Sequence:         wire.MaxTxInSequenceNum,
 	}
 	txOut := &wire.TxOut{
-		PkScript: blockdag.OpTrueScript,
-		Value:    uint64(1),
+		ScriptPubKey: blockdag.OpTrueScript,
+		Value:        uint64(1),
 	}
 	tx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
 
 	chainedTxIn := &wire.TxIn{
-		PreviousOutPoint: wire.OutPoint{TxID: tx.TxID(), Index: 0},
-		SignatureScript:  nil,
+		PreviousOutpoint: wire.Outpoint{TxID: *tx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
 		Sequence:         wire.MaxTxInSequenceNum,
 	}
+
+	scriptPubKey, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript)
+	if err != nil {
+		t.Fatalf("Failed to build public key script: %s", err)
+	}
 	chainedTxOut := &wire.TxOut{
-		PkScript: blockdag.OpTrueScript,
-		Value:    uint64(1),
+		ScriptPubKey: scriptPubKey,
+		Value:        uint64(1),
 	}
 	chainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{chainedTxIn}, []*wire.TxOut{chainedTxOut})
 
-	block2, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{tx, chainedTx}, true, 1)
+	block2, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{tx, chainedTx}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
 	//Checks that dag.ProcessBlock fails because we don't allow a transaction to spend another transaction from the same block
-	isOrphan, err = dag.ProcessBlock(util.NewBlock(block2), blockdag.BFNoPoWCheck)
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(block2), blockdag.BFNoPoWCheck)
 	if err == nil {
 		t.Errorf("ProcessBlock expected an error")
 	} else if rErr, ok := err.(blockdag.RuleError); ok {
@@ -230,41 +276,121 @@ func TestChainedTransactions(t *testing.T) {
 	} else {
 		t.Errorf("ProcessBlock expected a blockdag.RuleError but got %v", err)
 	}
+	if delay != 0 {
+		t.Fatalf("ProcessBlock: block2 " +
+			"is too far in the future")
+	}
 	if isOrphan {
 		t.Errorf("ProcessBlock: block2 got unexpectedly orphaned")
 	}
 
 	nonChainedTxIn := &wire.TxIn{
-		PreviousOutPoint: wire.OutPoint{TxID: cbTx.TxID(), Index: 0},
-		SignatureScript:  nil,
+		PreviousOutpoint: wire.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
 		Sequence:         wire.MaxTxInSequenceNum,
 	}
 	nonChainedTxOut := &wire.TxOut{
-		PkScript: blockdag.OpTrueScript,
-		Value:    uint64(1),
+		ScriptPubKey: scriptPubKey,
+		Value:        uint64(1),
 	}
 	nonChainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{nonChainedTxIn}, []*wire.TxOut{nonChainedTxOut})
 
-	block3, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{nonChainedTx}, false, 1)
+	block3, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{nonChainedTx}, false)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
 	//Checks that dag.ProcessBlock doesn't fail because all of its transaction are dependant on transactions from previous blocks
-	isOrphan, err = dag.ProcessBlock(util.NewBlock(block3), blockdag.BFNoPoWCheck)
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(block3), blockdag.BFNoPoWCheck)
 	if err != nil {
 		t.Errorf("ProcessBlock: %v", err)
 	}
+	if delay != 0 {
+		t.Fatalf("ProcessBlock: block3 " +
+			"is too far in the future")
+	}
 	if isOrphan {
 		t.Errorf("ProcessBlock: block3 got unexpectedly orphaned")
 	}
 }
 
+// TestOrderInDiffFromAcceptanceData makes sure that the order of transactions in
+// dag.diffFromAcceptanceData is such that if txA is spent by txB then txA is processed
+// before txB.
+func TestOrderInDiffFromAcceptanceData(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	params := dagconfig.SimNetParams
+	params.K = math.MaxUint32
+	dag, teardownFunc, err := blockdag.DAGSetup("TestOrderInDiffFromAcceptanceData", blockdag.Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+	dag.TestSetCoinbaseMaturity(0)
+
+	createBlock := func(previousBlock *util.Block) *util.Block {
+		// Prepare a transaction that spends the previous block's coinbase transaction
+		var txs []*wire.MsgTx
+		if !previousBlock.IsGenesis() {
+			previousCoinbaseTx := previousBlock.MsgBlock().Transactions[0]
+			signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+			if err != nil {
+				t.Fatalf("TestOrderInDiffFromAcceptanceData: Failed to build signature script: %s", err)
+			}
+			txIn := &wire.TxIn{
+				PreviousOutpoint: wire.Outpoint{TxID: *previousCoinbaseTx.TxID(), Index: 0},
+				SignatureScript:  signatureScript,
+				Sequence:         wire.MaxTxInSequenceNum,
+			}
+			txOut := &wire.TxOut{
+				ScriptPubKey: blockdag.OpTrueScript,
+				Value:        uint64(1),
+			}
+			txs = append(txs, wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut}))
+		}
+
+		// Create the block
+		msgBlock, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{previousBlock.Hash()}, txs, false)
+		if err != nil {
+			t.Fatalf("TestOrderInDiffFromAcceptanceData: Failed to prepare block: %s", err)
+		}
+
+		// Add the block to the DAG
+		newBlock := util.NewBlock(msgBlock)
+		isOrphan, delay, err := dag.ProcessBlock(newBlock, blockdag.BFNoPoWCheck)
+		if err != nil {
+			t.Errorf("TestOrderInDiffFromAcceptanceData: %s", err)
+		}
+		if delay != 0 {
+			t.Fatalf("TestOrderInDiffFromAcceptanceData: block is too far in the future")
+		}
+		if isOrphan {
+			t.Fatalf("TestOrderInDiffFromAcceptanceData: block got unexpectedly orphaned")
+		}
+		return newBlock
+	}
+
+	// Create two block chains starting from the genesis block. Every time a block is added
+	// one of the chains is selected as the selected parent chain while all the blocks in
+	// the other chain (and their transactions) get accepted by the new virtual. If the
+	// transactions in the non-selected parent chain get processed in the wrong order then
+	// diffFromAcceptanceData panics.
+	blockAmountPerChain := 100
+	chainATip := util.NewBlock(params.GenesisBlock)
+	chainBTip := chainATip
+	for i := 0; i < blockAmountPerChain; i++ {
+		chainATip = createBlock(chainATip)
+		chainBTip = createBlock(chainBTip)
+	}
+}
+
 // TestGasLimit tests the gas limit rules
 func TestGasLimit(t *testing.T) {
 	params := dagconfig.SimNetParams
 	params.K = 1
-	params.BlockRewardMaturity = 1
+	params.BlockCoinbaseMaturity = 0
 	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", blockdag.Config{
 		DAGParams: &params,
 	})
@@ -273,56 +399,74 @@ func TestGasLimit(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	// First we prepare a subnetwrok and a block with coinbase outputs to fund our tests
+	// First we prepare a subnetwork and a block with coinbase outputs to fund our tests
 	gasLimit := uint64(12345)
 	subnetworkID, err := testtools.RegisterSubnetworkForTest(dag, &params, gasLimit)
 	if err != nil {
 		t.Fatalf("could not register network: %s", err)
 	}
 
-	fundsBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), nil, false, 2)
-	if err != nil {
-		t.Fatalf("PrepareBlockForTest: %v", err)
-	}
-	isOrphan, err := dag.ProcessBlock(util.NewBlock(fundsBlock), blockdag.BFNoPoWCheck)
-	if err != nil {
-		t.Fatalf("ProcessBlock: %v", err)
-	}
-	if isOrphan {
-		t.Fatalf("ProcessBlock: funds block got unexpectedly orphan")
+	cbTxs := []*wire.MsgTx{}
+	for i := 0; i < 4; i++ {
+		fundsBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), nil, false)
+		if err != nil {
+			t.Fatalf("PrepareBlockForTest: %v", err)
+		}
+		isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(fundsBlock), blockdag.BFNoPoWCheck)
+		if err != nil {
+			t.Fatalf("ProcessBlock: %v", err)
+		}
+		if delay != 0 {
+			t.Fatalf("ProcessBlock: the funds block " +
+				"is too far in the future")
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock: fundsBlock got unexpectedly orphan")
+		}
+
+		cbTxs = append(cbTxs, fundsBlock.Transactions[util.CoinbaseTransactionIndex])
 	}
 
-	cbTxValue := fundsBlock.Transactions[0].TxOut[0].Value
-	cbTxID := fundsBlock.Transactions[0].TxID()
+	signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+	if err != nil {
+		t.Fatalf("Failed to build signature script: %s", err)
+	}
+
+	scriptPubKey, err := txscript.PayToScriptHashScript(blockdag.OpTrueScript)
+	if err != nil {
+		t.Fatalf("Failed to build public key script: %s", err)
+	}
 
 	tx1In := &wire.TxIn{
-		PreviousOutPoint: *wire.NewOutPoint(&cbTxID, 0),
+		PreviousOutpoint: *wire.NewOutpoint(cbTxs[0].TxID(), 0),
 		Sequence:         wire.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
 	}
 	tx1Out := &wire.TxOut{
-		Value:    cbTxValue,
-		PkScript: blockdag.OpTrueScript,
+		Value:        cbTxs[0].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
 	}
 	tx1 := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{tx1In}, []*wire.TxOut{tx1Out}, subnetworkID, 10000, []byte{})
 
 	tx2In := &wire.TxIn{
-		PreviousOutPoint: *wire.NewOutPoint(&cbTxID, 1),
+		PreviousOutpoint: *wire.NewOutpoint(cbTxs[1].TxID(), 0),
 		Sequence:         wire.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
 	}
 	tx2Out := &wire.TxOut{
-		Value:    cbTxValue,
-		PkScript: blockdag.OpTrueScript,
+		Value:        cbTxs[1].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
 	}
 	tx2 := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{tx2In}, []*wire.TxOut{tx2Out}, subnetworkID, 10000, []byte{})
 
 	// Here we check that we can't process a block that has transactions that exceed the gas limit
-	overLimitBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1, tx2}, true, 1)
+	overLimitBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1, tx2}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, err = dag.ProcessBlock(util.NewBlock(overLimitBlock), blockdag.BFNoPoWCheck)
+	isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(overLimitBlock), blockdag.BFNoPoWCheck)
 	if err == nil {
-		t.Fatalf("ProcessBlock expected to have an error")
+		t.Fatalf("ProcessBlock expected to have an error in block that exceeds gas limit")
 	}
 	rErr, ok := err.(blockdag.RuleError)
 	if !ok {
@@ -330,27 +474,32 @@ func TestGasLimit(t *testing.T) {
 	} else if rErr.ErrorCode != blockdag.ErrInvalidGas {
 		t.Fatalf("ProcessBlock expected error code %s but got %s", blockdag.ErrInvalidGas, rErr.ErrorCode)
 	}
+	if delay != 0 {
+		t.Fatalf("ProcessBlock: overLimitBlock " +
+			"is too far in the future")
+	}
 	if isOrphan {
 		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
 	}
 
 	overflowGasTxIn := &wire.TxIn{
-		PreviousOutPoint: *wire.NewOutPoint(&cbTxID, 1),
+		PreviousOutpoint: *wire.NewOutpoint(cbTxs[2].TxID(), 0),
 		Sequence:         wire.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
 	}
 	overflowGasTxOut := &wire.TxOut{
-		Value:    cbTxValue,
-		PkScript: blockdag.OpTrueScript,
+		Value:        cbTxs[2].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
 	}
 	overflowGasTx := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{overflowGasTxIn}, []*wire.TxOut{overflowGasTxOut},
 		subnetworkID, math.MaxUint64, []byte{})
 
 	// Here we check that we can't process a block that its transactions' gas overflows uint64
-	overflowGasBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1, overflowGasTx}, true, 1)
+	overflowGasBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1, overflowGasTx}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, err = dag.ProcessBlock(util.NewBlock(overflowGasBlock), blockdag.BFNoPoWCheck)
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(overflowGasBlock), blockdag.BFNoPoWCheck)
 	if err == nil {
 		t.Fatalf("ProcessBlock expected to have an error")
 	}
@@ -366,37 +515,43 @@ func TestGasLimit(t *testing.T) {
 
 	nonExistentSubnetwork := &subnetworkid.SubnetworkID{123}
 	nonExistentSubnetworkTxIn := &wire.TxIn{
-		PreviousOutPoint: *wire.NewOutPoint(&cbTxID, 0),
+		PreviousOutpoint: *wire.NewOutpoint(cbTxs[3].TxID(), 0),
 		Sequence:         wire.MaxTxInSequenceNum,
+		SignatureScript:  signatureScript,
 	}
 	nonExistentSubnetworkTxOut := &wire.TxOut{
-		Value:    cbTxValue,
-		PkScript: blockdag.OpTrueScript,
+		Value:        cbTxs[3].TxOut[0].Value,
+		ScriptPubKey: scriptPubKey,
 	}
 	nonExistentSubnetworkTx := wire.NewSubnetworkMsgTx(wire.TxVersion, []*wire.TxIn{nonExistentSubnetworkTxIn},
 		[]*wire.TxOut{nonExistentSubnetworkTxOut}, nonExistentSubnetwork, 1, []byte{})
 
-	nonExistentSubnetworkBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{nonExistentSubnetworkTx, overflowGasTx}, true, 1)
+	nonExistentSubnetworkBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{nonExistentSubnetworkTx, overflowGasTx}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
 	// Here we check that we can't process a block with a transaction from a non-existent subnetwork
-	isOrphan, err = dag.ProcessBlock(util.NewBlock(nonExistentSubnetworkBlock), blockdag.BFNoPoWCheck)
-	expectedErrStr := fmt.Sprintf("subnetwork '%s' not found", nonExistentSubnetwork)
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(nonExistentSubnetworkBlock), blockdag.BFNoPoWCheck)
+	expectedErrStr := fmt.Sprintf("Error getting gas limit for subnetworkID '%s': subnetwork '%s' not found",
+		nonExistentSubnetwork, nonExistentSubnetwork)
 	if err.Error() != expectedErrStr {
-		t.Fatalf("ProcessBlock expected error %v but got %v", expectedErrStr, err)
+		t.Fatalf("ProcessBlock expected error \"%v\" but got \"%v\"", expectedErrStr, err)
 	}
 
 	// Here we check that we can process a block with a transaction that doesn't exceed the gas limit
-	validBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1}, true, 1)
+	validBlock, err := mining.PrepareBlockForTest(dag, &params, dag.TipHashes(), []*wire.MsgTx{tx1}, true)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
-	isOrphan, err = dag.ProcessBlock(util.NewBlock(validBlock), blockdag.BFNoPoWCheck)
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(validBlock), blockdag.BFNoPoWCheck)
 	if err != nil {
 		t.Fatalf("ProcessBlock: %v", err)
 	}
+	if delay != 0 {
+		t.Fatalf("ProcessBlock: overLimitBlock " +
+			"is too far in the future")
+	}
 	if isOrphan {
 		t.Fatalf("ProcessBlock: overLimitBlock got unexpectedly orphan")
 	}

blockdag/fees.go

@@ -1,219 +0,0 @@
-package blockdag
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/util/txsort"
-	"github.com/daglabs/btcd/wire"
-)
-
-// compactFeeData is a specialized data type to store a compact list of fees
-// inside a block.
-// Every transaction gets a single uint64 value, stored as a plain binary list.
-// The transactions are ordered the same way they are ordered inside the block, making it easy
-// to traverse every transaction in a block and extract its fee.
-//
-// compactFeeFactory is used to create such a list.
-// compactFeeIterator is used to iterate over such a list.
-
-type compactFeeData []byte
-
-func (cfd compactFeeData) Len() int {
-	return len(cfd) / 8
-}
-
-type compactFeeFactory struct {
-	buffer *bytes.Buffer
-	writer *bufio.Writer
-}
-
-func newCompactFeeFactory() *compactFeeFactory {
-	buffer := bytes.NewBuffer([]byte{})
-	return &compactFeeFactory{
-		buffer: buffer,
-		writer: bufio.NewWriter(buffer),
-	}
-}
-
-func (cfw *compactFeeFactory) add(txFee uint64) error {
-	return binary.Write(cfw.writer, binary.LittleEndian, txFee)
-}
-
-func (cfw *compactFeeFactory) data() (compactFeeData, error) {
-	err := cfw.writer.Flush()
-
-	return compactFeeData(cfw.buffer.Bytes()), err
-}
-
-type compactFeeIterator struct {
-	reader io.Reader
-}
-
-func (cfd compactFeeData) iterator() *compactFeeIterator {
-	return &compactFeeIterator{
-		reader: bufio.NewReader(bytes.NewBuffer(cfd)),
-	}
-}
-
-func (cfr *compactFeeIterator) next() (uint64, error) {
-	var txFee uint64
-
-	err := binary.Read(cfr.reader, binary.LittleEndian, &txFee)
-
-	return txFee, err
-}
-
-// The following functions relate to storing and retrieving fee data from the database
-var feeBucket = []byte("fees")
-
-// getBluesFeeData returns the compactFeeData for all nodes's blues,
-// used to calculate the fees this blockNode needs to pay
-func (node *blockNode) getBluesFeeData(dag *BlockDAG) (map[daghash.Hash]compactFeeData, error) {
-	bluesFeeData := make(map[daghash.Hash]compactFeeData)
-
-	dag.db.View(func(dbTx database.Tx) error {
-		for _, blueBlock := range node.blues {
-			feeData, err := dbFetchFeeData(dbTx, blueBlock.hash)
-			if err != nil {
-				return fmt.Errorf("Error getting fee data for block %s: %s", blueBlock.hash, err)
-			}
-
-			bluesFeeData[*blueBlock.hash] = feeData
-		}
-
-		return nil
-	})
-
-	return bluesFeeData, nil
-}
-
-func dbStoreFeeData(dbTx database.Tx, blockHash *daghash.Hash, feeData compactFeeData) error {
-	feeBucket, err := dbTx.Metadata().CreateBucketIfNotExists(feeBucket)
-	if err != nil {
-		return fmt.Errorf("Error creating or retrieving fee bucket: %s", err)
-	}
-
-	return feeBucket.Put(blockHash.CloneBytes(), feeData)
-}
-
-func dbFetchFeeData(dbTx database.Tx, blockHash *daghash.Hash) (compactFeeData, error) {
-	feeBucket := dbTx.Metadata().Bucket(feeBucket)
-	if feeBucket == nil {
-		return nil, errors.New("Fee bucket does not exist")
-	}
-
-	feeData := feeBucket.Get(blockHash.CloneBytes())
-	if feeData == nil {
-		return nil, fmt.Errorf("No fee data found for block %s", blockHash)
-	}
-
-	return feeData, nil
-}
-
-// The following functions deal with building and validating the fee transaction
-
-func (node *blockNode) validateFeeTransaction(dag *BlockDAG, block *util.Block, txsAcceptanceData MultiBlockTxsAcceptanceData) error {
-	if node.isGenesis() {
-		return nil
-	}
-	expectedFeeTransaction, err := node.buildFeeTransaction(dag, txsAcceptanceData)
-	if err != nil {
-		return err
-	}
-
-	if !expectedFeeTransaction.TxHash().IsEqual(block.FeeTransaction().Hash()) {
-		return ruleError(ErrBadFeeTransaction, "Fee transaction is not built as expected")
-	}
-
-	return nil
-}
-
-// buildFeeTransaction returns the expected fee transaction for the current block
-func (node *blockNode) buildFeeTransaction(dag *BlockDAG, txsAcceptanceData MultiBlockTxsAcceptanceData) (*wire.MsgTx, error) {
-	bluesFeeData, err := node.getBluesFeeData(dag)
-	if err != nil {
-		return nil, err
-	}
-
-	txIns := []*wire.TxIn{}
-	txOuts := []*wire.TxOut{}
-
-	for _, blue := range node.blues {
-		txIn, txOut, err := feeInputAndOutputForBlueBlock(blue, txsAcceptanceData, bluesFeeData)
-		if err != nil {
-			return nil, err
-		}
-		txIns = append(txIns, txIn)
-		if txOut != nil {
-			txOuts = append(txOuts, txOut)
-		}
-	}
-	feeTx := wire.NewNativeMsgTx(wire.TxVersion, txIns, txOuts)
-	return txsort.Sort(feeTx), nil
-}
-
-// feeInputAndOutputForBlueBlock calculates the input and output that should go into the fee transaction of blueBlock
-// If blueBlock gets no fee - returns only txIn and nil for txOut
-func feeInputAndOutputForBlueBlock(blueBlock *blockNode, txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (
-	*wire.TxIn, *wire.TxOut, error) {
-
-	blockTxsAcceptanceData, ok := txsAcceptanceData[*blueBlock.hash]
-	if !ok {
-		return nil, nil, fmt.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
-	}
-	blockFeeData, ok := feeData[*blueBlock.hash]
-	if !ok {
-		return nil, nil, fmt.Errorf("No feeData for block %s", blueBlock.hash)
-	}
-
-	if len(blockTxsAcceptanceData) != blockFeeData.Len() {
-		return nil, nil, fmt.Errorf(
-			"length of accepted transaction data(%d) and fee data(%d) is not equal for block %s",
-			len(blockTxsAcceptanceData), blockFeeData.Len(), blueBlock.hash)
-	}
-
-	txIn := &wire.TxIn{
-		SignatureScript: []byte{},
-		PreviousOutPoint: wire.OutPoint{
-			TxID:  daghash.TxID(*blueBlock.hash),
-			Index: math.MaxUint32,
-		},
-		Sequence: wire.MaxTxInSequenceNum,
-	}
-
-	totalFees := uint64(0)
-	feeIterator := blockFeeData.iterator()
-
-	for _, txAcceptanceData := range blockTxsAcceptanceData {
-		fee, err := feeIterator.next()
-		if err != nil {
-			return nil, nil, fmt.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
-		}
-		if txAcceptanceData.IsAccepted {
-			totalFees += fee
-		}
-	}
-
-	if totalFees == 0 {
-		return txIn, nil, nil
-	}
-
-	// the scriptPubKey for the fee is the same as the coinbase's first scriptPubKey
-	pkScript := blockTxsAcceptanceData[0].Tx.MsgTx().TxOut[0].PkScript
-
-	txOut := &wire.TxOut{
-		Value:    totalFees,
-		PkScript: pkScript,
-	}
-
-	return txIn, txOut, nil
-}

blockdag/fullblocks_test.go

@@ -7,20 +7,20 @@ package blockdag_test
 
 import (
 	"bytes"
-	"fmt"
+	"github.com/pkg/errors"
 	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/blockdag/fullblocktests"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	_ "github.com/daglabs/btcd/database/ffldb"
-	"github.com/daglabs/btcd/txscript"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/blockdag/fullblocktests"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
+	_ "github.com/kaspanet/kaspad/database/ffldb"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 const (
@@ -62,7 +62,7 @@ func isSupportedDbType(dbType string) bool {
 // a teardown function the caller should invoke when done testing to clean up.
 func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func(), error) {
 	if !isSupportedDbType(testDbType) {
-		return nil, nil, fmt.Errorf("unsupported db type %v", testDbType)
+		return nil, nil, errors.Errorf("unsupported db type %v", testDbType)
 	}
 
 	// Handle memory database specially since it doesn't need the disk
@@ -72,7 +72,7 @@ func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func
 	if testDbType == "memdb" {
 		ndb, err := database.Create(testDbType)
 		if err != nil {
-			return nil, nil, fmt.Errorf("error creating db: %v", err)
+			return nil, nil, errors.Errorf("error creating db: %v", err)
 		}
 		db = ndb
 
@@ -85,7 +85,7 @@ func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func
 		// Create the root directory for test databases.
 		if !fileExists(testDbRoot) {
 			if err := os.MkdirAll(testDbRoot, 0700); err != nil {
-				err := fmt.Errorf("unable to create test db "+
+				err := errors.Errorf("unable to create test db "+
 					"root: %v", err)
 				return nil, nil, err
 			}
@@ -96,7 +96,7 @@ func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func
 		_ = os.RemoveAll(dbPath)
 		ndb, err := database.Create(testDbType, dbPath, blockDataNet)
 		if err != nil {
-			return nil, nil, fmt.Errorf("error creating db: %v", err)
+			return nil, nil, errors.Errorf("error creating db: %v", err)
 		}
 		db = ndb
 
@@ -110,7 +110,7 @@ func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func
 	}
 
 	// Copy the chain params to ensure any modifications the tests do to
-	// the chain parameters do not affect the global instance.
+	// the DAG parameters do not affect the global instance.
 	paramsCopy := *params
 
 	// Create the main chain instance.
@@ -123,7 +123,7 @@ func DAGSetup(dbName string, params *dagconfig.Params) (*blockdag.BlockDAG, func
 	})
 	if err != nil {
 		teardown()
-		err := fmt.Errorf("failed to create chain instance: %v", err)
+		err := errors.Errorf("failed to create chain instance: %v", err)
 		return nil, nil, err
 	}
 	return chain, teardown, nil
@@ -156,11 +156,11 @@ func TestFullBlocks(t *testing.T) {
 	testAcceptedBlock := func(item fullblocktests.AcceptedBlock) {
 		blockHeight := item.Height
 		block := util.NewBlock(item.Block)
-		block.SetHeight(blockHeight)
+		block.SetChainHeight(blockHeight)
 		t.Logf("Testing block %s (hash %s, height %d)",
 			item.Name, block.Hash(), blockHeight)
 
-		isOrphan, err := dag.ProcessBlock(block,
+		isOrphan, delay, err := dag.ProcessBlock(block,
 			blockdag.BFNone)
 		if err != nil {
 			t.Fatalf("block %q (hash %s, height %d) should "+
@@ -168,6 +168,13 @@ func TestFullBlocks(t *testing.T) {
 				block.Hash(), blockHeight, err)
 		}
 
+		if delay != item.Delay {
+			t.Fatalf("block %q (hash %s, height %d) unexpected "+
+				"delay -- got %v, want %v", item.Name,
+				block.Hash(), blockHeight, delay,
+				item.Delay)
+		}
+
 		if isOrphan != item.IsOrphan {
 			t.Fatalf("block %q (hash %s, height %d) unexpected "+
 				"orphan flag -- got %v, want %v", item.Name,
@@ -182,11 +189,11 @@ func TestFullBlocks(t *testing.T) {
 	testRejectedBlock := func(item fullblocktests.RejectedBlock) {
 		blockHeight := item.Height
 		block := util.NewBlock(item.Block)
-		block.SetHeight(blockHeight)
+		block.SetChainHeight(blockHeight)
 		t.Logf("Testing block %s (hash %s, height %d)",
 			item.Name, block.Hash(), blockHeight)
 
-		_, err := dag.ProcessBlock(block, blockdag.BFNone)
+		_, _, err := dag.ProcessBlock(block, blockdag.BFNone)
 		if err == nil {
 			t.Fatalf("block %q (hash %s, height %d) should not "+
 				"have been accepted", item.Name, block.Hash(),
@@ -239,11 +246,11 @@ func TestFullBlocks(t *testing.T) {
 	testOrphanOrRejectedBlock := func(item fullblocktests.OrphanOrRejectedBlock) {
 		blockHeight := item.Height
 		block := util.NewBlock(item.Block)
-		block.SetHeight(blockHeight)
+		block.SetChainHeight(blockHeight)
 		t.Logf("Testing block %s (hash %s, height %d)",
 			item.Name, block.Hash(), blockHeight)
 
-		isOrphan, err := dag.ProcessBlock(block, blockdag.BFNone)
+		isOrphan, delay, err := dag.ProcessBlock(block, blockdag.BFNone)
 		if err != nil {
 			// Ensure the error code is of the expected type.
 			if _, ok := err.(blockdag.RuleError); !ok {
@@ -255,6 +262,12 @@ func TestFullBlocks(t *testing.T) {
 			}
 		}
 
+		if delay != 0 {
+			t.Fatalf("block %q (hash %s, height %d) "+
+				"is too far in the future",
+				item.Name, block.Hash(), blockHeight)
+		}
+
 		if !isOrphan {
 			t.Fatalf("block %q (hash %s, height %d) was accepted, "+
 				"but is not considered an orphan", item.Name,
@@ -267,18 +280,18 @@ func TestFullBlocks(t *testing.T) {
 	testExpectedTip := func(item fullblocktests.ExpectedTip) {
 		blockHeight := item.Height
 		block := util.NewBlock(item.Block)
-		block.SetHeight(blockHeight)
+		block.SetChainHeight(blockHeight)
 		t.Logf("Testing tip for block %s (hash %s, height %d)",
 			item.Name, block.Hash(), blockHeight)
 
 		// Ensure hash and height match.
-		if dag.HighestTipHash() != item.Block.BlockHash() ||
-			dag.Height() != blockHeight { //TODO: (Ori) the use of dag.Height() and virtualBlock.HighestTipHash() is wrong, and was done only for compilation
+		if dag.SelectedTipHash() != item.Block.BlockHash() ||
+			dag.ChainHeight() != blockHeight { //TODO: (Ori) the use of dag.ChainHeight() and virtualBlock.HighestTipHash() is wrong, and was done only for compilation
 
 			t.Fatalf("block %q (hash %s, height %d) should be "+
 				"the current tip -- got (hash %s, height %d)",
-				item.Name, block.Hash(), blockHeight, dag.HighestTipHash(),
-				dag.Height()) //TODO: (Ori) the use of dag.Height() and virtualBlock.HighestTipHash() is wrong, and was done only for compilation
+				item.Name, block.Hash(), blockHeight, dag.SelectedTipHash(),
+				dag.ChainHeight()) //TODO: (Ori) the use of dag.ChainHeight() and virtualBlock.HighestTipHash() is wrong, and was done only for compilation
 		}
 	}
 

blockdag/fullblocktests/README.md

@@ -3,7 +3,7 @@ fullblocktests
 
 [![Build Status](http://img.shields.io/travis/btcsuite/btcd.svg)](https://travis-ci.org/btcsuite/btcd)
 [![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/daglabs/btcd/blockchain/fullblocktests)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/blockchain/fullblocktests)
 
 Package fullblocktests provides a set of full block tests to be used for testing
 the consensus validation rules.  The tests are intended to be flexible enough to
@@ -20,7 +20,7 @@ of blocks that exercise the consensus validation rules.
 ## Installation and Updating
 
 ```bash
-$ go get -u github.com/daglabs/btcd/blockchain/fullblocktests
+$ go get -u github.com/kaspanet/kaspad/blockchain/fullblocktests
 ```
 
 ## License

blockdag/fullblocktests/generate.go

@@ -12,20 +12,20 @@ package fullblocktests
 import (
 	"bytes"
 	"encoding/binary"
-	"errors"
 	"fmt"
+	"github.com/pkg/errors"
 	"math"
 	"runtime"
 	"time"
 
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/btcec"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/txscript"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/util/random"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/btcec"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/random"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 const (
@@ -67,8 +67,9 @@ type TestInstance interface {
 type AcceptedBlock struct {
 	Name     string
 	Block    *wire.MsgBlock
-	Height   int32
+	Height   uint64
 	IsOrphan bool
+	Delay    time.Duration
 }
 
 // Ensure AcceptedBlock implements the TestInstance interface.
@@ -85,7 +86,7 @@ func (b AcceptedBlock) FullBlockTestInstance() {}
 type RejectedBlock struct {
 	Name       string
 	Block      *wire.MsgBlock
-	Height     int32
+	Height     uint64
 	RejectCode blockdag.ErrorCode
 }
 
@@ -107,7 +108,7 @@ func (b RejectedBlock) FullBlockTestInstance() {}
 type OrphanOrRejectedBlock struct {
 	Name   string
 	Block  *wire.MsgBlock
-	Height int32
+	Height uint64
 }
 
 // Ensure ExpectedTip implements the TestInstance interface.
@@ -124,7 +125,7 @@ func (b OrphanOrRejectedBlock) FullBlockTestInstance() {}
 type ExpectedTip struct {
 	Name   string
 	Block  *wire.MsgBlock
-	Height int32
+	Height uint64
 }
 
 // Ensure ExpectedTip implements the TestInstance interface.
@@ -141,7 +142,7 @@ func (b ExpectedTip) FullBlockTestInstance() {}
 type RejectedNonCanonicalBlock struct {
 	Name     string
 	RawBlock []byte
-	Height   int32
+	Height   uint64
 }
 
 // FullBlockTestInstance only exists to allow RejectedNonCanonicalBlock to be treated as
@@ -153,7 +154,7 @@ func (b RejectedNonCanonicalBlock) FullBlockTestInstance() {}
 // spendableOut represents a transaction output that is spendable along with
 // additional metadata such as the block its in and how much it pays.
 type spendableOut struct {
-	prevOut wire.OutPoint
+	prevOut wire.Outpoint
 	amount  util.Amount
 }
 
@@ -161,8 +162,8 @@ type spendableOut struct {
 // and transaction output index within the transaction.
 func makeSpendableOutForTx(tx *wire.MsgTx, txOutIndex uint32) spendableOut {
 	return spendableOut{
-		prevOut: wire.OutPoint{
-			TxID:  tx.TxID(),
+		prevOut: wire.Outpoint{
+			TxID:  *tx.TxID(),
 			Index: txOutIndex,
 		},
 		amount: util.Amount(tx.TxOut[txOutIndex].Value),
@@ -182,10 +183,10 @@ type testGenerator struct {
 	params       *dagconfig.Params
 	tip          *wire.MsgBlock
 	tipName      string
-	tipHeight    int32
+	tipHeight    uint64
 	blocks       map[daghash.Hash]*wire.MsgBlock
 	blocksByName map[string]*wire.MsgBlock
-	blockHeights map[string]int32
+	blockHeights map[string]uint64
 
 	// Used for tracking spendable coinbase outputs.
 	spendableOuts     []spendableOut
@@ -193,6 +194,8 @@ type testGenerator struct {
 
 	// Common key for any tests which require signed transactions.
 	privKey *btcec.PrivateKey
+
+	powMaxBits uint32
 }
 
 // makeTestGenerator returns a test generator instance initialized with the
@@ -205,11 +208,12 @@ func makeTestGenerator(params *dagconfig.Params) (testGenerator, error) {
 		params:       params,
 		blocks:       map[daghash.Hash]*wire.MsgBlock{*genesisHash: genesis},
 		blocksByName: map[string]*wire.MsgBlock{"genesis": genesis},
-		blockHeights: map[string]int32{"genesis": 0},
+		blockHeights: map[string]uint64{"genesis": 0},
 		tip:          genesis,
 		tipName:      "genesis",
 		tipHeight:    0,
 		privKey:      privKey,
+		powMaxBits:   util.BigToCompact(params.PowMax),
 	}, nil
 }
 
@@ -243,8 +247,8 @@ func pushDataScript(items ...[]byte) []byte {
 // standardCoinbaseScript returns a standard script suitable for use as the
 // signature script of the coinbase transaction of a new block.  In particular,
 // it starts with the block height that is required by version 2 blocks.
-func standardCoinbaseScript(blockHeight int32, extraNonce uint64) ([]byte, error) {
-	return txscript.NewScriptBuilder().AddInt64(int64(blockHeight)).
+func standardCoinbaseScript(extraNonce uint64) ([]byte, error) {
+	return txscript.NewScriptBuilder().
 		AddInt64(int64(extraNonce)).Script()
 }
 
@@ -273,11 +277,10 @@ func uniqueOpReturnScript() []byte {
 }
 
 // createCoinbaseTx returns a coinbase transaction paying an appropriate
-// subsidy based on the passed block height.  The coinbase signature script
-// conforms to the requirements of version 2 blocks.
-func (g *testGenerator) createCoinbaseTx(blockHeight int32) *wire.MsgTx {
+// subsidy based on the passed block height.
+func (g *testGenerator) createCoinbaseTx(blueScore uint64) *wire.MsgTx {
 	extraNonce := uint64(0)
-	coinbaseScript, err := standardCoinbaseScript(blockHeight, extraNonce)
+	coinbaseScript, err := standardCoinbaseScript(extraNonce)
 	if err != nil {
 		panic(err)
 	}
@@ -285,14 +288,14 @@ func (g *testGenerator) createCoinbaseTx(blockHeight int32) *wire.MsgTx {
 	txIn := &wire.TxIn{
 		// Coinbase transactions have no inputs, so previous outpoint is
 		// zero hash and max index.
-		PreviousOutPoint: *wire.NewOutPoint(&daghash.TxID{},
+		PreviousOutpoint: *wire.NewOutpoint(&daghash.TxID{},
 			wire.MaxPrevOutIndex),
 		Sequence:        wire.MaxTxInSequenceNum,
 		SignatureScript: coinbaseScript,
 	}
 	txOut := &wire.TxOut{
-		Value:    blockdag.CalcBlockSubsidy(blockHeight, g.params),
-		PkScript: opTrueScript,
+		Value:        blockdag.CalcBlockSubsidy(blueScore, g.params),
+		ScriptPubKey: opTrueScript,
 	}
 	return wire.NewNativeMsgTx(1, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
 }
@@ -407,9 +410,9 @@ func additionalSpendFee(fee util.Amount) func(*wire.MsgBlock) {
 
 // replaceSpendScript returns a function that itself takes a block and modifies
 // it by replacing the public key script of the spending transaction.
-func replaceSpendScript(pkScript []byte) func(*wire.MsgBlock) {
+func replaceSpendScript(scriptPubKey []byte) func(*wire.MsgBlock) {
 	return func(b *wire.MsgBlock) {
-		b.Transactions[1].TxOut[0].PkScript = pkScript
+		b.Transactions[1].TxOut[0].ScriptPubKey = scriptPubKey
 	}
 }
 
@@ -436,7 +439,7 @@ func additionalTx(tx *wire.MsgTx) func(*wire.MsgBlock) {
 // tests.
 func createSpendTx(spend *spendableOut, fee util.Amount) *wire.MsgTx {
 	txIn := &wire.TxIn{
-		PreviousOutPoint: spend.prevOut,
+		PreviousOutpoint: spend.prevOut,
 		Sequence:         wire.MaxTxInSequenceNum,
 		SignatureScript:  nil,
 	}
@@ -512,7 +515,7 @@ func (g *testGenerator) nextBlock(blockName string, spend *spendableOut, mungers
 			Version:        1,
 			ParentHashes:   []*daghash.Hash{g.tip.BlockHash()}, // TODO: (Stas) This is wrong. Modified only to satisfy compilation.
 			HashMerkleRoot: calcHashMerkleRoot(txns),
-			Bits:           g.params.PowLimitBits,
+			Bits:           g.powMaxBits,
 			Timestamp:      ts,
 			Nonce:          0, // To be solved.
 		},
@@ -606,7 +609,8 @@ func (g *testGenerator) saveSpendableCoinbaseOuts() {
 	// reaching the block that has already had the coinbase outputs
 	// collected.
 	var collectBlocks []*wire.MsgBlock
-	for b := g.tip; b != nil; b = g.blocks[*b.Header.SelectedParentHash()] {
+	// TODO: (Evgeny) This is wrong. Modified only to satisfy compilation.
+	for b := g.tip; b != nil; b = g.blocks[*b.Header.ParentHashes[0]] {
 		if b.BlockHash() == g.prevCollectedHash {
 			break
 		}
@@ -680,7 +684,7 @@ func countBlockSigOps(block *wire.MsgBlock) int {
 			totalSigOps += numSigOps
 		}
 		for _, txOut := range tx.TxOut {
-			numSigOps := txscript.GetSigOpCount(txOut.PkScript)
+			numSigOps := txscript.GetSigOpCount(txOut.ScriptPubKey)
 			totalSigOps += numSigOps
 		}
 	}
@@ -773,7 +777,7 @@ func (g *testGenerator) assertTipBlockTxOutOpReturn(txIndex, txOutIndex uint32)
 	}
 
 	txOut := tx.TxOut[txOutIndex]
-	if txOut.PkScript[0] != txscript.OpReturn {
+	if txOut.ScriptPubKey[0] != txscript.OpReturn {
 		panic(fmt.Sprintf("transaction index %d output %d in block %q "+
 			"(height %d) is not an OP_RETURN", txIndex, txOutIndex,
 			g.tipName, g.tipHeight))
@@ -836,7 +840,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	// block to be the current tip of the block chain.
 	acceptBlock := func(blockName string, block *wire.MsgBlock, isOrphan bool) TestInstance {
 		blockHeight := g.blockHeights[blockName]
-		return AcceptedBlock{blockName, block, blockHeight, isOrphan}
+		return AcceptedBlock{blockName, block, blockHeight, isOrphan, 0}
 	}
 	rejectBlock := func(blockName string, block *wire.MsgBlock, code blockdag.ErrorCode) TestInstance {
 		blockHeight := g.blockHeights[blockName]
@@ -910,9 +914,9 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	//   genesis -> bm0 -> bm1 -> ... -> bm99
 	// ---------------------------------------------------------------------
 
-	coinbaseMaturity := g.params.BlockRewardMaturity
+	coinbaseMaturity := g.params.BlockCoinbaseMaturity
 	var testInstances []TestInstance
-	for i := uint16(0); i < coinbaseMaturity; i++ {
+	for i := uint64(0); i < coinbaseMaturity; i++ {
 		blockName := fmt.Sprintf("bm%d", i)
 		g.nextBlock(blockName, nil)
 		g.saveTipCoinbaseOut()
@@ -923,7 +927,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 
 	// Collect spendable outputs.  This simplifies the code below.
 	var outs []*spendableOut
-	for i := uint16(0); i < coinbaseMaturity; i++ {
+	for i := uint64(0); i < coinbaseMaturity; i++ {
 		op := g.oldestCoinbaseOut()
 		outs = append(outs, &op)
 	}
@@ -1001,45 +1005,6 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	// Too much proof-of-work coinbase tests.
 	// ---------------------------------------------------------------------
 
-	// Create a block that generates too coinbase.
-	//
-	//   ... -> b1(0) -> b2(1) -> b5(2) -> b6(3)
-	//                                         \-> b9(4)
-	//               \-> b3(1) -> b4(2)
-	g.setTip("b6")
-	g.nextBlock("b9", outs[4], additionalCoinbase(1))
-	rejected(blockdag.ErrBadCoinbaseValue)
-
-	// Create a fork that ends with block that generates too much coinbase.
-	//
-	//   ... -> b1(0) -> b2(1) -> b5(2) -> b6(3)
-	//                                 \-> b10(3) -> b11(4)
-	//               \-> b3(1) -> b4(2)
-	g.setTip("b5")
-	g.nextBlock("b10", outs[3])
-	acceptedToSideChainWithExpectedTip("b6")
-
-	g.nextBlock("b11", outs[4], additionalCoinbase(1))
-	rejected(blockdag.ErrBadCoinbaseValue)
-
-	// Create a fork that ends with block that generates too much coinbase
-	// as before, but with a valid fork first.
-	//
-	//   ... -> b1(0) -> b2(1) -> b5(2) -> b6(3)
-	//              |                  \-> b12(3) -> b13(4) -> b14(5)
-	//              |                      (b12 added last)
-	//               \-> b3(1) -> b4(2)
-	g.setTip("b5")
-	b12 := g.nextBlock("b12", outs[3])
-	b13 := g.nextBlock("b13", outs[4])
-	b14 := g.nextBlock("b14", outs[5], additionalCoinbase(1))
-	tests = append(tests, []TestInstance{
-		acceptBlock("b13", b13, true),
-		acceptBlock("b14", b14, true),
-		rejectBlock("b12", b12, blockdag.ErrBadCoinbaseValue),
-		expectTipBlock("b13", b13),
-	})
-
 	// ---------------------------------------------------------------------
 	// Checksig signature operation count tests.
 	// ---------------------------------------------------------------------
@@ -1141,7 +1106,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 		replaceSpendScript(sizePadScript)(b)
 	})
 	g.assertTipBlockSize(maxBlockSize + 1)
-	rejected(blockdag.ErrBlockTooBig)
+	rejected(blockdag.ErrBlockMassTooHigh)
 
 	// Parent was rejected, so this block must either be an orphan or
 	// outright rejected due to an invalid parent.
@@ -1161,7 +1126,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	g.setTip("b15")
 	tooSmallCbScript := repeatOpcode(0x00, minCoinbaseScriptLen-1)
 	g.nextBlock("b26", outs[6], replaceCoinbaseSigScript(tooSmallCbScript))
-	rejected(blockdag.ErrBadCoinbaseScriptLen)
+	rejected(blockdag.ErrBadCoinbasePayloadLen)
 
 	// Parent was rejected, so this block must either be an orphan or
 	// outright rejected due to an invalid parent.
@@ -1177,7 +1142,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	g.setTip("b15")
 	tooLargeCbScript := repeatOpcode(0x00, maxCoinbaseScriptLen+1)
 	g.nextBlock("b28", outs[6], replaceCoinbaseSigScript(tooLargeCbScript))
-	rejected(blockdag.ErrBadCoinbaseScriptLen)
+	rejected(blockdag.ErrBadCoinbasePayloadLen)
 
 	// Parent was rejected, so this block must either be an orphan or
 	// outright rejected due to an invalid parent.
@@ -1349,7 +1314,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 		fill := maxBlockSigOps - (txnsNeeded * redeemScriptSigOps) + 1
 		finalTx := b.Transactions[len(b.Transactions)-1]
 		tx := createSpendTxForTx(finalTx, lowFee)
-		tx.TxOut[0].PkScript = repeatOpcode(txscript.OpCheckSig, fill)
+		tx.TxOut[0].ScriptPubKey = repeatOpcode(txscript.OpCheckSig, fill)
 		b.AddTransaction(tx)
 	})
 	rejected(blockdag.ErrTooManySigOps)
@@ -1383,7 +1348,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 		}
 		finalTx := b.Transactions[len(b.Transactions)-1]
 		tx := createSpendTxForTx(finalTx, lowFee)
-		tx.TxOut[0].PkScript = repeatOpcode(txscript.OpCheckSig, fill)
+		tx.TxOut[0].ScriptPubKey = repeatOpcode(txscript.OpCheckSig, fill)
 		b.AddTransaction(tx)
 	})
 	accepted()
@@ -1444,7 +1409,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 			b46.Header.Nonce++
 			blockHash := b46.BlockHash()
 			hashNum := daghash.HashToBig(blockHash)
-			if hashNum.Cmp(g.params.PowLimit) >= 0 {
+			if hashNum.Cmp(g.params.PowMax) >= 0 {
 				break
 			}
 		}
@@ -1531,8 +1496,8 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	g.nextBlock("b52", outs[14], func(b *wire.MsgBlock) {
 		txID := newTxIDFromStr("00000000000000000000000000000000" +
 			"00000000000000000123456789abcdef")
-		b.Transactions[1].TxIn[0].PreviousOutPoint.TxID = *txID
-		b.Transactions[1].TxIn[0].PreviousOutPoint.Index = 0
+		b.Transactions[1].TxIn[0].PreviousOutpoint.TxID = *txID
+		b.Transactions[1].TxIn[0].PreviousOutpoint.Index = 0
 	})
 	rejected(blockdag.ErrMissingTxOut)
 
@@ -1553,9 +1518,11 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	//   ... -> b33(9) -> b35(10) -> b39(11) -> b42(12) -> b43(13) -> b53(14)
 	//                                                                       \-> b54(15)
 	g.nextBlock("b54", outs[15], func(b *wire.MsgBlock) {
-		medianBlock := g.blocks[*b.Header.SelectedParentHash()]
+		// TODO: (Evgeny) This is wrong. Modified only to satisfy compilation.
+		medianBlock := g.blocks[*b.Header.ParentHashes[0]]
 		for i := 0; i < medianTimeBlocks/2; i++ {
-			medianBlock = g.blocks[*medianBlock.Header.SelectedParentHash()]
+			// TODO: (Evgeny) This is wrong. Modified only to satisfy compilation.
+			medianBlock = g.blocks[*medianBlock.Header.ParentHashes[0]]
 		}
 		b.Header.Timestamp = medianBlock.Header.Timestamp
 	})
@@ -1567,9 +1534,11 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	//   ... -> b33(9) -> b35(10) -> b39(11) -> b42(12) -> b43(13) -> b53(14) -> b55(15)
 	g.setTip("b53")
 	g.nextBlock("b55", outs[15], func(b *wire.MsgBlock) {
-		medianBlock := g.blocks[*b.Header.SelectedParentHash()]
+		// TODO: (Evgeny) This is wrong. Modified only to satisfy compilation.
+		medianBlock := g.blocks[*b.Header.ParentHashes[0]]
 		for i := 0; i < medianTimeBlocks/2; i++ {
-			medianBlock = g.blocks[*medianBlock.Header.SelectedParentHash()]
+			// TODO: (Evgeny) This is wrong. Modified only to satisfy compilation.
+			medianBlock = g.blocks[*medianBlock.Header.ParentHashes[0]]
 		}
 		medianBlockTime := medianBlock.Header.Timestamp
 		b.Header.Timestamp = medianBlockTime.Add(time.Second)
@@ -1684,7 +1653,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	//                 \-> b58(17)
 	g.setTip("b57")
 	g.nextBlock("b58", outs[17], func(b *wire.MsgBlock) {
-		b.Transactions[1].TxIn[0].PreviousOutPoint.Index = 42
+		b.Transactions[1].TxIn[0].PreviousOutpoint.Index = 42
 	})
 	rejected(blockdag.ErrMissingTxOut)
 
@@ -1717,7 +1686,8 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	g.nextBlock("b61", outs[18], func(b *wire.MsgBlock) {
 		// Duplicate the coinbase of the parent block to force the
 		// condition.
-		parent := g.blocks[*b.Header.SelectedParentHash()]
+		// TODO: (Evgeny) This is wrong. Modified only to satisfy compilation.
+		parent := g.blocks[*b.Header.ParentHashes[0]]
 		b.Transactions[0] = parent.Transactions[0]
 	})
 	rejected(blockdag.ErrOverwriteTx)
@@ -1842,7 +1812,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	//                 \-> b68(20)
 	g.setTip("b65")
 	g.nextBlock("b68", outs[20], additionalCoinbase(10), additionalSpendFee(9))
-	rejected(blockdag.ErrBadCoinbaseValue)
+	rejected(blockdag.ErrBadCoinbaseTransaction)
 
 	// Create block that pays 10 extra to the coinbase and a tx that pays
 	// the extra 10 fee.
@@ -2059,7 +2029,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	// Collect all of the spendable coinbase outputs from the previous
 	// collection point up to the current tip.
 	g.saveSpendableCoinbaseOuts()
-	spendableOutOffset := g.tipHeight - int32(coinbaseMaturity)
+	spendableOutOffset := g.tipHeight - coinbaseMaturity
 
 	// Extend the main chain by a large number of max size blocks.
 	//
@@ -2068,7 +2038,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 	reorgSpend := *outs[spendableOutOffset]
 	reorgStartBlockName := g.tipName
 	chain1TipName := g.tipName
-	for i := int32(0); i < numLargeReorgBlocks; i++ {
+	for i := uint64(0); i < numLargeReorgBlocks; i++ {
 		chain1TipName = fmt.Sprintf("br%d", i)
 		g.nextBlock(chain1TipName, &reorgSpend, func(b *wire.MsgBlock) {
 			bytesToMaxSize := maxBlockSize - b.SerializeSize() - 3
@@ -2083,7 +2053,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) {
 		// Use the next available spendable output.  First use up any
 		// remaining spendable outputs that were already popped into the
 		// outs slice, then just pop them from the stack.
-		if spendableOutOffset+1+i < int32(len(outs)) {
+		if spendableOutOffset+1+i < uint64(len(outs)) {
 			reorgSpend = *outs[spendableOutOffset+1+i]
 		} else {
 			reorgSpend = g.oldestCoinbaseOut()

blockdag/fullblocktests/params.go

@@ -10,12 +10,12 @@ import (
 	"math/big"
 	"time"
 
-	"github.com/daglabs/btcd/util/hdkeychain"
-	"github.com/daglabs/btcd/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/hdkeychain"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // newHashFromStr converts the passed big-endian hex string into a
@@ -77,7 +77,7 @@ var (
 		Transactions: []*wire.MsgTx{{
 			Version: 1,
 			TxIn: []*wire.TxIn{{
-				PreviousOutPoint: wire.OutPoint{
+				PreviousOutpoint: wire.Outpoint{
 					TxID:  daghash.TxID{},
 					Index: 0xffffffff,
 				},
@@ -90,7 +90,7 @@ var (
 			}},
 			TxOut: []*wire.TxOut{{
 				Value: 0,
-				PkScript: fromHex("4104678afdb0fe5548271967f1" +
+				ScriptPubKey: fromHex("4104678afdb0fe5548271967f1" +
 					"a67130b7105cd6a828e03909a67962e0ea1f" +
 					"61deb649f6bc3f4cef38c4f35504e51ec138" +
 					"c4f35504e51ec112de5c384df7ba0b8d578a" +
@@ -111,22 +111,19 @@ var (
 // allow them to change out from under the tests potentially invalidating them.
 var regressionNetParams = &dagconfig.Params{
 	Name:        "regtest",
-	Net:         wire.TestNet,
+	Net:         wire.RegTest,
 	DefaultPort: "18444",
 
-	// Chain parameters
-	GenesisBlock:             &regTestGenesisBlock,
-	GenesisHash:              newHashFromStr("5bec7567af40504e0994db3b573c186fffcc4edefe096ff2e58d00523bd7e8a6"),
-	PowLimit:                 regressionPowLimit,
-	PowLimitBits:             0x207fffff,
-	BlockRewardMaturity:      100,
-	SubsidyReductionInterval: 150,
-	TargetTimespan:           time.Hour * 24 * 14, // 14 days
-	TargetTimePerBlock:       time.Second * 10,    // 10 seconds
-	RetargetAdjustmentFactor: 4,                   // 25% less, 400% more
-	ReduceMinDifficulty:      true,
-	MinDiffReductionTime:     time.Minute * 20, // TargetTimePerBlock * 2
-	GenerateSupported:        true,
+	// DAG parameters
+	GenesisBlock:                   &regTestGenesisBlock,
+	GenesisHash:                    newHashFromStr("5bec7567af40504e0994db3b573c186fffcc4edefe096ff2e58d00523bd7e8a6"),
+	PowMax:                         regressionPowLimit,
+	BlockCoinbaseMaturity:          100,
+	SubsidyReductionInterval:       150,
+	TargetTimePerBlock:             time.Second * 10, // 10 seconds
+	DifficultyAdjustmentWindowSize: 2640,
+	TimestampDeviationTolerance:    132,
+	GenerateSupported:              true,
 
 	// Checkpoints ordered from oldest to newest.
 	Checkpoints: nil,

blockdag/indexers/README.md

@@ -3,7 +3,7 @@ indexers
 
 [![Build Status](https://travis-ci.org/btcsuite/btcd.png?branch=master)](https://travis-ci.org/btcsuite/btcd)
 [![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
-[![GoDoc](https://godoc.org/github.com/daglabs/btcd/blockchain/indexers?status.png)](http://godoc.org/github.com/daglabs/btcd/blockchain/indexers)
+[![GoDoc](https://godoc.org/github.com/kaspanet/kaspad/blockchain/indexers?status.png)](http://godoc.org/github.com/kaspanet/kaspad/blockchain/indexers)
 
 Package indexers implements optional block chain indexes.
 
@@ -23,7 +23,7 @@ via an RPC interface.
 ## Installation
 
 ```bash
-$ go get -u github.com/daglabs/btcd/blockchain/indexers
+$ go get -u github.com/kaspanet/kaspad/blockchain/indexers
 ```
 
 ## License

blockdag/indexers/acceptanceindex.go

@@ -0,0 +1,234 @@
+package indexers
+
+import (
+	"bytes"
+	"encoding/gob"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
+)
+
+const (
+	// acceptanceIndexName is the human-readable name for the index.
+	acceptanceIndexName = "acceptance index"
+)
+
+var (
+	// acceptanceIndexKey is the key of the acceptance index and the db bucket used
+	// to house it.
+	acceptanceIndexKey = []byte("acceptanceidx")
+)
+
+// AcceptanceIndex implements a txAcceptanceData by block hash index. That is to say,
+// it stores a mapping between a block's hash and the set of transactions that the
+// block accepts among its blue blocks.
+type AcceptanceIndex struct {
+	db  database.DB
+	dag *blockdag.BlockDAG
+}
+
+// Ensure the AcceptanceIndex type implements the Indexer interface.
+var _ Indexer = (*AcceptanceIndex)(nil)
+
+// NewAcceptanceIndex returns a new instance of an indexer that is used to create a
+// mapping between block hashes and their txAcceptanceData.
+//
+// It implements the Indexer interface which plugs into the IndexManager that in
+// turn is used by the blockdag package. This allows the index to be
+// seamlessly maintained along with the DAG.
+func NewAcceptanceIndex() *AcceptanceIndex {
+	return &AcceptanceIndex{}
+}
+
+// DropAcceptanceIndex drops the acceptance index from the provided database if it
+// exists.
+func DropAcceptanceIndex(db database.DB, interrupt <-chan struct{}) error {
+	return dropIndex(db, acceptanceIndexKey, acceptanceIndexName, interrupt)
+}
+
+// Key returns the database key to use for the index as a byte slice.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) Key() []byte {
+	return acceptanceIndexKey
+}
+
+// Name returns the human-readable name of the index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) Name() string {
+	return acceptanceIndexName
+}
+
+// Create is invoked when the indexer manager determines the index needs
+// to be created for the first time.  It creates the bucket for the
+// acceptance index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) Create(dbTx database.Tx) error {
+	_, err := dbTx.Metadata().CreateBucket(acceptanceIndexKey)
+	return err
+}
+
+// Init initializes the hash-based acceptance index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) Init(db database.DB, dag *blockdag.BlockDAG) error {
+	idx.db = db
+	idx.dag = dag
+	return nil
+}
+
+// ConnectBlock is invoked by the index manager when a new block has been
+// connected to the DAG.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) ConnectBlock(dbTx database.Tx, _ *util.Block, blockID uint64, _ *blockdag.BlockDAG,
+	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, _ blockdag.MultiBlockTxsAcceptanceData) error {
+	return dbPutTxsAcceptanceData(dbTx, blockID, txsAcceptanceData)
+}
+
+// TxsAcceptanceData returns the acceptance data of all the transactions that
+// were accepted by the block with hash blockHash.
+func (idx *AcceptanceIndex) TxsAcceptanceData(blockHash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	var txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData
+	err := idx.db.View(func(dbTx database.Tx) error {
+		var err error
+		txsAcceptanceData, err = dbFetchTxsAcceptanceDataByHash(dbTx, blockHash)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return txsAcceptanceData, nil
+}
+
+// Recover is invoked when the indexer wasn't turned on for several blocks
+// and the indexer needs to close the gaps.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
+	for blockID := currentBlockID + 1; blockID <= lastKnownBlockID; blockID++ {
+		hash, err := blockdag.DBFetchBlockHashByID(dbTx, currentBlockID)
+		if err != nil {
+			return err
+		}
+		txAcceptanceData, err := idx.dag.TxsAcceptedByBlockHash(hash)
+		if err != nil {
+			return err
+		}
+		err = idx.ConnectBlock(dbTx, nil, blockID, nil, txAcceptanceData, nil)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func dbPutTxsAcceptanceData(dbTx database.Tx, blockID uint64,
+	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+	serializedTxsAcceptanceData, err := serializeMultiBlockTxsAcceptanceData(txsAcceptanceData)
+	if err != nil {
+		return err
+	}
+
+	bucket := dbTx.Metadata().Bucket(acceptanceIndexKey)
+	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedTxsAcceptanceData)
+}
+
+func dbFetchTxsAcceptanceDataByHash(dbTx database.Tx,
+	hash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
+
+	blockID, err := blockdag.DBFetchBlockIDByHash(dbTx, hash)
+	if err != nil {
+		return nil, err
+	}
+
+	return dbFetchTxsAcceptanceDataByID(dbTx, blockID)
+}
+
+func dbFetchTxsAcceptanceDataByID(dbTx database.Tx,
+	blockID uint64) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	serializedBlockID := blockdag.SerializeBlockID(blockID)
+	bucket := dbTx.Metadata().Bucket(acceptanceIndexKey)
+	serializedTxsAcceptanceData := bucket.Get(serializedBlockID)
+	if serializedTxsAcceptanceData == nil {
+		return nil, errors.Errorf("no entry in the accpetance index for block id %d", blockID)
+	}
+
+	return deserializeMultiBlockTxsAcceptanceData(serializedTxsAcceptanceData)
+}
+
+type serializableTxAcceptanceData struct {
+	MsgTx      wire.MsgTx
+	IsAccepted bool
+}
+
+type serializableBlockTxsAcceptanceData struct {
+	BlockHash        daghash.Hash
+	TxAcceptanceData []serializableTxAcceptanceData
+}
+
+type serializableMultiBlockTxsAcceptanceData []serializableBlockTxsAcceptanceData
+
+func serializeMultiBlockTxsAcceptanceData(
+	multiBlockTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) ([]byte, error) {
+	// Convert MultiBlockTxsAcceptanceData to a serializable format
+	serializableData := make(serializableMultiBlockTxsAcceptanceData, len(multiBlockTxsAcceptanceData))
+	for i, blockTxsAcceptanceData := range multiBlockTxsAcceptanceData {
+		serializableBlockData := serializableBlockTxsAcceptanceData{
+			BlockHash:        blockTxsAcceptanceData.BlockHash,
+			TxAcceptanceData: make([]serializableTxAcceptanceData, len(blockTxsAcceptanceData.TxAcceptanceData)),
+		}
+		for i, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
+			serializableBlockData.TxAcceptanceData[i] = serializableTxAcceptanceData{
+				MsgTx:      *txAcceptanceData.Tx.MsgTx(),
+				IsAccepted: txAcceptanceData.IsAccepted,
+			}
+		}
+		serializableData[i] = serializableBlockData
+	}
+
+	// Serialize
+	var buffer bytes.Buffer
+	encoder := gob.NewEncoder(&buffer)
+	err := encoder.Encode(serializableData)
+	if err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+func deserializeMultiBlockTxsAcceptanceData(
+	serializedTxsAcceptanceData []byte) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	// Deserialize
+	buffer := bytes.NewBuffer(serializedTxsAcceptanceData)
+	decoder := gob.NewDecoder(buffer)
+	var serializedData serializableMultiBlockTxsAcceptanceData
+	err := decoder.Decode(&serializedData)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert serializable format to MultiBlockTxsAcceptanceData
+	multiBlockTxsAcceptanceData := make(blockdag.MultiBlockTxsAcceptanceData, len(serializedData))
+	for i, serializableBlockData := range serializedData {
+		blockTxsAcceptanceData := blockdag.BlockTxsAcceptanceData{
+			BlockHash:        serializableBlockData.BlockHash,
+			TxAcceptanceData: make([]blockdag.TxAcceptanceData, len(serializableBlockData.TxAcceptanceData)),
+		}
+		for i, txData := range serializableBlockData.TxAcceptanceData {
+			msgTx := txData.MsgTx
+			blockTxsAcceptanceData.TxAcceptanceData[i] = blockdag.TxAcceptanceData{
+				Tx:         util.NewTx(&msgTx),
+				IsAccepted: txData.IsAccepted,
+			}
+		}
+		multiBlockTxsAcceptanceData[i] = blockTxsAcceptanceData
+	}
+
+	return multiBlockTxsAcceptanceData, nil
+}

blockdag/indexers/acceptanceindex_test.go

@@ -0,0 +1,340 @@
+package indexers
+
+import (
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"syscall"
+	"testing"
+)
+
+func TestAcceptanceIndexSerializationAndDeserialization(t *testing.T) {
+	// Create test data
+	hash, _ := daghash.NewHashFromStr("1111111111111111111111111111111111111111111111111111111111111111")
+	txIn1 := &wire.TxIn{SignatureScript: []byte{1}, PreviousOutpoint: wire.Outpoint{Index: 1}, Sequence: 0}
+	txIn2 := &wire.TxIn{SignatureScript: []byte{2}, PreviousOutpoint: wire.Outpoint{Index: 2}, Sequence: 0}
+	txOut1 := &wire.TxOut{ScriptPubKey: []byte{1}, Value: 10}
+	txOut2 := &wire.TxOut{ScriptPubKey: []byte{2}, Value: 20}
+	blockTxsAcceptanceData := blockdag.BlockTxsAcceptanceData{
+		BlockHash: *hash,
+		TxAcceptanceData: []blockdag.TxAcceptanceData{
+			{
+				Tx:         util.NewTx(wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn1}, []*wire.TxOut{txOut1})),
+				IsAccepted: true,
+			},
+			{
+				Tx:         util.NewTx(wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn2}, []*wire.TxOut{txOut2})),
+				IsAccepted: false,
+			},
+		},
+	}
+	multiBlockTxsAcceptanceData := blockdag.MultiBlockTxsAcceptanceData{blockTxsAcceptanceData}
+
+	// Serialize
+	serializedTxsAcceptanceData, err := serializeMultiBlockTxsAcceptanceData(multiBlockTxsAcceptanceData)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexSerializationAndDeserialization: serialization failed: %s", err)
+	}
+
+	// Deserialize
+	deserializedTxsAcceptanceData, err := deserializeMultiBlockTxsAcceptanceData(serializedTxsAcceptanceData)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexSerializationAndDeserialization: deserialization failed: %s", err)
+	}
+
+	// Check that they're the same
+	if !reflect.DeepEqual(multiBlockTxsAcceptanceData, deserializedTxsAcceptanceData) {
+		t.Fatalf("TestAcceptanceIndexSerializationAndDeserialization: original data and deseralize data aren't equal")
+	}
+}
+
+// TestAcceptanceIndexRecover tests the recoverability of the
+// acceptance index.
+// It does it by following these steps:
+// * It creates a DAG with enabled acceptance index (let's call it dag1) and
+//   make it process some blocks.
+// * It creates a copy of dag1 (let's call it dag2), and disables the acceptance
+//   index in it.
+// * It processes two more blocks in both dag1 and dag2.
+// * A copy of dag2 is created (let's call it dag3) with enabled
+//   acceptance index
+// * It checks that the two missing blocks are added to dag3 acceptance index by
+//   comparing dag1's last block acceptance data and dag3's last block acceptance
+//   data.
+func TestAcceptanceIndexRecover(t *testing.T) {
+	params := &dagconfig.SimNetParams
+	params.BlockCoinbaseMaturity = 0
+
+	testFiles := []string{
+		"blk_0_to_4.dat",
+		"blk_3B.dat",
+	}
+
+	var blocks []*util.Block
+	for _, file := range testFiles {
+		blockTmp, err := blockdag.LoadBlocks(filepath.Join("../testdata/", file))
+		if err != nil {
+			t.Fatalf("Error loading file: %v\n", err)
+		}
+		blocks = append(blocks, blockTmp...)
+	}
+
+	db1AcceptanceIndex := NewAcceptanceIndex()
+	db1IndexManager := NewManager([]Indexer{db1AcceptanceIndex})
+	db1Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover1")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+	defer os.RemoveAll(db1Path)
+
+	db1, err := database.Create("ffldb", db1Path, params.Net)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	db1Config := blockdag.Config{
+		IndexManager: db1IndexManager,
+		DAGParams:    params,
+		DB:           db1,
+	}
+
+	db1DAG, teardown, err := blockdag.DAGSetup("", db1Config)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
+	}
+	if teardown != nil {
+		defer teardown()
+	}
+
+	for i := 1; i < len(blocks)-2; i++ {
+		isOrphan, delay, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		if err != nil {
+			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
+		}
+		if delay != 0 {
+			t.Fatalf("ProcessBlock: block %d "+
+				"is too far in the future", i)
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock incorrectly returned block %v "+
+				"is an orphan\n", i)
+		}
+	}
+
+	err = db1.FlushCache()
+	if err != nil {
+		t.Fatalf("Error flushing database to disk: %s", err)
+	}
+
+	db2Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover2")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+	defer os.RemoveAll(db2Path)
+
+	err = copyDirectory(db1Path, db2Path)
+	if err != nil {
+		t.Fatalf("copyDirectory: %s", err)
+	}
+
+	for i := len(blocks) - 2; i < len(blocks); i++ {
+		isOrphan, delay, err := db1DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		if err != nil {
+			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
+		}
+		if delay != 0 {
+			t.Fatalf("ProcessBlock: block %d "+
+				"is too far in the future", i)
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock incorrectly returned block %v "+
+				"is an orphan\n", i)
+		}
+	}
+
+	db1LastBlockAcceptanceData, err := db1AcceptanceIndex.TxsAcceptanceData(blocks[len(blocks)-1].Hash())
+	if err != nil {
+		t.Fatalf("Error fetching acceptance data: %s", err)
+	}
+
+	db2, err := database.Open("ffldb", db2Path, params.Net)
+	if err != nil {
+		t.Fatalf("Error opening database: %s", err)
+	}
+
+	db2Config := blockdag.Config{
+		DAGParams: params,
+		DB:        db2,
+	}
+
+	db2DAG, teardown, err := blockdag.DAGSetup("", db2Config)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
+	}
+	if teardown != nil {
+		defer teardown()
+	}
+
+	for i := len(blocks) - 2; i < len(blocks); i++ {
+		isOrphan, delay, err := db2DAG.ProcessBlock(blocks[i], blockdag.BFNone)
+		if err != nil {
+			t.Fatalf("ProcessBlock fail on block %v: %v\n", i, err)
+		}
+		if delay != 0 {
+			t.Fatalf("ProcessBlock: block %d "+
+				"is too far in the future", i)
+		}
+		if isOrphan {
+			t.Fatalf("ProcessBlock incorrectly returned block %v "+
+				"is an orphan\n", i)
+		}
+	}
+
+	err = db2.FlushCache()
+	if err != nil {
+		t.Fatalf("Error flushing database to disk: %s", err)
+	}
+	db3Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover3")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+	defer os.RemoveAll(db3Path)
+	err = copyDirectory(db2Path, db3Path)
+	if err != nil {
+		t.Fatalf("copyDirectory: %s", err)
+	}
+
+	db3, err := database.Open("ffldb", db3Path, params.Net)
+	if err != nil {
+		t.Fatalf("Error opening database: %s", err)
+	}
+
+	db3AcceptanceIndex := NewAcceptanceIndex()
+	db3IndexManager := NewManager([]Indexer{db3AcceptanceIndex})
+	db3Config := blockdag.Config{
+		IndexManager: db3IndexManager,
+		DAGParams:    params,
+		DB:           db3,
+	}
+
+	_, teardown, err = blockdag.DAGSetup("", db3Config)
+	if err != nil {
+		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
+	}
+	if teardown != nil {
+		defer teardown()
+	}
+
+	db3LastBlockAcceptanceData, err := db3AcceptanceIndex.TxsAcceptanceData(blocks[len(blocks)-1].Hash())
+	if err != nil {
+		t.Fatalf("Error fetching acceptance data: %s", err)
+	}
+	if !reflect.DeepEqual(db1LastBlockAcceptanceData, db3LastBlockAcceptanceData) {
+		t.Fatalf("recovery failed")
+	}
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func copyDirectory(scrDir, dest string) error {
+	entries, err := ioutil.ReadDir(scrDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(scrDir, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+		if !ok {
+			return errors.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
+		}
+
+		switch fileInfo.Mode() & os.ModeType {
+		case os.ModeDir:
+			if err := createIfNotExists(destPath, 0755); err != nil {
+				return err
+			}
+			if err := copyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		case os.ModeSymlink:
+			if err := copySymLink(sourcePath, destPath); err != nil {
+				return err
+			}
+		default:
+			if err := copyFile(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+
+		if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
+			return err
+		}
+
+		isSymlink := entry.Mode()&os.ModeSymlink != 0
+		if !isSymlink {
+			if err := os.Chmod(destPath, entry.Mode()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func copyFile(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	defer out.Close()
+	if err != nil {
+		return err
+	}
+
+	in, err := os.Open(srcFile)
+	defer in.Close()
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func createIfNotExists(dir string, perm os.FileMode) error {
+	if blockdag.FileExists(dir) {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return errors.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
+	}
+
+	return nil
+}
+
+// This function is copied and modified from this stackoverflow answer: https://stackoverflow.com/a/56314145/2413761
+func copySymLink(source, dest string) error {
+	link, err := os.Readlink(source)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(link, dest)
+}

blockdag/indexers/addrindex.go

@@ -5,17 +5,17 @@
 package indexers
 
 import (
-	"errors"
 	"fmt"
+	"github.com/pkg/errors"
 	"sync"
 
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/txscript"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 const (
@@ -51,9 +51,9 @@ const (
 	// hash.
 	addrKeyTypeScriptHash = 1
 
-	// Size of a transaction entry.  It consists of 4 bytes block id + 4
+	// Size of a transaction entry.  It consists of 8 bytes block id + 4
 	// bytes offset + 4 bytes length.
-	txEntrySize = 4 + 4 + 4
+	txEntrySize = 8 + 4 + 4
 )
 
 var (
@@ -117,11 +117,11 @@ var (
 //   [<block id><start offset><tx length>,...]
 //
 //   Field           Type      Size
-//   block id        uint32    4 bytes
+//   block id        uint64    8 bytes
 //   start offset    uint32    4 bytes
 //   tx length       uint32    4 bytes
 //   -----
-//   Total: 12 bytes per indexed tx
+//   Total: 16 bytes per indexed tx
 // -----------------------------------------------------------------------------
 
 // fetchBlockHashFunc defines a callback function to use in order to convert a
@@ -130,12 +130,12 @@ type fetchBlockHashFunc func(serializedID []byte) (*daghash.Hash, error)
 
 // serializeAddrIndexEntry serializes the provided block id and transaction
 // location according to the format described in detail above.
-func serializeAddrIndexEntry(blockID uint32, txLoc wire.TxLoc) []byte {
+func serializeAddrIndexEntry(blockID uint64, txLoc wire.TxLoc) []byte {
 	// Serialize the entry.
-	serialized := make([]byte, 12)
-	byteOrder.PutUint32(serialized, blockID)
-	byteOrder.PutUint32(serialized[4:], uint32(txLoc.TxStart))
-	byteOrder.PutUint32(serialized[8:], uint32(txLoc.TxLen))
+	serialized := make([]byte, 16)
+	byteOrder.PutUint64(serialized, blockID)
+	byteOrder.PutUint32(serialized[8:], uint32(txLoc.TxStart))
+	byteOrder.PutUint32(serialized[12:], uint32(txLoc.TxLen))
 	return serialized
 }
 
@@ -149,13 +149,13 @@ func deserializeAddrIndexEntry(serialized []byte, region *database.BlockRegion,
 		return errDeserialize("unexpected end of data")
 	}
 
-	hash, err := fetchBlockHash(serialized[0:4])
+	hash, err := fetchBlockHash(serialized[0:8])
 	if err != nil {
 		return err
 	}
 	region.Hash = hash
-	region.Offset = byteOrder.Uint32(serialized[4:8])
-	region.Len = byteOrder.Uint32(serialized[8:12])
+	region.Offset = byteOrder.Uint32(serialized[8:12])
+	region.Len = byteOrder.Uint32(serialized[12:16])
 	return nil
 }
 
@@ -170,7 +170,7 @@ func keyForLevel(addrKey [addrKeySize]byte, level uint8) [levelKeySize]byte {
 
 // dbPutAddrIndexEntry updates the address index to include the provided entry
 // according to the level-based scheme described in detail above.
-func dbPutAddrIndexEntry(bucket internalBucket, addrKey [addrKeySize]byte, blockID uint32, txLoc wire.TxLoc) error {
+func dbPutAddrIndexEntry(bucket internalBucket, addrKey [addrKeySize]byte, blockID uint64, txLoc wire.TxLoc) error {
 	// Start with level 0 and its initial max number of entries.
 	curLevel := uint8(0)
 	maxLevelBytes := level0MaxEntries * txEntrySize
@@ -528,12 +528,6 @@ func addrToKey(addr util.Address) ([addrKeySize]byte, error) {
 		result[0] = addrKeyTypeScriptHash
 		copy(result[1:], addr.Hash160()[:])
 		return result, nil
-
-	case *util.AddressPubKey:
-		var result [addrKeySize]byte
-		result[0] = addrKeyTypePubKeyHash
-		copy(result[1:], addr.AddressPubKeyHash().Hash160()[:])
-		return result, nil
 	}
 
 	return [addrKeySize]byte{}, errUnsupportedAddressType
@@ -591,7 +585,7 @@ func (idx *AddrIndex) NeedsInputs() bool {
 // initialize for this index.
 //
 // This is part of the Indexer interface.
-func (idx *AddrIndex) Init(db database.DB) error {
+func (idx *AddrIndex) Init(db database.DB, _ *blockdag.BlockDAG) error {
 	idx.db = db
 	return nil
 }
@@ -626,37 +620,35 @@ func (idx *AddrIndex) Create(dbTx database.Tx) error {
 // stored in the order they appear in the block.
 type writeIndexData map[[addrKeySize]byte][]int
 
-// indexPkScript extracts all standard addresses from the passed public key
+// indexScriptPubKey extracts all standard addresses from the passed public key
 // script and maps each of them to the associated transaction using the passed
 // map.
-func (idx *AddrIndex) indexPkScript(data writeIndexData, pkScript []byte, txIdx int) {
+func (idx *AddrIndex) indexScriptPubKey(data writeIndexData, scriptPubKey []byte, txIdx int) {
 	// Nothing to index if the script is non-standard or otherwise doesn't
 	// contain any addresses.
-	_, addrs, _, err := txscript.ExtractPkScriptAddrs(pkScript,
+	_, addr, err := txscript.ExtractScriptPubKeyAddress(scriptPubKey,
 		idx.dagParams)
-	if err != nil || len(addrs) == 0 {
+	if err != nil || addr == nil {
 		return
 	}
 
-	for _, addr := range addrs {
-		addrKey, err := addrToKey(addr)
-		if err != nil {
-			// Ignore unsupported address types.
-			continue
-		}
-
-		// Avoid inserting the transaction more than once.  Since the
-		// transactions are indexed serially any duplicates will be
-		// indexed in a row, so checking the most recent entry for the
-		// address is enough to detect duplicates.
-		indexedTxns := data[addrKey]
-		numTxns := len(indexedTxns)
-		if numTxns > 0 && indexedTxns[numTxns-1] == txIdx {
-			continue
-		}
-		indexedTxns = append(indexedTxns, txIdx)
-		data[addrKey] = indexedTxns
+	addrKey, err := addrToKey(addr)
+	if err != nil {
+		// Ignore unsupported address types.
+		return
 	}
+
+	// Avoid inserting the transaction more than once.  Since the
+	// transactions are indexed serially any duplicates will be
+	// indexed in a row, so checking the most recent entry for the
+	// address is enough to detect duplicates.
+	indexedTxns := data[addrKey]
+	numTxns := len(indexedTxns)
+	if numTxns > 0 && indexedTxns[numTxns-1] == txIdx {
+		return
+	}
+	indexedTxns = append(indexedTxns, txIdx)
+	data[addrKey] = indexedTxns
 }
 
 // indexBlock extract all of the standard addresses from all of the transactions
@@ -667,23 +659,23 @@ func (idx *AddrIndex) indexBlock(data writeIndexData, block *util.Block, dag *bl
 		// Coinbases do not reference any inputs.  Since the block is
 		// required to have already gone through full validation, it has
 		// already been proven on the first transaction in the block is
-		// a coinbase, and the second one is a fee transaction.
-		if txIdx > 1 {
+		// a coinbase.
+		if txIdx > util.CoinbaseTransactionIndex {
 			for _, txIn := range tx.MsgTx().TxIn {
 				// The UTXO should always have the input since
 				// the index contract requires it, however, be
 				// safe and simply ignore any missing entries.
-				entry, ok := dag.GetUTXOEntry(txIn.PreviousOutPoint)
+				entry, ok := dag.GetUTXOEntry(txIn.PreviousOutpoint)
 				if !ok {
 					continue
 				}
 
-				idx.indexPkScript(data, entry.PkScript(), txIdx)
+				idx.indexScriptPubKey(data, entry.ScriptPubKey(), txIdx)
 			}
 		}
 
 		for _, txOut := range tx.MsgTx().TxOut {
-			idx.indexPkScript(data, txOut.PkScript, txIdx)
+			idx.indexScriptPubKey(data, txOut.ScriptPubKey, txIdx)
 		}
 	}
 }
@@ -693,7 +685,9 @@ func (idx *AddrIndex) indexBlock(data writeIndexData, block *util.Block, dag *bl
 // the transactions in the block involve.
 //
 // This is part of the Indexer interface.
-func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block *util.Block, dag *blockdag.BlockDAG, _ blockdag.MultiBlockTxsAcceptanceData) error {
+func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
+	_ blockdag.MultiBlockTxsAcceptanceData, _ blockdag.MultiBlockTxsAcceptanceData) error {
+
 	// The offset and length of the transactions within the serialized
 	// block.
 	txLocs, err := block.TxLoc()
@@ -701,12 +695,6 @@ func (idx *AddrIndex) ConnectBlock(dbTx database.Tx, block *util.Block, dag *blo
 		return err
 	}
 
-	// Get the internal block ID associated with the block.
-	blockID, err := dbFetchBlockIDByHash(dbTx, block.Hash())
-	if err != nil {
-		return err
-	}
-
 	// Build all of the address to transaction mappings in a local map.
 	addrsToTxns := make(writeIndexData)
 	idx.indexBlock(addrsToTxns, block, dag)
@@ -772,7 +760,7 @@ func (idx *AddrIndex) TxRegionsForAddress(dbTx database.Tx, addr util.Address, n
 		// the database transaction.
 		fetchBlockHash := func(id []byte) (*daghash.Hash, error) {
 			// Deserialize and populate the result.
-			return dbFetchBlockHashBySerializedID(dbTx, id)
+			return blockdag.DBFetchBlockHashBySerializedID(dbTx, id)
 		}
 
 		var err error
@@ -791,37 +779,35 @@ func (idx *AddrIndex) TxRegionsForAddress(dbTx database.Tx, addr util.Address, n
 // script to the transaction.
 //
 // This function is safe for concurrent access.
-func (idx *AddrIndex) indexUnconfirmedAddresses(pkScript []byte, tx *util.Tx) {
+func (idx *AddrIndex) indexUnconfirmedAddresses(scriptPubKey []byte, tx *util.Tx) {
 	// The error is ignored here since the only reason it can fail is if the
 	// script fails to parse and it was already validated before being
 	// admitted to the mempool.
-	_, addresses, _, _ := txscript.ExtractPkScriptAddrs(pkScript,
+	_, addr, _ := txscript.ExtractScriptPubKeyAddress(scriptPubKey,
 		idx.dagParams)
-	for _, addr := range addresses {
-		// Ignore unsupported address types.
-		addrKey, err := addrToKey(addr)
-		if err != nil {
-			continue
-		}
-
-		// Add a mapping from the address to the transaction.
-		idx.unconfirmedLock.Lock()
-		addrIndexEntry := idx.txnsByAddr[addrKey]
-		if addrIndexEntry == nil {
-			addrIndexEntry = make(map[daghash.TxID]*util.Tx)
-			idx.txnsByAddr[addrKey] = addrIndexEntry
-		}
-		addrIndexEntry[*tx.ID()] = tx
-
-		// Add a mapping from the transaction to the address.
-		addrsByTxEntry := idx.addrsByTx[*tx.ID()]
-		if addrsByTxEntry == nil {
-			addrsByTxEntry = make(map[[addrKeySize]byte]struct{})
-			idx.addrsByTx[*tx.ID()] = addrsByTxEntry
-		}
-		addrsByTxEntry[addrKey] = struct{}{}
-		idx.unconfirmedLock.Unlock()
+	// Ignore unsupported address types.
+	addrKey, err := addrToKey(addr)
+	if err != nil {
+		return
 	}
+
+	// Add a mapping from the address to the transaction.
+	idx.unconfirmedLock.Lock()
+	addrIndexEntry := idx.txnsByAddr[addrKey]
+	if addrIndexEntry == nil {
+		addrIndexEntry = make(map[daghash.TxID]*util.Tx)
+		idx.txnsByAddr[addrKey] = addrIndexEntry
+	}
+	addrIndexEntry[*tx.ID()] = tx
+
+	// Add a mapping from the transaction to the address.
+	addrsByTxEntry := idx.addrsByTx[*tx.ID()]
+	if addrsByTxEntry == nil {
+		addrsByTxEntry = make(map[[addrKeySize]byte]struct{})
+		idx.addrsByTx[*tx.ID()] = addrsByTxEntry
+	}
+	addrsByTxEntry[addrKey] = struct{}{}
+	idx.unconfirmedLock.Unlock()
 }
 
 // AddUnconfirmedTx adds all addresses related to the transaction to the
@@ -840,19 +826,19 @@ func (idx *AddrIndex) AddUnconfirmedTx(tx *util.Tx, utxoSet blockdag.UTXOSet) {
 	// transaction has already been validated and thus all inputs are
 	// already known to exist.
 	for _, txIn := range tx.MsgTx().TxIn {
-		entry, ok := utxoSet.Get(txIn.PreviousOutPoint)
+		entry, ok := utxoSet.Get(txIn.PreviousOutpoint)
 		if !ok {
 			// Ignore missing entries.  This should never happen
 			// in practice since the function comments specifically
 			// call out all inputs must be available.
 			continue
 		}
-		idx.indexUnconfirmedAddresses(entry.PkScript(), tx)
+		idx.indexUnconfirmedAddresses(entry.ScriptPubKey(), tx)
 	}
 
 	// Index addresses of all created outputs.
 	for _, txOut := range tx.MsgTx().TxOut {
-		idx.indexUnconfirmedAddresses(txOut.PkScript, tx)
+		idx.indexUnconfirmedAddresses(txOut.ScriptPubKey, tx)
 	}
 }
 
@@ -907,6 +893,15 @@ func (idx *AddrIndex) UnconfirmedTxnsForAddress(addr util.Address) []*util.Tx {
 	return nil
 }
 
+// Recover is invoked when the indexer wasn't turned on for several blocks
+// and the indexer needs to close the gaps.
+//
+// This is part of the Indexer interface.
+func (idx *AddrIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
+	return errors.Errorf("addrindex was turned off for %d blocks and can't be recovered."+
+		" To resume working drop the addrindex with --dropaddrindex", lastKnownBlockID-currentBlockID)
+}
+
 // NewAddrIndex returns a new instance of an indexer that is used to create a
 // mapping of all addresses in the blockchain to the respective transactions
 // that involve them.

blockdag/indexers/addrindex_test.go

@@ -7,9 +7,10 @@ package indexers
 import (
 	"bytes"
 	"fmt"
+	"github.com/pkg/errors"
 	"testing"
 
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // addrIndexBucket provides a mock address index database bucket by implementing
@@ -127,17 +128,17 @@ func (b *addrIndexBucket) sanityCheck(addrKey [addrKeySize]byte, expectedTotal i
 			if (highestLevel != 0 && numEntries == 0) ||
 				numEntries > maxEntries {
 
-				return fmt.Errorf("level %d has %d entries",
+				return errors.Errorf("level %d has %d entries",
 					level, numEntries)
 			}
 		} else if numEntries != maxEntries && numEntries != maxEntries/2 {
-			return fmt.Errorf("level %d has %d entries", level,
+			return errors.Errorf("level %d has %d entries", level,
 				numEntries)
 		}
 		maxEntries *= 2
 	}
 	if totalEntries != expectedTotal {
-		return fmt.Errorf("expected %d entries - got %d", expectedTotal,
+		return errors.Errorf("expected %d entries - got %d", expectedTotal,
 			totalEntries)
 	}
 
@@ -151,7 +152,7 @@ func (b *addrIndexBucket) sanityCheck(addrKey [addrKeySize]byte, expectedTotal i
 			start := i * txEntrySize
 			num := byteOrder.Uint32(data[start:])
 			if num != expectedNum {
-				return fmt.Errorf("level %d offset %d does "+
+				return errors.Errorf("level %d offset %d does "+
 					"not contain the expected number of "+
 					"%d - got %d", level, i, num,
 					expectedNum)
@@ -222,7 +223,7 @@ nextTest:
 		for i := 0; i < test.numInsert; i++ {
 			txLoc := wire.TxLoc{TxStart: i * 2}
 			err := dbPutAddrIndexEntry(populatedBucket, test.key,
-				uint32(i), txLoc)
+				uint64(i), txLoc)
 			if err != nil {
 				t.Errorf("dbPutAddrIndexEntry #%d (%s) - "+
 					"unexpected error: %v", testNum,

blockdag/indexers/blocklogger.go

@@ -1,76 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package indexers
-
-import (
-	"sync"
-	"time"
-
-	"github.com/btcsuite/btclog"
-	"github.com/daglabs/btcd/util"
-)
-
-// blockProgressLogger provides periodic logging for other services in order
-// to show users progress of certain "actions" involving some or all current
-// blocks. Ex: syncing to best chain, indexing all blocks, etc.
-type blockProgressLogger struct {
-	receivedLogBlocks int64
-	receivedLogTx     int64
-	lastBlockLogTime  time.Time
-
-	subsystemLogger btclog.Logger
-	progressAction  string
-	sync.Mutex
-}
-
-// newBlockProgressLogger returns a new block progress logger.
-// The progress message is templated as follows:
-//  {progressAction} {numProcessed} {blocks|block} in the last {timePeriod}
-//  ({numTxs}, height {lastBlockHeight}, {lastBlockTimeStamp})
-func newBlockProgressLogger(progressMessage string, logger btclog.Logger) *blockProgressLogger {
-	return &blockProgressLogger{
-		lastBlockLogTime: time.Now(),
-		progressAction:   progressMessage,
-		subsystemLogger:  logger,
-	}
-}
-
-// LogBlockHeight logs a new block height as an information message to show
-// progress to the user. In order to prevent spam, it limits logging to one
-// message every 10 seconds with duration and totals included.
-func (b *blockProgressLogger) LogBlockHeight(block *util.Block) {
-	b.Lock()
-	defer b.Unlock()
-
-	b.receivedLogBlocks++
-	b.receivedLogTx += int64(len(block.MsgBlock().Transactions))
-
-	now := time.Now()
-	duration := now.Sub(b.lastBlockLogTime)
-	if duration < time.Second*10 {
-		return
-	}
-
-	// Truncate the duration to 10s of milliseconds.
-	durationMillis := int64(duration / time.Millisecond)
-	tDuration := 10 * time.Millisecond * time.Duration(durationMillis/10)
-
-	// Log information about new block height.
-	blockStr := "blocks"
-	if b.receivedLogBlocks == 1 {
-		blockStr = "block"
-	}
-	txStr := "transactions"
-	if b.receivedLogTx == 1 {
-		txStr = "transaction"
-	}
-	b.subsystemLogger.Infof("%s %d %s in the last %s (%d %s, height %d, %s)",
-		b.progressAction, b.receivedLogBlocks, blockStr, tDuration, b.receivedLogTx,
-		txStr, block.Height(), block.MsgBlock().Header.Timestamp)
-
-	b.receivedLogBlocks = 0
-	b.receivedLogTx = 0
-	b.lastBlockLogTime = now
-}

blockdag/indexers/cfindex.go

@@ -1,352 +0,0 @@
-// Copyright (c) 2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package indexers
-
-import (
-	"errors"
-
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/util/gcs"
-	"github.com/daglabs/btcd/util/gcs/builder"
-	"github.com/daglabs/btcd/wire"
-)
-
-const (
-	// cfIndexName is the human-readable name for the index.
-	cfIndexName = "committed filter index"
-)
-
-// Committed filters come in two flavours: basic and extended. They are
-// generated and dropped in pairs, and both are indexed by a block's hash.
-// Besides holding different content, they also live in different buckets.
-var (
-	// cfIndexParentBucketKey is the name of the parent bucket used to house
-	// the index. The rest of the buckets live below this bucket.
-	cfIndexParentBucketKey = []byte("cfindexparentbucket")
-
-	// cfIndexKeys is an array of db bucket names used to house indexes of
-	// block hashes to cfilters.
-	cfIndexKeys = [][]byte{
-		[]byte("cf0byhashidx"),
-		[]byte("cf1byhashidx"),
-	}
-
-	// cfHeaderKeys is an array of db bucket names used to house indexes of
-	// block hashes to cf headers.
-	cfHeaderKeys = [][]byte{
-		[]byte("cf0headerbyhashidx"),
-		[]byte("cf1headerbyhashidx"),
-	}
-
-	// cfHashKeys is an array of db bucket names used to house indexes of
-	// block hashes to cf hashes.
-	cfHashKeys = [][]byte{
-		[]byte("cf0hashbyhashidx"),
-		[]byte("cf1hashbyhashidx"),
-	}
-
-	maxFilterType = uint8(len(cfHeaderKeys) - 1)
-)
-
-// dbFetchFilterIdxEntry retrieves a data blob from the filter index database.
-// An entry's absence is not considered an error.
-func dbFetchFilterIdxEntry(dbTx database.Tx, key []byte, h *daghash.Hash) ([]byte, error) {
-	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
-	return idx.Get(h[:]), nil
-}
-
-// dbStoreFilterIdxEntry stores a data blob in the filter index database.
-func dbStoreFilterIdxEntry(dbTx database.Tx, key []byte, h *daghash.Hash, f []byte) error {
-	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
-	return idx.Put(h[:], f)
-}
-
-// dbDeleteFilterIdxEntry deletes a data blob from the filter index database.
-func dbDeleteFilterIdxEntry(dbTx database.Tx, key []byte, h *daghash.Hash) error {
-	idx := dbTx.Metadata().Bucket(cfIndexParentBucketKey).Bucket(key)
-	return idx.Delete(h[:])
-}
-
-// CfIndex implements a committed filter (cf) by hash index.
-type CfIndex struct {
-	db        database.DB
-	dagParams *dagconfig.Params
-}
-
-// Ensure the CfIndex type implements the Indexer interface.
-var _ Indexer = (*CfIndex)(nil)
-
-// Init initializes the hash-based cf index. This is part of the Indexer
-// interface.
-func (idx *CfIndex) Init(db database.DB) error {
-	idx.db = db
-	return nil
-}
-
-// Key returns the database key to use for the index as a byte slice. This is
-// part of the Indexer interface.
-func (idx *CfIndex) Key() []byte {
-	return cfIndexParentBucketKey
-}
-
-// Name returns the human-readable name of the index. This is part of the
-// Indexer interface.
-func (idx *CfIndex) Name() string {
-	return cfIndexName
-}
-
-// Create is invoked when the indexer manager determines the index needs to
-// be created for the first time. It creates buckets for the two hash-based cf
-// indexes (simple, extended).
-func (idx *CfIndex) Create(dbTx database.Tx) error {
-	meta := dbTx.Metadata()
-
-	cfIndexParentBucket, err := meta.CreateBucket(cfIndexParentBucketKey)
-	if err != nil {
-		return err
-	}
-
-	for _, bucketName := range cfIndexKeys {
-		_, err = cfIndexParentBucket.CreateBucket(bucketName)
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, bucketName := range cfHeaderKeys {
-		_, err = cfIndexParentBucket.CreateBucket(bucketName)
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, bucketName := range cfHashKeys {
-		_, err = cfIndexParentBucket.CreateBucket(bucketName)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// storeFilter stores a given filter, and performs the steps needed to
-// generate the filter's header.
-func storeFilter(dbTx database.Tx, block *util.Block, f *gcs.Filter,
-	filterType wire.FilterType) error {
-	if uint8(filterType) > maxFilterType {
-		return errors.New("unsupported filter type")
-	}
-
-	// Figure out which buckets to use.
-	fkey := cfIndexKeys[filterType]
-	hkey := cfHeaderKeys[filterType]
-	hashkey := cfHashKeys[filterType]
-
-	// Start by storing the filter.
-	h := block.Hash()
-	filterBytes, err := f.NBytes()
-	if err != nil {
-		return err
-	}
-	err = dbStoreFilterIdxEntry(dbTx, fkey, h, filterBytes)
-	if err != nil {
-		return err
-	}
-
-	// Next store the filter hash.
-	filterHash, err := builder.GetFilterHash(f)
-	if err != nil {
-		return err
-	}
-	err = dbStoreFilterIdxEntry(dbTx, hashkey, h, filterHash[:])
-	if err != nil {
-		return err
-	}
-
-	// Then fetch the previous block's filter header.
-	var prevHeader *daghash.Hash
-	header := block.MsgBlock().Header
-	if header.IsGenesis() {
-		prevHeader = &daghash.ZeroHash
-	} else {
-		ph := header.SelectedParentHash()
-		pfh, err := dbFetchFilterIdxEntry(dbTx, hkey, ph)
-		if err != nil {
-			return err
-		}
-
-		// Construct the new block's filter header, and store it.
-		prevHeader, err = daghash.NewHash(pfh)
-		if err != nil {
-			return err
-		}
-	}
-
-	fh, err := builder.MakeHeaderForFilter(f, prevHeader)
-	if err != nil {
-		return err
-	}
-	return dbStoreFilterIdxEntry(dbTx, hkey, h, fh[:])
-}
-
-// ConnectBlock is invoked by the index manager when a new block has been
-// connected to the main chain. This indexer adds a hash-to-cf mapping for
-// every passed block. This is part of the Indexer interface.
-func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *util.Block,
-	_ *blockdag.BlockDAG, _ blockdag.MultiBlockTxsAcceptanceData) error {
-
-	f, err := builder.BuildBasicFilter(block.MsgBlock())
-	if err != nil {
-		return err
-	}
-
-	err = storeFilter(dbTx, block, f, wire.GCSFilterRegular)
-	if err != nil {
-		return err
-	}
-
-	f, err = builder.BuildExtFilter(block.MsgBlock())
-	if err != nil {
-		return err
-	}
-
-	return storeFilter(dbTx, block, f, wire.GCSFilterExtended)
-}
-
-// DisconnectBlock is invoked by the index manager when a block has been
-// disconnected from the main chain.  This indexer removes the hash-to-cf
-// mapping for every passed block. This is part of the Indexer interface.
-func (idx *CfIndex) DisconnectBlock(dbTx database.Tx, block *util.Block,
-	_ *blockdag.BlockDAG) error {
-
-	for _, key := range cfIndexKeys {
-		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, key := range cfHeaderKeys {
-		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, key := range cfHashKeys {
-		err := dbDeleteFilterIdxEntry(dbTx, key, block.Hash())
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// entryByBlockHash fetches a filter index entry of a particular type
-// (eg. filter, filter header, etc) for a filter type and block hash.
-func (idx *CfIndex) entryByBlockHash(filterTypeKeys [][]byte,
-	filterType wire.FilterType, h *daghash.Hash) ([]byte, error) {
-
-	if uint8(filterType) > maxFilterType {
-		return nil, errors.New("unsupported filter type")
-	}
-	key := filterTypeKeys[filterType]
-
-	var entry []byte
-	err := idx.db.View(func(dbTx database.Tx) error {
-		var err error
-		entry, err = dbFetchFilterIdxEntry(dbTx, key, h)
-		return err
-	})
-	return entry, err
-}
-
-// entriesByBlockHashes batch fetches a filter index entry of a particular type
-// (eg. filter, filter header, etc) for a filter type and slice of block hashes.
-func (idx *CfIndex) entriesByBlockHashes(filterTypeKeys [][]byte,
-	filterType wire.FilterType, blockHashes []*daghash.Hash) ([][]byte, error) {
-
-	if uint8(filterType) > maxFilterType {
-		return nil, errors.New("unsupported filter type")
-	}
-	key := filterTypeKeys[filterType]
-
-	entries := make([][]byte, 0, len(blockHashes))
-	err := idx.db.View(func(dbTx database.Tx) error {
-		for _, blockHash := range blockHashes {
-			entry, err := dbFetchFilterIdxEntry(dbTx, key, blockHash)
-			if err != nil {
-				return err
-			}
-			entries = append(entries, entry)
-		}
-		return nil
-	})
-	return entries, err
-}
-
-// FilterByBlockHash returns the serialized contents of a block's basic or
-// extended committed filter.
-func (idx *CfIndex) FilterByBlockHash(h *daghash.Hash,
-	filterType wire.FilterType) ([]byte, error) {
-	return idx.entryByBlockHash(cfIndexKeys, filterType, h)
-}
-
-// FiltersByBlockHashes returns the serialized contents of a block's basic or
-// extended committed filter for a set of blocks by hash.
-func (idx *CfIndex) FiltersByBlockHashes(blockHashes []*daghash.Hash,
-	filterType wire.FilterType) ([][]byte, error) {
-	return idx.entriesByBlockHashes(cfIndexKeys, filterType, blockHashes)
-}
-
-// FilterHeaderByBlockHash returns the serialized contents of a block's basic
-// or extended committed filter header.
-func (idx *CfIndex) FilterHeaderByBlockHash(h *daghash.Hash,
-	filterType wire.FilterType) ([]byte, error) {
-	return idx.entryByBlockHash(cfHeaderKeys, filterType, h)
-}
-
-// FilterHeadersByBlockHashes returns the serialized contents of a block's basic
-// or extended committed filter header for a set of blocks by hash.
-func (idx *CfIndex) FilterHeadersByBlockHashes(blockHashes []*daghash.Hash,
-	filterType wire.FilterType) ([][]byte, error) {
-	return idx.entriesByBlockHashes(cfHeaderKeys, filterType, blockHashes)
-}
-
-// FilterHashByBlockHash returns the serialized contents of a block's basic
-// or extended committed filter hash.
-func (idx *CfIndex) FilterHashByBlockHash(h *daghash.Hash,
-	filterType wire.FilterType) ([]byte, error) {
-	return idx.entryByBlockHash(cfHashKeys, filterType, h)
-}
-
-// FilterHashesByBlockHashes returns the serialized contents of a block's basic
-// or extended committed filter hash for a set of blocks by hash.
-func (idx *CfIndex) FilterHashesByBlockHashes(blockHashes []*daghash.Hash,
-	filterType wire.FilterType) ([][]byte, error) {
-	return idx.entriesByBlockHashes(cfHashKeys, filterType, blockHashes)
-}
-
-// NewCfIndex returns a new instance of an indexer that is used to create a
-// mapping of the hashes of all blocks in the blockchain to their respective
-// committed filters.
-//
-// It implements the Indexer interface which plugs into the IndexManager that
-// in turn is used by the blockchain package. This allows the index to be
-// seamlessly maintained along with the chain.
-func NewCfIndex(dagParams *dagconfig.Params) *CfIndex {
-	return &CfIndex{dagParams: dagParams}
-}
-
-// DropCfIndex drops the CF index from the provided database if exists.
-func DropCfIndex(db database.DB, interrupt <-chan struct{}) error {
-	return dropIndex(db, cfIndexParentBucketKey, cfIndexName, interrupt)
-}

blockdag/indexers/common.go

@@ -9,11 +9,10 @@ package indexers
 
 import (
 	"encoding/binary"
-	"errors"
-
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
 )
 
 var (
@@ -48,11 +47,20 @@ type Indexer interface {
 	// Init is invoked when the index manager is first initializing the
 	// index.  This differs from the Create method in that it is called on
 	// every load, including the case the index was just created.
-	Init(db database.DB) error
+	Init(db database.DB, dag *blockdag.BlockDAG) error
 
 	// ConnectBlock is invoked when the index manager is notified that a new
 	// block has been connected to the DAG.
-	ConnectBlock(dbTx database.Tx, block *util.Block, dag *blockdag.BlockDAG, _ blockdag.MultiBlockTxsAcceptanceData) error
+	ConnectBlock(dbTx database.Tx,
+		block *util.Block,
+		blockID uint64,
+		dag *blockdag.BlockDAG,
+		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData,
+		virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error
+
+	// Recover is invoked when the indexer wasn't turned on for several blocks
+	// and the indexer needs to close the gaps.
+	Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error
 }
 
 // AssertError identifies an error that indicates an internal code consistency

blockdag/indexers/log.go

@@ -5,16 +5,9 @@
 package indexers
 
 import (
-	"github.com/btcsuite/btclog"
-	"github.com/daglabs/btcd/logger"
+	"github.com/kaspanet/kaspad/logger"
+	"github.com/kaspanet/kaspad/util/panics"
 )
 
-// log is a logger that is initialized with no output filters.  This
-// means the package will not perform any logging by default until the caller
-// requests it.
-var log btclog.Logger
-
-// The default amount of logging is none.
-func init() {
-	log, _ = logger.Get(logger.SubsystemTags.INDX)
-}
+var log, _ = logger.Get(logger.SubsystemTags.INDX)
+var spawn = panics.GoroutineWrapperFunc(log)

blockdag/indexers/manager.go

@@ -5,15 +5,18 @@
 package indexers
 
 import (
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 var (
 	// indexTipsBucketName is the name of the db bucket used to house the
 	// current tip of each index.
 	indexTipsBucketName = []byte("idxtips")
+
+	indexCurrentBlockIDBucketName = []byte("idxcurrentblockid")
 )
 
 // Manager defines an index manager that manages multiple optional indexes and
@@ -146,6 +149,9 @@ func (m *Manager) Init(db database.DB, blockDAG *blockdag.BlockDAG, interrupt <-
 		if err != nil {
 			return err
 		}
+		if _, err := meta.CreateBucketIfNotExists(indexCurrentBlockIDBucketName); err != nil {
+			return err
+		}
 
 		return m.maybeCreateIndexes(dbTx)
 	})
@@ -155,12 +161,35 @@ func (m *Manager) Init(db database.DB, blockDAG *blockdag.BlockDAG, interrupt <-
 
 	// Initialize each of the enabled indexes.
 	for _, indexer := range m.enabledIndexes {
-		if err := indexer.Init(db); err != nil {
+		if err := indexer.Init(db, blockDAG); err != nil {
 			return err
 		}
 	}
 
-	return nil
+	return m.recoverIfNeeded()
+}
+
+// recoverIfNeeded checks if the node worked for some time
+// without one of the current enabled indexes, and if it's
+// the case, recovers the missing blocks from the index.
+func (m *Manager) recoverIfNeeded() error {
+	return m.db.Update(func(dbTx database.Tx) error {
+		lastKnownBlockID := blockdag.DBFetchCurrentBlockID(dbTx)
+		for _, indexer := range m.enabledIndexes {
+			serializedCurrentIdxBlockID := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Get(indexer.Key())
+			currentIdxBlockID := uint64(0)
+			if serializedCurrentIdxBlockID != nil {
+				currentIdxBlockID = blockdag.DeserializeBlockID(serializedCurrentIdxBlockID)
+			}
+			if lastKnownBlockID > currentIdxBlockID {
+				err := indexer.Recover(dbTx, currentIdxBlockID, lastKnownBlockID)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
 }
 
 // ConnectBlock must be invoked when a block is extending the main chain.  It
@@ -168,12 +197,32 @@ func (m *Manager) Init(db database.DB, blockDAG *blockdag.BlockDAG, interrupt <-
 // checks, and invokes each indexer.
 //
 // This is part of the blockchain.IndexManager interface.
-func (m *Manager) ConnectBlock(dbTx database.Tx, block *util.Block, dag *blockdag.BlockDAG, txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+func (m *Manager) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
+	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+
 	// Call each of the currently active optional indexes with the block
 	// being connected so they can update accordingly.
 	for _, index := range m.enabledIndexes {
 		// Notify the indexer with the connected block so it can index it.
-		if err := index.ConnectBlock(dbTx, block, dag, txsAcceptanceData); err != nil {
+		if err := index.ConnectBlock(dbTx, block, blockID, dag, txsAcceptanceData, virtualTxsAcceptanceData); err != nil {
+			return err
+		}
+	}
+
+	// Add the new block ID index entry for the block being connected and
+	// update the current internal block ID accordingly.
+	err := m.updateIndexersWithCurrentBlockID(dbTx, block.Hash(), blockID)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Manager) updateIndexersWithCurrentBlockID(dbTx database.Tx, blockHash *daghash.Hash, blockID uint64) error {
+	serializedBlockID := blockdag.SerializeBlockID(blockID)
+	for _, index := range m.enabledIndexes {
+		err := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Put(index.Key(), serializedBlockID)
+		if err != nil {
 			return err
 		}
 	}
@@ -316,13 +365,6 @@ func dropIndex(db database.DB, idxKey []byte, idxName string, interrupt <-chan s
 		})
 	}
 
-	// Call extra index specific deinitialization for the transaction index.
-	if idxName == txIndexName {
-		if err := dropBlockIDIndex(db); err != nil {
-			return err
-		}
-	}
-
 	// Remove the index tip, index bucket, and in-progress drop flag now
 	// that all index entries have been removed.
 	err = db.Update(func(dbTx database.Tx) error {
@@ -332,6 +374,10 @@ func dropIndex(db database.DB, idxKey []byte, idxName string, interrupt <-chan s
 			return err
 		}
 
+		if err := meta.Bucket(indexCurrentBlockIDBucketName).Delete(idxKey); err != nil {
+			return err
+		}
+
 		return indexesBucket.Delete(indexDropKey(idxKey))
 	})
 	if err != nil {

blockdag/indexers/txindex.go

@@ -6,12 +6,12 @@ package indexers
 
 import (
 	"fmt"
-
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -19,40 +19,26 @@ const (
 	txIndexName = "transaction index"
 
 	includingBlocksIndexKeyEntrySize = 8 // 4 bytes for offset + 4 bytes for transaction length
-
-	acceptingBlocksIndexKeyEntrySize = 4 // 4 bytes for accepting block ID
 )
 
 var (
 	includingBlocksIndexKey = []byte("includingblocksidx")
 
 	acceptingBlocksIndexKey = []byte("acceptingblocksidx")
-
-	// idByHashIndexBucketName is the name of the db bucket used to house
-	// the block id -> block hash index.
-	idByHashIndexBucketName = []byte("idbyhashidx")
-
-	// hashByIDIndexBucketName is the name of the db bucket used to house
-	// the block hash -> block id index.
-	hashByIDIndexBucketName = []byte("hashbyididx")
 )
 
+// txsAcceptedByVirtual is the in-memory index of txIDs that were accepted
+// by the current virtual
+var txsAcceptedByVirtual map[daghash.TxID]bool
+
 // -----------------------------------------------------------------------------
 // The transaction index consists of an entry for every transaction in the DAG.
-// In order to significantly optimize the space requirements a separate
-// index which provides an internal mapping between each block that has been
-// indexed and a unique ID for use within the hash to location mappings.  The ID
-// is simply a sequentially incremented uint32.  This is useful because it is
-// only 4 bytes versus 32 bytes hashes and thus saves a ton of space in the
-// index.
 //
-// There are four buckets used in total. The first bucket maps the hash of
+// There are two buckets used in total. The first bucket maps the hash of
 // each transaction to its location in each block it's included in. The second bucket
 // contains all of the blocks that from their viewpoint the transaction has been
 // accepted (i.e. the transaction is found in their blue set without double spends),
-// and their blue block (or themselves) that included the transaction. The third
-// bucket maps the hash of each block to the unique ID and the fourth maps
-// that ID back to the block hash.
+// and their blue block (or themselves) that included the transaction.
 //
 // NOTE: Although it is technically possible for multiple transactions to have
 // the same hash as long as the previous transaction with the same hash is fully
@@ -67,123 +53,43 @@ var (
 //   <block id> = <start offset><tx length>
 //
 //   Field           Type              Size
-//   block id        uint32          4 bytes
+//   block id        uint64          8 bytes
 //   start offset    uint32          4 bytes
 //   tx length       uint32          4 bytes
 //   -----
-//   Total: 12 bytes
+//   Total: 16 bytes
 //
 // The accepting blocks index contains a sub bucket for each transaction hash (32 byte each), that its serialized format is:
 //
 //   <accepting block id> = <including block id>
 //
 //   Field           Type              Size
-//   accepting block id        uint32          4 bytes
-//   including block id        uint32          4 bytes
+//   accepting block id        uint64          8 bytes
+//   including block id        uint64          8 bytes
 //   -----
-//   Total: 8 bytes
-//
-// The serialized format for keys and values in the block hash to ID bucket is:
-//   <hash> = <ID>
-//
-//   Field           Type              Size
-//   hash            daghash.Hash    32 bytes
-//   ID              uint32            4 bytes
-//   -----
-//   Total: 36 bytes
-//
-// The serialized format for keys and values in the ID to block hash bucket is:
-//   <ID> = <hash>
-//
-//   Field           Type              Size
-//   ID              uint32            4 bytes
-//   hash            daghash.Hash    32 bytes
-//   -----
-//   Total: 36 bytes
+//   Total: 16 bytes
 //
 // -----------------------------------------------------------------------------
 
-// dbPutBlockIDIndexEntry uses an existing database transaction to update or add
-// the index entries for the hash to id and id to hash mappings for the provided
-// values.
-func dbPutBlockIDIndexEntry(dbTx database.Tx, hash *daghash.Hash, id uint32) error {
-	// Serialize the height for use in the index entries.
-	var serializedID [4]byte
-	byteOrder.PutUint32(serializedID[:], id)
-
-	// Add the block hash to ID mapping to the index.
-	meta := dbTx.Metadata()
-	hashIndex := meta.Bucket(idByHashIndexBucketName)
-	if err := hashIndex.Put(hash[:], serializedID[:]); err != nil {
-		return err
-	}
-
-	// Add the block ID to hash mapping to the index.
-	idIndex := meta.Bucket(hashByIDIndexBucketName)
-	return idIndex.Put(serializedID[:], hash[:])
-}
-
-// dbFetchBlockIDByHash uses an existing database transaction to retrieve the
-// block id for the provided hash from the index.
-func dbFetchBlockIDByHash(dbTx database.Tx, hash *daghash.Hash) (uint32, error) {
-	hashIndex := dbTx.Metadata().Bucket(idByHashIndexBucketName)
-	serializedID := hashIndex.Get(hash[:])
-	if serializedID == nil {
-		return 0, fmt.Errorf("No entry in the block ID index for block with hash %s", hash)
-	}
-
-	return byteOrder.Uint32(serializedID), nil
-}
-
-// dbFetchBlockHashBySerializedID uses an existing database transaction to
-// retrieve the hash for the provided serialized block id from the index.
-func dbFetchBlockHashBySerializedID(dbTx database.Tx, serializedID []byte) (*daghash.Hash, error) {
-	idIndex := dbTx.Metadata().Bucket(hashByIDIndexBucketName)
-	hashBytes := idIndex.Get(serializedID)
-	if hashBytes == nil {
-		return nil, fmt.Errorf("No entry in the block ID index for block with id %d", byteOrder.Uint32(serializedID))
-	}
-
-	var hash daghash.Hash
-	copy(hash[:], hashBytes)
-	return &hash, nil
-}
-
-// dbFetchBlockHashByID uses an existing database transaction to retrieve the
-// hash for the provided block id from the index.
-func dbFetchBlockHashByID(dbTx database.Tx, id uint32) (*daghash.Hash, error) {
-	var serializedID [4]byte
-	byteOrder.PutUint32(serializedID[:], id)
-	return dbFetchBlockHashBySerializedID(dbTx, serializedID[:])
-}
-
 func putIncludingBlocksEntry(target []byte, txLoc wire.TxLoc) {
 	byteOrder.PutUint32(target, uint32(txLoc.TxStart))
 	byteOrder.PutUint32(target[4:], uint32(txLoc.TxLen))
 }
 
-func putAcceptingBlocksEntry(target []byte, includingBlockID uint32) {
-	byteOrder.PutUint32(target, includingBlockID)
-}
-
-func dbPutIncludingBlocksEntry(dbTx database.Tx, txID *daghash.TxID, blockID uint32, serializedData []byte) error {
+func dbPutIncludingBlocksEntry(dbTx database.Tx, txID *daghash.TxID, blockID uint64, serializedData []byte) error {
 	bucket, err := dbTx.Metadata().Bucket(includingBlocksIndexKey).CreateBucketIfNotExists(txID[:])
 	if err != nil {
 		return err
 	}
-	blockIDBytes := make([]byte, 4)
-	byteOrder.PutUint32(blockIDBytes, uint32(blockID))
-	return bucket.Put(blockIDBytes, serializedData)
+	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedData)
 }
 
-func dbPutAcceptingBlocksEntry(dbTx database.Tx, txID *daghash.TxID, blockID uint32, serializedData []byte) error {
+func dbPutAcceptingBlocksEntry(dbTx database.Tx, txID *daghash.TxID, blockID uint64, serializedData []byte) error {
 	bucket, err := dbTx.Metadata().Bucket(acceptingBlocksIndexKey).CreateBucketIfNotExists(txID[:])
 	if err != nil {
 		return err
 	}
-	blockIDBytes := make([]byte, 4)
-	byteOrder.PutUint32(blockIDBytes, uint32(blockID))
-	return bucket.Put(blockIDBytes, serializedData)
+	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedData)
 }
 
 // dbFetchFirstTxRegion uses an existing database transaction to fetch the block
@@ -199,7 +105,7 @@ func dbFetchFirstTxRegion(dbTx database.Tx, txID *daghash.TxID) (*database.Block
 	if txBucket == nil {
 		return nil, database.Error{
 			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No block region"+
+			Description: fmt.Sprintf("No block region "+
 				"was found for %s", txID),
 		}
 	}
@@ -207,11 +113,11 @@ func dbFetchFirstTxRegion(dbTx database.Tx, txID *daghash.TxID) (*database.Block
 	if ok := cursor.First(); !ok {
 		return nil, database.Error{
 			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No block region"+
+			Description: fmt.Sprintf("No block region "+
 				"was found for %s", txID),
 		}
 	}
-	blockIDBytes := cursor.Key()
+	serializedBlockID := cursor.Key()
 	serializedData := cursor.Value()
 	if len(serializedData) == 0 {
 		return nil, nil
@@ -227,7 +133,7 @@ func dbFetchFirstTxRegion(dbTx database.Tx, txID *daghash.TxID) (*database.Block
 	}
 
 	// Load the block hash associated with the block ID.
-	hash, err := dbFetchBlockHashBySerializedID(dbTx, blockIDBytes)
+	hash, err := blockdag.DBFetchBlockHashBySerializedID(dbTx, serializedBlockID)
 	if err != nil {
 		return nil, database.Error{
 			ErrorCode: database.ErrCorruption,
@@ -247,7 +153,7 @@ func dbFetchFirstTxRegion(dbTx database.Tx, txID *daghash.TxID) (*database.Block
 
 // dbAddTxIndexEntries uses an existing database transaction to add a
 // transaction index entry for every transaction in the passed block.
-func dbAddTxIndexEntries(dbTx database.Tx, block *util.Block, blockID uint32, txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+func dbAddTxIndexEntries(dbTx database.Tx, block *util.Block, blockID uint64, multiBlockTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
 	// The offset and length of the transactions within the serialized
 	// block.
 	txLocs, err := block.TxLoc()
@@ -273,22 +179,21 @@ func dbAddTxIndexEntries(dbTx database.Tx, block *util.Block, blockID uint32, tx
 		includingBlocksOffset += includingBlocksIndexKeyEntrySize
 	}
 
-	for includingBlockHash, blockTxsAcceptanceData := range txsAcceptanceData {
-		var includingBlockID uint32
-		if includingBlockHash.IsEqual(block.Hash()) {
+	for _, blockTxsAcceptanceData := range multiBlockTxsAcceptanceData {
+		var includingBlockID uint64
+		if blockTxsAcceptanceData.BlockHash.IsEqual(block.Hash()) {
 			includingBlockID = blockID
 		} else {
-			includingBlockID, err = dbFetchBlockIDByHash(dbTx, &includingBlockHash)
+			includingBlockID, err = blockdag.DBFetchBlockIDByHash(dbTx, &blockTxsAcceptanceData.BlockHash)
 			if err != nil {
 				return err
 			}
 		}
 
-		includingBlockIDBytes := make([]byte, 4)
-		byteOrder.PutUint32(includingBlockIDBytes, uint32(includingBlockID))
+		serializedIncludingBlockID := blockdag.SerializeBlockID(includingBlockID)
 
-		for _, txAcceptanceData := range blockTxsAcceptanceData {
-			err = dbPutAcceptingBlocksEntry(dbTx, txAcceptanceData.Tx.ID(), blockID, includingBlockIDBytes)
+		for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
+			err = dbPutAcceptingBlocksEntry(dbTx, txAcceptanceData.Tx.ID(), blockID, serializedIncludingBlockID)
 			if err != nil {
 				return err
 			}
@@ -298,11 +203,28 @@ func dbAddTxIndexEntries(dbTx database.Tx, block *util.Block, blockID uint32, tx
 	return nil
 }
 
+func updateTxsAcceptedByVirtual(virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+	// Initialize a new txsAcceptedByVirtual
+	entries := 0
+	for _, blockTxsAcceptanceData := range virtualTxsAcceptanceData {
+		entries += len(blockTxsAcceptanceData.TxAcceptanceData)
+	}
+	txsAcceptedByVirtual = make(map[daghash.TxID]bool, entries)
+
+	// Copy virtualTxsAcceptanceData to txsAcceptedByVirtual
+	for _, blockTxsAcceptanceData := range virtualTxsAcceptanceData {
+		for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
+			txsAcceptedByVirtual[*txAcceptanceData.Tx.ID()] = true
+		}
+	}
+
+	return nil
+}
+
 // TxIndex implements a transaction by hash index.  That is to say, it supports
 // querying all transactions by their hash.
 type TxIndex struct {
-	db         database.DB
-	curBlockID uint32
+	db database.DB
 }
 
 // Ensure the TxIndex type implements the Indexer interface.
@@ -313,63 +235,18 @@ var _ Indexer = (*TxIndex)(nil)
 // disconnecting blocks.
 //
 // This is part of the Indexer interface.
-func (idx *TxIndex) Init(db database.DB) error {
+func (idx *TxIndex) Init(db database.DB, dag *blockdag.BlockDAG) error {
 	idx.db = db
 
-	// Find the latest known block id field for the internal block id
-	// index and initialize it.  This is done because it's a lot more
-	// efficient to do a single search at initialize time than it is to
-	// write another value to the database on every update.
-	err := idx.db.View(func(dbTx database.Tx) error {
-		// Scan forward in large gaps to find a block id that doesn't
-		// exist yet to serve as an upper bound for the binary search
-		// below.
-		var highestKnown, nextUnknown uint32
-		testBlockID := uint32(1)
-		increment := uint32(100000)
-		for {
-			_, err := dbFetchBlockHashByID(dbTx, testBlockID)
-			if err != nil {
-				nextUnknown = testBlockID
-				break
-			}
-
-			highestKnown = testBlockID
-			testBlockID += increment
-		}
-		log.Tracef("Forward scan (highest known %d, next unknown %d)",
-			highestKnown, nextUnknown)
-
-		// No used block IDs due to new database.
-		if nextUnknown == 1 {
-			return nil
-		}
-
-		// Use a binary search to find the final highest used block id.
-		// This will take at most ceil(log_2(increment)) attempts.
-		for {
-			testBlockID = (highestKnown + nextUnknown) / 2
-			_, err := dbFetchBlockHashByID(dbTx, testBlockID)
-			if err != nil {
-				nextUnknown = testBlockID
-			} else {
-				highestKnown = testBlockID
-			}
-			log.Tracef("Binary scan (highest known %d, next "+
-				"unknown %d)", highestKnown, nextUnknown)
-			if highestKnown+1 == nextUnknown {
-				break
-			}
-		}
-
-		idx.curBlockID = highestKnown
-		return nil
-	})
+	// Initialize the txsAcceptedByVirtual index
+	virtualTxsAcceptanceData, err := dag.TxsAcceptedByVirtual()
+	if err != nil {
+		return err
+	}
+	err = updateTxsAcceptedByVirtual(virtualTxsAcceptanceData)
 	if err != nil {
 		return err
 	}
-
-	log.Debugf("Current internal block ID: %d", idx.curBlockID)
 	return nil
 }
 
@@ -394,12 +271,6 @@ func (idx *TxIndex) Name() string {
 // This is part of the Indexer interface.
 func (idx *TxIndex) Create(dbTx database.Tx) error {
 	meta := dbTx.Metadata()
-	if _, err := meta.CreateBucket(idByHashIndexBucketName); err != nil {
-		return err
-	}
-	if _, err := meta.CreateBucket(hashByIDIndexBucketName); err != nil {
-		return err
-	}
 	if _, err := meta.CreateBucket(includingBlocksIndexKey); err != nil {
 		return err
 	}
@@ -413,24 +284,16 @@ func (idx *TxIndex) Create(dbTx database.Tx) error {
 // for every transaction in the passed block.
 //
 // This is part of the Indexer interface.
-func (idx *TxIndex) ConnectBlock(dbTx database.Tx, block *util.Block, _ *blockdag.BlockDAG, acceptedTxsData blockdag.MultiBlockTxsAcceptanceData) error {
-	// Increment the internal block ID to use for the block being connected
-	// and add all of the transactions in the block to the index.
-	newBlockID := idx.curBlockID + 1
-	if block.MsgBlock().Header.IsGenesis() {
-		newBlockID = 0
-	}
-	if err := dbAddTxIndexEntries(dbTx, block, newBlockID, acceptedTxsData); err != nil {
+func (idx *TxIndex) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
+	acceptedTxsData blockdag.MultiBlockTxsAcceptanceData, virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+	if err := dbAddTxIndexEntries(dbTx, block, blockID, acceptedTxsData); err != nil {
 		return err
 	}
 
-	// Add the new block ID index entry for the block being connected and
-	// update the current internal block ID accordingly.
-	err := dbPutBlockIDIndexEntry(dbTx, block.Hash(), newBlockID)
+	err := updateTxsAcceptedByVirtual(virtualTxsAcceptanceData)
 	if err != nil {
 		return err
 	}
-	idx.curBlockID = newBlockID
 	return nil
 }
 
@@ -474,9 +337,8 @@ func dbFetchTxBlocks(dbTx database.Tx, txHash *daghash.Hash) ([]*daghash.Hash, e
 				"were found for %s", txHash),
 		}
 	}
-	err := bucket.ForEach(func(blockIDBytes, _ []byte) error {
-		blockID := byteOrder.Uint32(blockIDBytes)
-		blockHash, err := dbFetchBlockHashByID(dbTx, blockID)
+	err := bucket.ForEach(func(serializedBlockID, _ []byte) error {
+		blockHash, err := blockdag.DBFetchBlockHashBySerializedID(dbTx, serializedBlockID)
 		if err != nil {
 			return err
 		}
@@ -501,29 +363,30 @@ func (idx *TxIndex) BlockThatAcceptedTx(dag *blockdag.BlockDAG, txID *daghash.Tx
 }
 
 func dbFetchTxAcceptingBlock(dbTx database.Tx, txID *daghash.TxID, dag *blockdag.BlockDAG) (*daghash.Hash, error) {
+	// If the transaction was accepted by the current virtual,
+	// return the zeroHash immediately
+	if _, ok := txsAcceptedByVirtual[*txID]; ok {
+		return &daghash.ZeroHash, nil
+	}
+
 	bucket := dbTx.Metadata().Bucket(acceptingBlocksIndexKey).Bucket(txID[:])
 	if bucket == nil {
-		return nil, database.Error{
-			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No accepting blocks "+
-				"were found for %s", txID),
-		}
+		return nil, nil
 	}
 	cursor := bucket.Cursor()
 	if !cursor.First() {
 		return nil, database.Error{
 			ErrorCode: database.ErrCorruption,
-			Description: fmt.Sprintf("No accepting blocks "+
-				"were found for %s", txID),
+			Description: fmt.Sprintf("Accepting blocks bucket is "+
+				"empty for %s", txID),
 		}
 	}
 	for ; cursor.Key() != nil; cursor.Next() {
-		blockID := byteOrder.Uint32(cursor.Key())
-		blockHash, err := dbFetchBlockHashByID(dbTx, blockID)
+		blockHash, err := blockdag.DBFetchBlockHashBySerializedID(dbTx, cursor.Key())
 		if err != nil {
 			return nil, err
 		}
-		if dag.IsInSelectedPathChain(blockHash) {
+		if dag.IsInSelectedParentChain(blockHash) {
 			return blockHash, nil
 		}
 	}
@@ -541,19 +404,6 @@ func NewTxIndex() *TxIndex {
 	return &TxIndex{}
 }
 
-// dropBlockIDIndex drops the internal block id index.
-func dropBlockIDIndex(db database.DB) error {
-	return db.Update(func(dbTx database.Tx) error {
-		meta := dbTx.Metadata()
-		err := meta.DeleteBucket(idByHashIndexBucketName)
-		if err != nil {
-			return err
-		}
-
-		return meta.DeleteBucket(hashByIDIndexBucketName)
-	})
-}
-
 // DropTxIndex drops the transaction index from the provided database if it
 // exists.  Since the address index relies on it, the address index will also be
 // dropped when it exists.
@@ -570,3 +420,12 @@ func DropTxIndex(db database.DB, interrupt <-chan struct{}) error {
 
 	return dropIndex(db, acceptingBlocksIndexKey, txIndexName, interrupt)
 }
+
+// Recover is invoked when the indexer wasn't turned on for several blocks
+// and the indexer needs to close the gaps.
+//
+// This is part of the Indexer interface.
+func (idx *TxIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
+	return errors.Errorf("txindex was turned off for %d blocks and can't be recovered."+
+		" To resume working drop the txindex with --droptxindex", lastKnownBlockID-currentBlockID)
+}

blockdag/indexers/txindex_test.go

@@ -5,21 +5,27 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/daglabs/btcd/blockdag"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/mining"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/mining"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
-func createTransaction(value uint64, originTx *wire.MsgTx, outputIndex uint32) *wire.MsgTx {
+func createTransaction(t *testing.T, value uint64, originTx *wire.MsgTx, outputIndex uint32) *wire.MsgTx {
+	signatureScript, err := txscript.PayToScriptHashSignatureScript(blockdag.OpTrueScript, nil)
+	if err != nil {
+		t.Fatalf("Error creating signature script: %s", err)
+	}
 	txIn := &wire.TxIn{
-		PreviousOutPoint: wire.OutPoint{
-			TxID:  originTx.TxID(),
+		PreviousOutpoint: wire.Outpoint{
+			TxID:  *originTx.TxID(),
 			Index: outputIndex,
 		},
-		Sequence: wire.MaxTxInSequenceNum,
+		Sequence:        wire.MaxTxInSequenceNum,
+		SignatureScript: signatureScript,
 	}
 	txOut := wire.NewTxOut(value, blockdag.OpTrueScript)
 	tx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
@@ -34,7 +40,7 @@ func TestTxIndexConnectBlock(t *testing.T) {
 	indexManager := NewManager([]Indexer{txIndex})
 
 	params := dagconfig.SimNetParams
-	params.BlockRewardMaturity = 1
+	params.BlockCoinbaseMaturity = 0
 	params.K = 1
 
 	config := blockdag.Config{
@@ -51,16 +57,20 @@ func TestTxIndexConnectBlock(t *testing.T) {
 	}
 
 	prepareAndProcessBlock := func(parentHashes []*daghash.Hash, transactions []*wire.MsgTx, blockName string) *wire.MsgBlock {
-		block, err := mining.PrepareBlockForTest(dag, &params, parentHashes, transactions, false, 1)
+		block, err := mining.PrepareBlockForTest(dag, &params, parentHashes, transactions, false)
 		if err != nil {
 			t.Fatalf("TestTxIndexConnectBlock: block %v got unexpected error from PrepareBlockForTest: %v", blockName, err)
 		}
 		utilBlock := util.NewBlock(block)
 		blocks[*block.BlockHash()] = utilBlock
-		isOrphan, err := dag.ProcessBlock(utilBlock, blockdag.BFNoPoWCheck)
+		isOrphan, delay, err := dag.ProcessBlock(utilBlock, blockdag.BFNoPoWCheck)
 		if err != nil {
 			t.Fatalf("TestTxIndexConnectBlock: dag.ProcessBlock got unexpected error for block %v: %v", blockName, err)
 		}
+		if delay != 0 {
+			t.Fatalf("TestTxIndexConnectBlock: block %s "+
+				"is too far in the future", blockName)
+		}
 		if isOrphan {
 			t.Fatalf("TestTxIndexConnectBlock: block %v was unexpectedly orphan", blockName)
 		}
@@ -68,37 +78,47 @@ func TestTxIndexConnectBlock(t *testing.T) {
 	}
 
 	block1 := prepareAndProcessBlock([]*daghash.Hash{params.GenesisHash}, nil, "1")
-	block2Tx := createTransaction(block1.Transactions[0].TxOut[0].Value, block1.Transactions[0], 0)
+	block2Tx := createTransaction(t, block1.Transactions[0].TxOut[0].Value, block1.Transactions[0], 0)
 	block2 := prepareAndProcessBlock([]*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{block2Tx}, "2")
-	block3Tx := createTransaction(block2.Transactions[0].TxOut[0].Value, block2.Transactions[0], 0)
+	block3Tx := createTransaction(t, block2.Transactions[0].TxOut[0].Value, block2.Transactions[0], 0)
 	block3 := prepareAndProcessBlock([]*daghash.Hash{block2.BlockHash()}, []*wire.MsgTx{block3Tx}, "3")
 
-	block3TxID := block3Tx.TxID()
-	block3TxNewAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, &block3TxID)
+	block2TxID := block2Tx.TxID()
+	block2TxNewAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, block2TxID)
 	if err != nil {
 		t.Errorf("TestTxIndexConnectBlock: TxAcceptedInBlock: %v", err)
 	}
 	block3Hash := block3.BlockHash()
-	if !block3TxNewAcceptedBlock.IsEqual(block3Hash) {
+	if !block2TxNewAcceptedBlock.IsEqual(block3Hash) {
+		t.Errorf("TestTxIndexConnectBlock: block2Tx should've "+
+			"been accepted in block %v but instead got accepted in block %v", block3Hash, block2TxNewAcceptedBlock)
+	}
+
+	block3TxID := block3Tx.TxID()
+	block3TxNewAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, block3TxID)
+	if err != nil {
+		t.Errorf("TestTxIndexConnectBlock: TxAcceptedInBlock: %v", err)
+	}
+	if !block3TxNewAcceptedBlock.IsEqual(&daghash.ZeroHash) {
 		t.Errorf("TestTxIndexConnectBlock: block3Tx should've "+
-			"been accepted in block %v but instead got accepted in block %v", block3Hash, block3TxNewAcceptedBlock)
+			"been accepted by the virtual block but instead got accepted in block %v", block3TxNewAcceptedBlock)
 	}
 
 	block3A := prepareAndProcessBlock([]*daghash.Hash{block2.BlockHash()}, []*wire.MsgTx{block3Tx}, "3A")
 	block4 := prepareAndProcessBlock([]*daghash.Hash{block3.BlockHash()}, nil, "4")
 	prepareAndProcessBlock([]*daghash.Hash{block3A.BlockHash(), block4.BlockHash()}, nil, "5")
 
-	block3TxAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, &block3TxID)
+	block2TxAcceptedBlock, err := txIndex.BlockThatAcceptedTx(dag, block2TxID)
 	if err != nil {
 		t.Errorf("TestTxIndexConnectBlock: TxAcceptedInBlock: %v", err)
 	}
 	block3AHash := block3A.BlockHash()
-	if !block3TxAcceptedBlock.IsEqual(block3AHash) {
-		t.Errorf("TestTxIndexConnectBlock: block3Tx should've "+
-			"been accepted in block %v but instead got accepted in block %v", block3AHash, block3TxAcceptedBlock)
+	if !block2TxAcceptedBlock.IsEqual(block3AHash) {
+		t.Errorf("TestTxIndexConnectBlock: block2Tx should've "+
+			"been accepted in block %v but instead got accepted in block %v", block3AHash, block2TxAcceptedBlock)
 	}
 
-	region, err := txIndex.TxFirstBlockRegion(&block3TxID)
+	region, err := txIndex.TxFirstBlockRegion(block3TxID)
 	if err != nil {
 		t.Fatalf("TestTxIndexConnectBlock: no block region was found for block3Tx")
 	}

blockdag/log.go

@@ -5,16 +5,9 @@
 package blockdag
 
 import (
-	"github.com/btcsuite/btclog"
-	"github.com/daglabs/btcd/logger"
+	"github.com/kaspanet/kaspad/logger"
+	"github.com/kaspanet/kaspad/util/panics"
 )
 
-// log is a logger that is initialized with no output filters.  This
-// means the package will not perform any logging by default until the caller
-// requests it.
-var log btclog.Logger
-
-// The default amount of logging is none.
-func init() {
-	log, _ = logger.Get(logger.SubsystemTags.CHAN)
-}
+var log, _ = logger.Get(logger.SubsystemTags.BDAG)
+var spawn = panics.GoroutineWrapperFunc(log)

blockdag/merkle.go

@@ -7,8 +7,8 @@ package blockdag
 import (
 	"math"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // MerkleTree holds the hashes of a merkle tree

blockdag/merkle_test.go

@@ -5,9 +5,10 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/util/daghash"
 	"testing"
 
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/util"
 )
 
 // TestMerkle tests the BuildHashMerkleTreeStore API.
@@ -24,8 +25,11 @@ func TestMerkle(t *testing.T) {
 
 	idMerkleTree := BuildIDMerkleTreeStore(block.Transactions())
 	calculatedIDMerkleRoot := idMerkleTree.Root()
-	wantIDMerkleRoot := Block100000.Header.IDMerkleRoot
-	if !wantIDMerkleRoot.IsEqual(calculatedIDMerkleRoot) {
+	wantIDMerkleRoot, err := daghash.NewHashFromStr("3f69feb7edf5d0d67930afc990c8ec931e3428d7c7a65d7af6b81079319eb110")
+	if err != nil {
+		t.Errorf("BuildIDMerkleTreeStore: unexpected error: %s", err)
+	}
+	if !calculatedIDMerkleRoot.IsEqual(wantIDMerkleRoot) {
 		t.Errorf("BuildIDMerkleTreeStore: ID merkle root mismatch - "+
 			"got %v, want %v", calculatedIDMerkleRoot, wantIDMerkleRoot)
 	}

blockdag/notifications.go

@@ -6,6 +6,8 @@ package blockdag
 
 import (
 	"fmt"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // NotificationType represents the type of a notification message.
@@ -20,12 +22,17 @@ const (
 	// NTBlockAdded indicates the associated block was added into
 	// the blockDAG.
 	NTBlockAdded NotificationType = iota
+
+	// NTChainChanged indicates that selected parent
+	// chain had changed.
+	NTChainChanged
 )
 
 // notificationTypeStrings is a map of notification types back to their constant
 // names for pretty printing.
 var notificationTypeStrings = map[NotificationType]string{
-	NTBlockAdded: "NTBlockAdded",
+	NTBlockAdded:   "NTBlockAdded",
+	NTChainChanged: "NTChainChanged",
 }
 
 // String returns the NotificationType in human-readable form.
@@ -66,3 +73,17 @@ func (dag *BlockDAG) sendNotification(typ NotificationType, data interface{}) {
 	}
 	dag.notificationsLock.RUnlock()
 }
+
+// BlockAddedNotificationData defines data to be sent along with a BlockAdded
+// notification
+type BlockAddedNotificationData struct {
+	Block         *util.Block
+	WasUnorphaned bool
+}
+
+// ChainChangedNotificationData defines data to be sent along with a ChainChanged
+// notification
+type ChainChangedNotificationData struct {
+	RemovedChainBlockHashes []*daghash.Hash
+	AddedChainBlockHashes   []*daghash.Hash
+}

blockdag/notifications_test.go

@@ -5,14 +5,15 @@
 package blockdag
 
 import (
+	"path/filepath"
 	"testing"
 
-	"github.com/daglabs/btcd/dagconfig"
+	"github.com/kaspanet/kaspad/dagconfig"
 )
 
 // TestNotifications ensures that notification callbacks are fired on events.
 func TestNotifications(t *testing.T) {
-	blocks, err := loadBlocks("blk_0_to_4.dat")
+	blocks, err := LoadBlocks(filepath.Join("testdata/blk_0_to_4.dat"))
 	if err != nil {
 		t.Fatalf("Error loading file: %v\n", err)
 	}
@@ -40,14 +41,18 @@ func TestNotifications(t *testing.T) {
 		dag.Subscribe(callback)
 	}
 
-	isOrphan, err := dag.ProcessBlock(blocks[1], BFNone)
+	isOrphan, delay, err := dag.ProcessBlock(blocks[1], BFNone)
+	if err != nil {
+		t.Fatalf("ProcessBlock fail on block 1: %v\n", err)
+	}
+	if delay != 0 {
+		t.Fatalf("ProcessBlock: block 1 " +
+			"is too far in the future")
+	}
 	if isOrphan {
 		t.Fatalf("ProcessBlock incorrectly returned block " +
 			"is an orphan\n")
 	}
-	if err != nil {
-		t.Fatalf("ProcessBlock fail on block 1: %v\n", err)
-	}
 
 	if notificationCount != numSubscribers {
 		t.Fatalf("Expected notification callback to be executed %d "+

blockdag/phantom.go

@@ -1,7 +1,7 @@
 package blockdag
 
 import (
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // phantom calculates and returns the block's blue set, selected parent and blue score.
@@ -77,7 +77,7 @@ func blueCandidates(chainStart *blockNode) blockSet {
 func traverseCandidates(newBlock *blockNode, candidates blockSet, selectedParent *blockNode) []*blockNode {
 	blues := []*blockNode{}
 	selectedParentPast := newSet()
-	queue := NewDownHeap()
+	queue := newDownHeap()
 	visited := newSet()
 
 	for _, parent := range newBlock.parents {

blockdag/phantom_test.go

@@ -7,9 +7,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 
-	"github.com/daglabs/btcd/dagconfig"
+	"github.com/kaspanet/kaspad/dagconfig"
 )
 
 type testBlockData struct {
@@ -20,11 +20,6 @@ type testBlockData struct {
 	expectedBlues          []string
 }
 
-type hashIDPair struct {
-	hash *daghash.Hash
-	id   string
-}
-
 //TestPhantom iterate over several dag simulations, and checks
 //that the blue score, blue set and selected parent of each
 //block calculated as expected
@@ -113,7 +108,7 @@ func TestPhantom(t *testing.T) {
 					id:                     "K",
 					expectedScore:          9,
 					expectedSelectedParent: "H",
-					expectedBlues:          []string{"I", "J", "G", "F", "H"},
+					expectedBlues:          []string{"I", "G", "J", "F", "H"},
 				},
 			},
 		},

blockdag/process.go

@@ -8,13 +8,12 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // BehaviorFlags is a bitmask defining tweaks to the normal behavior when
-// performing chain processing and consensus rules checks.
+// performing DAG processing and consensus rules checks.
 type BehaviorFlags uint32
 
 const (
@@ -29,6 +28,22 @@ const (
 	// not be performed.
 	BFNoPoWCheck
 
+	// BFWasUnorphaned may be set to indicate that a block was just now
+	// unorphaned
+	BFWasUnorphaned
+
+	// BFAfterDelay may be set to indicate that a block had timestamp too far
+	// in the future, just finished the delay
+	BFAfterDelay
+
+	// BFIsSync may be set to indicate that the block was sent as part of the
+	// netsync process
+	BFIsSync
+
+	// BFWasStored is set to indicate that the block was previously stored
+	// in the block index but was never fully processed
+	BFWasStored
+
 	// BFNone is a convenience value to specifically indicate no flags.
 	BFNone BehaviorFlags = 0
 )
@@ -37,38 +52,8 @@ const (
 // the DAG.
 //
 // This function is safe for concurrent access.
-func (dag *BlockDAG) BlockExists(hash *daghash.Hash) (bool, error) {
-	// Check block index first (could be main chain or side chain blocks).
-	if dag.index.HaveBlock(hash) {
-		return true, nil
-	}
-
-	// Check in the database.
-	var exists bool
-	err := dag.db.View(func(dbTx database.Tx) error {
-		var err error
-		exists, err = dbTx.HasBlock(hash)
-		if err != nil || !exists {
-			return err
-		}
-
-		// Ignore side chain blocks in the database.  This is necessary
-		// because there is not currently any record of the associated
-		// block index data such as its block height, so it's not yet
-		// possible to efficiently load the block and do anything useful
-		// with it.
-		//
-		// Ultimately the entire block index should be serialized
-		// instead of only the current main chain so it can be consulted
-		// directly.
-		_, err = dbFetchHeightByHash(dbTx, hash)
-		if isNotInDAGErr(err) {
-			exists = false
-			return nil
-		}
-		return err
-	})
-	return exists, err
+func (dag *BlockDAG) BlockExists(hash *daghash.Hash) bool {
+	return dag.index.HaveBlock(hash)
 }
 
 // processOrphans determines if there are any orphans which depend on the passed
@@ -109,15 +94,30 @@ func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) err
 				continue
 			}
 
+			// Skip this orphan if one or more of its parents are
+			// still missing.
+			_, err := lookupParentNodes(orphan.block, dag)
+			if err != nil {
+				if ruleErr, ok := err.(RuleError); ok && ruleErr.ErrorCode == ErrParentBlockUnknown {
+					continue
+				}
+				return err
+			}
+
 			// Remove the orphan from the orphan pool.
 			orphanHash := orphan.block.Hash()
 			dag.removeOrphanBlock(orphan)
 			i--
 
-			// Potentially accept the block into the block chain.
-			err := dag.maybeAcceptBlock(orphan.block, flags)
+			// Potentially accept the block into the block DAG.
+			err = dag.maybeAcceptBlock(orphan.block, flags|BFWasUnorphaned)
 			if err != nil {
-				return err
+				// Since we don't want to reject the original block because of
+				// a bad unorphaned child, only return an error if it's not a RuleError.
+				if _, ok := err.(RuleError); !ok {
+					return err
+				}
+				log.Warnf("Verification failed for orphan block %s: %s", orphanHash, err)
 			}
 
 			// Add this block to the list of blocks to process so
@@ -138,102 +138,71 @@ func (dag *BlockDAG) processOrphans(hash *daghash.Hash, flags BehaviorFlags) err
 // whether or not the block is an orphan.
 //
 // This function is safe for concurrent access.
-func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (bool, error) {
+func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (isOrphan bool, delay time.Duration, err error) {
 	dag.dagLock.Lock()
 	defer dag.dagLock.Unlock()
 
-	fastAdd := flags&BFFastAdd == BFFastAdd
+	isDelayedBlock := flags&BFAfterDelay == BFAfterDelay
+	wasBlockStored := flags&BFWasStored == BFWasStored
 
 	blockHash := block.Hash()
 	log.Tracef("Processing block %s", blockHash)
 
-	// The block must not already exist in the main chain or side chains.
-	exists, err := dag.BlockExists(blockHash)
-	if err != nil {
-		return false, err
-	}
-	if exists {
+	// The block must not already exist in the DAG.
+	if dag.BlockExists(blockHash) && !wasBlockStored {
 		str := fmt.Sprintf("already have block %s", blockHash)
-		return false, ruleError(ErrDuplicateBlock, str)
+		return false, 0, ruleError(ErrDuplicateBlock, str)
 	}
 
 	// The block must not already exist as an orphan.
 	if _, exists := dag.orphans[*blockHash]; exists {
 		str := fmt.Sprintf("already have block (orphan) %s", blockHash)
-		return false, ruleError(ErrDuplicateBlock, str)
+		return false, 0, ruleError(ErrDuplicateBlock, str)
 	}
 
-	// Perform preliminary sanity checks on the block and its transactions.
-	err = dag.checkBlockSanity(block, flags)
-	if err != nil {
-		return false, err
-	}
-
-	// Find the previous checkpoint and perform some additional checks based
-	// on the checkpoint.  This provides a few nice properties such as
-	// preventing old side chain blocks before the last checkpoint,
-	// rejecting easy to mine, but otherwise bogus, blocks that could be
-	// used to eat memory, and ensuring expected (versus claimed) proof of
-	// work requirements since the previous checkpoint are met.
-	blockHeader := &block.MsgBlock().Header
-	checkpointNode, err := dag.findPreviousCheckpoint()
-	if err != nil {
-		return false, err
-	}
-	if checkpointNode != nil {
-		// Ensure the block timestamp is after the checkpoint timestamp.
-		checkpointTime := time.Unix(checkpointNode.timestamp, 0)
-		if blockHeader.Timestamp.Before(checkpointTime) {
-			str := fmt.Sprintf("block %s has timestamp %s before "+
-				"last checkpoint timestamp %s", blockHash,
-				blockHeader.Timestamp, checkpointTime)
-			return false, ruleError(ErrCheckpointTimeTooOld, str)
+	if !isDelayedBlock {
+		// Perform preliminary sanity checks on the block and its transactions.
+		delay, err := dag.checkBlockSanity(block, flags)
+		if err != nil {
+			return false, 0, err
 		}
-		if !fastAdd {
-			// Even though the checks prior to now have already ensured the
-			// proof of work exceeds the claimed amount, the claimed amount
-			// is a field in the block header which could be forged.  This
-			// check ensures the proof of work is at least the minimum
-			// expected based on elapsed time since the last checkpoint and
-			// maximum adjustment allowed by the retarget rules.
-			duration := blockHeader.Timestamp.Sub(checkpointTime)
-			requiredTarget := util.CompactToBig(dag.calcEasiestDifficulty(
-				checkpointNode.bits, duration))
-			currentTarget := util.CompactToBig(blockHeader.Bits)
-			if currentTarget.Cmp(requiredTarget) > 0 {
-				str := fmt.Sprintf("block target difficulty of %064x "+
-					"is too low when compared to the previous "+
-					"checkpoint", currentTarget)
-				return false, ruleError(ErrDifficultyTooLow, str)
-			}
+
+		if delay != 0 {
+			return false, delay, err
 		}
 	}
 
 	// Handle orphan blocks.
 	allParentsExist := true
-	for _, parentHash := range blockHeader.ParentHashes {
-		parentExists, err := dag.BlockExists(parentHash)
-		if err != nil {
-			return false, err
-		}
-
-		if !parentExists {
-			log.Infof("Adding orphan block %s with parent %s", blockHash, parentHash)
-			dag.addOrphanBlock(block)
-
+	for _, parentHash := range block.MsgBlock().Header.ParentHashes {
+		if !dag.BlockExists(parentHash) {
 			allParentsExist = false
 		}
 	}
 
 	if !allParentsExist {
-		return true, nil
+		// Some orphans during netsync are a normal part of the process, since the anticone
+		// of the chain-split is never explicitly requested.
+		// Therefore, if we are during netsync - don't report orphans to default logs.
+		//
+		// The number K*2 was chosen since in peace times anticone is limited to K blocks,
+		// while some red block can make it a bit bigger, but much more than that indicates
+		// there might be some problem with the netsync process.
+		if flags&BFIsSync == BFIsSync && uint32(len(dag.orphans)) < dag.dagParams.K*2 {
+			log.Debugf("Adding orphan block %s. This is normal part of netsync process", blockHash)
+		} else {
+			log.Infof("Adding orphan block %s", blockHash)
+		}
+		dag.addOrphanBlock(block)
+
+		return true, 0, nil
 	}
 
 	// The block has passed all context independent checks and appears sane
 	// enough to potentially accept it into the block DAG.
 	err = dag.maybeAcceptBlock(block, flags)
 	if err != nil {
-		return false, err
+		return false, 0, err
 	}
 
 	// Accept any orphan blocks that depend on this block (they are
@@ -241,10 +210,10 @@ func (dag *BlockDAG) ProcessBlock(block *util.Block, flags BehaviorFlags) (bool,
 	// there are no more.
 	err = dag.processOrphans(blockHash, flags)
 	if err != nil {
-		return false, err
+		return false, 0, err
 	}
 
 	log.Debugf("Accepted block %s", blockHash)
 
-	return false, nil
+	return false, 0, nil
 }

blockdag/process_test.go

@@ -0,0 +1,131 @@
+package blockdag
+
+import (
+	"bou.ke/monkey"
+	"fmt"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestProcessBlock(t *testing.T) {
+	dag, teardownFunc, err := DAGSetup("TestProcessBlock", Config{
+		DAGParams: &dagconfig.SimNetParams,
+	})
+	if err != nil {
+		t.Errorf("Failed to setup dag instance: %v", err)
+		return
+	}
+	defer teardownFunc()
+
+	// Check that BFAfterDelay skip checkBlockSanity
+	called := false
+	guard := monkey.Patch((*BlockDAG).checkBlockSanity, func(_ *BlockDAG, _ *util.Block, _ BehaviorFlags) (time.Duration, error) {
+		called = true
+		return 0, nil
+	})
+	defer guard.Unpatch()
+
+	isOrphan, delay, err := dag.ProcessBlock(util.NewBlock(&Block100000), BFNoPoWCheck)
+	if err != nil {
+		t.Errorf("ProcessBlock: %s", err)
+	}
+	if delay != 0 {
+		t.Errorf("ProcessBlock: block is too far in the future")
+	}
+	if !isOrphan {
+		t.Errorf("ProcessBlock: unexpected returned non orphan block")
+	}
+	if !called {
+		t.Errorf("ProcessBlock: expected checkBlockSanity to be called")
+	}
+
+	Block100000Copy := Block100000
+	// Change nonce to change block hash
+	Block100000Copy.Header.Nonce++
+	called = false
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(&Block100000Copy), BFAfterDelay|BFNoPoWCheck)
+	if err != nil {
+		t.Errorf("ProcessBlock: %s", err)
+	}
+	if delay != 0 {
+		t.Errorf("ProcessBlock: block is too far in the future")
+	}
+	if !isOrphan {
+		t.Errorf("ProcessBlock: unexpected returned non orphan block")
+	}
+	if called {
+		t.Errorf("ProcessBlock: Didn't expected checkBlockSanity to be called")
+	}
+
+	isOrphan, delay, err = dag.ProcessBlock(util.NewBlock(dagconfig.SimNetParams.GenesisBlock), BFNone)
+	expectedErrMsg := fmt.Sprintf("already have block %s", dagconfig.SimNetParams.GenesisHash)
+	if err == nil || err.Error() != expectedErrMsg {
+		t.Errorf("ProcessBlock: Expected error \"%s\" but got \"%s\"", expectedErrMsg, err)
+	}
+}
+
+func TestProcessOrphans(t *testing.T) {
+	dag, teardownFunc, err := DAGSetup("TestProcessOrphans", Config{
+		DAGParams: &dagconfig.SimNetParams,
+	})
+	if err != nil {
+		t.Errorf("Failed to setup dag instance: %v", err)
+		return
+	}
+	defer teardownFunc()
+
+	dag.TestSetCoinbaseMaturity(0)
+
+	blocksFile := "blk_0_to_4.dat"
+	blocks, err := LoadBlocks(filepath.Join("testdata/", blocksFile))
+	if err != nil {
+		t.Fatalf("TestProcessOrphans: "+
+			"Error loading file '%s': %s\n", blocksFile, err)
+	}
+
+	// Get a reference to a parent block
+	parentBlock := blocks[1]
+
+	// Get a reference to a child block and mess with it so that:
+	// a. It gets added to the orphan pool
+	// b. It gets rejected once it's unorphaned
+	childBlock := blocks[2]
+	childBlock.MsgBlock().Header.UTXOCommitment = &daghash.ZeroHash
+
+	// Process the child block so that it gets added to the orphan pool
+	isOrphan, delay, err := dag.ProcessBlock(childBlock, BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("TestProcessOrphans: child block unexpectedly returned an error: %s", err)
+	}
+	if delay != 0 {
+		t.Fatalf("TestProcessOrphans: child block is too far in the future")
+	}
+	if !isOrphan {
+		t.Fatalf("TestProcessOrphans: incorrectly returned that child block is not an orphan")
+	}
+
+	// Process the parent block. Note that this will attempt to unorphan the child block
+	isOrphan, delay, err = dag.ProcessBlock(parentBlock, BFNone)
+	if err != nil {
+		t.Fatalf("TestProcessOrphans: parent block unexpectedly returned an error: %s", err)
+	}
+	if delay != 0 {
+		t.Fatalf("TestProcessOrphans: parent block is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("TestProcessOrphans: incorrectly returned that parent block is an orphan")
+	}
+
+	// Make sure that the child block had been rejected
+	node := dag.index.LookupNode(childBlock.Hash())
+	if node == nil {
+		t.Fatalf("TestProcessOrphans: child block missing from block index")
+	}
+	if !dag.index.NodeStatus(node).KnownInvalid() {
+		t.Fatalf("TestProcessOrphans: child block erroneously not marked as invalid")
+	}
+}

blockdag/scriptval.go

@@ -6,13 +6,12 @@ package blockdag
 
 import (
 	"fmt"
-	"math"
 	"runtime"
 	"time"
 
-	"github.com/daglabs/btcd/txscript"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // txValidateItem holds a transaction along with which input to validate.
@@ -55,12 +54,12 @@ out:
 		case txVI := <-v.validateChan:
 			// Ensure the referenced input utxo is available.
 			txIn := txVI.txIn
-			entry, ok := v.utxoSet.Get(txIn.PreviousOutPoint)
+			entry, ok := v.utxoSet.Get(txIn.PreviousOutpoint)
 			if !ok {
 				str := fmt.Sprintf("unable to find unspent "+
 					"output %s referenced from "+
-					"transaction %s:%d",
-					txIn.PreviousOutPoint, txVI.tx.ID(),
+					"transaction %s input %d",
+					txIn.PreviousOutpoint, txVI.tx.ID(),
 					txVI.txInIndex)
 				err := ruleError(ErrMissingTxOut, str)
 				v.sendResult(err)
@@ -69,8 +68,8 @@ out:
 
 			// Create a new script engine for the script pair.
 			sigScript := txIn.SignatureScript
-			pkScript := entry.PkScript()
-			vm, err := txscript.NewEngine(pkScript, txVI.tx.MsgTx(),
+			scriptPubKey := entry.ScriptPubKey()
+			vm, err := txscript.NewEngine(scriptPubKey, txVI.tx.MsgTx(),
 				txVI.txInIndex, v.flags, v.sigCache)
 			if err != nil {
 				str := fmt.Sprintf("failed to parse input "+
@@ -78,7 +77,7 @@ out:
 					"%s (input script bytes %x, prev "+
 					"output script bytes %x)",
 					txVI.tx.ID(), txVI.txInIndex,
-					txIn.PreviousOutPoint, err, sigScript, pkScript)
+					txIn.PreviousOutpoint, err, sigScript, scriptPubKey)
 				err := ruleError(ErrScriptMalformed, str)
 				v.sendResult(err)
 				break out
@@ -91,7 +90,7 @@ out:
 					"%s (input script bytes %x, prev output "+
 					"script bytes %x)",
 					txVI.tx.ID(), txVI.txInIndex,
-					txIn.PreviousOutPoint, err, sigScript, pkScript)
+					txIn.PreviousOutpoint, err, sigScript, scriptPubKey)
 				err := ruleError(ErrScriptValidation, str)
 				v.sendResult(err)
 				break out
@@ -127,7 +126,7 @@ func (v *txValidator) Validate(items []*txValidateItem) error {
 	// Start up validation handlers that are used to asynchronously
 	// validate each transaction input.
 	for i := 0; i < maxGoRoutines; i++ {
-		go v.validateHandler()
+		spawn(v.validateHandler)
 	}
 
 	// Validate each of the inputs.  The quit channel is closed when any
@@ -180,16 +179,16 @@ func newTxValidator(utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscr
 // ValidateTransactionScripts validates the scripts for the passed transaction
 // using multiple goroutines.
 func ValidateTransactionScripts(tx *util.Tx, utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscript.SigCache) error {
+	// Don't validate coinbase transaction scripts.
+	if tx.IsCoinBase() {
+		return nil
+	}
+
 	// Collect all of the transaction inputs and required information for
 	// validation.
 	txIns := tx.MsgTx().TxIn
 	txValItems := make([]*txValidateItem, 0, len(txIns))
 	for txInIdx, txIn := range txIns {
-		// Skip block reward transactions.
-		if txIn.PreviousOutPoint.Index == math.MaxUint32 {
-			continue
-		}
-
 		txVI := &txValidateItem{
 			txInIndex: txInIdx,
 			txIn:      txIn,
@@ -214,12 +213,11 @@ func checkBlockScripts(block *blockNode, utxoSet UTXOSet, transactions []*util.T
 	}
 	txValItems := make([]*txValidateItem, 0, numInputs)
 	for _, tx := range transactions {
+		// Skip coinbase transactions.
+		if tx.IsCoinBase() {
+			continue
+		}
 		for txInIdx, txIn := range tx.MsgTx().TxIn {
-			// Skip block reward transactions.
-			if txIn.PreviousOutPoint.Index == math.MaxUint32 {
-				continue
-			}
-
 			txVI := &txValidateItem{
 				txInIndex: txInIdx,
 				txIn:      txIn,

blockdag/scriptval_test.go

@@ -6,10 +6,11 @@ package blockdag
 
 import (
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"testing"
 
-	"github.com/daglabs/btcd/txscript"
+	"github.com/kaspanet/kaspad/txscript"
 )
 
 // TestCheckBlockScripts ensures that validating the all of the scripts in a
@@ -20,7 +21,7 @@ func TestCheckBlockScripts(t *testing.T) {
 
 	testBlockNum := 277647
 	blockDataFile := fmt.Sprintf("%d.dat", testBlockNum)
-	blocks, err := loadBlocks(blockDataFile)
+	blocks, err := LoadBlocks(filepath.Join("testdata/", blockDataFile))
 	if err != nil {
 		t.Errorf("Error loading file: %v\n", err)
 		return

blockdag/subnetworks.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"github.com/pkg/errors"
 
-	"github.com/daglabs/btcd/util"
+	"github.com/kaspanet/kaspad/util"
 
-	"github.com/daglabs/btcd/database"
-	"github.com/daglabs/btcd/util/subnetworkid"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // SubnetworkStore stores the subnetworks data
@@ -23,38 +24,28 @@ func newSubnetworkStore(db database.DB) *SubnetworkStore {
 	}
 }
 
-// registerSubnetworks scans a list of accepted transactions, singles out
+// registerSubnetworks scans a list of transactions, singles out
 // subnetwork registry transactions, validates them, and registers a new
 // subnetwork based on it.
 // This function returns an error if one or more transactions are invalid
-func registerSubnetworks(dbTx database.Tx, txsAcceptanceData MultiBlockTxsAcceptanceData) error {
-	validSubnetworkRegistryTxs := make([]*wire.MsgTx, 0)
+func registerSubnetworks(dbTx database.Tx, txs []*util.Tx) error {
+	subnetworkRegistryTxs := make([]*wire.MsgTx, 0)
+	for _, tx := range txs {
+		msgTx := tx.MsgTx()
 
-	for _, txs := range txsAcceptanceData {
-		for _, txData := range txs {
-			if !txData.IsAccepted {
-				continue
-			}
+		if msgTx.SubnetworkID.IsEqual(subnetworkid.SubnetworkIDRegistry) {
+			subnetworkRegistryTxs = append(subnetworkRegistryTxs, msgTx)
+		}
 
-			tx := txData.Tx.MsgTx()
-			if tx.SubnetworkID.IsEqual(subnetworkid.SubnetworkIDRegistry) {
-				err := validateSubnetworkRegistryTransaction(tx)
-				if err != nil {
-					return err
-				}
-				validSubnetworkRegistryTxs = append(validSubnetworkRegistryTxs, tx)
-			}
-
-			if subnetworkid.Less(subnetworkid.SubnetworkIDRegistry, &tx.SubnetworkID) {
-				// Transactions are ordered by subnetwork, so we can safely assume
-				// that the rest of the transactions will not be subnetwork registry
-				// transactions.
-				break
-			}
+		if subnetworkid.Less(subnetworkid.SubnetworkIDRegistry, &msgTx.SubnetworkID) {
+			// Transactions are ordered by subnetwork, so we can safely assume
+			// that the rest of the transactions will not be subnetwork registry
+			// transactions.
+			break
 		}
 	}
 
-	for _, registryTx := range validSubnetworkRegistryTxs {
+	for _, registryTx := range subnetworkRegistryTxs {
 		subnetworkID, err := TxToSubnetworkID(registryTx)
 		if err != nil {
 			return err
@@ -67,7 +58,7 @@ func registerSubnetworks(dbTx database.Tx, txsAcceptanceData MultiBlockTxsAccept
 			createdSubnetwork := newSubnetwork(registryTx)
 			err := dbRegisterSubnetwork(dbTx, subnetworkID, createdSubnetwork)
 			if err != nil {
-				return fmt.Errorf("failed registering subnetwork"+
+				return errors.Errorf("failed registering subnetwork"+
 					"for tx '%s': %s", registryTx.TxHash(), err)
 			}
 		}
@@ -104,10 +95,10 @@ func (s *SubnetworkStore) subnetwork(subnetworkID *subnetworkid.SubnetworkID) (*
 		return nil
 	})
 	if dbErr != nil {
-		return nil, fmt.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, dbErr)
+		return nil, errors.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, dbErr)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, err)
+		return nil, errors.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, err)
 	}
 
 	return sNet, nil
@@ -121,7 +112,7 @@ func (s *SubnetworkStore) GasLimit(subnetworkID *subnetworkid.SubnetworkID) (uin
 		return 0, err
 	}
 	if sNet == nil {
-		return 0, fmt.Errorf("subnetwork '%s' not found", subnetworkID)
+		return 0, errors.Errorf("subnetwork '%s' not found", subnetworkID)
 	}
 
 	return sNet.gasLimit, nil
@@ -132,14 +123,14 @@ func dbRegisterSubnetwork(dbTx database.Tx, subnetworkID *subnetworkid.Subnetwor
 	// Serialize the subnetwork
 	serializedSubnetwork, err := serializeSubnetwork(network)
 	if err != nil {
-		return fmt.Errorf("failed to serialize sub-netowrk '%s': %s", subnetworkID, err)
+		return errors.Errorf("failed to serialize sub-netowrk '%s': %s", subnetworkID, err)
 	}
 
 	// Store the subnetwork
 	subnetworksBucket := dbTx.Metadata().Bucket(subnetworksBucketName)
 	err = subnetworksBucket.Put(subnetworkID[:], serializedSubnetwork)
 	if err != nil {
-		return fmt.Errorf("failed to write sub-netowrk '%s': %s", subnetworkID, err)
+		return errors.Errorf("failed to write sub-netowrk '%s': %s", subnetworkID, err)
 	}
 
 	return nil
@@ -161,13 +152,16 @@ type subnetwork struct {
 }
 
 func newSubnetwork(tx *wire.MsgTx) *subnetwork {
-	gasLimit := binary.LittleEndian.Uint64(tx.Payload[:8])
-
 	return &subnetwork{
-		gasLimit: gasLimit,
+		gasLimit: ExtractGasLimit(tx),
 	}
 }
 
+// ExtractGasLimit extracts the gas limit from the transaction payload
+func ExtractGasLimit(tx *wire.MsgTx) uint64 {
+	return binary.LittleEndian.Uint64(tx.Payload[:8])
+}
+
 // serializeSubnetwork serializes a subnetwork into the following binary format:
 // | gasLimit (8 bytes) |
 func serializeSubnetwork(sNet *subnetwork) ([]byte, error) {
@@ -176,7 +170,7 @@ func serializeSubnetwork(sNet *subnetwork) ([]byte, error) {
 	// Write the gas limit
 	err := binary.Write(serializedSNet, byteOrder, sNet.gasLimit)
 	if err != nil {
-		return nil, fmt.Errorf("failed to serialize subnetwork: %s", err)
+		return nil, errors.Errorf("failed to serialize subnetwork: %s", err)
 	}
 
 	return serializedSNet.Bytes(), nil
@@ -191,7 +185,7 @@ func deserializeSubnetwork(serializedSNetBytes []byte) (*subnetwork, error) {
 	var gasLimit uint64
 	err := binary.Read(serializedSNet, byteOrder, &gasLimit)
 	if err != nil {
-		return nil, fmt.Errorf("failed to deserialize subnetwork: %s", err)
+		return nil, errors.Errorf("failed to deserialize subnetwork: %s", err)
 	}
 
 	return &subnetwork{

blockdag/test_utils.go

@@ -3,18 +3,23 @@ package blockdag
 // This file functions are not considered safe for regular use, and should be used for test purposes only.
 
 import (
-	"fmt"
+	"compress/bzip2"
+	"encoding/binary"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/pkg/errors"
+	"io"
 	"os"
 	"path/filepath"
+	"strings"
+	"sync"
 
-	"github.com/daglabs/btcd/util/subnetworkid"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/database"
-	_ "github.com/daglabs/btcd/database/ffldb" // blank import ffldb so that its init() function runs before tests
-	"github.com/daglabs/btcd/txscript"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/database"
+	_ "github.com/kaspanet/kaspad/database/ffldb" // blank import ffldb so that its init() function runs before tests
+	"github.com/kaspanet/kaspad/txscript"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 const (
@@ -41,8 +46,8 @@ func isSupportedDbType(dbType string) bool {
 	return false
 }
 
-// filesExists returns whether or not the named file or directory exists.
-func fileExists(name string) bool {
+// FileExists returns whether or not the named file or directory exists.
+func FileExists(name string) bool {
 	if _, err := os.Stat(name); err != nil {
 		if os.IsNotExist(err) {
 			return false
@@ -56,16 +61,28 @@ func fileExists(name string) bool {
 // a teardown function the caller should invoke when done testing to clean up.
 func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
 	if !isSupportedDbType(testDbType) {
-		return nil, nil, fmt.Errorf("unsupported db type %s", testDbType)
+		return nil, nil, errors.Errorf("unsupported db type %s", testDbType)
 	}
 
 	var teardown func()
 
+	// To make sure that the teardown function is not called before any goroutines finished to run -
+	// overwrite `spawn` to count the number of running goroutines
+	spawnWaitGroup := sync.WaitGroup{}
+	realSpawn := spawn
+	spawn = func(f func()) {
+		spawnWaitGroup.Add(1)
+		realSpawn(func() {
+			f()
+			spawnWaitGroup.Done()
+		})
+	}
+
 	if config.DB == nil {
 		// Create the root directory for test databases.
-		if !fileExists(testDbRoot) {
+		if !FileExists(testDbRoot) {
 			if err := os.MkdirAll(testDbRoot, 0700); err != nil {
-				err := fmt.Errorf("unable to create test db "+
+				err := errors.Errorf("unable to create test db "+
 					"root: %s", err)
 				return nil, nil, err
 			}
@@ -76,16 +93,24 @@ func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
 		var err error
 		config.DB, err = database.Create(testDbType, dbPath, blockDataNet)
 		if err != nil {
-			return nil, nil, fmt.Errorf("error creating db: %s", err)
+			return nil, nil, errors.Errorf("error creating db: %s", err)
 		}
 
 		// Setup a teardown function for cleaning up.  This function is
 		// returned to the caller to be invoked when it is done testing.
 		teardown = func() {
+			spawnWaitGroup.Wait()
+			spawn = realSpawn
 			config.DB.Close()
 			os.RemoveAll(dbPath)
 			os.RemoveAll(testDbRoot)
 		}
+	} else {
+		teardown = func() {
+			spawnWaitGroup.Wait()
+			spawn = realSpawn
+			config.DB.Close()
+		}
 	}
 
 	config.TimeSource = NewMedianTime()
@@ -95,7 +120,7 @@ func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
 	dag, err := New(&config)
 	if err != nil {
 		teardown()
-		err := fmt.Errorf("failed to create dag instance: %s", err)
+		err := errors.Errorf("failed to create dag instance: %s", err)
 		return nil, nil, err
 	}
 	return dag, teardown, nil
@@ -116,7 +141,7 @@ func createTxForTest(numInputs uint32, numOutputs uint32, outputValue uint64, su
 
 	for i := uint32(0); i < numInputs; i++ {
 		txIns = append(txIns, &wire.TxIn{
-			PreviousOutPoint: *wire.NewOutPoint(&daghash.TxID{}, i),
+			PreviousOutpoint: *wire.NewOutpoint(&daghash.TxID{}, i),
 			SignatureScript:  []byte{},
 			Sequence:         wire.MaxTxInSequenceNum,
 		})
@@ -124,8 +149,8 @@ func createTxForTest(numInputs uint32, numOutputs uint32, outputValue uint64, su
 
 	for i := uint32(0); i < numOutputs; i++ {
 		txOuts = append(txOuts, &wire.TxOut{
-			PkScript: OpTrueScript,
-			Value:    outputValue,
+			ScriptPubKey: OpTrueScript,
+			Value:        outputValue,
 		})
 	}
 
@@ -136,74 +161,102 @@ func createTxForTest(numInputs uint32, numOutputs uint32, outputValue uint64, su
 	return wire.NewNativeMsgTx(wire.TxVersion, txIns, txOuts)
 }
 
-// createCoinbaseTxForTest returns a coinbase transaction with the requested number of
-// outputs paying an appropriate subsidy based on the passed block height to the
-// address associated with the harness.  It automatically uses a standard
-// signature script that starts with the block height
-func createCoinbaseTxForTest(blockHeight int32, numOutputs uint32, extraNonce int64, params *dagconfig.Params) (*wire.MsgTx, error) {
-	// Create standard coinbase script.
-	coinbaseScript, err := txscript.NewScriptBuilder().
-		AddInt64(int64(blockHeight)).AddInt64(extraNonce).Script()
-	if err != nil {
-		return nil, err
-	}
-
-	txIns := []*wire.TxIn{&wire.TxIn{
-		// Coinbase transactions have no inputs, so previous outpoint is
-		// zero hash and max index.
-		PreviousOutPoint: *wire.NewOutPoint(&daghash.TxID{},
-			wire.MaxPrevOutIndex),
-		SignatureScript: coinbaseScript,
-		Sequence:        wire.MaxTxInSequenceNum,
-	}}
-
-	txOuts := []*wire.TxOut{}
-
-	totalInput := CalcBlockSubsidy(blockHeight, params)
-	amountPerOutput := totalInput / uint64(numOutputs)
-	remainder := totalInput - amountPerOutput*uint64(numOutputs)
-	for i := uint32(0); i < numOutputs; i++ {
-		// Ensure the final output accounts for any remainder that might
-		// be left from splitting the input amount.
-		amount := amountPerOutput
-		if i == numOutputs-1 {
-			amount = amountPerOutput + remainder
-		}
-		txOuts = append(txOuts, &wire.TxOut{
-			PkScript: OpTrueScript,
-			Value:    amount,
-		})
-	}
-
-	return wire.NewNativeMsgTx(wire.TxVersion, txIns, txOuts), nil
-}
+// VirtualForTest is an exported version for virtualBlock, so that it can be returned by exported test_util methods
+type VirtualForTest *virtualBlock
 
 // SetVirtualForTest replaces the dag's virtual block. This function is used for test purposes only
-func SetVirtualForTest(dag *BlockDAG, virtual *virtualBlock) *virtualBlock {
+func SetVirtualForTest(dag *BlockDAG, virtual VirtualForTest) VirtualForTest {
 	oldVirtual := dag.virtual
 	dag.virtual = virtual
-	return oldVirtual
+	return VirtualForTest(oldVirtual)
 }
 
 // GetVirtualFromParentsForTest generates a virtual block with the given parents.
-func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) (*virtualBlock, error) {
+func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) (VirtualForTest, error) {
 	parents := newSet()
 	for _, hash := range parentHashes {
 		parent := dag.index.LookupNode(hash)
 		if parent == nil {
-			return nil, fmt.Errorf("GetVirtualFromParentsForTest: didn't found node for hash %s", hash)
+			return nil, errors.Errorf("GetVirtualFromParentsForTest: didn't found node for hash %s", hash)
 		}
 		parents.add(parent)
 	}
 	virtual := newVirtualBlock(parents, dag.dagParams.K)
 
-	pastUTXO, _, err := virtual.pastUTXO(dag.virtual, dag.db)
+	pastUTXO, acceptanceData, err := dag.pastUTXO(&virtual.blockNode)
 	if err != nil {
 		return nil, err
 	}
-	diffPastUTXO := pastUTXO.clone().(*DiffUTXOSet)
-	diffPastUTXO.meldToBase()
-	virtual.utxoSet = diffPastUTXO.base
+	diffFromAcceptanceData, err := virtual.blockNode.diffFromAcceptanceData(pastUTXO, acceptanceData)
+	if err != nil {
+		return nil, err
+	}
+	utxo, err := pastUTXO.WithDiff(diffFromAcceptanceData)
+	if err != nil {
+		return nil, err
+	}
+	diffUTXO := utxo.clone().(*DiffUTXOSet)
+	err = diffUTXO.meldToBase()
+	if err != nil {
+		return nil, err
+	}
+	virtual.utxoSet = diffUTXO.base
 
-	return virtual, nil
+	return VirtualForTest(virtual), nil
+}
+
+// LoadBlocks reads files containing bitcoin block data (gzipped but otherwise
+// in the format bitcoind writes) from disk and returns them as an array of
+// util.Block.  This is largely borrowed from the test code in btcdb.
+func LoadBlocks(filename string) (blocks []*util.Block, err error) {
+	var network = wire.MainNet
+	var dr io.Reader
+	var fi io.ReadCloser
+
+	fi, err = os.Open(filename)
+	if err != nil {
+		return
+	}
+
+	if strings.HasSuffix(filename, ".bz2") {
+		dr = bzip2.NewReader(fi)
+	} else {
+		dr = fi
+	}
+	defer fi.Close()
+
+	var block *util.Block
+
+	err = nil
+	for height := uint64(0); err == nil; height++ {
+		var rintbuf uint32
+		err = binary.Read(dr, binary.LittleEndian, &rintbuf)
+		if err == io.EOF {
+			// hit end of file at expected offset: no warning
+			height--
+			err = nil
+			break
+		}
+		if err != nil {
+			break
+		}
+		if rintbuf != uint32(network) {
+			break
+		}
+		err = binary.Read(dr, binary.LittleEndian, &rintbuf)
+		blocklen := rintbuf
+
+		rbytes := make([]byte, blocklen)
+
+		// read block
+		dr.Read(rbytes)
+
+		block, err = util.NewBlockFromBytes(rbytes)
+		if err != nil {
+			return
+		}
+		blocks = append(blocks, block)
+	}
+
+	return
 }

blockdag/test_utils_test.go

@@ -1,14 +1,14 @@
 package blockdag
 
 import (
-	"errors"
+	"github.com/pkg/errors"
 	"os"
 	"strings"
 	"testing"
 
 	"bou.ke/monkey"
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/database"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
 )
 
 func TestIsSupportedDbType(t *testing.T) {

blockdag/thresholdstate.go

@@ -7,7 +7,7 @@ package blockdag
 import (
 	"fmt"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // ThresholdState define the various threshold states used when voting on
@@ -77,11 +77,11 @@ type thresholdConditionChecker interface {
 
 	// RuleChangeActivationThreshold is the number of blocks for which the
 	// condition must be true in order to lock in a rule change.
-	RuleChangeActivationThreshold() uint32
+	RuleChangeActivationThreshold() uint64
 
 	// MinerConfirmationWindow is the number of blocks in each threshold
 	// state retarget window.
-	MinerConfirmationWindow() uint32
+	MinerConfirmationWindow() uint64
 
 	// Condition returns whether or not the rule change activation condition
 	// has been met.  This typically involves checking whether or not the
@@ -129,16 +129,16 @@ func newThresholdCaches(numCaches uint32) []thresholdStateCache {
 func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdConditionChecker, cache *thresholdStateCache) (ThresholdState, error) {
 	// The threshold state for the window that contains the genesis block is
 	// defined by definition.
-	confirmationWindow := int32(checker.MinerConfirmationWindow())
-	if prevNode == nil || (prevNode.height+1) < confirmationWindow {
+	confirmationWindow := checker.MinerConfirmationWindow()
+	if prevNode == nil || (prevNode.chainHeight+1) < confirmationWindow {
 		return ThresholdDefined, nil
 	}
 
 	// Get the ancestor that is the last block of the previous confirmation
 	// window in order to get its threshold state.  This can be done because
 	// the state is the same for all blocks within a given window.
-	prevNode = prevNode.SelectedAncestor(prevNode.height -
-		(prevNode.height+1)%confirmationWindow)
+	prevNode = prevNode.SelectedAncestor(prevNode.chainHeight -
+		(prevNode.chainHeight+1)%confirmationWindow)
 
 	// Iterate backwards through each of the previous confirmation windows
 	// to find the most recently cached threshold state.
@@ -152,7 +152,7 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 
 		// The start and expiration times are based on the median block
 		// time, so calculate it now.
-		medianTime := prevNode.PastMedianTime()
+		medianTime := prevNode.PastMedianTime(dag)
 
 		// The state is simply defined if the start time hasn't been
 		// been reached yet.
@@ -192,7 +192,7 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 		case ThresholdDefined:
 			// The deployment of the rule change fails if it expires
 			// before it is accepted and locked in.
-			medianTime := prevNode.PastMedianTime()
+			medianTime := prevNode.PastMedianTime(dag)
 			medianTimeUnix := uint64(medianTime.Unix())
 			if medianTimeUnix >= checker.EndTime() {
 				state = ThresholdFailed
@@ -209,7 +209,7 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 		case ThresholdStarted:
 			// The deployment of the rule change fails if it expires
 			// before it is accepted and locked in.
-			medianTime := prevNode.PastMedianTime()
+			medianTime := prevNode.PastMedianTime(dag)
 			if uint64(medianTime.Unix()) >= checker.EndTime() {
 				state = ThresholdFailed
 				break
@@ -218,9 +218,9 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 			// At this point, the rule change is still being voted
 			// on by the miners, so iterate backwards through the
 			// confirmation window to count all of the votes in it.
-			var count uint32
+			var count uint64
 			countNode := prevNode
-			for i := int32(0); i < confirmationWindow; i++ {
+			for i := uint64(0); i < confirmationWindow; i++ {
 				condition, err := checker.Condition(countNode)
 				if err != nil {
 					return ThresholdFailed, err

blockdag/thresholdstate_test.go

@@ -7,7 +7,7 @@ package blockdag
 import (
 	"testing"
 
-	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/kaspanet/kaspad/util/daghash"
 )
 
 // TestThresholdStateStringer tests the stringized output for the

blockdag/utxo_ecmh.go

@@ -0,0 +1,32 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/golang/groupcache/lru"
+	"github.com/kaspanet/kaspad/btcec"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
+)
+
+const ecmhCacheSize = 4_000_000
+
+var (
+	utxoToECMHCache = lru.New(ecmhCacheSize)
+)
+
+func utxoMultiset(entry *UTXOEntry, outpoint *wire.Outpoint) (*btcec.Multiset, error) {
+	w := &bytes.Buffer{}
+	err := serializeUTXO(w, entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	serializedUTXO := w.Bytes()
+	utxoHash := daghash.DoubleHashH(serializedUTXO)
+
+	if cachedMSPoint, ok := utxoToECMHCache.Get(utxoHash); ok {
+		return cachedMSPoint.(*btcec.Multiset), nil
+	}
+	msPoint := btcec.NewMultiset(btcec.S256()).Add(serializedUTXO)
+	utxoToECMHCache.Add(utxoHash, msPoint)
+	return msPoint, nil
+}

blockdag/utxodiffstore.go

@@ -0,0 +1,191 @@
+package blockdag
+
+import (
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/locks"
+	"github.com/pkg/errors"
+)
+
+var multisetPointSize = 32
+
+type blockUTXODiffData struct {
+	diff      *UTXODiff
+	diffChild *blockNode
+}
+
+type utxoDiffStore struct {
+	dag    *BlockDAG
+	dirty  map[daghash.Hash]struct{}
+	loaded map[daghash.Hash]*blockUTXODiffData
+	mtx    *locks.PriorityMutex
+}
+
+func newUTXODiffStore(dag *BlockDAG) *utxoDiffStore {
+	return &utxoDiffStore{
+		dag:    dag,
+		dirty:  make(map[daghash.Hash]struct{}),
+		loaded: make(map[daghash.Hash]*blockUTXODiffData),
+		mtx:    locks.NewPriorityMutex(),
+	}
+}
+
+func (diffStore *utxoDiffStore) setBlockDiff(node *blockNode, diff *UTXODiff) error {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+	// load the diff data from DB to diffStore.loaded
+	_, exists, err := diffStore.diffDataByHash(node.hash)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		diffStore.loaded[*node.hash] = &blockUTXODiffData{}
+	}
+
+	diffStore.loaded[*node.hash].diff = diff
+	diffStore.setBlockAsDirty(node.hash)
+	return nil
+}
+
+func (diffStore *utxoDiffStore) setBlockDiffChild(node *blockNode, diffChild *blockNode) error {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+	// load the diff data from DB to diffStore.loaded
+	_, exists, err := diffStore.diffDataByHash(node.hash)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return diffNotFoundError(node)
+	}
+
+	diffStore.loaded[*node.hash].diffChild = diffChild
+	diffStore.setBlockAsDirty(node.hash)
+	return nil
+}
+
+func (diffStore *utxoDiffStore) removeBlocksDiffData(dbTx database.Tx, blockHashes []*daghash.Hash) error {
+	for _, hash := range blockHashes {
+		err := diffStore.removeBlockDiffData(dbTx, hash)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (diffStore *utxoDiffStore) removeBlockDiffData(dbTx database.Tx, blockHash *daghash.Hash) error {
+	diffStore.mtx.LowPriorityWriteLock()
+	defer diffStore.mtx.LowPriorityWriteUnlock()
+	delete(diffStore.loaded, *blockHash)
+	err := dbRemoveDiffData(dbTx, blockHash)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (diffStore *utxoDiffStore) setBlockAsDirty(blockHash *daghash.Hash) {
+	diffStore.dirty[*blockHash] = struct{}{}
+}
+
+func (diffStore *utxoDiffStore) diffDataByHash(hash *daghash.Hash) (*blockUTXODiffData, bool, error) {
+	if diffData, ok := diffStore.loaded[*hash]; ok {
+		return diffData, true, nil
+	}
+	diffData, err := diffStore.diffDataFromDB(hash)
+	if err != nil {
+		return nil, false, err
+	}
+	exists := diffData != nil
+	if exists {
+		diffStore.loaded[*hash] = diffData
+	}
+	return diffData, exists, nil
+}
+
+func diffNotFoundError(node *blockNode) error {
+	return errors.Errorf("Couldn't find diff data for block %s", node.hash)
+}
+
+func (diffStore *utxoDiffStore) diffByNode(node *blockNode) (*UTXODiff, error) {
+	diffStore.mtx.HighPriorityReadLock()
+	defer diffStore.mtx.HighPriorityReadUnlock()
+	diffData, exists, err := diffStore.diffDataByHash(node.hash)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, diffNotFoundError(node)
+	}
+	return diffData.diff, nil
+}
+
+func (diffStore *utxoDiffStore) diffChildByNode(node *blockNode) (*blockNode, error) {
+	diffStore.mtx.HighPriorityReadLock()
+	defer diffStore.mtx.HighPriorityReadUnlock()
+	diffData, exists, err := diffStore.diffDataByHash(node.hash)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, diffNotFoundError(node)
+	}
+	return diffData.diffChild, nil
+}
+
+func (diffStore *utxoDiffStore) diffDataFromDB(hash *daghash.Hash) (*blockUTXODiffData, error) {
+	var diffData *blockUTXODiffData
+	err := diffStore.dag.db.View(func(dbTx database.Tx) error {
+		bucket := dbTx.Metadata().Bucket(utxoDiffsBucketName)
+		serializedBlockDiffData := bucket.Get(hash[:])
+		if serializedBlockDiffData != nil {
+			var err error
+			diffData, err = diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return diffData, nil
+}
+
+// flushToDB writes all dirty diff data to the database. If all writes
+// succeed, this clears the dirty set.
+func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+	if len(diffStore.dirty) == 0 {
+		return nil
+	}
+
+	for hash := range diffStore.dirty {
+		diffData := diffStore.loaded[hash]
+		err := dbStoreDiffData(dbTx, &hash, diffData)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (diffStore *utxoDiffStore) clearDirtyEntries() {
+	diffStore.dirty = make(map[daghash.Hash]struct{})
+}
+
+// dbStoreDiffData stores the UTXO diff data to the database.
+// This overwrites the current entry if there exists one.
+func dbStoreDiffData(dbTx database.Tx, hash *daghash.Hash, diffData *blockUTXODiffData) error {
+	serializedDiffData, err := serializeBlockUTXODiffData(diffData)
+	if err != nil {
+		return err
+	}
+
+	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Put(hash[:], serializedDiffData)
+}
+
+func dbRemoveDiffData(dbTx database.Tx, hash *daghash.Hash) error {
+	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Delete(hash[:])
+}

blockdag/utxodiffstore_test.go

@@ -0,0 +1,86 @@
+package blockdag
+
+import (
+	"fmt"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
+	"reflect"
+	"testing"
+)
+
+func TestUTXODiffStore(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", Config{
+		DAGParams: &dagconfig.SimNetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestUTXODiffStore: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	nodeCounter := byte(0)
+	createNode := func() *blockNode {
+		nodeCounter++
+		node := &blockNode{hash: &daghash.Hash{nodeCounter}}
+		dag.index.AddNode(node)
+		return node
+	}
+
+	// Check that an error is returned when asking for non existing node
+	nonExistingNode := createNode()
+	_, err = dag.utxoDiffStore.diffByNode(nonExistingNode)
+	expectedErrString := fmt.Sprintf("Couldn't find diff data for block %s", nonExistingNode.hash)
+	if err == nil || err.Error() != expectedErrString {
+		t.Errorf("diffByNode: expected error %s but got %s", expectedErrString, err)
+	}
+
+	// Add node's diff data to the utxoDiffStore and check if it's checked correctly.
+	node := createNode()
+	diff := NewUTXODiff()
+	diff.toAdd.add(wire.Outpoint{TxID: daghash.TxID{0x01}, Index: 0}, &UTXOEntry{amount: 1, scriptPubKey: []byte{0x01}})
+	diff.toRemove.add(wire.Outpoint{TxID: daghash.TxID{0x02}, Index: 0}, &UTXOEntry{amount: 2, scriptPubKey: []byte{0x02}})
+	if err := dag.utxoDiffStore.setBlockDiff(node, diff); err != nil {
+		t.Fatalf("setBlockDiff: unexpected error: %s", err)
+	}
+	diffChild := createNode()
+	if err := dag.utxoDiffStore.setBlockDiffChild(node, diffChild); err != nil {
+		t.Fatalf("setBlockDiffChild: unexpected error: %s", err)
+	}
+
+	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
+		t.Fatalf("diffByNode: unexpected error: %s", err)
+	} else if !reflect.DeepEqual(storeDiff, diff) {
+		t.Errorf("Expected diff and storeDiff to be equal")
+	}
+
+	if storeDiffChild, err := dag.utxoDiffStore.diffChildByNode(node); err != nil {
+		t.Fatalf("diffByNode: unexpected error: %s", err)
+	} else if !reflect.DeepEqual(storeDiffChild, diffChild) {
+		t.Errorf("Expected diff and storeDiff to be equal")
+	}
+
+	// Flush changes to db, delete them from the dag.utxoDiffStore.loaded
+	// map, and check if the diff data is re-fetched from the database.
+	err = dag.db.Update(func(dbTx database.Tx) error {
+		return dag.utxoDiffStore.flushToDB(dbTx)
+	})
+	if err != nil {
+		t.Fatalf("Error flushing utxoDiffStore data to DB: %s", err)
+	}
+	delete(dag.utxoDiffStore.loaded, *node.hash)
+
+	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
+		t.Fatalf("diffByNode: unexpected error: %s", err)
+	} else if !reflect.DeepEqual(storeDiff, diff) {
+		t.Errorf("Expected diff and storeDiff to be equal")
+	}
+
+	// Check if getBlockDiff caches the result in dag.utxoDiffStore.loaded
+	if loadedDiffData, ok := dag.utxoDiffStore.loaded[*node.hash]; !ok {
+		t.Errorf("the diff data wasn't added to loaded map after requesting it")
+	} else if !reflect.DeepEqual(loadedDiffData.diff, diff) {
+		t.Errorf("Expected diff and loadedDiff to be equal")
+	}
+}

blockdag/utxoio.go

@@ -0,0 +1,314 @@
+package blockdag
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
+	"math/big"
+
+	"github.com/kaspanet/kaspad/btcec"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
+)
+
+// serializeBlockUTXODiffData serializes diff data in the following format:
+// 	Name         | Data type | Description
+//  ------------ | --------- | -----------
+// 	hasDiffChild | Boolean   | Indicates if a diff child exist
+//  diffChild    | Hash      | The diffChild's hash. Empty if hasDiffChild is true.
+//  diff		 | UTXODiff  | The diff data's diff
+func serializeBlockUTXODiffData(diffData *blockUTXODiffData) ([]byte, error) {
+	w := &bytes.Buffer{}
+	hasDiffChild := diffData.diffChild != nil
+	err := wire.WriteElement(w, hasDiffChild)
+	if err != nil {
+		return nil, err
+	}
+	if hasDiffChild {
+		err := wire.WriteElement(w, diffData.diffChild.hash)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err = serializeUTXODiff(w, diffData.diff)
+	if err != nil {
+		return nil, err
+	}
+
+	return w.Bytes(), nil
+}
+
+// utxoEntryHeaderCode returns the calculated header code to be used when
+// serializing the provided utxo entry.
+func utxoEntryHeaderCode(entry *UTXOEntry) uint64 {
+	// As described in the serialization format comments, the header code
+	// encodes the blue score shifted over one bit and the block reward flag
+	// in the lowest bit.
+	headerCode := uint64(entry.BlockBlueScore()) << 1
+	if entry.IsCoinbase() {
+		headerCode |= 0x01
+	}
+
+	return headerCode
+}
+
+func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataBytes []byte) (*blockUTXODiffData, error) {
+	diffData := &blockUTXODiffData{}
+	serializedDiffData := bytes.NewBuffer(serializedDiffDataBytes)
+
+	var hasDiffChild bool
+	err := wire.ReadElement(serializedDiffData, &hasDiffChild)
+	if err != nil {
+		return nil, err
+	}
+
+	if hasDiffChild {
+		hash := &daghash.Hash{}
+		err := wire.ReadElement(serializedDiffData, hash)
+		if err != nil {
+			return nil, err
+		}
+		diffData.diffChild = diffStore.dag.index.LookupNode(hash)
+	}
+
+	diffData.diff = &UTXODiff{
+		useMultiset: true,
+	}
+
+	diffData.diff.toAdd, err = deserializeDiffEntries(serializedDiffData)
+	if err != nil {
+		return nil, err
+	}
+
+	diffData.diff.toRemove, err = deserializeDiffEntries(serializedDiffData)
+	if err != nil {
+		return nil, err
+	}
+
+	diffData.diff.diffMultiset, err = deserializeMultiset(serializedDiffData)
+	if err != nil {
+		return nil, err
+	}
+
+	return diffData, nil
+}
+
+func deserializeDiffEntries(r io.Reader) (utxoCollection, error) {
+	count, err := wire.ReadVarInt(r)
+	if err != nil {
+		return nil, err
+	}
+	collection := utxoCollection{}
+	for i := uint64(0); i < count; i++ {
+		outpointSize, err := wire.ReadVarInt(r)
+		if err != nil {
+			return nil, err
+		}
+
+		serializedOutpoint := make([]byte, outpointSize)
+		err = binary.Read(r, byteOrder, serializedOutpoint)
+		if err != nil {
+			return nil, err
+		}
+		outpoint, err := deserializeOutpoint(serializedOutpoint)
+		if err != nil {
+			return nil, err
+		}
+
+		utxoEntrySize, err := wire.ReadVarInt(r)
+		if err != nil {
+			return nil, err
+		}
+		serializedEntry := make([]byte, utxoEntrySize)
+		err = binary.Read(r, byteOrder, serializedEntry)
+		if err != nil {
+			return nil, err
+		}
+		utxoEntry, err := deserializeUTXOEntry(serializedEntry)
+		if err != nil {
+			return nil, err
+		}
+		collection.add(*outpoint, utxoEntry)
+	}
+	return collection, nil
+}
+
+// deserializeMultiset deserializes an EMCH multiset.
+// See serializeMultiset for more details.
+func deserializeMultiset(r io.Reader) (*btcec.Multiset, error) {
+	xBytes := make([]byte, multisetPointSize)
+	yBytes := make([]byte, multisetPointSize)
+	err := binary.Read(r, byteOrder, xBytes)
+	if err != nil {
+		return nil, err
+	}
+	err = binary.Read(r, byteOrder, yBytes)
+	if err != nil {
+		return nil, err
+	}
+	var x, y big.Int
+	x.SetBytes(xBytes)
+	y.SetBytes(yBytes)
+	return btcec.NewMultisetFromPoint(btcec.S256(), &x, &y), nil
+}
+
+// serializeUTXODiff serializes UTXODiff by serializing
+// UTXODiff.toAdd, UTXODiff.toRemove and UTXODiff.Multiset one after the other.
+func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
+	if !diff.useMultiset {
+		return errors.New("Cannot serialize a UTXO diff without a multiset")
+	}
+	err := serializeUTXOCollection(w, diff.toAdd)
+	if err != nil {
+		return err
+	}
+
+	err = serializeUTXOCollection(w, diff.toRemove)
+	if err != nil {
+		return err
+	}
+	err = serializeMultiset(w, diff.diffMultiset)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// serializeUTXOCollection serializes utxoCollection by iterating over
+// the utxo entries and serializing them and their corresponding outpoint
+// prefixed by a varint that indicates their size.
+func serializeUTXOCollection(w io.Writer, collection utxoCollection) error {
+	err := wire.WriteVarInt(w, uint64(len(collection)))
+	if err != nil {
+		return err
+	}
+	for outpoint, utxoEntry := range collection {
+		err := serializeUTXO(w, utxoEntry, &outpoint)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// serializeMultiset serializes an ECMH multiset. The serialization
+// is done by taking the (x,y) coordinnates of the multiset point and
+// padding each one of them with 32 byte (it'll be 32 byte in most
+// cases anyway except one of the coordinates is zero) and writing
+// them one after the other.
+func serializeMultiset(w io.Writer, ms *btcec.Multiset) error {
+	x, y := ms.Point()
+	xBytes := make([]byte, multisetPointSize)
+	copy(xBytes, x.Bytes())
+	yBytes := make([]byte, multisetPointSize)
+	copy(yBytes, y.Bytes())
+
+	err := binary.Write(w, byteOrder, xBytes)
+	if err != nil {
+		return err
+	}
+	err = binary.Write(w, byteOrder, yBytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// serializeUTXO serializes a utxo entry-outpoint pair
+func serializeUTXO(w io.Writer, entry *UTXOEntry, outpoint *wire.Outpoint) error {
+	serializedOutpoint := *outpointKey(*outpoint)
+	err := wire.WriteVarInt(w, uint64(len(serializedOutpoint)))
+	if err != nil {
+		return err
+	}
+
+	err = binary.Write(w, byteOrder, serializedOutpoint)
+	if err != nil {
+		return err
+	}
+
+	serializedUTXOEntry := serializeUTXOEntry(entry)
+	err = wire.WriteVarInt(w, uint64(len(serializedUTXOEntry)))
+	if err != nil {
+		return err
+	}
+	err = binary.Write(w, byteOrder, serializedUTXOEntry)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// serializeUTXOEntry returns the entry serialized to a format that is suitable
+// for long-term storage.  The format is described in detail above.
+func serializeUTXOEntry(entry *UTXOEntry) []byte {
+	// Encode the header code.
+	headerCode := utxoEntryHeaderCode(entry)
+
+	// Calculate the size needed to serialize the entry.
+	size := serializeSizeVLQ(headerCode) +
+		compressedTxOutSize(uint64(entry.Amount()), entry.ScriptPubKey())
+
+	// Serialize the header code followed by the compressed unspent
+	// transaction output.
+	serialized := make([]byte, size)
+	offset := putVLQ(serialized, headerCode)
+	offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()),
+		entry.ScriptPubKey())
+
+	return serialized
+}
+
+// deserializeOutpoint decodes an outpoint from the passed serialized byte
+// slice into a new wire.Outpoint using a format that is suitable for long-
+// term storage. this format is described in detail above.
+func deserializeOutpoint(serialized []byte) (*wire.Outpoint, error) {
+	if len(serialized) <= daghash.HashSize {
+		return nil, errDeserialize("unexpected end of data")
+	}
+
+	txID := daghash.TxID{}
+	txID.SetBytes(serialized[:daghash.HashSize])
+	index, _ := deserializeVLQ(serialized[daghash.HashSize:])
+	return wire.NewOutpoint(&txID, uint32(index)), nil
+}
+
+// deserializeUTXOEntry decodes a UTXO entry from the passed serialized byte
+// slice into a new UTXOEntry using a format that is suitable for long-term
+// storage.  The format is described in detail above.
+func deserializeUTXOEntry(serialized []byte) (*UTXOEntry, error) {
+	// Deserialize the header code.
+	code, offset := deserializeVLQ(serialized)
+	if offset >= len(serialized) {
+		return nil, errDeserialize("unexpected end of data after header")
+	}
+
+	// Decode the header code.
+	//
+	// Bit 0 indicates whether the containing transaction is a coinbase.
+	// Bits 1-x encode blue score of the containing transaction.
+	isCoinbase := code&0x01 != 0
+	blockBlueScore := code >> 1
+
+	// Decode the compressed unspent transaction output.
+	amount, scriptPubKey, _, err := decodeCompressedTxOut(serialized[offset:])
+	if err != nil {
+		return nil, errDeserialize(fmt.Sprintf("unable to decode "+
+			"UTXO: %s", err))
+	}
+
+	entry := &UTXOEntry{
+		amount:         amount,
+		scriptPubKey:   scriptPubKey,
+		blockBlueScore: blockBlueScore,
+		packedFlags:    0,
+	}
+	if isCoinbase {
+		entry.packedFlags |= tfCoinbase
+	}
+
+	return entry, nil
+}

blockdag/utxoset.go

@@ -1,18 +1,27 @@
 package blockdag
 
 import (
-	"errors"
 	"fmt"
+	"github.com/pkg/errors"
+	"math"
 	"sort"
 	"strings"
 
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/btcec"
+	"github.com/kaspanet/kaspad/wire"
+)
+
+const (
+	// UnacceptedBlueScore is the blue score used for the "block" blueScore
+	// field of the contextual transaction information provided in a
+	// transaction store when it has not yet been accepted by a block.
+	UnacceptedBlueScore uint64 = math.MaxUint64
 )
 
 // UTXOEntry houses details about an individual transaction output in a utxo
-// set such as whether or not it was contained in a block reward tx, the height of
-// the block that contains the tx, whether or not it is spent, its public key
-// script, and how much it pays.
+// set such as whether or not it was contained in a coinbase tx, the blue
+// score of the block that accepts the tx, its public key script, and how
+// much it pays.
 type UTXOEntry struct {
 	// NOTE: Additions, deletions, or modifications to the order of the
 	// definitions in this struct should not be changed without considering
@@ -20,26 +29,26 @@ type UTXOEntry struct {
 	// specifically crafted to result in minimal padding.  There will be a
 	// lot of these in memory, so a few extra bytes of padding adds up.
 
-	amount      uint64
-	pkScript    []byte // The public key script for the output.
-	blockHeight int32  // Height of block containing tx.
+	amount         uint64
+	scriptPubKey   []byte // The public key script for the output.
+	blockBlueScore uint64 // Blue score of the block accepting the tx.
 
 	// packedFlags contains additional info about output such as whether it
-	// is a block reward, and whether it has been modified
+	// is a coinbase, and whether it has been modified
 	// since it was loaded.  This approach is used in order to reduce memory
 	// usage since there will be a lot of these in memory.
 	packedFlags txoFlags
 }
 
-// IsBlockReward returns whether or not the output was contained in a block
+// IsCoinbase returns whether or not the output was contained in a block
 // reward transaction.
-func (entry *UTXOEntry) IsBlockReward() bool {
-	return entry.packedFlags&tfBlockReward == tfBlockReward
+func (entry *UTXOEntry) IsCoinbase() bool {
+	return entry.packedFlags&tfCoinbase == tfCoinbase
 }
 
-// BlockHeight returns the height of the block containing the output.
-func (entry *UTXOEntry) BlockHeight() int32 {
-	return entry.blockHeight
+// BlockBlueScore returns the blue score of the block accepting the output.
+func (entry *UTXOEntry) BlockBlueScore() uint64 {
+	return entry.blockBlueScore
 }
 
 // Amount returns the amount of the output.
@@ -47,9 +56,15 @@ func (entry *UTXOEntry) Amount() uint64 {
 	return entry.amount
 }
 
-// PkScript returns the public key script for the output.
-func (entry *UTXOEntry) PkScript() []byte {
-	return entry.pkScript
+// ScriptPubKey returns the public key script for the output.
+func (entry *UTXOEntry) ScriptPubKey() []byte {
+	return entry.scriptPubKey
+}
+
+// IsUnaccepted returns true iff this UTXOEntry has been included in a block
+// but has not yet been accepted by any block.
+func (entry *UTXOEntry) IsUnaccepted() bool {
+	return entry.blockBlueScore == UnacceptedBlueScore
 }
 
 // txoFlags is a bitmask defining additional information and state for a
@@ -57,19 +72,35 @@ func (entry *UTXOEntry) PkScript() []byte {
 type txoFlags uint8
 
 const (
-	// tfBlockReward indicates that a txout was contained in a block reward tx (coinbase or fee transaction).
-	tfBlockReward txoFlags = 1 << iota
+	// tfCoinbase indicates that a txout was contained in a coinbase tx.
+	tfCoinbase txoFlags = 1 << iota
 )
 
-// utxoCollection represents a set of UTXOs indexed by their outPoints
-type utxoCollection map[wire.OutPoint]*UTXOEntry
+// NewUTXOEntry creates a new utxoEntry representing the given txOut
+func NewUTXOEntry(txOut *wire.TxOut, isCoinbase bool, blockBlueScore uint64) *UTXOEntry {
+	entry := &UTXOEntry{
+		amount:         txOut.Value,
+		scriptPubKey:   txOut.ScriptPubKey,
+		blockBlueScore: blockBlueScore,
+	}
+
+	if isCoinbase {
+		entry.packedFlags |= tfCoinbase
+	}
+
+	return entry
+}
+
+// utxoCollection represents a set of UTXOs indexed by their outpoints
+type utxoCollection map[wire.Outpoint]*UTXOEntry
 
 func (uc utxoCollection) String() string {
 	utxoStrings := make([]string, len(uc))
 
 	i := 0
-	for outPoint, utxoEntry := range uc {
-		utxoStrings[i] = fmt.Sprintf("(%s, %d) => %d", outPoint.TxID, outPoint.Index, utxoEntry.amount)
+	for outpoint, utxoEntry := range uc {
+		utxoStrings[i] = fmt.Sprintf("(%s, %d) => %d, blueScore: %d",
+			outpoint.TxID, outpoint.Index, utxoEntry.amount, utxoEntry.blockBlueScore)
 		i++
 	}
 
@@ -80,33 +111,40 @@ func (uc utxoCollection) String() string {
 }
 
 // add adds a new UTXO entry to this collection
-func (uc utxoCollection) add(outPoint wire.OutPoint, entry *UTXOEntry) {
-	uc[outPoint] = entry
+func (uc utxoCollection) add(outpoint wire.Outpoint, entry *UTXOEntry) {
+	uc[outpoint] = entry
 }
 
 // remove removes a UTXO entry from this collection if it exists
-func (uc utxoCollection) remove(outPoint wire.OutPoint) {
-	delete(uc, outPoint)
+func (uc utxoCollection) remove(outpoint wire.Outpoint) {
+	delete(uc, outpoint)
 }
 
-// get returns the UTXOEntry represented by provided outPoint,
+// get returns the UTXOEntry represented by provided outpoint,
 // and a boolean value indicating if said UTXOEntry is in the set or not
-func (uc utxoCollection) get(outPoint wire.OutPoint) (*UTXOEntry, bool) {
-	entry, ok := uc[outPoint]
+func (uc utxoCollection) get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
+	entry, ok := uc[outpoint]
 	return entry, ok
 }
 
 // contains returns a boolean value indicating whether a UTXO entry is in the set
-func (uc utxoCollection) contains(outPoint wire.OutPoint) bool {
-	_, ok := uc[outPoint]
+func (uc utxoCollection) contains(outpoint wire.Outpoint) bool {
+	_, ok := uc[outpoint]
 	return ok
 }
 
+// containsWithBlueScore returns a boolean value indicating whether a UTXOEntry
+// is in the set and its blue score is equal to the given blue score.
+func (uc utxoCollection) containsWithBlueScore(outpoint wire.Outpoint, blueScore uint64) bool {
+	entry, ok := uc.get(outpoint)
+	return ok && entry.blockBlueScore == blueScore
+}
+
 // clone returns a clone of this collection
 func (uc utxoCollection) clone() utxoCollection {
 	clone := utxoCollection{}
-	for outPoint, entry := range uc {
-		clone.add(outPoint, entry)
+	for outpoint, entry := range uc {
+		clone.add(outpoint, entry)
 	}
 
 	return clone
@@ -114,15 +152,29 @@ func (uc utxoCollection) clone() utxoCollection {
 
 // UTXODiff represents a diff between two UTXO Sets.
 type UTXODiff struct {
-	toAdd    utxoCollection
-	toRemove utxoCollection
+	toAdd        utxoCollection
+	toRemove     utxoCollection
+	diffMultiset *btcec.Multiset
+	useMultiset  bool
 }
 
-// NewUTXODiff creates a new, empty utxoDiff
+// NewUTXODiffWithoutMultiset creates a new, empty utxoDiff
+// without a multiset.
+func NewUTXODiffWithoutMultiset() *UTXODiff {
+	return &UTXODiff{
+		toAdd:       utxoCollection{},
+		toRemove:    utxoCollection{},
+		useMultiset: false,
+	}
+}
+
+// NewUTXODiff creates a new, empty utxoDiff.
 func NewUTXODiff() *UTXODiff {
 	return &UTXODiff{
-		toAdd:    utxoCollection{},
-		toRemove: utxoCollection{},
+		toAdd:        utxoCollection{},
+		toRemove:     utxoCollection{},
+		useMultiset:  true,
+		diffMultiset: btcec.NewMultiset(btcec.S256()),
 	}
 }
 
@@ -155,7 +207,11 @@ func NewUTXODiff() *UTXODiff {
 // 2. This diff contains a UTXO in toRemove, and the other diff does not contain it
 //    diffFrom results in the UTXO being added to toAdd
 func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
-	result := NewUTXODiff()
+	result := UTXODiff{
+		toAdd:       make(utxoCollection, len(d.toRemove)+len(other.toAdd)),
+		toRemove:    make(utxoCollection, len(d.toAdd)+len(other.toRemove)),
+		useMultiset: d.useMultiset,
+	}
 
 	// Note that the following cases are not accounted for, as they are impossible
 	// as long as the base utxoSet is the same:
@@ -165,44 +221,67 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 	// All transactions in d.toAdd:
 	// If they are not in other.toAdd - should be added in result.toRemove
 	// If they are in other.toRemove - base utxoSet is not the same
-	for outPoint, utxoEntry := range d.toAdd {
-		if !other.toAdd.contains(outPoint) {
-			result.toRemove.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range d.toAdd {
+		if !other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toRemove.add(outpoint, utxoEntry)
 		}
-		if other.toRemove.contains(outPoint) {
-			return nil, fmt.Errorf("diffFrom: outpoint %s both in d.toAdd and in other.toRemove", outPoint)
+		if diffEntry, ok := other.toRemove.get(outpoint); ok {
+			// An exception is made for entries with unequal blue scores
+			// as long as the appropriate entry exists in either d.toRemove
+			// or other.toAdd.
+			// These are just "updates" to accepted blue score
+			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
+				(d.toRemove.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
+					other.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
+				continue
+			}
+			return nil, errors.Errorf("diffFrom: outpoint %s both in d.toAdd and in other.toRemove", outpoint)
 		}
 	}
 
 	// All transactions in d.toRemove:
 	// If they are not in other.toRemove - should be added in result.toAdd
 	// If they are in other.toAdd - base utxoSet is not the same
-	for outPoint, utxoEntry := range d.toRemove {
-		if !other.toRemove.contains(outPoint) {
-			result.toAdd.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range d.toRemove {
+		if !other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toAdd.add(outpoint, utxoEntry)
 		}
-		if other.toAdd.contains(outPoint) {
+		if diffEntry, ok := other.toAdd.get(outpoint); ok {
+			// An exception is made for entries with unequal blue scores
+			// as long as the appropriate entry exists in either d.toAdd
+			// or other.toRemove.
+			// These are just "updates" to accepted blue score
+			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
+				(d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
+					other.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
+				continue
+			}
 			return nil, errors.New("diffFrom: transaction both in d.toRemove and in other.toAdd")
 		}
 	}
 
 	// All transactions in other.toAdd:
 	// If they are not in d.toAdd - should be added in result.toAdd
-	for outPoint, utxoEntry := range other.toAdd {
-		if !d.toAdd.contains(outPoint) {
-			result.toAdd.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range other.toAdd {
+		if !d.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toAdd.add(outpoint, utxoEntry)
 		}
 	}
 
 	// All transactions in other.toRemove:
 	// If they are not in d.toRemove - should be added in result.toRemove
-	for outPoint, utxoEntry := range other.toRemove {
-		if !d.toRemove.contains(outPoint) {
-			result.toRemove.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range other.toRemove {
+		if !d.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toRemove.add(outpoint, utxoEntry)
 		}
 	}
 
-	return result, nil
+	if d.useMultiset {
+		// Create a new diffMultiset as the subtraction of the two diffs.
+		result.diffMultiset = other.diffMultiset.Subtract(d.diffMultiset)
+	}
+
+	return &result, nil
 }
 
 // WithDiff applies provided diff to this diff, creating a new utxoDiff, that would be the result if
@@ -232,18 +311,31 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 // 2. This diff contains a UTXO in toRemove, and the other diff does not contain it
 //    WithDiff results in the UTXO being added to toRemove
 func (d *UTXODiff) WithDiff(diff *UTXODiff) (*UTXODiff, error) {
-	result := NewUTXODiff()
+	result := UTXODiff{
+		toAdd:       make(utxoCollection, len(d.toAdd)+len(diff.toAdd)),
+		toRemove:    make(utxoCollection, len(d.toRemove)+len(diff.toRemove)),
+		useMultiset: d.useMultiset,
+	}
 
 	// All transactions in d.toAdd:
 	// If they are not in diff.toRemove - should be added in result.toAdd
 	// If they are in diff.toAdd - should throw an error
 	// Otherwise - should be ignored
-	for outPoint, utxoEntry := range d.toAdd {
-		if !diff.toRemove.contains(outPoint) {
-			result.toAdd.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range d.toAdd {
+		if !diff.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toAdd.add(outpoint, utxoEntry)
 		}
-		if diff.toAdd.contains(outPoint) {
-			return nil, ruleError(ErrWithDiff, fmt.Sprintf("WithDiff: outpoint %s both in d.toAdd and in other.toAdd", outPoint))
+		if diffEntry, ok := diff.toAdd.get(outpoint); ok {
+			// An exception is made for entries with unequal blue scores
+			// as long as the appropriate entry exists in either d.toRemove
+			// or diff.toRemove.
+			// These are just "updates" to accepted blue score
+			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
+				(d.toRemove.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
+					diff.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
+				continue
+			}
+			return nil, ruleError(ErrWithDiff, fmt.Sprintf("WithDiff: outpoint %s both in d.toAdd and in other.toAdd", outpoint))
 		}
 	}
 
@@ -251,74 +343,114 @@ func (d *UTXODiff) WithDiff(diff *UTXODiff) (*UTXODiff, error) {
 	// If they are not in diff.toAdd - should be added in result.toRemove
 	// If they are in diff.toRemove - should throw an error
 	// Otherwise - should be ignored
-	for outPoint, utxoEntry := range d.toRemove {
-		if !diff.toAdd.contains(outPoint) {
-			result.toRemove.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range d.toRemove {
+		if !diff.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toRemove.add(outpoint, utxoEntry)
 		}
-		if diff.toRemove.contains(outPoint) {
+		if diffEntry, ok := diff.toRemove.get(outpoint); ok {
+			// An exception is made for entries with unequal blue scores
+			// as long as the appropriate entry exists in either d.toAdd
+			// or diff.toAdd.
+			// These are just "updates" to accepted blue score
+			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
+				(d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) ||
+					diff.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore)) {
+				continue
+			}
 			return nil, ruleError(ErrWithDiff, "WithDiff: transaction both in d.toRemove and in other.toRemove")
 		}
 	}
 
 	// All transactions in diff.toAdd:
 	// If they are not in d.toRemove - should be added in result.toAdd
-	for outPoint, utxoEntry := range diff.toAdd {
-		if !d.toRemove.contains(outPoint) {
-			result.toAdd.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range diff.toAdd {
+		if !d.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toAdd.add(outpoint, utxoEntry)
 		}
 	}
 
 	// All transactions in diff.toRemove:
 	// If they are not in d.toAdd - should be added in result.toRemove
-	for outPoint, utxoEntry := range diff.toRemove {
-		if !d.toAdd.contains(outPoint) {
-			result.toRemove.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range diff.toRemove {
+		if !d.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
+			result.toRemove.add(outpoint, utxoEntry)
 		}
 	}
 
-	return result, nil
+	// Apply diff.diffMultiset to d.diffMultiset
+	if d.useMultiset {
+		result.diffMultiset = d.diffMultiset.Union(diff.diffMultiset)
+	}
+
+	return &result, nil
 }
 
 // clone returns a clone of this utxoDiff
 func (d *UTXODiff) clone() *UTXODiff {
-	return &UTXODiff{
-		toAdd:    d.toAdd.clone(),
-		toRemove: d.toRemove.clone(),
+	clone := &UTXODiff{
+		toAdd:       d.toAdd.clone(),
+		toRemove:    d.toRemove.clone(),
+		useMultiset: d.useMultiset,
 	}
+	if d.useMultiset {
+		clone.diffMultiset = d.diffMultiset.Clone()
+	}
+	return clone
 }
 
-//RemoveTxOuts marks the transaction's outputs to removal
-func (d *UTXODiff) RemoveTxOuts(tx *wire.MsgTx) {
-	for idx := range tx.TxOut {
-		hash := tx.TxID()
-		d.toRemove.add(*wire.NewOutPoint(&hash, uint32(idx)), nil)
+// AddEntry adds a UTXOEntry to the diff
+//
+// If d.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (d *UTXODiff) AddEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
+	if d.toRemove.containsWithBlueScore(outpoint, entry.blockBlueScore) {
+		d.toRemove.remove(outpoint)
+	} else if _, exists := d.toAdd[outpoint]; exists {
+		return errors.Errorf("AddEntry: Cannot add outpoint %s twice", outpoint)
+	} else {
+		d.toAdd.add(outpoint, entry)
 	}
+
+	if d.useMultiset {
+		newMs, err := addUTXOToMultiset(d.diffMultiset, entry, &outpoint)
+		if err != nil {
+			return err
+		}
+		d.diffMultiset = newMs
+	}
+	return nil
 }
 
-//AddEntry adds an UTXOEntry to the diff
-func (d *UTXODiff) AddEntry(outpoint wire.OutPoint, entry *UTXOEntry) {
-	d.toAdd.add(outpoint, entry)
+// RemoveEntry removes a UTXOEntry from the diff.
+//
+// If d.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (d *UTXODiff) RemoveEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
+	if d.toAdd.containsWithBlueScore(outpoint, entry.blockBlueScore) {
+		d.toAdd.remove(outpoint)
+	} else if _, exists := d.toRemove[outpoint]; exists {
+		return errors.Errorf("removeEntry: Cannot remove outpoint %s twice", outpoint)
+	} else {
+		d.toRemove.add(outpoint, entry)
+	}
+
+	if d.useMultiset {
+		newMs, err := removeUTXOFromMultiset(d.diffMultiset, entry, &outpoint)
+		if err != nil {
+			return err
+		}
+		d.diffMultiset = newMs
+	}
+	return nil
 }
 
 func (d UTXODiff) String() string {
+	if d.useMultiset {
+		return fmt.Sprintf("toAdd: %s; toRemove: %s, Multiset-Hash: %s", d.toAdd, d.toRemove, d.diffMultiset.Hash())
+	}
 	return fmt.Sprintf("toAdd: %s; toRemove: %s", d.toAdd, d.toRemove)
 }
 
-// NewUTXOEntry creates a new utxoEntry representing the given txOut
-func NewUTXOEntry(txOut *wire.TxOut, isBlockReward bool, blockHeight int32) *UTXOEntry {
-	entry := &UTXOEntry{
-		amount:      txOut.Value,
-		pkScript:    txOut.PkScript,
-		blockHeight: blockHeight,
-	}
-
-	if isBlockReward {
-		entry.packedFlags |= tfBlockReward
-	}
-
-	return entry
-}
-
 // UTXOSet represents a set of unspent transaction outputs
 // Every DAG has exactly one fullUTXOSet.
 // When a new block arrives, it is validated and applied to the fullUTXOSet in the following manner:
@@ -335,35 +467,75 @@ type UTXOSet interface {
 	fmt.Stringer
 	diffFrom(other UTXOSet) (*UTXODiff, error)
 	WithDiff(utxoDiff *UTXODiff) (UTXOSet, error)
-	diffFromTx(tx *wire.MsgTx, node *blockNode) (*UTXODiff, error)
-	AddTx(tx *wire.MsgTx, blockHeight int32) (ok bool)
+	diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error)
+	diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error)
+	AddTx(tx *wire.MsgTx, blockBlueScore uint64) (ok bool, err error)
 	clone() UTXOSet
-	Get(outPoint wire.OutPoint) (*UTXOEntry, bool)
+	Get(outpoint wire.Outpoint) (*UTXOEntry, bool)
+	Multiset() *btcec.Multiset
+	WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error)
 }
 
 // diffFromTx is a common implementation for diffFromTx, that works
 // for both diff-based and full UTXO sets
 // Returns a diff that is equivalent to provided transaction,
 // or an error if provided transaction is not valid in the context of this UTXOSet
-func diffFromTx(u UTXOSet, tx *wire.MsgTx, containingNode *blockNode) (*UTXODiff, error) {
+func diffFromTx(u UTXOSet, tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
 	diff := NewUTXODiff()
-	isBlockReward := tx.IsBlockReward()
-	if !isBlockReward {
+	isCoinbase := tx.IsCoinBase()
+	if !isCoinbase {
 		for _, txIn := range tx.TxIn {
-			if entry, ok := u.Get(txIn.PreviousOutPoint); ok {
-				diff.toRemove.add(txIn.PreviousOutPoint, entry)
+			if entry, ok := u.Get(txIn.PreviousOutpoint); ok {
+				err := diff.RemoveEntry(txIn.PreviousOutpoint, entry)
+				if err != nil {
+					return nil, err
+				}
 			} else {
 				return nil, ruleError(ErrMissingTxOut, fmt.Sprintf(
 					"Transaction %s is invalid because spends outpoint %s that is not in utxo set",
-					tx.TxID(), txIn.PreviousOutPoint))
+					tx.TxID(), txIn.PreviousOutpoint))
 			}
 		}
 	}
 	for i, txOut := range tx.TxOut {
-		hash := tx.TxID()
-		entry := NewUTXOEntry(txOut, isBlockReward, containingNode.height)
-		outPoint := *wire.NewOutPoint(&hash, uint32(i))
-		diff.toAdd.add(outPoint, entry)
+		entry := NewUTXOEntry(txOut, isCoinbase, acceptingBlueScore)
+		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
+		err := diff.AddEntry(outpoint, entry)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return diff, nil
+}
+
+// diffFromAcceptedTx is a common implementation for diffFromAcceptedTx, that works
+// for both diff-based and full UTXO sets.
+// Returns a diff that replaces an entry's blockBlueScore with the given acceptingBlueScore.
+// Returns an error if the provided transaction's entry is not valid in the context
+// of this UTXOSet.
+func diffFromAcceptedTx(u UTXOSet, tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
+	diff := NewUTXODiff()
+	isCoinbase := tx.IsCoinBase()
+	for i, txOut := range tx.TxOut {
+		// Fetch any unaccepted transaction
+		existingOutpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
+		existingEntry, ok := u.Get(existingOutpoint)
+		if !ok {
+			return nil, errors.Errorf("cannot accept outpoint %s because it doesn't exist in the given UTXO", existingOutpoint)
+		}
+
+		// Remove unaccepted entries
+		err := diff.RemoveEntry(existingOutpoint, existingEntry)
+		if err != nil {
+			return nil, err
+		}
+
+		// Add new entries with their accepting blue score
+		newEntry := NewUTXOEntry(txOut, isCoinbase, acceptingBlueScore)
+		err = diff.AddEntry(existingOutpoint, newEntry)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return diff, nil
 }
@@ -371,15 +543,33 @@ func diffFromTx(u UTXOSet, tx *wire.MsgTx, containingNode *blockNode) (*UTXODiff
 // FullUTXOSet represents a full list of transaction outputs and their values
 type FullUTXOSet struct {
 	utxoCollection
+	UTXOMultiset *btcec.Multiset
 }
 
 // NewFullUTXOSet creates a new utxoSet with full list of transaction outputs and their values
 func NewFullUTXOSet() *FullUTXOSet {
 	return &FullUTXOSet{
 		utxoCollection: utxoCollection{},
+		UTXOMultiset:   btcec.NewMultiset(btcec.S256()),
 	}
 }
 
+// newFullUTXOSetFromUTXOCollection converts a utxoCollection to a FullUTXOSet
+func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet, error) {
+	var err error
+	multiset := btcec.NewMultiset(btcec.S256())
+	for outpoint, utxoEntry := range collection {
+		multiset, err = addUTXOToMultiset(multiset, utxoEntry, &outpoint)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &FullUTXOSet{
+		utxoCollection: collection,
+		UTXOMultiset:   multiset,
+	}, nil
+}
+
 // diffFrom returns the difference between this utxoSet and another
 // diffFrom can only work when other is a diffUTXOSet, and its base utxoSet is this.
 func (fus *FullUTXOSet) diffFrom(other UTXOSet) (*UTXODiff, error) {
@@ -400,41 +590,50 @@ func (fus *FullUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
 	return NewDiffUTXOSet(fus, other.clone()), nil
 }
 
-// AddTx adds a transaction to this utxoSet and returns true iff it's valid in this UTXO's context
-func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blockHeight int32) bool {
-	isBlockReward := tx.IsBlockReward()
-	if !isBlockReward {
+// AddTx adds a transaction to this utxoSet and returns isAccepted=true iff it's valid in this UTXO's context.
+// It returns error if something unexpected happens, such as serialization error (isAccepted=false doesn't
+// necessarily means there's an error).
+//
+// This function MUST be called with the DAG lock held.
+func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blueScore uint64) (isAccepted bool, err error) {
+	isCoinbase := tx.IsCoinBase()
+	if !isCoinbase {
 		if !fus.containsInputs(tx) {
-			return false
+			return false, nil
 		}
 
 		for _, txIn := range tx.TxIn {
-			outPoint := *wire.NewOutPoint(&txIn.PreviousOutPoint.TxID, txIn.PreviousOutPoint.Index)
-			fus.remove(outPoint)
+			outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+			err := fus.removeAndUpdateMultiset(outpoint)
+			if err != nil {
+				return false, err
+			}
 		}
 	}
 
 	for i, txOut := range tx.TxOut {
-		hash := tx.TxID()
-		outPoint := *wire.NewOutPoint(&hash, uint32(i))
-		entry := NewUTXOEntry(txOut, isBlockReward, blockHeight)
+		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
+		entry := NewUTXOEntry(txOut, isCoinbase, blueScore)
 
-		fus.add(outPoint, entry)
+		err := fus.addAndUpdateMultiset(outpoint, entry)
+		if err != nil {
+			return false, err
+		}
 	}
 
-	return true
+	return true, nil
 }
 
 // diffFromTx returns a diff that is equivalent to provided transaction,
 // or an error if provided transaction is not valid in the context of this UTXOSet
-func (fus *FullUTXOSet) diffFromTx(tx *wire.MsgTx, node *blockNode) (*UTXODiff, error) {
-	return diffFromTx(fus, tx, node)
+func (fus *FullUTXOSet) diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
+	return diffFromTx(fus, tx, acceptingBlueScore)
 }
 
 func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 	for _, txIn := range tx.TxIn {
-		outPoint := *wire.NewOutPoint(&txIn.PreviousOutPoint.TxID, txIn.PreviousOutPoint.Index)
-		if !fus.contains(outPoint) {
+		outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+		if !fus.contains(outpoint) {
 			return false
 		}
 	}
@@ -442,17 +641,70 @@ func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 	return true
 }
 
-// clone returns a clone of this utxoSet
-func (fus *FullUTXOSet) clone() UTXOSet {
-	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone()}
+func (fus *FullUTXOSet) diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
+	return diffFromAcceptedTx(fus, tx, acceptingBlueScore)
 }
 
-// Get returns the UTXOEntry associated with the given OutPoint, and a boolean indicating if such entry was found
-func (fus *FullUTXOSet) Get(outPoint wire.OutPoint) (*UTXOEntry, bool) {
-	utxoEntry, ok := fus.utxoCollection[outPoint]
+// clone returns a clone of this utxoSet
+func (fus *FullUTXOSet) clone() UTXOSet {
+	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone(), UTXOMultiset: fus.UTXOMultiset.Clone()}
+}
+
+// Get returns the UTXOEntry associated with the given Outpoint, and a boolean indicating if such entry was found
+func (fus *FullUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
+	utxoEntry, ok := fus.utxoCollection[outpoint]
 	return utxoEntry, ok
 }
 
+// Multiset returns the ecmh-Multiset of this utxoSet
+func (fus *FullUTXOSet) Multiset() *btcec.Multiset {
+	return fus.UTXOMultiset
+}
+
+// addAndUpdateMultiset adds a UTXOEntry to this utxoSet and updates its multiset accordingly
+func (fus *FullUTXOSet) addAndUpdateMultiset(outpoint wire.Outpoint, entry *UTXOEntry) error {
+	fus.add(outpoint, entry)
+	newMs, err := addUTXOToMultiset(fus.UTXOMultiset, entry, &outpoint)
+	if err != nil {
+		return err
+	}
+	fus.UTXOMultiset = newMs
+	return nil
+}
+
+// removeAndUpdateMultiset removes a UTXOEntry from this utxoSet and updates its multiset accordingly
+func (fus *FullUTXOSet) removeAndUpdateMultiset(outpoint wire.Outpoint) error {
+	entry, ok := fus.Get(outpoint)
+	if !ok {
+		return errors.Errorf("Couldn't find outpoint %s", outpoint)
+	}
+	fus.remove(outpoint)
+	var err error
+	newMs, err := removeUTXOFromMultiset(fus.UTXOMultiset, entry, &outpoint)
+	if err != nil {
+		return err
+	}
+	fus.UTXOMultiset = newMs
+	return nil
+}
+
+// WithTransactions returns a new UTXO Set with the added transactions.
+//
+// This function MUST be called with the DAG lock held.
+func (fus *FullUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error) {
+	diffSet := NewDiffUTXOSet(fus, NewUTXODiff())
+	for _, tx := range transactions {
+		isAccepted, err := diffSet.AddTx(tx, blockBlueScore)
+		if err != nil {
+			return nil, err
+		}
+		if !ignoreDoubleSpends && !isAccepted {
+			return nil, errors.Errorf("Transaction %s is not valid with the current UTXO set", tx.TxID())
+		}
+	}
+	return UTXOSet(diffSet), nil
+}
+
 // DiffUTXOSet represents a utxoSet with a base fullUTXOSet and a UTXODiff
 type DiffUTXOSet struct {
 	base     *FullUTXOSet
@@ -492,52 +744,58 @@ func (dus *DiffUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
 	return NewDiffUTXOSet(dus.base, diff), nil
 }
 
-// AddTx adds a transaction to this utxoSet and returns true iff it's valid in this UTXO's context
-func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockHeight int32) bool {
-	isBlockReward := tx.IsBlockReward()
-	if !isBlockReward && !dus.containsInputs(tx) {
-		return false
+// AddTx adds a transaction to this utxoSet and returns true iff it's valid in this UTXO's context.
+//
+// If dus.UTXODiff.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockBlueScore uint64) (bool, error) {
+	isCoinbase := tx.IsCoinBase()
+	if !isCoinbase && !dus.containsInputs(tx) {
+		return false, nil
 	}
 
-	dus.appendTx(tx, blockHeight, isBlockReward)
+	err := dus.appendTx(tx, blockBlueScore, isCoinbase)
+	if err != nil {
+		return false, err
+	}
 
-	return true
+	return true, nil
 }
 
-func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockHeight int32, isBlockReward bool) {
-	if !isBlockReward {
-
+func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockBlueScore uint64, isCoinbase bool) error {
+	if !isCoinbase {
 		for _, txIn := range tx.TxIn {
-			outPoint := *wire.NewOutPoint(&txIn.PreviousOutPoint.TxID, txIn.PreviousOutPoint.Index)
-			if dus.UTXODiff.toAdd.contains(outPoint) {
-				dus.UTXODiff.toAdd.remove(outPoint)
-			} else {
-				prevUTXOEntry := dus.base.utxoCollection[outPoint]
-				dus.UTXODiff.toRemove.add(outPoint, prevUTXOEntry)
+			outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+			entry, ok := dus.Get(outpoint)
+			if !ok {
+				return errors.Errorf("Couldn't find entry for outpoint %s", outpoint)
+			}
+			err := dus.UTXODiff.RemoveEntry(outpoint, entry)
+			if err != nil {
+				return err
 			}
 		}
 	}
 
 	for i, txOut := range tx.TxOut {
-		hash := tx.TxID()
-		outPoint := *wire.NewOutPoint(&hash, uint32(i))
-		entry := NewUTXOEntry(txOut, isBlockReward, blockHeight)
+		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
+		entry := NewUTXOEntry(txOut, isCoinbase, blockBlueScore)
 
-		if dus.UTXODiff.toRemove.contains(outPoint) {
-			dus.UTXODiff.toRemove.remove(outPoint)
-		} else {
-			dus.UTXODiff.toAdd.add(outPoint, entry)
+		err := dus.UTXODiff.AddEntry(outpoint, entry)
+		if err != nil {
+			return err
 		}
 	}
+	return nil
 }
 
 func (dus *DiffUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 	for _, txIn := range tx.TxIn {
-		outPoint := *wire.NewOutPoint(&txIn.PreviousOutPoint.TxID, txIn.PreviousOutPoint.Index)
-		isInBase := dus.base.contains(outPoint)
-		isInDiffToAdd := dus.UTXODiff.toAdd.contains(outPoint)
-		isInDiffToRemove := dus.UTXODiff.toRemove.contains(outPoint)
-		if (!isInBase && !isInDiffToAdd) || isInDiffToRemove {
+		outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
+		isInBase := dus.base.contains(outpoint)
+		isInDiffToAdd := dus.UTXODiff.toAdd.contains(outpoint)
+		isInDiffToRemove := dus.UTXODiff.toRemove.contains(outpoint)
+		if (!isInBase && !isInDiffToAdd) || (isInDiffToRemove && !(isInBase && isInDiffToAdd)) {
 			return false
 		}
 	}
@@ -546,26 +804,43 @@ func (dus *DiffUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 }
 
 // meldToBase updates the base fullUTXOSet with all changes in diff
-func (dus *DiffUTXOSet) meldToBase() {
-	for outPoint := range dus.UTXODiff.toRemove {
-		dus.base.remove(outPoint)
+func (dus *DiffUTXOSet) meldToBase() error {
+	for outpoint := range dus.UTXODiff.toRemove {
+		if _, ok := dus.base.Get(outpoint); ok {
+			dus.base.remove(outpoint)
+		} else {
+			return errors.Errorf("Couldn't remove outpoint %s because it doesn't exist in the DiffUTXOSet base", outpoint)
+		}
 	}
 
-	for outPoint, utxoEntry := range dus.UTXODiff.toAdd {
-		dus.base.add(outPoint, utxoEntry)
+	for outpoint, utxoEntry := range dus.UTXODiff.toAdd {
+		dus.base.add(outpoint, utxoEntry)
 	}
 
-	dus.UTXODiff = NewUTXODiff()
+	if dus.UTXODiff.useMultiset {
+		dus.base.UTXOMultiset = dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset)
+	}
+
+	if dus.UTXODiff.useMultiset {
+		dus.UTXODiff = NewUTXODiff()
+	} else {
+		dus.UTXODiff = NewUTXODiffWithoutMultiset()
+	}
+	return nil
 }
 
 // diffFromTx returns a diff that is equivalent to provided transaction,
 // or an error if provided transaction is not valid in the context of this UTXOSet
-func (dus *DiffUTXOSet) diffFromTx(tx *wire.MsgTx, node *blockNode) (*UTXODiff, error) {
-	return diffFromTx(dus, tx, node)
+func (dus *DiffUTXOSet) diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
+	return diffFromTx(dus, tx, acceptingBlueScore)
+}
+
+func (dus *DiffUTXOSet) diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
+	return diffFromAcceptedTx(dus, tx, acceptingBlueScore)
 }
 
 func (dus *DiffUTXOSet) String() string {
-	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove)
+	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s, Multiset-Hash:%s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove, dus.Multiset().Hash())
 }
 
 // clone returns a clone of this UTXO Set
@@ -573,15 +848,59 @@ func (dus *DiffUTXOSet) clone() UTXOSet {
 	return NewDiffUTXOSet(dus.base.clone().(*FullUTXOSet), dus.UTXODiff.clone())
 }
 
-// Get returns the UTXOEntry associated with provided outPoint in this UTXOSet.
+// Get returns the UTXOEntry associated with provided outpoint in this UTXOSet.
 // Returns false in second output if this UTXOEntry was not found
-func (dus *DiffUTXOSet) Get(outPoint wire.OutPoint) (*UTXOEntry, bool) {
-	if dus.UTXODiff.toRemove.contains(outPoint) {
+func (dus *DiffUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
+	if toRemoveEntry, ok := dus.UTXODiff.toRemove.get(outpoint); ok {
+		// An exception is made for entries with unequal blue scores
+		// These are just "updates" to accepted blue score
+		if toAddEntry, ok := dus.UTXODiff.toAdd.get(outpoint); ok && toAddEntry.blockBlueScore != toRemoveEntry.blockBlueScore {
+			return toAddEntry, true
+		}
 		return nil, false
 	}
-	if txOut, ok := dus.base.get(outPoint); ok {
+	if txOut, ok := dus.base.get(outpoint); ok {
 		return txOut, true
 	}
-	txOut, ok := dus.UTXODiff.toAdd.get(outPoint)
+	txOut, ok := dus.UTXODiff.toAdd.get(outpoint)
 	return txOut, ok
 }
+
+// Multiset returns the ecmh-Multiset of this utxoSet
+func (dus *DiffUTXOSet) Multiset() *btcec.Multiset {
+	return dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset)
+}
+
+// WithTransactions returns a new UTXO Set with the added transactions.
+//
+// If dus.UTXODiff.useMultiset is true, this function MUST be
+// called with the DAG lock held.
+func (dus *DiffUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error) {
+	diffSet := NewDiffUTXOSet(dus.base, dus.UTXODiff.clone())
+	for _, tx := range transactions {
+		isAccepted, err := diffSet.AddTx(tx, blockBlueScore)
+		if err != nil {
+			return nil, err
+		}
+		if !ignoreDoubleSpends && !isAccepted {
+			return nil, errors.Errorf("Transaction %s is not valid with the current UTXO set", tx.TxID())
+		}
+	}
+	return UTXOSet(diffSet), nil
+}
+
+func addUTXOToMultiset(ms *btcec.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*btcec.Multiset, error) {
+	utxoMS, err := utxoMultiset(entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	return ms.Union(utxoMS), nil
+}
+
+func removeUTXOFromMultiset(ms *btcec.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*btcec.Multiset, error) {
+	utxoMS, err := utxoMultiset(entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	return ms.Subtract(utxoMS), nil
+}

blockdag/validate_test.go

@@ -5,58 +5,60 @@
 package blockdag
 
 import (
+	"bou.ke/monkey"
+	"github.com/pkg/errors"
 	"math"
-	"reflect"
+	"path/filepath"
 	"testing"
 	"time"
 
-	"github.com/daglabs/btcd/dagconfig"
-	"github.com/daglabs/btcd/dagconfig/daghash"
-	"github.com/daglabs/btcd/util"
-	"github.com/daglabs/btcd/util/subnetworkid"
-	"github.com/daglabs/btcd/wire"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 // TestSequenceLocksActive tests the SequenceLockActive function to ensure it
 // works as expected in all possible combinations/scenarios.
 func TestSequenceLocksActive(t *testing.T) {
-	seqLock := func(h int32, s int64) *SequenceLock {
+	seqLock := func(h int64, s int64) *SequenceLock {
 		return &SequenceLock{
-			Seconds:     s,
-			BlockHeight: h,
+			Seconds:        s,
+			BlockBlueScore: h,
 		}
 	}
 
 	tests := []struct {
-		seqLock     *SequenceLock
-		blockHeight int32
-		mtp         time.Time
+		seqLock          *SequenceLock
+		blockChainHeight uint64
+		mtp              time.Time
 
 		want bool
 	}{
 		// Block based sequence lock with equal block height.
-		{seqLock: seqLock(1000, -1), blockHeight: 1001, mtp: time.Unix(9, 0), want: true},
+		{seqLock: seqLock(1000, -1), blockChainHeight: 1001, mtp: time.Unix(9, 0), want: true},
 
 		// Time based sequence lock with mtp past the absolute time.
-		{seqLock: seqLock(-1, 30), blockHeight: 2, mtp: time.Unix(31, 0), want: true},
+		{seqLock: seqLock(-1, 30), blockChainHeight: 2, mtp: time.Unix(31, 0), want: true},
 
 		// Block based sequence lock with current height below seq lock block height.
-		{seqLock: seqLock(1000, -1), blockHeight: 90, mtp: time.Unix(9, 0), want: false},
+		{seqLock: seqLock(1000, -1), blockChainHeight: 90, mtp: time.Unix(9, 0), want: false},
 
 		// Time based sequence lock with current time before lock time.
-		{seqLock: seqLock(-1, 30), blockHeight: 2, mtp: time.Unix(29, 0), want: false},
+		{seqLock: seqLock(-1, 30), blockChainHeight: 2, mtp: time.Unix(29, 0), want: false},
 
 		// Block based sequence lock at the same height, so shouldn't yet be active.
-		{seqLock: seqLock(1000, -1), blockHeight: 1000, mtp: time.Unix(9, 0), want: false},
+		{seqLock: seqLock(1000, -1), blockChainHeight: 1000, mtp: time.Unix(9, 0), want: false},
 
 		// Time based sequence lock with current time equal to lock time, so shouldn't yet be active.
-		{seqLock: seqLock(-1, 30), blockHeight: 2, mtp: time.Unix(30, 0), want: false},
+		{seqLock: seqLock(-1, 30), blockChainHeight: 2, mtp: time.Unix(30, 0), want: false},
 	}
 
 	t.Logf("Running %d sequence locks tests", len(tests))
 	for i, test := range tests {
 		got := SequenceLockActive(test.seqLock,
-			test.blockHeight, test.mtp)
+			test.blockChainHeight, test.mtp)
 		if got != test.want {
 			t.Fatalf("SequenceLockActive #%d got %v want %v", i,
 				got, test.want)
@@ -77,9 +79,9 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	// Since we're not dealing with the real block DAG, set the block reward
-	// maturity to 1.
-	dag.TestSetBlockRewardMaturity(1)
+	// Since we're not dealing with the real block DAG, set the coinbase
+	// maturity to 0.
+	dag.TestSetCoinbaseMaturity(0)
 
 	// Load up blocks such that there is a side chain.
 	// (genesis block) -> 1 -> 2 -> 3 -> 4
@@ -91,7 +93,7 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 
 	var blocks []*util.Block
 	for _, file := range testFiles {
-		blockTmp, err := loadBlocks(file)
+		blockTmp, err := LoadBlocks(filepath.Join("testdata/", file))
 		if err != nil {
 			t.Fatalf("Error loading file: %v\n", err)
 		}
@@ -99,22 +101,25 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 	}
 
 	for i := 1; i <= 3; i++ {
-		_, err := dag.ProcessBlock(blocks[i], BFNone)
+		_, delay, err := dag.ProcessBlock(blocks[i], BFNone)
 		if err != nil {
 			t.Fatalf("CheckConnectBlockTemplate: Received unexpected error "+
 				"processing block %d: %v", i, err)
 		}
+		if delay != 0 {
+			t.Fatalf("CheckConnectBlockTemplate: block %d is too far in the future", i)
+		}
 	}
 
 	// Block 3 should fail to connect since it's already inserted.
-	err = dag.CheckConnectBlockTemplate(blocks[3])
+	err = dag.CheckConnectBlockTemplateNoLock(blocks[3])
 	if err == nil {
 		t.Fatal("CheckConnectBlockTemplate: Did not received expected error " +
 			"on block 3")
 	}
 
 	// Block 4 should connect successfully to tip of chain.
-	err = dag.CheckConnectBlockTemplate(blocks[4])
+	err = dag.CheckConnectBlockTemplateNoLock(blocks[4])
 	if err != nil {
 		t.Fatalf("CheckConnectBlockTemplate: Received unexpected error on "+
 			"block 4: %v", err)
@@ -126,8 +131,7 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 	}
 
 	// Block 3a should connect even though it does not build on dag tips.
-	blocks[5].SetHeight(3) // set height manually because it was set to 0 in loadBlocks
-	err = dag.CheckConnectBlockTemplate(blocks[5])
+	err = dag.CheckConnectBlockTemplateNoLock(blocks[5])
 	if err != nil {
 		t.Fatal("CheckConnectBlockTemplate: Recieved unexpected error on " +
 			"block 3a that connects below the tips")
@@ -137,8 +141,7 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 	invalidPowMsgBlock := *blocks[4].MsgBlock()
 	invalidPowMsgBlock.Header.Nonce++
 	invalidPowBlock := util.NewBlock(&invalidPowMsgBlock)
-	invalidPowBlock.SetHeight(blocks[4].Height())
-	err = dag.CheckConnectBlockTemplate(invalidPowBlock)
+	err = dag.CheckConnectBlockTemplateNoLock(invalidPowBlock)
 	if err != nil {
 		t.Fatalf("CheckConnectBlockTemplate: Received unexpected error on "+
 			"block 4 with bad nonce: %v", err)
@@ -147,7 +150,7 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 	// Invalid block building on chain tip should fail to connect.
 	invalidBlock := *blocks[4].MsgBlock()
 	invalidBlock.Header.Bits--
-	err = dag.CheckConnectBlockTemplate(util.NewBlock(&invalidBlock))
+	err = dag.CheckConnectBlockTemplateNoLock(util.NewBlock(&invalidBlock))
 	if err == nil {
 		t.Fatal("CheckConnectBlockTemplate: Did not received expected error " +
 			"on block 4 with invalid difficulty bits")
@@ -167,19 +170,20 @@ func TestCheckBlockSanity(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	powLimit := dagconfig.MainNetParams.PowLimit
 	block := util.NewBlock(&Block100000)
-	timeSource := NewMedianTime()
 	if len(block.Transactions()) < 3 {
 		t.Fatalf("Too few transactions in block, expect at least 3, got %v", len(block.Transactions()))
 	}
-	err = dag.CheckBlockSanity(block, powLimit, timeSource)
+	delay, err := dag.checkBlockSanity(block, BFNone)
 	if err != nil {
 		t.Errorf("CheckBlockSanity: %v", err)
 	}
+	if delay != 0 {
+		t.Errorf("CheckBlockSanity: unexpected return %s delay", delay)
+	}
 	// Test with block with wrong transactions sorting order
 	blockWithWrongTxOrder := util.NewBlock(&BlockWithWrongTxOrder)
-	err = dag.CheckBlockSanity(blockWithWrongTxOrder, powLimit, timeSource)
+	delay, err = dag.checkBlockSanity(blockWithWrongTxOrder, BFNone)
 	if err == nil {
 		t.Errorf("CheckBlockSanity: transactions disorder is not detected")
 	}
@@ -187,57 +191,69 @@ func TestCheckBlockSanity(t *testing.T) {
 	if !ok {
 		t.Errorf("CheckBlockSanity: wrong error returned, expect RuleError, got %T", err)
 	} else if ruleErr.ErrorCode != ErrTransactionsNotSorted {
-		t.Errorf("CheckBlockSanity: wrong error returned, expect ErrTransactionsNotSorted, got %v", ruleErr.ErrorCode)
+		t.Errorf("CheckBlockSanity: wrong error returned, expect ErrTransactionsNotSorted, got %v, err %s", ruleErr.ErrorCode, err)
+	}
+	if delay != 0 {
+		t.Errorf("CheckBlockSanity: unexpected return %s delay", delay)
 	}
 
 	// Ensure a block that has a timestamp with a precision higher than one
 	// second fails.
 	timestamp := block.MsgBlock().Header.Timestamp
 	block.MsgBlock().Header.Timestamp = timestamp.Add(time.Nanosecond)
-	err = dag.CheckBlockSanity(block, powLimit, timeSource)
+	delay, err = dag.checkBlockSanity(block, BFNone)
 	if err == nil {
 		t.Errorf("CheckBlockSanity: error is nil when it shouldn't be")
 	}
+	if delay != 0 {
+		t.Errorf("CheckBlockSanity: unexpected return %s delay", delay)
+	}
 
 	var invalidParentsOrderBlock = wire.MsgBlock{
 		Header: wire.BlockHeader{
 			Version: 0x10000000,
 			ParentHashes: []*daghash.Hash{
-				{ // Make go vet happy.
+				{
 					0x4b, 0xb0, 0x75, 0x35, 0xdf, 0xd5, 0x8e, 0x0b,
 					0x3c, 0xd6, 0x4f, 0xd7, 0x15, 0x52, 0x80, 0x87,
 					0x2a, 0x04, 0x71, 0xbc, 0xf8, 0x30, 0x95, 0x52,
 					0x6a, 0xce, 0x0e, 0x38, 0xc6, 0x00, 0x00, 0x00,
 				},
-				{ // Make go vet happy.
+				{
 					0x16, 0x5e, 0x38, 0xe8, 0xb3, 0x91, 0x45, 0x95,
 					0xd9, 0xc6, 0x41, 0xf3, 0xb8, 0xee, 0xc2, 0xf3,
 					0x46, 0x11, 0x89, 0x6b, 0x82, 0x1a, 0x68, 0x3b,
 					0x7a, 0x4e, 0xde, 0xfe, 0x2c, 0x00, 0x00, 0x00,
 				},
 			},
-			HashMerkleRoot: &daghash.Hash{ // Make go vet happy.
+			HashMerkleRoot: &daghash.Hash{
 				0x2f, 0x4c, 0xc3, 0x0b, 0x0a, 0x84, 0xbb, 0x95,
 				0x56, 0x9d, 0x77, 0xa2, 0xee, 0x3e, 0xb1, 0xac,
 				0x48, 0x3e, 0x8b, 0xe1, 0xcf, 0xdc, 0x20, 0xba,
 				0xae, 0xec, 0x0a, 0x2f, 0xe4, 0x85, 0x31, 0x30,
 			},
-			IDMerkleRoot: &daghash.Hash{ // Make go vet happy.
-				0x4e, 0x06, 0xba, 0x64, 0xd7, 0x61, 0xda, 0x25,
-				0x1a, 0x0e, 0x21, 0xd4, 0x64, 0x49, 0x02, 0xa2,
+			AcceptedIDMerkleRoot: &daghash.Hash{
 				0x80, 0xf7, 0x00, 0xe3, 0x16, 0x3d, 0x04, 0x95,
 				0x5b, 0x7e, 0xaf, 0x84, 0x7e, 0x1b, 0x6b, 0x06,
+				0x4e, 0x06, 0xba, 0x64, 0xd7, 0x61, 0xda, 0x25,
+				0x1a, 0x0e, 0x21, 0xd4, 0x64, 0x49, 0x02, 0xa2,
 			},
-			Timestamp: time.Unix(0x5c40613a, 0),
+			UTXOCommitment: &daghash.Hash{
+				0x80, 0xf7, 0x00, 0xe3, 0x16, 0x3d, 0x04, 0x95,
+				0x5b, 0x7e, 0xaf, 0x84, 0x7e, 0x1b, 0x6b, 0x06,
+				0x4e, 0x06, 0xba, 0x64, 0xd7, 0x61, 0xda, 0x25,
+				0x1a, 0x0e, 0x21, 0xd4, 0x64, 0x49, 0x02, 0xa2,
+			},
+			Timestamp: time.Unix(0x5cd18053, 0),
 			Bits:      0x207fffff,
-			Nonce:     0x4000000000000001,
+			Nonce:     0x1,
 		},
 		Transactions: []*wire.MsgTx{
 			{
 				Version: 1,
 				TxIn: []*wire.TxIn{
 					{
-						PreviousOutPoint: wire.OutPoint{
+						PreviousOutpoint: wire.Outpoint{
 							TxID:  daghash.TxID{},
 							Index: 0xffffffff,
 						},
@@ -252,7 +268,7 @@ func TestCheckBlockSanity(t *testing.T) {
 				TxOut: []*wire.TxOut{
 					{
 						Value: 0x12a05f200, // 5000000000
-						PkScript: []byte{
+						ScriptPubKey: []byte{
 							0x51,
 						},
 					},
@@ -264,8 +280,8 @@ func TestCheckBlockSanity(t *testing.T) {
 				Version: 1,
 				TxIn: []*wire.TxIn{
 					{
-						PreviousOutPoint: wire.OutPoint{
-							TxID: daghash.TxID([32]byte{ // Make go vet happy.
+						PreviousOutpoint: wire.Outpoint{
+							TxID: daghash.TxID([32]byte{
 								0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
 								0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
 								0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
@@ -302,7 +318,7 @@ func TestCheckBlockSanity(t *testing.T) {
 				TxOut: []*wire.TxOut{
 					{
 						Value: 0x2123e300, // 556000000
-						PkScript: []byte{
+						ScriptPubKey: []byte{
 							0x76, // OP_DUP
 							0xa9, // OP_HASH160
 							0x14, // OP_DATA_20
@@ -315,7 +331,7 @@ func TestCheckBlockSanity(t *testing.T) {
 					},
 					{
 						Value: 0x108e20f00, // 4444000000
-						PkScript: []byte{
+						ScriptPubKey: []byte{
 							0x76, // OP_DUP
 							0xa9, // OP_HASH160
 							0x14, // OP_DATA_20
@@ -334,8 +350,8 @@ func TestCheckBlockSanity(t *testing.T) {
 				Version: 1,
 				TxIn: []*wire.TxIn{
 					{
-						PreviousOutPoint: wire.OutPoint{
-							TxID: daghash.TxID([32]byte{ // Make go vet happy.
+						PreviousOutpoint: wire.Outpoint{
+							TxID: daghash.TxID([32]byte{
 								0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
 								0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
 								0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
@@ -371,7 +387,7 @@ func TestCheckBlockSanity(t *testing.T) {
 				TxOut: []*wire.TxOut{
 					{
 						Value: 0xf4240, // 1000000
-						PkScript: []byte{
+						ScriptPubKey: []byte{
 							0x76, // OP_DUP
 							0xa9, // OP_HASH160
 							0x14, // OP_DATA_20
@@ -384,7 +400,7 @@ func TestCheckBlockSanity(t *testing.T) {
 					},
 					{
 						Value: 0x11d260c0, // 299000000
-						PkScript: []byte{
+						ScriptPubKey: []byte{
 							0x76, // OP_DUP
 							0xa9, // OP_HASH160
 							0x14, // OP_DATA_20
@@ -403,8 +419,8 @@ func TestCheckBlockSanity(t *testing.T) {
 				Version: 1,
 				TxIn: []*wire.TxIn{
 					{
-						PreviousOutPoint: wire.OutPoint{
-							TxID: daghash.TxID([32]byte{ // Make go vet happy.
+						PreviousOutpoint: wire.Outpoint{
+							TxID: daghash.TxID([32]byte{
 								0x0b, 0x60, 0x72, 0xb3, 0x86, 0xd4, 0xa7, 0x73,
 								0x23, 0x52, 0x37, 0xf6, 0x4c, 0x11, 0x26, 0xac,
 								0x3b, 0x24, 0x0c, 0x84, 0xb9, 0x17, 0xa3, 0x90,
@@ -441,7 +457,7 @@ func TestCheckBlockSanity(t *testing.T) {
 				TxOut: []*wire.TxOut{
 					{
 						Value: 0xf4240, // 1000000
-						PkScript: []byte{
+						ScriptPubKey: []byte{
 							0x76, // OP_DUP
 							0xa9, // OP_HASH160
 							0x14, // OP_DATA_20
@@ -460,7 +476,7 @@ func TestCheckBlockSanity(t *testing.T) {
 	}
 
 	btcutilInvalidBlock := util.NewBlock(&invalidParentsOrderBlock)
-	err = dag.CheckBlockSanity(btcutilInvalidBlock, powLimit, timeSource)
+	delay, err = dag.checkBlockSanity(btcutilInvalidBlock, BFNone)
 	if err == nil {
 		t.Errorf("CheckBlockSanity: error is nil when it shouldn't be")
 	}
@@ -468,73 +484,20 @@ func TestCheckBlockSanity(t *testing.T) {
 	if rError.ErrorCode != ErrWrongParentsOrder {
 		t.Errorf("CheckBlockSanity: Expected error was ErrWrongParentsOrder but got %v", err)
 	}
-}
-
-// TestCheckSerializedHeight tests the checkSerializedHeight function with
-// various serialized heights and also does negative tests to ensure errors
-// and handled properly.
-func TestCheckSerializedHeight(t *testing.T) {
-	// Create an empty coinbase template to be used in the tests below.
-	coinbaseOutpoint := wire.NewOutPoint(&daghash.TxID{}, math.MaxUint32)
-	coinbaseTx := wire.NewNativeMsgTx(1, []*wire.TxIn{wire.NewTxIn(coinbaseOutpoint, nil)}, nil)
-
-	// Expected rule errors.
-	missingHeightError := RuleError{
-		ErrorCode: ErrMissingCoinbaseHeight,
-	}
-	badHeightError := RuleError{
-		ErrorCode: ErrBadCoinbaseHeight,
+	if delay != 0 {
+		t.Errorf("CheckBlockSanity: unexpected return %s delay", delay)
 	}
 
-	tests := []struct {
-		sigScript  []byte // Serialized data
-		wantHeight int32  // Expected height
-		err        error  // Expected error type
-	}{
-		// No serialized height length.
-		{[]byte{}, 0, missingHeightError},
-		// Serialized height length with no height bytes.
-		{[]byte{0x02}, 0, missingHeightError},
-		// Serialized height length with too few height bytes.
-		{[]byte{0x02, 0x4a}, 0, missingHeightError},
-		// Serialized height that needs 2 bytes to encode.
-		{[]byte{0x02, 0x4a, 0x52}, 21066, nil},
-		// Serialized height that needs 2 bytes to encode, but backwards
-		// endianness.
-		{[]byte{0x02, 0x4a, 0x52}, 19026, badHeightError},
-		// Serialized height that needs 3 bytes to encode.
-		{[]byte{0x03, 0x40, 0x0d, 0x03}, 200000, nil},
-		// Serialized height that needs 3 bytes to encode, but backwards
-		// endianness.
-		{[]byte{0x03, 0x40, 0x0d, 0x03}, 1074594560, badHeightError},
+	blockInTheFuture := Block100000
+	expectedDelay := 10 * time.Second
+	now := time.Unix(time.Now().Unix(), 0)
+	blockInTheFuture.Header.Timestamp = now.Add(time.Duration(dag.TimestampDeviationTolerance)*time.Second + expectedDelay)
+	delay, err = dag.checkBlockSanity(util.NewBlock(&blockInTheFuture), BFNoPoWCheck)
+	if err != nil {
+		t.Errorf("CheckBlockSanity: %v", err)
 	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		msgTx := coinbaseTx.Copy()
-		msgTx.TxIn[0].SignatureScript = test.sigScript
-
-		msgBlock := wire.NewMsgBlock(wire.NewBlockHeader(1, []*daghash.Hash{}, &daghash.Hash{}, &daghash.Hash{}, 0, 0))
-		msgBlock.AddTransaction(msgTx)
-		block := util.NewBlock(msgBlock)
-		block.SetHeight(test.wantHeight)
-
-		err := checkSerializedHeight(block)
-		if reflect.TypeOf(err) != reflect.TypeOf(test.err) {
-			t.Errorf("checkSerializedHeight #%d wrong error type "+
-				"got: %v <%T>, want: %T", i, err, err, test.err)
-			continue
-		}
-
-		if rerr, ok := err.(RuleError); ok {
-			trerr := test.err.(RuleError)
-			if rerr.ErrorCode != trerr.ErrorCode {
-				t.Errorf("checkSerializedHeight #%d wrong "+
-					"error code got: %v, want: %v", i,
-					rerr.ErrorCode, trerr.ErrorCode)
-				continue
-			}
-		}
+	if delay != expectedDelay {
+		t.Errorf("CheckBlockSanity: expected %s delay but got %s", expectedDelay, delay)
 	}
 }
 
@@ -555,49 +518,60 @@ func TestPastMedianTime(t *testing.T) {
 	}
 
 	// Checks that a block is valid if it has timestamp equals to past median time
-	height := tip.height + 1
+	chainHeight := tip.chainHeight + 1
 	node := newTestNode(setFromSlice(tip),
 		blockVersion,
-		0,
-		tip.PastMedianTime(),
+		dag.powMaxBits,
+		tip.PastMedianTime(dag),
 		dagconfig.MainNetParams.K)
 
 	header := node.Header()
-	err := dag.checkBlockHeaderContext(header, node.parents.bluest(), height, false)
+	err := dag.checkBlockHeaderContext(header, node.parents.bluest(), chainHeight, false)
 	if err != nil {
 		t.Errorf("TestPastMedianTime: unexpected error from checkBlockHeaderContext: %v"+
 			"(a block with timestamp equals to past median time should be valid)", err)
 	}
 
 	// Checks that a block is valid if its timestamp is after past median time
-	height = tip.height + 1
+	chainHeight = tip.chainHeight + 1
 	node = newTestNode(setFromSlice(tip),
 		blockVersion,
-		0,
-		tip.PastMedianTime().Add(time.Second),
+		dag.powMaxBits,
+		tip.PastMedianTime(dag).Add(time.Second),
 		dagconfig.MainNetParams.K)
 
 	header = node.Header()
-	err = dag.checkBlockHeaderContext(header, node.parents.bluest(), height, false)
+	err = dag.checkBlockHeaderContext(header, node.parents.bluest(), chainHeight, false)
 	if err != nil {
 		t.Errorf("TestPastMedianTime: unexpected error from checkBlockHeaderContext: %v"+
 			"(a block with timestamp bigger than past median time should be valid)", err)
 	}
 
 	// Checks that a block is invalid if its timestamp is before past median time
-	height = tip.height + 1
+	chainHeight = tip.chainHeight + 1
 	node = newTestNode(setFromSlice(tip),
 		blockVersion,
 		0,
-		tip.PastMedianTime().Add(-time.Second),
+		tip.PastMedianTime(dag).Add(-time.Second),
 		dagconfig.MainNetParams.K)
 
 	header = node.Header()
-	err = dag.checkBlockHeaderContext(header, node.parents.bluest(), height, false)
+	err = dag.checkBlockHeaderContext(header, node.parents.bluest(), chainHeight, false)
 	if err == nil {
 		t.Errorf("TestPastMedianTime: unexpected success: block should be invalid if its timestamp is before past median time")
 	}
 
+	guard := monkey.Patch(blockWindow.medianTimestamp, func(_ blockWindow) (int64, error) {
+		return 0, errors.New("medianTimestamp error")
+	})
+	defer guard.Unpatch()
+	defer func() {
+		if recover() == nil {
+			t.Errorf("Got no panic on PastMedianTime, while expected panic")
+		}
+	}()
+	node.PastMedianTime(dag)
+
 }
 
 func TestValidateParents(t *testing.T) {
@@ -621,7 +595,11 @@ func TestValidateParents(t *testing.T) {
 	b := generateNode(a)
 	c := generateNode(genesisNode)
 
-	fakeBlockHeader := &wire.BlockHeader{IDMerkleRoot: &daghash.ZeroHash, HashMerkleRoot: &daghash.ZeroHash}
+	fakeBlockHeader := &wire.BlockHeader{
+		HashMerkleRoot:       &daghash.ZeroHash,
+		AcceptedIDMerkleRoot: &daghash.ZeroHash,
+		UTXOCommitment:       &daghash.ZeroHash,
+	}
 
 	// Check direct parents relation
 	err := validateParents(fakeBlockHeader, setFromSlice(a, b))
@@ -656,7 +634,7 @@ func TestCheckTransactionSanity(t *testing.T) {
 		{"good one", 1, 1, 1, *subnetworkid.SubnetworkIDNative, nil, nil, nil},
 		{"no inputs", 0, 1, 1, *subnetworkid.SubnetworkIDNative, nil, nil, ruleError(ErrNoTxInputs, "")},
 		{"no outputs", 1, 0, 1, *subnetworkid.SubnetworkIDNative, nil, nil, nil},
-		{"too big", 100000, 1, 1, *subnetworkid.SubnetworkIDNative, nil, nil, ruleError(ErrTxTooBig, "")},
+		{"too massive", 1, 1000000, 1, *subnetworkid.SubnetworkIDNative, nil, nil, ruleError(ErrTxMassTooHigh, "")},
 		{"too much satoshi in one output", 1, 1, util.MaxSatoshi + 1,
 			*subnetworkid.SubnetworkIDNative,
 			nil,
@@ -670,8 +648,32 @@ func TestCheckTransactionSanity(t *testing.T) {
 		{"duplicate inputs", 2, 1, 1,
 			*subnetworkid.SubnetworkIDNative,
 			nil,
-			func(tx *wire.MsgTx) { tx.TxIn[1].PreviousOutPoint.Index = 0 },
+			func(tx *wire.MsgTx) { tx.TxIn[1].PreviousOutpoint.Index = 0 },
 			ruleError(ErrDuplicateTxInputs, "")},
+		{"1 input coinbase",
+			1,
+			1,
+			1,
+			*subnetworkid.SubnetworkIDNative,
+			&txSubnetworkData{subnetworkid.SubnetworkIDCoinbase, 0, nil},
+			nil,
+			nil},
+		{"no inputs coinbase",
+			0,
+			1,
+			1,
+			*subnetworkid.SubnetworkIDNative,
+			&txSubnetworkData{subnetworkid.SubnetworkIDCoinbase, 0, nil},
+			nil,
+			nil},
+		{"too long payload coinbase",
+			1,
+			1,
+			1,
+			*subnetworkid.SubnetworkIDNative,
+			&txSubnetworkData{subnetworkid.SubnetworkIDCoinbase, 0, make([]byte, MaxCoinbasePayloadLen+1)},
+			nil,
+			ruleError(ErrBadCoinbasePayloadLen, "")},
 		{"non-zero gas in DAGCoin", 1, 1, 0,
 			*subnetworkid.SubnetworkIDNative,
 			&txSubnetworkData{subnetworkid.SubnetworkIDNative, 1, []byte{}},
@@ -720,7 +722,7 @@ func TestCheckTransactionSanity(t *testing.T) {
 			test.extraModificationsFunc(tx)
 		}
 
-		err := CheckTransactionSanity(util.NewTx(tx), &test.nodeSubnetworkID, false)
+		err := CheckTransactionSanity(util.NewTx(tx), &test.nodeSubnetworkID)
 		if e := checkRuleError(err, test.expectedErr); e != nil {
 			t.Errorf("TestCheckTransactionSanity: '%s': %v", test.name, e)
 			continue
@@ -734,68 +736,79 @@ var Block100000 = wire.MsgBlock{
 	Header: wire.BlockHeader{
 		Version: 0x10000000,
 		ParentHashes: []*daghash.Hash{
-			{ // Make go vet happy.
+			{
 				0x16, 0x5e, 0x38, 0xe8, 0xb3, 0x91, 0x45, 0x95,
 				0xd9, 0xc6, 0x41, 0xf3, 0xb8, 0xee, 0xc2, 0xf3,
 				0x46, 0x11, 0x89, 0x6b, 0x82, 0x1a, 0x68, 0x3b,
 				0x7a, 0x4e, 0xde, 0xfe, 0x2c, 0x00, 0x00, 0x00,
 			},
-			{ // Make go vet happy.
+			{
 				0x4b, 0xb0, 0x75, 0x35, 0xdf, 0xd5, 0x8e, 0x0b,
 				0x3c, 0xd6, 0x4f, 0xd7, 0x15, 0x52, 0x80, 0x87,
 				0x2a, 0x04, 0x71, 0xbc, 0xf8, 0x30, 0x95, 0x52,
 				0x6a, 0xce, 0x0e, 0x38, 0xc6, 0x00, 0x00, 0x00,
 			},
 		},
-		HashMerkleRoot: &daghash.Hash{ // Make go vet happy.
-			0x30, 0xed, 0xf5, 0xbd, 0xd1, 0x4f, 0x8f, 0xb2,
-			0x0b, 0x6c, 0x92, 0xac, 0xd2, 0x47, 0xb7, 0xd6,
-			0x6f, 0x22, 0xfa, 0x60, 0x36, 0x80, 0x99, 0xc3,
-			0x6e, 0x39, 0x14, 0x9b, 0xcc, 0x1f, 0x31, 0xa9,
+		HashMerkleRoot: &daghash.Hash{
+			0x32, 0x30, 0x46, 0x39, 0x5e, 0x27, 0x6d, 0x5a,
+			0xc9, 0x64, 0x16, 0x29, 0x5b, 0xa4, 0x5a, 0xf3,
+			0xc0, 0xfc, 0x1a, 0xa9, 0xcb, 0x2a, 0xd2, 0x9f,
+			0xbe, 0x07, 0x0c, 0x47, 0xc9, 0x84, 0x39, 0x15,
 		},
-		IDMerkleRoot: &daghash.Hash{ // Make go vet happy.
-			0x81, 0xb8, 0xa0, 0x68, 0x77, 0xc4, 0x02, 0x1e,
-			0x3c, 0xb1, 0x16, 0x8f, 0x5f, 0x6b, 0x45, 0x87,
+		AcceptedIDMerkleRoot: &daghash.Hash{
 			0x8a, 0xb7, 0xd6, 0x73, 0x1b, 0xe6, 0xc5, 0xd3,
 			0x5d, 0x4e, 0x2c, 0xc9, 0x57, 0x88, 0x30, 0x65,
+			0x81, 0xb8, 0xa0, 0x68, 0x77, 0xc4, 0x02, 0x1e,
+			0x3c, 0xb1, 0x16, 0x8f, 0x5f, 0x6b, 0x45, 0x87,
 		},
-		Timestamp: time.Unix(0x5c404bc3, 0),
-		Bits:      0x207fffff,
-		Nonce:     0xdffffffffffffff9,
+		UTXOCommitment: &daghash.ZeroHash,
+		Timestamp:      time.Unix(0x5cdac4b1, 0),
+		Bits:           0x207fffff,
+		Nonce:          0x00000001,
 	},
 	Transactions: []*wire.MsgTx{
 		{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID:  daghash.TxID{},
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID{
+							0x9b, 0x22, 0x59, 0x44, 0x66, 0xf0, 0xbe, 0x50,
+							0x7c, 0x1c, 0x8a, 0xf6, 0x06, 0x27, 0xe6, 0x33,
+							0x38, 0x7e, 0xd1, 0xd5, 0x8c, 0x42, 0x59, 0x1a,
+							0x31, 0xac, 0x9a, 0xa6, 0x2e, 0xd5, 0x2b, 0x0f,
+						},
 						Index: 0xffffffff,
 					},
-					SignatureScript: []byte{
-						0x02, 0x10, 0x27, 0x08, 0x8f, 0x22, 0xfb, 0x88,
-						0x45, 0x7b, 0xee, 0xeb, 0x0b, 0x2f, 0x50, 0x32,
-						0x53, 0x48, 0x2f, 0x62, 0x74, 0x63, 0x64, 0x2f,
-					},
-					Sequence: math.MaxUint64,
+					SignatureScript: nil,
+					Sequence:        math.MaxUint64,
 				},
 			},
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0x12a05f200, // 5000000000
-					PkScript: []byte{
-						0x51,
+					ScriptPubKey: []byte{
+						0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49,
+						0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7,
+						0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
 					},
 				},
 			},
 			LockTime:     0,
-			SubnetworkID: *subnetworkid.SubnetworkIDNative,
+			SubnetworkID: *subnetworkid.SubnetworkIDCoinbase,
+			Payload:      []byte{0x00},
+			PayloadHash: &daghash.Hash{
+				0x14, 0x06, 0xe0, 0x58, 0x81, 0xe2, 0x99, 0x36,
+				0x77, 0x66, 0xd3, 0x13, 0xe2, 0x6c, 0x05, 0x56,
+				0x4e, 0xc9, 0x1b, 0xf7, 0x21, 0xd3, 0x17, 0x26,
+				0xbd, 0x6e, 0x46, 0xe6, 0x06, 0x89, 0x53, 0x9a,
+			},
 		},
 		{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
+					PreviousOutpoint: wire.Outpoint{
 						TxID: daghash.TxID{
 							0x16, 0x5e, 0x38, 0xe8, 0xb3, 0x91, 0x45, 0x95,
 							0xd9, 0xc6, 0x41, 0xf3, 0xb8, 0xee, 0xc2, 0xf3,
@@ -807,7 +820,7 @@ var Block100000 = wire.MsgBlock{
 					Sequence: math.MaxUint64,
 				},
 				{
-					PreviousOutPoint: wire.OutPoint{
+					PreviousOutpoint: wire.Outpoint{
 						TxID: daghash.TxID{
 							0x4b, 0xb0, 0x75, 0x35, 0xdf, 0xd5, 0x8e, 0x0b,
 							0x3c, 0xd6, 0x4f, 0xd7, 0x15, 0x52, 0x80, 0x87,
@@ -825,8 +838,8 @@ var Block100000 = wire.MsgBlock{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID: daghash.TxID([32]byte{ // Make go vet happy.
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID([32]byte{
 							0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
 							0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
 							0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
@@ -863,7 +876,7 @@ var Block100000 = wire.MsgBlock{
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0x2123e300, // 556000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -876,7 +889,7 @@ var Block100000 = wire.MsgBlock{
 				},
 				{
 					Value: 0x108e20f00, // 4444000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -895,8 +908,8 @@ var Block100000 = wire.MsgBlock{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID: daghash.TxID([32]byte{ // Make go vet happy.
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID([32]byte{
 							0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
 							0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
 							0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
@@ -932,7 +945,7 @@ var Block100000 = wire.MsgBlock{
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0xf4240, // 1000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -945,7 +958,7 @@ var Block100000 = wire.MsgBlock{
 				},
 				{
 					Value: 0x11d260c0, // 299000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -964,8 +977,8 @@ var Block100000 = wire.MsgBlock{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID: daghash.TxID([32]byte{ // Make go vet happy.
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID([32]byte{
 							0x0b, 0x60, 0x72, 0xb3, 0x86, 0xd4, 0xa7, 0x73,
 							0x23, 0x52, 0x37, 0xf6, 0x4c, 0x11, 0x26, 0xac,
 							0x3b, 0x24, 0x0c, 0x84, 0xb9, 0x17, 0xa3, 0x90,
@@ -1002,7 +1015,7 @@ var Block100000 = wire.MsgBlock{
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0xf4240, // 1000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -1025,75 +1038,84 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 	Header: wire.BlockHeader{
 		Version: 1,
 		ParentHashes: []*daghash.Hash{
-			{ // Make go vet happy.
+			{
 				0x16, 0x5e, 0x38, 0xe8, 0xb3, 0x91, 0x45, 0x95,
 				0xd9, 0xc6, 0x41, 0xf3, 0xb8, 0xee, 0xc2, 0xf3,
 				0x46, 0x11, 0x89, 0x6b, 0x82, 0x1a, 0x68, 0x3b,
 				0x7a, 0x4e, 0xde, 0xfe, 0x2c, 0x00, 0x00, 0x00,
 			},
-			{ // Make go vet happy.
+			{
 				0x4b, 0xb0, 0x75, 0x35, 0xdf, 0xd5, 0x8e, 0x0b,
 				0x3c, 0xd6, 0x4f, 0xd7, 0x15, 0x52, 0x80, 0x87,
 				0x2a, 0x04, 0x71, 0xbc, 0xf8, 0x30, 0x95, 0x52,
 				0x6a, 0xce, 0x0e, 0x38, 0xc6, 0x00, 0x00, 0x00,
 			},
 		},
-		HashMerkleRoot: &daghash.Hash{ // Make go vet happy.
-			0x0b, 0x79, 0xf5, 0x29, 0x6d, 0x1c, 0xaa, 0x90,
-			0x2f, 0x01, 0xd4, 0x83, 0x9b, 0x2a, 0x04, 0x5e,
+		HashMerkleRoot: &daghash.Hash{
+			0xac, 0xa4, 0x21, 0xe1, 0xa6, 0xc3, 0xbe, 0x5d,
+			0x52, 0x66, 0xf3, 0x0b, 0x21, 0x87, 0xbc, 0xf3,
+			0xf3, 0x2d, 0xd1, 0x05, 0x64, 0xb5, 0x16, 0x76,
+			0xe4, 0x66, 0x7d, 0x51, 0x53, 0x18, 0x6d, 0xb1,
+		},
+		AcceptedIDMerkleRoot: &daghash.Hash{
 			0xa0, 0x69, 0x2d, 0x16, 0xb5, 0xd7, 0xe4, 0xf3,
 			0xcd, 0xc7, 0xc9, 0xaf, 0xfb, 0xd2, 0x1b, 0x85,
-		},
-		IDMerkleRoot: &daghash.Hash{ // Make go vet happy.
 			0x0b, 0x79, 0xf5, 0x29, 0x6d, 0x1c, 0xaa, 0x90,
 			0x2f, 0x01, 0xd4, 0x83, 0x9b, 0x2a, 0x04, 0x5e,
-			0xa0, 0x69, 0x2d, 0x16, 0xb5, 0xd7, 0xe4, 0xf3,
-			0xcd, 0xc7, 0xc9, 0xaf, 0xfb, 0xd2, 0x1b, 0x85,
 		},
-		Timestamp: time.Unix(0x5c22330f, 0),
+		UTXOCommitment: &daghash.Hash{
+			0x00, 0x69, 0x2d, 0x16, 0xb5, 0xd7, 0xe4, 0xf3,
+			0xcd, 0xc7, 0xc9, 0xaf, 0xfb, 0xd2, 0x1b, 0x85,
+			0x0b, 0x79, 0xf5, 0x29, 0x6d, 0x1c, 0xaa, 0x90,
+			0x2f, 0x01, 0xd4, 0x83, 0x9b, 0x2a, 0x04, 0x5e,
+		},
+		Timestamp: time.Unix(0x5cd16eaa, 0),
 		Bits:      0x207fffff,
-		Nonce:     0xdffffffffffffffc,
+		Nonce:     0x0,
 	},
 	Transactions: []*wire.MsgTx{
 		{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID:  daghash.TxID{},
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID{
+							0x9b, 0x22, 0x59, 0x44, 0x66, 0xf0, 0xbe, 0x50,
+							0x7c, 0x1c, 0x8a, 0xf6, 0x06, 0x27, 0xe6, 0x33,
+							0x38, 0x7e, 0xd1, 0xd5, 0x8c, 0x42, 0x59, 0x1a,
+							0x31, 0xac, 0x9a, 0xa6, 0x2e, 0xd5, 0x2b, 0x0f,
+						},
 						Index: 0xffffffff,
 					},
-					SignatureScript: []byte{
-						0x04, 0x4c, 0x86, 0x04, 0x1b, 0x02, 0x06, 0x02,
-					},
-					Sequence: math.MaxUint64,
+					SignatureScript: nil,
+					Sequence:        math.MaxUint64,
 				},
 			},
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0x12a05f200, // 5000000000
-					PkScript: []byte{
-						0x41, // OP_DATA_65
-						0x04, 0x1b, 0x0e, 0x8c, 0x25, 0x67, 0xc1, 0x25,
-						0x36, 0xaa, 0x13, 0x35, 0x7b, 0x79, 0xa0, 0x73,
-						0xdc, 0x44, 0x44, 0xac, 0xb8, 0x3c, 0x4e, 0xc7,
-						0xa0, 0xe2, 0xf9, 0x9d, 0xd7, 0x45, 0x75, 0x16,
-						0xc5, 0x81, 0x72, 0x42, 0xda, 0x79, 0x69, 0x24,
-						0xca, 0x4e, 0x99, 0x94, 0x7d, 0x08, 0x7f, 0xed,
-						0xf9, 0xce, 0x46, 0x7c, 0xb9, 0xf7, 0xc6, 0x28,
-						0x70, 0x78, 0xf8, 0x01, 0xdf, 0x27, 0x6f, 0xdf,
-						0x84, // 65-byte signature
-						0xac, // OP_CHECKSIG
+					ScriptPubKey: []byte{
+						0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49,
+						0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7,
+						0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
 					},
 				},
 			},
-			LockTime: 0,
+			LockTime:     0,
+			SubnetworkID: *subnetworkid.SubnetworkIDCoinbase,
+			Payload:      []byte{0x00},
+			PayloadHash: &daghash.Hash{
+				0x14, 0x06, 0xe0, 0x58, 0x81, 0xe2, 0x99, 0x36,
+				0x77, 0x66, 0xd3, 0x13, 0xe2, 0x6c, 0x05, 0x56,
+				0x4e, 0xc9, 0x1b, 0xf7, 0x21, 0xd3, 0x17, 0x26,
+				0xbd, 0x6e, 0x46, 0xe6, 0x06, 0x89, 0x53, 0x9a,
+			},
 		},
 		{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
+					PreviousOutpoint: wire.Outpoint{
 						TxID: daghash.TxID{
 							0x16, 0x5e, 0x38, 0xe8, 0xb3, 0x91, 0x45, 0x95,
 							0xd9, 0xc6, 0x41, 0xf3, 0xb8, 0xee, 0xc2, 0xf3,
@@ -1105,7 +1127,7 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 					Sequence: math.MaxUint64,
 				},
 				{
-					PreviousOutPoint: wire.OutPoint{
+					PreviousOutpoint: wire.Outpoint{
 						TxID: daghash.TxID{
 							0x4b, 0xb0, 0x75, 0x35, 0xdf, 0xd5, 0x8e, 0x0b,
 							0x3c, 0xd6, 0x4f, 0xd7, 0x15, 0x52, 0x80, 0x87,
@@ -1123,8 +1145,8 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID: daghash.TxID([32]byte{ // Make go vet happy.
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID([32]byte{
 							0x03, 0x2e, 0x38, 0xe9, 0xc0, 0xa8, 0x4c, 0x60,
 							0x46, 0xd6, 0x87, 0xd1, 0x05, 0x56, 0xdc, 0xac,
 							0xc4, 0x1d, 0x27, 0x5e, 0xc5, 0x5f, 0xc0, 0x07,
@@ -1161,7 +1183,7 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0x2123e300, // 556000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -1174,7 +1196,7 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 				},
 				{
 					Value: 0x108e20f00, // 4444000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -1188,13 +1210,15 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 			},
 			LockTime:     0,
 			SubnetworkID: subnetworkid.SubnetworkID{11},
+			Payload:      []byte{},
+			PayloadHash:  daghash.DoubleHashP([]byte{}),
 		},
 		{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID: daghash.TxID([32]byte{ // Make go vet happy.
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID([32]byte{
 							0xc3, 0x3e, 0xbf, 0xf2, 0xa7, 0x09, 0xf1, 0x3d,
 							0x9f, 0x9a, 0x75, 0x69, 0xab, 0x16, 0xa3, 0x27,
 							0x86, 0xaf, 0x7d, 0x7e, 0x2d, 0xe0, 0x92, 0x65,
@@ -1230,7 +1254,7 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0xf4240, // 1000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -1243,7 +1267,7 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 				},
 				{
 					Value: 0x11d260c0, // 299000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20
@@ -1262,8 +1286,8 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 			Version: 1,
 			TxIn: []*wire.TxIn{
 				{
-					PreviousOutPoint: wire.OutPoint{
-						TxID: daghash.TxID([32]byte{ // Make go vet happy.
+					PreviousOutpoint: wire.Outpoint{
+						TxID: daghash.TxID([32]byte{
 							0x0b, 0x60, 0x72, 0xb3, 0x86, 0xd4, 0xa7, 0x73,
 							0x23, 0x52, 0x37, 0xf6, 0x4c, 0x11, 0x26, 0xac,
 							0x3b, 0x24, 0x0c, 0x84, 0xb9, 0x17, 0xa3, 0x90,
@@ -1300,7 +1324,7 @@ var BlockWithWrongTxOrder = wire.MsgBlock{
 			TxOut: []*wire.TxOut{
 				{
 					Value: 0xf4240, // 1000000
-					PkScript: []byte{
+					ScriptPubKey: []byte{
 						0x76, // OP_DUP
 						0xa9, // OP_HASH160
 						0x14, // OP_DATA_20

blockdag/versionbits.go

@@ -7,11 +7,10 @@ package blockdag
 import (
 	"math"
 
-	"github.com/daglabs/btcd/dagconfig"
+	"github.com/kaspanet/kaspad/dagconfig"
 )
 
 const (
-
 	// vbTopBits defines the bits to set in the version to signal that the
 	// version bits scheme is being used.
 	vbTopBits = 0x10000000
@@ -78,7 +77,7 @@ func (c bitConditionChecker) EndTime() uint64 {
 // is associated with.
 //
 // This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) RuleChangeActivationThreshold() uint32 {
+func (c bitConditionChecker) RuleChangeActivationThreshold() uint64 {
 	return c.chain.dagParams.RuleChangeActivationThreshold
 }
 
@@ -89,7 +88,7 @@ func (c bitConditionChecker) RuleChangeActivationThreshold() uint32 {
 // is associated with.
 //
 // This is part of the thresholdConditionChecker interface implementation.
-func (c bitConditionChecker) MinerConfirmationWindow() uint32 {
+func (c bitConditionChecker) MinerConfirmationWindow() uint64 {
 	return c.chain.dagParams.MinerConfirmationWindow
 }
 
@@ -159,7 +158,7 @@ func (c deploymentChecker) EndTime() uint64 {
 // is associated with.
 //
 // This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) RuleChangeActivationThreshold() uint32 {
+func (c deploymentChecker) RuleChangeActivationThreshold() uint64 {
 	return c.chain.dagParams.RuleChangeActivationThreshold
 }
 
@@ -170,7 +169,7 @@ func (c deploymentChecker) RuleChangeActivationThreshold() uint32 {
 // is associated with.
 //
 // This is part of the thresholdConditionChecker interface implementation.
-func (c deploymentChecker) MinerConfirmationWindow() uint32 {
+func (c deploymentChecker) MinerConfirmationWindow() uint64 {
 	return c.chain.dagParams.MinerConfirmationWindow
 }
 
@@ -250,10 +249,10 @@ func (dag *BlockDAG) warnUnknownRuleActivations(node *blockNode) error {
 			}
 
 		case ThresholdLockedIn:
-			window := int32(checker.MinerConfirmationWindow())
-			activationHeight := window - (node.height % window)
+			window := checker.MinerConfirmationWindow()
+			activationChainHeight := window - (node.chainHeight % window)
 			log.Warnf("Unknown new rules are about to activate in "+
-				"%d blocks (bit %d)", activationHeight, bit)
+				"%d blocks (bit %d)", activationChainHeight, bit)
 		}
 	}
 

blockdag/virtualblock.go

@@ -5,6 +5,7 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/util/daghash"
 	"sync"
 )
 
@@ -14,8 +15,15 @@ type virtualBlock struct {
 	phantomK uint32
 	utxoSet  *FullUTXOSet
 	blockNode
-	// selectedPathChainSet is a block set that includes all the blocks that belong to the chain of selected parents from the virtual block.
-	selectedPathChainSet blockSet
+
+	// selectedParentChainSet is a block set that includes all the blocks
+	// that belong to the chain of selected parents from the virtual block.
+	selectedParentChainSet blockSet
+
+	// selectedParentChainSlice is an ordered slice that includes all the
+	// blocks that belong the the chain of selected parents from the
+	// virtual block.
+	selectedParentChainSlice []*blockNode
 }
 
 // newVirtualBlock creates and returns a new VirtualBlock.
@@ -24,7 +32,8 @@ func newVirtualBlock(tips blockSet, phantomK uint32) *virtualBlock {
 	var virtual virtualBlock
 	virtual.phantomK = phantomK
 	virtual.utxoSet = NewFullUTXOSet()
-	virtual.selectedPathChainSet = newSet()
+	virtual.selectedParentChainSet = newSet()
+	virtual.selectedParentChainSlice = nil
 	virtual.setTips(tips)
 
 	return &virtual
@@ -33,10 +42,10 @@ func newVirtualBlock(tips blockSet, phantomK uint32) *virtualBlock {
 // clone creates and returns a clone of the virtual block.
 func (v *virtualBlock) clone() *virtualBlock {
 	return &virtualBlock{
-		phantomK:             v.phantomK,
-		utxoSet:              v.utxoSet,
-		blockNode:            v.blockNode,
-		selectedPathChainSet: v.selectedPathChainSet,
+		phantomK:               v.phantomK,
+		utxoSet:                v.utxoSet,
+		blockNode:              v.blockNode,
+		selectedParentChainSet: v.selectedParentChainSet,
 	}
 }
 
@@ -45,13 +54,13 @@ func (v *virtualBlock) clone() *virtualBlock {
 // is up to the caller to ensure the lock is held.
 //
 // This function MUST be called with the view mutex locked (for writes).
-func (v *virtualBlock) setTips(tips blockSet) {
+func (v *virtualBlock) setTips(tips blockSet) *chainUpdates {
 	oldSelectedParent := v.selectedParent
 	v.blockNode = *newBlockNode(nil, tips, v.phantomK)
-	v.updateSelectedPathSet(oldSelectedParent)
+	return v.updateSelectedParentSet(oldSelectedParent)
 }
 
-// updateSelectedPathSet updates the selectedPathSet to match the
+// updateSelectedParentSet updates the selectedParentSet to match the
 // new selected parent of the virtual block.
 // Every time the new selected parent is not a child of
 // the old one, it updates the selected path by removing from
@@ -59,25 +68,52 @@ func (v *virtualBlock) setTips(tips blockSet) {
 // parent and are not selected ancestors of the new one, and adding
 // blocks that are selected ancestors of the new selected parent
 // and aren't selected ancestors of the old one.
-func (v *virtualBlock) updateSelectedPathSet(oldSelectedParent *blockNode) {
+func (v *virtualBlock) updateSelectedParentSet(oldSelectedParent *blockNode) *chainUpdates {
 	var intersectionNode *blockNode
+	nodesToAdd := make([]*blockNode, 0)
 	for node := v.blockNode.selectedParent; intersectionNode == nil && node != nil; node = node.selectedParent {
-		if v.selectedPathChainSet.contains(node) {
+		if v.selectedParentChainSet.contains(node) {
 			intersectionNode = node
 		} else {
-			v.selectedPathChainSet.add(node)
+			nodesToAdd = append(nodesToAdd, node)
 		}
 	}
 
 	if intersectionNode == nil && oldSelectedParent != nil {
-		panic("updateSelectedPathSet: Cannot find intersection node. The block index may be corrupted.")
+		panic("updateSelectedParentSet: Cannot find intersection node. The block index may be corrupted.")
 	}
 
+	// Remove the nodes in the set from the oldSelectedParent down to the intersectionNode
+	// Also, save the hashes of the removed blocks to removedChainBlockHashes
+	removeCount := 0
+	var removedChainBlockHashes []*daghash.Hash
 	if intersectionNode != nil {
 		for node := oldSelectedParent; !node.hash.IsEqual(intersectionNode.hash); node = node.selectedParent {
-			v.selectedPathChainSet.remove(node)
+			v.selectedParentChainSet.remove(node)
+			removedChainBlockHashes = append(removedChainBlockHashes, node.hash)
+			removeCount++
 		}
 	}
+	// Remove the last removeCount nodes from the slice
+	v.selectedParentChainSlice = v.selectedParentChainSlice[:len(v.selectedParentChainSlice)-removeCount]
+
+	// Reverse nodesToAdd, since we collected them in reverse order
+	for left, right := 0, len(nodesToAdd)-1; left < right; left, right = left+1, right-1 {
+		nodesToAdd[left], nodesToAdd[right] = nodesToAdd[right], nodesToAdd[left]
+	}
+	// Add the nodes to the set and to the slice
+	// Also, save the hashes of the added blocks to addedChainBlockHashes
+	var addedChainBlockHashes []*daghash.Hash
+	for _, node := range nodesToAdd {
+		v.selectedParentChainSet.add(node)
+		addedChainBlockHashes = append(addedChainBlockHashes, node.hash)
+	}
+	v.selectedParentChainSlice = append(v.selectedParentChainSlice, nodesToAdd...)
+
+	return &chainUpdates{
+		removedChainBlockHashes: removedChainBlockHashes,
+		addedChainBlockHashes:   addedChainBlockHashes,
+	}
 }
 
 // SetTips replaces the tips of the virtual block with the blocks in the
@@ -96,14 +132,14 @@ func (v *virtualBlock) SetTips(tips blockSet) {
 // is up to the caller to ensure the lock is held.
 //
 // This function MUST be called with the view mutex locked (for writes).
-func (v *virtualBlock) addTip(newTip *blockNode) {
+func (v *virtualBlock) addTip(newTip *blockNode) *chainUpdates {
 	updatedTips := v.tips().clone()
 	for _, parent := range newTip.parents {
 		updatedTips.remove(parent)
 	}
 
 	updatedTips.add(newTip)
-	v.setTips(updatedTips)
+	return v.setTips(updatedTips)
 }
 
 // AddTip adds the given tip to the set of tips in the virtual block.
@@ -111,10 +147,10 @@ func (v *virtualBlock) addTip(newTip *blockNode) {
 // from the set.
 //
 // This function is safe for concurrent access.
-func (v *virtualBlock) AddTip(newTip *blockNode) {
+func (v *virtualBlock) AddTip(newTip *blockNode) *chainUpdates {
 	v.mtx.Lock()
-	v.addTip(newTip)
-	v.mtx.Unlock()
+	defer v.mtx.Unlock()
+	return v.addTip(newTip)
 }
 
 // tips returns the current tip block nodes for the DAG.  It will return

blockdag/virtualblock_test.go

@@ -123,9 +123,15 @@ func TestSelectedPath(t *testing.T) {
 		virtual.AddTip(tip)
 	}
 	// For now we don't have any DAG, just chain, the selected path should include all the blocks on the chain.
-	if !reflect.DeepEqual(virtual.selectedPathChainSet, firstPath) {
+	if !reflect.DeepEqual(virtual.selectedParentChainSet, firstPath) {
 		t.Fatalf("TestSelectedPath: selectedPathSet doesn't include the expected values. got %v, want %v", virtual.selectedParent, firstPath)
 	}
+	// We expect that selectedParentChainSlice should have all the blocks we've added so far
+	wantLen := 11
+	gotLen := len(virtual.selectedParentChainSlice)
+	if wantLen != gotLen {
+		t.Fatalf("TestSelectedPath: selectedParentChainSlice doesn't have the expected length. got %d, want %d", gotLen, wantLen)
+	}
 
 	secondPath := initialPath.clone()
 	tip = initialTip
@@ -135,9 +141,16 @@ func TestSelectedPath(t *testing.T) {
 		virtual.AddTip(tip)
 	}
 	// Because we added a chain that is much longer than the previous chain, the selected path should be re-organized.
-	if !reflect.DeepEqual(virtual.selectedPathChainSet, secondPath) {
+	if !reflect.DeepEqual(virtual.selectedParentChainSet, secondPath) {
 		t.Fatalf("TestSelectedPath: selectedPathSet didn't handle the re-org as expected. got %v, want %v", virtual.selectedParent, firstPath)
 	}
+	// We expect that selectedParentChainSlice should have all the blocks we've added so far except the old chain
+	wantLen = 106
+	gotLen = len(virtual.selectedParentChainSlice)
+	if wantLen != gotLen {
+		t.Fatalf("TestSelectedPath: selectedParentChainSlice doesn't have"+
+			"the expected length, possibly because it didn't handle the re-org as expected. got %d, want %d", gotLen, wantLen)
+	}
 
 	tip = initialTip
 	for i := 0; i < 3; i++ {
@@ -145,16 +158,77 @@ func TestSelectedPath(t *testing.T) {
 		virtual.AddTip(tip)
 	}
 	// Because we added a very short chain, the selected path should not be affected.
-	if !reflect.DeepEqual(virtual.selectedPathChainSet, secondPath) {
+	if !reflect.DeepEqual(virtual.selectedParentChainSet, secondPath) {
 		t.Fatalf("TestSelectedPath: selectedPathSet did an unexpected re-org. got %v, want %v", virtual.selectedParent, firstPath)
 	}
+	// We expect that selectedParentChainSlice not to change
+	wantLen = 106
+	gotLen = len(virtual.selectedParentChainSlice)
+	if wantLen != gotLen {
+		t.Fatalf("TestSelectedPath: selectedParentChainSlice doesn't"+
+			"have the expected length, possibly due to unexpected did an unexpected re-org. got %d, want %d", gotLen, wantLen)
+	}
 
-	// We call updateSelectedPathSet manually without updating the tips, to check if it panics
+	// We call updateSelectedParentSet manually without updating the tips, to check if it panics
 	virtual2 := newVirtualBlock(nil, phantomK)
 	defer func() {
 		if r := recover(); r == nil {
-			t.Fatalf("updateSelectedPathSet didn't panic")
+			t.Fatalf("updateSelectedParentSet didn't panic")
 		}
 	}()
-	virtual2.updateSelectedPathSet(buildNode(setFromSlice()))
+	virtual2.updateSelectedParentSet(buildNode(setFromSlice()))
+}
+
+func TestChainUpdates(t *testing.T) {
+	phantomK := uint32(1)
+	buildNode := buildNodeGenerator(phantomK, false)
+	genesis := buildNode(setFromSlice())
+
+	// Create a chain to be removed
+	var toBeRemovedNodes []*blockNode
+	toBeRemovedTip := genesis
+	for i := 0; i < 5; i++ {
+		toBeRemovedTip = buildNode(setFromSlice(toBeRemovedTip))
+		toBeRemovedNodes = append(toBeRemovedNodes, toBeRemovedTip)
+	}
+
+	// Create a VirtualBlock with the toBeRemoved chain
+	virtual := newVirtualBlock(setFromSlice(toBeRemovedNodes...), phantomK)
+
+	// Create a chain to be added
+	var toBeAddedNodes []*blockNode
+	toBeAddedTip := genesis
+	for i := 0; i < 8; i++ {
+		toBeAddedTip = buildNode(setFromSlice(toBeAddedTip))
+		toBeAddedNodes = append(toBeAddedNodes, toBeAddedTip)
+	}
+
+	// Set the virtual tip to be the tip of the toBeAdded chain
+	chainUpdates := virtual.setTips(setFromSlice(toBeAddedTip))
+
+	// Make sure that the removed blocks are as expected (in reverse order)
+	if len(chainUpdates.removedChainBlockHashes) != len(toBeRemovedNodes) {
+		t.Fatalf("TestChainUpdates: wrong removed amount. "+
+			"Got: %d, want: %d", len(chainUpdates.removedChainBlockHashes), len(toBeRemovedNodes))
+	}
+	for i, removedHash := range chainUpdates.removedChainBlockHashes {
+		correspondingRemovedNode := toBeRemovedNodes[len(toBeRemovedNodes)-1-i]
+		if !removedHash.IsEqual(correspondingRemovedNode.hash) {
+			t.Fatalf("TestChainUpdates: wrong removed hash. "+
+				"Got: %s, want: %s", removedHash, correspondingRemovedNode.hash)
+		}
+	}
+
+	// Make sure that the added blocks are as expected (in forward order)
+	if len(chainUpdates.addedChainBlockHashes) != len(toBeAddedNodes) {
+		t.Fatalf("TestChainUpdates: wrong added amount. "+
+			"Got: %d, want: %d", len(chainUpdates.removedChainBlockHashes), len(toBeAddedNodes))
+	}
+	for i, addedHash := range chainUpdates.addedChainBlockHashes {
+		correspondingAddedNode := toBeAddedNodes[i]
+		if !addedHash.IsEqual(correspondingAddedNode.hash) {
+			t.Fatalf("TestChainUpdates: wrong added hash. "+
+				"Got: %s, want: %s", addedHash, correspondingAddedNode.hash)
+		}
+	}
 }

btcd.go

@@ -16,16 +16,16 @@ import (
 	"runtime/pprof"
 	"strings"
 
-	"github.com/daglabs/btcd/blockdag/indexers"
-	"github.com/daglabs/btcd/config"
-	"github.com/daglabs/btcd/database"
-	_ "github.com/daglabs/btcd/database/ffldb"
-	"github.com/daglabs/btcd/limits"
-	"github.com/daglabs/btcd/logger"
-	"github.com/daglabs/btcd/server"
-	"github.com/daglabs/btcd/signal"
-	"github.com/daglabs/btcd/util/fs"
-	"github.com/daglabs/btcd/version"
+	"github.com/kaspanet/kaspad/blockdag/indexers"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/database"
+	_ "github.com/kaspanet/kaspad/database/ffldb"
+	"github.com/kaspanet/kaspad/limits"
+	"github.com/kaspanet/kaspad/server"
+	"github.com/kaspanet/kaspad/signal"
+	"github.com/kaspanet/kaspad/util/fs"
+	"github.com/kaspanet/kaspad/util/panics"
+	"github.com/kaspanet/kaspad/version"
 )
 
 const (
@@ -51,16 +51,12 @@ var winServiceMain func() (bool, error)
 func btcdMain(serverChan chan<- *server.Server) error {
 	// Load configuration and parse command line.  This function also
 	// initializes logging and configures it accordingly.
-	err := config.LoadAndSetMainConfig()
+	err := config.LoadAndSetActiveConfig()
 	if err != nil {
 		return err
 	}
-	cfg = config.MainConfig()
-	defer func() {
-		if logger.LogRotator != nil {
-			logger.LogRotator.Close()
-		}
-	}()
+	cfg = config.ActiveConfig()
+	defer panics.HandlePanic(btcdLog, nil, nil)
 
 	// Get a channel that will be closed when a shutdown signal has been
 	// triggered either from an OS signal such as SIGINT (Ctrl+C) or from
@@ -73,14 +69,14 @@ func btcdMain(serverChan chan<- *server.Server) error {
 
 	// Enable http profiling server if requested.
 	if cfg.Profile != "" {
-		go func() {
+		spawn(func() {
 			listenAddr := net.JoinHostPort("", cfg.Profile)
 			btcdLog.Infof("Profile server listening on %s", listenAddr)
 			profileRedirect := http.RedirectHandler("/debug/pprof",
 				http.StatusSeeOther)
 			http.Handle("/", profileRedirect)
 			btcdLog.Errorf("%s", http.ListenAndServe(listenAddr, nil))
-		}()
+		})
 	}
 
 	// Write cpu profile if requested.
@@ -151,8 +147,8 @@ func btcdMain(serverChan chan<- *server.Server) error {
 
 		return nil
 	}
-	if cfg.DropCfIndex {
-		if err := indexers.DropCfIndex(db, interrupt); err != nil {
+	if cfg.DropAcceptanceIndex {
+		if err := indexers.DropAcceptanceIndex(db, interrupt); err != nil {
 			btcdLog.Errorf("%s", err)
 			return err
 		}
@@ -161,7 +157,7 @@ func btcdMain(serverChan chan<- *server.Server) error {
 	}
 
 	// Create server and start it.
-	server, err := server.NewServer(cfg.Listeners, db, config.ActiveNetParams(),
+	server, err := server.NewServer(cfg.Listeners, db, config.ActiveConfig().NetParams(),
 		interrupt)
 	if err != nil {
 		// TODO: this logging could do with some beautifying.
@@ -292,7 +288,7 @@ func loadBlockDB() (database.DB, error) {
 	removeRegressionDB(dbPath)
 
 	btcdLog.Infof("Loading block database from '%s'", dbPath)
-	db, err := database.Open(cfg.DbType, dbPath, config.ActiveNetParams().Net)
+	db, err := database.Open(cfg.DbType, dbPath, config.ActiveConfig().NetParams().Net)
 	if err != nil {
 		// Return the error if it's not because the database doesn't
 		// exist.
@@ -307,7 +303,7 @@ func loadBlockDB() (database.DB, error) {
 		if err != nil {
 			return nil, err
 		}
-		db, err = database.Create(cfg.DbType, dbPath, config.ActiveNetParams().Net)
+		db, err = database.Create(cfg.DbType, dbPath, config.ActiveConfig().NetParams().Net)
 		if err != nil {
 			return nil, err
 		}

btcec/README.md

@@ -3,7 +3,7 @@ btcec
 
 [![Build Status](https://travis-ci.org/btcsuite/btcd.png?branch=master)](https://travis-ci.org/btcsuite/btcec)
 [![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
-[![GoDoc](https://godoc.org/github.com/daglabs/btcd/btcec?status.png)](http://godoc.org/github.com/daglabs/btcd/btcec)
+[![GoDoc](https://godoc.org/github.com/kaspanet/kaspad/btcec?status.png)](http://godoc.org/github.com/kaspanet/kaspad/btcec)
 
 Package btcec implements elliptic curve cryptography needed for working with
 Bitcoin (secp256k1 only for now). It is designed so that it may be used with the
@@ -20,24 +20,24 @@ use secp256k1 elliptic curve cryptography.
 ## Installation and Updating
 
 ```bash
-$ go get -u github.com/daglabs/btcd/btcec
+$ go get -u github.com/kaspanet/kaspad/btcec
 ```
 
 ## Examples
 
-* [Sign Message](http://godoc.org/github.com/daglabs/btcd/btcec#example-package--SignMessage)  
+* [Sign Message](http://godoc.org/github.com/kaspanet/kaspad/btcec#example-package--SignMessage)  
   Demonstrates signing a message with a secp256k1 private key that is first
   parsed form raw bytes and serializing the generated signature.
 
-* [Verify Signature](http://godoc.org/github.com/daglabs/btcd/btcec#example-package--VerifySignature)  
+* [Verify Signature](http://godoc.org/github.com/kaspanet/kaspad/btcec#example-package--VerifySignature)  
   Demonstrates verifying a secp256k1 signature against a public key that is
   first parsed from raw bytes.  The signature is also parsed from raw bytes.
 
-* [Encryption](http://godoc.org/github.com/daglabs/btcd/btcec#example-package--EncryptMessage)
+* [Encryption](http://godoc.org/github.com/kaspanet/kaspad/btcec#example-package--EncryptMessage)
   Demonstrates encrypting a message for a public key that is first parsed from
   raw bytes, then decrypting it using the corresponding private key.
 
-* [Decryption](http://godoc.org/github.com/daglabs/btcd/btcec#example-package--DecryptMessage)
+* [Decryption](http://godoc.org/github.com/kaspanet/kaspad/btcec#example-package--DecryptMessage)
   Demonstrates decrypting a message using a private key that is first parsed
   from raw bytes.
 

btcec/btcec.go

@@ -857,6 +857,109 @@ func (curve *KoblitzCurve) ScalarMult(Bx, By *big.Int, k []byte) (*big.Int, *big
 	return curve.fieldJacobianToBigAffine(qx, qy, qz)
 }
 
+// scalarMultJacobian returns the Jacobian coordinates of k*(Bx, By) where k is a big endian integer.
+// Taken from https://github.com/gcash/bchd/blob/99ad9c81ae1c543b42d9049cbfaa164014219ec9/bchec/bchec.go
+func (curve *KoblitzCurve) scalarMultJacobian(Bx, By *big.Int, k []byte) (*fieldVal, *fieldVal, *fieldVal) {
+	// Point Q = ∞ (point at infinity).
+	qx, qy, qz := new(fieldVal), new(fieldVal), new(fieldVal)
+
+	// Decompose K into k1 and k2 in order to halve the number of EC ops.
+	// See Algorithm 3.74 in [GECC].
+	k1, k2, signK1, signK2 := curve.splitK(curve.moduloReduce(k))
+
+	// The main equation here to remember is:
+	//   k * P = k1 * P + k2 * ϕ(P)
+	//
+	// P1 below is P in the equation, P2 below is ϕ(P) in the equation
+	p1x, p1y := curve.bigAffineToField(Bx, By)
+	p1yNeg := new(fieldVal).NegateVal(p1y, 1)
+	p1z := new(fieldVal).SetInt(1)
+
+	// NOTE: ϕ(x,y) = (βx,y).  The Jacobian z coordinate is 1, so this math
+	// goes through.
+	p2x := new(fieldVal).Mul2(p1x, curve.beta)
+	p2y := new(fieldVal).Set(p1y)
+	p2yNeg := new(fieldVal).NegateVal(p2y, 1)
+	p2z := new(fieldVal).SetInt(1)
+
+	// Flip the positive and negative values of the points as needed
+	// depending on the signs of k1 and k2.  As mentioned in the equation
+	// above, each of k1 and k2 are multiplied by the respective point.
+	// Since -k * P is the same thing as k * -P, and the group law for
+	// elliptic curves states that P(x, y) = -P(x, -y), it's faster and
+	// simplifies the code to just make the point negative.
+	if signK1 == -1 {
+		p1y, p1yNeg = p1yNeg, p1y
+	}
+	if signK2 == -1 {
+		p2y, p2yNeg = p2yNeg, p2y
+	}
+
+	// NAF versions of k1 and k2 should have a lot more zeros.
+	//
+	// The Pos version of the bytes contain the +1s and the Neg versions
+	// contain the -1s.
+	k1PosNAF, k1NegNAF := NAF(k1)
+	k2PosNAF, k2NegNAF := NAF(k2)
+	k1Len := len(k1PosNAF)
+	k2Len := len(k2PosNAF)
+
+	m := k1Len
+	if m < k2Len {
+		m = k2Len
+	}
+
+	// Add left-to-right using the NAF optimization.  See algorithm 3.77
+	// from [GECC].  This should be faster overall since there will be a lot
+	// more instances of 0, hence reducing the number of Jacobian additions
+	// at the cost of 1 possible extra doubling.
+	var k1BytePos, k1ByteNeg, k2BytePos, k2ByteNeg byte
+	for i := 0; i < m; i++ {
+		// Since we're going left-to-right, pad the front with 0s.
+		if i < m-k1Len {
+			k1BytePos = 0
+			k1ByteNeg = 0
+		} else {
+			k1BytePos = k1PosNAF[i-(m-k1Len)]
+			k1ByteNeg = k1NegNAF[i-(m-k1Len)]
+		}
+		if i < m-k2Len {
+			k2BytePos = 0
+			k2ByteNeg = 0
+		} else {
+			k2BytePos = k2PosNAF[i-(m-k2Len)]
+			k2ByteNeg = k2NegNAF[i-(m-k2Len)]
+		}
+
+		for j := 7; j >= 0; j-- {
+			// Q = 2 * Q
+			curve.doubleJacobian(qx, qy, qz, qx, qy, qz)
+
+			if k1BytePos&0x80 == 0x80 {
+				curve.addJacobian(qx, qy, qz, p1x, p1y, p1z,
+					qx, qy, qz)
+			} else if k1ByteNeg&0x80 == 0x80 {
+				curve.addJacobian(qx, qy, qz, p1x, p1yNeg, p1z,
+					qx, qy, qz)
+			}
+
+			if k2BytePos&0x80 == 0x80 {
+				curve.addJacobian(qx, qy, qz, p2x, p2y, p2z,
+					qx, qy, qz)
+			} else if k2ByteNeg&0x80 == 0x80 {
+				curve.addJacobian(qx, qy, qz, p2x, p2yNeg, p2z,
+					qx, qy, qz)
+			}
+			k1BytePos <<= 1
+			k1ByteNeg <<= 1
+			k2BytePos <<= 1
+			k2ByteNeg <<= 1
+		}
+	}
+
+	return qx, qy, qz
+}
+
 // ScalarBaseMult returns k*G where G is the base point of the group and k is a
 // big endian integer.
 // Part of the elliptic.Curve interface.
@@ -879,6 +982,28 @@ func (curve *KoblitzCurve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) {
 	return curve.fieldJacobianToBigAffine(qx, qy, qz)
 }
 
+// scalarBaseMultJacobian returns the Jacobian coordinates k*G where G is the base point of
+// the group and k is a big endian integer.
+// Taken from https://github.com/gcash/bchd/blob/99ad9c81ae1c543b42d9049cbfaa164014219ec9/bchec/bchec.go
+func (curve *KoblitzCurve) scalarBaseMultJacobian(k []byte) (*fieldVal, *fieldVal, *fieldVal) {
+	newK := curve.moduloReduce(k)
+	diff := len(curve.bytePoints) - len(newK)
+
+	// Point Q = ∞ (point at infinity).
+	qx, qy, qz := new(fieldVal), new(fieldVal), new(fieldVal)
+
+	// curve.bytePoints has all 256 byte points for each 8-bit window. The
+	// strategy is to add up the byte points. This is best understood by
+	// expressing k in base-256 which it already sort of is.
+	// Each "digit" in the 8-bit window can be looked up using bytePoints
+	// and added together.
+	for i, byteVal := range newK {
+		p := curve.bytePoints[diff+i][byteVal]
+		curve.addJacobian(qx, qy, qz, &p[0], &p[1], &p[2], qx, qy, qz)
+	}
+	return qx, qy, qz
+}
+
 // QPlus1Div4 returns the Q+1/4 constant for the curve for use in calculating
 // square roots via exponention.
 func (curve *KoblitzCurve) QPlus1Div4() *big.Int {

btcec/ciphering.go

@@ -1,216 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package btcec
-
-import (
-	"bytes"
-	"crypto/aes"
-	"crypto/cipher"
-	"crypto/hmac"
-	"crypto/rand"
-	"crypto/sha256"
-	"crypto/sha512"
-	"errors"
-	"io"
-)
-
-var (
-	// ErrInvalidMAC occurs when Message Authentication Check (MAC) fails
-	// during decryption. This happens because of either invalid private key or
-	// corrupt ciphertext.
-	ErrInvalidMAC = errors.New("invalid mac hash")
-
-	// errInputTooShort occurs when the input ciphertext to the Decrypt
-	// function is less than 134 bytes long.
-	errInputTooShort = errors.New("ciphertext too short")
-
-	// errUnsupportedCurve occurs when the first two bytes of the encrypted
-	// text aren't 0x02CA (= 712 = secp256k1, from OpenSSL).
-	errUnsupportedCurve = errors.New("unsupported curve")
-
-	errInvalidXLength = errors.New("invalid X length, must be 32")
-	errInvalidYLength = errors.New("invalid Y length, must be 32")
-	errInvalidPadding = errors.New("invalid PKCS#7 padding")
-
-	// 0x02CA = 714
-	ciphCurveBytes = [2]byte{0x02, 0xCA}
-	// 0x20 = 32
-	ciphCoordLength = [2]byte{0x00, 0x20}
-)
-
-// GenerateSharedSecret generates a shared secret based on a private key and a
-// public key using Diffie-Hellman key exchange (ECDH) (RFC 4753).
-// RFC5903 Section 9 states we should only return x.
-func GenerateSharedSecret(privkey *PrivateKey, pubkey *PublicKey) []byte {
-	x, _ := pubkey.Curve.ScalarMult(pubkey.X, pubkey.Y, privkey.D.Bytes())
-	return x.Bytes()
-}
-
-// Encrypt encrypts data for the target public key using AES-256-CBC. It also
-// generates a private key (the pubkey of which is also in the output). The only
-// supported curve is secp256k1. The `structure' that it encodes everything into
-// is:
-//
-//	struct {
-//		// Initialization Vector used for AES-256-CBC
-//		IV [16]byte
-//		// Public Key: curve(2) + len_of_pubkeyX(2) + pubkeyX +
-//		// len_of_pubkeyY(2) + pubkeyY (curve = 714)
-//		PublicKey [70]byte
-//		// Cipher text
-//		Data []byte
-//		// HMAC-SHA-256 Message Authentication Code
-//		HMAC [32]byte
-//	}
-//
-// The primary aim is to ensure byte compatibility with Pyelliptic.  Also, refer
-// to section 5.8.1 of ANSI X9.63 for rationale on this format.
-func Encrypt(pubkey *PublicKey, in []byte) ([]byte, error) {
-	ephemeral, err := NewPrivateKey(S256())
-	if err != nil {
-		return nil, err
-	}
-	ecdhKey := GenerateSharedSecret(ephemeral, pubkey)
-	derivedKey := sha512.Sum512(ecdhKey)
-	keyE := derivedKey[:32]
-	keyM := derivedKey[32:]
-
-	paddedIn := addPKCSPadding(in)
-	// IV + Curve params/X/Y + padded plaintext/ciphertext + HMAC-256
-	out := make([]byte, aes.BlockSize+70+len(paddedIn)+sha256.Size)
-	iv := out[:aes.BlockSize]
-	if _, err = io.ReadFull(rand.Reader, iv); err != nil {
-		return nil, err
-	}
-	// start writing public key
-	pb := ephemeral.PubKey().SerializeUncompressed()
-	offset := aes.BlockSize
-
-	// curve and X length
-	copy(out[offset:offset+4], append(ciphCurveBytes[:], ciphCoordLength[:]...))
-	offset += 4
-	// X
-	copy(out[offset:offset+32], pb[1:33])
-	offset += 32
-	// Y length
-	copy(out[offset:offset+2], ciphCoordLength[:])
-	offset += 2
-	// Y
-	copy(out[offset:offset+32], pb[33:])
-	offset += 32
-
-	// start encryption
-	block, err := aes.NewCipher(keyE)
-	if err != nil {
-		return nil, err
-	}
-	mode := cipher.NewCBCEncrypter(block, iv)
-	mode.CryptBlocks(out[offset:len(out)-sha256.Size], paddedIn)
-
-	// start HMAC-SHA-256
-	hm := hmac.New(sha256.New, keyM)
-	hm.Write(out[:len(out)-sha256.Size])          // everything is hashed
-	copy(out[len(out)-sha256.Size:], hm.Sum(nil)) // write checksum
-
-	return out, nil
-}
-
-// Decrypt decrypts data that was encrypted using the Encrypt function.
-func Decrypt(priv *PrivateKey, in []byte) ([]byte, error) {
-	// IV + Curve params/X/Y + 1 block + HMAC-256
-	if len(in) < aes.BlockSize+70+aes.BlockSize+sha256.Size {
-		return nil, errInputTooShort
-	}
-
-	// read iv
-	iv := in[:aes.BlockSize]
-	offset := aes.BlockSize
-
-	// start reading pubkey
-	if !bytes.Equal(in[offset:offset+2], ciphCurveBytes[:]) {
-		return nil, errUnsupportedCurve
-	}
-	offset += 2
-
-	if !bytes.Equal(in[offset:offset+2], ciphCoordLength[:]) {
-		return nil, errInvalidXLength
-	}
-	offset += 2
-
-	xBytes := in[offset : offset+32]
-	offset += 32
-
-	if !bytes.Equal(in[offset:offset+2], ciphCoordLength[:]) {
-		return nil, errInvalidYLength
-	}
-	offset += 2
-
-	yBytes := in[offset : offset+32]
-	offset += 32
-
-	pb := make([]byte, 65)
-	pb[0] = byte(0x04) // uncompressed
-	copy(pb[1:33], xBytes)
-	copy(pb[33:], yBytes)
-	// check if (X, Y) lies on the curve and create a Pubkey if it does
-	pubkey, err := ParsePubKey(pb, S256())
-	if err != nil {
-		return nil, err
-	}
-
-	// check for cipher text length
-	if (len(in)-aes.BlockSize-offset-sha256.Size)%aes.BlockSize != 0 {
-		return nil, errInvalidPadding // not padded to 16 bytes
-	}
-
-	// read hmac
-	messageMAC := in[len(in)-sha256.Size:]
-
-	// generate shared secret
-	ecdhKey := GenerateSharedSecret(priv, pubkey)
-	derivedKey := sha512.Sum512(ecdhKey)
-	keyE := derivedKey[:32]
-	keyM := derivedKey[32:]
-
-	// verify mac
-	hm := hmac.New(sha256.New, keyM)
-	hm.Write(in[:len(in)-sha256.Size]) // everything is hashed
-	expectedMAC := hm.Sum(nil)
-	if !hmac.Equal(messageMAC, expectedMAC) {
-		return nil, ErrInvalidMAC
-	}
-
-	// start decryption
-	block, err := aes.NewCipher(keyE)
-	if err != nil {
-		return nil, err
-	}
-	mode := cipher.NewCBCDecrypter(block, iv)
-	// same length as ciphertext
-	plaintext := make([]byte, len(in)-offset-sha256.Size)
-	mode.CryptBlocks(plaintext, in[offset:len(in)-sha256.Size])
-
-	return removePKCSPadding(plaintext)
-}
-
-// Implement PKCS#7 padding with block size of 16 (AES block size).
-
-// addPKCSPadding adds padding to a block of data
-func addPKCSPadding(src []byte) []byte {
-	padding := aes.BlockSize - len(src)%aes.BlockSize
-	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
-	return append(src, padtext...)
-}
-
-// removePKCSPadding removes padding from data that was added with addPKCSPadding
-func removePKCSPadding(src []byte) ([]byte, error) {
-	length := len(src)
-	padLength := int(src[length-1])
-	if padLength > aes.BlockSize || length < aes.BlockSize {
-		return nil, errInvalidPadding
-	}
-
-	return src[:length-padLength], nil
-}

btcec/ciphering_test.go

@@ -1,174 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package btcec
-
-import (
-	"bytes"
-	"encoding/hex"
-	"testing"
-)
-
-func TestGenerateSharedSecret(t *testing.T) {
-	privKey1, err := NewPrivateKey(S256())
-	if err != nil {
-		t.Errorf("private key generation error: %s", err)
-		return
-	}
-	privKey2, err := NewPrivateKey(S256())
-	if err != nil {
-		t.Errorf("private key generation error: %s", err)
-		return
-	}
-
-	secret1 := GenerateSharedSecret(privKey1, privKey2.PubKey())
-	secret2 := GenerateSharedSecret(privKey2, privKey1.PubKey())
-
-	if !bytes.Equal(secret1, secret2) {
-		t.Errorf("ECDH failed, secrets mismatch - first: %x, second: %x",
-			secret1, secret2)
-	}
-}
-
-// Test 1: Encryption and decryption
-func TestCipheringBasic(t *testing.T) {
-	privkey, err := NewPrivateKey(S256())
-	if err != nil {
-		t.Fatal("failed to generate private key")
-	}
-
-	in := []byte("Hey there dude. How are you doing? This is a test.")
-
-	out, err := Encrypt(privkey.PubKey(), in)
-	if err != nil {
-		t.Fatal("failed to encrypt:", err)
-	}
-
-	dec, err := Decrypt(privkey, out)
-	if err != nil {
-		t.Fatal("failed to decrypt:", err)
-	}
-
-	if !bytes.Equal(in, dec) {
-		t.Error("decrypted data doesn't match original")
-	}
-}
-
-// Test 2: Byte compatibility with Pyelliptic
-func TestCiphering(t *testing.T) {
-	pb, _ := hex.DecodeString("fe38240982f313ae5afb3e904fb8215fb11af1200592b" +
-		"fca26c96c4738e4bf8f")
-	privkey, _ := PrivKeyFromBytes(S256(), pb)
-
-	in := []byte("This is just a test.")
-	out, _ := hex.DecodeString("b0d66e5adaa5ed4e2f0ca68e17b8f2fc02ca002009e3" +
-		"3487e7fa4ab505cf34d98f131be7bd258391588ca7804acb30251e71a04e0020ecf" +
-		"df0f84608f8add82d7353af780fbb28868c713b7813eb4d4e61f7b75d7534dd9856" +
-		"9b0ba77cf14348fcff80fee10e11981f1b4be372d93923e9178972f69937ec850ed" +
-		"6c3f11ff572ddd5b2bedf9f9c0b327c54da02a28fcdce1f8369ffec")
-
-	dec, err := Decrypt(privkey, out)
-	if err != nil {
-		t.Fatal("failed to decrypt:", err)
-	}
-
-	if !bytes.Equal(in, dec) {
-		t.Error("decrypted data doesn't match original")
-	}
-}
-
-func TestCipheringErrors(t *testing.T) {
-	privkey, err := NewPrivateKey(S256())
-	if err != nil {
-		t.Fatal("failed to generate private key")
-	}
-
-	tests1 := []struct {
-		ciphertext []byte // input ciphertext
-	}{
-		{bytes.Repeat([]byte{0x00}, 133)},                   // errInputTooShort
-		{bytes.Repeat([]byte{0x00}, 134)},                   // errUnsupportedCurve
-		{bytes.Repeat([]byte{0x02, 0xCA}, 134)},             // errInvalidXLength
-		{bytes.Repeat([]byte{0x02, 0xCA, 0x00, 0x20}, 134)}, // errInvalidYLength
-		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // IV
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x02, 0xCA, 0x00, 0x20, // curve and X length
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // X
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x20, // Y length
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Y
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ciphertext
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MAC
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		}}, // invalid pubkey
-		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // IV
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x02, 0xCA, 0x00, 0x20, // curve and X length
-			0x11, 0x5C, 0x42, 0xE7, 0x57, 0xB2, 0xEF, 0xB7, // X
-			0x67, 0x1C, 0x57, 0x85, 0x30, 0xEC, 0x19, 0x1A,
-			0x13, 0x59, 0x38, 0x1E, 0x6A, 0x71, 0x12, 0x7A,
-			0x9D, 0x37, 0xC4, 0x86, 0xFD, 0x30, 0xDA, 0xE5,
-			0x00, 0x20, // Y length
-			0x7E, 0x76, 0xDC, 0x58, 0xF6, 0x93, 0xBD, 0x7E, // Y
-			0x70, 0x10, 0x35, 0x8C, 0xE6, 0xB1, 0x65, 0xE4,
-			0x83, 0xA2, 0x92, 0x10, 0x10, 0xDB, 0x67, 0xAC,
-			0x11, 0xB1, 0xB5, 0x1B, 0x65, 0x19, 0x53, 0xD2,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ciphertext
-			// padding not aligned to 16 bytes
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MAC
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		}}, // errInvalidPadding
-		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // IV
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x02, 0xCA, 0x00, 0x20, // curve and X length
-			0x11, 0x5C, 0x42, 0xE7, 0x57, 0xB2, 0xEF, 0xB7, // X
-			0x67, 0x1C, 0x57, 0x85, 0x30, 0xEC, 0x19, 0x1A,
-			0x13, 0x59, 0x38, 0x1E, 0x6A, 0x71, 0x12, 0x7A,
-			0x9D, 0x37, 0xC4, 0x86, 0xFD, 0x30, 0xDA, 0xE5,
-			0x00, 0x20, // Y length
-			0x7E, 0x76, 0xDC, 0x58, 0xF6, 0x93, 0xBD, 0x7E, // Y
-			0x70, 0x10, 0x35, 0x8C, 0xE6, 0xB1, 0x65, 0xE4,
-			0x83, 0xA2, 0x92, 0x10, 0x10, 0xDB, 0x67, 0xAC,
-			0x11, 0xB1, 0xB5, 0x1B, 0x65, 0x19, 0x53, 0xD2,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ciphertext
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // MAC
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		}}, // ErrInvalidMAC
-	}
-
-	for i, test := range tests1 {
-		_, err = Decrypt(privkey, test.ciphertext)
-		if err == nil {
-			t.Errorf("Decrypt #%d did not get error", i)
-		}
-	}
-
-	// test error from removePKCSPadding
-	tests2 := []struct {
-		in []byte // input data
-	}{
-		{bytes.Repeat([]byte{0x11}, 17)},
-		{bytes.Repeat([]byte{0x07}, 15)},
-	}
-	for i, test := range tests2 {
-		_, err = removePKCSPadding(test.in)
-		if err == nil {
-			t.Errorf("removePKCSPadding #%d did not get error", i)
-		}
-	}
-}

btcec/ecmh.go

@@ -0,0 +1,152 @@
+package btcec
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"math/big"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// Multiset tracks the state of a multiset as used to calculate the ECMH
+// (elliptic curve multiset hash) hash of an unordered set. The state is
+// a point on the curve. New elements are hashed onto a point on the curve
+// and then added to the current state. Hence elements can be added in any
+// order and we can also remove elements to return to a prior hash.
+type Multiset struct {
+	curve *KoblitzCurve
+	x     *big.Int
+	y     *big.Int
+}
+
+// NewMultiset returns an empty multiset. The hash of an empty set
+// is the 32 byte value of zero.
+func NewMultiset(curve *KoblitzCurve) *Multiset {
+	return &Multiset{curve: curve, x: big.NewInt(0), y: big.NewInt(0)}
+}
+
+// NewMultisetFromPoint initializes a new multiset with the given x, y
+// coordinate.
+func NewMultisetFromPoint(curve *KoblitzCurve, x, y *big.Int) *Multiset {
+	var copyX, copyY big.Int
+	if x != nil {
+		copyX.Set(x)
+	}
+	if y != nil {
+		copyY.Set(y)
+	}
+	return &Multiset{curve: curve, x: &copyX, y: &copyY}
+}
+
+// NewMultisetFromDataSlice gets a curve and a slice of byte
+// slices, creates an empty multiset, hashes each data and
+// add it to the multiset, and return the resulting multiset.
+func NewMultisetFromDataSlice(curve *KoblitzCurve, datas [][]byte) *Multiset {
+	ms := NewMultiset(curve)
+	for _, data := range datas {
+		x, y := hashToPoint(curve, data)
+		ms.addPoint(x, y)
+	}
+	return ms
+}
+
+// Clone returns a clone of this multiset.
+func (ms *Multiset) Clone() *Multiset {
+	return NewMultisetFromPoint(ms.curve, ms.x, ms.y)
+}
+
+// Add hashes the data onto the curve and returns
+// a multiset with the new resulting point.
+func (ms *Multiset) Add(data []byte) *Multiset {
+	newMs := ms.Clone()
+	x, y := hashToPoint(ms.curve, data)
+	newMs.addPoint(x, y)
+	return newMs
+}
+
+func (ms *Multiset) addPoint(x, y *big.Int) {
+	ms.x, ms.y = ms.curve.Add(ms.x, ms.y, x, y)
+}
+
+// Remove hashes the data onto the curve, subtracts
+// the point from the existing multiset, and returns
+// a multiset with the new point. This function
+// will execute regardless of whether or not the passed
+// data was previously added to the set. Hence if you
+// remove an element that was never added and also remove
+// all the elements that were added, you will not get
+// back to the point at infinity (empty set).
+func (ms *Multiset) Remove(data []byte) *Multiset {
+	newMs := ms.Clone()
+	x, y := hashToPoint(ms.curve, data)
+	newMs.removePoint(x, y)
+	return newMs
+}
+
+func (ms *Multiset) removePoint(x, y *big.Int) {
+	y.Neg(y).Mod(y, ms.curve.P)
+	ms.x, ms.y = ms.curve.Add(ms.x, ms.y, x, y)
+}
+
+// Union will add the point of the passed multiset instance to the point
+// of this multiset and will return a multiset with the resulting point.
+func (ms *Multiset) Union(otherMultiset *Multiset) *Multiset {
+	newMs := ms.Clone()
+	otherMsCopy := otherMultiset.Clone()
+	newMs.addPoint(otherMsCopy.x, otherMsCopy.y)
+	return newMs
+}
+
+// Subtract will remove the point of the passed multiset instance from the point
+// of this multiset and will return a multiset with the resulting point.
+func (ms *Multiset) Subtract(otherMultiset *Multiset) *Multiset {
+	newMs := ms.Clone()
+	otherMsCopy := otherMultiset.Clone()
+	newMs.removePoint(otherMsCopy.x, otherMsCopy.y)
+	return newMs
+}
+
+// Hash serializes and returns the hash of the multiset. The hash of an empty
+// set is the 32 byte value of zero. The hash of a non-empty multiset is the
+// sha256 hash of the 32 byte x value concatenated with the 32 byte y value.
+func (ms *Multiset) Hash() *daghash.Hash {
+	if ms.x.Sign() == 0 && ms.y.Sign() == 0 {
+		return &daghash.Hash{}
+	}
+
+	hash := sha256.Sum256(append(ms.x.Bytes(), ms.y.Bytes()...))
+	castHash := daghash.Hash(hash)
+	return &castHash
+}
+
+// Point returns a copy of the x and y coordinates of the current multiset state.
+func (ms *Multiset) Point() (x *big.Int, y *big.Int) {
+	var copyX, copyY big.Int
+	copyX.Set(ms.x)
+	copyY.Set(ms.y)
+	return &copyX, &copyY
+}
+
+// hashToPoint hashes the passed data into a point on the curve. The x value
+// is sha256(n, sha256(data)) where n starts at zero. If the resulting x value
+// is not in the field or x^3+7 is not quadratic residue then n is incremented
+// and we try again. There is a 50% chance of success for any given iteration.
+func hashToPoint(curve *KoblitzCurve, data []byte) (x *big.Int, y *big.Int) {
+	i := uint64(0)
+	var err error
+	h := sha256.Sum256(data)
+	n := make([]byte, 8)
+	for {
+		binary.LittleEndian.PutUint64(n, i)
+		h2 := sha256.Sum256(append(n, h[:]...))
+
+		x = new(big.Int).SetBytes(h2[:])
+
+		y, err = decompressPoint(curve, x, false)
+		if err == nil && x.Cmp(curve.N) < 0 {
+			break
+		}
+		i++
+	}
+	return x, y
+}

btcec/ecmh_test.go

@@ -0,0 +1,213 @@
+package btcec
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+var testVectors = []struct {
+	dataElementHex string
+	point          [2]string
+	ecmhHash       string
+	cumulativeHash string
+}{
+	{
+		"982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e00000000010000000100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac",
+		[2]string{"4f9a5dce69067bf28603e73a7af4c3650b16539b95bad05eee95dfc94d1efe2c", "346d5b777881f2729e7f89b2de4e8e79c7f2f42d1a0b25a8f10becb66e2d0f98"},
+		"9378d88aa60cfba3032cb19f27891886e26fc6de1afa340c1787a633591983f8",
+		"",
+	},
+	{
+		"d5fdcc541e25de1c7a5addedf24858b8bb665c9f36ef744ee42c316022c90f9b00000000020000000100f2052a010000004341047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac",
+		[2]string{"68cf91eb2388a0287c13d46011c73fb8efb6be89c0867a47feccb2d11c390d2d", "f42ba72b1079d3d941881836f88b5dcd7c207a6a4839f129272c77ebb7194d42"},
+		"e2f3dc6f3aa867c50bd41b80aa3bdafcc9e1d13a6292ff8a5da95da123d185ef",
+		"afaa1f7ba0bd8a789422fdd6968639a4b8575baf7d54342a987073d038fdbafa",
+	},
+	{
+		"44f672226090d85db9a9f2fbfe5f0f9609b387af7be5b7fbb7a1767c831c9e9900000000030000000100f2052a0100000043410494b9d3e76c5b1629ecf97fff95d7a4bbdac87cc26099ada28066c6ff1eb9191223cd897194a08d0c2726c5747f1db49e8cf90e75dc3e3550ae9b30086f3cd5aaac",
+		[2]string{"359c6f59859d1d5af8e7081905cb6bb734c010be8680c14b5a89ee315694fc2b", "fb6ba531d4bd83b14c970ad1bec332a8ae9a05706cd5df7fd91a2f2cc32482fe"},
+		"ffed6804617a4a33b1037cdd26426e61fde0faa2c0cc045efffa17c00ff4adcf",
+		"e236a694532be6a4926ab8d5b1ff9cbfe638178e0008b0a8c5e87c3da2cdbc1c",
+	},
+}
+
+func TestHashToPoint(t *testing.T) {
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		x, y := hashToPoint(S256(), data)
+		if hex.EncodeToString(x.Bytes()) != test.point[0] || hex.EncodeToString(y.Bytes()) != test.point[1] {
+			t.Fatal("hashToPoint return incorrect point")
+		}
+	}
+}
+
+func TestMultiset_Hash(t *testing.T) {
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		x, y := hashToPoint(S256(), data)
+		m := NewMultisetFromPoint(S256(), x, y)
+		if m.Hash().String() != test.ecmhHash {
+			t.Fatal("Multiset-Hash returned incorrect hash serialization")
+		}
+	}
+	m := NewMultiset(S256())
+	emptySet := m.Hash()
+	zeroHash := daghash.Hash{}
+	if !bytes.Equal(emptySet[:], zeroHash[:]) {
+		t.Fatal("Empty set did not return zero hash")
+	}
+}
+
+func TestMultiset_AddRemove(t *testing.T) {
+	m := NewMultiset(S256())
+	for i, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m = m.Add(data)
+		if test.cumulativeHash != "" && m.Hash().String() != test.cumulativeHash {
+			t.Fatalf("Test #%d: Multiset-Add returned incorrect hash. Expected %s but got %s", i, test.cumulativeHash, m.Hash())
+		}
+	}
+
+	for i := len(testVectors) - 1; i > 0; i-- {
+		data, err := hex.DecodeString(testVectors[i].dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m = m.Remove(data)
+		if testVectors[i-1].cumulativeHash != "" && m.Hash().String() != testVectors[i-1].cumulativeHash {
+			t.Fatalf("Test #%d: Multiset-Remove returned incorrect hash. Expected %s but got %s", i, testVectors[i].cumulativeHash, m.Hash())
+		}
+	}
+}
+
+func TestMultiset_UnionSubtract(t *testing.T) {
+	m1 := NewMultiset(S256())
+	zeroHash := m1.Hash().String()
+
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m1 = m1.Add(data)
+	}
+
+	m2 := NewMultiset(S256())
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m2 = m2.Remove(data)
+	}
+	m1m2Union := m1.Union(m2)
+	if m1m2Union.Hash().String() != zeroHash {
+		t.Fatalf("m1m2Union was expected to return to have zero hash, but was %s instead", m1m2Union.Hash())
+	}
+
+	m1Inverse := NewMultiset(S256()).Subtract(m1)
+	m1InverseM1Union := m1.Union(m1Inverse)
+	if m1InverseM1Union.Hash().String() != zeroHash {
+		t.Fatalf("m1InverseM1Union was expected to have zero hash, but got %s instead", m1InverseM1Union.Hash())
+	}
+
+	m1SubtractM1 := m1.Subtract(m1)
+	if m1SubtractM1.Hash().String() != zeroHash {
+		t.Fatalf("m1SubtractM1 was expected to have zero hash, but got %s instead", m1SubtractM1.Hash())
+	}
+}
+
+func TestMultiset_Commutativity(t *testing.T) {
+	m := NewMultiset(S256())
+	zeroHash := m.Hash().String()
+
+	// Check that if we subtract values from zero and then re-add them, we return to zero.
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m = m.Remove(data)
+	}
+
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m = m.Add(data)
+	}
+	if m.Hash().String() != zeroHash {
+		t.Fatalf("m was expected to be zero hash, but was %s instead", m.Hash())
+	}
+
+	// Here we first remove an element from an empty multiset, and then add some other
+	// elements, and then we create a new empty multiset, then we add the same elements
+	// we added to the previous multiset, and then we remove the same element we remove
+	// the same element we removed from the previous multiset. According to commutativity
+	// laws, the result should be the same.
+	removeIndex := 0
+	removeData, err := hex.DecodeString(testVectors[removeIndex].dataElementHex)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m1 := NewMultiset(S256())
+	m1 = m1.Remove(removeData)
+
+	for i, test := range testVectors {
+		if i != removeIndex {
+			data, err := hex.DecodeString(test.dataElementHex)
+			if err != nil {
+				t.Fatal(err)
+			}
+			m1 = m1.Add(data)
+		}
+	}
+
+	m2 := NewMultiset(S256())
+	for i, test := range testVectors {
+		if i != removeIndex {
+			data, err := hex.DecodeString(test.dataElementHex)
+			if err != nil {
+				t.Fatal(err)
+			}
+			m2 = m2.Add(data)
+		}
+	}
+	m2 = m2.Remove(removeData)
+
+	if m1.Hash().String() != m2.Hash().String() {
+		t.Fatalf("m1 and m2 was exepcted to have the same hash, but got instead m1 %s and m2 %s", m1.Hash(), m2.Hash())
+	}
+}
+
+func TestMultiset_NewMultisetFromDataSlice(t *testing.T) {
+	m1 := NewMultiset(S256())
+	datas := make([][]byte, 0, len(testVectors))
+	for _, test := range testVectors {
+		data, err := hex.DecodeString(test.dataElementHex)
+		if err != nil {
+			t.Fatal(err)
+		}
+		datas = append(datas, data)
+		m1 = m1.Add(data)
+	}
+
+	m2 := NewMultisetFromDataSlice(S256(), datas)
+	if m1.Hash().String() != m2.Hash().String() {
+		t.Fatalf("m1 and m2 was exepcted to have the same hash, but got instead m1 %s and m2 %s", m1.Hash(), m2.Hash())
+	}
+}