[NOD-1105] When recovering acceptance index, use a database transaction per block instead of for the entire recovery (#781)

* [NOD-1105] Don't use a database transaction when recovering acceptance index.

* Revert "[NOD-1105] Don't use a database transaction when recovering acceptance index."

This reverts commit da550f8e

* [NOD-1105] When recovering acceptance index, use a database transaction per block instead of for the entire recovery.

addrmgr/addrmanager.go

@@ -5,16 +5,16 @@
 package addrmgr
 
 import (
+	"bytes"
 	"container/list"
 	crand "crypto/rand" // for seeding
 	"encoding/binary"
-	"encoding/json"
+	"encoding/gob"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/pkg/errors"
 	"io"
 	"math/rand"
 	"net"
-	"os"
-	"path/filepath"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -26,14 +26,13 @@ import (
 	"github.com/kaspanet/kaspad/wire"
 )
 
-type newBucket [newBucketCount]map[string]*KnownAddress
-type triedBucket [triedBucketCount]*list.List
+type newBucket [NewBucketCount]map[string]*KnownAddress
+type triedBucket [TriedBucketCount]*list.List
 
 // AddrManager provides a concurrency safe address manager for caching potential
 // peers on the Kaspa network.
 type AddrManager struct {
 	mtx                sync.Mutex
-	peersFile          string
 	lookupFunc         func(string) ([]net.IP, error)
 	rand               *rand.Rand
 	key                [32]byte
@@ -66,10 +65,12 @@ type serializedKnownAddress struct {
 	// no refcount or tried, that is available from context.
 }
 
-type serializedNewBucket [newBucketCount][]string
-type serializedTriedBucket [triedBucketCount][]string
+type serializedNewBucket [NewBucketCount][]string
+type serializedTriedBucket [TriedBucketCount][]string
 
-type serializedAddrManager struct {
+// PeersStateForSerialization is the data model that is used to
+// serialize the peers state to any encoding.
+type PeersStateForSerialization struct {
 	Version              int
 	Key                  [32]byte
 	Addresses            []*serializedKnownAddress
@@ -118,17 +119,17 @@ const (
 	// tried address bucket.
 	triedBucketSize = 256
 
-	// triedBucketCount is the number of buckets we split tried
+	// TriedBucketCount is the number of buckets we split tried
 	// addresses over.
-	triedBucketCount = 64
+	TriedBucketCount = 64
 
 	// newBucketSize is the maximum number of addresses in each new address
 	// bucket.
 	newBucketSize = 64
 
-	// newBucketCount is the number of buckets that we spread new addresses
+	// NewBucketCount is the number of buckets that we spread new addresses
 	// over.
-	newBucketCount = 1024
+	NewBucketCount = 1024
 
 	// triedBucketsPerGroup is the number of tried buckets over which an
 	// address group will be spread.
@@ -162,17 +163,17 @@ const (
 	// to a getAddr. If we have less than this amount, we send everything.
 	getAddrMin = 50
 
-	// getAddrMax is the most addresses that we will send in response
+	// GetAddrMax is the most addresses that we will send in response
 	// to a getAddr (in practise the most addresses we will return from a
 	// call to AddressCache()).
-	getAddrMax = 2500
+	GetAddrMax = 2500
 
 	// getAddrPercent is the percentage of total addresses known that we
 	// will share with a call to AddressCache.
 	getAddrPercent = 23
 
-	// serialisationVersion is the current version of the on-disk format.
-	serialisationVersion = 1
+	// serializationVersion is the current version of the on-disk format.
+	serializationVersion = 1
 )
 
 // updateAddress is a helper function to either update an address already known
@@ -392,7 +393,7 @@ func (a *AddrManager) getNewBucket(netAddr, srcAddr *wire.NetAddress) int {
 	data2 = append(data2, hashbuf[:]...)
 
 	hash2 := daghash.DoubleHashB(data2)
-	return int(binary.LittleEndian.Uint64(hash2) % newBucketCount)
+	return int(binary.LittleEndian.Uint64(hash2) % NewBucketCount)
 }
 
 func (a *AddrManager) getTriedBucket(netAddr *wire.NetAddress) int {
@@ -411,7 +412,7 @@ func (a *AddrManager) getTriedBucket(netAddr *wire.NetAddress) int {
 	data2 = append(data2, hashbuf[:]...)
 
 	hash2 := daghash.DoubleHashB(data2)
-	return int(binary.LittleEndian.Uint64(hash2) % triedBucketCount)
+	return int(binary.LittleEndian.Uint64(hash2) % TriedBucketCount)
 }
 
 // addressHandler is the main handler for the address manager. It must be run
@@ -423,30 +424,62 @@ out:
 	for {
 		select {
 		case <-dumpAddressTicker.C:
-			a.savePeers()
+			err := a.savePeers()
+			if err != nil {
+				panic(errors.Wrap(err, "error saving peers"))
+			}
 
 		case <-a.quit:
 			break out
 		}
 	}
-	a.savePeers()
+	err := a.savePeers()
+	if err != nil {
+		panic(errors.Wrap(err, "error saving peers"))
+	}
 	a.wg.Done()
 	log.Trace("Address handler done")
 }
 
-// savePeers saves all the known addresses to a file so they can be read back
+// savePeers saves all the known addresses to the database so they can be read back
 // in at next run.
-func (a *AddrManager) savePeers() {
+func (a *AddrManager) savePeers() error {
+	serializedPeersState, err := a.serializePeersState()
+	if err != nil {
+		return err
+	}
+
+	return dbaccess.StorePeersState(dbaccess.NoTx(), serializedPeersState)
+}
+
+func (a *AddrManager) serializePeersState() ([]byte, error) {
+	peersState, err := a.PeersStateForSerialization()
+	if err != nil {
+		return nil, err
+	}
+
+	w := &bytes.Buffer{}
+	encoder := gob.NewEncoder(w)
+	err = encoder.Encode(&peersState)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to encode peers state")
+	}
+
+	return w.Bytes(), nil
+}
+
+// PeersStateForSerialization returns the data model that is used to serialize the peers state to any encoding.
+func (a *AddrManager) PeersStateForSerialization() (*PeersStateForSerialization, error) {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 
-	// First we make a serialisable datastructure so we can encode it to
-	// json.
-	sam := new(serializedAddrManager)
-	sam.Version = serialisationVersion
-	copy(sam.Key[:], a.key[:])
+	// First we make a serializable data structure so we can encode it to
+	// gob.
+	peersState := new(PeersStateForSerialization)
+	peersState.Version = serializationVersion
+	copy(peersState.Key[:], a.key[:])
 
-	sam.Addresses = make([]*serializedKnownAddress, len(a.addrIndex))
+	peersState.Addresses = make([]*serializedKnownAddress, len(a.addrIndex))
 	i := 0
 	for k, v := range a.addrIndex {
 		ska := new(serializedKnownAddress)
@@ -463,119 +496,104 @@ func (a *AddrManager) savePeers() {
 		ska.LastSuccess = v.lastsuccess.Unix()
 		// Tried and refs are implicit in the rest of the structure
 		// and will be worked out from context on unserialisation.
-		sam.Addresses[i] = ska
+		peersState.Addresses[i] = ska
 		i++
 	}
 
-	sam.NewBuckets = make(map[string]*serializedNewBucket)
+	peersState.NewBuckets = make(map[string]*serializedNewBucket)
 	for subnetworkID := range a.addrNew {
 		subnetworkIDStr := subnetworkID.String()
-		sam.NewBuckets[subnetworkIDStr] = &serializedNewBucket{}
+		peersState.NewBuckets[subnetworkIDStr] = &serializedNewBucket{}
 
 		for i := range a.addrNew[subnetworkID] {
-			sam.NewBuckets[subnetworkIDStr][i] = make([]string, len(a.addrNew[subnetworkID][i]))
+			peersState.NewBuckets[subnetworkIDStr][i] = make([]string, len(a.addrNew[subnetworkID][i]))
 			j := 0
 			for k := range a.addrNew[subnetworkID][i] {
-				sam.NewBuckets[subnetworkIDStr][i][j] = k
+				peersState.NewBuckets[subnetworkIDStr][i][j] = k
 				j++
 			}
 		}
 	}
 
 	for i := range a.addrNewFullNodes {
-		sam.NewBucketFullNodes[i] = make([]string, len(a.addrNewFullNodes[i]))
+		peersState.NewBucketFullNodes[i] = make([]string, len(a.addrNewFullNodes[i]))
 		j := 0
 		for k := range a.addrNewFullNodes[i] {
-			sam.NewBucketFullNodes[i][j] = k
+			peersState.NewBucketFullNodes[i][j] = k
 			j++
 		}
 	}
 
-	sam.TriedBuckets = make(map[string]*serializedTriedBucket)
+	peersState.TriedBuckets = make(map[string]*serializedTriedBucket)
 	for subnetworkID := range a.addrTried {
 		subnetworkIDStr := subnetworkID.String()
-		sam.TriedBuckets[subnetworkIDStr] = &serializedTriedBucket{}
+		peersState.TriedBuckets[subnetworkIDStr] = &serializedTriedBucket{}
 
 		for i := range a.addrTried[subnetworkID] {
-			sam.TriedBuckets[subnetworkIDStr][i] = make([]string, a.addrTried[subnetworkID][i].Len())
+			peersState.TriedBuckets[subnetworkIDStr][i] = make([]string, a.addrTried[subnetworkID][i].Len())
 			j := 0
 			for e := a.addrTried[subnetworkID][i].Front(); e != nil; e = e.Next() {
 				ka := e.Value.(*KnownAddress)
-				sam.TriedBuckets[subnetworkIDStr][i][j] = NetAddressKey(ka.na)
+				peersState.TriedBuckets[subnetworkIDStr][i][j] = NetAddressKey(ka.na)
 				j++
 			}
 		}
 	}
 
 	for i := range a.addrTriedFullNodes {
-		sam.TriedBucketFullNodes[i] = make([]string, a.addrTriedFullNodes[i].Len())
+		peersState.TriedBucketFullNodes[i] = make([]string, a.addrTriedFullNodes[i].Len())
 		j := 0
 		for e := a.addrTriedFullNodes[i].Front(); e != nil; e = e.Next() {
 			ka := e.Value.(*KnownAddress)
-			sam.TriedBucketFullNodes[i][j] = NetAddressKey(ka.na)
+			peersState.TriedBucketFullNodes[i][j] = NetAddressKey(ka.na)
 			j++
 		}
 	}
 
-	w, err := os.Create(a.peersFile)
-	if err != nil {
-		log.Errorf("Error opening file %s: %s", a.peersFile, err)
-		return
-	}
-	enc := json.NewEncoder(w)
-	defer w.Close()
-	if err := enc.Encode(&sam); err != nil {
-		log.Errorf("Failed to encode file %s: %s", a.peersFile, err)
-		return
-	}
+	return peersState, nil
 }
 
-// loadPeers loads the known address from the saved file. If empty, missing, or
-// malformed file, just don't load anything and start fresh
-func (a *AddrManager) loadPeers() {
+// loadPeers loads the known address from the database. If missing,
+// just don't load anything and start fresh.
+func (a *AddrManager) loadPeers() error {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
 
-	err := a.deserializePeers(a.peersFile)
-	if err != nil {
-		log.Errorf("Failed to parse file %s: %s", a.peersFile, err)
-		// if it is invalid we nuke the old one unconditionally.
-		err = os.Remove(a.peersFile)
-		if err != nil {
-			log.Warnf("Failed to remove corrupt peers file %s: %s",
-				a.peersFile, err)
-		}
+	serializedPeerState, err := dbaccess.FetchPeersState(dbaccess.NoTx())
+	if dbaccess.IsNotFoundError(err) {
 		a.reset()
-		return
-	}
-	log.Infof("Loaded %d addresses from file '%s'", a.totalNumAddresses(), a.peersFile)
-}
-
-func (a *AddrManager) deserializePeers(filePath string) error {
-	_, err := os.Stat(filePath)
-	if os.IsNotExist(err) {
+		log.Info("No peers state was found in the database. Created a new one", a.totalNumAddresses())
 		return nil
 	}
-	r, err := os.Open(filePath)
 	if err != nil {
-		return errors.Errorf("%s error opening file: %s", filePath, err)
-	}
-	defer r.Close()
-
-	var sam serializedAddrManager
-	dec := json.NewDecoder(r)
-	err = dec.Decode(&sam)
-	if err != nil {
-		return errors.Errorf("error reading %s: %s", filePath, err)
+		return err
 	}
 
-	if sam.Version != serialisationVersion {
+	err = a.deserializePeersState(serializedPeerState)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Loaded %d addresses from database", a.totalNumAddresses())
+	return nil
+}
+
+func (a *AddrManager) deserializePeersState(serializedPeerState []byte) error {
+	var peersState PeersStateForSerialization
+	r := bytes.NewBuffer(serializedPeerState)
+	dec := gob.NewDecoder(r)
+	err := dec.Decode(&peersState)
+	if err != nil {
+		return errors.Wrap(err, "error deserializing peers state")
+	}
+
+	if peersState.Version != serializationVersion {
 		return errors.Errorf("unknown version %d in serialized "+
-			"addrmanager", sam.Version)
+			"peers state", peersState.Version)
 	}
-	copy(a.key[:], sam.Key[:])
+	copy(a.key[:], peersState.Key[:])
 
-	for _, v := range sam.Addresses {
+	for _, v := range peersState.Addresses {
 		ka := new(KnownAddress)
 		ka.na, err = a.DeserializeNetAddress(v.Addr)
 		if err != nil {
@@ -600,12 +618,12 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		a.addrIndex[NetAddressKey(ka.na)] = ka
 	}
 
-	for subnetworkIDStr := range sam.NewBuckets {
+	for subnetworkIDStr := range peersState.NewBuckets {
 		subnetworkID, err := subnetworkid.NewFromStr(subnetworkIDStr)
 		if err != nil {
 			return err
 		}
-		for i, subnetworkNewBucket := range sam.NewBuckets[subnetworkIDStr] {
+		for i, subnetworkNewBucket := range peersState.NewBuckets[subnetworkIDStr] {
 			for _, val := range subnetworkNewBucket {
 				ka, ok := a.addrIndex[val]
 				if !ok {
@@ -622,7 +640,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		}
 	}
 
-	for i, newBucket := range sam.NewBucketFullNodes {
+	for i, newBucket := range peersState.NewBucketFullNodes {
 		for _, val := range newBucket {
 			ka, ok := a.addrIndex[val]
 			if !ok {
@@ -638,12 +656,12 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		}
 	}
 
-	for subnetworkIDStr := range sam.TriedBuckets {
+	for subnetworkIDStr := range peersState.TriedBuckets {
 		subnetworkID, err := subnetworkid.NewFromStr(subnetworkIDStr)
 		if err != nil {
 			return err
 		}
-		for i, subnetworkTriedBucket := range sam.TriedBuckets[subnetworkIDStr] {
+		for i, subnetworkTriedBucket := range peersState.TriedBuckets[subnetworkIDStr] {
 			for _, val := range subnetworkTriedBucket {
 				ka, ok := a.addrIndex[val]
 				if !ok {
@@ -658,7 +676,7 @@ func (a *AddrManager) deserializePeers(filePath string) error {
 		}
 	}
 
-	for i, triedBucket := range sam.TriedBucketFullNodes {
+	for i, triedBucket := range peersState.TriedBucketFullNodes {
 		for _, val := range triedBucket {
 			ka, ok := a.addrIndex[val]
 			if !ok {
@@ -704,20 +722,24 @@ func (a *AddrManager) DeserializeNetAddress(addr string) (*wire.NetAddress, erro
 
 // Start begins the core address handler which manages a pool of known
 // addresses, timeouts, and interval based writes.
-func (a *AddrManager) Start() {
+func (a *AddrManager) Start() error {
 	// Already started?
 	if atomic.AddInt32(&a.started, 1) != 1 {
-		return
+		return nil
 	}
 
 	log.Trace("Starting address manager")
 
-	// Load peers we already know about from file.
-	a.loadPeers()
+	// Load peers we already know about from the database.
+	err := a.loadPeers()
+	if err != nil {
+		return err
+	}
 
 	// Start the address ticker to save addresses periodically.
 	a.wg.Add(1)
 	spawn(a.addressHandler)
+	return nil
 }
 
 // Stop gracefully shuts down the address manager by stopping the main handler.
@@ -839,8 +861,8 @@ func (a *AddrManager) AddressCache(includeAllSubnetworks bool, subnetworkID *sub
 	}
 
 	numAddresses := len(allAddr) * getAddrPercent / 100
-	if numAddresses > getAddrMax {
-		numAddresses = getAddrMax
+	if numAddresses > GetAddrMax {
+		numAddresses = GetAddrMax
 	}
 	if len(allAddr) < getAddrMin {
 		numAddresses = len(allAddr)
@@ -1333,9 +1355,8 @@ func (a *AddrManager) GetBestLocalAddress(remoteAddr *wire.NetAddress) *wire.Net
 
 // New returns a new Kaspa address manager.
 // Use Start to begin processing asynchronous address updates.
-func New(dataDir string, lookupFunc func(string) ([]net.IP, error), subnetworkID *subnetworkid.SubnetworkID) *AddrManager {
+func New(lookupFunc func(string) ([]net.IP, error), subnetworkID *subnetworkid.SubnetworkID) *AddrManager {
 	am := AddrManager{
-		peersFile:         filepath.Join(dataDir, "peers.json"),
 		lookupFunc:        lookupFunc,
 		rand:              rand.New(rand.NewSource(time.Now().UnixNano())),
 		quit:              make(chan struct{}),

addrmgr/addrmanager_test.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"github.com/kaspanet/kaspad/config"
 	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util/subnetworkid"
+	"io/ioutil"
 	"net"
 	"reflect"
 	"testing"
@@ -101,14 +103,41 @@ func addNaTest(ip string, port uint16, want string) {
 	naTests = append(naTests, test)
 }
 
-func lookupFunc(host string) ([]net.IP, error) {
+func lookupFuncForTest(host string) ([]net.IP, error) {
 	return nil, errors.New("not implemented")
 }
 
+func newAddrManagerForTest(t *testing.T, testName string,
+	localSubnetworkID *subnetworkid.SubnetworkID) (addressManager *AddrManager, teardown func()) {
+
+	dbPath, err := ioutil.TempDir("", testName)
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %s", err)
+	}
+
+	err = dbaccess.Open(dbPath)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	addressManager = New(lookupFuncForTest, localSubnetworkID)
+
+	return addressManager, func() {
+		err := dbaccess.Close()
+		if err != nil {
+			t.Fatalf("error closing the database: %s", err)
+		}
+	}
+}
+
 func TestStartStop(t *testing.T) {
-	n := New("teststartstop", lookupFunc, nil)
-	n.Start()
-	err := n.Stop()
+	amgr, teardown := newAddrManagerForTest(t, "TestStartStop", nil)
+	defer teardown()
+	err := amgr.Start()
+	if err != nil {
+		t.Fatalf("Address Manager failed to start: %v", err)
+	}
+	err = amgr.Stop()
 	if err != nil {
 		t.Fatalf("Address Manager failed to stop: %v", err)
 	}
@@ -148,7 +177,8 @@ func TestAddAddressByIP(t *testing.T) {
 		},
 	}
 
-	amgr := New("testaddressbyip", nil, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestAddAddressByIP", nil)
+	defer teardown()
 	for i, test := range tests {
 		err := amgr.AddAddressByIP(test.addrIP, nil)
 		if test.err != nil && err == nil {
@@ -213,7 +243,8 @@ func TestAddLocalAddress(t *testing.T) {
 			true,
 		},
 	}
-	amgr := New("testaddlocaladdress", nil, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestAddLocalAddress", nil)
+	defer teardown()
 	for x, test := range tests {
 		result := amgr.AddLocalAddress(&test.address, test.priority)
 		if result == nil && !test.valid {
@@ -239,21 +270,22 @@ func TestAttempt(t *testing.T) {
 	})
 	defer config.SetActiveConfig(originalActiveCfg)
 
-	n := New("testattempt", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestAttempt", nil)
+	defer teardown()
 
 	// Add a new address and get it
-	err := n.AddAddressByIP(someIP+":8333", nil)
+	err := amgr.AddAddressByIP(someIP+":8333", nil)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka := n.GetAddress()
+	ka := amgr.GetAddress()
 
 	if !ka.LastAttempt().IsZero() {
 		t.Errorf("Address should not have attempts, but does")
 	}
 
 	na := ka.NetAddress()
-	n.Attempt(na)
+	amgr.Attempt(na)
 
 	if ka.LastAttempt().IsZero() {
 		t.Errorf("Address should have an attempt, but does not")
@@ -270,19 +302,20 @@ func TestConnected(t *testing.T) {
 	})
 	defer config.SetActiveConfig(originalActiveCfg)
 
-	n := New("testconnected", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestConnected", nil)
+	defer teardown()
 
 	// Add a new address and get it
-	err := n.AddAddressByIP(someIP+":8333", nil)
+	err := amgr.AddAddressByIP(someIP+":8333", nil)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka := n.GetAddress()
+	ka := amgr.GetAddress()
 	na := ka.NetAddress()
 	// make it an hour ago
 	na.Timestamp = time.Unix(time.Now().Add(time.Hour*-1).Unix(), 0)
 
-	n.Connected(na)
+	amgr.Connected(na)
 
 	if !ka.NetAddress().Timestamp.After(na.Timestamp) {
 		t.Errorf("Address should have a new timestamp, but does not")
@@ -299,9 +332,10 @@ func TestNeedMoreAddresses(t *testing.T) {
 	})
 	defer config.SetActiveConfig(originalActiveCfg)
 
-	n := New("testneedmoreaddresses", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestNeedMoreAddresses", nil)
+	defer teardown()
 	addrsToAdd := 1500
-	b := n.NeedMoreAddresses()
+	b := amgr.NeedMoreAddresses()
 	if !b {
 		t.Errorf("Expected that we need more addresses")
 	}
@@ -310,7 +344,7 @@ func TestNeedMoreAddresses(t *testing.T) {
 	var err error
 	for i := 0; i < addrsToAdd; i++ {
 		s := fmt.Sprintf("%d.%d.173.147:8333", i/128+60, i%128+60)
-		addrs[i], err = n.DeserializeNetAddress(s)
+		addrs[i], err = amgr.DeserializeNetAddress(s)
 		if err != nil {
 			t.Errorf("Failed to turn %s into an address: %v", s, err)
 		}
@@ -318,13 +352,13 @@ func TestNeedMoreAddresses(t *testing.T) {
 
 	srcAddr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 
-	n.AddAddresses(addrs, srcAddr, nil)
-	numAddrs := n.TotalNumAddresses()
+	amgr.AddAddresses(addrs, srcAddr, nil)
+	numAddrs := amgr.TotalNumAddresses()
 	if numAddrs > addrsToAdd {
 		t.Errorf("Number of addresses is too many %d vs %d", numAddrs, addrsToAdd)
 	}
 
-	b = n.NeedMoreAddresses()
+	b = amgr.NeedMoreAddresses()
 	if b {
 		t.Errorf("Expected that we don't need more addresses")
 	}
@@ -340,7 +374,8 @@ func TestGood(t *testing.T) {
 	})
 	defer config.SetActiveConfig(originalActiveCfg)
 
-	n := New("testgood", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestGood", nil)
+	defer teardown()
 	addrsToAdd := 64 * 64
 	addrs := make([]*wire.NetAddress, addrsToAdd)
 	subnetworkCount := 32
@@ -349,7 +384,7 @@ func TestGood(t *testing.T) {
 	var err error
 	for i := 0; i < addrsToAdd; i++ {
 		s := fmt.Sprintf("%d.173.147.%d:8333", i/64+60, i%64+60)
-		addrs[i], err = n.DeserializeNetAddress(s)
+		addrs[i], err = amgr.DeserializeNetAddress(s)
 		if err != nil {
 			t.Errorf("Failed to turn %s into an address: %v", s, err)
 		}
@@ -361,24 +396,24 @@ func TestGood(t *testing.T) {
 
 	srcAddr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 
-	n.AddAddresses(addrs, srcAddr, nil)
+	amgr.AddAddresses(addrs, srcAddr, nil)
 	for i, addr := range addrs {
-		n.Good(addr, subnetworkIDs[i%subnetworkCount])
+		amgr.Good(addr, subnetworkIDs[i%subnetworkCount])
 	}
 
-	numAddrs := n.TotalNumAddresses()
+	numAddrs := amgr.TotalNumAddresses()
 	if numAddrs >= addrsToAdd {
 		t.Errorf("Number of addresses is too many: %d vs %d", numAddrs, addrsToAdd)
 	}
 
-	numCache := len(n.AddressCache(true, nil))
+	numCache := len(amgr.AddressCache(true, nil))
 	if numCache == 0 || numCache >= numAddrs/4 {
 		t.Errorf("Number of addresses in cache: got %d, want positive and less than %d",
 			numCache, numAddrs/4)
 	}
 
 	for i := 0; i < subnetworkCount; i++ {
-		numCache = len(n.AddressCache(false, subnetworkIDs[i]))
+		numCache = len(amgr.AddressCache(false, subnetworkIDs[i]))
 		if numCache == 0 || numCache >= numAddrs/subnetworkCount {
 			t.Errorf("Number of addresses in subnetwork cache: got %d, want positive and less than %d",
 				numCache, numAddrs/4/subnetworkCount)
@@ -396,17 +431,18 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	})
 	defer config.SetActiveConfig(originalActiveCfg)
 
-	n := New("test_good_change_subnetwork_id", lookupFunc, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestGoodChangeSubnetworkID", nil)
+	defer teardown()
 	addr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 	addrKey := NetAddressKey(addr)
 	srcAddr := wire.NewNetAddressIPPort(net.IPv4(173, 144, 173, 111), 8333, 0)
 
 	oldSubnetwork := subnetworkid.SubnetworkIDNative
-	n.AddAddress(addr, srcAddr, oldSubnetwork)
-	n.Good(addr, oldSubnetwork)
+	amgr.AddAddress(addr, srcAddr, oldSubnetwork)
+	amgr.Good(addr, oldSubnetwork)
 
 	// make sure address was saved to addrIndex under oldSubnetwork
-	ka := n.find(addr)
+	ka := amgr.find(addr)
 	if ka == nil {
 		t.Fatalf("Address was not found after first time .Good called")
 	}
@@ -415,7 +451,7 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	}
 
 	// make sure address was added to correct bucket under oldSubnetwork
-	bucket := n.addrTried[*oldSubnetwork][n.getTriedBucket(addr)]
+	bucket := amgr.addrTried[*oldSubnetwork][amgr.getTriedBucket(addr)]
 	wasFound := false
 	for e := bucket.Front(); e != nil; e = e.Next() {
 		if NetAddressKey(e.Value.(*KnownAddress).NetAddress()) == addrKey {
@@ -428,10 +464,10 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 
 	// now call .Good again with a different subnetwork
 	newSubnetwork := subnetworkid.SubnetworkIDRegistry
-	n.Good(addr, newSubnetwork)
+	amgr.Good(addr, newSubnetwork)
 
 	// make sure address was updated in addrIndex under newSubnetwork
-	ka = n.find(addr)
+	ka = amgr.find(addr)
 	if ka == nil {
 		t.Fatalf("Address was not found after second time .Good called")
 	}
@@ -440,7 +476,7 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	}
 
 	// make sure address was removed from bucket under oldSubnetwork
-	bucket = n.addrTried[*oldSubnetwork][n.getTriedBucket(addr)]
+	bucket = amgr.addrTried[*oldSubnetwork][amgr.getTriedBucket(addr)]
 	wasFound = false
 	for e := bucket.Front(); e != nil; e = e.Next() {
 		if NetAddressKey(e.Value.(*KnownAddress).NetAddress()) == addrKey {
@@ -452,7 +488,7 @@ func TestGoodChangeSubnetworkID(t *testing.T) {
 	}
 
 	// make sure address was added to correct bucket under newSubnetwork
-	bucket = n.addrTried[*newSubnetwork][n.getTriedBucket(addr)]
+	bucket = amgr.addrTried[*newSubnetwork][amgr.getTriedBucket(addr)]
 	wasFound = false
 	for e := bucket.Front(); e != nil; e = e.Next() {
 		if NetAddressKey(e.Value.(*KnownAddress).NetAddress()) == addrKey {
@@ -475,34 +511,35 @@ func TestGetAddress(t *testing.T) {
 	defer config.SetActiveConfig(originalActiveCfg)
 
 	localSubnetworkID := &subnetworkid.SubnetworkID{0xff}
-	n := New("testgetaddress", lookupFunc, localSubnetworkID)
+	amgr, teardown := newAddrManagerForTest(t, "TestGetAddress", localSubnetworkID)
+	defer teardown()
 
 	// Get an address from an empty set (should error)
-	if rv := n.GetAddress(); rv != nil {
+	if rv := amgr.GetAddress(); rv != nil {
 		t.Errorf("GetAddress failed: got: %v want: %v\n", rv, nil)
 	}
 
 	// Add a new address and get it
-	err := n.AddAddressByIP(someIP+":8332", localSubnetworkID)
+	err := amgr.AddAddressByIP(someIP+":8332", localSubnetworkID)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka := n.GetAddress()
+	ka := amgr.GetAddress()
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
-	n.Attempt(ka.NetAddress())
+	amgr.Attempt(ka.NetAddress())
 
 	// Checks that we don't get it if we find that it has other subnetwork ID than expected.
 	actualSubnetworkID := &subnetworkid.SubnetworkID{0xfe}
-	n.Good(ka.NetAddress(), actualSubnetworkID)
-	ka = n.GetAddress()
+	amgr.Good(ka.NetAddress(), actualSubnetworkID)
+	ka = amgr.GetAddress()
 	if ka != nil {
 		t.Errorf("Didn't expect to get an address because there shouldn't be any address from subnetwork ID %s or nil", localSubnetworkID)
 	}
 
 	// Checks that the total number of addresses incremented although the new address is not full node or a partial node of the same subnetwork as the local node.
-	numAddrs := n.TotalNumAddresses()
+	numAddrs := amgr.TotalNumAddresses()
 	if numAddrs != 1 {
 		t.Errorf("Wrong number of addresses: got %d, want %d", numAddrs, 1)
 	}
@@ -510,11 +547,11 @@ func TestGetAddress(t *testing.T) {
 	// Now we repeat the same process, but now the address has the expected subnetwork ID.
 
 	// Add a new address and get it
-	err = n.AddAddressByIP(someIP+":8333", localSubnetworkID)
+	err = amgr.AddAddressByIP(someIP+":8333", localSubnetworkID)
 	if err != nil {
 		t.Fatalf("Adding address failed: %v", err)
 	}
-	ka = n.GetAddress()
+	ka = amgr.GetAddress()
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
@@ -524,11 +561,11 @@ func TestGetAddress(t *testing.T) {
 	if !ka.SubnetworkID().IsEqual(localSubnetworkID) {
 		t.Errorf("Wrong Subnetwork ID: got %v, want %v", *ka.SubnetworkID(), localSubnetworkID)
 	}
-	n.Attempt(ka.NetAddress())
+	amgr.Attempt(ka.NetAddress())
 
 	// Mark this as a good address and get it
-	n.Good(ka.NetAddress(), localSubnetworkID)
-	ka = n.GetAddress()
+	amgr.Good(ka.NetAddress(), localSubnetworkID)
+	ka = amgr.GetAddress()
 	if ka == nil {
 		t.Fatalf("Did not get an address where there is one in the pool")
 	}
@@ -539,7 +576,7 @@ func TestGetAddress(t *testing.T) {
 		t.Errorf("Wrong Subnetwork ID: got %v, want %v", ka.SubnetworkID(), localSubnetworkID)
 	}
 
-	numAddrs = n.TotalNumAddresses()
+	numAddrs = amgr.TotalNumAddresses()
 	if numAddrs != 2 {
 		t.Errorf("Wrong number of addresses: got %d, want %d", numAddrs, 1)
 	}
@@ -604,7 +641,8 @@ func TestGetBestLocalAddress(t *testing.T) {
 		*/
 	}
 
-	amgr := New("testgetbestlocaladdress", nil, nil)
+	amgr, teardown := newAddrManagerForTest(t, "TestGetBestLocalAddress", nil)
+	defer teardown()
 
 	// Test against default when there's no address
 	for x, test := range tests {

blockdag/accept.go

@@ -6,7 +6,7 @@ package blockdag
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
 )
@@ -16,7 +16,17 @@ func (dag *BlockDAG) addNodeToIndexWithInvalidAncestor(block *util.Block) error
 	newNode, _ := dag.newBlockNode(blockHeader, newBlockSet())
 	newNode.status = statusInvalidAncestor
 	dag.index.AddNode(newNode)
-	return dag.index.flushToDB()
+
+	dbTx, err := dbaccess.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = dag.index.flushToDB(dbTx)
+	if err != nil {
+		return err
+	}
+	return dbTx.Commit()
 }
 
 // maybeAcceptBlock potentially accepts a block into the block DAG. It
@@ -62,13 +72,26 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	// expensive connection logic. It also has some other nice properties
 	// such as making blocks that never become part of the DAG or
 	// blocks that fail to connect available for further analysis.
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		err := dbStoreBlock(dbTx, block)
+	dbTx, err := dbaccess.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
+	blockExists, err := dbaccess.HasBlock(dbTx, block.Hash())
+	if err != nil {
+		return err
+	}
+	if !blockExists {
+		err := storeBlock(dbTx, block)
 		if err != nil {
 			return err
 		}
-		return dag.index.flushToDBWithTx(dbTx)
-	})
+	}
+	err = dag.index.flushToDB(dbTx)
+	if err != nil {
+		return err
+	}
+	err = dbTx.Commit()
 	if err != nil {
 		return err
 	}
@@ -82,8 +105,6 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 		}
 	}
 
-	block.SetBlueScore(newNode.blueScore)
-
 	// Connect the passed block to the DAG. This also handles validation of the
 	// transaction scripts.
 	chainUpdates, err := dag.addBlock(newNode, block, selectedParentAnticone, flags)
@@ -110,17 +131,17 @@ func (dag *BlockDAG) maybeAcceptBlock(block *util.Block, flags BehaviorFlags) er
 	return nil
 }
 
-func lookupParentNodes(block *util.Block, blockDAG *BlockDAG) (blockSet, error) {
+func lookupParentNodes(block *util.Block, dag *BlockDAG) (blockSet, error) {
 	header := block.MsgBlock().Header
 	parentHashes := header.ParentHashes
 
 	nodes := newBlockSet()
 	for _, parentHash := range parentHashes {
-		node := blockDAG.index.LookupNode(parentHash)
-		if node == nil {
+		node, ok := dag.index.LookupNode(parentHash)
+		if !ok {
 			str := fmt.Sprintf("parent block %s is unknown", parentHash)
 			return nil, ruleError(ErrParentBlockUnknown, str)
-		} else if blockDAG.index.NodeStatus(node).KnownInvalid() {
+		} else if dag.index.NodeStatus(node).KnownInvalid() {
 			str := fmt.Sprintf("parent block %s is known to be invalid", parentHash)
 			return nil, ruleError(ErrInvalidAncestorBlock, str)
 		}

blockdag/accept_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestMaybeAcceptBlockErrors(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestMaybeAcceptBlockErrors", Config{
+	dag, teardownFunc, err := DAGSetup("TestMaybeAcceptBlockErrors", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -63,7 +63,10 @@ func TestMaybeAcceptBlockErrors(t *testing.T) {
 	if isOrphan {
 		t.Fatalf("TestMaybeAcceptBlockErrors: incorrectly returned block 1 is an orphan")
 	}
-	blockNode1 := dag.index.LookupNode(block1.Hash())
+	blockNode1, ok := dag.index.LookupNode(block1.Hash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", block1.Hash())
+	}
 	dag.index.SetStatusFlags(blockNode1, statusValidateFailed)
 
 	block2 := blocks[2]

blockdag/blockheap_test.go

@@ -10,15 +10,15 @@ import (
 // TestBlockHeap tests pushing, popping, and determining the length of the heap.
 func TestBlockHeap(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestBlockHeap", Config{
-		DAGParams: &dagconfig.MainnetParams,
+	dag, teardownFunc, err := DAGSetup("TestBlockHeap", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Fatalf("TestBlockHeap: Failed to setup DAG instance: %s", err)
 	}
 	defer teardownFunc()
 
-	block0Header := dagconfig.MainnetParams.GenesisBlock.Header
+	block0Header := dagconfig.SimnetParams.GenesisBlock.Header
 	block0, _ := dag.newBlockNode(&block0Header, newBlockSet())
 
 	block100000Header := Block100000.Header

blockdag/blockidhash.go

@@ -1,136 +0,0 @@
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/pkg/errors"
-)
-
-var (
-	// idByHashIndexBucketName is the name of the db bucket used to house
-	// the block hash -> block id index.
-	idByHashIndexBucketName = []byte("idbyhashidx")
-
-	// hashByIDIndexBucketName is the name of the db bucket used to house
-	// the block id -> block hash index.
-	hashByIDIndexBucketName = []byte("hashbyididx")
-
-	currentBlockIDKey = []byte("currentblockid")
-)
-
-// -----------------------------------------------------------------------------
-// This is a mapping between block hashes and unique IDs. The ID
-// is simply a sequentially incremented uint64 that is used instead of block hash
-// for the indexers. This is useful because it is only 8 bytes versus 32 bytes
-// hashes and thus saves a ton of space when a block is referenced in an index.
-// It consists of three buckets: the first bucket maps the hash of each
-// block to the unique ID and the second maps that ID back to the block hash.
-// The third bucket contains the last received block ID, and is used
-// when starting the node to check that the enabled indexes are up to date
-// with the latest received block, and if not, initiate recovery process.
-//
-// The serialized format for keys and values in the block hash to ID bucket is:
-//   <hash> = <ID>
-//
-//   Field           Type              Size
-//   hash            daghash.Hash     32 bytes
-//   ID              uint64            8 bytes
-//   -----
-//   Total: 40 bytes
-//
-// The serialized format for keys and values in the ID to block hash bucket is:
-//   <ID> = <hash>
-//
-//   Field           Type              Size
-//   ID              uint64            8 bytes
-//   hash            daghash.Hash     32 bytes
-//   -----
-//   Total: 40 bytes
-//
-// -----------------------------------------------------------------------------
-
-const blockIDSize = 8 // 8 bytes for block ID
-
-// DBFetchBlockIDByHash uses an existing database transaction to retrieve the
-// block id for the provided hash from the index.
-func DBFetchBlockIDByHash(dbTx database.Tx, hash *daghash.Hash) (uint64, error) {
-	hashIndex := dbTx.Metadata().Bucket(idByHashIndexBucketName)
-	serializedID := hashIndex.Get(hash[:])
-	if serializedID == nil {
-		return 0, errors.Errorf("no entry in the block ID index for block with hash %s", hash)
-	}
-
-	return DeserializeBlockID(serializedID), nil
-}
-
-// DBFetchBlockHashBySerializedID uses an existing database transaction to
-// retrieve the hash for the provided serialized block id from the index.
-func DBFetchBlockHashBySerializedID(dbTx database.Tx, serializedID []byte) (*daghash.Hash, error) {
-	idIndex := dbTx.Metadata().Bucket(hashByIDIndexBucketName)
-	hashBytes := idIndex.Get(serializedID)
-	if hashBytes == nil {
-		return nil, errors.Errorf("no entry in the block ID index for block with id %d", byteOrder.Uint64(serializedID))
-	}
-
-	var hash daghash.Hash
-	copy(hash[:], hashBytes)
-	return &hash, nil
-}
-
-// dbPutBlockIDIndexEntry uses an existing database transaction to update or add
-// the index entries for the hash to id and id to hash mappings for the provided
-// values.
-func dbPutBlockIDIndexEntry(dbTx database.Tx, hash *daghash.Hash, serializedID []byte) error {
-	// Add the block hash to ID mapping to the index.
-	meta := dbTx.Metadata()
-	hashIndex := meta.Bucket(idByHashIndexBucketName)
-	if err := hashIndex.Put(hash[:], serializedID[:]); err != nil {
-		return err
-	}
-
-	// Add the block ID to hash mapping to the index.
-	idIndex := meta.Bucket(hashByIDIndexBucketName)
-	return idIndex.Put(serializedID[:], hash[:])
-}
-
-// DBFetchCurrentBlockID returns the last known block ID.
-func DBFetchCurrentBlockID(dbTx database.Tx) uint64 {
-	serializedID := dbTx.Metadata().Get(currentBlockIDKey)
-	if serializedID == nil {
-		return 0
-	}
-	return DeserializeBlockID(serializedID)
-}
-
-// DeserializeBlockID returns a deserialized block id
-func DeserializeBlockID(serializedID []byte) uint64 {
-	return byteOrder.Uint64(serializedID)
-}
-
-// SerializeBlockID returns a serialized block id
-func SerializeBlockID(blockID uint64) []byte {
-	serializedBlockID := make([]byte, blockIDSize)
-	byteOrder.PutUint64(serializedBlockID, blockID)
-	return serializedBlockID
-}
-
-// DBFetchBlockHashByID uses an existing database transaction to retrieve the
-// hash for the provided block id from the index.
-func DBFetchBlockHashByID(dbTx database.Tx, id uint64) (*daghash.Hash, error) {
-	return DBFetchBlockHashBySerializedID(dbTx, SerializeBlockID(id))
-}
-
-func createBlockID(dbTx database.Tx, blockHash *daghash.Hash) (uint64, error) {
-	currentBlockID := DBFetchCurrentBlockID(dbTx)
-	newBlockID := currentBlockID + 1
-	serializedNewBlockID := SerializeBlockID(newBlockID)
-	err := dbTx.Metadata().Put(currentBlockIDKey, serializedNewBlockID)
-	if err != nil {
-		return 0, err
-	}
-	err = dbPutBlockIDIndexEntry(dbTx, blockHash, serializedNewBlockID)
-	if err != nil {
-		return 0, err
-	}
-	return newBlockID, nil
-}

blockdag/blockindex.go

@@ -5,10 +5,10 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/dbaccess"
 	"sync"
 
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
 
@@ -18,7 +18,6 @@ type blockIndex struct {
 	// The following fields are set when the instance is created and can't
 	// be changed afterwards, so there is no need to protect them with a
 	// separate mutex.
-	db        database.DB
 	dagParams *dagconfig.Params
 
 	sync.RWMutex
@@ -29,9 +28,8 @@ type blockIndex struct {
 // newBlockIndex returns a new empty instance of a block index. The index will
 // be dynamically populated as block nodes are loaded from the database and
 // manually added.
-func newBlockIndex(db database.DB, dagParams *dagconfig.Params) *blockIndex {
+func newBlockIndex(dagParams *dagconfig.Params) *blockIndex {
 	return &blockIndex{
-		db:        db,
 		dagParams: dagParams,
 		index:     make(map[daghash.Hash]*blockNode),
 		dirty:     make(map[*blockNode]struct{}),
@@ -52,11 +50,11 @@ func (bi *blockIndex) HaveBlock(hash *daghash.Hash) bool {
 // return nil if there is no entry for the hash.
 //
 // This function is safe for concurrent access.
-func (bi *blockIndex) LookupNode(hash *daghash.Hash) *blockNode {
+func (bi *blockIndex) LookupNode(hash *daghash.Hash) (*blockNode, bool) {
 	bi.RLock()
 	defer bi.RUnlock()
-	node := bi.index[*hash]
-	return node
+	node, ok := bi.index[*hash]
+	return node, ok
 }
 
 // AddNode adds the provided node to the block index and marks it as dirty.
@@ -111,17 +109,8 @@ func (bi *blockIndex) UnsetStatusFlags(node *blockNode, flags blockStatus) {
 	bi.dirty[node] = struct{}{}
 }
 
-// flushToDB writes all dirty block nodes to the database. If all writes
-// succeed, this clears the dirty set.
-func (bi *blockIndex) flushToDB() error {
-	return bi.db.Update(func(dbTx database.Tx) error {
-		return bi.flushToDBWithTx(dbTx)
-	})
-}
-
-// flushToDBWithTx writes all dirty block nodes to the database. If all
-// writes succeed, this clears the dirty set.
-func (bi *blockIndex) flushToDBWithTx(dbTx database.Tx) error {
+// flushToDB writes all dirty block nodes to the database.
+func (bi *blockIndex) flushToDB(dbContext *dbaccess.TxContext) error {
 	bi.Lock()
 	defer bi.Unlock()
 	if len(bi.dirty) == 0 {
@@ -129,7 +118,12 @@ func (bi *blockIndex) flushToDBWithTx(dbTx database.Tx) error {
 	}
 
 	for node := range bi.dirty {
-		err := dbStoreBlockNode(dbTx, node)
+		serializedBlockNode, err := serializeBlockNode(node)
+		if err != nil {
+			return err
+		}
+		key := blockIndexKey(node.hash, node.blueScore)
+		err = dbaccess.StoreIndexBlock(dbContext, key, serializedBlockNode)
 		if err != nil {
 			return err
 		}

blockdag/blockindex_test.go

@@ -10,7 +10,7 @@ import (
 func TestAncestorErrors(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
-	dag, teardownFunc, err := DAGSetup("TestAncestorErrors", Config{
+	dag, teardownFunc, err := DAGSetup("TestAncestorErrors", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {

blockdag/blocklocator.go

@@ -29,8 +29,15 @@ func (dag *BlockDAG) BlockLocatorFromHashes(highHash, lowHash *daghash.Hash) (Bl
 	dag.dagLock.RLock()
 	defer dag.dagLock.RUnlock()
 
-	highNode := dag.index.LookupNode(highHash)
-	lowNode := dag.index.LookupNode(lowHash)
+	highNode, ok := dag.index.LookupNode(highHash)
+	if !ok {
+		return nil, errors.Errorf("block %s is unknown", highHash)
+	}
+
+	lowNode, ok := dag.index.LookupNode(lowHash)
+	if !ok {
+		return nil, errors.Errorf("block %s is unknown", lowHash)
+	}
 
 	return dag.blockLocator(highNode, lowNode)
 }
@@ -88,8 +95,8 @@ func (dag *BlockDAG) FindNextLocatorBoundaries(locator BlockLocator) (highHash,
 	lowNode := dag.genesis
 	nextBlockLocatorIndex := int64(len(locator) - 1)
 	for i, hash := range locator {
-		node := dag.index.LookupNode(hash)
-		if node != nil {
+		node, ok := dag.index.LookupNode(hash)
+		if ok {
 			lowNode = node
 			nextBlockLocatorIndex = int64(i) - 1
 			break

blockdag/blocknode.go

@@ -110,7 +110,7 @@ func (dag *BlockDAG) newBlockNode(blockHeader *wire.BlockHeader, parents blockSe
 		parents:            parents,
 		children:           make(blockSet),
 		blueScore:          math.MaxUint64, // Initialized to the max value to avoid collisions with the genesis block
-		timestamp:          dag.AdjustedTime().Unix(),
+		timestamp:          dag.Now().Unix(),
 		bluesAnticoneSizes: make(map[*blockNode]dagconfig.KType),
 	}
 

blockdag/blocknode_test.go

@@ -9,7 +9,7 @@ import (
 // This test is to ensure the size BlueAnticoneSizesSize is serialized to the size of KType.
 // We verify that by serializing and deserializing the block while making sure that we stay within the expected range.
 func TestBlueAnticoneSizesSize(t *testing.T) {
-	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizesSize", Config{
+	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizesSize", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {

blockdag/blockwindow.go

@@ -2,6 +2,7 @@ package blockdag
 
 import (
 	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/bigintpool"
 	"github.com/pkg/errors"
 	"math"
 	"math/big"
@@ -53,13 +54,19 @@ func (window blockWindow) minMaxTimestamps() (min, max int64) {
 	return
 }
 
-func (window blockWindow) averageTarget() *big.Int {
-	averageTarget := big.NewInt(0)
+func (window blockWindow) averageTarget(averageTarget *big.Int) {
+	averageTarget.SetInt64(0)
+
+	target := bigintpool.Acquire(0)
+	defer bigintpool.Release(target)
 	for _, node := range window {
-		target := util.CompactToBig(node.bits)
+		util.CompactToBigWithDestination(node.bits, target)
 		averageTarget.Add(averageTarget, target)
 	}
-	return averageTarget.Div(averageTarget, big.NewInt(int64(len(window))))
+
+	windowLen := bigintpool.Acquire(int64(len(window)))
+	defer bigintpool.Release(windowLen)
+	averageTarget.Div(averageTarget, windowLen)
 }
 
 func (window blockWindow) medianTimestamp() (int64, error) {

blockdag/blockwindow_test.go

@@ -12,7 +12,7 @@ import (
 func TestBlueBlockWindow(t *testing.T) {
 	params := dagconfig.SimnetParams
 	params.K = 1
-	dag, teardownFunc, err := DAGSetup("TestBlueBlockWindow", Config{
+	dag, teardownFunc, err := DAGSetup("TestBlueBlockWindow", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -53,12 +53,12 @@ func TestBlueBlockWindow(t *testing.T) {
 		{
 			parents:                          []string{"C", "D"},
 			id:                               "E",
-			expectedWindowWithGenesisPadding: []string{"C", "D", "B", "A", "A", "A", "A", "A", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
 		},
 		{
 			parents:                          []string{"C", "D"},
 			id:                               "F",
-			expectedWindowWithGenesisPadding: []string{"C", "D", "B", "A", "A", "A", "A", "A", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"D", "C", "B", "A", "A", "A", "A", "A", "A", "A"},
 		},
 		{
 			parents:                          []string{"A"},
@@ -73,37 +73,37 @@ func TestBlueBlockWindow(t *testing.T) {
 		{
 			parents:                          []string{"H", "F"},
 			id:                               "I",
-			expectedWindowWithGenesisPadding: []string{"F", "C", "D", "B", "A", "A", "A", "A", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"F", "D", "C", "B", "A", "A", "A", "A", "A", "A"},
 		},
 		{
 			parents:                          []string{"I"},
 			id:                               "J",
-			expectedWindowWithGenesisPadding: []string{"I", "F", "C", "D", "B", "A", "A", "A", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"I", "F", "D", "C", "B", "A", "A", "A", "A", "A"},
 		},
 		{
 			parents:                          []string{"J"},
 			id:                               "K",
-			expectedWindowWithGenesisPadding: []string{"J", "I", "F", "C", "D", "B", "A", "A", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"J", "I", "F", "D", "C", "B", "A", "A", "A", "A"},
 		},
 		{
 			parents:                          []string{"K"},
 			id:                               "L",
-			expectedWindowWithGenesisPadding: []string{"K", "J", "I", "F", "C", "D", "B", "A", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"K", "J", "I", "F", "D", "C", "B", "A", "A", "A"},
 		},
 		{
 			parents:                          []string{"L"},
 			id:                               "M",
-			expectedWindowWithGenesisPadding: []string{"L", "K", "J", "I", "F", "C", "D", "B", "A", "A"},
+			expectedWindowWithGenesisPadding: []string{"L", "K", "J", "I", "F", "D", "C", "B", "A", "A"},
 		},
 		{
 			parents:                          []string{"M"},
 			id:                               "N",
-			expectedWindowWithGenesisPadding: []string{"M", "L", "K", "J", "I", "F", "C", "D", "B", "A"},
+			expectedWindowWithGenesisPadding: []string{"M", "L", "K", "J", "I", "F", "D", "C", "B", "A"},
 		},
 		{
 			parents:                          []string{"N"},
 			id:                               "O",
-			expectedWindowWithGenesisPadding: []string{"N", "M", "L", "K", "J", "I", "F", "C", "D", "B"},
+			expectedWindowWithGenesisPadding: []string{"N", "M", "L", "K", "J", "I", "F", "D", "C", "B"},
 		},
 	}
 
@@ -133,7 +133,10 @@ func TestBlueBlockWindow(t *testing.T) {
 			t.Fatalf("block %v was unexpectedly orphan", blockData.id)
 		}
 
-		node := dag.index.LookupNode(utilBlock.Hash())
+		node, ok := dag.index.LookupNode(utilBlock.Hash())
+		if !ok {
+			t.Fatalf("block %s does not exist in the DAG", utilBlock.Hash())
+		}
 
 		blockByIDMap[blockData.id] = node
 		idByBlockMap[node] = blockData.id

blockdag/coinbase.go

@@ -4,16 +4,15 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
-	"github.com/kaspanet/kaspad/util/subnetworkid"
-	"github.com/pkg/errors"
-	"io"
-	"math"
-
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/coinbasepayload"
 	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/subnetworkid"
 	"github.com/kaspanet/kaspad/util/txsort"
 	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
+	"io"
 )
 
 // compactFeeData is a specialized data type to store a compact list of fees
@@ -73,55 +72,24 @@ func (cfr *compactFeeIterator) next() (uint64, error) {
 }
 
 // The following functions relate to storing and retrieving fee data from the database
-var feeBucket = []byte("fees")
 
 // getBluesFeeData returns the compactFeeData for all nodes's blues,
 // used to calculate the fees this blockNode needs to pay
 func (node *blockNode) getBluesFeeData(dag *BlockDAG) (map[daghash.Hash]compactFeeData, error) {
 	bluesFeeData := make(map[daghash.Hash]compactFeeData)
 
-	err := dag.db.View(func(dbTx database.Tx) error {
-		for _, blueBlock := range node.blues {
-			feeData, err := dbFetchFeeData(dbTx, blueBlock.hash)
-			if err != nil {
-				return errors.Errorf("Error getting fee data for block %s: %s", blueBlock.hash, err)
-			}
-
-			bluesFeeData[*blueBlock.hash] = feeData
+	for _, blueBlock := range node.blues {
+		feeData, err := dbaccess.FetchFeeData(dbaccess.NoTx(), blueBlock.hash)
+		if err != nil {
+			return nil, err
 		}
 
-		return nil
-	})
-	if err != nil {
-		return nil, err
+		bluesFeeData[*blueBlock.hash] = feeData
 	}
 
 	return bluesFeeData, nil
 }
 
-func dbStoreFeeData(dbTx database.Tx, blockHash *daghash.Hash, feeData compactFeeData) error {
-	feeBucket, err := dbTx.Metadata().CreateBucketIfNotExists(feeBucket)
-	if err != nil {
-		return errors.Errorf("Error creating or retrieving fee bucket: %s", err)
-	}
-
-	return feeBucket.Put(blockHash.CloneBytes(), feeData)
-}
-
-func dbFetchFeeData(dbTx database.Tx, blockHash *daghash.Hash) (compactFeeData, error) {
-	feeBucket := dbTx.Metadata().Bucket(feeBucket)
-	if feeBucket == nil {
-		return nil, errors.New("Fee bucket does not exist")
-	}
-
-	feeData := feeBucket.Get(blockHash.CloneBytes())
-	if feeData == nil {
-		return nil, errors.Errorf("No fee data found for block %s", blockHash)
-	}
-
-	return feeData, nil
-}
-
 // The following functions deal with building and validating the coinbase transaction
 
 func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Block, txsAcceptanceData MultiBlockTxsAcceptanceData) error {
@@ -129,7 +97,10 @@ func (node *blockNode) validateCoinbaseTransaction(dag *BlockDAG, block *util.Bl
 		return nil
 	}
 	blockCoinbaseTx := block.CoinbaseTransaction().MsgTx()
-	scriptPubKey, extraData, err := DeserializeCoinbasePayload(blockCoinbaseTx)
+	_, scriptPubKey, extraData, err := coinbasepayload.DeserializeCoinbasePayload(blockCoinbaseTx)
+	if errors.Is(err, coinbasepayload.ErrIncorrectScriptPubKeyLen) {
+		return ruleError(ErrBadCoinbaseTransaction, err.Error())
+	}
 	if err != nil {
 		return err
 	}
@@ -156,16 +127,15 @@ func (node *blockNode) expectedCoinbaseTransaction(dag *BlockDAG, txsAcceptanceD
 	txOuts := []*wire.TxOut{}
 
 	for _, blue := range node.blues {
-		txIn, txOut, err := coinbaseInputAndOutputForBlueBlock(dag, blue, txsAcceptanceData, bluesFeeData)
+		txOut, err := coinbaseOutputForBlueBlock(dag, blue, txsAcceptanceData, bluesFeeData)
 		if err != nil {
 			return nil, err
 		}
-		txIns = append(txIns, txIn)
 		if txOut != nil {
 			txOuts = append(txOuts, txOut)
 		}
 	}
-	payload, err := SerializeCoinbasePayload(scriptPubKey, extraData)
+	payload, err := coinbasepayload.SerializeCoinbasePayload(node.blueScore, scriptPubKey, extraData)
 	if err != nil {
 		return nil, err
 	}
@@ -174,83 +144,33 @@ func (node *blockNode) expectedCoinbaseTransaction(dag *BlockDAG, txsAcceptanceD
 	return util.NewTx(sortedCoinbaseTx), nil
 }
 
-// SerializeCoinbasePayload builds the coinbase payload based on the provided scriptPubKey and extra data.
-func SerializeCoinbasePayload(scriptPubKey []byte, extraData []byte) ([]byte, error) {
-	w := &bytes.Buffer{}
-	err := wire.WriteVarInt(w, uint64(len(scriptPubKey)))
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(scriptPubKey)
-	if err != nil {
-		return nil, err
-	}
-	_, err = w.Write(extraData)
-	if err != nil {
-		return nil, err
-	}
-	return w.Bytes(), nil
-}
-
-// DeserializeCoinbasePayload deserialize the coinbase payload to its component (scriptPubKey and extra data).
-func DeserializeCoinbasePayload(tx *wire.MsgTx) (scriptPubKey []byte, extraData []byte, err error) {
-	r := bytes.NewReader(tx.Payload)
-	scriptPubKeyLen, err := wire.ReadVarInt(r)
-	if err != nil {
-		return nil, nil, err
-	}
-	scriptPubKey = make([]byte, scriptPubKeyLen)
-	_, err = r.Read(scriptPubKey)
-	if err != nil {
-		return nil, nil, err
-	}
-	extraData = make([]byte, r.Len())
-	if r.Len() != 0 {
-		_, err = r.Read(extraData)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-	return scriptPubKey, extraData, nil
-}
-
-// feeInputAndOutputForBlueBlock calculates the input and output that should go into the coinbase transaction of blueBlock
-// If blueBlock gets no fee - returns only txIn and nil for txOut
-func coinbaseInputAndOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
-	txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (
-	*wire.TxIn, *wire.TxOut, error) {
+// coinbaseOutputForBlueBlock calculates the output that should go into the coinbase transaction of blueBlock
+// If blueBlock gets no fee - returns nil for txOut
+func coinbaseOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
+	txsAcceptanceData MultiBlockTxsAcceptanceData, feeData map[daghash.Hash]compactFeeData) (*wire.TxOut, error) {
 
 	blockTxsAcceptanceData, ok := txsAcceptanceData.FindAcceptanceData(blueBlock.hash)
 	if !ok {
-		return nil, nil, errors.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
+		return nil, errors.Errorf("No txsAcceptanceData for block %s", blueBlock.hash)
 	}
 	blockFeeData, ok := feeData[*blueBlock.hash]
 	if !ok {
-		return nil, nil, errors.Errorf("No feeData for block %s", blueBlock.hash)
+		return nil, errors.Errorf("No feeData for block %s", blueBlock.hash)
 	}
 
 	if len(blockTxsAcceptanceData.TxAcceptanceData) != blockFeeData.Len() {
-		return nil, nil, errors.Errorf(
+		return nil, errors.Errorf(
 			"length of accepted transaction data(%d) and fee data(%d) is not equal for block %s",
 			len(blockTxsAcceptanceData.TxAcceptanceData), blockFeeData.Len(), blueBlock.hash)
 	}
 
-	txIn := &wire.TxIn{
-		SignatureScript: []byte{},
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID(*blueBlock.hash),
-			Index: math.MaxUint32,
-		},
-		Sequence: wire.MaxTxInSequenceNum,
-	}
-
 	totalFees := uint64(0)
 	feeIterator := blockFeeData.iterator()
 
 	for _, txAcceptanceData := range blockTxsAcceptanceData.TxAcceptanceData {
 		fee, err := feeIterator.next()
 		if err != nil {
-			return nil, nil, errors.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
+			return nil, errors.Errorf("Error retrieving fee from compactFeeData iterator: %s", err)
 		}
 		if txAcceptanceData.IsAccepted {
 			totalFees += fee
@@ -260,13 +180,13 @@ func coinbaseInputAndOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
 	totalReward := CalcBlockSubsidy(blueBlock.blueScore, dag.dagParams) + totalFees
 
 	if totalReward == 0 {
-		return txIn, nil, nil
+		return nil, nil
 	}
 
 	// the ScriptPubKey for the coinbase is parsed from the coinbase payload
-	scriptPubKey, _, err := DeserializeCoinbasePayload(blockTxsAcceptanceData.TxAcceptanceData[0].Tx.MsgTx())
+	_, scriptPubKey, _, err := coinbasepayload.DeserializeCoinbasePayload(blockTxsAcceptanceData.TxAcceptanceData[0].Tx.MsgTx())
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	txOut := &wire.TxOut{
@@ -274,5 +194,5 @@ func coinbaseInputAndOutputForBlueBlock(dag *BlockDAG, blueBlock *blockNode,
 		ScriptPubKey: scriptPubKey,
 	}
 
-	return txIn, txOut, nil
+	return txOut, nil
 }

blockdag/common_test.go

@@ -7,17 +7,16 @@ package blockdag
 import (
 	"compress/bzip2"
 	"encoding/binary"
-	"github.com/pkg/errors"
 	"io"
 	"os"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/kaspanet/kaspad/dagconfig"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
@@ -73,15 +72,8 @@ func loadUTXOSet(filename string) (UTXOSet, error) {
 			return nil, err
 		}
 
-		// Serialized utxo entry.
-		serialized := make([]byte, numBytes)
-		_, err = io.ReadAtLeast(r, serialized, int(numBytes))
-		if err != nil {
-			return nil, err
-		}
-
-		// Deserialize it and add it to the view.
-		entry, err := deserializeUTXOEntry(serialized)
+		// Deserialize the UTXO entry and add it to the UTXO set.
+		entry, err := deserializeUTXOEntry(r)
 		if err != nil {
 			return nil, err
 		}
@@ -102,11 +94,11 @@ func (dag *BlockDAG) TestSetCoinbaseMaturity(maturity uint64) {
 // it is not usable with all functions and the tests must take care when making
 // use of it.
 func newTestDAG(params *dagconfig.Params) *BlockDAG {
-	index := newBlockIndex(nil, params)
+	index := newBlockIndex(params)
 	targetTimePerBlock := int64(params.TargetTimePerBlock / time.Second)
 	dag := &BlockDAG{
 		dagParams:                      params,
-		timeSource:                     NewMedianTime(),
+		timeSource:                     NewTimeSource(),
 		targetTimePerBlock:             targetTimePerBlock,
 		difficultyAdjustmentWindowSize: params.DifficultyAdjustmentWindowSize,
 		TimestampDeviationTolerance:    params.TimestampDeviationTolerance,
@@ -152,62 +144,55 @@ func addNodeAsChildToParents(node *blockNode) {
 // same type (either both nil or both of type RuleError) and their error codes
 // match when not nil.
 func checkRuleError(gotErr, wantErr error) error {
-	// Ensure the error code is of the expected type and the error
-	// code matches the value specified in the test instance.
-	if reflect.TypeOf(gotErr) != reflect.TypeOf(wantErr) {
-		return errors.Errorf("wrong error - got %T (%[1]v), want %T",
-			gotErr, wantErr)
-	}
-	if gotErr == nil {
+	if wantErr == nil && gotErr == nil {
 		return nil
 	}
 
-	// Ensure the want error type is a script error.
-	werr, ok := wantErr.(RuleError)
-	if !ok {
-		return errors.Errorf("unexpected test error type %T", wantErr)
+	var gotRuleErr RuleError
+	if ok := errors.As(gotErr, &gotRuleErr); !ok {
+		return errors.Errorf("gotErr expected to be RuleError, but got %+v instead", gotErr)
+	}
+
+	var wantRuleErr RuleError
+	if ok := errors.As(wantErr, &wantRuleErr); !ok {
+		return errors.Errorf("wantErr expected to be RuleError, but got %+v instead", wantErr)
 	}
 
 	// Ensure the error codes match. It's safe to use a raw type assert
 	// here since the code above already proved they are the same type and
 	// the want error is a script error.
-	gotErrorCode := gotErr.(RuleError).ErrorCode
-	if gotErrorCode != werr.ErrorCode {
+	if gotRuleErr.ErrorCode != wantRuleErr.ErrorCode {
 		return errors.Errorf("mismatched error code - got %v (%v), want %v",
-			gotErrorCode, gotErr, werr.ErrorCode)
+			gotRuleErr.ErrorCode, gotErr, wantRuleErr.ErrorCode)
 	}
 
 	return nil
 }
 
-func prepareAndProcessBlock(t *testing.T, dag *BlockDAG, parents ...*wire.MsgBlock) *wire.MsgBlock {
+func prepareAndProcessBlockByParentMsgBlocks(t *testing.T, dag *BlockDAG, parents ...*wire.MsgBlock) *wire.MsgBlock {
 	parentHashes := make([]*daghash.Hash, len(parents))
 	for i, parent := range parents {
 		parentHashes[i] = parent.BlockHash()
 	}
-	daghash.Sort(parentHashes)
-	block, err := PrepareBlockForTest(dag, parentHashes, nil)
-	if err != nil {
-		t.Fatalf("error in PrepareBlockForTest: %s", err)
-	}
-	utilBlock := util.NewBlock(block)
-	isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
-	if err != nil {
-		t.Fatalf("unexpected error in ProcessBlock: %s", err)
-	}
-	if isDelayed {
-		t.Fatalf("block is too far in the future")
-	}
-	if isOrphan {
-		t.Fatalf("block was unexpectedly orphan")
-	}
-	return block
+	return PrepareAndProcessBlockForTest(t, dag, parentHashes, nil)
 }
 
 func nodeByMsgBlock(t *testing.T, dag *BlockDAG, block *wire.MsgBlock) *blockNode {
-	node := dag.index.LookupNode(block.BlockHash())
-	if node == nil {
+	node, ok := dag.index.LookupNode(block.BlockHash())
+	if !ok {
 		t.Fatalf("couldn't find block node with hash %s", block.BlockHash())
 	}
 	return node
 }
+
+type fakeTimeSource struct {
+	time time.Time
+}
+
+func (fts *fakeTimeSource) Now() time.Time {
+	return time.Unix(fts.time.Unix(), 0)
+}
+
+func newFakeTimeSource(fakeTime time.Time) TimeSource {
+	return &fakeTimeSource{time: fakeTime}
+}

blockdag/compress.go

@@ -1,584 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"github.com/kaspanet/kaspad/ecc"
-	"github.com/kaspanet/kaspad/txscript"
-)
-
-// -----------------------------------------------------------------------------
-// A variable length quantity (VLQ) is an encoding that uses an arbitrary number
-// of binary octets to represent an arbitrarily large integer. The scheme
-// employs a most significant byte (MSB) base-128 encoding where the high bit in
-// each byte indicates whether or not the byte is the final one. In addition,
-// to ensure there are no redundant encodings, an offset is subtracted every
-// time a group of 7 bits is shifted out. Therefore each integer can be
-// represented in exactly one way, and each representation stands for exactly
-// one integer.
-//
-// Another nice property of this encoding is that it provides a compact
-// representation of values that are typically used to indicate sizes. For
-// example, the values 0 - 127 are represented with a single byte, 128 - 16511
-// with two bytes, and 16512 - 2113663 with three bytes.
-//
-// While the encoding allows arbitrarily large integers, it is artificially
-// limited in this code to an unsigned 64-bit integer for efficiency purposes.
-//
-// Example encodings:
-//           0 -> [0x00]
-//         127 -> [0x7f]                 * Max 1-byte value
-//         128 -> [0x80 0x00]
-//         129 -> [0x80 0x01]
-//         255 -> [0x80 0x7f]
-//         256 -> [0x81 0x00]
-//       16511 -> [0xff 0x7f]            * Max 2-byte value
-//       16512 -> [0x80 0x80 0x00]
-//       32895 -> [0x80 0xff 0x7f]
-//     2113663 -> [0xff 0xff 0x7f]       * Max 3-byte value
-//   270549119 -> [0xff 0xff 0xff 0x7f]  * Max 4-byte value
-//      2^64-1 -> [0x80 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0x7f]
-//
-// References:
-//   https://en.wikipedia.org/wiki/Variable-length_quantity
-//   http://www.codecodex.com/wiki/Variable-Length_Integers
-// -----------------------------------------------------------------------------
-
-// serializeSizeVLQ returns the number of bytes it would take to serialize the
-// passed number as a variable-length quantity according to the format described
-// above.
-func serializeSizeVLQ(n uint64) int {
-	size := 1
-	for ; n > 0x7f; n = (n >> 7) - 1 {
-		size++
-	}
-
-	return size
-}
-
-// putVLQ serializes the provided number to a variable-length quantity according
-// to the format described above and returns the number of bytes of the encoded
-// value. The result is placed directly into the passed byte slice which must
-// be at least large enough to handle the number of bytes returned by the
-// serializeSizeVLQ function or it will panic.
-func putVLQ(target []byte, n uint64) int {
-	offset := 0
-	for ; ; offset++ {
-		// The high bit is set when another byte follows.
-		highBitMask := byte(0x80)
-		if offset == 0 {
-			highBitMask = 0x00
-		}
-
-		target[offset] = byte(n&0x7f) | highBitMask
-		if n <= 0x7f {
-			break
-		}
-		n = (n >> 7) - 1
-	}
-
-	// Reverse the bytes so it is MSB-encoded.
-	for i, j := 0, offset; i < j; i, j = i+1, j-1 {
-		target[i], target[j] = target[j], target[i]
-	}
-
-	return offset + 1
-}
-
-// deserializeVLQ deserializes the provided variable-length quantity according
-// to the format described above. It also returns the number of bytes
-// deserialized.
-func deserializeVLQ(serialized []byte) (uint64, int) {
-	var n uint64
-	var size int
-	for _, val := range serialized {
-		size++
-		n = (n << 7) | uint64(val&0x7f)
-		if val&0x80 != 0x80 {
-			break
-		}
-		n++
-	}
-
-	return n, size
-}
-
-// -----------------------------------------------------------------------------
-// In order to reduce the size of stored scripts, a domain specific compression
-// algorithm is used which recognizes standard scripts and stores them using
-// less bytes than the original script.
-//
-// The general serialized format is:
-//
-//   <script size or type><script data>
-//
-//   Field                 Type     Size
-//   script size or type   VLQ      variable
-//   script data           []byte   variable
-//
-// The specific serialized format for each recognized standard script is:
-//
-// - Pay-to-pubkey-hash: (21 bytes) - <0><20-byte pubkey hash>
-// - Pay-to-script-hash: (21 bytes) - <1><20-byte script hash>
-// - Pay-to-pubkey**:    (33 bytes) - <2, 3, 4, or 5><32-byte pubkey X value>
-//   2, 3 = compressed pubkey with bit 0 specifying the y coordinate to use
-//   4, 5 = uncompressed pubkey with bit 0 specifying the y coordinate to use
-//   ** Only valid public keys starting with 0x02, 0x03, and 0x04 are supported.
-//
-// Any scripts which are not recognized as one of the aforementioned standard
-// scripts are encoded using the general serialized format and encode the script
-// size as the sum of the actual size of the script and the number of special
-// cases.
-// -----------------------------------------------------------------------------
-
-// The following constants specify the special constants used to identify a
-// special script type in the domain-specific compressed script encoding.
-//
-// NOTE: This section specifically does not use iota since these values are
-// serialized and must be stable for long-term storage.
-const (
-	// cstPayToPubKeyHash identifies a compressed pay-to-pubkey-hash script.
-	cstPayToPubKeyHash = 0
-
-	// cstPayToScriptHash identifies a compressed pay-to-script-hash script.
-	cstPayToScriptHash = 1
-
-	// cstPayToPubKeyComp2 identifies a compressed pay-to-pubkey script to
-	// a compressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyComp2 = 2
-
-	// cstPayToPubKeyComp3 identifies a compressed pay-to-pubkey script to
-	// a compressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyComp3 = 3
-
-	// cstPayToPubKeyUncomp4 identifies a compressed pay-to-pubkey script to
-	// an uncompressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyUncomp4 = 4
-
-	// cstPayToPubKeyUncomp5 identifies a compressed pay-to-pubkey script to
-	// an uncompressed pubkey. Bit 0 specifies which y-coordinate to use
-	// to reconstruct the full uncompressed pubkey.
-	cstPayToPubKeyUncomp5 = 5
-
-	// numSpecialScripts is the number of special scripts recognized by the
-	// domain-specific script compression algorithm.
-	numSpecialScripts = 6
-)
-
-// isPubKeyHash returns whether or not the passed public key script is a
-// standard pay-to-pubkey-hash script along with the pubkey hash it is paying to
-// if it is.
-func isPubKeyHash(script []byte) (bool, []byte) {
-	if len(script) == 25 && script[0] == txscript.OpDup &&
-		script[1] == txscript.OpHash160 &&
-		script[2] == txscript.OpData20 &&
-		script[23] == txscript.OpEqualVerify &&
-		script[24] == txscript.OpCheckSig {
-
-		return true, script[3:23]
-	}
-
-	return false, nil
-}
-
-// isScriptHash returns whether or not the passed public key script is a
-// standard pay-to-script-hash script along with the script hash it is paying to
-// if it is.
-func isScriptHash(script []byte) (bool, []byte) {
-	if len(script) == 23 && script[0] == txscript.OpHash160 &&
-		script[1] == txscript.OpData20 &&
-		script[22] == txscript.OpEqual {
-
-		return true, script[2:22]
-	}
-
-	return false, nil
-}
-
-// isPubKey returns whether or not the passed public key script is a standard
-// pay-to-pubkey script that pays to a valid compressed or uncompressed public
-// key along with the serialized pubkey it is paying to if it is.
-//
-// NOTE: This function ensures the public key is actually valid since the
-// compression algorithm requires valid pubkeys. It does not support hybrid
-// pubkeys. This means that even if the script has the correct form for a
-// pay-to-pubkey script, this function will only return true when it is paying
-// to a valid compressed or uncompressed pubkey.
-func isPubKey(script []byte) (bool, []byte) {
-	// Pay-to-compressed-pubkey script.
-	if len(script) == 35 && script[0] == txscript.OpData33 &&
-		script[34] == txscript.OpCheckSig && (script[1] == 0x02 ||
-		script[1] == 0x03) {
-
-		// Ensure the public key is valid.
-		serializedPubKey := script[1:34]
-		_, err := ecc.ParsePubKey(serializedPubKey, ecc.S256())
-		if err == nil {
-			return true, serializedPubKey
-		}
-	}
-
-	// Pay-to-uncompressed-pubkey script.
-	if len(script) == 67 && script[0] == txscript.OpData65 &&
-		script[66] == txscript.OpCheckSig && script[1] == 0x04 {
-
-		// Ensure the public key is valid.
-		serializedPubKey := script[1:66]
-		_, err := ecc.ParsePubKey(serializedPubKey, ecc.S256())
-		if err == nil {
-			return true, serializedPubKey
-		}
-	}
-
-	return false, nil
-}
-
-// compressedScriptSize returns the number of bytes the passed script would take
-// when encoded with the domain specific compression algorithm described above.
-func compressedScriptSize(scriptPubKey []byte) int {
-	// Pay-to-pubkey-hash script.
-	if valid, _ := isPubKeyHash(scriptPubKey); valid {
-		return 21
-	}
-
-	// Pay-to-script-hash script.
-	if valid, _ := isScriptHash(scriptPubKey); valid {
-		return 21
-	}
-
-	// Pay-to-pubkey (compressed or uncompressed) script.
-	if valid, _ := isPubKey(scriptPubKey); valid {
-		return 33
-	}
-
-	// When none of the above special cases apply, encode the script as is
-	// preceded by the sum of its size and the number of special cases
-	// encoded as a variable length quantity.
-	return serializeSizeVLQ(uint64(len(scriptPubKey)+numSpecialScripts)) +
-		len(scriptPubKey)
-}
-
-// decodeCompressedScriptSize treats the passed serialized bytes as a compressed
-// script, possibly followed by other data, and returns the number of bytes it
-// occupies taking into account the special encoding of the script size by the
-// domain specific compression algorithm described above.
-func decodeCompressedScriptSize(serialized []byte) int {
-	scriptSize, bytesRead := deserializeVLQ(serialized)
-	if bytesRead == 0 {
-		return 0
-	}
-
-	switch scriptSize {
-	case cstPayToPubKeyHash:
-		return 21
-
-	case cstPayToScriptHash:
-		return 21
-
-	case cstPayToPubKeyComp2, cstPayToPubKeyComp3, cstPayToPubKeyUncomp4,
-		cstPayToPubKeyUncomp5:
-		return 33
-	}
-
-	scriptSize -= numSpecialScripts
-	scriptSize += uint64(bytesRead)
-	return int(scriptSize)
-}
-
-// putCompressedScript compresses the passed script according to the domain
-// specific compression algorithm described above directly into the passed
-// target byte slice. The target byte slice must be at least large enough to
-// handle the number of bytes returned by the compressedScriptSize function or
-// it will panic.
-func putCompressedScript(target, scriptPubKey []byte) int {
-	// Pay-to-pubkey-hash script.
-	if valid, hash := isPubKeyHash(scriptPubKey); valid {
-		target[0] = cstPayToPubKeyHash
-		copy(target[1:21], hash)
-		return 21
-	}
-
-	// Pay-to-script-hash script.
-	if valid, hash := isScriptHash(scriptPubKey); valid {
-		target[0] = cstPayToScriptHash
-		copy(target[1:21], hash)
-		return 21
-	}
-
-	// Pay-to-pubkey (compressed or uncompressed) script.
-	if valid, serializedPubKey := isPubKey(scriptPubKey); valid {
-		pubKeyFormat := serializedPubKey[0]
-		switch pubKeyFormat {
-		case 0x02, 0x03:
-			target[0] = pubKeyFormat
-			copy(target[1:33], serializedPubKey[1:33])
-			return 33
-		case 0x04:
-			// Encode the oddness of the serialized pubkey into the
-			// compressed script type.
-			target[0] = pubKeyFormat | (serializedPubKey[64] & 0x01)
-			copy(target[1:33], serializedPubKey[1:33])
-			return 33
-		}
-	}
-
-	// When none of the above special cases apply, encode the unmodified
-	// script preceded by the sum of its size and the number of special
-	// cases encoded as a variable length quantity.
-	encodedSize := uint64(len(scriptPubKey) + numSpecialScripts)
-	vlqSizeLen := putVLQ(target, encodedSize)
-	copy(target[vlqSizeLen:], scriptPubKey)
-	return vlqSizeLen + len(scriptPubKey)
-}
-
-// decompressScript returns the original script obtained by decompressing the
-// passed compressed script according to the domain specific compression
-// algorithm described above.
-//
-// NOTE: The script parameter must already have been proven to be long enough
-// to contain the number of bytes returned by decodeCompressedScriptSize or it
-// will panic. This is acceptable since it is only an internal function.
-func decompressScript(compressedScriptPubKey []byte) []byte {
-	// In practice this function will not be called with a zero-length or
-	// nil script since the nil script encoding includes the length, however
-	// the code below assumes the length exists, so just return nil now if
-	// the function ever ends up being called with a nil script in the
-	// future.
-	if len(compressedScriptPubKey) == 0 {
-		return nil
-	}
-
-	// Decode the script size and examine it for the special cases.
-	encodedScriptSize, bytesRead := deserializeVLQ(compressedScriptPubKey)
-	switch encodedScriptSize {
-	// Pay-to-pubkey-hash script. The resulting script is:
-	// <OP_DUP><OP_HASH160><20 byte hash><OP_EQUALVERIFY><OP_CHECKSIG>
-	case cstPayToPubKeyHash:
-		scriptPubKey := make([]byte, 25)
-		scriptPubKey[0] = txscript.OpDup
-		scriptPubKey[1] = txscript.OpHash160
-		scriptPubKey[2] = txscript.OpData20
-		copy(scriptPubKey[3:], compressedScriptPubKey[bytesRead:bytesRead+20])
-		scriptPubKey[23] = txscript.OpEqualVerify
-		scriptPubKey[24] = txscript.OpCheckSig
-		return scriptPubKey
-
-	// Pay-to-script-hash script. The resulting script is:
-	// <OP_HASH160><20 byte script hash><OP_EQUAL>
-	case cstPayToScriptHash:
-		scriptPubKey := make([]byte, 23)
-		scriptPubKey[0] = txscript.OpHash160
-		scriptPubKey[1] = txscript.OpData20
-		copy(scriptPubKey[2:], compressedScriptPubKey[bytesRead:bytesRead+20])
-		scriptPubKey[22] = txscript.OpEqual
-		return scriptPubKey
-
-	// Pay-to-compressed-pubkey script. The resulting script is:
-	// <OP_DATA_33><33 byte compressed pubkey><OP_CHECKSIG>
-	case cstPayToPubKeyComp2, cstPayToPubKeyComp3:
-		scriptPubKey := make([]byte, 35)
-		scriptPubKey[0] = txscript.OpData33
-		scriptPubKey[1] = byte(encodedScriptSize)
-		copy(scriptPubKey[2:], compressedScriptPubKey[bytesRead:bytesRead+32])
-		scriptPubKey[34] = txscript.OpCheckSig
-		return scriptPubKey
-
-	// Pay-to-uncompressed-pubkey script. The resulting script is:
-	// <OP_DATA_65><65 byte uncompressed pubkey><OP_CHECKSIG>
-	case cstPayToPubKeyUncomp4, cstPayToPubKeyUncomp5:
-		// Change the leading byte to the appropriate compressed pubkey
-		// identifier (0x02 or 0x03) so it can be decoded as a
-		// compressed pubkey. This really should never fail since the
-		// encoding ensures it is valid before compressing to this type.
-		compressedKey := make([]byte, 33)
-		compressedKey[0] = byte(encodedScriptSize - 2)
-		copy(compressedKey[1:], compressedScriptPubKey[1:])
-		key, err := ecc.ParsePubKey(compressedKey, ecc.S256())
-		if err != nil {
-			return nil
-		}
-
-		scriptPubKey := make([]byte, 67)
-		scriptPubKey[0] = txscript.OpData65
-		copy(scriptPubKey[1:], key.SerializeUncompressed())
-		scriptPubKey[66] = txscript.OpCheckSig
-		return scriptPubKey
-	}
-
-	// When none of the special cases apply, the script was encoded using
-	// the general format, so reduce the script size by the number of
-	// special cases and return the unmodified script.
-	scriptSize := int(encodedScriptSize - numSpecialScripts)
-	scriptPubKey := make([]byte, scriptSize)
-	copy(scriptPubKey, compressedScriptPubKey[bytesRead:bytesRead+scriptSize])
-	return scriptPubKey
-}
-
-// -----------------------------------------------------------------------------
-// In order to reduce the size of stored amounts, a domain specific compression
-// algorithm is used which relies on there typically being a lot of zeroes at
-// end of the amounts.
-//
-// While this is simply exchanging one uint64 for another, the resulting value
-// for typical amounts has a much smaller magnitude which results in fewer bytes
-// when encoded as variable length quantity. For example, consider the amount
-// of 0.1 KAS which is 10000000 sompi. Encoding 10000000 as a VLQ would take
-// 4 bytes while encoding the compressed value of 8 as a VLQ only takes 1 byte.
-//
-// Essentially the compression is achieved by splitting the value into an
-// exponent in the range [0-9] and a digit in the range [1-9], when possible,
-// and encoding them in a way that can be decoded. More specifically, the
-// encoding is as follows:
-// - 0 is 0
-// - Find the exponent, e, as the largest power of 10 that evenly divides the
-//   value up to a maximum of 9
-// - When e < 9, the final digit can't be 0 so store it as d and remove it by
-//   dividing the value by 10 (call the result n). The encoded value is thus:
-//   1 + 10*(9*n + d-1) + e
-// - When e==9, the only thing known is the amount is not 0. The encoded value
-//   is thus:
-//   1 + 10*(n-1) + e   ==   10 + 10*(n-1)
-//
-// Example encodings:
-// (The numbers in parenthesis are the number of bytes when serialized as a VLQ)
-//            0 (1) -> 0        (1)           *  0.00000000 KAS
-//         1000 (2) -> 4        (1)           *  0.00001000 KAS
-//        10000 (2) -> 5        (1)           *  0.00010000 KAS
-//     12345678 (4) -> 111111101(4)           *  0.12345678 KAS
-//     50000000 (4) -> 47       (1)           *  0.50000000 KAS
-//    100000000 (4) -> 9        (1)           *  1.00000000 KAS
-//    500000000 (5) -> 49       (1)           *  5.00000000 KAS
-//   1000000000 (5) -> 10       (1)           * 10.00000000 KAS
-// -----------------------------------------------------------------------------
-
-// compressTxOutAmount compresses the passed amount according to the domain
-// specific compression algorithm described above.
-func compressTxOutAmount(amount uint64) uint64 {
-	// No need to do any work if it's zero.
-	if amount == 0 {
-		return 0
-	}
-
-	// Find the largest power of 10 (max of 9) that evenly divides the
-	// value.
-	exponent := uint64(0)
-	for amount%10 == 0 && exponent < 9 {
-		amount /= 10
-		exponent++
-	}
-
-	// The compressed result for exponents less than 9 is:
-	// 1 + 10*(9*n + d-1) + e
-	if exponent < 9 {
-		lastDigit := amount % 10
-		amount /= 10
-		return 1 + 10*(9*amount+lastDigit-1) + exponent
-	}
-
-	// The compressed result for an exponent of 9 is:
-	// 1 + 10*(n-1) + e   ==   10 + 10*(n-1)
-	return 10 + 10*(amount-1)
-}
-
-// decompressTxOutAmount returns the original amount the passed compressed
-// amount represents according to the domain specific compression algorithm
-// described above.
-func decompressTxOutAmount(amount uint64) uint64 {
-	// No need to do any work if it's zero.
-	if amount == 0 {
-		return 0
-	}
-
-	// The decompressed amount is either of the following two equations:
-	// x = 1 + 10*(9*n + d - 1) + e
-	// x = 1 + 10*(n - 1)       + 9
-	amount--
-
-	// The decompressed amount is now one of the following two equations:
-	// x = 10*(9*n + d - 1) + e
-	// x = 10*(n - 1)       + 9
-	exponent := amount % 10
-	amount /= 10
-
-	// The decompressed amount is now one of the following two equations:
-	// x = 9*n + d - 1  | where e < 9
-	// x = n - 1        | where e = 9
-	n := uint64(0)
-	if exponent < 9 {
-		lastDigit := amount%9 + 1
-		amount /= 9
-		n = amount*10 + lastDigit
-	} else {
-		n = amount + 1
-	}
-
-	// Apply the exponent.
-	for ; exponent > 0; exponent-- {
-		n *= 10
-	}
-
-	return n
-}
-
-// -----------------------------------------------------------------------------
-// Compressed transaction outputs consist of an amount and a public key script
-// both compressed using the domain specific compression algorithms previously
-// described.
-//
-// The serialized format is:
-//
-//   <compressed amount><compressed script>
-//
-//   Field                 Type     Size
-//     compressed amount   VLQ      variable
-//     compressed script   []byte   variable
-// -----------------------------------------------------------------------------
-
-// compressedTxOutSize returns the number of bytes the passed transaction output
-// fields would take when encoded with the format described above.
-func compressedTxOutSize(amount uint64, scriptPubKey []byte) int {
-	return serializeSizeVLQ(compressTxOutAmount(amount)) +
-		compressedScriptSize(scriptPubKey)
-}
-
-// putCompressedTxOut compresses the passed amount and script according to their
-// domain specific compression algorithms and encodes them directly into the
-// passed target byte slice with the format described above. The target byte
-// slice must be at least large enough to handle the number of bytes returned by
-// the compressedTxOutSize function or it will panic.
-func putCompressedTxOut(target []byte, amount uint64, scriptPubKey []byte) int {
-	offset := putVLQ(target, compressTxOutAmount(amount))
-	offset += putCompressedScript(target[offset:], scriptPubKey)
-	return offset
-}
-
-// decodeCompressedTxOut decodes the passed compressed txout, possibly followed
-// by other data, into its uncompressed amount and script and returns them along
-// with the number of bytes they occupied prior to decompression.
-func decodeCompressedTxOut(serialized []byte) (uint64, []byte, int, error) {
-	// Deserialize the compressed amount and ensure there are bytes
-	// remaining for the compressed script.
-	compressedAmount, bytesRead := deserializeVLQ(serialized)
-	if bytesRead >= len(serialized) {
-		return 0, nil, bytesRead, errDeserialize("unexpected end of " +
-			"data after compressed amount")
-	}
-
-	// Decode the compressed script size and ensure there are enough bytes
-	// left in the slice for it.
-	scriptSize := decodeCompressedScriptSize(serialized[bytesRead:])
-	if len(serialized[bytesRead:]) < scriptSize {
-		return 0, nil, bytesRead, errDeserialize("unexpected end of " +
-			"data after script size")
-	}
-
-	// Decompress and return the amount and script.
-	amount := decompressTxOutAmount(compressedAmount)
-	script := decompressScript(serialized[bytesRead : bytesRead+scriptSize])
-	return amount, script, bytesRead + scriptSize, nil
-}

blockdag/compress_test.go

@@ -1,436 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"bytes"
-	"encoding/hex"
-	"testing"
-)
-
-// hexToBytes converts the passed hex string into bytes and will panic if there
-// is an error. This is only provided for the hard-coded constants so errors in
-// the source code can be detected. It will only (and must only) be called with
-// hard-coded values.
-func hexToBytes(s string) []byte {
-	b, err := hex.DecodeString(s)
-	if err != nil {
-		panic("invalid hex in source file: " + s)
-	}
-	return b
-}
-
-// TestVLQ ensures the variable length quantity serialization, deserialization,
-// and size calculation works as expected.
-func TestVLQ(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		val        uint64
-		serialized []byte
-	}{
-		{0, hexToBytes("00")},
-		{1, hexToBytes("01")},
-		{127, hexToBytes("7f")},
-		{128, hexToBytes("8000")},
-		{129, hexToBytes("8001")},
-		{255, hexToBytes("807f")},
-		{256, hexToBytes("8100")},
-		{16383, hexToBytes("fe7f")},
-		{16384, hexToBytes("ff00")},
-		{16511, hexToBytes("ff7f")}, // Max 2-byte value
-		{16512, hexToBytes("808000")},
-		{16513, hexToBytes("808001")},
-		{16639, hexToBytes("80807f")},
-		{32895, hexToBytes("80ff7f")},
-		{2113663, hexToBytes("ffff7f")}, // Max 3-byte value
-		{2113664, hexToBytes("80808000")},
-		{270549119, hexToBytes("ffffff7f")}, // Max 4-byte value
-		{270549120, hexToBytes("8080808000")},
-		{2147483647, hexToBytes("86fefefe7f")},
-		{2147483648, hexToBytes("86fefeff00")},
-		{4294967295, hexToBytes("8efefefe7f")}, // Max uint32, 5 bytes
-		// Max uint64, 10 bytes
-		{18446744073709551615, hexToBytes("80fefefefefefefefe7f")},
-	}
-
-	for _, test := range tests {
-		// Ensure the function to calculate the serialized size without
-		// actually serializing the value is calculated properly.
-		gotSize := serializeSizeVLQ(test.val)
-		if gotSize != len(test.serialized) {
-			t.Errorf("serializeSizeVLQ: did not get expected size "+
-				"for %d - got %d, want %d", test.val, gotSize,
-				len(test.serialized))
-			continue
-		}
-
-		// Ensure the value serializes to the expected bytes.
-		gotBytes := make([]byte, gotSize)
-		gotBytesWritten := putVLQ(gotBytes, test.val)
-		if !bytes.Equal(gotBytes, test.serialized) {
-			t.Errorf("putVLQUnchecked: did not get expected bytes "+
-				"for %d - got %x, want %x", test.val, gotBytes,
-				test.serialized)
-			continue
-		}
-		if gotBytesWritten != len(test.serialized) {
-			t.Errorf("putVLQUnchecked: did not get expected number "+
-				"of bytes written for %d - got %d, want %d",
-				test.val, gotBytesWritten, len(test.serialized))
-			continue
-		}
-
-		// Ensure the serialized bytes deserialize to the expected
-		// value.
-		gotVal, gotBytesRead := deserializeVLQ(test.serialized)
-		if gotVal != test.val {
-			t.Errorf("deserializeVLQ: did not get expected value "+
-				"for %x - got %d, want %d", test.serialized,
-				gotVal, test.val)
-			continue
-		}
-		if gotBytesRead != len(test.serialized) {
-			t.Errorf("deserializeVLQ: did not get expected number "+
-				"of bytes read for %d - got %d, want %d",
-				test.serialized, gotBytesRead,
-				len(test.serialized))
-			continue
-		}
-	}
-}
-
-// TestScriptCompression ensures the domain-specific script compression and
-// decompression works as expected.
-func TestScriptCompression(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name         string
-		uncompressed []byte
-		compressed   []byte
-	}{
-		{
-			name:         "nil",
-			uncompressed: nil,
-			compressed:   hexToBytes("06"),
-		},
-		{
-			name:         "pay-to-pubkey-hash 1",
-			uncompressed: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
-			compressed:   hexToBytes("001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
-		},
-		{
-			name:         "pay-to-pubkey-hash 2",
-			uncompressed: hexToBytes("76a914e34cce70c86373273efcc54ce7d2a491bb4a0e8488ac"),
-			compressed:   hexToBytes("00e34cce70c86373273efcc54ce7d2a491bb4a0e84"),
-		},
-		{
-			name:         "pay-to-script-hash 1",
-			uncompressed: hexToBytes("a914da1745e9b549bd0bfa1a569971c77eba30cd5a4b87"),
-			compressed:   hexToBytes("01da1745e9b549bd0bfa1a569971c77eba30cd5a4b"),
-		},
-		{
-			name:         "pay-to-script-hash 2",
-			uncompressed: hexToBytes("a914f815b036d9bbbce5e9f2a00abd1bf3dc91e9551087"),
-			compressed:   hexToBytes("01f815b036d9bbbce5e9f2a00abd1bf3dc91e95510"),
-		},
-		{
-			name:         "pay-to-pubkey compressed 0x02",
-			uncompressed: hexToBytes("2102192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4ac"),
-			compressed:   hexToBytes("02192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
-		},
-		{
-			name:         "pay-to-pubkey compressed 0x03",
-			uncompressed: hexToBytes("2103b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e65ac"),
-			compressed:   hexToBytes("03b0bd634234abbb1ba1e986e884185c61cf43e001f9137f23c2c409273eb16e65"),
-		},
-		{
-			name:         "pay-to-pubkey uncompressed 0x04 even",
-			uncompressed: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
-			compressed:   hexToBytes("04192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
-		},
-		{
-			name:         "pay-to-pubkey uncompressed 0x04 odd",
-			uncompressed: hexToBytes("410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"),
-			compressed:   hexToBytes("0511db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5c"),
-		},
-		{
-			name:         "pay-to-pubkey invalid pubkey",
-			uncompressed: hexToBytes("3302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
-			compressed:   hexToBytes("293302aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"),
-		},
-		{
-			name:         "requires 2 size bytes - data push 200 bytes",
-			uncompressed: append(hexToBytes("4cc8"), bytes.Repeat([]byte{0x00}, 200)...),
-			// [0x80, 0x50] = 208 as a variable length quantity
-			// [0x4c, 0xc8] = OP_PUSHDATA1 200
-			compressed: append(hexToBytes("80504cc8"), bytes.Repeat([]byte{0x00}, 200)...),
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the function to calculate the serialized size without
-		// actually serializing the value is calculated properly.
-		gotSize := compressedScriptSize(test.uncompressed)
-		if gotSize != len(test.compressed) {
-			t.Errorf("compressedScriptSize (%s): did not get "+
-				"expected size - got %d, want %d", test.name,
-				gotSize, len(test.compressed))
-			continue
-		}
-
-		// Ensure the script compresses to the expected bytes.
-		gotCompressed := make([]byte, gotSize)
-		gotBytesWritten := putCompressedScript(gotCompressed,
-			test.uncompressed)
-		if !bytes.Equal(gotCompressed, test.compressed) {
-			t.Errorf("putCompressedScript (%s): did not get "+
-				"expected bytes - got %x, want %x", test.name,
-				gotCompressed, test.compressed)
-			continue
-		}
-		if gotBytesWritten != len(test.compressed) {
-			t.Errorf("putCompressedScript (%s): did not get "+
-				"expected number of bytes written - got %d, "+
-				"want %d", test.name, gotBytesWritten,
-				len(test.compressed))
-			continue
-		}
-
-		// Ensure the compressed script size is properly decoded from
-		// the compressed script.
-		gotDecodedSize := decodeCompressedScriptSize(test.compressed)
-		if gotDecodedSize != len(test.compressed) {
-			t.Errorf("decodeCompressedScriptSize (%s): did not get "+
-				"expected size - got %d, want %d", test.name,
-				gotDecodedSize, len(test.compressed))
-			continue
-		}
-
-		// Ensure the script decompresses to the expected bytes.
-		gotDecompressed := decompressScript(test.compressed)
-		if !bytes.Equal(gotDecompressed, test.uncompressed) {
-			t.Errorf("decompressScript (%s): did not get expected "+
-				"bytes - got %x, want %x", test.name,
-				gotDecompressed, test.uncompressed)
-			continue
-		}
-	}
-}
-
-// TestScriptCompressionErrors ensures calling various functions related to
-// script compression with incorrect data returns the expected results.
-func TestScriptCompressionErrors(t *testing.T) {
-	t.Parallel()
-
-	// A nil script must result in a decoded size of 0.
-	if gotSize := decodeCompressedScriptSize(nil); gotSize != 0 {
-		t.Fatalf("decodeCompressedScriptSize with nil script did not "+
-			"return 0 - got %d", gotSize)
-	}
-
-	// A nil script must result in a nil decompressed script.
-	if gotScript := decompressScript(nil); gotScript != nil {
-		t.Fatalf("decompressScript with nil script did not return nil "+
-			"decompressed script - got %x", gotScript)
-	}
-
-	// A compressed script for a pay-to-pubkey (uncompressed) that results
-	// in an invalid pubkey must result in a nil decompressed script.
-	compressedScript := hexToBytes("04012d74d0cb94344c9569c2e77901573d8d" +
-		"7903c3ebec3a957724895dca52c6b4")
-	if gotScript := decompressScript(compressedScript); gotScript != nil {
-		t.Fatalf("decompressScript with compressed pay-to-"+
-			"uncompressed-pubkey that is invalid did not return "+
-			"nil decompressed script - got %x", gotScript)
-	}
-}
-
-// TestAmountCompression ensures the domain-specific transaction output amount
-// compression and decompression works as expected.
-func TestAmountCompression(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name         string
-		uncompressed uint64
-		compressed   uint64
-	}{
-		{
-			name:         "0 KAS",
-			uncompressed: 0,
-			compressed:   0,
-		},
-		{
-			name:         "546 Sompi (current network dust value)",
-			uncompressed: 546,
-			compressed:   4911,
-		},
-		{
-			name:         "0.00001 KAS (typical transaction fee)",
-			uncompressed: 1000,
-			compressed:   4,
-		},
-		{
-			name:         "0.0001 KAS (typical transaction fee)",
-			uncompressed: 10000,
-			compressed:   5,
-		},
-		{
-			name:         "0.12345678 KAS",
-			uncompressed: 12345678,
-			compressed:   111111101,
-		},
-		{
-			name:         "0.5 KAS",
-			uncompressed: 50000000,
-			compressed:   48,
-		},
-		{
-			name:         "1 KAS",
-			uncompressed: 100000000,
-			compressed:   9,
-		},
-		{
-			name:         "5 KAS",
-			uncompressed: 500000000,
-			compressed:   49,
-		},
-		{
-			name:         "21000000 KAS (max minted coins)",
-			uncompressed: 2100000000000000,
-			compressed:   21000000,
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the amount compresses to the expected value.
-		gotCompressed := compressTxOutAmount(test.uncompressed)
-		if gotCompressed != test.compressed {
-			t.Errorf("compressTxOutAmount (%s): did not get "+
-				"expected value - got %d, want %d", test.name,
-				gotCompressed, test.compressed)
-			continue
-		}
-
-		// Ensure the value decompresses to the expected value.
-		gotDecompressed := decompressTxOutAmount(test.compressed)
-		if gotDecompressed != test.uncompressed {
-			t.Errorf("decompressTxOutAmount (%s): did not get "+
-				"expected value - got %d, want %d", test.name,
-				gotDecompressed, test.uncompressed)
-			continue
-		}
-	}
-}
-
-// TestCompressedTxOut ensures the transaction output serialization and
-// deserialization works as expected.
-func TestCompressedTxOut(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name         string
-		amount       uint64
-		scriptPubKey []byte
-		compressed   []byte
-	}{
-		{
-			name:         "pay-to-pubkey-hash dust",
-			amount:       546,
-			scriptPubKey: hexToBytes("76a9141018853670f9f3b0582c5b9ee8ce93764ac32b9388ac"),
-			compressed:   hexToBytes("a52f001018853670f9f3b0582c5b9ee8ce93764ac32b93"),
-		},
-		{
-			name:         "pay-to-pubkey uncompressed 1 KAS",
-			amount:       100000000,
-			scriptPubKey: hexToBytes("4104192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b40d45264838c0bd96852662ce6a847b197376830160c6d2eb5e6a4c44d33f453eac"),
-			compressed:   hexToBytes("0904192d74d0cb94344c9569c2e77901573d8d7903c3ebec3a957724895dca52c6b4"),
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the function to calculate the serialized size without
-		// actually serializing the txout is calculated properly.
-		gotSize := compressedTxOutSize(test.amount, test.scriptPubKey)
-		if gotSize != len(test.compressed) {
-			t.Errorf("compressedTxOutSize (%s): did not get "+
-				"expected size - got %d, want %d", test.name,
-				gotSize, len(test.compressed))
-			continue
-		}
-
-		// Ensure the txout compresses to the expected value.
-		gotCompressed := make([]byte, gotSize)
-		gotBytesWritten := putCompressedTxOut(gotCompressed,
-			test.amount, test.scriptPubKey)
-		if !bytes.Equal(gotCompressed, test.compressed) {
-			t.Errorf("compressTxOut (%s): did not get expected "+
-				"bytes - got %x, want %x", test.name,
-				gotCompressed, test.compressed)
-			continue
-		}
-		if gotBytesWritten != len(test.compressed) {
-			t.Errorf("compressTxOut (%s): did not get expected "+
-				"number of bytes written - got %d, want %d",
-				test.name, gotBytesWritten,
-				len(test.compressed))
-			continue
-		}
-
-		// Ensure the serialized bytes are decoded back to the expected
-		// uncompressed values.
-		gotAmount, gotScript, gotBytesRead, err := decodeCompressedTxOut(
-			test.compressed)
-		if err != nil {
-			t.Errorf("decodeCompressedTxOut (%s): unexpected "+
-				"error: %v", test.name, err)
-			continue
-		}
-		if gotAmount != test.amount {
-			t.Errorf("decodeCompressedTxOut (%s): did not get "+
-				"expected amount - got %d, want %d",
-				test.name, gotAmount, test.amount)
-			continue
-		}
-		if !bytes.Equal(gotScript, test.scriptPubKey) {
-			t.Errorf("decodeCompressedTxOut (%s): did not get "+
-				"expected script - got %x, want %x",
-				test.name, gotScript, test.scriptPubKey)
-			continue
-		}
-		if gotBytesRead != len(test.compressed) {
-			t.Errorf("decodeCompressedTxOut (%s): did not get "+
-				"expected number of bytes read - got %d, want %d",
-				test.name, gotBytesRead, len(test.compressed))
-			continue
-		}
-	}
-}
-
-// TestTxOutCompressionErrors ensures calling various functions related to
-// txout compression with incorrect data returns the expected results.
-func TestTxOutCompressionErrors(t *testing.T) {
-	t.Parallel()
-
-	// A compressed txout with missing compressed script must error.
-	compressedTxOut := hexToBytes("00")
-	_, _, _, err := decodeCompressedTxOut(compressedTxOut)
-	if !isDeserializeErr(err) {
-		t.Fatalf("decodeCompressedTxOut with missing compressed script "+
-			"did not return expected error type - got %T, want "+
-			"errDeserialize", err)
-	}
-
-	// A compressed txout with short compressed script must error.
-	compressedTxOut = hexToBytes("0010")
-	_, _, _, err = decodeCompressedTxOut(compressedTxOut)
-	if !isDeserializeErr(err) {
-		t.Fatalf("decodeCompressedTxOut with short compressed script "+
-			"did not return expected error type - got %T, want "+
-			"errDeserialize", err)
-	}
-}

blockdag/dag_test.go

@@ -6,14 +6,17 @@ package blockdag
 
 import (
 	"fmt"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/pkg/errors"
+	"math"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 	"time"
 
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
@@ -40,7 +43,7 @@ func TestBlockCount(t *testing.T) {
 	}
 
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestBlockCount", Config{
+	dag, teardownFunc, err := DAGSetup("TestBlockCount", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -93,7 +96,7 @@ func TestIsKnownBlock(t *testing.T) {
 	}
 
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("haveblock", Config{
+	dag, teardownFunc, err := DAGSetup("haveblock", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -204,7 +207,7 @@ func TestIsKnownBlock(t *testing.T) {
 		{hash: dagconfig.SimnetParams.GenesisHash.String(), want: true},
 
 		// Block 3b should be present (as a second child of Block 2).
-		{hash: "264176fb6072e2362db18f92d3f4b739cff071a206736df7c407c0bf9a1d7fef", want: true},
+		{hash: "2eb8903d3eb7f977ab329649f56f4125afa532662f7afe5dba0d4a3f1b93746f", want: true},
 
 		// Block 100000 should be present (as an orphan).
 		{hash: "65b20b048a074793ebfd1196e49341c8d194dabfc6b44a4fd0c607406e122baf", want: true},
@@ -550,18 +553,17 @@ func TestNew(t *testing.T) {
 
 	dbPath := filepath.Join(tempDir, "TestNew")
 	_ = os.RemoveAll(dbPath)
-	db, err := database.Create(testDbType, dbPath, blockDataNet)
+	err := dbaccess.Open(dbPath)
 	if err != nil {
 		t.Fatalf("error creating db: %s", err)
 	}
 	defer func() {
-		db.Close()
+		dbaccess.Close()
 		os.RemoveAll(dbPath)
 	}()
 	config := &Config{
 		DAGParams:  &dagconfig.SimnetParams,
-		DB:         db,
-		TimeSource: NewMedianTime(),
+		TimeSource: NewTimeSource(),
 		SigCache:   txscript.NewSigCache(1000),
 	}
 	_, err = New(config)
@@ -590,20 +592,19 @@ func TestAcceptingInInit(t *testing.T) {
 	// Create a test database
 	dbPath := filepath.Join(tempDir, "TestAcceptingInInit")
 	_ = os.RemoveAll(dbPath)
-	db, err := database.Create(testDbType, dbPath, blockDataNet)
+	err := dbaccess.Open(dbPath)
 	if err != nil {
 		t.Fatalf("error creating db: %s", err)
 	}
 	defer func() {
-		db.Close()
+		dbaccess.Close()
 		os.RemoveAll(dbPath)
 	}()
 
 	// Create a DAG to add the test block into
 	config := &Config{
 		DAGParams:  &dagconfig.SimnetParams,
-		DB:         db,
-		TimeSource: NewMedianTime(),
+		TimeSource: NewTimeSource(),
 		SigCache:   txscript.NewSigCache(1000),
 	}
 	dag, err := New(config)
@@ -620,21 +621,38 @@ func TestAcceptingInInit(t *testing.T) {
 	testBlock := blocks[1]
 
 	// Create a test blockNode with an unvalidated status
-	genesisNode := dag.index.LookupNode(genesisBlock.Hash())
+	genesisNode, ok := dag.index.LookupNode(genesisBlock.Hash())
+	if !ok {
+		t.Fatalf("genesis block does not exist in the DAG")
+	}
 	testNode, _ := dag.newBlockNode(&testBlock.MsgBlock().Header, blockSetFromSlice(genesisNode))
 	testNode.status = statusDataStored
 
 	// Manually add the test block to the database
-	err = db.Update(func(dbTx database.Tx) error {
-		err := dbStoreBlock(dbTx, testBlock)
-		if err != nil {
-			return err
-		}
-		return dbStoreBlockNode(dbTx, testNode)
-	})
+	dbTx, err := dbaccess.NewTx()
+	if err != nil {
+		t.Fatalf("Failed to open database "+
+			"transaction: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = storeBlock(dbTx, testBlock)
+	if err != nil {
+		t.Fatalf("Failed to store block: %s", err)
+	}
+	dbTestNode, err := serializeBlockNode(testNode)
+	if err != nil {
+		t.Fatalf("Failed to serialize blockNode: %s", err)
+	}
+	key := blockIndexKey(testNode.hash, testNode.blueScore)
+	err = dbaccess.StoreIndexBlock(dbTx, key, dbTestNode)
 	if err != nil {
 		t.Fatalf("Failed to update block index: %s", err)
 	}
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Failed to commit database "+
+			"transaction: %s", err)
+	}
 
 	// Create a new DAG. We expect this DAG to process the
 	// test node
@@ -644,7 +662,11 @@ func TestAcceptingInInit(t *testing.T) {
 	}
 
 	// Make sure that the test node's status is valid
-	testNode = dag.index.LookupNode(testBlock.Hash())
+	testNode, ok = dag.index.LookupNode(testBlock.Hash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", testBlock.Hash())
+	}
+
 	if testNode.status&statusValid == 0 {
 		t.Fatalf("testNode is unexpectedly invalid")
 	}
@@ -654,7 +676,7 @@ func TestConfirmations(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
 	params.K = 1
-	dag, teardownFunc, err := DAGSetup("TestConfirmations", Config{
+	dag, teardownFunc, err := DAGSetup("TestConfirmations", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -676,7 +698,7 @@ func TestConfirmations(t *testing.T) {
 	chainBlocks := make([]*wire.MsgBlock, 5)
 	chainBlocks[0] = dag.dagParams.GenesisBlock
 	for i := uint32(1); i < 5; i++ {
-		chainBlocks[i] = prepareAndProcessBlock(t, dag, chainBlocks[i-1])
+		chainBlocks[i] = prepareAndProcessBlockByParentMsgBlocks(t, dag, chainBlocks[i-1])
 	}
 
 	// Make sure that each one of the chain blocks has the expected confirmations number
@@ -695,8 +717,8 @@ func TestConfirmations(t *testing.T) {
 
 	branchingBlocks := make([]*wire.MsgBlock, 2)
 	// Add two branching blocks
-	branchingBlocks[0] = prepareAndProcessBlock(t, dag, chainBlocks[1])
-	branchingBlocks[1] = prepareAndProcessBlock(t, dag, branchingBlocks[0])
+	branchingBlocks[0] = prepareAndProcessBlockByParentMsgBlocks(t, dag, chainBlocks[1])
+	branchingBlocks[1] = prepareAndProcessBlockByParentMsgBlocks(t, dag, branchingBlocks[0])
 
 	// Check that the genesis has a confirmations number == len(chainBlocks)
 	genesisConfirmations, err = dag.blockConfirmations(dag.genesis)
@@ -726,7 +748,7 @@ func TestConfirmations(t *testing.T) {
 	// Generate 100 blocks to force the "main" chain to become red
 	branchingChainTip := branchingBlocks[1]
 	for i := uint32(0); i < 100; i++ {
-		nextBranchingChainTip := prepareAndProcessBlock(t, dag, branchingChainTip)
+		nextBranchingChainTip := prepareAndProcessBlockByParentMsgBlocks(t, dag, branchingChainTip)
 		branchingChainTip = nextBranchingChainTip
 	}
 
@@ -757,7 +779,7 @@ func TestAcceptingBlock(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
 	params.K = 3
-	dag, teardownFunc, err := DAGSetup("TestAcceptingBlock", Config{
+	dag, teardownFunc, err := DAGSetup("TestAcceptingBlock", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -785,7 +807,7 @@ func TestAcceptingBlock(t *testing.T) {
 	chainBlocks := make([]*wire.MsgBlock, numChainBlocks)
 	chainBlocks[0] = dag.dagParams.GenesisBlock
 	for i := uint32(1); i <= numChainBlocks-1; i++ {
-		chainBlocks[i] = prepareAndProcessBlock(t, dag, chainBlocks[i-1])
+		chainBlocks[i] = prepareAndProcessBlockByParentMsgBlocks(t, dag, chainBlocks[i-1])
 	}
 
 	// Make sure that each chain block (including the genesis) is accepted by its child
@@ -813,7 +835,7 @@ func TestAcceptingBlock(t *testing.T) {
 
 	// Generate a chain tip that will be in the anticone of the selected tip and
 	// in dag.virtual.blues.
-	branchingChainTip := prepareAndProcessBlock(t, dag, chainBlocks[len(chainBlocks)-3])
+	branchingChainTip := prepareAndProcessBlockByParentMsgBlocks(t, dag, chainBlocks[len(chainBlocks)-3])
 
 	// Make sure that branchingChainTip is not in the selected parent chain
 	isBranchingChainTipInSelectedParentChain, err := dag.IsInSelectedParentChain(branchingChainTip.BlockHash())
@@ -851,7 +873,7 @@ func TestAcceptingBlock(t *testing.T) {
 	intersectionBlock := chainBlocks[1]
 	sideChainTip := intersectionBlock
 	for i := 0; i < len(chainBlocks)-3; i++ {
-		sideChainTip = prepareAndProcessBlock(t, dag, sideChainTip)
+		sideChainTip = prepareAndProcessBlockByParentMsgBlocks(t, dag, sideChainTip)
 	}
 
 	// Make sure that the accepting block of the parent of the branching block didn't change
@@ -867,7 +889,7 @@ func TestAcceptingBlock(t *testing.T) {
 
 	// Make sure that a block that is found in the red set of the selected tip
 	// doesn't have an accepting block
-	prepareAndProcessBlock(t, dag, sideChainTip, chainBlocks[len(chainBlocks)-1])
+	prepareAndProcessBlockByParentMsgBlocks(t, dag, sideChainTip, chainBlocks[len(chainBlocks)-1])
 
 	sideChainTipAcceptingBlock, err := acceptingBlockByMsgBlock(sideChainTip)
 	if err != nil {
@@ -887,7 +909,7 @@ func TestFinalizeNodesBelowFinalityPoint(t *testing.T) {
 func testFinalizeNodesBelowFinalityPoint(t *testing.T, deleteDiffData bool) {
 	params := dagconfig.SimnetParams
 	params.K = 1
-	dag, teardownFunc, err := DAGSetup("testFinalizeNodesBelowFinalityPoint", Config{
+	dag, teardownFunc, err := DAGSetup("testFinalizeNodesBelowFinalityPoint", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -899,13 +921,20 @@ func testFinalizeNodesBelowFinalityPoint(t *testing.T, deleteDiffData bool) {
 	blockTime := dag.genesis.Header().Timestamp
 
 	flushUTXODiffStore := func() {
-		err := dag.db.Update(func(dbTx database.Tx) error {
-			return dag.utxoDiffStore.flushToDB(dbTx)
-		})
+		dbTx, err := dbaccess.NewTx()
+		if err != nil {
+			t.Fatalf("Failed to open database transaction: %s", err)
+		}
+		defer dbTx.RollbackUnlessClosed()
+		err = dag.utxoDiffStore.flushToDB(dbTx)
 		if err != nil {
 			t.Fatalf("Error flushing utxoDiffStore data to DB: %s", err)
 		}
 		dag.utxoDiffStore.clearDirtyEntries()
+		err = dbTx.Commit()
+		if err != nil {
+			t.Fatalf("Failed to commit database transaction: %s", err)
+		}
 	}
 
 	addNode := func(parent *blockNode) *blockNode {
@@ -934,6 +963,11 @@ func testFinalizeNodesBelowFinalityPoint(t *testing.T, deleteDiffData bool) {
 	// Manually set the last finality point
 	dag.lastFinalityPoint = nodes[finalityInterval-1]
 
+	// Don't unload diffData
+	currentDifference := maxBlueScoreDifferenceToKeepLoaded
+	maxBlueScoreDifferenceToKeepLoaded = math.MaxUint64
+	defer func() { maxBlueScoreDifferenceToKeepLoaded = currentDifference }()
+
 	dag.finalizeNodesBelowFinalityPoint(deleteDiffData)
 	flushUTXODiffStore()
 
@@ -941,17 +975,27 @@ func testFinalizeNodesBelowFinalityPoint(t *testing.T, deleteDiffData bool) {
 		if !node.isFinalized {
 			t.Errorf("Node with blue score %d expected to be finalized", node.blueScore)
 		}
-		if _, ok := dag.utxoDiffStore.loaded[*node.hash]; deleteDiffData && ok {
+		if _, ok := dag.utxoDiffStore.loaded[node]; deleteDiffData && ok {
 			t.Errorf("The diff data of node with blue score %d should have been unloaded if deleteDiffData is %T", node.blueScore, deleteDiffData)
 		} else if !deleteDiffData && !ok {
 			t.Errorf("The diff data of node with blue score %d shouldn't have been unloaded if deleteDiffData is %T", node.blueScore, deleteDiffData)
 		}
-		if diffData, err := dag.utxoDiffStore.diffDataFromDB(node.hash); err != nil {
+
+		_, err := dag.utxoDiffStore.diffDataFromDB(node.hash)
+		exists := !dbaccess.IsNotFoundError(err)
+		if exists && err != nil {
 			t.Errorf("diffDataFromDB: %s", err)
-		} else if deleteDiffData && diffData != nil {
+			continue
+		}
+
+		if deleteDiffData && exists {
 			t.Errorf("The diff data of node with blue score %d should have been deleted from the database if deleteDiffData is %T", node.blueScore, deleteDiffData)
-		} else if !deleteDiffData && diffData == nil {
+			continue
+		}
+
+		if !deleteDiffData && !exists {
 			t.Errorf("The diff data of node with blue score %d shouldn't have been deleted from the database if deleteDiffData is %T", node.blueScore, deleteDiffData)
+			continue
 		}
 	}
 
@@ -959,7 +1003,7 @@ func testFinalizeNodesBelowFinalityPoint(t *testing.T, deleteDiffData bool) {
 		if node.isFinalized {
 			t.Errorf("Node with blue score %d wasn't expected to be finalized", node.blueScore)
 		}
-		if _, ok := dag.utxoDiffStore.loaded[*node.hash]; !ok {
+		if _, ok := dag.utxoDiffStore.loaded[node]; !ok {
 			t.Errorf("The diff data of node with blue score %d shouldn't have been unloaded", node.blueScore)
 		}
 		if diffData, err := dag.utxoDiffStore.diffDataFromDB(node.hash); err != nil {
@@ -972,7 +1016,7 @@ func testFinalizeNodesBelowFinalityPoint(t *testing.T, deleteDiffData bool) {
 
 func TestDAGIndexFailedStatus(t *testing.T) {
 	params := dagconfig.SimnetParams
-	dag, teardownFunc, err := DAGSetup("TestDAGIndexFailedStatus", Config{
+	dag, teardownFunc, err := DAGSetup("TestDAGIndexFailedStatus", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -1008,8 +1052,8 @@ func TestDAGIndexFailedStatus(t *testing.T) {
 			"is an orphan\n")
 	}
 
-	invalidBlockNode := dag.index.LookupNode(invalidBlock.Hash())
-	if invalidBlockNode == nil {
+	invalidBlockNode, ok := dag.index.LookupNode(invalidBlock.Hash())
+	if !ok {
 		t.Fatalf("invalidBlockNode wasn't added to the block index as expected")
 	}
 	if invalidBlockNode.status&statusValidateFailed != statusValidateFailed {
@@ -1037,8 +1081,8 @@ func TestDAGIndexFailedStatus(t *testing.T) {
 		t.Fatalf("ProcessBlock incorrectly returned invalidBlockChild " +
 			"is an orphan\n")
 	}
-	invalidBlockChildNode := dag.index.LookupNode(invalidBlockChild.Hash())
-	if invalidBlockChildNode == nil {
+	invalidBlockChildNode, ok := dag.index.LookupNode(invalidBlockChild.Hash())
+	if !ok {
 		t.Fatalf("invalidBlockChild wasn't added to the block index as expected")
 	}
 	if invalidBlockChildNode.status&statusInvalidAncestor != statusInvalidAncestor {
@@ -1065,8 +1109,8 @@ func TestDAGIndexFailedStatus(t *testing.T) {
 		t.Fatalf("ProcessBlock incorrectly returned invalidBlockGrandChild " +
 			"is an orphan\n")
 	}
-	invalidBlockGrandChildNode := dag.index.LookupNode(invalidBlockGrandChild.Hash())
-	if invalidBlockGrandChildNode == nil {
+	invalidBlockGrandChildNode, ok := dag.index.LookupNode(invalidBlockGrandChild.Hash())
+	if !ok {
 		t.Fatalf("invalidBlockGrandChild wasn't added to the block index as expected")
 	}
 	if invalidBlockGrandChildNode.status&statusInvalidAncestor != statusInvalidAncestor {
@@ -1088,3 +1132,277 @@ func TestIsDAGCurrentMaxDiff(t *testing.T) {
 		}
 	}
 }
+
+func testProcessBlockRuleError(t *testing.T, dag *BlockDAG, block *wire.MsgBlock, expectedRuleErr error) {
+	isOrphan, isDelayed, err := dag.ProcessBlock(util.NewBlock(block), BFNoPoWCheck)
+
+	err = checkRuleError(err, expectedRuleErr)
+	if err != nil {
+		t.Errorf("checkRuleError: %s", err)
+	}
+
+	if isDelayed {
+		t.Fatalf("ProcessBlock: block " +
+			"is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("ProcessBlock: block got unexpectedly orphaned")
+	}
+}
+
+func TestDoubleSpends(t *testing.T) {
+	params := dagconfig.SimnetParams
+	params.BlockCoinbaseMaturity = 0
+	// Create a new database and dag instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestDoubleSpends", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	fundingBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{params.GenesisHash}, nil)
+	cbTx := fundingBlock.Transactions[0]
+
+	signatureScript, err := txscript.PayToScriptHashSignatureScript(OpTrueScript, nil)
+	if err != nil {
+		t.Fatalf("Failed to build signature script: %s", err)
+	}
+	txIn := &wire.TxIn{
+		PreviousOutpoint: wire.Outpoint{TxID: *cbTx.TxID(), Index: 0},
+		SignatureScript:  signatureScript,
+		Sequence:         wire.MaxTxInSequenceNum,
+	}
+	txOut := &wire.TxOut{
+		ScriptPubKey: OpTrueScript,
+		Value:        uint64(1),
+	}
+	tx1 := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
+
+	doubleSpendTxOut := &wire.TxOut{
+		ScriptPubKey: OpTrueScript,
+		Value:        uint64(2),
+	}
+	doubleSpendTx1 := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{doubleSpendTxOut})
+
+	blockWithTx1 := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{fundingBlock.BlockHash()}, []*wire.MsgTx{tx1})
+
+	// Check that a block will be rejected if it has a transaction that already exists in its past.
+	anotherBlockWithTx1, err := PrepareBlockForTest(dag, []*daghash.Hash{blockWithTx1.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Manually add tx1.
+	anotherBlockWithTx1.Transactions = append(anotherBlockWithTx1.Transactions, tx1)
+	anotherBlockWithTx1UtilTxs := make([]*util.Tx, len(anotherBlockWithTx1.Transactions))
+	for i, tx := range anotherBlockWithTx1.Transactions {
+		anotherBlockWithTx1UtilTxs[i] = util.NewTx(tx)
+	}
+	anotherBlockWithTx1.Header.HashMerkleRoot = BuildHashMerkleTreeStore(anotherBlockWithTx1UtilTxs).Root()
+
+	testProcessBlockRuleError(t, dag, anotherBlockWithTx1, ruleError(ErrOverwriteTx, ""))
+
+	// Check that a block will be rejected if it has a transaction that double spends
+	// a transaction from its past.
+	blockWithDoubleSpendForTx1, err := PrepareBlockForTest(dag, []*daghash.Hash{blockWithTx1.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Manually add a transaction that double spends the block past.
+	blockWithDoubleSpendForTx1.Transactions = append(blockWithDoubleSpendForTx1.Transactions, doubleSpendTx1)
+	blockWithDoubleSpendForTx1UtilTxs := make([]*util.Tx, len(blockWithDoubleSpendForTx1.Transactions))
+	for i, tx := range blockWithDoubleSpendForTx1.Transactions {
+		blockWithDoubleSpendForTx1UtilTxs[i] = util.NewTx(tx)
+	}
+	blockWithDoubleSpendForTx1.Header.HashMerkleRoot = BuildHashMerkleTreeStore(blockWithDoubleSpendForTx1UtilTxs).Root()
+
+	testProcessBlockRuleError(t, dag, blockWithDoubleSpendForTx1, ruleError(ErrMissingTxOut, ""))
+
+	blockInAnticoneOfBlockWithTx1, err := PrepareBlockForTest(dag, []*daghash.Hash{fundingBlock.BlockHash()}, []*wire.MsgTx{doubleSpendTx1})
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Check that a block will not get rejected if it has a transaction that double spends
+	// a transaction from its anticone.
+	testProcessBlockRuleError(t, dag, blockInAnticoneOfBlockWithTx1, nil)
+
+	// Check that a block will be rejected if it has two transactions that spend the same UTXO.
+	blockWithDoubleSpendWithItself, err := PrepareBlockForTest(dag, []*daghash.Hash{fundingBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Manually add tx1 and doubleSpendTx1.
+	blockWithDoubleSpendWithItself.Transactions = append(blockWithDoubleSpendWithItself.Transactions, tx1, doubleSpendTx1)
+	blockWithDoubleSpendWithItselfUtilTxs := make([]*util.Tx, len(blockWithDoubleSpendWithItself.Transactions))
+	for i, tx := range blockWithDoubleSpendWithItself.Transactions {
+		blockWithDoubleSpendWithItselfUtilTxs[i] = util.NewTx(tx)
+	}
+	blockWithDoubleSpendWithItself.Header.HashMerkleRoot = BuildHashMerkleTreeStore(blockWithDoubleSpendWithItselfUtilTxs).Root()
+
+	testProcessBlockRuleError(t, dag, blockWithDoubleSpendWithItself, ruleError(ErrDoubleSpendInSameBlock, ""))
+
+	// Check that a block will be rejected if it has the same transaction twice.
+	blockWithDuplicateTransaction, err := PrepareBlockForTest(dag, []*daghash.Hash{fundingBlock.BlockHash()}, nil)
+	if err != nil {
+		t.Fatalf("PrepareBlockForTest: %v", err)
+	}
+
+	// Manually add tx1 twice.
+	blockWithDuplicateTransaction.Transactions = append(blockWithDuplicateTransaction.Transactions, tx1, tx1)
+	blockWithDuplicateTransactionUtilTxs := make([]*util.Tx, len(blockWithDuplicateTransaction.Transactions))
+	for i, tx := range blockWithDuplicateTransaction.Transactions {
+		blockWithDuplicateTransactionUtilTxs[i] = util.NewTx(tx)
+	}
+	blockWithDuplicateTransaction.Header.HashMerkleRoot = BuildHashMerkleTreeStore(blockWithDuplicateTransactionUtilTxs).Root()
+	testProcessBlockRuleError(t, dag, blockWithDuplicateTransaction, ruleError(ErrDuplicateTx, ""))
+}
+
+func TestUTXOCommitment(t *testing.T) {
+	// Create a new database and dag instance to run tests against.
+	params := dagconfig.SimnetParams
+	params.BlockCoinbaseMaturity = 0
+	dag, teardownFunc, err := DAGSetup("TestUTXOCommitment", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestUTXOCommitment: Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	resetExtraNonceForTest()
+
+	createTx := func(txToSpend *wire.MsgTx) *wire.MsgTx {
+		scriptPubKey, err := txscript.PayToScriptHashScript(OpTrueScript)
+		if err != nil {
+			t.Fatalf("TestUTXOCommitment: failed to build script pub key: %s", err)
+		}
+		signatureScript, err := txscript.PayToScriptHashSignatureScript(OpTrueScript, nil)
+		if err != nil {
+			t.Fatalf("TestUTXOCommitment: failed to build signature script: %s", err)
+		}
+		txIn := &wire.TxIn{
+			PreviousOutpoint: wire.Outpoint{TxID: *txToSpend.TxID(), Index: 0},
+			SignatureScript:  signatureScript,
+			Sequence:         wire.MaxTxInSequenceNum,
+		}
+		txOut := &wire.TxOut{
+			ScriptPubKey: scriptPubKey,
+			Value:        uint64(1),
+		}
+		return wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{txIn}, []*wire.TxOut{txOut})
+	}
+
+	// Build the following DAG:
+	// G <- A <- B <- D
+	//        <- C <-
+	genesis := params.GenesisBlock
+
+	// Block A:
+	blockA := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{genesis.BlockHash()}, nil)
+
+	// Block B:
+	blockB := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockA.BlockHash()}, nil)
+
+	// Block C:
+	txSpendBlockACoinbase := createTx(blockA.Transactions[0])
+	blockCTxs := []*wire.MsgTx{txSpendBlockACoinbase}
+	blockC := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockA.BlockHash()}, blockCTxs)
+
+	// Block D:
+	txSpendTxInBlockC := createTx(txSpendBlockACoinbase)
+	blockDTxs := []*wire.MsgTx{txSpendTxInBlockC}
+	blockD := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockB.BlockHash(), blockC.BlockHash()}, blockDTxs)
+
+	// Get the pastUTXO of blockD
+	blockNodeD, ok := dag.index.LookupNode(blockD.BlockHash())
+	if !ok {
+		t.Fatalf("TestUTXOCommitment: blockNode for block D not found")
+	}
+	blockDPastUTXO, _, _, _ := dag.pastUTXO(blockNodeD)
+	blockDPastDiffUTXOSet := blockDPastUTXO.(*DiffUTXOSet)
+
+	// Build a Multiset for block D
+	multiset := secp256k1.NewMultiset()
+	for outpoint, entry := range blockDPastDiffUTXOSet.base.utxoCollection {
+		var err error
+		multiset, err = addUTXOToMultiset(multiset, entry, &outpoint)
+		if err != nil {
+			t.Fatalf("TestUTXOCommitment: addUTXOToMultiset unexpectedly failed")
+		}
+	}
+	for outpoint, entry := range blockDPastDiffUTXOSet.UTXODiff.toAdd {
+		var err error
+		multiset, err = addUTXOToMultiset(multiset, entry, &outpoint)
+		if err != nil {
+			t.Fatalf("TestUTXOCommitment: addUTXOToMultiset unexpectedly failed")
+		}
+	}
+	for outpoint, entry := range blockDPastDiffUTXOSet.UTXODiff.toRemove {
+		var err error
+		multiset, err = removeUTXOFromMultiset(multiset, entry, &outpoint)
+		if err != nil {
+			t.Fatalf("TestUTXOCommitment: removeUTXOFromMultiset unexpectedly failed")
+		}
+	}
+
+	// Turn the multiset into a UTXO commitment
+	utxoCommitment := daghash.Hash(*multiset.Finalize())
+
+	// Make sure that the two commitments are equal
+	if !utxoCommitment.IsEqual(blockNodeD.utxoCommitment) {
+		t.Fatalf("TestUTXOCommitment: calculated UTXO commitment and "+
+			"actual UTXO commitment don't match. Want: %s, got: %s",
+			utxoCommitment, blockNodeD.utxoCommitment)
+	}
+}
+
+func TestPastUTXOMultiSet(t *testing.T) {
+	// Create a new database and dag instance to run tests against.
+	params := dagconfig.SimnetParams
+	dag, teardownFunc, err := DAGSetup("TestPastUTXOMultiSet", true, Config{
+		DAGParams: &params,
+	})
+	if err != nil {
+		t.Fatalf("TestPastUTXOMultiSet: Failed to setup dag instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Build a short chain
+	genesis := params.GenesisBlock
+	blockA := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{genesis.BlockHash()}, nil)
+	blockB := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockA.BlockHash()}, nil)
+	blockC := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockB.BlockHash()}, nil)
+
+	// Take blockC's selectedParentMultiset
+	blockNodeC, ok := dag.index.LookupNode(blockC.BlockHash())
+	if !ok {
+		t.Fatalf("TestPastUTXOMultiSet: blockNode for blockC not found")
+	}
+	blockCSelectedParentMultiset, err := blockNodeC.selectedParentMultiset(dag)
+	if err != nil {
+		t.Fatalf("TestPastUTXOMultiSet: selectedParentMultiset unexpectedly failed: %s", err)
+	}
+
+	// Copy the multiset
+	blockCSelectedParentMultisetCopy := *blockCSelectedParentMultiset
+	blockCSelectedParentMultiset = &blockCSelectedParentMultisetCopy
+
+	// Add a block on top of blockC
+	PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockC.BlockHash()}, nil)
+
+	// Get blockC's selectedParentMultiset again
+	blockCSelectedParentMultiSetAfterAnotherBlock, err := blockNodeC.selectedParentMultiset(dag)
+	if err != nil {
+		t.Fatalf("TestPastUTXOMultiSet: selectedParentMultiset unexpectedly failed: %s", err)
+	}
+
+	// Make sure that blockC's selectedParentMultiset had not changed
+	if !reflect.DeepEqual(blockCSelectedParentMultiset, blockCSelectedParentMultiSetAfterAnotherBlock) {
+		t.Fatalf("TestPastUTXOMultiSet: selectedParentMultiset appears to have changed")
+	}
+}

blockdag/dagio_test.go

@@ -6,11 +6,11 @@ package blockdag
 
 import (
 	"bytes"
+	"encoding/hex"
 	"github.com/pkg/errors"
 	"reflect"
 	"testing"
 
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
 
@@ -36,9 +36,21 @@ func TestErrNotInDAG(t *testing.T) {
 	}
 }
 
-// TestUtxoSerialization ensures serializing and deserializing unspent
+// hexToBytes converts the passed hex string into bytes and will panic if there
+// is an error. This is only provided for the hard-coded constants so errors in
+// the source code can be detected. It will only (and must only) be called with
+// hard-coded values.
+func hexToBytes(s string) []byte {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		panic("invalid hex in source file: " + s)
+	}
+	return b
+}
+
+// TestUTXOSerialization ensures serializing and deserializing unspent
 // trasaction output entries works as expected.
-func TestUtxoSerialization(t *testing.T) {
+func TestUTXOSerialization(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
@@ -54,7 +66,7 @@ func TestUtxoSerialization(t *testing.T) {
 				blockBlueScore: 1,
 				packedFlags:    tfCoinbase,
 			},
-			serialized: hexToBytes("03320496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52"),
+			serialized: hexToBytes("01000000000000000100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac"),
 		},
 		{
 			name: "blue score 100001, not coinbase",
@@ -64,13 +76,21 @@ func TestUtxoSerialization(t *testing.T) {
 				blockBlueScore: 100001,
 				packedFlags:    0,
 			},
-			serialized: hexToBytes("8b99420700ee8bd501094a7d5ca318da2506de35e1cb025ddc"),
+			serialized: hexToBytes("a1860100000000000040420f00000000001976a914ee8bd501094a7d5ca318da2506de35e1cb025ddc88ac"),
 		},
 	}
 
 	for i, test := range tests {
 		// Ensure the utxo entry serializes to the expected value.
-		gotBytes := serializeUTXOEntry(test.entry)
+		w := &bytes.Buffer{}
+		err := serializeUTXOEntry(w, test.entry)
+		if err != nil {
+			t.Errorf("serializeUTXOEntry #%d (%s) unexpected "+
+				"error: %v", i, test.name, err)
+			continue
+		}
+
+		gotBytes := w.Bytes()
 		if !bytes.Equal(gotBytes, test.serialized) {
 			t.Errorf("serializeUTXOEntry #%d (%s): mismatched "+
 				"bytes - got %x, want %x", i, test.name,
@@ -78,8 +98,8 @@ func TestUtxoSerialization(t *testing.T) {
 			continue
 		}
 
-		// Deserialize to a utxo entry.
-		utxoEntry, err := deserializeUTXOEntry(test.serialized)
+		// Deserialize to a utxo entry.gotBytes
+		utxoEntry, err := deserializeUTXOEntry(bytes.NewReader(test.serialized))
 		if err != nil {
 			t.Errorf("deserializeUTXOEntry #%d (%s) unexpected "+
 				"error: %v", i, test.name, err)
@@ -124,28 +144,24 @@ func TestUtxoEntryDeserializeErrors(t *testing.T) {
 	tests := []struct {
 		name       string
 		serialized []byte
-		errType    error
 	}{
 		{
 			name:       "no data after header code",
 			serialized: hexToBytes("02"),
-			errType:    errDeserialize(""),
 		},
 		{
 			name:       "incomplete compressed txout",
 			serialized: hexToBytes("0232"),
-			errType:    errDeserialize(""),
 		},
 	}
 
 	for _, test := range tests {
 		// Ensure the expected error type is returned and the returned
 		// entry is nil.
-		entry, err := deserializeUTXOEntry(test.serialized)
-		if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
-			t.Errorf("deserializeUTXOEntry (%s): expected error "+
-				"type does not match - got %T, want %T",
-				test.name, err, test.errType)
+		entry, err := deserializeUTXOEntry(bytes.NewReader(test.serialized))
+		if err == nil {
+			t.Errorf("deserializeUTXOEntry (%s): didn't return an error",
+				test.name)
 			continue
 		}
 		if entry != nil {
@@ -172,7 +188,7 @@ func TestDAGStateSerialization(t *testing.T) {
 				TipHashes:         []*daghash.Hash{newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f")},
 				LastFinalityPoint: newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"),
 			},
-			serialized: []byte("{\"TipHashes\":[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]}"),
+			serialized: []byte("{\"TipHashes\":[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0],\"LocalSubnetworkID\":null}"),
 		},
 		{
 			name: "block 1",
@@ -180,7 +196,7 @@ func TestDAGStateSerialization(t *testing.T) {
 				TipHashes:         []*daghash.Hash{newHashFromStr("00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048")},
 				LastFinalityPoint: newHashFromStr("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"),
 			},
-			serialized: []byte("{\"TipHashes\":[[72,96,235,24,191,27,22,32,227,126,148,144,252,138,66,117,20,65,111,215,81,89,171,134,104,142,154,131,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0]}"),
+			serialized: []byte("{\"TipHashes\":[[72,96,235,24,191,27,22,32,227,126,148,144,252,138,66,117,20,65,111,215,81,89,171,134,104,142,154,131,0,0,0,0]],\"LastFinalityPoint\":[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,79,147,30,131,101,225,90,8,156,104,214,25,0,0,0,0,0],\"LocalSubnetworkID\":null}"),
 		},
 	}
 
@@ -217,51 +233,6 @@ func TestDAGStateSerialization(t *testing.T) {
 	}
 }
 
-// TestDAGStateDeserializeErrors performs negative tests against
-// deserializing the DAG state to ensure error paths work as expected.
-func TestDAGStateDeserializeErrors(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name       string
-		serialized []byte
-		errType    error
-	}{
-		{
-			name:       "nothing serialized",
-			serialized: hexToBytes(""),
-			errType:    database.Error{ErrorCode: database.ErrCorruption},
-		},
-		{
-			name:       "corrupted data",
-			serialized: []byte("[[111,226,140,10,182,241,179,114,193,166,162,70,174,99,247,7"),
-			errType:    database.Error{ErrorCode: database.ErrCorruption},
-		},
-	}
-
-	for _, test := range tests {
-		// Ensure the expected error type and code is returned.
-		_, err := deserializeDAGState(test.serialized)
-		if reflect.TypeOf(err) != reflect.TypeOf(test.errType) {
-			t.Errorf("deserializeDAGState (%s): expected "+
-				"error type does not match - got %T, want %T",
-				test.name, err, test.errType)
-			continue
-		}
-		var dbErr database.Error
-		if ok := errors.As(err, &dbErr); ok {
-			tderr := test.errType.(database.Error)
-			if dbErr.ErrorCode != tderr.ErrorCode {
-				t.Errorf("deserializeDAGState (%s): "+
-					"wrong error code got: %v, want: %v",
-					test.name, dbErr.ErrorCode,
-					tderr.ErrorCode)
-				continue
-			}
-		}
-	}
-}
-
 // newHashFromStr converts the passed big-endian hex string into a
 // daghash.Hash. It only differs from the one available in daghash in that
 // it panics in case of an error since it will only (and must only) be

blockdag/difficulty.go

@@ -5,7 +5,7 @@
 package blockdag
 
 import (
-	"math/big"
+	"github.com/kaspanet/kaspad/util/bigintpool"
 	"time"
 
 	"github.com/kaspanet/kaspad/util"
@@ -30,11 +30,20 @@ func (dag *BlockDAG) requiredDifficulty(bluestParent *blockNode, newBlockTime ti
 	// averageWindowTarget * (windowMinTimestamp / (targetTimePerBlock * windowSize))
 	// The result uses integer division which means it will be slightly
 	// rounded down.
-	newTarget := targetsWindow.averageTarget()
+	newTarget := bigintpool.Acquire(0)
+	defer bigintpool.Release(newTarget)
+	windowTimeStampDifference := bigintpool.Acquire(windowMaxTimeStamp - windowMinTimestamp)
+	defer bigintpool.Release(windowTimeStampDifference)
+	targetTimePerBlock := bigintpool.Acquire(dag.targetTimePerBlock)
+	defer bigintpool.Release(targetTimePerBlock)
+	difficultyAdjustmentWindowSize := bigintpool.Acquire(int64(dag.difficultyAdjustmentWindowSize))
+	defer bigintpool.Release(difficultyAdjustmentWindowSize)
+
+	targetsWindow.averageTarget(newTarget)
 	newTarget.
-		Mul(newTarget, big.NewInt(windowMaxTimeStamp-windowMinTimestamp)).
-		Div(newTarget, big.NewInt(dag.targetTimePerBlock)).
-		Div(newTarget, big.NewInt(int64(dag.difficultyAdjustmentWindowSize)))
+		Mul(newTarget, windowTimeStampDifference).
+		Div(newTarget, targetTimePerBlock).
+		Div(newTarget, difficultyAdjustmentWindowSize)
 	if newTarget.Cmp(dag.dagParams.PowMax) > 0 {
 		return dag.powMaxBits
 	}

blockdag/difficulty_test.go

@@ -82,7 +82,7 @@ func TestDifficulty(t *testing.T) {
 	params := dagconfig.SimnetParams
 	params.K = 1
 	params.DifficultyAdjustmentWindowSize = 264
-	dag, teardownFunc, err := DAGSetup("TestDifficulty", Config{
+	dag, teardownFunc, err := DAGSetup("TestDifficulty", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -94,7 +94,8 @@ func TestDifficulty(t *testing.T) {
 	addNode := func(parents blockSet, blockTime time.Time) *blockNode {
 		bluestParent := parents.bluest()
 		if blockTime == zeroTime {
-			blockTime = time.Unix(bluestParent.timestamp+1, 0)
+			blockTime = time.Unix(bluestParent.timestamp, 0)
+			blockTime = blockTime.Add(params.TargetTimePerBlock)
 		}
 		block, err := PrepareBlockForTest(dag, parents.hashes(), nil)
 		if err != nil {
@@ -113,13 +114,18 @@ func TestDifficulty(t *testing.T) {
 		if isOrphan {
 			t.Fatalf("block was unexpectedly orphan")
 		}
-		return dag.index.LookupNode(block.BlockHash())
+		node, ok := dag.index.LookupNode(block.BlockHash())
+		if !ok {
+			t.Fatalf("block %s does not exist in the DAG", block.BlockHash())
+		}
+		return node
 	}
 	tip := dag.genesis
 	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize; i++ {
 		tip = addNode(blockSetFromSlice(tip), zeroTime)
 		if tip.bits != dag.genesis.bits {
-			t.Fatalf("As long as the bluest parent's blue score is less then the difficulty adjustment window size, the difficulty should be the same as genesis'")
+			t.Fatalf("As long as the bluest parent's blue score is less then the difficulty adjustment " +
+				"window size, the difficulty should be the same as genesis'")
 		}
 	}
 	for i := uint64(0); i < dag.difficultyAdjustmentWindowSize+100; i++ {
@@ -140,7 +146,8 @@ func TestDifficulty(t *testing.T) {
 	}
 	tip = addNode(blockSetFromSlice(tip), zeroTime)
 	if compareBits(tip.bits, nodeInThePast.bits) >= 0 {
-		t.Fatalf("tip.bits should be smaller than nodeInThePast.bits because nodeInThePast increased the block rate, so the difficulty should increase as well")
+		t.Fatalf("tip.bits should be smaller than nodeInThePast.bits because nodeInThePast increased the " +
+			"block rate, so the difficulty should increase as well")
 	}
 	expectedBits := uint32(0x207f83df)
 	if tip.bits != expectedBits {
@@ -167,7 +174,9 @@ func TestDifficulty(t *testing.T) {
 			sameBitsCount = 0
 		}
 	}
-	slowNode := addNode(blockSetFromSlice(tip), time.Unix(tip.timestamp+2, 0))
+	slowBlockTime := time.Unix(tip.timestamp, 0)
+	slowBlockTime = slowBlockTime.Add(params.TargetTimePerBlock + time.Second)
+	slowNode := addNode(blockSetFromSlice(tip), slowBlockTime)
 	if slowNode.bits != tip.bits {
 		t.Fatalf("The difficulty should only change when slowNode is in the past of a block bluest parent")
 	}
@@ -180,7 +189,8 @@ func TestDifficulty(t *testing.T) {
 	}
 	tip = addNode(blockSetFromSlice(tip), zeroTime)
 	if compareBits(tip.bits, slowNode.bits) <= 0 {
-		t.Fatalf("tip.bits should be smaller than slowNode.bits because slowNode decreased the block rate, so the difficulty should decrease as well")
+		t.Fatalf("tip.bits should be smaller than slowNode.bits because slowNode decreased the block" +
+			" rate, so the difficulty should decrease as well")
 	}
 
 	splitNode := addNode(blockSetFromSlice(tip), zeroTime)
@@ -197,7 +207,8 @@ func TestDifficulty(t *testing.T) {
 	tipWithRedPast := addNode(blockSetFromSlice(redChainTip, blueTip), zeroTime)
 	tipWithoutRedPast := addNode(blockSetFromSlice(blueTip), zeroTime)
 	if tipWithoutRedPast.bits != tipWithRedPast.bits {
-		t.Fatalf("tipWithoutRedPast.bits should be the same as tipWithRedPast.bits because red blocks shouldn't affect the difficulty")
+		t.Fatalf("tipWithoutRedPast.bits should be the same as tipWithRedPast.bits because red blocks" +
+			" shouldn't affect the difficulty")
 	}
 }
 

blockdag/error.go

@@ -6,28 +6,10 @@ package blockdag
 
 import (
 	"fmt"
+
+	"github.com/pkg/errors"
 )
 
-// DeploymentError identifies an error that indicates a deployment ID was
-// specified that does not exist.
-type DeploymentError uint32
-
-// Error returns the assertion error as a human-readable string and satisfies
-// the error interface.
-func (e DeploymentError) Error() string {
-	return fmt.Sprintf("deployment ID %d does not exist", uint32(e))
-}
-
-// AssertError identifies an error that indicates an internal code consistency
-// issue and should be treated as a critical and unrecoverable error.
-type AssertError string
-
-// Error returns the assertion error as a human-readable string and satisfies
-// the error interface.
-func (e AssertError) Error() string {
-	return "assertion failed: " + string(e)
-}
-
 // ErrorCode identifies a kind of error.
 type ErrorCode int
 
@@ -87,6 +69,9 @@ const (
 	// the expected value.
 	ErrBadUTXOCommitment
 
+	// ErrInvalidSubnetwork indicates the subnetwork is now allowed.
+	ErrInvalidSubnetwork
+
 	// ErrFinalityPointTimeTooOld indicates a block has a timestamp before the
 	// last finality point.
 	ErrFinalityPointTimeTooOld
@@ -121,6 +106,11 @@ const (
 	// either does not exist or has already been spent.
 	ErrMissingTxOut
 
+	// ErrDoubleSpendInSameBlock indicates a transaction
+	// that spends an output that was already spent by another
+	// transaction in the same block.
+	ErrDoubleSpendInSameBlock
+
 	// ErrUnfinalizedTx indicates a transaction has not been finalized.
 	// A valid block may only contain finalized transactions.
 	ErrUnfinalizedTx
@@ -245,6 +235,7 @@ var errorCodeStrings = map[ErrorCode]string{
 	ErrDuplicateTxInputs:         "ErrDuplicateTxInputs",
 	ErrBadTxInput:                "ErrBadTxInput",
 	ErrMissingTxOut:              "ErrMissingTxOut",
+	ErrDoubleSpendInSameBlock:    "ErrDoubleSpendInSameBlock",
 	ErrUnfinalizedTx:             "ErrUnfinalizedTx",
 	ErrDuplicateTx:               "ErrDuplicateTx",
 	ErrOverwriteTx:               "ErrOverwriteTx",
@@ -294,7 +285,6 @@ func (e RuleError) Error() string {
 	return e.Description
 }
 
-// ruleError creates an RuleError given a set of arguments.
-func ruleError(c ErrorCode, desc string) RuleError {
-	return RuleError{ErrorCode: c, Description: desc}
+func ruleError(c ErrorCode, desc string) error {
+	return errors.WithStack(RuleError{ErrorCode: c, Description: desc})
 }

blockdag/error_test.go

@@ -5,7 +5,6 @@
 package blockdag
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -99,46 +98,3 @@ func TestRuleError(t *testing.T) {
 		}
 	}
 }
-
-// TestDeploymentError tests the stringized output for the DeploymentError type.
-func TestDeploymentError(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		in   DeploymentError
-		want string
-	}{
-		{
-			DeploymentError(0),
-			"deployment ID 0 does not exist",
-		},
-		{
-			DeploymentError(10),
-			"deployment ID 10 does not exist",
-		},
-		{
-			DeploymentError(123),
-			"deployment ID 123 does not exist",
-		},
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.Error()
-		if result != test.want {
-			t.Errorf("Error #%d\n got: %s want: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-func TestAssertError(t *testing.T) {
-	message := "abc 123"
-	err := AssertError(message)
-	expectedMessage := fmt.Sprintf("assertion failed: %s", message)
-	if expectedMessage != err.Error() {
-		t.Errorf("Unexpected AssertError message. "+
-			"Got: %s, want: %s", err.Error(), expectedMessage)
-	}
-}

blockdag/external_dag_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/pkg/errors"
 	"math"
+	"strings"
 	"testing"
 
 	"github.com/kaspanet/kaspad/util/subnetworkid"
@@ -40,7 +41,7 @@ func TestFinality(t *testing.T) {
 	params := dagconfig.SimnetParams
 	params.K = 1
 	params.FinalityInterval = 100
-	dag, teardownFunc, err := blockdag.DAGSetup("TestFinality", blockdag.Config{
+	dag, teardownFunc, err := blockdag.DAGSetup("TestFinality", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -185,7 +186,8 @@ func TestSubnetworkRegistry(t *testing.T) {
 	params := dagconfig.SimnetParams
 	params.K = 1
 	params.BlockCoinbaseMaturity = 0
-	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", blockdag.Config{
+	params.EnableNonNativeSubnetworks = true
+	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -198,7 +200,7 @@ func TestSubnetworkRegistry(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not register network: %s", err)
 	}
-	limit, err := dag.SubnetworkStore.GasLimit(subnetworkID)
+	limit, err := blockdag.GasLimit(subnetworkID)
 	if err != nil {
 		t.Fatalf("could not retrieve gas limit: %s", err)
 	}
@@ -211,7 +213,7 @@ func TestChainedTransactions(t *testing.T) {
 	params := dagconfig.SimnetParams
 	params.BlockCoinbaseMaturity = 0
 	// Create a new database and dag instance to run tests against.
-	dag, teardownFunc, err := blockdag.DAGSetup("TestChainedTransactions", blockdag.Config{
+	dag, teardownFunc, err := blockdag.DAGSetup("TestChainedTransactions", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -267,11 +269,19 @@ func TestChainedTransactions(t *testing.T) {
 	}
 	chainedTx := wire.NewNativeMsgTx(wire.TxVersion, []*wire.TxIn{chainedTxIn}, []*wire.TxOut{chainedTxOut})
 
-	block2, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{tx, chainedTx}, true)
+	block2, err := mining.PrepareBlockForTest(dag, &params, []*daghash.Hash{block1.BlockHash()}, []*wire.MsgTx{tx}, false)
 	if err != nil {
 		t.Fatalf("PrepareBlockForTest: %v", err)
 	}
 
+	// Manually add a chained transaction to block2
+	block2.Transactions = append(block2.Transactions, chainedTx)
+	block2UtilTxs := make([]*util.Tx, len(block2.Transactions))
+	for i, tx := range block2.Transactions {
+		block2UtilTxs[i] = util.NewTx(tx)
+	}
+	block2.Header.HashMerkleRoot = blockdag.BuildHashMerkleTreeStore(block2UtilTxs).Root()
+
 	//Checks that dag.ProcessBlock fails because we don't allow a transaction to spend another transaction from the same block
 	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(block2), blockdag.BFNoPoWCheck)
 	if err == nil {
@@ -331,7 +341,7 @@ func TestOrderInDiffFromAcceptanceData(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
 	params.K = math.MaxUint8
-	dag, teardownFunc, err := blockdag.DAGSetup("TestOrderInDiffFromAcceptanceData", blockdag.Config{
+	dag, teardownFunc, err := blockdag.DAGSetup("TestOrderInDiffFromAcceptanceData", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -401,7 +411,8 @@ func TestGasLimit(t *testing.T) {
 	params := dagconfig.SimnetParams
 	params.K = 1
 	params.BlockCoinbaseMaturity = 0
-	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", blockdag.Config{
+	params.EnableNonNativeSubnetworks = true
+	dag, teardownFunc, err := blockdag.DAGSetup("TestSubnetworkRegistry", true, blockdag.Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -548,7 +559,7 @@ func TestGasLimit(t *testing.T) {
 	isOrphan, isDelayed, err = dag.ProcessBlock(util.NewBlock(nonExistentSubnetworkBlock), blockdag.BFNoPoWCheck)
 	expectedErrStr := fmt.Sprintf("Error getting gas limit for subnetworkID '%s': subnetwork '%s' not found",
 		nonExistentSubnetwork, nonExistentSubnetwork)
-	if err.Error() != expectedErrStr {
+	if strings.Contains(err.Error(), expectedErrStr) {
 		t.Fatalf("ProcessBlock expected error \"%v\" but got \"%v\"", expectedErrStr, err)
 	}
 	if isDelayed {

blockdag/ghostdag.go

@@ -57,7 +57,7 @@ func (dag *BlockDAG) ghostdag(newNode *blockNode) (selectedParentAnticone []*blo
 			// newNode is always in the future of blueCandidate, so there's
 			// no point in checking it.
 			if chainBlock != newNode {
-				if isAncestorOfBlueCandidate, err := dag.isAncestorOf(chainBlock, blueCandidate); err != nil {
+				if isAncestorOfBlueCandidate, err := dag.isInPast(chainBlock, blueCandidate); err != nil {
 					return nil, err
 				} else if isAncestorOfBlueCandidate {
 					break
@@ -66,7 +66,7 @@ func (dag *BlockDAG) ghostdag(newNode *blockNode) (selectedParentAnticone []*blo
 
 			for _, block := range chainBlock.blues {
 				// Skip blocks that exist in the past of blueCandidate.
-				if isAncestorOfBlueCandidate, err := dag.isAncestorOf(block, blueCandidate); err != nil {
+				if isAncestorOfBlueCandidate, err := dag.isInPast(block, blueCandidate); err != nil {
 					return nil, err
 				} else if isAncestorOfBlueCandidate {
 					continue
@@ -148,7 +148,7 @@ func (dag *BlockDAG) selectedParentAnticone(node *blockNode) ([]*blockNode, erro
 			if anticoneSet.contains(parent) || selectedParentPast.contains(parent) {
 				continue
 			}
-			isAncestorOfSelectedParent, err := dag.isAncestorOf(parent, node.selectedParent)
+			isAncestorOfSelectedParent, err := dag.isInPast(parent, node.selectedParent)
 			if err != nil {
 				return nil, err
 			}

blockdag/ghostdag_test.go

@@ -3,7 +3,7 @@ package blockdag
 import (
 	"fmt"
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"reflect"
@@ -33,7 +33,7 @@ func TestGHOSTDAG(t *testing.T) {
 	}{
 		{
 			k:            3,
-			expectedReds: []string{"F", "G", "H", "I", "O", "P"},
+			expectedReds: []string{"F", "G", "H", "I", "N", "Q"},
 			dagData: []*testBlockData{
 				{
 					parents:                []string{"A"},
@@ -166,7 +166,7 @@ func TestGHOSTDAG(t *testing.T) {
 					id:                     "T",
 					expectedScore:          13,
 					expectedSelectedParent: "S",
-					expectedBlues:          []string{"S", "N", "Q"},
+					expectedBlues:          []string{"S", "O", "P"},
 				},
 			},
 		},
@@ -176,7 +176,7 @@ func TestGHOSTDAG(t *testing.T) {
 		func() {
 			resetExtraNonceForTest()
 			dagParams.K = test.k
-			dag, teardownFunc, err := DAGSetup(fmt.Sprintf("TestGHOSTDAG%d", i), Config{
+			dag, teardownFunc, err := DAGSetup(fmt.Sprintf("TestGHOSTDAG%d", i), true, Config{
 				DAGParams: &dagParams,
 			})
 			if err != nil {
@@ -215,7 +215,10 @@ func TestGHOSTDAG(t *testing.T) {
 					t.Fatalf("TestGHOSTDAG: block %v was unexpectedly orphan", blockData.id)
 				}
 
-				node := dag.index.LookupNode(utilBlock.Hash())
+				node, ok := dag.index.LookupNode(utilBlock.Hash())
+				if !ok {
+					t.Fatalf("block %s does not exist in the DAG", utilBlock.Hash())
+				}
 
 				blockByIDMap[blockData.id] = node
 				idByBlockMap[node] = blockData.id
@@ -282,7 +285,7 @@ func checkReds(expectedReds []string, reds map[string]bool) bool {
 
 func TestBlueAnticoneSizeErrors(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizeErrors", Config{
+	dag, teardownFunc, err := DAGSetup("TestBlueAnticoneSizeErrors", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -293,20 +296,27 @@ func TestBlueAnticoneSizeErrors(t *testing.T) {
 	// Prepare a block chain with size K beginning with the genesis block
 	currentBlockA := dag.dagParams.GenesisBlock
 	for i := dagconfig.KType(0); i < dag.dagParams.K; i++ {
-		newBlock := prepareAndProcessBlock(t, dag, currentBlockA)
+		newBlock := prepareAndProcessBlockByParentMsgBlocks(t, dag, currentBlockA)
 		currentBlockA = newBlock
 	}
 
 	// Prepare another block chain with size K beginning with the genesis block
 	currentBlockB := dag.dagParams.GenesisBlock
 	for i := dagconfig.KType(0); i < dag.dagParams.K; i++ {
-		newBlock := prepareAndProcessBlock(t, dag, currentBlockB)
+		newBlock := prepareAndProcessBlockByParentMsgBlocks(t, dag, currentBlockB)
 		currentBlockB = newBlock
 	}
 
 	// Get references to the tips of the two chains
-	blockNodeA := dag.index.LookupNode(currentBlockA.BlockHash())
-	blockNodeB := dag.index.LookupNode(currentBlockB.BlockHash())
+	blockNodeA, ok := dag.index.LookupNode(currentBlockA.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", currentBlockA.BlockHash())
+	}
+
+	blockNodeB, ok := dag.index.LookupNode(currentBlockB.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", currentBlockB.BlockHash())
+	}
 
 	// Try getting the blueAnticoneSize between them. Since the two
 	// blocks are not in the anticones of eachother, this should fail.
@@ -323,7 +333,7 @@ func TestBlueAnticoneSizeErrors(t *testing.T) {
 
 func TestGHOSTDAGErrors(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestGHOSTDAGErrors", Config{
+	dag, teardownFunc, err := DAGSetup("TestGHOSTDAGErrors", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -332,37 +342,42 @@ func TestGHOSTDAGErrors(t *testing.T) {
 	defer teardownFunc()
 
 	// Add two child blocks to the genesis
-	block1 := prepareAndProcessBlock(t, dag, dag.dagParams.GenesisBlock)
-	block2 := prepareAndProcessBlock(t, dag, dag.dagParams.GenesisBlock)
+	block1 := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.dagParams.GenesisBlock)
+	block2 := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.dagParams.GenesisBlock)
 
 	// Add a child block to the previous two blocks
-	block3 := prepareAndProcessBlock(t, dag, block1, block2)
+	block3 := prepareAndProcessBlockByParentMsgBlocks(t, dag, block1, block2)
 
 	// Clear the reachability store
-	dag.reachabilityStore.loaded = map[daghash.Hash]*reachabilityData{}
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		bucket := dbTx.Metadata().Bucket(reachabilityDataBucketName)
-		cursor := bucket.Cursor()
-		for ok := cursor.First(); ok; ok = cursor.Next() {
-			err := bucket.Delete(cursor.Key())
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
+	dag.reachabilityTree.store.loaded = map[daghash.Hash]*reachabilityData{}
+
+	dbTx, err := dbaccess.NewTx()
 	if err != nil {
-		t.Fatalf("TestGHOSTDAGErrors: db.Update failed: %s", err)
+		t.Fatalf("NewTx: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+
+	err = dbaccess.ClearReachabilityData(dbTx)
+	if err != nil {
+		t.Fatalf("ClearReachabilityData: %s", err)
+	}
+
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Commit: %s", err)
 	}
 
 	// Try to rerun GHOSTDAG on the last block. GHOSTDAG uses
 	// reachability data, so we expect it to fail.
-	blockNode3 := dag.index.LookupNode(block3.BlockHash())
+	blockNode3, ok := dag.index.LookupNode(block3.BlockHash())
+	if !ok {
+		t.Fatalf("block %s does not exist in the DAG", block3.BlockHash())
+	}
 	_, err = dag.ghostdag(blockNode3)
 	if err == nil {
 		t.Fatalf("TestGHOSTDAGErrors: ghostdag unexpectedly succeeded")
 	}
-	expectedErrSubstring := "Couldn't find reachability data"
+	expectedErrSubstring := "couldn't find reachability data"
 	if !strings.Contains(err.Error(), expectedErrSubstring) {
 		t.Fatalf("TestGHOSTDAGErrors: ghostdag returned wrong error. "+
 			"Want: %s, got: %s", expectedErrSubstring, err)

blockdag/indexers/acceptanceindex.go

@@ -4,29 +4,16 @@ import (
 	"bytes"
 	"encoding/gob"
 	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
-	"github.com/pkg/errors"
-)
-
-const (
-	// acceptanceIndexName is the human-readable name for the index.
-	acceptanceIndexName = "acceptance index"
-)
-
-var (
-	// acceptanceIndexKey is the key of the acceptance index and the db bucket used
-	// to house it.
-	acceptanceIndexKey = []byte("acceptanceidx")
 )
 
 // AcceptanceIndex implements a txAcceptanceData by block hash index. That is to say,
 // it stores a mapping between a block's hash and the set of transactions that the
 // block accepts among its blue blocks.
 type AcceptanceIndex struct {
-	db  database.DB
 	dag *blockdag.BlockDAG
 }
 
@@ -43,122 +30,82 @@ func NewAcceptanceIndex() *AcceptanceIndex {
 	return &AcceptanceIndex{}
 }
 
-// DropAcceptanceIndex drops the acceptance index from the provided database if it
-// exists.
-func DropAcceptanceIndex(db database.DB, interrupt <-chan struct{}) error {
-	return dropIndex(db, acceptanceIndexKey, acceptanceIndexName, interrupt)
-}
+// DropAcceptanceIndex drops the acceptance index.
+func DropAcceptanceIndex() error {
+	dbTx, err := dbaccess.NewTx()
+	if err != nil {
+		return err
+	}
+	defer dbTx.RollbackUnlessClosed()
 
-// Key returns the database key to use for the index as a byte slice.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Key() []byte {
-	return acceptanceIndexKey
-}
+	err = dbaccess.DropAcceptanceIndex(dbTx)
+	if err != nil {
+		return err
+	}
 
-// Name returns the human-readable name of the index.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Name() string {
-	return acceptanceIndexName
-}
-
-// Create is invoked when the indexer manager determines the index needs
-// to be created for the first time. It creates the bucket for the
-// acceptance index.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Create(dbTx database.Tx) error {
-	_, err := dbTx.Metadata().CreateBucket(acceptanceIndexKey)
-	return err
+	return dbTx.Commit()
 }
 
 // Init initializes the hash-based acceptance index.
 //
 // This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Init(db database.DB, dag *blockdag.BlockDAG) error {
-	idx.db = db
+func (idx *AcceptanceIndex) Init(dag *blockdag.BlockDAG) error {
 	idx.dag = dag
-	return nil
+	return idx.recover()
+}
+
+// recover attempts to insert any data that's missing from the
+// acceptance index.
+//
+// This is part of the Indexer interface.
+func (idx *AcceptanceIndex) recover() error {
+	return idx.dag.ForEachHash(func(hash daghash.Hash) error {
+		dbTx, err := dbaccess.NewTx()
+		if err != nil {
+			return err
+		}
+		defer dbTx.RollbackUnlessClosed()
+
+		exists, err := dbaccess.HasAcceptanceData(dbTx, &hash)
+		if err != nil {
+			return err
+		}
+		if exists {
+			return nil
+		}
+		txAcceptanceData, err := idx.dag.TxsAcceptedByBlockHash(&hash)
+		if err != nil {
+			return err
+		}
+		err = idx.ConnectBlock(dbTx, &hash, txAcceptanceData)
+		if err != nil {
+			return err
+		}
+
+		return dbTx.Commit()
+	})
 }
 
 // ConnectBlock is invoked by the index manager when a new block has been
 // connected to the DAG.
 //
 // This is part of the Indexer interface.
-func (idx *AcceptanceIndex) ConnectBlock(dbTx database.Tx, _ *util.Block, blockID uint64, _ *blockdag.BlockDAG,
-	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, _ blockdag.MultiBlockTxsAcceptanceData) error {
-	return dbPutTxsAcceptanceData(dbTx, blockID, txsAcceptanceData)
-}
-
-// TxsAcceptanceData returns the acceptance data of all the transactions that
-// were accepted by the block with hash blockHash.
-func (idx *AcceptanceIndex) TxsAcceptanceData(blockHash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
-	var txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData
-	err := idx.db.View(func(dbTx database.Tx) error {
-		var err error
-		txsAcceptanceData, err = dbFetchTxsAcceptanceDataByHash(dbTx, blockHash)
-		return err
-	})
-	if err != nil {
-		return nil, err
-	}
-	return txsAcceptanceData, nil
-}
-
-// Recover is invoked when the indexer wasn't turned on for several blocks
-// and the indexer needs to close the gaps.
-//
-// This is part of the Indexer interface.
-func (idx *AcceptanceIndex) Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error {
-	for blockID := currentBlockID + 1; blockID <= lastKnownBlockID; blockID++ {
-		hash, err := blockdag.DBFetchBlockHashByID(dbTx, currentBlockID)
-		if err != nil {
-			return err
-		}
-		txAcceptanceData, err := idx.dag.TxsAcceptedByBlockHash(hash)
-		if err != nil {
-			return err
-		}
-		err = idx.ConnectBlock(dbTx, nil, blockID, nil, txAcceptanceData, nil)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func dbPutTxsAcceptanceData(dbTx database.Tx, blockID uint64,
+func (idx *AcceptanceIndex) ConnectBlock(dbContext *dbaccess.TxContext, blockHash *daghash.Hash,
 	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
 	serializedTxsAcceptanceData, err := serializeMultiBlockTxsAcceptanceData(txsAcceptanceData)
 	if err != nil {
 		return err
 	}
-
-	bucket := dbTx.Metadata().Bucket(acceptanceIndexKey)
-	return bucket.Put(blockdag.SerializeBlockID(blockID), serializedTxsAcceptanceData)
+	return dbaccess.StoreAcceptanceData(dbContext, blockHash, serializedTxsAcceptanceData)
 }
 
-func dbFetchTxsAcceptanceDataByHash(dbTx database.Tx,
-	hash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
-
-	blockID, err := blockdag.DBFetchBlockIDByHash(dbTx, hash)
+// TxsAcceptanceData returns the acceptance data of all the transactions that
+// were accepted by the block with hash blockHash.
+func (idx *AcceptanceIndex) TxsAcceptanceData(blockHash *daghash.Hash) (blockdag.MultiBlockTxsAcceptanceData, error) {
+	serializedTxsAcceptanceData, err := dbaccess.FetchAcceptanceData(dbaccess.NoTx(), blockHash)
 	if err != nil {
 		return nil, err
 	}
-
-	return dbFetchTxsAcceptanceDataByID(dbTx, blockID)
-}
-
-func dbFetchTxsAcceptanceDataByID(dbTx database.Tx,
-	blockID uint64) (blockdag.MultiBlockTxsAcceptanceData, error) {
-	serializedBlockID := blockdag.SerializeBlockID(blockID)
-	bucket := dbTx.Metadata().Bucket(acceptanceIndexKey)
-	serializedTxsAcceptanceData := bucket.Get(serializedBlockID)
-	if serializedTxsAcceptanceData == nil {
-		return nil, errors.Errorf("no entry in the accpetance index for block id %d", blockID)
-	}
-
 	return deserializeMultiBlockTxsAcceptanceData(serializedTxsAcceptanceData)
 }
 

blockdag/indexers/acceptanceindex_test.go

@@ -3,7 +3,7 @@ package indexers
 import (
 	"github.com/kaspanet/kaspad/blockdag"
 	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
@@ -96,7 +96,7 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	}
 	defer os.RemoveAll(db1Path)
 
-	db1, err := database.Create("ffldb", db1Path, params.Net)
+	err = dbaccess.Open(db1Path)
 	if err != nil {
 		t.Fatalf("error creating db: %s", err)
 	}
@@ -104,10 +104,9 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	db1Config := blockdag.Config{
 		IndexManager: db1IndexManager,
 		DAGParams:    params,
-		DB:           db1,
 	}
 
-	db1DAG, teardown, err := blockdag.DAGSetup("", db1Config)
+	db1DAG, teardown, err := blockdag.DAGSetup("", false, db1Config)
 	if err != nil {
 		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
 	}
@@ -130,11 +129,6 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		}
 	}
 
-	err = db1.FlushCache()
-	if err != nil {
-		t.Fatalf("Error flushing database to disk: %s", err)
-	}
-
 	db2Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover2")
 	if err != nil {
 		t.Fatalf("Error creating temporary directory: %s", err)
@@ -166,17 +160,20 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		t.Fatalf("Error fetching acceptance data: %s", err)
 	}
 
-	db2, err := database.Open("ffldb", db2Path, params.Net)
+	err = dbaccess.Close()
 	if err != nil {
-		t.Fatalf("Error opening database: %s", err)
+		t.Fatalf("Error closing the database: %s", err)
+	}
+	err = dbaccess.Open(db2Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
 	}
 
 	db2Config := blockdag.Config{
 		DAGParams: params,
-		DB:        db2,
 	}
 
-	db2DAG, teardown, err := blockdag.DAGSetup("", db2Config)
+	db2DAG, teardown, err := blockdag.DAGSetup("", false, db2Config)
 	if err != nil {
 		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
 	}
@@ -199,10 +196,6 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		}
 	}
 
-	err = db2.FlushCache()
-	if err != nil {
-		t.Fatalf("Error flushing database to disk: %s", err)
-	}
 	db3Path, err := ioutil.TempDir("", "TestAcceptanceIndexRecover3")
 	if err != nil {
 		t.Fatalf("Error creating temporary directory: %s", err)
@@ -213,9 +206,13 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 		t.Fatalf("copyDirectory: %s", err)
 	}
 
-	db3, err := database.Open("ffldb", db3Path, params.Net)
+	err = dbaccess.Close()
 	if err != nil {
-		t.Fatalf("Error opening database: %s", err)
+		t.Fatalf("Error closing the database: %s", err)
+	}
+	err = dbaccess.Open(db3Path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
 	}
 
 	db3AcceptanceIndex := NewAcceptanceIndex()
@@ -223,10 +220,9 @@ func TestAcceptanceIndexRecover(t *testing.T) {
 	db3Config := blockdag.Config{
 		IndexManager: db3IndexManager,
 		DAGParams:    params,
-		DB:           db3,
 	}
 
-	_, teardown, err = blockdag.DAGSetup("", db3Config)
+	_, teardown, err = blockdag.DAGSetup("", false, db3Config)
 	if err != nil {
 		t.Fatalf("TestAcceptanceIndexRecover: Failed to setup DAG instance: %v", err)
 	}

blockdag/indexers/common.go

@@ -1,112 +0,0 @@
-// Copyright (c) 2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-Package indexers implements optional block DAG indexes.
-*/
-package indexers
-
-import (
-	"encoding/binary"
-	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/pkg/errors"
-)
-
-var (
-	// byteOrder is the preferred byte order used for serializing numeric
-	// fields for storage in the database.
-	byteOrder = binary.LittleEndian
-
-	// errInterruptRequested indicates that an operation was cancelled due
-	// to a user-requested interrupt.
-	errInterruptRequested = errors.New("interrupt requested")
-)
-
-// NeedsInputser provides a generic interface for an indexer to specify the it
-// requires the ability to look up inputs for a transaction.
-type NeedsInputser interface {
-	NeedsInputs() bool
-}
-
-// Indexer provides a generic interface for an indexer that is managed by an
-// index manager such as the Manager type provided by this package.
-type Indexer interface {
-	// Key returns the key of the index as a byte slice.
-	Key() []byte
-
-	// Name returns the human-readable name of the index.
-	Name() string
-
-	// Create is invoked when the indexer manager determines the index needs
-	// to be created for the first time.
-	Create(dbTx database.Tx) error
-
-	// Init is invoked when the index manager is first initializing the
-	// index. This differs from the Create method in that it is called on
-	// every load, including the case the index was just created.
-	Init(db database.DB, dag *blockdag.BlockDAG) error
-
-	// ConnectBlock is invoked when the index manager is notified that a new
-	// block has been connected to the DAG.
-	ConnectBlock(dbTx database.Tx,
-		block *util.Block,
-		blockID uint64,
-		dag *blockdag.BlockDAG,
-		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData,
-		virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error
-
-	// Recover is invoked when the indexer wasn't turned on for several blocks
-	// and the indexer needs to close the gaps.
-	Recover(dbTx database.Tx, currentBlockID, lastKnownBlockID uint64) error
-}
-
-// AssertError identifies an error that indicates an internal code consistency
-// issue and should be treated as a critical and unrecoverable error.
-type AssertError string
-
-// Error returns the assertion error as a huma-readable string and satisfies
-// the error interface.
-func (e AssertError) Error() string {
-	return "assertion failed: " + string(e)
-}
-
-// errDeserialize signifies that a problem was encountered when deserializing
-// data.
-type errDeserialize string
-
-// Error implements the error interface.
-func (e errDeserialize) Error() string {
-	return string(e)
-}
-
-// isDeserializeErr returns whether or not the passed error is an errDeserialize
-// error.
-func isDeserializeErr(err error) bool {
-	var deserializeErr errDeserialize
-	return errors.As(err, &deserializeErr)
-}
-
-// internalBucket is an abstraction over a database bucket. It is used to make
-// the code easier to test since it allows mock objects in the tests to only
-// implement these functions instead of everything a database.Bucket supports.
-type internalBucket interface {
-	Get(key []byte) []byte
-	Put(key []byte, value []byte) error
-	Delete(key []byte) error
-}
-
-// interruptRequested returns true when the provided channel has been closed.
-// This simplifies early shutdown slightly since the caller can just use an if
-// statement instead of a select.
-func interruptRequested(interrupted <-chan struct{}) bool {
-	select {
-	case <-interrupted:
-		return true
-	default:
-	}
-
-	return false
-}

blockdag/indexers/indexer.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2016 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+/*
+Package indexers implements optional block DAG indexes.
+*/
+package indexers
+
+import (
+	"github.com/kaspanet/kaspad/blockdag"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+)
+
+// Indexer provides a generic interface for an indexer that is managed by an
+// index manager such as the Manager type provided by this package.
+type Indexer interface {
+	// Init is invoked when the index manager is first initializing the
+	// index.
+	Init(dag *blockdag.BlockDAG) error
+
+	// ConnectBlock is invoked when the index manager is notified that a new
+	// block has been connected to the DAG.
+	ConnectBlock(dbContext *dbaccess.TxContext,
+		blockHash *daghash.Hash,
+		acceptedTxsData blockdag.MultiBlockTxsAcceptanceData) error
+}

blockdag/indexers/manager.go

@@ -6,190 +6,30 @@ package indexers
 
 import (
 	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util/daghash"
 )
 
-var (
-	// indexTipsBucketName is the name of the db bucket used to house the
-	// current tip of each index.
-	indexTipsBucketName = []byte("idxtips")
-
-	indexCurrentBlockIDBucketName = []byte("idxcurrentblockid")
-)
-
 // Manager defines an index manager that manages multiple optional indexes and
 // implements the blockdag.IndexManager interface so it can be seamlessly
 // plugged into normal DAG processing.
 type Manager struct {
-	db             database.DB
 	enabledIndexes []Indexer
 }
 
 // Ensure the Manager type implements the blockdag.IndexManager interface.
 var _ blockdag.IndexManager = (*Manager)(nil)
 
-// indexDropKey returns the key for an index which indicates it is in the
-// process of being dropped.
-func indexDropKey(idxKey []byte) []byte {
-	dropKey := make([]byte, len(idxKey)+1)
-	dropKey[0] = 'd'
-	copy(dropKey[1:], idxKey)
-	return dropKey
-}
-
-// maybeFinishDrops determines if each of the enabled indexes are in the middle
-// of being dropped and finishes dropping them when the are. This is necessary
-// because dropping and index has to be done in several atomic steps rather than
-// one big atomic step due to the massive number of entries.
-func (m *Manager) maybeFinishDrops(interrupt <-chan struct{}) error {
-	indexNeedsDrop := make([]bool, len(m.enabledIndexes))
-	err := m.db.View(func(dbTx database.Tx) error {
-		// None of the indexes needs to be dropped if the index tips
-		// bucket hasn't been created yet.
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		if indexesBucket == nil {
-			return nil
-		}
-
-		// Mark the indexer as requiring a drop if one is already in
-		// progress.
-		for i, indexer := range m.enabledIndexes {
-			dropKey := indexDropKey(indexer.Key())
-			if indexesBucket.Get(dropKey) != nil {
-				indexNeedsDrop[i] = true
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
-	if interruptRequested(interrupt) {
-		return errInterruptRequested
-	}
-
-	// Finish dropping any of the enabled indexes that are already in the
-	// middle of being dropped.
-	for i, indexer := range m.enabledIndexes {
-		if !indexNeedsDrop[i] {
-			continue
-		}
-
-		log.Infof("Resuming %s drop", indexer.Name())
-		err := dropIndex(m.db, indexer.Key(), indexer.Name(), interrupt)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// maybeCreateIndexes determines if each of the enabled indexes have already
-// been created and creates them if not.
-func (m *Manager) maybeCreateIndexes(dbTx database.Tx) error {
-	indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-	for _, indexer := range m.enabledIndexes {
-		// Nothing to do if the index tip already exists.
-		idxKey := indexer.Key()
-		if indexesBucket.Get(idxKey) != nil {
-			continue
-		}
-
-		// The tip for the index does not exist, so create it and
-		// invoke the create callback for the index so it can perform
-		// any one-time initialization it requires.
-		if err := indexer.Create(dbTx); err != nil {
-			return err
-		}
-
-		// TODO (Mike): this is temporary solution to prevent node from not starting
-		// because it thinks indexers are not initialized.
-		// Indexers, however, do not work properly, and a general solution to their work operation is required
-		indexesBucket.Put(idxKey, []byte{0})
-	}
-
-	return nil
-}
-
-// Init initializes the enabled indexes. This is called during DAG
-// initialization and primarily consists of catching up all indexes to the
-// current tips. This is necessary since each index can be disabled
-// and re-enabled at any time and attempting to catch-up indexes at the same
-// time new blocks are being downloaded would lead to an overall longer time to
-// catch up due to the I/O contention.
-//
+// Init initializes the enabled indexes.
 // This is part of the blockdag.IndexManager interface.
-func (m *Manager) Init(db database.DB, blockDAG *blockdag.BlockDAG, interrupt <-chan struct{}) error {
-	// Nothing to do when no indexes are enabled.
-	if len(m.enabledIndexes) == 0 {
-		return nil
-	}
-
-	if interruptRequested(interrupt) {
-		return errInterruptRequested
-	}
-
-	m.db = db
-
-	// Finish and drops that were previously interrupted.
-	if err := m.maybeFinishDrops(interrupt); err != nil {
-		return err
-	}
-
-	// Create the initial state for the indexes as needed.
-	err := m.db.Update(func(dbTx database.Tx) error {
-		// Create the bucket for the current tips as needed.
-		meta := dbTx.Metadata()
-		_, err := meta.CreateBucketIfNotExists(indexTipsBucketName)
-		if err != nil {
-			return err
-		}
-		if _, err := meta.CreateBucketIfNotExists(indexCurrentBlockIDBucketName); err != nil {
-			return err
-		}
-
-		return m.maybeCreateIndexes(dbTx)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Initialize each of the enabled indexes.
+func (m *Manager) Init(dag *blockdag.BlockDAG) error {
 	for _, indexer := range m.enabledIndexes {
-		if err := indexer.Init(db, blockDAG); err != nil {
+		if err := indexer.Init(dag); err != nil {
 			return err
 		}
 	}
 
-	return m.recoverIfNeeded()
-}
-
-// recoverIfNeeded checks if the node worked for some time
-// without one of the current enabled indexes, and if it's
-// the case, recovers the missing blocks from the index.
-func (m *Manager) recoverIfNeeded() error {
-	return m.db.Update(func(dbTx database.Tx) error {
-		lastKnownBlockID := blockdag.DBFetchCurrentBlockID(dbTx)
-		for _, indexer := range m.enabledIndexes {
-			serializedCurrentIdxBlockID := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Get(indexer.Key())
-			currentIdxBlockID := uint64(0)
-			if serializedCurrentIdxBlockID != nil {
-				currentIdxBlockID = blockdag.DeserializeBlockID(serializedCurrentIdxBlockID)
-			}
-			if lastKnownBlockID > currentIdxBlockID {
-				err := indexer.Recover(dbTx, currentIdxBlockID, lastKnownBlockID)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
+	return nil
 }
 
 // ConnectBlock must be invoked when a block is added to the DAG. It
@@ -197,32 +37,13 @@ func (m *Manager) recoverIfNeeded() error {
 // checks, and invokes each indexer.
 //
 // This is part of the blockdag.IndexManager interface.
-func (m *Manager) ConnectBlock(dbTx database.Tx, block *util.Block, blockID uint64, dag *blockdag.BlockDAG,
-	txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData, virtualTxsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
+func (m *Manager) ConnectBlock(dbContext *dbaccess.TxContext, blockHash *daghash.Hash, txsAcceptanceData blockdag.MultiBlockTxsAcceptanceData) error {
 
 	// Call each of the currently active optional indexes with the block
 	// being connected so they can update accordingly.
 	for _, index := range m.enabledIndexes {
 		// Notify the indexer with the connected block so it can index it.
-		if err := index.ConnectBlock(dbTx, block, blockID, dag, txsAcceptanceData, virtualTxsAcceptanceData); err != nil {
-			return err
-		}
-	}
-
-	// Add the new block ID index entry for the block being connected and
-	// update the current internal block ID accordingly.
-	err := m.updateIndexersWithCurrentBlockID(dbTx, block.Hash(), blockID)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (m *Manager) updateIndexersWithCurrentBlockID(dbTx database.Tx, blockHash *daghash.Hash, blockID uint64) error {
-	serializedBlockID := blockdag.SerializeBlockID(blockID)
-	for _, index := range m.enabledIndexes {
-		err := dbTx.Metadata().Bucket(indexCurrentBlockIDBucketName).Put(index.Key(), serializedBlockID)
-		if err != nil {
+		if err := index.ConnectBlock(dbContext, blockHash, txsAcceptanceData); err != nil {
 			return err
 		}
 	}
@@ -238,155 +59,3 @@ func NewManager(enabledIndexes []Indexer) *Manager {
 		enabledIndexes: enabledIndexes,
 	}
 }
-
-// dropIndex drops the passed index from the database. Since indexes can be
-// massive, it deletes the index in multiple database transactions in order to
-// keep memory usage to reasonable levels. It also marks the drop in progress
-// so the drop can be resumed if it is stopped before it is done before the
-// index can be used again.
-func dropIndex(db database.DB, idxKey []byte, idxName string, interrupt <-chan struct{}) error {
-	// Nothing to do if the index doesn't already exist.
-	var needsDelete bool
-	err := db.View(func(dbTx database.Tx) error {
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		if indexesBucket != nil && indexesBucket.Get(idxKey) != nil {
-			needsDelete = true
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	if !needsDelete {
-		log.Infof("Not dropping %s because it does not exist", idxName)
-		return nil
-	}
-
-	// Mark that the index is in the process of being dropped so that it
-	// can be resumed on the next start if interrupted before the process is
-	// complete.
-	log.Infof("Dropping all %s entries. This might take a while...",
-		idxName)
-	err = db.Update(func(dbTx database.Tx) error {
-		indexesBucket := dbTx.Metadata().Bucket(indexTipsBucketName)
-		return indexesBucket.Put(indexDropKey(idxKey), idxKey)
-	})
-	if err != nil {
-		return err
-	}
-
-	// Since the indexes can be so large, attempting to simply delete
-	// the bucket in a single database transaction would result in massive
-	// memory usage and likely crash many systems due to ulimits. In order
-	// to avoid this, use a cursor to delete a maximum number of entries out
-	// of the bucket at a time. Recurse buckets depth-first to delete any
-	// sub-buckets.
-	const maxDeletions = 2000000
-	var totalDeleted uint64
-
-	// Recurse through all buckets in the index, cataloging each for
-	// later deletion.
-	var subBuckets [][][]byte
-	var subBucketClosure func(database.Tx, []byte, [][]byte) error
-	subBucketClosure = func(dbTx database.Tx,
-		subBucket []byte, tlBucket [][]byte) error {
-		// Get full bucket name and append to subBuckets for later
-		// deletion.
-		var bucketName [][]byte
-		if (tlBucket == nil) || (len(tlBucket) == 0) {
-			bucketName = append(bucketName, subBucket)
-		} else {
-			bucketName = append(tlBucket, subBucket)
-		}
-		subBuckets = append(subBuckets, bucketName)
-		// Recurse sub-buckets to append to subBuckets slice.
-		bucket := dbTx.Metadata()
-		for _, subBucketName := range bucketName {
-			bucket = bucket.Bucket(subBucketName)
-		}
-		return bucket.ForEachBucket(func(k []byte) error {
-			return subBucketClosure(dbTx, k, bucketName)
-		})
-	}
-
-	// Call subBucketClosure with top-level bucket.
-	err = db.View(func(dbTx database.Tx) error {
-		return subBucketClosure(dbTx, idxKey, nil)
-	})
-	if err != nil {
-		return nil
-	}
-
-	// Iterate through each sub-bucket in reverse, deepest-first, deleting
-	// all keys inside them and then dropping the buckets themselves.
-	for i := range subBuckets {
-		bucketName := subBuckets[len(subBuckets)-1-i]
-		// Delete maxDeletions key/value pairs at a time.
-		for numDeleted := maxDeletions; numDeleted == maxDeletions; {
-			numDeleted = 0
-			err := db.Update(func(dbTx database.Tx) error {
-				subBucket := dbTx.Metadata()
-				for _, subBucketName := range bucketName {
-					subBucket = subBucket.Bucket(subBucketName)
-				}
-				cursor := subBucket.Cursor()
-				for ok := cursor.First(); ok; ok = cursor.Next() &&
-					numDeleted < maxDeletions {
-
-					if err := cursor.Delete(); err != nil {
-						return err
-					}
-					numDeleted++
-				}
-				return nil
-			})
-			if err != nil {
-				return err
-			}
-
-			if numDeleted > 0 {
-				totalDeleted += uint64(numDeleted)
-				log.Infof("Deleted %d keys (%d total) from %s",
-					numDeleted, totalDeleted, idxName)
-			}
-		}
-
-		if interruptRequested(interrupt) {
-			return errInterruptRequested
-		}
-
-		// Drop the bucket itself.
-		err = db.Update(func(dbTx database.Tx) error {
-			bucket := dbTx.Metadata()
-			for j := 0; j < len(bucketName)-1; j++ {
-				bucket = bucket.Bucket(bucketName[j])
-			}
-			return bucket.DeleteBucket(bucketName[len(bucketName)-1])
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	// Remove the index tip, index bucket, and in-progress drop flag now
-	// that all index entries have been removed.
-	err = db.Update(func(dbTx database.Tx) error {
-		meta := dbTx.Metadata()
-		indexesBucket := meta.Bucket(indexTipsBucketName)
-		if err := indexesBucket.Delete(idxKey); err != nil {
-			return err
-		}
-
-		if err := meta.Bucket(indexCurrentBlockIDBucketName).Delete(idxKey); err != nil {
-			return err
-		}
-
-		return indexesBucket.Delete(indexDropKey(idxKey))
-	})
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Dropped %s", idxName)
-	return nil
-}

blockdag/mediantime.go

@@ -1,206 +0,0 @@
-// Copyright (c) 2013-2014 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"math"
-	"sort"
-	"sync"
-	"time"
-)
-
-const (
-	// maxAllowedOffsetSeconds is the maximum number of seconds in either
-	// direction that local clock will be adjusted. When the median time
-	// of the network is outside of this range, no offset will be applied.
-	maxAllowedOffsetSecs = 70 * 60 // 1 hour 10 minutes
-
-	// similarTimeSecs is the number of seconds in either direction from the
-	// local clock that is used to determine that it is likley wrong and
-	// hence to show a warning.
-	similarTimeSecs = 5 * 60 // 5 minutes
-)
-
-var (
-	// maxMedianTimeEntries is the maximum number of entries allowed in the
-	// median time data. This is a variable as opposed to a constant so the
-	// test code can modify it.
-	maxMedianTimeEntries = 200
-)
-
-// MedianTimeSource provides a mechanism to add several time samples which are
-// used to determine a median time which is then used as an offset to the local
-// clock.
-type MedianTimeSource interface {
-	// AdjustedTime returns the current time adjusted by the median time
-	// offset as calculated from the time samples added by AddTimeSample.
-	AdjustedTime() time.Time
-
-	// AddTimeSample adds a time sample that is used when determining the
-	// median time of the added samples.
-	AddTimeSample(id string, timeVal time.Time)
-
-	// Offset returns the number of seconds to adjust the local clock based
-	// upon the median of the time samples added by AddTimeData.
-	Offset() time.Duration
-}
-
-// int64Sorter implements sort.Interface to allow a slice of 64-bit integers to
-// be sorted.
-type int64Sorter []int64
-
-// Len returns the number of 64-bit integers in the slice. It is part of the
-// sort.Interface implementation.
-func (s int64Sorter) Len() int {
-	return len(s)
-}
-
-// Swap swaps the 64-bit integers at the passed indices. It is part of the
-// sort.Interface implementation.
-func (s int64Sorter) Swap(i, j int) {
-	s[i], s[j] = s[j], s[i]
-}
-
-// Less returns whether the 64-bit integer with index i should sort before the
-// 64-bit integer with index j. It is part of the sort.Interface
-// implementation.
-func (s int64Sorter) Less(i, j int) bool {
-	return s[i] < s[j]
-}
-
-// medianTime provides an implementation of the MedianTimeSource interface.
-type medianTime struct {
-	mtx                sync.Mutex
-	knownIDs           map[string]struct{}
-	offsets            []int64
-	offsetSecs         int64
-	invalidTimeChecked bool
-}
-
-// Ensure the medianTime type implements the MedianTimeSource interface.
-var _ MedianTimeSource = (*medianTime)(nil)
-
-// AdjustedTime returns the current time adjusted by the median time offset as
-// calculated from the time samples added by AddTimeSample.
-//
-// This function is safe for concurrent access and is part of the
-// MedianTimeSource interface implementation.
-func (m *medianTime) AdjustedTime() time.Time {
-	m.mtx.Lock()
-	defer m.mtx.Unlock()
-
-	// Limit the adjusted time to 1 second precision.
-	now := time.Unix(time.Now().Unix(), 0)
-	return now.Add(time.Duration(m.offsetSecs) * time.Second)
-}
-
-// AddTimeSample adds a time sample that is used when determining the median
-// time of the added samples.
-//
-// This function is safe for concurrent access and is part of the
-// MedianTimeSource interface implementation.
-func (m *medianTime) AddTimeSample(sourceID string, timeVal time.Time) {
-	m.mtx.Lock()
-	defer m.mtx.Unlock()
-
-	// Don't add time data from the same source.
-	if _, exists := m.knownIDs[sourceID]; exists {
-		return
-	}
-	m.knownIDs[sourceID] = struct{}{}
-
-	// Truncate the provided offset to seconds and append it to the slice
-	// of offsets while respecting the maximum number of allowed entries by
-	// replacing the oldest entry with the new entry once the maximum number
-	// of entries is reached.
-	now := time.Unix(time.Now().Unix(), 0)
-	offsetSecs := int64(timeVal.Sub(now).Seconds())
-	numOffsets := len(m.offsets)
-	if numOffsets == maxMedianTimeEntries && maxMedianTimeEntries > 0 {
-		m.offsets = m.offsets[1:]
-		numOffsets--
-	}
-	m.offsets = append(m.offsets, offsetSecs)
-	numOffsets++
-
-	// Sort the offsets so the median can be obtained as needed later.
-	sortedOffsets := make([]int64, numOffsets)
-	copy(sortedOffsets, m.offsets)
-	sort.Sort(int64Sorter(sortedOffsets))
-
-	offsetDuration := time.Duration(offsetSecs) * time.Second
-	log.Debugf("Added time sample of %s (total: %d)", offsetDuration,
-		numOffsets)
-
-	// The median offset is only updated when there are enough offsets and
-	// the number of offsets is odd so the middle value is the true median.
-	// Thus, there is nothing to do when those conditions are not met.
-	if numOffsets < 5 || numOffsets&0x01 != 1 {
-		return
-	}
-
-	// At this point the number of offsets in the list is odd, so the
-	// middle value of the sorted offsets is the median.
-	median := sortedOffsets[numOffsets/2]
-
-	// Set the new offset when the median offset is within the allowed
-	// offset range.
-	if math.Abs(float64(median)) < maxAllowedOffsetSecs {
-		m.offsetSecs = median
-	} else {
-		// The median offset of all added time data is larger than the
-		// maximum allowed offset, so don't use an offset. This
-		// effectively limits how far the local clock can be skewed.
-		m.offsetSecs = 0
-
-		if !m.invalidTimeChecked {
-			m.invalidTimeChecked = true
-
-			// Find if any time samples have a time that is close
-			// to the local time.
-			var remoteHasCloseTime bool
-			for _, offset := range sortedOffsets {
-				if math.Abs(float64(offset)) < similarTimeSecs {
-					remoteHasCloseTime = true
-					break
-				}
-			}
-
-			// Warn if none of the time samples are close.
-			if !remoteHasCloseTime {
-				log.Warnf("Please check your date and time " +
-					"are correct!  kaspad will not work " +
-					"properly with an invalid time")
-			}
-		}
-	}
-
-	medianDuration := time.Duration(m.offsetSecs) * time.Second
-	log.Debugf("New time offset: %d", medianDuration)
-}
-
-// Offset returns the number of seconds to adjust the local clock based upon the
-// median of the time samples added by AddTimeData.
-//
-// This function is safe for concurrent access and is part of the
-// MedianTimeSource interface implementation.
-func (m *medianTime) Offset() time.Duration {
-	m.mtx.Lock()
-	defer m.mtx.Unlock()
-
-	return time.Duration(m.offsetSecs) * time.Second
-}
-
-// NewMedianTime returns a new instance of concurrency-safe implementation of
-// the MedianTimeSource interface. The returned implementation contains the
-// rules necessary for proper time handling in the DAG consensus rules and
-// expects the time samples to be added from the timestamp field of the version
-// message received from remote peers that successfully connect and negotiate.
-func NewMedianTime() MedianTimeSource {
-	return &medianTime{
-		knownIDs: make(map[string]struct{}),
-		offsets:  make([]int64, 0, maxMedianTimeEntries),
-	}
-}

blockdag/mediantime_test.go

@@ -1,102 +0,0 @@
-// Copyright (c) 2013-2017 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package blockdag
-
-import (
-	"strconv"
-	"testing"
-	"time"
-)
-
-// TestMedianTime tests the medianTime implementation.
-func TestMedianTime(t *testing.T) {
-	tests := []struct {
-		in         []int64
-		wantOffset int64
-		useDupID   bool
-	}{
-		// Not enough samples must result in an offset of 0.
-		{in: []int64{1}, wantOffset: 0},
-		{in: []int64{1, 2}, wantOffset: 0},
-		{in: []int64{1, 2, 3}, wantOffset: 0},
-		{in: []int64{1, 2, 3, 4}, wantOffset: 0},
-
-		// Various number of entries. The expected offset is only
-		// updated on odd number of elements.
-		{in: []int64{-13, 57, -4, -23, -12}, wantOffset: -12},
-		{in: []int64{55, -13, 61, -52, 39, 55}, wantOffset: 39},
-		{in: []int64{-62, -58, -30, -62, 51, -30, 15}, wantOffset: -30},
-		{in: []int64{29, -47, 39, 54, 42, 41, 8, -33}, wantOffset: 39},
-		{in: []int64{37, 54, 9, -21, -56, -36, 5, -11, -39}, wantOffset: -11},
-		{in: []int64{57, -28, 25, -39, 9, 63, -16, 19, -60, 25}, wantOffset: 9},
-		{in: []int64{-5, -4, -3, -2, -1}, wantOffset: -3, useDupID: true},
-
-		// The offset stops being updated once the max number of entries
-		// has been reached.
-		{in: []int64{-67, 67, -50, 24, 63, 17, 58, -14, 5, -32, -52}, wantOffset: 17},
-		{in: []int64{-67, 67, -50, 24, 63, 17, 58, -14, 5, -32, -52, 45}, wantOffset: 17},
-		{in: []int64{-67, 67, -50, 24, 63, 17, 58, -14, 5, -32, -52, 45, 4}, wantOffset: 17},
-
-		// Offsets that are too far away from the local time should
-		// be ignored.
-		{in: []int64{-4201, 4202, -4203, 4204, -4205}, wantOffset: 0},
-
-		// Exercise the condition where the median offset is greater
-		// than the max allowed adjustment, but there is at least one
-		// sample that is close enough to the current time to avoid
-		// triggering a warning about an invalid local clock.
-		{in: []int64{4201, 4202, 4203, 4204, -299}, wantOffset: 0},
-	}
-
-	// Modify the max number of allowed median time entries for these tests.
-	maxMedianTimeEntries = 10
-	defer func() { maxMedianTimeEntries = 200 }()
-
-	for i, test := range tests {
-		filter := NewMedianTime()
-		for j, offset := range test.in {
-			id := strconv.Itoa(j)
-			now := time.Unix(time.Now().Unix(), 0)
-			tOffset := now.Add(time.Duration(offset) * time.Second)
-			filter.AddTimeSample(id, tOffset)
-
-			// Ensure the duplicate IDs are ignored.
-			if test.useDupID {
-				// Modify the offsets to ensure the final median
-				// would be different if the duplicate is added.
-				tOffset = tOffset.Add(time.Duration(offset) *
-					time.Second)
-				filter.AddTimeSample(id, tOffset)
-			}
-		}
-
-		// Since it is possible that the time.Now call in AddTimeSample
-		// and the time.Now calls here in the tests will be off by one
-		// second, allow a fudge factor to compensate.
-		gotOffset := filter.Offset()
-		wantOffset := time.Duration(test.wantOffset) * time.Second
-		wantOffset2 := time.Duration(test.wantOffset-1) * time.Second
-		if gotOffset != wantOffset && gotOffset != wantOffset2 {
-			t.Errorf("Offset #%d: unexpected offset -- got %v, "+
-				"want %v or %v", i, gotOffset, wantOffset,
-				wantOffset2)
-			continue
-		}
-
-		// Since it is possible that the time.Now call in AdjustedTime
-		// and the time.Now call here in the tests will be off by one
-		// second, allow a fudge factor to compensate.
-		adjustedTime := filter.AdjustedTime()
-		now := time.Unix(time.Now().Unix(), 0)
-		wantTime := now.Add(filter.Offset())
-		wantTime2 := now.Add(filter.Offset() - time.Second)
-		if !adjustedTime.Equal(wantTime) && !adjustedTime.Equal(wantTime2) {
-			t.Errorf("AdjustedTime #%d: unexpected result -- got %v, "+
-				"want %v or %v", i, adjustedTime, wantTime,
-				wantTime2)
-			continue
-		}
-	}
-}

blockdag/mining.go

@@ -3,8 +3,10 @@ package blockdag
 import (
 	"bytes"
 	"encoding/binary"
+	"github.com/kaspanet/go-secp256k1"
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util"
+	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
 	"time"
 )
@@ -12,6 +14,8 @@ import (
 // BlockForMining returns a block with the given transactions
 // that points to the current DAG tips, that is valid from
 // all aspects except proof of work.
+//
+// This function MUST be called with the DAG state lock held (for reads).
 func (dag *BlockDAG) BlockForMining(transactions []*util.Tx) (*wire.MsgBlock, error) {
 	blockTimestamp := dag.NextBlockTime()
 	requiredDifficulty := dag.NextRequiredDifficulty(blockTimestamp)
@@ -34,18 +38,17 @@ func (dag *BlockDAG) BlockForMining(transactions []*util.Tx) (*wire.MsgBlock, er
 		msgBlock.AddTransaction(tx.MsgTx())
 	}
 
-	utxoWithTransactions, err := dag.UTXOSet().WithTransactions(msgBlock.Transactions, UnacceptedBlueScore, false)
+	multiset, err := dag.NextBlockMultiset()
 	if err != nil {
 		return nil, err
 	}
-	utxoCommitment := utxoWithTransactions.Multiset().Hash()
 
 	msgBlock.Header = wire.BlockHeader{
 		Version:              nextBlockVersion,
 		ParentHashes:         dag.TipHashes(),
 		HashMerkleRoot:       hashMerkleTree.Root(),
 		AcceptedIDMerkleRoot: acceptedIDMerkleRoot,
-		UTXOCommitment:       utxoCommitment,
+		UTXOCommitment:       (*daghash.Hash)(multiset.Finalize()),
 		Timestamp:            blockTimestamp,
 		Bits:                 requiredDifficulty,
 	}
@@ -53,6 +56,19 @@ func (dag *BlockDAG) BlockForMining(transactions []*util.Tx) (*wire.MsgBlock, er
 	return &msgBlock, nil
 }
 
+// NextBlockMultiset returns the multiset of an assumed next block
+// built on top of the current tips.
+//
+// This function MUST be called with the DAG state lock held (for reads).
+func (dag *BlockDAG) NextBlockMultiset() (*secp256k1.MultiSet, error) {
+	_, selectedParentPastUTXO, txsAcceptanceData, err := dag.pastUTXO(&dag.virtual.blockNode)
+	if err != nil {
+		return nil, err
+	}
+
+	return dag.virtual.blockNode.calcMultiset(dag, txsAcceptanceData, selectedParentPastUTXO)
+}
+
 // CoinbasePayloadExtraData returns coinbase payload extra data parameter
 // which is built from extra nonce and coinbase flags.
 func CoinbasePayloadExtraData(extraNonce uint64, coinbaseFlags string) ([]byte, error) {
@@ -101,7 +117,7 @@ func (dag *BlockDAG) NextBlockTime() time.Time {
 	// timestamp is truncated to a second boundary before comparison since a
 	// block timestamp does not supported a precision greater than one
 	// second.
-	newTimestamp := dag.AdjustedTime()
+	newTimestamp := dag.Now()
 	minTimestamp := dag.NextBlockMinimumTime()
 	if newTimestamp.Before(minTimestamp) {
 		newTimestamp = minTimestamp

blockdag/multisetio.go

@@ -0,0 +1,29 @@
+package blockdag
+
+import (
+	"encoding/binary"
+	"github.com/kaspanet/go-secp256k1"
+	"io"
+)
+
+const multisetPointSize = 32
+
+// serializeMultiset serializes an ECMH multiset.
+func serializeMultiset(w io.Writer, ms *secp256k1.MultiSet) error {
+	serialized := ms.Serialize()
+	err := binary.Write(w, byteOrder, serialized)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// deserializeMultiset deserializes an EMCH multiset.
+func deserializeMultiset(r io.Reader) (*secp256k1.MultiSet, error) {
+	serialized := &secp256k1.SerializedMultiSet{}
+	err := binary.Read(r, byteOrder, serialized[:])
+	if err != nil {
+		return nil, err
+	}
+	return secp256k1.DeserializeMultiSet(serialized)
+}

blockdag/multisetstore.go

@@ -0,0 +1,131 @@
+package blockdag
+
+import (
+	"bytes"
+	"github.com/kaspanet/go-secp256k1"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/util/locks"
+	"github.com/pkg/errors"
+)
+
+type multisetStore struct {
+	dag    *BlockDAG
+	new    map[daghash.Hash]struct{}
+	loaded map[daghash.Hash]secp256k1.MultiSet
+	mtx    *locks.PriorityMutex
+}
+
+func newMultisetStore(dag *BlockDAG) *multisetStore {
+	return &multisetStore{
+		dag:    dag,
+		new:    make(map[daghash.Hash]struct{}),
+		loaded: make(map[daghash.Hash]secp256k1.MultiSet),
+	}
+}
+
+func (store *multisetStore) setMultiset(node *blockNode, ms *secp256k1.MultiSet) {
+	store.loaded[*node.hash] = *ms
+	store.addToNewBlocks(node.hash)
+}
+
+func (store *multisetStore) addToNewBlocks(blockHash *daghash.Hash) {
+	store.new[*blockHash] = struct{}{}
+}
+
+func multisetNotFoundError(blockHash *daghash.Hash) error {
+	return errors.Errorf("Couldn't find multiset data for block %s", blockHash)
+}
+
+func (store *multisetStore) multisetByBlockNode(node *blockNode) (*secp256k1.MultiSet, error) {
+	ms, exists := store.multisetByBlockHash(node.hash)
+	if !exists {
+		return nil, multisetNotFoundError(node.hash)
+	}
+	return ms, nil
+}
+
+func (store *multisetStore) multisetByBlockHash(hash *daghash.Hash) (*secp256k1.MultiSet, bool) {
+	ms, ok := store.loaded[*hash]
+	return &ms, ok
+}
+
+// flushToDB writes all new multiset data to the database.
+func (store *multisetStore) flushToDB(dbContext *dbaccess.TxContext) error {
+	if len(store.new) == 0 {
+		return nil
+	}
+
+	w := &bytes.Buffer{}
+	for hash := range store.new {
+		hash := hash // Copy hash to a new variable to avoid passing the same pointer
+
+		w.Reset()
+		ms, exists := store.loaded[hash]
+		if !exists {
+			return multisetNotFoundError(&hash)
+		}
+
+		err := serializeMultiset(w, &ms)
+		if err != nil {
+			return err
+		}
+
+		err = store.storeMultiset(dbContext, &hash, w.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (store *multisetStore) clearNewEntries() {
+	store.new = make(map[daghash.Hash]struct{})
+}
+
+func (store *multisetStore) init(dbContext dbaccess.Context) error {
+	cursor, err := dbaccess.MultisetCursor(dbContext)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		key, err := cursor.Key()
+		if err != nil {
+			return err
+		}
+
+		hash, err := daghash.NewHash(key.Suffix())
+		if err != nil {
+			return err
+		}
+
+		serializedMS, err := cursor.Value()
+		if err != nil {
+			return err
+		}
+
+		ms, err := deserializeMultiset(bytes.NewReader(serializedMS))
+		if err != nil {
+			return err
+		}
+
+		store.loaded[*hash] = *ms
+	}
+	return nil
+}
+
+// storeMultiset stores the multiset data to the database.
+func (store *multisetStore) storeMultiset(dbContext dbaccess.Context, blockHash *daghash.Hash, serializedMS []byte) error {
+	exists, err := dbaccess.HasMultiset(dbContext, blockHash)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		return errors.Errorf("Can't override an existing multiset database entry for block %s", blockHash)
+	}
+
+	return dbaccess.StoreMultiset(dbContext, blockHash, serializedMS)
+}

blockdag/notifications_test.go

@@ -19,7 +19,7 @@ func TestNotifications(t *testing.T) {
 	}
 
 	// Create a new database and dag instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("notifications", Config{
+	dag, teardownFunc, err := DAGSetup("notifications", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {

blockdag/process.go

@@ -253,6 +253,8 @@ func (dag *BlockDAG) processBlockNoLock(block *util.Block, flags BehaviorFlags)
 		}
 	}
 
+	dag.addBlockProcessingTimestamp()
+
 	log.Debugf("Accepted block %s", blockHash)
 
 	return false, false, nil
@@ -264,7 +266,7 @@ func (dag *BlockDAG) maxDelayOfParents(parentHashes []*daghash.Hash) (delay time
 	for _, parentHash := range parentHashes {
 		if delayedParent, exists := dag.delayedBlocks[*parentHash]; exists {
 			isDelayed = true
-			parentDelay := delayedParent.processTime.Sub(dag.AdjustedTime())
+			parentDelay := delayedParent.processTime.Sub(dag.Now())
 			if parentDelay > delay {
 				delay = parentDelay
 			}

blockdag/process_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestProcessOrphans(t *testing.T) {
-	dag, teardownFunc, err := DAGSetup("TestProcessOrphans", Config{
+	dag, teardownFunc, err := DAGSetup("TestProcessOrphans", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -63,8 +63,8 @@ func TestProcessOrphans(t *testing.T) {
 	}
 
 	// Make sure that the child block had been rejected
-	node := dag.index.LookupNode(childBlock.Hash())
-	if node == nil {
+	node, ok := dag.index.LookupNode(childBlock.Hash())
+	if !ok {
 		t.Fatalf("TestProcessOrphans: child block missing from block index")
 	}
 	if !dag.index.NodeStatus(node).KnownInvalid() {
@@ -72,45 +72,35 @@ func TestProcessOrphans(t *testing.T) {
 	}
 }
 
-type fakeTimeSource struct {
-	time time.Time
-}
-
-func (fts *fakeTimeSource) AdjustedTime() time.Time {
-	return fts.time
-}
-
-func (fts *fakeTimeSource) AddTimeSample(_ string, _ time.Time) {
-}
-
-func (fts *fakeTimeSource) Offset() time.Duration {
-	return 0
-}
-
 func TestProcessDelayedBlocks(t *testing.T) {
 	// We use dag1 so we can build the test blocks with the proper
 	// block header (UTXO commitment, acceptedIDMerkleroot, etc), and
 	// then we use dag2 for the actual test.
-	dag1, teardownFunc, err := DAGSetup("TestProcessDelayedBlocks1", Config{
+	dag1, teardownFunc, err := DAGSetup("TestProcessDelayedBlocks1", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Fatalf("Failed to setup DAG instance: %v", err)
 	}
-	defer teardownFunc()
+	isDAG1Open := true
+	defer func() {
+		if isDAG1Open {
+			teardownFunc()
+		}
+	}()
 
 	initialTime := dag1.dagParams.GenesisBlock.Header.Timestamp
 	// Here we use a fake time source that returns a timestamp
 	// one hour into the future to make delayedBlock artificially
 	// valid.
-	dag1.timeSource = &fakeTimeSource{initialTime.Add(time.Hour)}
+	dag1.timeSource = newFakeTimeSource(initialTime.Add(time.Hour))
 
 	delayedBlock, err := PrepareBlockForTest(dag1, []*daghash.Hash{dag1.dagParams.GenesisBlock.BlockHash()}, nil)
 	if err != nil {
 		t.Fatalf("error in PrepareBlockForTest: %s", err)
 	}
 
-	blockDelay := time.Duration(dag1.dagParams.TimestampDeviationTolerance+5) * time.Second
+	blockDelay := time.Duration(dag1.dagParams.TimestampDeviationTolerance*uint64(dag1.targetTimePerBlock)+5) * time.Second
 	delayedBlock.Header.Timestamp = initialTime.Add(blockDelay)
 
 	isOrphan, isDelayed, err := dag1.ProcessBlock(util.NewBlock(delayedBlock), BFNoPoWCheck)
@@ -131,18 +121,21 @@ func TestProcessDelayedBlocks(t *testing.T) {
 		t.Fatalf("error in PrepareBlockForTest: %s", err)
 	}
 
+	teardownFunc()
+	isDAG1Open = false
+
 	// Here the actual test begins. We add a delayed block and
 	// its child and check that they are not added to the DAG,
 	// and check that they're added only if we add a new block
 	// after the delayed block timestamp is valid.
-	dag2, teardownFunc2, err := DAGSetup("TestProcessDelayedBlocks2", Config{
+	dag2, teardownFunc2, err := DAGSetup("TestProcessDelayedBlocks2", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
 		t.Fatalf("Failed to setup DAG instance: %v", err)
 	}
 	defer teardownFunc2()
-	dag2.timeSource = &fakeTimeSource{initialTime}
+	dag2.timeSource = newFakeTimeSource(initialTime)
 
 	isOrphan, isDelayed, err = dag2.ProcessBlock(util.NewBlock(delayedBlock), BFNoPoWCheck)
 	if err != nil {
@@ -209,10 +202,13 @@ func TestProcessDelayedBlocks(t *testing.T) {
 	}
 
 	// We advance the clock to the point where delayedBlock timestamp is valid.
-	secondsUntilDelayedBlockIsValid := delayedBlock.Header.Timestamp.Unix() - int64(dag2.TimestampDeviationTolerance) - dag2.AdjustedTime().Unix() + 1
-	dag2.timeSource = &fakeTimeSource{initialTime.Add(time.Duration(secondsUntilDelayedBlockIsValid) * time.Second)}
+	deviationTolerance := int64(dag2.TimestampDeviationTolerance) * dag2.targetTimePerBlock
+	secondsUntilDelayedBlockIsValid := delayedBlock.Header.Timestamp.Unix() - deviationTolerance - dag2.Now().Unix() + 1
+	dag2.timeSource = newFakeTimeSource(initialTime.Add(time.Duration(secondsUntilDelayedBlockIsValid) * time.Second))
 
-	blockAfterDelay, err := PrepareBlockForTest(dag2, []*daghash.Hash{dag2.dagParams.GenesisBlock.BlockHash()}, nil)
+	blockAfterDelay, err := PrepareBlockForTest(dag2,
+		[]*daghash.Hash{dag2.dagParams.GenesisBlock.BlockHash()},
+		nil)
 	if err != nil {
 		t.Fatalf("error in PrepareBlockForTest: %s", err)
 	}

blockdag/reachability_test.go

@@ -1,6 +1,8 @@
 package blockdag
 
 import (
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/util/daghash"
 	"reflect"
 	"strings"
 	"testing"
@@ -11,19 +13,20 @@ func TestAddChild(t *testing.T) {
 	//             root -> a -> b -> c...
 	// Create the root node of a new reachability tree
 	root := newReachabilityTreeNode(&blockNode{})
-	root.setInterval(newReachabilityInterval(1, 100))
+	root.interval = newReachabilityInterval(1, 100)
 
 	// Add a chain of child nodes just before a reindex occurs (2^6=64 < 100)
 	currentTip := root
 	for i := 0; i < 6; i++ {
 		node := newReachabilityTreeNode(&blockNode{})
-		modifiedNodes, err := currentTip.addChild(node)
+		modifiedNodes := newModifiedTreeNodes()
+		err := currentTip.addChild(node, root, modifiedNodes)
 		if err != nil {
 			t.Fatalf("TestAddChild: addChild failed: %s", err)
 		}
 
 		// Expect only the node and its parent to be affected
-		expectedModifiedNodes := []*reachabilityTreeNode{currentTip, node}
+		expectedModifiedNodes := newModifiedTreeNodes(currentTip, node)
 		if !reflect.DeepEqual(modifiedNodes, expectedModifiedNodes) {
 			t.Fatalf("TestAddChild: unexpected modifiedNodes. "+
 				"want: %s, got: %s", expectedModifiedNodes, modifiedNodes)
@@ -34,7 +37,8 @@ func TestAddChild(t *testing.T) {
 
 	// Add another node to the tip of the chain to trigger a reindex (100 < 2^7=128)
 	lastChild := newReachabilityTreeNode(&blockNode{})
-	modifiedNodes, err := currentTip.addChild(lastChild)
+	modifiedNodes := newModifiedTreeNodes()
+	err := currentTip.addChild(lastChild, root, modifiedNodes)
 	if err != nil {
 		t.Fatalf("TestAddChild: addChild failed: %s", err)
 	}
@@ -45,19 +49,23 @@ func TestAddChild(t *testing.T) {
 		t.Fatalf("TestAddChild: unexpected amount of modifiedNodes.")
 	}
 
-	// Expect the tip to have an interval of 1 and remaining interval of 0
+	// Expect the tip to have an interval of 1 and remaining interval of 0 both before and after
 	tipInterval := lastChild.interval.size()
 	if tipInterval != 1 {
 		t.Fatalf("TestAddChild: unexpected tip interval size: want: 1, got: %d", tipInterval)
 	}
-	tipRemainingInterval := lastChild.remainingInterval.size()
-	if tipRemainingInterval != 0 {
-		t.Fatalf("TestAddChild: unexpected tip interval size: want: 0, got: %d", tipRemainingInterval)
+	tipRemainingIntervalBefore := lastChild.remainingIntervalBefore().size()
+	if tipRemainingIntervalBefore != 0 {
+		t.Fatalf("TestAddChild: unexpected tip interval before size: want: 0, got: %d", tipRemainingIntervalBefore)
+	}
+	tipRemainingIntervalAfter := lastChild.remainingIntervalAfter().size()
+	if tipRemainingIntervalAfter != 0 {
+		t.Fatalf("TestAddChild: unexpected tip interval after size: want: 0, got: %d", tipRemainingIntervalAfter)
 	}
 
 	// Expect all nodes to be descendant nodes of root
 	currentNode := currentTip
-	for currentNode != nil {
+	for currentNode != root {
 		if !root.isAncestorOf(currentNode) {
 			t.Fatalf("TestAddChild: currentNode is not a descendant of root")
 		}
@@ -68,19 +76,20 @@ func TestAddChild(t *testing.T) {
 	//             root -> a, b, c...
 	// Create the root node of a new reachability tree
 	root = newReachabilityTreeNode(&blockNode{})
-	root.setInterval(newReachabilityInterval(1, 100))
+	root.interval = newReachabilityInterval(1, 100)
 
 	// Add child nodes to root just before a reindex occurs (2^6=64 < 100)
 	childNodes := make([]*reachabilityTreeNode, 6)
 	for i := 0; i < len(childNodes); i++ {
 		childNodes[i] = newReachabilityTreeNode(&blockNode{})
-		modifiedNodes, err := root.addChild(childNodes[i])
+		modifiedNodes := newModifiedTreeNodes()
+		err := root.addChild(childNodes[i], root, modifiedNodes)
 		if err != nil {
 			t.Fatalf("TestAddChild: addChild failed: %s", err)
 		}
 
 		// Expect only the node and the root to be affected
-		expectedModifiedNodes := []*reachabilityTreeNode{root, childNodes[i]}
+		expectedModifiedNodes := newModifiedTreeNodes(root, childNodes[i])
 		if !reflect.DeepEqual(modifiedNodes, expectedModifiedNodes) {
 			t.Fatalf("TestAddChild: unexpected modifiedNodes. "+
 				"want: %s, got: %s", expectedModifiedNodes, modifiedNodes)
@@ -89,7 +98,8 @@ func TestAddChild(t *testing.T) {
 
 	// Add another node to the root to trigger a reindex (100 < 2^7=128)
 	lastChild = newReachabilityTreeNode(&blockNode{})
-	modifiedNodes, err = root.addChild(lastChild)
+	modifiedNodes = newModifiedTreeNodes()
+	err = root.addChild(lastChild, root, modifiedNodes)
 	if err != nil {
 		t.Fatalf("TestAddChild: addChild failed: %s", err)
 	}
@@ -100,14 +110,18 @@ func TestAddChild(t *testing.T) {
 		t.Fatalf("TestAddChild: unexpected amount of modifiedNodes.")
 	}
 
-	// Expect the last-added child to have an interval of 1 and remaining interval of 0
+	// Expect the last-added child to have an interval of 1 and remaining interval of 0 both before and after
 	lastChildInterval := lastChild.interval.size()
 	if lastChildInterval != 1 {
 		t.Fatalf("TestAddChild: unexpected lastChild interval size: want: 1, got: %d", lastChildInterval)
 	}
-	lastChildRemainingInterval := lastChild.remainingInterval.size()
-	if lastChildRemainingInterval != 0 {
-		t.Fatalf("TestAddChild: unexpected lastChild interval size: want: 0, got: %d", lastChildRemainingInterval)
+	lastChildRemainingIntervalBefore := lastChild.remainingIntervalBefore().size()
+	if lastChildRemainingIntervalBefore != 0 {
+		t.Fatalf("TestAddChild: unexpected lastChild interval before size: want: 0, got: %d", lastChildRemainingIntervalBefore)
+	}
+	lastChildRemainingIntervalAfter := lastChild.remainingIntervalAfter().size()
+	if lastChildRemainingIntervalAfter != 0 {
+		t.Fatalf("TestAddChild: unexpected lastChild interval after size: want: 0, got: %d", lastChildRemainingIntervalAfter)
 	}
 
 	// Expect all nodes to be descendant nodes of root
@@ -118,6 +132,91 @@ func TestAddChild(t *testing.T) {
 	}
 }
 
+func TestReachabilityTreeNodeIsAncestorOf(t *testing.T) {
+	root := newReachabilityTreeNode(&blockNode{})
+	currentTip := root
+	const numberOfDescendants = 6
+	descendants := make([]*reachabilityTreeNode, numberOfDescendants)
+	for i := 0; i < numberOfDescendants; i++ {
+		node := newReachabilityTreeNode(&blockNode{})
+		err := currentTip.addChild(node, root, newModifiedTreeNodes())
+		if err != nil {
+			t.Fatalf("TestReachabilityTreeNodeIsAncestorOf: addChild failed: %s", err)
+		}
+		descendants[i] = node
+		currentTip = node
+	}
+
+	// Expect all descendants to be in the future of root
+	for _, node := range descendants {
+		if !root.isAncestorOf(node) {
+			t.Fatalf("TestReachabilityTreeNodeIsAncestorOf: node is not a descendant of root")
+		}
+	}
+
+	if !root.isAncestorOf(root) {
+		t.Fatalf("TestReachabilityTreeNodeIsAncestorOf: root is expected to be an ancestor of root")
+	}
+}
+
+func TestIntervalContains(t *testing.T) {
+	tests := []struct {
+		name              string
+		this, other       *reachabilityInterval
+		thisContainsOther bool
+	}{
+		{
+			name:              "this == other",
+			this:              newReachabilityInterval(10, 100),
+			other:             newReachabilityInterval(10, 100),
+			thisContainsOther: true,
+		},
+		{
+			name:              "this.start == other.start && this.end < other.end",
+			this:              newReachabilityInterval(10, 90),
+			other:             newReachabilityInterval(10, 100),
+			thisContainsOther: false,
+		},
+		{
+			name:              "this.start == other.start && this.end > other.end",
+			this:              newReachabilityInterval(10, 100),
+			other:             newReachabilityInterval(10, 90),
+			thisContainsOther: true,
+		},
+		{
+			name:              "this.start > other.start && this.end == other.end",
+			this:              newReachabilityInterval(20, 100),
+			other:             newReachabilityInterval(10, 100),
+			thisContainsOther: false,
+		},
+		{
+			name:              "this.start < other.start && this.end == other.end",
+			this:              newReachabilityInterval(10, 100),
+			other:             newReachabilityInterval(20, 100),
+			thisContainsOther: true,
+		},
+		{
+			name:              "this.start > other.start && this.end < other.end",
+			this:              newReachabilityInterval(20, 90),
+			other:             newReachabilityInterval(10, 100),
+			thisContainsOther: false,
+		},
+		{
+			name:              "this.start < other.start && this.end > other.end",
+			this:              newReachabilityInterval(10, 100),
+			other:             newReachabilityInterval(20, 90),
+			thisContainsOther: true,
+		},
+	}
+
+	for _, test := range tests {
+		if thisContainsOther := test.this.contains(test.other); thisContainsOther != test.thisContainsOther {
+			t.Errorf("test.this.contains(test.other) is expected to be %t but got %t",
+				test.thisContainsOther, thisContainsOther)
+		}
+	}
+}
+
 func TestSplitFraction(t *testing.T) {
 	tests := []struct {
 		interval      *reachabilityInterval
@@ -346,140 +445,140 @@ func TestSplitWithExponentialBias(t *testing.T) {
 	}
 }
 
-func TestIsInFuture(t *testing.T) {
-	blocks := futureCoveringBlockSet{
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(2, 3)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
+func TestHasAncestorOf(t *testing.T) {
+	treeNodes := futureCoveringTreeNodeSet{
+		&reachabilityTreeNode{interval: newReachabilityInterval(2, 3)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(4, 67)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(67, 77)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(657, 789)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)},
 	}
 
 	tests := []struct {
-		block          *futureCoveringBlock
+		treeNode       *reachabilityTreeNode
 		expectedResult bool
 	}{
 		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 1)}},
+			treeNode:       &reachabilityTreeNode{interval: newReachabilityInterval(1, 1)},
 			expectedResult: false,
 		},
 		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(5, 7)}},
+			treeNode:       &reachabilityTreeNode{interval: newReachabilityInterval(5, 7)},
 			expectedResult: true,
 		},
 		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 76)}},
+			treeNode:       &reachabilityTreeNode{interval: newReachabilityInterval(67, 76)},
 			expectedResult: true,
 		},
 		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(78, 100)}},
+			treeNode:       &reachabilityTreeNode{interval: newReachabilityInterval(78, 100)},
 			expectedResult: false,
 		},
 		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1980, 2000)}},
+			treeNode:       &reachabilityTreeNode{interval: newReachabilityInterval(1980, 2000)},
 			expectedResult: false,
 		},
 		{
-			block:          &futureCoveringBlock{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1920)}},
+			treeNode:       &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1920)},
 			expectedResult: true,
 		},
 	}
 
 	for i, test := range tests {
-		result := blocks.isInFuture(test.block)
+		result := treeNodes.hasAncestorOf(test.treeNode)
 		if result != test.expectedResult {
-			t.Errorf("TestIsInFuture: unexpected result in test #%d. Want: %t, got: %t",
+			t.Errorf("TestHasAncestorOf: unexpected result in test #%d. Want: %t, got: %t",
 				i, test.expectedResult, result)
 		}
 	}
 }
 
-func TestInsertBlock(t *testing.T) {
-	blocks := futureCoveringBlockSet{
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-		{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
+func TestInsertNode(t *testing.T) {
+	treeNodes := futureCoveringTreeNodeSet{
+		&reachabilityTreeNode{interval: newReachabilityInterval(1, 3)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(4, 67)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(67, 77)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(657, 789)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)},
+		&reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)},
 	}
 
 	tests := []struct {
-		toInsert       []*futureCoveringBlock
-		expectedResult futureCoveringBlockSet
+		toInsert       []*reachabilityTreeNode
+		expectedResult futureCoveringTreeNodeSet
 	}{
 		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(5, 7)}},
+			toInsert: []*reachabilityTreeNode{
+				{interval: newReachabilityInterval(5, 7)},
 			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
+			expectedResult: futureCoveringTreeNodeSet{
+				&reachabilityTreeNode{interval: newReachabilityInterval(1, 3)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(4, 67)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(67, 77)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(657, 789)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)},
 			},
 		},
 		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(65, 78)}},
+			toInsert: []*reachabilityTreeNode{
+				{interval: newReachabilityInterval(65, 78)},
 			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(65, 78)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
+			expectedResult: futureCoveringTreeNodeSet{
+				&reachabilityTreeNode{interval: newReachabilityInterval(1, 3)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(4, 67)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(65, 78)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(657, 789)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)},
 			},
 		},
 		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
+			toInsert: []*reachabilityTreeNode{
+				{interval: newReachabilityInterval(88, 97)},
 			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
+			expectedResult: futureCoveringTreeNodeSet{
+				&reachabilityTreeNode{interval: newReachabilityInterval(1, 3)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(4, 67)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(67, 77)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(88, 97)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(657, 789)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)},
 			},
 		},
 		{
-			toInsert: []*futureCoveringBlock{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(3000, 3010)}},
+			toInsert: []*reachabilityTreeNode{
+				{interval: newReachabilityInterval(88, 97)},
+				{interval: newReachabilityInterval(3000, 3010)},
 			},
-			expectedResult: futureCoveringBlockSet{
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1, 3)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(4, 67)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(67, 77)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(88, 97)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(657, 789)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)}},
-				{treeNode: &reachabilityTreeNode{interval: newReachabilityInterval(3000, 3010)}},
+			expectedResult: futureCoveringTreeNodeSet{
+				&reachabilityTreeNode{interval: newReachabilityInterval(1, 3)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(4, 67)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(67, 77)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(88, 97)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(657, 789)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1000, 1000)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(1920, 1921)},
+				&reachabilityTreeNode{interval: newReachabilityInterval(3000, 3010)},
 			},
 		},
 	}
 
 	for i, test := range tests {
-		// Create a clone of blocks so that we have a clean start for every test
-		blocksClone := make(futureCoveringBlockSet, len(blocks))
-		for i, block := range blocks {
-			blocksClone[i] = block
+		// Create a clone of treeNodes so that we have a clean start for every test
+		treeNodesClone := make(futureCoveringTreeNodeSet, len(treeNodes))
+		for i, treeNode := range treeNodes {
+			treeNodesClone[i] = treeNode
 		}
 
-		for _, block := range test.toInsert {
-			blocksClone.insertBlock(block)
+		for _, treeNode := range test.toInsert {
+			treeNodesClone.insertNode(treeNode)
 		}
-		if !reflect.DeepEqual(blocksClone, test.expectedResult) {
-			t.Errorf("TestInsertBlock: unexpected result in test #%d. Want: %s, got: %s",
-				i, test.expectedResult, blocksClone)
+		if !reflect.DeepEqual(treeNodesClone, test.expectedResult) {
+			t.Errorf("TestInsertNode: unexpected result in test #%d. Want: %s, got: %s",
+				i, test.expectedResult, treeNodesClone)
 		}
 	}
 }
@@ -580,14 +679,14 @@ func TestSplitWithExponentialBiasErrors(t *testing.T) {
 func TestReindexIntervalErrors(t *testing.T) {
 	// Create a treeNode and give it size = 100
 	treeNode := newReachabilityTreeNode(&blockNode{})
-	treeNode.setInterval(newReachabilityInterval(0, 99))
+	treeNode.interval = newReachabilityInterval(0, 99)
 
 	// Add a chain of 100 child treeNodes to treeNode
 	var err error
 	currentTreeNode := treeNode
 	for i := 0; i < 100; i++ {
 		childTreeNode := newReachabilityTreeNode(&blockNode{})
-		_, err = currentTreeNode.addChild(childTreeNode)
+		err = currentTreeNode.addChild(childTreeNode, treeNode, newModifiedTreeNodes())
 		if err != nil {
 			break
 		}
@@ -609,33 +708,70 @@ func TestReindexIntervalErrors(t *testing.T) {
 	}
 }
 
-func TestFutureCoveringBlockSetString(t *testing.T) {
-	treeNodeA := newReachabilityTreeNode(&blockNode{})
-	treeNodeA.setInterval(newReachabilityInterval(123, 456))
-	treeNodeB := newReachabilityTreeNode(&blockNode{})
-	treeNodeB.setInterval(newReachabilityInterval(457, 789))
-	futureCoveringSet := futureCoveringBlockSet{
-		&futureCoveringBlock{treeNode: treeNodeA},
-		&futureCoveringBlock{treeNode: treeNodeB},
+func BenchmarkReindexInterval(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		root := newReachabilityTreeNode(&blockNode{})
+
+		const subTreeSize = 70000
+		// We set the interval of the root to subTreeSize*2 because
+		// its first child gets half of the interval, so a reindex
+		// from the root should happen after adding subTreeSize
+		// nodes.
+		root.interval = newReachabilityInterval(0, subTreeSize*2)
+
+		currentTreeNode := root
+		for i := 0; i < subTreeSize; i++ {
+			childTreeNode := newReachabilityTreeNode(&blockNode{})
+			err := currentTreeNode.addChild(childTreeNode, root, newModifiedTreeNodes())
+			if err != nil {
+				b.Fatalf("addChild: %s", err)
+			}
+
+			currentTreeNode = childTreeNode
+		}
+
+		originalRemainingInterval := *root.remainingIntervalAfter()
+		// After we added subTreeSize nodes, adding the next
+		// node should lead to a reindex from root.
+		fullReindexTriggeringNode := newReachabilityTreeNode(&blockNode{})
+		b.StartTimer()
+		err := currentTreeNode.addChild(fullReindexTriggeringNode, root, newModifiedTreeNodes())
+		b.StopTimer()
+		if err != nil {
+			b.Fatalf("addChild: %s", err)
+		}
+
+		if *root.remainingIntervalAfter() == originalRemainingInterval {
+			b.Fatal("Expected a reindex from root, but it didn't happen")
+		}
 	}
+}
+
+func TestFutureCoveringTreeNodeSetString(t *testing.T) {
+	treeNodeA := newReachabilityTreeNode(&blockNode{})
+	treeNodeA.interval = newReachabilityInterval(123, 456)
+	treeNodeB := newReachabilityTreeNode(&blockNode{})
+	treeNodeB.interval = newReachabilityInterval(457, 789)
+	futureCoveringSet := futureCoveringTreeNodeSet{treeNodeA, treeNodeB}
 
 	str := futureCoveringSet.String()
 	expectedStr := "[123,456][457,789]"
 	if str != expectedStr {
-		t.Fatalf("TestFutureCoveringBlockSetString: unexpected "+
+		t.Fatalf("TestFutureCoveringTreeNodeSetString: unexpected "+
 			"string. Want: %s, got: %s", expectedStr, str)
 	}
 }
 
 func TestReachabilityTreeNodeString(t *testing.T) {
 	treeNodeA := newReachabilityTreeNode(&blockNode{})
-	treeNodeA.setInterval(newReachabilityInterval(100, 199))
+	treeNodeA.interval = newReachabilityInterval(100, 199)
 	treeNodeB1 := newReachabilityTreeNode(&blockNode{})
-	treeNodeB1.setInterval(newReachabilityInterval(100, 150))
+	treeNodeB1.interval = newReachabilityInterval(100, 150)
 	treeNodeB2 := newReachabilityTreeNode(&blockNode{})
-	treeNodeB2.setInterval(newReachabilityInterval(150, 199))
+	treeNodeB2.interval = newReachabilityInterval(150, 199)
 	treeNodeC := newReachabilityTreeNode(&blockNode{})
-	treeNodeC.setInterval(newReachabilityInterval(100, 149))
+	treeNodeC.interval = newReachabilityInterval(100, 149)
 	treeNodeA.children = []*reachabilityTreeNode{treeNodeB1, treeNodeB2}
 	treeNodeB2.children = []*reachabilityTreeNode{treeNodeC}
 
@@ -646,3 +782,268 @@ func TestReachabilityTreeNodeString(t *testing.T) {
 			"string. Want: %s, got: %s", expectedStr, str)
 	}
 }
+
+func TestIsInPast(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestIsInPast", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestIsInPast: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Add a chain of two blocks above the genesis. This will be the
+	// selected parent chain.
+	blockA := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	blockB := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockA.BlockHash()}, nil)
+
+	// Add another block above the genesis
+	blockC := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	nodeC, ok := dag.index.LookupNode(blockC.BlockHash())
+	if !ok {
+		t.Fatalf("TestIsInPast: block C is not in the block index")
+	}
+
+	// Add a block whose parents are the two tips
+	blockD := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{blockB.BlockHash(), blockC.BlockHash()}, nil)
+	nodeD, ok := dag.index.LookupNode(blockD.BlockHash())
+	if !ok {
+		t.Fatalf("TestIsInPast: block C is not in the block index")
+	}
+
+	// Make sure that node C is in the past of node D
+	isInFuture, err := dag.reachabilityTree.isInPast(nodeC, nodeD)
+	if err != nil {
+		t.Fatalf("TestIsInPast: isInPast unexpectedly failed: %s", err)
+	}
+	if !isInFuture {
+		t.Fatalf("TestIsInPast: node C is unexpectedly not the past of node D")
+	}
+}
+
+func TestAddChildThatPointsDirectlyToTheSelectedParentChainBelowReindexRoot(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestAddChildThatPointsDirectlyToTheSelectedParentChainBelowReindexRoot",
+		true, Config{DAGParams: &dagconfig.SimnetParams})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Set the reindex window to a low number to make this test run fast
+	originalReachabilityReindexWindow := reachabilityReindexWindow
+	reachabilityReindexWindow = 10
+	defer func() {
+		reachabilityReindexWindow = originalReachabilityReindexWindow
+	}()
+
+	// Add a block on top of the genesis block
+	chainRootBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+
+	// Add chain of reachabilityReindexWindow blocks above chainRootBlock.
+	// This should move the reindex root
+	chainRootBlockTipHash := chainRootBlock.BlockHash()
+	for i := uint64(0); i < reachabilityReindexWindow; i++ {
+		chainBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{chainRootBlockTipHash}, nil)
+		chainRootBlockTipHash = chainBlock.BlockHash()
+	}
+
+	// Add another block over genesis
+	PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+}
+
+func TestUpdateReindexRoot(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestUpdateReindexRoot", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Set the reindex window to a low number to make this test run fast
+	originalReachabilityReindexWindow := reachabilityReindexWindow
+	reachabilityReindexWindow = 10
+	defer func() {
+		reachabilityReindexWindow = originalReachabilityReindexWindow
+	}()
+
+	// Add two blocks on top of the genesis block
+	chain1RootBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	chain2RootBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+
+	// Add chain of reachabilityReindexWindow - 1 blocks above chain1RootBlock and
+	// chain2RootBlock, respectively. This should not move the reindex root
+	chain1RootBlockTipHash := chain1RootBlock.BlockHash()
+	chain2RootBlockTipHash := chain2RootBlock.BlockHash()
+	genesisTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(dag.genesis.hash)
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	for i := uint64(0); i < reachabilityReindexWindow-1; i++ {
+		chain1Block := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{chain1RootBlockTipHash}, nil)
+		chain1RootBlockTipHash = chain1Block.BlockHash()
+
+		chain2Block := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{chain2RootBlockTipHash}, nil)
+		chain2RootBlockTipHash = chain2Block.BlockHash()
+
+		if dag.reachabilityTree.reindexRoot != genesisTreeNode {
+			t.Fatalf("reindex root unexpectedly moved")
+		}
+	}
+
+	// Add another block over chain1. This will move the reindex root to chain1RootBlock
+	PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{chain1RootBlockTipHash}, nil)
+
+	// Make sure that chain1RootBlock is now the reindex root
+	chain1RootTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(chain1RootBlock.BlockHash())
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	if dag.reachabilityTree.reindexRoot != chain1RootTreeNode {
+		t.Fatalf("chain1RootBlock is not the reindex root after reindex")
+	}
+
+	// Make sure that tight intervals have been applied to chain2. Since
+	// we added reachabilityReindexWindow-1 blocks to chain2, the size
+	// of the interval at its root should be equal to reachabilityReindexWindow
+	chain2RootTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(chain2RootBlock.BlockHash())
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	if chain2RootTreeNode.interval.size() != reachabilityReindexWindow {
+		t.Fatalf("got unexpected chain2RootNode interval. Want: %d, got: %d",
+			chain2RootTreeNode.interval.size(), reachabilityReindexWindow)
+	}
+
+	// Make sure that the rest of the interval has been allocated to
+	// chain1RootNode, minus slack from both sides
+	expectedChain1RootIntervalSize := genesisTreeNode.interval.size() - 1 -
+		chain2RootTreeNode.interval.size() - 2*reachabilityReindexSlack
+	if chain1RootTreeNode.interval.size() != expectedChain1RootIntervalSize {
+		t.Fatalf("got unexpected chain1RootNode interval. Want: %d, got: %d",
+			chain1RootTreeNode.interval.size(), expectedChain1RootIntervalSize)
+	}
+}
+
+func TestReindexIntervalsEarlierThanReindexRoot(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestReindexIntervalsEarlierThanReindexRoot", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Set the reindex window and slack to low numbers to make this test
+	// run fast
+	originalReachabilityReindexWindow := reachabilityReindexWindow
+	originalReachabilityReindexSlack := reachabilityReindexSlack
+	reachabilityReindexWindow = 10
+	reachabilityReindexSlack = 5
+	defer func() {
+		reachabilityReindexWindow = originalReachabilityReindexWindow
+		reachabilityReindexSlack = originalReachabilityReindexSlack
+	}()
+
+	// Add three children to the genesis: leftBlock, centerBlock, rightBlock
+	leftBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	centerBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	rightBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+
+	// Add a chain of reachabilityReindexWindow blocks above centerBlock.
+	// This will move the reindex root to centerBlock
+	centerTipHash := centerBlock.BlockHash()
+	for i := uint64(0); i < reachabilityReindexWindow; i++ {
+		block := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{centerTipHash}, nil)
+		centerTipHash = block.BlockHash()
+	}
+
+	// Make sure that centerBlock is now the reindex root
+	centerTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(centerBlock.BlockHash())
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	if dag.reachabilityTree.reindexRoot != centerTreeNode {
+		t.Fatalf("centerBlock is not the reindex root after reindex")
+	}
+
+	// Get the current interval for leftBlock. The reindex should have
+	// resulted in a tight interval there
+	leftTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(leftBlock.BlockHash())
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	if leftTreeNode.interval.size() != 1 {
+		t.Fatalf("leftBlock interval not tight after reindex")
+	}
+
+	// Get the current interval for rightBlock. The reindex should have
+	// resulted in a tight interval there
+	rightTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(rightBlock.BlockHash())
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	if rightTreeNode.interval.size() != 1 {
+		t.Fatalf("rightBlock interval not tight after reindex")
+	}
+
+	// Get the current interval for centerBlock. Its interval should be:
+	// genesisInterval - 1 - leftInterval - leftSlack - rightInterval - rightSlack
+	genesisTreeNode, err := dag.reachabilityTree.store.treeNodeByBlockHash(dag.genesis.hash)
+	if err != nil {
+		t.Fatalf("failed to get tree node: %s", err)
+	}
+	expectedCenterInterval := genesisTreeNode.interval.size() - 1 -
+		leftTreeNode.interval.size() - reachabilityReindexSlack -
+		rightTreeNode.interval.size() - reachabilityReindexSlack
+	if centerTreeNode.interval.size() != expectedCenterInterval {
+		t.Fatalf("unexpected centerBlock interval. Want: %d, got: %d",
+			expectedCenterInterval, centerTreeNode.interval.size())
+	}
+
+	// Add a chain of reachabilityReindexWindow - 1 blocks above leftBlock.
+	// Each addition will trigger a low-than-reindex-root reindex. We
+	// expect the centerInterval to shrink by 1 each time, but its child
+	// to remain unaffected
+	treeChildOfCenterBlock := centerTreeNode.children[0]
+	treeChildOfCenterBlockOriginalIntervalSize := treeChildOfCenterBlock.interval.size()
+	leftTipHash := leftBlock.BlockHash()
+	for i := uint64(0); i < reachabilityReindexWindow-1; i++ {
+		block := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{leftTipHash}, nil)
+		leftTipHash = block.BlockHash()
+
+		expectedCenterInterval--
+		if centerTreeNode.interval.size() != expectedCenterInterval {
+			t.Fatalf("unexpected centerBlock interval. Want: %d, got: %d",
+				expectedCenterInterval, centerTreeNode.interval.size())
+		}
+
+		if treeChildOfCenterBlock.interval.size() != treeChildOfCenterBlockOriginalIntervalSize {
+			t.Fatalf("the interval of centerBlock's child unexpectedly changed")
+		}
+	}
+
+	// Add a chain of reachabilityReindexWindow - 1 blocks above rightBlock.
+	// Each addition will trigger a low-than-reindex-root reindex. We
+	// expect the centerInterval to shrink by 1 each time, but its child
+	// to remain unaffected
+	rightTipHash := rightBlock.BlockHash()
+	for i := uint64(0); i < reachabilityReindexWindow-1; i++ {
+		block := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{rightTipHash}, nil)
+		rightTipHash = block.BlockHash()
+
+		expectedCenterInterval--
+		if centerTreeNode.interval.size() != expectedCenterInterval {
+			t.Fatalf("unexpected centerBlock interval. Want: %d, got: %d",
+				expectedCenterInterval, centerTreeNode.interval.size())
+		}
+
+		if treeChildOfCenterBlock.interval.size() != treeChildOfCenterBlockOriginalIntervalSize {
+			t.Fatalf("the interval of centerBlock's child unexpectedly changed")
+		}
+	}
+}

blockdag/reachabilitystore.go

@@ -3,6 +3,7 @@ package blockdag
 import (
 	"bytes"
 	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
 	"github.com/pkg/errors"
@@ -11,7 +12,7 @@ import (
 
 type reachabilityData struct {
 	treeNode          *reachabilityTreeNode
-	futureCoveringSet futureCoveringBlockSet
+	futureCoveringSet futureCoveringTreeNodeSet
 }
 
 type reachabilityStore struct {
@@ -40,11 +41,11 @@ func (store *reachabilityStore) setTreeNode(treeNode *reachabilityTreeNode) {
 	store.setBlockAsDirty(node.hash)
 }
 
-func (store *reachabilityStore) setFutureCoveringSet(node *blockNode, futureCoveringSet futureCoveringBlockSet) error {
+func (store *reachabilityStore) setFutureCoveringSet(node *blockNode, futureCoveringSet futureCoveringTreeNodeSet) error {
 	// load the reachability data from DB to store.loaded
 	_, exists := store.reachabilityDataByHash(node.hash)
 	if !exists {
-		return reachabilityNotFoundError(node)
+		return reachabilityNotFoundError(node.hash)
 	}
 
 	store.loaded[*node.hash].futureCoveringSet = futureCoveringSet
@@ -56,22 +57,26 @@ func (store *reachabilityStore) setBlockAsDirty(blockHash *daghash.Hash) {
 	store.dirty[*blockHash] = struct{}{}
 }
 
-func reachabilityNotFoundError(node *blockNode) error {
-	return errors.Errorf("Couldn't find reachability data for block %s", node.hash)
+func reachabilityNotFoundError(hash *daghash.Hash) error {
+	return errors.Errorf("couldn't find reachability data for block %s", hash)
 }
 
-func (store *reachabilityStore) treeNodeByBlockNode(node *blockNode) (*reachabilityTreeNode, error) {
-	reachabilityData, exists := store.reachabilityDataByHash(node.hash)
+func (store *reachabilityStore) treeNodeByBlockHash(hash *daghash.Hash) (*reachabilityTreeNode, error) {
+	reachabilityData, exists := store.reachabilityDataByHash(hash)
 	if !exists {
-		return nil, reachabilityNotFoundError(node)
+		return nil, reachabilityNotFoundError(hash)
 	}
 	return reachabilityData.treeNode, nil
 }
 
-func (store *reachabilityStore) futureCoveringSetByBlockNode(node *blockNode) (futureCoveringBlockSet, error) {
+func (store *reachabilityStore) treeNodeByBlockNode(node *blockNode) (*reachabilityTreeNode, error) {
+	return store.treeNodeByBlockHash(node.hash)
+}
+
+func (store *reachabilityStore) futureCoveringSetByBlockNode(node *blockNode) (futureCoveringTreeNodeSet, error) {
 	reachabilityData, exists := store.reachabilityDataByHash(node.hash)
 	if !exists {
-		return nil, reachabilityNotFoundError(node)
+		return nil, reachabilityNotFoundError(node.hash)
 	}
 	return reachabilityData.futureCoveringSet, nil
 }
@@ -82,7 +87,7 @@ func (store *reachabilityStore) reachabilityDataByHash(hash *daghash.Hash) (*rea
 }
 
 // flushToDB writes all dirty reachability data to the database.
-func (store *reachabilityStore) flushToDB(dbTx database.Tx) error {
+func (store *reachabilityStore) flushToDB(dbContext *dbaccess.TxContext) error {
 	if len(store.dirty) == 0 {
 		return nil
 	}
@@ -90,7 +95,7 @@ func (store *reachabilityStore) flushToDB(dbTx database.Tx) error {
 	for hash := range store.dirty {
 		hash := hash // Copy hash to a new variable to avoid passing the same pointer
 		reachabilityData := store.loaded[hash]
-		err := store.dbStoreReachabilityData(dbTx, &hash, reachabilityData)
+		err := store.storeReachabilityData(dbContext, &hash, reachabilityData)
 		if err != nil {
 			return err
 		}
@@ -102,22 +107,25 @@ func (store *reachabilityStore) clearDirtyEntries() {
 	store.dirty = make(map[daghash.Hash]struct{})
 }
 
-func (store *reachabilityStore) init(dbTx database.Tx) error {
-	bucket := dbTx.Metadata().Bucket(reachabilityDataBucketName)
-
+func (store *reachabilityStore) init(dbContext dbaccess.Context) error {
 	// TODO: (Stas) This is a quick and dirty hack.
 	// We iterate over the entire bucket twice:
 	// * First, populate the loaded set with all entries
 	// * Second, connect the parent/children pointers in each entry
 	//   with other nodes, which are now guaranteed to exist
-	cursor := bucket.Cursor()
+	cursor, err := dbaccess.ReachabilityDataCursor(dbContext)
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
 	for ok := cursor.First(); ok; ok = cursor.Next() {
 		err := store.initReachabilityData(cursor)
 		if err != nil {
 			return err
 		}
 	}
-	cursor = bucket.Cursor()
+
 	for ok := cursor.First(); ok; ok = cursor.Next() {
 		err := store.loadReachabilityDataFromCursor(cursor)
 		if err != nil {
@@ -128,7 +136,12 @@ func (store *reachabilityStore) init(dbTx database.Tx) error {
 }
 
 func (store *reachabilityStore) initReachabilityData(cursor database.Cursor) error {
-	hash, err := daghash.NewHash(cursor.Key())
+	key, err := cursor.Key()
+	if err != nil {
+		return err
+	}
+
+	hash, err := daghash.NewHash(key.Suffix())
 	if err != nil {
 		return err
 	}
@@ -141,7 +154,12 @@ func (store *reachabilityStore) initReachabilityData(cursor database.Cursor) err
 }
 
 func (store *reachabilityStore) loadReachabilityDataFromCursor(cursor database.Cursor) error {
-	hash, err := daghash.NewHash(cursor.Key())
+	key, err := cursor.Key()
+	if err != nil {
+		return err
+	}
+
+	hash, err := daghash.NewHash(key.Suffix())
 	if err != nil {
 		return err
 	}
@@ -151,26 +169,34 @@ func (store *reachabilityStore) loadReachabilityDataFromCursor(cursor database.C
 		return errors.Errorf("cannot find reachability data for block hash: %s", hash)
 	}
 
-	err = store.deserializeReachabilityData(cursor.Value(), reachabilityData)
+	serializedReachabilityData, err := cursor.Value()
+	if err != nil {
+		return err
+	}
+
+	err = store.deserializeReachabilityData(serializedReachabilityData, reachabilityData)
 	if err != nil {
 		return err
 	}
 
 	// Connect the treeNode with its blockNode
-	reachabilityData.treeNode.blockNode = store.dag.index.LookupNode(hash)
+	reachabilityData.treeNode.blockNode, ok = store.dag.index.LookupNode(hash)
+	if !ok {
+		return errors.Errorf("block %s does not exist in the DAG", hash)
+	}
 
 	return nil
 }
 
-// dbStoreReachabilityData stores the reachability data to the database.
+// storeReachabilityData stores the reachability data to the database.
 // This overwrites the current entry if there exists one.
-func (store *reachabilityStore) dbStoreReachabilityData(dbTx database.Tx, hash *daghash.Hash, reachabilityData *reachabilityData) error {
+func (store *reachabilityStore) storeReachabilityData(dbContext dbaccess.Context, hash *daghash.Hash, reachabilityData *reachabilityData) error {
 	serializedReachabilyData, err := store.serializeReachabilityData(reachabilityData)
 	if err != nil {
 		return err
 	}
 
-	return dbTx.Metadata().Bucket(reachabilityDataBucketName).Put(hash[:], serializedReachabilyData)
+	return dbaccess.StoreReachabilityData(dbContext, hash, serializedReachabilyData)
 }
 
 func (store *reachabilityStore) serializeReachabilityData(reachabilityData *reachabilityData) ([]byte, error) {
@@ -193,12 +219,6 @@ func (store *reachabilityStore) serializeTreeNode(w io.Writer, treeNode *reachab
 		return err
 	}
 
-	// Serialize the remaining interval
-	err = store.serializeReachabilityInterval(w, treeNode.remainingInterval)
-	if err != nil {
-		return err
-	}
-
 	// Serialize the parent
 	// If this is the genesis block, write the zero hash instead
 	parentHash := &daghash.ZeroHash
@@ -243,16 +263,16 @@ func (store *reachabilityStore) serializeReachabilityInterval(w io.Writer, inter
 	return nil
 }
 
-func (store *reachabilityStore) serializeFutureCoveringSet(w io.Writer, futureCoveringSet futureCoveringBlockSet) error {
+func (store *reachabilityStore) serializeFutureCoveringSet(w io.Writer, futureCoveringSet futureCoveringTreeNodeSet) error {
 	// Serialize the set size
 	err := wire.WriteVarInt(w, uint64(len(futureCoveringSet)))
 	if err != nil {
 		return err
 	}
 
-	// Serialize each block in the set
-	for _, block := range futureCoveringSet {
-		err = wire.WriteElement(w, block.blockNode.hash)
+	// Serialize each node in the set
+	for _, node := range futureCoveringSet {
+		err = wire.WriteElement(w, node.blockNode.hash)
 		if err != nil {
 			return err
 		}
@@ -289,13 +309,6 @@ func (store *reachabilityStore) deserializeTreeNode(r io.Reader, destination *re
 	}
 	destination.treeNode.interval = interval
 
-	// Deserialize the remaining interval
-	remainingInterval, err := store.deserializeReachabilityInterval(r)
-	if err != nil {
-		return err
-	}
-	destination.treeNode.remainingInterval = remainingInterval
-
 	// Deserialize the parent
 	// If this is the zero hash, this node is the genesis and as such doesn't have a parent
 	parentHash := &daghash.Hash{}
@@ -366,25 +379,18 @@ func (store *reachabilityStore) deserializeFutureCoveringSet(r io.Reader, destin
 	}
 
 	// Deserialize each block in the set
-	futureCoveringSet := make(futureCoveringBlockSet, setSize)
+	futureCoveringSet := make(futureCoveringTreeNodeSet, setSize)
 	for i := uint64(0); i < setSize; i++ {
 		blockHash := &daghash.Hash{}
 		err = wire.ReadElement(r, blockHash)
 		if err != nil {
 			return err
 		}
-		blockNode := store.dag.index.LookupNode(blockHash)
-		if blockNode == nil {
-			return errors.Errorf("blockNode not found for hash %s", blockHash)
-		}
 		blockReachabilityData, ok := store.reachabilityDataByHash(blockHash)
 		if !ok {
 			return errors.Errorf("block reachability data not found for hash: %s", blockHash)
 		}
-		futureCoveringSet[i] = &futureCoveringBlock{
-			blockNode: blockNode,
-			treeNode:  blockReachabilityData.treeNode,
-		}
+		futureCoveringSet[i] = blockReachabilityData.treeNode
 	}
 	destination.futureCoveringSet = futureCoveringSet
 

blockdag/scriptval.go

@@ -179,11 +179,6 @@ func newTxValidator(utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscr
 // ValidateTransactionScripts validates the scripts for the passed transaction
 // using multiple goroutines.
 func ValidateTransactionScripts(tx *util.Tx, utxoSet UTXOSet, flags txscript.ScriptFlags, sigCache *txscript.SigCache) error {
-	// Don't validate coinbase transaction scripts.
-	if tx.IsCoinBase() {
-		return nil
-	}
-
 	// Collect all of the transaction inputs and required information for
 	// validation.
 	txIns := tx.MsgTx().TxIn
@@ -213,10 +208,6 @@ func checkBlockScripts(block *blockNode, utxoSet UTXOSet, transactions []*util.T
 	}
 	txValItems := make([]*txValidateItem, 0, numInputs)
 	for _, tx := range transactions {
-		// Skip coinbase transactions.
-		if tx.IsCoinBase() {
-			continue
-		}
 		for txInIdx, txIn := range tx.MsgTx().TxIn {
 			txVI := &txValidateItem{
 				txInIndex: txInIdx,

blockdag/subnetworks.go

@@ -4,31 +4,20 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/pkg/errors"
 
 	"github.com/kaspanet/kaspad/util"
 
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 	"github.com/kaspanet/kaspad/wire"
 )
 
-// SubnetworkStore stores the subnetworks data
-type SubnetworkStore struct {
-	db database.DB
-}
-
-func newSubnetworkStore(db database.DB) *SubnetworkStore {
-	return &SubnetworkStore{
-		db: db,
-	}
-}
-
 // registerSubnetworks scans a list of transactions, singles out
 // subnetwork registry transactions, validates them, and registers a new
 // subnetwork based on it.
 // This function returns an error if one or more transactions are invalid
-func registerSubnetworks(dbTx database.Tx, txs []*util.Tx) error {
+func registerSubnetworks(dbContext dbaccess.Context, txs []*util.Tx) error {
 	subnetworkRegistryTxs := make([]*wire.MsgTx, 0)
 	for _, tx := range txs {
 		msgTx := tx.MsgTx()
@@ -50,13 +39,13 @@ func registerSubnetworks(dbTx database.Tx, txs []*util.Tx) error {
 		if err != nil {
 			return err
 		}
-		sNet, err := dbGetSubnetwork(dbTx, subnetworkID)
+		exists, err := dbaccess.HasSubnetwork(dbContext, subnetworkID)
 		if err != nil {
 			return err
 		}
-		if sNet == nil {
+		if !exists {
 			createdSubnetwork := newSubnetwork(registryTx)
-			err := dbRegisterSubnetwork(dbTx, subnetworkID, createdSubnetwork)
+			err := registerSubnetwork(dbContext, subnetworkID, createdSubnetwork)
 			if err != nil {
 				return errors.Errorf("failed registering subnetwork"+
 					"for tx '%s': %s", registryTx.TxHash(), err)
@@ -85,66 +74,39 @@ func TxToSubnetworkID(tx *wire.MsgTx) (*subnetworkid.SubnetworkID, error) {
 	return subnetworkid.New(util.Hash160(txHash[:]))
 }
 
-// subnetwork returns a registered subnetwork. If the subnetwork does not exist
-// this method returns an error.
-func (s *SubnetworkStore) subnetwork(subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
-	var sNet *subnetwork
-	var err error
-	dbErr := s.db.View(func(dbTx database.Tx) error {
-		sNet, err = dbGetSubnetwork(dbTx, subnetworkID)
-		return nil
-	})
-	if dbErr != nil {
-		return nil, errors.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, dbErr)
-	}
+// fetchSubnetwork returns a registered subnetwork.
+func fetchSubnetwork(subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
+	serializedSubnetwork, err := dbaccess.FetchSubnetworkData(dbaccess.NoTx(), subnetworkID)
 	if err != nil {
-		return nil, errors.Errorf("could not retrieve subnetwork '%d': %s", subnetworkID, err)
+		return nil, err
 	}
 
-	return sNet, nil
+	subnet, err := deserializeSubnetwork(serializedSubnetwork)
+	if err != nil {
+		return nil, err
+	}
+
+	return subnet, nil
 }
 
 // GasLimit returns the gas limit of a registered subnetwork. If the subnetwork does not
 // exist this method returns an error.
-func (s *SubnetworkStore) GasLimit(subnetworkID *subnetworkid.SubnetworkID) (uint64, error) {
-	sNet, err := s.subnetwork(subnetworkID)
+func GasLimit(subnetworkID *subnetworkid.SubnetworkID) (uint64, error) {
+	sNet, err := fetchSubnetwork(subnetworkID)
 	if err != nil {
 		return 0, err
 	}
-	if sNet == nil {
-		return 0, errors.Errorf("subnetwork '%s' not found", subnetworkID)
-	}
 
 	return sNet.gasLimit, nil
 }
 
-// dbRegisterSubnetwork stores mappings from ID of the subnetwork to the subnetwork data.
-func dbRegisterSubnetwork(dbTx database.Tx, subnetworkID *subnetworkid.SubnetworkID, network *subnetwork) error {
-	// Serialize the subnetwork
+func registerSubnetwork(dbContext dbaccess.Context, subnetworkID *subnetworkid.SubnetworkID, network *subnetwork) error {
 	serializedSubnetwork, err := serializeSubnetwork(network)
 	if err != nil {
 		return errors.Errorf("failed to serialize sub-netowrk '%s': %s", subnetworkID, err)
 	}
 
-	// Store the subnetwork
-	subnetworksBucket := dbTx.Metadata().Bucket(subnetworksBucketName)
-	err = subnetworksBucket.Put(subnetworkID[:], serializedSubnetwork)
-	if err != nil {
-		return errors.Errorf("failed to write sub-netowrk '%s': %s", subnetworkID, err)
-	}
-
-	return nil
-}
-
-// dbGetSubnetwork returns the subnetwork associated with subnetworkID or nil if the subnetwork was not found.
-func dbGetSubnetwork(dbTx database.Tx, subnetworkID *subnetworkid.SubnetworkID) (*subnetwork, error) {
-	bucket := dbTx.Metadata().Bucket(subnetworksBucketName)
-	serializedSubnetwork := bucket.Get(subnetworkID[:])
-	if serializedSubnetwork == nil {
-		return nil, nil
-	}
-
-	return deserializeSubnetwork(serializedSubnetwork)
+	return dbaccess.StoreSubnetwork(dbContext, subnetworkID, serializedSubnetwork)
 }
 
 type subnetwork struct {

blockdag/sync_rate.go

@@ -0,0 +1,57 @@
+package blockdag
+
+import "time"
+
+const syncRateWindowDuration = 15 * time.Minute
+
+// addBlockProcessingTimestamp adds the last block processing timestamp in order to measure the recent sync rate.
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) addBlockProcessingTimestamp() {
+	now := time.Now()
+	dag.recentBlockProcessingTimestamps = append(dag.recentBlockProcessingTimestamps, now)
+	dag.removeNonRecentTimestampsFromRecentBlockProcessingTimestamps()
+}
+
+// removeNonRecentTimestampsFromRecentBlockProcessingTimestamps removes timestamps older than syncRateWindowDuration
+// from dag.recentBlockProcessingTimestamps
+//
+// This function MUST be called with the DAG state lock held (for writes).
+func (dag *BlockDAG) removeNonRecentTimestampsFromRecentBlockProcessingTimestamps() {
+	dag.recentBlockProcessingTimestamps = dag.recentBlockProcessingTimestampsRelevantWindow()
+}
+
+func (dag *BlockDAG) recentBlockProcessingTimestampsRelevantWindow() []time.Time {
+	minTime := time.Now().Add(-syncRateWindowDuration)
+	windowStartIndex := len(dag.recentBlockProcessingTimestamps)
+	for i, processTime := range dag.recentBlockProcessingTimestamps {
+		if processTime.After(minTime) {
+			windowStartIndex = i
+			break
+		}
+	}
+	return dag.recentBlockProcessingTimestamps[windowStartIndex:]
+}
+
+// syncRate returns the rate of processed
+// blocks in the last syncRateWindowDuration
+// duration.
+func (dag *BlockDAG) syncRate() float64 {
+	dag.RLock()
+	defer dag.RUnlock()
+	return float64(len(dag.recentBlockProcessingTimestampsRelevantWindow())) / syncRateWindowDuration.Seconds()
+}
+
+// IsSyncRateBelowThreshold checks whether the sync rate
+// is below the expected threshold.
+func (dag *BlockDAG) IsSyncRateBelowThreshold(maxDeviation float64) bool {
+	if dag.uptime() < syncRateWindowDuration {
+		return false
+	}
+
+	return dag.syncRate() < 1/dag.dagParams.TargetTimePerBlock.Seconds()*maxDeviation
+}
+
+func (dag *BlockDAG) uptime() time.Duration {
+	return time.Now().Sub(dag.startTime)
+}

blockdag/test_utils.go

@@ -5,45 +5,27 @@ package blockdag
 import (
 	"compress/bzip2"
 	"encoding/binary"
+	"github.com/kaspanet/kaspad/database/ffldb/ldb"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
+	"testing"
 
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb" // blank import ffldb so that its init() function runs before tests
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
 )
 
-const (
-	// testDbType is the database backend type to use for the tests.
-	testDbType = "ffldb"
-
-	// blockDataNet is the expected network in the test block data.
-	blockDataNet = wire.Mainnet
-)
-
-// isSupportedDbType returns whether or not the passed database type is
-// currently supported.
-func isSupportedDbType(dbType string) bool {
-	supportedDrivers := database.SupportedDrivers()
-	for _, driver := range supportedDrivers {
-		if dbType == driver {
-			return true
-		}
-	}
-
-	return false
-}
-
 // FileExists returns whether or not the named file or directory exists.
 func FileExists(name string) bool {
 	if _, err := os.Stat(name); err != nil {
@@ -57,11 +39,10 @@ func FileExists(name string) bool {
 // DAGSetup is used to create a new db and DAG instance with the genesis
 // block already inserted. In addition to the new DAG instance, it returns
 // a teardown function the caller should invoke when done testing to clean up.
-func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, errors.Errorf("unsupported db type %s", testDbType)
-	}
-
+// The openDB parameter instructs DAGSetup whether or not to also open the
+// database. Setting it to false is useful in tests that handle database
+// opening/closing by themselves.
+func DAGSetup(dbName string, openDb bool, config Config) (*BlockDAG, func(), error) {
 	var teardown func()
 
 	// To make sure that the teardown function is not called before any goroutines finished to run -
@@ -76,13 +57,25 @@ func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
 		})
 	}
 
-	if config.DB == nil {
-		tmpDir := os.TempDir()
+	if openDb {
+		var err error
+		tmpDir, err := ioutil.TempDir("", "DAGSetup")
+		if err != nil {
+			return nil, nil, errors.Errorf("error creating temp dir: %s", err)
+		}
+
+		// We set ldb.Options here to return nil because normally
+		// the database is initialized with very large caches that
+		// can make opening/closing the database for every test
+		// quite heavy.
+		originalLDBOptions := ldb.Options
+		ldb.Options = func() *opt.Options {
+			return nil
+		}
 
 		dbPath := filepath.Join(tmpDir, dbName)
 		_ = os.RemoveAll(dbPath)
-		var err error
-		config.DB, err = database.Create(testDbType, dbPath, blockDataNet)
+		err = dbaccess.Open(dbPath)
 		if err != nil {
 			return nil, nil, errors.Errorf("error creating db: %s", err)
 		}
@@ -92,18 +85,18 @@ func DAGSetup(dbName string, config Config) (*BlockDAG, func(), error) {
 		teardown = func() {
 			spawnWaitGroup.Wait()
 			spawn = realSpawn
-			config.DB.Close()
+			dbaccess.Close()
+			ldb.Options = originalLDBOptions
 			os.RemoveAll(dbPath)
 		}
 	} else {
 		teardown = func() {
 			spawnWaitGroup.Wait()
 			spawn = realSpawn
-			config.DB.Close()
 		}
 	}
 
-	config.TimeSource = NewMedianTime()
+	config.TimeSource = NewTimeSource()
 	config.SigCache = txscript.NewSigCache(1000)
 
 	// Create the DAG instance.
@@ -165,15 +158,15 @@ func SetVirtualForTest(dag *BlockDAG, virtual VirtualForTest) VirtualForTest {
 func GetVirtualFromParentsForTest(dag *BlockDAG, parentHashes []*daghash.Hash) (VirtualForTest, error) {
 	parents := newBlockSet()
 	for _, hash := range parentHashes {
-		parent := dag.index.LookupNode(hash)
-		if parent == nil {
+		parent, ok := dag.index.LookupNode(hash)
+		if !ok {
 			return nil, errors.Errorf("GetVirtualFromParentsForTest: didn't found node for hash %s", hash)
 		}
 		parents.add(parent)
 	}
 	virtual := newVirtualBlock(dag, parents)
 
-	pastUTXO, _, err := dag.pastUTXO(&virtual.blockNode)
+	pastUTXO, _, _, err := dag.pastUTXO(&virtual.blockNode)
 	if err != nil {
 		return nil, err
 	}
@@ -299,6 +292,28 @@ func PrepareBlockForTest(dag *BlockDAG, parentHashes []*daghash.Hash, transactio
 	return block, nil
 }
 
+// PrepareAndProcessBlockForTest prepares a block that points to the given parent
+// hashes and process it.
+func PrepareAndProcessBlockForTest(t *testing.T, dag *BlockDAG, parentHashes []*daghash.Hash, transactions []*wire.MsgTx) *wire.MsgBlock {
+	daghash.Sort(parentHashes)
+	block, err := PrepareBlockForTest(dag, parentHashes, transactions)
+	if err != nil {
+		t.Fatalf("error in PrepareBlockForTest: %s", err)
+	}
+	utilBlock := util.NewBlock(block)
+	isOrphan, isDelayed, err := dag.ProcessBlock(utilBlock, BFNoPoWCheck)
+	if err != nil {
+		t.Fatalf("unexpected error in ProcessBlock: %s", err)
+	}
+	if isDelayed {
+		t.Fatalf("block is too far in the future")
+	}
+	if isOrphan {
+		t.Fatalf("block was unexpectedly orphan")
+	}
+	return block
+}
+
 // generateDeterministicExtraNonceForTest returns a unique deterministic extra nonce for coinbase data, in order to create unique coinbase transactions.
 func generateDeterministicExtraNonceForTest() uint64 {
 	extraNonceForTest++

blockdag/test_utils_test.go

@@ -1,14 +0,0 @@
-package blockdag
-
-import (
-	"testing"
-)
-
-func TestIsSupportedDbType(t *testing.T) {
-	if !isSupportedDbType("ffldb") {
-		t.Errorf("ffldb should be a supported DB driver")
-	}
-	if isSupportedDbType("madeUpDb") {
-		t.Errorf("madeUpDb should not be a supported DB driver")
-	}
-}

blockdag/thresholdstate.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 
 	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/pkg/errors"
 )
 
 // ThresholdState define the various threshold states used when voting on
@@ -177,9 +178,9 @@ func (dag *BlockDAG) thresholdState(prevNode *blockNode, checker thresholdCondit
 		var ok bool
 		state, ok = cache.Lookup(prevNode.hash)
 		if !ok {
-			return ThresholdFailed, AssertError(fmt.Sprintf(
+			return ThresholdFailed, errors.Errorf(
 				"thresholdState: cache lookup failed for %s",
-				prevNode.hash))
+				prevNode.hash)
 		}
 	}
 
@@ -297,7 +298,7 @@ func (dag *BlockDAG) IsDeploymentActive(deploymentID uint32) (bool, error) {
 // This function MUST be called with the DAG state lock held (for writes).
 func (dag *BlockDAG) deploymentState(prevNode *blockNode, deploymentID uint32) (ThresholdState, error) {
 	if deploymentID > uint32(len(dag.dagParams.Deployments)) {
-		return ThresholdFailed, DeploymentError(deploymentID)
+		return ThresholdFailed, errors.Errorf("deployment ID %d does not exist", deploymentID)
 	}
 
 	deployment := &dag.dagParams.Deployments[deploymentID]
@@ -335,8 +336,8 @@ func (dag *BlockDAG) initThresholdCaches() error {
 	}
 
 	// No warnings about unknown rules or versions until the DAG is
-	// current.
-	if dag.isCurrent() {
+	// synced.
+	if dag.isSynced() {
 		// Warn if a high enough percentage of the last blocks have
 		// unexpected versions.
 		bestNode := dag.selectedTip()

blockdag/timesource.go

@@ -0,0 +1,25 @@
+package blockdag
+
+import (
+	"time"
+)
+
+// TimeSource is the interface to access time.
+type TimeSource interface {
+	// Now returns the current time.
+	Now() time.Time
+}
+
+// timeSource provides an implementation of the TimeSource interface
+// that simply returns the current local time.
+type timeSource struct{}
+
+// Now returns the current local time, with one second precision.
+func (m *timeSource) Now() time.Time {
+	return time.Unix(time.Now().Unix(), 0)
+}
+
+// NewTimeSource returns a new instance of a TimeSource
+func NewTimeSource() TimeSource {
+	return &timeSource{}
+}

blockdag/utxo_ecmh.go

@@ -2,31 +2,26 @@ package blockdag
 
 import (
 	"bytes"
-	"github.com/golang/groupcache/lru"
-	"github.com/kaspanet/kaspad/ecc"
-	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/go-secp256k1"
 	"github.com/kaspanet/kaspad/wire"
 )
 
-const ecmhCacheSize = 4_000_000
-
-var (
-	utxoToECMHCache = lru.New(ecmhCacheSize)
-)
-
-func utxoMultiset(entry *UTXOEntry, outpoint *wire.Outpoint) (*ecc.Multiset, error) {
+func addUTXOToMultiset(ms *secp256k1.MultiSet, entry *UTXOEntry, outpoint *wire.Outpoint) (*secp256k1.MultiSet, error) {
 	w := &bytes.Buffer{}
 	err := serializeUTXO(w, entry, outpoint)
 	if err != nil {
 		return nil, err
 	}
-	serializedUTXO := w.Bytes()
-	utxoHash := daghash.DoubleHashH(serializedUTXO)
-
-	if cachedMSPoint, ok := utxoToECMHCache.Get(utxoHash); ok {
-		return cachedMSPoint.(*ecc.Multiset), nil
-	}
-	msPoint := ecc.NewMultiset(ecc.S256()).Add(serializedUTXO)
-	utxoToECMHCache.Add(utxoHash, msPoint)
-	return msPoint, nil
+	ms.Add(w.Bytes())
+	return ms, nil
+}
+
+func removeUTXOFromMultiset(ms *secp256k1.MultiSet, entry *UTXOEntry, outpoint *wire.Outpoint) (*secp256k1.MultiSet, error) {
+	w := &bytes.Buffer{}
+	err := serializeUTXO(w, entry, outpoint)
+	if err != nil {
+		return nil, err
+	}
+	ms.Remove(w.Bytes())
+	return ms, nil
 }

blockdag/utxodiffstore.go

@@ -2,14 +2,11 @@ package blockdag
 
 import (
 	"bytes"
-	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/dbaccess"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/util/locks"
-	"github.com/pkg/errors"
 )
 
-var multisetPointSize = 32
-
 type blockUTXODiffData struct {
 	diff      *UTXODiff
 	diffChild *blockNode
@@ -17,16 +14,16 @@ type blockUTXODiffData struct {
 
 type utxoDiffStore struct {
 	dag    *BlockDAG
-	dirty  map[daghash.Hash]struct{}
-	loaded map[daghash.Hash]*blockUTXODiffData
+	dirty  map[*blockNode]struct{}
+	loaded map[*blockNode]*blockUTXODiffData
 	mtx    *locks.PriorityMutex
 }
 
 func newUTXODiffStore(dag *BlockDAG) *utxoDiffStore {
 	return &utxoDiffStore{
 		dag:    dag,
-		dirty:  make(map[daghash.Hash]struct{}),
-		loaded: make(map[daghash.Hash]*blockUTXODiffData),
+		dirty:  make(map[*blockNode]struct{}),
+		loaded: make(map[*blockNode]*blockUTXODiffData),
 		mtx:    locks.NewPriorityMutex(),
 	}
 }
@@ -35,16 +32,15 @@ func (diffStore *utxoDiffStore) setBlockDiff(node *blockNode, diff *UTXODiff) er
 	diffStore.mtx.HighPriorityWriteLock()
 	defer diffStore.mtx.HighPriorityWriteUnlock()
 	// load the diff data from DB to diffStore.loaded
-	_, exists, err := diffStore.diffDataByHash(node.hash)
-	if err != nil {
+	_, err := diffStore.diffDataByBlockNode(node)
+	if dbaccess.IsNotFoundError(err) {
+		diffStore.loaded[node] = &blockUTXODiffData{}
+	} else if err != nil {
 		return err
 	}
-	if !exists {
-		diffStore.loaded[*node.hash] = &blockUTXODiffData{}
-	}
 
-	diffStore.loaded[*node.hash].diff = diff
-	diffStore.setBlockAsDirty(node.hash)
+	diffStore.loaded[node].diff = diff
+	diffStore.setBlockAsDirty(node)
 	return nil
 }
 
@@ -52,22 +48,19 @@ func (diffStore *utxoDiffStore) setBlockDiffChild(node *blockNode, diffChild *bl
 	diffStore.mtx.HighPriorityWriteLock()
 	defer diffStore.mtx.HighPriorityWriteUnlock()
 	// load the diff data from DB to diffStore.loaded
-	_, exists, err := diffStore.diffDataByHash(node.hash)
+	_, err := diffStore.diffDataByBlockNode(node)
 	if err != nil {
 		return err
 	}
-	if !exists {
-		return diffNotFoundError(node)
-	}
 
-	diffStore.loaded[*node.hash].diffChild = diffChild
-	diffStore.setBlockAsDirty(node.hash)
+	diffStore.loaded[node].diffChild = diffChild
+	diffStore.setBlockAsDirty(node)
 	return nil
 }
 
-func (diffStore *utxoDiffStore) removeBlocksDiffData(dbTx database.Tx, blockHashes []*daghash.Hash) error {
-	for _, hash := range blockHashes {
-		err := diffStore.removeBlockDiffData(dbTx, hash)
+func (diffStore *utxoDiffStore) removeBlocksDiffData(dbContext dbaccess.Context, nodes []*blockNode) error {
+	for _, node := range nodes {
+		err := diffStore.removeBlockDiffData(dbContext, node)
 		if err != nil {
 			return err
 		}
@@ -75,87 +68,64 @@ func (diffStore *utxoDiffStore) removeBlocksDiffData(dbTx database.Tx, blockHash
 	return nil
 }
 
-func (diffStore *utxoDiffStore) removeBlockDiffData(dbTx database.Tx, blockHash *daghash.Hash) error {
+func (diffStore *utxoDiffStore) removeBlockDiffData(dbContext dbaccess.Context, node *blockNode) error {
 	diffStore.mtx.LowPriorityWriteLock()
 	defer diffStore.mtx.LowPriorityWriteUnlock()
-	delete(diffStore.loaded, *blockHash)
-	err := dbRemoveDiffData(dbTx, blockHash)
+	delete(diffStore.loaded, node)
+	err := dbaccess.RemoveDiffData(dbContext, node.hash)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func (diffStore *utxoDiffStore) setBlockAsDirty(blockHash *daghash.Hash) {
-	diffStore.dirty[*blockHash] = struct{}{}
+func (diffStore *utxoDiffStore) setBlockAsDirty(node *blockNode) {
+	diffStore.dirty[node] = struct{}{}
 }
 
-func (diffStore *utxoDiffStore) diffDataByHash(hash *daghash.Hash) (*blockUTXODiffData, bool, error) {
-	if diffData, ok := diffStore.loaded[*hash]; ok {
-		return diffData, true, nil
+func (diffStore *utxoDiffStore) diffDataByBlockNode(node *blockNode) (*blockUTXODiffData, error) {
+	if diffData, ok := diffStore.loaded[node]; ok {
+		return diffData, nil
 	}
-	diffData, err := diffStore.diffDataFromDB(hash)
+	diffData, err := diffStore.diffDataFromDB(node.hash)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
-	exists := diffData != nil
-	if exists {
-		diffStore.loaded[*hash] = diffData
-	}
-	return diffData, exists, nil
-}
-
-func diffNotFoundError(node *blockNode) error {
-	return errors.Errorf("Couldn't find diff data for block %s", node.hash)
+	diffStore.loaded[node] = diffData
+	return diffData, nil
 }
 
 func (diffStore *utxoDiffStore) diffByNode(node *blockNode) (*UTXODiff, error) {
 	diffStore.mtx.HighPriorityReadLock()
 	defer diffStore.mtx.HighPriorityReadUnlock()
-	diffData, exists, err := diffStore.diffDataByHash(node.hash)
+	diffData, err := diffStore.diffDataByBlockNode(node)
 	if err != nil {
 		return nil, err
 	}
-	if !exists {
-		return nil, diffNotFoundError(node)
-	}
 	return diffData.diff, nil
 }
 
 func (diffStore *utxoDiffStore) diffChildByNode(node *blockNode) (*blockNode, error) {
 	diffStore.mtx.HighPriorityReadLock()
 	defer diffStore.mtx.HighPriorityReadUnlock()
-	diffData, exists, err := diffStore.diffDataByHash(node.hash)
+	diffData, err := diffStore.diffDataByBlockNode(node)
 	if err != nil {
 		return nil, err
 	}
-	if !exists {
-		return nil, diffNotFoundError(node)
-	}
 	return diffData.diffChild, nil
 }
 
 func (diffStore *utxoDiffStore) diffDataFromDB(hash *daghash.Hash) (*blockUTXODiffData, error) {
-	var diffData *blockUTXODiffData
-	err := diffStore.dag.db.View(func(dbTx database.Tx) error {
-		bucket := dbTx.Metadata().Bucket(utxoDiffsBucketName)
-		serializedBlockDiffData := bucket.Get(hash[:])
-		if serializedBlockDiffData != nil {
-			var err error
-			diffData, err = diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
-			return err
-		}
-		return nil
-	})
+	serializedBlockDiffData, err := dbaccess.FetchUTXODiffData(dbaccess.NoTx(), hash)
 	if err != nil {
 		return nil, err
 	}
-	return diffData, nil
+
+	return diffStore.deserializeBlockUTXODiffData(serializedBlockDiffData)
 }
 
-// flushToDB writes all dirty diff data to the database. If all writes
-// succeed, this clears the dirty set.
-func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
+// flushToDB writes all dirty diff data to the database.
+func (diffStore *utxoDiffStore) flushToDB(dbContext *dbaccess.TxContext) error {
 	diffStore.mtx.HighPriorityWriteLock()
 	defer diffStore.mtx.HighPriorityWriteUnlock()
 	if len(diffStore.dirty) == 0 {
@@ -165,11 +135,10 @@ func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
 	// Allocate a buffer here to avoid needless allocations/grows
 	// while writing each entry.
 	buffer := &bytes.Buffer{}
-	for hash := range diffStore.dirty {
-		hash := hash // Copy hash to a new variable to avoid passing the same pointer
+	for node := range diffStore.dirty {
 		buffer.Reset()
-		diffData := diffStore.loaded[hash]
-		err := dbStoreDiffData(dbTx, buffer, &hash, diffData)
+		diffData := diffStore.loaded[node]
+		err := storeDiffData(dbContext, buffer, node.hash, diffData)
 		if err != nil {
 			return err
 		}
@@ -178,31 +147,53 @@ func (diffStore *utxoDiffStore) flushToDB(dbTx database.Tx) error {
 }
 
 func (diffStore *utxoDiffStore) clearDirtyEntries() {
-	diffStore.dirty = make(map[daghash.Hash]struct{})
+	diffStore.dirty = make(map[*blockNode]struct{})
 }
 
-// dbStoreDiffData stores the UTXO diff data to the database.
+// maxBlueScoreDifferenceToKeepLoaded is the maximum difference
+// between the virtual's blueScore and a blockNode's blueScore
+// under which to keep diff data loaded in memory.
+var maxBlueScoreDifferenceToKeepLoaded uint64 = 100
+
+// clearOldEntries removes entries whose blue score is lower than
+// virtual.blueScore - maxBlueScoreDifferenceToKeepLoaded. Note
+// that tips are not removed either even if their blue score is
+// lower than the above.
+func (diffStore *utxoDiffStore) clearOldEntries() {
+	diffStore.mtx.HighPriorityWriteLock()
+	defer diffStore.mtx.HighPriorityWriteUnlock()
+
+	virtualBlueScore := diffStore.dag.VirtualBlueScore()
+	minBlueScore := virtualBlueScore - maxBlueScoreDifferenceToKeepLoaded
+	if maxBlueScoreDifferenceToKeepLoaded > virtualBlueScore {
+		minBlueScore = 0
+	}
+
+	tips := diffStore.dag.virtual.tips()
+
+	toRemove := make(map[*blockNode]struct{})
+	for node := range diffStore.loaded {
+		if node.blueScore < minBlueScore && !tips.contains(node) {
+			toRemove[node] = struct{}{}
+		}
+	}
+	for node := range toRemove {
+		delete(diffStore.loaded, node)
+	}
+}
+
+// storeDiffData stores the UTXO diff data to the database.
 // This overwrites the current entry if there exists one.
-func dbStoreDiffData(dbTx database.Tx, writeBuffer *bytes.Buffer, hash *daghash.Hash, diffData *blockUTXODiffData) error {
-	// To avoid a ton of allocs, use the given writeBuffer
+func storeDiffData(dbContext dbaccess.Context, w *bytes.Buffer, hash *daghash.Hash, diffData *blockUTXODiffData) error {
+	// To avoid a ton of allocs, use the io.Writer
 	// instead of allocating one. We expect the buffer to
-	// already be initalized and, in most cases, to already
+	// already be initialized and, in most cases, to already
 	// be large enough to accommodate the serialized data
 	// without growing.
-	err := serializeBlockUTXODiffData(writeBuffer, diffData)
+	err := serializeBlockUTXODiffData(w, diffData)
 	if err != nil {
 		return err
 	}
 
-	// Bucket.Put doesn't copy on its own, so we manually
-	// copy here. We do so because we expect the buffer
-	// to be reused once we're done with it.
-	serializedDiffData := make([]byte, writeBuffer.Len())
-	copy(serializedDiffData, writeBuffer.Bytes())
-
-	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Put(hash[:], serializedDiffData)
-}
-
-func dbRemoveDiffData(dbTx database.Tx, hash *daghash.Hash) error {
-	return dbTx.Metadata().Bucket(utxoDiffsBucketName).Delete(hash[:])
+	return dbaccess.StoreUTXODiffData(dbContext, hash, w.Bytes())
 }

blockdag/utxodiffstore_test.go

@@ -1,18 +1,18 @@
 package blockdag
 
 import (
-	"fmt"
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-	"github.com/kaspanet/kaspad/wire"
 	"reflect"
 	"testing"
+
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/kaspanet/kaspad/util/daghash"
+	"github.com/kaspanet/kaspad/wire"
 )
 
 func TestUTXODiffStore(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", Config{
+	dag, teardownFunc, err := DAGSetup("TestUTXODiffStore", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -31,9 +31,12 @@ func TestUTXODiffStore(t *testing.T) {
 	// Check that an error is returned when asking for non existing node
 	nonExistingNode := createNode()
 	_, err = dag.utxoDiffStore.diffByNode(nonExistingNode)
-	expectedErrString := fmt.Sprintf("Couldn't find diff data for block %s", nonExistingNode.hash)
-	if err == nil || err.Error() != expectedErrString {
-		t.Errorf("diffByNode: expected error %s but got %s", expectedErrString, err)
+	if !dbaccess.IsNotFoundError(err) {
+		if err != nil {
+			t.Errorf("diffByNode: %s", err)
+		} else {
+			t.Errorf("diffByNode: unexpectedly found diff data")
+		}
 	}
 
 	// Add node's diff data to the utxoDiffStore and check if it's checked correctly.
@@ -63,13 +66,20 @@ func TestUTXODiffStore(t *testing.T) {
 
 	// Flush changes to db, delete them from the dag.utxoDiffStore.loaded
 	// map, and check if the diff data is re-fetched from the database.
-	err = dag.db.Update(func(dbTx database.Tx) error {
-		return dag.utxoDiffStore.flushToDB(dbTx)
-	})
+	dbTx, err := dbaccess.NewTx()
+	if err != nil {
+		t.Fatalf("Failed to open database transaction: %s", err)
+	}
+	defer dbTx.RollbackUnlessClosed()
+	err = dag.utxoDiffStore.flushToDB(dbTx)
 	if err != nil {
 		t.Fatalf("Error flushing utxoDiffStore data to DB: %s", err)
 	}
-	delete(dag.utxoDiffStore.loaded, *node.hash)
+	err = dbTx.Commit()
+	if err != nil {
+		t.Fatalf("Failed to commit database transaction: %s", err)
+	}
+	delete(dag.utxoDiffStore.loaded, node)
 
 	if storeDiff, err := dag.utxoDiffStore.diffByNode(node); err != nil {
 		t.Fatalf("diffByNode: unexpected error: %s", err)
@@ -78,9 +88,80 @@ func TestUTXODiffStore(t *testing.T) {
 	}
 
 	// Check if getBlockDiff caches the result in dag.utxoDiffStore.loaded
-	if loadedDiffData, ok := dag.utxoDiffStore.loaded[*node.hash]; !ok {
+	if loadedDiffData, ok := dag.utxoDiffStore.loaded[node]; !ok {
 		t.Errorf("the diff data wasn't added to loaded map after requesting it")
 	} else if !reflect.DeepEqual(loadedDiffData.diff, diff) {
 		t.Errorf("Expected diff and loadedDiff to be equal")
 	}
 }
+
+func TestClearOldEntries(t *testing.T) {
+	// Create a new database and DAG instance to run tests against.
+	dag, teardownFunc, err := DAGSetup("TestClearOldEntries", true, Config{
+		DAGParams: &dagconfig.SimnetParams,
+	})
+	if err != nil {
+		t.Fatalf("TestClearOldEntries: Failed to setup DAG instance: %v", err)
+	}
+	defer teardownFunc()
+
+	// Set maxBlueScoreDifferenceToKeepLoaded to 10 to make this test fast to run
+	currentDifference := maxBlueScoreDifferenceToKeepLoaded
+	maxBlueScoreDifferenceToKeepLoaded = 10
+	defer func() { maxBlueScoreDifferenceToKeepLoaded = currentDifference }()
+
+	// Add 10 blocks
+	blockNodes := make([]*blockNode, 10)
+	for i := 0; i < 10; i++ {
+		processedBlock := PrepareAndProcessBlockForTest(t, dag, dag.TipHashes(), nil)
+
+		node, ok := dag.index.LookupNode(processedBlock.BlockHash())
+		if !ok {
+			t.Fatalf("TestClearOldEntries: missing blockNode for hash %s", processedBlock.BlockHash())
+		}
+		blockNodes[i] = node
+	}
+
+	// Make sure that all of them exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if !ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is not in the loaded set", node.hash)
+		}
+	}
+
+	// Add 10 more blocks on top of the others
+	for i := 0; i < 10; i++ {
+		PrepareAndProcessBlockForTest(t, dag, dag.TipHashes(), nil)
+	}
+
+	// Make sure that all the old nodes no longer exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is in the loaded set", node.hash)
+		}
+	}
+
+	// Add a block on top of the genesis to force the retrieval of all diffData
+	processedBlock := PrepareAndProcessBlockForTest(t, dag, []*daghash.Hash{dag.genesis.hash}, nil)
+	node, ok := dag.index.LookupNode(processedBlock.BlockHash())
+	if !ok {
+		t.Fatalf("TestClearOldEntries: missing blockNode for hash %s", processedBlock.BlockHash())
+	}
+
+	// Make sure that the child-of-genesis node is in the loaded set, since it
+	// is a tip.
+	_, ok = dag.utxoDiffStore.loaded[node]
+	if !ok {
+		t.Fatalf("TestClearOldEntries: diffData for node %s is not in the loaded set", node.hash)
+	}
+
+	// Make sure that all the old nodes still do not exist in the loaded set
+	for _, node := range blockNodes {
+		_, ok := dag.utxoDiffStore.loaded[node]
+		if ok {
+			t.Fatalf("TestClearOldEntries: diffData for node %s is in the loaded set", node.hash)
+		}
+	}
+}

blockdag/utxoio.go

@@ -2,15 +2,11 @@ package blockdag
 
 import (
 	"bytes"
-	"encoding/binary"
-	"fmt"
-	"github.com/pkg/errors"
-	"io"
-	"math/big"
-
-	"github.com/kaspanet/kaspad/ecc"
+	"github.com/kaspanet/kaspad/util/binaryserializer"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
+	"github.com/pkg/errors"
+	"io"
 )
 
 // serializeBlockUTXODiffData serializes diff data in the following format:
@@ -40,54 +36,31 @@ func serializeBlockUTXODiffData(w io.Writer, diffData *blockUTXODiffData) error
 	return nil
 }
 
-// utxoEntryHeaderCode returns the calculated header code to be used when
-// serializing the provided utxo entry.
-func utxoEntryHeaderCode(entry *UTXOEntry) uint64 {
-	// As described in the serialization format comments, the header code
-	// encodes the blue score shifted over one bit and the block reward flag
-	// in the lowest bit.
-	headerCode := uint64(entry.BlockBlueScore()) << 1
-	if entry.IsCoinbase() {
-		headerCode |= 0x01
-	}
-
-	return headerCode
-}
-
-func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataBytes []byte) (*blockUTXODiffData, error) {
+func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffData []byte) (*blockUTXODiffData, error) {
 	diffData := &blockUTXODiffData{}
-	serializedDiffData := bytes.NewBuffer(serializedDiffDataBytes)
+	r := bytes.NewBuffer(serializedDiffData)
 
 	var hasDiffChild bool
-	err := wire.ReadElement(serializedDiffData, &hasDiffChild)
+	err := wire.ReadElement(r, &hasDiffChild)
 	if err != nil {
 		return nil, err
 	}
 
 	if hasDiffChild {
 		hash := &daghash.Hash{}
-		err := wire.ReadElement(serializedDiffData, hash)
+		err := wire.ReadElement(r, hash)
 		if err != nil {
 			return nil, err
 		}
-		diffData.diffChild = diffStore.dag.index.LookupNode(hash)
+
+		var ok bool
+		diffData.diffChild, ok = diffStore.dag.index.LookupNode(hash)
+		if !ok {
+			return nil, errors.Errorf("block %s does not exist in the DAG", hash)
+		}
 	}
 
-	diffData.diff = &UTXODiff{
-		useMultiset: true,
-	}
-
-	diffData.diff.toAdd, err = deserializeDiffEntries(serializedDiffData)
-	if err != nil {
-		return nil, err
-	}
-
-	diffData.diff.toRemove, err = deserializeDiffEntries(serializedDiffData)
-	if err != nil {
-		return nil, err
-	}
-
-	diffData.diff.diffMultiset, err = deserializeMultiset(serializedDiffData)
+	diffData.diff, err = deserializeUTXODiff(r)
 	if err != nil {
 		return nil, err
 	}
@@ -95,38 +68,31 @@ func (diffStore *utxoDiffStore) deserializeBlockUTXODiffData(serializedDiffDataB
 	return diffData, nil
 }
 
-func deserializeDiffEntries(r io.Reader) (utxoCollection, error) {
+func deserializeUTXODiff(r io.Reader) (*UTXODiff, error) {
+	diff := &UTXODiff{}
+
+	var err error
+	diff.toAdd, err = deserializeUTXOCollection(r)
+	if err != nil {
+		return nil, err
+	}
+
+	diff.toRemove, err = deserializeUTXOCollection(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return diff, nil
+}
+
+func deserializeUTXOCollection(r io.Reader) (utxoCollection, error) {
 	count, err := wire.ReadVarInt(r)
 	if err != nil {
 		return nil, err
 	}
 	collection := utxoCollection{}
 	for i := uint64(0); i < count; i++ {
-		outpointSize, err := wire.ReadVarInt(r)
-		if err != nil {
-			return nil, err
-		}
-
-		serializedOutpoint := make([]byte, outpointSize)
-		err = binary.Read(r, byteOrder, serializedOutpoint)
-		if err != nil {
-			return nil, err
-		}
-		outpoint, err := deserializeOutpoint(serializedOutpoint)
-		if err != nil {
-			return nil, err
-		}
-
-		utxoEntrySize, err := wire.ReadVarInt(r)
-		if err != nil {
-			return nil, err
-		}
-		serializedEntry := make([]byte, utxoEntrySize)
-		err = binary.Read(r, byteOrder, serializedEntry)
-		if err != nil {
-			return nil, err
-		}
-		utxoEntry, err := deserializeUTXOEntry(serializedEntry)
+		utxoEntry, outpoint, err := deserializeUTXO(r)
 		if err != nil {
 			return nil, err
 		}
@@ -135,31 +101,22 @@ func deserializeDiffEntries(r io.Reader) (utxoCollection, error) {
 	return collection, nil
 }
 
-// deserializeMultiset deserializes an EMCH multiset.
-// See serializeMultiset for more details.
-func deserializeMultiset(r io.Reader) (*ecc.Multiset, error) {
-	xBytes := make([]byte, multisetPointSize)
-	yBytes := make([]byte, multisetPointSize)
-	err := binary.Read(r, byteOrder, xBytes)
+func deserializeUTXO(r io.Reader) (*UTXOEntry, *wire.Outpoint, error) {
+	outpoint, err := deserializeOutpoint(r)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	err = binary.Read(r, byteOrder, yBytes)
+
+	utxoEntry, err := deserializeUTXOEntry(r)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	var x, y big.Int
-	x.SetBytes(xBytes)
-	y.SetBytes(yBytes)
-	return ecc.NewMultisetFromPoint(ecc.S256(), &x, &y), nil
+	return utxoEntry, outpoint, nil
 }
 
 // serializeUTXODiff serializes UTXODiff by serializing
 // UTXODiff.toAdd, UTXODiff.toRemove and UTXODiff.Multiset one after the other.
 func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
-	if !diff.useMultiset {
-		return errors.New("Cannot serialize a UTXO diff without a multiset")
-	}
 	err := serializeUTXOCollection(w, diff.toAdd)
 	if err != nil {
 		return err
@@ -169,10 +126,7 @@ func serializeUTXODiff(w io.Writer, diff *UTXODiff) error {
 	if err != nil {
 		return err
 	}
-	err = serializeMultiset(w, diff.diffMultiset)
-	if err != nil {
-		return err
-	}
+
 	return nil
 }
 
@@ -193,120 +147,93 @@ func serializeUTXOCollection(w io.Writer, collection utxoCollection) error {
 	return nil
 }
 
-// serializeMultiset serializes an ECMH multiset. The serialization
-// is done by taking the (x,y) coordinnates of the multiset point and
-// padding each one of them with 32 byte (it'll be 32 byte in most
-// cases anyway except one of the coordinates is zero) and writing
-// them one after the other.
-func serializeMultiset(w io.Writer, ms *ecc.Multiset) error {
-	x, y := ms.Point()
-	xBytes := make([]byte, multisetPointSize)
-	copy(xBytes, x.Bytes())
-	yBytes := make([]byte, multisetPointSize)
-	copy(yBytes, y.Bytes())
-
-	err := binary.Write(w, byteOrder, xBytes)
-	if err != nil {
-		return err
-	}
-	err = binary.Write(w, byteOrder, yBytes)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 // serializeUTXO serializes a utxo entry-outpoint pair
 func serializeUTXO(w io.Writer, entry *UTXOEntry, outpoint *wire.Outpoint) error {
-	serializedOutpoint := *outpointKey(*outpoint)
-	err := wire.WriteVarInt(w, uint64(len(serializedOutpoint)))
+	err := serializeOutpoint(w, outpoint)
 	if err != nil {
 		return err
 	}
 
-	err = binary.Write(w, byteOrder, serializedOutpoint)
-	if err != nil {
-		return err
-	}
-
-	serializedUTXOEntry := serializeUTXOEntry(entry)
-	err = wire.WriteVarInt(w, uint64(len(serializedUTXOEntry)))
-	if err != nil {
-		return err
-	}
-	err = binary.Write(w, byteOrder, serializedUTXOEntry)
+	err = serializeUTXOEntry(w, entry)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-// serializeUTXOEntry returns the entry serialized to a format that is suitable
-// for long-term storage. The format is described in detail above.
-func serializeUTXOEntry(entry *UTXOEntry) []byte {
-	// Encode the header code.
-	headerCode := utxoEntryHeaderCode(entry)
+// p2pkhUTXOEntrySerializeSize is the serialized size for a P2PKH UTXO entry.
+// 8 bytes (header code) + 8 bytes (amount) + varint for script pub key length of 25 (for P2PKH) + 25 bytes for P2PKH script.
+var p2pkhUTXOEntrySerializeSize = 8 + 8 + wire.VarIntSerializeSize(25) + 25
 
-	// Calculate the size needed to serialize the entry.
-	size := serializeSizeVLQ(headerCode) +
-		compressedTxOutSize(uint64(entry.Amount()), entry.ScriptPubKey())
-
-	// Serialize the header code followed by the compressed unspent
-	// transaction output.
-	serialized := make([]byte, size)
-	offset := putVLQ(serialized, headerCode)
-	offset += putCompressedTxOut(serialized[offset:], uint64(entry.Amount()),
-		entry.ScriptPubKey())
-
-	return serialized
-}
-
-// deserializeOutpoint decodes an outpoint from the passed serialized byte
-// slice into a new wire.Outpoint using a format that is suitable for long-
-// term storage. this format is described in detail above.
-func deserializeOutpoint(serialized []byte) (*wire.Outpoint, error) {
-	if len(serialized) <= daghash.HashSize {
-		return nil, errDeserialize("unexpected end of data")
-	}
-
-	txID := daghash.TxID{}
-	txID.SetBytes(serialized[:daghash.HashSize])
-	index, _ := deserializeVLQ(serialized[daghash.HashSize:])
-	return wire.NewOutpoint(&txID, uint32(index)), nil
-}
-
-// deserializeUTXOEntry decodes a UTXO entry from the passed serialized byte
-// slice into a new UTXOEntry using a format that is suitable for long-term
-// storage. The format is described in detail above.
-func deserializeUTXOEntry(serialized []byte) (*UTXOEntry, error) {
-	// Deserialize the header code.
-	code, offset := deserializeVLQ(serialized)
-	if offset >= len(serialized) {
-		return nil, errDeserialize("unexpected end of data after header")
-	}
-
-	// Decode the header code.
-	//
-	// Bit 0 indicates whether the containing transaction is a coinbase.
-	// Bits 1-x encode blue score of the containing transaction.
-	isCoinbase := code&0x01 != 0
-	blockBlueScore := code >> 1
-
-	// Decode the compressed unspent transaction output.
-	amount, scriptPubKey, _, err := decodeCompressedTxOut(serialized[offset:])
+// serializeUTXOEntry encodes the entry to the given io.Writer and use compression if useCompression is true.
+// The compression format is described in detail above.
+func serializeUTXOEntry(w io.Writer, entry *UTXOEntry) error {
+	// Encode the blueScore.
+	err := binaryserializer.PutUint64(w, byteOrder, entry.blockBlueScore)
 	if err != nil {
-		return nil, errDeserialize(fmt.Sprintf("unable to decode "+
-			"UTXO: %s", err))
+		return err
+	}
+
+	// Encode the packedFlags.
+	err = binaryserializer.PutUint8(w, uint8(entry.packedFlags))
+	if err != nil {
+		return err
+	}
+
+	err = binaryserializer.PutUint64(w, byteOrder, entry.Amount())
+	if err != nil {
+		return err
+	}
+
+	err = wire.WriteVarInt(w, uint64(len(entry.ScriptPubKey())))
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(entry.ScriptPubKey())
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+// deserializeUTXOEntry decodes a UTXO entry from the passed reader
+// into a new UTXOEntry. If isCompressed is used it will decompress
+// the entry according to the format that is described in detail
+// above.
+func deserializeUTXOEntry(r io.Reader) (*UTXOEntry, error) {
+	// Deserialize the blueScore.
+	blockBlueScore, err := binaryserializer.Uint64(r, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decode the packedFlags.
+	packedFlags, err := binaryserializer.Uint8(r)
+	if err != nil {
+		return nil, err
 	}
 
 	entry := &UTXOEntry{
-		amount:         amount,
-		scriptPubKey:   scriptPubKey,
 		blockBlueScore: blockBlueScore,
-		packedFlags:    0,
+		packedFlags:    txoFlags(packedFlags),
 	}
-	if isCoinbase {
-		entry.packedFlags |= tfCoinbase
+
+	entry.amount, err = binaryserializer.Uint64(r, byteOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	scriptPubKeyLen, err := wire.ReadVarInt(r)
+	if err != nil {
+		return nil, err
+	}
+
+	entry.scriptPubKey = make([]byte, scriptPubKeyLen)
+	_, err = r.Read(entry.scriptPubKey)
+	if err != nil {
+		return nil, errors.WithStack(err)
 	}
 
 	return entry, nil

blockdag/utxoset.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/kaspanet/kaspad/ecc"
+	"github.com/kaspanet/go-secp256k1"
 	"github.com/kaspanet/kaspad/wire"
 )
 
@@ -153,29 +153,16 @@ func (uc utxoCollection) clone() utxoCollection {
 
 // UTXODiff represents a diff between two UTXO Sets.
 type UTXODiff struct {
-	toAdd        utxoCollection
-	toRemove     utxoCollection
-	diffMultiset *ecc.Multiset
-	useMultiset  bool
+	toAdd    utxoCollection
+	toRemove utxoCollection
 }
 
-// NewUTXODiffWithoutMultiset creates a new, empty utxoDiff
+// NewUTXODiff creates a new, empty utxoDiff
 // without a multiset.
-func NewUTXODiffWithoutMultiset() *UTXODiff {
-	return &UTXODiff{
-		toAdd:       utxoCollection{},
-		toRemove:    utxoCollection{},
-		useMultiset: false,
-	}
-}
-
-// NewUTXODiff creates a new, empty utxoDiff.
 func NewUTXODiff() *UTXODiff {
 	return &UTXODiff{
-		toAdd:        utxoCollection{},
-		toRemove:     utxoCollection{},
-		useMultiset:  true,
-		diffMultiset: ecc.NewMultiset(ecc.S256()),
+		toAdd:    utxoCollection{},
+		toRemove: utxoCollection{},
 	}
 }
 
@@ -209,9 +196,8 @@ func NewUTXODiff() *UTXODiff {
 //    diffFrom results in the UTXO being added to toAdd
 func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 	result := UTXODiff{
-		toAdd:       make(utxoCollection, len(d.toRemove)+len(other.toAdd)),
-		toRemove:    make(utxoCollection, len(d.toAdd)+len(other.toRemove)),
-		useMultiset: d.useMultiset,
+		toAdd:    make(utxoCollection, len(d.toRemove)+len(other.toAdd)),
+		toRemove: make(utxoCollection, len(d.toAdd)+len(other.toRemove)),
 	}
 
 	// Note that the following cases are not accounted for, as they are impossible
@@ -293,17 +279,12 @@ func (d *UTXODiff) diffFrom(other *UTXODiff) (*UTXODiff, error) {
 		}
 	}
 
-	if d.useMultiset {
-		// Create a new diffMultiset as the subtraction of the two diffs.
-		result.diffMultiset = other.diffMultiset.Subtract(d.diffMultiset)
-	}
-
 	return &result, nil
 }
 
-// WithDiffInPlace applies provided diff to this diff in-place, that would be the result if
+// withDiffInPlace applies provided diff to this diff in-place, that would be the result if
 // first d, and than diff were applied to the same base
-func (d *UTXODiff) WithDiffInPlace(diff *UTXODiff) error {
+func (d *UTXODiff) withDiffInPlace(diff *UTXODiff) error {
 	for outpoint, entryToRemove := range diff.toRemove {
 		if d.toAdd.containsWithBlueScore(outpoint, entryToRemove.blockBlueScore) {
 			// If already exists in toAdd with the same blueScore - remove from toAdd
@@ -312,8 +293,8 @@ func (d *UTXODiff) WithDiffInPlace(diff *UTXODiff) error {
 		}
 		if d.toRemove.contains(outpoint) {
 			// If already exists - this is an error
-			return ruleError(ErrWithDiff, fmt.Sprintf(
-				"WithDiffInPlace: outpoint %s both in d.toRemove and in diff.toRemove", outpoint))
+			return errors.Errorf(
+				"withDiffInPlace: outpoint %s both in d.toRemove and in diff.toRemove", outpoint)
 		}
 
 		// If not exists neither in toAdd nor in toRemove - add to toRemove
@@ -324,9 +305,9 @@ func (d *UTXODiff) WithDiffInPlace(diff *UTXODiff) error {
 		if d.toRemove.containsWithBlueScore(outpoint, entryToAdd.blockBlueScore) {
 			// If already exists in toRemove with the same blueScore - remove from toRemove
 			if d.toAdd.contains(outpoint) && !diff.toRemove.contains(outpoint) {
-				return ruleError(ErrWithDiff, fmt.Sprintf(
-					"WithDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd with no "+
-						"corresponding entry in diff.toRemove", outpoint))
+				return errors.Errorf(
+					"withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd with no "+
+						"corresponding entry in diff.toRemove", outpoint)
 			}
 			d.toRemove.remove(outpoint)
 			continue
@@ -335,130 +316,35 @@ func (d *UTXODiff) WithDiffInPlace(diff *UTXODiff) error {
 			(existingEntry.blockBlueScore == entryToAdd.blockBlueScore ||
 				!diff.toRemove.containsWithBlueScore(outpoint, existingEntry.blockBlueScore)) {
 			// If already exists - this is an error
-			return ruleError(ErrWithDiff, fmt.Sprintf(
-				"WithDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd", outpoint))
+			return errors.Errorf(
+				"withDiffInPlace: outpoint %s both in d.toAdd and in diff.toAdd", outpoint)
 		}
 
 		// If not exists neither in toAdd nor in toRemove, or exists in toRemove with different blueScore - add to toAdd
 		d.toAdd.add(outpoint, entryToAdd)
 	}
 
-	// Apply diff.diffMultiset to d.diffMultiset
-	if d.useMultiset {
-		d.diffMultiset = d.diffMultiset.Union(diff.diffMultiset)
-	}
-
 	return nil
 }
 
 // WithDiff applies provided diff to this diff, creating a new utxoDiff, that would be the result if
-// first d, and than diff were applied to the same base
-//
-// WithDiff follows a set of rules represented by the following 3 by 3 table:
-//
-//          |           | this      |           |
-// ---------+-----------+-----------+-----------+-----------
-//          |           | toAdd     | toRemove  | None
-// ---------+-----------+-----------+-----------+-----------
-// other    | toAdd     | X         | -         | toAdd
-// ---------+-----------+-----------+-----------+-----------
-//          | toRemove  | -         | X         | toRemove
-// ---------+-----------+-----------+-----------+-----------
-//          | None      | toAdd     | toRemove  | -
-//
-// Key:
-// -		Don't add anything to the result
-// X		Return an error
-// toAdd	Add the UTXO into the toAdd collection of the result
-// toRemove	Add the UTXO into the toRemove collection of the result
-//
-// Examples:
-// 1. This diff contains a UTXO in toAdd, and the other diff contains it in toRemove
-//    WithDiff results in nothing being added
-// 2. This diff contains a UTXO in toRemove, and the other diff does not contain it
-//    WithDiff results in the UTXO being added to toRemove
+// first d, and than diff were applied to some base
 func (d *UTXODiff) WithDiff(diff *UTXODiff) (*UTXODiff, error) {
-	result := UTXODiff{
-		toAdd:       make(utxoCollection, len(d.toAdd)+len(diff.toAdd)),
-		toRemove:    make(utxoCollection, len(d.toRemove)+len(diff.toRemove)),
-		useMultiset: d.useMultiset,
+	clone := d.clone()
+
+	err := clone.withDiffInPlace(diff)
+	if err != nil {
+		return nil, err
 	}
 
-	// All transactions in d.toAdd:
-	// If they are not in diff.toRemove - should be added in result.toAdd
-	// If they are in diff.toAdd - should throw an error
-	// Otherwise - should be ignored
-	for outpoint, utxoEntry := range d.toAdd {
-		if !diff.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toAdd.add(outpoint, utxoEntry)
-		}
-		if diffEntry, ok := diff.toAdd.get(outpoint); ok {
-			// An exception is made for entries with unequal blue scores
-			// as long as the appropriate entry exists in either d.toRemove
-			// or diff.toRemove.
-			// These are just "updates" to accepted blue score
-			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
-				diff.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-				continue
-			}
-			return nil, ruleError(ErrWithDiff, fmt.Sprintf("WithDiff: outpoint %s both in d.toAdd and in other.toAdd", outpoint))
-		}
-	}
-
-	// All transactions in d.toRemove:
-	// If they are not in diff.toAdd - should be added in result.toRemove
-	// If they are in diff.toRemove - should throw an error
-	// Otherwise - should be ignored
-	for outpoint, utxoEntry := range d.toRemove {
-		if !diff.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toRemove.add(outpoint, utxoEntry)
-		}
-		if diffEntry, ok := diff.toRemove.get(outpoint); ok {
-			// An exception is made for entries with unequal blue scores
-			// as long as the appropriate entry exists in either d.toAdd
-			// or diff.toAdd.
-			// These are just "updates" to accepted blue score
-			if diffEntry.blockBlueScore != utxoEntry.blockBlueScore &&
-				d.toAdd.containsWithBlueScore(outpoint, diffEntry.blockBlueScore) {
-				continue
-			}
-			return nil, ruleError(ErrWithDiff, "WithDiff: outpoint both in d.toRemove and in other.toRemove")
-		}
-	}
-
-	// All transactions in diff.toAdd:
-	// If they are not in d.toRemove - should be added in result.toAdd
-	for outpoint, utxoEntry := range diff.toAdd {
-		if !d.toRemove.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toAdd.add(outpoint, utxoEntry)
-		}
-	}
-
-	// All transactions in diff.toRemove:
-	// If they are not in d.toAdd - should be added in result.toRemove
-	for outpoint, utxoEntry := range diff.toRemove {
-		if !d.toAdd.containsWithBlueScore(outpoint, utxoEntry.blockBlueScore) {
-			result.toRemove.add(outpoint, utxoEntry)
-		}
-	}
-
-	// Apply diff.diffMultiset to d.diffMultiset
-	if d.useMultiset {
-		result.diffMultiset = d.diffMultiset.Union(diff.diffMultiset)
-	}
-
-	return &result, nil
+	return clone, nil
 }
 
 // clone returns a clone of this utxoDiff
 func (d *UTXODiff) clone() *UTXODiff {
 	clone := &UTXODiff{
-		toAdd:       d.toAdd.clone(),
-		toRemove:    d.toRemove.clone(),
-		useMultiset: d.useMultiset,
-	}
-	if d.useMultiset {
-		clone.diffMultiset = d.diffMultiset.Clone()
+		toAdd:    d.toAdd.clone(),
+		toRemove: d.toRemove.clone(),
 	}
 	return clone
 }
@@ -475,14 +361,6 @@ func (d *UTXODiff) AddEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
 	} else {
 		d.toAdd.add(outpoint, entry)
 	}
-
-	if d.useMultiset {
-		newMs, err := addUTXOToMultiset(d.diffMultiset, entry, &outpoint)
-		if err != nil {
-			return err
-		}
-		d.diffMultiset = newMs
-	}
 	return nil
 }
 
@@ -498,21 +376,10 @@ func (d *UTXODiff) RemoveEntry(outpoint wire.Outpoint, entry *UTXOEntry) error {
 	} else {
 		d.toRemove.add(outpoint, entry)
 	}
-
-	if d.useMultiset {
-		newMs, err := removeUTXOFromMultiset(d.diffMultiset, entry, &outpoint)
-		if err != nil {
-			return err
-		}
-		d.diffMultiset = newMs
-	}
 	return nil
 }
 
 func (d UTXODiff) String() string {
-	if d.useMultiset {
-		return fmt.Sprintf("toAdd: %s; toRemove: %s, Multiset-Hash: %s", d.toAdd, d.toRemove, d.diffMultiset.Hash())
-	}
 	return fmt.Sprintf("toAdd: %s; toRemove: %s", d.toAdd, d.toRemove)
 }
 
@@ -532,97 +399,27 @@ type UTXOSet interface {
 	fmt.Stringer
 	diffFrom(other UTXOSet) (*UTXODiff, error)
 	WithDiff(utxoDiff *UTXODiff) (UTXOSet, error)
-	diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error)
-	diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error)
 	AddTx(tx *wire.MsgTx, blockBlueScore uint64) (ok bool, err error)
 	clone() UTXOSet
 	Get(outpoint wire.Outpoint) (*UTXOEntry, bool)
-	Multiset() *ecc.Multiset
-	WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error)
-}
-
-// diffFromTx is a common implementation for diffFromTx, that works
-// for both diff-based and full UTXO sets
-// Returns a diff that is equivalent to provided transaction,
-// or an error if provided transaction is not valid in the context of this UTXOSet
-func diffFromTx(u UTXOSet, tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	diff := NewUTXODiff()
-	isCoinbase := tx.IsCoinBase()
-	if !isCoinbase {
-		for _, txIn := range tx.TxIn {
-			if entry, ok := u.Get(txIn.PreviousOutpoint); ok {
-				err := diff.RemoveEntry(txIn.PreviousOutpoint, entry)
-				if err != nil {
-					return nil, err
-				}
-			} else {
-				return nil, ruleError(ErrMissingTxOut, fmt.Sprintf(
-					"Transaction %s is invalid because spends outpoint %s that is not in utxo set",
-					tx.TxID(), txIn.PreviousOutpoint))
-			}
-		}
-	}
-	for i, txOut := range tx.TxOut {
-		entry := NewUTXOEntry(txOut, isCoinbase, acceptingBlueScore)
-		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
-		err := diff.AddEntry(outpoint, entry)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return diff, nil
-}
-
-// diffFromAcceptedTx is a common implementation for diffFromAcceptedTx, that works
-// for both diff-based and full UTXO sets.
-// Returns a diff that replaces an entry's blockBlueScore with the given acceptingBlueScore.
-// Returns an error if the provided transaction's entry is not valid in the context
-// of this UTXOSet.
-func diffFromAcceptedTx(u UTXOSet, tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	diff := NewUTXODiff()
-	isCoinbase := tx.IsCoinBase()
-	for i, txOut := range tx.TxOut {
-		// Fetch any unaccepted transaction
-		existingOutpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
-		existingEntry, ok := u.Get(existingOutpoint)
-		if !ok {
-			return nil, errors.Errorf("cannot accept outpoint %s because it doesn't exist in the given UTXO", existingOutpoint)
-		}
-
-		// Remove unaccepted entries
-		err := diff.RemoveEntry(existingOutpoint, existingEntry)
-		if err != nil {
-			return nil, err
-		}
-
-		// Add new entries with their accepting blue score
-		newEntry := NewUTXOEntry(txOut, isCoinbase, acceptingBlueScore)
-		err = diff.AddEntry(existingOutpoint, newEntry)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return diff, nil
 }
 
 // FullUTXOSet represents a full list of transaction outputs and their values
 type FullUTXOSet struct {
 	utxoCollection
-	UTXOMultiset *ecc.Multiset
 }
 
 // NewFullUTXOSet creates a new utxoSet with full list of transaction outputs and their values
 func NewFullUTXOSet() *FullUTXOSet {
 	return &FullUTXOSet{
 		utxoCollection: utxoCollection{},
-		UTXOMultiset:   ecc.NewMultiset(ecc.S256()),
 	}
 }
 
 // newFullUTXOSetFromUTXOCollection converts a utxoCollection to a FullUTXOSet
 func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet, error) {
 	var err error
-	multiset := ecc.NewMultiset(ecc.S256())
+	multiset := secp256k1.NewMultiset()
 	for outpoint, utxoEntry := range collection {
 		multiset, err = addUTXOToMultiset(multiset, utxoEntry, &outpoint)
 		if err != nil {
@@ -631,7 +428,6 @@ func newFullUTXOSetFromUTXOCollection(collection utxoCollection) (*FullUTXOSet,
 	}
 	return &FullUTXOSet{
 		utxoCollection: collection,
-		UTXOMultiset:   multiset,
 	}, nil
 }
 
@@ -661,40 +457,24 @@ func (fus *FullUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
 //
 // This function MUST be called with the DAG lock held.
 func (fus *FullUTXOSet) AddTx(tx *wire.MsgTx, blueScore uint64) (isAccepted bool, err error) {
-	isCoinbase := tx.IsCoinBase()
-	if !isCoinbase {
-		if !fus.containsInputs(tx) {
-			return false, nil
-		}
-
-		for _, txIn := range tx.TxIn {
-			outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
-			err := fus.removeAndUpdateMultiset(outpoint)
-			if err != nil {
-				return false, err
-			}
-		}
+	if !fus.containsInputs(tx) {
+		return false, nil
 	}
 
+	for _, txIn := range tx.TxIn {
+		fus.remove(txIn.PreviousOutpoint)
+	}
+
+	isCoinbase := tx.IsCoinBase()
 	for i, txOut := range tx.TxOut {
 		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
 		entry := NewUTXOEntry(txOut, isCoinbase, blueScore)
-
-		err := fus.addAndUpdateMultiset(outpoint, entry)
-		if err != nil {
-			return false, err
-		}
+		fus.add(outpoint, entry)
 	}
 
 	return true, nil
 }
 
-// diffFromTx returns a diff that is equivalent to provided transaction,
-// or an error if provided transaction is not valid in the context of this UTXOSet
-func (fus *FullUTXOSet) diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromTx(fus, tx, acceptingBlueScore)
-}
-
 func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 	for _, txIn := range tx.TxIn {
 		outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
@@ -706,13 +486,9 @@ func (fus *FullUTXOSet) containsInputs(tx *wire.MsgTx) bool {
 	return true
 }
 
-func (fus *FullUTXOSet) diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromAcceptedTx(fus, tx, acceptingBlueScore)
-}
-
 // clone returns a clone of this utxoSet
 func (fus *FullUTXOSet) clone() UTXOSet {
-	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone(), UTXOMultiset: fus.UTXOMultiset.Clone()}
+	return &FullUTXOSet{utxoCollection: fus.utxoCollection.clone()}
 }
 
 // Get returns the UTXOEntry associated with the given Outpoint, and a boolean indicating if such entry was found
@@ -721,55 +497,6 @@ func (fus *FullUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
 	return utxoEntry, ok
 }
 
-// Multiset returns the ecmh-Multiset of this utxoSet
-func (fus *FullUTXOSet) Multiset() *ecc.Multiset {
-	return fus.UTXOMultiset
-}
-
-// addAndUpdateMultiset adds a UTXOEntry to this utxoSet and updates its multiset accordingly
-func (fus *FullUTXOSet) addAndUpdateMultiset(outpoint wire.Outpoint, entry *UTXOEntry) error {
-	fus.add(outpoint, entry)
-	newMs, err := addUTXOToMultiset(fus.UTXOMultiset, entry, &outpoint)
-	if err != nil {
-		return err
-	}
-	fus.UTXOMultiset = newMs
-	return nil
-}
-
-// removeAndUpdateMultiset removes a UTXOEntry from this utxoSet and updates its multiset accordingly
-func (fus *FullUTXOSet) removeAndUpdateMultiset(outpoint wire.Outpoint) error {
-	entry, ok := fus.Get(outpoint)
-	if !ok {
-		return errors.Errorf("Couldn't find outpoint %s", outpoint)
-	}
-	fus.remove(outpoint)
-	var err error
-	newMs, err := removeUTXOFromMultiset(fus.UTXOMultiset, entry, &outpoint)
-	if err != nil {
-		return err
-	}
-	fus.UTXOMultiset = newMs
-	return nil
-}
-
-// WithTransactions returns a new UTXO Set with the added transactions.
-//
-// This function MUST be called with the DAG lock held.
-func (fus *FullUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error) {
-	diffSet := NewDiffUTXOSet(fus, NewUTXODiff())
-	for _, tx := range transactions {
-		isAccepted, err := diffSet.AddTx(tx, blockBlueScore)
-		if err != nil {
-			return nil, err
-		}
-		if !ignoreDoubleSpends && !isAccepted {
-			return nil, errors.Errorf("Transaction %s is not valid with the current UTXO set", tx.TxID())
-		}
-	}
-	return UTXOSet(diffSet), nil
-}
-
 // DiffUTXOSet represents a utxoSet with a base fullUTXOSet and a UTXODiff
 type DiffUTXOSet struct {
 	base     *FullUTXOSet
@@ -814,12 +541,11 @@ func (dus *DiffUTXOSet) WithDiff(other *UTXODiff) (UTXOSet, error) {
 // If dus.UTXODiff.useMultiset is true, this function MUST be
 // called with the DAG lock held.
 func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockBlueScore uint64) (bool, error) {
-	isCoinbase := tx.IsCoinBase()
-	if !isCoinbase && !dus.containsInputs(tx) {
+	if !dus.containsInputs(tx) {
 		return false, nil
 	}
 
-	err := dus.appendTx(tx, blockBlueScore, isCoinbase)
+	err := dus.appendTx(tx, blockBlueScore)
 	if err != nil {
 		return false, err
 	}
@@ -827,21 +553,19 @@ func (dus *DiffUTXOSet) AddTx(tx *wire.MsgTx, blockBlueScore uint64) (bool, erro
 	return true, nil
 }
 
-func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockBlueScore uint64, isCoinbase bool) error {
-	if !isCoinbase {
-		for _, txIn := range tx.TxIn {
-			outpoint := *wire.NewOutpoint(&txIn.PreviousOutpoint.TxID, txIn.PreviousOutpoint.Index)
-			entry, ok := dus.Get(outpoint)
-			if !ok {
-				return errors.Errorf("Couldn't find entry for outpoint %s", outpoint)
-			}
-			err := dus.UTXODiff.RemoveEntry(outpoint, entry)
-			if err != nil {
-				return err
-			}
+func (dus *DiffUTXOSet) appendTx(tx *wire.MsgTx, blockBlueScore uint64) error {
+	for _, txIn := range tx.TxIn {
+		entry, ok := dus.Get(txIn.PreviousOutpoint)
+		if !ok {
+			return errors.Errorf("couldn't find entry for outpoint %s", txIn.PreviousOutpoint)
+		}
+		err := dus.UTXODiff.RemoveEntry(txIn.PreviousOutpoint, entry)
+		if err != nil {
+			return err
 		}
 	}
 
+	isCoinbase := tx.IsCoinBase()
 	for i, txOut := range tx.TxOut {
 		outpoint := *wire.NewOutpoint(tx.TxID(), uint32(i))
 		entry := NewUTXOEntry(txOut, isCoinbase, blockBlueScore)
@@ -881,31 +605,12 @@ func (dus *DiffUTXOSet) meldToBase() error {
 	for outpoint, utxoEntry := range dus.UTXODiff.toAdd {
 		dus.base.add(outpoint, utxoEntry)
 	}
-
-	if dus.UTXODiff.useMultiset {
-		dus.base.UTXOMultiset = dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset)
-	}
-
-	if dus.UTXODiff.useMultiset {
-		dus.UTXODiff = NewUTXODiff()
-	} else {
-		dus.UTXODiff = NewUTXODiffWithoutMultiset()
-	}
+	dus.UTXODiff = NewUTXODiff()
 	return nil
 }
 
-// diffFromTx returns a diff that is equivalent to provided transaction,
-// or an error if provided transaction is not valid in the context of this UTXOSet
-func (dus *DiffUTXOSet) diffFromTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromTx(dus, tx, acceptingBlueScore)
-}
-
-func (dus *DiffUTXOSet) diffFromAcceptedTx(tx *wire.MsgTx, acceptingBlueScore uint64) (*UTXODiff, error) {
-	return diffFromAcceptedTx(dus, tx, acceptingBlueScore)
-}
-
 func (dus *DiffUTXOSet) String() string {
-	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s, Multiset-Hash:%s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove, dus.Multiset().Hash())
+	return fmt.Sprintf("{Base: %s, To Add: %s, To Remove: %s}", dus.base, dus.UTXODiff.toAdd, dus.UTXODiff.toRemove)
 }
 
 // clone returns a clone of this UTXO Set
@@ -913,6 +618,12 @@ func (dus *DiffUTXOSet) clone() UTXOSet {
 	return NewDiffUTXOSet(dus.base.clone().(*FullUTXOSet), dus.UTXODiff.clone())
 }
 
+// cloneWithoutBase returns a *DiffUTXOSet with same
+// base as this *DiffUTXOSet and a cloned diff.
+func (dus *DiffUTXOSet) cloneWithoutBase() UTXOSet {
+	return NewDiffUTXOSet(dus.base, dus.UTXODiff.clone())
+}
+
 // Get returns the UTXOEntry associated with provided outpoint in this UTXOSet.
 // Returns false in second output if this UTXOEntry was not found
 func (dus *DiffUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
@@ -930,42 +641,3 @@ func (dus *DiffUTXOSet) Get(outpoint wire.Outpoint) (*UTXOEntry, bool) {
 	txOut, ok := dus.UTXODiff.toAdd.get(outpoint)
 	return txOut, ok
 }
-
-// Multiset returns the ecmh-Multiset of this utxoSet
-func (dus *DiffUTXOSet) Multiset() *ecc.Multiset {
-	return dus.base.UTXOMultiset.Union(dus.UTXODiff.diffMultiset)
-}
-
-// WithTransactions returns a new UTXO Set with the added transactions.
-//
-// If dus.UTXODiff.useMultiset is true, this function MUST be
-// called with the DAG lock held.
-func (dus *DiffUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueScore uint64, ignoreDoubleSpends bool) (UTXOSet, error) {
-	diffSet := NewDiffUTXOSet(dus.base, dus.UTXODiff.clone())
-	for _, tx := range transactions {
-		isAccepted, err := diffSet.AddTx(tx, blockBlueScore)
-		if err != nil {
-			return nil, err
-		}
-		if !ignoreDoubleSpends && !isAccepted {
-			return nil, errors.Errorf("Transaction %s is not valid with the current UTXO set", tx.TxID())
-		}
-	}
-	return UTXOSet(diffSet), nil
-}
-
-func addUTXOToMultiset(ms *ecc.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*ecc.Multiset, error) {
-	utxoMS, err := utxoMultiset(entry, outpoint)
-	if err != nil {
-		return nil, err
-	}
-	return ms.Union(utxoMS), nil
-}
-
-func removeUTXOFromMultiset(ms *ecc.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*ecc.Multiset, error) {
-	utxoMS, err := utxoMultiset(entry, outpoint)
-	if err != nil {
-		return nil, err
-	}
-	return ms.Subtract(utxoMS), nil
-}

blockdag/utxoset_test.go

@@ -1,13 +1,11 @@
 package blockdag
 
 import (
-	"math"
 	"reflect"
 	"testing"
 
 	"github.com/kaspanet/kaspad/util/subnetworkid"
 
-	"github.com/kaspanet/kaspad/ecc"
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
 )
@@ -80,49 +78,40 @@ func TestUTXODiff(t *testing.T) {
 	utxoEntry0 := NewUTXOEntry(&wire.TxOut{ScriptPubKey: []byte{}, Value: 10}, true, 0)
 	utxoEntry1 := NewUTXOEntry(&wire.TxOut{ScriptPubKey: []byte{}, Value: 20}, false, 1)
 
-	for i := 0; i < 2; i++ {
-		withMultiset := i == 0
-		// Test utxoDiff creation
-		var diff *UTXODiff
-		if withMultiset {
-			diff = NewUTXODiff()
-		} else {
-			diff = NewUTXODiffWithoutMultiset()
-		}
-		if len(diff.toAdd) != 0 || len(diff.toRemove) != 0 {
-			t.Errorf("new diff is not empty")
-		}
+	// Test utxoDiff creation
 
-		err := diff.AddEntry(outpoint0, utxoEntry0)
-		if err != nil {
-			t.Fatalf("error adding entry to utxo diff: %s", err)
-		}
+	diff := NewUTXODiff()
 
-		err = diff.RemoveEntry(outpoint1, utxoEntry1)
-		if err != nil {
-			t.Fatalf("error adding entry to utxo diff: %s", err)
-		}
+	if len(diff.toAdd) != 0 || len(diff.toRemove) != 0 {
+		t.Errorf("new diff is not empty")
+	}
 
-		// Test utxoDiff cloning
-		clonedDiff := diff.clone()
-		if clonedDiff == diff {
-			t.Errorf("cloned diff is reference-equal to the original")
-		}
-		if !reflect.DeepEqual(clonedDiff, diff) {
-			t.Errorf("cloned diff not equal to the original"+
-				"Original: \"%v\", cloned: \"%v\".", diff, clonedDiff)
-		}
+	err := diff.AddEntry(outpoint0, utxoEntry0)
+	if err != nil {
+		t.Fatalf("error adding entry to utxo diff: %s", err)
+	}
 
-		// Test utxoDiff string representation
-		expectedDiffString := "toAdd: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ]; toRemove: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20, blueScore: 1 ]"
-		if withMultiset {
-			expectedDiffString = "toAdd: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ]; toRemove: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20, blueScore: 1 ], Multiset-Hash: 7cb61e48005b0c817211d04589d719bff87d86a6a6ce2454515f57265382ded7"
-		}
-		diffString := clonedDiff.String()
-		if diffString != expectedDiffString {
-			t.Errorf("unexpected diff string. "+
-				"Expected: \"%s\", got: \"%s\".", expectedDiffString, diffString)
-		}
+	err = diff.RemoveEntry(outpoint1, utxoEntry1)
+	if err != nil {
+		t.Fatalf("error adding entry to utxo diff: %s", err)
+	}
+
+	// Test utxoDiff cloning
+	clonedDiff := diff.clone()
+	if clonedDiff == diff {
+		t.Errorf("cloned diff is reference-equal to the original")
+	}
+	if !reflect.DeepEqual(clonedDiff, diff) {
+		t.Errorf("cloned diff not equal to the original"+
+			"Original: \"%v\", cloned: \"%v\".", diff, clonedDiff)
+	}
+
+	// Test utxoDiff string representation
+	expectedDiffString := "toAdd: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ]; toRemove: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20, blueScore: 1 ]"
+	diffString := clonedDiff.String()
+	if diffString != expectedDiffString {
+		t.Errorf("unexpected diff string. "+
+			"Expected: \"%s\", got: \"%s\".", expectedDiffString, diffString)
 	}
 }
 
@@ -137,7 +126,7 @@ func TestUTXODiffRules(t *testing.T) {
 	// For each of the following test cases, we will:
 	// this.diffFrom(other) and compare it to expectedDiffFromResult
 	// this.WithDiff(other) and compare it to expectedWithDiffResult
-	// this.WithDiffInPlace(other) and compare it to expectedWithDiffResult
+	// this.withDiffInPlace(other) and compare it to expectedWithDiffResult
 	//
 	// Note: an expected nil result means that we expect the respective operation to fail
 	// See the following spreadsheet for a summary of all test-cases:
@@ -542,157 +531,101 @@ func TestUTXODiffRules(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		this := addMultisetToDiff(t, test.this)
-		other := addMultisetToDiff(t, test.other)
-		expectedDiffFromResult := addMultisetToDiff(t, test.expectedDiffFromResult)
-		expectedWithDiffResult := addMultisetToDiff(t, test.expectedWithDiffResult)
-
-		// diffFrom from this to other
-		diffResult, err := this.diffFrom(other)
+		// diffFrom from test.this to test.other
+		diffResult, err := test.this.diffFrom(test.other)
 
 		// Test whether diffFrom returned an error
 		isDiffFromOk := err == nil
-		expectedIsDiffFromOk := expectedDiffFromResult != nil
+		expectedIsDiffFromOk := test.expectedDiffFromResult != nil
 		if isDiffFromOk != expectedIsDiffFromOk {
 			t.Errorf("unexpected diffFrom error in test \"%s\". "+
 				"Expected: \"%t\", got: \"%t\".", test.name, expectedIsDiffFromOk, isDiffFromOk)
 		}
 
 		// If not error, test the diffFrom result
-		if isDiffFromOk && !expectedDiffFromResult.equal(diffResult) {
+		if isDiffFromOk && !test.expectedDiffFromResult.equal(diffResult) {
 			t.Errorf("unexpected diffFrom result in test \"%s\". "+
-				"Expected: \"%v\", got: \"%v\".", test.name, expectedDiffFromResult, diffResult)
+				"Expected: \"%v\", got: \"%v\".", test.name, test.expectedDiffFromResult, diffResult)
 		}
 
-		// Make sure that WithDiff after diffFrom results in the original other
+		// Make sure that WithDiff after diffFrom results in the original test.other
 		if isDiffFromOk {
-			otherResult, err := this.WithDiff(diffResult)
+			otherResult, err := test.this.WithDiff(diffResult)
 			if err != nil {
 				t.Errorf("WithDiff unexpectedly failed in test \"%s\": %s", test.name, err)
 			}
-			if !other.equal(otherResult) {
+			if !test.other.equal(otherResult) {
 				t.Errorf("unexpected WithDiff result in test \"%s\". "+
-					"Expected: \"%v\", got: \"%v\".", test.name, other, otherResult)
+					"Expected: \"%v\", got: \"%v\".", test.name, test.other, otherResult)
 			}
 		}
 
-		// WithDiff from this to other
-		withDiffResult, err := this.WithDiff(other)
+		// WithDiff from test.this to test.other
+		withDiffResult, err := test.this.WithDiff(test.other)
 
 		// Test whether WithDiff returned an error
 		isWithDiffOk := err == nil
-		expectedIsWithDiffOk := expectedWithDiffResult != nil
+		expectedIsWithDiffOk := test.expectedWithDiffResult != nil
 		if isWithDiffOk != expectedIsWithDiffOk {
 			t.Errorf("unexpected WithDiff error in test \"%s\". "+
 				"Expected: \"%t\", got: \"%t\".", test.name, expectedIsWithDiffOk, isWithDiffOk)
 		}
 
 		// If not error, test the WithDiff result
-		if isWithDiffOk && !withDiffResult.equal(expectedWithDiffResult) {
+		if isWithDiffOk && !withDiffResult.equal(test.expectedWithDiffResult) {
 			t.Errorf("unexpected WithDiff result in test \"%s\". "+
-				"Expected: \"%v\", got: \"%v\".", test.name, expectedWithDiffResult, withDiffResult)
+				"Expected: \"%v\", got: \"%v\".", test.name, test.expectedWithDiffResult, withDiffResult)
 		}
 
-		// Repeat WithDiff check this time using WithDiffInPlace
-		thisClone := this.clone()
-		err = thisClone.WithDiffInPlace(other)
+		// Repeat WithDiff check test.this time using withDiffInPlace
+		thisClone := test.this.clone()
+		err = thisClone.withDiffInPlace(test.other)
 
-		// Test whether WithDiffInPlace returned an error
+		// Test whether withDiffInPlace returned an error
 		isWithDiffInPlaceOk := err == nil
-		expectedIsWithDiffInPlaceOk := expectedWithDiffResult != nil
+		expectedIsWithDiffInPlaceOk := test.expectedWithDiffResult != nil
 		if isWithDiffInPlaceOk != expectedIsWithDiffInPlaceOk {
-			t.Errorf("unexpected WithDiffInPlace error in test \"%s\". "+
+			t.Errorf("unexpected withDiffInPlace error in test \"%s\". "+
 				"Expected: \"%t\", got: \"%t\".", test.name, expectedIsWithDiffInPlaceOk, isWithDiffInPlaceOk)
 		}
 
-		// If not error, test the WithDiffInPlace result
-		if isWithDiffInPlaceOk && !thisClone.equal(expectedWithDiffResult) {
-			t.Errorf("unexpected WithDiffInPlace result in test \"%s\". "+
-				"Expected: \"%v\", got: \"%v\".", test.name, expectedWithDiffResult, thisClone)
+		// If not error, test the withDiffInPlace result
+		if isWithDiffInPlaceOk && !thisClone.equal(test.expectedWithDiffResult) {
+			t.Errorf("unexpected withDiffInPlace result in test \"%s\". "+
+				"Expected: \"%v\", got: \"%v\".", test.name, test.expectedWithDiffResult, thisClone)
 		}
 
-		// Make sure that diffFrom after WithDiff results in the original other
+		// Make sure that diffFrom after WithDiff results in the original test.other
 		if isWithDiffOk {
-			otherResult, err := this.diffFrom(withDiffResult)
+			otherResult, err := test.this.diffFrom(withDiffResult)
 			if err != nil {
 				t.Errorf("diffFrom unexpectedly failed in test \"%s\": %s", test.name, err)
 			}
-			if !other.equal(otherResult) {
+			if !test.other.equal(otherResult) {
 				t.Errorf("unexpected diffFrom result in test \"%s\". "+
-					"Expected: \"%v\", got: \"%v\".", test.name, other, otherResult)
+					"Expected: \"%v\", got: \"%v\".", test.name, test.other, otherResult)
 			}
 		}
 	}
 }
 
-func areMultisetsEqual(a *ecc.Multiset, b *ecc.Multiset) bool {
-	aX, aY := a.Point()
-	bX, bY := b.Point()
-	return aX.Cmp(bX) == 0 && aY.Cmp(bY) == 0
-}
-
 func (d *UTXODiff) equal(other *UTXODiff) bool {
 	if d == nil || other == nil {
 		return d == other
 	}
 
 	return reflect.DeepEqual(d.toAdd, other.toAdd) &&
-		reflect.DeepEqual(d.toRemove, other.toRemove) &&
-		areMultisetsEqual(d.diffMultiset, other.diffMultiset)
+		reflect.DeepEqual(d.toRemove, other.toRemove)
 }
 
 func (fus *FullUTXOSet) equal(other *FullUTXOSet) bool {
-	return reflect.DeepEqual(fus.utxoCollection, other.utxoCollection) &&
-		areMultisetsEqual(fus.UTXOMultiset, other.UTXOMultiset)
+	return reflect.DeepEqual(fus.utxoCollection, other.utxoCollection)
 }
 
 func (dus *DiffUTXOSet) equal(other *DiffUTXOSet) bool {
 	return dus.base.equal(other.base) && dus.UTXODiff.equal(other.UTXODiff)
 }
 
-func addMultisetToDiff(t *testing.T, diff *UTXODiff) *UTXODiff {
-	if diff == nil {
-		return nil
-	}
-	diffWithMs := NewUTXODiff()
-	for outpoint, entry := range diff.toAdd {
-		err := diffWithMs.AddEntry(outpoint, entry)
-		if err != nil {
-			t.Fatalf("Error with diffWithMs.AddEntry: %s", err)
-		}
-	}
-	for outpoint, entry := range diff.toRemove {
-		err := diffWithMs.RemoveEntry(outpoint, entry)
-		if err != nil {
-			t.Fatalf("Error with diffWithMs.removeEntry: %s", err)
-		}
-	}
-	return diffWithMs
-}
-
-func addMultisetToFullUTXOSet(t *testing.T, fus *FullUTXOSet) *FullUTXOSet {
-	if fus == nil {
-		return nil
-	}
-	fusWithMs := NewFullUTXOSet()
-	for outpoint, entry := range fus.utxoCollection {
-		err := fusWithMs.addAndUpdateMultiset(outpoint, entry)
-		if err != nil {
-			t.Fatalf("Error with diffWithMs.AddEntry: %s", err)
-		}
-	}
-	return fusWithMs
-}
-
-func addMultisetToDiffUTXOSet(t *testing.T, diffSet *DiffUTXOSet) *DiffUTXOSet {
-	if diffSet == nil {
-		return nil
-	}
-	diffWithMs := addMultisetToDiff(t, diffSet.UTXODiff)
-	baseWithMs := addMultisetToFullUTXOSet(t, diffSet.base)
-	return NewDiffUTXOSet(baseWithMs, diffWithMs)
-}
-
 // TestFullUTXOSet makes sure that fullUTXOSet is working as expected.
 func TestFullUTXOSet(t *testing.T) {
 	txID0, _ := daghash.NewTxIDFromStr("0000000000000000000000000000000000000000000000000000000000000000")
@@ -703,10 +636,10 @@ func TestFullUTXOSet(t *testing.T) {
 	txOut1 := &wire.TxOut{ScriptPubKey: []byte{}, Value: 20}
 	utxoEntry0 := NewUTXOEntry(txOut0, true, 0)
 	utxoEntry1 := NewUTXOEntry(txOut1, false, 1)
-	diff := addMultisetToDiff(t, &UTXODiff{
+	diff := &UTXODiff{
 		toAdd:    utxoCollection{outpoint0: utxoEntry0},
 		toRemove: utxoCollection{outpoint1: utxoEntry1},
-	})
+	}
 
 	// Test fullUTXOSet creation
 	emptySet := NewFullUTXOSet()
@@ -735,7 +668,7 @@ func TestFullUTXOSet(t *testing.T) {
 	} else if isAccepted {
 		t.Errorf("addTx unexpectedly succeeded")
 	}
-	emptySet = addMultisetToFullUTXOSet(t, &FullUTXOSet{utxoCollection: utxoCollection{outpoint0: utxoEntry0}})
+	emptySet = &FullUTXOSet{utxoCollection: utxoCollection{outpoint0: utxoEntry0}}
 	if isAccepted, err := emptySet.AddTx(transaction0, 0); err != nil {
 		t.Errorf("addTx unexpectedly failed. Error: %s", err)
 	} else if !isAccepted {
@@ -767,10 +700,10 @@ func TestDiffUTXOSet(t *testing.T) {
 	txOut1 := &wire.TxOut{ScriptPubKey: []byte{}, Value: 20}
 	utxoEntry0 := NewUTXOEntry(txOut0, true, 0)
 	utxoEntry1 := NewUTXOEntry(txOut1, false, 1)
-	diff := addMultisetToDiff(t, &UTXODiff{
+	diff := &UTXODiff{
 		toAdd:    utxoCollection{outpoint0: utxoEntry0},
 		toRemove: utxoCollection{outpoint1: utxoEntry1},
-	})
+	}
 
 	// Test diffUTXOSet creation
 	emptySet := NewDiffUTXOSet(NewFullUTXOSet(), NewUTXODiff())
@@ -828,7 +761,7 @@ func TestDiffUTXOSet(t *testing.T) {
 					toRemove: utxoCollection{},
 				},
 			},
-			expectedString:     "{Base: [  ], To Add: [  ], To Remove: [  ], Multiset-Hash:0000000000000000000000000000000000000000000000000000000000000000}",
+			expectedString:     "{Base: [  ], To Add: [  ], To Remove: [  ]}",
 			expectedCollection: utxoCollection{},
 		},
 		{
@@ -847,7 +780,7 @@ func TestDiffUTXOSet(t *testing.T) {
 					toRemove: utxoCollection{},
 				},
 			},
-			expectedString:     "{Base: [  ], To Add: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], To Remove: [  ], Multiset-Hash:da4768bd0359c3426268d6707c1fc17a68c45ef1ea734331b07568418234487f}",
+			expectedString:     "{Base: [  ], To Add: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], To Remove: [  ]}",
 			expectedCollection: utxoCollection{outpoint0: utxoEntry0},
 		},
 		{
@@ -860,7 +793,7 @@ func TestDiffUTXOSet(t *testing.T) {
 				},
 			},
 			expectedMeldSet:         nil,
-			expectedString:          "{Base: [  ], To Add: [  ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], Multiset-Hash:046242cb1bb1e6d3fd91d0f181e1b2d4a597ac57fa2584fc3c2eb0e0f46c9369}",
+			expectedString:          "{Base: [  ], To Add: [  ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ]}",
 			expectedCollection:      utxoCollection{},
 			expectedMeldToBaseError: "Couldn't remove outpoint 0000000000000000000000000000000000000000000000000000000000000000:0 because it doesn't exist in the DiffUTXOSet base",
 		},
@@ -885,7 +818,7 @@ func TestDiffUTXOSet(t *testing.T) {
 					toRemove: utxoCollection{},
 				},
 			},
-			expectedString: "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], To Add: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20, blueScore: 1 ], To Remove: [  ], Multiset-Hash:556cc61fd4d7e74d7807ca2298c5320375a6a20310a18920e54667220924baff}",
+			expectedString: "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], To Add: [ (1111111111111111111111111111111111111111111111111111111111111111, 0) => 20, blueScore: 1 ], To Remove: [  ]}",
 			expectedCollection: utxoCollection{
 				outpoint0: utxoEntry0,
 				outpoint1: utxoEntry1,
@@ -909,24 +842,21 @@ func TestDiffUTXOSet(t *testing.T) {
 					toRemove: utxoCollection{},
 				},
 			},
-			expectedString:     "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], To Add: [  ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], Multiset-Hash:0000000000000000000000000000000000000000000000000000000000000000}",
+			expectedString:     "{Base: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ], To Add: [  ], To Remove: [ (0000000000000000000000000000000000000000000000000000000000000000, 0) => 10, blueScore: 0 ]}",
 			expectedCollection: utxoCollection{},
 		},
 	}
 
 	for _, test := range tests {
-		diffSet := addMultisetToDiffUTXOSet(t, test.diffSet)
-		expectedMeldSet := addMultisetToDiffUTXOSet(t, test.expectedMeldSet)
-
 		// Test string representation
-		setString := diffSet.String()
+		setString := test.diffSet.String()
 		if setString != test.expectedString {
 			t.Errorf("unexpected string in test \"%s\". "+
 				"Expected: \"%s\", got: \"%s\".", test.name, test.expectedString, setString)
 		}
 
 		// Test meldToBase
-		meldSet := diffSet.clone().(*DiffUTXOSet)
+		meldSet := test.diffSet.clone().(*DiffUTXOSet)
 		err := meldSet.meldToBase()
 		errString := ""
 		if err != nil {
@@ -938,27 +868,27 @@ func TestDiffUTXOSet(t *testing.T) {
 		if err != nil {
 			continue
 		}
-		if !meldSet.equal(expectedMeldSet) {
+		if !meldSet.equal(test.expectedMeldSet) {
 			t.Errorf("unexpected melded set in test \"%s\". "+
-				"Expected: \"%v\", got: \"%v\".", test.name, expectedMeldSet, meldSet)
+				"Expected: \"%v\", got: \"%v\".", test.name, test.expectedMeldSet, meldSet)
 		}
 
 		// Test collection
-		setCollection, err := diffSet.collection()
+		setCollection, err := test.diffSet.collection()
 		if err != nil {
-			t.Errorf("Error getting diffSet collection: %s", err)
+			t.Errorf("Error getting test.diffSet collection: %s", err)
 		} else if !reflect.DeepEqual(setCollection, test.expectedCollection) {
 			t.Errorf("unexpected set collection in test \"%s\". "+
 				"Expected: \"%v\", got: \"%v\".", test.name, test.expectedCollection, setCollection)
 		}
 
 		// Test cloning
-		clonedSet := diffSet.clone().(*DiffUTXOSet)
-		if !reflect.DeepEqual(clonedSet, diffSet) {
+		clonedSet := test.diffSet.clone().(*DiffUTXOSet)
+		if !reflect.DeepEqual(clonedSet, test.diffSet) {
 			t.Errorf("unexpected set clone in test \"%s\". "+
-				"Expected: \"%v\", got: \"%v\".", test.name, diffSet, clonedSet)
+				"Expected: \"%v\", got: \"%v\".", test.name, test.diffSet, clonedSet)
 		}
-		if clonedSet == diffSet {
+		if clonedSet == test.diffSet {
 			t.Errorf("cloned set is reference-equal to the original")
 		}
 	}
@@ -1016,12 +946,9 @@ func TestUTXOSetDiffRules(t *testing.T) {
 
 // TestDiffUTXOSet_addTx makes sure that diffUTXOSet addTx works as expected
 func TestDiffUTXOSet_addTx(t *testing.T) {
-	// coinbaseTX is coinbase. As such, it has exactly one input with hash zero and MaxUInt32 index
-	txID0, _ := daghash.NewTxIDFromStr("0000000000000000000000000000000000000000000000000000000000000000")
-	txIn0 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutpoint: wire.Outpoint{TxID: *txID0, Index: math.MaxUint32}, Sequence: 0}
 	txOut0 := &wire.TxOut{ScriptPubKey: []byte{0}, Value: 10}
 	utxoEntry0 := NewUTXOEntry(txOut0, true, 0)
-	coinbaseTX := wire.NewSubnetworkMsgTx(1, []*wire.TxIn{txIn0}, []*wire.TxOut{txOut0}, subnetworkid.SubnetworkIDCoinbase, 0, nil)
+	coinbaseTX := wire.NewSubnetworkMsgTx(1, []*wire.TxIn{}, []*wire.TxOut{txOut0}, subnetworkid.SubnetworkIDCoinbase, 0, nil)
 
 	// transaction1 spends coinbaseTX
 	id1 := coinbaseTX.TxID()
@@ -1159,10 +1086,7 @@ func TestDiffUTXOSet_addTx(t *testing.T) {
 
 testLoop:
 	for _, test := range tests {
-		startSet := addMultisetToDiffUTXOSet(t, test.startSet)
-		expectedSet := addMultisetToDiffUTXOSet(t, test.expectedSet)
-
-		diffSet := startSet.clone()
+		diffSet := test.startSet.clone()
 
 		// Apply all transactions to diffSet, in order, with the initial block height startHeight
 		for i, transaction := range test.toAdd {
@@ -1174,89 +1098,14 @@ testLoop:
 			}
 		}
 
-		// Make sure that the result diffSet equals to the expectedSet
-		if !diffSet.(*DiffUTXOSet).equal(expectedSet) {
+		// Make sure that the result diffSet equals to test.expectedSet
+		if !diffSet.(*DiffUTXOSet).equal(test.expectedSet) {
 			t.Errorf("unexpected diffSet in test \"%s\". "+
-				"Expected: \"%v\", got: \"%v\".", test.name, expectedSet, diffSet)
+				"Expected: \"%v\", got: \"%v\".", test.name, test.expectedSet, diffSet)
 		}
 	}
 }
 
-func TestDiffFromTx(t *testing.T) {
-	fus := addMultisetToFullUTXOSet(t, &FullUTXOSet{
-		utxoCollection: utxoCollection{},
-	})
-
-	txID0, _ := daghash.NewTxIDFromStr("0000000000000000000000000000000000000000000000000000000000000000")
-	txIn0 := &wire.TxIn{SignatureScript: []byte{}, PreviousOutpoint: wire.Outpoint{TxID: *txID0, Index: math.MaxUint32}, Sequence: 0}
-	txOut0 := &wire.TxOut{ScriptPubKey: []byte{0}, Value: 10}
-	cbTx := wire.NewSubnetworkMsgTx(1, []*wire.TxIn{txIn0}, []*wire.TxOut{txOut0}, subnetworkid.SubnetworkIDCoinbase, 0, nil)
-	if isAccepted, err := fus.AddTx(cbTx, 1); err != nil {
-		t.Fatalf("AddTx unexpectedly failed. Error: %s", err)
-	} else if !isAccepted {
-		t.Fatalf("AddTx unexpectedly didn't add tx %s", cbTx.TxID())
-	}
-	acceptingBlueScore := uint64(2)
-	cbOutpoint := wire.Outpoint{TxID: *cbTx.TxID(), Index: 0}
-	txIns := []*wire.TxIn{{
-		PreviousOutpoint: cbOutpoint,
-		SignatureScript:  nil,
-		Sequence:         wire.MaxTxInSequenceNum,
-	}}
-	txOuts := []*wire.TxOut{{
-		ScriptPubKey: OpTrueScript,
-		Value:        uint64(1),
-	}}
-	tx := wire.NewNativeMsgTx(wire.TxVersion, txIns, txOuts)
-	diff, err := fus.diffFromTx(tx, acceptingBlueScore)
-	if err != nil {
-		t.Errorf("diffFromTx: %v", err)
-	}
-	if !reflect.DeepEqual(diff.toAdd, utxoCollection{
-		wire.Outpoint{TxID: *tx.TxID(), Index: 0}: NewUTXOEntry(tx.TxOut[0], false, 2),
-	}) {
-		t.Errorf("diff.toAdd doesn't have the expected values")
-	}
-
-	if !reflect.DeepEqual(diff.toRemove, utxoCollection{
-		wire.Outpoint{TxID: *cbTx.TxID(), Index: 0}: NewUTXOEntry(cbTx.TxOut[0], true, 1),
-	}) {
-		t.Errorf("diff.toRemove doesn't have the expected values")
-	}
-
-	//Test that we get an error if we don't have the outpoint inside the utxo set
-	invalidTxIns := []*wire.TxIn{{
-		PreviousOutpoint: wire.Outpoint{TxID: daghash.TxID{}, Index: 0},
-		SignatureScript:  nil,
-		Sequence:         wire.MaxTxInSequenceNum,
-	}}
-	invalidTxOuts := []*wire.TxOut{{
-		ScriptPubKey: OpTrueScript,
-		Value:        uint64(1),
-	}}
-	invalidTx := wire.NewNativeMsgTx(wire.TxVersion, invalidTxIns, invalidTxOuts)
-	_, err = fus.diffFromTx(invalidTx, acceptingBlueScore)
-	if err == nil {
-		t.Errorf("diffFromTx: expected an error but got <nil>")
-	}
-
-	//Test that we get an error if the outpoint is inside diffUTXOSet's toRemove
-	diff2 := addMultisetToDiff(t, &UTXODiff{
-		toAdd:    utxoCollection{},
-		toRemove: utxoCollection{},
-	})
-	dus := NewDiffUTXOSet(fus, diff2)
-	if isAccepted, err := dus.AddTx(tx, 2); err != nil {
-		t.Fatalf("AddTx unexpectedly failed. Error: %s", err)
-	} else if !isAccepted {
-		t.Fatalf("AddTx unexpectedly didn't add tx %s", tx.TxID())
-	}
-	_, err = dus.diffFromTx(tx, acceptingBlueScore)
-	if err == nil {
-		t.Errorf("diffFromTx: expected an error but got <nil>")
-	}
-}
-
 // collection returns a collection of all UTXOs in this set
 func (fus *FullUTXOSet) collection() utxoCollection {
 	return fus.utxoCollection.clone()
@@ -1321,7 +1170,6 @@ func TestUTXOSetAddEntry(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		expectedUTXODiff := addMultisetToDiff(t, test.expectedUTXODiff)
 		err := utxoDiff.AddEntry(*test.outpointToAdd, test.utxoEntryToAdd)
 		errString := ""
 		if err != nil {
@@ -1330,9 +1178,9 @@ func TestUTXOSetAddEntry(t *testing.T) {
 		if errString != test.expectedError {
 			t.Fatalf("utxoDiff.AddEntry: unexpected err in test \"%s\". Expected: %s but got: %s", test.name, test.expectedError, err)
 		}
-		if err == nil && !utxoDiff.equal(expectedUTXODiff) {
+		if err == nil && !utxoDiff.equal(test.expectedUTXODiff) {
 			t.Fatalf("utxoDiff.AddEntry: unexpected utxoDiff in test \"%s\". "+
-				"Expected: %v, got: %v", test.name, expectedUTXODiff, utxoDiff)
+				"Expected: %v, got: %v", test.name, test.expectedUTXODiff, utxoDiff)
 		}
 	}
 }

blockdag/validate.go

@@ -400,10 +400,11 @@ func CalcTxMass(tx *util.Tx, previousScriptPubKeys [][]byte) uint64 {
 //
 // The flags do not modify the behavior of this function directly, however they
 // are needed to pass along to checkProofOfWork.
-func (dag *BlockDAG) checkBlockHeaderSanity(header *wire.BlockHeader, flags BehaviorFlags) (delay time.Duration, err error) {
+func (dag *BlockDAG) checkBlockHeaderSanity(block *util.Block, flags BehaviorFlags) (delay time.Duration, err error) {
 	// Ensure the proof of work bits in the block header is in min/max range
 	// and the block hash is less than the target value described by the
 	// bits.
+	header := &block.MsgBlock().Header
 	err = dag.checkProofOfWork(header, flags)
 	if err != nil {
 		return 0, err
@@ -435,7 +436,7 @@ func (dag *BlockDAG) checkBlockHeaderSanity(header *wire.BlockHeader, flags Beha
 	// the duration of time that should be waited before the block becomes valid.
 	// This check needs to be last as it does not return an error but rather marks the
 	// header as delayed (and valid).
-	maxTimestamp := dag.AdjustedTime().Add(time.Second *
+	maxTimestamp := dag.Now().Add(time.Second *
 		time.Duration(int64(dag.TimestampDeviationTolerance)*dag.targetTimePerBlock))
 	if header.Timestamp.After(maxTimestamp) {
 		return header.Timestamp.Sub(maxTimestamp), nil
@@ -465,87 +466,182 @@ func checkBlockParentsOrder(header *wire.BlockHeader) error {
 // The flags do not modify the behavior of this function directly, however they
 // are needed to pass along to checkBlockHeaderSanity.
 func (dag *BlockDAG) checkBlockSanity(block *util.Block, flags BehaviorFlags) (time.Duration, error) {
-	msgBlock := block.MsgBlock()
-	header := &msgBlock.Header
-	delay, err := dag.checkBlockHeaderSanity(header, flags)
+	delay, err := dag.checkBlockHeaderSanity(block, flags)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkBlockContainsAtLeastOneTransaction(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkBlockContainsLessThanMaxBlockMassTransactions(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkFirstBlockTransactionIsCoinbase(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkBlockContainsOnlyOneCoinbase(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkBlockTransactionOrder(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkNoNonNativeTransactions(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkBlockTransactionSanity(block)
+	if err != nil {
+		return 0, err
+	}
+	err = dag.checkBlockHashMerkleRoot(block)
 	if err != nil {
 		return 0, err
 	}
 
-	// A block must have at least one transaction.
-	numTx := len(msgBlock.Transactions)
-	if numTx == 0 {
-		return 0, ruleError(ErrNoTransactions, "block does not contain "+
-			"any transactions")
+	// The following check will be fairly quick since the transaction IDs
+	// are already cached due to building the merkle tree above.
+	err = dag.checkBlockDuplicateTransactions(block)
+	if err != nil {
+		return 0, err
 	}
 
+	err = dag.checkBlockDoubleSpends(block)
+	if err != nil {
+		return 0, err
+	}
+	return delay, nil
+}
+
+func (dag *BlockDAG) checkBlockContainsAtLeastOneTransaction(block *util.Block) error {
+	transactions := block.Transactions()
+	numTx := len(transactions)
+	if numTx == 0 {
+		return ruleError(ErrNoTransactions, "block does not contain "+
+			"any transactions")
+	}
+	return nil
+}
+
+func (dag *BlockDAG) checkBlockContainsLessThanMaxBlockMassTransactions(block *util.Block) error {
 	// A block must not have more transactions than the max block mass or
 	// else it is certainly over the block mass limit.
+	transactions := block.Transactions()
+	numTx := len(transactions)
 	if numTx > wire.MaxMassPerBlock {
 		str := fmt.Sprintf("block contains too many transactions - "+
 			"got %d, max %d", numTx, wire.MaxMassPerBlock)
-		return 0, ruleError(ErrBlockMassTooHigh, str)
+		return ruleError(ErrBlockMassTooHigh, str)
 	}
+	return nil
+}
 
-	// The first transaction in a block must be a coinbase.
+func (dag *BlockDAG) checkFirstBlockTransactionIsCoinbase(block *util.Block) error {
 	transactions := block.Transactions()
 	if !transactions[util.CoinbaseTransactionIndex].IsCoinBase() {
-		return 0, ruleError(ErrFirstTxNotCoinbase, "first transaction in "+
+		return ruleError(ErrFirstTxNotCoinbase, "first transaction in "+
 			"block is not a coinbase")
 	}
+	return nil
+}
 
-	txOffset := util.CoinbaseTransactionIndex + 1
-
-	// A block must not have more than one coinbase. And transactions must be
-	// ordered by subnetwork
-	for i, tx := range transactions[txOffset:] {
+func (dag *BlockDAG) checkBlockContainsOnlyOneCoinbase(block *util.Block) error {
+	transactions := block.Transactions()
+	for i, tx := range transactions[util.CoinbaseTransactionIndex+1:] {
 		if tx.IsCoinBase() {
 			str := fmt.Sprintf("block contains second coinbase at "+
 				"index %d", i+2)
-			return 0, ruleError(ErrMultipleCoinbases, str)
-		}
-		if i != 0 && subnetworkid.Less(&tx.MsgTx().SubnetworkID, &transactions[i].MsgTx().SubnetworkID) {
-			return 0, ruleError(ErrTransactionsNotSorted, "transactions must be sorted by subnetwork")
+			return ruleError(ErrMultipleCoinbases, str)
 		}
 	}
+	return nil
+}
 
-	// Do some preliminary checks on each transaction to ensure they are
-	// sane before continuing.
+func (dag *BlockDAG) checkBlockTransactionOrder(block *util.Block) error {
+	transactions := block.Transactions()
+	for i, tx := range transactions[util.CoinbaseTransactionIndex+1:] {
+		if i != 0 && subnetworkid.Less(&tx.MsgTx().SubnetworkID, &transactions[i].MsgTx().SubnetworkID) {
+			return ruleError(ErrTransactionsNotSorted, "transactions must be sorted by subnetwork")
+		}
+	}
+	return nil
+}
+
+func (dag *BlockDAG) checkNoNonNativeTransactions(block *util.Block) error {
+	// Disallow non-native/coinbase subnetworks in networks that don't allow them
+	if !dag.dagParams.EnableNonNativeSubnetworks {
+		transactions := block.Transactions()
+		for _, tx := range transactions {
+			if !(tx.MsgTx().SubnetworkID.IsEqual(subnetworkid.SubnetworkIDNative) ||
+				tx.MsgTx().SubnetworkID.IsEqual(subnetworkid.SubnetworkIDCoinbase)) {
+				return ruleError(ErrInvalidSubnetwork, "non-native/coinbase subnetworks are not allowed")
+			}
+		}
+	}
+	return nil
+}
+
+func (dag *BlockDAG) checkBlockTransactionSanity(block *util.Block) error {
+	transactions := block.Transactions()
 	for _, tx := range transactions {
 		err := CheckTransactionSanity(tx, dag.subnetworkID)
 		if err != nil {
-			return 0, err
+			return err
 		}
 	}
+	return nil
+}
 
+func (dag *BlockDAG) checkBlockHashMerkleRoot(block *util.Block) error {
 	// Build merkle tree and ensure the calculated merkle root matches the
 	// entry in the block header. This also has the effect of caching all
 	// of the transaction hashes in the block to speed up future hash
 	// checks.
 	hashMerkleTree := BuildHashMerkleTreeStore(block.Transactions())
 	calculatedHashMerkleRoot := hashMerkleTree.Root()
-	if !header.HashMerkleRoot.IsEqual(calculatedHashMerkleRoot) {
+	if !block.MsgBlock().Header.HashMerkleRoot.IsEqual(calculatedHashMerkleRoot) {
 		str := fmt.Sprintf("block hash merkle root is invalid - block "+
 			"header indicates %s, but calculated value is %s",
-			header.HashMerkleRoot, calculatedHashMerkleRoot)
-		return 0, ruleError(ErrBadMerkleRoot, str)
+			block.MsgBlock().Header.HashMerkleRoot, calculatedHashMerkleRoot)
+		return ruleError(ErrBadMerkleRoot, str)
 	}
+	return nil
+}
 
-	// Check for duplicate transactions. This check will be fairly quick
-	// since the transaction IDs are already cached due to building the
-	// merkle tree above.
+func (dag *BlockDAG) checkBlockDuplicateTransactions(block *util.Block) error {
 	existingTxIDs := make(map[daghash.TxID]struct{})
+	transactions := block.Transactions()
 	for _, tx := range transactions {
 		id := tx.ID()
 		if _, exists := existingTxIDs[*id]; exists {
 			str := fmt.Sprintf("block contains duplicate "+
 				"transaction %s", id)
-			return 0, ruleError(ErrDuplicateTx, str)
+			return ruleError(ErrDuplicateTx, str)
 		}
 		existingTxIDs[*id] = struct{}{}
 	}
+	return nil
+}
 
-	return delay, nil
+func (dag *BlockDAG) checkBlockDoubleSpends(block *util.Block) error {
+	usedOutpoints := make(map[wire.Outpoint]*daghash.TxID)
+	transactions := block.Transactions()
+	for _, tx := range transactions {
+		for _, txIn := range tx.MsgTx().TxIn {
+			if spendingTxID, exists := usedOutpoints[txIn.PreviousOutpoint]; exists {
+				str := fmt.Sprintf("transaction %s spends "+
+					"outpoint %s that was already spent by "+
+					"transaction %s in this block", tx.ID(), txIn.PreviousOutpoint, spendingTxID)
+				return ruleError(ErrDoubleSpendInSameBlock, str)
+			}
+			usedOutpoints[txIn.PreviousOutpoint] = tx.ID()
+		}
+	}
+	return nil
 }
 
 // checkBlockHeaderContext performs several validation checks on the block header
@@ -602,7 +698,7 @@ func (dag *BlockDAG) validateParents(blockHeader *wire.BlockHeader, parents bloc
 	for parentA := range parents {
 		// isFinalized might be false-negative because node finality status is
 		// updated in a separate goroutine. This is why later the block is
-		// checked more thoroughly on the finality rules in dag.checkFinalityRules.
+		// checked more thoroughly on the finality rules in dag.checkFinalityViolation.
 		if parentA.isFinalized {
 			return ruleError(ErrFinality, fmt.Sprintf("block %s is a finalized "+
 				"parent of block %s", parentA.hash, blockHeader.BlockHash()))
@@ -613,7 +709,7 @@ func (dag *BlockDAG) validateParents(blockHeader *wire.BlockHeader, parents bloc
 				continue
 			}
 
-			isAncestorOf, err := dag.isAncestorOf(parentA, parentB)
+			isAncestorOf, err := dag.isInPast(parentA, parentB)
 			if err != nil {
 				return err
 			}
@@ -838,6 +934,11 @@ func (dag *BlockDAG) checkConnectToPastUTXO(block *blockNode, pastUTXO UTXOSet,
 			return nil, err
 		}
 
+		err = checkDoubleSpendsWithBlockPast(pastUTXO, transactions)
+		if err != nil {
+			return nil, err
+		}
+
 		if err := validateBlockMass(pastUTXO, transactions); err != nil {
 			return nil, err
 		}
@@ -913,7 +1014,7 @@ func (dag *BlockDAG) checkConnectToPastUTXO(block *blockNode, pastUTXO UTXOSet,
 
 		// Now that the inexpensive checks are done and have passed, verify the
 		// transactions are actually allowed to spend the coins by running the
-		// expensive ECDSA signature check scripts. Doing this last helps
+		// expensive SCHNORR signature check scripts. Doing this last helps
 		// prevent CPU exhaustion attacks.
 		err := checkBlockScripts(block, pastUTXO, transactions, scriptFlags, dag.sigCache)
 		if err != nil {

blockdag/validate_test.go

@@ -5,12 +5,13 @@
 package blockdag
 
 import (
-	"github.com/pkg/errors"
 	"math"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/kaspanet/kaspad/dagconfig"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/daghash"
@@ -69,7 +70,7 @@ func TestSequenceLocksActive(t *testing.T) {
 // ensure it fails.
 func TestCheckConnectBlockTemplate(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("checkconnectblocktemplate", Config{
+	dag, teardownFunc, err := DAGSetup("checkconnectblocktemplate", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -124,12 +125,6 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 			"block 4: %v", err)
 	}
 
-	blockNode3 := dag.index.LookupNode(blocks[3].Hash())
-	blockNode4 := dag.index.LookupNode(blocks[4].Hash())
-	if blockNode3.children.contains(blockNode4) {
-		t.Errorf("Block 4 wasn't successfully detached as a child from block3")
-	}
-
 	// Block 3a should connect even though it does not build on dag tips.
 	err = dag.CheckConnectBlockTemplateNoLock(blocks[5])
 	if err != nil {
@@ -161,7 +156,7 @@ func TestCheckConnectBlockTemplate(t *testing.T) {
 // as expected.
 func TestCheckBlockSanity(t *testing.T) {
 	// Create a new database and dag instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestCheckBlockSanity", Config{
+	dag, teardownFunc, err := DAGSetup("TestCheckBlockSanity", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -169,6 +164,7 @@ func TestCheckBlockSanity(t *testing.T) {
 		return
 	}
 	defer teardownFunc()
+	dag.timeSource = newFakeTimeSource(time.Now())
 
 	block := util.NewBlock(&Block100000)
 	if len(block.Transactions()) < 3 {
@@ -191,7 +187,8 @@ func TestCheckBlockSanity(t *testing.T) {
 	if !errors.As(err, &ruleErr) {
 		t.Errorf("CheckBlockSanity: wrong error returned, expect RuleError, got %T", err)
 	} else if ruleErr.ErrorCode != ErrTransactionsNotSorted {
-		t.Errorf("CheckBlockSanity: wrong error returned, expect ErrTransactionsNotSorted, got %v, err %s", ruleErr.ErrorCode, err)
+		t.Errorf("CheckBlockSanity: wrong error returned, expect ErrTransactionsNotSorted, got"+
+			" %v, err %s", ruleErr.ErrorCode, err)
 	}
 	if delay != 0 {
 		t.Errorf("CheckBlockSanity: unexpected return %s delay", delay)
@@ -492,8 +489,8 @@ func TestCheckBlockSanity(t *testing.T) {
 
 	blockInTheFuture := Block100000
 	expectedDelay := 10 * time.Second
-	now := time.Unix(time.Now().Unix(), 0)
-	blockInTheFuture.Header.Timestamp = now.Add(time.Duration(dag.TimestampDeviationTolerance)*time.Second + expectedDelay)
+	deviationTolerance := time.Duration(dag.TimestampDeviationTolerance*uint64(dag.targetTimePerBlock)) * time.Second
+	blockInTheFuture.Header.Timestamp = dag.Now().Add(deviationTolerance + expectedDelay)
 	delay, err = dag.checkBlockSanity(util.NewBlock(&blockInTheFuture), BFNoPoWCheck)
 	if err != nil {
 		t.Errorf("CheckBlockSanity: %v", err)
@@ -559,7 +556,7 @@ func TestPastMedianTime(t *testing.T) {
 
 func TestValidateParents(t *testing.T) {
 	// Create a new database and dag instance to run tests against.
-	dag, teardownFunc, err := DAGSetup("TestCheckBlockSanity", Config{
+	dag, teardownFunc, err := DAGSetup("TestCheckBlockSanity", true, Config{
 		DAGParams: &dagconfig.SimnetParams,
 	})
 	if err != nil {
@@ -568,9 +565,9 @@ func TestValidateParents(t *testing.T) {
 	}
 	defer teardownFunc()
 
-	a := prepareAndProcessBlock(t, dag, dag.dagParams.GenesisBlock)
-	b := prepareAndProcessBlock(t, dag, a)
-	c := prepareAndProcessBlock(t, dag, dag.dagParams.GenesisBlock)
+	a := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.dagParams.GenesisBlock)
+	b := prepareAndProcessBlockByParentMsgBlocks(t, dag, a)
+	c := prepareAndProcessBlockByParentMsgBlocks(t, dag, dag.dagParams.GenesisBlock)
 
 	aNode := nodeByMsgBlock(t, dag, a)
 	bNode := nodeByMsgBlock(t, dag, b)

blockdag/virtualblock_test.go

@@ -35,7 +35,7 @@ func TestVirtualBlock(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
 	params.K = 1
-	dag, teardownFunc, err := DAGSetup("TestVirtualBlock", Config{
+	dag, teardownFunc, err := DAGSetup("TestVirtualBlock", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -134,7 +134,7 @@ func TestSelectedPath(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
 	params.K = 1
-	dag, teardownFunc, err := DAGSetup("TestSelectedPath", Config{
+	dag, teardownFunc, err := DAGSetup("TestSelectedPath", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {
@@ -222,7 +222,7 @@ func TestChainUpdates(t *testing.T) {
 	// Create a new database and DAG instance to run tests against.
 	params := dagconfig.SimnetParams
 	params.K = 1
-	dag, teardownFunc, err := DAGSetup("TestChainUpdates", Config{
+	dag, teardownFunc, err := DAGSetup("TestChainUpdates", true, Config{
 		DAGParams: &params,
 	})
 	if err != nil {

cmd/addblock/addblock.go

@@ -5,12 +5,9 @@
 package main
 
 import (
-	"github.com/pkg/errors"
 	"os"
-	"path/filepath"
 	"runtime"
 
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/limits"
 	"github.com/kaspanet/kaspad/logs"
 	"github.com/kaspanet/kaspad/util/panics"
@@ -27,39 +24,6 @@ var (
 	spawn func(func())
 )
 
-// loadBlockDB opens the block database and returns a handle to it.
-func loadBlockDB() (database.DB, error) {
-	// The database name is based on the database type.
-	dbName := blockDBNamePrefix + "_" + cfg.DBType
-	dbPath := filepath.Join(cfg.DataDir, dbName)
-
-	log.Infof("Loading block database from '%s'", dbPath)
-	db, err := database.Open(cfg.DBType, dbPath, ActiveConfig().NetParams().Net)
-	if err != nil {
-		// Return the error if it's not because the database doesn't
-		// exist.
-		var dbErr database.Error
-		if ok := errors.As(err, &dbErr); !ok || dbErr.ErrorCode !=
-			database.ErrDbDoesNotExist {
-
-			return nil, err
-		}
-
-		// Create the db if it does not exist.
-		err = os.MkdirAll(cfg.DataDir, 0700)
-		if err != nil {
-			return nil, err
-		}
-		db, err = database.Create(cfg.DBType, dbPath, ActiveConfig().NetParams().Net)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	log.Info("Block database loaded")
-	return db, nil
-}
-
 // realMain is the real main function for the utility. It is necessary to work
 // around the fact that deferred functions do not run when os.Exit() is called.
 func realMain() error {
@@ -76,14 +40,6 @@ func realMain() error {
 	log = backendLogger.Logger("MAIN")
 	spawn = panics.GoroutineWrapperFunc(log)
 
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		log.Errorf("Failed to load database: %s", err)
-		return err
-	}
-	defer db.Close()
-
 	fi, err := os.Open(cfg.InFile)
 	if err != nil {
 		log.Errorf("Failed to open file %s: %s", cfg.InFile, err)
@@ -94,7 +50,7 @@ func realMain() error {
 	// Create a block importer for the database and input file and start it.
 	// The done channel returned from start will contain an error if
 	// anything went wrong.
-	importer, err := newBlockImporter(db, fi)
+	importer, err := newBlockImporter(fi)
 	if err != nil {
 		log.Errorf("Failed create block importer: %s", err)
 		return err

cmd/addblock/config.go

@@ -6,20 +6,15 @@ package main
 
 import (
 	"fmt"
+	flags "github.com/jessevdk/go-flags"
 	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
 	"os"
 	"path/filepath"
-	"strings"
-
-	flags "github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
 )
 
 const (
-	defaultDBType   = "ffldb"
 	defaultDataFile = "bootstrap.dat"
 	defaultProgress = 10
 )
@@ -27,7 +22,6 @@ const (
 var (
 	kaspadHomeDir  = util.AppDataDir("kaspad", false)
 	defaultDataDir = filepath.Join(kaspadHomeDir, "data")
-	knownDbTypes   = database.SupportedDrivers()
 	activeConfig   *ConfigFlags
 )
 
@@ -41,7 +35,6 @@ func ActiveConfig() *ConfigFlags {
 // See loadConfig for details on the configuration load process.
 type ConfigFlags struct {
 	DataDir         string `short:"b" long:"datadir" description:"Location of the kaspad data directory"`
-	DBType          string `long:"dbtype" description:"Database backend to use for the Block DAG"`
 	InFile          string `short:"i" long:"infile" description:"File containing the block(s)"`
 	Progress        int    `short:"p" long:"progress" description:"Show a progress message each time this number of seconds have passed -- Use 0 to disable progress announcements"`
 	AcceptanceIndex bool   `long:"acceptanceindex" description:"Maintain a full hash-based acceptance index which makes the getChainFromBlock RPC available"`
@@ -58,23 +51,11 @@ func fileExists(name string) bool {
 	return true
 }
 
-// validDbType returns whether or not dbType is a supported database type.
-func validDbType(dbType string) bool {
-	for _, knownType := range knownDbTypes {
-		if dbType == knownType {
-			return true
-		}
-	}
-
-	return false
-}
-
 // loadConfig initializes and parses the config using command line options.
 func loadConfig() (*ConfigFlags, []string, error) {
 	// Default config.
 	activeConfig = &ConfigFlags{
 		DataDir:  defaultDataDir,
-		DBType:   defaultDBType,
 		InFile:   defaultDataFile,
 		Progress: defaultProgress,
 	}
@@ -95,16 +76,6 @@ func loadConfig() (*ConfigFlags, []string, error) {
 		return nil, nil, err
 	}
 
-	// Validate database type.
-	if !validDbType(activeConfig.DBType) {
-		str := "%s: The specified database type [%s] is invalid -- " +
-			"supported types %s"
-		err := errors.Errorf(str, "loadConfig", activeConfig.DBType, strings.Join(knownDbTypes, ", "))
-		fmt.Fprintln(os.Stderr, err)
-		parser.WriteHelp(os.Stderr)
-		return nil, nil, err
-	}
-
 	// Append the network type to the data directory so it is "namespaced"
 	// per network. In addition to the block database, there are other
 	// pieces of data that are saved to disk such as address manager state.

cmd/addblock/import.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/kaspanet/kaspad/blockdag"
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/wire"
 )
@@ -28,7 +27,6 @@ type importResults struct {
 // blockImporter houses information about an ongoing import from a block data
 // file to the block database.
 type blockImporter struct {
-	db                database.DB
 	dag               *blockdag.BlockDAG
 	r                 io.ReadSeeker
 	processQueue      chan []byte
@@ -287,7 +285,7 @@ func (bi *blockImporter) Import() chan *importResults {
 
 // newBlockImporter returns a new importer for the provided file reader seeker
 // and database.
-func newBlockImporter(db database.DB, r io.ReadSeeker) (*blockImporter, error) {
+func newBlockImporter(r io.ReadSeeker) (*blockImporter, error) {
 	// Create the acceptance index if needed.
 	var indexes []indexers.Indexer
 	if cfg.AcceptanceIndex {
@@ -302,9 +300,8 @@ func newBlockImporter(db database.DB, r io.ReadSeeker) (*blockImporter, error) {
 	}
 
 	dag, err := blockdag.New(&blockdag.Config{
-		DB:           db,
 		DAGParams:    ActiveConfig().NetParams(),
-		TimeSource:   blockdag.NewMedianTime(),
+		TimeSource:   blockdag.NewTimeSource(),
 		IndexManager: indexManager,
 	})
 	if err != nil {
@@ -312,7 +309,6 @@ func newBlockImporter(db database.DB, r io.ReadSeeker) (*blockImporter, error) {
 	}
 
 	return &blockImporter{
-		db:           db,
 		r:            r,
 		processQueue: make(chan []byte, 2),
 		doneChan:     make(chan bool),

cmd/kaspaminer/config.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/config"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
+	"github.com/kaspanet/kaspad/config"
+
 	"github.com/kaspanet/kaspad/util"
 	"github.com/pkg/errors"
 
@@ -28,15 +30,18 @@ var (
 )
 
 type configFlags struct {
-	ShowVersion    bool   `short:"V" long:"version" description:"Display version information and exit"`
-	RPCUser        string `short:"u" long:"rpcuser" description:"RPC username"`
-	RPCPassword    string `short:"P" long:"rpcpass" default-mask:"-" description:"RPC password"`
-	RPCServer      string `short:"s" long:"rpcserver" description:"RPC server to connect to"`
-	RPCCert        string `short:"c" long:"rpccert" description:"RPC server certificate chain for validation"`
-	DisableTLS     bool   `long:"notls" description:"Disable TLS"`
-	Verbose        bool   `long:"verbose" short:"v" description:"Enable logging of RPC requests"`
-	NumberOfBlocks uint64 `short:"n" long:"numblocks" description:"Number of blocks to mine. If omitted, will mine until the process is interrupted."`
-	BlockDelay     uint64 `long:"block-delay" description:"Delay for block submission (in milliseconds). This is used only for testing purposes."`
+	ShowVersion       bool   `short:"V" long:"version" description:"Display version information and exit"`
+	RPCUser           string `short:"u" long:"rpcuser" description:"RPC username"`
+	RPCPassword       string `short:"P" long:"rpcpass" default-mask:"-" description:"RPC password"`
+	RPCServer         string `short:"s" long:"rpcserver" description:"RPC server to connect to"`
+	RPCCert           string `short:"c" long:"rpccert" description:"RPC server certificate chain for validation"`
+	DisableTLS        bool   `long:"notls" description:"Disable TLS"`
+	MiningAddr        string `long:"miningaddr" description:"Address to mine to"`
+	Verbose           bool   `long:"verbose" short:"v" description:"Enable logging of RPC requests"`
+	NumberOfBlocks    uint64 `short:"n" long:"numblocks" description:"Number of blocks to mine. If omitted, will mine until the process is interrupted."`
+	BlockDelay        uint64 `long:"block-delay" description:"Delay for block submission (in milliseconds). This is used only for testing purposes."`
+	MineWhenNotSynced bool   `long:"mine-when-not-synced" description:"Mine even if the node is not synced with the rest of the network."`
+	Profile           string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
 	config.NetworkFlags
 }
 
@@ -72,12 +77,19 @@ func parseConfig() (*configFlags, error) {
 	}
 
 	if cfg.RPCCert == "" && !cfg.DisableTLS {
-		return nil, errors.New("--notls has to be disabled if --cert is used")
+		return nil, errors.New("either --notls or --rpccert must be specified")
 	}
 	if cfg.RPCCert != "" && cfg.DisableTLS {
 		return nil, errors.New("--rpccert should be omitted if --notls is used")
 	}
 
+	if cfg.Profile != "" {
+		profilePort, err := strconv.Atoi(cfg.Profile)
+		if err != nil || profilePort < 1024 || profilePort > 65535 {
+			return nil, errors.New("The profile port must be between 1024 and 65535")
+		}
+	}
+
 	initLog(defaultLogFile, defaultErrLogFile)
 
 	return cfg, nil

cmd/kaspaminer/docker/Dockerfile

@@ -1,5 +1,5 @@
 # -- multistage docker build: stage #1: build stage
-FROM golang:1.13-alpine AS build
+FROM golang:1.14-alpine AS build
 
 RUN mkdir -p /go/src/github.com/kaspanet/kaspad
 
@@ -20,7 +20,7 @@ WORKDIR /go/src/github.com/kaspanet/kaspad/cmd/kaspaminer
 RUN GOFMT_RESULT=`go fmt ./...`; echo $GOFMT_RESULT; test -z "$GOFMT_RESULT"
 RUN go vet ./...
 RUN golint -set_exit_status ./...
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o kaspaminer .
+RUN GOOS=linux go build -a -installsuffix cgo -o kaspaminer .
 
 # --- multistage docker build: stage #2: runtime image
 FROM alpine

cmd/kaspaminer/main.go

@@ -2,13 +2,18 @@ package main
 
 import (
 	"fmt"
-	"github.com/kaspanet/kaspad/version"
+	"github.com/kaspanet/kaspad/util"
 	"os"
 
+	"github.com/kaspanet/kaspad/version"
+
 	"github.com/pkg/errors"
 
+	_ "net/http/pprof"
+
 	"github.com/kaspanet/kaspad/signal"
 	"github.com/kaspanet/kaspad/util/panics"
+	"github.com/kaspanet/kaspad/util/profiling"
 )
 
 func main() {
@@ -28,17 +33,27 @@ func main() {
 		enableRPCLogging()
 	}
 
+	// Enable http profiling server if requested.
+	if cfg.Profile != "" {
+		profiling.Start(cfg.Profile, log)
+	}
+
 	client, err := connectToServer(cfg)
 	if err != nil {
-		panic(errors.Wrap(err, "Error connecting to the RPC server"))
+		panic(errors.Wrap(err, "error connecting to the RPC server"))
 	}
 	defer client.Disconnect()
 
+	miningAddr, err := util.DecodeAddress(cfg.MiningAddr, cfg.ActiveNetParams.Prefix)
+	if err != nil {
+		panic(errors.Wrap(err, "error decoding mining address"))
+	}
+
 	doneChan := make(chan struct{})
 	spawn(func() {
-		err = mineLoop(client, cfg.NumberOfBlocks, cfg.BlockDelay)
+		err = mineLoop(client, cfg.NumberOfBlocks, cfg.BlockDelay, cfg.MineWhenNotSynced, miningAddr)
 		if err != nil {
-			panic(errors.Errorf("Error in mine loop: %s", err))
+			panic(errors.Wrap(err, "error in mine loop"))
 		}
 		doneChan <- struct{}{}
 	})

cmd/kaspaminer/mineloop.go

@@ -25,7 +25,9 @@ var hashesTried uint64
 
 const logHashRateInterval = 10 * time.Second
 
-func mineLoop(client *minerClient, numberOfBlocks uint64, blockDelay uint64) error {
+func mineLoop(client *minerClient, numberOfBlocks uint64, blockDelay uint64, mineWhenNotSynced bool,
+	miningAddr util.Address) error {
+
 	errChan := make(chan error)
 
 	templateStopChan := make(chan struct{})
@@ -35,7 +37,7 @@ func mineLoop(client *minerClient, numberOfBlocks uint64, blockDelay uint64) err
 		wg := sync.WaitGroup{}
 		for i := uint64(0); numberOfBlocks == 0 || i < numberOfBlocks; i++ {
 			foundBlock := make(chan *util.Block)
-			mineNextBlock(client, foundBlock, templateStopChan, errChan)
+			mineNextBlock(client, miningAddr, foundBlock, mineWhenNotSynced, templateStopChan, errChan)
 			block := <-foundBlock
 			templateStopChan <- struct{}{}
 			wg.Add(1)
@@ -80,13 +82,15 @@ func logHashRate() {
 	})
 }
 
-func mineNextBlock(client *minerClient, foundBlock chan *util.Block, templateStopChan chan struct{}, errChan chan error) {
+func mineNextBlock(client *minerClient, miningAddr util.Address, foundBlock chan *util.Block, mineWhenNotSynced bool,
+	templateStopChan chan struct{}, errChan chan error) {
+
 	newTemplateChan := make(chan *rpcmodel.GetBlockTemplateResult)
 	spawn(func() {
-		templatesLoop(client, newTemplateChan, errChan, templateStopChan)
+		templatesLoop(client, miningAddr, newTemplateChan, errChan, templateStopChan)
 	})
 	spawn(func() {
-		solveLoop(newTemplateChan, foundBlock, errChan)
+		solveLoop(newTemplateChan, foundBlock, mineWhenNotSynced, errChan)
 	})
 }
 
@@ -132,7 +136,7 @@ func parseBlock(template *rpcmodel.GetBlockTemplateResult) (*util.Block, error)
 		wire.NewBlockHeader(template.Version, parentHashes, &daghash.Hash{},
 			acceptedIDMerkleRoot, utxoCommitment, bits, 0))
 
-	for i, txResult := range append([]rpcmodel.GetBlockTemplateResultTx{*template.CoinbaseTxn}, template.Transactions...) {
+	for i, txResult := range template.Transactions {
 		reader := hex.NewDecoder(strings.NewReader(txResult.Data))
 		tx := &wire.MsgTx{}
 		if err := tx.KaspaDecode(reader, 0); err != nil {
@@ -167,7 +171,9 @@ func solveBlock(block *util.Block, stopChan chan struct{}, foundBlock chan *util
 
 }
 
-func templatesLoop(client *minerClient, newTemplateChan chan *rpcmodel.GetBlockTemplateResult, errChan chan error, stopChan chan struct{}) {
+func templatesLoop(client *minerClient, miningAddr util.Address,
+	newTemplateChan chan *rpcmodel.GetBlockTemplateResult, errChan chan error, stopChan chan struct{}) {
+
 	longPollID := ""
 	getBlockTemplateLongPoll := func() {
 		if longPollID != "" {
@@ -175,7 +181,7 @@ func templatesLoop(client *minerClient, newTemplateChan chan *rpcmodel.GetBlockT
 		} else {
 			log.Infof("Requesting template without longPollID from %s", client.Host())
 		}
-		template, err := getBlockTemplate(client, longPollID)
+		template, err := getBlockTemplate(client, miningAddr, longPollID)
 		if nativeerrors.Is(err, rpcclient.ErrResponseTimedOut) {
 			log.Infof("Got timeout while requesting template '%s' from %s", longPollID, client.Host())
 			return
@@ -203,16 +209,27 @@ func templatesLoop(client *minerClient, newTemplateChan chan *rpcmodel.GetBlockT
 	}
 }
 
-func getBlockTemplate(client *minerClient, longPollID string) (*rpcmodel.GetBlockTemplateResult, error) {
-	return client.GetBlockTemplate([]string{"coinbasetxn"}, longPollID)
+func getBlockTemplate(client *minerClient, miningAddr util.Address, longPollID string) (*rpcmodel.GetBlockTemplateResult, error) {
+	return client.GetBlockTemplate(miningAddr.String(), longPollID)
 }
 
-func solveLoop(newTemplateChan chan *rpcmodel.GetBlockTemplateResult, foundBlock chan *util.Block, errChan chan error) {
+func solveLoop(newTemplateChan chan *rpcmodel.GetBlockTemplateResult, foundBlock chan *util.Block,
+	mineWhenNotSynced bool, errChan chan error) {
+
 	var stopOldTemplateSolving chan struct{}
 	for template := range newTemplateChan {
 		if stopOldTemplateSolving != nil {
 			close(stopOldTemplateSolving)
 		}
+
+		if !template.IsSynced {
+			if !mineWhenNotSynced {
+				errChan <- errors.Errorf("got template with isSynced=false")
+				return
+			}
+			log.Warnf("Got template with isSynced=false")
+		}
+
 		stopOldTemplateSolving = make(chan struct{})
 		block, err := parseBlock(template)
 		if err != nil {

cmd/txsigner/txsigner.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
-	"github.com/kaspanet/kaspad/ecc"
+	"github.com/kaspanet/go-secp256k1"
 	"github.com/kaspanet/kaspad/txscript"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/wire"
@@ -28,7 +28,11 @@ func main() {
 		printErrorAndExit(err, "Failed to decode transaction")
 	}
 
-	scriptPubKey, err := createScriptPubKey(privateKey.PubKey())
+	pubkey, err := privateKey.SchnorrPublicKey()
+	if err != nil {
+		printErrorAndExit(err, "Failed to generate a public key")
+	}
+	scriptPubKey, err := createScriptPubKey(pubkey)
 	if err != nil {
 		printErrorAndExit(err, "Failed to create scriptPubKey")
 	}
@@ -46,10 +50,12 @@ func main() {
 	fmt.Printf("Signed Transaction (hex): %s\n\n", serializedTransaction)
 }
 
-func parsePrivateKey(privateKeyHex string) (*ecc.PrivateKey, error) {
+func parsePrivateKey(privateKeyHex string) (*secp256k1.PrivateKey, error) {
 	privateKeyBytes, err := hex.DecodeString(privateKeyHex)
-	privateKey, _ := ecc.PrivKeyFromBytes(ecc.S256(), privateKeyBytes)
-	return privateKey, err
+	if err != nil {
+		return nil, errors.Errorf("'%s' isn't a valid hex. err: '%s' ", privateKeyHex, err)
+	}
+	return secp256k1.DeserializePrivateKeyFromSlice(privateKeyBytes)
 }
 
 func parseTransaction(transactionHex string) (*wire.MsgTx, error) {
@@ -62,8 +68,12 @@ func parseTransaction(transactionHex string) (*wire.MsgTx, error) {
 	return &transaction, err
 }
 
-func createScriptPubKey(publicKey *ecc.PublicKey) ([]byte, error) {
-	p2pkhAddress, err := util.NewAddressPubKeyHashFromPublicKey(publicKey.SerializeCompressed(), ActiveConfig().NetParams().Prefix)
+func createScriptPubKey(publicKey *secp256k1.SchnorrPublicKey) ([]byte, error) {
+	serializedKey, err := publicKey.SerializeCompressed()
+	if err != nil {
+		return nil, err
+	}
+	p2pkhAddress, err := util.NewAddressPubKeyHashFromPublicKey(serializedKey, ActiveConfig().NetParams().Prefix)
 	if err != nil {
 		return nil, err
 	}
@@ -71,7 +81,7 @@ func createScriptPubKey(publicKey *ecc.PublicKey) ([]byte, error) {
 	return scriptPubKey, err
 }
 
-func signTransaction(transaction *wire.MsgTx, privateKey *ecc.PrivateKey, scriptPubKey []byte) error {
+func signTransaction(transaction *wire.MsgTx, privateKey *secp256k1.PrivateKey, scriptPubKey []byte) error {
 	for i, transactionInput := range transaction.TxIn {
 		signatureScript, err := txscript.SignatureScript(transaction, i, scriptPubKey, txscript.SigHashAll, privateKey, true)
 		if err != nil {

config/config.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/btcsuite/go-socks/socks"
 	"github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/database"
 	"github.com/kaspanet/kaspad/logger"
 	"github.com/kaspanet/kaspad/util"
 	"github.com/kaspanet/kaspad/util/network"
@@ -46,7 +45,6 @@ const (
 	defaultMaxRPCClients         = 10
 	defaultMaxRPCWebsockets      = 25
 	defaultMaxRPCConcurrentReqs  = 20
-	defaultDbType                = "ffldb"
 	defaultBlockMaxMass          = 10000000
 	blockMaxMassMin              = 1000
 	blockMaxMassMax              = 10000000
@@ -65,7 +63,6 @@ var (
 
 	defaultConfigFile  = filepath.Join(DefaultHomeDir, defaultConfigFilename)
 	defaultDataDir     = filepath.Join(DefaultHomeDir, defaultDataDirname)
-	knownDbTypes       = database.SupportedDrivers()
 	defaultRPCKeyFile  = filepath.Join(DefaultHomeDir, "rpc.key")
 	defaultRPCCertFile = filepath.Join(DefaultHomeDir, "rpc.cert")
 	defaultLogDir      = filepath.Join(DefaultHomeDir, defaultLogDirname)
@@ -120,7 +117,6 @@ type Flags struct {
 	Upnp                 bool          `long:"upnp" description:"Use UPnP to map our listening port outside of NAT"`
 	MinRelayTxFee        float64       `long:"minrelaytxfee" description:"The minimum transaction fee in KAS/kB to be considered a non-zero fee."`
 	MaxOrphanTxs         int           `long:"maxorphantx" description:"Max number of orphan transactions to keep in memory"`
-	MiningAddrs          []string      `long:"miningaddr" description:"Add the specified payment address to the list of addresses to use for generated blocks -- At least one address is required if the generate option is set"`
 	BlockMaxMass         uint64        `long:"blockmaxmass" description:"Maximum transaction mass to be used when creating a block"`
 	UserAgentComments    []string      `long:"uacomment" description:"Comment to add to the user agent -- See BIP 14 for more information."`
 	NoPeerBloomFilters   bool          `long:"nopeerbloomfilters" description:"Disable bloom filtering support"`
@@ -130,7 +126,6 @@ type Flags struct {
 	DropAcceptanceIndex  bool          `long:"dropacceptanceindex" description:"Deletes the hash-based acceptance index from the database on start up and then exits."`
 	RelayNonStd          bool          `long:"relaynonstd" description:"Relay non-standard transactions regardless of the default settings for the active network."`
 	RejectNonStd         bool          `long:"rejectnonstd" description:"Reject non-standard transactions regardless of the default settings for the active network."`
-	Subnetwork           string        `long:"subnetwork" description:"If subnetwork ID is specified, than node will request and process only payloads from specified subnetwork. And if subnetwork ID is ommited, than payloads of all subnetworks are processed. Subnetworks with IDs 2 through 255 are reserved for future use and are currently not allowed."`
 	ResetDatabase        bool          `long:"reset-db" description:"Reset database before starting node. It's needed when switching between subnetworks."`
 	NetworkFlags
 }
@@ -168,17 +163,6 @@ func cleanAndExpandPath(path string) string {
 	return filepath.Clean(os.ExpandEnv(path))
 }
 
-// validDbType returns whether or not dbType is a supported database type.
-func validDbType(dbType string) bool {
-	for _, knownType := range knownDbTypes {
-		if dbType == knownType {
-			return true
-		}
-	}
-
-	return false
-}
-
 // newConfigParser returns a new command line flags parser.
 func newConfigParser(cfgFlags *Flags, so *serviceOptions, options flags.Options) *flags.Parser {
 	parser := flags.NewParser(cfgFlags, options)
@@ -235,7 +219,6 @@ func loadConfig() (*Config, []string, error) {
 		RPCMaxConcurrentReqs: defaultMaxRPCConcurrentReqs,
 		DataDir:              defaultDataDir,
 		LogDir:               defaultLogDir,
-		DbType:               defaultDbType,
 		RPCKey:               defaultRPCKeyFile,
 		RPCCert:              defaultRPCCertFile,
 		BlockMaxMass:         defaultBlockMaxMass,
@@ -424,16 +407,6 @@ func loadConfig() (*Config, []string, error) {
 		return nil, nil, err
 	}
 
-	// Validate database type.
-	if !validDbType(activeConfig.DbType) {
-		str := "%s: The specified database type [%s] is invalid -- " +
-			"supported types %s"
-		err := errors.Errorf(str, funcName, activeConfig.DbType, knownDbTypes)
-		fmt.Fprintln(os.Stderr, err)
-		fmt.Fprintln(os.Stderr, usageMessage)
-		return nil, nil, err
-	}
-
 	// Validate profile port number
 	if activeConfig.Profile != "" {
 		profilePort, err := strconv.Atoi(activeConfig.Profile)
@@ -633,36 +606,6 @@ func loadConfig() (*Config, []string, error) {
 		return nil, nil, err
 	}
 
-	// Check mining addresses are valid and saved parsed versions.
-	activeConfig.MiningAddrs = make([]util.Address, 0, len(activeConfig.Flags.MiningAddrs))
-	for _, strAddr := range activeConfig.Flags.MiningAddrs {
-		addr, err := util.DecodeAddress(strAddr, activeConfig.NetParams().Prefix)
-		if err != nil {
-			str := "%s: mining address '%s' failed to decode: %s"
-			err := errors.Errorf(str, funcName, strAddr, err)
-			fmt.Fprintln(os.Stderr, err)
-			fmt.Fprintln(os.Stderr, usageMessage)
-			return nil, nil, err
-		}
-		if !addr.IsForPrefix(activeConfig.NetParams().Prefix) {
-			str := "%s: mining address '%s' is on the wrong network"
-			err := errors.Errorf(str, funcName, strAddr)
-			fmt.Fprintln(os.Stderr, err)
-			fmt.Fprintln(os.Stderr, usageMessage)
-			return nil, nil, err
-		}
-		activeConfig.MiningAddrs = append(activeConfig.MiningAddrs, addr)
-	}
-
-	if activeConfig.Flags.Subnetwork != "" {
-		activeConfig.SubnetworkID, err = subnetworkid.NewFromStr(activeConfig.Flags.Subnetwork)
-		if err != nil {
-			return nil, nil, err
-		}
-	} else {
-		activeConfig.SubnetworkID = nil
-	}
-
 	// Add default port to all listener addresses if needed and remove
 	// duplicate addresses.
 	activeConfig.Listeners, err = network.NormalizeAddresses(activeConfig.Listeners,

connmgr/connmanager.go

@@ -7,6 +7,9 @@ package connmgr
 import (
 	nativeerrors "errors"
 	"fmt"
+	"github.com/kaspanet/kaspad/addrmgr"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/wire"
 	"net"
 	"sync"
 	"sync/atomic"
@@ -30,10 +33,6 @@ var (
 	// defaultRetryDuration is the default duration of time for retrying
 	// persistent connections.
 	defaultRetryDuration = time.Second * 5
-
-	// defaultTargetOutbound is the default number of outbound connections to
-	// maintain.
-	defaultTargetOutbound = uint32(8)
 )
 
 var (
@@ -54,6 +53,9 @@ var (
 
 	// ErrPeerNotFound is an error that is thrown if the peer was not found.
 	ErrPeerNotFound = errors.New("peer not found")
+
+	//ErrAddressManagerNil is used to indicate that Address Manager cannot be nil in the configuration.
+	ErrAddressManagerNil = errors.New("Config: Address manager cannot be nil")
 )
 
 // ConnState represents the state of the requested connection.
@@ -77,7 +79,7 @@ type ConnReq struct {
 	// The following variables must only be used atomically.
 	id uint64
 
-	Addr      net.Addr
+	Addr      *net.TCPAddr
 	Permanent bool
 
 	conn       net.Conn
@@ -151,13 +153,15 @@ type Config struct {
 	// connection is established.
 	OnConnection func(*ConnReq, net.Conn)
 
+	// OnConnectionFailed is a callback that is fired when a new outbound
+	// connection has failed to be established.
+	OnConnectionFailed func(*ConnReq)
+
 	// OnDisconnection is a callback that is fired when an outbound
 	// connection is disconnected.
 	OnDisconnection func(*ConnReq)
 
-	// GetNewAddress is a way to get an address to make a network connection
-	// to. If nil, no new connections will be made automatically.
-	GetNewAddress func() (net.Addr, error)
+	AddrManager *addrmgr.AddrManager
 
 	// Dial connects to the address on the named network. It cannot be nil.
 	Dial func(net.Addr) (net.Conn, error)
@@ -197,7 +201,9 @@ type ConnManager struct {
 	start        int32
 	stop         int32
 
-	newConnReqMtx sync.Mutex
+	addressMtx         sync.Mutex
+	usedOutboundGroups map[string]int64
+	usedAddresses      map[string]struct{}
 
 	cfg            Config
 	wg             sync.WaitGroup
@@ -233,9 +239,12 @@ func (cm *ConnManager) handleFailedConn(c *ConnReq, err error) {
 			log.Debugf("Retrying further connections to %s every %s", c, d)
 		}
 		spawnAfter(d, func() {
-			cm.Connect(c)
+			cm.connect(c)
 		})
-	} else if cm.cfg.GetNewAddress != nil {
+	} else {
+		if c.Addr != nil {
+			cm.releaseAddress(c.Addr)
+		}
 		cm.failedAttempts++
 		if cm.failedAttempts >= maxFailedAttempts {
 			if shouldWriteLog {
@@ -250,6 +259,43 @@ func (cm *ConnManager) handleFailedConn(c *ConnReq, err error) {
 	}
 }
 
+func (cm *ConnManager) releaseAddress(addr *net.TCPAddr) {
+	cm.addressMtx.Lock()
+	defer cm.addressMtx.Unlock()
+
+	groupKey := usedOutboundGroupsKey(addr)
+	cm.usedOutboundGroups[groupKey]--
+	if cm.usedOutboundGroups[groupKey] < 0 {
+		panic(fmt.Errorf("cm.usedOutboundGroups[%s] has a negative value of %d. This should never happen", groupKey, cm.usedOutboundGroups[groupKey]))
+	}
+	delete(cm.usedAddresses, usedAddressesKey(addr))
+}
+
+func (cm *ConnManager) markAddressAsUsed(addr *net.TCPAddr) {
+	cm.usedOutboundGroups[usedOutboundGroupsKey(addr)]++
+	cm.usedAddresses[usedAddressesKey(addr)] = struct{}{}
+}
+
+func (cm *ConnManager) isOutboundGroupUsed(addr *net.TCPAddr) bool {
+	_, ok := cm.usedOutboundGroups[usedOutboundGroupsKey(addr)]
+	return ok
+}
+
+func (cm *ConnManager) isAddressUsed(addr *net.TCPAddr) bool {
+	_, ok := cm.usedAddresses[usedAddressesKey(addr)]
+	return ok
+}
+
+func usedOutboundGroupsKey(addr *net.TCPAddr) string {
+	// A fake service flag is used since it doesn't affect the group key.
+	na := wire.NewNetAddress(addr, wire.SFNodeNetwork)
+	return addrmgr.GroupKey(na)
+}
+
+func usedAddressesKey(addr *net.TCPAddr) string {
+	return addr.String()
+}
+
 // throttledError defines an error type whose logs get throttled. This is to
 // prevent flooding the logs with identical errors.
 type throttledError error
@@ -388,21 +434,16 @@ out:
 					continue
 				}
 
-				// Otherwise, we will attempt a reconnection if
-				// we do not have enough peers, or if this is a
-				// persistent peer. The connection request is
-				// re added to the pending map, so that
-				// subsequent processing of connections and
-				// failures do not ignore the request.
-				if uint32(len(conns)) < cm.cfg.TargetOutbound ||
-					connReq.Permanent {
-
-					connReq.updateState(ConnPending)
-					log.Debugf("Reconnecting to %s",
-						connReq)
-					pending[msg.id] = connReq
-					cm.handleFailedConn(connReq, nil)
-				}
+				// Otherwise, we will attempt a reconnection.
+				// The connection request is re added to the
+				// pending map, so that subsequent processing
+				// of connections and failures do not ignore
+				// the request.
+				connReq.updateState(ConnPending)
+				log.Debugf("Reconnecting to %s",
+					connReq)
+				pending[msg.id] = connReq
+				cm.handleFailedConn(connReq, nil)
 
 			case handleFailed:
 				connReq := msg.c
@@ -419,6 +460,10 @@ out:
 						connReq, msg.err)
 				}
 				cm.handleFailedConn(connReq, msg.err)
+
+				if cm.cfg.OnConnectionFailed != nil {
+					cm.cfg.OnConnectionFailed(connReq)
+				}
 			}
 
 		case <-cm.quit:
@@ -440,14 +485,9 @@ func (cm *ConnManager) NotifyConnectionRequestComplete() {
 // NewConnReq creates a new connection request and connects to the
 // corresponding address.
 func (cm *ConnManager) NewConnReq() {
-	cm.newConnReqMtx.Lock()
-	defer cm.newConnReqMtx.Unlock()
 	if atomic.LoadInt32(&cm.stop) != 0 {
 		return
 	}
-	if cm.cfg.GetNewAddress == nil {
-		return
-	}
 
 	c := &ConnReq{}
 	atomic.StoreUint64(&c.id, atomic.AddUint64(&cm.connReqCount, 1))
@@ -470,8 +510,7 @@ func (cm *ConnManager) NewConnReq() {
 	case <-cm.quit:
 		return
 	}
-
-	addr, err := cm.cfg.GetNewAddress()
+	err := cm.associateAddressToConnReq(c)
 	if err != nil {
 		select {
 		case cm.requests <- handleFailed{c, err}:
@@ -480,17 +519,52 @@ func (cm *ConnManager) NewConnReq() {
 		return
 	}
 
-	c.Addr = addr
+	cm.connect(c)
+}
 
-	cm.Connect(c)
+func (cm *ConnManager) associateAddressToConnReq(c *ConnReq) error {
+	cm.addressMtx.Lock()
+	defer cm.addressMtx.Unlock()
+
+	addr, err := cm.getNewAddress()
+	if err != nil {
+		return err
+	}
+
+	cm.markAddressAsUsed(addr)
+	c.Addr = addr
+	return nil
 }
 
 // Connect assigns an id and dials a connection to the address of the
 // connection request.
-func (cm *ConnManager) Connect(c *ConnReq) {
+func (cm *ConnManager) Connect(c *ConnReq) error {
+	err := func() error {
+		cm.addressMtx.Lock()
+		defer cm.addressMtx.Unlock()
+
+		if cm.isAddressUsed(c.Addr) {
+			return fmt.Errorf("address %s is already in use", c.Addr)
+		}
+		cm.markAddressAsUsed(c.Addr)
+		return nil
+	}()
+	if err != nil {
+		return err
+	}
+
+	cm.connect(c)
+	return nil
+}
+
+// connect assigns an id and dials a connection to the address of the
+// connection request. This function assumes that the connection address
+// has checked and already marked as used.
+func (cm *ConnManager) connect(c *ConnReq) {
 	if atomic.LoadInt32(&cm.stop) != 0 {
 		return
 	}
+
 	if atomic.LoadUint64(&c.id) == 0 {
 		atomic.StoreUint64(&c.id, atomic.AddUint64(&cm.connReqCount, 1))
 
@@ -637,23 +711,69 @@ func (cm *ConnManager) Stop() {
 	log.Trace("Connection manager stopped")
 }
 
+func (cm *ConnManager) getNewAddress() (*net.TCPAddr, error) {
+	for tries := 0; tries < 100; tries++ {
+		addr := cm.cfg.AddrManager.GetAddress()
+		if addr == nil {
+			break
+		}
+
+		// Check if there's already a connection to the same address.
+		netAddr := addr.NetAddress().TCPAddress()
+		if cm.isAddressUsed(netAddr) {
+			continue
+		}
+
+		// Address will not be invalid, local or unroutable
+		// because addrmanager rejects those on addition.
+		// Just check that we don't already have an address
+		// in the same group so that we are not connecting
+		// to the same network segment at the expense of
+		// others.
+		//
+		// Networks that accept unroutable connections are exempt
+		// from this rule, since they're meant to run within a
+		// private subnet, like 10.0.0.0/16.
+		if !config.ActiveConfig().NetParams().AcceptUnroutable && cm.isOutboundGroupUsed(netAddr) {
+			continue
+		}
+
+		// only allow recent nodes (10mins) after we failed 30
+		// times
+		if tries < 30 && time.Since(addr.LastAttempt()) < 10*time.Minute {
+			continue
+		}
+
+		// allow nondefault ports after 50 failed tries.
+		if tries < 50 && fmt.Sprintf("%d", netAddr.Port) !=
+			config.ActiveConfig().NetParams().DefaultPort {
+			continue
+		}
+
+		return netAddr, nil
+	}
+	return nil, ErrNoAddress
+}
+
 // New returns a new connection manager.
 // Use Start to start connecting to the network.
 func New(cfg *Config) (*ConnManager, error) {
 	if cfg.Dial == nil {
-		return nil, ErrDialNil
+		return nil, errors.WithStack(ErrDialNil)
+	}
+	if cfg.AddrManager == nil {
+		return nil, errors.WithStack(ErrAddressManagerNil)
 	}
 	// Default to sane values
 	if cfg.RetryDuration <= 0 {
 		cfg.RetryDuration = defaultRetryDuration
 	}
-	if cfg.TargetOutbound == 0 {
-		cfg.TargetOutbound = defaultTargetOutbound
-	}
 	cm := ConnManager{
-		cfg:      *cfg, // Copy so caller can't mutate
-		requests: make(chan interface{}),
-		quit:     make(chan struct{}),
+		cfg:                *cfg, // Copy so caller can't mutate
+		requests:           make(chan interface{}),
+		quit:               make(chan struct{}),
+		usedAddresses:      make(map[string]struct{}),
+		usedOutboundGroups: make(map[string]int64),
 	}
 	return &cm, nil
 }

connmgr/connmanager_test.go

@@ -5,12 +5,19 @@
 package connmgr
 
 import (
-	"github.com/pkg/errors"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"sync/atomic"
 	"testing"
 	"time"
+
+	"github.com/kaspanet/kaspad/addrmgr"
+	"github.com/kaspanet/kaspad/config"
+	"github.com/kaspanet/kaspad/dagconfig"
+	"github.com/kaspanet/kaspad/dbaccess"
+	"github.com/pkg/errors"
 )
 
 func init() {
@@ -70,13 +77,28 @@ func mockDialer(addr net.Addr) (net.Conn, error) {
 
 // TestNewConfig tests that new ConnManager config is validated as expected.
 func TestNewConfig(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	_, err := New(&Config{})
-	if err == nil {
-		t.Fatalf("New expected error: 'Dial can't be nil', got nil")
+	if !errors.Is(err, ErrDialNil) {
+		t.Fatalf("New expected error: %s, got %s", ErrDialNil, err)
 	}
+
 	_, err = New(&Config{
 		Dial: mockDialer,
 	})
+	if !errors.Is(err, ErrAddressManagerNil) {
+		t.Fatalf("New expected error: %s, got %s", ErrAddressManagerNil, err)
+	}
+
+	amgr, teardown := addressManagerForTest(t, "TestNewConfig", 10)
+	defer teardown()
+
+	_, err = New(&Config{
+		Dial:        mockDialer,
+		AddrManager: amgr,
+	})
 	if err != nil {
 		t.Fatalf("New unexpected error: %v", err)
 	}
@@ -85,17 +107,19 @@ func TestNewConfig(t *testing.T) {
 // TestStartStop tests that the connection manager starts and stops as
 // expected.
 func TestStartStop(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	connected := make(chan *ConnReq)
 	disconnected := make(chan *ConnReq)
+
+	amgr, teardown := addressManagerForTest(t, "TestStartStop", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
 		TargetOutbound: 1,
-		GetNewAddress: func() (net.Addr, error) {
-			return &net.TCPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: 18555,
-			}, nil
-		},
-		Dial: mockDialer,
+		AddrManager:    amgr,
+		Dial:           mockDialer,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			connected <- c
 		},
@@ -104,7 +128,7 @@ func TestStartStop(t *testing.T) {
 		},
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
 	gotConnReq := <-connected
@@ -119,7 +143,10 @@ func TestStartStop(t *testing.T) {
 		},
 		Permanent: true,
 	}
-	cmgr.Connect(cr)
+	err = cmgr.Connect(cr)
+	if err != nil {
+		t.Fatalf("Connect error: %s", err)
+	}
 	if cr.ID() != 0 {
 		t.Fatalf("start/stop: got id: %v, want: 0", cr.ID())
 	}
@@ -133,21 +160,85 @@ func TestStartStop(t *testing.T) {
 	}
 }
 
+func overrideActiveConfig() func() {
+	originalActiveCfg := config.ActiveConfig()
+	config.SetActiveConfig(&config.Config{
+		Flags: &config.Flags{
+			NetworkFlags: config.NetworkFlags{
+				ActiveNetParams: &dagconfig.SimnetParams},
+		},
+	})
+	return func() {
+		// Give some extra time to all open NewConnReq goroutines
+		// to finish before restoring the active config to prevent
+		// potential panics.
+		time.Sleep(10 * time.Millisecond)
+
+		config.SetActiveConfig(originalActiveCfg)
+	}
+}
+
+func addressManagerForTest(t *testing.T, testName string, numAddresses uint8) (*addrmgr.AddrManager, func()) {
+	amgr, teardown := createEmptyAddressManagerForTest(t, testName)
+
+	for i := uint8(0); i < numAddresses; i++ {
+		ip := fmt.Sprintf("173.%d.115.66:16511", i)
+		err := amgr.AddAddressByIP(ip, nil)
+		if err != nil {
+			t.Fatalf("AddAddressByIP unexpectedly failed to add IP %s: %s", ip, err)
+		}
+	}
+
+	return amgr, teardown
+}
+
+func createEmptyAddressManagerForTest(t *testing.T, testName string) (*addrmgr.AddrManager, func()) {
+	path, err := ioutil.TempDir("", fmt.Sprintf("%s-database", testName))
+	if err != nil {
+		t.Fatalf("createEmptyAddressManagerForTest: TempDir unexpectedly "+
+			"failed: %s", err)
+	}
+
+	err = dbaccess.Open(path)
+	if err != nil {
+		t.Fatalf("error creating db: %s", err)
+	}
+
+	return addrmgr.New(nil, nil), func() {
+		// Wait for the connection manager to finish, so it'll
+		// have access to the address manager as long as it's
+		// alive.
+		time.Sleep(10 * time.Millisecond)
+
+		err := dbaccess.Close()
+		if err != nil {
+			t.Fatalf("error closing the database: %s", err)
+		}
+	}
+}
+
 // TestConnectMode tests that the connection manager works in the connect mode.
 //
 // In connect mode, automatic connections are disabled, so we test that
 // requests using Connect are handled and that no other connections are made.
 func TestConnectMode(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	connected := make(chan *ConnReq)
+	amgr, teardown := addressManagerForTest(t, "TestConnectMode", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
-		TargetOutbound: 2,
+		TargetOutbound: 0,
 		Dial:           mockDialer,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			connected <- c
 		},
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cr := &ConnReq{
 		Addr: &net.TCPAddr{
@@ -176,6 +267,7 @@ func TestConnectMode(t *testing.T) {
 		break
 	}
 	cmgr.Stop()
+	cmgr.Wait()
 }
 
 // TestTargetOutbound tests the target number of outbound connections.
@@ -183,23 +275,26 @@ func TestConnectMode(t *testing.T) {
 // We wait until all connections are established, then test they there are the
 // only connections made.
 func TestTargetOutbound(t *testing.T) {
-	targetOutbound := uint32(10)
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
+	const numAddressesInAddressManager = 10
+	targetOutbound := uint32(numAddressesInAddressManager - 2)
 	connected := make(chan *ConnReq)
+
+	amgr, teardown := addressManagerForTest(t, "TestTargetOutbound", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
 		TargetOutbound: targetOutbound,
 		Dial:           mockDialer,
-		GetNewAddress: func() (net.Addr, error) {
-			return &net.TCPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: 18555,
-			}, nil
-		},
+		AddrManager:    amgr,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			connected <- c
 		},
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
 	for i := uint32(0); i < targetOutbound; i++ {
@@ -213,6 +308,146 @@ func TestTargetOutbound(t *testing.T) {
 		break
 	}
 	cmgr.Stop()
+	cmgr.Wait()
+}
+
+// TestDuplicateOutboundConnections tests that connection requests cannot use an already used address.
+// It checks it by creating one connection request for each address in the address manager, so that
+// the next connection request will have to fail because no unused address will be available.
+func TestDuplicateOutboundConnections(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
+	const numAddressesInAddressManager = 10
+	targetOutbound := uint32(numAddressesInAddressManager - 1)
+	connected := make(chan struct{})
+	failedConnections := make(chan struct{})
+
+	amgr, teardown := addressManagerForTest(t, "TestDuplicateOutboundConnections", 10)
+	defer teardown()
+
+	cmgr, err := New(&Config{
+		TargetOutbound: targetOutbound,
+		Dial:           mockDialer,
+		AddrManager:    amgr,
+		OnConnection: func(c *ConnReq, conn net.Conn) {
+			connected <- struct{}{}
+		},
+		OnConnectionFailed: func(_ *ConnReq) {
+			failedConnections <- struct{}{}
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected error from New: %s", err)
+	}
+	cmgr.Start()
+	for i := uint32(0); i < targetOutbound; i++ {
+		<-connected
+	}
+
+	time.Sleep(time.Millisecond)
+
+	// Here we check that making a manual connection request beyond the target outbound connection
+	// doesn't fail, so we can know that the reason such connection request will fail is an address
+	// related issue.
+	cmgr.NewConnReq()
+	select {
+	case <-connected:
+		break
+	case <-time.After(time.Millisecond):
+		t.Fatalf("connection request unexpectedly didn't connect")
+	}
+
+	select {
+	case <-failedConnections:
+		t.Fatalf("a connection request unexpectedly failed")
+	case <-time.After(time.Millisecond):
+		break
+	}
+
+	// After we created numAddressesInAddressManager connection requests, this request should fail
+	// because there aren't any more available addresses.
+	cmgr.NewConnReq()
+	select {
+	case <-connected:
+		t.Fatalf("connection request unexpectedly succeeded")
+	case <-time.After(time.Millisecond):
+		t.Fatalf("connection request didn't fail as expected")
+	case <-failedConnections:
+		break
+	}
+
+	cmgr.Stop()
+	cmgr.Wait()
+}
+
+// TestSameOutboundGroupConnections tests that connection requests cannot use an address with an already used
+// address CIDR group.
+// It checks it by creating an address manager with only two addresses, that both belong to the same CIDR group
+// and checks that the second connection request fails.
+func TestSameOutboundGroupConnections(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
+	amgr, teardown := createEmptyAddressManagerForTest(t, "TestSameOutboundGroupConnections")
+	defer teardown()
+
+	err := amgr.AddAddressByIP("173.190.115.66:16511", nil)
+	if err != nil {
+		t.Fatalf("AddAddressByIP unexpectedly failed: %s", err)
+	}
+
+	err = amgr.AddAddressByIP("173.190.115.67:16511", nil)
+	if err != nil {
+		t.Fatalf("AddAddressByIP unexpectedly failed: %s", err)
+	}
+
+	connected := make(chan struct{})
+	failedConnections := make(chan struct{})
+	cmgr, err := New(&Config{
+		TargetOutbound: 0,
+		Dial:           mockDialer,
+		AddrManager:    amgr,
+		OnConnection: func(c *ConnReq, conn net.Conn) {
+			connected <- struct{}{}
+		},
+		OnConnectionFailed: func(_ *ConnReq) {
+			failedConnections <- struct{}{}
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected error from New: %s", err)
+	}
+
+	cmgr.Start()
+
+	cmgr.NewConnReq()
+	select {
+	case <-connected:
+		break
+	case <-time.After(time.Millisecond):
+		t.Fatalf("connection request unexpectedly didn't connect")
+	}
+
+	select {
+	case <-failedConnections:
+		t.Fatalf("a connection request unexpectedly failed")
+	case <-time.After(time.Millisecond):
+		break
+	}
+
+	cmgr.NewConnReq()
+	select {
+	case <-connected:
+		t.Fatalf("connection request unexpectedly succeeded")
+	case <-time.After(time.Millisecond):
+		t.Fatalf("connection request didn't fail as expected")
+	case <-failedConnections:
+		break
+	}
+
+	cmgr.Stop()
+	cmgr.Wait()
 }
 
 // TestRetryPermanent tests that permanent connection requests are retried.
@@ -220,11 +455,18 @@ func TestTargetOutbound(t *testing.T) {
 // We make a permanent connection request using Connect, disconnect it using
 // Disconnect and we wait for it to be connected back.
 func TestRetryPermanent(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	connected := make(chan *ConnReq)
 	disconnected := make(chan *ConnReq)
+
+	amgr, teardown := addressManagerForTest(t, "TestRetryPermanent", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
 		RetryDuration:  time.Millisecond,
-		TargetOutbound: 1,
+		TargetOutbound: 0,
 		Dial:           mockDialer,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			connected <- c
@@ -232,9 +474,10 @@ func TestRetryPermanent(t *testing.T) {
 		OnDisconnection: func(c *ConnReq) {
 			disconnected <- c
 		},
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 
 	cr := &ConnReq{
@@ -289,6 +532,9 @@ func TestRetryPermanent(t *testing.T) {
 
 	cmgr.Remove(cr.ID())
 	gotConnReq = <-disconnected
+
+	// Wait for status to be updated
+	time.Sleep(10 * time.Millisecond)
 	wantID = cr.ID()
 	gotID = gotConnReq.ID()
 	if gotID != wantID {
@@ -300,6 +546,7 @@ func TestRetryPermanent(t *testing.T) {
 		t.Fatalf("retry: %v - want state %v, got state %v", cr.Addr, wantState, gotState)
 	}
 	cmgr.Stop()
+	cmgr.Wait()
 }
 
 // TestMaxRetryDuration tests the maximum retry duration.
@@ -307,6 +554,9 @@ func TestRetryPermanent(t *testing.T) {
 // We have a timed dialer which initially returns err but after RetryDuration
 // hits maxRetryDuration returns a mock conn.
 func TestMaxRetryDuration(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	networkUp := make(chan struct{})
 	time.AfterFunc(5*time.Millisecond, func() {
 		close(networkUp)
@@ -320,17 +570,21 @@ func TestMaxRetryDuration(t *testing.T) {
 		}
 	}
 
+	amgr, teardown := addressManagerForTest(t, "TestMaxRetryDuration", 10)
+	defer teardown()
+
 	connected := make(chan *ConnReq)
 	cmgr, err := New(&Config{
 		RetryDuration:  time.Millisecond,
-		TargetOutbound: 1,
+		TargetOutbound: 0,
 		Dial:           timedDialer,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			connected <- c
 		},
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 
 	cr := &ConnReq{
@@ -350,35 +604,40 @@ func TestMaxRetryDuration(t *testing.T) {
 	case <-time.Tick(100 * time.Millisecond):
 		t.Fatalf("max retry duration: connection timeout")
 	}
+	cmgr.Stop()
+	cmgr.Wait()
 }
 
 // TestNetworkFailure tests that the connection manager handles a network
 // failure gracefully.
 func TestNetworkFailure(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	var dials uint32
 	errDialer := func(net net.Addr) (net.Conn, error) {
 		atomic.AddUint32(&dials, 1)
 		return nil, errors.New("network down")
 	}
+
+	amgr, teardown := addressManagerForTest(t, "TestNetworkFailure", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
 		TargetOutbound: 5,
 		RetryDuration:  5 * time.Millisecond,
 		Dial:           errDialer,
-		GetNewAddress: func() (net.Addr, error) {
-			return &net.TCPAddr{
-				IP:   net.ParseIP("127.0.0.1"),
-				Port: 18555,
-			}, nil
-		},
+		AddrManager:    amgr,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			t.Fatalf("network failure: got unexpected connection - %v", c.Addr)
 		},
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
-	time.AfterFunc(10*time.Millisecond, cmgr.Stop)
+	time.Sleep(10 * time.Millisecond)
+	cmgr.Stop()
 	cmgr.Wait()
 	wantMaxDials := uint32(75)
 	if atomic.LoadUint32(&dials) > wantMaxDials {
@@ -394,17 +653,25 @@ func TestNetworkFailure(t *testing.T) {
 // err so that the handler assumes that the conn manager is stopped and ignores
 // the failure.
 func TestStopFailed(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	done := make(chan struct{}, 1)
 	waitDialer := func(addr net.Addr) (net.Conn, error) {
 		done <- struct{}{}
 		time.Sleep(time.Millisecond)
 		return nil, errors.New("network down")
 	}
+
+	amgr, teardown := addressManagerForTest(t, "TestStopFailed", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
-		Dial: waitDialer,
+		Dial:        waitDialer,
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
 	go func() {
@@ -428,6 +695,9 @@ func TestStopFailed(t *testing.T) {
 // TestRemovePendingConnection tests that it's possible to cancel a pending
 // connection, removing its internal state from the ConnMgr.
 func TestRemovePendingConnection(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	// Create a ConnMgr instance with an instance of a dialer that'll never
 	// succeed.
 	wait := make(chan struct{})
@@ -435,11 +705,16 @@ func TestRemovePendingConnection(t *testing.T) {
 		<-wait
 		return nil, errors.Errorf("error")
 	}
+
+	amgr, teardown := addressManagerForTest(t, "TestRemovePendingConnection", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
-		Dial: indefiniteDialer,
+		Dial:        indefiniteDialer,
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
 
@@ -474,12 +749,16 @@ func TestRemovePendingConnection(t *testing.T) {
 
 	close(wait)
 	cmgr.Stop()
+	cmgr.Wait()
 }
 
 // TestCancelIgnoreDelayedConnection tests that a canceled connection request will
 // not execute the on connection callback, even if an outstanding retry
 // succeeds.
 func TestCancelIgnoreDelayedConnection(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	retryTimeout := 10 * time.Millisecond
 
 	// Setup a dialer that will continue to return an error until the
@@ -497,18 +776,22 @@ func TestCancelIgnoreDelayedConnection(t *testing.T) {
 	}
 
 	connected := make(chan *ConnReq)
+
+	amgr, teardown := addressManagerForTest(t, "TestCancelIgnoreDelayedConnection", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
 		Dial:          failingDialer,
 		RetryDuration: retryTimeout,
 		OnConnection: func(c *ConnReq, conn net.Conn) {
 			connected <- c
 		},
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
-	defer cmgr.Stop()
 
 	// Establish a connection request to a random IP we've chosen.
 	cr := &ConnReq{
@@ -552,7 +835,8 @@ func TestCancelIgnoreDelayedConnection(t *testing.T) {
 		t.Fatalf("on-connect should not be called for canceled req")
 	case <-time.After(5 * retryTimeout):
 	}
-
+	cmgr.Stop()
+	cmgr.Wait()
 }
 
 // mockListener implements the net.Listener interface and is used to test
@@ -617,21 +901,29 @@ func newMockListener(localAddr string) *mockListener {
 // TestListeners ensures providing listeners to the connection manager along
 // with an accept callback works properly.
 func TestListeners(t *testing.T) {
+	restoreConfig := overrideActiveConfig()
+	defer restoreConfig()
+
 	// Setup a connection manager with a couple of mock listeners that
 	// notify a channel when they receive mock connections.
 	receivedConns := make(chan net.Conn)
 	listener1 := newMockListener("127.0.0.1:16111")
 	listener2 := newMockListener("127.0.0.1:9333")
 	listeners := []net.Listener{listener1, listener2}
+
+	amgr, teardown := addressManagerForTest(t, "TestListeners", 10)
+	defer teardown()
+
 	cmgr, err := New(&Config{
 		Listeners: listeners,
 		OnAccept: func(conn net.Conn) {
 			receivedConns <- conn
 		},
-		Dial: mockDialer,
+		Dial:        mockDialer,
+		AddrManager: amgr,
 	})
 	if err != nil {
-		t.Fatalf("New error: %v", err)
+		t.Fatalf("unexpected error from New: %s", err)
 	}
 	cmgr.Start()
 

dagconfig/genesis.go

@@ -5,7 +5,6 @@
 package dagconfig
 
 import (
-	"math"
 	"time"
 
 	"github.com/kaspanet/kaspad/util/daghash"
@@ -13,22 +12,10 @@ import (
 	"github.com/kaspanet/kaspad/wire"
 )
 
-var genesisTxIns = []*wire.TxIn{
-	{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID{},
-			Index: math.MaxUint32,
-		},
-		SignatureScript: []byte{
-			0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48,
-			0x2f, 0x62, 0x74, 0x63, 0x64, 0x2f,
-		},
-		Sequence: math.MaxUint64,
-	},
-}
 var genesisTxOuts = []*wire.TxOut{}
 
 var genesisTxPayload = []byte{
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Blue score
 	0x17,                                           // Varint
 	0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, // OP-TRUE p2sh
 	0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7,
@@ -37,58 +24,46 @@ var genesisTxPayload = []byte{
 
 // genesisCoinbaseTx is the coinbase transaction for the genesis blocks for
 // the main network.
-var genesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, genesisTxIns, genesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, genesisTxPayload)
+var genesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, []*wire.TxIn{}, genesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, genesisTxPayload)
 
 // genesisHash is the hash of the first block in the block DAG for the main
 // network (genesis block).
 var genesisHash = daghash.Hash{
-	0x9b, 0x22, 0x59, 0x44, 0x66, 0xf0, 0xbe, 0x50,
-	0x7c, 0x1c, 0x8a, 0xf6, 0x06, 0x27, 0xe6, 0x33,
-	0x38, 0x7e, 0xd1, 0xd5, 0x8c, 0x42, 0x59, 0x1a,
-	0x31, 0xac, 0x9a, 0xa6, 0x2e, 0xd5, 0x2b, 0x0f,
+	0xb3, 0x5d, 0x34, 0x3c, 0xf6, 0xbb, 0xd5, 0xaf,
+	0x40, 0x4b, 0xff, 0x3f, 0x83, 0x27, 0x71, 0x1e,
+	0xe1, 0x83, 0xf6, 0x41, 0x32, 0x8c, 0xba, 0xe6,
+	0xd3, 0xba, 0x13, 0xef, 0x7b, 0x7e, 0x61, 0x65,
 }
 
 // genesisMerkleRoot is the hash of the first transaction in the genesis block
 // for the main network.
 var genesisMerkleRoot = daghash.Hash{
-	0x72, 0x10, 0x35, 0x85, 0xdd, 0xac, 0x82, 0x5c,
-	0x49, 0x13, 0x9f, 0xc0, 0x0e, 0x37, 0xc0, 0x45,
-	0x71, 0xdf, 0xd9, 0xf6, 0x36, 0xdf, 0x4c, 0x42,
-	0x72, 0x7b, 0x9e, 0x86, 0xdd, 0x37, 0xd2, 0xbd,
+	0xca, 0x85, 0x56, 0x27, 0xc7, 0x6a, 0xb5, 0x7a,
+	0x26, 0x1d, 0x63, 0x62, 0x1e, 0x57, 0x21, 0xf0,
+	0x5e, 0x60, 0x1f, 0xee, 0x1d, 0x4d, 0xaa, 0x53,
+	0x72, 0xe1, 0x16, 0xda, 0x4b, 0xb3, 0xd8, 0x0e,
 }
 
 // genesisBlock defines the genesis block of the block DAG which serves as the
 // public transaction ledger for the main network.
 var genesisBlock = wire.MsgBlock{
 	Header: wire.BlockHeader{
-		Version:              1,
+		Version:              0x10000000,
 		ParentHashes:         []*daghash.Hash{},
 		HashMerkleRoot:       &genesisMerkleRoot,
 		AcceptedIDMerkleRoot: &daghash.Hash{},
 		UTXOCommitment:       &daghash.ZeroHash,
-		Timestamp:            time.Unix(0x5cdac4b0, 0),
+		Timestamp:            time.Unix(0x5edf4ce0, 0),
 		Bits:                 0x207fffff,
-		Nonce:                0x1,
+		Nonce:                0,
 	},
 	Transactions: []*wire.MsgTx{genesisCoinbaseTx},
 }
 
-var devnetGenesisTxIns = []*wire.TxIn{
-	{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID{},
-			Index: math.MaxUint32,
-		},
-		SignatureScript: []byte{
-			0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48,
-			0x2f, 0x62, 0x74, 0x63, 0x64, 0x2f,
-		},
-		Sequence: math.MaxUint64,
-	},
-}
 var devnetGenesisTxOuts = []*wire.TxOut{}
 
 var devnetGenesisTxPayload = []byte{
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Blue score
 	0x17,                                           // Varint
 	0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, // OP-TRUE p2sh
 	0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7,
@@ -98,58 +73,46 @@ var devnetGenesisTxPayload = []byte{
 
 // devnetGenesisCoinbaseTx is the coinbase transaction for the genesis blocks for
 // the development network.
-var devnetGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, devnetGenesisTxIns, devnetGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, devnetGenesisTxPayload)
+var devnetGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, []*wire.TxIn{}, devnetGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, devnetGenesisTxPayload)
 
 // devGenesisHash is the hash of the first block in the block DAG for the development
 // network (genesis block).
 var devnetGenesisHash = daghash.Hash{
-	0x17, 0x59, 0x5c, 0x09, 0xdd, 0x1a, 0x51, 0x65,
-	0x14, 0xbc, 0x19, 0xff, 0x29, 0xea, 0xf3, 0xcb,
-	0xe2, 0x76, 0xf0, 0xc7, 0x86, 0xf8, 0x0c, 0x53,
-	0x59, 0xbe, 0xee, 0x0c, 0x2b, 0x5d, 0x00, 0x00,
+	0x50, 0x92, 0xd1, 0x1f, 0xaa, 0xba, 0xd3, 0x58,
+	0xa8, 0x22, 0xd7, 0xec, 0x8e, 0xe3, 0xf4, 0x26,
+	0x17, 0x18, 0x74, 0xd7, 0x87, 0x05, 0x9d, 0xed,
+	0x33, 0xcd, 0xe1, 0x26, 0x1a, 0x69, 0x00, 0x00,
 }
 
 // devnetGenesisMerkleRoot is the hash of the first transaction in the genesis block
 // for the devopment network.
 var devnetGenesisMerkleRoot = daghash.Hash{
-	0x16, 0x0a, 0xc6, 0x8b, 0x77, 0x08, 0xf4, 0x96,
-	0xa3, 0x07, 0x05, 0xbc, 0x92, 0xda, 0xee, 0x73,
-	0x26, 0x5e, 0xd0, 0x85, 0x78, 0xa2, 0x5d, 0x02,
-	0x49, 0x8a, 0x2a, 0x22, 0xef, 0x41, 0xc9, 0xc3,
+	0x68, 0x60, 0xe7, 0x77, 0x47, 0x74, 0x7f, 0xd5,
+	0x55, 0x58, 0x8a, 0xb5, 0xc2, 0x29, 0x0c, 0xa6,
+	0x65, 0x44, 0xb4, 0x4f, 0xfa, 0x31, 0x7a, 0xfa,
+	0x55, 0xe0, 0xcf, 0xac, 0x9c, 0x86, 0x30, 0x2a,
 }
 
 // devnetGenesisBlock defines the genesis block of the block DAG which serves as the
 // public transaction ledger for the development network.
 var devnetGenesisBlock = wire.MsgBlock{
 	Header: wire.BlockHeader{
-		Version:              1,
+		Version:              0x10000000,
 		ParentHashes:         []*daghash.Hash{},
 		HashMerkleRoot:       &devnetGenesisMerkleRoot,
 		AcceptedIDMerkleRoot: &daghash.Hash{},
 		UTXOCommitment:       &daghash.ZeroHash,
-		Timestamp:            time.Unix(0x5e15e758, 0),
+		Timestamp:            time.Unix(0x5edf4ce0, 0),
 		Bits:                 0x1e7fffff,
-		Nonce:                0x282ac,
+		Nonce:                0xb3ed,
 	},
 	Transactions: []*wire.MsgTx{devnetGenesisCoinbaseTx},
 }
 
-var regtestGenesisTxIns = []*wire.TxIn{
-	{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID{},
-			Index: math.MaxUint32,
-		},
-		SignatureScript: []byte{
-			0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48,
-			0x2f, 0x62, 0x74, 0x63, 0x64, 0x2f,
-		},
-		Sequence: math.MaxUint64,
-	},
-}
 var regtestGenesisTxOuts = []*wire.TxOut{}
 
 var regtestGenesisTxPayload = []byte{
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Blue score
 	0x17,                                           // Varint
 	0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, // OP-TRUE p2sh
 	0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7,
@@ -159,58 +122,46 @@ var regtestGenesisTxPayload = []byte{
 
 // regtestGenesisCoinbaseTx is the coinbase transaction for
 // the genesis blocks for the regtest network.
-var regtestGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, regtestGenesisTxIns, regtestGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, regtestGenesisTxPayload)
+var regtestGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, []*wire.TxIn{}, regtestGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, regtestGenesisTxPayload)
 
 // devGenesisHash is the hash of the first block in the block DAG for the development
 // network (genesis block).
 var regtestGenesisHash = daghash.Hash{
-	0xfc, 0x02, 0x19, 0x6f, 0x79, 0x7a, 0xed, 0x2d,
-	0x0f, 0x31, 0xa5, 0xbd, 0x32, 0x13, 0x29, 0xc7,
-	0x7c, 0x0c, 0x5c, 0x1a, 0x5b, 0x7c, 0x20, 0x68,
-	0xb7, 0xc9, 0x9f, 0x61, 0x13, 0x11, 0x00, 0x00,
+	0xf8, 0x1d, 0xe9, 0x86, 0xa5, 0x60, 0xe0, 0x34,
+	0x0f, 0x02, 0xaa, 0x8d, 0xea, 0x6f, 0x1f, 0xc6,
+	0x2a, 0xb4, 0x77, 0xbd, 0xca, 0xed, 0xad, 0x3c,
+	0x99, 0xe6, 0x98, 0x7c, 0x7b, 0x5e, 0x00, 0x00,
 }
 
 // regtestGenesisMerkleRoot is the hash of the first transaction in the genesis block
 // for the regtest.
 var regtestGenesisMerkleRoot = daghash.Hash{
-	0x3a, 0x9f, 0x62, 0xc9, 0x2b, 0x16, 0x17, 0xb3,
-	0x41, 0x6d, 0x9e, 0x2d, 0x87, 0x93, 0xfd, 0x72,
-	0x77, 0x4d, 0x1d, 0x6f, 0x6d, 0x38, 0x5b, 0xf1,
-	0x24, 0x1b, 0xdc, 0x96, 0xce, 0xbf, 0xa1, 0x09,
+	0x1e, 0x08, 0xae, 0x1f, 0x43, 0xf5, 0xfc, 0x24,
+	0xe6, 0xec, 0x54, 0x5b, 0xf7, 0x52, 0x99, 0xe4,
+	0xcc, 0x4c, 0xa0, 0x79, 0x41, 0xfc, 0xbe, 0x76,
+	0x72, 0x4c, 0x7e, 0xd8, 0xa3, 0x43, 0x65, 0x94,
 }
 
 // regtestGenesisBlock defines the genesis block of the block DAG which serves as the
 // public transaction ledger for the development network.
 var regtestGenesisBlock = wire.MsgBlock{
 	Header: wire.BlockHeader{
-		Version:              1,
+		Version:              0x10000000,
 		ParentHashes:         []*daghash.Hash{},
 		HashMerkleRoot:       &regtestGenesisMerkleRoot,
 		AcceptedIDMerkleRoot: &daghash.Hash{},
 		UTXOCommitment:       &daghash.ZeroHash,
-		Timestamp:            time.Unix(0x5e15e2d8, 0),
+		Timestamp:            time.Unix(0x5edf4ce0, 0),
 		Bits:                 0x1e7fffff,
-		Nonce:                0x15a6,
+		Nonce:                0x4a78,
 	},
 	Transactions: []*wire.MsgTx{regtestGenesisCoinbaseTx},
 }
 
-var simnetGenesisTxIns = []*wire.TxIn{
-	{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID{},
-			Index: math.MaxUint32,
-		},
-		SignatureScript: []byte{
-			0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48,
-			0x2f, 0x62, 0x74, 0x63, 0x64, 0x2f,
-		},
-		Sequence: math.MaxUint64,
-	},
-}
 var simnetGenesisTxOuts = []*wire.TxOut{}
 
 var simnetGenesisTxPayload = []byte{
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Blue score
 	0x17,                                           // Varint
 	0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, // OP-TRUE p2sh
 	0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7,
@@ -219,96 +170,84 @@ var simnetGenesisTxPayload = []byte{
 }
 
 // simnetGenesisCoinbaseTx is the coinbase transaction for the simnet genesis block.
-var simnetGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, simnetGenesisTxIns, simnetGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, simnetGenesisTxPayload)
+var simnetGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, []*wire.TxIn{}, simnetGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, simnetGenesisTxPayload)
 
 // simnetGenesisHash is the hash of the first block in the block DAG for
 // the simnet (genesis block).
 var simnetGenesisHash = daghash.Hash{
-	0xff, 0x69, 0xcc, 0x45, 0x45, 0x74, 0x5b, 0xf9,
-	0xd5, 0x4e, 0x43, 0x56, 0x4f, 0x1b, 0xdf, 0x31,
-	0x09, 0xb7, 0x76, 0xaa, 0x2a, 0x33, 0x35, 0xc9,
-	0xa1, 0x80, 0xe0, 0x92, 0xbb, 0xae, 0xcd, 0x49,
+	0x34, 0x43, 0xed, 0xdc, 0xab, 0x0c, 0x39, 0x53,
+	0xa2, 0xc5, 0x6d, 0x12, 0x4b, 0xc2, 0x41, 0x1c,
+	0x1a, 0x05, 0x24, 0xb4, 0xff, 0xeb, 0xe8, 0xbd,
+	0xee, 0x6e, 0x9a, 0x77, 0xc7, 0xbb, 0x70, 0x7d,
 }
 
 // simnetGenesisMerkleRoot is the hash of the first transaction in the genesis block
 // for the devopment network.
 var simnetGenesisMerkleRoot = daghash.Hash{
-	0xb0, 0x1c, 0x3b, 0x9e, 0x0d, 0x9a, 0xc0, 0x80,
-	0x0a, 0x08, 0x42, 0x50, 0x02, 0xa3, 0xea, 0xdb,
-	0xed, 0xc8, 0xd0, 0xad, 0x35, 0x03, 0xd8, 0x0e,
-	0x11, 0x3c, 0x7b, 0xb2, 0xb5, 0x20, 0xe5, 0x84,
+	0x47, 0x52, 0xc7, 0x23, 0x70, 0x4d, 0x89, 0x17,
+	0xbd, 0x44, 0x26, 0xfa, 0x82, 0x7e, 0x1b, 0xa9,
+	0xc6, 0x46, 0x1a, 0x37, 0x5a, 0x73, 0x88, 0x09,
+	0xe8, 0x17, 0xff, 0xb1, 0xdb, 0x1a, 0xb3, 0x3f,
 }
 
 // simnetGenesisBlock defines the genesis block of the block DAG which serves as the
 // public transaction ledger for the development network.
 var simnetGenesisBlock = wire.MsgBlock{
 	Header: wire.BlockHeader{
-		Version:              1,
+		Version:              0x10000000,
 		ParentHashes:         []*daghash.Hash{},
 		HashMerkleRoot:       &simnetGenesisMerkleRoot,
 		AcceptedIDMerkleRoot: &daghash.Hash{},
 		UTXOCommitment:       &daghash.ZeroHash,
-		Timestamp:            time.Unix(0x5e15d31c, 0),
+		Timestamp:            time.Unix(0x5ede5261, 0),
 		Bits:                 0x207fffff,
-		Nonce:                0x3,
+		Nonce:                0x2,
 	},
 	Transactions: []*wire.MsgTx{simnetGenesisCoinbaseTx},
 }
 
-var testnetGenesisTxIns = []*wire.TxIn{
-	{
-		PreviousOutpoint: wire.Outpoint{
-			TxID:  daghash.TxID{},
-			Index: math.MaxUint32,
-		},
-		SignatureScript: []byte{
-			0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48,
-			0x2f, 0x62, 0x74, 0x63, 0x64, 0x2f,
-		},
-		Sequence: math.MaxUint64,
-	},
-}
 var testnetGenesisTxOuts = []*wire.TxOut{}
 
 var testnetGenesisTxPayload = []byte{
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Blue score
 	0x01,                                                                         // Varint
 	0x00,                                                                         // OP-FALSE
 	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x74, 0x65, 0x73, 0x74, 0x6e, 0x65, 0x74, // kaspa-testnet
 }
 
 // testnetGenesisCoinbaseTx is the coinbase transaction for the testnet genesis block.
-var testnetGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, testnetGenesisTxIns, testnetGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, testnetGenesisTxPayload)
+var testnetGenesisCoinbaseTx = wire.NewSubnetworkMsgTx(1, []*wire.TxIn{}, testnetGenesisTxOuts, subnetworkid.SubnetworkIDCoinbase, 0, testnetGenesisTxPayload)
 
 // testnetGenesisHash is the hash of the first block in the block DAG for the test
 // network (genesis block).
 var testnetGenesisHash = daghash.Hash{
-	0x22, 0x15, 0x34, 0xa9, 0xff, 0x10, 0xdd, 0x47,
-	0xcd, 0x21, 0x11, 0x25, 0xc5, 0x6d, 0x85, 0x9a,
-	0x97, 0xc8, 0x63, 0x63, 0x79, 0x40, 0x80, 0x04,
-	0x74, 0xe6, 0x29, 0x7b, 0xbc, 0x08, 0x00, 0x00,
+	0xFC, 0x21, 0x64, 0x1A, 0xB5, 0x59, 0x61, 0x8E,
+	0xF3, 0x9A, 0x95, 0xF1, 0xDA, 0x07, 0x79, 0xBD,
+	0x11, 0x2F, 0x90, 0xFC, 0x8B, 0x33, 0x14, 0x8A,
+	0x90, 0x6B, 0x76, 0x08, 0x4B, 0x52, 0x00, 0x00,
 }
 
 // testnetGenesisMerkleRoot is the hash of the first transaction in the genesis block
 // for testnet.
 var testnetGenesisMerkleRoot = daghash.Hash{
-	0x88, 0x05, 0xd0, 0xe7, 0x8f, 0x41, 0x77, 0x39,
-	0x2c, 0xb6, 0xbb, 0xb4, 0x19, 0xa8, 0x48, 0x4a,
-	0xdf, 0x77, 0xb0, 0x82, 0xd6, 0x70, 0xd8, 0x24,
-	0x6a, 0x36, 0x05, 0xaa, 0xbd, 0x7a, 0xd1, 0x62,
+	0xA0, 0xA1, 0x3D, 0xFD, 0x86, 0x41, 0x35, 0xC8,
+	0xBD, 0xBB, 0xE6, 0x37, 0x35, 0xBB, 0x4C, 0x51,
+	0x11, 0x7B, 0x26, 0x90, 0x15, 0x64, 0x0F, 0x42,
+	0x6D, 0x2B, 0x6F, 0x37, 0x4D, 0xC1, 0xA9, 0x72,
 }
 
 // testnetGenesisBlock defines the genesis block of the block DAG which serves as the
 // public transaction ledger for testnet.
 var testnetGenesisBlock = wire.MsgBlock{
 	Header: wire.BlockHeader{
-		Version:              1,
+		Version:              0x10000000,
 		ParentHashes:         []*daghash.Hash{},
 		HashMerkleRoot:       &testnetGenesisMerkleRoot,
 		AcceptedIDMerkleRoot: &daghash.ZeroHash,
 		UTXOCommitment:       &daghash.ZeroHash,
-		Timestamp:            time.Unix(0x5e15adfe, 0),
+		Timestamp:            time.Unix(0x5efc2128, 0),
 		Bits:                 0x1e7fffff,
-		Nonce:                0x20a1,
+		Nonce:                0x1124,
 	},
 	Transactions: []*wire.MsgTx{testnetGenesisCoinbaseTx},
 }

dagconfig/genesis_test.go

@@ -148,187 +148,101 @@ func TestDevnetGenesisBlock(t *testing.T) {
 // genesisBlockBytes are the wire encoded bytes for the genesis block of the
 // main network as of protocol version 1.
 var genesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x72, 0x10, 0x35, 0x85, 0xdd, 0xac, 0x82, 0x5c, 0x49, 0x13, 0x9f,
-	0xc0, 0x0e, 0x37, 0xc0, 0x45, 0x71, 0xdf, 0xd9, 0xf6, 0x36, 0xdf, 0x4c, 0x42, 0x72, 0x7b, 0x9e,
-	0x86, 0xdd, 0x37, 0xd2, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0xca, 0x85, 0x56, 0x27, 0xc7, 0x6a, 0xb5, 0x7a, 0x26, 0x1d, 0x63,
+	0x62, 0x1e, 0x57, 0x21, 0xf0, 0x5e, 0x60, 0x1f, 0xee, 0x1d, 0x4d, 0xaa, 0x53, 0x72, 0xe1, 0x16,
+	0xda, 0x4b, 0xb3, 0xd8, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xc4, 0xda, 0x5c, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x20, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x4c, 0xdf, 0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
+	0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f, 0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2,
-	0xea, 0x82, 0x4e, 0xb8, 0x87, 0x42, 0xd0, 0x6d, 0x1f, 0x8d, 0xc3, 0xad, 0x9f, 0x43, 0x9e, 0xed,
-	0x6f, 0x43, 0x3c, 0x02, 0x71, 0x71, 0x69, 0xfb, 0xbc, 0x91, 0x44, 0xac, 0xf1, 0x93, 0xd3, 0x18,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
+	0x00, 0x00, 0x00, 0x00, 0xc4, 0x41, 0xe6, 0x78, 0x1d, 0xf7, 0xb3, 0x39, 0x66, 0x4d, 0x1a, 0x03,
+	0x97, 0x63, 0xc7, 0x2c, 0xfc, 0x70, 0xd7, 0x75, 0xb6, 0xd9, 0xfc, 0x1a, 0x96, 0xf0, 0xac, 0x07,
+	0xef, 0xfa, 0x26, 0x38, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xa9, 0x14,
+	0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7, 0x7e, 0xba,
+	0x30, 0xcd, 0x5a, 0x4b, 0x87,
 }
 
 // regtestGenesisBlockBytes are the wire encoded bytes for the genesis block of
 // the regression test network as of protocol version 1.
 var regtestGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x3a, 0x9f, 0x62,
-	0xc9, 0x2b, 0x16, 0x17, 0xb3, 0x41, 0x6d, 0x9e,
-	0x2d, 0x87, 0x93, 0xfd, 0x72, 0x77, 0x4d, 0x1d,
-	0x6f, 0x6d, 0x38, 0x5b, 0xf1, 0x24, 0x1b, 0xdc,
-	0x96, 0xce, 0xbf, 0xa1, 0x09, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0xe2, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x1e, 0xa6, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed,
-	0x32, 0xec, 0xb4, 0xf8, 0x3c, 0x7a, 0x32, 0x0f,
-	0xd2, 0xe5, 0x24, 0x77, 0x89, 0x43, 0x3a, 0x78,
-	0x0a, 0xda, 0x68, 0x2d, 0xf6, 0xaa, 0xb1, 0x19,
-	0xdd, 0xd8, 0x97, 0x15, 0x4b, 0xcb, 0x42, 0x25,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5,
-	0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x72, 0x65,
-	0x67, 0x74, 0x65, 0x73, 0x74,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x1e, 0x08, 0xae, 0x1f, 0x43, 0xf5, 0xfc, 0x24, 0xe6, 0xec, 0x54,
+	0x5b, 0xf7, 0x52, 0x99, 0xe4, 0xcc, 0x4c, 0xa0, 0x79, 0x41, 0xfc, 0xbe, 0x76, 0x72, 0x4c, 0x7e,
+	0xd8, 0xa3, 0x43, 0x65, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x4c, 0xdf, 0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
+	0x1e, 0x78, 0x4a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0xd4, 0xc4, 0x87, 0x77, 0xf2, 0xe7, 0x5d, 0xf7, 0xff, 0x2d, 0xbb, 0xb6,
+	0x2a, 0x73, 0x1f, 0x54, 0x36, 0x33, 0xa7, 0x99, 0xad, 0xb1, 0x09, 0x65, 0xc0, 0xf0, 0xf4, 0x53,
+	0xba, 0xfb, 0x88, 0xae, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xa9, 0x14,
+	0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7, 0x7e, 0xba,
+	0x30, 0xcd, 0x5a, 0x4b, 0x87, 0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x72, 0x65, 0x67, 0x74, 0x65,
+	0x73, 0x74,
 }
 
 // testnetGenesisBlockBytes are the wire encoded bytes for the genesis block of
 // the test network as of protocol version 1.
 var testnetGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x88, 0x05, 0xd0,
-	0xe7, 0x8f, 0x41, 0x77, 0x39, 0x2c, 0xb6, 0xbb,
-	0xb4, 0x19, 0xa8, 0x48, 0x4a, 0xdf, 0x77, 0xb0,
-	0x82, 0xd6, 0x70, 0xd8, 0x24, 0x6a, 0x36, 0x05,
-	0xaa, 0xbd, 0x7a, 0xd1, 0x62, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xad, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x1e, 0xa1, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc,
-	0x72, 0xe6, 0x7e, 0x37, 0xa1, 0x34, 0x89, 0x23,
-	0x24, 0xaf, 0xae, 0x99, 0x1f, 0x89, 0x09, 0x41,
-	0x1a, 0x4d, 0x58, 0xfe, 0x5a, 0x04, 0xb0, 0x3e,
-	0xeb, 0x1b, 0x5b, 0xb8, 0x65, 0xa8, 0x65, 0x0f,
-	0x01, 0x00, 0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d,
-	0x74, 0x65, 0x73, 0x74, 0x6e, 0x65, 0x74,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0xa0, 0xa1, 0x3d, 0xfd, 0x86, 0x41, 0x35, 0xc8, 0xbd, 0xbb, 0xe6,
+	0x37, 0x35, 0xbb, 0x4c, 0x51, 0x11, 0x7b, 0x26, 0x90, 0x15, 0x64, 0x0f, 0x42, 0x6d, 0x2b, 0x6f,
+	0x37, 0x4d, 0xc1, 0xa9, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x21, 0xfc, 0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
+	0x1e, 0x24, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0xf5, 0x41, 0x4c, 0xf4, 0xa8, 0xa2, 0x8c, 0x47, 0x9d, 0xb5, 0x75, 0x5e,
+	0x0f, 0x38, 0xd3, 0x27, 0x82, 0xc6, 0xd1, 0x89, 0xc1, 0x60, 0x49, 0xd9, 0x99, 0xc6, 0x2e, 0xbf,
+	0x4b, 0x5a, 0x3a, 0xcf, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x6b,
+	0x61, 0x73, 0x70, 0x61, 0x2d, 0x74, 0x65, 0x73, 0x74, 0x6e, 0x65, 0x74,
 }
 
 // simnetGenesisBlockBytes are the wire encoded bytes for the genesis block of
 // the simulation test network as of protocol version 1.
 var simnetGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0xb0, 0x1c, 0x3b,
-	0x9e, 0x0d, 0x9a, 0xc0, 0x80, 0x0a, 0x08, 0x42,
-	0x50, 0x02, 0xa3, 0xea, 0xdb, 0xed, 0xc8, 0xd0,
-	0xad, 0x35, 0x03, 0xd8, 0x0e, 0x11, 0x3c, 0x7b,
-	0xb2, 0xb5, 0x20, 0xe5, 0x84, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xd3, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89,
-	0x48, 0xd3, 0x23, 0x9c, 0xf9, 0x88, 0x2b, 0x63,
-	0xc7, 0x33, 0x0f, 0xa3, 0x64, 0xf2, 0xdb, 0x39,
-	0x73, 0x5f, 0x2b, 0xa8, 0xd5, 0x7b, 0x5c, 0x31,
-	0x68, 0xc9, 0x63, 0x37, 0x5c, 0xe7, 0x41, 0x24,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5,
-	0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x73, 0x69,
-	0x6d, 0x6e, 0x65, 0x74,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x47, 0x52, 0xc7, 0x23, 0x70, 0x4d, 0x89, 0x17, 0xbd, 0x44, 0x26,
+	0xfa, 0x82, 0x7e, 0x1b, 0xa9, 0xc6, 0x46, 0x1a, 0x37, 0x5a, 0x73, 0x88, 0x09, 0xe8, 0x17, 0xff,
+	0xb1, 0xdb, 0x1a, 0xb3, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x52, 0xde, 0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
+	0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0xd9, 0x39, 0x5f, 0x40, 0x2a, 0x5e, 0x24, 0x09, 0x1b, 0x9a, 0x4b, 0xdf,
+	0x7f, 0x0c, 0x03, 0x7f, 0xf1, 0xd2, 0x48, 0x8c, 0x26, 0xb0, 0xa3, 0x74, 0x60, 0xd9, 0x48, 0x18,
+	0x2b, 0x33, 0x22, 0x64, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xa9, 0x14,
+	0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7, 0x7e, 0xba,
+	0x30, 0xcd, 0x5a, 0x4b, 0x87, 0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x73, 0x69, 0x6d, 0x6e, 0x65,
+	0x74,
 }
 
 // devnetGenesisBlockBytes are the wire encoded bytes for the genesis block of
 // the development network as of protocol version 1.
 var devnetGenesisBlockBytes = []byte{
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x16, 0x0a, 0xc6,
-	0x8b, 0x77, 0x08, 0xf4, 0x96, 0xa3, 0x07, 0x05,
-	0xbc, 0x92, 0xda, 0xee, 0x73, 0x26, 0x5e, 0xd0,
-	0x85, 0x78, 0xa2, 0x5d, 0x02, 0x49, 0x8a, 0x2a,
-	0x22, 0xef, 0x41, 0xc9, 0xc3, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0xe7, 0x15,
-	0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
-	0x1e, 0xac, 0x82, 0x02, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
-	0xff, 0xff, 0xff, 0x0e, 0x00, 0x00, 0x0b, 0x2f,
-	0x50, 0x32, 0x53, 0x48, 0x2f, 0x62, 0x74, 0x63,
-	0x64, 0x2f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11,
-	0xc7, 0x0c, 0x02, 0x9e, 0xb2, 0x2e, 0xb3, 0xad,
-	0x24, 0x10, 0xfe, 0x2c, 0xdb, 0x8e, 0x1d, 0xde,
-	0x81, 0x5b, 0xbb, 0x42, 0xfe, 0xb4, 0x93, 0xd6,
-	0xe3, 0xbe, 0x86, 0x02, 0xe6, 0x3a, 0x65, 0x24,
-	0x17, 0xa9, 0x14, 0xda, 0x17, 0x45, 0xe9, 0xb5,
-	0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71,
-	0xc7, 0x7e, 0xba, 0x30, 0xcd, 0x5a, 0x4b, 0x87,
-	0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x64, 0x65,
-	0x76, 0x6e, 0x65, 0x74,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x68, 0x60, 0xe7, 0x77, 0x47, 0x74, 0x7f, 0xd5, 0x55, 0x58, 0x8a,
+	0xb5, 0xc2, 0x29, 0x0c, 0xa6, 0x65, 0x44, 0xb4, 0x4f, 0xfa, 0x31, 0x7a, 0xfa, 0x55, 0xe0, 0xcf,
+	0xac, 0x9c, 0x86, 0x30, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x4c, 0xdf, 0x5e, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f,
+	0x1e, 0xed, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x1d, 0x1c, 0x05, 0x21, 0x10, 0x45, 0x61, 0xed, 0xc6, 0x0b, 0xdc, 0x85,
+	0xc0, 0x0a, 0x70, 0x2b, 0x15, 0xd5, 0x3c, 0x07, 0xb0, 0x54, 0x4f, 0x5b, 0x1a, 0x04, 0xcd, 0x49,
+	0xf1, 0x7b, 0xd6, 0x27, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xa9, 0x14,
+	0xda, 0x17, 0x45, 0xe9, 0xb5, 0x49, 0xbd, 0x0b, 0xfa, 0x1a, 0x56, 0x99, 0x71, 0xc7, 0x7e, 0xba,
+	0x30, 0xcd, 0x5a, 0x4b, 0x87, 0x6b, 0x61, 0x73, 0x70, 0x61, 0x2d, 0x64, 0x65, 0x76, 0x6e, 0x65,
+	0x74,
 }

dagconfig/params.go

@@ -13,7 +13,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/util/hdkeychain"
 
 	"github.com/kaspanet/kaspad/util/daghash"
 	"github.com/kaspanet/kaspad/wire"
@@ -49,7 +48,7 @@ var (
 )
 
 const (
-	ghostdagK                      = 10
+	ghostdagK                      = 15
 	difficultyAdjustmentWindowSize = 2640
 	timestampDeviationTolerance    = 132
 	finalityDuration               = 24 * time.Hour
@@ -178,12 +177,8 @@ type Params struct {
 	// Address encoding magics
 	PrivateKeyID byte // First byte of a WIF private key
 
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair hdkeychain.HDKeyIDPair
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType uint32
+	// EnableNonNativeSubnetworks enables non-native/coinbase transactions
+	EnableNonNativeSubnetworks bool
 }
 
 // NormalizeRPCServerAddress returns addr with the current network default
@@ -239,12 +234,8 @@ var MainnetParams = Params{
 	// Address encoding magics
 	PrivateKeyID: 0x80, // starts with 5 (uncompressed) or K (compressed)
 
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair: hdkeychain.HDKeyPairMainnet,
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType: 0,
+	// EnableNonNativeSubnetworks enables non-native/coinbase transactions
+	EnableNonNativeSubnetworks: false,
 }
 
 // RegressionNetParams defines the network parameters for the regression test
@@ -296,12 +287,8 @@ var RegressionNetParams = Params{
 	// Address encoding magics
 	PrivateKeyID: 0xef, // starts with 9 (uncompressed) or c (compressed)
 
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair: hdkeychain.HDKeyPairRegressionNet,
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType: 1,
+	// EnableNonNativeSubnetworks enables non-native/coinbase transactions
+	EnableNonNativeSubnetworks: false,
 }
 
 // TestnetParams defines the network parameters for the test Kaspa network.
@@ -351,12 +338,8 @@ var TestnetParams = Params{
 	// Address encoding magics
 	PrivateKeyID: 0xef, // starts with 9 (uncompressed) or c (compressed)
 
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair: hdkeychain.HDKeyPairTestnet,
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType: 1,
+	// EnableNonNativeSubnetworks enables non-native/coinbase transactions
+	EnableNonNativeSubnetworks: false,
 }
 
 // SimnetParams defines the network parameters for the simulation test Kaspa
@@ -410,12 +393,8 @@ var SimnetParams = Params{
 	// Human-readable part for Bech32 encoded addresses
 	Prefix: util.Bech32PrefixKaspaSim,
 
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair: hdkeychain.HDKeyPairSimnet,
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType: 115, // ASCII for s
+	// EnableNonNativeSubnetworks enables non-native/coinbase transactions
+	EnableNonNativeSubnetworks: false,
 }
 
 // DevnetParams defines the network parameters for the development Kaspa network.
@@ -465,12 +444,8 @@ var DevnetParams = Params{
 	// Address encoding magics
 	PrivateKeyID: 0xef, // starts with 9 (uncompressed) or c (compressed)
 
-	// BIP32 hierarchical deterministic extended key magics
-	HDKeyIDPair: hdkeychain.HDKeyPairDevnet,
-
-	// BIP44 coin type used in the hierarchical deterministic path for
-	// address generation.
-	HDCoinType: 1,
+	// EnableNonNativeSubnetworks enables non-native/coinbase transactions
+	EnableNonNativeSubnetworks: false,
 }
 
 var (

dagconfig/register_test.go

@@ -1,9 +1,6 @@
 package dagconfig_test
 
 import (
-	"bytes"
-	"github.com/kaspanet/kaspad/util/hdkeychain"
-	"reflect"
 	"testing"
 
 	. "github.com/kaspanet/kaspad/dagconfig"
@@ -15,10 +12,6 @@ import (
 var mockNetParams = Params{
 	Name: "mocknet",
 	Net:  1<<32 - 1,
-	HDKeyIDPair: hdkeychain.HDKeyIDPair{
-		PrivateKeyID: [4]byte{0x01, 0x02, 0x03, 0x04},
-		PublicKeyID:  [4]byte{0x05, 0x06, 0x07, 0x08},
-	},
 }
 
 func TestRegister(t *testing.T) {
@@ -27,16 +20,10 @@ func TestRegister(t *testing.T) {
 		params *Params
 		err    error
 	}
-	type hdTest struct {
-		priv []byte
-		want []byte
-		err  error
-	}
 
 	tests := []struct {
 		name     string
 		register []registerTest
-		hdMagics []hdTest
 	}{
 		{
 			name: "default networks",
@@ -62,40 +49,6 @@ func TestRegister(t *testing.T) {
 					err:    ErrDuplicateNet,
 				},
 			},
-			hdMagics: []hdTest{
-				{
-					priv: MainnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: MainnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: TestnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: TestnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: RegressionNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: RegressionNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: SimnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: SimnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: mockNetParams.HDKeyIDPair.PrivateKeyID[:],
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-				{
-					priv: []byte{0xff, 0xff, 0xff, 0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-				{
-					priv: []byte{0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-			},
 		},
 		{
 			name: "register mocknet",
@@ -106,13 +59,6 @@ func TestRegister(t *testing.T) {
 					err:    nil,
 				},
 			},
-			hdMagics: []hdTest{
-				{
-					priv: mockNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: mockNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-			},
 		},
 		{
 			name: "more duplicates",
@@ -143,41 +89,6 @@ func TestRegister(t *testing.T) {
 					err:    ErrDuplicateNet,
 				},
 			},
-			hdMagics: []hdTest{
-				{
-					priv: MainnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: MainnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: TestnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: TestnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: RegressionNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: RegressionNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: SimnetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: SimnetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: mockNetParams.HDKeyIDPair.PrivateKeyID[:],
-					want: mockNetParams.HDKeyIDPair.PublicKeyID[:],
-					err:  nil,
-				},
-				{
-					priv: []byte{0xff, 0xff, 0xff, 0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-				{
-					priv: []byte{0xff},
-					err:  hdkeychain.ErrUnknownHDKeyID,
-				},
-			},
 		},
 	}
 
@@ -185,25 +96,10 @@ func TestRegister(t *testing.T) {
 		for _, regtest := range test.register {
 			err := Register(regtest.params)
 
-			// HDKeyIDPairs must be registered separately
-			hdkeychain.RegisterHDKeyIDPair(regtest.params.HDKeyIDPair)
-
 			if err != regtest.err {
 				t.Errorf("%s:%s: Registered network with unexpected error: got %v expected %v",
 					test.name, regtest.name, err, regtest.err)
 			}
 		}
-		for i, magTest := range test.hdMagics {
-			pubKey, err := hdkeychain.HDPrivateKeyToPublicKeyID(magTest.priv[:])
-			if !reflect.DeepEqual(err, magTest.err) {
-				t.Errorf("%s: HD magic %d mismatched error: got %v expected %v ",
-					test.name, i, err, magTest.err)
-				continue
-			}
-			if magTest.err == nil && !bytes.Equal(pubKey, magTest.want[:]) {
-				t.Errorf("%s: HD magic %d private and public mismatch: got %v expected %v ",
-					test.name, i, pubKey, magTest.want[:])
-			}
-		}
 	}
 }

database/README.md

@@ -4,29 +4,35 @@ database
 [![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](https://choosealicense.com/licenses/isc/)
 [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/kaspanet/kaspad/database)
 
-Package database provides a block and metadata storage database.
+Package database provides a database for kaspad.
 
-Please note that this package is intended to enable kaspad to support different
-database backends and is not something that a client can directly access as only
-one entity can have the database open at a time (for most database backends),
-and that entity will be kaspad.
+Overview
+--------
+This package provides a database layer to store and retrieve data in a simple
+and efficient manner.
 
-When a client wants programmatic access to the data provided by kaspad, they'll
-likely want to use the [rpcclient](https://github.com/kaspanet/kaspad/tree/master/rpcclient)
-package which makes use of the [JSON-RPC API](https://github.com/kaspanet/kaspad/tree/master/docs/json_rpc_api.md).
+The current backend is ffldb, which makes use of leveldb, flat files, and strict
+checksums in key areas to ensure data integrity.
 
-The default backend, ffldb, has a strong focus on speed, efficiency, and
-robustness. It makes use of leveldb for the metadata, flat files for block
-storage, and strict checksums in key areas to ensure data integrity.
+Implementors of additional backends are required to implement the following interfaces:
 
-## Feature Overview
+DataAccessor
+------------
+This defines the common interface by which data gets accessed in a generic kaspad
+database. Both the Database and the Transaction interfaces (see below) implement it.
 
-- Key/value metadata store
-- Kaspa block storage
-- Efficient retrieval of block headers and regions (transactions, scripts, etc)
-- Read-only and read-write transactions with both manual and managed modes
-- Nested buckets
-- Iteration support including cursors with seek capability
-- Supports registration of backend databases
-- Comprehensive test coverage
+Database
+--------
+This defines the interface of a database that can begin transactions and close itself.
 
+Transaction
+-----------
+This defines the interface of a generic kaspad database transaction.
+
+Note: Transactions provide data consistency over the state of the database as it was
+when the transaction started. There is NO guarantee that if one puts data into the
+transaction then it will be available to get within the same transaction.
+
+Cursor
+------
+This iterates over database entries given some bucket.

database/cmd/dbtool/fetchblock.go

@@ -1,62 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"encoding/hex"
-	"github.com/pkg/errors"
-	"time"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// fetchBlockCmd defines the configuration options for the fetchblock command.
-type fetchBlockCmd struct{}
-
-var (
-	// fetchBlockCfg defines the configuration options for the command.
-	fetchBlockCfg = fetchBlockCmd{}
-)
-
-// Execute is the main entry point for the command. It's invoked by the parser.
-func (cmd *fetchBlockCmd) Execute(args []string) error {
-	// Setup the global config options and ensure they are valid.
-	if err := setupGlobalConfig(); err != nil {
-		return err
-	}
-
-	if len(args) < 1 {
-		return errors.New("required block hash parameter not specified")
-	}
-	blockHash, err := daghash.NewHashFromStr(args[0])
-	if err != nil {
-		return err
-	}
-
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	return db.View(func(dbTx database.Tx) error {
-		log.Infof("Fetching block %s", blockHash)
-		startTime := time.Now()
-		blockBytes, err := dbTx.FetchBlock(blockHash)
-		if err != nil {
-			return err
-		}
-		log.Infof("Loaded block in %s", time.Since(startTime))
-		log.Infof("Block Hex: %s", hex.EncodeToString(blockBytes))
-		return nil
-	})
-}
-
-// Usage overrides the usage display for the command.
-func (cmd *fetchBlockCmd) Usage() string {
-	return "<block-hash>"
-}

database/cmd/dbtool/fetchblockregion.go

@@ -1,90 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"encoding/hex"
-	"github.com/pkg/errors"
-	"strconv"
-	"time"
-
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/util/daghash"
-)
-
-// blockRegionCmd defines the configuration options for the fetchblockregion
-// command.
-type blockRegionCmd struct{}
-
-var (
-	// blockRegionCfg defines the configuration options for the command.
-	blockRegionCfg = blockRegionCmd{}
-)
-
-// Execute is the main entry point for the command. It's invoked by the parser.
-func (cmd *blockRegionCmd) Execute(args []string) error {
-	// Setup the global config options and ensure they are valid.
-	if err := setupGlobalConfig(); err != nil {
-		return err
-	}
-
-	// Ensure expected arguments.
-	if len(args) < 1 {
-		return errors.New("required block hash parameter not specified")
-	}
-	if len(args) < 2 {
-		return errors.New("required start offset parameter not " +
-			"specified")
-	}
-	if len(args) < 3 {
-		return errors.New("required region length parameter not " +
-			"specified")
-	}
-
-	// Parse arguments.
-	blockHash, err := daghash.NewHashFromStr(args[0])
-	if err != nil {
-		return err
-	}
-	startOffset, err := strconv.ParseUint(args[1], 10, 32)
-	if err != nil {
-		return err
-	}
-	regionLen, err := strconv.ParseUint(args[2], 10, 32)
-	if err != nil {
-		return err
-	}
-
-	// Load the block database.
-	db, err := loadBlockDB()
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-
-	return db.View(func(dbTx database.Tx) error {
-		log.Infof("Fetching block region %s<%d:%d>", blockHash,
-			startOffset, startOffset+regionLen-1)
-		region := database.BlockRegion{
-			Hash:   blockHash,
-			Offset: uint32(startOffset),
-			Len:    uint32(regionLen),
-		}
-		startTime := time.Now()
-		regionBytes, err := dbTx.FetchBlockRegion(&region)
-		if err != nil {
-			return err
-		}
-		log.Infof("Loaded block region in %s", time.Since(startTime))
-		log.Infof("Double Hash: %s", daghash.DoubleHashH(regionBytes))
-		log.Infof("Region Hex: %s", hex.EncodeToString(regionBytes))
-		return nil
-	})
-}
-
-// Usage overrides the usage display for the command.
-func (cmd *blockRegionCmd) Usage() string {
-	return "<block-hash> <start-offset> <length-of-region>"
-}

database/cmd/dbtool/globalconfig.go

@@ -1,111 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
-)
-
-var (
-	kaspadHomeDir   = util.AppDataDir("kaspad", false)
-	knownDbTypes    = database.SupportedDrivers()
-	activeNetParams = &dagconfig.MainnetParams
-
-	// Default global config.
-	cfg = &config{
-		DataDir: filepath.Join(kaspadHomeDir, "data"),
-		DbType:  "ffldb",
-	}
-)
-
-// config defines the global configuration options.
-type config struct {
-	DataDir        string `short:"b" long:"datadir" description:"Location of the kaspad data directory"`
-	DbType         string `long:"dbtype" description:"Database backend to use for the Block DAG"`
-	Testnet        bool   `long:"testnet" description:"Use the test network"`
-	RegressionTest bool   `long:"regtest" description:"Use the regression test network"`
-	Simnet         bool   `long:"simnet" description:"Use the simulation test network"`
-	Devnet         bool   `long:"devnet" description:"Use the development test network"`
-}
-
-// fileExists reports whether the named file or directory exists.
-func fileExists(name string) bool {
-	if _, err := os.Stat(name); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
-
-// validDbType returns whether or not dbType is a supported database type.
-func validDbType(dbType string) bool {
-	for _, knownType := range knownDbTypes {
-		if dbType == knownType {
-			return true
-		}
-	}
-
-	return false
-}
-
-// setupGlobalConfig examine the global configuration options for any conditions
-// which are invalid as well as performs any addition setup necessary after the
-// initial parse.
-func setupGlobalConfig() error {
-	// Multiple networks can't be selected simultaneously.
-	// Count number of network flags passed; assign active network params
-	// while we're at it
-	numNets := 0
-	if cfg.Testnet {
-		numNets++
-		activeNetParams = &dagconfig.TestnetParams
-	}
-	if cfg.RegressionTest {
-		numNets++
-		activeNetParams = &dagconfig.RegressionNetParams
-	}
-	if cfg.Simnet {
-		numNets++
-		activeNetParams = &dagconfig.SimnetParams
-	}
-	if cfg.Devnet {
-		numNets++
-		activeNetParams = &dagconfig.DevnetParams
-	}
-	if numNets > 1 {
-		return errors.New("The testnet, regtest, simnet and devnet params " +
-			"can't be used together -- choose one of the four")
-	}
-
-	if numNets == 0 {
-		return errors.New("Mainnet has not launched yet, use --testnet to run in testnet mode")
-	}
-
-	// Validate database type.
-	if !validDbType(cfg.DbType) {
-		str := "The specified database type [%s] is invalid -- " +
-			"supported types: %s"
-		return errors.Errorf(str, cfg.DbType, strings.Join(knownDbTypes, ", "))
-	}
-
-	// Append the network type to the data directory so it is "namespaced"
-	// per network. In addition to the block database, there are other
-	// pieces of data that are saved to disk such as address manager state.
-	// All data is specific to a network, so namespacing the data directory
-	// means each individual piece of serialized data does not have to
-	// worry about changing names per network and such.
-	cfg.DataDir = filepath.Join(cfg.DataDir, activeNetParams.Name)
-
-	return nil
-}

database/cmd/dbtool/main.go

@@ -1,113 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"github.com/kaspanet/kaspad/util/panics"
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-
-	"github.com/jessevdk/go-flags"
-	"github.com/kaspanet/kaspad/database"
-	"github.com/kaspanet/kaspad/logger"
-	"github.com/kaspanet/kaspad/logs"
-)
-
-const (
-	// blockDbNamePrefix is the prefix for the kaspad block database.
-	blockDbNamePrefix = "blocks"
-)
-
-var (
-	log             *logs.Logger
-	spawn           func(func())
-	shutdownChannel = make(chan error)
-)
-
-// loadBlockDB opens the block database and returns a handle to it.
-func loadBlockDB() (database.DB, error) {
-	// The database name is based on the database type.
-	dbName := blockDbNamePrefix + "_" + cfg.DbType
-	dbPath := filepath.Join(cfg.DataDir, dbName)
-
-	log.Infof("Loading block database from '%s'", dbPath)
-	db, err := database.Open(cfg.DbType, dbPath, activeNetParams.Net)
-	if err != nil {
-		// Return the error if it's not because the database doesn't
-		// exist.
-		var dbErr database.Error
-		if ok := errors.As(err, &dbErr); !ok || dbErr.ErrorCode !=
-			database.ErrDbDoesNotExist {
-
-			return nil, err
-		}
-
-		// Create the db if it does not exist.
-		err = os.MkdirAll(cfg.DataDir, 0700)
-		if err != nil {
-			return nil, err
-		}
-		db, err = database.Create(cfg.DbType, dbPath, activeNetParams.Net)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	log.Info("Block database loaded")
-	return db, nil
-}
-
-// realMain is the real main function for the utility. It is necessary to work
-// around the fact that deferred functions do not run when os.Exit() is called.
-func realMain() error {
-	// Setup logging.
-	backendLogger := logs.NewBackend()
-	defer os.Stdout.Sync()
-	log = backendLogger.Logger("MAIN")
-	spawn = panics.GoroutineWrapperFunc(log)
-	dbLog, _ := logger.Get(logger.SubsystemTags.KSDB)
-	dbLog.SetLevel(logs.LevelDebug)
-
-	// Setup the parser options and commands.
-	appName := filepath.Base(os.Args[0])
-	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
-	parserFlags := flags.Options(flags.HelpFlag | flags.PassDoubleDash)
-	parser := flags.NewNamedParser(appName, parserFlags)
-	parser.AddGroup("Global Options", "", cfg)
-	parser.AddCommand("fetchblock",
-		"Fetch the specific block hash from the database", "",
-		&fetchBlockCfg)
-	parser.AddCommand("fetchblockregion",
-		"Fetch the specified block region from the database", "",
-		&blockRegionCfg)
-
-	// Parse command line and invoke the Execute function for the specified
-	// command.
-	if _, err := parser.Parse(); err != nil {
-		var flagsErr *flags.Error
-		if ok := errors.As(err, &flagsErr); ok && flagsErr.Type == flags.ErrHelp {
-			parser.WriteHelp(os.Stderr)
-		} else {
-			log.Error(err)
-		}
-
-		return err
-	}
-
-	return nil
-}
-
-func main() {
-	// Use all processor cores.
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	// Work around defer not working after os.Exit()
-	if err := realMain(); err != nil {
-		os.Exit(1)
-	}
-}

database/cmd/dbtool/signal.go

@@ -1,82 +0,0 @@
-// Copyright (c) 2013-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"os"
-	"os/signal"
-)
-
-// interruptChannel is used to receive SIGINT (Ctrl+C) signals.
-var interruptChannel chan os.Signal
-
-// addHandlerChannel is used to add an interrupt handler to the list of handlers
-// to be invoked on SIGINT (Ctrl+C) signals.
-var addHandlerChannel = make(chan func())
-
-// mainInterruptHandler listens for SIGINT (Ctrl+C) signals on the
-// interruptChannel and invokes the registered interruptCallbacks accordingly.
-// It also listens for callback registration. It must be run as a goroutine.
-func mainInterruptHandler() {
-	// interruptCallbacks is a list of callbacks to invoke when a
-	// SIGINT (Ctrl+C) is received.
-	var interruptCallbacks []func()
-
-	// isShutdown is a flag which is used to indicate whether or not
-	// the shutdown signal has already been received and hence any future
-	// attempts to add a new interrupt handler should invoke them
-	// immediately.
-	var isShutdown bool
-
-	for {
-		select {
-		case <-interruptChannel:
-			// Ignore more than one shutdown signal.
-			if isShutdown {
-				log.Infof("Received SIGINT (Ctrl+C). " +
-					"Already shutting down...")
-				continue
-			}
-
-			isShutdown = true
-			log.Infof("Received SIGINT (Ctrl+C). Shutting down...")
-
-			// Run handlers in LIFO order.
-			for i := range interruptCallbacks {
-				idx := len(interruptCallbacks) - 1 - i
-				callback := interruptCallbacks[idx]
-				callback()
-			}
-
-			// Signal the main goroutine to shutdown.
-			spawn(func() {
-				shutdownChannel <- nil
-			})
-
-		case handler := <-addHandlerChannel:
-			// The shutdown signal has already been received, so
-			// just invoke and new handlers immediately.
-			if isShutdown {
-				handler()
-			}
-
-			interruptCallbacks = append(interruptCallbacks, handler)
-		}
-	}
-}
-
-// addInterruptHandler adds a handler to call when a SIGINT (Ctrl+C) is
-// received.
-func addInterruptHandler(handler func()) {
-	// Create the channel and start the main interrupt handler which invokes
-	// all other callbacks and exits if not already done.
-	if interruptChannel == nil {
-		interruptChannel = make(chan os.Signal, 1)
-		signal.Notify(interruptChannel, os.Interrupt)
-		spawn(mainInterruptHandler)
-	}
-
-	addHandlerChannel <- handler
-}

database/common_test.go

@@ -0,0 +1,84 @@
+package database_test
+
+import (
+	"fmt"
+	"github.com/kaspanet/kaspad/database"
+	"github.com/kaspanet/kaspad/database/ffldb"
+	"io/ioutil"
+	"testing"
+)
+
+type databasePrepareFunc func(t *testing.T, testName string) (db database.Database, name string, teardownFunc func())
+
+// databasePrepareFuncs is a set of functions, in which each function
+// prepares a separate database type for testing.
+// See testForAllDatabaseTypes for further details.
+var databasePrepareFuncs = []databasePrepareFunc{
+	prepareFFLDBForTest,
+}
+
+func prepareFFLDBForTest(t *testing.T, testName string) (db database.Database, name string, teardownFunc func()) {
+	// Create a temp db to run tests against
+	path, err := ioutil.TempDir("", testName)
+	if err != nil {
+		t.Fatalf("%s: TempDir unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	db, err = ffldb.Open(path)
+	if err != nil {
+		t.Fatalf("%s: Open unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	teardownFunc = func() {
+		err = db.Close()
+		if err != nil {
+			t.Fatalf("%s: Close unexpectedly "+
+				"failed: %s", testName, err)
+		}
+	}
+	return db, "ffldb", teardownFunc
+}
+
+// testForAllDatabaseTypes runs the given testFunc for every database
+// type defined in databasePrepareFuncs. This is to make sure that
+// all supported database types adhere to the assumptions defined in
+// the interfaces in this package.
+func testForAllDatabaseTypes(t *testing.T, testName string,
+	testFunc func(t *testing.T, db database.Database, testName string)) {
+
+	for _, prepareDatabase := range databasePrepareFuncs {
+		func() {
+			db, dbType, teardownFunc := prepareDatabase(t, testName)
+			defer teardownFunc()
+
+			testName := fmt.Sprintf("%s: %s", dbType, testName)
+			testFunc(t, db, testName)
+		}()
+	}
+}
+
+type keyValuePair struct {
+	key   *database.Key
+	value []byte
+}
+
+func populateDatabaseForTest(t *testing.T, db database.Database, testName string) []keyValuePair {
+	// Prepare a list of key/value pairs
+	entries := make([]keyValuePair, 10)
+	for i := 0; i < 10; i++ {
+		key := database.MakeBucket().Key([]byte(fmt.Sprintf("key%d", i)))
+		value := []byte("value")
+		entries[i] = keyValuePair{key: key, value: value}
+	}
+
+	// Put the pairs into the database
+	for _, entry := range entries {
+		err := db.Put(entry.key, entry.value)
+		if err != nil {
+			t.Fatalf("%s: Put unexpectedly "+
+				"failed: %s", testName, err)
+		}
+	}
+
+	return entries
+}

database/cursor.go

@@ -0,0 +1,30 @@
+package database
+
+// Cursor iterates over database entries given some bucket.
+type Cursor interface {
+	// Next moves the iterator to the next key/value pair. It returns whether the
+	// iterator is exhausted. Panics if the cursor is closed.
+	Next() bool
+
+	// First moves the iterator to the first key/value pair. It returns false if
+	// such a pair does not exist. Panics if the cursor is closed.
+	First() bool
+
+	// Seek moves the iterator to the first key/value pair whose key is greater
+	// than or equal to the given key. It returns ErrNotFound if such pair does not
+	// exist.
+	Seek(key *Key) error
+
+	// Key returns the key of the current key/value pair, or ErrNotFound if done.
+	// The caller should not modify the contents of the returned key, and
+	// its contents may change on the next call to Next.
+	Key() (*Key, error)
+
+	// Value returns the value of the current key/value pair, or ErrNotFound if done.
+	// The caller should not modify the contents of the returned slice, and its
+	// contents may change on the next call to Next.
+	Value() ([]byte, error)
+
+	// Close releases associated resources.
+	Close() error
+}

database/cursor_test.go

@@ -0,0 +1,345 @@
+// All tests within this file should call testForAllDatabaseTypes
+// over the actual test. This is to make sure that all supported
+// database types adhere to the assumptions defined in the
+// interfaces in this package.
+
+package database_test
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/kaspanet/kaspad/database"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func prepareCursorForTest(t *testing.T, db database.Database, testName string) database.Cursor {
+	cursor, err := db.Cursor(database.MakeBucket())
+	if err != nil {
+		t.Fatalf("%s: Cursor unexpectedly "+
+			"failed: %s", testName, err)
+	}
+
+	return cursor
+}
+
+func recoverFromClosedCursorPanic(t *testing.T, testName string) {
+	panicErr := recover()
+	if panicErr == nil {
+		t.Fatalf("%s: cursor unexpectedly "+
+			"didn't panic after being closed", testName)
+	}
+	expectedPanicErr := "closed cursor"
+	if !strings.Contains(fmt.Sprintf("%v", panicErr), expectedPanicErr) {
+		t.Fatalf("%s: cursor panicked "+
+			"with wrong message. Want: %v, got: %s",
+			testName, expectedPanicErr, panicErr)
+	}
+}
+
+func TestCursorNext(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestCursorNext", testCursorNext)
+}
+
+func testCursorNext(t *testing.T, db database.Database, testName string) {
+	entries := populateDatabaseForTest(t, db, testName)
+	cursor := prepareCursorForTest(t, db, testName)
+
+	// Make sure that all the entries exist in the cursor, in their
+	// correct order
+	for _, entry := range entries {
+		hasNext := cursor.Next()
+		if !hasNext {
+			t.Fatalf("%s: cursor unexpectedly "+
+				"done", testName)
+		}
+		cursorKey, err := cursor.Key()
+		if err != nil {
+			t.Fatalf("%s: Key unexpectedly "+
+				"failed: %s", testName, err)
+		}
+		if !reflect.DeepEqual(cursorKey, entry.key) {
+			t.Fatalf("%s: Cursor returned "+
+				"wrong key. Want: %s, got: %s", testName, entry.key, cursorKey)
+		}
+		cursorValue, err := cursor.Value()
+		if err != nil {
+			t.Fatalf("%s: Value unexpectedly "+
+				"failed: %s", testName, err)
+		}
+		if !bytes.Equal(cursorValue, entry.value) {
+			t.Fatalf("%s: Cursor returned "+
+				"wrong value. Want: %s, got: %s", testName, entry.value, cursorValue)
+		}
+	}
+
+	// The cursor should now be exhausted. Make sure Next now
+	// returns false
+	hasNext := cursor.Next()
+	if hasNext {
+		t.Fatalf("%s: cursor unexpectedly "+
+			"not done", testName)
+	}
+
+	// Rewind the cursor and close it
+	cursor.First()
+	err := cursor.Close()
+	if err != nil {
+		t.Fatalf("%s: Close unexpectedly "+
+			"failed: %s", testName, err)
+	}
+
+	// Call Next on the cursor. This time it should panic
+	// because it's closed.
+	func() {
+		defer recoverFromClosedCursorPanic(t, testName)
+		cursor.Next()
+	}()
+}
+
+func TestCursorFirst(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestCursorFirst", testCursorFirst)
+}
+
+func testCursorFirst(t *testing.T, db database.Database, testName string) {
+	entries := populateDatabaseForTest(t, db, testName)
+	cursor := prepareCursorForTest(t, db, testName)
+
+	// Make sure that First returns true when the cursor is not empty
+	exists := cursor.First()
+	if !exists {
+		t.Fatalf("%s: Cursor unexpectedly "+
+			"returned false", testName)
+	}
+
+	// Make sure that the first key and value are as expected
+	firstEntryKey := entries[0].key
+	firstCursorKey, err := cursor.Key()
+	if err != nil {
+		t.Fatalf("%s: Key unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	if !reflect.DeepEqual(firstCursorKey, firstEntryKey) {
+		t.Fatalf("%s: Cursor returned "+
+			"wrong key. Want: %s, got: %s", testName, firstEntryKey, firstCursorKey)
+	}
+	firstEntryValue := entries[0].value
+	firstCursorValue, err := cursor.Value()
+	if err != nil {
+		t.Fatalf("%s: Value unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	if !bytes.Equal(firstCursorValue, firstEntryValue) {
+		t.Fatalf("%s: Cursor returned "+
+			"wrong value. Want: %s, got: %s", testName, firstEntryValue, firstCursorValue)
+	}
+
+	// Exhaust the cursor
+	for cursor.Next() {
+		// Do nothing
+	}
+
+	// Call first again and make sure it still returns true
+	exists = cursor.First()
+	if !exists {
+		t.Fatalf("%s: First unexpectedly "+
+			"returned false", testName)
+	}
+
+	// Call next and make sure it returns true as well
+	exists = cursor.Next()
+	if !exists {
+		t.Fatalf("%s: Next unexpectedly "+
+			"returned false", testName)
+	}
+
+	// Remove all the entries from the database
+	for _, entry := range entries {
+		err := db.Delete(entry.key)
+		if err != nil {
+			t.Fatalf("%s: Delete unexpectedly "+
+				"failed: %s", testName, err)
+		}
+	}
+
+	// Create a new cursor over an empty dataset
+	cursor = prepareCursorForTest(t, db, testName)
+
+	// Make sure that First returns false when the cursor is empty
+	exists = cursor.First()
+	if exists {
+		t.Fatalf("%s: Cursor unexpectedly "+
+			"returned true", testName)
+	}
+}
+
+func TestCursorSeek(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestCursorSeek", testCursorSeek)
+}
+
+func testCursorSeek(t *testing.T, db database.Database, testName string) {
+	entries := populateDatabaseForTest(t, db, testName)
+	cursor := prepareCursorForTest(t, db, testName)
+
+	// Seek to the fourth entry and make sure it exists
+	fourthEntry := entries[3]
+	err := cursor.Seek(fourthEntry.key)
+	if err != nil {
+		t.Fatalf("%s: Cursor unexpectedly "+
+			"failed: %s", testName, err)
+	}
+
+	// Make sure that the key and value are as expected
+	fourthEntryKey := entries[3].key
+	fourthCursorKey, err := cursor.Key()
+	if err != nil {
+		t.Fatalf("%s: Key unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	if !reflect.DeepEqual(fourthCursorKey, fourthEntryKey) {
+		t.Fatalf("%s: Cursor returned "+
+			"wrong key. Want: %s, got: %s", testName, fourthEntryKey, fourthCursorKey)
+	}
+	fourthEntryValue := entries[3].value
+	fourthCursorValue, err := cursor.Value()
+	if err != nil {
+		t.Fatalf("%s: Value unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	if !bytes.Equal(fourthCursorValue, fourthEntryValue) {
+		t.Fatalf("%s: Cursor returned "+
+			"wrong value. Want: %s, got: %s", testName, fourthEntryValue, fourthCursorValue)
+	}
+
+	// Call Next and make sure that we are now on the fifth entry
+	exists := cursor.Next()
+	if !exists {
+		t.Fatalf("%s: Next unexpectedly "+
+			"returned false", testName)
+	}
+	fifthEntryKey := entries[4].key
+	fifthCursorKey, err := cursor.Key()
+	if err != nil {
+		t.Fatalf("%s: Key unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	if !reflect.DeepEqual(fifthCursorKey, fifthEntryKey) {
+		t.Fatalf("%s: Cursor returned "+
+			"wrong key. Want: %s, got: %s", testName, fifthEntryKey, fifthCursorKey)
+	}
+	fifthEntryValue := entries[4].value
+	fifthCursorValue, err := cursor.Value()
+	if err != nil {
+		t.Fatalf("%s: Value unexpectedly "+
+			"failed: %s", testName, err)
+	}
+	if !bytes.Equal(fifthCursorValue, fifthEntryValue) {
+		t.Fatalf("%s: Cursor returned "+
+			"wrong value. Want: %s, got: %s", testName, fifthEntryValue, fifthCursorValue)
+	}
+
+	// Seek to a value that doesn't exist and make sure that
+	// the returned error is ErrNotFound
+	err = cursor.Seek(database.MakeBucket().Key([]byte("doesn't exist")))
+	if err == nil {
+		t.Fatalf("%s: Seek unexpectedly "+
+			"succeeded", testName)
+	}
+	if !database.IsNotFoundError(err) {
+		t.Fatalf("%s: Seek returned "+
+			"wrong error: %s", testName, err)
+	}
+}
+
+func TestCursorCloseErrors(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestCursorCloseErrors", testCursorCloseErrors)
+}
+
+func testCursorCloseErrors(t *testing.T, db database.Database, testName string) {
+	populateDatabaseForTest(t, db, testName)
+	cursor := prepareCursorForTest(t, db, testName)
+
+	// Close the cursor
+	err := cursor.Close()
+	if err != nil {
+		t.Fatalf("%s: Close "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	tests := []struct {
+		name     string
+		function func() error
+	}{
+		{
+			name: "Seek",
+			function: func() error {
+				return cursor.Seek(database.MakeBucket().Key([]byte{}))
+			},
+		},
+		{
+			name: "Key",
+			function: func() error {
+				_, err := cursor.Key()
+				return err
+			},
+		},
+		{
+			name: "Value",
+			function: func() error {
+				_, err := cursor.Value()
+				return err
+			},
+		},
+		{
+			name: "Close",
+			function: func() error {
+				return cursor.Close()
+			},
+		},
+	}
+
+	for _, test := range tests {
+		expectedErrContainsString := "closed cursor"
+
+		// Make sure that the test function returns a "closed cursor" error
+		err = test.function()
+		if err == nil {
+			t.Fatalf("%s: %s "+
+				"unexpectedly succeeded", testName, test.name)
+		}
+		if !strings.Contains(err.Error(), expectedErrContainsString) {
+			t.Fatalf("%s: %s "+
+				"returned wrong error. Want: %s, got: %s",
+				testName, test.name, expectedErrContainsString, err)
+		}
+	}
+}
+
+func TestCursorCloseFirstAndNext(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestCursorCloseFirstAndNext", testCursorCloseFirstAndNext)
+}
+
+func testCursorCloseFirstAndNext(t *testing.T, db database.Database, testName string) {
+	populateDatabaseForTest(t, db, testName)
+	cursor := prepareCursorForTest(t, db, testName)
+
+	// Close the cursor
+	err := cursor.Close()
+	if err != nil {
+		t.Fatalf("%s: Close "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// We expect First to panic
+	func() {
+		defer recoverFromClosedCursorPanic(t, testName)
+		cursor.First()
+	}()
+
+	// We expect Next to panic
+	func() {
+		defer recoverFromClosedCursorPanic(t, testName)
+		cursor.Next()
+	}()
+}

database/dataaccessor.go

@@ -0,0 +1,36 @@
+package database
+
+// DataAccessor defines the common interface by which data gets
+// accessed in a generic kaspad database.
+type DataAccessor interface {
+	// Put sets the value for the given key. It overwrites
+	// any previous value for that key.
+	Put(key *Key, value []byte) error
+
+	// Get gets the value for the given key. It returns
+	// ErrNotFound if the given key does not exist.
+	Get(key *Key) ([]byte, error)
+
+	// Has returns true if the database does contains the
+	// given key.
+	Has(key *Key) (bool, error)
+
+	// Delete deletes the value for the given key. Will not
+	// return an error if the key doesn't exist.
+	Delete(key *Key) error
+
+	// AppendToStore appends the given data to the store
+	// defined by storeName. This function returns a serialized
+	// location handle that's meant to be stored and later used
+	// when querying the data that has just now been inserted.
+	AppendToStore(storeName string, data []byte) ([]byte, error)
+
+	// RetrieveFromStore retrieves data from the store defined by
+	// storeName using the given serialized location handle. It
+	// returns ErrNotFound if the location does not exist. See
+	// AppendToStore for further details.
+	RetrieveFromStore(storeName string, location []byte) ([]byte, error)
+
+	// Cursor begins a new cursor over the given bucket.
+	Cursor(bucket *Bucket) (Cursor, error)
+}

database/database.go

@@ -0,0 +1,19 @@
+package database
+
+// Database defines the interface of a database that can begin
+// transactions and close itself.
+//
+// Important: This is not part of the DataAccessor interface
+// because the Transaction interface includes it. Were we to
+// merge Database with DataAccessor, implementors of the
+// Transaction interface would be forced to implement methods
+// such as Begin and Close, which is undesirable.
+type Database interface {
+	DataAccessor
+
+	// Begin begins a new database transaction.
+	Begin() (Transaction, error)
+
+	// Close closes the database.
+	Close() error
+}

database/database_test.go

@@ -0,0 +1,207 @@
+// All tests within this file should call testForAllDatabaseTypes
+// over the actual test. This is to make sure that all supported
+// database types adhere to the assumptions defined in the
+// interfaces in this package.
+
+package database_test
+
+import (
+	"bytes"
+	"github.com/kaspanet/kaspad/database"
+	"testing"
+)
+
+func TestDatabasePut(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestDatabasePut", testDatabasePut)
+}
+
+func testDatabasePut(t *testing.T, db database.Database, testName string) {
+	// Put value1 into the database
+	key := database.MakeBucket().Key([]byte("key"))
+	value1 := []byte("value1")
+	err := db.Put(key, value1)
+	if err != nil {
+		t.Fatalf("%s: Put "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Make sure that the returned value is value1
+	returnedValue, err := db.Get(key)
+	if err != nil {
+		t.Fatalf("%s: Get "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if !bytes.Equal(returnedValue, value1) {
+		t.Fatalf("%s: Get "+
+			"returned wrong value. Want: %s, got: %s",
+			testName, string(value1), string(returnedValue))
+	}
+
+	// Put value2 into the database with the same key
+	value2 := []byte("value2")
+	err = db.Put(key, value2)
+	if err != nil {
+		t.Fatalf("%s: Put "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Make sure that the returned value is value2
+	returnedValue, err = db.Get(key)
+	if err != nil {
+		t.Fatalf("%s: Get "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if !bytes.Equal(returnedValue, value2) {
+		t.Fatalf("%s: Get "+
+			"returned wrong value. Want: %s, got: %s",
+			testName, string(value2), string(returnedValue))
+	}
+}
+
+func TestDatabaseGet(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestDatabaseGet", testDatabaseGet)
+}
+
+func testDatabaseGet(t *testing.T, db database.Database, testName string) {
+	// Put a value into the database
+	key := database.MakeBucket().Key([]byte("key"))
+	value := []byte("value")
+	err := db.Put(key, value)
+	if err != nil {
+		t.Fatalf("%s: Put "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Get the value back and make sure it's the same one
+	returnedValue, err := db.Get(key)
+	if err != nil {
+		t.Fatalf("%s: Get "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if !bytes.Equal(returnedValue, value) {
+		t.Fatalf("%s: Get "+
+			"returned wrong value. Want: %s, got: %s",
+			testName, string(value), string(returnedValue))
+	}
+
+	// Try getting a non-existent value and make sure
+	// the returned error is ErrNotFound
+	_, err = db.Get(database.MakeBucket().Key([]byte("doesn't exist")))
+	if err == nil {
+		t.Fatalf("%s: Get "+
+			"unexpectedly succeeded", testName)
+	}
+	if !database.IsNotFoundError(err) {
+		t.Fatalf("%s: Get "+
+			"returned wrong error: %s", testName, err)
+	}
+}
+
+func TestDatabaseHas(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestDatabaseHas", testDatabaseHas)
+}
+
+func testDatabaseHas(t *testing.T, db database.Database, testName string) {
+	// Put a value into the database
+	key := database.MakeBucket().Key([]byte("key"))
+	value := []byte("value")
+	err := db.Put(key, value)
+	if err != nil {
+		t.Fatalf("%s: Put "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Make sure that Has returns true for the value we just put
+	exists, err := db.Has(key)
+	if err != nil {
+		t.Fatalf("%s: Has "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if !exists {
+		t.Fatalf("%s: Has "+
+			"unexpectedly returned that the value does not exist", testName)
+	}
+
+	// Make sure that Has returns false for a non-existent value
+	exists, err = db.Has(database.MakeBucket().Key([]byte("doesn't exist")))
+	if err != nil {
+		t.Fatalf("%s: Has "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if exists {
+		t.Fatalf("%s: Has "+
+			"unexpectedly returned that the value exists", testName)
+	}
+}
+
+func TestDatabaseDelete(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestDatabaseDelete", testDatabaseDelete)
+}
+
+func testDatabaseDelete(t *testing.T, db database.Database, testName string) {
+	// Put a value into the database
+	key := database.MakeBucket().Key([]byte("key"))
+	value := []byte("value")
+	err := db.Put(key, value)
+	if err != nil {
+		t.Fatalf("%s: Put "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Delete the value
+	err = db.Delete(key)
+	if err != nil {
+		t.Fatalf("%s: Delete "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Make sure that Has returns false for the deleted value
+	exists, err := db.Has(key)
+	if err != nil {
+		t.Fatalf("%s: Has "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if exists {
+		t.Fatalf("%s: Has "+
+			"unexpectedly returned that the value exists", testName)
+	}
+}
+
+func TestDatabaseAppendToStoreAndRetrieveFromStore(t *testing.T) {
+	testForAllDatabaseTypes(t, "TestDatabaseAppendToStoreAndRetrieveFromStore", testDatabaseAppendToStoreAndRetrieveFromStore)
+}
+
+func testDatabaseAppendToStoreAndRetrieveFromStore(t *testing.T, db database.Database, testName string) {
+	// Append some data into the store
+	storeName := "store"
+	data := []byte("data")
+	location, err := db.AppendToStore(storeName, data)
+	if err != nil {
+		t.Fatalf("%s: AppendToStore "+
+			"unexpectedly failed: %s", testName, err)
+	}
+
+	// Retrieve the data and make sure it's equal to what was appended
+	retrievedData, err := db.RetrieveFromStore(storeName, location)
+	if err != nil {
+		t.Fatalf("%s: RetrieveFromStore "+
+			"unexpectedly failed: %s", testName, err)
+	}
+	if !bytes.Equal(retrievedData, data) {
+		t.Fatalf("%s: RetrieveFromStore "+
+			"returned unexpected data. Want: %s, got: %s",
+			testName, string(data), string(retrievedData))
+	}
+
+	// Make sure that an invalid location returns ErrNotFound
+	fakeLocation := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}
+	_, err = db.RetrieveFromStore(storeName, fakeLocation)
+	if err == nil {
+		t.Fatalf("%s: RetrieveFromStore "+
+			"unexpectedly succeeded", testName)
+	}
+	if !database.IsNotFoundError(err) {
+		t.Fatalf("%s: RetrieveFromStore "+
+			"returned wrong error: %s", testName, err)
+	}
+}

database/doc.go

@@ -1,85 +1,34 @@
 /*
-Package database provides a block and metadata storage database.
+Package database provides a database for kaspad.
 
 Overview
 
-This package provides a database layer to store and retrieve this data in a
-simple and efficient manner.
+This package provides a database layer to store and retrieve data in a simple
+and efficient manner.
 
-The default backend, ffldb, has a strong focus on speed, efficiency, and
-robustness. It makes use leveldb for the metadata, flat files for block
-storage, and strict checksums in key areas to ensure data integrity.
+The current backend is ffldb, which makes use of leveldb, flat files, and strict
+checksums in key areas to ensure data integrity.
 
-A quick overview of the features database provides are as follows:
+Implementors of additional backends are required to implement the following interfaces:
 
- - Key/value metadata store
- - Kaspa block storage
- - Efficient retrieval of block headers and regions (transactions, scripts, etc)
- - Read-only and read-write transactions with both manual and managed modes
- - Nested buckets
- - Supports registration of backend databases
- - Comprehensive test coverage
+DataAccessor
+
+This defines the common interface by which data gets accessed in a generic kaspad
+database. Both the Database and the Transaction interfaces (see below) implement it.
 
 Database
 
-The main entry point is the DB interface. It exposes functionality for
-transactional-based access and storage of metadata and block data. It is
-obtained via the Create and Open functions which take a database type string
-that identifies the specific database driver (backend) to use as well as
-arguments specific to the specified driver.
+This defines the interface of a database that can begin transactions and close itself.
 
-The interface provides facilities for obtaining transactions (the Tx interface)
-that are the basis of all database reads and writes. Unlike some database
-interfaces that support reading and writing without transactions, this interface
-requires transactions even when only reading or writing a single key.
+Transaction
 
-The Begin function provides an unmanaged transaction while the View and Update
-functions provide a managed transaction. These are described in more detail
-below.
+This defines the interface of a generic kaspad database transaction.
+Note: transactions provide data consistency over the state of the database as it was
+when the transaction started. There is NO guarantee that if one puts data into the
+transaction then it will be available to get within the same transaction.
 
-Transactions
+Cursor
 
-The Tx interface provides facilities for rolling back or committing changes that
-took place while the transaction was active. It also provides the root metadata
-bucket under which all keys, values, and nested buckets are stored. A
-transaction can either be read-only or read-write and managed or unmanaged.
-
-Managed versus Unmanaged Transactions
-
-A managed transaction is one where the caller provides a function to execute
-within the context of the transaction and the commit or rollback is handled
-automatically depending on whether or not the provided function returns an
-error. Attempting to manually call Rollback or Commit on the managed
-transaction will result in a panic.
-
-An unmanaged transaction, on the other hand, requires the caller to manually
-call Commit or Rollback when they are finished with it. Leaving transactions
-open for long periods of time can have several adverse effects, so it is
-recommended that managed transactions are used instead.
-
-Buckets
-
-The Bucket interface provides the ability to manipulate key/value pairs and
-nested buckets as well as iterate through them.
-
-The Get, Put, and Delete functions work with key/value pairs, while the Bucket,
-CreateBucket, CreateBucketIfNotExists, and DeleteBucket functions work with
-buckets. The ForEach function allows the caller to provide a function to be
-called with each key/value pair and nested bucket in the current bucket.
-
-Metadata Bucket
-
-As discussed above, all of the functions which are used to manipulate key/value
-pairs and nested buckets exist on the Bucket interface. The root metadata
-bucket is the upper-most bucket in which data is stored and is created at the
-same time as the database. Use the Metadata function on the Tx interface
-to retrieve it.
-
-Nested Buckets
-
-The CreateBucket and CreateBucketIfNotExists functions on the Bucket interface
-provide the ability to create an arbitrary number of nested buckets. It is
-a good idea to avoid a lot of buckets with little data in them as it could lead
-to poor page utilization depending on the specific driver in use.
+This iterates over database entries given some bucket.
 */
 package database

database/driver.go

@@ -1,84 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"fmt"
-)
-
-// Driver defines a structure for backend drivers to use when they registered
-// themselves as a backend which implements the DB interface.
-type Driver struct {
-	// DbType is the identifier used to uniquely identify a specific
-	// database driver. There can be only one driver with the same name.
-	DbType string
-
-	// Create is the function that will be invoked with all user-specified
-	// arguments to create the database. This function must return
-	// ErrDbExists if the database already exists.
-	Create func(args ...interface{}) (DB, error)
-
-	// Open is the function that will be invoked with all user-specified
-	// arguments to open the database. This function must return
-	// ErrDbDoesNotExist if the database has not already been created.
-	Open func(args ...interface{}) (DB, error)
-}
-
-// driverList holds all of the registered database backends.
-var drivers = make(map[string]*Driver)
-
-// RegisterDriver adds a backend database driver to available interfaces.
-// ErrDbTypeRegistered will be returned if the database type for the driver has
-// already been registered.
-func RegisterDriver(driver Driver) error {
-	if _, exists := drivers[driver.DbType]; exists {
-		str := fmt.Sprintf("driver %q is already registered",
-			driver.DbType)
-		return makeError(ErrDbTypeRegistered, str, nil)
-	}
-
-	drivers[driver.DbType] = &driver
-	return nil
-}
-
-// SupportedDrivers returns a slice of strings that represent the database
-// drivers that have been registered and are therefore supported.
-func SupportedDrivers() []string {
-	supportedDBs := make([]string, 0, len(drivers))
-	for _, drv := range drivers {
-		supportedDBs = append(supportedDBs, drv.DbType)
-	}
-	return supportedDBs
-}
-
-// Create initializes and opens a database for the specified type. The
-// arguments are specific to the database type driver. See the documentation
-// for the database driver for further details.
-//
-// ErrDbUnknownType will be returned if the the database type is not registered.
-func Create(dbType string, args ...interface{}) (DB, error) {
-	drv, exists := drivers[dbType]
-	if !exists {
-		str := fmt.Sprintf("driver %q is not registered", dbType)
-		return nil, makeError(ErrDbUnknownType, str, nil)
-	}
-
-	return drv.Create(args...)
-}
-
-// Open opens an existing database for the specified type. The arguments are
-// specific to the database type driver. See the documentation for the database
-// driver for further details.
-//
-// ErrDbUnknownType will be returned if the the database type is not registered.
-func Open(dbType string, args ...interface{}) (DB, error) {
-	drv, exists := drivers[dbType]
-	if !exists {
-		str := fmt.Sprintf("driver %q is not registered", dbType)
-		return nil, makeError(ErrDbUnknownType, str, nil)
-	}
-
-	return drv.Open(args...)
-}

database/driver_test.go

@@ -1,128 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database_test
-
-import (
-	"github.com/pkg/errors"
-	"testing"
-
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-)
-
-// checkDbError ensures the passed error is a database.Error with an error code
-// that matches the passed  error code.
-func checkDbError(t *testing.T, testName string, gotErr error, wantErrCode database.ErrorCode) bool {
-	dbErr, ok := gotErr.(database.Error)
-	if !ok {
-		t.Errorf("%s: unexpected error type - got %T, want %T",
-			testName, gotErr, database.Error{})
-		return false
-	}
-	if dbErr.ErrorCode != wantErrCode {
-		t.Errorf("%s: unexpected error code - got %s (%s), want %s",
-			testName, dbErr.ErrorCode, dbErr.Description,
-			wantErrCode)
-		return false
-	}
-
-	return true
-}
-
-// TestAddDuplicateDriver ensures that adding a duplicate driver does not
-// overwrite an existing one.
-func TestAddDuplicateDriver(t *testing.T) {
-	supportedDrivers := database.SupportedDrivers()
-	if len(supportedDrivers) == 0 {
-		t.Errorf("no backends to test")
-		return
-	}
-	dbType := supportedDrivers[0]
-
-	// bogusCreateDB is a function which acts as a bogus create and open
-	// driver function and intentionally returns a failure that can be
-	// detected if the interface allows a duplicate driver to overwrite an
-	// existing one.
-	bogusCreateDB := func(args ...interface{}) (database.DB, error) {
-		return nil, errors.Errorf("duplicate driver allowed for database "+
-			"type [%v]", dbType)
-	}
-
-	// Create a driver that tries to replace an existing one. Set its
-	// create and open functions to a function that causes a test failure if
-	// they are invoked.
-	driver := database.Driver{
-		DbType: dbType,
-		Create: bogusCreateDB,
-		Open:   bogusCreateDB,
-	}
-	testName := "duplicate driver registration"
-	err := database.RegisterDriver(driver)
-	if !checkDbError(t, testName, err, database.ErrDbTypeRegistered) {
-		return
-	}
-}
-
-// TestCreateOpenFail ensures that errors which occur while opening or closing
-// a database are handled properly.
-func TestCreateOpenFail(t *testing.T) {
-	// bogusCreateDB is a function which acts as a bogus create and open
-	// driver function that intentionally returns a failure which can be
-	// detected.
-	dbType := "createopenfail"
-	openError := errors.Errorf("failed to create or open database for "+
-		"database type [%v]", dbType)
-	bogusCreateDB := func(args ...interface{}) (database.DB, error) {
-		return nil, openError
-	}
-
-	// Create and add driver that intentionally fails when created or opened
-	// to ensure errors on database open and create are handled properly.
-	driver := database.Driver{
-		DbType: dbType,
-		Create: bogusCreateDB,
-		Open:   bogusCreateDB,
-	}
-	database.RegisterDriver(driver)
-
-	// Ensure creating a database with the new type fails with the expected
-	// error.
-	_, err := database.Create(dbType)
-	if err != openError {
-		t.Errorf("expected error not received - got: %v, want %v", err,
-			openError)
-		return
-	}
-
-	// Ensure opening a database with the new type fails with the expected
-	// error.
-	_, err = database.Open(dbType)
-	if err != openError {
-		t.Errorf("expected error not received - got: %v, want %v", err,
-			openError)
-		return
-	}
-}
-
-// TestCreateOpenUnsupported ensures that attempting to create or open an
-// unsupported database type is handled properly.
-func TestCreateOpenUnsupported(t *testing.T) {
-	// Ensure creating a database with an unsupported type fails with the
-	// expected error.
-	testName := "create with unsupported database type"
-	dbType := "unsupported"
-	_, err := database.Create(dbType)
-	if !checkDbError(t, testName, err, database.ErrDbUnknownType) {
-		return
-	}
-
-	// Ensure opening a database with the an unsupported type fails with the
-	// expected error.
-	testName = "open with unsupported database type"
-	_, err = database.Open(dbType)
-	if !checkDbError(t, testName, err, database.ErrDbUnknownType) {
-		return
-	}
-}

database/error.go

@@ -1,211 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"fmt"
-	"github.com/pkg/errors"
-)
-
-// ErrorCode identifies a kind of error.
-type ErrorCode int
-
-// These constants are used to identify a specific database Error.
-const (
-	// **************************************
-	// Errors related to driver registration.
-	// **************************************
-
-	// ErrDbTypeRegistered indicates two different database drivers
-	// attempt to register with the name database type.
-	ErrDbTypeRegistered ErrorCode = iota
-
-	// *************************************
-	// Errors related to database functions.
-	// *************************************
-
-	// ErrDbUnknownType indicates there is no driver registered for
-	// the specified database type.
-	ErrDbUnknownType
-
-	// ErrDbDoesNotExist indicates open is called for a database that
-	// does not exist.
-	ErrDbDoesNotExist
-
-	// ErrDbExists indicates create is called for a database that
-	// already exists.
-	ErrDbExists
-
-	// ErrDbNotOpen indicates a database instance is accessed before
-	// it is opened or after it is closed.
-	ErrDbNotOpen
-
-	// ErrDbAlreadyOpen indicates open was called on a database that
-	// is already open.
-	ErrDbAlreadyOpen
-
-	// ErrInvalid indicates the specified database is not valid.
-	ErrInvalid
-
-	// ErrCorruption indicates a checksum failure occurred which invariably
-	// means the database is corrupt.
-	ErrCorruption
-
-	// ****************************************
-	// Errors related to database transactions.
-	// ****************************************
-
-	// ErrTxClosed indicates an attempt was made to commit or rollback a
-	// transaction that has already had one of those operations performed.
-	ErrTxClosed
-
-	// ErrTxNotWritable indicates an operation that requires write access to
-	// the database was attempted against a read-only transaction.
-	ErrTxNotWritable
-
-	// **************************************
-	// Errors related to metadata operations.
-	// **************************************
-
-	// ErrBucketNotFound indicates an attempt to access a bucket that has
-	// not been created yet.
-	ErrBucketNotFound
-
-	// ErrBucketExists indicates an attempt to create a bucket that already
-	// exists.
-	ErrBucketExists
-
-	// ErrBucketNameRequired indicates an attempt to create a bucket with a
-	// blank name.
-	ErrBucketNameRequired
-
-	// ErrKeyRequired indicates at attempt to insert a zero-length key.
-	ErrKeyRequired
-
-	// ErrKeyTooLarge indicates an attmempt to insert a key that is larger
-	// than the max allowed key size. The max key size depends on the
-	// specific backend driver being used. As a general rule, key sizes
-	// should be relatively, so this should rarely be an issue.
-	ErrKeyTooLarge
-
-	// ErrValueTooLarge indicates an attmpt to insert a value that is larger
-	// than max allowed value size. The max key size depends on the
-	// specific backend driver being used.
-	ErrValueTooLarge
-
-	// ErrIncompatibleValue indicates the value in question is invalid for
-	// the specific requested operation. For example, trying create or
-	// delete a bucket with an existing non-bucket key, attempting to create
-	// or delete a non-bucket key with an existing bucket key, or trying to
-	// delete a value via a cursor when it points to a nested bucket.
-	ErrIncompatibleValue
-
-	// ***************************************
-	// Errors related to block I/O operations.
-	// ***************************************
-
-	// ErrBlockNotFound indicates a block with the provided hash does not
-	// exist in the database.
-	ErrBlockNotFound
-
-	// ErrBlockExists indicates a block with the provided hash already
-	// exists in the database.
-	ErrBlockExists
-
-	// ErrBlockRegionInvalid indicates a region that exceeds the bounds of
-	// the specified block was requested. When the hash provided by the
-	// region does not correspond to an existing block, the error will be
-	// ErrBlockNotFound instead.
-	ErrBlockRegionInvalid
-
-	// ***********************************
-	// Support for driver-specific errors.
-	// ***********************************
-
-	// ErrDriverSpecific indicates the Err field is a driver-specific error.
-	// This provides a mechanism for drivers to plug-in their own custom
-	// errors for any situations which aren't already covered by the error
-	// codes provided by this package.
-	ErrDriverSpecific
-
-	// numErrorCodes is the maximum error code number used in tests.
-	numErrorCodes
-)
-
-// Map of ErrorCode values back to their constant names for pretty printing.
-var errorCodeStrings = map[ErrorCode]string{
-	ErrDbTypeRegistered:   "ErrDbTypeRegistered",
-	ErrDbUnknownType:      "ErrDbUnknownType",
-	ErrDbDoesNotExist:     "ErrDbDoesNotExist",
-	ErrDbExists:           "ErrDbExists",
-	ErrDbNotOpen:          "ErrDbNotOpen",
-	ErrDbAlreadyOpen:      "ErrDbAlreadyOpen",
-	ErrInvalid:            "ErrInvalid",
-	ErrCorruption:         "ErrCorruption",
-	ErrTxClosed:           "ErrTxClosed",
-	ErrTxNotWritable:      "ErrTxNotWritable",
-	ErrBucketNotFound:     "ErrBucketNotFound",
-	ErrBucketExists:       "ErrBucketExists",
-	ErrBucketNameRequired: "ErrBucketNameRequired",
-	ErrKeyRequired:        "ErrKeyRequired",
-	ErrKeyTooLarge:        "ErrKeyTooLarge",
-	ErrValueTooLarge:      "ErrValueTooLarge",
-	ErrIncompatibleValue:  "ErrIncompatibleValue",
-	ErrBlockNotFound:      "ErrBlockNotFound",
-	ErrBlockExists:        "ErrBlockExists",
-	ErrBlockRegionInvalid: "ErrBlockRegionInvalid",
-	ErrDriverSpecific:     "ErrDriverSpecific",
-}
-
-// String returns the ErrorCode as a human-readable name.
-func (e ErrorCode) String() string {
-	if s := errorCodeStrings[e]; s != "" {
-		return s
-	}
-	return fmt.Sprintf("Unknown ErrorCode (%d)", int(e))
-}
-
-// Error provides a single type for errors that can happen during database
-// operation. It is used to indicate several types of failures including errors
-// with caller requests such as specifying invalid block regions or attempting
-// to access data against closed database transactions, driver errors, errors
-// retrieving data, and errors communicating with database servers.
-//
-// The caller can use type assertions to determine if an error is an Error and
-// access the ErrorCode field to ascertain the specific reason for the failure.
-//
-// The ErrDriverSpecific error code will also have the Err field set with the
-// underlying error. Depending on the backend driver, the Err field might be
-// set to the underlying error for other error codes as well.
-type Error struct {
-	ErrorCode   ErrorCode // Describes the kind of error
-	Description string    // Human readable description of the issue
-	Err         error     // Underlying error
-}
-
-// Error satisfies the error interface and prints human-readable errors.
-func (e Error) Error() string {
-	if e.Err != nil {
-		return e.Description + ": " + e.Err.Error()
-	}
-	return e.Description
-}
-
-// makeError creates an Error given a set of arguments. The error code must
-// be one of the error codes provided by this package.
-func makeError(c ErrorCode, desc string, err error) Error {
-	return Error{ErrorCode: c, Description: desc, Err: err}
-}
-
-// IsErrorCode returns whether or not the provided error is a script error with
-// the provided error code.
-func IsErrorCode(err error, c ErrorCode) bool {
-	var errError Error
-	if ok := errors.As(err, &errError); ok {
-		return errError.ErrorCode == c
-	}
-
-	return false
-}

database/error_test.go

@@ -1,118 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database
-
-import (
-	"github.com/pkg/errors"
-	"testing"
-)
-
-// TestErrorCodeStringer tests the stringized output for the ErrorCode type.
-func TestErrorCodeStringer(t *testing.T) {
-	tests := []struct {
-		in   ErrorCode
-		want string
-	}{
-		{ErrDbTypeRegistered, "ErrDbTypeRegistered"},
-		{ErrDbUnknownType, "ErrDbUnknownType"},
-		{ErrDbDoesNotExist, "ErrDbDoesNotExist"},
-		{ErrDbExists, "ErrDbExists"},
-		{ErrDbNotOpen, "ErrDbNotOpen"},
-		{ErrDbAlreadyOpen, "ErrDbAlreadyOpen"},
-		{ErrInvalid, "ErrInvalid"},
-		{ErrCorruption, "ErrCorruption"},
-		{ErrTxClosed, "ErrTxClosed"},
-		{ErrTxNotWritable, "ErrTxNotWritable"},
-		{ErrBucketNotFound, "ErrBucketNotFound"},
-		{ErrBucketExists, "ErrBucketExists"},
-		{ErrBucketNameRequired, "ErrBucketNameRequired"},
-		{ErrKeyRequired, "ErrKeyRequired"},
-		{ErrKeyTooLarge, "ErrKeyTooLarge"},
-		{ErrValueTooLarge, "ErrValueTooLarge"},
-		{ErrIncompatibleValue, "ErrIncompatibleValue"},
-		{ErrBlockNotFound, "ErrBlockNotFound"},
-		{ErrBlockExists, "ErrBlockExists"},
-		{ErrBlockRegionInvalid, "ErrBlockRegionInvalid"},
-		{ErrDriverSpecific, "ErrDriverSpecific"},
-
-		{0xffff, "Unknown ErrorCode (65535)"},
-	}
-
-	// Detect additional error codes that don't have the stringer added.
-	if len(tests)-1 != int(TstNumErrorCodes) {
-		t.Errorf("It appears an error code was added without adding " +
-			"an associated stringer test")
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.String()
-		if result != test.want {
-			t.Errorf("String #%d\ngot: %s\nwant: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-// TestError tests the error output for the Error type.
-func TestError(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		in   Error
-		want string
-	}{
-		{
-			Error{Description: "some error"},
-			"some error",
-		},
-		{
-			Error{Description: "human-readable error"},
-			"human-readable error",
-		},
-		{
-			Error{
-				ErrorCode:   ErrDriverSpecific,
-				Description: "some error",
-				Err:         errors.New("driver-specific error"),
-			},
-			"some error: driver-specific error",
-		},
-	}
-
-	t.Logf("Running %d tests", len(tests))
-	for i, test := range tests {
-		result := test.in.Error()
-		if result != test.want {
-			t.Errorf("Error #%d\n got: %s want: %s", i, result,
-				test.want)
-			continue
-		}
-	}
-}
-
-func TestIsErrorCode(t *testing.T) {
-	dummyError := errors.New("")
-
-	tests := []struct {
-		err            error
-		code           ErrorCode
-		expectedResult bool
-	}{
-		{makeError(ErrBucketExists, "", dummyError), ErrBucketExists, true},
-		{makeError(ErrBucketExists, "", dummyError), ErrBlockExists, false},
-		{dummyError, ErrBlockExists, false},
-		{nil, ErrBlockExists, false},
-	}
-
-	for i, test := range tests {
-		actualResult := IsErrorCode(test.err, test.code)
-		if test.expectedResult != actualResult {
-			t.Errorf("TestIsErrorCode: %d: Expected: %t, but got: %t",
-				i, test.expectedResult, actualResult)
-		}
-	}
-}

database/errors.go

@@ -0,0 +1,12 @@
+package database
+
+import "errors"
+
+// ErrNotFound denotes that the requested item was not
+// found in the database.
+var ErrNotFound = errors.New("not found")
+
+// IsNotFoundError checks whether an error is an ErrNotFound.
+func IsNotFoundError(err error) bool {
+	return errors.Is(err, ErrNotFound)
+}

database/example_test.go

@@ -1,180 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package database_test
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/pkg/errors"
-	"os"
-	"path/filepath"
-
-	"github.com/kaspanet/kaspad/dagconfig"
-	"github.com/kaspanet/kaspad/database"
-	_ "github.com/kaspanet/kaspad/database/ffldb"
-	"github.com/kaspanet/kaspad/util"
-	"github.com/kaspanet/kaspad/wire"
-)
-
-// This example demonstrates creating a new database.
-func ExampleCreate() {
-	// This example assumes the ffldb driver is imported.
-	//
-	// import (
-	// 	"github.com/kaspanet/kaspad/database"
-	// 	_ "github.com/kaspanet/kaspad/database/ffldb"
-	// )
-
-	// Create a database and schedule it to be closed and removed on exit.
-	// Typically you wouldn't want to remove the database right away like
-	// this, nor put it in the temp directory, but it's done here to ensure
-	// the example cleans up after itself.
-	dbPath := filepath.Join(os.TempDir(), "examplecreate")
-	db, err := database.Create("ffldb", dbPath, wire.Mainnet)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Output:
-}
-
-// This example demonstrates creating a new database and using a managed
-// read-write transaction to store and retrieve metadata.
-func Example_basicUsage() {
-	// This example assumes the ffldb driver is imported.
-	//
-	// import (
-	// 	"github.com/kaspanet/kaspad/database"
-	// 	_ "github.com/kaspanet/kaspad/database/ffldb"
-	// )
-
-	// Create a database and schedule it to be closed and removed on exit.
-	// Typically you wouldn't want to remove the database right away like
-	// this, nor put it in the temp directory, but it's done here to ensure
-	// the example cleans up after itself.
-	dbPath := filepath.Join(os.TempDir(), "exampleusage")
-	// ensure that DB does not exist before test starts
-	os.RemoveAll(dbPath)
-	db, err := database.Create("ffldb", dbPath, wire.Mainnet)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Use the Update function of the database to perform a managed
-	// read-write transaction. The transaction will automatically be rolled
-	// back if the supplied inner function returns a non-nil error.
-	err = db.Update(func(dbTx database.Tx) error {
-		// Store a key/value pair directly in the metadata bucket.
-		// Typically a nested bucket would be used for a given feature,
-		// but this example is using the metadata bucket directly for
-		// simplicity.
-		key := []byte("mykey")
-		value := []byte("myvalue")
-		if err := dbTx.Metadata().Put(key, value); err != nil {
-			return err
-		}
-
-		// Read the key back and ensure it matches.
-		if !bytes.Equal(dbTx.Metadata().Get(key), value) {
-			return errors.Errorf("unexpected value for key '%s'", key)
-		}
-
-		// Create a new nested bucket under the metadata bucket.
-		nestedBucketKey := []byte("mybucket")
-		nestedBucket, err := dbTx.Metadata().CreateBucket(nestedBucketKey)
-		if err != nil {
-			return err
-		}
-
-		// The key from above that was set in the metadata bucket does
-		// not exist in this new nested bucket.
-		if nestedBucket.Get(key) != nil {
-			return errors.Errorf("key '%s' is not expected nil", key)
-		}
-
-		return nil
-	})
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	// Output:
-}
-
-// This example demonstrates creating a new database, using a managed read-write
-// transaction to store a block, and using a managed read-only transaction to
-// fetch the block.
-func Example_blockStorageAndRetrieval() {
-	// This example assumes the ffldb driver is imported.
-	//
-	// import (
-	// 	"github.com/kaspanet/kaspad/database"
-	// 	_ "github.com/kaspanet/kaspad/database/ffldb"
-	// )
-
-	// Create a database and schedule it to be closed and removed on exit.
-	// Typically you wouldn't want to remove the database right away like
-	// this, nor put it in the temp directory, but it's done here to ensure
-	// the example cleans up after itself.
-	dbPath := filepath.Join(os.TempDir(), "exampleblkstorage")
-	db, err := database.Create("ffldb", dbPath, wire.Mainnet)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-	defer os.RemoveAll(dbPath)
-	defer db.Close()
-
-	// Use the Update function of the database to perform a managed
-	// read-write transaction and store a genesis block in the database as
-	// and example.
-	err = db.Update(func(dbTx database.Tx) error {
-		genesisBlock := dagconfig.MainnetParams.GenesisBlock
-		return dbTx.StoreBlock(util.NewBlock(genesisBlock))
-	})
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	// Use the View function of the database to perform a managed read-only
-	// transaction and fetch the block stored above.
-	var loadedBlockBytes []byte
-	err = db.Update(func(dbTx database.Tx) error {
-		genesisHash := dagconfig.MainnetParams.GenesisHash
-		blockBytes, err := dbTx.FetchBlock(genesisHash)
-		if err != nil {
-			return err
-		}
-
-		// As documented, all data fetched from the database is only
-		// valid during a database transaction in order to support
-		// zero-copy backends. Thus, make a copy of the data so it
-		// can be used outside of the transaction.
-		loadedBlockBytes = make([]byte, len(blockBytes))
-		copy(loadedBlockBytes, blockBytes)
-		return nil
-	})
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	// Typically at this point, the block could be deserialized via the
-	// wire.MsgBlock.Deserialize function or used in its serialized form
-	// depending on need. However, for this example, just display the
-	// number of serialized bytes to show it was loaded as expected.
-	fmt.Printf("Serialized block size: %d bytes\n", len(loadedBlockBytes))
-
-	// Output:
-	// Serialized block size: 280 bytes
-}

database/export_test.go

@@ -1,17 +0,0 @@
-// Copyright (c) 2015-2016 The btcsuite developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-/*
-This test file is part of the database package rather than than the
-database_test package so it can bridge access to the internals to properly test
-cases which are either not possible or can't reliably be tested via the public
-interface. The functions, constants, and variables are only exported while the
-tests are being run.
-*/
-
-package database
-
-// TstNumErrorCodes makes the internal numErrorCodes parameter available to the
-// test package.
-const TstNumErrorCodes = numErrorCodes