[NOD-70] Added mining simluator (#228)

* [NOD-70] Added GetBlockTemplate method to rpcclient

* [NOD-70] Basic infrastructure for mining simulator

* [NOD-70] Fix txFees in NewBlockTempalte: include value for fee transaction + don't set fee for coinbase = -totalFees.

* [NOD-70] Added capabilities parameter to Client.GetBlockTemplate call

* [NOD-70] Dirty version of mining simulator complete

* [NOD-70] cleaned up mining simulator

* [NOD-70] Added dockerfile to mining simulator

* [NOD-70] Updated base config path of mining simulator to mining_simulator

* [NOD-70] Remove error return from msgblock.AddTransaction - it never returns one

* [NOD-70] Renamed r -> random

* [NOD-70] Move paths initialization of mining simulator to main

* [NOD-70] Cleaned up mining simulator dockerfile

* [NOD-70] Add '--' to tini argument

integration/rpctest/blockgen.go

@@ -194,9 +194,7 @@ func CreateBlock(parentBlock *util.Block, inclusionTxs []*util.Tx,
 		Bits:           net.PowLimitBits,
 	}
 	for _, tx := range blockTxns {
-		if err := block.AddTransaction(tx.MsgTx()); err != nil {
-			return nil, err
-		}
+		block.AddTransaction(tx.MsgTx())
 	}
 
 	found := solveBlock(&block.Header, net.PowLimit)

mining/mining.go

@@ -447,9 +447,9 @@ func (g *BlkTmplGenerator) NewBlockTemplate(payToAddress util.Address) (*BlockTe
 	// a transaction as it is selected for inclusion in the final block.
 	// However, since the total fees aren't known yet, use a dummy value for
 	// the coinbase fee which will be updated later.
-	txFees := make([]uint64, 0, len(sourceTxns))
-	txSigOpCounts := make([]int64, 0, len(sourceTxns))
-	txFees = append(txFees, 0) // Updated once known
+	txFees := make([]uint64, 0, len(sourceTxns)+2)
+	txSigOpCounts := make([]int64, 0, len(sourceTxns)+2)
+	txFees = append(txFees, 0, 0) // For coinbase and fee txs
 	txSigOpCounts = append(txSigOpCounts, numCoinbaseSigOps, feeTxSigOps)
 
 	log.Debugf("Considering %d transactions for inclusion to new block",
@@ -629,7 +629,6 @@ func (g *BlkTmplGenerator) NewBlockTemplate(payToAddress util.Address) (*BlockTe
 	blockSize -= wire.MaxVarIntPayload -
 		uint32(wire.VarIntSerializeSize(uint64(len(blockTxns))))
 	coinbaseTx.MsgTx().TxOut[0].Value += totalFees
-	txFees[0] = -totalFees
 
 	// Calculate the required difficulty for the block.  The timestamp
 	// is potentially adjusted to ensure it comes after the median time of
@@ -665,9 +664,7 @@ func (g *BlkTmplGenerator) NewBlockTemplate(payToAddress util.Address) (*BlockTe
 		Bits:           reqDifficulty,
 	}
 	for _, tx := range blockTxns {
-		if err := msgBlock.AddTransaction(tx.MsgTx()); err != nil {
-			return nil, err
-		}
+		msgBlock.AddTransaction(tx.MsgTx())
 	}
 
 	// Finally, perform a full check on the created block against the chain

mining/simulator/addresslist.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"bufio"
+	"os"
+)
+
+var addressListPath string
+
+func getAddressList() ([]string, error) {
+	file, err := os.Open(addressListPath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	addressList := []string{}
+	for scanner.Scan() {
+		addressList = append(addressList, scanner.Text())
+	}
+
+	return addressList, nil
+}

mining/simulator/connect.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+
+	"github.com/daglabs/btcd/rpcclient"
+)
+
+var certificatePath string
+
+func connectToServers(addressList []string) ([]*rpcclient.Client, error) {
+	clients := make([]*rpcclient.Client, len(addressList))
+
+	cert, err := ioutil.ReadFile(certificatePath)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading certificates file: %s", err)
+	}
+
+	for i, address := range addressList {
+		connCfg := &rpcclient.ConnConfig{
+			Host:         address,
+			Endpoint:     "ws",
+			User:         "user",
+			Pass:         "pass",
+			Certificates: cert,
+		}
+
+		client, err := rpcclient.New(connCfg, nil)
+		if err != nil {
+			return nil, fmt.Errorf("Error connecting to address %s: %s", address, err)
+		}
+
+		clients[i] = client
+
+		log.Printf("Connected to server %s", address)
+	}
+
+	return clients, nil
+}

mining/simulator/docker/Dockerfile

@@ -0,0 +1,29 @@
+# -- multistage docker build: stage #1: build stage
+FROM golang:1.12-alpine AS build
+
+RUN mkdir -p /go/src/github.com/daglabs/btcd
+
+WORKDIR /go/src/github.com/daglabs/btcd
+
+RUN apk add --no-cache curl git 
+RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+
+COPY ./Gopkg.* ./
+
+RUN dep ensure -v -vendor-only
+
+COPY . .
+
+RUN cd mining/simulator && CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o mining_simulator .
+
+# --- multistage docker build: stage #2: runtime image
+FROM alpine
+WORKDIR /app
+
+RUN apk add --no-cache tini
+
+COPY --from=build /go/src/github.com/daglabs/btcd/mining/simulator/mining_simulator /app/
+
+ENTRYPOINT ["/sbin/tini", "--"]
+
+CMD ["/app/mining_simulator"]

mining/simulator/docker/README

@@ -0,0 +1,10 @@
+1. To build docker image invoke following command from btcd root directory:
+    docker build -t mining_simulator -f ./mining/simulator/docker/Dockerfile .
+
+2. To run:
+    a. create folder ~/.btcd/mining_simulator with the following files:
+        rpc.cert - certificate file that all rpc nodes accept
+        addresses - list of node addresses in the format [hostname]:[port]. One node per line
+    b. run:
+         docker run -v ~/.btcd:/root/.btcd -t mining_simulator 
+

mining/simulator/main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os/user"
+	"path"
+	"runtime/debug"
+	"sync/atomic"
+
+	"github.com/daglabs/btcd/rpcclient"
+)
+
+var isRunning int32
+
+func main() {
+	defer handlePanic()
+
+	err := initPaths()
+	if err != nil {
+		panic(fmt.Errorf("Error initializing paths: %s", err))
+	}
+
+	addressList, err := getAddressList()
+	if err != nil {
+		panic(fmt.Errorf("Couldn't load address list: %s", err))
+	}
+
+	clients, err := connectToServers(addressList)
+	if err != nil {
+		panic(fmt.Errorf("Error connecting to servers: %s", err))
+	}
+	defer disconnect(clients)
+
+	atomic.StoreInt32(&isRunning, 1)
+
+	err = mineLoop(clients)
+	if err != nil {
+		panic(fmt.Errorf("Error in main loop: %s", err))
+	}
+}
+
+func initPaths() error {
+	usr, err := user.Current()
+	if err != nil {
+		return fmt.Errorf("Error getting current user: %s", err)
+	}
+
+	basePath := ".btcd/mining_simulator"
+
+	certificatePath = path.Join(usr.HomeDir, basePath, "rpc.cert")
+	addressListPath = path.Join(usr.HomeDir, basePath, "addresses")
+
+	return nil
+}
+
+func disconnect(clients []*rpcclient.Client) {
+	for _, client := range clients {
+		client.Disconnect()
+	}
+}
+
+func handlePanic() {
+	err := recover()
+	if err != nil {
+		log.Printf("Fatal error: %s", err)
+		log.Printf("Stack trace: %s", debug.Stack())
+	}
+}

mining/simulator/mineloop.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"log"
+	"math/rand"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/daglabs/btcd/blockdag"
+	"github.com/daglabs/btcd/btcjson"
+	"github.com/daglabs/btcd/dagconfig/daghash"
+	"github.com/daglabs/btcd/rpcclient"
+	"github.com/daglabs/btcd/util"
+	"github.com/daglabs/btcd/wire"
+)
+
+var random = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func parseBlock(template *btcjson.GetBlockTemplateResult) (*util.Block, error) {
+	// parse parent hashes
+	parentHashes := make([]daghash.Hash, len(template.ParentHashes))
+	for i, parentHash := range template.ParentHashes {
+		hash, err := daghash.NewHashFromStr(parentHash)
+		if err != nil {
+			return nil, fmt.Errorf("Error decoding hash %s: %s", parentHash, err)
+		}
+		parentHashes[i] = *hash
+	}
+
+	// parse Bits
+	bitsInt64, err := strconv.ParseInt(template.Bits, 16, 32)
+	if err != nil {
+		return nil, fmt.Errorf("Error decoding bits %s: %s", template.Bits, err)
+	}
+	bits := uint32(bitsInt64)
+
+	// parse rest of block
+	msgBlock := wire.NewMsgBlock(wire.NewBlockHeader(template.Version, parentHashes, &daghash.Hash{}, &daghash.Hash{}, uint32(bits), 0))
+
+	for i, txResult := range append([]btcjson.GetBlockTemplateResultTx{*template.CoinbaseTxn}, template.Transactions...) {
+		reader := hex.NewDecoder(strings.NewReader(txResult.Data))
+		tx := &wire.MsgTx{}
+		if err := tx.BtcDecode(reader, 0); err != nil {
+			return nil, fmt.Errorf("Error decoding tx #%d: %s", i, err)
+		}
+		msgBlock.AddTransaction(tx)
+	}
+
+	return util.NewBlock(msgBlock), nil
+}
+
+func solveBlock(msgBlock *wire.MsgBlock) {
+	maxNonce := ^uint64(0) // 2^64 - 1
+	targetDifficulty := util.CompactToBig(msgBlock.Header.Bits)
+	for i := uint64(0); i < maxNonce; i++ {
+		msgBlock.Header.Nonce = i
+		hash := msgBlock.BlockHash()
+		if daghash.HashToBig(&hash).Cmp(targetDifficulty) <= 0 {
+			break
+		}
+	}
+
+}
+
+func mineLoop(clients []*rpcclient.Client) error {
+	clientsCount := int64(len(clients))
+
+	for atomic.LoadInt32(&isRunning) == 1 {
+		var currentClient *rpcclient.Client
+		if clientsCount == 1 {
+			currentClient = clients[0]
+		} else {
+			currentClient = clients[random.Int63n(clientsCount)]
+		}
+		log.Printf("Next block will be mined by: %s", currentClient.Host())
+
+		template, err := currentClient.GetBlockTemplate([]string{"coinbasetxn"})
+		if err != nil {
+			return fmt.Errorf("Error getting block template: %s", err)
+		}
+
+		block, err := parseBlock(template)
+		if err != nil {
+			return fmt.Errorf("Error parsing block: %s", err)
+		}
+
+		msgBlock := block.MsgBlock()
+
+		msgBlock.Header.HashMerkleRoot = *blockdag.BuildHashMerkleTreeStore(block.Transactions()).Root()
+		msgBlock.Header.IDMerkleRoot = *blockdag.BuildIDMerkleTreeStore(block.Transactions()).Root()
+
+		solveBlock(msgBlock)
+
+		log.Printf("Found block %s! Submitting", block.Hash())
+
+		err = currentClient.SubmitBlock(block, &btcjson.SubmitBlockOptions{})
+		if err != nil {
+			return fmt.Errorf("Error submitting block: %s", err)
+		}
+	}
+
+	return nil
+}

rpcclient/infrastructure.go

@@ -158,6 +158,11 @@ type Client struct {
 	wg              sync.WaitGroup
 }
 
+// Host returns the host name of the server this client is connected to
+func (c *Client) Host() string {
+	return c.config.Host
+}
+
 // NextID returns the next id to be used when sending a JSON-RPC message.  This
 // ID allows responses to be associated with particular requests per the
 // JSON-RPC specification.  Typically the consumer of the client does not need

rpcclient/mining.go

@@ -307,7 +307,6 @@ func (r FutureSubmitBlockResult) Receive() error {
 	}
 
 	return nil
-
 }
 
 // SubmitBlockAsync returns an instance of a type that can be used to get the
@@ -335,4 +334,40 @@ func (c *Client) SubmitBlock(block *util.Block, options *btcjson.SubmitBlockOpti
 	return c.SubmitBlockAsync(block, options).Receive()
 }
 
-// TODO(davec): Implement GetBlockTemplate
+// FutureGetBlockTemplateResult is a future promise to deliver the result of a
+// GetBlockTemplate RPC invocation (or an applicable error).
+type FutureGetBlockTemplateResult chan *response
+
+// GetBlockTemplateAsync returns an instance of a type that can be used to get
+// the result of the RPC at some future time by invoking the Receive function on
+// the returned instance.
+//
+// See GetBlockTemplate for the blocking version and more details
+func (c *Client) GetBlockTemplateAsync(capabilities []string) FutureGetBlockTemplateResult {
+	request := &btcjson.TemplateRequest{
+		Mode:         "template",
+		Capabilities: capabilities,
+	}
+	cmd := btcjson.NewGetBlockTemplateCmd(request)
+	return c.sendCmd(cmd)
+}
+
+// Receive waits for the response promised by the future and returns an error if
+// any occurred when submitting the block.
+func (r FutureGetBlockTemplateResult) Receive() (*btcjson.GetBlockTemplateResult, error) {
+	res, err := receiveFuture(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var result btcjson.GetBlockTemplateResult
+	if err := json.Unmarshal(res, &result); err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// GetBlockTemplate request a block template from the server, to mine upon
+func (c *Client) GetBlockTemplate(capabilities []string) (*btcjson.GetBlockTemplateResult, error) {
+	return c.GetBlockTemplateAsync(capabilities).Receive()
+}

wire/msgblock.go

@@ -7,9 +7,10 @@ package wire
 import (
 	"bytes"
 	"fmt"
-	"github.com/daglabs/btcd/util/subnetworkid"
 	"io"
 
+	"github.com/daglabs/btcd/util/subnetworkid"
+
 	"github.com/daglabs/btcd/dagconfig/daghash"
 )
 
@@ -46,10 +47,8 @@ type MsgBlock struct {
 }
 
 // AddTransaction adds a transaction to the message.
-func (msg *MsgBlock) AddTransaction(tx *MsgTx) error {
+func (msg *MsgBlock) AddTransaction(tx *MsgTx) {
 	msg.Transactions = append(msg.Transactions, tx)
-	return nil
-
 }
 
 // ClearTransactions removes all transactions from the message.