From 1da045d1b970be42b9b8eab57645ccc55c0edbbc Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Tue, 12 Mar 2019 13:03:08 +0200 Subject: [PATCH] [NOD-30] move wire.RandomUint64() and binarySerializer to util (#204) * [NOD-30] move wire.RandomUint64() and binarySerializer to util * [NOD-30] change const style --- blockdag/fullblocktests/generate.go | 3 +- mining/cpuminer/cpuminer.go | 3 +- mining/mining.go | 3 +- peer/peer.go | 3 +- server/rpc/rpcserver.go | 3 +- server/rpc/rpcwebsocket.go | 3 +- util/binaryserializer/binaryserializer.go | 146 +++++++++++ .../binaryserializer/binaryserializer_test.go | 58 +++++ util/random/random.go | 25 ++ util/random/random_test.go | 73 ++++++ wire/blockheader_test.go | 5 +- wire/common.go | 231 +++--------------- wire/common_test.go | 115 --------- wire/msgping_test.go | 9 +- wire/msgpong_test.go | 7 +- wire/msgtx.go | 21 +- wire/msgversion_test.go | 5 +- wire/netaddress.go | 4 +- 18 files changed, 379 insertions(+), 338 deletions(-) create mode 100644 util/binaryserializer/binaryserializer.go create mode 100644 util/binaryserializer/binaryserializer_test.go create mode 100644 util/random/random.go create mode 100644 util/random/random_test.go diff --git a/blockdag/fullblocktests/generate.go b/blockdag/fullblocktests/generate.go index fe5954622..e6c2a85b1 100644 --- a/blockdag/fullblocktests/generate.go +++ b/blockdag/fullblocktests/generate.go @@ -24,6 +24,7 @@ import ( "github.com/daglabs/btcd/dagconfig/daghash" "github.com/daglabs/btcd/txscript" "github.com/daglabs/btcd/util" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/wire" ) @@ -261,7 +262,7 @@ func opReturnScript(data []byte) []byte { // uniqueOpReturnScript returns a standard provably-pruneable OP_RETURN script // with a random uint64 encoded as the data. func uniqueOpReturnScript() []byte { - rand, err := wire.RandomUint64() + rand, err := random.Uint64() if err != nil { panic(err) } diff --git a/mining/cpuminer/cpuminer.go b/mining/cpuminer/cpuminer.go index 0a8025d85..d3f2f122e 100644 --- a/mining/cpuminer/cpuminer.go +++ b/mining/cpuminer/cpuminer.go @@ -17,6 +17,7 @@ import ( "github.com/daglabs/btcd/dagconfig/daghash" "github.com/daglabs/btcd/mining" "github.com/daglabs/btcd/util" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/wire" ) @@ -213,7 +214,7 @@ func (m *CPUMiner) solveBlock(msgBlock *wire.MsgBlock, blockHeight int32, hashesCompleted := uint64(0) // Choose a random extra nonce for this block template and worker. - extraNonce, err := wire.RandomUint64() + extraNonce, err := random.Uint64() if err != nil { log.Errorf("Unexpected error while generating random "+ "extra nonce offset: %s", err) diff --git a/mining/mining.go b/mining/mining.go index 14d609245..de5e6cb5b 100644 --- a/mining/mining.go +++ b/mining/mining.go @@ -15,6 +15,7 @@ import ( "github.com/daglabs/btcd/dagconfig/daghash" "github.com/daglabs/btcd/txscript" "github.com/daglabs/btcd/util" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/util/subnetworkid" "github.com/daglabs/btcd/wire" ) @@ -395,7 +396,7 @@ func (g *BlkTmplGenerator) NewBlockTemplate(payToAddress util.Address) (*BlockTe // ensure the transaction is not a duplicate transaction (paying the // same value to the same public key address would otherwise be an // identical transaction for block version 1). - extraNonce, err := wire.RandomUint64() + extraNonce, err := random.Uint64() if err != nil { return nil, err } diff --git a/peer/peer.go b/peer/peer.go index 38b498d80..2bb7bad05 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -17,6 +17,7 @@ import ( "sync/atomic" "time" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/util/subnetworkid" "github.com/btcsuite/go-socks/socks" @@ -1891,7 +1892,7 @@ out: for { select { case <-pingTicker.C: - nonce, err := wire.RandomUint64() + nonce, err := random.Uint64() if err != nil { log.Errorf("Not sending ping to %s: %s", p, err) continue diff --git a/server/rpc/rpcserver.go b/server/rpc/rpcserver.go index 768f799af..3c16c30e0 100644 --- a/server/rpc/rpcserver.go +++ b/server/rpc/rpcserver.go @@ -48,6 +48,7 @@ import ( "github.com/daglabs/btcd/util" "github.com/daglabs/btcd/util/fs" "github.com/daglabs/btcd/util/network" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/util/subnetworkid" "github.com/daglabs/btcd/version" "github.com/daglabs/btcd/wire" @@ -2761,7 +2762,7 @@ func handleHelp(s *Server, cmd interface{}, closeChan <-chan struct{}) (interfac // handlePing implements the ping command. func handlePing(s *Server, cmd interface{}, closeChan <-chan struct{}) (interface{}, error) { // Ask server to ping \o_ - nonce, err := wire.RandomUint64() + nonce, err := random.Uint64() if err != nil { return nil, internalRPCError("Not sending ping - failed to "+ "generate nonce: "+err.Error(), "") diff --git a/server/rpc/rpcwebsocket.go b/server/rpc/rpcwebsocket.go index 7d248f992..5dc99e928 100644 --- a/server/rpc/rpcwebsocket.go +++ b/server/rpc/rpcwebsocket.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/util/subnetworkid" "golang.org/x/crypto/ripemd160" @@ -1747,7 +1748,7 @@ func (c *wsClient) WaitForShutdown() { func newWebsocketClient(server *Server, conn *websocket.Conn, remoteAddr string, authenticated bool, isAdmin bool) (*wsClient, error) { - sessionID, err := wire.RandomUint64() + sessionID, err := random.Uint64() if err != nil { return nil, err } diff --git a/util/binaryserializer/binaryserializer.go b/util/binaryserializer/binaryserializer.go new file mode 100644 index 000000000..f1a0d692c --- /dev/null +++ b/util/binaryserializer/binaryserializer.go @@ -0,0 +1,146 @@ +package binaryserializer + +import ( + "encoding/binary" + "io" +) + +// maxItems is the number of buffers to keep in the free +// list to use for binary serialization and deserialization. +const maxItems = 1024 + +// Borrow returns a byte slice from the free list with a length of 8. A new +// buffer is allocated if there are not any available on the free list. +func Borrow() []byte { + var buf []byte + select { + case buf = <-binaryFreeList: + default: + buf = make([]byte, 8) + } + return buf[:8] +} + +// Return puts the provided byte slice back on the free list. The buffer MUST +// have been obtained via the Borrow function and therefore have a cap of 8. +func Return(buf []byte) { + select { + case binaryFreeList <- buf: + default: + // Let it go to the garbage collector. + } +} + +// Uint8 reads a single byte from the provided reader using a buffer from the +// free list and returns it as a uint8. +func Uint8(r io.Reader) (uint8, error) { + buf := Borrow()[:1] + if _, err := io.ReadFull(r, buf); err != nil { + Return(buf) + return 0, err + } + rv := buf[0] + Return(buf) + return rv, nil +} + +// Uint16 reads two bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint16. +func Uint16(r io.Reader, byteOrder binary.ByteOrder) (uint16, error) { + buf := Borrow()[:2] + if _, err := io.ReadFull(r, buf); err != nil { + Return(buf) + return 0, err + } + rv := byteOrder.Uint16(buf) + Return(buf) + return rv, nil +} + +// Uint32 reads four bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint32. +func Uint32(r io.Reader, byteOrder binary.ByteOrder) (uint32, error) { + buf := Borrow()[:4] + if _, err := io.ReadFull(r, buf); err != nil { + Return(buf) + return 0, err + } + rv := byteOrder.Uint32(buf) + Return(buf) + return rv, nil +} + +// Uint64 reads eight bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint64. +func Uint64(r io.Reader, byteOrder binary.ByteOrder) (uint64, error) { + buf := Borrow()[:8] + if _, err := io.ReadFull(r, buf); err != nil { + Return(buf) + return 0, err + } + rv := byteOrder.Uint64(buf) + Return(buf) + return rv, nil +} + +// PutUint8 copies the provided uint8 into a buffer from the free list and +// writes the resulting byte to the given writer. +func PutUint8(w io.Writer, val uint8) error { + buf := Borrow()[:1] + buf[0] = val + _, err := w.Write(buf) + Return(buf) + return err +} + +// PutUint16 serializes the provided uint16 using the given byte order into a +// buffer from the free list and writes the resulting two bytes to the given +// writer. +func PutUint16(w io.Writer, byteOrder binary.ByteOrder, val uint16) error { + buf := Borrow()[:2] + byteOrder.PutUint16(buf, val) + _, err := w.Write(buf) + Return(buf) + return err +} + +// PutUint32 serializes the provided uint32 using the given byte order into a +// buffer from the free list and writes the resulting four bytes to the given +// writer. +func PutUint32(w io.Writer, byteOrder binary.ByteOrder, val uint32) error { + buf := Borrow()[:4] + byteOrder.PutUint32(buf, val) + _, err := w.Write(buf) + Return(buf) + return err +} + +// PutUint64 serializes the provided uint64 using the given byte order into a +// buffer from the free list and writes the resulting eight bytes to the given +// writer. +func PutUint64(w io.Writer, byteOrder binary.ByteOrder, val uint64) error { + buf := Borrow()[:8] + byteOrder.PutUint64(buf, val) + _, err := w.Write(buf) + Return(buf) + return err +} + +// binaryFreeList provides a free list of buffers to use for serializing and +// deserializing primitive integer values to and from io.Readers and io.Writers. +// +// It defines a concurrent safe free list of byte slices (up to the +// maximum number defined by the maxItems constant) that have a +// cap of 8 (thus it supports up to a uint64). It is used to provide temporary +// buffers for serializing and deserializing primitive numbers to and from their +// binary encoding in order to greatly reduce the number of allocations +// required. +// +// For convenience, functions are provided for each of the primitive unsigned +// integers that automatically obtain a buffer from the free list, perform the +// necessary binary conversion, read from or write to the given io.Reader or +// io.Writer, and return the buffer to the free list. +var binaryFreeList = make(chan []byte, maxItems) diff --git a/util/binaryserializer/binaryserializer_test.go b/util/binaryserializer/binaryserializer_test.go new file mode 100644 index 000000000..5b2a6452c --- /dev/null +++ b/util/binaryserializer/binaryserializer_test.go @@ -0,0 +1,58 @@ +package binaryserializer + +import ( + "reflect" + "testing" + "unsafe" +) + +func TestBinaryFreeList(t *testing.T) { + + expectedCapacity := 8 + expectedLength := 8 + + first := Borrow() + if cap(first) != expectedCapacity { + t.Errorf("MsgTx.TestBinaryFreeList: Expected capacity for first %d, but got %d", + expectedCapacity, cap(first)) + } + if len(first) != expectedLength { + t.Errorf("MsgTx.TestBinaryFreeList: Expected length for first %d, but got %d", + expectedLength, len(first)) + } + Return(first) + + // Borrow again, and check that the underlying array is re-used for second + second := Borrow() + if cap(second) != expectedCapacity { + t.Errorf("TestBinaryFreeList: Expected capacity for second %d, but got %d", + expectedCapacity, cap(second)) + } + if len(second) != expectedLength { + t.Errorf("TestBinaryFreeList: Expected length for second %d, but got %d", + expectedLength, len(second)) + } + + firstArrayAddress := underlyingArrayAddress(first) + secondArrayAddress := underlyingArrayAddress(second) + + if firstArrayAddress != secondArrayAddress { + t.Errorf("First underlying array is at address %d and second at address %d, "+ + "which means memory was not re-used", firstArrayAddress, secondArrayAddress) + } + + Return(second) + + // test there's no crash when channel is full because borrowed too much + buffers := make([][]byte, maxItems+1) + for i := 0; i < maxItems+1; i++ { + buffers[i] = Borrow() + } + for i := 0; i < maxItems+1; i++ { + Return(buffers[i]) + } +} + +func underlyingArrayAddress(buf []byte) uint64 { + return uint64((*reflect.SliceHeader)(unsafe.Pointer(&buf)).Data) +} diff --git a/util/random/random.go b/util/random/random.go new file mode 100644 index 000000000..aec8e9bfb --- /dev/null +++ b/util/random/random.go @@ -0,0 +1,25 @@ +package random + +import ( + "crypto/rand" + "encoding/binary" + "io" + + "github.com/daglabs/btcd/util/binaryserializer" +) + +// randomUint64 returns a cryptographically random uint64 value. This +// unexported version takes a reader primarily to ensure the error paths +// can be properly tested by passing a fake reader in the tests. +func randomUint64(r io.Reader) (uint64, error) { + rv, err := binaryserializer.Uint64(r, binary.BigEndian) + if err != nil { + return 0, err + } + return rv, nil +} + +// Uint64 returns a cryptographically random uint64 value. +func Uint64() (uint64, error) { + return randomUint64(rand.Reader) +} diff --git a/util/random/random_test.go b/util/random/random_test.go new file mode 100644 index 000000000..66186afd3 --- /dev/null +++ b/util/random/random_test.go @@ -0,0 +1,73 @@ +package random + +import ( + "fmt" + "io" + "testing" +) + +// fakeRandReader implements the io.Reader interface and is used to force +// errors in the RandomUint64 function. +type fakeRandReader struct { + n int + err error +} + +// Read returns the fake reader error and the lesser of the fake reader value +// and the length of p. +func (r *fakeRandReader) Read(p []byte) (int, error) { + n := r.n + if n > len(p) { + n = len(p) + } + return n, r.err +} + +// TestRandomUint64 exercises the randomness of the random number generator on +// the system by ensuring the probability of the generated numbers. If the RNG +// is evenly distributed as a proper cryptographic RNG should be, there really +// should only be 1 number < 2^56 in 2^8 tries for a 64-bit number. However, +// use a higher number of 5 to really ensure the test doesn't fail unless the +// RNG is just horrendous. +func TestRandomUint64(t *testing.T) { + tries := 1 << 8 // 2^8 + watermark := uint64(1 << 56) // 2^56 + maxHits := 5 + badRNG := "The random number generator on this system is clearly " + + "terrible since we got %d values less than %d in %d runs " + + "when only %d was expected" + + numHits := 0 + for i := 0; i < tries; i++ { + nonce, err := Uint64() + if err != nil { + t.Errorf("RandomUint64 iteration %d failed - err %v", + i, err) + return + } + if nonce < watermark { + numHits++ + } + if numHits > maxHits { + str := fmt.Sprintf(badRNG, numHits, watermark, tries, maxHits) + t.Errorf("Random Uint64 iteration %d failed - %v %v", i, + str, numHits) + return + } + } +} + +// TestRandomUint64Errors uses a fake reader to force error paths to be executed +// and checks the results accordingly. +func TestRandomUint64Errors(t *testing.T) { + // Test short reads. + fr := &fakeRandReader{n: 2, err: io.EOF} + nonce, err := randomUint64(fr) + if err != io.ErrUnexpectedEOF { + t.Errorf("Error not expected value of %v [%v]", + io.ErrUnexpectedEOF, err) + } + if nonce != 0 { + t.Errorf("Nonce is not 0 [%v]", nonce) + } +} diff --git a/wire/blockheader_test.go b/wire/blockheader_test.go index fa7b2a8da..e6aac2faf 100644 --- a/wire/blockheader_test.go +++ b/wire/blockheader_test.go @@ -11,14 +11,15 @@ import ( "time" "github.com/daglabs/btcd/dagconfig/daghash" + "github.com/daglabs/btcd/util/random" "github.com/davecgh/go-spew/spew" ) // TestBlockHeader tests the BlockHeader API. func TestBlockHeader(t *testing.T) { - nonce, err := RandomUint64() + nonce, err := random.Uint64() if err != nil { - t.Errorf("RandomUint64: Error generating nonce: %v", err) + t.Errorf("random.Uint64: Error generating nonce: %v", err) } hashes := []daghash.Hash{mainNetGenesisHash, simNetGenesisHash} diff --git a/wire/common.go b/wire/common.go index 939a36275..3a8bb29f8 100644 --- a/wire/common.go +++ b/wire/common.go @@ -5,7 +5,6 @@ package wire import ( - "crypto/rand" "encoding/binary" "fmt" "io" @@ -13,17 +12,12 @@ import ( "time" "github.com/daglabs/btcd/dagconfig/daghash" + "github.com/daglabs/btcd/util/binaryserializer" "github.com/daglabs/btcd/util/subnetworkid" ) -const ( - // MaxVarIntPayload is the maximum payload size for a variable length integer. - MaxVarIntPayload = 9 - - // binaryFreeListMaxItems is the number of buffers to keep in the free - // list to use for binary serialization and deserialization. - binaryFreeListMaxItems = 1024 -) +// MaxVarIntPayload is the maximum payload size for a variable length integer. +const MaxVarIntPayload = 9 var ( // littleEndian is a convenience variable since binary.LittleEndian is @@ -35,143 +29,6 @@ var ( bigEndian = binary.BigEndian ) -// binaryFreeList defines a concurrent safe free list of byte slices (up to the -// maximum number defined by the binaryFreeListMaxItems constant) that have a -// cap of 8 (thus it supports up to a uint64). It is used to provide temporary -// buffers for serializing and deserializing primitive numbers to and from their -// binary encoding in order to greatly reduce the number of allocations -// required. -// -// For convenience, functions are provided for each of the primitive unsigned -// integers that automatically obtain a buffer from the free list, perform the -// necessary binary conversion, read from or write to the given io.Reader or -// io.Writer, and return the buffer to the free list. -type binaryFreeList chan []byte - -// Borrow returns a byte slice from the free list with a length of 8. A new -// buffer is allocated if there are not any available on the free list. -func (l binaryFreeList) Borrow() []byte { - var buf []byte - select { - case buf = <-l: - default: - buf = make([]byte, 8) - } - return buf[:8] -} - -// Return puts the provided byte slice back on the free list. The buffer MUST -// have been obtained via the Borrow function and therefore have a cap of 8. -func (l binaryFreeList) Return(buf []byte) { - select { - case l <- buf: - default: - // Let it go to the garbage collector. - } -} - -// Uint8 reads a single byte from the provided reader using a buffer from the -// free list and returns it as a uint8. -func (l binaryFreeList) Uint8(r io.Reader) (uint8, error) { - buf := l.Borrow()[:1] - if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) - return 0, err - } - rv := buf[0] - l.Return(buf) - return rv, nil -} - -// Uint16 reads two bytes from the provided reader using a buffer from the -// free list, converts it to a number using the provided byte order, and returns -// the resulting uint16. -func (l binaryFreeList) Uint16(r io.Reader, byteOrder binary.ByteOrder) (uint16, error) { - buf := l.Borrow()[:2] - if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) - return 0, err - } - rv := byteOrder.Uint16(buf) - l.Return(buf) - return rv, nil -} - -// Uint32 reads four bytes from the provided reader using a buffer from the -// free list, converts it to a number using the provided byte order, and returns -// the resulting uint32. -func (l binaryFreeList) Uint32(r io.Reader, byteOrder binary.ByteOrder) (uint32, error) { - buf := l.Borrow()[:4] - if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) - return 0, err - } - rv := byteOrder.Uint32(buf) - l.Return(buf) - return rv, nil -} - -// Uint64 reads eight bytes from the provided reader using a buffer from the -// free list, converts it to a number using the provided byte order, and returns -// the resulting uint64. -func (l binaryFreeList) Uint64(r io.Reader, byteOrder binary.ByteOrder) (uint64, error) { - buf := l.Borrow()[:8] - if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) - return 0, err - } - rv := byteOrder.Uint64(buf) - l.Return(buf) - return rv, nil -} - -// PutUint8 copies the provided uint8 into a buffer from the free list and -// writes the resulting byte to the given writer. -func (l binaryFreeList) PutUint8(w io.Writer, val uint8) error { - buf := l.Borrow()[:1] - buf[0] = val - _, err := w.Write(buf) - l.Return(buf) - return err -} - -// PutUint16 serializes the provided uint16 using the given byte order into a -// buffer from the free list and writes the resulting two bytes to the given -// writer. -func (l binaryFreeList) PutUint16(w io.Writer, byteOrder binary.ByteOrder, val uint16) error { - buf := l.Borrow()[:2] - byteOrder.PutUint16(buf, val) - _, err := w.Write(buf) - l.Return(buf) - return err -} - -// PutUint32 serializes the provided uint32 using the given byte order into a -// buffer from the free list and writes the resulting four bytes to the given -// writer. -func (l binaryFreeList) PutUint32(w io.Writer, byteOrder binary.ByteOrder, val uint32) error { - buf := l.Borrow()[:4] - byteOrder.PutUint32(buf, val) - _, err := w.Write(buf) - l.Return(buf) - return err -} - -// PutUint64 serializes the provided uint64 using the given byte order into a -// buffer from the free list and writes the resulting eight bytes to the given -// writer. -func (l binaryFreeList) PutUint64(w io.Writer, byteOrder binary.ByteOrder, val uint64) error { - buf := l.Borrow()[:8] - byteOrder.PutUint64(buf, val) - _, err := w.Write(buf) - l.Return(buf) - return err -} - -// binarySerializer provides a free list of buffers to use for serializing and -// deserializing primitive integer values to and from io.Readers and io.Writers. -var binarySerializer binaryFreeList = make(chan []byte, binaryFreeListMaxItems) - // errNonCanonicalVarInt is the common format string used for non-canonically // encoded variable length integer errors. var errNonCanonicalVarInt = "non-canonical varint %x - discriminant %x must " + @@ -189,7 +46,7 @@ func readElement(r io.Reader, element interface{}) error { // type assertions first. switch e := element.(type) { case *int32: - rv, err := binarySerializer.Uint32(r, littleEndian) + rv, err := binaryserializer.Uint32(r, littleEndian) if err != nil { return err } @@ -197,7 +54,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *uint32: - rv, err := binarySerializer.Uint32(r, littleEndian) + rv, err := binaryserializer.Uint32(r, littleEndian) if err != nil { return err } @@ -205,7 +62,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *int64: - rv, err := binarySerializer.Uint64(r, littleEndian) + rv, err := binaryserializer.Uint64(r, littleEndian) if err != nil { return err } @@ -213,7 +70,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *uint64: - rv, err := binarySerializer.Uint64(r, littleEndian) + rv, err := binaryserializer.Uint64(r, littleEndian) if err != nil { return err } @@ -221,7 +78,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *bool: - rv, err := binarySerializer.Uint8(r) + rv, err := binaryserializer.Uint8(r) if err != nil { return err } @@ -234,7 +91,7 @@ func readElement(r io.Reader, element interface{}) error { // Unix timestamp encoded as an int64. case *int64Time: - rv, err := binarySerializer.Uint64(r, binary.LittleEndian) + rv, err := binaryserializer.Uint64(r, binary.LittleEndian) if err != nil { return err } @@ -280,7 +137,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *ServiceFlag: - rv, err := binarySerializer.Uint64(r, littleEndian) + rv, err := binaryserializer.Uint64(r, littleEndian) if err != nil { return err } @@ -288,7 +145,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *InvType: - rv, err := binarySerializer.Uint32(r, littleEndian) + rv, err := binaryserializer.Uint32(r, littleEndian) if err != nil { return err } @@ -296,7 +153,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *BitcoinNet: - rv, err := binarySerializer.Uint32(r, littleEndian) + rv, err := binaryserializer.Uint32(r, littleEndian) if err != nil { return err } @@ -304,7 +161,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *BloomUpdateType: - rv, err := binarySerializer.Uint8(r) + rv, err := binaryserializer.Uint8(r) if err != nil { return err } @@ -312,7 +169,7 @@ func readElement(r io.Reader, element interface{}) error { return nil case *RejectCode: - rv, err := binarySerializer.Uint8(r) + rv, err := binaryserializer.Uint8(r) if err != nil { return err } @@ -343,28 +200,28 @@ func writeElement(w io.Writer, element interface{}) error { // type assertions first. switch e := element.(type) { case int32: - err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) + err := binaryserializer.PutUint32(w, littleEndian, uint32(e)) if err != nil { return err } return nil case uint32: - err := binarySerializer.PutUint32(w, littleEndian, e) + err := binaryserializer.PutUint32(w, littleEndian, e) if err != nil { return err } return nil case int64: - err := binarySerializer.PutUint64(w, littleEndian, uint64(e)) + err := binaryserializer.PutUint64(w, littleEndian, uint64(e)) if err != nil { return err } return nil case uint64: - err := binarySerializer.PutUint64(w, littleEndian, e) + err := binaryserializer.PutUint64(w, littleEndian, e) if err != nil { return err } @@ -373,9 +230,9 @@ func writeElement(w io.Writer, element interface{}) error { case bool: var err error if e { - err = binarySerializer.PutUint8(w, 0x01) + err = binaryserializer.PutUint8(w, 0x01) } else { - err = binarySerializer.PutUint8(w, 0x00) + err = binaryserializer.PutUint8(w, 0x00) } if err != nil { return err @@ -421,35 +278,35 @@ func writeElement(w io.Writer, element interface{}) error { return nil case ServiceFlag: - err := binarySerializer.PutUint64(w, littleEndian, uint64(e)) + err := binaryserializer.PutUint64(w, littleEndian, uint64(e)) if err != nil { return err } return nil case InvType: - err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) + err := binaryserializer.PutUint32(w, littleEndian, uint32(e)) if err != nil { return err } return nil case BitcoinNet: - err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) + err := binaryserializer.PutUint32(w, littleEndian, uint32(e)) if err != nil { return err } return nil case BloomUpdateType: - err := binarySerializer.PutUint8(w, uint8(e)) + err := binaryserializer.PutUint8(w, uint8(e)) if err != nil { return err } return nil case RejectCode: - err := binarySerializer.PutUint8(w, uint8(e)) + err := binaryserializer.PutUint8(w, uint8(e)) if err != nil { return err } @@ -475,7 +332,7 @@ func writeElements(w io.Writer, elements ...interface{}) error { // ReadVarInt reads a variable length integer from r and returns it as a uint64. func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { - discriminant, err := binarySerializer.Uint8(r) + discriminant, err := binaryserializer.Uint8(r) if err != nil { return 0, err } @@ -483,7 +340,7 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { var rv uint64 switch discriminant { case 0xff: - sv, err := binarySerializer.Uint64(r, littleEndian) + sv, err := binaryserializer.Uint64(r, littleEndian) if err != nil { return 0, err } @@ -498,7 +355,7 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { } case 0xfe: - sv, err := binarySerializer.Uint32(r, littleEndian) + sv, err := binaryserializer.Uint32(r, littleEndian) if err != nil { return 0, err } @@ -513,7 +370,7 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { } case 0xfd: - sv, err := binarySerializer.Uint16(r, littleEndian) + sv, err := binaryserializer.Uint16(r, littleEndian) if err != nil { return 0, err } @@ -538,30 +395,30 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { // on its value. func WriteVarInt(w io.Writer, pver uint32, val uint64) error { if val < 0xfd { - return binarySerializer.PutUint8(w, uint8(val)) + return binaryserializer.PutUint8(w, uint8(val)) } if val <= math.MaxUint16 { - err := binarySerializer.PutUint8(w, 0xfd) + err := binaryserializer.PutUint8(w, 0xfd) if err != nil { return err } - return binarySerializer.PutUint16(w, littleEndian, uint16(val)) + return binaryserializer.PutUint16(w, littleEndian, uint16(val)) } if val <= math.MaxUint32 { - err := binarySerializer.PutUint8(w, 0xfe) + err := binaryserializer.PutUint8(w, 0xfe) if err != nil { return err } - return binarySerializer.PutUint32(w, littleEndian, uint32(val)) + return binaryserializer.PutUint32(w, littleEndian, uint32(val)) } - err := binarySerializer.PutUint8(w, 0xff) + err := binaryserializer.PutUint8(w, 0xff) if err != nil { return err } - return binarySerializer.PutUint64(w, littleEndian, val) + return binaryserializer.PutUint64(w, littleEndian, val) } // VarIntSerializeSize returns the number of bytes it would take to serialize @@ -672,19 +529,3 @@ func WriteVarBytes(w io.Writer, pver uint32, bytes []byte) error { _, err = w.Write(bytes) return err } - -// randomUint64 returns a cryptographically random uint64 value. This -// unexported version takes a reader primarily to ensure the error paths -// can be properly tested by passing a fake reader in the tests. -func randomUint64(r io.Reader) (uint64, error) { - rv, err := binarySerializer.Uint64(r, bigEndian) - if err != nil { - return 0, err - } - return rv, nil -} - -// RandomUint64 returns a cryptographically random uint64 value. -func RandomUint64() (uint64, error) { - return randomUint64(rand.Reader) -} diff --git a/wire/common_test.go b/wire/common_test.go index bdec08e0f..7472c8a9f 100644 --- a/wire/common_test.go +++ b/wire/common_test.go @@ -6,7 +6,6 @@ package wire import ( "bytes" - "fmt" "io" "reflect" "strings" @@ -50,23 +49,6 @@ var exampleIDMerkleRoot = daghash.Hash{ 0x30, 0xC1, 0xF8, 0xFD, 0xD0, 0xD9, 0x72, 0x87, } -// fakeRandReader implements the io.Reader interface and is used to force -// errors in the RandomUint64 function. -type fakeRandReader struct { - n int - err error -} - -// Read returns the fake reader error and the lesser of the fake reader value -// and the length of p. -func (r *fakeRandReader) Read(p []byte) (int, error) { - n := r.n - if n > len(p) { - n = len(p) - } - return n, r.err -} - // TestElementWire tests wire encode and decode for various element types. This // is mainly to test the "fast" paths in readElement and writeElement which use // type assertions to avoid reflection when possible. @@ -727,100 +709,3 @@ func TestVarBytesOverflowErrors(t *testing.T) { } } - -// TestRandomUint64 exercises the randomness of the random number generator on -// the system by ensuring the probability of the generated numbers. If the RNG -// is evenly distributed as a proper cryptographic RNG should be, there really -// should only be 1 number < 2^56 in 2^8 tries for a 64-bit number. However, -// use a higher number of 5 to really ensure the test doesn't fail unless the -// RNG is just horrendous. -func TestRandomUint64(t *testing.T) { - tries := 1 << 8 // 2^8 - watermark := uint64(1 << 56) // 2^56 - maxHits := 5 - badRNG := "The random number generator on this system is clearly " + - "terrible since we got %d values less than %d in %d runs " + - "when only %d was expected" - - numHits := 0 - for i := 0; i < tries; i++ { - nonce, err := RandomUint64() - if err != nil { - t.Errorf("RandomUint64 iteration %d failed - err %v", - i, err) - return - } - if nonce < watermark { - numHits++ - } - if numHits > maxHits { - str := fmt.Sprintf(badRNG, numHits, watermark, tries, maxHits) - t.Errorf("Random Uint64 iteration %d failed - %v %v", i, - str, numHits) - return - } - } -} - -// TestRandomUint64Errors uses a fake reader to force error paths to be executed -// and checks the results accordingly. -func TestRandomUint64Errors(t *testing.T) { - // Test short reads. - fr := &fakeRandReader{n: 2, err: io.EOF} - nonce, err := randomUint64(fr) - if err != io.ErrUnexpectedEOF { - t.Errorf("Error not expected value of %v [%v]", - io.ErrUnexpectedEOF, err) - } - if nonce != 0 { - t.Errorf("Nonce is not 0 [%v]", nonce) - } -} - -func TestBinaryFreeList(t *testing.T) { - var list binaryFreeList = make(chan []byte, binaryFreeListMaxItems) - - expectedCapacity := 8 - expectedLength := 8 - - first := list.Borrow() - if cap(first) != expectedCapacity { - t.Errorf("MsgTx.TestBinaryFreeList: Expected capacity for first %d, but got %d", - expectedCapacity, cap(first)) - } - if len(first) != expectedLength { - t.Errorf("MsgTx.TestBinaryFreeList: Expected length for first %d, but got %d", - expectedLength, len(first)) - } - list.Return(first) - - // Borrow again, and check that the underlying array is re-used for second - second := list.Borrow() - if cap(second) != expectedCapacity { - t.Errorf("TestBinaryFreeList: Expected capacity for second %d, but got %d", - expectedCapacity, cap(second)) - } - if len(second) != expectedLength { - t.Errorf("TestBinaryFreeList: Expected length for second %d, but got %d", - expectedLength, len(second)) - } - - firstArrayAddress := underlyingArrayAddress(first) - secondArrayAddress := underlyingArrayAddress(second) - - if firstArrayAddress != secondArrayAddress { - t.Errorf("First underlying array is at address %d and second at address %d, "+ - "which means memory was not re-used", firstArrayAddress, secondArrayAddress) - } - - list.Return(second) - - // test there's no crash when channel is full because borrowed too much - buffers := make([][]byte, binaryFreeListMaxItems+1) - for i := 0; i < binaryFreeListMaxItems+1; i++ { - buffers[i] = list.Borrow() - } - for i := 0; i < binaryFreeListMaxItems+1; i++ { - list.Return(buffers[i]) - } -} diff --git a/wire/msgping_test.go b/wire/msgping_test.go index b44c21fe6..64d89e726 100644 --- a/wire/msgping_test.go +++ b/wire/msgping_test.go @@ -10,6 +10,7 @@ import ( "reflect" "testing" + "github.com/daglabs/btcd/util/random" "github.com/davecgh/go-spew/spew" ) @@ -18,9 +19,9 @@ func TestPing(t *testing.T) { pver := ProtocolVersion // Ensure we get the same nonce back out. - nonce, err := RandomUint64() + nonce, err := random.Uint64() if err != nil { - t.Errorf("RandomUint64: Error generating nonce: %v", err) + t.Errorf("random.Uint64: Error generating nonce: %v", err) } msg := NewMsgPing(nonce) if msg.Nonce != nonce { @@ -48,9 +49,9 @@ func TestPing(t *testing.T) { // TestPingCrossProtocol tests the MsgPing API when encoding with the latest // protocol version and decoding with BIP0031Version. func TestPingCrossProtocol(t *testing.T) { - nonce, err := RandomUint64() + nonce, err := random.Uint64() if err != nil { - t.Errorf("RandomUint64: Error generating nonce: %v", err) + t.Errorf("random.Uint64: Error generating nonce: %v", err) } msg := NewMsgPing(nonce) if msg.Nonce != nonce { diff --git a/wire/msgpong_test.go b/wire/msgpong_test.go index 7df9b9d50..ac0904c64 100644 --- a/wire/msgpong_test.go +++ b/wire/msgpong_test.go @@ -10,6 +10,7 @@ import ( "reflect" "testing" + "github.com/daglabs/btcd/util/random" "github.com/davecgh/go-spew/spew" ) @@ -17,9 +18,9 @@ import ( func TestPongLatest(t *testing.T) { pver := ProtocolVersion - nonce, err := RandomUint64() + nonce, err := random.Uint64() if err != nil { - t.Errorf("RandomUint64: error generating nonce: %v", err) + t.Errorf("random.Uint64: error generating nonce: %v", err) } msg := NewMsgPong(nonce) if msg.Nonce != nonce { @@ -66,7 +67,7 @@ func TestPongLatest(t *testing.T) { // TestPongCrossProtocol tests the MsgPong API when encoding with the latest // protocol version and decoding with BIP0031Version. func TestPongCrossProtocol(t *testing.T) { - nonce, err := RandomUint64() + nonce, err := random.Uint64() if err != nil { t.Errorf("Error generating nonce: %v", err) } diff --git a/wire/msgtx.go b/wire/msgtx.go index 975ef3032..823b2335d 100644 --- a/wire/msgtx.go +++ b/wire/msgtx.go @@ -13,6 +13,7 @@ import ( "strconv" "github.com/daglabs/btcd/dagconfig/daghash" + "github.com/daglabs/btcd/util/binaryserializer" "github.com/daglabs/btcd/util/subnetworkid" ) @@ -429,7 +430,7 @@ func (msg *MsgTx) Copy() *MsgTx { // See Deserialize for decoding transactions stored to disk, such as in a // database, as opposed to decoding transactions from the wire. func (msg *MsgTx) BtcDecode(r io.Reader, pver uint32) error { - version, err := binarySerializer.Uint32(r, littleEndian) + version, err := binaryserializer.Uint32(r, littleEndian) if err != nil { return err } @@ -520,7 +521,7 @@ func (msg *MsgTx) BtcDecode(r io.Reader, pver uint32) error { totalScriptSize += uint64(len(to.PkScript)) } - lockTime, err := binarySerializer.Uint64(r, littleEndian) + lockTime, err := binaryserializer.Uint64(r, littleEndian) msg.LockTime = lockTime if err != nil { returnScriptBuffers() @@ -539,7 +540,7 @@ func (msg *MsgTx) BtcDecode(r io.Reader, pver uint32) error { } if !msg.SubnetworkID.IsEqual(subnetworkid.SubnetworkIDNative) { - msg.Gas, err = binarySerializer.Uint64(r, littleEndian) + msg.Gas, err = binaryserializer.Uint64(r, littleEndian) if err != nil { returnScriptBuffers() return err @@ -641,7 +642,7 @@ func (msg *MsgTx) BtcEncode(w io.Writer, pver uint32) error { } func (msg *MsgTx) encode(w io.Writer, pver uint32, encodingFlags txEncoding) error { - err := binarySerializer.PutUint32(w, littleEndian, uint32(msg.Version)) + err := binaryserializer.PutUint32(w, littleEndian, uint32(msg.Version)) if err != nil { return err } @@ -672,7 +673,7 @@ func (msg *MsgTx) encode(w io.Writer, pver uint32, encodingFlags txEncoding) err } } - err = binarySerializer.PutUint64(w, littleEndian, msg.LockTime) + err = binaryserializer.PutUint64(w, littleEndian, msg.LockTime) if err != nil { return err } @@ -688,7 +689,7 @@ func (msg *MsgTx) encode(w io.Writer, pver uint32, encodingFlags txEncoding) err return messageError("MsgTx.BtcEncode", str) } - err = binarySerializer.PutUint64(w, littleEndian, msg.Gas) + err = binaryserializer.PutUint64(w, littleEndian, msg.Gas) if err != nil { return err } @@ -877,7 +878,7 @@ func readOutPoint(r io.Reader, pver uint32, version int32, op *OutPoint) error { return err } - op.Index, err = binarySerializer.Uint32(r, littleEndian) + op.Index, err = binaryserializer.Uint32(r, littleEndian) return err } @@ -889,7 +890,7 @@ func writeOutPoint(w io.Writer, pver uint32, version int32, op *OutPoint) error return err } - return binarySerializer.PutUint32(w, littleEndian, op.Index) + return binaryserializer.PutUint32(w, littleEndian, op.Index) } // readScript reads a variable length byte array that represents a transaction @@ -957,7 +958,7 @@ func writeTxIn(w io.Writer, pver uint32, version int32, ti *TxIn, encodingFlags return err } - return binarySerializer.PutUint64(w, littleEndian, ti.Sequence) + return binaryserializer.PutUint64(w, littleEndian, ti.Sequence) } // readTxOut reads the next sequence of bytes from r as a transaction output @@ -976,7 +977,7 @@ func readTxOut(r io.Reader, pver uint32, version int32, to *TxOut) error { // WriteTxOut encodes to into the bitcoin protocol encoding for a transaction // output (TxOut) to w. func WriteTxOut(w io.Writer, pver uint32, version int32, to *TxOut) error { - err := binarySerializer.PutUint64(w, littleEndian, uint64(to.Value)) + err := binaryserializer.PutUint64(w, littleEndian, uint64(to.Value)) if err != nil { return err } diff --git a/wire/msgversion_test.go b/wire/msgversion_test.go index d6fe0030a..f927986a9 100644 --- a/wire/msgversion_test.go +++ b/wire/msgversion_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/daglabs/btcd/util/random" "github.com/daglabs/btcd/util/subnetworkid" "github.com/davecgh/go-spew/spew" ) @@ -27,9 +28,9 @@ func TestVersion(t *testing.T) { me := NewNetAddress(tcpAddrMe, SFNodeNetwork) tcpAddrYou := &net.TCPAddr{IP: net.ParseIP("192.168.0.1"), Port: 8333} you := NewNetAddress(tcpAddrYou, SFNodeNetwork) - nonce, err := RandomUint64() + nonce, err := random.Uint64() if err != nil { - t.Errorf("RandomUint64: error generating nonce: %v", err) + t.Errorf("random.Uint64: error generating nonce: %v", err) } // Ensure we get the correct data back out. diff --git a/wire/netaddress.go b/wire/netaddress.go index 92c1f5d21..6b521ce3f 100644 --- a/wire/netaddress.go +++ b/wire/netaddress.go @@ -9,6 +9,8 @@ import ( "io" "net" "time" + + "github.com/daglabs/btcd/util/binaryserializer" ) // maxNetAddressPayload returns the max payload size for a bitcoin NetAddress @@ -92,7 +94,7 @@ func readNetAddress(r io.Reader, pver uint32, na *NetAddress, ts bool) error { return err } // Sigh. Bitcoin protocol mixes little and big endian. - port, err := binarySerializer.Uint16(r, bigEndian) + port, err := binaryserializer.Uint16(r, bigEndian) if err != nil { return err }