From d3b1953deb510b51a4082132e26f3b9f1f0ebd0a Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Tue, 24 Mar 2020 16:44:41 +0200 Subject: [PATCH] [NOD-848] optimize utxo diffs serialize allocations (#666) * [NOD-848] Optimize allocations when serializing UTXO diffs * [NOD-848] Use same UTXO serialization everywhere, and use compression as well * [NOD-848] Fix usage of wrong buffer * [NOD-848] Fix tests * [NOD-848] Fix wire tests * [NOD-848] Fix tests * [NOD-848] Remove VLQ * [NOD-848] Fix comments * [NOD-848] Add varint for big endian encoding * [NOD-848] In TestVarIntWire, assume the expected decoded value is the same as the serialization input * [NOD-848] Serialize outpoint index with big endian varint * [NOD-848] Remove p2pk from compression support * [NOD-848] Fix comments * [NOD-848] Remove p2pk from decompression support * [NOD-848] Make entry compression optional * [NOD-848] Fix tests * [NOD-848] Fix comments and var names * [NOD-848] Remove UTXO compression * [NOD-848] Fix tests * [NOD-848] Remove big endian varint * [NOD-848] Fix comments * [NOD-848] Rename ReadVarIntLittleEndian->ReadVarInt and fix WriteVarInt comment * [NOD-848] Add outpointIndexByteOrder variable * [NOD-848] Remove redundant comment * [NOD-848] Fix outpointMaxSerializeSize to the correct value * [NOD-848] Move subBuffer to utils --- blockdag/common_test.go | 11 +- blockdag/compress.go | 584 ---------------------- blockdag/compress_test.go | 436 ---------------- blockdag/dag_test.go | 2 +- blockdag/dagio.go | 206 +++----- blockdag/dagio_test.go | 47 +- blockdag/ghostdag_test.go | 4 +- blockdag/testdata/blk_0_to_4.dat | Bin 2055 -> 2055 bytes blockdag/testdata/blk_3A.dat | Bin 467 -> 467 bytes blockdag/testdata/blk_3B.dat | Bin 354 -> 354 bytes blockdag/testdata/blk_3C.dat | Bin 382 -> 382 bytes blockdag/testdata/blk_3D.dat | Bin 508 -> 508 bytes blockdag/utxoio.go | 204 ++++---- blockdag/utxoset_test.go | 8 +- util/binaryserializer/binaryserializer.go | 17 +- util/block_test.go | 4 +- util/buffers/sub_buffer.go | 48 ++ util/random/random_test.go | 3 +- util/tx_test.go | 3 +- wire/common.go | 6 +- wire/common_test.go | 47 +- wire/message_test.go | 17 +- wire/msgaddr_test.go | 28 +- wire/msgblock_test.go | 11 +- wire/msgblocklocator_test.go | 28 +- wire/msgfeefilter_test.go | 28 +- wire/msgfilteradd_test.go | 29 +- wire/msgfilterload_test.go | 28 +- wire/msggetblockinvs_test.go | 6 +- wire/msggetblocklocator_test.go | 6 +- wire/msggetdata_test.go | 28 +- wire/msginv_test.go | 28 +- wire/msgmerkleblock_test.go | 28 +- wire/msgnotfound_test.go | 29 +- wire/msgping_test.go | 5 +- wire/msgpong_test.go | 28 +- wire/msgreject_test.go | 28 +- wire/msgtx_test.go | 9 +- wire/msgversion_test.go | 28 +- wire/netaddress_test.go | 5 +- 40 files changed, 495 insertions(+), 1532 deletions(-) delete mode 100644 blockdag/compress.go delete mode 100644 blockdag/compress_test.go create mode 100644 util/buffers/sub_buffer.go diff --git a/blockdag/common_test.go b/blockdag/common_test.go index 910c98e80..627a992d2 100644 --- a/blockdag/common_test.go +++ b/blockdag/common_test.go @@ -73,15 +73,8 @@ func loadUTXOSet(filename string) (UTXOSet, error) { return nil, err } - // Serialized utxo entry. - serialized := make([]byte, numBytes) - _, err = io.ReadAtLeast(r, serialized, int(numBytes)) - if err != nil { - return nil, err - } - - // Deserialize it and add it to the view. - entry, err := deserializeUTXOEntry(serialized) + // Deserialize the UTXO entry and add it to the UTXO set. + entry, err := deserializeUTXOEntry(r) if err != nil { return nil, err } diff --git a/blockdag/compress.go b/blockdag/compress.go deleted file mode 100644 index 6e9db0312..000000000 --- a/blockdag/compress.go +++ /dev/null @@ -1,584 +0,0 @@ -// Copyright (c) 2015-2016 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package blockdag - -import ( - "github.com/kaspanet/kaspad/ecc" - "github.com/kaspanet/kaspad/txscript" -) - -// ----------------------------------------------------------------------------- -// A variable length quantity (VLQ) is an encoding that uses an arbitrary number -// of binary octets to represent an arbitrarily large integer. The scheme -// employs a most significant byte (MSB) base-128 encoding where the high bit in -// each byte indicates whether or not the byte is the final one. In addition, -// to ensure there are no redundant encodings, an offset is subtracted every -// time a group of 7 bits is shifted out. Therefore each integer can be -// represented in exactly one way, and each representation stands for exactly -// one integer. -// -// Another nice property of this encoding is that it provides a compact -// representation of values that are typically used to indicate sizes. For -// example, the values 0 - 127 are represented with a single byte, 128 - 16511 -// with two bytes, and 16512 - 2113663 with three bytes. -// -// While the encoding allows arbitrarily large integers, it is artificially -// limited in this code to an unsigned 64-bit integer for efficiency purposes. -// -// Example encodings: -// 0 -> [0x00] -// 127 -> [0x7f] * Max 1-byte value -// 128 -> [0x80 0x00] -// 129 -> [0x80 0x01] -// 255 -> [0x80 0x7f] -// 256 -> [0x81 0x00] -// 16511 -> [0xff 0x7f] * Max 2-byte value -// 16512 -> [0x80 0x80 0x00] -// 32895 -> [0x80 0xff 0x7f] -// 2113663 -> [0xff 0xff 0x7f] * Max 3-byte value -// 270549119 -> [0xff 0xff 0xff 0x7f] * Max 4-byte value -// 2^64-1 -> [0x80 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0xfe 0x7f] -// -// References: -// https://en.wikipedia.org/wiki/Variable-length_quantity -// http://www.codecodex.com/wiki/Variable-Length_Integers -// ----------------------------------------------------------------------------- - -// serializeSizeVLQ returns the number of bytes it would take to serialize the -// passed number as a variable-length quantity according to the format described -// above. -func serializeSizeVLQ(n uint64) int { - size := 1 - for ; n > 0x7f; n = (n >> 7) - 1 { - size++ - } - - return size -} - -// putVLQ serializes the provided number to a variable-length quantity according -// to the format described above and returns the number of bytes of the encoded -// value. The result is placed directly into the passed byte slice which must -// be at least large enough to handle the number of bytes returned by the -// serializeSizeVLQ function or it will panic. -func putVLQ(target []byte, n uint64) int { - offset := 0 - for ; ; offset++ { - // The high bit is set when another byte follows. - highBitMask := byte(0x80) - if offset == 0 { - highBitMask = 0x00 - } - - target[offset] = byte(n&0x7f) | highBitMask - if n <= 0x7f { - break - } - n = (n >> 7) - 1 - } - - // Reverse the bytes so it is MSB-encoded. - for i, j := 0, offset; i < j; i, j = i+1, j-1 { - target[i], target[j] = target[j], target[i] - } - - return offset + 1 -} - -// deserializeVLQ deserializes the provided variable-length quantity according -// to the format described above. It also returns the number of bytes -// deserialized. -func deserializeVLQ(serialized []byte) (uint64, int) { - var n uint64 - var size int - for _, val := range serialized { - size++ - n = (n << 7) | uint64(val&0x7f) - if val&0x80 != 0x80 { - break - } - n++ - } - - return n, size -} - -// ----------------------------------------------------------------------------- -// In order to reduce the size of stored scripts, a domain specific compression -// algorithm is used which recognizes standard scripts and stores them using -// less bytes than the original script. -// -// The general serialized format is: -// -//