From 40657a83f5c5d591d199a69e64f3e6309ccfbf6e Mon Sep 17 00:00:00 2001 From: Ori Newman Date: Sun, 3 Nov 2019 12:29:55 +0200 Subject: [PATCH] [NOD-344] Cache ECMH (#445) * [NOD-134] Change newConnMtx to newConnReqMtx * [NOD-344] Change ECMH cache size to 4e6 * [NOD-344] Refactor * [NOD-344] Fix go.mod --- blockdag/utxo_ecmh.go | 29 +++++++++++++++++++++++++++++ blockdag/utxoset.go | 11 ++++------- go.mod | 5 +++-- go.sum | 8 ++------ 4 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 blockdag/utxo_ecmh.go diff --git a/blockdag/utxo_ecmh.go b/blockdag/utxo_ecmh.go new file mode 100644 index 000000000..944175a59 --- /dev/null +++ b/blockdag/utxo_ecmh.go @@ -0,0 +1,29 @@ +package blockdag + +import ( + "bytes" + "github.com/daglabs/btcd/btcec" + "github.com/daglabs/btcd/util/daghash" + "github.com/daglabs/btcd/wire" + "github.com/golang/groupcache/lru" +) + +const ecmhCacheSize = 4_000_000 + +var ecmhCache = lru.New(ecmhCacheSize) + +func utxoMultiset(entry *UTXOEntry, outpoint *wire.Outpoint) (*btcec.Multiset, error) { + w := &bytes.Buffer{} + err := serializeUTXO(w, entry, outpoint) + if err != nil { + return nil, err + } + serializedUTXO := w.Bytes() + utxoHash := daghash.DoubleHashH(serializedUTXO) + if cachedMSPoint, ok := ecmhCache.Get(utxoHash); ok { + return cachedMSPoint.(*btcec.Multiset), nil + } + msPoint := btcec.NewMultiset(btcec.S256()).Add(serializedUTXO) + ecmhCache.Add(utxoHash, msPoint) + return msPoint, nil +} diff --git a/blockdag/utxoset.go b/blockdag/utxoset.go index aab1bc33a..e5065a308 100644 --- a/blockdag/utxoset.go +++ b/blockdag/utxoset.go @@ -1,7 +1,6 @@ package blockdag import ( - "bytes" "errors" "fmt" "math" @@ -843,19 +842,17 @@ func (dus *DiffUTXOSet) WithTransactions(transactions []*wire.MsgTx, blockBlueSc } func addUTXOToMultiset(ms *btcec.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*btcec.Multiset, error) { - w := &bytes.Buffer{} - err := serializeUTXO(w, entry, outpoint) + utxoMS, err := utxoMultiset(entry, outpoint) if err != nil { return nil, err } - return ms.Add(w.Bytes()), nil + return ms.Union(utxoMS), nil } func removeUTXOFromMultiset(ms *btcec.Multiset, entry *UTXOEntry, outpoint *wire.Outpoint) (*btcec.Multiset, error) { - w := &bytes.Buffer{} - err := serializeUTXO(w, entry, outpoint) + utxoMS, err := utxoMultiset(entry, outpoint) if err != nil { return nil, err } - return ms.Remove(w.Bytes()), nil + return ms.Subtract(utxoMS), nil } diff --git a/go.mod b/go.mod index e43c6b04e..1a8f8388d 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/daglabs/btcd -go 1.12 +go 1.13 require ( bou.ke/monkey v1.0.1 github.com/aead/siphash v1.0.1 - github.com/aws/aws-lambda-go v1.13.2 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd github.com/btcsuite/goleveldb v1.0.0 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 github.com/btcsuite/winsvc v1.0.0 github.com/davecgh/go-spew v1.1.1 github.com/golang-migrate/migrate/v4 v4.6.1 + github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.1 github.com/jessevdk/go-flags v1.4.0 @@ -20,5 +20,6 @@ require ( github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec github.com/miekg/dns v1.1.6 github.com/pkg/errors v0.8.1 + github.com/stretchr/testify v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 ) diff --git a/go.sum b/go.sum index 6b493b542..ca81ae42d 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,6 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/aws/aws-lambda-go v1.13.2 h1:8lYuRVn6rESoUNZXdbCmtGB4bBk4vcVYojiHjE4mMrM= -github.com/aws/aws-lambda-go v1.13.2/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= @@ -38,7 +36,6 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk= github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= @@ -87,6 +84,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang-migrate/migrate/v4 v4.6.1 h1:wTTtB3B+HMT1faZOpDgPd6LcwBZ/VwALjzqQ6PPS5G4= github.com/golang-migrate/migrate/v4 v4.6.1/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw= +github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -188,10 +187,8 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -203,7 +200,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=