Mirroristas/kaspad
2
0
Fork 0
mirror of https://github.com/kaspanet/kaspad.git synced 2025-06-04 13:16:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[NOD-448] Fix initial sync in API Server crashing due to misaligned getBlocks calls (#482)

Browse Source 
* [NOD-448] Change GetBlocksCmd to be able to include both raw and verbose block data.

* [NOD-448] Update sync logic to only make one getBlocks call per page.

* [NOD-448] Make GetBlocks get each block only once.
This commit is contained in:
stasatdaglabs 2019-11-21 12:21:19 +02:00 committed by Svarog
parent 47214121a7
commit db6e9c773f
7 changed files with 116 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
apiserver/sync.go
View File

@ -107,10 +107,10 @@ func syncBlocks(client *jsonrpc.Client) error {
return err
}
var blocks []string
var rawBlocks []btcjson.GetBlockVerboseResult
var rawBlocks []string
var verboseBlocks []btcjson.GetBlockVerboseResult
for {
blocksResult, err := client.GetBlocks(true, false, startHash)
blocksResult, err := client.GetBlocks(true, true, startHash)
if err != nil {
return err
}
@ -118,17 +118,12 @@ func syncBlocks(client *jsonrpc.Client) error {
break
}
rawBlocksResult, err := client.GetBlocks(true, true, startHash)
if err != nil {
return err
}
startHash = &blocksResult.Hashes[len(blocksResult.Hashes)-1]
blocks = append(blocks, blocksResult.Blocks...)
rawBlocks = append(rawBlocks, rawBlocksResult.RawBlocks...)
rawBlocks = append(rawBlocks, blocksResult.RawBlocks...)
verboseBlocks = append(verboseBlocks, blocksResult.VerboseBlocks...)
}
return addBlocks(client, blocks, rawBlocks)
return addBlocks(client, rawBlocks, verboseBlocks)
}
// syncSelectedParentChain attempts to download the selected parent
@ -191,7 +186,7 @@ func findHashOfBluestBlock(mustBeChainBlock bool) (*string, error) {
// fetchBlock downloads the serialized block and raw block data of
// the block with hash blockHash.
func fetchBlock(client *jsonrpc.Client, blockHash *daghash.Hash) (
block string, rawBlock *btcjson.GetBlockVerboseResult, err error) {
rawBlock string, verboseBlock *btcjson.GetBlockVerboseResult, err error) {
msgBlock, err := client.GetBlock(blockHash, nil)
if err != nil {
return "", nil, err
@ -201,21 +196,21 @@ func fetchBlock(client *jsonrpc.Client, blockHash *daghash.Hash) (
if err != nil {
return "", nil, err
}
block = hex.EncodeToString(writer.Bytes())
rawBlock = hex.EncodeToString(writer.Bytes())
rawBlock, err = client.GetBlockVerboseTx(blockHash, nil)
verboseBlock, err = client.GetBlockVerboseTx(blockHash, nil)
if err != nil {
return "", nil, err
}
return block, rawBlock, nil
return rawBlock, verboseBlock, nil
}
// addBlocks inserts data in the given blocks and rawBlocks pairwise
// addBlocks inserts data in the given rawBlocks and verboseBlocks pairwise
// into the database. See addBlock for further details.
func addBlocks(client *jsonrpc.Client, blocks []string, rawBlocks []btcjson.GetBlockVerboseResult) error {
func addBlocks(client *jsonrpc.Client, rawBlocks []string, verboseBlocks []btcjson.GetBlockVerboseResult) error {
for i, rawBlock := range rawBlocks {
block := blocks[i]
err := addBlock(client, block, rawBlock)
verboseBlock := verboseBlocks[i]
err := addBlock(client, rawBlock, verboseBlock)
if err != nil {
return err
}
@ -235,12 +230,12 @@ func doesBlockExist(dbTx *gorm.DB, blockHash string) (bool, error) {
return !httpserverutils.IsDBRecordNotFoundError(dbErrors), nil
}
// addBlocks inserts all the data that could be gleaned out of the serialized
// addBlocks inserts all the data that could be gleaned out of the verbose
// block and raw block data into the database. This includes transactions,
// subnetworks, and addresses.
// Note that if this function may take a nil dbTx, in which case it would start
// a database transaction by itself and commit it before returning.
func addBlock(client *jsonrpc.Client, block string, rawBlock btcjson.GetBlockVerboseResult) error {
func addBlock(client *jsonrpc.Client, rawBlock string, verboseBlock btcjson.GetBlockVerboseResult) error {
db, err := database.DB()
if err != nil {
return err
@ -249,7 +244,7 @@ func addBlock(client *jsonrpc.Client, block string, rawBlock btcjson.GetBlockVer
defer dbTx.RollbackUnlessCommitted()
// Skip this block if it already exists.
blockExists, err := doesBlockExist(dbTx, rawBlock.Hash)
blockExists, err := doesBlockExist(dbTx, verboseBlock.Hash)
if err != nil {
return err
}
@ -258,21 +253,21 @@ func addBlock(client *jsonrpc.Client, block string, rawBlock btcjson.GetBlockVer
return nil
}
dbBlock, err := insertBlock(dbTx, rawBlock)
dbBlock, err := insertBlock(dbTx, verboseBlock)
if err != nil {
return err
}
err = insertBlockParents(dbTx, rawBlock, dbBlock)
err = insertBlockParents(dbTx, verboseBlock, dbBlock)
if err != nil {
return err
}
err = insertBlockData(dbTx, block, dbBlock)
err = insertRawBlockData(dbTx, rawBlock, dbBlock)
if err != nil {
return err
}
blockMass := uint64(0)
for i, transaction := range rawBlock.RawTx {
for i, transaction := range verboseBlock.RawTx {
dbSubnetwork, err := insertSubnetwork(dbTx, &transaction, client)
if err != nil {
return err
@ -307,32 +302,32 @@ func addBlock(client *jsonrpc.Client, block string, rawBlock btcjson.GetBlockVer
// If the block was previously missing, remove it from
// the missing blocks collection.
if _, ok := missingBlocks[rawBlock.Hash]; ok {
delete(missingBlocks, rawBlock.Hash)
if _, ok := missingBlocks[verboseBlock.Hash]; ok {
delete(missingBlocks, verboseBlock.Hash)
}
return nil
}
func insertBlock(dbTx *gorm.DB, rawBlock btcjson.GetBlockVerboseResult) (*dbmodels.Block, error) {
bits, err := strconv.ParseUint(rawBlock.Bits, 16, 32)
func insertBlock(dbTx *gorm.DB, verboseBlock btcjson.GetBlockVerboseResult) (*dbmodels.Block, error) {
bits, err := strconv.ParseUint(verboseBlock.Bits, 16, 32)
if err != nil {
return nil, err
}
dbBlock := dbmodels.Block{
BlockHash: rawBlock.Hash,
Version: rawBlock.Version,
HashMerkleRoot: rawBlock.HashMerkleRoot,
AcceptedIDMerkleRoot: rawBlock.AcceptedIDMerkleRoot,
UTXOCommitment: rawBlock.UTXOCommitment,
Timestamp: time.Unix(rawBlock.Time, 0),
BlockHash: verboseBlock.Hash,
Version: verboseBlock.Version,
HashMerkleRoot: verboseBlock.HashMerkleRoot,
AcceptedIDMerkleRoot: verboseBlock.AcceptedIDMerkleRoot,
UTXOCommitment: verboseBlock.UTXOCommitment,
Timestamp: time.Unix(verboseBlock.Time, 0),
Bits: uint32(bits),
Nonce: rawBlock.Nonce,
BlueScore: rawBlock.BlueScore,
Nonce: verboseBlock.Nonce,
BlueScore: verboseBlock.BlueScore,
IsChainBlock: false, // This must be false for updateSelectedParentChain to work properly
}
// Set genesis block as the initial chain block
if len(rawBlock.ParentHashes) == 0 {
if len(verboseBlock.ParentHashes) == 0 {
dbBlock.IsChainBlock = true
}
dbResult := dbTx.Create(&dbBlock)
@ -343,14 +338,14 @@ func insertBlock(dbTx *gorm.DB, rawBlock btcjson.GetBlockVerboseResult) (*dbmode
return &dbBlock, nil
}
func insertBlockParents(dbTx *gorm.DB, rawBlock btcjson.GetBlockVerboseResult, dbBlock *dbmodels.Block) error {
func insertBlockParents(dbTx *gorm.DB, verboseBlock btcjson.GetBlockVerboseResult, dbBlock *dbmodels.Block) error {
// Exit early if this is the genesis block
if len(rawBlock.ParentHashes) == 0 {
if len(verboseBlock.ParentHashes) == 0 {
return nil
}
hashesIn := make([]string, len(rawBlock.ParentHashes))
for i, parentHash := range rawBlock.ParentHashes {
hashesIn := make([]string, len(verboseBlock.ParentHashes))
for i, parentHash := range verboseBlock.ParentHashes {
hashesIn[i] = parentHash
}
var dbParents []dbmodels.Block
@ -361,10 +356,10 @@ func insertBlockParents(dbTx *gorm.DB, rawBlock btcjson.GetBlockVerboseResult, d
if httpserverutils.HasDBError(dbErrors) {
return httpserverutils.NewErrorFromDBErrors("failed to find blocks: ", dbErrors)
}
if len(dbParents) != len(rawBlock.ParentHashes) {
missingParents := make([]string, 0, len(rawBlock.ParentHashes)-len(dbParents))
if len(dbParents) != len(verboseBlock.ParentHashes) {
missingParents := make([]string, 0, len(verboseBlock.ParentHashes)-len(dbParents))
outerLoop:
for _, parentHash := range rawBlock.ParentHashes {
for _, parentHash := range verboseBlock.ParentHashes {
for _, dbParent := range dbParents {
if dbParent.BlockHash == parentHash {
continue outerLoop
@ -372,7 +367,7 @@ func insertBlockParents(dbTx *gorm.DB, rawBlock btcjson.GetBlockVerboseResult, d
}
missingParents = append(missingParents, parentHash)
}
return errors.Errorf("some parents are missing for block %s: %s", rawBlock.Hash, strings.Join(missingParents, ", "))
return errors.Errorf("some parents are missing for block %s: %s", verboseBlock.Hash, strings.Join(missingParents, ", "))
}
for _, dbParent := range dbParents {
@ -389,8 +384,8 @@ func insertBlockParents(dbTx *gorm.DB, rawBlock btcjson.GetBlockVerboseResult, d
return nil
}
func insertBlockData(dbTx *gorm.DB, block string, dbBlock *dbmodels.Block) error {
blockData, err := hex.DecodeString(block)
func insertRawBlockData(dbTx *gorm.DB, rawBlock string, dbBlock *dbmodels.Block) error {
blockData, err := hex.DecodeString(rawBlock)
if err != nil {
return err
}
@ -950,12 +945,12 @@ func processBlockAddedMsgs(client *jsonrpc.Client) {
hash := blockAdded.Header.BlockHash()
log.Debugf("Getting block %s from the RPC server", hash)
block, rawBlock, err := fetchBlock(client, hash)
rawBlock, verboseBlock, err := fetchBlock(client, hash)
if err != nil {
log.Warnf("Could not fetch block %s: %s", hash, err)
return
}
err = addBlock(client, block, *rawBlock)
err = addBlock(client, rawBlock, *verboseBlock)
if err != nil {
log.Errorf("Could not insert block %s: %s", hash, err)
return
@ -1023,11 +1018,11 @@ func handleMissingParent(client *jsonrpc.Client, missingParentHash string) error
if err != nil {
return errors.Errorf("Could not create hash: %s", err)
}
block, rawBlock, err := fetchBlock(client, hash)
rawBlock, verboseBlock, err := fetchBlock(client, hash)
if err != nil {
return errors.Errorf("Could not fetch block %s: %s", hash, err)
}
err = addBlock(client, block, *rawBlock)
err = addBlock(client, rawBlock, *verboseBlock)
if err != nil {
return errors.Errorf("Could not insert block %s: %s", hash, err)
}

14
btcjson/dagsvrcmds.go
View File

@ -157,18 +157,18 @@ func NewGetBlockCmd(hash string, verbose, verboseTx *bool, subnetworkID *string)
// GetBlocksCmd defines the getBlocks JSON-RPC command.
type GetBlocksCmd struct {
IncludeBlocks bool `json:"includeBlocks"`
VerboseBlocks bool `json:"verboseBlocks"`
StartHash *string `json:"startHash"`
IncludeRawBlockData bool `json:"includeRawBlockData"`
IncludeVerboseBlockData bool `json:"includeVerboseBlockData"`
StartHash *string `json:"startHash"`
}
// NewGetBlocksCmd returns a new instance which can be used to issue a
// GetGetBlocks JSON-RPC command.
func NewGetBlocksCmd(includeBlocks bool, verboseBlocks bool, startHash *string) *GetBlocksCmd {
func NewGetBlocksCmd(includeRawBlockData bool, includeVerboseBlockData bool, startHash *string) *GetBlocksCmd {
return &GetBlocksCmd{
IncludeBlocks: includeBlocks,
VerboseBlocks: verboseBlocks,
StartHash: startHash,
IncludeRawBlockData: includeRawBlockData,
IncludeVerboseBlockData: includeVerboseBlockData,
StartHash: startHash,
}
}

6
btcjson/dagsvrcmds_test.go
View File

@ -200,9 +200,9 @@ func TestDAGSvrCmds(t *testing.T) {
},
marshalled: `{"jsonrpc":"1.0","method":"getBlocks","params":[true,true,"123"],"id":1}`,
unmarshalled: &btcjson.GetBlocksCmd{
IncludeBlocks: true,
VerboseBlocks: true,
StartHash: btcjson.String("123"),
IncludeRawBlockData: true,
IncludeVerboseBlockData: true,
StartHash: btcjson.String("123"),
},
},
{

6
btcjson/dagsvrresults.go
View File

@ -507,7 +507,7 @@ type GetChainFromBlockResult struct {
// GetBlocksResult models the data from the getBlocks command.
type GetBlocksResult struct {
Hashes []string `json:"hashes"`
Blocks []string `json:"blocks"`
RawBlocks []GetBlockVerboseResult `json:"rawBlocks"`
Hashes []string `json:"hashes"`
RawBlocks []string `json:"rawBlocks"`
VerboseBlocks []GetBlockVerboseResult `json:"verboseBlocks"`
}

8
rpcclient/dag.go
View File

@ -133,15 +133,15 @@ func (r FutureGetBlocksResult) Receive() (*btcjson.GetBlocksResult, error) {
// returned instance.
//
// See GetBlocks for the blocking version and more details.
func (c *Client) GetBlocksAsync(includeBlocks bool, verboseBlocks bool, startHash *string) FutureGetBlocksResult {
cmd := btcjson.NewGetBlocksCmd(includeBlocks, verboseBlocks, startHash)
func (c *Client) GetBlocksAsync(includeRawBlockData bool, IncludeVerboseBlockData bool, startHash *string) FutureGetBlocksResult {
cmd := btcjson.NewGetBlocksCmd(includeRawBlockData, IncludeVerboseBlockData, startHash)
return c.sendCmd(cmd)
}
// GetBlocks returns the blocks starting from startHash up to the virtual ordered
// by blue score.
func (c *Client) GetBlocks(includeBlocks bool, verboseBlocks bool, startHash *string) (*btcjson.GetBlocksResult, error) {
return c.GetBlocksAsync(includeBlocks, verboseBlocks, startHash).Receive()
func (c *Client) GetBlocks(includeRawBlockData bool, includeVerboseBlockData bool, startHash *string) (*btcjson.GetBlocksResult, error) {
return c.GetBlocksAsync(includeRawBlockData, includeVerboseBlockData, startHash).Receive()
}
// FutureGetBlockVerboseResult is a future promise to deliver the result of a

59
server/rpc/handle_get_blocks.go
View File

@ -4,6 +4,7 @@ import (
"encoding/hex"
"github.com/daglabs/btcd/btcjson"
"github.com/daglabs/btcd/database"
"github.com/daglabs/btcd/util"
"github.com/daglabs/btcd/util/daghash"
)
@ -48,39 +49,41 @@ func handleGetBlocks(s *Server, cmd interface{}, closeChan <-chan struct{}) (int
}
result := &btcjson.GetBlocksResult{
Hashes: hashes,
Blocks: nil,
Hashes: hashes,
RawBlocks: nil,
VerboseBlocks: nil,
}
// If the user specified to include the blocks, collect them as well.
if c.IncludeBlocks {
if c.VerboseBlocks {
getBlockVerboseResults, err := hashesToGetBlockVerboseResults(s, blockHashes)
// Include more data if requested
if c.IncludeRawBlockData || c.IncludeVerboseBlockData {
blockBytesSlice, err := hashesToBlockBytes(s, blockHashes)
if err != nil {
return nil, err
}
if c.IncludeRawBlockData {
result.RawBlocks = blockBytesToStrings(blockBytesSlice)
}
if c.IncludeVerboseBlockData {
verboseBlocks, err := blockBytesToBlockVerboseResults(s, blockBytesSlice)
if err != nil {
return nil, err
}
result.RawBlocks = getBlockVerboseResults
} else {
blocks, err := hashesToBlockStrings(s, blockHashes)
if err != nil {
return nil, err
}
result.Blocks = blocks
result.VerboseBlocks = verboseBlocks
}
}
return result, nil
}
func hashesToBlockStrings(s *Server, hashes []*daghash.Hash) ([]string, error) {
blocks := make([]string, len(hashes))
func hashesToBlockBytes(s *Server, hashes []*daghash.Hash) ([][]byte, error) {
blocks := make([][]byte, len(hashes))
err := s.cfg.DB.View(func(dbTx database.Tx) error {
for i, hash := range hashes {
blockBytes, err := dbTx.FetchBlock(hash)
if err != nil {
return err
}
blocks[i] = hex.EncodeToString(blockBytes)
blocks[i] = blockBytes
}
return nil
})
@ -89,3 +92,27 @@ func hashesToBlockStrings(s *Server, hashes []*daghash.Hash) ([]string, error) {
}
return blocks, nil
}
func blockBytesToStrings(blockBytesSlice [][]byte) []string {
rawBlocks := make([]string, len(blockBytesSlice))
for i, blockBytes := range blockBytesSlice {
rawBlocks[i] = hex.EncodeToString(blockBytes)
}
return rawBlocks
}
func blockBytesToBlockVerboseResults(s *Server, blockBytesSlice [][]byte) ([]btcjson.GetBlockVerboseResult, error) {
verboseBlocks := make([]btcjson.GetBlockVerboseResult, len(blockBytesSlice))
for i, blockBytes := range blockBytesSlice {
block, err := util.NewBlockFromBytes(blockBytes)
if err != nil {
return nil, err
}
getBlockVerboseResult, err := buildGetBlockVerboseResult(s, block, false)
if err != nil {
return nil, err
}
verboseBlocks[i] = *getBlockVerboseResult
}
return verboseBlocks, nil
}

16
server/rpc/rpcserverhelp.go
View File

@ -178,16 +178,16 @@ var helpDescsEnUS = map[string]string{
"getBlock--result0": "Hex-encoded bytes of the serialized block",
// GetBlocksCmd help.
"getBlocks--synopsis": "Return the blocks starting from startHash up to the virtual ordered by blue score.",
"getBlocks-includeBlocks": "If set to true - the block contents would be also included.",
"getBlocks-verboseBlocks": "If set to true - each block is returned as a JSON object",
"getBlocks-startHash": "Hash of the block with the bottom blue score. If this hash is unknown - returns an error.",
"getBlocks--result0": "Blocks starting from startHash. The result may contains up to 1000 blocks. For the remainder, call the command again with the bluest block's hash.",
"getBlocks--synopsis": "Return the blocks starting from startHash up to the virtual ordered by blue score.",
"getBlocks-includeRawBlockData": "If set to true - the raw block data would be also included.",
"getBlocks-includeVerboseBlockData": "If set to true - the verbose block data would also be included.",
"getBlocks-startHash": "Hash of the block with the bottom blue score. If this hash is unknown - returns an error.",
"getBlocks--result0": "Blocks starting from startHash. The result may contains up to 1000 blocks. For the remainder, call the command again with the bluest block's hash.",
// GetChainFromBlockResult help.
"getBlocksResult-hashes": "List of hashes from StartHash (excluding StartHash) ordered by smallest blue score to greatest.",
"getBlocksResult-blocks": "If includeBlocks=true - contains the block contents. Otherwise - omitted.",
"getBlocksResult-rawBlocks": "If includeBlocks=true and verboseBlocks=true - each block is returned as a JSON object. Otherwise - hex encoded string.",
"getBlocksResult-hashes": "List of hashes from StartHash (excluding StartHash) ordered by smallest blue score to greatest.",
"getBlocksResult-rawBlocks": "If includeBlocks=true - contains the block contents. Otherwise - omitted.",
"getBlocksResult-verboseBlocks": "If includeBlocks=true and verboseBlocks=true - each block is returned as a JSON object. Otherwise - hex encoded string.",
// GetBlockChainInfoCmd help.
"getBlockDagInfo--synopsis": "Returns information about the current blockDAG state and the status of any active soft-fork deployments.",