From d401caced5dbd4a9347a8eb8e8fba81f2dd2576e Mon Sep 17 00:00:00 2001 From: D-Stacks Date: Fri, 14 Oct 2022 00:47:14 +0200 Subject: [PATCH] store accepting + including block + index --- domain/txindex/model.go | 43 ++--- domain/txindex/serialization.go | 33 +++- domain/txindex/serialization_test.go | 57 ++++--- domain/txindex/store.go | 90 +++++----- domain/txindex/txindex.go | 239 ++++++++++++++++++--------- 5 files changed, 288 insertions(+), 174 deletions(-) diff --git a/domain/txindex/model.go b/domain/txindex/model.go index c7b268a1c..e5d1fa1fa 100644 --- a/domain/txindex/model.go +++ b/domain/txindex/model.go @@ -1,43 +1,34 @@ package txindex import ( - "encoding/hex" - "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" ) // TXAcceptanceChange is the set of changes made to the TX index after // a successful update type TXAcceptanceChange struct { - Added map[externalapi.DomainTransactionID]*externalapi.DomainHash - Removed map[externalapi.DomainTransactionID]*externalapi.DomainHash + Added map[externalapi.DomainTransactionID]*TxData + Removed map[externalapi.DomainTransactionID]*TxData } +//TxData holds tx data stored in the TXIndex database +type TxData struct { + IncludingBlockHash *externalapi.DomainHash + AcceptingBlockHash *externalapi.DomainHash + IncludingIndex uint32 +} + +//TxIDsToTxIndexData is a map of TxIDs to corrospnding TxIndexData +type TxIDsToTxIndexData map[externalapi.DomainTransactionID]*TxData + //TxIDsToBlockHashes is a map of TxIDs to corrospnding blockHashes -type TxIDsToBlockHashes map[*externalapi.DomainTransactionID]*externalapi.DomainHash +type TxIDsToBlockHashes map[externalapi.DomainTransactionID]*externalapi.DomainHash //TxIDsToBlocks is a map of TxIDs to corrospnding blocks -type TxIDsToBlocks map[*externalapi.DomainTransactionID]*externalapi.DomainBlock +type TxIDsToBlocks map[externalapi.DomainTransactionID]*externalapi.DomainBlock //TxIDsToConfirmations is a map of TxIDs to corrospnding Confirmations -type TxIDsToConfirmations map[*externalapi.DomainTransactionID]int64 +type TxIDsToConfirmations map[externalapi.DomainTransactionID]int64 -// ConvertDomainHashToString converts the given DomainHash to a string -func ConvertDomainHashToString(blockHash *externalapi.DomainHash) string { - return hex.EncodeToString(blockHash.ByteSlice()) -} - -// ConvertStringToDomainHash converts the given string to a domainHash -func ConvertStringToDomainHash(stringDomainHash string) (*externalapi.DomainHash, error) { - return externalapi.NewDomainHashFromString(stringDomainHash) -} - -// ConvertTXIDToString converts the given DomainHash to a string -func ConvertTXIDToString(txID *externalapi.DomainTransactionID) string { - return hex.EncodeToString(txID.ByteSlice()) -} - -// ConvertStringTXID converts the given string to a domainHash -func ConvertStringTXID(stringDomainTransactionID string) (*externalapi.DomainTransactionID, error) { - return externalapi.NewDomainTransactionIDFromString(stringDomainTransactionID) -} +//TxIDsToBlueScores is a map of TxIDs to corrospnding Confirmations +type TxIDsToBlueScores map[externalapi.DomainTransactionID]uint64 diff --git a/domain/txindex/serialization.go b/domain/txindex/serialization.go index 14a95c76f..f5e0897b1 100644 --- a/domain/txindex/serialization.go +++ b/domain/txindex/serialization.go @@ -2,9 +2,10 @@ package txindex import ( "encoding/binary" + "io" + "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" "github.com/pkg/errors" - "io" ) func serializeHashes(hashes []*externalapi.DomainHash) []byte { @@ -40,3 +41,33 @@ func deserializeHashes(serializedHashes []byte) ([]*externalapi.DomainHash, erro return hashes, nil } + +func deserializeTxIndexData(serializedTxIndexData []byte) (*TxData, error) { + var err error + + deserializedTxIndexData := &TxData{} + deserializedTxIndexData.IncludingBlockHash, err = externalapi.NewDomainHashFromByteSlice(serializedTxIndexData[:32]) + if err != nil { + return nil, err + } + deserializedTxIndexData.AcceptingBlockHash, err = externalapi.NewDomainHashFromByteSlice(serializedTxIndexData[32:64]) + if err != nil { + return nil, err + } + deserializedTxIndexData.IncludingIndex = binary.BigEndian.Uint32(serializedTxIndexData[64:68]) + + return deserializedTxIndexData, nil +} + +func serializeTxIndexData(blockTxIndexData *TxData) []byte { + indexBytes := make([]byte, 4) + binary.BigEndian.PutUint32(indexBytes, blockTxIndexData.IncludingIndex) + serializedTxIndexData := append( + append( + blockTxIndexData.IncludingBlockHash.ByteSlice(), + blockTxIndexData.AcceptingBlockHash.ByteSlice()..., + ), + indexBytes..., + ) + return serializedTxIndexData +} diff --git a/domain/txindex/serialization_test.go b/domain/txindex/serialization_test.go index 34ff59dc3..665fc0deb 100644 --- a/domain/txindex/serialization_test.go +++ b/domain/txindex/serialization_test.go @@ -2,43 +2,42 @@ package txindex import ( "encoding/binary" - "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" - "github.com/pkg/errors" - "io" "math/rand" "testing" + + "github.com/kaspanet/kaspad/domain/consensus/model/externalapi" ) -func Test_serializeHashes(t *testing.T) { +func Test_serializeTxIndexData(t *testing.T) { r := rand.New(rand.NewSource(0)) - for length := 0; length < 32; length++ { - hashes := make([]*externalapi.DomainHash, length) - for i := range hashes { - var hashBytes [32]byte - r.Read(hashBytes[:]) - hashes[i] = externalapi.NewDomainHashFromByteArray(&hashBytes) - } - result, err := deserializeHashes(serializeHashes(hashes)) - if err != nil { - t.Fatalf("Failed deserializing hashes: %v", err) - } - if !externalapi.HashesEqual(hashes, result) { - t.Fatalf("Expected \n %s \n==\n %s\n", hashes, result) - } + serializedtxIndex := make([]byte, 68) // 32 bytes including block hash 32 bytes accepting blockhash and 4 bytes uint32 + r.Read(serializedtxIndex[:]) + includingBlockHash, err := externalapi.NewDomainHashFromByteSlice(serializedtxIndex[:32]) + if err != nil { + t.Fatalf(err.Error()) } -} + acceptingBlockHash, err := externalapi.NewDomainHashFromByteSlice(serializedtxIndex[32:64]) + if err != nil { + t.Fatalf(err.Error()) + } + includingIndex := binary.BigEndian.Uint32(serializedtxIndex[64:68]) -func Test_deserializeHashesFailure(t *testing.T) { - hashes := []*externalapi.DomainHash{ - externalapi.NewDomainHashFromByteArray(&[externalapi.DomainHashSize]byte{1}), - externalapi.NewDomainHashFromByteArray(&[externalapi.DomainHashSize]byte{2}), - externalapi.NewDomainHashFromByteArray(&[externalapi.DomainHashSize]byte{3}), + testdeserializedtxIndex := &TxData{ + IncludingBlockHash: includingBlockHash, + AcceptingBlockHash: acceptingBlockHash, + IncludingIndex: includingIndex, } - serialized := serializeHashes(hashes) - binary.LittleEndian.PutUint64(serialized[:8], uint64(len(hashes)+1)) - _, err := deserializeHashes(serialized) - if !errors.Is(err, io.ErrUnexpectedEOF) { - t.Fatalf("Expected error to be EOF, instead got: %v", err) + + result, err := deserializeTxIndexData(serializeTxIndexData(testdeserializedtxIndex)) + if err != nil { + t.Fatalf("Failed deserializing txIndexData: %v", err) + } + if !testdeserializedtxIndex.IncludingBlockHash.Equal(result.IncludingBlockHash) { + t.Fatalf("Expected including block hash: \n %s \n Got: \n %s\n", testdeserializedtxIndex.IncludingBlockHash.String(), result.IncludingBlockHash.String()) + } else if !testdeserializedtxIndex.AcceptingBlockHash.Equal(result.AcceptingBlockHash) { + t.Fatalf("Expected accepting block hash \n %s \n Got: \n %s\n", testdeserializedtxIndex.AcceptingBlockHash.String(), result.AcceptingBlockHash.String()) + } else if testdeserializedtxIndex.IncludingIndex != result.IncludingIndex { + t.Fatalf("Expected including index \n %d \n Got: \n %d\n", testdeserializedtxIndex.IncludingIndex, result.IncludingIndex) } } diff --git a/domain/txindex/store.go b/domain/txindex/store.go index 38be9d19d..4bcb01f49 100644 --- a/domain/txindex/store.go +++ b/domain/txindex/store.go @@ -13,8 +13,8 @@ var pruningPointKey = database.MakeBucket([]byte("")).Key([]byte("tx-index-prunn type txIndexStore struct { database database.Database - toAdd map[externalapi.DomainTransactionID]*externalapi.DomainHash - toRemove map[externalapi.DomainTransactionID]*externalapi.DomainHash + toAdd map[externalapi.DomainTransactionID]*TxData + toRemove map[externalapi.DomainTransactionID]*TxData virtualParents []*externalapi.DomainHash pruningPoint *externalapi.DomainHash } @@ -22,8 +22,8 @@ type txIndexStore struct { func newTXIndexStore(database database.Database) *txIndexStore { return &txIndexStore{ database: database, - toAdd: make(map[externalapi.DomainTransactionID]*externalapi.DomainHash), - toRemove: make(map[externalapi.DomainTransactionID]*externalapi.DomainHash), + toAdd: make(map[externalapi.DomainTransactionID]*TxData), + toRemove: make(map[externalapi.DomainTransactionID]*TxData), virtualParents: nil, pruningPoint: nil, } @@ -60,22 +60,32 @@ func (tis *txIndexStore) deleteAll() error { return nil } -func (tis *txIndexStore) add(txID externalapi.DomainTransactionID, blockHash *externalapi.DomainHash) { - log.Tracef("Adding %s Txs from blockHash %s", txID.String(), ConvertDomainHashToString(blockHash)) +func (tis *txIndexStore) add(txID externalapi.DomainTransactionID, includingIndex uint32, + includingBlockHash *externalapi.DomainHash, acceptingBlockHash *externalapi.DomainHash) { + log.Tracef("Adding %s Txs from blockHash %s", txID.String(), includingBlockHash.String()) delete(tis.toRemove, txID) //adding takes precedence - tis.toAdd[txID] = blockHash + tis.toAdd[txID] = &TxData{ + IncludingBlockHash: includingBlockHash, + IncludingIndex: includingIndex, + AcceptingBlockHash: acceptingBlockHash, + } } -func (tis *txIndexStore) remove(txID externalapi.DomainTransactionID, blockHash *externalapi.DomainHash) { - log.Tracef("Removing %s Txs from blockHash %s", txID.String(), ConvertDomainHashToString(blockHash)) +func (tis *txIndexStore) remove(txID externalapi.DomainTransactionID, includingIndex uint32, + includingBlockHash *externalapi.DomainHash, acceptingBlockHash *externalapi.DomainHash) { + log.Tracef("Removing %s Txs from blockHash %s", txID.String(), includingBlockHash.String()) if _, found := tis.toAdd[txID]; !found { //adding takes precedence - tis.toRemove[txID] = blockHash + tis.toRemove[txID] = &TxData{ + IncludingBlockHash: includingBlockHash, + IncludingIndex: includingIndex, + AcceptingBlockHash: acceptingBlockHash, + } } } func (tis *txIndexStore) discardAllButPruningPoint() { - tis.toAdd = make(map[externalapi.DomainTransactionID]*externalapi.DomainHash) - tis.toRemove = make(map[externalapi.DomainTransactionID]*externalapi.DomainHash) + tis.toAdd = make(map[externalapi.DomainTransactionID]*TxData) + tis.toRemove = make(map[externalapi.DomainTransactionID]*TxData) tis.virtualParents = nil } @@ -90,10 +100,10 @@ func (tis *txIndexStore) commit() error { defer dbTransaction.RollbackUnlessClosed() - for toAddTxID, blockHash := range tis.toAdd { + for toAddTxID, txData := range tis.toAdd { delete(tis.toRemove, toAddTxID) //safeguard key := tis.convertTxIDToKey(txAcceptedIndexBucket, toAddTxID) - dbTransaction.Put(key, blockHash.ByteSlice()) + dbTransaction.Put(key, serializeTxIndexData(txData)) if err != nil { return err } @@ -138,10 +148,10 @@ func (tis *txIndexStore) updateAndCommitPruningPointWithoutTransaction(pruningPo } func (tis *txIndexStore) commitTxIDsWithoutTransaction() error { - for txID, blockHash := range tis.toAdd { + for txID, txData := range tis.toAdd { delete(tis.toRemove, txID) //adding takes precedence key := tis.convertTxIDToKey(txAcceptedIndexBucket, txID) - err := tis.database.Put(key, blockHash.ByteSlice()) + err := tis.database.Put(key, serializeTxIndexData(txData)) if err != nil { return err } @@ -189,18 +199,18 @@ func (tis *txIndexStore) convertTxIDToKey(bucket *database.Bucket, txID external } func (tis *txIndexStore) stagedData() ( - toAdd map[externalapi.DomainTransactionID]*externalapi.DomainHash, - toRemove map[externalapi.DomainTransactionID]*externalapi.DomainHash, + toAdd map[externalapi.DomainTransactionID]*TxData, + toRemove map[externalapi.DomainTransactionID]*TxData, virtualParents []*externalapi.DomainHash, pruningPoint *externalapi.DomainHash) { - toAddClone := make(map[externalapi.DomainTransactionID]*externalapi.DomainHash) - toRemoveClone := make(map[externalapi.DomainTransactionID]*externalapi.DomainHash) - for txID, blockHash := range tis.toAdd { - toAddClone[txID] = blockHash + toAddClone := make(map[externalapi.DomainTransactionID]*TxData) + toRemoveClone := make(map[externalapi.DomainTransactionID]*TxData) + for txID, txData := range tis.toAdd { + toAddClone[txID] = txData } - for txID, blockHash := range tis.toRemove { - toRemoveClone[txID] = blockHash + for txID, txData := range tis.toRemove { + toRemoveClone[txID] = txData } return toAddClone, toRemoveClone, tis.virtualParents, tis.pruningPoint } @@ -209,14 +219,14 @@ func (tis *txIndexStore) isAnythingStaged() bool { return len(tis.toAdd) > 0 || len(tis.toRemove) > 0 } -func (tis *txIndexStore) getTxAcceptingBlockHash(txID *externalapi.DomainTransactionID) (blockHash *externalapi.DomainHash, found bool, err error) { +func (tis *txIndexStore) getTxData(txID *externalapi.DomainTransactionID) (txData *TxData, found bool, err error) { if tis.isAnythingStaged() { return nil, false, errors.Errorf("cannot get TX accepting Block hash while staging isn't empty") } key := tis.convertTxIDToKey(txAcceptedIndexBucket, *txID) - serializedAcceptingBlockHash, err := tis.database.Get(key) + serializedTxData, err := tis.database.Get(key) if err != nil { if database.IsNotFoundError(err) { return nil, false, nil @@ -224,39 +234,43 @@ func (tis *txIndexStore) getTxAcceptingBlockHash(txID *externalapi.DomainTransac return nil, false, err } - acceptingBlockHash, err := externalapi.NewDomainHashFromByteSlice(serializedAcceptingBlockHash) + deserializedTxData, err := deserializeTxIndexData(serializedTxData) if err != nil { return nil, false, err } - return acceptingBlockHash, true, nil + return deserializedTxData, true, nil } -func (tis *txIndexStore) getTxAcceptingBlockHashes(txIDs []*externalapi.DomainTransactionID) (acceptingBlockHashes TxIDsToBlockHashes, found bool, err error) { +func (tis *txIndexStore) getTxsData(txIDs []*externalapi.DomainTransactionID) ( + txsData TxIDsToTxIndexData, notFoundTxIDs []*externalapi.DomainTransactionID, err error) { if tis.isAnythingStaged() { - return nil, false, errors.Errorf("cannot get TX accepting Block hash while staging isn't empty") + return nil, nil, errors.Errorf("cannot get TX accepting Block hash while staging isn't empty") } keys := make([]*database.Key, len(txIDs)) - acceptingBlockHashes = make(TxIDsToBlockHashes) + txsData = make(TxIDsToTxIndexData) + notFoundTxIDs = make([]*externalapi.DomainTransactionID, 0) for i, key := range keys { key = tis.convertTxIDToKey(txAcceptedIndexBucket, *txIDs[i]) - serializedAcceptingBlockHash, err := tis.database.Get(key) + serializedTxData, err := tis.database.Get(key) if err != nil { if database.IsNotFoundError(err) { - continue //ignore not found errors we expect this to happen frequently with queries + notFoundTxIDs = append(notFoundTxIDs, txIDs[i]) + } else { + return nil, nil, err } - return nil, false, err } - acceptingBlockHash, err := externalapi.NewDomainHashFromByteSlice(serializedAcceptingBlockHash) + deserializedTxData, err := deserializeTxIndexData(serializedTxData) if err != nil { - return nil, false, err + return nil, nil, err } - acceptingBlockHashes[txIDs[i]] = acceptingBlockHash + + txsData[*txIDs[i]] = deserializedTxData } - return acceptingBlockHashes, true, nil + return txsData, notFoundTxIDs, nil } diff --git a/domain/txindex/txindex.go b/domain/txindex/txindex.go index 4c6d3f154..182fc939b 100644 --- a/domain/txindex/txindex.go +++ b/domain/txindex/txindex.go @@ -187,15 +187,21 @@ func (ti *TXIndex) addTXIDs(selectedParentChainChanges *externalapi.SelectedChai if err != nil { return err } - for i := range chainBlocksChunk { + for i, acceptingBlock := range chainBlocksChunk { chainBlockAcceptanceData := chainBlocksAcceptanceData[i] for _, blockAcceptanceData := range chainBlockAcceptanceData { - for _, transactionAcceptanceData := range blockAcceptanceData.TransactionAcceptanceData { - log.Tracef("TX index Adding: %d transactions", len(blockAcceptanceData.TransactionAcceptanceData)) + for j, transactionAcceptanceData := range blockAcceptanceData.TransactionAcceptanceData { + log.Warnf("TX index Adding: %d transactions", len(blockAcceptanceData.TransactionAcceptanceData)) if transactionAcceptanceData.IsAccepted { + if err != nil { + return err + } + transactionID := consensushashing.TransactionID(transactionAcceptanceData.Transaction) ti.store.add( - *consensushashing.TransactionID(transactionAcceptanceData.Transaction), - blockAcceptanceData.BlockHash, + *transactionID, + uint32(j), //index of including block where transaction is found + blockAcceptanceData.BlockHash, //this is the including block + acceptingBlock, //this is the accepting block ) } } @@ -222,15 +228,17 @@ func (ti *TXIndex) removeTXIDs(selectedParentChainChanges *externalapi.SelectedC if err != nil { return err } - for i := range chainBlocksChunk { + for i, acceptingBlockHash := range chainBlocksChunk { chainBlockAcceptanceData := chainBlocksAcceptanceData[i] for _, blockAcceptanceData := range chainBlockAcceptanceData { log.Tracef("TX index Removing: %d transactions", len(blockAcceptanceData.TransactionAcceptanceData)) - for _, transactionAcceptanceData := range blockAcceptanceData.TransactionAcceptanceData { + for j, transactionAcceptanceData := range blockAcceptanceData.TransactionAcceptanceData { if transactionAcceptanceData.IsAccepted { ti.store.remove( *consensushashing.TransactionID(transactionAcceptanceData.Transaction), + uint32(j), blockAcceptanceData.BlockHash, + acceptingBlockHash, ) } } @@ -242,14 +250,15 @@ func (ti *TXIndex) removeTXIDs(selectedParentChainChanges *externalapi.SelectedC } // TXAcceptingBlockHash returns the accepting block hash for for the given txID -func (ti *TXIndex) TXAcceptingBlockHash(txID *externalapi.DomainTransactionID) (acceptingBlockHash *externalapi.DomainHash, found bool, err error) { +func (ti *TXIndex) TXAcceptingBlockHash(txID *externalapi.DomainTransactionID) ( + acceptingBlockHash *externalapi.DomainHash, found bool, err error) { onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXAcceptingBlockHash") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHash, found, err = ti.store.getTxAcceptingBlockHash(txID) + txData, found, err := ti.store.getTxData(txID) if err != nil { return nil, false, err } @@ -257,26 +266,29 @@ func (ti *TXIndex) TXAcceptingBlockHash(txID *externalapi.DomainTransactionID) ( return nil, false, nil } - return acceptingBlockHash, found, nil + return txData.AcceptingBlockHash, found, nil } // TXAcceptingBlockHashes returns the accepting block hashes for for the given txIDs -func (ti *TXIndex) TXAcceptingBlockHashes(txIDs []*externalapi.DomainTransactionID) (acceptingBlockHashes TxIDsToBlockHashes, found bool, err error) { - onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXAcceptingBlockHash") +func (ti *TXIndex) TXAcceptingBlockHashes(txIDs []*externalapi.DomainTransactionID) ( + txIDsToAcceptingBlockHashes TxIDsToBlockHashes, missingTxIds []*externalapi.DomainTransactionID, err error) { + onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXAcceptingBlockHashes") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHashes, found, err = ti.store.getTxAcceptingBlockHashes(txIDs) + txIDsToTxIndexData, missingTxIds, err := ti.store.getTxsData(txIDs) if err != nil { - return nil, false, err - } - if !found { - return nil, false, nil + return nil, nil, err } - return acceptingBlockHashes, found, nil + txIDsToAcceptingBlockHashes = make(TxIDsToBlockHashes) + for txID, txIndexData := range txIDsToTxIndexData { + txIDsToAcceptingBlockHashes[txID] = txIndexData.AcceptingBlockHash + } + + return txIDsToAcceptingBlockHashes, missingTxIds, nil } // TXAcceptingBlock returns the accepting block for for the given txID @@ -288,13 +300,16 @@ func (ti *TXIndex) TXAcceptingBlock(txID *externalapi.DomainTransactionID) ( ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHash, found, err := ti.store.getTxAcceptingBlockHash(txID) + txIndexData, found, err := ti.store.getTxData(txID) if err != nil { return nil, false, err } - acceptingBlock, err := ti.domain.Consensus().GetBlock(acceptingBlockHash) + acceptingBlock, err := ti.domain.Consensus().GetBlockEvenIfHeaderOnly(txIndexData.AcceptingBlockHash) if err != nil { + if database.IsNotFoundError(err) { + return nil, false, fmt.Errorf("accepting block %s missing for txID %s ", txIndexData.AcceptingBlockHash.String(), txID.String()) + } return nil, false, err } return acceptingBlock, true, nil @@ -302,113 +317,104 @@ func (ti *TXIndex) TXAcceptingBlock(txID *externalapi.DomainTransactionID) ( // TXAcceptingBlocks returns the accepting blocks for for the given txIDs func (ti *TXIndex) TXAcceptingBlocks(txIDs []*externalapi.DomainTransactionID) ( - acceptingBlocks TxIDsToBlocks, found bool, err error) { - onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXAcceptingBlock") + txIDsToAcceptingBlocks TxIDsToBlocks, notFound []*externalapi.DomainTransactionID, err error) { + onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXAcceptingBlocks") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHashTxIDPairs, found, err := ti.store.getTxAcceptingBlockHashes(txIDs) + txIDsToTxIndexData, notFound, err := ti.store.getTxsData(txIDs) if err != nil { - return nil, false, err + return nil, nil, err } - acceptingBlocks = make(TxIDsToBlocks) - i := 0 - for txID, blockHash := range acceptingBlockHashTxIDPairs { - acceptingBlocks[txID], err = ti.domain.Consensus().GetBlock(blockHash) + txIDsToAcceptingBlocks = make(TxIDsToBlocks) + + for txID, txIndexData := range txIDsToTxIndexData { + txIDsToAcceptingBlocks[txID], err = ti.domain.Consensus().GetBlockEvenIfHeaderOnly(txIndexData.AcceptingBlockHash) if err != nil { if database.IsNotFoundError(err) { - continue // ignore - } else { - return nil, false, err + return nil, nil, fmt.Errorf("accepting block %s missing for txID %s ", txIndexData.IncludingBlockHash.String(), txID.String()) } + return nil, notFound, err } - i++ } - return acceptingBlocks, true, nil + return txIDsToAcceptingBlocks, notFound, nil } // GetTX returns the domain transaction for for the given txID func (ti *TXIndex) GetTX(txID *externalapi.DomainTransactionID) ( - block *externalapi.DomainTransaction, found bool, err error) { + tx *externalapi.DomainTransaction, found bool, err error) { onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.GetTX") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHash, found, err := ti.store.getTxAcceptingBlockHash(txID) + txIndexData, found, err := ti.store.getTxData(txID) if err != nil { return nil, false, err } - acceptingBlock, err := ti.domain.Consensus().GetBlock(acceptingBlockHash) + acceptingBlock, err := ti.domain.Consensus().GetBlock(txIndexData.AcceptingBlockHash) if err != nil { return nil, false, err } - for i := range acceptingBlock.Transactions { - if consensushashing.TransactionID(acceptingBlock.Transactions[i]).Equal(txID) { - return acceptingBlock.Transactions[i].Clone(), true, nil - } - } - - return nil, false, fmt.Errorf("Could not find transaction with ID %s in Txindex database", txID.String()) + return acceptingBlock.Transactions[txIndexData.IncludingIndex], true, nil } // GetTXs returns the domain transaction for for the given txIDs func (ti *TXIndex) GetTXs(txIDs []*externalapi.DomainTransactionID) ( - txs []*externalapi.DomainTransaction, found bool, err error) { + txs []*externalapi.DomainTransaction, notFound []*externalapi.DomainTransactionID, err error) { onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.GetTXs") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHashes, found, err := ti.store.getTxAcceptingBlockHashes(txIDs) + txIDsToTxIndexData, notFound, err := ti.store.getTxsData(txIDs) if err != nil { - return nil, false, err + return nil, nil, err } - txs = make([]*externalapi.DomainTransaction, 0) - for txID, acceptingBlockHash := range acceptingBlockHashes { - acceptingBlock, err := ti.domain.Consensus().GetBlock(acceptingBlockHash) + txs = make([]*externalapi.DomainTransaction, len(txIDsToTxIndexData)) + i := 0 + + for txID, txIndexData := range txIDsToTxIndexData { + includingBlock, err := ti.domain.Consensus().GetBlockEvenIfHeaderOnly(txIndexData.IncludingBlockHash) + if err != nil { if database.IsNotFoundError(err) { - continue // ignore - } else { - return nil, false, err - } - } - for _, tx := range acceptingBlock.Transactions { - if consensushashing.TransactionID(tx).Equal(txID) { - txs = append(txs, tx) + return nil, nil, fmt.Errorf("including block %s missing for txID %s ", txIndexData.IncludingBlockHash.String(), txID.String()) } + return nil, nil, err } + txs[i] = includingBlock.Transactions[txIndexData.IncludingIndex] + i++ } - return txs, true, nil + return txs, notFound, nil } // GetTXConfirmations returns the tx confirmations for for the given txID func (ti *TXIndex) GetTXConfirmations(txID *externalapi.DomainTransactionID) ( - Confirmations int64, found bool, err error) { + confirmations int64, found bool, err error) { onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.GetTXConfirmations") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHash, found, err := ti.store.getTxAcceptingBlockHash(txID) + txdata, found, err := ti.store.getTxData(txID) if err != nil { return 0, false, err } - acceptingBlockHeader, err := ti.domain.Consensus().GetBlockHeader(acceptingBlockHash) + acceptingBlockHeader, err := ti.domain.Consensus().GetBlockHeader(txdata.AcceptingBlockHash) if err != nil { return -1, false, err } @@ -423,7 +429,7 @@ func (ti *TXIndex) GetTXConfirmations(txID *externalapi.DomainTransactionID) ( // GetTXsConfirmations returns the tx confirmations for for the given txIDs func (ti *TXIndex) GetTXsConfirmations(txIDs []*externalapi.DomainTransactionID) ( - Confirmations TxIDsToConfirmations, found bool, err error) { + txIDsToConfirmations TxIDsToConfirmations, notFound []*externalapi.DomainTransactionID, err error) { onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.GetTXsConfirmations") defer onEnd() @@ -432,51 +438,124 @@ func (ti *TXIndex) GetTXsConfirmations(txIDs []*externalapi.DomainTransactionID) virtualBlock, err := ti.domain.Consensus().GetVirtualInfo() if err != nil { - return nil, false, err + return nil, nil, err } - acceptingBlockHashes, _, err := ti.store.getTxAcceptingBlockHashes(txIDs) + txIDsToTxIndexData, _, err := ti.store.getTxsData(txIDs) if err != nil { - return nil, false, err + return nil, nil, err } - Confirmations = make(TxIDsToConfirmations) - for txID, acceptingBlockHash := range acceptingBlockHashes { - acceptingBlockHeader, err := ti.domain.Consensus().GetBlockHeader(acceptingBlockHash) + txIDsToConfirmations = make(TxIDsToConfirmations) + for txID, txIndexData := range txIDsToTxIndexData { + acceptingBlockHeader, err := ti.domain.Consensus().GetBlockHeader(txIndexData.AcceptingBlockHash) if err != nil { - return nil, false, err + if database.IsNotFoundError(err) { + return nil, nil, fmt.Errorf("including block %s missing for txID %s ", txIndexData.IncludingBlockHash.String(), txID.String()) + } + return nil, nil, err } - Confirmations[txID] = int64(virtualBlock.BlueScore - acceptingBlockHeader.BlueScore()) + txIDsToConfirmations[txID] = int64(virtualBlock.BlueScore - acceptingBlockHeader.BlueScore()) } - return Confirmations, true, nil + return txIDsToConfirmations, notFound, nil } // TXIncludingBlockHash returns the including block hash for the given txID func (ti *TXIndex) TXIncludingBlockHash(txID *externalapi.DomainTransactionID) (includingBlockHash *externalapi.DomainHash, found bool, err error) { - onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXAcceptingBlock") + onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXIncludingBlockHash") defer onEnd() ti.mutex.Lock() defer ti.mutex.Unlock() - acceptingBlockHash, found, err := ti.store.getTxAcceptingBlockHash(txID) + txIndexData, found, err := ti.store.getTxData(txID) if err != nil { return nil, false, err } - acceptanceData, err := ti.domain.Consensus().GetBlockAcceptanceData(acceptingBlockHash) + return txIndexData.IncludingBlockHash, true, nil +} + +// TXIncludingBlockHashes returns the including block hashes for for the given txI +func (ti *TXIndex) TXIncludingBlockHashes(txIDs []*externalapi.DomainTransactionID) ( + txIDsToIncludinglockHashes TxIDsToBlockHashes, missingTxIds []*externalapi.DomainTransactionID, err error) { + onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXIncludingBlockHashes") + defer onEnd() + + ti.mutex.Lock() + defer ti.mutex.Unlock() + + txIDsToTxIndexData, notFound, err := ti.store.getTxsData(txIDs) if err != nil { - return nil, false, err + return nil, nil, err } - for _, blockAcceptanceData := range acceptanceData { - for _, transactionAcceptanceData := range blockAcceptanceData.TransactionAcceptanceData { - if consensushashing.TransactionID(transactionAcceptanceData.Transaction).Equal(txID) { - return blockAcceptanceData.BlockHash, true, nil + txIDsToIncludinglockHashes = make(TxIDsToBlockHashes) + + for txID, txIndexData := range txIDsToTxIndexData { + txIDsToIncludinglockHashes[txID] = txIndexData.IncludingBlockHash + } + + return txIDsToIncludinglockHashes, notFound, nil +} + +// TXIncludingBlocks returns the including block hashes for for the given txIDs +func (ti *TXIndex) TXIncludingBlocks(txIDs []*externalapi.DomainTransactionID) ( + txIDsToIncludingBlocks TxIDsToBlocks, notFound []*externalapi.DomainTransactionID, err error) { + onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.TXIncludingBlocks") + defer onEnd() + + ti.mutex.Lock() + defer ti.mutex.Unlock() + + txIDsToTxIndexData, notFound, err := ti.store.getTxsData(txIDs) + if err != nil { + return nil, nil, err + } + + txIDsToIncludingBlocks = make(TxIDsToBlocks) + + for txID, txIndexData := range txIDsToTxIndexData { + txIDsToIncludingBlocks[txID], err = ti.domain.Consensus().GetBlockEvenIfHeaderOnly(txIndexData.IncludingBlockHash) + if err != nil { + if database.IsNotFoundError(err) { + return nil, nil, fmt.Errorf("including block %s missing for txID %s ", txIndexData.IncludingBlockHash.String(), txID.String()) } + return nil, nil, err } } - return nil, false, fmt.Errorf("Could not find including blockHash for transaction with ID %s in Txindex database", txID.String()) + return txIDsToIncludingBlocks, notFound, nil +} + +// GetTXsBlueScores returns the tx's accepting bluescore for for the given txID +// Note: this is a optimization function to store and dynamically calc. tx confirmations with access to to virtual bluescore +// such as in the case of rpc confirmation notification listeners +func (ti *TXIndex) GetTXsBlueScores(txIDs []*externalapi.DomainTransactionID) ( + txIDsToBlueScores TxIDsToBlueScores, notFound []*externalapi.DomainTransactionID, err error) { + onEnd := logger.LogAndMeasureExecutionTime(log, "TXIndex.GetTXsBlueScores") + defer onEnd() + + ti.mutex.Lock() + defer ti.mutex.Unlock() + + txIDsToTxIndexData, notFound, err := ti.store.getTxsData(txIDs) + if err != nil { + return nil, nil, err + } + + txIDsToBlueScores = make(TxIDsToBlueScores) + for txID, txIndexData := range txIDsToTxIndexData { + acceptingBlockHeader, err := ti.domain.Consensus().GetBlockHeader(txIndexData.AcceptingBlockHash) + if err != nil { + if database.IsNotFoundError(err) { + return nil, nil, fmt.Errorf("Accepting block %s missing for txID %s ", txIndexData.AcceptingBlockHash.String(), txID.String()) + } + return nil, nil, err + } + txIDsToBlueScores[txID] = acceptingBlockHeader.BlueScore() + } + + return txIDsToBlueScores, notFound, nil }