From c2ea25860a6eb8c182a3d9e9fc0aea6a9abb2dfd Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Fri, 31 Aug 2018 16:52:59 +0200 Subject: [PATCH] Problem: CI build fails when using memoized from_dict Solution: Clear from_dict cache --- bigchaindb/backend/localmongodb/query.py | 6 ++-- bigchaindb/common/memoize.py | 30 ++++++++++++++----- bigchaindb/common/transaction.py | 5 ++-- bigchaindb/lib.py | 4 +-- .../upsert_validator/validator_election.py | 10 +++---- tests/common/test_transaction.py | 4 +-- tests/conftest.py | 7 +++-- 7 files changed, 41 insertions(+), 25 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index c85e0854..e5b1c723 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -91,10 +91,8 @@ def get_assets(conn, asset_ids): @register_query(LocalMongoDBConnection) def get_spent(conn, transaction_id, output): - query = {'inputs.fulfills': { - 'transaction_id': transaction_id, - 'output_index': output}} - + query = {'inputs.fulfills': {'transaction_id': transaction_id, + 'output_index': output}} return conn.run( conn.collection('transactions') .find(query, {'_id': 0})) diff --git a/bigchaindb/common/memoize.py b/bigchaindb/common/memoize.py index 9342ea84..b814e512 100644 --- a/bigchaindb/common/memoize.py +++ b/bigchaindb/common/memoize.py @@ -17,25 +17,41 @@ def memoize_from_dict(func): @functools.wraps(func) def memoized_func(*args, **kwargs): - args = list(args) - args[1] = HDict(args[1]) - new_args = tuple(args) - return from_dict(func, *new_args, **kwargs) + + if args[1].get('id', None): + args = list(args) + args[1] = HDict(args[1]) + new_args = tuple(args) + return from_dict(func, *new_args, **kwargs) + else: + return func(*args, **kwargs) return memoized_func +class ToDictWrapper(): + def __init__(self, tx): + self.tx = tx + + def __eq__(self, other): + return self.tx.id == other.tx.id + + def __hash__(self): + return hash(self.tx.id) + + @lru_cache(maxsize=16384) -def to_dict(func, *args, **kwargs): - return func(*args, **kwargs) +def to_dict(func, tx_wrapped): + return func(tx_wrapped.tx) def memoize_to_dict(func): @functools.wraps(func) def memoized_func(*args, **kwargs): + if args[0].id: - return to_dict(func, *args, **kwargs) + return to_dict(func, ToDictWrapper(args[0])) else: return func(*args, **kwargs) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index cccc9078..1097deea 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1012,7 +1012,6 @@ class Transaction(object): return all(validate(i, cond) for i, cond in enumerate(output_condition_uris)) - # @memoize_input_valid @lru_cache(maxsize=16384) def _input_valid(self, input_, operation, message, output_condition_uri=None): """Validates a single Input against a single Output. @@ -1063,7 +1062,7 @@ class Transaction(object): def __hash__(self): return hash(self.id) - @memoize_to_dict + # @memoize_to_dict def to_dict(self): """Transforms the object to a Python dictionary. @@ -1185,7 +1184,7 @@ class Transaction(object): raise InvalidHash(err_msg.format(proposed_tx_id)) @classmethod - @memoize_from_dict + # @memoize_from_dict def from_dict(cls, tx, skip_schema_validation=True): """Transforms a Python dictionary to a Transaction object. diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py index 444a5a95..7142c0c5 100644 --- a/bigchaindb/lib.py +++ b/bigchaindb/lib.py @@ -9,6 +9,7 @@ MongoDB. import logging from collections import namedtuple from uuid import uuid4 +import rapidjson try: from hashlib import sha3_256 @@ -124,8 +125,7 @@ class BigchainDB(object): assets = [] txn_metadatas = [] for t in transactions: - # self.update_utxoset(transaction) - transaction = t.tx_dict if t.tx_dict else t.to_dict() + transaction = t.tx_dict if t.tx_dict else rapidjson.loads(rapidjson.dumps(t.to_dict())) if transaction['operation'] == t.CREATE: asset = transaction.pop('asset') asset['id'] = transaction['id'] diff --git a/bigchaindb/upsert_validator/validator_election.py b/bigchaindb/upsert_validator/validator_election.py index f007c38b..d7f4b3d2 100644 --- a/bigchaindb/upsert_validator/validator_election.py +++ b/bigchaindb/upsert_validator/validator_election.py @@ -30,11 +30,11 @@ class ValidatorElection(Transaction): CREATE = VALIDATOR_ELECTION ALLOWED_OPERATIONS = (VALIDATOR_ELECTION,) - def __init__(self, operation, asset, inputs, outputs, - metadata=None, version=None, hash_id=None): - # operation `CREATE` is being passed as argument as `VALIDATOR_ELECTION` is an extension - # of `CREATE` and any validation on `CREATE` in the parent class should apply to it - super().__init__(operation, asset, inputs, outputs, metadata, version, hash_id) + # def __init__(self, operation, asset, inputs, outputs, + # metadata=None, version=None, hash_id=None, tx_): + # # operation `CREATE` is being passed as argument as `VALIDATOR_ELECTION` is an extension + # # of `CREATE` and any validation on `CREATE` in the parent class should apply to it + # super().__init__(operation, asset, inputs, outputs, metadata, version, hash_id) @classmethod def get_validators(cls, bigchain, height=None): diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 7b8dde99..426d6c34 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -13,7 +13,7 @@ from cryptoconditions import Ed25519Sha256 from pytest import mark, raises from sha3 import sha3_256 -pytestmark = mark.tendermint +pytestmark = [mark.tendermint, mark.bdb] def test_input_serialization(ffill_uri, user_pub): @@ -533,7 +533,7 @@ def test_validate_input_with_invalid_parameters(utx): input_conditions = [out.fulfillment.condition_uri for out in utx.outputs] tx_dict = utx.to_dict() tx_serialized = Transaction._to_str(tx_dict) - valid = utx._input_valid(utx.inputs[0], tx_serialized, input_conditions) + valid = utx._input_valid(utx.inputs[0], tx_serialized, input_conditions[0]) assert not valid diff --git a/tests/conftest.py b/tests/conftest.py index cc985d96..6e59abb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -144,11 +144,15 @@ def _bdb(_setup_database, _configure_bigchaindb): from bigchaindb import config from bigchaindb.backend import connect from .utils import flush_db + from bigchaindb.common.memoize import to_dict, from_dict conn = connect() yield dbname = config['database']['name'] flush_db(conn, dbname) + to_dict.cache_clear() + from_dict.cache_clear() + # We need this function to avoid loading an existing # conf file located in the home of the user running @@ -256,8 +260,7 @@ def b(): def create_tx(alice, user_pk): from bigchaindb.models import Transaction name = f'I am created by the create_tx fixture. My random identifier is {random.random()}.' - return Transaction.create([alice.public_key], [([user_pk], 1)], asset={'name': name})\ - .sign([alice.private_key]) + return Transaction.create([alice.public_key], [([user_pk], 1)], asset={'name': name}) @pytest.fixture