From 86c526feb18e110e771ed8eab95b43649d036e70 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 17 Feb 2022 16:18:35 +0200 Subject: [PATCH] Rewrited return type for some queries --- planetmint/backend/connection_tarantool.py | 45 +--------- planetmint/backend/query.py | 7 +- planetmint/backend/tarantool/init_db.lua | 3 + planetmint/backend/tarantool/query.py | 96 +++++++++++----------- tests/backend/tarantool/test_queries.py | 62 +++++++------- tests/conftest.py | 27 ++---- 6 files changed, 102 insertions(+), 138 deletions(-) diff --git a/planetmint/backend/connection_tarantool.py b/planetmint/backend/connection_tarantool.py index 5343976..64da960 100644 --- a/planetmint/backend/connection_tarantool.py +++ b/planetmint/backend/connection_tarantool.py @@ -51,52 +51,15 @@ def drop_tarantool(): class TarantoolDB: def __init__(self, host: str, port: int, user: str, password: str): - init_tarantool() - sleep(3) # For test case + # init_tarantool() + self.db_connect = None self.db_connect = tarantool.connect(host=host, port=port, user=user, password=password) - self._spaces = { - "abci_chains": self.db_connect.space("abci_chains"), - "assets": self.db_connect.space("assets"), - "blocks": {"blocks": self.db_connect.space("blocks"), "blocks_tx": self.db_connect.space("blocks_tx")}, - "elections": self.db_connect.space("elections"), - "meta_data": self.db_connect.space("meta_data"), - "pre_commits": self.db_connect.space("pre_commits"), - "validators": self.db_connect.space("validators"), - "transactions": { - "transactions": self.db_connect.space("transactions"), - "inputs": self.db_connect.space("inputs"), - "outputs": self.db_connect.space("outputs"), - "keys": self.db_connect.space("keys") - } - } - def get_space(self, spacename: str): - return self._spaces[spacename] + def get_connection(self): + return self.db_connect def connect(host: str = None, port: int = None, username: str = "admin", password: str = "pass", backend: str = None): - """Create a new connection to the database backend. - - All arguments default to the current configuration's values if not - given. - - Args: - backend (str): the name of the backend to use. - host (str): the host to connect to. - port (int): the port to connect to. - - Returns: - An instance of :class:`~planetmint.backend.connection.Connection` - based on the given (or defaulted) :attr:`backend`. - - Raises: - :exc:`~ConnectionError`: If the connection to the database fails. - :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend` - is not supported or could not be loaded. - :exc:`~AuthenticationError`: If there is a OperationFailure due to - Authentication failure after connecting to the database. - """ - backend = backend or get_planetmint_config_value_or_key_error('backend') # TODO Rewrite Configs host = host or get_planetmint_config_value_or_key_error('host') port = port or get_planetmint_config_value_or_key_error('port') diff --git a/planetmint/backend/query.py b/planetmint/backend/query.py index 2c26bfa..4c6156a 100644 --- a/planetmint/backend/query.py +++ b/planetmint/backend/query.py @@ -10,8 +10,9 @@ from functools import singledispatch from planetmint.backend.exceptions import OperationError +# FIXME ADD HERE HINT FOR RETURNING TYPE @singledispatch -def store_asset(connection, asset): +def store_asset(asset: dict, connection): """Write an asset to the asset table. Args: @@ -25,7 +26,7 @@ def store_asset(connection, asset): @singledispatch -def store_assets(connection, assets): +def store_assets(assets: list, connection): """Write a list of assets to the assets table. Args: @@ -191,7 +192,7 @@ def get_metadata(connection, transaction_ids): @singledispatch -def get_assets(connection, asset_ids): +def get_assets(connection, asset_ids) -> list: """Get a list of assets from the assets table. Args: asset_ids (list): a list of ids for the assets to be retrieved from diff --git a/planetmint/backend/tarantool/init_db.lua b/planetmint/backend/tarantool/init_db.lua index 40fa814..b2d1361 100644 --- a/planetmint/backend/tarantool/init_db.lua +++ b/planetmint/backend/tarantool/init_db.lua @@ -64,3 +64,6 @@ keys:format({{name = 'transaction_id', type = 'string'} ,{name = 'output_id', ty keys:create_index('keys_search', {type = 'hash', parts={'public_key'}}) keys:create_index('txid_search', {type = 'tree', unique=false, parts={'transaction_id'}}) keys:create_index('id_search', {type = 'tree', unique=false, parts={'output_id'}}) + +local console = require('console') +console.start() \ No newline at end of file diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 6d7b3c0..b31b6a6 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -7,6 +7,9 @@ from pymongo import DESCENDING +from secrets import token_hex +from operator import itemgetter + from planetmint import backend from planetmint.backend.exceptions import DuplicateKeyError from planetmint.backend.utils import module_dispatch_registrar @@ -41,7 +44,7 @@ def _group_transaction_by_ids(txids: list, connection): { "owners_before": _in[2], "fulfills": {"transaction_id": _in[3], "output_index": _in[4]} if len(_in[3]) > 0 and len( - _int[4]) > 0 else None, + _in[4]) > 0 else None, "fulfillment": _in[1] } for _in in _txinputs ], @@ -62,7 +65,7 @@ def _group_transaction_by_ids(txids: list, connection): return _transactions -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_transactions(signed_transactions: list, connection): txspace = connection.space("transactions") @@ -73,7 +76,7 @@ def store_transactions(signed_transactions: list, txspace.insert((transaction["id"], transaction["operation"], transaction["version"], - transaction["asset"]["id"] if "asset" in transaction.keys() else "" + transaction["asset"]["id"] if transaction["operation"] == "TRANSFER" else "" )) for _in in transaction["inputs"]: input_id = token_hex(7) @@ -93,29 +96,30 @@ def store_transactions(signed_transactions: list, output_id )) for _key in _out["public_keys"]: - keysxspace.insert((transaction["id"], output_id, _key)) + unique_id = token_hex(8) + keysxspace.insert((unique_id, transaction["id"], output_id, _key)) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_transaction(transaction_id: str, connection): _transactions = _group_transaction_by_ids(txids=[transaction_id], connection=connection) return next(iter(_transactions), None) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_transactions(transactions_ids: list, connection): _transactions = _group_transaction_by_ids(txids=transactions_ids, connection=connection) return _transactions -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_metadatas(metadata: dict, connection): space = connection.space("meta_data") for meta in metadata: space.insert((meta["id"], meta)) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_metadata(transaction_ids: list, space): _returned_data = [] for _id in transaction_ids: @@ -124,31 +128,33 @@ def get_metadata(transaction_ids: list, space): return _returned_data -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_asset(asset: dict, connection): space = connection.space("assets") unique = token_hex(8) - space.insert((asset["id"], unique, asset["data"])) + space.insert((str(asset["id"]), unique, asset["data"])) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_assets(assets: list, connection): space = connection.space("assets") for asset in assets: unique = token_hex(8) - space.insert((asset["id"], unique, asset["data"])) + space.insert((str(asset["id"]), unique, asset["data"])) -@register_query(LocalMongoDBConnection) -def get_asset(asset_id: str, space): +# @register_query(LocalMongoDBConnection) +def get_asset(asset_id: str, connection): + space = connection.space("assets") _data = space.select(asset_id, index="assetid_search") _data = _data.data[0] return {"data": _data[1]} -@register_query(LocalMongoDBConnection) -def get_assets(assets_ids: list, space): +# @register_query(LocalMongoDBConnection) +def get_assets(assets_ids: list, connection) -> list: _returned_data = [] + space = connection.space("assets") for _id in assets_ids: asset = space.select(_id, index="assetid_search") asset = asset.data[0] @@ -156,7 +162,7 @@ def get_assets(assets_ids: list, space): return _returned_data -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_spent(fullfil_transaction_id: str, fullfil_output_index: str, connection): space = connection.space("inputs") _inputs = space.select([fullfil_transaction_id, str(fullfil_output_index)], index="spent_search") @@ -165,7 +171,7 @@ def get_spent(fullfil_transaction_id: str, fullfil_output_index: str, connection return _transactions -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def latest_block(connection): # TODO Here is used DESCENDING OPERATOR space = connection.space("blocks") _all_blocks = space.select() @@ -177,7 +183,7 @@ def latest_block(connection): # TODO Here is used DESCENDING OPERATOR return {"app_hash": _block[1], "height": _block[1], "transactions": [tx[0] for tx in _txids]} -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_block(block: dict, connection): space = connection.space("blocks") block_unique_id = token_hex(8) @@ -189,7 +195,7 @@ def store_block(block: dict, connection): space.insert((txid, block_unique_id)) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_txids_filtered(connection, asset_id: str, operation: str = None, last_tx: any = None): # TODO here is used 'OR' operator actions = { @@ -209,10 +215,8 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, else: _tx_ids = space.select([asset_id], index="id_search") _assets_ids = space.select([asset_id], index="only_asset_search") - - return tuple( - set([item for sublist in _assets_ids.data for item in sublist] + [item for sublist in _tx_ids.data for item - in sublist])) + _result = [sublist[0] for sublist in _assets_ids.data] + [sublist[0] for sublist in _tx_ids.data] + return tuple(set(_result)) if last_tx: return tuple(next(iter(_transactions))) @@ -220,7 +224,7 @@ def get_txids_filtered(connection, asset_id: str, operation: str = None, return tuple([elem[0] for elem in _transactions]) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def text_search(conn, search, *, language='english', case_sensitive=False, # TODO review text search in tarantool (maybe, remove) diacritic_sensitive=False, text_score=False, limit=0, table='assets'): @@ -246,7 +250,7 @@ def _remove_text_score(asset): return asset -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_owned_ids(connection, owner: str): space = connection.space("keys") _keys = space.select(owner, index="keys_search", limit=1) @@ -257,7 +261,7 @@ def get_owned_ids(connection, owner: str): return _transactions -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_spending_transactions(inputs, connection): _transactions = [] @@ -270,7 +274,7 @@ def get_spending_transactions(inputs, connection): return _transactions -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_block(block_id: str, connection): space = connection.space("blocks") _block = space.select(block_id, index="block_search", limit=1) @@ -280,7 +284,7 @@ def get_block(block_id: str, connection): return {"app_hash": _block[0], "height": _block[1], "transactions": [_tx[0] for _tx in _txblock]} -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_block_with_transaction(txid: str, connection): space = connection.space("blocks_tx") _all_blocks_tx = space.select(txid, index="id_search") @@ -292,7 +296,7 @@ def get_block_with_transaction(txid: str, connection): return {"app_hash": _block[0], "height": _block[1], "transactions": [_tx[0] for _tx in _all_blocks_tx]} -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def delete_transactions(connection, txn_ids: list): space = connection.space("transactions") for _id in txn_ids: @@ -312,7 +316,7 @@ def delete_transactions(connection, txn_ids: list): outputs_space.delete(_outpID[5], index="unique_search") -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_unspent_outputs(conn, *unspent_outputs: list): if unspent_outputs: try: @@ -327,7 +331,7 @@ def store_unspent_outputs(conn, *unspent_outputs: list): pass -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def delete_unspent_outputs(conn, *unspent_outputs: list): if unspent_outputs: return conn.run( @@ -342,7 +346,7 @@ def delete_unspent_outputs(conn, *unspent_outputs: list): ) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_unspent_outputs(conn, *, query=None): if query is None: query = {} @@ -350,7 +354,7 @@ def get_unspent_outputs(conn, *, query=None): projection={'_id': False})) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_pre_commit_state(state: dict, connection): space = connection.space("pre_commits") _precommit = space.select(state["height"], index="height_search", limit=1) @@ -362,7 +366,7 @@ def store_pre_commit_state(state: dict, connection): limit=1) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_pre_commit_state(connection): space = connection.space("pre_commit_tx") _commits_tx = space.select(limit=1) @@ -373,7 +377,7 @@ def get_pre_commit_state(connection): return {"height": _commit[0], "transactions": [_cmt[0] for _cmt in _commits_tx]} -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_validator_set(validators_update: dict, connection): space = connection.space("validators") _validator = space.select(validators_update["height"], index="height_search", limit=1) @@ -385,7 +389,7 @@ def store_validator_set(validators_update: dict, connection): limit=1) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def delete_validator_set(connection, height: int): space = connection.space("validators") _validators = space.select(height, index="height_search") @@ -393,7 +397,7 @@ def delete_validator_set(connection, height: int): space.delete(_valid[0]) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_election(election_id: str, height: int, is_concluded: bool, connection): space = connection.space("elections") space.upsert((election_id, height, is_concluded), @@ -403,7 +407,7 @@ def store_election(election_id: str, height: int, is_concluded: bool, connection limit=1) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_elections(elections: list, connection): space = connection.space("elections") for election in elections: @@ -412,7 +416,7 @@ def store_elections(elections: list, connection): election["is_concluded"])) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def delete_elections(connection, height: int): space = connection.space("elections") _elections = space.select(height, index="height_search") @@ -420,7 +424,7 @@ def delete_elections(connection, height: int): space.delete(_elec[0]) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_validator_set(connection, height: int = None): space = connection.space("validators") _validators = space.select() @@ -432,7 +436,7 @@ def get_validator_set(connection, height: int = None): return next(iter(sorted(_validators, key=itemgetter(1))), None) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_election(election_id: str, connection): space = connection.space("elections") _elections = space.select(election_id, index="id_search") @@ -441,7 +445,7 @@ def get_election(election_id: str, connection): return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]} -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_asset_tokens_for_public_key(connection, asset_id: str, public_key: str): space = connection.space("keys") _keys = space.select([public_key], index="keys_search") @@ -453,7 +457,7 @@ def get_asset_tokens_for_public_key(connection, asset_id: str, public_key: str): return _grouped_transactions -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def store_abci_chain(height: int, chain_id: str, connection, is_synced: bool = True): space = connection.space("abci_chains") space.upsert((height, chain_id, is_synced), @@ -463,7 +467,7 @@ def store_abci_chain(height: int, chain_id: str, connection, is_synced: bool = T limit=1) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def delete_abci_chain(connection, height: int): space = connection.space("abci_chains") _chains = space.select(height, index="height_search") @@ -471,7 +475,7 @@ def delete_abci_chain(connection, height: int): space.delete(_chain[2]) -@register_query(LocalMongoDBConnection) +# @register_query(LocalMongoDBConnection) def get_latest_abci_chain(connection): space = connection.space("abci_chains") _all_chains = space.select() diff --git a/tests/backend/tarantool/test_queries.py b/tests/backend/tarantool/test_queries.py index fb7d5ba..e728c65 100644 --- a/tests/backend/tarantool/test_queries.py +++ b/tests/backend/tarantool/test_queries.py @@ -8,39 +8,42 @@ from copy import deepcopy import pytest # import pymongo -from planetmint.backend import connect, query +# from planetmint.backend import connect, query pytestmark = pytest.mark.bdb def test_get_txids_filtered(signed_create_tx, signed_transfer_tx): - from planetmint.backend import connect, query + from planetmint.backend import connect + from planetmint.backend.tarantool import query from planetmint.models import Transaction - conn = connect() # TODO First rewrite to get here tarantool connection - # # create and insert two blocks, one for the create and one for the - # # transfer transaction - # conn.db.transactions.insert_one(signed_create_tx.to_dict()) - # conn.db.transactions.insert_one(signed_transfer_tx.to_dict()) - # - # asset_id = Transaction.get_asset_id([signed_create_tx, signed_transfer_tx]) - # - # # Test get by just asset id - # txids = set(query.get_txids_filtered(conn, asset_id)) - # assert txids == {signed_create_tx.id, signed_transfer_tx.id} - # - # # Test get by asset and CREATE - # txids = set(query.get_txids_filtered(conn, asset_id, Transaction.CREATE)) - # assert txids == {signed_create_tx.id} - # - # # Test get by asset and TRANSFER - # txids = set(query.get_txids_filtered(conn, asset_id, Transaction.TRANSFER)) - # assert txids == {signed_transfer_tx.id} + conn = connect().get_connection() # TODO First rewrite to get here tarantool connection + # create and insert two blocks, one for the create and one for the + # transfer transaction + create_tx_dict = signed_create_tx.to_dict() + transfer_tx_dict = signed_transfer_tx.to_dict() + query.store_transactions(signed_transactions=[create_tx_dict], connection=conn) + query.store_transactions(signed_transactions=[transfer_tx_dict], connection=conn) + + asset_id = Transaction.get_asset_id([signed_create_tx, signed_transfer_tx]) + + # Test get by just asset id + txids = set(query.get_txids_filtered(connection=conn, asset_id=asset_id)) + assert txids == {signed_create_tx.id, signed_transfer_tx.id} + + # Test get by asset and CREATE + txids = set(query.get_txids_filtered(connection=conn, asset_id=asset_id, operation=Transaction.CREATE)) + assert txids == {signed_create_tx.id} + + # Test get by asset and TRANSFER + txids = set(query.get_txids_filtered(connection=conn, asset_id=asset_id, operation=Transaction.TRANSFER)) + assert txids == {signed_transfer_tx.id} def test_write_assets(): - from planetmint.backend import connect, query - conn = connect() - + from planetmint.backend import connect + from planetmint.backend.tarantool import query + conn = connect().get_connection() assets = [ {'id': 1, 'data': '1'}, {'id': 2, 'data': '2'}, @@ -51,14 +54,15 @@ def test_write_assets(): # write the assets for asset in assets: - query.store_asset(conn, deepcopy(asset)) + query.store_asset(connection=conn, asset=asset) # check that 3 assets were written to the database - cursor = conn.db.assets.find({}, projection={'_id': False}) \ - .sort('id', pymongo.ASCENDING) + # cursor = conn.db.assets.find({}, projection={'_id': False}) \ + # .sort('id', pymongo.ASCENDING) + documents = query.get_assets(assets_ids=[asset["id"] for asset in assets], connection=conn) - assert cursor.collection.count_documents({}) == 3 - assert list(cursor) == assets[:-1] + assert len(documents) == 3 + # assert list(cursor) == assets[:-1] # TODO To change from id 'string' to 'unsigned' def test_get_assets(): diff --git a/tests/conftest.py b/tests/conftest.py index b6ceb4f..9195c47 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -112,37 +112,26 @@ def _configure_planetmint(request): @pytest.fixture(scope='session') -def _setup_database(_configure_planetmint): - from planetmint import config - from planetmint.backend import connect - print('Initializing test db') - dbname = config['database']['name'] - conn = connect() - - _drop_db(conn, dbname) - schema.init_database(conn) - print('Finishing init database') +def _setup_database(_configure_planetmint): # TODO Here is located setup database + from planetmint.backend.connection_tarantool import init_tarantool, drop_tarantool + # print('Initializing test db') + # init_tarantool() + # print('Finishing init database') yield - print('Deleting `{}` database'.format(dbname)) - conn = connect() - _drop_db(conn, dbname) - - print('Finished deleting `{}`'.format(dbname)) + # print('Deleting `{}` database') + # drop_tarantool() + # print('Finished deleting ``') @pytest.fixture def _bdb(_setup_database, _configure_planetmint): - from planetmint import config from planetmint.backend import connect - from .utils import flush_db from planetmint.common.memoize import to_dict, from_dict from planetmint.models import Transaction conn = connect() yield - dbname = config['database']['name'] - flush_db(conn, dbname) to_dict.cache_clear() from_dict.cache_clear()