From a864f8b7e1e53e2e712a2b67a959cdf1da5f0dc7 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 13:04:46 +0300 Subject: [PATCH 1/9] Added exception for NetworkError, to get specific error from connection class. --- planetmint/backend/connection.py | 22 ++++++++++++++-------- planetmint/backend/tarantool/connection.py | 3 +++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/planetmint/backend/connection.py b/planetmint/backend/connection.py index c001ef0..38e496f 100644 --- a/planetmint/backend/connection.py +++ b/planetmint/backend/connection.py @@ -5,11 +5,13 @@ import logging from importlib import import_module + +import tarantool + from planetmint.config import Config from planetmint.backend.exceptions import ConnectionError from planetmint.transactions.common.exceptions import ConfigurationError - BACKENDS = { # This is path to MongoDBClass 'tarantool_db': 'planetmint.backend.tarantool.connection.TarantoolDB', 'localmongodb': 'planetmint.backend.localmongodb.connection.LocalMongoDBConnection' @@ -48,7 +50,7 @@ def Connection(host: str = None, port: int = None, login: str = None, password: replicaset = _kwargs_parser(key="replicaset", kwargs=kwargs) or Config().get()['database']['replicaset'] ssl = _kwargs_parser(key="ssl", kwargs=kwargs) or Config().get()['database']['ssl'] login = login or Config().get()['database']['login'] if _kwargs_parser(key="login", - kwargs=kwargs) is None else _kwargs_parser( + kwargs=kwargs) is None else _kwargs_parser( key="login", kwargs=kwargs) password = password or Config().get()['database']['password'] if _kwargs_parser(key="password", kwargs=kwargs) is None else _kwargs_parser( @@ -63,13 +65,17 @@ def Connection(host: str = None, port: int = None, login: str = None, password: connection_timeout = _kwargs_parser(key="connection_timeout", kwargs=kwargs) return Class(host=host, port=port, dbname=dbname, - max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset, ssl=ssl, login=login, password=password, - ca_cert=ca_cert, certfile=certfile, keyfile=keyfile, - keyfile_passphrase=keyfile_passphrase, crlfile=crlfile) + max_tries=max_tries, connection_timeout=connection_timeout, + replicaset=replicaset, ssl=ssl, login=login, password=password, + ca_cert=ca_cert, certfile=certfile, keyfile=keyfile, + keyfile_passphrase=keyfile_passphrase, crlfile=crlfile) + except tarantool.error.NetworkError as network_err: + print(f"Host {host}:{port} can't be reached.\n{network_err}") + raise network_err except: - logger.info('Exception in _connect(): {}') - raise ConfigurationError + logger.info('Exception in _connect(): {}') + raise ConfigurationError + def _kwargs_parser(key, kwargs): if kwargs.get(key): diff --git a/planetmint/backend/tarantool/connection.py b/planetmint/backend/tarantool/connection.py index 15a6ea3..e3df560 100644 --- a/planetmint/backend/tarantool/connection.py +++ b/planetmint/backend/tarantool/connection.py @@ -33,6 +33,9 @@ class TarantoolDB: self.SPACE_NAMES = ["abci_chains", "assets", "blocks", "blocks_tx", "elections", "meta_data", "pre_commits", "validators", "transactions", "inputs", "outputs", "keys"] + except tarantool.error.NetworkError as network_err: + logger.info('Host cant be reached') + raise network_err except: logger.info('Exception in _connect(): {}') raise ConfigurationError From 8e88c12722f98ea6fe4e39c1c20551972a7f8765 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 14:59:59 +0300 Subject: [PATCH 2/9] _setup_database fixture restored.(STILL NOT WORKING in this commit) --- tests/conftest.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7e02cf7..9062447 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -123,20 +123,37 @@ def _configure_planetmint(request): @pytest.fixture(scope='session') def _setup_database(_configure_planetmint): # TODO Here is located setup database - from planetmint.backend.tarantool.connection import TarantoolDB + from planetmint.backend.connection import Connection + from planetmint.config import Config - print('Deleting `{}` database') - db_conn = Connection() - db_conn.drop_database() - db_conn.init_database() - print('Finished deleting ``') + # print('Deleting `{}` database') + # db_conn = Connection() + # db_conn.drop_database() + # db_conn.init_database() + # print('Finished deleting ``') + # + # yield + # + # print('Initializing test db') + # db_conn2 = Connection() + # db_conn2.drop_database() + # print('Finishing init database') + + print('Initializing test db') + dbname = Config().get()['database']['name'] + conn = Connection() + + _drop_db(conn, dbname) + schema.init_database(conn) + print('Finishing init database') yield - print('Initializing test db') - db_conn2 = Connection() - db_conn2.drop_database() - print('Finishing init database') + print('Deleting `{}` database'.format(dbname)) + conn = Connection() + _drop_db(conn, dbname) + + print('Finished deleting `{}`'.format(dbname)) @pytest.fixture From bf3ece309e5de8ba6f5613be1aeecea3a532df9d Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 15:00:33 +0300 Subject: [PATCH 3/9] added 'universe' name as database name for tarantool --- planetmint/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetmint/config.py b/planetmint/config.py index 583d4c6..efa2355 100644 --- a/planetmint/config.py +++ b/planetmint/config.py @@ -59,7 +59,7 @@ class Config(metaclass=Singleton): 'backend': 'tarantool_db', 'connection_timeout': 5000, 'max_tries': 3, - 'name': 'bigchain', + 'name': 'universe', "reconnect_delay": 0.5, 'host': 'localhost', 'port': 3303, From cc6f8eea16ff3b1d1cce39d1e832035c1e606716 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 15:24:39 +0300 Subject: [PATCH 4/9] created flush_db for tarantool connection type --- planetmint/backend/schema.py | 4 ++++ tests/conftest.py | 18 ++++-------------- tests/utils.py | 20 +++++++++++++++----- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/planetmint/backend/schema.py b/planetmint/backend/schema.py index 2677df0..89f5872 100644 --- a/planetmint/backend/schema.py +++ b/planetmint/backend/schema.py @@ -20,6 +20,10 @@ logger = logging.getLogger(__name__) TABLES = ('transactions', 'blocks', 'assets', 'metadata', 'validators', 'elections', 'pre_commit', 'utxos', 'abci_chains') +SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx", + "elections", "meta_data", "pre_commits", "validators", + "transactions", "inputs", "outputs", "keys") + VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', 'russian', 'spanish', 'swedish', 'turkish', 'none', diff --git a/tests/conftest.py b/tests/conftest.py index 9062447..1b87cb1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -126,19 +126,6 @@ def _setup_database(_configure_planetmint): # TODO Here is located setup databa from planetmint.backend.connection import Connection from planetmint.config import Config - # print('Deleting `{}` database') - # db_conn = Connection() - # db_conn.drop_database() - # db_conn.init_database() - # print('Finished deleting ``') - # - # yield - # - # print('Initializing test db') - # db_conn2 = Connection() - # db_conn2.drop_database() - # print('Finishing init database') - print('Initializing test db') dbname = Config().get()['database']['name'] conn = Connection() @@ -161,8 +148,12 @@ def _bdb(_setup_database): from planetmint.backend import Connection from planetmint.transactions.common.memoize import to_dict, from_dict from planetmint.models import Transaction + from .utils import flush_db + from planetmint.config import Config conn = Connection() yield + dbname = Config().get()['database']['name'] + flush_db(conn, dbname) to_dict.cache_clear() from_dict.cache_clear() @@ -372,7 +363,6 @@ def inputs(user_pk, b, alice): def _drop_db(conn, dbname): try: - conn.drop_database() schema.drop_database(conn, dbname) except DatabaseDoesNotExist: pass diff --git a/tests/utils.py b/tests/utils.py index ceffff0..811f6fd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,7 +10,8 @@ import random from functools import singledispatch from planetmint.backend.localmongodb.connection import LocalMongoDBConnection -from planetmint.backend.schema import TABLES +from planetmint.backend.tarantool.connection import TarantoolDB +from planetmint.backend.schema import TABLES, SPACE_NAMES from planetmint.transactions.common import crypto from planetmint.transactions.common.transaction_mode_types import BROADCAST_TX_COMMIT from planetmint.transactions.types.assets.create import Create @@ -29,14 +30,23 @@ def flush_localmongo_db(connection, dbname): getattr(connection.conn[dbname], t).delete_many({}) +@flush_db.register(TarantoolDB) +def flush_tarantool_db(connection, dbname): + for s in SPACE_NAMES: + _space = connection.space(space_name=s) + _all_data = _space.select([]).data + for _id in _all_data: + _space.delete(_id[0]) + + def generate_block(planet): from planetmint.transactions.common.crypto import generate_key_pair alice = generate_key_pair() tx = Create.generate([alice.public_key], - [([alice.public_key], 1)], - asset=None)\ - .sign([alice.private_key]) + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]) code, message = planet.write_transaction(tx, BROADCAST_TX_COMMIT) assert code == 202 @@ -55,7 +65,7 @@ def gen_vote(election, i, ed25519_node_keys): election_pub_key = Election.to_public_key(election.id) return Vote.generate([input_i], [([election_pub_key], votes_i)], - election_id=election.id)\ + election_id=election.id) \ .sign([key_i.private_key]) From a905de679201e2287583e420fabbedac9ae39285 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 15:32:10 +0300 Subject: [PATCH 5/9] function create_database will call create_tables only for tarantool connection --- planetmint/backend/tarantool/schema.py | 7 +++---- tests/conftest.py | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index da553b4..070f8c5 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -165,12 +165,11 @@ def drop_database(connection, not_used=None): def create_database(connection, not_used=None): ''' - This function 'create_database' cannot be used with TarantoolDB connection Class. - It will be ignored if called. No Errors. + For tarantool implementation, this function runs + create_tables, to initiate spaces, schema and indexes. ''' - # connection.init_database() - warnings.warn("Function schema.'create_database', ignored. Cannot be used using TarantoolDB") + create_tables(None, None) def run_command_with_output(command): diff --git a/tests/conftest.py b/tests/conftest.py index 1b87cb1..320a636 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -145,6 +145,7 @@ def _setup_database(_configure_planetmint): # TODO Here is located setup databa @pytest.fixture def _bdb(_setup_database): + print(f"BDB CALL") from planetmint.backend import Connection from planetmint.transactions.common.memoize import to_dict, from_dict from planetmint.models import Transaction From d4ed0512f990e649087953ea0a2a737356e53e19 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 15:37:09 +0300 Subject: [PATCH 6/9] _bdb, setup_database restored as it was before --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 320a636..604e276 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -144,7 +144,7 @@ def _setup_database(_configure_planetmint): # TODO Here is located setup databa @pytest.fixture -def _bdb(_setup_database): +def _bdb(_setup_database, _configure_bigchaindb): print(f"BDB CALL") from planetmint.backend import Connection from planetmint.transactions.common.memoize import to_dict, from_dict From ad6c9926e27ad4022ea7147f349340ceb265fa4b Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 26 May 2022 16:07:26 +0300 Subject: [PATCH 7/9] Fixed flush_dbtarantool --- tests/conftest.py | 2 +- tests/utils.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 604e276..9e85f95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -144,7 +144,7 @@ def _setup_database(_configure_planetmint): # TODO Here is located setup databa @pytest.fixture -def _bdb(_setup_database, _configure_bigchaindb): +def _bdb(_setup_database, _configure_planetmint): print(f"BDB CALL") from planetmint.backend import Connection from planetmint.transactions.common.memoize import to_dict, from_dict diff --git a/tests/utils.py b/tests/utils.py index 811f6fd..786b02d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -36,7 +36,12 @@ def flush_tarantool_db(connection, dbname): _space = connection.space(space_name=s) _all_data = _space.select([]).data for _id in _all_data: - _space.delete(_id[0]) + if "assets" == s: + _space.delete(_id[1]) + elif "abci_chains" == s: + _space.delete(_id[2]) + else: + _space.delete(_id[0]) def generate_block(planet): From bcddc754be7222e6f3c8f87bbd047a64b80483ff Mon Sep 17 00:00:00 2001 From: andrei Date: Fri, 27 May 2022 14:10:15 +0300 Subject: [PATCH 8/9] Fixed fatal crashing at beginning of starting planetmint. problem was solved by returning None value if where is no space. --- planetmint/backend/tarantool/query.py | 17 +++++-- planetmint/backend/tarantool/schema.py | 2 +- planetmint/lib.py | 1 + requirements_old.txt | 64 -------------------------- tests/commands/test_commands.py | 1 - 5 files changed, 14 insertions(+), 71 deletions(-) delete mode 100644 requirements_old.txt diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index d866afb..3773a19 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -8,6 +8,8 @@ from secrets import token_hex from operator import itemgetter +import tarantool.error + from planetmint.backend import query from planetmint.backend.utils import module_dispatch_registrar from planetmint.backend.tarantool.connection import TarantoolDB @@ -377,12 +379,17 @@ def store_pre_commit_state(connection, state: dict): @register_query(TarantoolDB) def get_pre_commit_state(connection): - space = connection.space("pre_commits") - _commit = space.select([], index="id_search").data - if len(_commit) == 0: + try: + space = connection.space("pre_commits") + _commit = space.select([], index="id_search").data + if len(_commit) == 0: + return None + _commit = sorted(_commit, key=itemgetter(1), reverse=True)[0] + return {"height": _commit[1], "transactions": _commit[2]} + except tarantool.error.SchemaError: return None - _commit = sorted(_commit, key=itemgetter(1), reverse=True)[0] - return {"height": _commit[1], "transactions": _commit[2]} + except Exception as err: + raise err @register_query(TarantoolDB) diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index 070f8c5..224c67b 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -9,7 +9,7 @@ register_schema = module_dispatch_registrar(backend.schema) SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx", "elections", "meta_data", "pre_commits", "validators", - "transactions", "inputs", "outputs", "keys") + "transactions", "inputs", "outputs", "keys", "utxos") SPACE_COMMANDS = { "abci_chains": "abci_chains = box.schema.space.create('abci_chains', {engine='memtx', is_sync = false})", diff --git a/planetmint/lib.py b/planetmint/lib.py index 1297e94..0de282a 100644 --- a/planetmint/lib.py +++ b/planetmint/lib.py @@ -513,4 +513,5 @@ class Planetmint(object): return backend.query.delete_elections(self.connection, height) + Block = namedtuple('Block', ('app_hash', 'height', 'transactions')) diff --git a/requirements_old.txt b/requirements_old.txt deleted file mode 100644 index 4b26d68..0000000 --- a/requirements_old.txt +++ /dev/null @@ -1,64 +0,0 @@ -aiohttp==3.6.2 -aniso8601==9.0.1 -asn1crypto==1.4.0 -async-timeout==3.0.1 -attrs==21.4.0 -base58==1.0.3 -BigchainDB==2.2.2 -bigchaindb-abci==1.0.5 -capturer==3.0 -certifi==2021.10.8 -cffi==1.15.0 -chardet==3.0.4 -charset-normalizer==2.0.12 -click==8.0.3 -colorlog==4.1.0 -cryptoconditions==0.8.0 -cryptography==36.0.1 -Flask==1.1.2 -Flask-Cors==3.0.8 -Flask-RESTful==0.3.8 -gevent==20.6.2 -greenlet==0.4.16 -gunicorn==20.0.4 -humanfriendly==10.0 -hypothesis==6.39.3 -idna==2.10 -iniconfig==1.1.1 -itsdangerous==2.0.1 -Jinja2==3.0.3 -jsonschema==3.2.0 -logstats==0.3.0 -MarkupSafe==2.0.1 -msgpack==1.0.3 -multidict==4.7.6 -packaging==21.3 -Planetmint==0.9.0 -planetmint-cryptoconditions==0.9.3 -pluggy==1.0.0 -protobuf==3.6.1 -py==1.11.0 -pyasn1==0.4.8 -pycparser==2.21 -pymongo==3.7.2 -PyNaCl==1.1.2 -pyOpenSSL==22.0.0 -pyparsing==3.0.7 -pyrsistent==0.18.1 -pytest==7.0.1 -python-rapidjson==0.9.1 -pytz==2021.3 -PyYAML==5.3.1 -requests==2.23.0 -setproctitle==1.1.10 -six==1.16.0 -sortedcontainers==2.4.0 -tarantool==0.7.1 -tomli==2.0.1 -typing_extensions==4.1.1 -urllib3==1.25.11 -Werkzeug==2.0.3 -yarl==1.7.2 -zenroom==2.1.0.dev1647359536 -zope.event==4.5.0 -zope.interface==5.5.0.dev0 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 73c7dd3..a269c5d 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -20,7 +20,6 @@ from planetmint.migrations.chain_migration_election import ChainMigrationElectio from tests.utils import generate_election, generate_validators - def test_make_sure_we_dont_remove_any_command(): # thanks to: http://stackoverflow.com/a/18161115/597097 from planetmint.commands.planetmint import create_parser From 18d0e8fbcf534bf3a6e704e45c729171acc23319 Mon Sep 17 00:00:00 2001 From: andrei Date: Mon, 30 May 2022 12:56:56 +0300 Subject: [PATCH 9/9] adding try except to some queries from tarantool/query.py. --- planetmint/backend/tarantool/query.py | 92 ++++++++++++++++----------- 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 3773a19..02b6c4b 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -167,22 +167,27 @@ def get_spent(connection, fullfil_transaction_id: str, fullfil_output_index: str @register_query(TarantoolDB) def get_latest_block(connection): # TODO Here is used DESCENDING OPERATOR - space = connection.space("blocks") - _all_blocks = space.select() - _all_blocks = _all_blocks.data - block = {"app_hash": '', "height": 0, "transactions": []} + try: + space = connection.space("blocks") + _all_blocks = space.select() + _all_blocks = _all_blocks.data + block = {"app_hash": '', "height": 0, "transactions": []} - if len(_all_blocks) > 0: - _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0] - space = connection.space("blocks_tx") - _txids = space.select(_block[2], index="block_search") - _txids = _txids.data - block["app_hash"] = _block[0] - block["height"] = _block[1] - block["transactions"] = [tx[0] for tx in _txids] - else: - block = None - return block + if len(_all_blocks) > 0: + _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0] + space = connection.space("blocks_tx") + _txids = space.select(_block[2], index="block_search") + _txids = _txids.data + block["app_hash"] = _block[0] + block["height"] = _block[1] + block["transactions"] = [tx[0] for tx in _txids] + else: + block = None + return block + except tarantool.error.SchemaError: + return None + except Exception as err: + raise err @register_query(TarantoolDB) @@ -441,27 +446,37 @@ def delete_elections(connection, height: int): @register_query(TarantoolDB) def get_validator_set(connection, height: int = None): - space = connection.space("validators") - _validators = space.select() - _validators = _validators.data - if height is not None: - _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators if - validator[1] <= height] - return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) - else: - _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators] - return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) + try: + space = connection.space("validators") + _validators = space.select() + _validators = _validators.data + if height is not None: + _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators if + validator[1] <= height] + return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) + else: + _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators] + return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) + except tarantool.error.SchemaError: + return None + except Exception as err: + raise err @register_query(TarantoolDB) def get_election(connection, election_id: str): - space = connection.space("elections") - _elections = space.select(election_id, index="id_search") - _elections = _elections.data - if len(_elections) == 0: + try: + space = connection.space("elections") + _elections = space.select(election_id, index="id_search") + _elections = _elections.data + if len(_elections) == 0: + return None + _election = sorted(_elections, key=itemgetter(0), reverse=True)[0] + return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]} + except tarantool.error.SchemaError: return None - _election = sorted(_elections, key=itemgetter(0), reverse=True)[0] - return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]} + except Exception as err: + raise err @register_query(TarantoolDB) @@ -496,9 +511,14 @@ def delete_abci_chain(connection, height: int): @register_query(TarantoolDB) def get_latest_abci_chain(connection): - space = connection.space("abci_chains") - _all_chains = space.select().data - if len(_all_chains) == 0: + try: + space = connection.space("abci_chains") + _all_chains = space.select().data + if len(_all_chains) == 0: + return None + _chain = sorted(_all_chains, key=itemgetter(0), reverse=True)[0] + return {"height": _chain[0], "is_synced": _chain[1], "chain_id": _chain[2]} + except tarantool.error.SchemaError: return None - _chain = sorted(_all_chains, key=itemgetter(0), reverse=True)[0] - return {"height": _chain[0], "is_synced": _chain[1], "chain_id": _chain[2]} + except Exception as err: + raise err