diff --git a/planetmint/backend/connection.py b/planetmint/backend/connection.py index 5aa2e77..e764cfd 100644 --- a/planetmint/backend/connection.py +++ b/planetmint/backend/connection.py @@ -5,11 +5,13 @@ import logging from importlib import import_module + +import tarantool + from planetmint.config import Config from planetmint.backend.exceptions import ConnectionError from planetmint.transactions.common.exceptions import ConfigurationError - BACKENDS = { # This is path to MongoDBClass 'tarantool_db': 'planetmint.backend.tarantool.connection.TarantoolDBConnection', 'localmongodb': 'planetmint.backend.localmongodb.connection.LocalMongoDBConnection' @@ -48,7 +50,7 @@ def connect(host: str = None, port: int = None, login: str = None, password: str replicaset = _kwargs_parser(key="replicaset", kwargs=kwargs) or Config().get()['database']['replicaset'] ssl = _kwargs_parser(key="ssl", kwargs=kwargs) or Config().get()['database']['ssl'] login = login or Config().get()['database']['login'] if _kwargs_parser(key="login", - kwargs=kwargs) is None else _kwargs_parser( + kwargs=kwargs) is None else _kwargs_parser( key="login", kwargs=kwargs) password = password or Config().get()['database']['password'] if _kwargs_parser(key="password", kwargs=kwargs) is None else _kwargs_parser( @@ -63,13 +65,17 @@ def connect(host: str = None, port: int = None, login: str = None, password: str connection_timeout = _kwargs_parser(key="connection_timeout", kwargs=kwargs) return Class(host=host, port=port, dbname=dbname, - max_tries=max_tries, connection_timeout=connection_timeout, - replicaset=replicaset, ssl=ssl, login=login, password=password, - ca_cert=ca_cert, certfile=certfile, keyfile=keyfile, - keyfile_passphrase=keyfile_passphrase, crlfile=crlfile) + max_tries=max_tries, connection_timeout=connection_timeout, + replicaset=replicaset, ssl=ssl, login=login, password=password, + ca_cert=ca_cert, certfile=certfile, keyfile=keyfile, + keyfile_passphrase=keyfile_passphrase, crlfile=crlfile) + except tarantool.error.NetworkError as network_err: + print(f"Host {host}:{port} can't be reached.\n{network_err}") + raise network_err except: - logger.info('Exception in _connect(): {}') - raise ConfigurationError + logger.info('Exception in _connect(): {}') + raise ConfigurationError + def _kwargs_parser(key, kwargs): if kwargs.get(key): diff --git a/planetmint/backend/schema.py b/planetmint/backend/schema.py index a47cbb2..0a4a34c 100644 --- a/planetmint/backend/schema.py +++ b/planetmint/backend/schema.py @@ -20,6 +20,10 @@ logger = logging.getLogger(__name__) TABLES = ('transactions', 'blocks', 'assets', 'metadata', 'validators', 'elections', 'pre_commit', 'utxos', 'abci_chains') +SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx", + "elections", "meta_data", "pre_commits", "validators", + "transactions", "inputs", "outputs", "keys") + VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', 'russian', 'spanish', 'swedish', 'turkish', 'none', diff --git a/planetmint/backend/tarantool/connection.py b/planetmint/backend/tarantool/connection.py index 84324fd..d2c640e 100644 --- a/planetmint/backend/tarantool/connection.py +++ b/planetmint/backend/tarantool/connection.py @@ -34,6 +34,9 @@ class TarantoolDBConnection(Connection): self.SPACE_NAMES = ["abci_chains", "assets", "blocks", "blocks_tx", "elections", "meta_data", "pre_commits", "validators", "transactions", "inputs", "outputs", "keys"] + except tarantool.error.NetworkError as network_err: + logger.info('Host cant be reached') + raise network_err except: logger.info('Exception in _connect(): {}') raise ConfigurationError diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index f4e2aa6..c9ab8a4 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -8,6 +8,8 @@ from secrets import token_hex from operator import itemgetter +import tarantool.error + from planetmint.backend import query from planetmint.backend.utils import module_dispatch_registrar from planetmint.backend.tarantool.connection import TarantoolDBConnection @@ -165,22 +167,27 @@ def get_spent(connection, fullfil_transaction_id: str, fullfil_output_index: str @register_query(TarantoolDBConnection) def get_latest_block(connection): # TODO Here is used DESCENDING OPERATOR - space = connection.space("blocks") - _all_blocks = space.select() - _all_blocks = _all_blocks.data - block = {"app_hash": '', "height": 0, "transactions": []} + try: + space = connection.space("blocks") + _all_blocks = space.select() + _all_blocks = _all_blocks.data + block = {"app_hash": '', "height": 0, "transactions": []} - if len(_all_blocks) > 0: - _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0] - space = connection.space("blocks_tx") - _txids = space.select(_block[2], index="block_search") - _txids = _txids.data - block["app_hash"] = _block[0] - block["height"] = _block[1] - block["transactions"] = [tx[0] for tx in _txids] - else: - block = None - return block + if len(_all_blocks) > 0: + _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0] + space = connection.space("blocks_tx") + _txids = space.select(_block[2], index="block_search") + _txids = _txids.data + block["app_hash"] = _block[0] + block["height"] = _block[1] + block["transactions"] = [tx[0] for tx in _txids] + else: + block = None + return block + except tarantool.error.SchemaError: + return None + except Exception as err: + raise err @register_query(TarantoolDBConnection) @@ -377,12 +384,17 @@ def store_pre_commit_state(connection, state: dict): @register_query(TarantoolDBConnection) def get_pre_commit_state(connection): - space = connection.space("pre_commits") - _commit = space.select([], index="id_search").data - if len(_commit) == 0: + try: + space = connection.space("pre_commits") + _commit = space.select([], index="id_search").data + if len(_commit) == 0: + return None + _commit = sorted(_commit, key=itemgetter(1), reverse=True)[0] + return {"height": _commit[1], "transactions": _commit[2]} + except tarantool.error.SchemaError: return None - _commit = sorted(_commit, key=itemgetter(1), reverse=True)[0] - return {"height": _commit[1], "transactions": _commit[2]} + except Exception as err: + raise err @register_query(TarantoolDBConnection) @@ -434,27 +446,37 @@ def delete_elections(connection, height: int): @register_query(TarantoolDBConnection) def get_validator_set(connection, height: int = None): - space = connection.space("validators") - _validators = space.select() - _validators = _validators.data - if height is not None: - _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators if - validator[1] <= height] - return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) - else: - _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators] - return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) + try: + space = connection.space("validators") + _validators = space.select() + _validators = _validators.data + if height is not None: + _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators if + validator[1] <= height] + return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) + else: + _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators] + return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None) + except tarantool.error.SchemaError: + return None + except Exception as err: + raise err @register_query(TarantoolDBConnection) def get_election(connection, election_id: str): - space = connection.space("elections") - _elections = space.select(election_id, index="id_search") - _elections = _elections.data - if len(_elections) == 0: + try: + space = connection.space("elections") + _elections = space.select(election_id, index="id_search") + _elections = _elections.data + if len(_elections) == 0: + return None + _election = sorted(_elections, key=itemgetter(0), reverse=True)[0] + return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]} + except tarantool.error.SchemaError: return None - _election = sorted(_elections, key=itemgetter(0), reverse=True)[0] - return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]} + except Exception as err: + raise err @register_query(TarantoolDBConnection) @@ -489,9 +511,14 @@ def delete_abci_chain(connection, height: int): @register_query(TarantoolDBConnection) def get_latest_abci_chain(connection): - space = connection.space("abci_chains") - _all_chains = space.select().data - if len(_all_chains) == 0: + try: + space = connection.space("abci_chains") + _all_chains = space.select().data + if len(_all_chains) == 0: + return None + _chain = sorted(_all_chains, key=itemgetter(0), reverse=True)[0] + return {"height": _chain[0], "is_synced": _chain[1], "chain_id": _chain[2]} + except tarantool.error.SchemaError: return None - _chain = sorted(_all_chains, key=itemgetter(0), reverse=True)[0] - return {"height": _chain[0], "is_synced": _chain[1], "chain_id": _chain[2]} + except Exception as err: + raise err diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index 5745232..e79ed3e 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -9,7 +9,7 @@ register_schema = module_dispatch_registrar(backend.schema) SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx", "elections", "meta_data", "pre_commits", "validators", - "transactions", "inputs", "outputs", "keys") + "transactions", "inputs", "outputs", "keys", "utxos") SPACE_COMMANDS = { "abci_chains": "abci_chains = box.schema.space.create('abci_chains', {engine='memtx', is_sync = false})", @@ -165,12 +165,11 @@ def drop_database(connection, not_used=None): def create_database(connection, not_used=None): ''' - This function 'create_database' cannot be used with TarantoolDB connection Class. - It will be ignored if called. No Errors. + For tarantool implementation, this function runs + create_tables, to initiate spaces, schema and indexes. ''' - # connection.init_database() - warnings.warn("Function schema.'create_database', ignored. Cannot be used using TarantoolDB") + create_tables(None, None) def run_command_with_output(command): diff --git a/planetmint/config.py b/planetmint/config.py index 583d4c6..efa2355 100644 --- a/planetmint/config.py +++ b/planetmint/config.py @@ -59,7 +59,7 @@ class Config(metaclass=Singleton): 'backend': 'tarantool_db', 'connection_timeout': 5000, 'max_tries': 3, - 'name': 'bigchain', + 'name': 'universe', "reconnect_delay": 0.5, 'host': 'localhost', 'port': 3303, diff --git a/planetmint/lib.py b/planetmint/lib.py index bbab923..e4009d0 100644 --- a/planetmint/lib.py +++ b/planetmint/lib.py @@ -511,4 +511,5 @@ class Planetmint(object): return backend.query.delete_elections(self.connection, height) + Block = namedtuple('Block', ('app_hash', 'height', 'transactions')) diff --git a/requirements_old.txt b/requirements_old.txt deleted file mode 100644 index 4b26d68..0000000 --- a/requirements_old.txt +++ /dev/null @@ -1,64 +0,0 @@ -aiohttp==3.6.2 -aniso8601==9.0.1 -asn1crypto==1.4.0 -async-timeout==3.0.1 -attrs==21.4.0 -base58==1.0.3 -BigchainDB==2.2.2 -bigchaindb-abci==1.0.5 -capturer==3.0 -certifi==2021.10.8 -cffi==1.15.0 -chardet==3.0.4 -charset-normalizer==2.0.12 -click==8.0.3 -colorlog==4.1.0 -cryptoconditions==0.8.0 -cryptography==36.0.1 -Flask==1.1.2 -Flask-Cors==3.0.8 -Flask-RESTful==0.3.8 -gevent==20.6.2 -greenlet==0.4.16 -gunicorn==20.0.4 -humanfriendly==10.0 -hypothesis==6.39.3 -idna==2.10 -iniconfig==1.1.1 -itsdangerous==2.0.1 -Jinja2==3.0.3 -jsonschema==3.2.0 -logstats==0.3.0 -MarkupSafe==2.0.1 -msgpack==1.0.3 -multidict==4.7.6 -packaging==21.3 -Planetmint==0.9.0 -planetmint-cryptoconditions==0.9.3 -pluggy==1.0.0 -protobuf==3.6.1 -py==1.11.0 -pyasn1==0.4.8 -pycparser==2.21 -pymongo==3.7.2 -PyNaCl==1.1.2 -pyOpenSSL==22.0.0 -pyparsing==3.0.7 -pyrsistent==0.18.1 -pytest==7.0.1 -python-rapidjson==0.9.1 -pytz==2021.3 -PyYAML==5.3.1 -requests==2.23.0 -setproctitle==1.1.10 -six==1.16.0 -sortedcontainers==2.4.0 -tarantool==0.7.1 -tomli==2.0.1 -typing_extensions==4.1.1 -urllib3==1.25.11 -Werkzeug==2.0.3 -yarl==1.7.2 -zenroom==2.1.0.dev1647359536 -zope.event==4.5.0 -zope.interface==5.5.0.dev0 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 92ee787..2b5a3ee 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -20,7 +20,6 @@ from planetmint.migrations.chain_migration_election import ChainMigrationElectio from tests.utils import generate_election, generate_validators - def test_make_sure_we_dont_remove_any_command(): # thanks to: http://stackoverflow.com/a/18161115/597097 from planetmint.commands.planetmint import create_parser diff --git a/tests/conftest.py b/tests/conftest.py index e0bf2a4..5c1d2a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -123,29 +123,38 @@ def _configure_planetmint(request): @pytest.fixture(scope='session') def _setup_database(_configure_planetmint): # TODO Here is located setup database - from planetmint.backend.tarantool.connection import TarantoolDBConnection + from planetmint.backend.connection import Connection + from planetmint.config import Config - print('Deleting `{}` database') - db_conn = connect() - db_conn.drop_database() - db_conn.init_database() - print('Finished deleting ``') + print('Initializing test db') + dbname = Config().get()['database']['name'] + conn = Connection() + + _drop_db(conn, dbname) + schema.init_database(conn) + print('Finishing init database') yield - print('Initializing test db') - db_conn2 = connect() - db_conn2.drop_database() - print('Finishing init database') + print('Deleting `{}` database'.format(dbname)) + conn = Connection() + _drop_db(conn, dbname) + + print('Finished deleting `{}`'.format(dbname)) @pytest.fixture -def _bdb(_setup_database): - from planetmint.backend import connect +def _bdb(_setup_database, _configure_planetmint): + print(f"BDB CALL") + from planetmint.backend import Connection from planetmint.transactions.common.memoize import to_dict, from_dict from planetmint.models import Transaction - conn = connect() + from .utils import flush_db + from planetmint.config import Config + conn = Connection() yield + dbname = Config().get()['database']['name'] + flush_db(conn, dbname) to_dict.cache_clear() from_dict.cache_clear() @@ -355,7 +364,6 @@ def inputs(user_pk, b, alice): def _drop_db(conn, dbname): try: - conn.drop_database() schema.drop_database(conn, dbname) except DatabaseDoesNotExist: pass diff --git a/tests/utils.py b/tests/utils.py index ceffff0..786b02d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,7 +10,8 @@ import random from functools import singledispatch from planetmint.backend.localmongodb.connection import LocalMongoDBConnection -from planetmint.backend.schema import TABLES +from planetmint.backend.tarantool.connection import TarantoolDB +from planetmint.backend.schema import TABLES, SPACE_NAMES from planetmint.transactions.common import crypto from planetmint.transactions.common.transaction_mode_types import BROADCAST_TX_COMMIT from planetmint.transactions.types.assets.create import Create @@ -29,14 +30,28 @@ def flush_localmongo_db(connection, dbname): getattr(connection.conn[dbname], t).delete_many({}) +@flush_db.register(TarantoolDB) +def flush_tarantool_db(connection, dbname): + for s in SPACE_NAMES: + _space = connection.space(space_name=s) + _all_data = _space.select([]).data + for _id in _all_data: + if "assets" == s: + _space.delete(_id[1]) + elif "abci_chains" == s: + _space.delete(_id[2]) + else: + _space.delete(_id[0]) + + def generate_block(planet): from planetmint.transactions.common.crypto import generate_key_pair alice = generate_key_pair() tx = Create.generate([alice.public_key], - [([alice.public_key], 1)], - asset=None)\ - .sign([alice.private_key]) + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]) code, message = planet.write_transaction(tx, BROADCAST_TX_COMMIT) assert code == 202 @@ -55,7 +70,7 @@ def gen_vote(election, i, ed25519_node_keys): election_pub_key = Election.to_public_key(election.id) return Vote.generate([input_i], [([election_pub_key], votes_i)], - election_id=election.id)\ + election_id=election.id) \ .sign([key_i.private_key])