Merge branch 'liviu-lesan-planetmint-tarantool' into planetmint-tarantool

planetmint/backend/connection.py

@@ -5,11 +5,13 @@
 
 import logging
 from importlib import import_module
+
+import tarantool
+
 from planetmint.config import Config
 from planetmint.backend.exceptions import ConnectionError
 from planetmint.transactions.common.exceptions import ConfigurationError
 
-
 BACKENDS = {  # This is path to MongoDBClass
     'tarantool_db': 'planetmint.backend.tarantool.connection.TarantoolDBConnection',
     'localmongodb': 'planetmint.backend.localmongodb.connection.LocalMongoDBConnection'
@@ -48,7 +50,7 @@ def connect(host: str = None, port: int = None, login: str = None, password: str
             replicaset = _kwargs_parser(key="replicaset", kwargs=kwargs) or Config().get()['database']['replicaset']
             ssl = _kwargs_parser(key="ssl", kwargs=kwargs) or Config().get()['database']['ssl']
             login = login or Config().get()['database']['login'] if _kwargs_parser(key="login",
-                                                                                kwargs=kwargs) is None else _kwargs_parser(
+                                                                                   kwargs=kwargs) is None else _kwargs_parser(
                 key="login", kwargs=kwargs)
             password = password or Config().get()['database']['password'] if _kwargs_parser(key="password",
                                                                                             kwargs=kwargs) is None else _kwargs_parser(
@@ -63,13 +65,17 @@ def connect(host: str = None, port: int = None, login: str = None, password: str
             connection_timeout = _kwargs_parser(key="connection_timeout", kwargs=kwargs)
 
             return Class(host=host, port=port, dbname=dbname,
-                        max_tries=max_tries, connection_timeout=connection_timeout,
+                         max_tries=max_tries, connection_timeout=connection_timeout,
-                        replicaset=replicaset, ssl=ssl, login=login, password=password,
+                         replicaset=replicaset, ssl=ssl, login=login, password=password,
-                        ca_cert=ca_cert, certfile=certfile, keyfile=keyfile,
+                         ca_cert=ca_cert, certfile=certfile, keyfile=keyfile,
-                        keyfile_passphrase=keyfile_passphrase, crlfile=crlfile)
+                         keyfile_passphrase=keyfile_passphrase, crlfile=crlfile)
+    except tarantool.error.NetworkError as network_err:
+        print(f"Host {host}:{port} can't be reached.\n{network_err}")
+        raise network_err
     except:
-            logger.info('Exception in _connect(): {}')
+        logger.info('Exception in _connect(): {}')
-            raise ConfigurationError 
+        raise ConfigurationError
+
 
 def _kwargs_parser(key, kwargs):
     if kwargs.get(key):

planetmint/backend/schema.py

@@ -20,6 +20,10 @@ logger = logging.getLogger(__name__)
 TABLES = ('transactions', 'blocks', 'assets', 'metadata',
           'validators', 'elections', 'pre_commit', 'utxos', 'abci_chains')
 
+SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx",
+               "elections", "meta_data", "pre_commits", "validators",
+               "transactions", "inputs", "outputs", "keys")
+
 VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german',
                    'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian',
                    'russian', 'spanish', 'swedish', 'turkish', 'none',

planetmint/backend/tarantool/connection.py

@@ -34,6 +34,9 @@ class TarantoolDBConnection(Connection):
             self.SPACE_NAMES = ["abci_chains", "assets", "blocks", "blocks_tx",
                                 "elections", "meta_data", "pre_commits", "validators",
                                 "transactions", "inputs", "outputs", "keys"]
+        except tarantool.error.NetworkError as network_err:
+            logger.info('Host cant be reached')
+            raise network_err
         except:
             logger.info('Exception in _connect(): {}')
             raise ConfigurationError

planetmint/backend/tarantool/query.py

@@ -8,6 +8,8 @@
 from secrets import token_hex
 from operator import itemgetter
 
+import tarantool.error
+
 from planetmint.backend import query
 from planetmint.backend.utils import module_dispatch_registrar
 from planetmint.backend.tarantool.connection import TarantoolDBConnection
@@ -165,22 +167,27 @@ def get_spent(connection, fullfil_transaction_id: str, fullfil_output_index: str
 
 @register_query(TarantoolDBConnection)
 def get_latest_block(connection):  # TODO Here is used DESCENDING OPERATOR
-    space = connection.space("blocks")
+    try:
-    _all_blocks = space.select()
+        space = connection.space("blocks")
-    _all_blocks = _all_blocks.data
+        _all_blocks = space.select()
-    block = {"app_hash": '', "height": 0, "transactions": []}
+        _all_blocks = _all_blocks.data
+        block = {"app_hash": '', "height": 0, "transactions": []}
 
-    if len(_all_blocks) > 0:
+        if len(_all_blocks) > 0:
-        _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0]
+            _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0]
-        space = connection.space("blocks_tx")
+            space = connection.space("blocks_tx")
-        _txids = space.select(_block[2], index="block_search")
+            _txids = space.select(_block[2], index="block_search")
-        _txids = _txids.data
+            _txids = _txids.data
-        block["app_hash"] = _block[0]
+            block["app_hash"] = _block[0]
-        block["height"] = _block[1]
+            block["height"] = _block[1]
-        block["transactions"] = [tx[0] for tx in _txids]
+            block["transactions"] = [tx[0] for tx in _txids]
-    else:
+        else:
-        block = None
+            block = None
-    return block
+        return block
+    except tarantool.error.SchemaError:
+        return None
+    except Exception as err:
+        raise err
 
 
 @register_query(TarantoolDBConnection)
@@ -377,12 +384,17 @@ def store_pre_commit_state(connection, state: dict):
 
 @register_query(TarantoolDBConnection)
 def get_pre_commit_state(connection):
-    space = connection.space("pre_commits")
+    try:
-    _commit = space.select([], index="id_search").data
+        space = connection.space("pre_commits")
-    if len(_commit) == 0:
+        _commit = space.select([], index="id_search").data
+        if len(_commit) == 0:
+            return None
+        _commit = sorted(_commit, key=itemgetter(1), reverse=True)[0]
+        return {"height": _commit[1], "transactions": _commit[2]}
+    except tarantool.error.SchemaError:
         return None
-    _commit = sorted(_commit, key=itemgetter(1), reverse=True)[0]
+    except Exception as err:
-    return {"height": _commit[1], "transactions": _commit[2]}
+        raise err
 
 
 @register_query(TarantoolDBConnection)
@@ -434,27 +446,37 @@ def delete_elections(connection, height: int):
 
 @register_query(TarantoolDBConnection)
 def get_validator_set(connection, height: int = None):
-    space = connection.space("validators")
+    try:
-    _validators = space.select()
+        space = connection.space("validators")
-    _validators = _validators.data
+        _validators = space.select()
-    if height is not None:
+        _validators = _validators.data
-        _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators if
+        if height is not None:
-                       validator[1] <= height]
+            _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators if
-        return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None)
+                           validator[1] <= height]
-    else:
+            return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None)
-        _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators]
+        else:
-        return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None)
+            _validators = [{"height": validator[1], "validators": validator[2]} for validator in _validators]
+            return next(iter(sorted(_validators, key=lambda k: k["height"], reverse=True)), None)
+    except tarantool.error.SchemaError:
+        return None
+    except Exception as err:
+        raise err
 
 
 @register_query(TarantoolDBConnection)
 def get_election(connection, election_id: str):
-    space = connection.space("elections")
+    try:
-    _elections = space.select(election_id, index="id_search")
+        space = connection.space("elections")
-    _elections = _elections.data
+        _elections = space.select(election_id, index="id_search")
-    if len(_elections) == 0:
+        _elections = _elections.data
+        if len(_elections) == 0:
+            return None
+        _election = sorted(_elections, key=itemgetter(0), reverse=True)[0]
+        return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]}
+    except tarantool.error.SchemaError:
         return None
-    _election = sorted(_elections, key=itemgetter(0), reverse=True)[0]
+    except Exception as err:
-    return {"election_id": _election[0], "height": _election[1], "is_concluded": _election[2]}
+        raise err
 
 
 @register_query(TarantoolDBConnection)
@@ -489,9 +511,14 @@ def delete_abci_chain(connection, height: int):
 
 @register_query(TarantoolDBConnection)
 def get_latest_abci_chain(connection):
-    space = connection.space("abci_chains")
+    try:
-    _all_chains = space.select().data
+        space = connection.space("abci_chains")
-    if len(_all_chains) == 0:
+        _all_chains = space.select().data
+        if len(_all_chains) == 0:
+            return None
+        _chain = sorted(_all_chains, key=itemgetter(0), reverse=True)[0]
+        return {"height": _chain[0], "is_synced": _chain[1], "chain_id": _chain[2]}
+    except tarantool.error.SchemaError:
         return None
-    _chain = sorted(_all_chains, key=itemgetter(0), reverse=True)[0]
+    except Exception as err:
-    return {"height": _chain[0], "is_synced": _chain[1], "chain_id": _chain[2]}
+        raise err

planetmint/backend/tarantool/schema.py

@@ -9,7 +9,7 @@ register_schema = module_dispatch_registrar(backend.schema)
 
 SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx",
                "elections", "meta_data", "pre_commits", "validators",
-               "transactions", "inputs", "outputs", "keys")
+               "transactions", "inputs", "outputs", "keys", "utxos")
 
 SPACE_COMMANDS = {
     "abci_chains": "abci_chains = box.schema.space.create('abci_chains', {engine='memtx', is_sync = false})",
@@ -165,12 +165,11 @@ def drop_database(connection, not_used=None):
 def create_database(connection, not_used=None):
     '''
 
-    This function 'create_database' cannot be used with TarantoolDB connection Class.
+    For tarantool implementation, this function runs
-    It will be ignored if called. No Errors.
+    create_tables, to initiate spaces, schema and indexes.
 
     '''
-    # connection.init_database()
+    create_tables(None, None)
-    warnings.warn("Function schema.'create_database', ignored. Cannot be used using TarantoolDB")
 
 
 def run_command_with_output(command):

planetmint/config.py

@@ -59,7 +59,7 @@ class Config(metaclass=Singleton):
             'backend': 'tarantool_db',
             'connection_timeout': 5000,
             'max_tries': 3,
-            'name': 'bigchain',
+            'name': 'universe',
             "reconnect_delay": 0.5,
             'host': 'localhost',
             'port': 3303,

planetmint/lib.py

@@ -511,4 +511,5 @@ class Planetmint(object):
         return backend.query.delete_elections(self.connection, height)
 
 
+
 Block = namedtuple('Block', ('app_hash', 'height', 'transactions'))

requirements_old.txt

@@ -1,64 +0,0 @@
-aiohttp==3.6.2
-aniso8601==9.0.1
-asn1crypto==1.4.0
-async-timeout==3.0.1
-attrs==21.4.0
-base58==1.0.3
-BigchainDB==2.2.2
-bigchaindb-abci==1.0.5
-capturer==3.0
-certifi==2021.10.8
-cffi==1.15.0
-chardet==3.0.4
-charset-normalizer==2.0.12
-click==8.0.3
-colorlog==4.1.0
-cryptoconditions==0.8.0
-cryptography==36.0.1
-Flask==1.1.2
-Flask-Cors==3.0.8
-Flask-RESTful==0.3.8
-gevent==20.6.2
-greenlet==0.4.16
-gunicorn==20.0.4
-humanfriendly==10.0
-hypothesis==6.39.3
-idna==2.10
-iniconfig==1.1.1
-itsdangerous==2.0.1
-Jinja2==3.0.3
-jsonschema==3.2.0
-logstats==0.3.0
-MarkupSafe==2.0.1
-msgpack==1.0.3
-multidict==4.7.6
-packaging==21.3
-Planetmint==0.9.0
-planetmint-cryptoconditions==0.9.3
-pluggy==1.0.0
-protobuf==3.6.1
-py==1.11.0
-pyasn1==0.4.8
-pycparser==2.21
-pymongo==3.7.2
-PyNaCl==1.1.2
-pyOpenSSL==22.0.0
-pyparsing==3.0.7
-pyrsistent==0.18.1
-pytest==7.0.1
-python-rapidjson==0.9.1
-pytz==2021.3
-PyYAML==5.3.1
-requests==2.23.0
-setproctitle==1.1.10
-six==1.16.0
-sortedcontainers==2.4.0
-tarantool==0.7.1
-tomli==2.0.1
-typing_extensions==4.1.1
-urllib3==1.25.11
-Werkzeug==2.0.3
-yarl==1.7.2
-zenroom==2.1.0.dev1647359536
-zope.event==4.5.0
-zope.interface==5.5.0.dev0

tests/commands/test_commands.py

@@ -20,7 +20,6 @@ from planetmint.migrations.chain_migration_election import ChainMigrationElectio
 
 from tests.utils import generate_election, generate_validators
 
-
 def test_make_sure_we_dont_remove_any_command():
     # thanks to: http://stackoverflow.com/a/18161115/597097
     from planetmint.commands.planetmint import create_parser

tests/conftest.py

@@ -123,29 +123,38 @@ def _configure_planetmint(request):
 
 @pytest.fixture(scope='session')
 def _setup_database(_configure_planetmint):  # TODO Here is located setup database
-    from planetmint.backend.tarantool.connection import TarantoolDBConnection
+    from planetmint.backend.connection import Connection
+    from planetmint.config import Config
 
-    print('Deleting `{}` database')
+    print('Initializing test db')
-    db_conn = connect()
+    dbname = Config().get()['database']['name']
-    db_conn.drop_database()
+    conn = Connection()
-    db_conn.init_database()
+
-    print('Finished deleting ``')
+    _drop_db(conn, dbname)
+    schema.init_database(conn)
+    print('Finishing init database')
 
     yield
 
-    print('Initializing test db')
+    print('Deleting `{}` database'.format(dbname))
-    db_conn2 = connect()
+    conn = Connection()
-    db_conn2.drop_database()
+    _drop_db(conn, dbname)
-    print('Finishing init database')
+
+    print('Finished deleting `{}`'.format(dbname))
 
 
 @pytest.fixture
-def _bdb(_setup_database):
+def _bdb(_setup_database, _configure_planetmint):
-    from planetmint.backend import connect
+    print(f"BDB CALL")
+    from planetmint.backend import Connection
     from planetmint.transactions.common.memoize import to_dict, from_dict
     from planetmint.models import Transaction
-    conn = connect()
+    from .utils import flush_db
+    from planetmint.config import Config
+    conn = Connection()
     yield
+    dbname = Config().get()['database']['name']
+    flush_db(conn, dbname)
 
     to_dict.cache_clear()
     from_dict.cache_clear()
@@ -355,7 +364,6 @@ def inputs(user_pk, b, alice):
 
 def _drop_db(conn, dbname):
     try:
-        conn.drop_database()
         schema.drop_database(conn, dbname)
     except DatabaseDoesNotExist:
         pass

tests/utils.py

@@ -10,7 +10,8 @@ import random
 from functools import singledispatch
 
 from planetmint.backend.localmongodb.connection import LocalMongoDBConnection
-from planetmint.backend.schema import TABLES
+from planetmint.backend.tarantool.connection import TarantoolDB
+from planetmint.backend.schema import TABLES, SPACE_NAMES
 from planetmint.transactions.common import crypto
 from planetmint.transactions.common.transaction_mode_types import BROADCAST_TX_COMMIT
 from planetmint.transactions.types.assets.create import Create
@@ -29,14 +30,28 @@ def flush_localmongo_db(connection, dbname):
         getattr(connection.conn[dbname], t).delete_many({})
 
 
+@flush_db.register(TarantoolDB)
+def flush_tarantool_db(connection, dbname):
+    for s in SPACE_NAMES:
+        _space = connection.space(space_name=s)
+        _all_data = _space.select([]).data
+        for _id in _all_data:
+            if "assets" == s:
+                _space.delete(_id[1])
+            elif "abci_chains" == s:
+                _space.delete(_id[2])
+            else:
+                _space.delete(_id[0])
+
+
 def generate_block(planet):
     from planetmint.transactions.common.crypto import generate_key_pair
 
     alice = generate_key_pair()
     tx = Create.generate([alice.public_key],
-                            [([alice.public_key], 1)],
+                         [([alice.public_key], 1)],
-                            asset=None)\
+                         asset=None) \
-                    .sign([alice.private_key])
+        .sign([alice.private_key])
 
     code, message = planet.write_transaction(tx, BROADCAST_TX_COMMIT)
     assert code == 202
@@ -55,7 +70,7 @@ def gen_vote(election, i, ed25519_node_keys):
     election_pub_key = Election.to_public_key(election.id)
     return Vote.generate([input_i],
                          [([election_pub_key], votes_i)],
-                         election_id=election.id)\
+                         election_id=election.id) \
         .sign([key_i.private_key])