From 1e9c4c28f7f9cabbb22e17af928101a650fcca00 Mon Sep 17 00:00:00 2001 From: andrei Date: Tue, 15 Feb 2022 16:44:40 +0200 Subject: [PATCH] rewrited init_tarantool functions and others --- planetmint/backend/connection_tarantool.py | 67 ++++++++++++++++++++-- planetmint/backend/tarantool/database.py | 38 ++++++------ planetmint/backend/tarantool/drop_db.lua | 5 -- planetmint/backend/tarantool/init.lua | 19 ------ planetmint/backend/tarantool/utils.py | 10 ++-- requirements_old.txt | 47 +++++++++++++++ tests/backend/tarantool/test_queries.py | 4 -- tests/conftest.py | 7 ++- 8 files changed, 137 insertions(+), 60 deletions(-) delete mode 100644 planetmint/backend/tarantool/drop_db.lua delete mode 100644 planetmint/backend/tarantool/init.lua create mode 100644 requirements_old.txt diff --git a/planetmint/backend/connection_tarantool.py b/planetmint/backend/connection_tarantool.py index d939ec0..889da55 100644 --- a/planetmint/backend/connection_tarantool.py +++ b/planetmint/backend/connection_tarantool.py @@ -7,19 +7,75 @@ import logging from importlib import import_module from itertools import repeat +import tarantool + +import os +import pathlib + +from planetmint.backend.tarantool.utils import run + import planetmint from planetmint.backend.exceptions import ConnectionError from planetmint.backend.utils import get_planetmint_config_value, get_planetmint_config_value_or_key_error from planetmint.common.exceptions import ConfigurationError BACKENDS = { # This is path to MongoDBClass - 'tarantool_db': 'planetmint.backend.tarantool.connection_tarantool.TarantoolDB', + 'tarantool_db': 'planetmint.backend.connection_tarantool.TarantoolDB', } logger = logging.getLogger(__name__) -def connect(host: str, port: int, username: str, password: str, backend: str): +def init_tarantool(): + tarantool_root = os.path.join(pathlib.Path.home(), 'tarantool') + snap = os.path.join(pathlib.Path.home(), 'tarantool_snap') + init_lua = os.path.join(tarantool_root, 'init.lua') + if os.path.exists(tarantool_root) is not True: + run(["mkdir", tarantool_root]) + run(["mkdir", snap]) + run(["ln", "-s", init_lua, "init.lua"], snap) + run(["tarantool", "init.lua"], tarantool_root) + else: + raise Exception("There is a instance of tarantool already created in %s" + snap) + + +def drop_tarantool(): + tarantool_root = os.path.join(pathlib.Path.home(), 'tarantool') + snap = os.path.join(pathlib.Path.home(), 'tarantool_snap') + init_lua = os.path.join(tarantool_root, 'init.lua') + drop_lua = os.path.join(tarantool_root, "/drop_db.lua") + if os.path.exists(init_lua) is not True: + run(["ln", "-s", drop_lua, "drop_db.lua"], snap) + run(["tarantool", "drop_db.lua"]) + else: + raise Exception("There is no tarantool spaces to drop") + + +class TarantoolDB: + def __init__(self, host: str, port: int, username: str, password: str): + init_tarantool() + self.db_connect = tarantool.connect(host=host, port=port, user=username, password=password) + self._spaces = { + "abci_chains": self.db_connect.space("abci_chains"), + "assets": self.db_connect.space("assets"), + "blocks": {"blocks": self.db_connect.space("blocks"), "blocks_tx": self.db_connect.space("blocks_tx")}, + "elections": self.db_connect.space("elections"), + "meta_data": self.db_connect.space("meta_data"), + "pre_commits": self.db_connect.space("pre_commits"), + "validators": self.db_connect.space("validators"), + "transactions": { + "transactions": self.db_connect.space("transactions"), + "inputs": self.db_connect.space("inputs"), + "outputs": self.db_connect.space("outputs"), + "keys": self.db_connect.space("keys") + } + } + + def get_space(self, spacename: str): + return self._spaces[spacename] + + +def connect(host: str = None, port: int = None, username: str = "admin", password: str = "pass", backend: str = None): """Create a new connection to the database backend. All arguments default to the current configuration's values if not @@ -68,8 +124,7 @@ class Connection: from and implements this class. """ - def __init__(self, host=None, port=None, dbname=None, - connection_timeout=None, max_tries=None, + def __init__(self, host=None, port=None, connection_timeout=None, max_tries=None, **kwargs): """Create a new :class:`~.Connection` instance. @@ -90,7 +145,6 @@ class Connection: self.host = host or dbconf['host'] self.port = port or dbconf['port'] - self.dbname = dbname or dbconf['name'] self.connection_timeout = connection_timeout if connection_timeout is not None \ else dbconf['connection_timeout'] self.max_tries = max_tries if max_tries is not None else dbconf['max_tries'] @@ -99,11 +153,13 @@ class Connection: @property def conn(self): + pass if self._conn is None: self.connect() return self._conn def run(self, query): + pass """Run a query. Args: @@ -120,6 +176,7 @@ class Connection: raise NotImplementedError() def connect(self): + pass """Try to connect to the database. Raises: diff --git a/planetmint/backend/tarantool/database.py b/planetmint/backend/tarantool/database.py index 43c9734..aeb2584 100644 --- a/planetmint/backend/tarantool/database.py +++ b/planetmint/backend/tarantool/database.py @@ -3,6 +3,25 @@ import os from planetmint.backend.tarantool.utils import run +def init_tarantool(): + if os.path.exists(os.path.join(os.getcwd(), 'tarantool', 'init.lua')) is not True: + path = os.getcwd() + run(["mkdir", "tarantool_snap"]) + run(["ln", "-s", path + "/init.lua", "init.lua"], path + "/tarantool_snap") + run(["tarantool", "init.lua"], path + "/tarantool") + else: + raise Exception("There is a instance of tarantool already created in %s" + os.getcwd() + "/tarantool_snap") + + +def drop_tarantool(): + if os.path.exists(os.path.join(os.getcwd(), 'tarantool', 'init.lua')) is not True: + path = os.getcwd() + run(["ln", "-s", path + "/drop_db.lua", "drop_db.lua"], path + "/tarantool_snap") + run(["tarantool", "drop_db.lua"]) + else: + raise Exception("There is no tarantool spaces to drop") + + class TarantoolDB: def __init__(self, host: str, port: int, username: str, password: str): self.db_connect = tarantool.connect(host=host, port=port, user=username, password=password) @@ -24,22 +43,3 @@ class TarantoolDB: def get_space(self, spacename: str): return self._spaces[spacename] - - -def init_tarantool(): - if os.path.exists(os.path.join(os.getcwd(), 'tarantool', 'init.lua')) is not True: - path = os.getcwd() - run(["mkdir", "tarantool_snap"]) - run(["ln", "-s", path + "/init.lua", "init.lua"], path + "/tarantool_snap") - run(["tarantool", "init.lua"], path + "/tarantool") - else: - raise Exception("There is a instance of tarantool already created in %s" + os.getcwd() + "/tarantool_snap") - - -def drop_tarantool(): - if os.path.exists(os.path.join(os.getcwd(), 'tarantool', 'init.lua')) is not True: - path = os.getcwd() - run(["ln", "-s", path + "/drop_db.lua", "drop_db.lua"], path + "/tarantool_snap") - run(["tarantool", "drop_db.lua"]) - else: - raise Exception("There is no tarantool spaces to drop") diff --git a/planetmint/backend/tarantool/drop_db.lua b/planetmint/backend/tarantool/drop_db.lua deleted file mode 100644 index aca9792..0000000 --- a/planetmint/backend/tarantool/drop_db.lua +++ /dev/null @@ -1,5 +0,0 @@ -box.space.transactions.drop() -box.space.output.drop() -box.space.inputs.drop() -box.space.keys.drop() -box.snapshot() \ No newline at end of file diff --git a/planetmint/backend/tarantool/init.lua b/planetmint/backend/tarantool/init.lua deleted file mode 100644 index e8adeab..0000000 --- a/planetmint/backend/tarantool/init.lua +++ /dev/null @@ -1,19 +0,0 @@ -box.cfg{listen=3301} - -transactions = box.schema.space.create('transactions',{engine='memtx' , is_sync=false,if_not_exists = true}) -transactions:format({{name='transaction_id' , type='string'},{name='operation' , type='string'}, {name='version' ,type='string'}}) -transactions:create_index('id_search' , {type = 'hash' , parts={'transaction_id'},if_not_exists=true}) - -inputs = box.schema.space.create('inputs',{engine='memtx' , is_sync=false,if_not_exists = true}) -inputs:format({{name='transaction_id' , type='string'},{name='fullfilment' , type='string'},{name='owners_before' , type='array'}, {name='fulfills_transaction_id', type = 'string'}, {name='fulfills_output_index', type = 'string'}}) -inputs:create_index('spent_search' , {type = 'hash' , parts={'fulfills_transaction_id', 'fulfills_output_index'},if_not_exists=true}) - -outputs = box.schema.space.create('outputs',{engine='memtx' , is_sync=false,if_not_exists = true}) -outputs:format({{name='transaction_id' , type='string'}, {name='amount' , type='string'}, {name='uri', type='string'}, {name='details_type', type='string'}, {name='details_public_key', type='string'}, {name = 'public_keys', type = 'array'}}) -outputs:create_index('id_search' ,{type='hash' , parts={'transaction_id'},if_not_exists=true}) -outputs:create_index('keys_search' ,{type='rtree' , parts={'public_keys'},if_not_exists=true}) - -keys = box.schema.space.create('keys',{engine='memtx' , is_sync=false,if_not_exists = true}) -keys:format({{name='transaction_id' , type='string'}, {name='public_keys' , type='array'}, {name = 'output_id', type = 'string'}}) -keys:create_index('id_search' ,{type='hash' , parts={'transaction_id', 'output_id'},if_not_exists=true}) -keys:create_index('keys_search', {type='rtree', parts={'public_keys'},if_not_exists=true}) \ No newline at end of file diff --git a/planetmint/backend/tarantool/utils.py b/planetmint/backend/tarantool/utils.py index 1c10063..52c042a 100644 --- a/planetmint/backend/tarantool/utils.py +++ b/planetmint/backend/tarantool/utils.py @@ -10,17 +10,17 @@ def run(command , path=None): stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output , error = p.communicate() + output, error = p.communicate() if p.returncode != 0: - print(p.returncode + "\n" + output + "\n" +error) + print(str(p.returncode) + "\n" + str(output) + "\n" + str(error)) else: - p=subprocess.run( + p=subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output , error = p.communicate() + output, error = p.communicate() if p.returncode != 0: - print(p.returncode + "\n" + output + "\n" +error) + print(str(p.returncode) + "\n" + str(output) + "\n" + str(error)) diff --git a/requirements_old.txt b/requirements_old.txt new file mode 100644 index 0000000..20f32a4 --- /dev/null +++ b/requirements_old.txt @@ -0,0 +1,47 @@ +aiohttp==3.6.2 +aniso8601==9.0.1 +asn1crypto==1.4.0 +async-timeout==3.0.1 +attrs==21.4.0 +base58==1.0.3 +BigchainDB==2.2.2 +bigchaindb-abci==1.0.5 +certifi==2021.10.8 +cffi==1.15.0 +chardet==3.0.4 +click==8.0.3 +colorlog==4.1.0 +cryptoconditions==0.8.0 +cryptography==2.3.1 +Flask==1.1.2 +Flask-Cors==3.0.8 +Flask-RESTful==0.3.8 +gevent==20.6.2 +greenlet==0.4.16 +gunicorn==20.0.4 +idna==2.10 +itsdangerous==2.0.1 +Jinja2==3.0.3 +jsonschema==3.2.0 +logstats==0.3.0 +MarkupSafe==2.0.1 +multidict==4.7.6 +packaging==21.3 +protobuf==3.6.1 +pyasn1==0.4.8 +pycparser==2.21 +pymongo==3.7.2 +PyNaCl==1.1.2 +pyparsing==3.0.7 +pyrsistent==0.18.1 +python-rapidjson==0.9.1 +pytz==2021.3 +PyYAML==5.3.1 +requests==2.23.0 +setproctitle==1.1.10 +six==1.16.0 +urllib3==1.25.11 +Werkzeug==2.0.3 +yarl==1.7.2 +zope.event==4.5.0 +zope.interface==5.5.0.dev0 diff --git a/tests/backend/tarantool/test_queries.py b/tests/backend/tarantool/test_queries.py index 2831243..fb7d5ba 100644 --- a/tests/backend/tarantool/test_queries.py +++ b/tests/backend/tarantool/test_queries.py @@ -12,15 +12,11 @@ from planetmint.backend import connect, query pytestmark = pytest.mark.bdb -conn = connect() -print(conn) - def test_get_txids_filtered(signed_create_tx, signed_transfer_tx): from planetmint.backend import connect, query from planetmint.models import Transaction conn = connect() # TODO First rewrite to get here tarantool connection - print(conn) # # create and insert two blocks, one for the create and one for the # # transfer transaction # conn.db.transactions.insert_one(signed_create_tx.to_dict()) diff --git a/tests/conftest.py b/tests/conftest.py index 77e22cf..b6ceb4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ from logging import getLogger from logging.config import dictConfig import pytest -from pymongo import MongoClient +# from pymongo import MongoClient from planetmint import ValidatorElection from planetmint.common import crypto @@ -48,7 +48,7 @@ def pytest_addoption(parser): parser.addoption( '--database-backend', action='store', - default=os.environ.get('PLANETMINT_DATABASE_BACKEND', 'localmongodb'), + default=os.environ.get('PLANETMINT_DATABASE_BACKEND', 'tarantool_db'), help='Defines the backend to use (available: {})'.format(backends), ) @@ -96,7 +96,8 @@ def _configure_planetmint(request): if xdist_suffix: test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) - backend = request.config.getoption('--database-backend') + # backend = request.config.getoption('--database-backend') + backend = "tarantool_db" config = { 'database': planetmint._database_map[backend],