From ec074553fd127d4929f0b0962600fb44b9a4b3f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Eckel?= Date: Fri, 8 Apr 2022 11:46:42 +0200 Subject: [PATCH] added some basic fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jürgen Eckel --- docker-compose.yml | 17 ++-- planetmint/backend/tarantool/connection.py | 20 ++-- planetmint/backend/tarantool/utils.py | 62 ++++++++---- planetmint/common/transaction.py | 9 +- planetmint/config.py | 2 +- planetmint/core.py | 4 +- tests/backend/tarantool/Pipfile | 12 +++ tests/backend/tarantool/Pipfile.lock | 78 +++++++++++++++ tests/backend/tarantool/conftest.py | 31 +++++- tests/backend/tarantool/test_schema.py | 107 ++++++++++----------- tests/commands/test_commands.py | 3 +- tests/conftest.py | 2 + 12 files changed, 248 insertions(+), 99 deletions(-) create mode 100644 tests/backend/tarantool/Pipfile create mode 100644 tests/backend/tarantool/Pipfile.lock diff --git a/docker-compose.yml b/docker-compose.yml index 9230301..378ad7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,16 +15,21 @@ services: command: mongod restart: always tarantool: - image: tarantool/tarantool:2.3.1 + image: tarantool/tarantool:2.8.3 ports: - "5200:5200" - "3301:3301" - command: tarantool + - "3303:3303" + - "8081:8081" + volumes: + - ./planetmint/backend/tarantool/basic.lua:/opt/tarantool/basic.lua + command: tarantool /opt/tarantool/basic.lua restart: always planetmint: depends_on: - - mongodb + #- mongodb - tendermint + - tarantool build: context: . dockerfile: Dockerfile-dev @@ -38,9 +43,9 @@ services: - ./pytest.ini:/usr/src/app/pytest.ini - ./tox.ini:/usr/src/app/tox.ini environment: - PLANETMINT_DATABASE_BACKEND: localmongodb - PLANETMINT_DATABASE_HOST: mongodb - PLANETMINT_DATABASE_PORT: 27017 + PLANETMINT_DATABASE_BACKEND: tarantooldb + PLANETMINT_DATABASE_HOST: tarantool + PLANETMINT_DATABASE_PORT: 3303 PLANETMINT_SERVER_BIND: 0.0.0.0:9984 PLANETMINT_WSSERVER_HOST: 0.0.0.0 PLANETMINT_WSSERVER_ADVERTISED_HOST: planetmint diff --git a/planetmint/backend/tarantool/connection.py b/planetmint/backend/tarantool/connection.py index 2e99600..2f0767a 100644 --- a/planetmint/backend/tarantool/connection.py +++ b/planetmint/backend/tarantool/connection.py @@ -16,17 +16,19 @@ class TarantoolDB: reset_database: bool = False): self.host = host self.port = port - self.db_connect = tarantool.connect(host=host, port=port, user=user, password=password) + # TODO add user support later on + #self.db_connect = tarantool.connect(host=host, port=port, user=user, password=password) + self.db_connect = tarantool.connect(host=host, port=port) self._load_setup_files() if reset_database: self.drop_database() self.init_database() def _load_setup_files(self): - init_path = Config().get()["database"]["init_config"]["absolute_path"] - drop_path = Config().get()["database"]["drop_config"]["absolute_path"] - self.drop_commands = self.__read_commands(file_path=init_path) - self.init_commands = self.__read_commands(file_path=drop_path) + self.init_path = Config().get()["database"]["init_config"]["absolute_path"] + self.drop_path = Config().get()["database"]["drop_config"]["absolute_path"] + #self.drop_commands = self.__read_commands(file_path=init_path) + #self.init_commands = self.__read_commands(file_path=drop_path) def space(self, space_name: str): return self.db_connect.space(space_name) @@ -41,17 +43,17 @@ class TarantoolDB: return commands def drop_database(self): - from planetmint.backend.tarantool.utils import run + from planetmint.backend.tarantool.utils import run2 db_config = Config().get()["database"] # drop_config = db_config["drop_config"] # f_path = "%s%s" % (drop_config["relative_path"], drop_config["drop_file"]) # commands = self.__read_commands(file_path=f_path) - run(commands=self.drop_commands, config=db_config) + run2(commands=self.drop_path, config=db_config) def init_database(self): - from planetmint.backend.tarantool.utils import run + from planetmint.backend.tarantool.utils import run2 db_config = Config().get()["database"] # init_config = db_config["init_config"] # f_path = "%s%s" % (init_config["relative_path"], init_config["init_file"]) # commands = self.__read_commands(file_path=f_path) - run(commands=self.init_commands, config=db_config) + run2(commands=self.init_path, config=db_config) diff --git a/planetmint/backend/tarantool/utils.py b/planetmint/backend/tarantool/utils.py index 6c417e3..e4894f3 100644 --- a/planetmint/backend/tarantool/utils.py +++ b/planetmint/backend/tarantool/utils.py @@ -2,9 +2,11 @@ import subprocess from planetmint.config import Config -def run(commands: list, config: dict): + + +def run2(commands: list, config: dict): sshProcess = subprocess.Popen( - ['%s %s:%s@%s:%s' % (config["service"], config["login"], config["password"], config["host"], config["port"])], + ['%s %s:%s < %s' % ("tarantoolctl connect", "localhost", "3303", "planetmint/backend/tarantool/init.lua")], stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True, @@ -20,21 +22,41 @@ def run(commands: list, config: dict): # TODO To add here Exception Handler for stdout # for line in sshProcess.stdout: # print(line) - - -def __read_commands(file_path): - with open(file_path, "r") as cmd_file: - commands = [line.strip()+'\n' for line in cmd_file.readlines() if len(str(line)) > 1] - cmd_file.close() - return commands - - -def _load_setup_files(): - drop_commands = __read_commands(file_path="planetmint/backend/tarantool/drop.lua") - init_commands = __read_commands(file_path="planetmint/backend/tarantool/init.lua") - return init_commands, drop_commands - - -init, drop = _load_setup_files() -db_config = Config().get()["database"] -run(commands=drop, config=db_config) +# +#def run(commands: list, config: dict): +# sshProcess = subprocess.Popen( +# ['%s %s:%s@%s:%s' % (config["service"], config["login"], config["password"], config["host"], config["port"])], +# stdin=subprocess.PIPE, +# stdout=subprocess.PIPE, +# universal_newlines=True, +# bufsize=0, +# shell=True) +# +# for cmd in commands: +# try: +# sshProcess.stdin.write(cmd) +# except Exception as cmd_err: +# print(str(cmd_err)) +# sshProcess.stdin.close() +# # TODO To add here Exception Handler for stdout +# # for line in sshProcess.stdout: +# # print(line) +# +# +#def __read_commands(file_path): +# with open(file_path, "r") as cmd_file: +# commands = [line.strip()+'\n' for line in cmd_file.readlines() if len(str(line)) > 1] +# cmd_file.close() +# return commands +# +# +#def _load_setup_files(): +# drop_commands = __read_commands(file_path="planetmint/backend/tarantool/drop.lua") +# init_commands = __read_commands(file_path="planetmint/backend/tarantool/init.lua") +# return init_commands, drop_commands +# +# +#init, drop = _load_setup_files() +#db_config = Config().get()["database"] +#run(commands=drop, config=db_config) +# \ No newline at end of file diff --git a/planetmint/common/transaction.py b/planetmint/common/transaction.py index 1e5c9cd..c1c5600 100644 --- a/planetmint/common/transaction.py +++ b/planetmint/common/transaction.py @@ -1123,7 +1123,8 @@ class Transaction(object): # TODO: This method shouldn't call `_remove_signatures` def __str__(self): - tx = Transaction._remove_signatures(self.to_dict()) + _tx = self.to_dict() + tx = Transaction._remove_signatures(_tx) return Transaction._to_str(tx) @classmethod @@ -1182,6 +1183,10 @@ class Transaction(object): tx_body_serialized = Transaction._to_str(tx_body) valid_tx_id = Transaction._to_hash(tx_body_serialized) + print( f" valid TX : {valid_tx_id}") + print( f" proposed TX id : {proposed_tx_id}") + print( f" tx body : {tx_body}") + print( f" tx serialized : {tx_body_serialized}") if proposed_tx_id != valid_tx_id: err_msg = ("The transaction's id '{}' isn't equal to " "the hash of its body, i.e. it's not valid.") @@ -1200,7 +1205,7 @@ class Transaction(object): """ operation = tx.get('operation', Transaction.CREATE) if isinstance(tx, dict) else Transaction.CREATE cls = Transaction.resolve_class(operation) - + print( f" Schema validation {tx}") if not skip_schema_validation: cls.validate_id(tx) cls.validate_schema(tx) diff --git a/planetmint/config.py b/planetmint/config.py index 5c8c3dc..583d4c6 100644 --- a/planetmint/config.py +++ b/planetmint/config.py @@ -65,7 +65,7 @@ class Config(metaclass=Singleton): 'port': 3303, "connect_now": True, "encoding": "utf-8", - "login": "", + "login": "guest", 'password': "", "service": "tarantoolctl connect", "init_config": self.__private_init_config, diff --git a/planetmint/core.py b/planetmint/core.py index 5ed429b..e9d2160 100644 --- a/planetmint/core.py +++ b/planetmint/core.py @@ -44,7 +44,8 @@ class App(BaseApplication): self.validators = None self.new_height = None self.chain = self.planetmint.get_latest_abci_chain() - + print( f"chain: {self.chain}") + def log_abci_migration_error(self, chain_id, validators): logger.error('An ABCI chain migration is in process. ' 'Download the new ABCI client and configure it with ' @@ -65,6 +66,7 @@ class App(BaseApplication): height = 0 known_chain = self.planetmint.get_latest_abci_chain() + print( f" known_chain: {known_chain}") if known_chain is not None: chain_id = known_chain['chain_id'] diff --git a/tests/backend/tarantool/Pipfile b/tests/backend/tarantool/Pipfile new file mode 100644 index 0000000..27fc644 --- /dev/null +++ b/tests/backend/tarantool/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pytest = "*" + +[dev-packages] + +[requires] +python_version = "3.8" diff --git a/tests/backend/tarantool/Pipfile.lock b/tests/backend/tarantool/Pipfile.lock new file mode 100644 index 0000000..bb541ae --- /dev/null +++ b/tests/backend/tarantool/Pipfile.lock @@ -0,0 +1,78 @@ +{ + "_meta": { + "hash": { + "sha256": "97a0be44f6d5351e166a90d91c789c8100486c7cc30d922ef7f7e3541838acae" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "version": "==21.4.0" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "version": "==21.3" + }, + "pluggy": { + "hashes": [ + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + ], + "version": "==1.0.0" + }, + "py": { + "hashes": [ + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" + ], + "version": "==1.11.0" + }, + "pyparsing": { + "hashes": [ + "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", + "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + ], + "version": "==3.0.7" + }, + "pytest": { + "hashes": [ + "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db", + "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171" + ], + "index": "pypi", + "version": "==7.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "version": "==2.0.1" + } + }, + "develop": {} +} diff --git a/tests/backend/tarantool/conftest.py b/tests/backend/tarantool/conftest.py index 41a7355..d4bb7d2 100644 --- a/tests/backend/tarantool/conftest.py +++ b/tests/backend/tarantool/conftest.py @@ -1,8 +1,35 @@ import pytest -from planetmint.backend import connection +from planetmint.backend.connection import Connection +# +# +# +@pytest.fixture +def dummy_db(request): + from planetmint.backend import Connection + conn = Connection() + dbname = request.fixturename + xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid') + if xdist_suffix: + dbname = '{}_{}'.format(dbname, xdist_suffix) + + conn.drop_database() + #_drop_db(conn, dbname) # make sure we start with a clean DB + #schema.init_database(conn, dbname) + conn.init_database() + yield dbname + + conn.drop_database() + #_drop_db(conn, dbname) + +#def _drop_db(conn, dbname): +# try: +# conn.drop_database() +# schema.drop_database(conn, dbname) +# except DatabaseDoesNotExist: +# pass @pytest.fixture def db_conn(): - conn = connection.Connection() + conn = Connection() return conn diff --git a/tests/backend/tarantool/test_schema.py b/tests/backend/tarantool/test_schema.py index e0c531a..2460eba 100644 --- a/tests/backend/tarantool/test_schema.py +++ b/tests/backend/tarantool/test_schema.py @@ -4,75 +4,68 @@ # Code is Apache-2.0 and docs are CC-BY-4.0 from planetmint.config import Config - - +from planetmint.backend import Connection +from planetmint.backend.tarantool.connection import TarantoolDB def test_init_database_is_graceful_if_db_exists(): - import planetmint - from planetmint import backend - from planetmint.backend.schema import init_database - - conn = backend.connect() - dbname = Config().get()['database']['name'] - - # The db is set up by the fixtures - assert dbname in conn.conn.list_database_names() - - init_database() + + conn = TarantoolDB('localhost', 3303) + conn.drop_database() + conn.init_database() def test_create_tables(): - import planetmint - from planetmint import backend from planetmint.backend import schema - conn = backend.connect() - dbname = Config().get()['database']['name'] + conn = TarantoolDB('localhost', 3303) + #conn = Connection() + #dbname = Config().get()['database']['name'] # The db is set up by the fixtures so we need to remove it - conn.conn.drop_database(dbname) - schema.create_database(conn, dbname) - schema.create_tables(conn, dbname) + conn.drop_database() + conn.init_database() - collection_names = conn.conn[dbname].list_collection_names() - assert set(collection_names) == { - 'transactions', 'assets', 'metadata', 'blocks', 'utxos', 'validators', 'elections', - 'pre_commit', 'abci_chains', - } - - indexes = conn.conn[dbname]['assets'].index_information().keys() - assert set(indexes) == {'_id_', 'asset_id', 'text'} - - index_info = conn.conn[dbname]['transactions'].index_information() - indexes = index_info.keys() - assert set(indexes) == { - '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} - assert index_info['transaction_id']['unique'] - - index_info = conn.conn[dbname]['blocks'].index_information() - indexes = index_info.keys() - assert set(indexes) == {'_id_', 'height'} - assert index_info['height']['unique'] - - index_info = conn.conn[dbname]['utxos'].index_information() - assert set(index_info.keys()) == {'_id_', 'utxo'} - assert index_info['utxo']['unique'] - assert index_info['utxo']['key'] == [('transaction_id', 1), - ('output_index', 1)] - - indexes = conn.conn[dbname]['elections'].index_information() - assert set(indexes.keys()) == {'_id_', 'election_id_height'} - assert indexes['election_id_height']['unique'] - - indexes = conn.conn[dbname]['pre_commit'].index_information() - assert set(indexes.keys()) == {'_id_', 'height'} - assert indexes['height']['unique'] + # TOTO verify spaces + #collection_names = conn.conn[dbname].list_collection_names() + #assert set(collection_names) == { + # 'transactions', 'assets', 'metadata', 'blocks', 'utxos', 'validators', 'elections', + # 'pre_commit', 'abci_chains', + #} +# + #indexes = conn.conn[dbname]['assets'].index_information().keys() + #assert set(indexes) == {'_id_', 'asset_id', 'text'} +# + #index_info = conn.conn[dbname]['transactions'].index_information() + #indexes = index_info.keys() + #assert set(indexes) == { + # '_id_', 'transaction_id', 'asset_id', 'outputs', 'inputs'} + #assert index_info['transaction_id']['unique'] +# + #index_info = conn.conn[dbname]['blocks'].index_information() + #indexes = index_info.keys() + #assert set(indexes) == {'_id_', 'height'} + #assert index_info['height']['unique'] +# + #index_info = conn.conn[dbname]['utxos'].index_information() + #assert set(index_info.keys()) == {'_id_', 'utxo'} + #assert index_info['utxo']['unique'] + #assert index_info['utxo']['key'] == [('transaction_id', 1), + # ('output_index', 1)] +# + #indexes = conn.conn[dbname]['elections'].index_information() + #assert set(indexes.keys()) == {'_id_', 'election_id_height'} + #assert indexes['election_id_height']['unique'] +# + #indexes = conn.conn[dbname]['pre_commit'].index_information() + #assert set(indexes.keys()) == {'_id_', 'height'} + #assert indexes['height']['unique'] def test_drop(dummy_db): from planetmint import backend from planetmint.backend import schema - conn = backend.connect() - assert dummy_db in conn.conn.list_database_names() - schema.drop_database(conn, dummy_db) - assert dummy_db not in conn.conn.list_database_names() + conn = TarantoolDB('localhost', 3303) + assert assets in conn.spaces('assets') + conn.drop_database() + assert assets not in conn.conn.list_database_names() + diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index cd274c1..d464f1e 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -275,7 +275,8 @@ def test_run_recover(b, alice, bob): asset={'cycle': 'hero'}, metadata={'name': 'hohenheim'}) \ .sign([bob.private_key]) - + print( tx1.id) + print( tx2.id) # store the transactions b.store_bulk_transactions([tx1, tx2]) diff --git a/tests/conftest.py b/tests/conftest.py index cb2ff74..ac71731 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -326,6 +326,7 @@ def dummy_db(request): if xdist_suffix: dbname = '{}_{}'.format(dbname, xdist_suffix) + _drop_db(conn, dbname) # make sure we start with a clean DB schema.init_database(conn, dbname) yield dbname @@ -335,6 +336,7 @@ def dummy_db(request): def _drop_db(conn, dbname): try: + conn.drop_database() schema.drop_database(conn, dbname) except DatabaseDoesNotExist: pass