From 09efa319c058510ede636068ad61c1e872137d0f Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 9 Jun 2022 10:41:46 +0300 Subject: [PATCH 1/8] Small fixes in .upsert() functions for tarantool_db --- planetmint/backend/tarantool/connection.py | 3 +-- planetmint/backend/tarantool/query.py | 24 +++++++--------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/planetmint/backend/tarantool/connection.py b/planetmint/backend/tarantool/connection.py index 73db49b..06ff3bc 100644 --- a/planetmint/backend/tarantool/connection.py +++ b/planetmint/backend/tarantool/connection.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class TarantoolDBConnection(Connection): - def __init__(self, host: str = "localhost", port: int = 3303, user: str = None, password: str = None, **kwargs): + def __init__(self, host: str = "localhost", port: int = 3303, user: str = None, password: str = None, **kwargs): try: super().__init__(**kwargs) self.host = host @@ -62,7 +62,6 @@ class TarantoolDBConnection(Connection): except tarantool.error.NetworkError as net_error: raise net_error - def get_connection(self): return self.conn diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index 83f4c1d..f302c3f 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -390,11 +390,11 @@ def store_pre_commit_state(connection, state: dict): _precommit = connection.run( connection.space("pre_commits").select([], limit=1) ) - _precommitTuple = (token_hex(8), state["height"], state["transactions"]) if _precommit is None or len(_precommit) == 0 else _precommit[0] + _precommitTuple = (token_hex(8), state["height"], state["transactions"]) if _precommit is None or len( + _precommit) == 0 else _precommit[0] connection.run( connection.space("pre_commits").upsert(_precommitTuple, - op_list=[('=', 0, _precommitTuple[0]), - ('=', 1, state["height"]), + op_list=[('=', 1, state["height"]), ('=', 2, state["transactions"])], limit=1), only_data=False @@ -420,8 +420,7 @@ def store_validator_set(conn, validators_update: dict): unique_id = token_hex(8) if _validator is None or len(_validator) == 0 else _validator[0][0] conn.run( conn.space("validators").upsert((unique_id, validators_update["height"], validators_update["validators"]), - op_list=[('=', 0, unique_id), - ('=', 1, validators_update["height"]), + op_list=[('=', 1, validators_update["height"]), ('=', 2, validators_update["validators"])], limit=1), only_data=False @@ -444,8 +443,7 @@ def delete_validator_set(connection, height: int): def store_election(connection, election_id: str, height: int, is_concluded: bool): connection.run( connection.space("elections").upsert((election_id, height, is_concluded), - op_list=[('=', 0, election_id), - ('=', 1, height), + op_list=[('=', 1, height), ('=', 2, is_concluded)], limit=1), only_data=False @@ -517,16 +515,8 @@ def get_asset_tokens_for_public_key(connection, asset_id: str, @register_query(TarantoolDBConnection) def store_abci_chain(connection, height: int, chain_id: str, is_synced: bool = True): - _chain = connection.run(connection.space("abci_chains").select(height, index="height_search", limit=1)) - _chainTuple = (height, is_synced, chain_id) if _chain is None or len(_chain) == 0 else _chain[0] - connection.run( - connection.space("abci_chains").upsert(_chainTuple, - op_list=[('=', 0, height), - ('=', 1, is_synced), - ('=', 2, chain_id)], - limit=1), - only_data=False - ) + connection.run(connection.space("abci_chains").delete(chain_id), only_data=False) + connection.run(connection.space("abci_chains").insert((height, is_synced, chain_id)), only_data=False) @register_query(TarantoolDBConnection) From 4ca2c53cb7310ddb1e24c7a30ad68e1501b7e36d Mon Sep 17 00:00:00 2001 From: Sangat Das Date: Thu, 9 Jun 2022 01:41:42 -0700 Subject: [PATCH 2/8] Fixes some test cases in tendermint/test_lib.py --- tests/conftest.py | 6 ++++-- tests/tendermint/test_lib.py | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cacaf6d..7072596 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -552,7 +552,7 @@ def tarantool_client(db_context): # TODO Here add TarantoolConnectionClass # @pytest.fixture -def utxo_collection(tarantool_client): +def utxo_collection(tarantool_client, _setup_database): return tarantool_client.get_space("utxos") @@ -568,10 +568,12 @@ def dummy_unspent_outputs(): @pytest.fixture def utxoset(dummy_unspent_outputs, utxo_collection): from json import dumps + num_rows_before_operation = utxo_collection.select().rowcount for utxo in dummy_unspent_outputs: res = utxo_collection.insert((utxo["transaction_id"], utxo["output_index"], dumps(utxo))) assert res - assert len(utxo_collection.select()) == 3 + num_rows_after_operation = utxo_collection.select().rowcount + assert num_rows_after_operation == num_rows_before_operation + 3 return dummy_unspent_outputs, utxo_collection diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index d016867..82f1eea 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -288,9 +288,11 @@ def test_store_bulk_transaction(mocker, b, signed_create_tx, @pytest.mark.bdb def test_delete_zero_unspent_outputs(b, utxoset): unspent_outputs, utxo_collection = utxoset + num_rows_before_operation = utxo_collection.select().rowcount delete_res = b.delete_unspent_outputs() + num_rows_after_operation = utxo_collection.select().rowcount # assert delete_res is None - assert utxo_collection.select().rowcount == 3 + assert num_rows_before_operation == num_rows_after_operation # assert utxo_collection.count_documents( # {'$or': [ # {'transaction_id': 'a', 'output_index': 0}, @@ -350,9 +352,11 @@ def test_delete_many_unspent_outputs(b, utxoset): @pytest.mark.bdb def test_store_zero_unspent_output(b, utxo_collection): + num_rows_before_operation = utxo_collection.select().rowcount res = b.store_unspent_outputs() + num_rows_after_operation = utxo_collection.select().rowcount assert res is None - assert utxo_collection.select().rowcount == 0 + assert num_rows_before_operation == num_rows_after_operation @pytest.mark.bdb From 06b1244ca5db3dee1dd4770cc1d26cdcaa30bddb Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 9 Jun 2022 13:57:59 +0300 Subject: [PATCH 3/8] Fixed flush_tarantool_db Function. --- tests/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 08b16b8..1dc9806 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,13 +40,15 @@ def flush_tarantool_db(connection, dbname): if "assets" == s: connection.run(connection.space(s).delete(_id[1]), only_data=False) elif s in ["blocks", "abci_chains"]: - connection.run(connection.space(s).delete(_id[2], only_data=False)) + connection.run(connection.space(s).delete(_id[2]), only_data=False) elif s == "inputs": - connection.run(connection.space(s).delete(_id[-2], only_data=False)) + connection.run(connection.space(s).delete(_id[-2]), only_data=False) elif s == "outputs": - connection.run(connection.space(s).delete(_id[-4], only_data=False)) + connection.run(connection.space(s).delete(_id[-4]), only_data=False) + elif s == "utxos": + connection.run(connection.space(s).delete([_id[0], _id[1]]), only_data=False) else: - connection.run(connection.space(s).delete(_id[0], only_data=False)) + connection.run(connection.space(s).delete(_id[0]), only_data=False) def generate_block(planet): From dd5060aeba997686487bee5028d7cbdab39220c6 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 9 Jun 2022 17:17:23 +0300 Subject: [PATCH 4/8] added utxos to SPACE_NAMES to delete data from utxo space --- planetmint/backend/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetmint/backend/schema.py b/planetmint/backend/schema.py index 0a4a34c..7204ea8 100644 --- a/planetmint/backend/schema.py +++ b/planetmint/backend/schema.py @@ -22,7 +22,7 @@ TABLES = ('transactions', 'blocks', 'assets', 'metadata', SPACE_NAMES = ("abci_chains", "assets", "blocks", "blocks_tx", "elections", "meta_data", "pre_commits", "validators", - "transactions", "inputs", "outputs", "keys") + "transactions", "inputs", "outputs", "keys", "utxos") VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', From 0d5791dd50133aedc2fa4a3a5d88387d10c90583 Mon Sep 17 00:00:00 2001 From: andrei Date: Thu, 9 Jun 2022 17:21:53 +0300 Subject: [PATCH 5/8] fixed this module by restoring original function instructions. --- tests/tendermint/test_lib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 82f1eea..65994fd 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -518,11 +518,11 @@ def test_get_spent_key_order(b, user_pk, user_sk, user2_pk, user2_sk): bob = generate_key_pair() tx1 = Create.generate([user_pk], - [([alice.public_key], 3), ([user_pk], 2)], - asset=None) \ + [([alice.public_key], 3), ([user_pk], 2)], + asset=None) \ .sign([user_sk]) b.store_bulk_transactions([tx1]) - assert tx1.validate(b) + inputs = tx1.to_inputs() tx2 = Transfer.generate([inputs[1]], [([user2_pk], 2)], tx1.id).sign([user_sk]) assert tx2.validate(b) From feb5ad319b079b129eb0069556ed3a138d6ba344 Mon Sep 17 00:00:00 2001 From: andrei Date: Mon, 13 Jun 2022 15:04:03 +0300 Subject: [PATCH 6/8] added hash generation to create primary key for abci_chain tuple. --- planetmint/backend/tarantool/query.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/planetmint/backend/tarantool/query.py b/planetmint/backend/tarantool/query.py index f302c3f..6a1b5b0 100644 --- a/planetmint/backend/tarantool/query.py +++ b/planetmint/backend/tarantool/query.py @@ -5,6 +5,7 @@ """Query implementation for Tarantool""" from secrets import token_hex +from hashlib import sha256 from operator import itemgetter import tarantool.error @@ -15,6 +16,7 @@ from planetmint.backend.tarantool.connection import TarantoolDBConnection from planetmint.backend.tarantool.transaction.tools import TransactionCompose, TransactionDecompose from json import dumps, loads + register_query = module_dispatch_registrar(query) @@ -515,20 +517,25 @@ def get_asset_tokens_for_public_key(connection, asset_id: str, @register_query(TarantoolDBConnection) def store_abci_chain(connection, height: int, chain_id: str, is_synced: bool = True): - connection.run(connection.space("abci_chains").delete(chain_id), only_data=False) - connection.run(connection.space("abci_chains").insert((height, is_synced, chain_id)), only_data=False) + hash_id_primarykey = sha256(dumps(obj={"height": height}).encode()).hexdigest() + connection.run( + connection.space("abci_chains").upsert((height, is_synced, chain_id, hash_id_primarykey), + op_list=[ + ('=', 0, height), + ('=', 1, is_synced), + ('=', 2, chain_id) + ]), + only_data=False + ) @register_query(TarantoolDBConnection) def delete_abci_chain(connection, height: int): - _chains = connection.run( - connection.space("abci_chains").select(height, index="height_search") + hash_id_primarykey = sha256(dumps(obj={"height": height}).encode()).hexdigest() + connection.run( + connection.space("abci_chains").delete(hash_id_primarykey), + only_data=False ) - for _chain in _chains: - connection.run( - connection.space("abci_chains").delete(_chain[2]), - only_data=False - ) @register_query(TarantoolDBConnection) From bf557eee9c52e6c8029b000be4f700b90a5a2493 Mon Sep 17 00:00:00 2001 From: andrei Date: Mon, 13 Jun 2022 15:04:49 +0300 Subject: [PATCH 7/8] Added field 'id' to abci_chains --- planetmint/backend/tarantool/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/planetmint/backend/tarantool/schema.py b/planetmint/backend/tarantool/schema.py index d51148d..73b13dc 100644 --- a/planetmint/backend/tarantool/schema.py +++ b/planetmint/backend/tarantool/schema.py @@ -32,8 +32,8 @@ SPACE_COMMANDS = { INDEX_COMMANDS = { "abci_chains": { - "id_search": "abci_chains:create_index('id_search' ,{type='hash', parts={'chain_id'}})", - "height_search": "abci_chains:create_index('height_search' ,{type='tree',unique=false, parts={'height'}})" + "id_search": "abci_chains:create_index('id_search' ,{type='hash', parts={'id'}})", + "height_search": "abci_chains:create_index('height_search' ,{type='tree', unique=false, parts={'height'}})" }, "assets": { @@ -105,7 +105,7 @@ INDEX_COMMANDS = { SCHEMA_COMMANDS = { "abci_chains": - "abci_chains:format({{name='height' , type='integer'},{name='is_synched' , type='boolean'},{name='chain_id',type='string'}})", + "abci_chains:format({{name='height' , type='integer'},{name='is_synched' , type='boolean'},{name='chain_id',type='string'}, {name='id', type='string'}})", "assets": "assets:format({{name='data' , type='any'}, {name='tx_id', type='string'}, {name='asset_id', type='string'}})", "blocks": From f34885748cf40c64e96b363652085f832e6ed40f Mon Sep 17 00:00:00 2001 From: andrei Date: Mon, 13 Jun 2022 15:06:33 +0300 Subject: [PATCH 8/8] changed flush function to delete abci_chains tuples --- tests/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 1dc9806..1355da6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -39,7 +39,7 @@ def flush_tarantool_db(connection, dbname): for _id in _all_data: if "assets" == s: connection.run(connection.space(s).delete(_id[1]), only_data=False) - elif s in ["blocks", "abci_chains"]: + elif s == "blocks": connection.run(connection.space(s).delete(_id[2]), only_data=False) elif s == "inputs": connection.run(connection.space(s).delete(_id[-2]), only_data=False) @@ -47,6 +47,8 @@ def flush_tarantool_db(connection, dbname): connection.run(connection.space(s).delete(_id[-4]), only_data=False) elif s == "utxos": connection.run(connection.space(s).delete([_id[0], _id[1]]), only_data=False) + elif s == "abci_chains": + connection.run(connection.space(s).delete(_id[-1]), only_data=False) else: connection.run(connection.space(s).delete(_id[0]), only_data=False)