From 2bb0539b78fc03d7979ea45875f16d857ee27b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Eckel?= Date: Wed, 1 Feb 2023 13:43:39 +0100 Subject: [PATCH] =?UTF-8?q?catching=20Tarantool=20exceptions=20in=20case?= =?UTF-8?q?=20of=20concurrency=20(implicitly=20issu=E2=80=A6=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * catching Tarantool exceptions in case of concurrency (implicitly issued by the planetmint-diver-ts tests) * fixed black version * blackified (new version) --------- Signed-off-by: Jürgen Eckel --- CHANGELOG.md | 3 +++ integration/python/src/test_naughty_strings.py | 2 -- integration/python/src/test_zenroom.py | 1 - planetmint/backend/localmongodb/query.py | 1 - planetmint/version.py | 2 +- planetmint/web/routes.py | 4 ++-- planetmint/web/views/base.py | 7 +++++-- planetmint/web/views/outputs.py | 10 +++++++++- planetmint/web/views/transactions.py | 7 ++++++- setup.py | 6 +++--- tests/assets/test_divisible_assets.py | 10 ---------- tests/assets/test_zenroom_signing.py | 1 - tests/db/test_planetmint_api.py | 1 - tests/test_config_utils.py | 1 - tests/upsert_validator/test_upsert_validator_vote.py | 4 +--- tests/web/test_metadata.py | 1 - tests/web/test_transactions.py | 1 - tests/web/test_websocket_server.py | 2 -- 18 files changed, 30 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a30970..71bfe01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ For reference, the possible headings are: * **Known Issues** * **Notes** +## [2.2.2] - 2023-31-01 +* **Fixed** catching tarantool exceptions in case tarantool drivers throw execeptions due to concurrency issues. This issue got idenitfied during the testing of the planetmint-driver-ts. + ## [2.2.0] - 2023-31-01 * **Changed** standardized blocks API diff --git a/integration/python/src/test_naughty_strings.py b/integration/python/src/test_naughty_strings.py index 921d718..20f89e3 100644 --- a/integration/python/src/test_naughty_strings.py +++ b/integration/python/src/test_naughty_strings.py @@ -115,7 +115,6 @@ def send_naughty_tx(assets, metadata): @pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings) def test_naughty_keys(naughty_string): - assets = [{"data": {naughty_string: "nice_value"}}] metadata = {naughty_string: "nice_value"} @@ -124,7 +123,6 @@ def test_naughty_keys(naughty_string): @pytest.mark.parametrize("naughty_string", naughty_strings, ids=naughty_strings) def test_naughty_values(naughty_string): - assets = [{"data": {"nice_key": naughty_string}}] metadata = {"nice_key": naughty_string} diff --git a/integration/python/src/test_zenroom.py b/integration/python/src/test_zenroom.py index 1f33ca0..cc0f6b3 100644 --- a/integration/python/src/test_zenroom.py +++ b/integration/python/src/test_zenroom.py @@ -17,7 +17,6 @@ def test_zenroom_signing( zenroom_script_input, condition_script_zencode, ): - biolabs = generate_keypair() version = "2.0" diff --git a/planetmint/backend/localmongodb/query.py b/planetmint/backend/localmongodb/query.py index 3351164..feaf145 100644 --- a/planetmint/backend/localmongodb/query.py +++ b/planetmint/backend/localmongodb/query.py @@ -102,7 +102,6 @@ def store_block(conn, block): @register_query(LocalMongoDBConnection) def get_txids_filtered(conn, asset_ids, operation=None, last_tx=None): - match = { Transaction.CREATE: {"operation": "CREATE", "id": {"$in": asset_ids}}, Transaction.TRANSFER: {"operation": "TRANSFER", "asset.id": {"$in": asset_ids}}, diff --git a/planetmint/version.py b/planetmint/version.py index 4b94d70..0c3bc4d 100644 --- a/planetmint/version.py +++ b/planetmint/version.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) # Code is Apache-2.0 and docs are CC-BY-4.0 -__version__ = "2.2.1" +__version__ = "2.2.2" __short_version__ = "2.2" # Supported Tendermint versions diff --git a/planetmint/web/routes.py b/planetmint/web/routes.py index 2dd9022..dab52a0 100644 --- a/planetmint/web/routes.py +++ b/planetmint/web/routes.py @@ -18,9 +18,9 @@ from planetmint.web.views import ( def add_routes(app): """Add the routes to an app""" - for (prefix, routes) in API_SECTIONS: + for prefix, routes in API_SECTIONS: api = Api(app, prefix=prefix) - for ((pattern, resource, *args), kwargs) in routes: + for (pattern, resource, *args), kwargs in routes: kwargs.setdefault("strict_slashes", False) api.add_resource(resource, pattern, *args, **kwargs) diff --git a/planetmint/web/views/base.py b/planetmint/web/views/base.py index d4b7f36..1572c8b 100644 --- a/planetmint/web/views/base.py +++ b/planetmint/web/views/base.py @@ -14,7 +14,7 @@ from planetmint.config import Config logger = logging.getLogger(__name__) -def make_error(status_code, message=None): +def make_error(status_code, message=None, level: str = "debug"): if status_code == 404 and message is None: message = "Not found" @@ -22,7 +22,10 @@ def make_error(status_code, message=None): request_info = {"method": request.method, "path": request.path} request_info.update(response_content) - logger.debug("HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s", request_info) + if level == "error": + logger.error("HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s", request_info) + else: + logger.debug("HTTP API error: %(status)s - %(method)s:%(path)s - %(message)s", request_info) response = jsonify(response_content) response.status_code = status_code diff --git a/planetmint/web/views/outputs.py b/planetmint/web/views/outputs.py index 9b4992c..6a3ae9e 100644 --- a/planetmint/web/views/outputs.py +++ b/planetmint/web/views/outputs.py @@ -6,6 +6,7 @@ from flask import current_app from flask_restful import reqparse, Resource from planetmint.web.views import parameters +from planetmint.web.views.base import make_error class OutputListApi(Resource): @@ -23,5 +24,12 @@ class OutputListApi(Resource): pool = current_app.config["bigchain_pool"] with pool() as planet: - outputs = planet.get_outputs_filtered(args["public_key"], args["spent"]) + try: + outputs = planet.get_outputs_filtered(args["public_key"], args["spent"]) + except Exception as e: + return make_error( + 500, + "Invalid output ({}): {} : {} - {}".format(type(e).__name__, e, args["public_key"], args["spent"]), + level="error", + ) return [{"transaction_id": output.txid, "output_index": output.output} for output in outputs] diff --git a/planetmint/web/views/transactions.py b/planetmint/web/views/transactions.py index 39fe56c..d0ba1a7 100644 --- a/planetmint/web/views/transactions.py +++ b/planetmint/web/views/transactions.py @@ -73,7 +73,6 @@ class TransactionListApi(Resource): # `force` will try to format the body of the POST request even if the # `content-type` header is not set to `application/json` tx = request.get_json(force=True) - try: tx_obj = Transaction.from_dict(tx, False) except SchemaValidationError as e: @@ -85,12 +84,18 @@ class TransactionListApi(Resource): return make_error(400, "Invalid transaction ({}): {}".format(type(e).__name__, e)) except ValidationError as e: return make_error(400, "Invalid transaction ({}): {}".format(type(e).__name__, e)) + except Exception as e: + return make_error(500, "Invalid transaction ({}): {} - {}".format(type(e).__name__, e, tx), level="error") with pool() as planet: try: planet.validate_transaction(tx_obj) except ValidationError as e: return make_error(400, "Invalid transaction ({}): {}".format(type(e).__name__, e)) + except Exception as e: + return make_error( + 500, "Invalid transaction ({}): {} : {}".format(type(e).__name__, e, tx), level="error" + ) else: if tx_obj.version != Transaction.VERSION: return make_error( diff --git a/setup.py b/setup.py index bdb0ecb..e17bd51 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ docs_require = [ "mdit-py-plugins==0.3.0", "mdurl==0.1.1", "myst-parser==0.17.2", - "packaging==21.3", + "packaging>=22.0", "pockets==0.9.1", "Pygments==2.12.0", "pyparsing==3.0.8", @@ -94,7 +94,7 @@ dev_require = ["ipdb", "ipython", "watchdog", "logging_tree", "pre-commit", "twi tests_require = [ "coverage", "pep8", - "black", + "black>=23.1.0", "hypothesis>=5.3.0", "pytest>=3.0.0", "pytest-cov==2.8.1", @@ -116,7 +116,7 @@ install_requires = [ "gunicorn==20.1.0", "jsonschema==4.16.0", "logstats==0.3.0", - "packaging>=20.9", + "packaging>=22.0", "pymongo==3.11.4", "tarantool==0.7.1", "python-rapidjson>=1.0", diff --git a/tests/assets/test_divisible_assets.py b/tests/assets/test_divisible_assets.py index 60254bf..94f7030 100644 --- a/tests/assets/test_divisible_assets.py +++ b/tests/assets/test_divisible_assets.py @@ -34,7 +34,6 @@ def test_single_in_single_own_single_out_single_own_create(alice, user_pk, b): # Multiple outputs # Single owners_after per output def test_single_in_single_own_multiple_out_single_own_create(alice, user_pk, b): - tx = Create.generate( [alice.public_key], [([user_pk], 50), ([user_pk], 50)], @@ -55,7 +54,6 @@ def test_single_in_single_own_multiple_out_single_own_create(alice, user_pk, b): # Single output # Multiple owners_after def test_single_in_single_own_single_out_multiple_own_create(alice, user_pk, b): - tx = Create.generate( [alice.public_key], [([user_pk, user_pk], 100)], @@ -81,7 +79,6 @@ def test_single_in_single_own_single_out_multiple_own_create(alice, user_pk, b): # Mix: one output with a single owners_after, one output with multiple # owners_after def test_single_in_single_own_multiple_out_mix_own_create(alice, user_pk, b): - tx = Create.generate( [alice.public_key], [([user_pk], 50), ([user_pk, user_pk], 50)], @@ -130,7 +127,6 @@ def test_single_in_multiple_own_single_out_single_own_create(alice, b, user_pk, # Single output # Single owners_after def test_single_in_single_own_single_out_single_own_transfer(alice, b, user_pk, user_sk): - # CREATE divisible asset tx_create = Create.generate( [alice.public_key], [([user_pk], 100)], assets=[{"data": "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4"}] @@ -155,7 +151,6 @@ def test_single_in_single_own_single_out_single_own_transfer(alice, b, user_pk, # Multiple output # Single owners_after def test_single_in_single_own_multiple_out_single_own_transfer(alice, b, user_pk, user_sk): - # CREATE divisible asset tx_create = Create.generate( [alice.public_key], [([user_pk], 100)], assets=[{"data": "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4"}] @@ -183,7 +178,6 @@ def test_single_in_single_own_multiple_out_single_own_transfer(alice, b, user_pk # Single output # Multiple owners_after def test_single_in_single_own_single_out_multiple_own_transfer(alice, b, user_pk, user_sk): - # CREATE divisible asset tx_create = Create.generate( [alice.public_key], [([user_pk], 100)], assets=[{"data": "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4"}] @@ -219,7 +213,6 @@ def test_single_in_single_own_single_out_multiple_own_transfer(alice, b, user_pk # Mix: one output with a single owners_after, one output with multiple # owners_after def test_single_in_single_own_multiple_out_mix_own_transfer(alice, b, user_pk, user_sk): - # CREATE divisible asset tx_create = Create.generate( [alice.public_key], [([user_pk], 100)], assets=[{"data": "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4"}] @@ -448,7 +441,6 @@ def test_muiltiple_in_mix_own_multiple_out_mix_own_transfer(alice, b, user_pk, u # Single output # Single owners_after def test_multiple_in_different_transactions(alice, b, user_pk, user_sk): - # CREATE divisible asset # `b` creates a divisible asset and assigns 50 shares to `b` and # 50 shares to `user_pk` @@ -549,7 +541,6 @@ def test_threshold_same_public_key(alice, b, user_pk, user_sk): def test_sum_amount(alice, b, user_pk, user_sk): - # CREATE divisible asset with 3 outputs with amount 1 tx_create = Create.generate( [alice.public_key], @@ -575,7 +566,6 @@ def test_sum_amount(alice, b, user_pk, user_sk): def test_divide(alice, b, user_pk, user_sk): - # CREATE divisible asset with 1 output with amount 3 tx_create = Create.generate( [alice.public_key], [([user_pk], 3)], assets=[{"data": "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4"}] diff --git a/tests/assets/test_zenroom_signing.py b/tests/assets/test_zenroom_signing.py index 655d4fb..85263bf 100644 --- a/tests/assets/test_zenroom_signing.py +++ b/tests/assets/test_zenroom_signing.py @@ -52,7 +52,6 @@ metadata = {"units": 300, "type": "KG"} def test_zenroom_signing(): - biolabs = generate_key_pair() version = "3.0" diff --git a/tests/db/test_planetmint_api.py b/tests/db/test_planetmint_api.py index e0e92eb..c1da92d 100644 --- a/tests/db/test_planetmint_api.py +++ b/tests/db/test_planetmint_api.py @@ -106,7 +106,6 @@ class TestBigchainApi(object): b.validate_transaction(tx) def test_write_transaction(self, b, user_sk, user_pk, alice, create_tx): - asset1 = {"data": "QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4"} tx = Create.generate([alice.public_key], [([alice.public_key], 1)], assets=[asset1]).sign([alice.private_key]) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index f537830..382066b 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -315,7 +315,6 @@ def test_write_config(): ), ) def test_database_envs(env_name, env_value, config_key, monkeypatch): - monkeypatch.setattr("os.environ", {env_name: env_value}) planetmint.config_utils.autoconfigure() diff --git a/tests/upsert_validator/test_upsert_validator_vote.py b/tests/upsert_validator/test_upsert_validator_vote.py index 5620160..2afa9d0 100644 --- a/tests/upsert_validator/test_upsert_validator_vote.py +++ b/tests/upsert_validator/test_upsert_validator_vote.py @@ -156,7 +156,6 @@ def test_valid_election_votes_received(b_mock, valid_upsert_validator_election, @pytest.mark.bdb def test_valid_election_conclude(b_mock, valid_upsert_validator_election, ed25519_node_keys): - # Node 0: cast vote tx_vote0 = gen_vote(valid_upsert_validator_election, 0, ed25519_node_keys) @@ -215,7 +214,6 @@ def test_valid_election_conclude(b_mock, valid_upsert_validator_election, ed2551 @pytest.mark.abci def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys): - if b.get_latest_block()["height"] == 0: generate_block(b) @@ -344,6 +342,6 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys): def reset_validator_set(b, node_keys, height): validators = [] - for (node_pub, _) in node_keys.items(): + for node_pub, _ in node_keys.items(): validators.append({"public_key": {"type": "ed25519-base64", "value": node_pub}, "voting_power": 10}) b.store_validator_set(height, validators) diff --git a/tests/web/test_metadata.py b/tests/web/test_metadata.py index f65f4b9..ba0d5b1 100644 --- a/tests/web/test_metadata.py +++ b/tests/web/test_metadata.py @@ -37,7 +37,6 @@ def test_get_metadata_tendermint(client, b, alice): @pytest.mark.bdb def test_get_metadata_limit_tendermint(client, b, alice): - # create two assets assets1 = [{"data": multihash(marshal({"msg": "abc 1"}))}] meta = multihash(marshal({"key": "meta 1"})) diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 04e9101..21e99f3 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -340,7 +340,6 @@ def test_post_invalid_transaction( @pytest.mark.abci def test_post_transfer_transaction_endpoint(client, user_pk, user_sk, posted_create_tx): - transfer_tx = Transfer.generate(posted_create_tx.to_inputs(), [([user_pk], 1)], asset_ids=[posted_create_tx.id]) transfer_tx = transfer_tx.sign([user_sk]) diff --git a/tests/web/test_websocket_server.py b/tests/web/test_websocket_server.py index c332663..3f72ddc 100644 --- a/tests/web/test_websocket_server.py +++ b/tests/web/test_websocket_server.py @@ -138,7 +138,6 @@ async def test_bridge_sync_async_queue(event_loop): @pytest.mark.asyncio async def test_websocket_block_event(aiohttp_client, event_loop): - user_priv, user_pub = crypto.generate_key_pair() tx = Create.generate([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv]) @@ -169,7 +168,6 @@ async def test_websocket_block_event(aiohttp_client, event_loop): @pytest.mark.asyncio async def test_websocket_transaction_event(aiohttp_client, event_loop): - user_priv, user_pub = crypto.generate_key_pair() tx = Create.generate([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv])