Merge with existing changes

.idea/misc.xml

@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (planetmint)" project-jdk-type="Python SDK" />
+  <component name="PyCharmProfessionalAdvertiser">
+    <option name="shown" value="true" />
+  </component>
 </project>

planetmint/backend/tarantool/init.lua

@@ -64,7 +64,7 @@ keys:create_index('txid_search', {type = 'tree', unique=false, parts={'transacti
 keys:create_index('output_search', {type = 'tree', unique=false, parts={'output_id'}})
 
 utxos = box.schema.space.create('utxos', {engine = 'memtx' , is_sync = false})
-utxos:format({{name='transaction_id' , type='string'}, {name='output_index' , type='integer'}})
+utxos:format({{name='transaction_id' , type='string'}, {name='output_index' , type='integer'}, {name='utxo_dict', type='string'}})
 utxos:create_index('id_search', {type='hash' , parts={'transaction_id', 'output_index'}})
 utxos:create_index('transaction_search', {type='tree', unique=false, parts={'transaction_id'}})
 utxos:create_index('index_search', {type='tree', unique=false, parts={'output_index'}})

planetmint/backend/tarantool/query.py

@@ -9,11 +9,10 @@ from secrets import token_hex
 from operator import itemgetter
 
 from planetmint.backend import query
-from planetmint.backend.exceptions import DuplicateKeyError
-from planetmint.backend.exceptions import OperationError
 from planetmint.backend.utils import module_dispatch_registrar
 from planetmint.backend.tarantool.connection import TarantoolDB
 from planetmint.backend.tarantool.transaction.tools import TransactionCompose, TransactionDecompose
+from json import dumps, loads
 
 register_query = module_dispatch_registrar(query)
 
@@ -118,7 +117,7 @@ def get_metadata(connection, transaction_ids: list):
 
 
 @register_query(TarantoolDB)
-def store_asset(connection, asset: dict):
+def store_asset(connection, asset):
     space = connection.space("assets")
     convert = lambda obj: obj if isinstance(obj, tuple) else (obj, obj["id"], obj["id"])
     try:
@@ -169,18 +168,19 @@ def get_latest_block(connection):  # TODO Here is used DESCENDING OPERATOR
     space = connection.space("blocks")
     _all_blocks = space.select()
     _all_blocks = _all_blocks.data
-    hash = ''
+    block = {"app_hash": '', "height": 0, "transactions": []}
-    heigth = 0
+
-    txs = []
     if len(_all_blocks) > 0:
         _block = sorted(_all_blocks, key=itemgetter(1), reverse=True)[0]
         space = connection.space("blocks_tx")
         _txids = space.select(_block[2], index="block_search")
         _txids = _txids.data
-        hash = _block[0]
+        block["app_hash"] = _block[0]
-        heigth = _block[1]
+        block["height"] = _block[1]
-        txs = [tx[0] for tx in _txids]
+        block["transactions"] = [tx[0] for tx in _txids]
-    return {"app_hash": hash, "height": heigth, "transactions": txs}
+    else:
+        block = None
+    return block
 
 
 @register_query(TarantoolDB)
@@ -337,11 +337,10 @@ def store_unspent_outputs(connection, *unspent_outputs: list):
     result = []
     if unspent_outputs:
         for utxo in unspent_outputs:
-            output = space.insert((utxo['transaction_id'], utxo['output_index']))
+            output = space.insert((utxo['transaction_id'], utxo['output_index'], dumps(utxo)))
             result.append(output.data)
     return result
 
-
 @register_query(TarantoolDB)
 def delete_unspent_outputs(connection, *unspent_outputs: list):
     space = connection.space('utxos')
@@ -354,9 +353,10 @@ def delete_unspent_outputs(connection, *unspent_outputs: list):
 
 
 @register_query(TarantoolDB)
-def get_unspent_outputs(connection):
+def get_unspent_outputs(connection, query=None):  # for now we don't have implementation for 'query'.
     space = connection.space('utxos')
-    return space.select()
+    _utxos = space.select([]).data
+    return [loads(utx[2]) for utx in _utxos]
 
 
 @register_query(TarantoolDB)

tests/conftest.py

@@ -540,8 +540,9 @@ def dummy_unspent_outputs():
 
 @pytest.fixture
 def utxoset(dummy_unspent_outputs, utxo_collection):
+    from json import dumps
     for utxo in dummy_unspent_outputs:
-        res = utxo_collection.insert((utxo["transaction_id"], utxo["output_index"]))
+        res = utxo_collection.insert((utxo["transaction_id"], utxo["output_index"], dumps(utxo)))
         assert res
     assert len(utxo_collection.select()) == 3
     return dummy_unspent_outputs, utxo_collection

tests/tendermint/test_lib.py

@@ -7,7 +7,6 @@ from operator import index
 import os
 from unittest.mock import patch
 
-
 try:
     from hashlib import sha3_256
 except ImportError:
@@ -29,6 +28,10 @@ def test_asset_is_separated_from_transaciton(b):
     import copy
     from planetmint.models import Transaction
     from planetmint.common.crypto import generate_key_pair
+    from planetmint.backend.tarantool.connection import TarantoolDB
+
+    if isinstance(b.connection, TarantoolDB):
+        pytest.skip("This specific function is skipped because, assets are stored differently if using Tarantool")
 
     alice = generate_key_pair()
     bob = generate_key_pair()
@@ -45,8 +48,8 @@ def test_asset_is_separated_from_transaciton(b):
     tx = Transaction.create([alice.public_key],
                             [([bob.public_key], 1)],
                             metadata=None,
-                            asset=asset)\
+                            asset=asset) \
-                    .sign([alice.private_key])
+        .sign([alice.private_key])
 
     # with store_bulk_transactions we use `insert_many` where PyMongo
     # automatically adds an `_id` field to the tx, therefore we need the
@@ -89,8 +92,8 @@ def test_validation_error(b):
     alice = generate_key_pair()
     tx = Transaction.create([alice.public_key],
                             [([alice.public_key], 1)],
-                            asset=None)\
+                            asset=None) \
-                    .sign([alice.private_key]).to_dict()
+        .sign([alice.private_key]).to_dict()
 
     tx['metadata'] = ''
     assert not b.validate_transaction(tx)
@@ -105,8 +108,8 @@ def test_write_and_post_transaction(mock_post, b):
     alice = generate_key_pair()
     tx = Transaction.create([alice.public_key],
                             [([alice.public_key], 1)],
-                            asset=None)\
+                            asset=None) \
-                    .sign([alice.private_key]).to_dict()
+        .sign([alice.private_key]).to_dict()
 
     tx = b.validate_transaction(tx)
     b.write_transaction(tx, BROADCAST_TX_ASYNC)
@@ -171,23 +174,30 @@ def test_update_utxoset(b, signed_create_tx, signed_transfer_tx, db_conn):
 @pytest.mark.bdb
 def test_store_transaction(mocker, b, signed_create_tx,
                            signed_transfer_tx, db_context):
+    from planetmint.backend.tarantool.connection import TarantoolDB
     mocked_store_asset = mocker.patch('planetmint.backend.query.store_assets')
     mocked_store_metadata = mocker.patch(
         'planetmint.backend.query.store_metadatas')
     mocked_store_transaction = mocker.patch(
         'planetmint.backend.query.store_transactions')
     b.store_bulk_transactions([signed_create_tx])
-    # mongo_client = MongoClient(host=db_context.host, port=db_context.port)
+    if not isinstance(b.connection, TarantoolDB):
-    # utxoset = mongo_client[db_context.name]['utxos']
+        mongo_client = MongoClient(host=db_context.host, port=db_context.port)
-    # assert utxoset.count_documents({}) == 1
+        utxoset = mongo_client[db_context.name]['utxos']
-    # utxo = utxoset.find_one()
+        assert utxoset.count_documents({}) == 1
-    # assert utxo['transaction_id'] == signed_create_tx.id
+        utxo = utxoset.find_one()
-    # assert utxo['output_index'] == 0
+        assert utxo['transaction_id'] == signed_create_tx.id
+        assert utxo['output_index'] == 0
+        mocked_store_asset.assert_called_once_with(
+            b.connection,
+            [{'data': signed_create_tx.asset['data'], 'tx_id': signed_create_tx.id, 'asset_id': signed_create_tx.id}]
+        )
+    else:
+        mocked_store_asset.assert_called_once_with(
+            b.connection,
+            [(signed_create_tx.asset, signed_create_tx.id, signed_create_tx.id)]
+        )
 
-    mocked_store_asset.assert_called_once_with(
-        b.connection,
-        [{'data': signed_create_tx.asset['data'], 'tx_id': signed_create_tx.id, 'asset_id': signed_create_tx.id}]
-    )
     mocked_store_metadata.assert_called_once_with(
         b.connection,
         [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}],
@@ -195,31 +205,34 @@ def test_store_transaction(mocker, b, signed_create_tx,
     mocked_store_transaction.assert_called_once_with(
         b.connection,
         [{k: v for k, v in signed_create_tx.to_dict().items()
-         if k not in ('asset', 'metadata')}],
+          if k not in ('asset', 'metadata')}],
     )
     mocked_store_asset.reset_mock()
     mocked_store_metadata.reset_mock()
     mocked_store_transaction.reset_mock()
     b.store_bulk_transactions([signed_transfer_tx])
-    # assert utxoset.count_documents({}) == 1
+    if not isinstance(b.connection, TarantoolDB):
-    # utxo = utxoset.find_one()
+        assert utxoset.count_documents({}) == 1
-    # assert utxo['transaction_id'] == signed_transfer_tx.id
+        utxo = utxoset.find_one()
-    # assert utxo['output_index'] == 0
+        assert utxo['transaction_id'] == signed_transfer_tx.id
-    assert not mocked_store_asset.called
+        assert utxo['output_index'] == 0
+        assert not mocked_store_asset.called
     mocked_store_metadata.asser_called_once_with(
         b.connection,
         [{'id': signed_transfer_tx.id, 'metadata': signed_transfer_tx.metadata}],
     )
-    mocked_store_transaction.assert_called_once_with(
+    if not isinstance(b.connection, TarantoolDB):
-        b.connection,
+        mocked_store_transaction.assert_called_once_with(
-        [{k: v for k, v in signed_transfer_tx.to_dict().items()
+            b.connection,
-         if k != 'metadata'}],
+            [{k: v for k, v in signed_transfer_tx.to_dict().items()
-    )
+              if k != 'metadata'}],
+        )
 
 
 @pytest.mark.bdb
 def test_store_bulk_transaction(mocker, b, signed_create_tx,
                                 signed_transfer_tx, db_context):
+    from planetmint.backend.tarantool.connection import TarantoolDB
     mocked_store_assets = mocker.patch(
         'planetmint.backend.query.store_assets')
     mocked_store_metadata = mocker.patch(
@@ -227,16 +240,23 @@ def test_store_bulk_transaction(mocker, b, signed_create_tx,
     mocked_store_transactions = mocker.patch(
         'planetmint.backend.query.store_transactions')
     b.store_bulk_transactions((signed_create_tx,))
-    # mongo_client = MongoClient(host=db_context.host, port=db_context.port)
+    if not isinstance(b.connection, TarantoolDB):
-    # utxoset = mongo_client[db_context.name]['utxos']
+        mongo_client = MongoClient(host=db_context.host, port=db_context.port)
-    # assert utxoset.count_documents({}) == 1
+        utxoset = mongo_client[db_context.name]['utxos']
-    # utxo = utxoset.find_one()
+        assert utxoset.count_documents({}) == 1
-    # assert utxo['transaction_id'] == signed_create_tx.id
+        utxo = utxoset.find_one()
-    # assert utxo['output_index'] == 0
+        assert utxo['transaction_id'] == signed_create_tx.id
-    mocked_store_assets.assert_called_once_with(
+        assert utxo['output_index'] == 0
-        b.connection,
+    if isinstance(b.connection, TarantoolDB):
-        [ ( signed_create_tx.asset['data'], signed_create_tx.id, signed_create_tx.id )],
+        mocked_store_assets.assert_called_once_with(
-    )
+            b.connection,  # signed_create_tx.asset['data'] this was before
+            [(signed_create_tx.asset, signed_create_tx.id, signed_create_tx.id)],
+        )
+    else:
+        mocked_store_assets.assert_called_once_with(
+            b.connection,  # signed_create_tx.asset['data'] this was before
+            [(signed_create_tx.asset["data"], signed_create_tx.id, signed_create_tx.id)],
+        )
     mocked_store_metadata.assert_called_once_with(
         b.connection,
         [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}],
@@ -244,27 +264,29 @@ def test_store_bulk_transaction(mocker, b, signed_create_tx,
     mocked_store_transactions.assert_called_once_with(
         b.connection,
         [{k: v for k, v in signed_create_tx.to_dict().items()
-         if k not in ('asset', 'metadata')}],
+          if k not in ('asset', 'metadata')}],
     )
     mocked_store_assets.reset_mock()
     mocked_store_metadata.reset_mock()
     mocked_store_transactions.reset_mock()
     b.store_bulk_transactions((signed_transfer_tx,))
-    # assert utxoset.count_documents({}) == 1
+    if not isinstance(b.connection, TarantoolDB):
-    # utxo = utxoset.find_one()
+        assert utxoset.count_documents({}) == 1
-    # assert utxo['transaction_id'] == signed_transfer_tx.id
+        utxo = utxoset.find_one()
-    # assert utxo['output_index'] == 0
+        assert utxo['transaction_id'] == signed_transfer_tx.id
-    assert not mocked_store_assets.called
+        assert utxo['output_index'] == 0
+        assert not mocked_store_assets.called
     mocked_store_metadata.asser_called_once_with(
         b.connection,
         [{'id': signed_transfer_tx.id,
           'metadata': signed_transfer_tx.metadata}],
     )
-    mocked_store_transactions.assert_called_once_with(
+    if not isinstance(b.connection, TarantoolDB):
-        b.connection,
+        mocked_store_transactions.assert_called_once_with(
-        [{k: v for k, v in signed_transfer_tx.to_dict().items()
+            b.connection,
-          if k != 'metadata'}],
+            [{k: v for k, v in signed_transfer_tx.to_dict().items()
-    )
+              if k != 'metadata'}],
+        )
 
 
 @pytest.mark.bdb
@@ -284,32 +306,51 @@ def test_delete_zero_unspent_outputs(b, utxoset):
 
 @pytest.mark.bdb
 def test_delete_one_unspent_outputs(b, utxoset):
+    from planetmint.backend.tarantool.connection import TarantoolDB
     unspent_outputs, utxo_collection = utxoset
     delete_res = b.delete_unspent_outputs(unspent_outputs[0])
-    assert len(list(delete_res)) == 1
+    if not isinstance(b.connection, TarantoolDB):
-    # assert utxo_collection.count_documents(
+        assert len(list(delete_res)) == 1
-    #     {'$or': [
+        assert utxo_collection.count_documents(
-    #         {'transaction_id': 'a', 'output_index': 1},
+            {'$or': [
-    #         {'transaction_id': 'b', 'output_index': 0},
+                {'transaction_id': 'a', 'output_index': 1},
-    #     ]}
+                {'transaction_id': 'b', 'output_index': 0},
-    # ) == 2
+            ]}
-    # assert utxo_collection.count_documents(
+        ) == 2
-    #         {'transaction_id': 'a', 'output_index': 0}) == 0
+        assert utxo_collection.count_documents(
+            {'transaction_id': 'a', 'output_index': 0}) == 0
+    else:
+        utx_space = b.connection.space("utxos")
+        res1 = utx_space.select(['a', 1], index="id_search").data
+        res2 = utx_space.select(['b', 0], index="id_search").data
+        assert len(res1) + len(res2) == 2
+        res3 = utx_space.select(['a', 0], index="id_search").data
+        assert len(res3) == 0
 
 
 @pytest.mark.bdb
 def test_delete_many_unspent_outputs(b, utxoset):
+    from planetmint.backend.tarantool.connection import TarantoolDB
     unspent_outputs, utxo_collection = utxoset
     delete_res = b.delete_unspent_outputs(*unspent_outputs[::2])
-    assert len(list(delete_res)) == 2
+    if not isinstance(b.connection, TarantoolDB):
-    # assert utxo_collection.count_documents(
+        assert len(list(delete_res)) == 2
-    #     {'$or': [
+        assert utxo_collection.count_documents(
-    #         {'transaction_id': 'a', 'output_index': 0},
+            {'$or': [
-    #         {'transaction_id': 'b', 'output_index': 0},
+                {'transaction_id': 'a', 'output_index': 0},
-    #     ]}
+                {'transaction_id': 'b', 'output_index': 0},
-    # ) == 0
+            ]}
-    # assert utxo_collection.count_documents(
+        ) == 0
-    #         {'transaction_id': 'a', 'output_index': 1}) == 1
+        assert utxo_collection.count_documents(
+            {'transaction_id': 'a', 'output_index': 1}) == 1
+    else:  # TODO It looks ugly because query.get_unspent_outputs function, has not yet implemented query parameter.
+        utx_space = b.connection.space("utxos")
+        res1 = utx_space.select(['a', 0], index="id_search").data
+        res2 = utx_space.select(['b', 0], index="id_search").data
+        assert len(res1) + len(res2) == 0
+        res3 = utx_space.select([], index="id_search").data
+        assert len(res3) == 1
+
 
 
 @pytest.mark.bdb
@@ -321,23 +362,36 @@ def test_store_zero_unspent_output(b, utxo_collection):
 
 @pytest.mark.bdb
 def test_store_one_unspent_output(b, unspent_output_1, utxo_collection):
+    from planetmint.backend.tarantool.connection import TarantoolDB
     res = b.store_unspent_outputs(unspent_output_1)
-    # assert res.acknowledged
+    if not isinstance(b.connection, TarantoolDB):
-    assert len(list(res)) == 1
+        assert res.acknowledged
-    # assert utxo_collection.count_documents(
+        assert len(list(res)) == 1
-    #     {'transaction_id': unspent_output_1['transaction_id'],
+        assert utxo_collection.count_documents(
-    #      'output_index': unspent_output_1['output_index']}
+            {'transaction_id': unspent_output_1['transaction_id'],
-    # ) == 1
+             'output_index': unspent_output_1['output_index']}
+        ) == 1
+    else:
+        utx_space = b.connection.space("utxos")
+        res = utx_space.select([unspent_output_1["transaction_id"], unspent_output_1["output_index"]], index="id_search")
+        assert len(res.data) == 1
+
 
 
 @pytest.mark.bdb
 def test_store_many_unspent_outputs(b, unspent_outputs, utxo_collection):
+    from planetmint.backend.tarantool.connection import TarantoolDB
     res = b.store_unspent_outputs(*unspent_outputs)
-    # assert res.acknowledged
+    if not isinstance(b.connection, TarantoolDB):
-    assert len(list(res)) == 3
+        assert res.acknowledged
-    # assert utxo_collection.count_documents(
+        assert len(list(res)) == 3
-    #     {'transaction_id': unspent_outputs[0]['transaction_id']}
+        assert utxo_collection.count_documents(
-    # ) == 3
+            {'transaction_id': unspent_outputs[0]['transaction_id']}
+        ) == 3
+    else:
+        utxo_space = b.connection.space("utxos")  # .select([], index="transaction_search").data
+        res = utxo_space.select([unspent_outputs[0]["transaction_id"]], index="transaction_search")
+        assert len(res.data) == 3
 
 
 def test_get_utxoset_merkle_root_when_no_utxo(b):
@@ -363,23 +417,23 @@ def test_get_spent_transaction_critical_double_spend(b, alice, bob, carol):
 
     tx = Transaction.create([alice.public_key],
                             [([alice.public_key], 1)],
-                            asset=asset)\
+                            asset=asset) \
-                    .sign([alice.private_key])
+        .sign([alice.private_key])
 
     tx_transfer = Transaction.transfer(tx.to_inputs(),
                                        [([bob.public_key], 1)],
-                                       asset_id=tx.id)\
+                                       asset_id=tx.id) \
-                             .sign([alice.private_key])
+        .sign([alice.private_key])
 
     double_spend = Transaction.transfer(tx.to_inputs(),
                                         [([carol.public_key], 1)],
-                                        asset_id=tx.id)\
+                                        asset_id=tx.id) \
-                              .sign([alice.private_key])
+        .sign([alice.private_key])
 
     same_input_double_spend = Transaction.transfer(tx.to_inputs() + tx.to_inputs(),
                                                    [([bob.public_key], 1)],
-                                                   asset_id=tx.id)\
+                                                   asset_id=tx.id) \
-                                         .sign([alice.private_key])
+        .sign([alice.private_key])
 
     b.store_bulk_transactions([tx])
 
@@ -435,16 +489,16 @@ def test_migrate_abci_chain_yields_on_genesis(b):
 @pytest.mark.bdb
 @pytest.mark.parametrize('chain,block_height,expected', [
     (
-        (1, 'chain-XYZ', True),
+            (1, 'chain-XYZ', True),
-        4,
+            4,
-        {'height': 5, 'chain_id': 'chain-XYZ-migrated-at-height-4',
+            {'height': 5, 'chain_id': 'chain-XYZ-migrated-at-height-4',
-         'is_synced': False},
+             'is_synced': False},
     ),
     (
-        (5, 'chain-XYZ-migrated-at-height-4', True),
+            (5, 'chain-XYZ-migrated-at-height-4', True),
-        13,
+            13,
-        {'height': 14, 'chain_id': 'chain-XYZ-migrated-at-height-13',
+            {'height': 14, 'chain_id': 'chain-XYZ-migrated-at-height-13',
-         'is_synced': False},
+             'is_synced': False},
     ),
 ])
 def test_migrate_abci_chain_generates_new_chains(b, chain, block_height,
@@ -469,8 +523,8 @@ def test_get_spent_key_order(b, user_pk, user_sk, user2_pk, user2_sk):
 
     tx1 = Transaction.create([user_pk],
                              [([alice.public_key], 3), ([user_pk], 2)],
-                             asset=None)\
+                             asset=None) \
-                     .sign([user_sk])
+        .sign([user_sk])
     b.store_bulk_transactions([tx1])
     assert tx1.validate(b)
     inputs = tx1.to_inputs()