Problem: utxoset needs to be updated

Solution: update utxoset via store_transaction

bigchaindb/backend/localmongodb/query.py

@@ -256,4 +256,5 @@ def delete_unspent_outputs(conn, *unspent_outputs):
 def get_unspent_outputs(conn, *, query=None):
     if query is None:
         query = {}
-    return conn.run(conn.collection('utxos').find(query))
+    return conn.run(conn.collection('utxos').find(query,
+                                                  projection={'_id': False}))

bigchaindb/tendermint/lib.py

@@ -61,6 +61,7 @@ class BigchainDB(Bigchain):
     def store_transaction(self, transaction):
         """Store a valid transaction to the transactions collection."""
 
+        self.update_utxoset(transaction)
         transaction = deepcopy(transaction.to_dict())
         if transaction['operation'] == 'CREATE':
             asset = transaction.pop('asset')
@@ -81,6 +82,7 @@ class BigchainDB(Bigchain):
         assets = []
         txn_metadatas = []
         for transaction in transactions:
+            self.update_utxoset(transaction)
             transaction = transaction.to_dict()
             if transaction['operation'] == 'CREATE':
                 asset = transaction.pop('asset')
@@ -98,6 +100,56 @@ class BigchainDB(Bigchain):
             backend.query.store_assets(self.connection, assets)
         return backend.query.store_transactions(self.connection, txns)
 
+    def update_utxoset(self, transaction):
+        """Update the UTXO set given ``transaction``. That is, remove
+        the outputs that the given ``transaction`` spends, and add the
+        outputs that the given ``transaction`` creates.
+
+        Args:
+            transaction (:obj:`~bigchaindb.models.Transaction`): A new
+                transaction incoming into the system for which the UTXO
+                set needs to be updated.
+        """
+        spent_outputs = [
+            spent_output for spent_output in transaction.spent_outputs
+        ]
+        if spent_outputs:
+            self.delete_unspent_outputs(*spent_outputs)
+        self.store_unspent_outputs(
+            *[utxo._asdict() for utxo in transaction.unspent_outputs]
+        )
+
+    def store_unspent_outputs(self, *unspent_outputs):
+        """Store the given ``unspent_outputs`` (utxos).
+
+        Args:
+            *unspent_outputs (:obj:`tuple` of :obj:`dict`): Variable
+                length tuple or list of unspent outputs.
+        """
+        if unspent_outputs:
+            return backend.query.store_unspent_outputs(
+                                            self.connection, *unspent_outputs)
+
+    def get_unspent_outputs(self):
+        """Get the utxoset.
+
+        Returns:
+            generator of unspent_outputs.
+        """
+        cursor = backend.query.get_unspent_outputs(self.connection)
+        return (record for record in cursor)
+
+    def delete_unspent_outputs(self, *unspent_outputs):
+        """Deletes the given ``unspent_outputs`` (utxos).
+
+        Args:
+            *unspent_outputs (:obj:`tuple` of :obj:`dict`): Variable
+                length tuple or list of unspent outputs.
+        """
+        if unspent_outputs:
+            return backend.query.delete_unspent_outputs(
+                                        self.connection, *unspent_outputs)
+
     def get_transaction(self, transaction_id, include_status=False):
         transaction = backend.query.get_transaction(self.connection, transaction_id)
         asset = backend.query.get_asset(self.connection, transaction_id)

tests/backend/localmongodb/test_queries.py

@@ -6,23 +6,6 @@ import pymongo
 pytestmark = [pytest.mark.tendermint, pytest.mark.localmongodb, pytest.mark.bdb]
 
 
-@pytest.fixture
-def dummy_unspent_outputs():
-    return [
-        {'transaction_id': 'a', 'output_index': 0},
-        {'transaction_id': 'a', 'output_index': 1},
-        {'transaction_id': 'b', 'output_index': 0},
-    ]
-
-
-@pytest.fixture
-def utxoset(dummy_unspent_outputs, utxo_collection):
-    insert_res = utxo_collection.insert_many(deepcopy(dummy_unspent_outputs))
-    assert insert_res.acknowledged
-    assert len(insert_res.inserted_ids) == 3
-    return dummy_unspent_outputs, utxo_collection
-
-
 def test_get_txids_filtered(signed_create_tx, signed_transfer_tx):
     from bigchaindb.backend import connect, query
     from bigchaindb.models import Transaction
@@ -286,7 +269,6 @@ def test_get_unspent_outputs(db_context, utxoset):
     assert cursor.count() == 3
     retrieved_utxoset = list(cursor)
     unspent_outputs, utxo_collection = utxoset
-    assert retrieved_utxoset == list(utxo_collection.find())
+    assert retrieved_utxoset == list(
-    for utxo in retrieved_utxoset:
+        utxo_collection.find(projection={'_id': False}))
-        del utxo['_id']
     assert retrieved_utxoset == unspent_outputs

tests/conftest.py

@@ -11,6 +11,7 @@ import random
 from collections import namedtuple
 
 import pytest
+from pymongo import MongoClient
 
 from logging import getLogger
 from logging.config import dictConfig
@@ -675,3 +676,30 @@ def unspent_output_2():
 @pytest.fixture
 def unspent_outputs(unspent_output_0, unspent_output_1, unspent_output_2):
     return unspent_output_0, unspent_output_1, unspent_output_2
+
+
+@pytest.fixture
+def mongo_client(db_context):
+    return MongoClient(host=db_context.host, port=db_context.port)
+
+
+@pytest.fixture
+def utxo_collection(db_context, mongo_client):
+    return mongo_client[db_context.name].utxos
+
+
+@pytest.fixture
+def dummy_unspent_outputs():
+    return [
+        {'transaction_id': 'a', 'output_index': 0},
+        {'transaction_id': 'a', 'output_index': 1},
+        {'transaction_id': 'b', 'output_index': 0},
+    ]
+
+
+@pytest.fixture
+def utxoset(dummy_unspent_outputs, utxo_collection):
+    res = utxo_collection.insert_many(copy.deepcopy(dummy_unspent_outputs))
+    assert res.acknowledged
+    assert len(res.inserted_ids) == 3
+    return dummy_unspent_outputs, utxo_collection

tests/tendermint/test_core.py

@@ -66,6 +66,12 @@ def test_deliver_tx__valid_create_updates_db(b):
     app.end_block(99)
     app.commit()
     assert b.get_transaction(tx.id).id == tx.id
+    unspent_outputs = b.get_unspent_outputs()
+    unspent_output = next(unspent_outputs)
+    expected_unspent_output = next(tx.unspent_outputs)._asdict()
+    assert unspent_output == expected_unspent_output
+    with pytest.raises(StopIteration):
+        next(unspent_outputs)
 
 
 def test_deliver_tx__double_spend_fails(b):

tests/tendermint/test_lib.py

@@ -2,6 +2,7 @@ import os
 from unittest.mock import patch
 
 import pytest
+from pymongo import MongoClient
 
 from bigchaindb import backend
 
@@ -9,6 +10,7 @@ from bigchaindb import backend
 pytestmark = pytest.mark.tendermint
 
 
+@pytest.mark.bdb
 def test_asset_is_separated_from_transaciton(b):
     from bigchaindb.models import Transaction
     from bigchaindb.common.crypto import generate_key_pair
@@ -122,3 +124,190 @@ def test_post_transaction_invalid_mode(b):
     tx = b.validate_transaction(tx)
     with pytest.raises(ValidationError):
         b.write_transaction(tx, 'nope')
+
+
+@pytest.mark.bdb
+def test_update_utxoset(tb, signed_create_tx, signed_transfer_tx, db_context):
+    mongo_client = MongoClient(host=db_context.host, port=db_context.port)
+    tb.update_utxoset(signed_create_tx)
+    utxoset = mongo_client[db_context.name]['utxos']
+    assert utxoset.count() == 1
+    utxo = utxoset.find_one()
+    assert utxo['transaction_id'] == signed_create_tx.id
+    assert utxo['output_index'] == 0
+    tb.update_utxoset(signed_transfer_tx)
+    assert utxoset.count() == 1
+    utxo = utxoset.find_one()
+    assert utxo['transaction_id'] == signed_transfer_tx.id
+    assert utxo['output_index'] == 0
+
+
+@pytest.mark.bdb
+def test_store_transaction(mocker, tb, signed_create_tx,
+                           signed_transfer_tx, db_context):
+    mocked_store_asset = mocker.patch('bigchaindb.backend.query.store_asset')
+    mocked_store_metadata = mocker.patch(
+        'bigchaindb.backend.query.store_metadata')
+    mocked_store_transaction = mocker.patch(
+        'bigchaindb.backend.query.store_transaction')
+    mongo_client = MongoClient(host=db_context.host, port=db_context.port)
+    tb.store_transaction(signed_create_tx)
+    utxoset = mongo_client[db_context.name]['utxos']
+    assert utxoset.count() == 1
+    utxo = utxoset.find_one()
+    assert utxo['transaction_id'] == signed_create_tx.id
+    assert utxo['output_index'] == 0
+    mocked_store_asset.assert_called_once_with(
+        tb.connection,
+        {'id': signed_create_tx.id, 'data': signed_create_tx.asset['data']},
+    )
+    mocked_store_metadata.assert_called_once_with(
+        tb.connection,
+        [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}],
+    )
+    mocked_store_transaction.assert_called_once_with(
+        tb.connection,
+        {k: v for k, v in signed_create_tx.to_dict().items()
+         if k not in ('asset', 'metadata')},
+    )
+    mocked_store_asset.reset_mock()
+    mocked_store_metadata.reset_mock()
+    mocked_store_transaction.reset_mock()
+    tb.store_transaction(signed_transfer_tx)
+    assert utxoset.count() == 1
+    utxo = utxoset.find_one()
+    assert utxo['transaction_id'] == signed_transfer_tx.id
+    assert utxo['output_index'] == 0
+    assert not mocked_store_asset.called
+    mocked_store_metadata.asser_called_once_with(
+        tb.connection,
+        {'id': signed_transfer_tx.id, 'metadata': signed_transfer_tx.metadata},
+    )
+    mocked_store_transaction.assert_called_once_with(
+        tb.connection,
+        {k: v for k, v in signed_transfer_tx.to_dict().items()
+         if k != 'metadata'},
+    )
+
+
+@pytest.mark.bdb
+def test_store_bulk_transaction(mocker, tb, signed_create_tx,
+                                signed_transfer_tx, db_context):
+    mocked_store_assets = mocker.patch(
+        'bigchaindb.backend.query.store_assets')
+    mocked_store_metadata = mocker.patch(
+        'bigchaindb.backend.query.store_metadatas')
+    mocked_store_transactions = mocker.patch(
+        'bigchaindb.backend.query.store_transactions')
+    mongo_client = MongoClient(host=db_context.host, port=db_context.port)
+    tb.store_bulk_transactions((signed_create_tx,))
+    utxoset = mongo_client[db_context.name]['utxos']
+    assert utxoset.count() == 1
+    utxo = utxoset.find_one()
+    assert utxo['transaction_id'] == signed_create_tx.id
+    assert utxo['output_index'] == 0
+    mocked_store_assets.assert_called_once_with(
+        tb.connection,
+        [{'id': signed_create_tx.id, 'data': signed_create_tx.asset['data']}],
+    )
+    mocked_store_metadata.assert_called_once_with(
+        tb.connection,
+        [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}],
+    )
+    mocked_store_transactions.assert_called_once_with(
+        tb.connection,
+        [{k: v for k, v in signed_create_tx.to_dict().items()
+         if k not in ('asset', 'metadata')}],
+    )
+    mocked_store_assets.reset_mock()
+    mocked_store_metadata.reset_mock()
+    mocked_store_transactions.reset_mock()
+    tb.store_bulk_transactions((signed_transfer_tx,))
+    assert utxoset.count() == 1
+    utxo = utxoset.find_one()
+    assert utxo['transaction_id'] == signed_transfer_tx.id
+    assert utxo['output_index'] == 0
+    assert not mocked_store_assets.called
+    mocked_store_metadata.asser_called_once_with(
+        tb.connection,
+        [{'id': signed_transfer_tx.id,
+          'metadata': signed_transfer_tx.metadata}],
+    )
+    mocked_store_transactions.assert_called_once_with(
+        tb.connection,
+        [{k: v for k, v in signed_transfer_tx.to_dict().items()
+          if k != 'metadata'}],
+    )
+
+
+@pytest.mark.bdb
+def test_delete_zero_unspent_outputs(b, utxoset):
+    unspent_outputs, utxo_collection = utxoset
+    delete_res = b.delete_unspent_outputs()
+    assert delete_res is None
+    assert utxo_collection.count() == 3
+    assert utxo_collection.find(
+        {'$or': [
+            {'transaction_id': 'a', 'output_index': 0},
+            {'transaction_id': 'b', 'output_index': 0},
+            {'transaction_id': 'a', 'output_index': 1},
+        ]}
+    ).count() == 3
+
+
+@pytest.mark.bdb
+def test_delete_one_unspent_outputs(b, utxoset):
+    unspent_outputs, utxo_collection = utxoset
+    delete_res = b.delete_unspent_outputs(unspent_outputs[0])
+    assert delete_res['n'] == 1
+    assert utxo_collection.find(
+        {'$or': [
+            {'transaction_id': 'a', 'output_index': 1},
+            {'transaction_id': 'b', 'output_index': 0},
+        ]}
+    ).count() == 2
+    assert utxo_collection.find(
+            {'transaction_id': 'a', 'output_index': 0}).count() == 0
+
+
+@pytest.mark.bdb
+def test_delete_many_unspent_outputs(b, utxoset):
+    unspent_outputs, utxo_collection = utxoset
+    delete_res = b.delete_unspent_outputs(*unspent_outputs[::2])
+    assert delete_res['n'] == 2
+    assert utxo_collection.find(
+        {'$or': [
+            {'transaction_id': 'a', 'output_index': 0},
+            {'transaction_id': 'b', 'output_index': 0},
+        ]}
+    ).count() == 0
+    assert utxo_collection.find(
+            {'transaction_id': 'a', 'output_index': 1}).count() == 1
+
+
+@pytest.mark.bdb
+def test_store_zero_unspent_output(b, utxo_collection):
+    res = b.store_unspent_outputs()
+    assert res is None
+    assert utxo_collection.count() == 0
+
+
+@pytest.mark.bdb
+def test_store_one_unspent_output(b, unspent_output_1, utxo_collection):
+    res = b.store_unspent_outputs(unspent_output_1)
+    assert res.acknowledged
+    assert len(res.inserted_ids) == 1
+    assert utxo_collection.find(
+        {'transaction_id': unspent_output_1['transaction_id'],
+         'output_index': unspent_output_1['output_index']}
+    ).count() == 1
+
+
+@pytest.mark.bdb
+def test_store_many_unspent_outputs(b, unspent_outputs, utxo_collection):
+    res = b.store_unspent_outputs(*unspent_outputs)
+    assert res.acknowledged
+    assert len(res.inserted_ids) == 3
+    assert utxo_collection.find(
+        {'transaction_id': unspent_outputs[0]['transaction_id']}
+    ).count() == 3