Problem: Outputs API responds with incorrect outputs (#2567).

* Problem: Outputs API doesn't respond with correct unspent/spent outputs

Solution: Fix fastquery such that embedded document is queried properly

* Problem: key order agnostic queries not implemented

Solution: get_spent queries embedded documents which respect key order. This is
not expected by the application hence the query should be altered to query any
kind of key order

* Problem: Mongo query for get_spent too complicated

Solution: Simplify query using $elemMatch

* Problem: No test for checking mixed spent outputs

Solution: Add test for `get_spending_transactions` to check that correct
matching is done when querying documents with multiple inputs

* Problem: tranasction ids not assert when getting spent outputs

Solution: assert tranasction ids

bigchaindb/backend/localmongodb/query.py

@@ -91,8 +91,11 @@ def get_assets(conn, asset_ids):
 
 @register_query(LocalMongoDBConnection)
 def get_spent(conn, transaction_id, output):
-    query = {'inputs.fulfills': {'transaction_id': transaction_id,
-                                 'output_index': output}}
+    query = {'inputs':
+             {'$elemMatch':
+              {'$and': [{'fulfills.transaction_id': transaction_id},
+                        {'fulfills.output_index': output}]}}}
+
     return conn.run(
         conn.collection('transactions')
             .find(query, {'_id': 0}))
@@ -180,15 +183,18 @@ def get_owned_ids(conn, owner):
 
 @register_query(LocalMongoDBConnection)
 def get_spending_transactions(conn, inputs):
+    transaction_ids = [i['transaction_id'] for i in inputs]
+    output_indexes = [i['output_index'] for i in inputs]
+    query = {'inputs':
+             {'$elemMatch':
+              {'$and':
+               [
+                   {'fulfills.transaction_id': {'$in': transaction_ids}},
+                   {'fulfills.output_index': {'$in': output_indexes}}
+               ]}}}
+
     cursor = conn.run(
-        conn.collection('transactions').aggregate([
-            {'$match': {
-                'inputs.fulfills': {
-                    '$in': inputs,
-                },
-            }},
-            {'$project': {'_id': False}}
-        ]))
+        conn.collection('transactions').find(query, {'_id': False}))
     return cursor
 
 

tests/backend/localmongodb/test_queries.py

@@ -234,6 +234,49 @@ def test_get_spending_transactions(user_pk, user_sk):
     assert txns == [tx2.to_dict(), tx4.to_dict()]
 
 
+def test_get_spending_transactions_multiple_inputs():
+    from bigchaindb.backend import connect, query
+    from bigchaindb.models import Transaction
+    from bigchaindb.common.crypto import generate_key_pair
+    conn = connect()
+    (alice_sk, alice_pk) = generate_key_pair()
+    (bob_sk, bob_pk) = generate_key_pair()
+    (carol_sk, carol_pk) = generate_key_pair()
+
+    out = [([alice_pk], 9)]
+    tx1 = Transaction.create([alice_pk], out).sign([alice_sk])
+
+    inputs1 = tx1.to_inputs()
+    tx2 = Transaction.transfer([inputs1[0]],
+                               [([alice_pk], 6), ([bob_pk], 3)],
+                               tx1.id).sign([alice_sk])
+
+    inputs2 = tx2.to_inputs()
+    tx3 = Transaction.transfer([inputs2[0]],
+                               [([bob_pk], 3), ([carol_pk], 3)],
+                               tx1.id).sign([alice_sk])
+
+    inputs3 = tx3.to_inputs()
+    tx4 = Transaction.transfer([inputs2[1], inputs3[0]],
+                               [([carol_pk], 6)],
+                               tx1.id).sign([bob_sk])
+
+    txns = [deepcopy(tx.to_dict()) for tx in [tx1, tx2, tx3, tx4]]
+    conn.db.transactions.insert_many(txns)
+
+    links = [
+        ({'transaction_id': tx2.id, 'output_index': 0}, 1, [tx3.id]),
+        ({'transaction_id': tx2.id, 'output_index': 1}, 1, [tx4.id]),
+        ({'transaction_id': tx3.id, 'output_index': 0}, 1, [tx4.id]),
+        ({'transaction_id': tx3.id, 'output_index': 1}, 0, None),
+    ]
+    for l, num, match in links:
+        txns = list(query.get_spending_transactions(conn, [l]))
+        assert len(txns) == num
+        if len(txns):
+            assert [tx['id'] for tx in txns] == match
+
+
 def test_store_block():
     from bigchaindb.backend import connect, query
     from bigchaindb.lib import Block

tests/tendermint/test_fastquery.py

@@ -73,3 +73,47 @@ def test_filter_unspent_outputs(b, user_pk, user_sk):
     assert set(sp for sp in spents) == {
         inputs[0].fulfills,
     }
+
+
+def test_outputs_query_key_order(b, user_pk, user_sk, user2_pk, user2_sk):
+    from bigchaindb import backend
+    from bigchaindb.backend import connect
+
+    tx1 = Transaction.create([user_pk],
+                             [([user_pk], 3), ([user_pk], 2), ([user_pk], 1)])\
+                     .sign([user_sk])
+    b.store_bulk_transactions([tx1])
+
+    inputs = tx1.to_inputs()
+    tx2 = Transaction.transfer([inputs[1]], [([user2_pk], 2)], tx1.id).sign([user_sk])
+    assert tx2.validate(b)
+
+    tx2_dict = tx2.to_dict()
+    fulfills = tx2_dict['inputs'][0]['fulfills']
+    tx2_dict['inputs'][0]['fulfills'] = {'transaction_id': fulfills['transaction_id'],
+                                         'output_index': fulfills['output_index']}
+    backend.query.store_transactions(b.connection, [tx2_dict])
+
+    outputs = b.get_outputs_filtered(user_pk, spent=False)
+    assert len(outputs) == 2
+
+    outputs = b.get_outputs_filtered(user2_pk, spent=False)
+    assert len(outputs) == 1
+
+    # clean the transaction, metdata and asset collection
+    conn = connect()
+    conn.run(conn.collection('transactions').delete_many({}))
+    conn.run(conn.collection('metadata').delete_many({}))
+    conn.run(conn.collection('assets').delete_many({}))
+
+    b.store_bulk_transactions([tx1])
+    tx2_dict = tx2.to_dict()
+    tx2_dict['inputs'][0]['fulfills'] = {'output_index': fulfills['output_index'],
+                                         'transaction_id': fulfills['transaction_id']}
+
+    backend.query.store_transactions(b.connection, [tx2_dict])
+    outputs = b.get_outputs_filtered(user_pk, spent=False)
+    assert len(outputs) == 2
+
+    outputs = b.get_outputs_filtered(user2_pk, spent=False)
+    assert len(outputs) == 1

tests/tendermint/test_lib.py

@@ -471,3 +471,36 @@ def test_migrate_abci_chain_generates_new_chains(b, chain, block_height,
     b.migrate_abci_chain()
     latest_chain = b.get_latest_abci_chain()
     assert latest_chain == expected
+
+
+@pytest.mark.bdb
+def test_get_spent_key_order(b, user_pk, user_sk, user2_pk, user2_sk):
+    from bigchaindb import backend
+    from bigchaindb.models import Transaction
+    from bigchaindb.common.crypto import generate_key_pair
+    from bigchaindb.common.exceptions import DoubleSpend
+
+    alice = generate_key_pair()
+    bob = generate_key_pair()
+
+    tx1 = Transaction.create([user_pk],
+                             [([alice.public_key], 3), ([user_pk], 2)],
+                             asset=None)\
+                     .sign([user_sk])
+    b.store_bulk_transactions([tx1])
+
+    inputs = tx1.to_inputs()
+    tx2 = Transaction.transfer([inputs[1]], [([user2_pk], 2)], tx1.id).sign([user_sk])
+    assert tx2.validate(b)
+
+    tx2_dict = tx2.to_dict()
+    fulfills = tx2_dict['inputs'][0]['fulfills']
+    tx2_dict['inputs'][0]['fulfills'] = {'output_index': fulfills['output_index'],
+                                         'transaction_id': fulfills['transaction_id']}
+
+    backend.query.store_transactions(b.connection, [tx2_dict])
+
+    tx3 = Transaction.transfer([inputs[1]], [([bob.public_key], 2)], tx1.id).sign([user_sk])
+
+    with pytest.raises(DoubleSpend):
+        tx3.validate(b)

tests/test_core.py

@@ -64,6 +64,7 @@ def test_bigchain_class_initialization_with_parameters():
     assert bigchain.validation == BaseValidationRules
 
 
+@pytest.mark.bdb
 def test_get_spent_issue_1271(b, alice, bob, carol):
     from bigchaindb.models import Transaction
 
@@ -71,7 +72,7 @@ def test_get_spent_issue_1271(b, alice, bob, carol):
         [carol.public_key],
         [([carol.public_key], 8)],
     ).sign([carol.private_key])
-    assert b.validate_transaction(tx_1)
+    assert tx_1.validate(b)
     b.store_bulk_transactions([tx_1])
 
     tx_2 = Transaction.transfer(
@@ -81,7 +82,7 @@ def test_get_spent_issue_1271(b, alice, bob, carol):
          ([carol.public_key], 4)],
         asset_id=tx_1.id,
     ).sign([carol.private_key])
-    assert b.validate_transaction(tx_2)
+    assert tx_2.validate(b)
     b.store_bulk_transactions([tx_2])
 
     tx_3 = Transaction.transfer(
@@ -90,7 +91,7 @@ def test_get_spent_issue_1271(b, alice, bob, carol):
          ([carol.public_key], 3)],
         asset_id=tx_1.id,
     ).sign([carol.private_key])
-    assert b.validate_transaction(tx_3)
+    assert tx_3.validate(b)
     b.store_bulk_transactions([tx_3])
 
     tx_4 = Transaction.transfer(
@@ -98,7 +99,7 @@ def test_get_spent_issue_1271(b, alice, bob, carol):
         [([bob.public_key], 3)],
         asset_id=tx_1.id,
     ).sign([alice.private_key])
-    assert b.validate_transaction(tx_4)
+    assert tx_4.validate(b)
     b.store_bulk_transactions([tx_4])
 
     tx_5 = Transaction.transfer(
@@ -106,7 +107,8 @@ def test_get_spent_issue_1271(b, alice, bob, carol):
         [([alice.public_key], 2)],
         asset_id=tx_1.id,
     ).sign([bob.private_key])
-    assert b.validate_transaction(tx_5)
+    assert tx_5.validate(b)
+
     b.store_bulk_transactions([tx_5])
 
     assert b.get_spent(tx_2.id, 0) == tx_5