Merge branch 'master' into implement-connection-run-for-mongodb

CHANGELOG.md

@@ -16,6 +16,27 @@ For reference, the possible headings are:
 * **Notes**
 
 
+## [0.8.2] - 2017-01-27
+Tag name: v0.8.2
+
+### Fixed
+- Fix spend input twice in same transaction
+  (https://github.com/bigchaindb/bigchaindb/issues/1099).
+
+
+## [0.8.1] - 2017-01-16
+Tag name: v0.8.1
+= commit:
+committed:
+
+### Changed
+- Upgrade pysha3 to 1.0.0 (supports official NIST standard).
+
+### Fixed
+- Workaround for rapidjson problem with package metadata extraction
+  (https://github.com/kenrobbins/python-rapidjson/pull/52). 
+
+
 ## [0.8.0] - 2016-11-29
 Tag name: v0.8.0
 = commit:

bigchaindb/backend/mongodb/query.py

@@ -1,6 +1,7 @@
 """Query implementation for MongoDB"""
 
 from time import time
+from itertools import chain
 
 from pymongo import ReturnDocument
 from pymongo import errors
@@ -8,6 +9,7 @@ from pymongo import errors
 
 from bigchaindb import backend
 from bigchaindb.common.exceptions import CyclicBlockchainError
+from bigchaindb.common.transaction import Transaction
 from bigchaindb.backend.utils import module_dispatch_registrar
 from bigchaindb.backend.mongodb.connection import MongoDBConnection, collection
 
@@ -97,6 +99,43 @@ def get_blocks_status_from_transaction(conn, transaction_id):
               projection=['id', 'block.voters']))
 
 
+@register_query(MongoDBConnection)
+def get_txids_filtered(conn, asset_id, operation=None):
+    parts = []
+
+    if operation in (Transaction.CREATE, None):
+        # get the txid of the create transaction for asset_id
+        cursor = conn.db['bigchain'].aggregate([
+            {'$match': {
+                'block.transactions.id': asset_id,
+                'block.transactions.operation': 'CREATE'
+            }},
+            {'$unwind': '$block.transactions'},
+            {'$match': {
+                'block.transactions.id': asset_id,
+                'block.transactions.operation': 'CREATE'
+            }},
+            {'$project': {'block.transactions.id': True}}
+        ])
+        parts.append(elem['block']['transactions']['id'] for elem in cursor)
+
+    if operation in (Transaction.TRANSFER, None):
+        # get txids of transfer transaction with asset_id
+        cursor = conn.db['bigchain'].aggregate([
+            {'$match': {
+                'block.transactions.asset.id': asset_id
+            }},
+            {'$unwind': '$block.transactions'},
+            {'$match': {
+                'block.transactions.asset.id': asset_id
+            }},
+            {'$project': {'block.transactions.id': True}}
+        ])
+        parts.append(elem['block']['transactions']['id'] for elem in cursor)
+
+    return chain(*parts)
+
+
 @register_query(MongoDBConnection)
 def get_asset_by_id(conn, asset_id):
     cursor = conn.run(
@@ -253,37 +292,18 @@ def get_last_voted_block(conn, node_pubkey):
 
 @register_query(MongoDBConnection)
 def get_unvoted_blocks(conn, node_pubkey):
-    return conn.run(
+    return conn.db['bigchain'].aggregate([
-        collection('bigchain').aggregate([
+        {'$lookup': {
-            {'$lookup': {
+            'from': 'votes',
-                'from': 'votes',
+            'localField': 'id',
-                'localField': 'id',
+            'foreignField': 'vote.voting_for_block',
-                'foreignField': 'vote.voting_for_block',
+            'as': 'votes'
-                'as': 'votes'
+        }},
-            }},
+        {'$match': {
-            {'$match': {
+            'votes.node_pubkey': {'$ne': node_pubkey},
-                'votes.node_pubkey': {'$ne': node_pubkey},
+            'block.transactions.operation': {'$ne': 'GENESIS'}
-                'block.transactions.operation': {'$ne': 'GENESIS'}
+        }},
-            }},
+        {'$project': {
-            {'$project': {
+            'votes': False, '_id': False
-                'votes': False, '_id': False
+        }}
-            }}
+    ])
-        ]))
-
-
-@register_query(MongoDBConnection)
-def get_txids_filtered(conn, asset_id, operation=None):
-    match = {'block.transactions.asset.id': asset_id}
-
-    if operation:
-        match['block.transactions.operation'] = operation
-
-    cursor = conn.run(
-        collection('bigchain')
-        .aggregate([
-            {'$match': match},
-            {'$unwind': '$block.transactions'},
-            {'$match': match},
-            {'$project': {'block.transactions.id': True}}
-        ]))
-    return (r['block']['transactions']['id'] for r in cursor)

bigchaindb/backend/rethinkdb/connection.py

@@ -77,3 +77,5 @@ class RethinkDBConnection(Connection):
                 wait_time = 2**i
                 logging.debug('Error connecting to database, waiting %ss', wait_time)
                 time.sleep(wait_time)
+            else:
+                break

bigchaindb/backend/rethinkdb/query.py

@@ -1,9 +1,11 @@
+from itertools import chain
 from time import time
 
 import rethinkdb as r
 
 from bigchaindb import backend, utils
 from bigchaindb.common import exceptions
+from bigchaindb.common.transaction import Transaction
 from bigchaindb.backend.utils import module_dispatch_registrar
 from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection
 
@@ -71,6 +73,30 @@ def get_blocks_status_from_transaction(connection, transaction_id):
             .pluck('votes', 'id', {'block': ['voters']}))
 
 
+@register_query(RethinkDBConnection)
+def get_txids_filtered(connection, asset_id, operation=None):
+    # here we only want to return the transaction ids since later on when
+    # we are going to retrieve the transaction with status validation
+
+    parts = []
+
+    if operation in (Transaction.CREATE, None):
+        # First find the asset's CREATE transaction
+        parts.append(connection.run(
+            _get_asset_create_tx_query(asset_id).get_field('id')))
+
+    if operation in (Transaction.TRANSFER, None):
+        # Then find any TRANSFER transactions related to the asset
+        parts.append(connection.run(
+            r.table('bigchain')
+            .get_all(asset_id, index='asset_id')
+            .concat_map(lambda block: block['block']['transactions'])
+            .filter(lambda transaction: transaction['asset']['id'] == asset_id)
+            .get_field('id')))
+
+    return chain(*parts)
+
+
 @register_query(RethinkDBConnection)
 def get_asset_by_id(connection, asset_id):
     return connection.run(_get_asset_create_tx_query(asset_id).pluck('asset'))
@@ -233,20 +259,3 @@ def get_unvoted_blocks(connection, node_pubkey):
     #        database level. Solving issue #444 can help untangling the situation
     unvoted_blocks = filter(lambda block: not utils.is_genesis_block(block), unvoted)
     return unvoted_blocks
-
-
-@register_query(RethinkDBConnection)
-def get_txids_filtered(connection, asset_id, operation=None):
-    # here we only want to return the transaction ids since later on when
-    # we are going to retrieve the transaction with status validation
-
-    tx_filter = r.row['asset']['id'] == asset_id
-    if operation:
-        tx_filter &= r.row['operation'] == operation
-
-    return connection.run(
-        r.table('bigchain')
-         .get_all(asset_id, index='asset_id')
-         .concat_map(lambda block: block['block']['transactions'])
-         .filter(tx_filter)
-         .get_field('id'))

bigchaindb/common/schema/transaction.yaml

@@ -103,8 +103,8 @@ definitions:
     description: |
         Description of the asset being transacted. In the case of a ``TRANSFER``
         transaction, this field contains only the ID of asset. In the case
-        of a ``CREATE`` transaction, this field contains the user-defined
+        of a ``CREATE`` transaction, this field contains only the user-defined
-        payload and the asset ID (duplicated from the Transaction ID).
+        payload.
     additionalProperties: false
     properties:
       id:

bigchaindb/common/transaction.py

@@ -444,7 +444,6 @@ class Transaction(object):
                 asset is not None and not (isinstance(asset, dict) and 'data' in asset)):
             raise TypeError(('`asset` must be None or a dict holding a `data` '
                              " property instance for '{}' Transactions".format(operation)))
-            asset.pop('id', None)  # Remove duplicated asset ID if there is one
         elif (operation == Transaction.TRANSFER and
                 not (isinstance(asset, dict) and 'id' in asset)):
             raise TypeError(('`asset` must be a dict holding an `id` property '
@@ -927,11 +926,9 @@ class Transaction(object):
 
         tx_no_signatures = Transaction._remove_signatures(tx)
         tx_serialized = Transaction._to_str(tx_no_signatures)
-        tx['id'] = Transaction._to_hash(tx_serialized)
+        tx_id = Transaction._to_hash(tx_serialized)
-        if self.operation == Transaction.CREATE:
+
-            # Duplicate asset into asset for consistency with TRANSFER
+        tx['id'] = tx_id
-            # transactions
-            tx['asset']['id'] = tx['id']
         return tx
 
     @staticmethod
@@ -955,9 +952,6 @@ class Transaction(object):
             #       case could yield incorrect signatures. This is why we only
             #       set it to `None` if it's set in the dict.
             input_['fulfillment'] = None
-        # Pop duplicated asset_id from CREATE tx
-        if tx_dict['operation'] == Transaction.CREATE:
-            tx_dict['asset'].pop('id', None)
         return tx_dict
 
     @staticmethod
@@ -1037,10 +1031,6 @@ class Transaction(object):
                        "the hash of its body, i.e. it's not valid.")
             raise InvalidHash(err_msg.format(proposed_tx_id))
 
-        if tx_body.get('operation') == Transaction.CREATE:
-            if proposed_tx_id != tx_body['asset'].get('id'):
-                raise InvalidHash('CREATE tx has wrong asset_id')
-
     @classmethod
     def from_dict(cls, tx):
         """Transforms a Python dictionary to a Transaction object.

bigchaindb/models.py

@@ -88,6 +88,11 @@ class Transaction(Transaction):
                 if output.amount < 1:
                     raise AmountError('`amount` needs to be greater than zero')
 
+            # Validate that all inputs are distinct
+            links = [i.fulfills.to_uri() for i in self.inputs]
+            if len(links) != len(set(links)):
+                raise DoubleSpend('tx "{}" spends inputs twice'.format(self.id))
+
             # validate asset id
             asset_id = Transaction.get_asset_id(input_txs)
             if asset_id != self.asset['id']:

bigchaindb/web/views/parameters.py

@@ -8,6 +8,7 @@ def valid_txid(txid):
 
 
 def valid_bool(val):
+    val = val.lower()
     if val == 'true':
         return True
     if val == 'false':
@@ -23,6 +24,7 @@ def valid_ed25519(key):
 
 
 def valid_operation(op):
+    op = op.upper()
     if op == 'CREATE':
         return 'CREATE'
     if op == 'TRANSFER':

docs/server/source/data-models/asset-model.md

@@ -1,11 +1,10 @@
 # The Digital Asset Model
 
-The asset ID is the same as the ID of the CREATE transaction that defined the asset.
+To avoid redundant data in transactions, the digital asset model is different for `CREATE` and `TRANSFER` transactions.
 
-In the case of a CREATE transaction, the transaction ID is duplicated into the asset object for clarity and consistency in the database. The CREATE transaction also contains a user definable payload to describe the asset:
+A digital asset's properties are defined in a `CREATE` transaction with the following model:
 ```json
 {
-    "id": "<same as transaction ID (sha3-256 hash)>",
     "data": "<json document>"
 }
 ```

docs/server/source/drivers-clients/http-client-server-api.rst

@@ -165,7 +165,7 @@ not already been spent.
    Returns a list of links to transaction outputs.
 
    :param public_key: Base58 encoded public key associated with output ownership. This parameter is mandatory and without it the endpoint will return a ``400`` response code.
-   :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend.
+   :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend. Defaults to "false".
 
 
    **Example request**:

tests/backend/rethinkdb/test_connection.py

@@ -1,6 +1,7 @@
 import time
 import multiprocessing as mp
 from threading import Thread
+from unittest.mock import patch
 
 import pytest
 import rethinkdb as r
@@ -118,3 +119,15 @@ def test_changefeed_reconnects_when_connection_lost(monkeypatch):
 
     fact = changefeed.outqueue.get()['fact']
     assert fact == 'Cats sleep 70% of their lives.'
+
+
+@patch('rethinkdb.connect')
+def test_connection_happens_one_time_if_successful(mock_connect):
+    from bigchaindb.backend import connect
+
+    query = r.expr('1')
+    conn = connect('rethinkdb', 'localhost', 1337, 'whatev')
+    conn.run(query)
+    mock_connect.assert_called_once_with(host='localhost',
+                                         port=1337,
+                                         db='whatev')

tests/common/test_transaction.py

@@ -300,7 +300,6 @@ def test_transaction_serialization(user_input, user_output, data):
         'operation': Transaction.CREATE,
         'metadata': None,
         'asset': {
-            'id': tx_id,
             'data': data,
         }
     }
@@ -308,7 +307,7 @@ def test_transaction_serialization(user_input, user_output, data):
     tx = Transaction(Transaction.CREATE, {'data': data}, [user_input],
                      [user_output])
     tx_dict = tx.to_dict()
-    tx_dict['id'] = tx_dict['asset']['id'] = tx_id
+    tx_dict['id'] = tx_id
 
     assert tx_dict == expected
 
@@ -335,7 +334,6 @@ def test_transaction_deserialization(user_input, user_output, data):
     }
     tx_no_signatures = Transaction._remove_signatures(tx)
     tx['id'] = Transaction._to_hash(Transaction._to_str(tx_no_signatures))
-    tx['asset']['id'] = tx['id']
     tx = Transaction.from_dict(tx)
 
     assert tx == expected
@@ -691,7 +689,6 @@ def test_create_create_transaction_single_io(user_output, user_pub, data):
     tx_dict = tx.to_dict()
     tx_dict['inputs'][0]['fulfillment'] = None
     tx_dict.pop('id')
-    tx_dict['asset'].pop('id')
 
     assert tx_dict == expected
 
@@ -775,7 +772,6 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub,
                             metadata=data, asset=data)
     tx_dict = tx.to_dict()
     tx_dict.pop('id')
-    tx_dict['asset'].pop('id')
     tx_dict['inputs'][0]['fulfillment'] = None
 
     assert tx_dict == expected
@@ -989,25 +985,3 @@ def test_validate_version(utx):
     utx.version = '1.0.0'
     with raises(SchemaValidationError):
         validate_transaction_model(utx)
-
-
-def test_create_tx_has_asset_id(tx):
-    tx = tx.to_dict()
-    assert tx['id'] == tx['asset']['id']
-
-
-def test_create_tx_validates_asset_id(tx):
-    from bigchaindb.common.transaction import Transaction
-    from bigchaindb.common.exceptions import InvalidHash
-
-    tx = tx.to_dict()
-
-    # Test fails with wrong asset_id
-    tx['asset']['id'] = tx['asset']['id'][::-1]
-    with raises(InvalidHash):
-        Transaction.from_dict(tx)
-
-    # Test fails with no asset_id
-    tx['asset'].pop('id')
-    with raises(InvalidHash):
-        Transaction.from_dict(tx)

tests/db/test_bigchain_api.py

@@ -1192,3 +1192,33 @@ def test_get_outputs_filtered():
     get_outputs.assert_called_once_with('abc')
     get_spent.assert_not_called()
     assert out == get_outputs.return_value
+
+
+@pytest.mark.bdb
+def test_cant_spend_same_input_twice_in_tx(b, genesis_block):
+    """
+    Recreate duplicated fulfillments bug
+    https://github.com/bigchaindb/bigchaindb/issues/1099
+    """
+    from bigchaindb.models import Transaction
+    from bigchaindb.common.exceptions import DoubleSpend
+
+    # create a divisible asset
+    tx_create = Transaction.create([b.me], [([b.me], 100)])
+    tx_create_signed = tx_create.sign([b.me_private])
+    assert b.validate_transaction(tx_create_signed) == tx_create_signed
+
+    # create a block and valid vote
+    block = b.create_block([tx_create_signed])
+    b.write_block(block)
+    vote = b.vote(block.id, genesis_block.id, True)
+    b.write_vote(vote)
+
+    # Create a transfer transaction with duplicated fulfillments
+    dup_inputs = tx_create.to_inputs() + tx_create.to_inputs()
+    tx_transfer = Transaction.transfer(dup_inputs, [([b.me], 200)],
+                                       asset_id=tx_create.id)
+    tx_transfer_signed = tx_transfer.sign([b.me_private])
+    assert b.is_valid_transaction(tx_transfer_signed) is False
+    with pytest.raises(DoubleSpend):
+        tx_transfer_signed.validate(b)

tests/web/test_parameters.py

@@ -24,11 +24,9 @@ def test_valid_bool():
 
     assert valid_bool('true') is True
     assert valid_bool('false') is False
+    assert valid_bool('tRUE') is True
+    assert valid_bool('fALSE') is False
 
-    with pytest.raises(ValueError):
-        valid_bool('TRUE')
-    with pytest.raises(ValueError):
-        valid_bool('FALSE')
     with pytest.raises(ValueError):
         valid_bool('0')
     with pytest.raises(ValueError):
@@ -64,13 +62,11 @@ def test_valid_ed25519():
 def test_valid_operation():
     from bigchaindb.web.views.parameters import valid_operation
 
-    assert valid_operation('CREATE') == 'CREATE'
+    assert valid_operation('create') == 'CREATE'
-    assert valid_operation('TRANSFER') == 'TRANSFER'
+    assert valid_operation('transfer') == 'TRANSFER'
+    assert valid_operation('CREATe') == 'CREATE'
+    assert valid_operation('TRANSFEr') == 'TRANSFER'
 
-    with pytest.raises(ValueError):
-        valid_operation('create')
-    with pytest.raises(ValueError):
-        valid_operation('transfer')
     with pytest.raises(ValueError):
         valid_operation('GENESIS')
     with pytest.raises(ValueError):