Integrate blocks API (#1970)

* Integrate blocks api

* Update docs

* Fix docs

* Fixed mismatch between code and documentation

* Fixed docs

bigchaindb/backend/localmongodb/query.py

@@ -32,6 +32,17 @@ def get_transaction(conn, transaction_id):
         pass
 
 
+@register_query(LocalMongoDBConnection)
+def get_transactions(conn, transaction_ids):
+    try:
+        return conn.run(
+            conn.collection('transactions')
+            .find({'id': {'$in': transaction_ids}},
+                  projection={'_id': False}))
+    except IndexError:
+        pass
+
+
 @register_query(LocalMongoDBConnection)
 def store_metadata(conn, metadata):
     try:
@@ -153,3 +164,11 @@ def get_spending_transactions(conn, inputs):
             {'$project': {'_id': False}}
         ]))
     return cursor
+
+
+@register_query(LocalMongoDBConnection)
+def get_block(conn, block_id):
+    return conn.run(
+        conn.collection('blocks')
+        .find_one({'height': block_id},
+                  projection={'_id': False}))

bigchaindb/backend/query.py

@@ -68,6 +68,20 @@ def get_transaction(connection, transaction_id):
     raise NotImplementedError
 
 
+@singledispatch
+def get_transactions(connection, transaction_ids):
+    """Get transactions from the transactions table.
+
+    Args:
+        transaction_ids (list): list of transaction ids to fetch
+
+    Returns:
+        The result of the operation.
+    """
+
+    raise NotImplementedError
+
+
 @singledispatch
 def get_asset(connection, asset_id):
     """Get a transaction from the transactions table.

bigchaindb/models.py

@@ -96,7 +96,7 @@ class Transaction(Transaction):
         return super().from_dict(tx_body)
 
     @classmethod
-    def from_db(cls, bigchain, tx_dict):
+    def from_db(cls, bigchain, tx_dict_list):
         """Helper method that reconstructs a transaction dict that was returned
         from the database. It checks what asset_id to retrieve, retrieves the
         asset from the asset table and reconstructs the transaction.
@@ -104,29 +104,46 @@ class Transaction(Transaction):
         Args:
             bigchain (:class:`~bigchaindb.Bigchain`): An instance of Bigchain
                 used to perform database queries.
-            tx_dict (:obj:`dict`): The transaction dict as returned from the
-                database.
+            tx_dict_list (:list:`dict` or :obj:`dict`): The transaction dict or
+                list of transaction dict as returned from the database.
 
         Returns:
             :class:`~Transaction`
 
         """
-        if tx_dict['operation'] in [Transaction.CREATE, Transaction.GENESIS]:
-            # TODO: Maybe replace this call to a call to get_asset_by_id
-            asset = list(bigchain.get_assets([tx_dict['id']]))[0]
+        return_list = True
+        if isinstance(tx_dict_list, dict):
+            tx_dict_list = [tx_dict_list]
+            return_list = False
+
+        tx_map = {}
+        tx_ids = []
+        for tx in tx_dict_list:
+            tx.update({'metadata': None})
+            tx_map[tx['id']] = tx
+            if tx['operation'] in [Transaction.CREATE, Transaction.GENESIS]:
+                tx_ids.append(tx['id'])
+
+        assets = list(bigchain.get_assets(tx_ids))
+        for asset in assets:
+            tx = tx_map[asset['id']]
             del asset['id']
-            tx_dict.update({'asset': asset})
+            tx.update({'asset': asset})
 
-        # get metadata of the transaction
-        metadata = list(bigchain.get_metadata([tx_dict['id']]))
-        if 'metadata' not in tx_dict:
-            metadata = metadata[0] if metadata else None
-            if metadata:
-                metadata = metadata.get('metadata')
+        tx_ids = list(tx_map.keys())
+        metadata_list = list(bigchain.get_metadata(tx_ids))
+        for metadata in metadata_list:
+            tx = tx_map[metadata['id']]
+            tx.update({'metadata': metadata.get('metadata')})
 
-            tx_dict.update({'metadata': metadata})
-
-        return cls.from_dict(tx_dict)
+        if return_list:
+            tx_list = []
+            for tx_id, tx in tx_map.items():
+                tx_list.append(cls.from_dict(tx))
+            return tx_list
+        else:
+            tx = list(tx_map.values())[0]
+            return cls.from_dict(tx)
 
 
 class Block(object):

bigchaindb/tendermint/core.py

@@ -31,7 +31,7 @@ class App(BaseApplication):
     def init_chain(self, validators):
         """Initialize chain with block of height 0"""
 
-        block = Block(app_hash='', height=0)
+        block = Block(app_hash='', height=0, transactions=[])
         self.bigchaindb.store_block(block._asdict())
 
     def info(self):
@@ -112,7 +112,9 @@ class App(BaseApplication):
 
         # register a new block only when new transactions are received
         if self.block_txn_ids:
-            block = Block(app_hash=self.block_txn_hash, height=self.new_height)
+            block = Block(app_hash=self.block_txn_hash,
+                          height=self.new_height,
+                          transactions=self.block_txn_ids)
             self.bigchaindb.store_block(block._asdict())
 
         data = self.block_txn_hash.encode('utf-8')

bigchaindb/tendermint/lib.py

@@ -116,6 +116,42 @@ class BigchainDB(Bigchain):
 
         return backend.query.get_latest_block(self.connection)
 
+    def get_block(self, block_id, include_status=False):
+        """Get the block with the specified `block_id` (and optionally its status)
+
+        Returns the block corresponding to `block_id` or None if no match is
+        found.
+
+        Args:
+            block_id (str): block id of the block to get
+            include_status (bool): also return the status of the block
+                       the return value is then a tuple: (block, status)
+        """
+        # get block from database
+        if isinstance(block_id, str):
+            block_id = int(block_id)
+
+        block = backend.query.get_block(self.connection, block_id)
+        if block:
+            transactions = backend.query.get_transactions(self.connection, block['transactions'])
+            transactions = Transaction.from_db(self, transactions)
+
+            block = {'height': block['height'],
+                     'transactions': []}
+            block_txns = block['transactions']
+            for txn in transactions:
+                block_txns.append(txn.to_dict())
+
+        status = None
+        if include_status:
+            # NOTE: (In Tendermint) a block is an abstract entity which
+            # exists only after it has been validated
+            if block:
+                status = self.BLOCK_VALID
+            return block, status
+        else:
+            return block
+
     def validate_transaction(self, tx):
         """Validate a transaction against the current status of the database."""
 
@@ -142,4 +178,4 @@ class BigchainDB(Bigchain):
         return fastquery.FastQuery(self.connection, self.me)
 
 
-Block = namedtuple('Block', ('app_hash', 'height'))
+Block = namedtuple('Block', ('app_hash', 'height', 'transactions'))

docs/server/generate_http_server_api_documentation.py

@@ -7,6 +7,7 @@ import os.path
 from bigchaindb.common.transaction import Transaction, Input, TransactionLink
 from bigchaindb.core import Bigchain
 from bigchaindb.models import Block
+from bigchaindb.tendermint import lib
 from bigchaindb.web import server
 
 
@@ -242,18 +243,24 @@ def main():
     node_private = "5G2kE1zJAgTajkVSbPAQWo4c2izvtwqaNHYsaNpbbvxX"
     node_public = "DngBurxfeNVKZWCEcDnLj1eMPAS7focUZTE5FndFGuHT"
     signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA"
-    block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature)
-    ctx['block'] = pretty_json(block.to_dict())
-    ctx['blockid'] = block.id
 
+    app_hash = 'f6e0c49c6d94d6924351f25bb334cf2a99af4206339bf784e741d1a5ab599056'
+    block = lib.Block(height=1, transactions=[tx.to_dict()], app_hash=app_hash)
+    block_dict = block._asdict()
+    block_dict.pop('app_hash')
+    ctx['block'] = pretty_json(block_dict)
+    ctx['blockid'] = block.height
+
+    block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature)
     block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public,
                            voters=[node_public], signature=signature)
-    ctx['block_transfer'] = pretty_json(block.to_dict())
+    ctx['block_transfer'] = pretty_json(block_transfer.to_dict())
 
     # vote
+    vblock = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature)
     DUMMY_SHA3 = '0123456789abcdef' * 4
     b = Bigchain(public_key=node_public, private_key=node_private)
-    vote = b.vote(block.id, DUMMY_SHA3, True)
+    vote = b.vote(vblock.id, DUMMY_SHA3, True)
     ctx['vote'] = pretty_json(vote)
 
     # block status

docs/server/source/data-models/block-model.rst

@@ -9,82 +9,23 @@ A block must contain the following JSON keys
 .. code-block:: json
 
     {
-      "id": "<ID of the block>",
-      "block": {
-        "timestamp": "<Block-creation timestamp>",
-        "transactions": ["<List of transactions>"],
-        "node_pubkey": "<Public key of the node which created the block>",
-        "voters": ["<List of public keys of all nodes in the cluster>"]
-      },
-      "signature": "<Signature of inner block object>"
+      "height": "<Height of the block>",
+      "transactions": ["<List of transactions>"]
     }
 
 
 The JSON Keys in a Block
 ------------------------
 
-**id**
+**height**
 
-The transaction ID and also the SHA3-256 hash
-of the inner ``block`` object, loosely speaking.
-It's a string.
-To compute it, 1) construct an :term:`associative array` ``d`` containing
-``block.timestamp``, ``block.transactions``, ``block.node_pubkey``,
-``block.voters``, and their values. 2) compute ``id = hash_of_aa(d)``.
-There's pseudocode for the ``hash_of_aa()`` function
-in the `IPDB Transaction Spec page about cryptographic hashes
-<https://the-ipdb-transaction-spec.readthedocs.io/en/latest/common-operations/crypto-hashes.html#computing-the-hash-of-an-associative-array>`_.
-The result (``id``) is a string: the block ID.
-An example is ``"b60adf655932bf47ef58c0bfb2dd276d4795b94346b36cbb477e10d7eb02cea8"``
+The block ``"height"`` (``integer``) denotes the height of the blockchain when the given block was committed.
+Since the blockchain height increases monotonically the height of block can be regarded as its id.
+
+**NOTE**: The genesis block has height ``0``
 
 
-**block.timestamp**
-
-The `Unix time <https://en.wikipedia.org/wiki/Unix_time>`_
-when the block was created, according to the node which created it.
-It's a string representation of an integer.
-An example is ``"1507294217"``.
-
-
-**block.transactions**
+**transactions**
 
 A list of the :ref:`transactions <The Transaction Model>` included in the block.
 (Each transaction is a JSON object.)
-
-
-**block.node_pubkey**
-
-The public key of the node that created the block.
-It's a string.
-See the `IPDB Transaction Spec page about cryptographic keys & signatures
-<https://the-ipdb-transaction-spec.readthedocs.io/en/latest/common-operations/crypto-keys-and-sigs.html>`_.
-
-
-**block.voters**
-
-A list of the public keys of all cluster nodes at the time the block was created.
-It's a list of strings.
-This list can change from block to block, as nodes join and leave the cluster.
-
-
-**signature**
-
-The cryptographic signature of the inner ``block``
-by the node that created the block
-(i.e. the node with public key ``node_pubkey``).
-To compute that:
-
-#. Construct an :term:`associative array` ``d`` containing the contents
-   of the inner ``block``
-   (i.e. ``block.timestamp``, ``block.transactions``, ``block.node_pubkey``,
-   ``block.voters``, and their values).
-#. Compute ``signature = sig_of_aa(d, private_key)``,
-   where ``private_key`` is the node's private key
-   (i.e. ``node_pubkey`` and ``private_key`` are a key pair). There's pseudocode
-   for the ``sig_of_aa()`` function
-   on `the IPDB Transaction Spec page about cryptographic keys and signatures 
-   <https://the-ipdb-transaction-spec.readthedocs.io/en/latest/common-operations/crypto-keys-and-sigs.html#computing-the-signature-of-an-associative-array>`_.
-
-.. note::
-
-   The ``d_bytes`` computed when computing the block ID will be the *same* as the ``d_bytes`` computed when computing the block signature. This can be used to avoid redundant calculations.

docs/server/source/http-client-server-api.rst

@@ -597,14 +597,12 @@ The `votes endpoint <#votes>`_ contains all the voting information for a specifi
 Blocks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. http:get:: /api/v1/blocks/{block_id}
+.. http:get:: /api/v1/blocks/{block_height}
 
-   Get the block with the ID ``block_id``. Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be
-   returned. To check a block's status independently, use the `Statuses endpoint <#status>`_.
-   To check the votes on a block, have a look at the `votes endpoint <#votes>`_.
+   Get the block with the height ``block_height``.
 
-   :param block_id: block ID
-   :type block_id: hex string
+   :param block_height: block ID
+   :type block_height: integer
 
    **Example request**:
 
@@ -620,7 +618,7 @@ Blocks
    :resheader Content-Type: ``application/json``
 
    :statuscode 200: A block with that ID was found.
-   :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``.
+   :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_height``.
    :statuscode 404: A block with that ID was not found.
 
 

tests/backend/localmongodb/test_queries.py

@@ -152,3 +152,31 @@ def test_get_spending_transactions(user_pk, user_sk):
 
     # tx3 not a member because input 1 not asked for
     assert txns == [tx2.to_dict(), tx4.to_dict()]
+
+
+def test_store_block():
+    from bigchaindb.backend import connect, query
+    from bigchaindb.tendermint.lib import Block
+    conn = connect()
+
+    block = Block(app_hash='random_utxo',
+                  height=3,
+                  transactions=[])
+    query.store_block(conn, block._asdict())
+    cursor = conn.db.blocks.find({}, projection={'_id': False})
+    assert cursor.count() == 1
+
+
+def test_get_block():
+    from bigchaindb.backend import connect, query
+    from bigchaindb.tendermint.lib import Block
+    conn = connect()
+
+    block = Block(app_hash='random_utxo',
+                  height=3,
+                  transactions=[])
+
+    conn.db.blocks.insert_one(block._asdict())
+
+    block = dict(query.get_block(conn, 3))
+    assert block['height'] == 3

tests/tendermint/test_integration.py

@@ -12,12 +12,13 @@ pytestmark = pytest.mark.tendermint
 
 
 @pytest.mark.bdb
-def test_app(b):
+def test_app(tb):
     from bigchaindb.tendermint import App
     from bigchaindb.tendermint.utils import calculate_hash
     from bigchaindb.common.crypto import generate_key_pair
     from bigchaindb.models import Transaction
 
+    b = tb
     app = App(b)
     p = ProtocolHandler(app)
 

tests/tendermint/test_lib.py

@@ -37,12 +37,15 @@ def test_asset_is_separated_from_transaciton(b):
     assert b.get_transaction(tx.id) == tx
 
 
-def test_get_latest_block(b):
+def test_get_latest_block(tb):
     from bigchaindb.tendermint.lib import Block
 
+    b = tb
     for i in range(10):
         app_hash = os.urandom(16).hex()
-        block = Block(app_hash=app_hash, height=i)._asdict()
+        txn_id = os.urandom(16).hex()
+        block = Block(app_hash=app_hash, height=i,
+                      transactions=[txn_id])._asdict()
         b.store_block(block)
 
     block = b.get_latest_block()

tests/utils.py

@@ -4,6 +4,7 @@ import rethinkdb as r
 
 from bigchaindb.backend.mongodb.connection import MongoDBConnection
 from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection
+from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection
 
 
 @singledispatch
@@ -47,6 +48,15 @@ def flush_mongo_db(connection, dbname):
     connection.conn[dbname].metadata.delete_many({})
 
 
+@flush_db.register(LocalMongoDBConnection)
+def flush_localmongo_db(connection, dbname):
+    connection.conn[dbname].bigchain.delete_many({})
+    connection.conn[dbname].blocks.delete_many({})
+    connection.conn[dbname].transactions.delete_many({})
+    connection.conn[dbname].assets.delete_many({})
+    connection.conn[dbname].metadata.delete_many({})
+
+
 @singledispatch
 def update_table_config(connection, table, **kwrgas):
     raise NotImplementedError

tests/web/test_block_tendermint.py

@@ -0,0 +1,37 @@
+import pytest
+
+from bigchaindb.models import Transaction
+from bigchaindb.tendermint.lib import Block
+
+BLOCKS_ENDPOINT = '/api/v1/blocks/'
+
+pytestmark = pytest.mark.tendermint
+
+
+@pytest.mark.bdb
+@pytest.mark.usefixtures('inputs')
+def test_get_block_endpoint(tb, client):
+    b = tb
+    tx = Transaction.create([b.me], [([b.me], 1)], asset={'cycle': 'hero'})
+    tx = tx.sign([b.me_private])
+    b.store_transaction(tx)
+
+    block = Block(app_hash='random_utxo',
+                  height=31,
+                  transactions=[tx.id])
+    b.store_block(block._asdict())
+
+    res = client.get(BLOCKS_ENDPOINT + str(block.height))
+    expected_response = {'height': block.height, 'transactions': [tx.to_dict()]}
+    assert res.json == expected_response
+    assert res.status_code == 200
+
+
+@pytest.mark.bdb
+@pytest.mark.usefixtures('inputs')
+def test_get_block_returns_404_if_not_found(client):
+    res = client.get(BLOCKS_ENDPOINT + '123')
+    assert res.status_code == 404
+
+    res = client.get(BLOCKS_ENDPOINT + '123/')
+    assert res.status_code == 404