Return commit hash to tendermint (#1851)

* Aggregate transaction ids for commit hash

* Setup chain and return commit hash to tendermint

* Fix function naming

bigchaindb/backend/localmongodb/query.py

@@ -4,6 +4,7 @@ from bigchaindb import backend
 from bigchaindb.backend.exceptions import DuplicateKeyError
 from bigchaindb.backend.utils import module_dispatch_registrar
 from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection
+from pymongo import DESCENDING
 
 
 register_query = module_dispatch_registrar(backend.query)
@@ -59,3 +60,20 @@ def get_spent(conn, transaction_id, output):
                       {'_id': 0}))
     except IndexError:
         pass
+
+
+@register_query(LocalMongoDBConnection)
+def get_latest_block(conn):
+    return conn.run(
+        conn.collection('blocks')
+        .find_one(sort=[('height', DESCENDING)]))
+
+
+@register_query(LocalMongoDBConnection)
+def store_block(conn, block):
+    try:
+        return conn.run(
+            conn.collection('blocks')
+            .insert_one(block))
+    except DuplicateKeyError:
+        pass

bigchaindb/backend/localmongodb/schema.py

@@ -2,7 +2,7 @@
 
 import logging
 
-from pymongo import ASCENDING, TEXT
+from pymongo import ASCENDING, DESCENDING, TEXT
 
 from bigchaindb import backend
 from bigchaindb.common import exceptions
@@ -27,7 +27,7 @@ def create_database(conn, dbname):
 
 @register_schema(LocalMongoDBConnection)
 def create_tables(conn, dbname):
-    for table_name in ['transactions', 'assets']:
+    for table_name in ['transactions', 'assets', 'blocks']:
         logger.info('Create `%s` table.', table_name)
         # create the table
         # TODO: read and write concerns can be declared here
@@ -38,6 +38,7 @@ def create_tables(conn, dbname):
 def create_indexes(conn, dbname):
     create_transactions_secondary_index(conn, dbname)
     create_assets_secondary_index(conn, dbname)
+    create_blocks_secondary_index(conn, dbname)
 
 
 @register_schema(LocalMongoDBConnection)
@@ -80,3 +81,8 @@ def create_assets_secondary_index(conn, dbname):
 
     # full text search index
     conn.conn[dbname]['assets'].create_index([('$**', TEXT)], name='text')
+
+
+def create_blocks_secondary_index(conn, dbname):
+    conn.conn[dbname]['blocks']\
+        .create_index([('height', DESCENDING)], name='height')

bigchaindb/backend/query.py

@@ -438,3 +438,24 @@ def text_search(conn, search, *, language='english', case_sensitive=False,
 
     raise OperationError('This query is only supported when running '
                          'BigchainDB with MongoDB as the backend.')
+
+
+@singledispatch
+def get_latest_block(conn):
+    """Get the latest commited block i.e. block with largest height """
+
+    raise NotImplementedError
+
+
+@singledispatch
+def store_block(conn, block):
+    """Write a new block to the `blocks` table
+
+    Args:
+        block (dict): block with current height and block hash.
+
+    Returns:
+        The result of the operation.
+    """
+
+    raise NotImplementedError

bigchaindb/tendermint/core.py

@@ -1,11 +1,12 @@
 """This module contains all the goodness to integrate BigchainDB
 with Tendermint."""
 
-
 from abci import BaseApplication, Result
+from abci.types_pb2 import ResponseEndBlock, ResponseInfo
 
 from bigchaindb.tendermint import BigchainDB
-from bigchaindb.tendermint.utils import decode_transaction
+from bigchaindb.tendermint.utils import decode_transaction, calculate_hash
+from bigchaindb.tendermint.lib import Block
 
 
 class App(BaseApplication):
@@ -19,6 +20,29 @@ class App(BaseApplication):
         if not bigchaindb:
             bigchaindb = BigchainDB()
         self.bigchaindb = bigchaindb
+        self.block_txn_ids = []
+        self.block_txn_hash = ''
+        self.validators = None
+        self.new_height = None
+
+    def init_chain(self, validators):
+        """Initialize chain with block of height 0"""
+
+        block = Block(hash='', height=0)
+        self.bigchaindb.store_block(block.to_dict())
+
+    def info(self):
+        """Return height of the latest committed block."""
+
+        r = ResponseInfo()
+        block = self.bigchaindb.get_latest_block()
+        if block:
+            r.last_block_height = block['height']
+            r.last_block_app_hash = block['hash'].encode('utf-8')
+        else:
+            r.last_block_height = 0
+            r.last_block_app_hash = b''
+        return r
 
     def check_tx(self, raw_transaction):
         """Validate the transaction before entry into
@@ -33,12 +57,16 @@ class App(BaseApplication):
         else:
             return Result.error()
 
+    def begin_block(self, block_hash, header):
+        """Initialize list of transaction."""
+
+        self.block_txn_ids = []
+
     def deliver_tx(self, raw_transaction):
         """Validate the transaction before mutating the state.
 
         Args:
             raw_tx: a raw string (in bytes) transaction."""
-
         transaction = self.bigchaindb.validate_transaction(
                 decode_transaction(raw_transaction))
 
@@ -46,4 +74,34 @@ class App(BaseApplication):
             return Result.error(log='Invalid transaction')
         else:
             self.bigchaindb.store_transaction(transaction)
+            self.block_txn_ids.append(transaction.id)
             return Result.ok()
+
+    def end_block(self, height):
+        """Calculate block hash using transaction ids and previous block
+        hash to be stored in the next block.
+
+        Args:
+            height (int): new height of the chain."""
+
+        self.new_height = height
+        block_txn_hash = calculate_hash(self.block_txn_ids)
+        block = self.bigchaindb.get_latest_block()
+
+        if self.block_txn_ids:
+            self.block_txn_hash = calculate_hash([block['hash'], block_txn_hash])
+        else:
+            self.block_txn_hash = block['hash']
+
+        return ResponseEndBlock()
+
+    def commit(self):
+        """Store the new height and along with block hash."""
+
+        # register a new block only when new transactions are received
+        if self.block_txn_ids:
+            block = Block(hash=self.block_txn_hash, height=self.new_height)
+            self.bigchaindb.store_block(block.to_dict())
+
+        data = self.block_txn_hash.encode('utf-8')
+        return Result.ok(data=data)

bigchaindb/tendermint/lib.py

@@ -70,6 +70,16 @@ class BigchainDB(Bigchain):
                                               output)
         return Transaction.from_dict(transaction)
 
+    def store_block(self, block):
+        """Create a new block."""
+
+        return backend.query.store_block(self.connection, block)
+
+    def get_latest_block(self):
+        """Get the block with largest height."""
+
+        return backend.query.get_latest_block(self.connection)
+
     def validate_transaction(self, tx):
         """Validate a transaction against the current status of the database."""
 
@@ -90,3 +100,15 @@ class BigchainDB(Bigchain):
             logger.warning('Invalid transaction (%s): %s', type(e).__name__, e)
             return False
         return transaction
+
+
+class Block(object):
+
+    def __init__(self, hash='', height=0):
+        self.hash = hash
+        self.height = height
+
+    def to_dict(self):
+        block = {'hash': self.hash,
+                 'height': self.height}
+        return block

bigchaindb/tendermint/utils.py

@@ -1,5 +1,6 @@
 import base64
 import json
+import sha3
 
 
 def encode_transaction(value):
@@ -12,3 +13,14 @@ def decode_transaction(raw):
     """Decode a transaction from Base64 to a dict."""
 
     return json.loads(raw.decode('utf8'))
+
+
+def calculate_hash(key_list):
+    if not key_list:
+        return ''
+
+    full_hash = sha3.sha3_256()
+    for key in key_list:
+        full_hash.update(key.encode('utf8'))
+
+    return full_hash.hexdigest()

tests/tendermint/test_integration.py

@@ -0,0 +1,78 @@
+import json
+import pytest
+
+
+@pytest.mark.bdb
+@pytest.mark.tapp
+def test_app(b):
+    from bigchaindb.tendermint import App
+    from bigchaindb.tendermint.utils import calculate_hash
+    from abci.server import ProtocolHandler
+    from io import BytesIO
+    import abci.types_pb2 as types
+    from abci.wire import read_message
+    from bigchaindb.common.crypto import generate_key_pair
+    from bigchaindb.models import Transaction
+    from abci.messages import to_request_deliver_tx, to_request_check_tx
+
+    app = App(b)
+    p = ProtocolHandler(app)
+
+    data = p.process('info', None)
+    res, err = read_message(BytesIO(data), types.Response)
+    assert res
+    assert res.info.last_block_app_hash == b''
+    assert res.info.last_block_height == 0
+    assert not b.get_latest_block()
+
+    p.process('init_chain', None)
+    block0 = b.get_latest_block()
+    assert block0
+    assert block0['height'] == 0
+    assert block0['hash'] == ''
+
+    alice = generate_key_pair()
+    bob = generate_key_pair()
+    tx = Transaction.create([alice.public_key],
+                            [([bob.public_key], 1)])\
+                    .sign([alice.private_key])
+    etxn = json.dumps(tx.to_dict()).encode('utf8')
+
+    r = to_request_check_tx(etxn)
+    data = p.process('check_tx', r)
+    res, err = read_message(BytesIO(data), types.Response)
+    assert res
+    assert res.check_tx.code == 0
+
+    r = types.Request()
+    r.begin_block.hash = b''
+    p.process('begin_block', r)
+
+    r = to_request_deliver_tx(etxn)
+    data = p.process('deliver_tx', r)
+    res, err = read_message(BytesIO(data), types.Response)
+    assert res
+    assert res.deliver_tx.code == 0
+    assert b.get_transaction(tx.id).id == tx.id
+
+    new_block_txn_hash = calculate_hash([tx.id])
+
+    r = types.Request()
+    r.end_block.height = 1
+    data = p.process('end_block', r)
+    res, err = read_message(BytesIO(data), types.Response)
+    assert res
+    assert 'end_block' == res.WhichOneof("value")
+
+    new_block_hash = calculate_hash([block0['hash'], new_block_txn_hash])
+
+    data = p.process('commit', None)
+    res, err = read_message(BytesIO(data), types.Response)
+    assert res
+    assert res.commit.code == 0
+    assert res.commit.data == new_block_hash.encode('utf-8')
+
+    block0 = b.get_latest_block()
+    assert block0
+    assert block0['height'] == 1
+    assert block0['hash'] == new_block_hash