From 779d31ef3e27a467a95afc4f16c148542162119a Mon Sep 17 00:00:00 2001 From: Roman Sokolkov Date: Sat, 30 Dec 2017 17:31:55 +0100 Subject: [PATCH 01/22] Fix issue with STORAGE_ENGINE_CACHE_SIZE parameter In mongodb docker image STORAGE_ENGINE_CACHE_SIZE will always be set to '' (two quotation marks) and verification will fail. So it's impossible to run image without --storage-engine-cache-size set. This change removes double quotes while assigning default value to STORAGE_ENGINE_CACHE_SIZE. --- k8s/mongodb/container/mongod_entrypoint.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index 213f9989..bde5259a 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -66,7 +66,7 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ - -z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then + -z ${STORAGE_ENGINE_CACHE_SIZE:=''} ]] ; then #-z "${MONGODB_KEY_FILE_PASSWORD:?MongoDB Key File Password not specified. Exiting!}" || \ exit 1 else From e2316a3de857e67a44a932e92cac28f090bf78a7 Mon Sep 17 00:00:00 2001 From: muawiakh Date: Thu, 4 Jan 2018 13:58:28 +0100 Subject: [PATCH 02/22] Fix error checking of optional parameter `storage-engine-cache-size` - Keep the parameter optional - Remove redundant checking of the same parameter - Do not exit if the parameter is not specified --- k8s/mongodb/container/Dockerfile | 2 +- k8s/mongodb/container/docker_build_and_push.bash | 4 ++-- k8s/mongodb/container/mongod_entrypoint.bash | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile index 9d9d35f5..cf4aad42 100644 --- a/k8s/mongodb/container/Dockerfile +++ b/k8s/mongodb/container/Dockerfile @@ -1,4 +1,4 @@ -FROM mongo:3.4.4 +FROM mongo:3.4.10 LABEL maintainer "dev@bigchaindb.com" WORKDIR / RUN apt-get update \ diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash index 680fab22..bb48dd2f 100755 --- a/k8s/mongodb/container/docker_build_and_push.bash +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/mongodb:3.2 . +docker build -t bigchaindb/mongodb:3.3 . -docker push bigchaindb/mongodb:3.2 +docker push bigchaindb/mongodb:3.3 diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index bde5259a..e8516a82 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -65,9 +65,8 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ - -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ - -z ${STORAGE_ENGINE_CACHE_SIZE:=''} ]] ; then - #-z "${MONGODB_KEY_FILE_PASSWORD:?MongoDB Key File Password not specified. Exiting!}" || \ + -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then + echo "Missing required enviroment variable(s)." exit 1 else echo REPLICA_SET_NAME="$REPLICA_SET_NAME" From 9d45f735fd48a9da4d398d8161a5f29fb9539bce Mon Sep 17 00:00:00 2001 From: muawiakh Date: Thu, 4 Jan 2018 14:53:23 +0100 Subject: [PATCH 03/22] Improving code readability --- k8s/mongodb/container/mongod_entrypoint.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index e8516a82..6a6fe309 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -66,6 +66,9 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then + # Not handling the STORAGE_ENGINE_CACHE_SIZE because + # it is optional. If not specified the default cache + # size is: max((50% RAM - 1GB), 256MB) echo "Missing required enviroment variable(s)." exit 1 else From 06a091e2157a136195873ed9170a2b8ab4f44b9a Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Thu, 11 Jan 2018 13:26:06 +0100 Subject: [PATCH 04/22] add mode parameter to transaction endpoint, fixes #1995 --- bigchaindb/core.py | 2 +- bigchaindb/tendermint/lib.py | 16 +++++--- bigchaindb/web/views/parameters.py | 12 +++++- bigchaindb/web/views/transactions.py | 9 ++++- docs/server/source/http-client-server-api.rst | 23 ++++++++++++ tests/tendermint/test_lib.py | 37 ++++++++++++++++++- tests/web/test_transactions.py | 36 ++++++++++++++++++ 7 files changed, 126 insertions(+), 9 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 4fb84722..fae16e9d 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -77,7 +77,7 @@ class Bigchain(object): federation = property(lambda self: set(self.nodes_except_me + [self.me])) """ Set of federation member public keys """ - def write_transaction(self, signed_transaction): + def write_transaction(self, signed_transaction, **kwargs): """Write the transaction to bigchain. When first writing a transaction to the bigchain the transaction will be kept in a backlog until diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 37fb9d6e..3cf93ce5 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -23,11 +23,17 @@ ENDPOINT = 'http://{}:{}/'.format(TENDERMINT_HOST, TENDERMINT_PORT) class BigchainDB(Bigchain): - def post_transaction(self, transaction): + def post_transaction(self, transaction, mode): """Submit a valid transaction to the mempool.""" + mode_list = ('broadcast_tx_async', + 'broadcast_tx_sync', + 'broadcast_tx_commit') + if not mode or mode['mode'] not in mode_list: + raise ValidationError(('Mode must be one of the following {}.') + .format(', '.join(mode_list))) payload = { - 'method': 'broadcast_tx_async', + 'method': mode['mode'], 'jsonrpc': '2.0', 'params': [encode_transaction(transaction.to_dict())], 'id': str(uuid4()) @@ -35,11 +41,11 @@ class BigchainDB(Bigchain): # TODO: handle connection errors! requests.post(ENDPOINT, json=payload) - def write_transaction(self, transaction): + def write_transaction(self, transaction, **kwargs): # This method offers backward compatibility with the Web API. """Submit a valid transaction to the mempool.""" - - self.post_transaction(transaction) + mode = kwargs + self.post_transaction(transaction, mode) def store_transaction(self, transaction): """Store a valid transaction to the transactions collection.""" diff --git a/bigchaindb/web/views/parameters.py b/bigchaindb/web/views/parameters.py index 6eca4a97..58eb8a5f 100644 --- a/bigchaindb/web/views/parameters.py +++ b/bigchaindb/web/views/parameters.py @@ -29,4 +29,14 @@ def valid_operation(op): return 'CREATE' if op == 'TRANSFER': return 'TRANSFER' - raise ValueError('Operation must be "CREATE" or "TRANSFER') + raise ValueError('Operation must be "CREATE" or "TRANSFER"') + + +def valid_mode(mode): + if mode == 'async': + return 'broadcast_tx_async' + if mode == 'sync': + return 'broadcast_tx_sync' + if mode == 'commit': + return 'broadcast_tx_commit' + raise ValueError('Mode must be "async", "sync" or "commit"') diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index 39c6d529..37912140 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -55,6 +55,13 @@ class TransactionListApi(Resource): Return: A ``dict`` containing the data about the transaction. """ + parser = reqparse.RequestParser() + parser.add_argument('mode', type=parameters.valid_mode, + choices=['async', 'sync', 'commit']) + args = parser.parse_args() + if not str(args['mode']) or str(args['mode']) == 'None': + args['mode'] = 'broadcast_tx_async' + pool = current_app.config['bigchain_pool'] # `force` will try to format the body of the POST request even if the @@ -85,7 +92,7 @@ class TransactionListApi(Resource): 'Invalid transaction ({}): {}'.format(type(e).__name__, e) ) else: - bigchain.write_transaction(tx_obj) + bigchain.write_transaction(tx_obj, **args) response = jsonify(tx) response.status_code = 202 diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 58ec5617..fce27c15 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -137,6 +137,10 @@ Transactions `_ to build a valid transaction. + A generalization of the parameter follows: + + :param mode: (Optional) Set the broadcast method. + **Example request**: .. literalinclude:: http-samples/post-tx-request.http @@ -161,6 +165,25 @@ Transactions :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. +.. http:post:: /api/v1/transactions?mode={mode} + + .. note:: + + This option is only available when using BigchainDB with Tendermint. + + Tendermint offers a `broadcast API + `_ with three different modes to send transactions. + By setting the mode, a new transaction can be pushed with a different mode than the default. The default mode is ``broadcast_tx_async``, which + will return immediately and not wait to see if the transaction is valid. + + :query string mode: (Optional) One of the three supported modes to send a transaction: ``broadcast_tx_async``, ``broadcast_tx_sync``, ``broadcast_tx_commit``. + + The only change for a request, compared to the one above, is to set the mode + + .. sourcecode:: http + + GET /api/v1/transactions?mode={mode} HTTP/1.1 + Transaction Outputs ------------------- diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 6cf28719..76c7c744 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -76,10 +76,45 @@ def test_write_and_post_transaction(mock_post, b): .sign([alice.private_key]).to_dict() tx = b.validate_transaction(tx) - b.write_transaction(tx) + b.write_transaction(tx, **{'mode': 'broadcast_tx_async'}) assert mock_post.called args, kwargs = mock_post.call_args assert 'broadcast_tx_async' == kwargs['json']['method'] encoded_tx = [encode_transaction(tx.to_dict())] assert encoded_tx == kwargs['json']['params'] + + +@patch('requests.post') +@pytest.mark.parametrize('mode', [ + {'mode': 'broadcast_tx_async'}, + {'mode': 'broadcast_tx_sync'}, + {'mode': 'broadcast_tx_commit'} +]) +def test_post_transaction_valid_modes(mock_post, b, mode): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]).to_dict() + tx = b.validate_transaction(tx) + b.write_transaction(tx, **mode) + + args, kwargs = mock_post.call_args + assert mode['mode'] == kwargs['json']['method'] + + +def test_post_transaction_invalid_mode(b): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + from bigchaindb.common.exceptions import ValidationError + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]).to_dict() + tx = b.validate_transaction(tx) + with pytest.raises(ValidationError): + b.write_transaction(tx, **{'mode': 'nope'}) diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 95e1ff4b..39e1a00b 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -410,3 +410,39 @@ def test_return_only_valid_transaction(client): get_transaction_patched(Bigchain.TX_IN_BACKLOG)): url = '{}{}'.format(TX_ENDPOINT, '123') assert client.get(url).status_code == 404 + + +@pytest.mark.tendermint +@patch('requests.post') +@pytest.mark.parametrize('mode', [ + ('', 'broadcast_tx_async'), + ('?mode=async', 'broadcast_tx_async'), + ('?mode=sync', 'broadcast_tx_sync'), + ('?mode=commit', 'broadcast_tx_commit'), +]) +def test_post_transaction_valid_modes(mock_post, client, mode): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]) + mode_endpoint = TX_ENDPOINT + mode[0] + client.post(mode_endpoint, data=json.dumps(tx.to_dict())) + args, kwargs = mock_post.call_args + assert mode[1] == kwargs['json']['method'] + + +@pytest.mark.tendermint +def test_post_transaction_invalid_mode(client): + from bigchaindb.models import Transaction + from bigchaindb.common.crypto import generate_key_pair + alice = generate_key_pair() + tx = Transaction.create([alice.public_key], + [([alice.public_key], 1)], + asset=None) \ + .sign([alice.private_key]) + mode_endpoint = TX_ENDPOINT + '?mode=nope' + response = client.post(mode_endpoint, data=json.dumps(tx.to_dict())) + assert '400 BAD REQUEST' in response.status From b7d235379a69126accfc37ff81d9f70c656473ab Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Thu, 11 Jan 2018 13:44:54 +0100 Subject: [PATCH 05/22] fix tests --- bigchaindb/web/views/transactions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index 37912140..e49cd189 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -56,8 +56,7 @@ class TransactionListApi(Resource): A ``dict`` containing the data about the transaction. """ parser = reqparse.RequestParser() - parser.add_argument('mode', type=parameters.valid_mode, - choices=['async', 'sync', 'commit']) + parser.add_argument('mode', type=parameters.valid_mode) args = parser.parse_args() if not str(args['mode']) or str(args['mode']) == 'None': args['mode'] = 'broadcast_tx_async' From 411d1963bb1832ab73c237d04dc990109bcd7b7c Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Fri, 12 Jan 2018 11:06:27 +0100 Subject: [PATCH 06/22] requested changes --- bigchaindb/tendermint/lib.py | 3 +-- bigchaindb/web/views/transactions.py | 5 ++--- tests/web/test_transactions.py | 2 ++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 3cf93ce5..66b568e2 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -41,10 +41,9 @@ class BigchainDB(Bigchain): # TODO: handle connection errors! requests.post(ENDPOINT, json=payload) - def write_transaction(self, transaction, **kwargs): + def write_transaction(self, transaction, **mode): # This method offers backward compatibility with the Web API. """Submit a valid transaction to the mempool.""" - mode = kwargs self.post_transaction(transaction, mode) def store_transaction(self, transaction): diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index e49cd189..e53058b4 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -56,10 +56,9 @@ class TransactionListApi(Resource): A ``dict`` containing the data about the transaction. """ parser = reqparse.RequestParser() - parser.add_argument('mode', type=parameters.valid_mode) + parser.add_argument('mode', type=parameters.valid_mode, + default='broadcast_tx_async') args = parser.parse_args() - if not str(args['mode']) or str(args['mode']) == 'None': - args['mode'] = 'broadcast_tx_async' pool = current_app.config['bigchain_pool'] diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 39e1a00b..a67d790e 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -446,3 +446,5 @@ def test_post_transaction_invalid_mode(client): mode_endpoint = TX_ENDPOINT + '?mode=nope' response = client.post(mode_endpoint, data=json.dumps(tx.to_dict())) assert '400 BAD REQUEST' in response.status + assert 'Mode must be "async", "sync" or "commit"' ==\ + json.loads(response.data.decode('utf8'))['message']['mode'] From 76ac3b3c6c760f2e2b74b75e3b1ffa66fe994cf2 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 12 Jan 2018 16:32:32 +0100 Subject: [PATCH 07/22] Order import as per PEP 8 --- bigchaindb/tendermint/event_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index f8f01346..7398e86f 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -1,8 +1,8 @@ +import asyncio import json import logging import time -import asyncio import aiohttp from bigchaindb.common.utils import gen_timestamp From 2d0e83657ef12882432c6a5cde6fc4672d8d713f Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 12 Jan 2018 16:33:08 +0100 Subject: [PATCH 08/22] Parametrize host and port used by event stream closes #1994 --- bigchaindb/tendermint/event_stream.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index 7398e86f..765fb190 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -2,6 +2,7 @@ import asyncio import json import logging import time +from os import getenv import aiohttp @@ -10,8 +11,8 @@ from bigchaindb.events import EventTypes, Event from bigchaindb.tendermint.utils import decode_transaction_base64 -HOST = 'localhost' -PORT = 46657 +HOST = getenv('TENDERMINT_HOST', 'localhost') +PORT = int(getenv('TENDERMINT_PORT', 46657)) URL = f'ws://{HOST}:{PORT}/websocket' logger = logging.getLogger(__name__) From 1d99d6e8a8eebb88b9168ab69e05b372ab270a94 Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Mon, 15 Jan 2018 17:12:17 +0100 Subject: [PATCH 09/22] requested changes --- bigchaindb/core.py | 2 +- bigchaindb/tendermint/lib.py | 14 +++++++------- bigchaindb/web/views/transactions.py | 3 ++- tests/tendermint/test_lib.py | 14 +++++++------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index fae16e9d..4fb84722 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -77,7 +77,7 @@ class Bigchain(object): federation = property(lambda self: set(self.nodes_except_me + [self.me])) """ Set of federation member public keys """ - def write_transaction(self, signed_transaction, **kwargs): + def write_transaction(self, signed_transaction): """Write the transaction to bigchain. When first writing a transaction to the bigchain the transaction will be kept in a backlog until diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 66b568e2..61d1834b 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -19,21 +19,21 @@ logger = logging.getLogger(__name__) TENDERMINT_HOST = getenv('TENDERMINT_HOST', 'localhost') TENDERMINT_PORT = getenv('TENDERMINT_PORT', '46657') ENDPOINT = 'http://{}:{}/'.format(TENDERMINT_HOST, TENDERMINT_PORT) +MODE_LIST = ('broadcast_tx_async', + 'broadcast_tx_sync', + 'broadcast_tx_commit') class BigchainDB(Bigchain): def post_transaction(self, transaction, mode): """Submit a valid transaction to the mempool.""" - mode_list = ('broadcast_tx_async', - 'broadcast_tx_sync', - 'broadcast_tx_commit') - if not mode or mode['mode'] not in mode_list: + if not mode or mode not in MODE_LIST: raise ValidationError(('Mode must be one of the following {}.') - .format(', '.join(mode_list))) + .format(', '.join(MODE_LIST))) payload = { - 'method': mode['mode'], + 'method': mode, 'jsonrpc': '2.0', 'params': [encode_transaction(transaction.to_dict())], 'id': str(uuid4()) @@ -41,7 +41,7 @@ class BigchainDB(Bigchain): # TODO: handle connection errors! requests.post(ENDPOINT, json=payload) - def write_transaction(self, transaction, **mode): + def write_transaction(self, transaction, mode): # This method offers backward compatibility with the Web API. """Submit a valid transaction to the mempool.""" self.post_transaction(transaction, mode) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index e53058b4..c746f2ad 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -59,6 +59,7 @@ class TransactionListApi(Resource): parser.add_argument('mode', type=parameters.valid_mode, default='broadcast_tx_async') args = parser.parse_args() + mode = str(args['mode']) pool = current_app.config['bigchain_pool'] @@ -90,7 +91,7 @@ class TransactionListApi(Resource): 'Invalid transaction ({}): {}'.format(type(e).__name__, e) ) else: - bigchain.write_transaction(tx_obj, **args) + bigchain.write_transaction(tx_obj, mode) response = jsonify(tx) response.status_code = 202 diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 76c7c744..3355683c 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -76,7 +76,7 @@ def test_write_and_post_transaction(mock_post, b): .sign([alice.private_key]).to_dict() tx = b.validate_transaction(tx) - b.write_transaction(tx, **{'mode': 'broadcast_tx_async'}) + b.write_transaction(tx, 'broadcast_tx_async') assert mock_post.called args, kwargs = mock_post.call_args @@ -87,9 +87,9 @@ def test_write_and_post_transaction(mock_post, b): @patch('requests.post') @pytest.mark.parametrize('mode', [ - {'mode': 'broadcast_tx_async'}, - {'mode': 'broadcast_tx_sync'}, - {'mode': 'broadcast_tx_commit'} + 'broadcast_tx_async', + 'broadcast_tx_sync', + 'broadcast_tx_commit' ]) def test_post_transaction_valid_modes(mock_post, b, mode): from bigchaindb.models import Transaction @@ -100,10 +100,10 @@ def test_post_transaction_valid_modes(mock_post, b, mode): asset=None) \ .sign([alice.private_key]).to_dict() tx = b.validate_transaction(tx) - b.write_transaction(tx, **mode) + b.write_transaction(tx, mode) args, kwargs = mock_post.call_args - assert mode['mode'] == kwargs['json']['method'] + assert mode == kwargs['json']['method'] def test_post_transaction_invalid_mode(b): @@ -117,4 +117,4 @@ def test_post_transaction_invalid_mode(b): .sign([alice.private_key]).to_dict() tx = b.validate_transaction(tx) with pytest.raises(ValidationError): - b.write_transaction(tx, **{'mode': 'nope'}) + b.write_transaction(tx, 'nope') From ab9e1f71c70da4b5fd8f2e14b8b8048f702a8918 Mon Sep 17 00:00:00 2001 From: muawiakh Date: Mon, 15 Jan 2018 17:33:54 +0100 Subject: [PATCH 10/22] Pin yarl version to 0.18.* in setup.py - We need to pin this dependency because version 1.0.0 of yarl does not expose `unquote` and if we do not pin it, the latest version >=1.*.* is downloaded and `aiohttp` expects it. - This can also be fixed on `aiohttp`, but meanwhile :) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 049d182b..49bb88c1 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ install_requires = [ 'aiohttp~=2.0', 'python-rapidjson-schema==0.1.1', 'statsd==3.2.1', + 'yarl>=0.11,<1.0', ] setup( From 9bed8f363a4e62c29e492ac24dd816f1ef5c9258 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 5 Dec 2017 23:01:29 +0100 Subject: [PATCH 11/22] Fixes #1930 --- bigchaindb/common/transaction.py | 30 ++++++++++++++++++++-------- tests/common/test_transaction.py | 34 +++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index bf47588d..e4738ca1 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -9,6 +9,7 @@ import base58 from cryptoconditions import Fulfillment, ThresholdSha256, Ed25519Sha256 from cryptoconditions.exceptions import ( ParsingError, ASN1DecodeError, ASN1EncodeError, UnsupportedTypeError) +from sha3 import sha3_256 from bigchaindb.common.crypto import PrivateKey, hash_data from bigchaindb.common.exceptions import (KeypairMismatchException, @@ -812,13 +813,16 @@ class Transaction(object): # this should never happen, but then again, never say never. input_ = deepcopy(input_) public_key = input_.owners_before[0] + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + try: # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings input_.fulfillment.sign( - message.encode(), - base58.b58decode(key_pairs[public_key].encode()), - ) + message.digest(), base58.b58decode(key_pairs[public_key].encode())) except KeyError: raise KeypairMismatchException('Public key {} is not a pair to ' 'any of the private keys' @@ -836,6 +840,11 @@ class Transaction(object): key_pairs (dict): The keys to sign the Transaction with. """ input_ = deepcopy(input_) + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + for owner_before in set(input_.owners_before): # TODO: CC should throw a KeypairMismatchException, instead of # our manual mapping here @@ -863,7 +872,8 @@ class Transaction(object): # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings for subffill in subffills: - subffill.sign(message.encode(), base58.b58decode(private_key.encode())) + subffill.sign( + message.digest(), base58.b58decode(private_key.encode())) return input_ def inputs_valid(self, outputs=None): @@ -931,7 +941,7 @@ class Transaction(object): for i, cond in enumerate(output_condition_uris)) @staticmethod - def _input_valid(input_, operation, tx_serialized, output_condition_uri=None): + def _input_valid(input_, operation, message, output_condition_uri=None): """Validates a single Input against a single Output. Note: @@ -942,8 +952,7 @@ class Transaction(object): input_ (:class:`~bigchaindb.common.transaction. Input`) The Input to be signed. operation (str): The type of Transaction. - tx_serialized (str): The Transaction used as a message when - initially signing it. + message (str): The fulfillment message. output_condition_uri (str, optional): An Output to check the Input against. @@ -964,12 +973,17 @@ class Transaction(object): else: output_valid = output_condition_uri == ccffill.condition_uri + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + # NOTE: We pass a timestamp to `.validate`, as in case of a timeout # condition we'll have to validate against it # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings - ffill_valid = parsed_ffill.validate(message=tx_serialized.encode()) + ffill_valid = parsed_ffill.validate(message=message.digest()) return output_valid and ffill_valid def to_dict(self): diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 0b113c05..39048e19 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -479,7 +479,11 @@ def test_validate_tx_simple_create_signature(user_input, user_output, user_priv, tx = Transaction(Transaction.CREATE, asset_definition, [user_input], [user_output]) expected = deepcopy(user_output) - message = str(tx).encode() + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() expected.fulfillment.sign(message, b58decode(user_priv)) tx.sign([user_priv]) @@ -539,7 +543,11 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_input, tx = Transaction(Transaction.CREATE, asset_definition, [user_user2_threshold_input], [user_user2_threshold_output]) - message = str(tx).encode() + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() expected = deepcopy(user_user2_threshold_output) expected.fulfillment.subconditions[0]['body'].sign( message, b58decode(user_priv)) @@ -570,11 +578,18 @@ def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, tx = Transaction(Transaction.CREATE, asset_definition, [threshold_input], [threshold_output]) + + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() + expected = deepcopy(threshold_input) expected.fulfillment.subconditions[0]['body'].sign( - str(tx).encode(), b58decode(user_priv)) + message, b58decode(user_priv)) expected.fulfillment.subconditions[1]['body'].sign( - str(tx).encode(), b58decode(user_priv)) + message, b58decode(user_priv)) tx.sign([user_priv, user_priv]) @@ -807,7 +822,6 @@ def test_outputs_to_inputs(tx): def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, user2_output, user_priv): from bigchaindb.common.transaction import Transaction - from bigchaindb.common.utils import serialize from .utils import validate_transaction_model expected = { @@ -839,8 +853,14 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, transfer_tx = transfer_tx.to_dict() expected_input = deepcopy(inputs[0]) - expected_input.fulfillment.sign( - serialize(expected).encode(), b58decode(user_priv)) + json_serialized_tx = json.dumps(expected, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(json_serialized_tx.encode()) + message.update('{}{}'.format( + expected['inputs'][0]['fulfills']['transaction_id'], + expected['inputs'][0]['fulfills']['output_index'], + ).encode()) + expected_input.fulfillment.sign(message.digest(), b58decode(user_priv)) expected_ffill = expected_input.fulfillment.serialize_uri() transfer_ffill = transfer_tx['inputs'][0]['fulfillment'] From afda1686e692384386f9a62e9171ebd37f5f258b Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 16 Jan 2018 11:54:17 +0100 Subject: [PATCH 12/22] Upgrade aiohttp related to https://github.com/aio-libs/aiohttp/pull/2667 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 49bb88c1..b506527d 100644 --- a/setup.py +++ b/setup.py @@ -80,10 +80,9 @@ install_requires = [ 'multipipes~=0.1.0', 'jsonschema~=2.5.1', 'pyyaml~=3.12', - 'aiohttp~=2.0', + 'aiohttp~=2.3', 'python-rapidjson-schema==0.1.1', 'statsd==3.2.1', - 'yarl>=0.11,<1.0', ] setup( From 8c600e71376a3884af60c0c5288289a087d22041 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 16 Jan 2018 13:37:31 +0100 Subject: [PATCH 13/22] Pin tendermint to 0.13 --- docker-compose.tendermint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 2c43b36a..7c4489aa 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -31,7 +31,7 @@ services: - "9984" command: bigchaindb -l DEBUG start tendermint: - image: tendermint/tendermint + image: tendermint/tendermint:0.13 volumes: - ./tmdata:/tendermint entrypoint: '' From e74fb096fcb67fe0a6620542723b46f862dba96a Mon Sep 17 00:00:00 2001 From: codegeschrei Date: Wed, 24 Jan 2018 11:59:50 +0100 Subject: [PATCH 14/22] requested changes --- .../generate_http_server_api_documentation.py | 2 +- docs/server/source/http-client-server-api.rst | 47 ++++++++----------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 276c7f44..52520f47 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -58,7 +58,7 @@ Content-Type: application/json """ TPLS['post-tx-request'] = """\ -POST /api/v1/transactions/ HTTP/1.1 +POST /api/v1/transactions?mode=async HTTP/1.1 Host: example.com Content-Type: application/json diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index fce27c15..c088f926 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -124,22 +124,19 @@ Transactions :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. -.. http:post:: /api/v1/transactions +.. http:post:: /api/v1/transactions?mode={mode} - Push a new transaction. + Tendermint offers a `broadcast API + `_ with three different modes to post transactions. + By setting the mode, a new transaction can be pushed with a different mode than the default. The default mode is ``async``, which + will return immediately and not wait to see if the transaction is valid. The ``sync`` mode will return after the transactions is validated, while ``commit`` + returns after a transaction is committed to a block. .. note:: - The posted `transaction - `_ - should be structurally valid and not spending an already spent output. - The steps to build a valid transaction are beyond the scope of this page. - One would normally use a driver such as the `BigchainDB Python Driver - `_ - to build a valid transaction. - A generalization of the parameter follows: + This option is only available when using BigchainDB with Tendermint. - :param mode: (Optional) Set the broadcast method. + :query string mode: (Optional) One of the three supported modes to send a transaction: ``async``, ``sync``, ``commit``. **Example request**: @@ -165,25 +162,19 @@ Transactions :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. -.. http:post:: /api/v1/transactions?mode={mode} + +.. http:post:: /api/v1/transactions + + The endpoint without any parameters will push a new transaction. If BigchainDB is used with Tendermint, the default mode ``async`` is used. .. note:: - - This option is only available when using BigchainDB with Tendermint. - - Tendermint offers a `broadcast API - `_ with three different modes to send transactions. - By setting the mode, a new transaction can be pushed with a different mode than the default. The default mode is ``broadcast_tx_async``, which - will return immediately and not wait to see if the transaction is valid. - - :query string mode: (Optional) One of the three supported modes to send a transaction: ``broadcast_tx_async``, ``broadcast_tx_sync``, ``broadcast_tx_commit``. - - The only change for a request, compared to the one above, is to set the mode - - .. sourcecode:: http - - GET /api/v1/transactions?mode={mode} HTTP/1.1 - + The posted `transaction + `_ + should be structurally valid and not spending an already spent output. + The steps to build a valid transaction are beyond the scope of this page. + One would normally use a driver such as the `BigchainDB Python Driver + `_ + to build a valid transaction. Transaction Outputs ------------------- From 328f2ed1933d760fd8f09efee0184e2d9583960a Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 24 Jan 2018 16:43:35 +0100 Subject: [PATCH 15/22] Some minor copy-editing --- docs/server/source/http-client-server-api.rst | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index c088f926..2a17220d 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -129,13 +129,23 @@ Transactions Tendermint offers a `broadcast API `_ with three different modes to post transactions. By setting the mode, a new transaction can be pushed with a different mode than the default. The default mode is ``async``, which - will return immediately and not wait to see if the transaction is valid. The ``sync`` mode will return after the transactions is validated, while ``commit`` - returns after a transaction is committed to a block. + will return immediately and not wait to see if the transaction is valid. The ``sync`` mode will return after the transaction is validated, while ``commit`` + returns after the transaction is committed to a block. .. note:: This option is only available when using BigchainDB with Tendermint. + .. note:: + + The posted `transaction + `_ + should be structurally valid and not spending an already spent output. + The steps to build a valid transaction are beyond the scope of this page. + One would normally use a driver such as the `BigchainDB Python Driver + `_ + to build a valid transaction. + :query string mode: (Optional) One of the three supported modes to send a transaction: ``async``, ``sync``, ``commit``. **Example request**: @@ -165,16 +175,8 @@ Transactions .. http:post:: /api/v1/transactions - The endpoint without any parameters will push a new transaction. If BigchainDB is used with Tendermint, the default mode ``async`` is used. + This endpoint (without any parameters) will push a new transaction. If BigchainDB is used with Tendermint, the default mode ``async`` is used. - .. note:: - The posted `transaction - `_ - should be structurally valid and not spending an already spent output. - The steps to build a valid transaction are beyond the scope of this page. - One would normally use a driver such as the `BigchainDB Python Driver - `_ - to build a valid transaction. Transaction Outputs ------------------- From e25d36582812b35aadd985d1d26fd4684b1a57a9 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Wed, 31 Jan 2018 19:12:41 +0530 Subject: [PATCH 16/22] Integrate blocks API (#1970) * Integrate blocks api * Update docs * Fix docs * Fixed mismatch between code and documentation * Fixed docs --- bigchaindb/backend/localmongodb/query.py | 19 +++++ bigchaindb/backend/query.py | 14 ++++ bigchaindb/models.py | 49 ++++++++---- bigchaindb/tendermint/core.py | 6 +- bigchaindb/tendermint/lib.py | 38 +++++++++- .../generate_http_server_api_documentation.py | 17 +++-- .../server/source/data-models/block-model.rst | 75 ++----------------- docs/server/source/http-client-server-api.rst | 12 ++- tests/backend/localmongodb/test_queries.py | 28 +++++++ tests/tendermint/test_integration.py | 3 +- tests/tendermint/test_lib.py | 7 +- tests/utils.py | 10 +++ tests/web/test_block_tendermint.py | 37 +++++++++ 13 files changed, 214 insertions(+), 101 deletions(-) create mode 100644 tests/web/test_block_tendermint.py diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index f99f3815..5238055a 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -32,6 +32,17 @@ def get_transaction(conn, transaction_id): pass +@register_query(LocalMongoDBConnection) +def get_transactions(conn, transaction_ids): + try: + return conn.run( + conn.collection('transactions') + .find({'id': {'$in': transaction_ids}}, + projection={'_id': False})) + except IndexError: + pass + + @register_query(LocalMongoDBConnection) def store_metadata(conn, metadata): try: @@ -153,3 +164,11 @@ def get_spending_transactions(conn, inputs): {'$project': {'_id': False}} ])) return cursor + + +@register_query(LocalMongoDBConnection) +def get_block(conn, block_id): + return conn.run( + conn.collection('blocks') + .find_one({'height': block_id}, + projection={'_id': False})) diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 766d6668..1cdea12c 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -68,6 +68,20 @@ def get_transaction(connection, transaction_id): raise NotImplementedError +@singledispatch +def get_transactions(connection, transaction_ids): + """Get transactions from the transactions table. + + Args: + transaction_ids (list): list of transaction ids to fetch + + Returns: + The result of the operation. + """ + + raise NotImplementedError + + @singledispatch def get_asset(connection, asset_id): """Get a transaction from the transactions table. diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 0c2b4418..6a7a1acc 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -96,7 +96,7 @@ class Transaction(Transaction): return super().from_dict(tx_body) @classmethod - def from_db(cls, bigchain, tx_dict): + def from_db(cls, bigchain, tx_dict_list): """Helper method that reconstructs a transaction dict that was returned from the database. It checks what asset_id to retrieve, retrieves the asset from the asset table and reconstructs the transaction. @@ -104,29 +104,46 @@ class Transaction(Transaction): Args: bigchain (:class:`~bigchaindb.Bigchain`): An instance of Bigchain used to perform database queries. - tx_dict (:obj:`dict`): The transaction dict as returned from the - database. + tx_dict_list (:list:`dict` or :obj:`dict`): The transaction dict or + list of transaction dict as returned from the database. Returns: :class:`~Transaction` """ - if tx_dict['operation'] in [Transaction.CREATE, Transaction.GENESIS]: - # TODO: Maybe replace this call to a call to get_asset_by_id - asset = list(bigchain.get_assets([tx_dict['id']]))[0] + return_list = True + if isinstance(tx_dict_list, dict): + tx_dict_list = [tx_dict_list] + return_list = False + + tx_map = {} + tx_ids = [] + for tx in tx_dict_list: + tx.update({'metadata': None}) + tx_map[tx['id']] = tx + if tx['operation'] in [Transaction.CREATE, Transaction.GENESIS]: + tx_ids.append(tx['id']) + + assets = list(bigchain.get_assets(tx_ids)) + for asset in assets: + tx = tx_map[asset['id']] del asset['id'] - tx_dict.update({'asset': asset}) + tx.update({'asset': asset}) - # get metadata of the transaction - metadata = list(bigchain.get_metadata([tx_dict['id']])) - if 'metadata' not in tx_dict: - metadata = metadata[0] if metadata else None - if metadata: - metadata = metadata.get('metadata') + tx_ids = list(tx_map.keys()) + metadata_list = list(bigchain.get_metadata(tx_ids)) + for metadata in metadata_list: + tx = tx_map[metadata['id']] + tx.update({'metadata': metadata.get('metadata')}) - tx_dict.update({'metadata': metadata}) - - return cls.from_dict(tx_dict) + if return_list: + tx_list = [] + for tx_id, tx in tx_map.items(): + tx_list.append(cls.from_dict(tx)) + return tx_list + else: + tx = list(tx_map.values())[0] + return cls.from_dict(tx) class Block(object): diff --git a/bigchaindb/tendermint/core.py b/bigchaindb/tendermint/core.py index ea08dfdd..de5b9266 100644 --- a/bigchaindb/tendermint/core.py +++ b/bigchaindb/tendermint/core.py @@ -31,7 +31,7 @@ class App(BaseApplication): def init_chain(self, validators): """Initialize chain with block of height 0""" - block = Block(app_hash='', height=0) + block = Block(app_hash='', height=0, transactions=[]) self.bigchaindb.store_block(block._asdict()) def info(self): @@ -112,7 +112,9 @@ class App(BaseApplication): # register a new block only when new transactions are received if self.block_txn_ids: - block = Block(app_hash=self.block_txn_hash, height=self.new_height) + block = Block(app_hash=self.block_txn_hash, + height=self.new_height, + transactions=self.block_txn_ids) self.bigchaindb.store_block(block._asdict()) data = self.block_txn_hash.encode('utf-8') diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 61d1834b..107c6dcf 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -116,6 +116,42 @@ class BigchainDB(Bigchain): return backend.query.get_latest_block(self.connection) + def get_block(self, block_id, include_status=False): + """Get the block with the specified `block_id` (and optionally its status) + + Returns the block corresponding to `block_id` or None if no match is + found. + + Args: + block_id (str): block id of the block to get + include_status (bool): also return the status of the block + the return value is then a tuple: (block, status) + """ + # get block from database + if isinstance(block_id, str): + block_id = int(block_id) + + block = backend.query.get_block(self.connection, block_id) + if block: + transactions = backend.query.get_transactions(self.connection, block['transactions']) + transactions = Transaction.from_db(self, transactions) + + block = {'height': block['height'], + 'transactions': []} + block_txns = block['transactions'] + for txn in transactions: + block_txns.append(txn.to_dict()) + + status = None + if include_status: + # NOTE: (In Tendermint) a block is an abstract entity which + # exists only after it has been validated + if block: + status = self.BLOCK_VALID + return block, status + else: + return block + def validate_transaction(self, tx): """Validate a transaction against the current status of the database.""" @@ -142,4 +178,4 @@ class BigchainDB(Bigchain): return fastquery.FastQuery(self.connection, self.me) -Block = namedtuple('Block', ('app_hash', 'height')) +Block = namedtuple('Block', ('app_hash', 'height', 'transactions')) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 52520f47..da89abeb 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -7,6 +7,7 @@ import os.path from bigchaindb.common.transaction import Transaction, Input, TransactionLink from bigchaindb.core import Bigchain from bigchaindb.models import Block +from bigchaindb.tendermint import lib from bigchaindb.web import server @@ -242,18 +243,24 @@ def main(): node_private = "5G2kE1zJAgTajkVSbPAQWo4c2izvtwqaNHYsaNpbbvxX" node_public = "DngBurxfeNVKZWCEcDnLj1eMPAS7focUZTE5FndFGuHT" signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" - block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) - ctx['block'] = pretty_json(block.to_dict()) - ctx['blockid'] = block.id + app_hash = 'f6e0c49c6d94d6924351f25bb334cf2a99af4206339bf784e741d1a5ab599056' + block = lib.Block(height=1, transactions=[tx.to_dict()], app_hash=app_hash) + block_dict = block._asdict() + block_dict.pop('app_hash') + ctx['block'] = pretty_json(block_dict) + ctx['blockid'] = block.height + + block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, voters=[node_public], signature=signature) - ctx['block_transfer'] = pretty_json(block.to_dict()) + ctx['block_transfer'] = pretty_json(block_transfer.to_dict()) # vote + vblock = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) DUMMY_SHA3 = '0123456789abcdef' * 4 b = Bigchain(public_key=node_public, private_key=node_private) - vote = b.vote(block.id, DUMMY_SHA3, True) + vote = b.vote(vblock.id, DUMMY_SHA3, True) ctx['vote'] = pretty_json(vote) # block status diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index 0537bfd8..7073baa3 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -9,82 +9,23 @@ A block must contain the following JSON keys .. code-block:: json { - "id": "", - "block": { - "timestamp": "", - "transactions": [""], - "node_pubkey": "", - "voters": [""] - }, - "signature": "" + "height": "", + "transactions": [""] } The JSON Keys in a Block ------------------------ -**id** +**height** -The transaction ID and also the SHA3-256 hash -of the inner ``block`` object, loosely speaking. -It's a string. -To compute it, 1) construct an :term:`associative array` ``d`` containing -``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, -``block.voters``, and their values. 2) compute ``id = hash_of_aa(d)``. -There's pseudocode for the ``hash_of_aa()`` function -in the `IPDB Transaction Spec page about cryptographic hashes -`_. -The result (``id``) is a string: the block ID. -An example is ``"b60adf655932bf47ef58c0bfb2dd276d4795b94346b36cbb477e10d7eb02cea8"`` +The block ``"height"`` (``integer``) denotes the height of the blockchain when the given block was committed. +Since the blockchain height increases monotonically the height of block can be regarded as its id. + +**NOTE**: The genesis block has height ``0`` -**block.timestamp** - -The `Unix time `_ -when the block was created, according to the node which created it. -It's a string representation of an integer. -An example is ``"1507294217"``. - - -**block.transactions** +**transactions** A list of the :ref:`transactions ` included in the block. (Each transaction is a JSON object.) - - -**block.node_pubkey** - -The public key of the node that created the block. -It's a string. -See the `IPDB Transaction Spec page about cryptographic keys & signatures -`_. - - -**block.voters** - -A list of the public keys of all cluster nodes at the time the block was created. -It's a list of strings. -This list can change from block to block, as nodes join and leave the cluster. - - -**signature** - -The cryptographic signature of the inner ``block`` -by the node that created the block -(i.e. the node with public key ``node_pubkey``). -To compute that: - -#. Construct an :term:`associative array` ``d`` containing the contents - of the inner ``block`` - (i.e. ``block.timestamp``, ``block.transactions``, ``block.node_pubkey``, - ``block.voters``, and their values). -#. Compute ``signature = sig_of_aa(d, private_key)``, - where ``private_key`` is the node's private key - (i.e. ``node_pubkey`` and ``private_key`` are a key pair). There's pseudocode - for the ``sig_of_aa()`` function - on `the IPDB Transaction Spec page about cryptographic keys and signatures - `_. - -.. note:: - - The ``d_bytes`` computed when computing the block ID will be the *same* as the ``d_bytes`` computed when computing the block signature. This can be used to avoid redundant calculations. diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 2a17220d..06bb9632 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -597,14 +597,12 @@ The `votes endpoint <#votes>`_ contains all the voting information for a specifi Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. http:get:: /api/v1/blocks/{block_id} +.. http:get:: /api/v1/blocks/{block_height} - Get the block with the ID ``block_id``. Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be - returned. To check a block's status independently, use the `Statuses endpoint <#status>`_. - To check the votes on a block, have a look at the `votes endpoint <#votes>`_. + Get the block with the height ``block_height``. - :param block_id: block ID - :type block_id: hex string + :param block_height: block ID + :type block_height: integer **Example request**: @@ -620,7 +618,7 @@ Blocks :resheader Content-Type: ``application/json`` :statuscode 200: A block with that ID was found. - :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_height``. :statuscode 404: A block with that ID was not found. diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index 67643fff..be44a0eb 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -152,3 +152,31 @@ def test_get_spending_transactions(user_pk, user_sk): # tx3 not a member because input 1 not asked for assert txns == [tx2.to_dict(), tx4.to_dict()] + + +def test_store_block(): + from bigchaindb.backend import connect, query + from bigchaindb.tendermint.lib import Block + conn = connect() + + block = Block(app_hash='random_utxo', + height=3, + transactions=[]) + query.store_block(conn, block._asdict()) + cursor = conn.db.blocks.find({}, projection={'_id': False}) + assert cursor.count() == 1 + + +def test_get_block(): + from bigchaindb.backend import connect, query + from bigchaindb.tendermint.lib import Block + conn = connect() + + block = Block(app_hash='random_utxo', + height=3, + transactions=[]) + + conn.db.blocks.insert_one(block._asdict()) + + block = dict(query.get_block(conn, 3)) + assert block['height'] == 3 diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py index 2cda7e19..a366c6a3 100644 --- a/tests/tendermint/test_integration.py +++ b/tests/tendermint/test_integration.py @@ -12,12 +12,13 @@ pytestmark = pytest.mark.tendermint @pytest.mark.bdb -def test_app(b): +def test_app(tb): from bigchaindb.tendermint import App from bigchaindb.tendermint.utils import calculate_hash from bigchaindb.common.crypto import generate_key_pair from bigchaindb.models import Transaction + b = tb app = App(b) p = ProtocolHandler(app) diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 3355683c..81523dfc 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -37,12 +37,15 @@ def test_asset_is_separated_from_transaciton(b): assert b.get_transaction(tx.id) == tx -def test_get_latest_block(b): +def test_get_latest_block(tb): from bigchaindb.tendermint.lib import Block + b = tb for i in range(10): app_hash = os.urandom(16).hex() - block = Block(app_hash=app_hash, height=i)._asdict() + txn_id = os.urandom(16).hex() + block = Block(app_hash=app_hash, height=i, + transactions=[txn_id])._asdict() b.store_block(block) block = b.get_latest_block() diff --git a/tests/utils.py b/tests/utils.py index 5b3b5242..50dedae9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,6 +4,7 @@ import rethinkdb as r from bigchaindb.backend.mongodb.connection import MongoDBConnection from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection +from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection @singledispatch @@ -47,6 +48,15 @@ def flush_mongo_db(connection, dbname): connection.conn[dbname].metadata.delete_many({}) +@flush_db.register(LocalMongoDBConnection) +def flush_localmongo_db(connection, dbname): + connection.conn[dbname].bigchain.delete_many({}) + connection.conn[dbname].blocks.delete_many({}) + connection.conn[dbname].transactions.delete_many({}) + connection.conn[dbname].assets.delete_many({}) + connection.conn[dbname].metadata.delete_many({}) + + @singledispatch def update_table_config(connection, table, **kwrgas): raise NotImplementedError diff --git a/tests/web/test_block_tendermint.py b/tests/web/test_block_tendermint.py new file mode 100644 index 00000000..1b12228b --- /dev/null +++ b/tests/web/test_block_tendermint.py @@ -0,0 +1,37 @@ +import pytest + +from bigchaindb.models import Transaction +from bigchaindb.tendermint.lib import Block + +BLOCKS_ENDPOINT = '/api/v1/blocks/' + +pytestmark = pytest.mark.tendermint + + +@pytest.mark.bdb +@pytest.mark.usefixtures('inputs') +def test_get_block_endpoint(tb, client): + b = tb + tx = Transaction.create([b.me], [([b.me], 1)], asset={'cycle': 'hero'}) + tx = tx.sign([b.me_private]) + b.store_transaction(tx) + + block = Block(app_hash='random_utxo', + height=31, + transactions=[tx.id]) + b.store_block(block._asdict()) + + res = client.get(BLOCKS_ENDPOINT + str(block.height)) + expected_response = {'height': block.height, 'transactions': [tx.to_dict()]} + assert res.json == expected_response + assert res.status_code == 200 + + +@pytest.mark.bdb +@pytest.mark.usefixtures('inputs') +def test_get_block_returns_404_if_not_found(client): + res = client.get(BLOCKS_ENDPOINT + '123') + assert res.status_code == 404 + + res = client.get(BLOCKS_ENDPOINT + '123/') + assert res.status_code == 404 From a7ed8cf4cbd794d9c66578e305074b2335b1d3e0 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Fri, 2 Feb 2018 19:10:24 +0530 Subject: [PATCH 17/22] Write transactions on commit (#2017) * Write transacitons on commit * Fix docs strings, lazy check for duplicates --- bigchaindb/backend/localmongodb/query.py | 21 ++++++++- bigchaindb/backend/query.py | 27 ++++++++++-- bigchaindb/models.py | 16 +++++-- bigchaindb/tendermint/core.py | 8 +++- bigchaindb/tendermint/lib.py | 51 +++++++++++++++++++--- tests/backend/localmongodb/test_queries.py | 2 +- tests/tendermint/test_core.py | 19 +++++++- tests/tendermint/test_integration.py | 2 +- tests/tendermint/test_lib.py | 1 + 9 files changed, 129 insertions(+), 18 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 5238055a..4c7ef16e 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -22,6 +22,15 @@ def store_transaction(conn, signed_transaction): pass +@register_query(LocalMongoDBConnection) +def store_transactions(conn, signed_transactions): + try: + return conn.run(conn.collection('transactions') + .insert_many(signed_transactions)) + except DuplicateKeyError: + pass + + @register_query(LocalMongoDBConnection) def get_transaction(conn, transaction_id): try: @@ -44,7 +53,7 @@ def get_transactions(conn, transaction_ids): @register_query(LocalMongoDBConnection) -def store_metadata(conn, metadata): +def store_metadatas(conn, metadata): try: return conn.run( conn.collection('metadata') @@ -71,6 +80,16 @@ def store_asset(conn, asset): pass +@register_query(LocalMongoDBConnection) +def store_assets(conn, assets): + try: + return conn.run( + conn.collection('assets') + .insert_many(assets, ordered=False)) + except DuplicateKeyError: + pass + + @register_query(LocalMongoDBConnection) def get_asset(conn, asset_id): try: diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 1cdea12c..92f18f24 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -34,11 +34,25 @@ def store_asset(connection, asset): @singledispatch -def store_metadata(connection, metadata): - """Write metadata to the metadata table. +def store_assets(connection, assets): + """Write a list of assets to the assets table. Args: - metadata (dict): transaction metadata. + assets (list): a list of assets to write. + + Returns: + The database response. + """ + + raise NotImplementedError + + +@singledispatch +def store_metadatas(connection, metadata): + """Write a list of metadata to metadata table. + + Args: + metadata (list): list of metadata. Returns: The result of the operation. @@ -54,6 +68,13 @@ def store_transaction(connection, signed_transaction): raise NotImplementedError +@singledispatch +def store_transactions(connection, signed_transactions): + """Store list of transactions.""" + + raise NotImplementedError + + @singledispatch def get_transaction(connection, transaction_id): """Get a transaction from the transactions table. diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 6a7a1acc..a91ccc24 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -14,7 +14,7 @@ from bigchaindb.backend.schema import validate_language_key class Transaction(Transaction): - def validate(self, bigchain): + def validate(self, bigchain, current_transactions=[]): """Validate transaction spend Args: @@ -31,7 +31,8 @@ class Transaction(Transaction): input_conditions = [] if self.operation == Transaction.CREATE: - if bigchain.get_transaction(self.to_dict()['id']): + duplicates = any(txn for txn in current_transactions if txn.id == self.id) + if bigchain.get_transaction(self.to_dict()['id']) or duplicates: raise DuplicateTransaction('transaction `{}` already exists' .format(self.id)) elif self.operation == Transaction.TRANSFER: @@ -42,6 +43,14 @@ class Transaction(Transaction): input_tx, status = bigchain.\ get_transaction(input_txid, include_status=True) + if input_tx is None: + for ctxn in current_transactions: + # assume that the status as valid for previously validated + # transactions in current round + if ctxn.id == input_txid: + input_tx = ctxn + status = bigchain.TX_VALID + if input_tx is None: raise InputDoesNotExist("input `{}` doesn't exist" .format(input_txid)) @@ -51,7 +60,8 @@ class Transaction(Transaction): 'input `{}` does not exist in a valid block'.format( input_txid)) - spent = bigchain.get_spent(input_txid, input_.fulfills.output) + spent = bigchain.get_spent(input_txid, input_.fulfills.output, + current_transactions) if spent and spent.id != self.id: raise DoubleSpend('input `{}` was already spent' .format(input_txid)) diff --git a/bigchaindb/tendermint/core.py b/bigchaindb/tendermint/core.py index de5b9266..704dd592 100644 --- a/bigchaindb/tendermint/core.py +++ b/bigchaindb/tendermint/core.py @@ -25,6 +25,7 @@ class App(BaseApplication): self.bigchaindb = bigchaindb self.block_txn_ids = [] self.block_txn_hash = '' + self.block_transactions = [] self.validators = None self.new_height = None @@ -70,6 +71,7 @@ class App(BaseApplication): """ self.block_txn_ids = [] + self.block_transactions = [] def deliver_tx(self, raw_transaction): """Validate the transaction before mutating the state. @@ -78,15 +80,16 @@ class App(BaseApplication): raw_tx: a raw string (in bytes) transaction.""" logger.debug('deliver_tx: %s', raw_transaction) transaction = self.bigchaindb.validate_transaction( - decode_transaction(raw_transaction)) + decode_transaction(raw_transaction), self.block_transactions) if not transaction: logger.debug('deliver_tx: INVALID') return Result.error(log='Invalid transaction') else: logger.debug('storing tx') - self.bigchaindb.store_transaction(transaction) + # self.bigchaindb.store_transaction(transaction) self.block_txn_ids.append(transaction.id) + self.block_transactions.append(transaction) return Result.ok() def end_block(self, height): @@ -112,6 +115,7 @@ class App(BaseApplication): # register a new block only when new transactions are received if self.block_txn_ids: + self.bigchaindb.store_bulk_transactions(self.block_transactions) block = Block(app_hash=self.block_txn_hash, height=self.new_height, transactions=self.block_txn_ids) diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 107c6dcf..397da16f 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -12,6 +12,7 @@ from bigchaindb.models import Transaction from bigchaindb.common.exceptions import SchemaValidationError, ValidationError from bigchaindb.tendermint.utils import encode_transaction from bigchaindb.tendermint import fastquery +from bigchaindb import exceptions as core_exceptions logger = logging.getLogger(__name__) @@ -60,10 +61,32 @@ class BigchainDB(Bigchain): transaction_metadata = {'id': transaction['id'], 'metadata': metadata} - backend.query.store_metadata(self.connection, [transaction_metadata]) + backend.query.store_metadatas(self.connection, [transaction_metadata]) return backend.query.store_transaction(self.connection, transaction) + def store_bulk_transactions(self, transactions): + txns = [] + assets = [] + txn_metadatas = [] + for transaction in transactions: + transaction = transaction.to_dict() + if transaction['operation'] == 'CREATE': + asset = transaction.pop('asset') + asset['id'] = transaction['id'] + if asset['data'] is not None: + assets.append(asset) + + metadata = transaction.pop('metadata') + txn_metadatas.append({'id': transaction['id'], + 'metadata': metadata}) + txns.append(transaction) + + backend.query.store_metadatas(self.connection, txn_metadatas) + if assets: + backend.query.store_assets(self.connection, assets) + return backend.query.store_transactions(self.connection, txns) + def get_transaction(self, transaction_id, include_status=False): transaction = backend.query.get_transaction(self.connection, transaction_id) asset = backend.query.get_asset(self.connection, transaction_id) @@ -89,9 +112,25 @@ class BigchainDB(Bigchain): else: return transaction - def get_spent(self, txid, output): - transaction = backend.query.get_spent(self.connection, txid, - output) + def get_spent(self, txid, output, current_transactions=[]): + transactions = backend.query.get_spent(self.connection, txid, + output) + transactions = list(transactions) if transactions else [] + + for ctxn in current_transactions: + for ctxn_input in ctxn.inputs: + if ctxn_input.fulfills.txid == txid and\ + ctxn_input.fulfills.output == output: + transactions.append(ctxn.to_dict()) + + transaction = None + if len(transactions) > 1: + raise core_exceptions.CriticalDoubleSpend( + '`{}` was spent more than once. There is a problem' + ' with the chain'.format(txid)) + elif transactions: + transaction = transactions[0] + if transaction and transaction['operation'] == 'CREATE': asset = backend.query.get_asset(self.connection, transaction['id']) @@ -152,7 +191,7 @@ class BigchainDB(Bigchain): else: return block - def validate_transaction(self, tx): + def validate_transaction(self, tx, current_transactions=[]): """Validate a transaction against the current status of the database.""" transaction = tx @@ -167,7 +206,7 @@ class BigchainDB(Bigchain): logger.warning('Invalid transaction (%s): %s', type(e).__name__, e) return False try: - return transaction.validate(self) + return transaction.validate(self, current_transactions) except ValidationError as e: logger.warning('Invalid transaction (%s): %s', type(e).__name__, e) return False diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index be44a0eb..75f5190c 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -88,7 +88,7 @@ def test_write_metadata(): ] # write the assets - query.store_metadata(conn, deepcopy(metadata)) + query.store_metadatas(conn, deepcopy(metadata)) # check that 3 assets were written to the database cursor = conn.db.metadata.find({}, projection={'_id': False})\ diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py index 40b0310e..56c871e6 100644 --- a/tests/tendermint/test_core.py +++ b/tests/tendermint/test_core.py @@ -3,7 +3,7 @@ import json import pytest -pytestmark = pytest.mark.tendermint +pytestmark = [pytest.mark.tendermint, pytest.mark.bdb] def encode_tx_to_bytes(transaction): @@ -43,6 +43,7 @@ def test_check_tx__unsigned_create_is_error(b): assert result.is_error() +@pytest.mark.bdb def test_deliver_tx__valid_create_updates_db(b): from bigchaindb.tendermint import App from bigchaindb.models import Transaction @@ -56,8 +57,14 @@ def test_deliver_tx__valid_create_updates_db(b): .sign([alice.private_key]) app = App(b) + app.init_chain(["ignore"]) + app.begin_block("ignore") + result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_ok() + + app.end_block(99) + app.commit() assert b.get_transaction(tx.id).id == tx.id @@ -74,8 +81,15 @@ def test_deliver_tx__double_spend_fails(b): .sign([alice.private_key]) app = App(b) + app.init_chain(["ignore"]) + app.begin_block("ignore") + result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_ok() + + app.end_block(99) + app.commit() + assert b.get_transaction(tx.id).id == tx.id result = app.deliver_tx(encode_tx_to_bytes(tx)) assert result.is_error() @@ -87,6 +101,9 @@ def test_deliver_transfer_tx__double_spend_fails(b): from bigchaindb.common.crypto import generate_key_pair app = App(b) + app.init_chain(["ignore"]) + app.begin_block("ignore") + alice = generate_key_pair() bob = generate_key_pair() carly = generate_key_pair() diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py index a366c6a3..7b582446 100644 --- a/tests/tendermint/test_integration.py +++ b/tests/tendermint/test_integration.py @@ -57,7 +57,6 @@ def test_app(tb): res, err = read_message(BytesIO(data), types.Response) assert res assert res.deliver_tx.code == 0 - assert b.get_transaction(tx.id).id == tx.id new_block_txn_hash = calculate_hash([tx.id]) @@ -75,6 +74,7 @@ def test_app(tb): assert res assert res.commit.code == 0 assert res.commit.data == new_block_hash.encode('utf-8') + assert b.get_transaction(tx.id).id == tx.id block0 = b.get_latest_block() assert block0 diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 81523dfc..c684368c 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -37,6 +37,7 @@ def test_asset_is_separated_from_transaciton(b): assert b.get_transaction(tx.id) == tx +@pytest.mark.bdb def test_get_latest_block(tb): from bigchaindb.tendermint.lib import Block From fa33fc26afb01f5460ef819fa26ef54650f47755 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Thu, 8 Feb 2018 19:02:21 +0530 Subject: [PATCH 18/22] Integrate get block with transaction id api (#2021) * Integrate api, get block with transaction id * Fixed docs and docstrings * Fix docs * Remove status from tendermint, fix mongo query --- bigchaindb/backend/localmongodb/query.py | 8 +++ bigchaindb/backend/query.py | 14 +++++ bigchaindb/tendermint/lib.py | 16 ++++++ bigchaindb/web/views/blocks.py | 8 +-- .../generate_http_server_api_documentation.py | 16 +++--- docs/server/source/http-client-server-api.rst | 15 +++-- tests/web/test_block_tendermint.py | 55 +++++++++++++++++++ 7 files changed, 111 insertions(+), 21 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 4c7ef16e..e167f99e 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -191,3 +191,11 @@ def get_block(conn, block_id): conn.collection('blocks') .find_one({'height': block_id}, projection={'_id': False})) + + +@register_query(LocalMongoDBConnection) +def get_block_with_transaction(conn, txid): + return conn.run( + conn.collection('blocks') + .find({'transactions': txid}, + projection={'_id': False, 'height': True})) diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 92f18f24..96a27888 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -339,6 +339,20 @@ def get_block(connection, block_id): raise NotImplementedError +@singledispatch +def get_block_with_transaction(connection, txid): + """Get a block containing transaction id `txid` + + Args: + txid (str): id of transaction to be searched. + + Returns: + block_id (int): the block id or `None` + """ + + raise NotImplementedError + + @singledispatch def write_assets(connection, assets): """Write a list of assets to the assets table. diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 397da16f..37788b51 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -191,6 +191,22 @@ class BigchainDB(Bigchain): else: return block + def get_block_containing_tx(self, txid): + """Retrieve the list of blocks (block ids) containing a + transaction with transaction id `txid` + + Args: + txid (str): transaction id of the transaction to query + + Returns: + Block id list (list(int)) + """ + blocks = list(backend.query.get_block_with_transaction(self.connection, txid)) + if len(blocks) > 1: + logger.critical('Transaction id %s exists in multiple blocks', txid) + + return [block['height'] for block in blocks] + def validate_transaction(self, tx, current_transactions=[]): """Validate a transaction against the current status of the database.""" diff --git a/bigchaindb/web/views/blocks.py b/bigchaindb/web/views/blocks.py index 676601c9..b00a478f 100644 --- a/bigchaindb/web/views/blocks.py +++ b/bigchaindb/web/views/blocks.py @@ -5,7 +5,6 @@ For more information please refer to the documentation: http://bigchaindb.com/ht from flask import current_app from flask_restful import Resource, reqparse -from bigchaindb import Bigchain from bigchaindb.web.views.base import make_error @@ -42,18 +41,13 @@ class BlockListApi(Resource): """ parser = reqparse.RequestParser() parser.add_argument('transaction_id', type=str, required=True) - parser.add_argument('status', type=str, case_sensitive=False, - choices=[Bigchain.BLOCK_VALID, Bigchain.BLOCK_INVALID, Bigchain.BLOCK_UNDECIDED]) args = parser.parse_args(strict=True) tx_id = args['transaction_id'] - status = args['status'] pool = current_app.config['bigchain_pool'] with pool() as bigchain: - block_statuses = bigchain.get_blocks_status_containing_tx(tx_id) - blocks = [block_id for block_id, block_status in block_statuses.items() - if not status or block_status == status] + blocks = bigchain.get_block_containing_tx(tx_id) return blocks diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index da89abeb..7e683bb6 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -251,7 +251,14 @@ def main(): ctx['block'] = pretty_json(block_dict) ctx['blockid'] = block.height - block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) + # block status + block_list = [ + block.height + ] + ctx['block_list'] = pretty_json(block_list) + + + # block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, voters=[node_public], signature=signature) ctx['block_transfer'] = pretty_json(block_transfer.to_dict()) @@ -263,13 +270,6 @@ def main(): vote = b.vote(vblock.id, DUMMY_SHA3, True) ctx['vote'] = pretty_json(vote) - # block status - block_list = [ - block_transfer.id, - block.id - ] - ctx['block_list'] = pretty_json(block_list) - base_path = os.path.join(os.path.dirname(__file__), 'source/http-samples') if not os.path.exists(base_path): diff --git a/docs/server/source/http-client-server-api.rst b/docs/server/source/http-client-server-api.rst index 06bb9632..68ad5d7c 100644 --- a/docs/server/source/http-client-server-api.rst +++ b/docs/server/source/http-client-server-api.rst @@ -645,19 +645,22 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. -.. http:get:: /api/v1/blocks?transaction_id={transaction_id}&status={UNDECIDED|VALID|INVALID} +.. http:get:: /api/v1/blocks?transaction_id={transaction_id} - Retrieve a list of ``block_id`` with their corresponding status that contain a transaction with the ID ``transaction_id``. + Retrieve a list of block IDs (block heights), such that the blocks with those IDs contain a transaction with the ID ``transaction_id``. A correct response may consist of an empty list or a list with one block ID. - Any blocks, be they ``UNDECIDED``, ``VALID`` or ``INVALID`` will be - returned if no status filter is provided. + .. note:: + The query parameter ``status`` has been deprecated. It allowed + users to filter blocks based on their status i.e. only blocks with the specified + status were included in the response. Since then this behavior has changed + and now block are created only after the transactions are accepted by the + network i.e. blocks have only one status ``VALID`` .. note:: In case no block was found, an empty list and an HTTP status code ``200 OK`` is returned, as the request was still successful. :query string transaction_id: transaction ID *(required)* - :query string status: Filter blocks by their status. One of ``VALID``, ``UNDECIDED`` or ``INVALID``. **Example request**: @@ -671,7 +674,7 @@ Blocks :resheader Content-Type: ``application/json`` - :statuscode 200: A list of blocks containing a transaction with ID ``transaction_id`` was found and returned. + :statuscode 200: The request was properly formed and zero or more blocks were found containing the specified ``transaction_id``. :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``transaction_id``. diff --git a/tests/web/test_block_tendermint.py b/tests/web/test_block_tendermint.py index 1b12228b..1bf4a5c3 100644 --- a/tests/web/test_block_tendermint.py +++ b/tests/web/test_block_tendermint.py @@ -35,3 +35,58 @@ def test_get_block_returns_404_if_not_found(client): res = client.get(BLOCKS_ENDPOINT + '123/') assert res.status_code == 404 + + +@pytest.mark.bdb +def test_get_block_containing_transaction(tb, client): + b = tb + tx = Transaction.create([b.me], [([b.me], 1)], asset={'cycle': 'hero'}) + tx = tx.sign([b.me_private]) + b.store_transaction(tx) + + block = Block(app_hash='random_utxo', + height=13, + transactions=[tx.id]) + b.store_block(block._asdict()) + + res = client.get('{}?transaction_id={}'.format(BLOCKS_ENDPOINT, tx.id)) + expected_response = [block.height] + assert res.json == expected_response + assert res.status_code == 200 + + +@pytest.mark.bdb +def test_get_blocks_by_txid_endpoint_returns_empty_list_not_found(client): + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=') + assert res.status_code == 200 + assert len(res.json) == 0 + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123') + assert res.status_code == 200 + assert len(res.json) == 0 + + +@pytest.mark.bdb +def test_get_blocks_by_txid_endpoint_returns_400_bad_query_params(client): + res = client.get(BLOCKS_ENDPOINT) + assert res.status_code == 400 + + res = client.get(BLOCKS_ENDPOINT + '?ts_id=123') + assert res.status_code == 400 + assert res.json == { + 'message': { + 'transaction_id': 'Missing required parameter in the JSON body or the post body or the query string' + } + } + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123&foo=123') + assert res.status_code == 400 + assert res.json == { + 'message': 'Unknown arguments: foo' + } + + res = client.get(BLOCKS_ENDPOINT + '?transaction_id=123&status=123') + assert res.status_code == 400 + assert res.json == { + 'message': 'Unknown arguments: status' + } From b2584dd0fa71a547516176905c23090d075eef22 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 3 Jan 2018 23:33:37 +0100 Subject: [PATCH 19/22] Order imports as per pep 8 --- tests/tendermint/test_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index c684368c..13638b02 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -1,9 +1,9 @@ import os +from unittest.mock import patch import pytest from bigchaindb import backend -from unittest.mock import patch pytestmark = pytest.mark.tendermint From cc069fa6409c3b7662c3c4563fff5edbccfdda37 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Sat, 23 Dec 2017 07:43:46 +0100 Subject: [PATCH 20/22] Add db_context fixture --- tests/conftest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index acf309aa..90894491 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -486,6 +486,19 @@ def db_conn(): return connect() +@pytest.fixture +def db_context(db_config, db_host, db_port, db_name, db_conn): + DBContext = namedtuple( + 'DBContext', ('config', 'host', 'port', 'name', 'conn')) + return DBContext( + config=db_config, + host=db_host, + port=db_port, + name=db_name, + conn=db_conn, + ) + + @pytest.fixture def mocked_setup_pub_logger(mocker): return mocker.patch( From 5d0463282ba96d4beeaf9c9cd9b6b50bd8cc2206 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Feb 2018 15:23:36 +0100 Subject: [PATCH 21/22] Add tendermint marker to pytest.ini --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 72e63a06..e604b0b9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] testpaths = tests/ norecursedirs = .* *.egg *.egg-info env* devenv* docs +addopts = -m tendermint From 1611e6e04b225336cef4c02c007bedfedad39e34 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Feb 2018 15:24:19 +0100 Subject: [PATCH 22/22] Mount other volumes for docker-compose dev --- docker-compose.tendermint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 7c4489aa..0796b562 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -18,6 +18,11 @@ services: volumes: - ./bigchaindb:/usr/src/app/bigchaindb - ./tests:/usr/src/app/tests + - ./docs:/usr/src/app/docs + - ./setup.py:/usr/src/app/setup.py + - ./setup.cfg:/usr/src/app/setup.cfg + - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini environment: BIGCHAINDB_DATABASE_BACKEND: localmongodb BIGCHAINDB_DATABASE_HOST: mdb