From 779d31ef3e27a467a95afc4f16c148542162119a Mon Sep 17 00:00:00 2001 From: Roman Sokolkov Date: Sat, 30 Dec 2017 17:31:55 +0100 Subject: [PATCH 1/9] Fix issue with STORAGE_ENGINE_CACHE_SIZE parameter In mongodb docker image STORAGE_ENGINE_CACHE_SIZE will always be set to '' (two quotation marks) and verification will fail. So it's impossible to run image without --storage-engine-cache-size set. This change removes double quotes while assigning default value to STORAGE_ENGINE_CACHE_SIZE. --- k8s/mongodb/container/mongod_entrypoint.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index 213f9989..bde5259a 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -66,7 +66,7 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ - -z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then + -z ${STORAGE_ENGINE_CACHE_SIZE:=''} ]] ; then #-z "${MONGODB_KEY_FILE_PASSWORD:?MongoDB Key File Password not specified. Exiting!}" || \ exit 1 else From e2316a3de857e67a44a932e92cac28f090bf78a7 Mon Sep 17 00:00:00 2001 From: muawiakh Date: Thu, 4 Jan 2018 13:58:28 +0100 Subject: [PATCH 2/9] Fix error checking of optional parameter `storage-engine-cache-size` - Keep the parameter optional - Remove redundant checking of the same parameter - Do not exit if the parameter is not specified --- k8s/mongodb/container/Dockerfile | 2 +- k8s/mongodb/container/docker_build_and_push.bash | 4 ++-- k8s/mongodb/container/mongod_entrypoint.bash | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile index 9d9d35f5..cf4aad42 100644 --- a/k8s/mongodb/container/Dockerfile +++ b/k8s/mongodb/container/Dockerfile @@ -1,4 +1,4 @@ -FROM mongo:3.4.4 +FROM mongo:3.4.10 LABEL maintainer "dev@bigchaindb.com" WORKDIR / RUN apt-get update \ diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash index 680fab22..bb48dd2f 100755 --- a/k8s/mongodb/container/docker_build_and_push.bash +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -1,5 +1,5 @@ #!/bin/bash -docker build -t bigchaindb/mongodb:3.2 . +docker build -t bigchaindb/mongodb:3.3 . -docker push bigchaindb/mongodb:3.2 +docker push bigchaindb/mongodb:3.3 diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index bde5259a..e8516a82 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -65,9 +65,8 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ - -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ - -z ${STORAGE_ENGINE_CACHE_SIZE:=''} ]] ; then - #-z "${MONGODB_KEY_FILE_PASSWORD:?MongoDB Key File Password not specified. Exiting!}" || \ + -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then + echo "Missing required enviroment variable(s)." exit 1 else echo REPLICA_SET_NAME="$REPLICA_SET_NAME" From 9d45f735fd48a9da4d398d8161a5f29fb9539bce Mon Sep 17 00:00:00 2001 From: muawiakh Date: Thu, 4 Jan 2018 14:53:23 +0100 Subject: [PATCH 3/9] Improving code readability --- k8s/mongodb/container/mongod_entrypoint.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index e8516a82..6a6fe309 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -66,6 +66,9 @@ if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then + # Not handling the STORAGE_ENGINE_CACHE_SIZE because + # it is optional. If not specified the default cache + # size is: max((50% RAM - 1GB), 256MB) echo "Missing required enviroment variable(s)." exit 1 else From 76ac3b3c6c760f2e2b74b75e3b1ffa66fe994cf2 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 12 Jan 2018 16:32:32 +0100 Subject: [PATCH 4/9] Order import as per PEP 8 --- bigchaindb/tendermint/event_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index f8f01346..7398e86f 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -1,8 +1,8 @@ +import asyncio import json import logging import time -import asyncio import aiohttp from bigchaindb.common.utils import gen_timestamp From 2d0e83657ef12882432c6a5cde6fc4672d8d713f Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 12 Jan 2018 16:33:08 +0100 Subject: [PATCH 5/9] Parametrize host and port used by event stream closes #1994 --- bigchaindb/tendermint/event_stream.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index 7398e86f..765fb190 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -2,6 +2,7 @@ import asyncio import json import logging import time +from os import getenv import aiohttp @@ -10,8 +11,8 @@ from bigchaindb.events import EventTypes, Event from bigchaindb.tendermint.utils import decode_transaction_base64 -HOST = 'localhost' -PORT = 46657 +HOST = getenv('TENDERMINT_HOST', 'localhost') +PORT = int(getenv('TENDERMINT_PORT', 46657)) URL = f'ws://{HOST}:{PORT}/websocket' logger = logging.getLogger(__name__) From ab9e1f71c70da4b5fd8f2e14b8b8048f702a8918 Mon Sep 17 00:00:00 2001 From: muawiakh Date: Mon, 15 Jan 2018 17:33:54 +0100 Subject: [PATCH 6/9] Pin yarl version to 0.18.* in setup.py - We need to pin this dependency because version 1.0.0 of yarl does not expose `unquote` and if we do not pin it, the latest version >=1.*.* is downloaded and `aiohttp` expects it. - This can also be fixed on `aiohttp`, but meanwhile :) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 049d182b..49bb88c1 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ install_requires = [ 'aiohttp~=2.0', 'python-rapidjson-schema==0.1.1', 'statsd==3.2.1', + 'yarl>=0.11,<1.0', ] setup( From 9bed8f363a4e62c29e492ac24dd816f1ef5c9258 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 5 Dec 2017 23:01:29 +0100 Subject: [PATCH 7/9] Fixes #1930 --- bigchaindb/common/transaction.py | 30 ++++++++++++++++++++-------- tests/common/test_transaction.py | 34 +++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index bf47588d..e4738ca1 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -9,6 +9,7 @@ import base58 from cryptoconditions import Fulfillment, ThresholdSha256, Ed25519Sha256 from cryptoconditions.exceptions import ( ParsingError, ASN1DecodeError, ASN1EncodeError, UnsupportedTypeError) +from sha3 import sha3_256 from bigchaindb.common.crypto import PrivateKey, hash_data from bigchaindb.common.exceptions import (KeypairMismatchException, @@ -812,13 +813,16 @@ class Transaction(object): # this should never happen, but then again, never say never. input_ = deepcopy(input_) public_key = input_.owners_before[0] + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + try: # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings input_.fulfillment.sign( - message.encode(), - base58.b58decode(key_pairs[public_key].encode()), - ) + message.digest(), base58.b58decode(key_pairs[public_key].encode())) except KeyError: raise KeypairMismatchException('Public key {} is not a pair to ' 'any of the private keys' @@ -836,6 +840,11 @@ class Transaction(object): key_pairs (dict): The keys to sign the Transaction with. """ input_ = deepcopy(input_) + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + for owner_before in set(input_.owners_before): # TODO: CC should throw a KeypairMismatchException, instead of # our manual mapping here @@ -863,7 +872,8 @@ class Transaction(object): # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings for subffill in subffills: - subffill.sign(message.encode(), base58.b58decode(private_key.encode())) + subffill.sign( + message.digest(), base58.b58decode(private_key.encode())) return input_ def inputs_valid(self, outputs=None): @@ -931,7 +941,7 @@ class Transaction(object): for i, cond in enumerate(output_condition_uris)) @staticmethod - def _input_valid(input_, operation, tx_serialized, output_condition_uri=None): + def _input_valid(input_, operation, message, output_condition_uri=None): """Validates a single Input against a single Output. Note: @@ -942,8 +952,7 @@ class Transaction(object): input_ (:class:`~bigchaindb.common.transaction. Input`) The Input to be signed. operation (str): The type of Transaction. - tx_serialized (str): The Transaction used as a message when - initially signing it. + message (str): The fulfillment message. output_condition_uri (str, optional): An Output to check the Input against. @@ -964,12 +973,17 @@ class Transaction(object): else: output_valid = output_condition_uri == ccffill.condition_uri + message = sha3_256(message.encode()) + if input_.fulfills: + message.update('{}{}'.format( + input_.fulfills.txid, input_.fulfills.output).encode()) + # NOTE: We pass a timestamp to `.validate`, as in case of a timeout # condition we'll have to validate against it # cryptoconditions makes no assumptions of the encoding of the # message to sign or verify. It only accepts bytestrings - ffill_valid = parsed_ffill.validate(message=tx_serialized.encode()) + ffill_valid = parsed_ffill.validate(message=message.digest()) return output_valid and ffill_valid def to_dict(self): diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 0b113c05..39048e19 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -479,7 +479,11 @@ def test_validate_tx_simple_create_signature(user_input, user_output, user_priv, tx = Transaction(Transaction.CREATE, asset_definition, [user_input], [user_output]) expected = deepcopy(user_output) - message = str(tx).encode() + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() expected.fulfillment.sign(message, b58decode(user_priv)) tx.sign([user_priv]) @@ -539,7 +543,11 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_input, tx = Transaction(Transaction.CREATE, asset_definition, [user_user2_threshold_input], [user_user2_threshold_output]) - message = str(tx).encode() + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() expected = deepcopy(user_user2_threshold_output) expected.fulfillment.subconditions[0]['body'].sign( message, b58decode(user_priv)) @@ -570,11 +578,18 @@ def test_validate_tx_threshold_duplicated_pk(user_pub, user_priv, tx = Transaction(Transaction.CREATE, asset_definition, [threshold_input], [threshold_output]) + + tx_dict = tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = None + serialized_tx = json.dumps(tx_dict, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(serialized_tx.encode()).digest() + expected = deepcopy(threshold_input) expected.fulfillment.subconditions[0]['body'].sign( - str(tx).encode(), b58decode(user_priv)) + message, b58decode(user_priv)) expected.fulfillment.subconditions[1]['body'].sign( - str(tx).encode(), b58decode(user_priv)) + message, b58decode(user_priv)) tx.sign([user_priv, user_priv]) @@ -807,7 +822,6 @@ def test_outputs_to_inputs(tx): def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, user2_output, user_priv): from bigchaindb.common.transaction import Transaction - from bigchaindb.common.utils import serialize from .utils import validate_transaction_model expected = { @@ -839,8 +853,14 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, transfer_tx = transfer_tx.to_dict() expected_input = deepcopy(inputs[0]) - expected_input.fulfillment.sign( - serialize(expected).encode(), b58decode(user_priv)) + json_serialized_tx = json.dumps(expected, sort_keys=True, + separators=(',', ':'), ensure_ascii=True) + message = sha3_256(json_serialized_tx.encode()) + message.update('{}{}'.format( + expected['inputs'][0]['fulfills']['transaction_id'], + expected['inputs'][0]['fulfills']['output_index'], + ).encode()) + expected_input.fulfillment.sign(message.digest(), b58decode(user_priv)) expected_ffill = expected_input.fulfillment.serialize_uri() transfer_ffill = transfer_tx['inputs'][0]['fulfillment'] From afda1686e692384386f9a62e9171ebd37f5f258b Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 16 Jan 2018 11:54:17 +0100 Subject: [PATCH 8/9] Upgrade aiohttp related to https://github.com/aio-libs/aiohttp/pull/2667 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 49bb88c1..b506527d 100644 --- a/setup.py +++ b/setup.py @@ -80,10 +80,9 @@ install_requires = [ 'multipipes~=0.1.0', 'jsonschema~=2.5.1', 'pyyaml~=3.12', - 'aiohttp~=2.0', + 'aiohttp~=2.3', 'python-rapidjson-schema==0.1.1', 'statsd==3.2.1', - 'yarl>=0.11,<1.0', ] setup( From 8c600e71376a3884af60c0c5288289a087d22041 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 16 Jan 2018 13:37:31 +0100 Subject: [PATCH 9/9] Pin tendermint to 0.13 --- docker-compose.tendermint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 2c43b36a..7c4489aa 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -31,7 +31,7 @@ services: - "9984" command: bigchaindb -l DEBUG start tendermint: - image: tendermint/tendermint + image: tendermint/tendermint:0.13 volumes: - ./tmdata:/tendermint entrypoint: ''