From a3fa8393ace07c3e50923beda0d7902ed7860dd0 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 May 2016 16:26:05 +0200 Subject: [PATCH 01/12] rename condition to input_condition in fulfillment_message --- bigchaindb/util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 59f658ef..9d318485 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -330,7 +330,7 @@ def sign_tx(transaction, signing_keys): for fulfillment in tx['transaction']['fulfillments']: fulfillment_message = get_fulfillment_message(transaction, fulfillment) - parsed_fulfillment = cc.Fulfillment.from_json(fulfillment_message['condition']['condition']['details']) + parsed_fulfillment = cc.Fulfillment.from_json(fulfillment_message['input_condition']['condition']['details']) # for the case in which the type of fulfillment is not covered by this method parsed_fulfillment_signed = parsed_fulfillment @@ -448,7 +448,7 @@ def validate_fulfillments(signed_transaction): # if transaction has an input (i.e. not a `CREATE` transaction) if fulfillment['input']: is_valid &= parsed_fulfillment.condition.serialize_uri() == \ - fulfillment_message['condition']['condition']['uri'] + fulfillment_message['input_condition']['condition']['uri'] if not is_valid: return False @@ -488,13 +488,13 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): # get previous condition previous_tx = b.get_transaction(fulfillment['input']['txid']) conditions = sorted(previous_tx['transaction']['conditions'], key=lambda d: d['cid']) - fulfillment_message['condition'] = conditions[fulfillment['input']['cid']] + fulfillment_message['input_condition'] = conditions[fulfillment['input']['cid']] # if `CREATE` transaction # there is no previous transaction so we need to create one on the fly else: current_owner = transaction['transaction']['fulfillments'][0]['current_owners'][0] condition = json.loads(cc.Ed25519Fulfillment(public_key=current_owner).serialize_json()) - fulfillment_message['condition'] = {'condition': {'details': condition}} + fulfillment_message['input_condition'] = {'condition': {'details': condition}} if serialized: return serialize(fulfillment_message) return fulfillment_message From c623ca56d0ea0308d9a1d6af5df6f40e661ec841 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 May 2016 19:02:37 +0200 Subject: [PATCH 02/12] fulfillment_message with output_condition added tests --- bigchaindb/util.py | 5 +- tests/db/test_bigchain_api.py | 92 +++++++++++++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 9d318485..68d97eab 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -480,7 +480,8 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): # or created on the fly it this is a `CREATE` transaction fulfillment_message.update({ 'input': fulfillment['input'], - 'condition': None, + 'input_condition': None, + 'output_condition': None }) # if `TRANSFER` transaction @@ -489,12 +490,14 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): previous_tx = b.get_transaction(fulfillment['input']['txid']) conditions = sorted(previous_tx['transaction']['conditions'], key=lambda d: d['cid']) fulfillment_message['input_condition'] = conditions[fulfillment['input']['cid']] + fulfillment_message['output_condition'] = transaction['transaction']['conditions'][fulfillment['input']['cid']] # if `CREATE` transaction # there is no previous transaction so we need to create one on the fly else: current_owner = transaction['transaction']['fulfillments'][0]['current_owners'][0] condition = json.loads(cc.Ed25519Fulfillment(public_key=current_owner).serialize_json()) fulfillment_message['input_condition'] = {'condition': {'details': condition}} + fulfillment_message['output_condition'] = transaction['transaction']['conditions'][0] if serialized: return serialize(fulfillment_message) return fulfillment_message diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 0537e9ca..4b7d77c4 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1202,6 +1202,86 @@ class TestMultipleInputs(object): assert b.get_spent(inp) is None +class TestFulfillmentMessage(object): + def test_fulfillment_message_create(self, b, user_vk): + tx = b.create_transaction(b.me, user_vk, None, 'CREATE', payload={'pay': 'load'}) + original_fulfillment = tx['transaction']['fulfillments'][0] + fulfillment_message = util.get_fulfillment_message(tx, original_fulfillment) + + assert sorted(fulfillment_message) == ['data', 'id', 'input', 'input_condition', + 'operation', 'output_condition', 'timestamp', 'version'] + + assert fulfillment_message['data']['payload'] == tx['transaction']['data']['payload'] + assert fulfillment_message['id'] == tx['id'] + assert fulfillment_message['input'] == original_fulfillment['input'] + assert fulfillment_message['input_condition']['condition']['details']['public_key'] == \ + original_fulfillment['current_owners'][0] + assert fulfillment_message['operation'] == tx['transaction']['operation'] + assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] + assert fulfillment_message['version'] == tx['version'] + assert fulfillment_message['output_condition'] == tx['transaction']['conditions'][0] + + @pytest.mark.usefixtures('inputs') + def test_fulfillment_message_transfer(self, b, user_vk): + input_tx = b.get_owned_ids(user_vk).pop() + assert b.validate_fulfillments(b.get_transaction(input_tx['txid'])) == True + + tx = b.create_transaction(user_vk, b.me, input_tx, 'TRANSFER', payload={'pay': 'load'}) + + original_fulfillment = tx['transaction']['fulfillments'][0] + fulfillment_message = util.get_fulfillment_message(tx, original_fulfillment) + + assert sorted(fulfillment_message) == ['data', 'id', 'input', 'input_condition', + 'operation', 'output_condition', 'timestamp', 'version'] + + assert fulfillment_message['data']['payload'] == tx['transaction']['data']['payload'] + assert fulfillment_message['id'] == tx['id'] + assert fulfillment_message['input'] == original_fulfillment['input'] + assert fulfillment_message['input_condition']['new_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['operation'] == tx['transaction']['operation'] + assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] + assert fulfillment_message['version'] == tx['version'] + assert fulfillment_message['output_condition'] == tx['transaction']['conditions'][0] + + def test_fulfillment_message_multiple_current_owners_multiple_new_owners_multiple_inputs(self, b, user_vk): + # create a new users + user2_sk, user2_vk = crypto.generate_key_pair() + user3_sk, user3_vk = crypto.generate_key_pair() + user4_sk, user4_vk = crypto.generate_key_pair() + + # create inputs to spend + transactions = [] + for i in range(5): + tx = b.create_transaction(b.me, [user_vk, user2_vk], None, 'CREATE') + tx_signed = b.sign_transaction(tx, b.me_private) + transactions.append(tx_signed) + block = b.create_block(transactions) + b.write_block(block, durability='hard') + + # get input + owned_inputs = b.get_owned_ids(user_vk) + inp = owned_inputs[:3] + + # create a transaction + tx = b.create_transaction([user_vk, user2_vk], [user3_vk, user4_vk], inp, 'TRANSFER', payload={'pay': 'load'}) + + for original_fulfillment in tx['transaction']['fulfillments']: + fulfillment_message = util.get_fulfillment_message(tx, original_fulfillment) + + assert sorted(fulfillment_message) == ['data', 'id', 'input', 'input_condition', + 'operation', 'output_condition', 'timestamp', 'version'] + + assert fulfillment_message['data']['payload'] == tx['transaction']['data']['payload'] + assert fulfillment_message['id'] == tx['id'] + assert fulfillment_message['input'] == original_fulfillment['input'] + assert fulfillment_message['input_condition']['new_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['operation'] == tx['transaction']['operation'] + assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] + assert fulfillment_message['version'] == tx['version'] + assert fulfillment_message['output_condition'] == \ + tx['transaction']['conditions'][original_fulfillment['input']['cid']] + + class TestCryptoconditions(object): def test_fulfillment_transaction_create(self, b, user_vk): tx = b.create_transaction(b.me, user_vk, None, 'CREATE') @@ -1306,7 +1386,7 @@ class TestCryptoconditions(object): assert b.is_valid_transaction(tx) == tx @pytest.mark.usefixtures('inputs') - def test_override_fulfillment_transfer(self, b, user_vk, user_sk): + def test_override_fulfillment_transfer(self, b, user_vk, user_sk): # create valid transaction other_sk, other_vk = crypto.generate_key_pair() prev_tx_id = b.get_owned_ids(user_vk).pop() @@ -1323,7 +1403,7 @@ class TestCryptoconditions(object): assert b.is_valid_transaction(tx) == tx @pytest.mark.usefixtures('inputs') - def test_override_condition_and_fulfillment_transfer(self, b, user_vk, user_sk): + def test_override_condition_and_fulfillment_transfer(self, b, user_vk, user_sk): other_sk, other_vk = crypto.generate_key_pair() first_input_tx = b.get_owned_ids(user_vk).pop() first_tx = b.create_transaction(user_vk, other_vk, first_input_tx, 'TRANSFER') @@ -1560,7 +1640,8 @@ class TestCryptoconditions(object): tx_transfer_signed = b.sign_transaction(tx_transfer, [user_sk, user2_sk]) # expected fulfillment - expected_fulfillment = cc.Fulfillment.from_json(tx_create['transaction']['conditions'][0]['condition']['details']) + expected_fulfillment = cc.Fulfillment.from_json( + tx_create['transaction']['conditions'][0]['condition']['details']) subfulfillment1 = expected_fulfillment.subconditions[0]['body'] subfulfillment2 = expected_fulfillment.subconditions[1]['body'] @@ -1571,7 +1652,7 @@ class TestCryptoconditions(object): subfulfillment2.sign(util.serialize(expected_fulfillment_message), crypto.SigningKey(user2_sk)) assert tx_transfer_signed['transaction']['fulfillments'][0]['fulfillment'] \ - == expected_fulfillment.serialize_uri() + == expected_fulfillment.serialize_uri() assert b.validate_fulfillments(tx_transfer_signed) is True @@ -1696,7 +1777,8 @@ class TestCryptoconditions(object): # try doublespending user2_sk, user2_vk = crypto.generate_key_pair() - hashlock_doublespend_tx = b.create_transaction(None, user2_vk, {'txid': hashlock_tx['id'], 'cid': 0}, 'TRANSFER') + hashlock_doublespend_tx = b.create_transaction(None, user2_vk, {'txid': hashlock_tx['id'], 'cid': 0}, + 'TRANSFER') hashlock_doublespend_tx_fulfillment = cc.PreimageSha256Fulfillment(preimage=secret) hashlock_doublespend_tx['transaction']['fulfillments'][0]['fulfillment'] = \ From 0c9777686da2b6aaef1bb463d8a313c24f934b17 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 May 2016 19:18:56 +0200 Subject: [PATCH 03/12] output_condition from fid instead of cid --- bigchaindb/util.py | 4 +--- tests/db/test_bigchain_api.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 68d97eab..50b1acdf 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -481,7 +481,7 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): fulfillment_message.update({ 'input': fulfillment['input'], 'input_condition': None, - 'output_condition': None + 'output_condition': transaction['transaction']['conditions'][fulfillment['fid']] }) # if `TRANSFER` transaction @@ -490,14 +490,12 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): previous_tx = b.get_transaction(fulfillment['input']['txid']) conditions = sorted(previous_tx['transaction']['conditions'], key=lambda d: d['cid']) fulfillment_message['input_condition'] = conditions[fulfillment['input']['cid']] - fulfillment_message['output_condition'] = transaction['transaction']['conditions'][fulfillment['input']['cid']] # if `CREATE` transaction # there is no previous transaction so we need to create one on the fly else: current_owner = transaction['transaction']['fulfillments'][0]['current_owners'][0] condition = json.loads(cc.Ed25519Fulfillment(public_key=current_owner).serialize_json()) fulfillment_message['input_condition'] = {'condition': {'details': condition}} - fulfillment_message['output_condition'] = transaction['transaction']['conditions'][0] if serialized: return serialize(fulfillment_message) return fulfillment_message diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 4b7d77c4..b094c404 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1279,7 +1279,7 @@ class TestFulfillmentMessage(object): assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] assert fulfillment_message['version'] == tx['version'] assert fulfillment_message['output_condition'] == \ - tx['transaction']['conditions'][original_fulfillment['input']['cid']] + tx['transaction']['conditions'][original_fulfillment['fid']] class TestCryptoconditions(object): From 2c766d7bf2d79e671b900671e673b1cecf007ac4 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 12 May 2016 15:08:30 +0200 Subject: [PATCH 04/12] Display useful message when setuptools is outdated --- setup.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/setup.py b/setup.py index 3905799d..42ca96a3 100644 --- a/setup.py +++ b/setup.py @@ -6,11 +6,28 @@ For full docs visit https://bigchaindb.readthedocs.org """ from setuptools import setup, find_packages + # get the version version = {} with open('bigchaindb/version.py') as fp: exec(fp.read(), version) + +# check if setuptools is up to date +def check_setuptools_features(): + import pkg_resources + try: + list(pkg_resources.parse_requirements('foo~=1.0')) + except pkg_resources.RequirementParseError: + exit('Your Python distribution comes with an incompatible version ' + 'of `setuptools`. Please run:\n' + ' $ pip install --update setuptools\n' + 'and then run this command again') + + +check_setuptools_features() + + tests_require = [ 'pytest', 'coverage', From 7630ec7fc6481e466b1d15462043d11f33d17c2f Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 12 May 2016 15:30:58 +0200 Subject: [PATCH 05/12] refactor input_condition and get_fulfillment_message test malleability --- bigchaindb/util.py | 59 ++++++++++++------ tests/db/test_bigchain_api.py | 113 +++++++++++++++++++++++++++++----- 2 files changed, 135 insertions(+), 37 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 50b1acdf..40dfc7fb 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -330,7 +330,10 @@ def sign_tx(transaction, signing_keys): for fulfillment in tx['transaction']['fulfillments']: fulfillment_message = get_fulfillment_message(transaction, fulfillment) - parsed_fulfillment = cc.Fulfillment.from_json(fulfillment_message['input_condition']['condition']['details']) + # TODO: avoid instantiation, pass as argument! + bigchain = bigchaindb.Bigchain() + input_condition = get_input_condition(bigchain, fulfillment) + parsed_fulfillment = cc.Fulfillment.from_json(input_condition['condition']['details']) # for the case in which the type of fulfillment is not covered by this method parsed_fulfillment_signed = parsed_fulfillment @@ -436,19 +439,23 @@ def validate_fulfillments(signed_transaction): Returns: bool: True if the signature is correct, False otherwise. """ - for fulfillment in signed_transaction['transaction']['fulfillments']: fulfillment_message = get_fulfillment_message(signed_transaction, fulfillment) try: parsed_fulfillment = cc.Fulfillment.from_uri(fulfillment['fulfillment']) except (TypeError, ValueError, ParsingError): return False + is_valid = parsed_fulfillment.validate(serialize(fulfillment_message)) # if transaction has an input (i.e. not a `CREATE` transaction) if fulfillment['input']: + # TODO: avoid instantiation, pass as argument! + bigchain = bigchaindb.Bigchain() + input_condition = get_input_condition(bigchain, fulfillment) is_valid &= parsed_fulfillment.condition.serialize_uri() == \ - fulfillment_message['input_condition']['condition']['uri'] + input_condition['condition']['uri'] + if not is_valid: return False @@ -466,8 +473,6 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): Returns: str|dict: fulfillment message """ - b = bigchaindb.Bigchain() - # data to sign contains common transaction data fulfillment_message = { 'operation': transaction['transaction']['operation'], @@ -479,28 +484,42 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): # and the condition which needs to be retrieved from the output of a previous transaction # or created on the fly it this is a `CREATE` transaction fulfillment_message.update({ - 'input': fulfillment['input'], - 'input_condition': None, - 'output_condition': transaction['transaction']['conditions'][fulfillment['fid']] + 'fulfillment': copy.deepcopy(fulfillment), + 'condition': transaction['transaction']['conditions'][fulfillment['fid']] }) - # if `TRANSFER` transaction - if fulfillment['input']: - # get previous condition - previous_tx = b.get_transaction(fulfillment['input']['txid']) - conditions = sorted(previous_tx['transaction']['conditions'], key=lambda d: d['cid']) - fulfillment_message['input_condition'] = conditions[fulfillment['input']['cid']] - # if `CREATE` transaction - # there is no previous transaction so we need to create one on the fly - else: - current_owner = transaction['transaction']['fulfillments'][0]['current_owners'][0] - condition = json.loads(cc.Ed25519Fulfillment(public_key=current_owner).serialize_json()) - fulfillment_message['input_condition'] = {'condition': {'details': condition}} + # remove any fulfillment, as a fulfillment cannot sign itself + fulfillment_message['fulfillment']['fulfillment'] = None + if serialized: return serialize(fulfillment_message) return fulfillment_message +def get_input_condition(bigchain, fulfillment): + """ + + Args: + bigchain: + fulfillment: + Returns: + """ + input_tx = fulfillment['input'] + # if `TRANSFER` transaction + if input_tx: + # get previous condition + previous_tx = bigchain.get_transaction(input_tx['txid']) + conditions = sorted(previous_tx['transaction']['conditions'], key=lambda d: d['cid']) + return conditions[input_tx['cid']] + + # if `CREATE` transaction + # there is no previous transaction so we need to create one on the fly + else: + current_owner = fulfillment['current_owners'][0] + condition = json.loads(cc.Ed25519Fulfillment(public_key=current_owner).serialize_json()) + return {'condition': {'details': condition}} + + def get_hash_data(transaction): """ Get the hashed data that (should) correspond to the `transaction['id']` diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index b094c404..7e100912 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1208,18 +1208,18 @@ class TestFulfillmentMessage(object): original_fulfillment = tx['transaction']['fulfillments'][0] fulfillment_message = util.get_fulfillment_message(tx, original_fulfillment) - assert sorted(fulfillment_message) == ['data', 'id', 'input', 'input_condition', - 'operation', 'output_condition', 'timestamp', 'version'] + assert sorted(fulfillment_message) == \ + ['condition', 'data', 'fulfillment', 'id', 'operation', 'timestamp', 'version'] assert fulfillment_message['data']['payload'] == tx['transaction']['data']['payload'] assert fulfillment_message['id'] == tx['id'] - assert fulfillment_message['input'] == original_fulfillment['input'] - assert fulfillment_message['input_condition']['condition']['details']['public_key'] == \ - original_fulfillment['current_owners'][0] + assert fulfillment_message['condition'] == tx['transaction']['conditions'][0] + assert fulfillment_message['fulfillment']['current_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['fulfillment']['fid'] == original_fulfillment['fid'] + assert fulfillment_message['fulfillment']['input'] == original_fulfillment['input'] assert fulfillment_message['operation'] == tx['transaction']['operation'] assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] assert fulfillment_message['version'] == tx['version'] - assert fulfillment_message['output_condition'] == tx['transaction']['conditions'][0] @pytest.mark.usefixtures('inputs') def test_fulfillment_message_transfer(self, b, user_vk): @@ -1231,17 +1231,18 @@ class TestFulfillmentMessage(object): original_fulfillment = tx['transaction']['fulfillments'][0] fulfillment_message = util.get_fulfillment_message(tx, original_fulfillment) - assert sorted(fulfillment_message) == ['data', 'id', 'input', 'input_condition', - 'operation', 'output_condition', 'timestamp', 'version'] + assert sorted(fulfillment_message) == \ + ['condition', 'data', 'fulfillment', 'id', 'operation', 'timestamp', 'version'] assert fulfillment_message['data']['payload'] == tx['transaction']['data']['payload'] assert fulfillment_message['id'] == tx['id'] - assert fulfillment_message['input'] == original_fulfillment['input'] - assert fulfillment_message['input_condition']['new_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['condition'] == tx['transaction']['conditions'][0] + assert fulfillment_message['fulfillment']['current_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['fulfillment']['fid'] == original_fulfillment['fid'] + assert fulfillment_message['fulfillment']['input'] == original_fulfillment['input'] assert fulfillment_message['operation'] == tx['transaction']['operation'] assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] assert fulfillment_message['version'] == tx['version'] - assert fulfillment_message['output_condition'] == tx['transaction']['conditions'][0] def test_fulfillment_message_multiple_current_owners_multiple_new_owners_multiple_inputs(self, b, user_vk): # create a new users @@ -1268,18 +1269,96 @@ class TestFulfillmentMessage(object): for original_fulfillment in tx['transaction']['fulfillments']: fulfillment_message = util.get_fulfillment_message(tx, original_fulfillment) - assert sorted(fulfillment_message) == ['data', 'id', 'input', 'input_condition', - 'operation', 'output_condition', 'timestamp', 'version'] + assert sorted(fulfillment_message) == \ + ['condition', 'data', 'fulfillment', 'id', 'operation', 'timestamp', 'version'] assert fulfillment_message['data']['payload'] == tx['transaction']['data']['payload'] assert fulfillment_message['id'] == tx['id'] - assert fulfillment_message['input'] == original_fulfillment['input'] - assert fulfillment_message['input_condition']['new_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['condition'] == tx['transaction']['conditions'][original_fulfillment['fid']] + assert fulfillment_message['fulfillment']['current_owners'] == original_fulfillment['current_owners'] + assert fulfillment_message['fulfillment']['fid'] == original_fulfillment['fid'] + assert fulfillment_message['fulfillment']['input'] == original_fulfillment['input'] assert fulfillment_message['operation'] == tx['transaction']['operation'] assert fulfillment_message['timestamp'] == tx['transaction']['timestamp'] assert fulfillment_message['version'] == tx['version'] - assert fulfillment_message['output_condition'] == \ - tx['transaction']['conditions'][original_fulfillment['fid']] + + +class TestTransactionMalleability(object): + @pytest.mark.usefixtures('inputs') + def test_create_transaction_transfer(self, b, user_vk, user_sk): + input_tx = b.get_owned_ids(user_vk).pop() + assert b.validate_fulfillments(b.get_transaction(input_tx['txid'])) is True + + tx = b.create_transaction(user_vk, b.me, input_tx, 'TRANSFER') + + tx_signed = b.sign_transaction(tx, user_sk) + + assert b.validate_fulfillments(tx_signed) is True + assert b.is_valid_transaction(tx_signed) == tx_signed + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['id'] = 'dsdasd' + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['version'] = '0' + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['operation'] = 'CREATE' + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['timestamp'] = '1463033192.123456' + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['data'] = { + "hash": "872fa6e6f46246cd44afdb2ee9cfae0e72885fb0910e2bcf9a5a2a4eadb417b8", + "payload": { + "msg": "Hello BigchainDB!" + } + } + assert b.validate_fulfillments(tx_changed) == False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['fulfillments'] = [ + { + "current_owners": [ + "AFbofwJYEB7Cx2fgrPrCJzbdDVRzRKysoGXt4DsvuTGN" + ], + "fid": 0, + "fulfillment": "cf:4:iXaq3UbandDj4DgBhFDcfHjkm2639RwgLmwAHUmuDFMfMEKMZ71eQw2qCMK951kBaNNJel_FCDuYnacn_MsWzYXOUJs6DGW3lYfXI_d55xuqpH2BenvRWKNp98tRRr4B", + "input": None + } + ] + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['fulfillments'][0]['fid'] = 1 + with pytest.raises(IndexError): + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['fulfillments'][0]['current_owners'] = [ + "AFbofwJYEB7Cx2fgrPrCJzbdDVRzRKysoGXt4DsvuTGN"] + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False + + tx_changed = copy.deepcopy(tx_signed) + tx_changed['transaction']['fulfillments'][0]['input'] = { + "cid": 0, + "txid": "3055348675fc6f23b75f13c55db6d112b66eee068e99d30a802883d3b1784203" + } + with pytest.raises(TypeError): + assert b.validate_fulfillments(tx_changed) is False + assert b.is_valid_transaction(tx_changed) is False class TestCryptoconditions(object): From 37f0477c890fad463e601e293888b9b17b80b836 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 12 May 2016 15:37:16 +0200 Subject: [PATCH 06/12] check condition_uri of CREATE on validate_fulfillment --- bigchaindb/util.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 40dfc7fb..78b46ad7 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -276,7 +276,7 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): 'new_owners': new_owners, 'condition': { 'details': json.loads(condition.serialize_json()), - 'uri': condition.condition.serialize_uri() + 'uri': condition.condition_uri }, 'cid': fulfillment['fid'] }) @@ -449,12 +449,11 @@ def validate_fulfillments(signed_transaction): is_valid = parsed_fulfillment.validate(serialize(fulfillment_message)) # if transaction has an input (i.e. not a `CREATE` transaction) - if fulfillment['input']: - # TODO: avoid instantiation, pass as argument! - bigchain = bigchaindb.Bigchain() - input_condition = get_input_condition(bigchain, fulfillment) - is_valid &= parsed_fulfillment.condition.serialize_uri() == \ - input_condition['condition']['uri'] + # TODO: avoid instantiation, pass as argument! + bigchain = bigchaindb.Bigchain() + input_condition = get_input_condition(bigchain, fulfillment) + is_valid &= parsed_fulfillment.condition.serialize_uri() == \ + input_condition['condition']['uri'] if not is_valid: return False @@ -516,8 +515,14 @@ def get_input_condition(bigchain, fulfillment): # there is no previous transaction so we need to create one on the fly else: current_owner = fulfillment['current_owners'][0] - condition = json.loads(cc.Ed25519Fulfillment(public_key=current_owner).serialize_json()) - return {'condition': {'details': condition}} + condition = cc.Ed25519Fulfillment(public_key=current_owner) + + return { + 'condition': { + 'details': json.loads(condition.serialize_json()), + 'uri': condition.condition_uri + } + } def get_hash_data(transaction): From 3dad488344a0fdffe75e73e3b62fe28176dbd402 Mon Sep 17 00:00:00 2001 From: diminator Date: Tue, 17 May 2016 15:12:20 +0200 Subject: [PATCH 07/12] condition.serialize_uri => condition_uri --- bigchaindb/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 78b46ad7..ab0a9d72 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -452,7 +452,7 @@ def validate_fulfillments(signed_transaction): # TODO: avoid instantiation, pass as argument! bigchain = bigchaindb.Bigchain() input_condition = get_input_condition(bigchain, fulfillment) - is_valid &= parsed_fulfillment.condition.serialize_uri() == \ + is_valid &= parsed_fulfillment.condition_uri == \ input_condition['condition']['uri'] if not is_valid: From 3aba164bfd8c8714e3b4b6df5d4140347618e890 Mon Sep 17 00:00:00 2001 From: diminator Date: Tue, 17 May 2016 15:23:00 +0200 Subject: [PATCH 08/12] logical and rather than bitwise --- bigchaindb/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index ab0a9d72..65362a17 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -446,14 +446,14 @@ def validate_fulfillments(signed_transaction): except (TypeError, ValueError, ParsingError): return False + # TODO: might already break on a False here is_valid = parsed_fulfillment.validate(serialize(fulfillment_message)) # if transaction has an input (i.e. not a `CREATE` transaction) # TODO: avoid instantiation, pass as argument! bigchain = bigchaindb.Bigchain() input_condition = get_input_condition(bigchain, fulfillment) - is_valid &= parsed_fulfillment.condition_uri == \ - input_condition['condition']['uri'] + is_valid = is_valid and parsed_fulfillment.condition_uri == input_condition['condition']['uri'] if not is_valid: return False From 491dc29b41d979efe0a8ef7f65000c7868f5da16 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 19 May 2016 11:08:26 +0200 Subject: [PATCH 09/12] Use more generic and compatible Exception --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42ca96a3..4cdbe20f 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def check_setuptools_features(): import pkg_resources try: list(pkg_resources.parse_requirements('foo~=1.0')) - except pkg_resources.RequirementParseError: + except ValueError: exit('Your Python distribution comes with an incompatible version ' 'of `setuptools`. Please run:\n' ' $ pip install --update setuptools\n' From dbbdb2f0172800dc96f56f9e6026d6803d18a2d3 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 19 May 2016 11:30:58 +0200 Subject: [PATCH 11/12] Fix help message --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4cdbe20f..3639bd7e 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def check_setuptools_features(): except ValueError: exit('Your Python distribution comes with an incompatible version ' 'of `setuptools`. Please run:\n' - ' $ pip install --update setuptools\n' + ' $ pip3 install --upgrade setuptools\n' 'and then run this command again') From 9f03b2b771d73af6b247e8d97b0092fa00467a59 Mon Sep 17 00:00:00 2001 From: Ryan Henderson Date: Thu, 19 May 2016 14:03:52 +0200 Subject: [PATCH 12/12] add election class, process starts (#221) * add election class, process starts * remove else clause * resolve splitvote * move quorum check to core * remove comments * fixed vote counting * add test * poison pill * queue test * generalize election status * get correct part of block * clean shut down * test block liquidation * add vote signature checking * add tests * user_public_key to user_vk * python3 style floordivs * correct verdict for odd number of voters, and tests * remove whitespace * simplify verify_vote_signature * election checks pubkey is in block voters * tests for elector pubkey * count invalid votes as votes for invalid block * test update * test mismatched vote, block sigs * variable name change * comments on voting logic * comments on voting logic * remove unused queue * correct block model * imperative docstring * election status class constants * election tests use constants --- bigchaindb/consensus.py | 22 ++++++ bigchaindb/core.py | 32 +++++++++ bigchaindb/processes.py | 10 ++- bigchaindb/util.py | 24 +++++++ bigchaindb/voter.py | 65 +++++++++++++++++ tests/db/test_voter.py | 151 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 299 insertions(+), 5 deletions(-) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 29cca142..c54fbc1e 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -89,6 +89,19 @@ class AbstractConsensusRules(metaclass=ABCMeta): and correct, False otherwise. """ + @abstractmethod + def verify_vote_signature(block, signed_vote): + """Verify a cast vote. + + Args: + block (dict): block under election + signed_vote (dict): signed vote to verify + + Returns: + bool: True if the votes's required signature data is present + and correct, False otherwise. + """ + raise NotImplementedError class BaseConsensusRules(AbstractConsensusRules): """Base consensus rules for Bigchain. @@ -223,3 +236,12 @@ class BaseConsensusRules(AbstractConsensusRules): """ return util.validate_fulfillments(signed_transaction) + + @staticmethod + def verify_vote_signature(block, signed_vote): + """Verify the signature of a vote. + + Refer to the documentation of ``bigchaindb.util.verify_signature`` + """ + + return util.verify_vote_signature(block, signed_vote) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index b7ed151a..9a433697 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,4 +1,6 @@ import random +import math +import operator import rethinkdb as r import rapidjson @@ -20,6 +22,10 @@ class Bigchain(object): Create, read, sign, write transactions to the database """ + BLOCK_INVALID = 'invalid' + BLOCK_VALID = 'valid' + BLOCK_UNDECIDED = 'undecided' + def __init__(self, host=None, port=None, dbname=None, public_key=None, private_key=None, keyring=[], consensus_plugin=None): @@ -487,3 +493,29 @@ class Bigchain(object): unvoted.pop(0) return unvoted + + def block_election_status(self, block): + """Tally the votes on a block, and return the status: valid, invalid, or undecided.""" + + n_voters = len(block['block']['voters']) + vote_cast = [vote['vote']['is_block_valid'] for vote in block['votes']] + vote_validity = [self.consensus.verify_vote_signature(block, vote) for vote in block['votes']] + + # element-wise product of stated vote and validity of vote + vote_list = list(map(operator.mul, vote_cast, vote_validity)) + + # validate votes here + n_valid_votes = sum(vote_list) + n_invalid_votes = len(vote_list) - n_valid_votes + + # The use of ceiling and floor is to account for the case of an + # even number of voters where half the voters have voted 'invalid' + # and half 'valid'. In this case, the block should be marked invalid + # to avoid a tie. In the case of an odd number of voters this is not + # relevant, since one side must be a majority. + if n_invalid_votes >= math.ceil(n_voters / 2): + return Bigchain.BLOCK_INVALID + elif n_valid_votes > math.floor(n_voters / 2): + return Bigchain.BLOCK_VALID + else: + return Bigchain.BLOCK_UNDECIDED diff --git a/bigchaindb/processes.py b/bigchaindb/processes.py index 20b8df8d..b51df516 100644 --- a/bigchaindb/processes.py +++ b/bigchaindb/processes.py @@ -5,7 +5,7 @@ import rethinkdb as r import bigchaindb from bigchaindb import Bigchain -from bigchaindb.voter import Voter +from bigchaindb.voter import Voter, Election from bigchaindb.block import Block from bigchaindb.web import server @@ -31,6 +31,7 @@ class Processes(object): # initialize the class self.q_new_block = mp.Queue() self.q_new_transaction = mp.Queue() + self.q_block_new_vote = mp.Queue() def map_backlog(self): # listen to changes on the backlog and redirect the changes @@ -70,9 +71,9 @@ class Processes(object): elif change['new_val'] is None: pass - # update + # update (new vote) elif change['new_val'] is not None and change['old_val'] is not None: - pass + self.q_block_new_vote.put(change['new_val']) def start(self): logger.info('Initializing BigchainDB...') @@ -90,6 +91,7 @@ class Processes(object): p_map_backlog = mp.Process(name='backlog_mapper', target=self.map_backlog) p_block = mp.Process(name='block', target=block.start) p_voter = Voter(self.q_new_block) + p_election = Election(self.q_block_new_vote) # start the processes logger.info('starting bigchain mapper') @@ -101,6 +103,8 @@ class Processes(object): logger.info('starting voter') p_voter.start() + logger.info('starting election') + p_election.start() # start message block.initialized.wait() diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 65362a17..5f664a40 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -545,6 +545,30 @@ def get_hash_data(transaction): return crypto.hash_data(serialize(tx)) +def verify_vote_signature(block, signed_vote): + """Verify the signature of a vote + + A valid vote should have been signed `current_owner` corresponding private key. + + Args: + block (dict): block under election + signed_vote (dict): a vote with the `signature` included. + + Returns: + bool: True if the signature is correct, False otherwise. + """ + + signature = signed_vote['signature'] + vk_base58 = signed_vote['node_pubkey'] + + # immediately return False if the voter is not in the block voter list + if vk_base58 not in block['block']['voters']: + return False + + public_key = crypto.VerifyingKey(vk_base58) + return public_key.verify(serialize(signed_vote['vote']), signature) + + def transform_create(tx): """Change the owner and signature for a ``CREATE`` transaction created by a node""" diff --git a/bigchaindb/voter.py b/bigchaindb/voter.py index 9a34c0d8..b1b3b618 100644 --- a/bigchaindb/voter.py +++ b/bigchaindb/voter.py @@ -196,3 +196,68 @@ class Voter(object): p_validate.start() p_vote.start() p_update.start() + + +class Election(object): + + def __init__(self, q_block_new_vote): + """ + Initialize the class with the needed queues. + + Initialize a queue where blocks with new votes will be held + """ + self.q_block_new_vote = q_block_new_vote + self.q_invalid_blocks = mp.Queue() + + def check_for_quorum(self): + """ + Checks if block has enough invalid votes to make a decision + """ + b = Bigchain() + + while True: + next_block = self.q_block_new_vote.get() + + # poison pill + if next_block == 'stop': + self.q_invalid_blocks.put('stop') + logger.info('clean exit') + return + + if b.block_election_status(next_block) == 'invalid': + self.q_invalid_blocks.put(next_block) + + def requeue_transactions(self): + """ + Liquidates transactions from invalid blocks so they can be processed again + """ + while True: + invalid_block = self.q_invalid_blocks.get() + + # poison pill + if invalid_block == 'stop': + logger.info('clean exit') + return + + b = Bigchain() + for tx in invalid_block['block']['transactions']: + b.write_transaction(tx) + + def kill(self): + """ + Terminate processes + """ + self.q_block_new_vote.put('stop') + + def start(self): + """ + Initialize, spawn, and start the processes + """ + + # initialize the processes + p_quorum_check = mp.Process(name='check_for_quorum', target=self.check_for_quorum) + p_requeue_tx = mp.Process(name='requeue_tx', target=self.requeue_transactions) + + # start the processes + p_quorum_check.start() + p_requeue_tx.start() diff --git a/tests/db/test_voter.py b/tests/db/test_voter.py index ce1b42e1..8ad397a1 100644 --- a/tests/db/test_voter.py +++ b/tests/db/test_voter.py @@ -5,8 +5,8 @@ import multiprocessing as mp from bigchaindb import util -from bigchaindb.voter import Voter, BlockStream -from bigchaindb import crypto +from bigchaindb.voter import Voter, Election, BlockStream +from bigchaindb import crypto, Bigchain class TestBigchainVoter(object): @@ -297,6 +297,153 @@ class TestBigchainVoter(object): pass +class TestBlockElection(object): + + def test_quorum(self, b): + # create a new block + test_block = b.create_block([]) + + # simulate a federation with four voters + key_pairs = [crypto.generate_key_pair() for _ in range(4)] + test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs] + + # dummy block with test federation public keys as voters + test_block['block']['voters'] = [key_pair[1] for key_pair in key_pairs] + + # fake "yes" votes + valid_vote = [member.vote(test_block, 'abc', True) + for member in test_federation] + + # fake "no" votes + invalid_vote = [member.vote(test_block, 'abc', False) + for member in test_federation] + + # fake "yes" votes with incorrect signatures + improperly_signed_valid_vote = [member.vote(test_block, 'abc', True) for + member in test_federation] + [vote['vote'].update(this_should_ruin_things='lol') + for vote in improperly_signed_valid_vote] + + # test unanimously valid block + test_block['votes'] = valid_vote + assert b.block_election_status(test_block) == Bigchain.BLOCK_VALID + + # test partial quorum situations + test_block['votes'] = valid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_UNDECIDED + # + test_block['votes'] = valid_vote[:3] + assert b.block_election_status(test_block) == Bigchain.BLOCK_VALID + # + test_block['votes'] = invalid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_INVALID + + # test unanimously valid block with one improperly signed vote -- should still succeed + test_block['votes'] = valid_vote[:3] + improperly_signed_valid_vote[:1] + assert b.block_election_status(test_block) == Bigchain.BLOCK_VALID + + # test unanimously valid block with two improperly signed votes -- should fail + test_block['votes'] = valid_vote[:2] + improperly_signed_valid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_INVALID + + # test block with minority invalid vote + test_block['votes'] = invalid_vote[:1] + valid_vote[:3] + assert b.block_election_status(test_block) == Bigchain.BLOCK_VALID + + # test split vote + test_block['votes'] = invalid_vote[:2] + valid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_INVALID + + # test undecided + test_block['votes'] = valid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_UNDECIDED + + # change signatures in block, should fail + test_block['block']['voters'][0] = 'abc' + test_block['block']['voters'][1] = 'abc' + test_block['votes'] = valid_vote + assert b.block_election_status(test_block) == Bigchain.BLOCK_INVALID + + def test_quorum_odd(self, b): + # test partial quorum situations for odd numbers of voters + # create a new block + test_block = b.create_block([]) + + # simulate a federation with four voters + key_pairs = [crypto.generate_key_pair() for _ in range(5)] + test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs] + + # dummy block with test federation public keys as voters + test_block['block']['voters'] = [key_pair[1] for key_pair in key_pairs] + + # fake "yes" votes + valid_vote = [member.vote(test_block, 'abc', True) + for member in test_federation] + + # fake "no" votes + invalid_vote = [member.vote(test_block, 'abc', False) + for member in test_federation] + + test_block['votes'] = valid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_UNDECIDED + + test_block['votes'] = invalid_vote[:2] + assert b.block_election_status(test_block) == Bigchain.BLOCK_UNDECIDED + + test_block['votes'] = valid_vote[:3] + assert b.block_election_status(test_block) == Bigchain.BLOCK_VALID + + test_block['votes'] = invalid_vote[:3] + assert b.block_election_status(test_block) == Bigchain.BLOCK_INVALID + + def test_tx_rewritten_after_invalid(self, b, user_vk): + q_block_new_vote = mp.Queue() + + # create blocks with transactions + tx1 = b.create_transaction(b.me, user_vk, None, 'CREATE') + tx2 = b.create_transaction(b.me, user_vk, None, 'CREATE') + test_block_1 = b.create_block([tx1]) + test_block_2 = b.create_block([tx2]) + + # simulate a federation with four voters + key_pairs = [crypto.generate_key_pair() for _ in range(4)] + test_federation = [Bigchain(public_key=key_pair[1], private_key=key_pair[0]) + for key_pair in key_pairs] + + # simulate a federation with four voters + test_block_1['block']['voters'] = [key_pair[1] for key_pair in key_pairs] + test_block_2['block']['voters'] = [key_pair[1] for key_pair in key_pairs] + + # votes for block one + vote_1 = [member.vote(test_block_1, 'abc', True) + for member in test_federation] + + # votes for block two + vote_2 = [member.vote(test_block_2, 'abc', True) for member in test_federation[:2]] + \ + [member.vote(test_block_2, 'abc', False) for member in test_federation[2:]] + + # construct valid block + test_block_1['votes'] = vote_1 + q_block_new_vote.put(test_block_1) + + # construct invalid block + test_block_2['votes'] = vote_2 + q_block_new_vote.put(test_block_2) + + election = Election(q_block_new_vote) + election.start() + time.sleep(1) + election.kill() + + # tx1 was in a valid block, and should not be in the backlog + assert r.table('backlog').get(tx1['id']).run(b.conn) is None + + # tx2 was in an invalid block and SHOULD be in the backlog + assert r.table('backlog').get(tx2['id']).run(b.conn)['id'] == tx2['id'] + + class TestBlockStream(object): def test_if_federation_size_is_greater_than_one_ignore_past_blocks(self, b):