From 8a9030e5c0269d989221446d1e3ec8f105a2df9c Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 18:38:33 -0800 Subject: [PATCH 01/16] Added a simple plugin system for consensus rules using setuputils entry_points --- bigchaindb/__init__.py | 5 ++++- bigchaindb/config_utils.py | 20 ++++++++++++++++++++ bigchaindb/consensus/__init__.py | 0 bigchaindb/core.py | 4 +++- setup.py | 10 +++++++++- 5 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 bigchaindb/consensus/__init__.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 4bb76c44..b728e103 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -39,7 +39,10 @@ config = { 'host': e('BIGCHAIN_STATSD_HOST', default='localhost'), 'port': e('BIGCHAIN_STATSD_PORT', default=8125), 'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01) - } + }, + 'consensus_plugins': [ + 'base' + ], } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index bf3ed529..c273245d 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -17,6 +17,8 @@ import json import logging import collections +from pkg_resources import iter_entry_points + import bigchaindb logger = logging.getLogger(__name__) @@ -100,3 +102,21 @@ def autoconfigure(): except FileNotFoundError: logger.warning('Cannot find your config file. Run `bigchaindb configure` to create one') + +def get_plugins(plugin_names): + if not plugin_names: + plugin_names = bigchaindb.config.get('consensus_plugins', []) + + plugins = [] + + # It's important to maintain plugin ordering as stated in the config file. + # e.g. Expensive validation tasks should happen after cheap ones. + # + # TODO: We might want to add some sort of priority system, but for now we + # simply assume everything in a given plugin is designed to run at the + # same time. + for name in plugin_names: + for entry_point in iter_entry_points('bigchaindb.plugins', name): + plugins.append(entry_point.load()) + + return plugins diff --git a/bigchaindb/consensus/__init__.py b/bigchaindb/consensus/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bigchaindb/core.py b/bigchaindb/core.py index f2ba61dd..89cccd7e 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -30,7 +30,8 @@ class Bigchain(object): """ def __init__(self, host=None, port=None, dbname=None, - public_key=None, private_key=None, keyring=[]): + public_key=None, private_key=None, keyring=[], + consensus_plugins=['base']): """Initialize the Bigchain instance There are three ways in which the Bigchain instance can get its parameters. @@ -56,6 +57,7 @@ class Bigchain(object): self.me = public_key or bigchaindb.config['keypair']['public'] self.me_private = private_key or bigchaindb.config['keypair']['private'] self.federation_nodes = keyring or bigchaindb.config['keyring'] + self.consensus_plugins = config_utils.get_plugins(consensus_plugins) if not self.me or not self.me_private: raise KeypairNotFoundException() diff --git a/setup.py b/setup.py index 3fd8dae4..26f111fa 100644 --- a/setup.py +++ b/setup.py @@ -54,13 +54,21 @@ setup( 'Operating System :: POSIX :: Linux', ], - packages=['bigchaindb', 'bigchaindb.commands', 'bigchaindb.db'], + packages=[ + 'bigchaindb', + 'bigchaindb.commands', + 'bigchaindb.db', + 'bigchaindb.consensus' + ], entry_points={ 'console_scripts': [ 'bigchaindb=bigchaindb.commands.bigchain:main', 'bigchaindb-benchmark=bigchaindb.commands.bigchain_benchmark:main' ], + 'bigchaindb.plugins': [ + 'base=bigchaindb.consensus.base:ConsensusRules' + ] }, install_requires=[ 'rethinkdb==2.2.0.post4', From ee4720d1a59b6fdaadd27a8b170818b8ef0fef1a Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 18:39:23 -0800 Subject: [PATCH 02/16] Pulled validate_transaction and validate_block out into a base ConsensusRules plugin class --- bigchaindb/consensus/base.py | 107 +++++++++++++++++++++++++++++++++++ bigchaindb/core.py | 62 ++++---------------- 2 files changed, 118 insertions(+), 51 deletions(-) create mode 100644 bigchaindb/consensus/base.py diff --git a/bigchaindb/consensus/base.py b/bigchaindb/consensus/base.py new file mode 100644 index 00000000..1943c543 --- /dev/null +++ b/bigchaindb/consensus/base.py @@ -0,0 +1,107 @@ +import bigchaindb.exceptions as exceptions +from bigchaindb.crypto import hash_data + +class ConsensusRules(object): + """Base consensus rules for Bigchain. + + This class can be copied to write your own consensus rules! + + Note: Consensus plugins will be executed in the order that they're listed in + the bigchain config file. + """ + + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not found + TransactionOwnerError: if the new transaction is using an input it doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong + """ + + # If the operation is CREATE the transaction should have no inputs and + # should be signed by a federation node + if transaction['transaction']['operation'] == 'CREATE': + if transaction['transaction']['input']: + raise ValueError('A CREATE operation has no inputs') + if transaction['transaction']['current_owner'] not in ( + bigchain.federation_nodes + [bigchain.me]): + raise exceptions.OperationError( + 'Only federation nodes can use the operation `CREATE`') + + else: + # check if the input exists, is owned by the current_owner + if not transaction['transaction']['input']: + raise ValueError( + 'Only `CREATE` transactions can have null inputs') + + tx_input = bigchain.get_transaction( + transaction['transaction']['input']) + + if not tx_input: + raise exceptions.TransactionDoesNotExist( + 'input `{}` does not exist in the bigchain'.format( + transaction['transaction']['input'])) + + if (tx_input['transaction']['new_owner'] != + transaction['transaction']['current_owner']): + raise exceptions.TransactionOwnerError( + 'current_owner `{}` does not own the input `{}`'.format( + transaction['transaction']['current_owner'], + transaction['transaction']['input'])) + + # check if the input was already spent by a transaction other than + # this one. + spent = bigchain.get_spent(tx_input['id']) + if spent and spent['id'] != transaction['id']: + raise exceptions.DoubleSpend( + 'input `{}` was already spent'.format( + transaction['transaction']['input'])) + + # Check hash of the transaction + calculated_hash = hash_data(bigchain.serialize( + transaction['transaction'])) + if calculated_hash != transaction['id']: + raise exceptions.InvalidHash() + + # Check signature + if not bigchain.verify_signature(transaction): + raise exceptions.InvalidSignature() + + return transaction + + # TODO: check that the votings structure is correctly constructed + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + InvalidHash: if the hash of the block is wrong. + """ + + # Check if current hash is correct + calculated_hash = hash_data(bigchain.serialize(block['block'])) + if calculated_hash != block['id']: + raise exceptions.InvalidHash() + + return block diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 89cccd7e..7df5ed85 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -327,57 +327,16 @@ class Bigchain(object): transaction (dict): transaction to validate. Returns: - The transaction if the transaction is valid else it raises and exception - describing the reason why the transaction is invalid. - - Raises: - OperationError: if the transaction operation is not supported - TransactionDoesNotExist: if the input of the transaction is not found - TransactionOwnerError: if the new transaction is using an input it doesn't own - DoubleSpend: if the transaction is a double spend - InvalidHash: if the hash of the transaction is wrong - InvalidSignature: if the signature of the transaction is wrong + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. """ - # If the operation is CREATE the transaction should have no inputs and should be signed by a - # federation node - if transaction['transaction']['operation'] == 'CREATE': - if transaction['transaction']['input']: - raise ValueError('A CREATE operation has no inputs') - if transaction['transaction']['current_owner'] not in self.federation_nodes + [self.me]: - raise exceptions.OperationError('Only federation nodes can use the operation `CREATE`') - else: - # check if the input exists, is owned by the current_owner - if not transaction['transaction']['input']: - raise ValueError('Only `CREATE` transactions can have null inputs') - - tx_input = self.get_transaction(transaction['transaction']['input']) - if not tx_input: - raise exceptions.TransactionDoesNotExist('input `{}` does not exist in the bigchain'.format( - transaction['transaction']['input'])) - - if tx_input['transaction']['new_owner'] != transaction['transaction']['current_owner']: - raise exceptions.TransactionOwnerError('current_owner `{}` does not own the input `{}`'.format( - transaction['transaction']['current_owner'], transaction['transaction']['input'])) - - # check if the input was already spent by a transaction other then this one. - spent = self.get_spent(tx_input['id']) - if spent: - if spent['id'] != transaction['id']: - raise exceptions.DoubleSpend('input `{}` was already spent'.format( - transaction['transaction']['input'])) - - # Check hash of the transaction - calculated_hash = hash_data(self.serialize(transaction['transaction'])) - if calculated_hash != transaction['id']: - raise exceptions.InvalidHash() - - # Check signature - if not self.verify_signature(transaction): - raise exceptions.InvalidSignature() + for plugin in self.consensus_plugins: + transaction = plugin.validate_transaction(self, transaction) return transaction + def is_valid_transaction(self, transaction): """Check whether a transacion is valid or invalid. @@ -447,12 +406,13 @@ class Bigchain(object): """ - # 1. Check if current hash is correct - calculated_hash = hash_data(self.serialize(block['block'])) - if calculated_hash != block['id']: - raise exceptions.InvalidHash() + # First run all of the plugin block validation logic + for plugin in self.consensus_plugins: + transaction = plugin.validate_block(self, block) - # 2. Validate all transactions in the block + + # Finally: Tentative assumption that every blockchain will want to + # validate all transactions in each block for transaction in block['block']['transactions']: if not self.is_valid_transaction(transaction): # this will raise the exception From 380482b9674125d79be558373f43ef50ebaadf4b Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 19:19:08 -0800 Subject: [PATCH 03/16] Added an abstract base class so plugins don't need to add stubs for functions they don't care about. --- bigchaindb/consensus/__init__.py | 43 ++++++++++++++++++++++++++++++++ bigchaindb/consensus/base.py | 3 ++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/bigchaindb/consensus/__init__.py b/bigchaindb/consensus/__init__.py index e69de29b..24806bd0 100644 --- a/bigchaindb/consensus/__init__.py +++ b/bigchaindb/consensus/__init__.py @@ -0,0 +1,43 @@ +# TODO: no real reason to use abc yet, but later we can enforce inheritance from +# this class when loading plugins if that's desirable. +# from abc import ABCMeta + +class AbstractConsensusRules: + + # TODO: rather than having plugin-authors inherit and override, + # it'd be cleaner to make a `transactionrule` decorator and etc + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + Descriptive exceptions indicating the reason the transaction failed. + See the `exceptions` module for bigchain-native error classes. + """ + return transaction + + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + Descriptive exceptions indicating the reason the block failed. + See the `exceptions` module for bigchain-native error classes. + """ + return block diff --git a/bigchaindb/consensus/base.py b/bigchaindb/consensus/base.py index 1943c543..ab01d26e 100644 --- a/bigchaindb/consensus/base.py +++ b/bigchaindb/consensus/base.py @@ -1,7 +1,8 @@ import bigchaindb.exceptions as exceptions from bigchaindb.crypto import hash_data +from bigchaindb.consensus import AbstractConsensusRules -class ConsensusRules(object): +class ConsensusRules(AbstractConsensusRules): """Base consensus rules for Bigchain. This class can be copied to write your own consensus rules! From 2c8cea11ebb28f56a8c7ed49100fff04fd7330fb Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 19:22:04 -0800 Subject: [PATCH 04/16] Added some nonsense consensus rules to demonstrate a simpler plugin. --- bigchaindb/consensus/silly.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 bigchaindb/consensus/silly.py diff --git a/bigchaindb/consensus/silly.py b/bigchaindb/consensus/silly.py new file mode 100644 index 00000000..879b1328 --- /dev/null +++ b/bigchaindb/consensus/silly.py @@ -0,0 +1,21 @@ +import bigchaindb.exceptions as exceptions +from bigchaindb.crypto import hash_data +from bigchaindb.consensus import AbstractConsensusRules + + +class SillyConsensusRules(AbstractConsensusRules): + + @classmethod + def validate_transaction(cls, bigchain, transaction): + # I only like transactions whose timestamps are even. + if transaction['transaction']['timestamp'] % 2 != 0: + raise StandardError("Odd... very odd indeed.") + return transaction + + @classmethod + def validate_block(cls, bigchain, transaction): + # I don't trust Alice, I think she's shady. + if block['block']['node_pubkey'] == '': + raise StandardError("Alice is shady, everybody ignore her blocks!") + + return block From 01d706ac56f44887c34ae294a8d67950af1020d0 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:24:21 -0800 Subject: [PATCH 05/16] Moved consensus package files into single consensus.py module --- bigchaindb/consensus.py | 152 +++++++++++++++++++++++++++++++ bigchaindb/consensus/__init__.py | 43 --------- bigchaindb/consensus/base.py | 109 ---------------------- bigchaindb/consensus/silly.py | 21 ----- setup.py | 3 +- 5 files changed, 153 insertions(+), 175 deletions(-) create mode 100644 bigchaindb/consensus.py delete mode 100644 bigchaindb/consensus/__init__.py delete mode 100644 bigchaindb/consensus/base.py delete mode 100644 bigchaindb/consensus/silly.py diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py new file mode 100644 index 00000000..4bf5fe66 --- /dev/null +++ b/bigchaindb/consensus.py @@ -0,0 +1,152 @@ +import bigchaindb.exceptions as exceptions +from bigchaindb import util +from bigchaindb.crypto import hash_data + +# TODO: no real reason to use abc yet, but later we can enforce inheritance from +# this class when loading plugins if that's desirable. +# from abc import ABCMeta + +class AbstractConsensusRules: + + # TODO: rather than having plugin-authors inherit and override, + # it'd be cleaner to make a `transactionrule` decorator and etc + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + Descriptive exceptions indicating the reason the transaction failed. + See the `exceptions` module for bigchain-native error classes. + """ + return transaction + + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + Descriptive exceptions indicating the reason the block failed. + See the `exceptions` module for bigchain-native error classes. + """ + return block + + class ConsensusRules(AbstractConsensusRules): + """Base consensus rules for Bigchain. + + This class can be copied to write your own consensus rules! + + Note: Consensus plugins will be executed in the order that they're listed in + the bigchain config file. + """ + + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not found + TransactionOwnerError: if the new transaction is using an input it doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong + """ + + # If the operation is CREATE the transaction should have no inputs and + # should be signed by a federation node + if transaction['transaction']['operation'] == 'CREATE': + if transaction['transaction']['input']: + raise ValueError('A CREATE operation has no inputs') + if transaction['transaction']['current_owner'] not in ( + bigchain.federation_nodes + [bigchain.me]): + raise exceptions.OperationError( + 'Only federation nodes can use the operation `CREATE`') + + else: + # check if the input exists, is owned by the current_owner + if not transaction['transaction']['input']: + raise ValueError( + 'Only `CREATE` transactions can have null inputs') + + tx_input = bigchain.get_transaction( + transaction['transaction']['input']) + + if not tx_input: + raise exceptions.TransactionDoesNotExist( + 'input `{}` does not exist in the bigchain'.format( + transaction['transaction']['input'])) + + if (tx_input['transaction']['new_owner'] != + transaction['transaction']['current_owner']): + raise exceptions.TransactionOwnerError( + 'current_owner `{}` does not own the input `{}`'.format( + transaction['transaction']['current_owner'], + transaction['transaction']['input'])) + + # check if the input was already spent by a transaction other than + # this one. + spent = bigchain.get_spent(tx_input['id']) + if spent and spent['id'] != transaction['id']: + raise exceptions.DoubleSpend( + 'input `{}` was already spent'.format( + transaction['transaction']['input'])) + + # Check hash of the transaction + calculated_hash = hash_data(util.serialize( + transaction['transaction'])) + if calculated_hash != transaction['id']: + raise exceptions.InvalidHash() + + # Check signature + if not bigchain.verify_signature(transaction): + raise exceptions.InvalidSignature() + + return transaction + + # TODO: check that the votings structure is correctly constructed + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + InvalidHash: if the hash of the block is wrong. + """ + + # Check if current hash is correct + calculated_hash = hash_data(util.serialize(block['block'])) + if calculated_hash != block['id']: + raise exceptions.InvalidHash() + + return block diff --git a/bigchaindb/consensus/__init__.py b/bigchaindb/consensus/__init__.py deleted file mode 100644 index 24806bd0..00000000 --- a/bigchaindb/consensus/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -# TODO: no real reason to use abc yet, but later we can enforce inheritance from -# this class when loading plugins if that's desirable. -# from abc import ABCMeta - -class AbstractConsensusRules: - - # TODO: rather than having plugin-authors inherit and override, - # it'd be cleaner to make a `transactionrule` decorator and etc - @classmethod - def validate_transaction(cls, bigchain, transaction): - """Validate a transaction. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - transaction (dict): transaction to validate. - - Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. - - Raises: - Descriptive exceptions indicating the reason the transaction failed. - See the `exceptions` module for bigchain-native error classes. - """ - return transaction - - @classmethod - def validate_block(cls, bigchain, block): - """Validate a block. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - block (dict): block to validate. - - Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. - - Raises: - Descriptive exceptions indicating the reason the block failed. - See the `exceptions` module for bigchain-native error classes. - """ - return block diff --git a/bigchaindb/consensus/base.py b/bigchaindb/consensus/base.py deleted file mode 100644 index ed3f05c9..00000000 --- a/bigchaindb/consensus/base.py +++ /dev/null @@ -1,109 +0,0 @@ -import bigchaindb.exceptions as exceptions -from bigchaindb import util -from bigchaindb.crypto import hash_data -from bigchaindb.consensus import AbstractConsensusRules - -class ConsensusRules(AbstractConsensusRules): - """Base consensus rules for Bigchain. - - This class can be copied to write your own consensus rules! - - Note: Consensus plugins will be executed in the order that they're listed in - the bigchain config file. - """ - - @classmethod - def validate_transaction(cls, bigchain, transaction): - """Validate a transaction. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - transaction (dict): transaction to validate. - - Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. - - Raises: - OperationError: if the transaction operation is not supported - TransactionDoesNotExist: if the input of the transaction is not found - TransactionOwnerError: if the new transaction is using an input it doesn't own - DoubleSpend: if the transaction is a double spend - InvalidHash: if the hash of the transaction is wrong - InvalidSignature: if the signature of the transaction is wrong - """ - - # If the operation is CREATE the transaction should have no inputs and - # should be signed by a federation node - if transaction['transaction']['operation'] == 'CREATE': - if transaction['transaction']['input']: - raise ValueError('A CREATE operation has no inputs') - if transaction['transaction']['current_owner'] not in ( - bigchain.federation_nodes + [bigchain.me]): - raise exceptions.OperationError( - 'Only federation nodes can use the operation `CREATE`') - - else: - # check if the input exists, is owned by the current_owner - if not transaction['transaction']['input']: - raise ValueError( - 'Only `CREATE` transactions can have null inputs') - - tx_input = bigchain.get_transaction( - transaction['transaction']['input']) - - if not tx_input: - raise exceptions.TransactionDoesNotExist( - 'input `{}` does not exist in the bigchain'.format( - transaction['transaction']['input'])) - - if (tx_input['transaction']['new_owner'] != - transaction['transaction']['current_owner']): - raise exceptions.TransactionOwnerError( - 'current_owner `{}` does not own the input `{}`'.format( - transaction['transaction']['current_owner'], - transaction['transaction']['input'])) - - # check if the input was already spent by a transaction other than - # this one. - spent = bigchain.get_spent(tx_input['id']) - if spent and spent['id'] != transaction['id']: - raise exceptions.DoubleSpend( - 'input `{}` was already spent'.format( - transaction['transaction']['input'])) - - # Check hash of the transaction - calculated_hash = hash_data(util.serialize( - transaction['transaction'])) - if calculated_hash != transaction['id']: - raise exceptions.InvalidHash() - - # Check signature - if not bigchain.verify_signature(transaction): - raise exceptions.InvalidSignature() - - return transaction - - # TODO: check that the votings structure is correctly constructed - @classmethod - def validate_block(cls, bigchain, block): - """Validate a block. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - block (dict): block to validate. - - Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. - - Raises: - InvalidHash: if the hash of the block is wrong. - """ - - # Check if current hash is correct - calculated_hash = hash_data(util.serialize(block['block'])) - if calculated_hash != block['id']: - raise exceptions.InvalidHash() - - return block diff --git a/bigchaindb/consensus/silly.py b/bigchaindb/consensus/silly.py deleted file mode 100644 index 879b1328..00000000 --- a/bigchaindb/consensus/silly.py +++ /dev/null @@ -1,21 +0,0 @@ -import bigchaindb.exceptions as exceptions -from bigchaindb.crypto import hash_data -from bigchaindb.consensus import AbstractConsensusRules - - -class SillyConsensusRules(AbstractConsensusRules): - - @classmethod - def validate_transaction(cls, bigchain, transaction): - # I only like transactions whose timestamps are even. - if transaction['transaction']['timestamp'] % 2 != 0: - raise StandardError("Odd... very odd indeed.") - return transaction - - @classmethod - def validate_block(cls, bigchain, transaction): - # I don't trust Alice, I think she's shady. - if block['block']['node_pubkey'] == '': - raise StandardError("Alice is shady, everybody ignore her blocks!") - - return block diff --git a/setup.py b/setup.py index 083dc7a8..475cb3cc 100644 --- a/setup.py +++ b/setup.py @@ -59,8 +59,7 @@ setup( packages=[ 'bigchaindb', 'bigchaindb.commands', - 'bigchaindb.db', - 'bigchaindb.consensus' + 'bigchaindb.db' ], entry_points={ From 14b71537d6b0231f0125f0c861c8486ef6faa78b Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:32:25 -0800 Subject: [PATCH 06/16] Made AbstractConsensusRules use python's abc and enforcing that ConsensusRules plugins subclass it --- bigchaindb/__init__.py | 4 +--- bigchaindb/config_utils.py | 47 ++++++++++++++++++++++++++------------ bigchaindb/consensus.py | 28 +++++++++++++---------- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 0a69c150..9283913f 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -41,9 +41,7 @@ config = { 'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01) }, 'api_endpoint': 'http://localhost:8008/api/v1', - 'consensus_plugins': [ - 'base' - ] + 'consensus_plugin': e('BIGCHAIN_CONSENSUS_PLUGIN', default='default') } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index c273245d..7f312d0c 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -17,7 +17,7 @@ import json import logging import collections -from pkg_resources import iter_entry_points +from pkg_resources import iter_entry_points, ResolutionError import bigchaindb @@ -103,20 +103,37 @@ def autoconfigure(): logger.warning('Cannot find your config file. Run `bigchaindb configure` to create one') -def get_plugins(plugin_names): - if not plugin_names: - plugin_names = bigchaindb.config.get('consensus_plugins', []) +def load_consensus_plugin(name=None): + """Find and load the chosen consensus plugin. - plugins = [] + Args: + name (string): the name of the entry_point, as advertised in the + setup.py of the providing package. - # It's important to maintain plugin ordering as stated in the config file. - # e.g. Expensive validation tasks should happen after cheap ones. - # - # TODO: We might want to add some sort of priority system, but for now we - # simply assume everything in a given plugin is designed to run at the - # same time. - for name in plugin_names: - for entry_point in iter_entry_points('bigchaindb.plugins', name): - plugins.append(entry_point.load()) + Returns: + an uninstantiated subclass of ``bigchaindb.consensus.AbstractConsensusRules`` + """ + if not name: + name = bigchaindb.config.get('consensus_plugin', 'default') - return plugins + # TODO: This will return the first plugin with group `bigchaindb.plugins` + # and name `name` in the active WorkingSet. + # We should probably support Requirements specs in the config, e.g. + # consensus_plugin: 'my-plugin-package==0.0.1;default' + plugin = None + for entry_point in iter_entry_points('bigchaindb.consensus', name): + plugin = entry_point.load() + + # No matching entry_point found + if not plugin: + raise ResolutionError( + 'No plugin found in group `bigchaindb.plugins` with name `{}`'. + format(name)) + + # Is this strictness desireable? + # It will probably reduce developer headaches in the wild. + if not issubclass(plugin, (AbstractConsensusRules)): + raise TypeError("object of type '{}' does not implement `bigchaindb." + "consensus.AbstractConsensusRules`".format(type(plugin))) + + return plugin diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 4bf5fe66..1c0b1aec 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -1,17 +1,21 @@ +from abc import ABCMeta, abstractmethod + import bigchaindb.exceptions as exceptions from bigchaindb import util -from bigchaindb.crypto import hash_data +from bigchaindb.crypto import hash_data, PublicKey -# TODO: no real reason to use abc yet, but later we can enforce inheritance from -# this class when loading plugins if that's desirable. -# from abc import ABCMeta -class AbstractConsensusRules: +class AbstractConsensusRules(metaclass=ABCMeta): + """Abstract base class for Bigchain plugins which implement consensus logic. - # TODO: rather than having plugin-authors inherit and override, - # it'd be cleaner to make a `transactionrule` decorator and etc - @classmethod - def validate_transaction(cls, bigchain, transaction): + A consensus plugin must expose a class inheriting from this one via an + entry_point. + + All methods listed below must be implemented. + """ + + @abstractmethod + def validate_transaction(bigchain, transaction): """Validate a transaction. Args: @@ -26,10 +30,10 @@ class AbstractConsensusRules: Descriptive exceptions indicating the reason the transaction failed. See the `exceptions` module for bigchain-native error classes. """ - return transaction + raise NotImplementedError - @classmethod - def validate_block(cls, bigchain, block): + @abstractmethod + def validate_block(bigchain, block): """Validate a block. Args: From a5243e43f68661d99ea68c6b788fa1838eff7895 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:33:31 -0800 Subject: [PATCH 07/16] This large (sorry) commit 1. switches from a composable plugin model to a single-plugin model 2. switches class methods to static methods in the BaseConsensusRules class 3. adds create_transaction, sign_transaction, and verify_transaction to the plugin API TODO: If we adopt this model, all references in e.g. client.py to util methods like `sign_tx` need to be routed through the plugin methods, and possibly need to be added to the plugin interface. --- bigchaindb/config_utils.py | 1 + bigchaindb/consensus.py | 189 +++++++++++++++++++++++++------------ bigchaindb/core.py | 60 ++++++------ setup.py | 4 +- 4 files changed, 159 insertions(+), 95 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 7f312d0c..26afa3c8 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -20,6 +20,7 @@ import collections from pkg_resources import iter_entry_points, ResolutionError import bigchaindb +from bigchaindb.consensus import AbstractConsensusRules logger = logging.getLogger(__name__) CONFIG_DEFAULT_PATH = os.environ.setdefault( diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 1c0b1aec..818249b8 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -19,7 +19,7 @@ class AbstractConsensusRules(metaclass=ABCMeta): """Validate a transaction. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + bigchain (Bigchain): an instantiated ``bigchaindb.Bigchain`` object. transaction (dict): transaction to validate. Returns: @@ -37,7 +37,7 @@ class AbstractConsensusRules(metaclass=ABCMeta): """Validate a block. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + bigchain (Bigchain): an instantiated ``bigchaindb.Bigchain`` object. block (dict): block to validate. Returns: @@ -48,109 +48,174 @@ class AbstractConsensusRules(metaclass=ABCMeta): Descriptive exceptions indicating the reason the block failed. See the `exceptions` module for bigchain-native error classes. """ - return block + raise NotImplementedError - class ConsensusRules(AbstractConsensusRules): + @abstractmethod + def create_transaction(*args, **kwargs): + """Create a new transaction. + + Args: + The signature of this method is left to plugin authors to decide. + + Returns: + dict: newly constructed transaction. + """ + raise NotImplementedError + + @abstractmethod + def sign_transaction(transaction, *args, **kwargs): + """Sign a transaction. + + Args: + transaction (dict): transaction to sign. + any other arguments are left to plugin authors to decide. + + Returns: + dict: transaction with any signatures applied. + """ + raise NotImplementedError + + @abstractmethod + def verify_signature(signed_transaction): + """Verify the signature of a transaction. + + Args: + signed_transaction (dict): signed transaction to verify + + Returns: + bool: True if the transaction's required signature data is present + and correct, False otherwise. + """ + raise NotImplementedError + + +class BaseConsensusRules(AbstractConsensusRules): """Base consensus rules for Bigchain. - This class can be copied to write your own consensus rules! - - Note: Consensus plugins will be executed in the order that they're listed in - the bigchain config file. + This class can be copied or overridden to write your own consensus rules! """ - @classmethod - def validate_transaction(cls, bigchain, transaction): + @staticmethod + def validate_transaction(bigchain, transaction): """Validate a transaction. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - transaction (dict): transaction to validate. + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. Raises: - OperationError: if the transaction operation is not supported - TransactionDoesNotExist: if the input of the transaction is not found - TransactionOwnerError: if the new transaction is using an input it doesn't own - DoubleSpend: if the transaction is a double spend - InvalidHash: if the hash of the transaction is wrong - InvalidSignature: if the signature of the transaction is wrong + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not found + TransactionOwnerError: if the new transaction is using an input it doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong """ # If the operation is CREATE the transaction should have no inputs and # should be signed by a federation node if transaction['transaction']['operation'] == 'CREATE': - if transaction['transaction']['input']: - raise ValueError('A CREATE operation has no inputs') - if transaction['transaction']['current_owner'] not in ( - bigchain.federation_nodes + [bigchain.me]): - raise exceptions.OperationError( - 'Only federation nodes can use the operation `CREATE`') + if transaction['transaction']['input']: + raise ValueError('A CREATE operation has no inputs') + if transaction['transaction']['current_owner'] not in ( + bigchain.federation_nodes + [bigchain.me]): + raise exceptions.OperationError( + 'Only federation nodes can use the operation `CREATE`') else: - # check if the input exists, is owned by the current_owner - if not transaction['transaction']['input']: - raise ValueError( - 'Only `CREATE` transactions can have null inputs') + # check if the input exists, is owned by the current_owner + if not transaction['transaction']['input']: + raise ValueError( + 'Only `CREATE` transactions can have null inputs') - tx_input = bigchain.get_transaction( - transaction['transaction']['input']) + tx_input = bigchain.get_transaction( + transaction['transaction']['input']) - if not tx_input: - raise exceptions.TransactionDoesNotExist( - 'input `{}` does not exist in the bigchain'.format( - transaction['transaction']['input'])) + if not tx_input: + raise exceptions.TransactionDoesNotExist( + 'input `{}` does not exist in the bigchain'.format( + transaction['transaction']['input'])) - if (tx_input['transaction']['new_owner'] != - transaction['transaction']['current_owner']): - raise exceptions.TransactionOwnerError( - 'current_owner `{}` does not own the input `{}`'.format( - transaction['transaction']['current_owner'], - transaction['transaction']['input'])) + if (tx_input['transaction']['new_owner'] != + transaction['transaction']['current_owner']): + raise exceptions.TransactionOwnerError( + 'current_owner `{}` does not own the input `{}`'.format( + transaction['transaction']['current_owner'], + transaction['transaction']['input'])) - # check if the input was already spent by a transaction other than - # this one. - spent = bigchain.get_spent(tx_input['id']) - if spent and spent['id'] != transaction['id']: - raise exceptions.DoubleSpend( - 'input `{}` was already spent'.format( - transaction['transaction']['input'])) + # check if the input was already spent by a transaction other than + # this one. + spent = bigchain.get_spent(tx_input['id']) + if spent and spent['id'] != transaction['id']: + raise exceptions.DoubleSpend( + 'input `{}` was already spent'.format( + transaction['transaction']['input'])) # Check hash of the transaction calculated_hash = hash_data(util.serialize( - transaction['transaction'])) + transaction['transaction'])) if calculated_hash != transaction['id']: - raise exceptions.InvalidHash() + raise exceptions.InvalidHash() # Check signature - if not bigchain.verify_signature(transaction): - raise exceptions.InvalidSignature() + if not util.verify_signature(transaction): + raise exceptions.InvalidSignature() return transaction - # TODO: check that the votings structure is correctly constructed - @classmethod - def validate_block(cls, bigchain, block): + # TODO: Unsure if a bigchain parameter is really necessary here? + @staticmethod + def validate_block(bigchain, block): """Validate a block. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - block (dict): block to validate. + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. Raises: - InvalidHash: if the hash of the block is wrong. + InvalidHash: if the hash of the block is wrong. """ # Check if current hash is correct calculated_hash = hash_data(util.serialize(block['block'])) if calculated_hash != block['id']: - raise exceptions.InvalidHash() + raise exceptions.InvalidHash() return block + + @staticmethod + def create_transaction(current_owner, new_owner, tx_input, operation, + payload=None): + """Create a new transaction + + Refer to the documentation of ``bigchaindb.util.create_tx`` + """ + + return util.create_tx(current_owner, new_owner, tx_input, operation, + payload) + + @staticmethod + def sign_transaction(transaction, private_key): + """Sign a transaction + + Refer to the documentation of ``bigchaindb.util.sign_tx`` + """ + + return util.sign_tx(transaction, private_key) + + @staticmethod + def verify_signature(signed_transaction): + """Verify the signature of a transaction. + + Refer to the documentation of ``bigchaindb.util.verify_signature`` + """ + + return util.verify_signature(signed_transaction) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index ffdffa9b..f76dd2d6 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -27,7 +27,7 @@ class Bigchain(object): def __init__(self, host=None, port=None, dbname=None, public_key=None, private_key=None, keyring=[], - consensus_plugins=['base']): + consensus_plugin=None): """Initialize the Bigchain instance There are three ways in which the Bigchain instance can get its parameters. @@ -53,7 +53,7 @@ class Bigchain(object): self.me = public_key or bigchaindb.config['keypair']['public'] self.me_private = private_key or bigchaindb.config['keypair']['private'] self.federation_nodes = keyring or bigchaindb.config['keyring'] - self.consensus_plugins = config_utils.get_plugins(consensus_plugins) + self.consensus = config_utils.load_consensus_plugin(consensus_plugin) if not self.me or not self.me_private: raise exceptions.KeypairNotFoundException() @@ -70,38 +70,40 @@ class Bigchain(object): return r.connect(host=self.host, port=self.port, db=self.dbname) @monitor.timer('create_transaction', rate=bigchaindb.config['statsd']['rate']) - def create_transaction(self, current_owner, new_owner, tx_input, operation, payload=None): + def create_transaction(self, *args, **kwargs): """Create a new transaction - Refer to the documentation of ``bigchaindb.util.create_tx`` + Refer to the documentation of your consensus plugin. + + Returns: + dict: newly constructed transaction. """ - return util.create_tx(current_owner, new_owner, tx_input, operation, payload) + return self.consensus.create_transaction(*args, **kwargs) - def sign_transaction(self, transaction, private_key): + def sign_transaction(self, transaction, *args, **kwargs): """Sign a transaction - Refer to the documentation of ``bigchaindb.util.sign_tx`` + Refer to the documentation of your consensus plugin. + + Returns: + dict: transaction with any signatures applied. """ - return util.sign_tx(transaction, private_key) + return self.consensus.sign_transaction(transaction, *args, **kwargs) - def verify_signature(self, signed_transaction): - """Verify the signature of a transaction. + def verify_signature(self, signed_transaction, *args, **kwargs): + """Verify the signature(s) of a transaction. - Refer to the documentation of ``bigchaindb.crypto.verify_signature`` + Refer to the documentation of your consensus plugin. + + Returns: + bool: True if the transaction's required signature data is present + and correct, False otherwise. """ - data = signed_transaction.copy() - - # if assignee field in the transaction, remove it - if 'assignee' in data: - data.pop('assignee') - - signature = data.pop('signature') - public_key_base58 = signed_transaction['transaction']['current_owner'] - public_key = crypto.PublicKey(public_key_base58) - return public_key.verify(util.serialize(data), signature) + return self.consensus.verify_signature( + signed_transaction, *args, **kwargs) @monitor.timer('write_transaction', rate=bigchaindb.config['statsd']['rate']) def write_transaction(self, signed_transaction, durability='soft'): @@ -111,7 +113,7 @@ class Bigchain(object): it has been validated by the nodes of the federation. Args: - singed_transaction (dict): transaction with the `signature` included. + signed_transaction (dict): transaction with the `signature` included. Returns: dict: database response @@ -250,11 +252,7 @@ class Bigchain(object): exception describing the reason why the transaction is invalid. """ - for plugin in self.consensus_plugins: - transaction = plugin.validate_transaction(self, transaction) - - return transaction - + return self.consensus.validate_transaction(self, transaction) def is_valid_transaction(self, transaction): """Check whether a transacion is valid or invalid. @@ -324,10 +322,8 @@ class Bigchain(object): describing the reason why the block is invalid. """ - # First run all of the plugin block validation logic - for plugin in self.consensus_plugins: - transaction = plugin.validate_block(self, block) - + # First: Run the plugin block validation logic + self.consensus.validate_block(self, block) # Finally: Tentative assumption that every blockchain will want to # validate all transactions in each block @@ -372,6 +368,8 @@ class Bigchain(object): response = r.table('bigchain').get_all(transaction_id, index='transaction_id').run(self.conn) return True if len(response.items) > 0 else False + # TODO: Unless we prescribe the signature of create_transaction, this will + # also need to be moved into the plugin API. def create_genesis_block(self): """Create the genesis block diff --git a/setup.py b/setup.py index 475cb3cc..f6fb0c43 100644 --- a/setup.py +++ b/setup.py @@ -67,8 +67,8 @@ setup( 'bigchaindb=bigchaindb.commands.bigchain:main', 'bigchaindb-benchmark=bigchaindb.commands.bigchain_benchmark:main' ], - 'bigchaindb.plugins': [ - 'base=bigchaindb.consensus.base:ConsensusRules' + 'bigchaindb.consensus': [ + 'default=bigchaindb.consensus:BaseConsensusRules' ] }, install_requires=[ From 79a66bdf9b7212c34b2a2f453a72913a54ecb322 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:42:26 -0800 Subject: [PATCH 08/16] Added consensus doc stub --- docs/source/consensus.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docs/source/consensus.md diff --git a/docs/source/consensus.md b/docs/source/consensus.md new file mode 100644 index 00000000..b753068d --- /dev/null +++ b/docs/source/consensus.md @@ -0,0 +1,27 @@ +# BigchainDB Consensus Plugins +TODO: Write this section + +A quick example of a plugin that adds nonsense rules: + +```python +from bigchaindb.consensus import BaseConsensusRules + +class SillyConsensusRules(BaseConsensusRules): + + @staticmethod + def validate_transaction(bigchain, transaction): + transaction = super().validate_transaction(bigchain, transaction) + # I only like transactions whose timestamps are even. + if transaction['transaction']['timestamp'] % 2 != 0: + raise StandardError("Odd... very odd indeed.") + return transaction + + @staticmethod + def validate_block(bigchain, block): + block = super().validate_block(bigchain, block) + # I don't trust Alice, I think she's shady. + if block['block']['node_pubkey'] == '': + raise StandardError("Alice is shady, everybody ignore her blocks!") + + return block +``` From 037b5110fea7ff071810aec32f96502187b6ffd3 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sun, 13 Mar 2016 22:26:35 -0700 Subject: [PATCH 09/16] Modified Client module and web views to use plublic consensus methods for creating/signing/verifying instead of the functions in utils.py -- those are now only called by the the BaseConsensusRules class in consensus.py. --- bigchaindb/client.py | 33 ++++++++++++++++++++++++++------- bigchaindb/web/views.py | 5 +++-- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/bigchaindb/client.py b/bigchaindb/client.py index 0a9eed01..f7b37ad4 100644 --- a/bigchaindb/client.py +++ b/bigchaindb/client.py @@ -15,7 +15,8 @@ class Client: In the future, a Client might connect to >1 hosts. """ - def __init__(self, public_key=None, private_key=None, api_endpoint=None): + def __init__(self, public_key=None, private_key=None, api_endpoint=None, + consensus_plugin=None): """Initialize the Client instance There are three ways in which the Client instance can get its parameters. @@ -28,8 +29,11 @@ class Client: Args: public_key (str): the base58 encoded public key for the ECDSA secp256k1 curve. private_key (str): the base58 encoded private key for the ECDSA secp256k1 curve. - host (str): hostname where the rethinkdb is running. - port (int): port in which rethinkb is running (usually 28015). + api_endpoint (str): a URL where rethinkdb is running. + format: scheme://hostname:port + consensus_plugin (str): the registered name of your installed + consensus plugin. The `core` plugin is built into BigchainDB; + others must be installed via pip. """ config_utils.autoconfigure() @@ -37,6 +41,7 @@ class Client: self.public_key = public_key or bigchaindb.config['keypair']['public'] self.private_key = private_key or bigchaindb.config['keypair']['private'] self.api_endpoint = api_endpoint or bigchaindb.config['api_endpoint'] + self.consensus = config_utils.load_consensus_plugin(consensus_plugin) if not self.public_key or not self.private_key: raise exceptions.KeypairNotFoundException() @@ -51,8 +56,15 @@ class Client: The transaction pushed to the Federation. """ - tx = util.create_tx(self.public_key, self.public_key, None, operation='CREATE', payload=payload) - signed_tx = util.sign_tx(tx, self.private_key) + tx = self.consensus.create_transaction( + current_owner=self.public_key, + new_owner=self.public_key, + tx_input=None, + operation='CREATE', + payload=payload) + + signed_tx = self.consensus.sign_transaction( + tx, private_key=self.private_key) return self._push(signed_tx) def transfer(self, new_owner, tx_input, payload=None): @@ -67,8 +79,15 @@ class Client: The transaction pushed to the Federation. """ - tx = util.create_tx(self.public_key, new_owner, tx_input, operation='TRANSFER', payload=payload) - signed_tx = util.sign_tx(tx, self.private_key) + tx = self.consensus.create_transaction( + current_owner=self.public_key, + new_owner=new_owner, + tx_input=tx_input, + operation='TRANSFER', + payload=payload) + + signed_tx = self.consensus.sign_transaction( + tx, private_key=self.private_key) return self._push(signed_tx) def _push(self, tx): diff --git a/bigchaindb/web/views.py b/bigchaindb/web/views.py index 04d3992d..53db6f66 100644 --- a/bigchaindb/web/views.py +++ b/bigchaindb/web/views.py @@ -57,9 +57,10 @@ def create_transaction(): if tx['transaction']['operation'] == 'CREATE': tx = util.transform_create(tx) - tx = util.sign_tx(tx, bigchain.me_private) + tx = bigchain.consensus.sign_transaction( + tx, private_key=bigchain.me_private) - if not util.verify_signature(tx): + if not bigchain.consensus.verify_signature(tx): val['error'] = 'Invalid transaction signature' val = bigchain.write_transaction(tx) From 7253e96796fbdb48e85b0f6cf18485a70607cee5 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sun, 13 Mar 2016 23:28:24 -0700 Subject: [PATCH 10/16] Fleshed out consensus plugin docs --- docs/source/consensus.md | 61 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/docs/source/consensus.md b/docs/source/consensus.md index b753068d..552ce0d6 100644 --- a/docs/source/consensus.md +++ b/docs/source/consensus.md @@ -1,7 +1,50 @@ # BigchainDB Consensus Plugins -TODO: Write this section -A quick example of a plugin that adds nonsense rules: +BigchainDB has a pluggable block/transaction validation architecture. The default consensus rules can be extended or replaced entirely. + + +## Installing a plugin + +Plugins can be installed via pip! + +```bash +$ pip install bigchaindb-plugin-demo +``` + +Or using setuptools: + +```bash +$ cd bigchaindb-plugin-demo/ +$ python setup.py install # (or develop) +``` + +To activate your plugin, you can either set the `consensus_plugin` field in your config file (usually `~/.bigchaindb`) or by setting the `BIGCHAIN_CONSENSUS_PLUGIN` environement variable to the name of your plugin (see the section on [Packaging a plugin](#packaging-a-plugin) for more about plugin names). + + +## Plugin API + +BigchainDB's [current plugin API](../../bigchaindb/consensus.py) exposes five functions in an `AbstractConsensusRules` class: + +```python +validate_transaction(bigchain, transaction) +validate_block(bigchain, block) +create_transaction(*args, **kwargs) +sign_transaction(transaction, *args, **kwargs) +verify_signature(transaction) +``` + +Together, these functions are sufficient for most customizations. For example: +- Replace the crypto-system with one your hardware can accelerate +- Re-implement an existing protocol +- Delegate validation to another application +- etc... + + +## Extending BigchainDB behavior + +A default installation of BigchainDB will use the rules in the `BaseConsensusRules` class. If you only need to modify this behavior slightly, you can inherit from that class and call `super()` in any methods you change, so long as the return values remain the same. + +Here's a quick example of a plugin that adds nonsense rules: ```python from bigchaindb.consensus import BaseConsensusRules @@ -22,6 +65,18 @@ class SillyConsensusRules(BaseConsensusRules): # I don't trust Alice, I think she's shady. if block['block']['node_pubkey'] == '': raise StandardError("Alice is shady, everybody ignore her blocks!") - return block ``` + + +## Packaging a plugin + +BigchainDB uses [setuptool's entry_point](https://pythonhosted.org/setuptools/setuptools.html) system to provide the plugin functionality. Any custom plugin needs to add this section to the `setup()` call in their `setup.py`: + +```python +entry_points={ + 'bigchaindb.consensus': [ + 'PLUGIN_NAME=package.module:ConsensusRulesClass' + ] +}, +``` From ad535c78951f42281e8fd10a1a0be06155930d0e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 14 Mar 2016 13:27:17 -0700 Subject: [PATCH 11/16] Tests for loading consensus plugins --- tests/test_core.py | 6 ++++++ tests/utils/test_config_utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 9857b37f..f136f4e3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -15,6 +15,7 @@ def config(request, monkeypatch): }, 'keyring': [], 'CONFIGURED': True, + 'consensus_plugin': 'default' } monkeypatch.setattr('bigchaindb.config', config) @@ -24,6 +25,7 @@ def config(request, monkeypatch): def test_bigchain_class_default_initialization(config): from bigchaindb.core import Bigchain + from bigchaindb.consensus import BaseConsensusRules bigchain = Bigchain() assert bigchain.host == config['database']['host'] assert bigchain.port == config['database']['port'] @@ -31,11 +33,13 @@ def test_bigchain_class_default_initialization(config): assert bigchain.me == config['keypair']['public'] assert bigchain.me_private == config['keypair']['private'] assert bigchain.federation_nodes == config['keyring'] + assert bigchain.consensus == BaseConsensusRules assert bigchain._conn is None def test_bigchain_class_initialization_with_parameters(config): from bigchaindb.core import Bigchain + from bigchaindb.consensus import BaseConsensusRules init_kwargs = { 'host': 'some_node', 'port': '12345', @@ -43,6 +47,7 @@ def test_bigchain_class_initialization_with_parameters(config): 'public_key': 'white', 'private_key': 'black', 'keyring': ['key_one', 'key_two'], + 'consensus_plugin': 'default' } bigchain = Bigchain(**init_kwargs) assert bigchain.host == init_kwargs['host'] @@ -51,4 +56,5 @@ def test_bigchain_class_initialization_with_parameters(config): assert bigchain.me == init_kwargs['public_key'] assert bigchain.me_private == init_kwargs['private_key'] assert bigchain.federation_nodes == init_kwargs['keyring'] + assert bigchain.consensus == BaseConsensusRules assert bigchain._conn is None diff --git a/tests/utils/test_config_utils.py b/tests/utils/test_config_utils.py index 2cf57242..fa63158d 100644 --- a/tests/utils/test_config_utils.py +++ b/tests/utils/test_config_utils.py @@ -37,3 +37,32 @@ def test_bigchain_instance_raises_when_not_configured(monkeypatch): with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() + + +def test_load_consensus_plugin_loads_default_rules_without_name(): + from bigchaindb import config_utils + from bigchaindb.consensus import BaseConsensusRules + + assert config_utils.load_consensus_plugin() == BaseConsensusRules + + +def test_load_consensus_plugin_raises_with_unknown_name(): + from pkg_resources import ResolutionError + from bigchaindb import config_utils + + with pytest.raises(ResolutionError): + config_utils.load_consensus_plugin('bogus') + + +def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): + # Monkeypatch entry_point.load to return something other than a + # ConsensusRules instance + from bigchaindb import config_utils + monkeypatch.setattr(config_utils, + 'iter_entry_points', + lambda *args: [ type('entry_point', + (object), + {'load': lambda: object}) ]) + + with pytest.raises(TypeError): + config_utils.load_consensus_plugin() From 3d50459b84a462b4819f9389a23b12a203817658 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 14 Mar 2016 20:02:18 -0700 Subject: [PATCH 12/16] Corrected plugin group name in comments --- bigchaindb/config_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 26afa3c8..645780eb 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -117,7 +117,7 @@ def load_consensus_plugin(name=None): if not name: name = bigchaindb.config.get('consensus_plugin', 'default') - # TODO: This will return the first plugin with group `bigchaindb.plugins` + # TODO: This will return the first plugin with group `bigchaindb.consensus` # and name `name` in the active WorkingSet. # We should probably support Requirements specs in the config, e.g. # consensus_plugin: 'my-plugin-package==0.0.1;default' @@ -128,7 +128,7 @@ def load_consensus_plugin(name=None): # No matching entry_point found if not plugin: raise ResolutionError( - 'No plugin found in group `bigchaindb.plugins` with name `{}`'. + 'No plugin found in group `bigchaindb.consensus` with name `{}`'. format(name)) # Is this strictness desireable? From 678cd2dd2968ceeaa65c70a6916a591289f369ae Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 17 Mar 2016 11:05:15 +0100 Subject: [PATCH 13/16] Updated ROADMAP.md based on March 16 meeting --- ROADMAP.md | 97 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 19 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 79fadf02..09f1b7ac 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,30 +5,89 @@ See also: * [Milestones](https://github.com/bigchaindb/bigchaindb/milestones) (i.e. issues to be closed before various releases) * [Open issues](https://github.com/bigchaindb/bigchaindb/issues) and [open pull requests](https://github.com/bigchaindb/bigchaindb/pulls) -## BigchainDB Protocols -* Validation of other nodes -* Fault tolerance -* Permissions framework -* Protocol audits including security audits +Note: Below, #345 refers to Issue #345 in the BigchainDB repository on GitHub. #N refers to Issue #N. -## HTTP Client-Server API -* Validate the structure of the transaction + +## Deployment and Federation Configuration/Management +* Define how a federation is managed - [#126](https://github.com/bigchaindb/bigchaindb/issues/126) +* Review current configuration mechanism - [#49](https://github.com/bigchaindb/bigchaindb/issues/49) +* Make the configuration easier for Docker-based setup - [#36](https://github.com/bigchaindb/bigchaindb/issues/36) + + +## Testing +* (Unit-test writing and unit testing are ongoing.) +* More Integration Testing, Validation Testing, System Testing, Benchmarking +* Define some standard test systems (e.g. local virtual cluster, data center, WAN) +* Develop standardized test descriptions and documentation (test setup, inputs, outputs) +* Build up a suite of tests to test each identified fault +* More tools for cluster benchmarking +* Identify bottlenecks using profiling and monitoring +* Fault-testing framework +* Clean exit for the bigchaindb-benchmark process - [#122](https://github.com/bigchaindb/bigchaindb/issues/122) +* Tool to bulk-upload transactions into biacklog table - [#114](https://github.com/bigchaindb/bigchaindb/issues/114) +* Tool to deploy multiple clients for testing - [#113](https://github.com/bigchaindb/bigchaindb/issues/113) +* Tool to read transactions from files for testing - [#112](https://github.com/bigchaindb/bigchaindb/issues/112) + + +## Specific Bugs/Faults and Related Tests +* Validation of other nodes +* Changefeed watchdog +* Non-deterministic assignment of tx in S is a DoS vulnerability - [#20](https://github.com/bigchaindb/bigchaindb/issues/20) +* Queues are unbounded - [#124](https://github.com/bigchaindb/bigchaindb/issues/124) +* Better handling of timeouts in block creation - [#123](https://github.com/bigchaindb/bigchaindb/issues/123) +* Secure node-node communication - [#77](https://github.com/bigchaindb/bigchaindb/issues/77) +* Checking if transactions are in a decided_valid block (or otherwise) when necessary - [#134](https://github.com/bigchaindb/bigchaindb/issues/134) +* When validating an incoming transaction, check to ensure it isn't a duplicate - [#131](https://github.com/bigchaindb/bigchaindb/issues/131) +* Consider secondary indexes on some queries - [#105](https://github.com/bigchaindb/bigchaindb/issues/105) + + +## Transactions / Assets +* Current Top-Level Goal: Define and implement "v2 transactions", that is, support multisig (done) and: +* Support for multiple inputs and outputs - [#128](https://github.com/bigchaindb/bigchaindb/issues/128) +* Crypto-conditions specific to ILP - [#127](https://github.com/bigchaindb/bigchaindb/issues/127) +* Support divisible assets - [#129](https://github.com/bigchaindb/bigchaindb/issues/129) +* Define a JSON template for digital assets - [#125](https://github.com/bigchaindb/bigchaindb/issues/125) +* Revisit timestamps - [#132](https://github.com/bigchaindb/bigchaindb/issues/132) +* Refactor structure of a transaction - [#98](https://github.com/bigchaindb/bigchaindb/issues/98) +* Plugin or hook architecture e.g. for validate_transaction - [#90](https://github.com/bigchaindb/bigchaindb/issues/90) + + +## Web API (HTTP Client-Server API) +* Current Top-Level Goal: Support v2 transactions (see above) +* Validate the structure of incoming transactions * Return the correct error code if something goes wrong +* Validate transaction before writing it to the backlog - [#109](https://github.com/bigchaindb/bigchaindb/issues/109) +* Better organization of transaction-related code - [#108](https://github.com/bigchaindb/bigchaindb/issues/108) * Add an endpoint to query unspents for a given public key * More endpoints -* See [the relevant open issues](https://github.com/bigchaindb/bigchaindb/issues?q=is%3Aissue+is%3Aopen+label%3Arest-api) +* See [open issues with the "rest-api" label](https://github.com/bigchaindb/bigchaindb/issues?q=is%3Aissue+is%3Aopen+label%3Arest-api) -## Implementation/Code -* Node validation framework (inspect and agree or not with what the other nodes are doing) -* Open public testing cluster (for people to try out a BigchainDB cluster and to test client software) -* Federation management tools -* More tools for benchmarking a cluster -* Descriptions and results of more benchmarking tests -* AWS image and other easy deployment options + +## Drivers +* Update the reference driver (Python) to support v2 transactions and web API (see above) * Drivers/SDKs for more client-side languages (e.g. JavaScript, Ruby, Java) -* ORM to better-decouple BigchainDB from its data store (will make it easy to try other databases) -* Code audits including security audits -## Other/Future -* Byzantine fault tolerance + +## Public Testnet and Public BigchainDB +* Deploy a 3-node public testnet in a data center, open to all external users, refreshing daily +* Deploy Public BigchaindB Testnet with more than 3 nodes and with nodes more globally-distributed +* Public BigchainDB governance/voting system +* Transaction (or usage) accounting +* Billing system + + +## Other +* Get BigchainDB production-ready for submission to AWS Marketplace (as an AMI) + + +## Future +* Permissions framework +* More Byzantine fault tolerance (BFT) * Better support for smart contract frameworks +* Algorithm audits +* Protocol audits +* Code (implementation) audits +* Security audits +* IPFS interoperability - [#100](https://github.com/bigchaindb/bigchaindb/issues/100) +* ORM to better-decouple BigchainDB from its data store (will make it easy to try other databases) +* Support more server operating systems From 785ee388fefba8fb1769dc0de0330f30ea9bc891 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 17 Mar 2016 16:20:11 +0100 Subject: [PATCH 14/16] Move info on running all unit tests into the Sphinx docs --- PYTHON_STYLE_GUIDE.md | 42 +++---------------------------- docs/source/index.rst | 1 + docs/source/running-unit-tests.md | 41 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 docs/source/running-unit-tests.md diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index dc7e68d7..4d4d4888 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -57,46 +57,10 @@ we use the `format()` version. The [official Python documentation says](https:// ## Writing (Python) Tests -We write tests for our Python code using the [pytest](http://pytest.org/latest/) framework. +We write unit tests for our Python code using the [pytest](http://pytest.org/latest/) framework. All tests go in the `bigchaindb/tests` directory or one of its subdirectories. You can use the tests already in there as templates or examples. -### Standard Ways to Run All Tests +The BigchainDB Documentation has a [section explaining how to run all unit tests](http://bigchaindb.readthedocs.org/en/develop/running-unit-tests.html). -To run all the tests, first make sure you have RethinkDB running: -```text -$ rethinkdb -``` - -then in another terminal, do: -```text -$ py.test -v -``` - -If that doesn't work (e.g. maybe you are running in a conda virtual environment), try: -```text -$ python -m pytest -v -``` - -You can also run all tests via `setup.py`, using: -```text -$ python setup.py test -``` - -### Using `docker-compose` to Run the Tests - -You can use `docker-compose` to run the tests. (You don't have to start RethinkDB first: `docker-compose` does that on its own, when it reads the `docker-compose.yml` file.) - -First, build the images (~once), using: -```text -$ docker-compose build -``` - -then run the tests using: -```text -$ docker-compose run --rm bigchaindb py.test -v -``` - -### Automated Testing of All Pull Requests - -We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) +**Automated testing of pull requests.** We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) diff --git a/docs/source/index.rst b/docs/source/index.rst index 44523c7d..fd2c991e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,6 +14,7 @@ Table of Contents introduction installing-server + running-unit-tests python-server-api-examples bigchaindb-cli http-client-server-api diff --git a/docs/source/running-unit-tests.md b/docs/source/running-unit-tests.md new file mode 100644 index 00000000..9e53dc77 --- /dev/null +++ b/docs/source/running-unit-tests.md @@ -0,0 +1,41 @@ +# Running Unit Tests + +Once you've installed BigchainDB Server, you may want to run all the unit tests. This section explains how. + +First of all, if you installed BigchainDB Server using `pip` (i.e. by getting the package from PyPI), then you didn't install the tests. Before you can run all the unit tests, you must [install BigchainDB from source](http://bigchaindb.readthedocs.org/en/develop/installing-server.html#how-to-install-bigchaindb-from-source). + +To run all the unit tests, first make sure you have RethinkDB running: +```text +$ rethinkdb +``` + +then in another terminal, do: +```text +$ py.test -v +``` + +If the above command doesn't work (e.g. maybe you are running in a conda virtual environment), try: +```text +$ python -m pytest -v +``` + +(We write our unit tests using the [pytest](http://pytest.org/latest/) framework.) + +You can also run all unit tests via `setup.py`, using: +```text +$ python setup.py test +``` + +### Using `docker-compose` to Run the Tests + +You can also use `docker-compose` to run the unit tests. (You don't have to start RethinkDB first: `docker-compose` does that on its own, when it reads the `docker-compose.yml` file.) + +First, build the images (~once), using: +```text +$ docker-compose build +``` + +then run the unit tests using: +```text +$ docker-compose run --rm bigchaindb py.test -v +``` From cb5c14f9f1f35965b6182248e35cf2a1916fabc1 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 18 Mar 2016 11:55:20 +0100 Subject: [PATCH 15/16] Public Testnet --> Public Sandbox Testnet --- ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 09f1b7ac..9d319994 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -68,8 +68,8 @@ Note: Below, #345 refers to Issue #345 in the BigchainDB repository on GitHub. # * Drivers/SDKs for more client-side languages (e.g. JavaScript, Ruby, Java) -## Public Testnet and Public BigchainDB -* Deploy a 3-node public testnet in a data center, open to all external users, refreshing daily +## Public Sandbox Testnet and Public BigchainDB +* Deploy a 3-node Public Sandbox Testnet in a data center, open to all external users, refreshing daily * Deploy Public BigchaindB Testnet with more than 3 nodes and with nodes more globally-distributed * Public BigchainDB governance/voting system * Transaction (or usage) accounting From 2aa003578ccec83e13e1abf810c1eba5f97d690c Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 18 Mar 2016 16:50:21 -0700 Subject: [PATCH 16/16] Typos --- bigchaindb/consensus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 818249b8..b65ab9ad 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -41,7 +41,7 @@ class AbstractConsensusRules(metaclass=ABCMeta): block (dict): block to validate. Returns: - The block if the block is valid else it raises and exception + The block if the block is valid else it raises an exception describing the reason why the block is invalid. Raises: @@ -177,7 +177,7 @@ class BaseConsensusRules(AbstractConsensusRules): block (dict): block to validate. Returns: - The block if the block is valid else it raises and exception + The block if the block is valid else it raises an exception describing the reason why the block is invalid. Raises: