From 8a9030e5c0269d989221446d1e3ec8f105a2df9c Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 18:38:33 -0800 Subject: [PATCH 01/22] Added a simple plugin system for consensus rules using setuputils entry_points --- bigchaindb/__init__.py | 5 ++++- bigchaindb/config_utils.py | 20 ++++++++++++++++++++ bigchaindb/consensus/__init__.py | 0 bigchaindb/core.py | 4 +++- setup.py | 10 +++++++++- 5 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 bigchaindb/consensus/__init__.py diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 4bb76c44..b728e103 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -39,7 +39,10 @@ config = { 'host': e('BIGCHAIN_STATSD_HOST', default='localhost'), 'port': e('BIGCHAIN_STATSD_PORT', default=8125), 'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01) - } + }, + 'consensus_plugins': [ + 'base' + ], } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index bf3ed529..c273245d 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -17,6 +17,8 @@ import json import logging import collections +from pkg_resources import iter_entry_points + import bigchaindb logger = logging.getLogger(__name__) @@ -100,3 +102,21 @@ def autoconfigure(): except FileNotFoundError: logger.warning('Cannot find your config file. Run `bigchaindb configure` to create one') + +def get_plugins(plugin_names): + if not plugin_names: + plugin_names = bigchaindb.config.get('consensus_plugins', []) + + plugins = [] + + # It's important to maintain plugin ordering as stated in the config file. + # e.g. Expensive validation tasks should happen after cheap ones. + # + # TODO: We might want to add some sort of priority system, but for now we + # simply assume everything in a given plugin is designed to run at the + # same time. + for name in plugin_names: + for entry_point in iter_entry_points('bigchaindb.plugins', name): + plugins.append(entry_point.load()) + + return plugins diff --git a/bigchaindb/consensus/__init__.py b/bigchaindb/consensus/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bigchaindb/core.py b/bigchaindb/core.py index f2ba61dd..89cccd7e 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -30,7 +30,8 @@ class Bigchain(object): """ def __init__(self, host=None, port=None, dbname=None, - public_key=None, private_key=None, keyring=[]): + public_key=None, private_key=None, keyring=[], + consensus_plugins=['base']): """Initialize the Bigchain instance There are three ways in which the Bigchain instance can get its parameters. @@ -56,6 +57,7 @@ class Bigchain(object): self.me = public_key or bigchaindb.config['keypair']['public'] self.me_private = private_key or bigchaindb.config['keypair']['private'] self.federation_nodes = keyring or bigchaindb.config['keyring'] + self.consensus_plugins = config_utils.get_plugins(consensus_plugins) if not self.me or not self.me_private: raise KeypairNotFoundException() diff --git a/setup.py b/setup.py index 3fd8dae4..26f111fa 100644 --- a/setup.py +++ b/setup.py @@ -54,13 +54,21 @@ setup( 'Operating System :: POSIX :: Linux', ], - packages=['bigchaindb', 'bigchaindb.commands', 'bigchaindb.db'], + packages=[ + 'bigchaindb', + 'bigchaindb.commands', + 'bigchaindb.db', + 'bigchaindb.consensus' + ], entry_points={ 'console_scripts': [ 'bigchaindb=bigchaindb.commands.bigchain:main', 'bigchaindb-benchmark=bigchaindb.commands.bigchain_benchmark:main' ], + 'bigchaindb.plugins': [ + 'base=bigchaindb.consensus.base:ConsensusRules' + ] }, install_requires=[ 'rethinkdb==2.2.0.post4', From ee4720d1a59b6fdaadd27a8b170818b8ef0fef1a Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 18:39:23 -0800 Subject: [PATCH 02/22] Pulled validate_transaction and validate_block out into a base ConsensusRules plugin class --- bigchaindb/consensus/base.py | 107 +++++++++++++++++++++++++++++++++++ bigchaindb/core.py | 62 ++++---------------- 2 files changed, 118 insertions(+), 51 deletions(-) create mode 100644 bigchaindb/consensus/base.py diff --git a/bigchaindb/consensus/base.py b/bigchaindb/consensus/base.py new file mode 100644 index 00000000..1943c543 --- /dev/null +++ b/bigchaindb/consensus/base.py @@ -0,0 +1,107 @@ +import bigchaindb.exceptions as exceptions +from bigchaindb.crypto import hash_data + +class ConsensusRules(object): + """Base consensus rules for Bigchain. + + This class can be copied to write your own consensus rules! + + Note: Consensus plugins will be executed in the order that they're listed in + the bigchain config file. + """ + + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not found + TransactionOwnerError: if the new transaction is using an input it doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong + """ + + # If the operation is CREATE the transaction should have no inputs and + # should be signed by a federation node + if transaction['transaction']['operation'] == 'CREATE': + if transaction['transaction']['input']: + raise ValueError('A CREATE operation has no inputs') + if transaction['transaction']['current_owner'] not in ( + bigchain.federation_nodes + [bigchain.me]): + raise exceptions.OperationError( + 'Only federation nodes can use the operation `CREATE`') + + else: + # check if the input exists, is owned by the current_owner + if not transaction['transaction']['input']: + raise ValueError( + 'Only `CREATE` transactions can have null inputs') + + tx_input = bigchain.get_transaction( + transaction['transaction']['input']) + + if not tx_input: + raise exceptions.TransactionDoesNotExist( + 'input `{}` does not exist in the bigchain'.format( + transaction['transaction']['input'])) + + if (tx_input['transaction']['new_owner'] != + transaction['transaction']['current_owner']): + raise exceptions.TransactionOwnerError( + 'current_owner `{}` does not own the input `{}`'.format( + transaction['transaction']['current_owner'], + transaction['transaction']['input'])) + + # check if the input was already spent by a transaction other than + # this one. + spent = bigchain.get_spent(tx_input['id']) + if spent and spent['id'] != transaction['id']: + raise exceptions.DoubleSpend( + 'input `{}` was already spent'.format( + transaction['transaction']['input'])) + + # Check hash of the transaction + calculated_hash = hash_data(bigchain.serialize( + transaction['transaction'])) + if calculated_hash != transaction['id']: + raise exceptions.InvalidHash() + + # Check signature + if not bigchain.verify_signature(transaction): + raise exceptions.InvalidSignature() + + return transaction + + # TODO: check that the votings structure is correctly constructed + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + InvalidHash: if the hash of the block is wrong. + """ + + # Check if current hash is correct + calculated_hash = hash_data(bigchain.serialize(block['block'])) + if calculated_hash != block['id']: + raise exceptions.InvalidHash() + + return block diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 89cccd7e..7df5ed85 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -327,57 +327,16 @@ class Bigchain(object): transaction (dict): transaction to validate. Returns: - The transaction if the transaction is valid else it raises and exception - describing the reason why the transaction is invalid. - - Raises: - OperationError: if the transaction operation is not supported - TransactionDoesNotExist: if the input of the transaction is not found - TransactionOwnerError: if the new transaction is using an input it doesn't own - DoubleSpend: if the transaction is a double spend - InvalidHash: if the hash of the transaction is wrong - InvalidSignature: if the signature of the transaction is wrong + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. """ - # If the operation is CREATE the transaction should have no inputs and should be signed by a - # federation node - if transaction['transaction']['operation'] == 'CREATE': - if transaction['transaction']['input']: - raise ValueError('A CREATE operation has no inputs') - if transaction['transaction']['current_owner'] not in self.federation_nodes + [self.me]: - raise exceptions.OperationError('Only federation nodes can use the operation `CREATE`') - else: - # check if the input exists, is owned by the current_owner - if not transaction['transaction']['input']: - raise ValueError('Only `CREATE` transactions can have null inputs') - - tx_input = self.get_transaction(transaction['transaction']['input']) - if not tx_input: - raise exceptions.TransactionDoesNotExist('input `{}` does not exist in the bigchain'.format( - transaction['transaction']['input'])) - - if tx_input['transaction']['new_owner'] != transaction['transaction']['current_owner']: - raise exceptions.TransactionOwnerError('current_owner `{}` does not own the input `{}`'.format( - transaction['transaction']['current_owner'], transaction['transaction']['input'])) - - # check if the input was already spent by a transaction other then this one. - spent = self.get_spent(tx_input['id']) - if spent: - if spent['id'] != transaction['id']: - raise exceptions.DoubleSpend('input `{}` was already spent'.format( - transaction['transaction']['input'])) - - # Check hash of the transaction - calculated_hash = hash_data(self.serialize(transaction['transaction'])) - if calculated_hash != transaction['id']: - raise exceptions.InvalidHash() - - # Check signature - if not self.verify_signature(transaction): - raise exceptions.InvalidSignature() + for plugin in self.consensus_plugins: + transaction = plugin.validate_transaction(self, transaction) return transaction + def is_valid_transaction(self, transaction): """Check whether a transacion is valid or invalid. @@ -447,12 +406,13 @@ class Bigchain(object): """ - # 1. Check if current hash is correct - calculated_hash = hash_data(self.serialize(block['block'])) - if calculated_hash != block['id']: - raise exceptions.InvalidHash() + # First run all of the plugin block validation logic + for plugin in self.consensus_plugins: + transaction = plugin.validate_block(self, block) - # 2. Validate all transactions in the block + + # Finally: Tentative assumption that every blockchain will want to + # validate all transactions in each block for transaction in block['block']['transactions']: if not self.is_valid_transaction(transaction): # this will raise the exception From 380482b9674125d79be558373f43ef50ebaadf4b Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 19:19:08 -0800 Subject: [PATCH 03/22] Added an abstract base class so plugins don't need to add stubs for functions they don't care about. --- bigchaindb/consensus/__init__.py | 43 ++++++++++++++++++++++++++++++++ bigchaindb/consensus/base.py | 3 ++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/bigchaindb/consensus/__init__.py b/bigchaindb/consensus/__init__.py index e69de29b..24806bd0 100644 --- a/bigchaindb/consensus/__init__.py +++ b/bigchaindb/consensus/__init__.py @@ -0,0 +1,43 @@ +# TODO: no real reason to use abc yet, but later we can enforce inheritance from +# this class when loading plugins if that's desirable. +# from abc import ABCMeta + +class AbstractConsensusRules: + + # TODO: rather than having plugin-authors inherit and override, + # it'd be cleaner to make a `transactionrule` decorator and etc + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + Descriptive exceptions indicating the reason the transaction failed. + See the `exceptions` module for bigchain-native error classes. + """ + return transaction + + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + Descriptive exceptions indicating the reason the block failed. + See the `exceptions` module for bigchain-native error classes. + """ + return block diff --git a/bigchaindb/consensus/base.py b/bigchaindb/consensus/base.py index 1943c543..ab01d26e 100644 --- a/bigchaindb/consensus/base.py +++ b/bigchaindb/consensus/base.py @@ -1,7 +1,8 @@ import bigchaindb.exceptions as exceptions from bigchaindb.crypto import hash_data +from bigchaindb.consensus import AbstractConsensusRules -class ConsensusRules(object): +class ConsensusRules(AbstractConsensusRules): """Base consensus rules for Bigchain. This class can be copied to write your own consensus rules! From 2c8cea11ebb28f56a8c7ed49100fff04fd7330fb Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 29 Feb 2016 19:22:04 -0800 Subject: [PATCH 04/22] Added some nonsense consensus rules to demonstrate a simpler plugin. --- bigchaindb/consensus/silly.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 bigchaindb/consensus/silly.py diff --git a/bigchaindb/consensus/silly.py b/bigchaindb/consensus/silly.py new file mode 100644 index 00000000..879b1328 --- /dev/null +++ b/bigchaindb/consensus/silly.py @@ -0,0 +1,21 @@ +import bigchaindb.exceptions as exceptions +from bigchaindb.crypto import hash_data +from bigchaindb.consensus import AbstractConsensusRules + + +class SillyConsensusRules(AbstractConsensusRules): + + @classmethod + def validate_transaction(cls, bigchain, transaction): + # I only like transactions whose timestamps are even. + if transaction['transaction']['timestamp'] % 2 != 0: + raise StandardError("Odd... very odd indeed.") + return transaction + + @classmethod + def validate_block(cls, bigchain, transaction): + # I don't trust Alice, I think she's shady. + if block['block']['node_pubkey'] == '': + raise StandardError("Alice is shady, everybody ignore her blocks!") + + return block From 01d706ac56f44887c34ae294a8d67950af1020d0 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:24:21 -0800 Subject: [PATCH 05/22] Moved consensus package files into single consensus.py module --- bigchaindb/consensus.py | 152 +++++++++++++++++++++++++++++++ bigchaindb/consensus/__init__.py | 43 --------- bigchaindb/consensus/base.py | 109 ---------------------- bigchaindb/consensus/silly.py | 21 ----- setup.py | 3 +- 5 files changed, 153 insertions(+), 175 deletions(-) create mode 100644 bigchaindb/consensus.py delete mode 100644 bigchaindb/consensus/__init__.py delete mode 100644 bigchaindb/consensus/base.py delete mode 100644 bigchaindb/consensus/silly.py diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py new file mode 100644 index 00000000..4bf5fe66 --- /dev/null +++ b/bigchaindb/consensus.py @@ -0,0 +1,152 @@ +import bigchaindb.exceptions as exceptions +from bigchaindb import util +from bigchaindb.crypto import hash_data + +# TODO: no real reason to use abc yet, but later we can enforce inheritance from +# this class when loading plugins if that's desirable. +# from abc import ABCMeta + +class AbstractConsensusRules: + + # TODO: rather than having plugin-authors inherit and override, + # it'd be cleaner to make a `transactionrule` decorator and etc + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + Descriptive exceptions indicating the reason the transaction failed. + See the `exceptions` module for bigchain-native error classes. + """ + return transaction + + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + Descriptive exceptions indicating the reason the block failed. + See the `exceptions` module for bigchain-native error classes. + """ + return block + + class ConsensusRules(AbstractConsensusRules): + """Base consensus rules for Bigchain. + + This class can be copied to write your own consensus rules! + + Note: Consensus plugins will be executed in the order that they're listed in + the bigchain config file. + """ + + @classmethod + def validate_transaction(cls, bigchain, transaction): + """Validate a transaction. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. + + Returns: + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. + + Raises: + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not found + TransactionOwnerError: if the new transaction is using an input it doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong + """ + + # If the operation is CREATE the transaction should have no inputs and + # should be signed by a federation node + if transaction['transaction']['operation'] == 'CREATE': + if transaction['transaction']['input']: + raise ValueError('A CREATE operation has no inputs') + if transaction['transaction']['current_owner'] not in ( + bigchain.federation_nodes + [bigchain.me]): + raise exceptions.OperationError( + 'Only federation nodes can use the operation `CREATE`') + + else: + # check if the input exists, is owned by the current_owner + if not transaction['transaction']['input']: + raise ValueError( + 'Only `CREATE` transactions can have null inputs') + + tx_input = bigchain.get_transaction( + transaction['transaction']['input']) + + if not tx_input: + raise exceptions.TransactionDoesNotExist( + 'input `{}` does not exist in the bigchain'.format( + transaction['transaction']['input'])) + + if (tx_input['transaction']['new_owner'] != + transaction['transaction']['current_owner']): + raise exceptions.TransactionOwnerError( + 'current_owner `{}` does not own the input `{}`'.format( + transaction['transaction']['current_owner'], + transaction['transaction']['input'])) + + # check if the input was already spent by a transaction other than + # this one. + spent = bigchain.get_spent(tx_input['id']) + if spent and spent['id'] != transaction['id']: + raise exceptions.DoubleSpend( + 'input `{}` was already spent'.format( + transaction['transaction']['input'])) + + # Check hash of the transaction + calculated_hash = hash_data(util.serialize( + transaction['transaction'])) + if calculated_hash != transaction['id']: + raise exceptions.InvalidHash() + + # Check signature + if not bigchain.verify_signature(transaction): + raise exceptions.InvalidSignature() + + return transaction + + # TODO: check that the votings structure is correctly constructed + @classmethod + def validate_block(cls, bigchain, block): + """Validate a block. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. + + Returns: + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. + + Raises: + InvalidHash: if the hash of the block is wrong. + """ + + # Check if current hash is correct + calculated_hash = hash_data(util.serialize(block['block'])) + if calculated_hash != block['id']: + raise exceptions.InvalidHash() + + return block diff --git a/bigchaindb/consensus/__init__.py b/bigchaindb/consensus/__init__.py deleted file mode 100644 index 24806bd0..00000000 --- a/bigchaindb/consensus/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -# TODO: no real reason to use abc yet, but later we can enforce inheritance from -# this class when loading plugins if that's desirable. -# from abc import ABCMeta - -class AbstractConsensusRules: - - # TODO: rather than having plugin-authors inherit and override, - # it'd be cleaner to make a `transactionrule` decorator and etc - @classmethod - def validate_transaction(cls, bigchain, transaction): - """Validate a transaction. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - transaction (dict): transaction to validate. - - Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. - - Raises: - Descriptive exceptions indicating the reason the transaction failed. - See the `exceptions` module for bigchain-native error classes. - """ - return transaction - - @classmethod - def validate_block(cls, bigchain, block): - """Validate a block. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - block (dict): block to validate. - - Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. - - Raises: - Descriptive exceptions indicating the reason the block failed. - See the `exceptions` module for bigchain-native error classes. - """ - return block diff --git a/bigchaindb/consensus/base.py b/bigchaindb/consensus/base.py deleted file mode 100644 index ed3f05c9..00000000 --- a/bigchaindb/consensus/base.py +++ /dev/null @@ -1,109 +0,0 @@ -import bigchaindb.exceptions as exceptions -from bigchaindb import util -from bigchaindb.crypto import hash_data -from bigchaindb.consensus import AbstractConsensusRules - -class ConsensusRules(AbstractConsensusRules): - """Base consensus rules for Bigchain. - - This class can be copied to write your own consensus rules! - - Note: Consensus plugins will be executed in the order that they're listed in - the bigchain config file. - """ - - @classmethod - def validate_transaction(cls, bigchain, transaction): - """Validate a transaction. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - transaction (dict): transaction to validate. - - Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. - - Raises: - OperationError: if the transaction operation is not supported - TransactionDoesNotExist: if the input of the transaction is not found - TransactionOwnerError: if the new transaction is using an input it doesn't own - DoubleSpend: if the transaction is a double spend - InvalidHash: if the hash of the transaction is wrong - InvalidSignature: if the signature of the transaction is wrong - """ - - # If the operation is CREATE the transaction should have no inputs and - # should be signed by a federation node - if transaction['transaction']['operation'] == 'CREATE': - if transaction['transaction']['input']: - raise ValueError('A CREATE operation has no inputs') - if transaction['transaction']['current_owner'] not in ( - bigchain.federation_nodes + [bigchain.me]): - raise exceptions.OperationError( - 'Only federation nodes can use the operation `CREATE`') - - else: - # check if the input exists, is owned by the current_owner - if not transaction['transaction']['input']: - raise ValueError( - 'Only `CREATE` transactions can have null inputs') - - tx_input = bigchain.get_transaction( - transaction['transaction']['input']) - - if not tx_input: - raise exceptions.TransactionDoesNotExist( - 'input `{}` does not exist in the bigchain'.format( - transaction['transaction']['input'])) - - if (tx_input['transaction']['new_owner'] != - transaction['transaction']['current_owner']): - raise exceptions.TransactionOwnerError( - 'current_owner `{}` does not own the input `{}`'.format( - transaction['transaction']['current_owner'], - transaction['transaction']['input'])) - - # check if the input was already spent by a transaction other than - # this one. - spent = bigchain.get_spent(tx_input['id']) - if spent and spent['id'] != transaction['id']: - raise exceptions.DoubleSpend( - 'input `{}` was already spent'.format( - transaction['transaction']['input'])) - - # Check hash of the transaction - calculated_hash = hash_data(util.serialize( - transaction['transaction'])) - if calculated_hash != transaction['id']: - raise exceptions.InvalidHash() - - # Check signature - if not bigchain.verify_signature(transaction): - raise exceptions.InvalidSignature() - - return transaction - - # TODO: check that the votings structure is correctly constructed - @classmethod - def validate_block(cls, bigchain, block): - """Validate a block. - - Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - block (dict): block to validate. - - Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. - - Raises: - InvalidHash: if the hash of the block is wrong. - """ - - # Check if current hash is correct - calculated_hash = hash_data(util.serialize(block['block'])) - if calculated_hash != block['id']: - raise exceptions.InvalidHash() - - return block diff --git a/bigchaindb/consensus/silly.py b/bigchaindb/consensus/silly.py deleted file mode 100644 index 879b1328..00000000 --- a/bigchaindb/consensus/silly.py +++ /dev/null @@ -1,21 +0,0 @@ -import bigchaindb.exceptions as exceptions -from bigchaindb.crypto import hash_data -from bigchaindb.consensus import AbstractConsensusRules - - -class SillyConsensusRules(AbstractConsensusRules): - - @classmethod - def validate_transaction(cls, bigchain, transaction): - # I only like transactions whose timestamps are even. - if transaction['transaction']['timestamp'] % 2 != 0: - raise StandardError("Odd... very odd indeed.") - return transaction - - @classmethod - def validate_block(cls, bigchain, transaction): - # I don't trust Alice, I think she's shady. - if block['block']['node_pubkey'] == '': - raise StandardError("Alice is shady, everybody ignore her blocks!") - - return block diff --git a/setup.py b/setup.py index 083dc7a8..475cb3cc 100644 --- a/setup.py +++ b/setup.py @@ -59,8 +59,7 @@ setup( packages=[ 'bigchaindb', 'bigchaindb.commands', - 'bigchaindb.db', - 'bigchaindb.consensus' + 'bigchaindb.db' ], entry_points={ From 14b71537d6b0231f0125f0c861c8486ef6faa78b Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:32:25 -0800 Subject: [PATCH 06/22] Made AbstractConsensusRules use python's abc and enforcing that ConsensusRules plugins subclass it --- bigchaindb/__init__.py | 4 +--- bigchaindb/config_utils.py | 47 ++++++++++++++++++++++++++------------ bigchaindb/consensus.py | 28 +++++++++++++---------- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 0a69c150..9283913f 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -41,9 +41,7 @@ config = { 'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01) }, 'api_endpoint': 'http://localhost:8008/api/v1', - 'consensus_plugins': [ - 'base' - ] + 'consensus_plugin': e('BIGCHAIN_CONSENSUS_PLUGIN', default='default') } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index c273245d..7f312d0c 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -17,7 +17,7 @@ import json import logging import collections -from pkg_resources import iter_entry_points +from pkg_resources import iter_entry_points, ResolutionError import bigchaindb @@ -103,20 +103,37 @@ def autoconfigure(): logger.warning('Cannot find your config file. Run `bigchaindb configure` to create one') -def get_plugins(plugin_names): - if not plugin_names: - plugin_names = bigchaindb.config.get('consensus_plugins', []) +def load_consensus_plugin(name=None): + """Find and load the chosen consensus plugin. - plugins = [] + Args: + name (string): the name of the entry_point, as advertised in the + setup.py of the providing package. - # It's important to maintain plugin ordering as stated in the config file. - # e.g. Expensive validation tasks should happen after cheap ones. - # - # TODO: We might want to add some sort of priority system, but for now we - # simply assume everything in a given plugin is designed to run at the - # same time. - for name in plugin_names: - for entry_point in iter_entry_points('bigchaindb.plugins', name): - plugins.append(entry_point.load()) + Returns: + an uninstantiated subclass of ``bigchaindb.consensus.AbstractConsensusRules`` + """ + if not name: + name = bigchaindb.config.get('consensus_plugin', 'default') - return plugins + # TODO: This will return the first plugin with group `bigchaindb.plugins` + # and name `name` in the active WorkingSet. + # We should probably support Requirements specs in the config, e.g. + # consensus_plugin: 'my-plugin-package==0.0.1;default' + plugin = None + for entry_point in iter_entry_points('bigchaindb.consensus', name): + plugin = entry_point.load() + + # No matching entry_point found + if not plugin: + raise ResolutionError( + 'No plugin found in group `bigchaindb.plugins` with name `{}`'. + format(name)) + + # Is this strictness desireable? + # It will probably reduce developer headaches in the wild. + if not issubclass(plugin, (AbstractConsensusRules)): + raise TypeError("object of type '{}' does not implement `bigchaindb." + "consensus.AbstractConsensusRules`".format(type(plugin))) + + return plugin diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 4bf5fe66..1c0b1aec 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -1,17 +1,21 @@ +from abc import ABCMeta, abstractmethod + import bigchaindb.exceptions as exceptions from bigchaindb import util -from bigchaindb.crypto import hash_data +from bigchaindb.crypto import hash_data, PublicKey -# TODO: no real reason to use abc yet, but later we can enforce inheritance from -# this class when loading plugins if that's desirable. -# from abc import ABCMeta -class AbstractConsensusRules: +class AbstractConsensusRules(metaclass=ABCMeta): + """Abstract base class for Bigchain plugins which implement consensus logic. - # TODO: rather than having plugin-authors inherit and override, - # it'd be cleaner to make a `transactionrule` decorator and etc - @classmethod - def validate_transaction(cls, bigchain, transaction): + A consensus plugin must expose a class inheriting from this one via an + entry_point. + + All methods listed below must be implemented. + """ + + @abstractmethod + def validate_transaction(bigchain, transaction): """Validate a transaction. Args: @@ -26,10 +30,10 @@ class AbstractConsensusRules: Descriptive exceptions indicating the reason the transaction failed. See the `exceptions` module for bigchain-native error classes. """ - return transaction + raise NotImplementedError - @classmethod - def validate_block(cls, bigchain, block): + @abstractmethod + def validate_block(bigchain, block): """Validate a block. Args: From a5243e43f68661d99ea68c6b788fa1838eff7895 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:33:31 -0800 Subject: [PATCH 07/22] This large (sorry) commit 1. switches from a composable plugin model to a single-plugin model 2. switches class methods to static methods in the BaseConsensusRules class 3. adds create_transaction, sign_transaction, and verify_transaction to the plugin API TODO: If we adopt this model, all references in e.g. client.py to util methods like `sign_tx` need to be routed through the plugin methods, and possibly need to be added to the plugin interface. --- bigchaindb/config_utils.py | 1 + bigchaindb/consensus.py | 189 +++++++++++++++++++++++++------------ bigchaindb/core.py | 60 ++++++------ setup.py | 4 +- 4 files changed, 159 insertions(+), 95 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 7f312d0c..26afa3c8 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -20,6 +20,7 @@ import collections from pkg_resources import iter_entry_points, ResolutionError import bigchaindb +from bigchaindb.consensus import AbstractConsensusRules logger = logging.getLogger(__name__) CONFIG_DEFAULT_PATH = os.environ.setdefault( diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 1c0b1aec..818249b8 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -19,7 +19,7 @@ class AbstractConsensusRules(metaclass=ABCMeta): """Validate a transaction. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + bigchain (Bigchain): an instantiated ``bigchaindb.Bigchain`` object. transaction (dict): transaction to validate. Returns: @@ -37,7 +37,7 @@ class AbstractConsensusRules(metaclass=ABCMeta): """Validate a block. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + bigchain (Bigchain): an instantiated ``bigchaindb.Bigchain`` object. block (dict): block to validate. Returns: @@ -48,109 +48,174 @@ class AbstractConsensusRules(metaclass=ABCMeta): Descriptive exceptions indicating the reason the block failed. See the `exceptions` module for bigchain-native error classes. """ - return block + raise NotImplementedError - class ConsensusRules(AbstractConsensusRules): + @abstractmethod + def create_transaction(*args, **kwargs): + """Create a new transaction. + + Args: + The signature of this method is left to plugin authors to decide. + + Returns: + dict: newly constructed transaction. + """ + raise NotImplementedError + + @abstractmethod + def sign_transaction(transaction, *args, **kwargs): + """Sign a transaction. + + Args: + transaction (dict): transaction to sign. + any other arguments are left to plugin authors to decide. + + Returns: + dict: transaction with any signatures applied. + """ + raise NotImplementedError + + @abstractmethod + def verify_signature(signed_transaction): + """Verify the signature of a transaction. + + Args: + signed_transaction (dict): signed transaction to verify + + Returns: + bool: True if the transaction's required signature data is present + and correct, False otherwise. + """ + raise NotImplementedError + + +class BaseConsensusRules(AbstractConsensusRules): """Base consensus rules for Bigchain. - This class can be copied to write your own consensus rules! - - Note: Consensus plugins will be executed in the order that they're listed in - the bigchain config file. + This class can be copied or overridden to write your own consensus rules! """ - @classmethod - def validate_transaction(cls, bigchain, transaction): + @staticmethod + def validate_transaction(bigchain, transaction): """Validate a transaction. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - transaction (dict): transaction to validate. + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + transaction (dict): transaction to validate. Returns: - The transaction if the transaction is valid else it raises an - exception describing the reason why the transaction is invalid. + The transaction if the transaction is valid else it raises an + exception describing the reason why the transaction is invalid. Raises: - OperationError: if the transaction operation is not supported - TransactionDoesNotExist: if the input of the transaction is not found - TransactionOwnerError: if the new transaction is using an input it doesn't own - DoubleSpend: if the transaction is a double spend - InvalidHash: if the hash of the transaction is wrong - InvalidSignature: if the signature of the transaction is wrong + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not found + TransactionOwnerError: if the new transaction is using an input it doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong """ # If the operation is CREATE the transaction should have no inputs and # should be signed by a federation node if transaction['transaction']['operation'] == 'CREATE': - if transaction['transaction']['input']: - raise ValueError('A CREATE operation has no inputs') - if transaction['transaction']['current_owner'] not in ( - bigchain.federation_nodes + [bigchain.me]): - raise exceptions.OperationError( - 'Only federation nodes can use the operation `CREATE`') + if transaction['transaction']['input']: + raise ValueError('A CREATE operation has no inputs') + if transaction['transaction']['current_owner'] not in ( + bigchain.federation_nodes + [bigchain.me]): + raise exceptions.OperationError( + 'Only federation nodes can use the operation `CREATE`') else: - # check if the input exists, is owned by the current_owner - if not transaction['transaction']['input']: - raise ValueError( - 'Only `CREATE` transactions can have null inputs') + # check if the input exists, is owned by the current_owner + if not transaction['transaction']['input']: + raise ValueError( + 'Only `CREATE` transactions can have null inputs') - tx_input = bigchain.get_transaction( - transaction['transaction']['input']) + tx_input = bigchain.get_transaction( + transaction['transaction']['input']) - if not tx_input: - raise exceptions.TransactionDoesNotExist( - 'input `{}` does not exist in the bigchain'.format( - transaction['transaction']['input'])) + if not tx_input: + raise exceptions.TransactionDoesNotExist( + 'input `{}` does not exist in the bigchain'.format( + transaction['transaction']['input'])) - if (tx_input['transaction']['new_owner'] != - transaction['transaction']['current_owner']): - raise exceptions.TransactionOwnerError( - 'current_owner `{}` does not own the input `{}`'.format( - transaction['transaction']['current_owner'], - transaction['transaction']['input'])) + if (tx_input['transaction']['new_owner'] != + transaction['transaction']['current_owner']): + raise exceptions.TransactionOwnerError( + 'current_owner `{}` does not own the input `{}`'.format( + transaction['transaction']['current_owner'], + transaction['transaction']['input'])) - # check if the input was already spent by a transaction other than - # this one. - spent = bigchain.get_spent(tx_input['id']) - if spent and spent['id'] != transaction['id']: - raise exceptions.DoubleSpend( - 'input `{}` was already spent'.format( - transaction['transaction']['input'])) + # check if the input was already spent by a transaction other than + # this one. + spent = bigchain.get_spent(tx_input['id']) + if spent and spent['id'] != transaction['id']: + raise exceptions.DoubleSpend( + 'input `{}` was already spent'.format( + transaction['transaction']['input'])) # Check hash of the transaction calculated_hash = hash_data(util.serialize( - transaction['transaction'])) + transaction['transaction'])) if calculated_hash != transaction['id']: - raise exceptions.InvalidHash() + raise exceptions.InvalidHash() # Check signature - if not bigchain.verify_signature(transaction): - raise exceptions.InvalidSignature() + if not util.verify_signature(transaction): + raise exceptions.InvalidSignature() return transaction - # TODO: check that the votings structure is correctly constructed - @classmethod - def validate_block(cls, bigchain, block): + # TODO: Unsure if a bigchain parameter is really necessary here? + @staticmethod + def validate_block(bigchain, block): """Validate a block. Args: - bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. - block (dict): block to validate. + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + block (dict): block to validate. Returns: - The block if the block is valid else it raises and exception - describing the reason why the block is invalid. + The block if the block is valid else it raises and exception + describing the reason why the block is invalid. Raises: - InvalidHash: if the hash of the block is wrong. + InvalidHash: if the hash of the block is wrong. """ # Check if current hash is correct calculated_hash = hash_data(util.serialize(block['block'])) if calculated_hash != block['id']: - raise exceptions.InvalidHash() + raise exceptions.InvalidHash() return block + + @staticmethod + def create_transaction(current_owner, new_owner, tx_input, operation, + payload=None): + """Create a new transaction + + Refer to the documentation of ``bigchaindb.util.create_tx`` + """ + + return util.create_tx(current_owner, new_owner, tx_input, operation, + payload) + + @staticmethod + def sign_transaction(transaction, private_key): + """Sign a transaction + + Refer to the documentation of ``bigchaindb.util.sign_tx`` + """ + + return util.sign_tx(transaction, private_key) + + @staticmethod + def verify_signature(signed_transaction): + """Verify the signature of a transaction. + + Refer to the documentation of ``bigchaindb.util.verify_signature`` + """ + + return util.verify_signature(signed_transaction) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index ffdffa9b..f76dd2d6 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -27,7 +27,7 @@ class Bigchain(object): def __init__(self, host=None, port=None, dbname=None, public_key=None, private_key=None, keyring=[], - consensus_plugins=['base']): + consensus_plugin=None): """Initialize the Bigchain instance There are three ways in which the Bigchain instance can get its parameters. @@ -53,7 +53,7 @@ class Bigchain(object): self.me = public_key or bigchaindb.config['keypair']['public'] self.me_private = private_key or bigchaindb.config['keypair']['private'] self.federation_nodes = keyring or bigchaindb.config['keyring'] - self.consensus_plugins = config_utils.get_plugins(consensus_plugins) + self.consensus = config_utils.load_consensus_plugin(consensus_plugin) if not self.me or not self.me_private: raise exceptions.KeypairNotFoundException() @@ -70,38 +70,40 @@ class Bigchain(object): return r.connect(host=self.host, port=self.port, db=self.dbname) @monitor.timer('create_transaction', rate=bigchaindb.config['statsd']['rate']) - def create_transaction(self, current_owner, new_owner, tx_input, operation, payload=None): + def create_transaction(self, *args, **kwargs): """Create a new transaction - Refer to the documentation of ``bigchaindb.util.create_tx`` + Refer to the documentation of your consensus plugin. + + Returns: + dict: newly constructed transaction. """ - return util.create_tx(current_owner, new_owner, tx_input, operation, payload) + return self.consensus.create_transaction(*args, **kwargs) - def sign_transaction(self, transaction, private_key): + def sign_transaction(self, transaction, *args, **kwargs): """Sign a transaction - Refer to the documentation of ``bigchaindb.util.sign_tx`` + Refer to the documentation of your consensus plugin. + + Returns: + dict: transaction with any signatures applied. """ - return util.sign_tx(transaction, private_key) + return self.consensus.sign_transaction(transaction, *args, **kwargs) - def verify_signature(self, signed_transaction): - """Verify the signature of a transaction. + def verify_signature(self, signed_transaction, *args, **kwargs): + """Verify the signature(s) of a transaction. - Refer to the documentation of ``bigchaindb.crypto.verify_signature`` + Refer to the documentation of your consensus plugin. + + Returns: + bool: True if the transaction's required signature data is present + and correct, False otherwise. """ - data = signed_transaction.copy() - - # if assignee field in the transaction, remove it - if 'assignee' in data: - data.pop('assignee') - - signature = data.pop('signature') - public_key_base58 = signed_transaction['transaction']['current_owner'] - public_key = crypto.PublicKey(public_key_base58) - return public_key.verify(util.serialize(data), signature) + return self.consensus.verify_signature( + signed_transaction, *args, **kwargs) @monitor.timer('write_transaction', rate=bigchaindb.config['statsd']['rate']) def write_transaction(self, signed_transaction, durability='soft'): @@ -111,7 +113,7 @@ class Bigchain(object): it has been validated by the nodes of the federation. Args: - singed_transaction (dict): transaction with the `signature` included. + signed_transaction (dict): transaction with the `signature` included. Returns: dict: database response @@ -250,11 +252,7 @@ class Bigchain(object): exception describing the reason why the transaction is invalid. """ - for plugin in self.consensus_plugins: - transaction = plugin.validate_transaction(self, transaction) - - return transaction - + return self.consensus.validate_transaction(self, transaction) def is_valid_transaction(self, transaction): """Check whether a transacion is valid or invalid. @@ -324,10 +322,8 @@ class Bigchain(object): describing the reason why the block is invalid. """ - # First run all of the plugin block validation logic - for plugin in self.consensus_plugins: - transaction = plugin.validate_block(self, block) - + # First: Run the plugin block validation logic + self.consensus.validate_block(self, block) # Finally: Tentative assumption that every blockchain will want to # validate all transactions in each block @@ -372,6 +368,8 @@ class Bigchain(object): response = r.table('bigchain').get_all(transaction_id, index='transaction_id').run(self.conn) return True if len(response.items) > 0 else False + # TODO: Unless we prescribe the signature of create_transaction, this will + # also need to be moved into the plugin API. def create_genesis_block(self): """Create the genesis block diff --git a/setup.py b/setup.py index 475cb3cc..f6fb0c43 100644 --- a/setup.py +++ b/setup.py @@ -67,8 +67,8 @@ setup( 'bigchaindb=bigchaindb.commands.bigchain:main', 'bigchaindb-benchmark=bigchaindb.commands.bigchain_benchmark:main' ], - 'bigchaindb.plugins': [ - 'base=bigchaindb.consensus.base:ConsensusRules' + 'bigchaindb.consensus': [ + 'default=bigchaindb.consensus:BaseConsensusRules' ] }, install_requires=[ From 79a66bdf9b7212c34b2a2f453a72913a54ecb322 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 8 Mar 2016 18:42:26 -0800 Subject: [PATCH 08/22] Added consensus doc stub --- docs/source/consensus.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docs/source/consensus.md diff --git a/docs/source/consensus.md b/docs/source/consensus.md new file mode 100644 index 00000000..b753068d --- /dev/null +++ b/docs/source/consensus.md @@ -0,0 +1,27 @@ +# BigchainDB Consensus Plugins +TODO: Write this section + +A quick example of a plugin that adds nonsense rules: + +```python +from bigchaindb.consensus import BaseConsensusRules + +class SillyConsensusRules(BaseConsensusRules): + + @staticmethod + def validate_transaction(bigchain, transaction): + transaction = super().validate_transaction(bigchain, transaction) + # I only like transactions whose timestamps are even. + if transaction['transaction']['timestamp'] % 2 != 0: + raise StandardError("Odd... very odd indeed.") + return transaction + + @staticmethod + def validate_block(bigchain, block): + block = super().validate_block(bigchain, block) + # I don't trust Alice, I think she's shady. + if block['block']['node_pubkey'] == '': + raise StandardError("Alice is shady, everybody ignore her blocks!") + + return block +``` From 037b5110fea7ff071810aec32f96502187b6ffd3 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sun, 13 Mar 2016 22:26:35 -0700 Subject: [PATCH 09/22] Modified Client module and web views to use plublic consensus methods for creating/signing/verifying instead of the functions in utils.py -- those are now only called by the the BaseConsensusRules class in consensus.py. --- bigchaindb/client.py | 33 ++++++++++++++++++++++++++------- bigchaindb/web/views.py | 5 +++-- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/bigchaindb/client.py b/bigchaindb/client.py index 0a9eed01..f7b37ad4 100644 --- a/bigchaindb/client.py +++ b/bigchaindb/client.py @@ -15,7 +15,8 @@ class Client: In the future, a Client might connect to >1 hosts. """ - def __init__(self, public_key=None, private_key=None, api_endpoint=None): + def __init__(self, public_key=None, private_key=None, api_endpoint=None, + consensus_plugin=None): """Initialize the Client instance There are three ways in which the Client instance can get its parameters. @@ -28,8 +29,11 @@ class Client: Args: public_key (str): the base58 encoded public key for the ECDSA secp256k1 curve. private_key (str): the base58 encoded private key for the ECDSA secp256k1 curve. - host (str): hostname where the rethinkdb is running. - port (int): port in which rethinkb is running (usually 28015). + api_endpoint (str): a URL where rethinkdb is running. + format: scheme://hostname:port + consensus_plugin (str): the registered name of your installed + consensus plugin. The `core` plugin is built into BigchainDB; + others must be installed via pip. """ config_utils.autoconfigure() @@ -37,6 +41,7 @@ class Client: self.public_key = public_key or bigchaindb.config['keypair']['public'] self.private_key = private_key or bigchaindb.config['keypair']['private'] self.api_endpoint = api_endpoint or bigchaindb.config['api_endpoint'] + self.consensus = config_utils.load_consensus_plugin(consensus_plugin) if not self.public_key or not self.private_key: raise exceptions.KeypairNotFoundException() @@ -51,8 +56,15 @@ class Client: The transaction pushed to the Federation. """ - tx = util.create_tx(self.public_key, self.public_key, None, operation='CREATE', payload=payload) - signed_tx = util.sign_tx(tx, self.private_key) + tx = self.consensus.create_transaction( + current_owner=self.public_key, + new_owner=self.public_key, + tx_input=None, + operation='CREATE', + payload=payload) + + signed_tx = self.consensus.sign_transaction( + tx, private_key=self.private_key) return self._push(signed_tx) def transfer(self, new_owner, tx_input, payload=None): @@ -67,8 +79,15 @@ class Client: The transaction pushed to the Federation. """ - tx = util.create_tx(self.public_key, new_owner, tx_input, operation='TRANSFER', payload=payload) - signed_tx = util.sign_tx(tx, self.private_key) + tx = self.consensus.create_transaction( + current_owner=self.public_key, + new_owner=new_owner, + tx_input=tx_input, + operation='TRANSFER', + payload=payload) + + signed_tx = self.consensus.sign_transaction( + tx, private_key=self.private_key) return self._push(signed_tx) def _push(self, tx): diff --git a/bigchaindb/web/views.py b/bigchaindb/web/views.py index 04d3992d..53db6f66 100644 --- a/bigchaindb/web/views.py +++ b/bigchaindb/web/views.py @@ -57,9 +57,10 @@ def create_transaction(): if tx['transaction']['operation'] == 'CREATE': tx = util.transform_create(tx) - tx = util.sign_tx(tx, bigchain.me_private) + tx = bigchain.consensus.sign_transaction( + tx, private_key=bigchain.me_private) - if not util.verify_signature(tx): + if not bigchain.consensus.verify_signature(tx): val['error'] = 'Invalid transaction signature' val = bigchain.write_transaction(tx) From 7253e96796fbdb48e85b0f6cf18485a70607cee5 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Sun, 13 Mar 2016 23:28:24 -0700 Subject: [PATCH 10/22] Fleshed out consensus plugin docs --- docs/source/consensus.md | 61 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/docs/source/consensus.md b/docs/source/consensus.md index b753068d..552ce0d6 100644 --- a/docs/source/consensus.md +++ b/docs/source/consensus.md @@ -1,7 +1,50 @@ # BigchainDB Consensus Plugins -TODO: Write this section -A quick example of a plugin that adds nonsense rules: +BigchainDB has a pluggable block/transaction validation architecture. The default consensus rules can be extended or replaced entirely. + + +## Installing a plugin + +Plugins can be installed via pip! + +```bash +$ pip install bigchaindb-plugin-demo +``` + +Or using setuptools: + +```bash +$ cd bigchaindb-plugin-demo/ +$ python setup.py install # (or develop) +``` + +To activate your plugin, you can either set the `consensus_plugin` field in your config file (usually `~/.bigchaindb`) or by setting the `BIGCHAIN_CONSENSUS_PLUGIN` environement variable to the name of your plugin (see the section on [Packaging a plugin](#packaging-a-plugin) for more about plugin names). + + +## Plugin API + +BigchainDB's [current plugin API](../../bigchaindb/consensus.py) exposes five functions in an `AbstractConsensusRules` class: + +```python +validate_transaction(bigchain, transaction) +validate_block(bigchain, block) +create_transaction(*args, **kwargs) +sign_transaction(transaction, *args, **kwargs) +verify_signature(transaction) +``` + +Together, these functions are sufficient for most customizations. For example: +- Replace the crypto-system with one your hardware can accelerate +- Re-implement an existing protocol +- Delegate validation to another application +- etc... + + +## Extending BigchainDB behavior + +A default installation of BigchainDB will use the rules in the `BaseConsensusRules` class. If you only need to modify this behavior slightly, you can inherit from that class and call `super()` in any methods you change, so long as the return values remain the same. + +Here's a quick example of a plugin that adds nonsense rules: ```python from bigchaindb.consensus import BaseConsensusRules @@ -22,6 +65,18 @@ class SillyConsensusRules(BaseConsensusRules): # I don't trust Alice, I think she's shady. if block['block']['node_pubkey'] == '': raise StandardError("Alice is shady, everybody ignore her blocks!") - return block ``` + + +## Packaging a plugin + +BigchainDB uses [setuptool's entry_point](https://pythonhosted.org/setuptools/setuptools.html) system to provide the plugin functionality. Any custom plugin needs to add this section to the `setup()` call in their `setup.py`: + +```python +entry_points={ + 'bigchaindb.consensus': [ + 'PLUGIN_NAME=package.module:ConsensusRulesClass' + ] +}, +``` From ad535c78951f42281e8fd10a1a0be06155930d0e Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 14 Mar 2016 13:27:17 -0700 Subject: [PATCH 11/22] Tests for loading consensus plugins --- tests/test_core.py | 6 ++++++ tests/utils/test_config_utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 9857b37f..f136f4e3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -15,6 +15,7 @@ def config(request, monkeypatch): }, 'keyring': [], 'CONFIGURED': True, + 'consensus_plugin': 'default' } monkeypatch.setattr('bigchaindb.config', config) @@ -24,6 +25,7 @@ def config(request, monkeypatch): def test_bigchain_class_default_initialization(config): from bigchaindb.core import Bigchain + from bigchaindb.consensus import BaseConsensusRules bigchain = Bigchain() assert bigchain.host == config['database']['host'] assert bigchain.port == config['database']['port'] @@ -31,11 +33,13 @@ def test_bigchain_class_default_initialization(config): assert bigchain.me == config['keypair']['public'] assert bigchain.me_private == config['keypair']['private'] assert bigchain.federation_nodes == config['keyring'] + assert bigchain.consensus == BaseConsensusRules assert bigchain._conn is None def test_bigchain_class_initialization_with_parameters(config): from bigchaindb.core import Bigchain + from bigchaindb.consensus import BaseConsensusRules init_kwargs = { 'host': 'some_node', 'port': '12345', @@ -43,6 +47,7 @@ def test_bigchain_class_initialization_with_parameters(config): 'public_key': 'white', 'private_key': 'black', 'keyring': ['key_one', 'key_two'], + 'consensus_plugin': 'default' } bigchain = Bigchain(**init_kwargs) assert bigchain.host == init_kwargs['host'] @@ -51,4 +56,5 @@ def test_bigchain_class_initialization_with_parameters(config): assert bigchain.me == init_kwargs['public_key'] assert bigchain.me_private == init_kwargs['private_key'] assert bigchain.federation_nodes == init_kwargs['keyring'] + assert bigchain.consensus == BaseConsensusRules assert bigchain._conn is None diff --git a/tests/utils/test_config_utils.py b/tests/utils/test_config_utils.py index 2cf57242..fa63158d 100644 --- a/tests/utils/test_config_utils.py +++ b/tests/utils/test_config_utils.py @@ -37,3 +37,32 @@ def test_bigchain_instance_raises_when_not_configured(monkeypatch): with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() + + +def test_load_consensus_plugin_loads_default_rules_without_name(): + from bigchaindb import config_utils + from bigchaindb.consensus import BaseConsensusRules + + assert config_utils.load_consensus_plugin() == BaseConsensusRules + + +def test_load_consensus_plugin_raises_with_unknown_name(): + from pkg_resources import ResolutionError + from bigchaindb import config_utils + + with pytest.raises(ResolutionError): + config_utils.load_consensus_plugin('bogus') + + +def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): + # Monkeypatch entry_point.load to return something other than a + # ConsensusRules instance + from bigchaindb import config_utils + monkeypatch.setattr(config_utils, + 'iter_entry_points', + lambda *args: [ type('entry_point', + (object), + {'load': lambda: object}) ]) + + with pytest.raises(TypeError): + config_utils.load_consensus_plugin() From 3d50459b84a462b4819f9389a23b12a203817658 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 14 Mar 2016 20:02:18 -0700 Subject: [PATCH 12/22] Corrected plugin group name in comments --- bigchaindb/config_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 26afa3c8..645780eb 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -117,7 +117,7 @@ def load_consensus_plugin(name=None): if not name: name = bigchaindb.config.get('consensus_plugin', 'default') - # TODO: This will return the first plugin with group `bigchaindb.plugins` + # TODO: This will return the first plugin with group `bigchaindb.consensus` # and name `name` in the active WorkingSet. # We should probably support Requirements specs in the config, e.g. # consensus_plugin: 'my-plugin-package==0.0.1;default' @@ -128,7 +128,7 @@ def load_consensus_plugin(name=None): # No matching entry_point found if not plugin: raise ResolutionError( - 'No plugin found in group `bigchaindb.plugins` with name `{}`'. + 'No plugin found in group `bigchaindb.consensus` with name `{}`'. format(name)) # Is this strictness desireable? From 678cd2dd2968ceeaa65c70a6916a591289f369ae Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 17 Mar 2016 11:05:15 +0100 Subject: [PATCH 13/22] Updated ROADMAP.md based on March 16 meeting --- ROADMAP.md | 97 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 19 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 79fadf02..09f1b7ac 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,30 +5,89 @@ See also: * [Milestones](https://github.com/bigchaindb/bigchaindb/milestones) (i.e. issues to be closed before various releases) * [Open issues](https://github.com/bigchaindb/bigchaindb/issues) and [open pull requests](https://github.com/bigchaindb/bigchaindb/pulls) -## BigchainDB Protocols -* Validation of other nodes -* Fault tolerance -* Permissions framework -* Protocol audits including security audits +Note: Below, #345 refers to Issue #345 in the BigchainDB repository on GitHub. #N refers to Issue #N. -## HTTP Client-Server API -* Validate the structure of the transaction + +## Deployment and Federation Configuration/Management +* Define how a federation is managed - [#126](https://github.com/bigchaindb/bigchaindb/issues/126) +* Review current configuration mechanism - [#49](https://github.com/bigchaindb/bigchaindb/issues/49) +* Make the configuration easier for Docker-based setup - [#36](https://github.com/bigchaindb/bigchaindb/issues/36) + + +## Testing +* (Unit-test writing and unit testing are ongoing.) +* More Integration Testing, Validation Testing, System Testing, Benchmarking +* Define some standard test systems (e.g. local virtual cluster, data center, WAN) +* Develop standardized test descriptions and documentation (test setup, inputs, outputs) +* Build up a suite of tests to test each identified fault +* More tools for cluster benchmarking +* Identify bottlenecks using profiling and monitoring +* Fault-testing framework +* Clean exit for the bigchaindb-benchmark process - [#122](https://github.com/bigchaindb/bigchaindb/issues/122) +* Tool to bulk-upload transactions into biacklog table - [#114](https://github.com/bigchaindb/bigchaindb/issues/114) +* Tool to deploy multiple clients for testing - [#113](https://github.com/bigchaindb/bigchaindb/issues/113) +* Tool to read transactions from files for testing - [#112](https://github.com/bigchaindb/bigchaindb/issues/112) + + +## Specific Bugs/Faults and Related Tests +* Validation of other nodes +* Changefeed watchdog +* Non-deterministic assignment of tx in S is a DoS vulnerability - [#20](https://github.com/bigchaindb/bigchaindb/issues/20) +* Queues are unbounded - [#124](https://github.com/bigchaindb/bigchaindb/issues/124) +* Better handling of timeouts in block creation - [#123](https://github.com/bigchaindb/bigchaindb/issues/123) +* Secure node-node communication - [#77](https://github.com/bigchaindb/bigchaindb/issues/77) +* Checking if transactions are in a decided_valid block (or otherwise) when necessary - [#134](https://github.com/bigchaindb/bigchaindb/issues/134) +* When validating an incoming transaction, check to ensure it isn't a duplicate - [#131](https://github.com/bigchaindb/bigchaindb/issues/131) +* Consider secondary indexes on some queries - [#105](https://github.com/bigchaindb/bigchaindb/issues/105) + + +## Transactions / Assets +* Current Top-Level Goal: Define and implement "v2 transactions", that is, support multisig (done) and: +* Support for multiple inputs and outputs - [#128](https://github.com/bigchaindb/bigchaindb/issues/128) +* Crypto-conditions specific to ILP - [#127](https://github.com/bigchaindb/bigchaindb/issues/127) +* Support divisible assets - [#129](https://github.com/bigchaindb/bigchaindb/issues/129) +* Define a JSON template for digital assets - [#125](https://github.com/bigchaindb/bigchaindb/issues/125) +* Revisit timestamps - [#132](https://github.com/bigchaindb/bigchaindb/issues/132) +* Refactor structure of a transaction - [#98](https://github.com/bigchaindb/bigchaindb/issues/98) +* Plugin or hook architecture e.g. for validate_transaction - [#90](https://github.com/bigchaindb/bigchaindb/issues/90) + + +## Web API (HTTP Client-Server API) +* Current Top-Level Goal: Support v2 transactions (see above) +* Validate the structure of incoming transactions * Return the correct error code if something goes wrong +* Validate transaction before writing it to the backlog - [#109](https://github.com/bigchaindb/bigchaindb/issues/109) +* Better organization of transaction-related code - [#108](https://github.com/bigchaindb/bigchaindb/issues/108) * Add an endpoint to query unspents for a given public key * More endpoints -* See [the relevant open issues](https://github.com/bigchaindb/bigchaindb/issues?q=is%3Aissue+is%3Aopen+label%3Arest-api) +* See [open issues with the "rest-api" label](https://github.com/bigchaindb/bigchaindb/issues?q=is%3Aissue+is%3Aopen+label%3Arest-api) -## Implementation/Code -* Node validation framework (inspect and agree or not with what the other nodes are doing) -* Open public testing cluster (for people to try out a BigchainDB cluster and to test client software) -* Federation management tools -* More tools for benchmarking a cluster -* Descriptions and results of more benchmarking tests -* AWS image and other easy deployment options + +## Drivers +* Update the reference driver (Python) to support v2 transactions and web API (see above) * Drivers/SDKs for more client-side languages (e.g. JavaScript, Ruby, Java) -* ORM to better-decouple BigchainDB from its data store (will make it easy to try other databases) -* Code audits including security audits -## Other/Future -* Byzantine fault tolerance + +## Public Testnet and Public BigchainDB +* Deploy a 3-node public testnet in a data center, open to all external users, refreshing daily +* Deploy Public BigchaindB Testnet with more than 3 nodes and with nodes more globally-distributed +* Public BigchainDB governance/voting system +* Transaction (or usage) accounting +* Billing system + + +## Other +* Get BigchainDB production-ready for submission to AWS Marketplace (as an AMI) + + +## Future +* Permissions framework +* More Byzantine fault tolerance (BFT) * Better support for smart contract frameworks +* Algorithm audits +* Protocol audits +* Code (implementation) audits +* Security audits +* IPFS interoperability - [#100](https://github.com/bigchaindb/bigchaindb/issues/100) +* ORM to better-decouple BigchainDB from its data store (will make it easy to try other databases) +* Support more server operating systems From 785ee388fefba8fb1769dc0de0330f30ea9bc891 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 17 Mar 2016 16:20:11 +0100 Subject: [PATCH 14/22] Move info on running all unit tests into the Sphinx docs --- PYTHON_STYLE_GUIDE.md | 42 +++---------------------------- docs/source/index.rst | 1 + docs/source/running-unit-tests.md | 41 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 docs/source/running-unit-tests.md diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index dc7e68d7..4d4d4888 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -57,46 +57,10 @@ we use the `format()` version. The [official Python documentation says](https:// ## Writing (Python) Tests -We write tests for our Python code using the [pytest](http://pytest.org/latest/) framework. +We write unit tests for our Python code using the [pytest](http://pytest.org/latest/) framework. All tests go in the `bigchaindb/tests` directory or one of its subdirectories. You can use the tests already in there as templates or examples. -### Standard Ways to Run All Tests +The BigchainDB Documentation has a [section explaining how to run all unit tests](http://bigchaindb.readthedocs.org/en/develop/running-unit-tests.html). -To run all the tests, first make sure you have RethinkDB running: -```text -$ rethinkdb -``` - -then in another terminal, do: -```text -$ py.test -v -``` - -If that doesn't work (e.g. maybe you are running in a conda virtual environment), try: -```text -$ python -m pytest -v -``` - -You can also run all tests via `setup.py`, using: -```text -$ python setup.py test -``` - -### Using `docker-compose` to Run the Tests - -You can use `docker-compose` to run the tests. (You don't have to start RethinkDB first: `docker-compose` does that on its own, when it reads the `docker-compose.yml` file.) - -First, build the images (~once), using: -```text -$ docker-compose build -``` - -then run the tests using: -```text -$ docker-compose run --rm bigchaindb py.test -v -``` - -### Automated Testing of All Pull Requests - -We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) +**Automated testing of pull requests.** We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) diff --git a/docs/source/index.rst b/docs/source/index.rst index 44523c7d..fd2c991e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,6 +14,7 @@ Table of Contents introduction installing-server + running-unit-tests python-server-api-examples bigchaindb-cli http-client-server-api diff --git a/docs/source/running-unit-tests.md b/docs/source/running-unit-tests.md new file mode 100644 index 00000000..9e53dc77 --- /dev/null +++ b/docs/source/running-unit-tests.md @@ -0,0 +1,41 @@ +# Running Unit Tests + +Once you've installed BigchainDB Server, you may want to run all the unit tests. This section explains how. + +First of all, if you installed BigchainDB Server using `pip` (i.e. by getting the package from PyPI), then you didn't install the tests. Before you can run all the unit tests, you must [install BigchainDB from source](http://bigchaindb.readthedocs.org/en/develop/installing-server.html#how-to-install-bigchaindb-from-source). + +To run all the unit tests, first make sure you have RethinkDB running: +```text +$ rethinkdb +``` + +then in another terminal, do: +```text +$ py.test -v +``` + +If the above command doesn't work (e.g. maybe you are running in a conda virtual environment), try: +```text +$ python -m pytest -v +``` + +(We write our unit tests using the [pytest](http://pytest.org/latest/) framework.) + +You can also run all unit tests via `setup.py`, using: +```text +$ python setup.py test +``` + +### Using `docker-compose` to Run the Tests + +You can also use `docker-compose` to run the unit tests. (You don't have to start RethinkDB first: `docker-compose` does that on its own, when it reads the `docker-compose.yml` file.) + +First, build the images (~once), using: +```text +$ docker-compose build +``` + +then run the unit tests using: +```text +$ docker-compose run --rm bigchaindb py.test -v +``` From 6ea63903cbf5852f5aea67ab682e1aae62b3a537 Mon Sep 17 00:00:00 2001 From: Alberto Granzotto Date: Thu, 17 Mar 2016 16:59:56 +0100 Subject: [PATCH 15/22] Revert "Add missing package bigchaindb.web to setup.py" --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d888e6a..0cb4aea3 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( 'Operating System :: POSIX :: Linux', ], - packages=['bigchaindb', 'bigchaindb.commands', 'bigchaindb.db', 'bigchaindb.web'], + packages=['bigchaindb', 'bigchaindb.commands', 'bigchaindb.db'], entry_points={ 'console_scripts': [ From 6e60a38a9958f26f3191c12bfacb2092bbb38ea5 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 17 Mar 2016 17:03:08 +0100 Subject: [PATCH 16/22] Use find_packages --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0cb4aea3..608eed27 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ BigchainDB: A Scalable Blockchain Database For full docs visit https://bigchaindb.readthedocs.org """ -from setuptools import setup +from setuptools import setup, find_packages tests_require = [ 'pytest', @@ -56,7 +56,7 @@ setup( 'Operating System :: POSIX :: Linux', ], - packages=['bigchaindb', 'bigchaindb.commands', 'bigchaindb.db'], + packages=find_packages(), entry_points={ 'console_scripts': [ From 5d338483198cc75587798c252d834f5a84ef7a7c Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 17 Mar 2016 17:17:27 +0100 Subject: [PATCH 17/22] Exclude tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 608eed27..63ce9428 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( 'Operating System :: POSIX :: Linux', ], - packages=find_packages(), + packages=find_packages(exclude=['tests*']), entry_points={ 'console_scripts': [ From 55ad60097da46da44a33892d9c4f1f8ce797e774 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 18 Mar 2016 11:40:30 +0100 Subject: [PATCH 18/22] Fixed minor errors in Introduction (in docs) --- docs/source/introduction.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/introduction.md b/docs/source/introduction.md index 4583d941..3c1230ad 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -1,10 +1,10 @@ # Introduction -BigchainDB is a scalable blockchain database. You can read about its motivations, goals and high-level architecture in the [BigchainDB whitepaper](https://www.bigchaindb.com/whitepaper/). This document, the _BigchainDB Documentation_, is intended primarily for three groups of people: +BigchainDB is a scalable blockchain database. You can read about its motivations, goals and high-level architecture in the [BigchainDB whitepaper](https://www.bigchaindb.com/whitepaper/). This document, the _BigchainDB Documentation_, is intended primarily for: 1. Developers of BigchainDB server software. 2. People deploying and managing BigchainDB clusters. 3. Developers of BigchainDB driver software (SDKs used to develop client software). 4. App developers who are developing client apps to talk to one or more live, operational BigchainDB clusters. They would use one of the BigchainDB drivers. -If you're curious about what's coming in our roadmap, see [the ROADMAP.md file](https://github.com/bigchaindb/bigchaindb/blob/develop/ROADMAP.md) and [the list of open issues](https://github.com/bigchaindb/bigchaindb/issues). If you want to request a feature, file a bug report, or make a pull request, see [the CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/develop/CONTRIBUTING.md). +If you're curious about what's in our roadmap, see [the ROADMAP.md file](https://github.com/bigchaindb/bigchaindb/blob/develop/ROADMAP.md) and [the list of open issues](https://github.com/bigchaindb/bigchaindb/issues). If you want to request a feature, file a bug report, or make a pull request, see [the CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/develop/CONTRIBUTING.md). From cb5c14f9f1f35965b6182248e35cf2a1916fabc1 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 18 Mar 2016 11:55:20 +0100 Subject: [PATCH 19/22] Public Testnet --> Public Sandbox Testnet --- ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 09f1b7ac..9d319994 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -68,8 +68,8 @@ Note: Below, #345 refers to Issue #345 in the BigchainDB repository on GitHub. # * Drivers/SDKs for more client-side languages (e.g. JavaScript, Ruby, Java) -## Public Testnet and Public BigchainDB -* Deploy a 3-node public testnet in a data center, open to all external users, refreshing daily +## Public Sandbox Testnet and Public BigchainDB +* Deploy a 3-node Public Sandbox Testnet in a data center, open to all external users, refreshing daily * Deploy Public BigchaindB Testnet with more than 3 nodes and with nodes more globally-distributed * Public BigchainDB governance/voting system * Transaction (or usage) accounting From 2aa003578ccec83e13e1abf810c1eba5f97d690c Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 18 Mar 2016 16:50:21 -0700 Subject: [PATCH 20/22] Typos --- bigchaindb/consensus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 818249b8..b65ab9ad 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -41,7 +41,7 @@ class AbstractConsensusRules(metaclass=ABCMeta): block (dict): block to validate. Returns: - The block if the block is valid else it raises and exception + The block if the block is valid else it raises an exception describing the reason why the block is invalid. Raises: @@ -177,7 +177,7 @@ class BaseConsensusRules(AbstractConsensusRules): block (dict): block to validate. Returns: - The block if the block is valid else it raises and exception + The block if the block is valid else it raises an exception describing the reason why the block is invalid. Raises: From 39ba990447604ace297f190e70d8be187f7104ee Mon Sep 17 00:00:00 2001 From: troymc Date: Sat, 19 Mar 2016 11:33:29 +0100 Subject: [PATCH 21/22] Revised/renamed admin.md in docs --- docs/source/admin.md | 62 -------------------------- docs/source/index.rst | 2 +- docs/source/installing-server.md | 4 +- docs/source/local-rethinkdb-cluster.md | 54 ++++++++++++++++++++++ docs/source/running-unit-tests.md | 2 +- 5 files changed, 58 insertions(+), 66 deletions(-) delete mode 100644 docs/source/admin.md create mode 100644 docs/source/local-rethinkdb-cluster.md diff --git a/docs/source/admin.md b/docs/source/admin.md deleted file mode 100644 index 4745ee41..00000000 --- a/docs/source/admin.md +++ /dev/null @@ -1,62 +0,0 @@ -# Server/Cluster Deployment and Administration -This section covers everything which might concern a BigchainDB server/cluster administrator: -* deployment -* security -* monitoring -* troubleshooting - - - -## Deploying a local cluster -One of the advantages of RethinkDB as the storage backend is the easy installation. Developers like to have everything locally, so let's install a local storage backend cluster from scratch. -Here is an example to run a cluster assuming rethinkdb is already [installed](installing-server.html#installing-and-running-rethinkdb-server-on-ubuntu-12-04) in -your system: - - # preparing two additional nodes - # remember, that the user who starts rethinkdb must have write access to the paths - mkdir -p /path/to/node2 - mkdir -p /path/to/node3 - - # then start your additional nodes - rethinkdb --port-offset 1 --directory /path/to/node2 --join localhost:29015 - rethinkdb --port-offset 2 --directory /path/to/node3 --join localhost:29015 - -That's all, folks! Cluster is up and running. Check it out in your browser at http://localhost:8080, which opens the console. - -Now you can install BigchainDB and run it against the storage backend! - -## Securing the storage backend -We have turned on the bind=all option for connecting other nodes and making RethinkDB accessible from outside the server. Unfortunately this is insecure. So, we will need to block RethinkDB off from the Internet. But we need to allow access to its services from authorized computers. - -For the cluster port, we will use a firewall to enclose our cluster. For the web management console and the driver port, we will use SSH tunnels to access them from outside the server. SSH tunnels redirect requests on a client computer to a remote computer over SSH, giving the client access to all of the services only available on the remote server's localhost name space. - -Repeat these steps on all your RethinkDB servers. - -First, block all outside connections: - - # the web management console - sudo iptables -A INPUT -i eth0 -p tcp --dport 8080 -j DROP - sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 8080 -j ACCEPT - - # the driver port - sudo iptables -A INPUT -i eth0 -p tcp --dport 28015 -j DROP - sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 28015 -j ACCEPT - - # the communication port - sudo iptables -A INPUT -i eth0 -p tcp --dport 29015 -j DROP - sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 29015 -j ACCEPT - -Save the iptables config: - - sudo apt-get update - sudo apt-get install iptables-persistent - -More about iptables you can find in the [man pages](http://linux.die.net/man/8/iptables). - -## Monitoring the storage backend -Monitoring is pretty simple. You can do this via the [monitoring url](http://localhost:8080). Here you see the complete behaviour of all nodes. -One annotation: if you play around with replication the number of transaction will increase. So for the real throughput you should devide the number of transactions by the number of replicas. - -## Troubleshooting -Since every software may have some minor issues we are not responsible for the storage backend. -If your nodes in a sharded and replicated cluster are not in sync, it may help if you delete the data of the affected node and restart the node. If there are no other problems your node will come back and sync within a couple of minutes. You can verify this by monitoring the cluster via the [monitoring url](http://localhost:8080). diff --git a/docs/source/index.rst b/docs/source/index.rst index fd2c991e..deab9cec 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,7 @@ Table of Contents bigchaindb-cli http-client-server-api python-driver-api-examples - admin + local-rethinkdb-cluster cryptography models json-serialization diff --git a/docs/source/installing-server.md b/docs/source/installing-server.md index 18c62d1e..b16b0f8d 100644 --- a/docs/source/installing-server.md +++ b/docs/source/installing-server.md @@ -4,9 +4,9 @@ We're developing BigchainDB Server on Ubuntu 14.04, but it should work on any OS BigchainDB Server is intended to be run on each server in a large distributed cluster of servers; it's not very useful running by itself on a single computer. That's _why_ we're developing it on Ubuntu 14.04: it's one of the more common server operating systems. -Mac OS X users may get the best results [running BigchainDB Server with Docker](https://bigchaindb.readthedocs.org/en/develop/installing-server.html#run-bigchaindb-with-docker). +Mac OS X users may get the best results [running BigchainDB Server with Docker](installing-server.html#run-bigchaindb-with-docker). -Windows users may get the best results [running BigchainDB Server in a VM with Vagrant](https://bigchaindb.readthedocs.org/en/develop/installing-server.html#how-to-install-bigchaindb-on-a-vm-with-vagrant). +Windows users may get the best results [running BigchainDB Server in a VM with Vagrant](installing-server.html#how-to-install-bigchaindb-on-a-vm-with-vagrant). (BigchainDB clients should run on a much larger array of operating systems.) diff --git a/docs/source/local-rethinkdb-cluster.md b/docs/source/local-rethinkdb-cluster.md new file mode 100644 index 00000000..38685311 --- /dev/null +++ b/docs/source/local-rethinkdb-cluster.md @@ -0,0 +1,54 @@ +# Deploying a Local Multi-Node RethinkDB Cluster + +This section explains one way to deploy a multi-node RethinkDB cluster on one machine. You could use such a cluster as a storage backend for a BigchaindB process running on that machine. + +## Launching More RethinkDB Nodes + +Assuming you've already [installed RethinkDB](installing-server.html#install-and-run-rethinkdb-server) and have one RethinkDB node running, here's how you can launch two more nodes on the same machine. First, prepare two additional nodes. Note that the user who starts RethinkDB must have write access to the created directories: + + mkdir -p /path/to/node2 + mkdir -p /path/to/node3 + +then launch two more nodes: + + rethinkdb --port-offset 1 --directory /path/to/node2 --join localhost:29015 + rethinkdb --port-offset 2 --directory /path/to/node3 --join localhost:29015 + +You should now have a three-node RethinkDB cluster up and running. You should be able to monitor it at [http://localhost:8080](http://localhost:8080). + +Note: if you play around with replication, the number of transactions will increase. For the real throughput, you should divide the number of transactions by the number of replicas. + +## Securing the Cluster + +We have turned on the `bind=all` option for connecting other nodes and making RethinkDB accessible from outside the server. Unfortunately, that is insecure, so we need to block access to the RethinkDB cluster from the Internet. At the same time, we need to allow access to its services from authorized computers. + +For the cluster port, we will use a firewall to enclose our cluster. For the web management console and the driver port, we will use SSH tunnels to access them from outside the server. SSH tunnels redirect requests on a client computer to a remote computer over SSH, giving the client access to all of the services only available on the remote server's localhost name space. + +Repeat the following steps on all your RethinkDB servers. + +First, block all outside connections: + + # the web management console + sudo iptables -A INPUT -i eth0 -p tcp --dport 8080 -j DROP + sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 8080 -j ACCEPT + + # the driver port + sudo iptables -A INPUT -i eth0 -p tcp --dport 28015 -j DROP + sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 28015 -j ACCEPT + + # the communication port + sudo iptables -A INPUT -i eth0 -p tcp --dport 29015 -j DROP + sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 29015 -j ACCEPT + +Save the `iptables` config: + + sudo apt-get update + sudo apt-get install iptables-persistent + +You can find out more about `iptables` in the [man pages](http://linux.die.net/man/8/iptables). + +## Troubleshooting + +You can get help with RethinkDB from [RethinkDB experts](https://rethinkdb.com/services/) and [the RethinkDB community](https://rethinkdb.com/community/). + +If your nodes in a sharded and replicated cluster are not in sync, it may help if you delete the data of the affected node and restart the node. If there are no other problems, your node will come back and sync within a couple of minutes. You can verify this by monitoring the cluster via the monitoring url: [http://localhost:8080](http://localhost:8080). diff --git a/docs/source/running-unit-tests.md b/docs/source/running-unit-tests.md index 9e53dc77..c04ccb22 100644 --- a/docs/source/running-unit-tests.md +++ b/docs/source/running-unit-tests.md @@ -2,7 +2,7 @@ Once you've installed BigchainDB Server, you may want to run all the unit tests. This section explains how. -First of all, if you installed BigchainDB Server using `pip` (i.e. by getting the package from PyPI), then you didn't install the tests. Before you can run all the unit tests, you must [install BigchainDB from source](http://bigchaindb.readthedocs.org/en/develop/installing-server.html#how-to-install-bigchaindb-from-source). +First of all, if you installed BigchainDB Server using `pip` (i.e. by getting the package from PyPI), then you didn't install the tests. Before you can run all the unit tests, you must [install BigchainDB from source](installing-server.html#how-to-install-bigchaindb-from-source). To run all the unit tests, first make sure you have RethinkDB running: ```text From b12f77edbf48a31f03411d9e4c94f9f6e4fe9c36 Mon Sep 17 00:00:00 2001 From: troymc Date: Sat, 19 Mar 2016 11:36:43 +0100 Subject: [PATCH 22/22] Add consensus.md section to docs TOC --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index deab9cec..ac7b3be9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -24,6 +24,7 @@ Table of Contents models json-serialization developer-interface + consensus monitoring licenses contributing