diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..e53e35b5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: +- repo: git://github.com/pre-commit/pre-commit-hooks + sha: v1.1.1 + hooks: + - id: trailing-whitespace + args: ['--no-markdown-linebreak-ext'] + - id: check-merge-conflict + - id: debug-statements + - id: check-added-large-files + - id: flake8 + +- repo: git://github.com/chewse/pre-commit-mirrors-pydocstyle + sha: v2.1.1 + hooks: + - id: pydocstyle + # list of error codes to check, see: http://www.pydocstyle.org/en/latest/error_codes.html + args: ['--select=D204,D201,D209,D210,D212,D300,D403'] + +# negate the exclude to only apply the hooks to 'bigchaindb' and 'tests' folder +exclude: '^(?!bigchaindb/)(?!tests/)' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 145efe95..4aad9d23 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,6 +115,9 @@ git add new-or-changed-file-1 git add new-or-changed-file-2 git commit -m "Short description of new or changed things" ``` +We use [pre-commit](http://pre-commit.com/) which should be triggered with every commit. Some hooks will change files but others will give errors that needs to be fixed. Every time a hook is failing you need to add the changed files again. +The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file. + You will want to merge changes from upstream (i.e. the original repository) into your new branch from time to time, using something like: ```text @@ -133,7 +136,7 @@ Make sure you've commited all the additions or changes you want to include in yo git push origin new-branch-name ``` -### Step 10 - Create a Pull Request +### Step 10 - Create a Pull Request Go to the GitHub website and to _your_ remote bigchaindb repository (i.e. something like https://github.com/your-user-name/bigchaindb). diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index 3fa35391..492235b1 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -8,6 +8,9 @@ Our starting point is [PEP8](https://www.python.org/dev/peps/pep-0008/), the sta BigchainDB uses Python 3.5+, so you can ignore all PEP8 guidelines specific to Python 2. +We use [pre-commit](http://pre-commit.com/) to check some of the rules below before every commit but not everything is realized yet. +The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file. + ### Python Docstrings PEP8 says some things about docstrings, but not what to put in them or how to structure them. [PEP257](https://www.python.org/dev/peps/pep-0257/) was one proposal for docstring conventions, but we prefer [Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) instead: they're easier to read and the [napoleon extension](http://www.sphinx-doc.org/en/stable/ext/napoleon.html) for Sphinx lets us turn them into nice-looking documentation. Here are some references on Google-style docstrings: diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 4c08827b..dbd072d5 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -111,3 +111,27 @@ def get_txids_filtered(conn, asset_id, operation=None): @register_query(LocalMongoDBConnection) def text_search(*args, **kwargs): return mongodb.query.text_search(*args, **kwargs) + + +@register_query(LocalMongoDBConnection) +def get_owned_ids(conn, owner): + cursor = conn.run( + conn.collection('transactions').aggregate([ + {'$match': {'outputs.public_keys': owner}}, + {'$project': {'_id': False}} + ])) + return cursor + + +@register_query(LocalMongoDBConnection) +def get_spending_transactions(conn, inputs): + cursor = conn.run( + conn.collection('transactions').aggregate([ + {'$match': { + 'inputs.fulfills': { + '$in': inputs, + }, + }}, + {'$project': {'_id': False}} + ])) + return cursor diff --git a/bigchaindb/backend/mongodb/changefeed.py b/bigchaindb/backend/mongodb/changefeed.py index c155bb6f..e1b3bb51 100644 --- a/bigchaindb/backend/mongodb/changefeed.py +++ b/bigchaindb/backend/mongodb/changefeed.py @@ -20,6 +20,7 @@ class MongoDBChangeFeed(ChangeFeed): We emulate the behaviour of the RethinkDB changefeed by using a tailable cursor that listens for events on the oplog. """ + def run_forever(self): for element in self.prefeed: self.outqueue.put(element) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 998d5171..92f784d5 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -20,7 +20,6 @@ class MongoDBConnection(Connection): def __init__(self, replicaset=None, ssl=None, login=None, password=None, ca_cert=None, certfile=None, keyfile=None, keyfile_passphrase=None, crlfile=None, **kwargs): - """Create a new Connection instance. Args: diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 198647d6..efbcfcd7 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -408,8 +408,7 @@ def get_last_voted_block_id(connection, node_pubkey): @singledispatch def get_txids_filtered(connection, asset_id, operation=None): - """ - Return all transactions for a particular asset id and optional operation. + """Return all transactions for a particular asset id and optional operation. Args: asset_id (str): ID of transaction that defined the asset @@ -421,8 +420,7 @@ def get_txids_filtered(connection, asset_id, operation=None): @singledispatch def get_new_blocks_feed(connection, start_block_id): - """ - Return a generator that yields change events of the blocks feed + """Return a generator that yields change events of the blocks feed Args: start_block_id (str): ID of block to resume from diff --git a/bigchaindb/backend/rethinkdb/connection.py b/bigchaindb/backend/rethinkdb/connection.py index 3b2459ac..8c1f98ae 100644 --- a/bigchaindb/backend/rethinkdb/connection.py +++ b/bigchaindb/backend/rethinkdb/connection.py @@ -4,8 +4,7 @@ from bigchaindb.backend.exceptions import ConnectionError, OperationError class RethinkDBConnection(Connection): - """ - This class is a proxy to run queries against the database, it is: + """This class is a proxy to run queries against the database, it is: - lazy, since it creates a connection only when needed - resilient, because before raising exceptions it tries diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index 6a572d0c..5887d841 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -293,7 +293,7 @@ def get_votes_for_blocks_by_voter(connection, block_ids, node_pubkey): def unwind_block_transactions(block): - """ Yield a block for each transaction in given block """ + """Yield a block for each transaction in given block""" return block['block']['transactions'].map(lambda tx: block.merge({'tx': tx})) diff --git a/bigchaindb/backend/utils.py b/bigchaindb/backend/utils.py index 4a0c4095..e84d31f4 100644 --- a/bigchaindb/backend/utils.py +++ b/bigchaindb/backend/utils.py @@ -1,6 +1,7 @@ class ModuleDispatchRegistrationError(Exception): """Raised when there is a problem registering dispatched functions for a - module""" + module + """ def module_dispatch_registrar(module): diff --git a/bigchaindb/common/exceptions.py b/bigchaindb/common/exceptions.py index 869aa2c7..697d2d69 100644 --- a/bigchaindb/common/exceptions.py +++ b/bigchaindb/common/exceptions.py @@ -29,7 +29,8 @@ class KeypairNotFoundException(BigchainDBError): class KeypairMismatchException(BigchainDBError): """Raised if the private key(s) provided for signing don't match any of the - current owner(s)""" + current owner(s) + """ class OperationError(BigchainDBError): @@ -55,7 +56,8 @@ class DoubleSpend(ValidationError): class InvalidHash(ValidationError): """Raised if there was an error checking the hash for a particular - operation""" + operation + """ class SchemaValidationError(ValidationError): @@ -64,7 +66,8 @@ class SchemaValidationError(ValidationError): class InvalidSignature(ValidationError): """Raised if there was an error checking the signature for a particular - operation""" + operation + """ class ImproperVoteError(ValidationError): @@ -77,7 +80,8 @@ class MultipleVotesError(ValidationError): class TransactionNotInValidBlock(ValidationError): """Raised when a transfer transaction is attempting to fulfill the - outputs of a transaction that is in an invalid or undecided block""" + outputs of a transaction that is in an invalid or undecided block + """ class AssetIdMismatch(ValidationError): diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index f7f0aa80..0f4990dd 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -1,4 +1,4 @@ -""" Schema validation related functions and data """ +"""Schema validation related functions and data""" import os.path import logging @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) def _load_schema(name): - """ Load a schema from disk """ + """Load a schema from disk""" path = os.path.join(os.path.dirname(__file__), name + '.yaml') with open(path) as handle: schema = yaml.safe_load(handle) @@ -34,7 +34,7 @@ VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote') def _validate_schema(schema, body): - """ Validate data against a schema """ + """Validate data against a schema""" # Note # @@ -59,8 +59,7 @@ def _validate_schema(schema, body): def validate_transaction_schema(tx): - """ - Validate a transaction dict. + """Validate a transaction dict. TX_SCHEMA_COMMON contains properties that are common to all types of transaction. TX_SCHEMA_[TRANSFER|CREATE] add additional constraints on top. @@ -73,5 +72,5 @@ def validate_transaction_schema(tx): def validate_vote_schema(vote): - """ Validate a vote dict """ + """Validate a vote dict""" _validate_schema(VOTE_SCHEMA, vote) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 2399185d..c1cc760a 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -128,8 +128,7 @@ class Input(object): def _fulfillment_to_details(fulfillment): - """ - Encode a fulfillment as a details dictionary + """Encode a fulfillment as a details dictionary Args: fulfillment: Crypto-conditions Fulfillment object @@ -156,8 +155,7 @@ def _fulfillment_to_details(fulfillment): def _fulfillment_from_details(data, _depth=0): - """ - Load a fulfillment for a signing spec dictionary + """Load a fulfillment for a signing spec dictionary Args: data: tx.output[].condition.details dictionary @@ -471,6 +469,7 @@ class Transaction(object): Metadata to be stored along with the Transaction. version (string): Defines the version number of a Transaction. """ + CREATE = 'CREATE' TRANSFER = 'TRANSFER' GENESIS = 'GENESIS' @@ -911,7 +910,7 @@ class Transaction(object): tx_serialized = Transaction._to_str(tx_dict) def validate(i, output_condition_uri=None): - """ Validate input against output condition URI """ + """Validate input against output condition URI""" return self._input_valid(self.inputs[i], self.operation, tx_serialized, output_condition_uri) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index d4657769..74a95dc2 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -135,7 +135,8 @@ def env_config(config): def update_types(config, reference, list_sep=':'): """Return a new configuration where all the values types - are aligned with the ones in the default configuration""" + are aligned with the ones in the default configuration + """ def _coerce(current, value): # Coerce a value to the `current` type. @@ -226,7 +227,8 @@ def is_configured(): def autoconfigure(filename=None, config=None, force=False): """Run ``file_config`` and ``env_config`` if the module has not - been initialized.""" + been initialized. + """ if not force and is_configured(): logger.debug('System already configured, skipping autoconfiguration') return diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index a0672577..0612d6ad 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -9,12 +9,14 @@ class BaseConsensusRules(): All methods listed below must be implemented. """ + voting = Voting @staticmethod def validate_transaction(bigchain, transaction): """See :meth:`bigchaindb.models.Transaction.validate` - for documentation.""" + for documentation. + """ return transaction.validate(bigchain) @staticmethod diff --git a/bigchaindb/core.py b/bigchaindb/core.py index f5ae567f..4fb84722 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -159,8 +159,7 @@ class Bigchain(object): return self.consensus.validate_transaction(self, transaction) def is_new_transaction(self, txid, exclude_block_id=None): - """ - Return True if the transaction does not exist in any + """Return True if the transaction does not exist in any VALID or UNDECIDED block. Return False otherwise. Args: @@ -421,8 +420,7 @@ class Bigchain(object): return fastquery.FastQuery(self.connection, self.me) def get_outputs_filtered(self, owner, spent=None): - """ - Get a list of output links filtered on some criteria + """Get a list of output links filtered on some criteria Args: owner (str): base58 encoded public_key. @@ -443,8 +441,7 @@ class Bigchain(object): return self.fastquery.filter_spent_outputs(outputs) def get_transactions_filtered(self, asset_id, operation=None): - """ - Get a list of transactions filtered on some criteria + """Get a list of transactions filtered on some criteria """ txids = backend.query.get_txids_filtered(self.connection, asset_id, operation) @@ -617,12 +614,12 @@ class Bigchain(object): def block_election_status(self, block): """Tally the votes on a block, and return the status: - valid, invalid, or undecided.""" + valid, invalid, or undecided. + """ return self.block_election(block)['status'] def get_assets(self, asset_ids): - """ - Return a list of assets that match the asset_ids + """Return a list of assets that match the asset_ids Args: asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to @@ -634,8 +631,7 @@ class Bigchain(object): return backend.query.get_assets(self.connection, asset_ids) def get_metadata(self, txn_ids): - """ - Return a list of metadata that match the transaction ids (txn_ids) + """Return a list of metadata that match the transaction ids (txn_ids) Args: txn_ids (:obj:`list` of :obj:`str`): A list of txn_ids to @@ -647,8 +643,7 @@ class Bigchain(object): return backend.query.get_metadata(self.connection, txn_ids) def write_assets(self, assets): - """ - Writes a list of assets into the database. + """Writes a list of assets into the database. Args: assets (:obj:`list` of :obj:`dict`): A list of assets to write to @@ -657,8 +652,7 @@ class Bigchain(object): return backend.query.write_assets(self.connection, assets) def write_metadata(self, metadata): - """ - Writes a list of metadata into the database. + """Writes a list of metadata into the database. Args: metadata (:obj:`list` of :obj:`dict`): A list of metadata to write to @@ -667,8 +661,7 @@ class Bigchain(object): return backend.query.write_metadata(self.connection, metadata) def text_search(self, search, *, limit=0, table='assets'): - """ - Return an iterator of assets that match the text search + """Return an iterator of assets that match the text search Args: search (str): Text search string to query the text index diff --git a/bigchaindb/events.py b/bigchaindb/events.py index 3e3237bf..8b1da2e5 100644 --- a/bigchaindb/events.py +++ b/bigchaindb/events.py @@ -7,7 +7,8 @@ POISON_PILL = 'POISON_PILL' class EventTypes: """Container class that holds all the possible - events BigchainDB manages.""" + events BigchainDB manages. + """ # If you add a new Event Type, make sure to add it # to the docs in docs/server/source/event-plugin-api.rst diff --git a/bigchaindb/fastquery.py b/bigchaindb/fastquery.py index e0431a71..4aed18be 100644 --- a/bigchaindb/fastquery.py +++ b/bigchaindb/fastquery.py @@ -4,8 +4,7 @@ from bigchaindb.common.transaction import TransactionLink class FastQuery: - """ - Database queries that join on block results from a single node. + """Database queries that join on block results from a single node. * Votes are not validated for security (security is a replication concern) * Votes come from only one node, and as such, non-byzantine fault tolerance @@ -18,14 +17,13 @@ class FastQuery: query multiple nodes to insure against getting an incorrect response from a byzantine node. """ + def __init__(self, connection, me): self.connection = connection self.me = me def filter_valid_block_ids(self, block_ids, include_undecided=False): - """ - Given block ids, return only the ones that are valid. - """ + """Given block ids, return only the ones that are valid.""" block_ids = list(set(block_ids)) votes = query.get_votes_for_blocks_by_voter( self.connection, block_ids, self.me) @@ -35,8 +33,7 @@ class FastQuery: if votes.get(block_id, include_undecided)] def filter_valid_items(self, items, block_id_key=lambda b: b[0]): - """ - Given items with block ids, return only the ones that are valid or undecided. + """Given items with block ids, return only the ones that are valid or undecided. """ items = list(items) block_ids = map(block_id_key, items) @@ -44,9 +41,7 @@ class FastQuery: return [b for b in items if block_id_key(b) in valid_block_ids] def get_outputs_by_public_key(self, public_key): - """ - Get outputs for a public key - """ + """Get outputs for a public key""" res = list(query.get_owned_ids(self.connection, public_key)) txs = [tx for _, tx in self.filter_valid_items(res)] return [TransactionLink(tx['id'], index) @@ -56,8 +51,7 @@ class FastQuery: public_key)] def filter_spent_outputs(self, outputs): - """ - Remove outputs that have been spent + """Remove outputs that have been spent Args: outputs: list of TransactionLink @@ -71,8 +65,7 @@ class FastQuery: return [ff for ff in outputs if ff not in spends] def filter_unspent_outputs(self, outputs): - """ - Remove outputs that have not been spent + """Remove outputs that have not been spent Args: outputs: list of TransactionLink diff --git a/bigchaindb/log/loggers.py b/bigchaindb/log/loggers.py index 156a4c41..3b5b08e0 100644 --- a/bigchaindb/log/loggers.py +++ b/bigchaindb/log/loggers.py @@ -12,6 +12,7 @@ class HttpServerLogger(Logger): configuration setting on gunicorn. """ + def setup(self, cfg): """Setup the gunicorn access and error loggers. This overrides the parent method. Its main goal is to simply pipe all the logs to diff --git a/bigchaindb/log/setup.py b/bigchaindb/log/setup.py index 61ada66f..d5df4a8f 100644 --- a/bigchaindb/log/setup.py +++ b/bigchaindb/log/setup.py @@ -116,8 +116,7 @@ class LogRecordStreamHandler(StreamRequestHandler): """ def handle(self): - """ - Handle multiple requests - each expected to be a 4-byte length, + """Handle multiple requests - each expected to be a 4-byte length, followed by the LogRecord in pickle format. Logs the record according to whatever policy is configured locally. """ @@ -152,10 +151,10 @@ class LogRecordStreamHandler(StreamRequestHandler): class LogRecordSocketServer(ThreadingTCPServer): - """ - Simple TCP socket-based logging server. + """Simple TCP socket-based logging server. """ + allow_reuse_address = True def __init__(self, diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 8d45000a..03f16e72 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -96,8 +96,7 @@ class Transaction(Transaction): @classmethod def from_db(cls, bigchain, tx_dict): - """ - Helper method that reconstructs a transaction dict that was returned + """Helper method that reconstructs a transaction dict that was returned from the database. It checks what asset_id to retrieve, retrieves the asset from the asset table and reconstructs the transaction. @@ -346,8 +345,7 @@ class Block(object): @classmethod def from_db(cls, bigchain, block_dict, from_dict_kwargs=None): - """ - Helper method that reconstructs a block_dict that was returned from + """Helper method that reconstructs a block_dict that was returned from the database. It checks what asset_ids to retrieve, retrieves the assets from the assets table and reconstructs the block. @@ -373,8 +371,7 @@ class Block(object): return cls.from_dict(block_dict, **kwargs) def decouple_assets(self, block_dict=None): - """ - Extracts the assets from the ``CREATE`` transactions in the block. + """Extracts the assets from the ``CREATE`` transactions in the block. Returns: tuple: (assets, block) with the assets being a list of dicts and @@ -395,8 +392,7 @@ class Block(object): return (assets, block_dict) def decouple_metadata(self, block_dict=None): - """ - Extracts the metadata from transactions in the block. + """Extracts the metadata from transactions in the block. Returns: tuple: (metadatas, block) with the metadatas being a list of dict/null and @@ -417,8 +413,7 @@ class Block(object): @staticmethod def couple_assets(block_dict, assets): - """ - Given a block_dict with no assets (as returned from a database call) + """Given a block_dict with no assets (as returned from a database call) and a list of assets, reconstruct the original block by putting the assets back into the ``CREATE`` transactions in the block. @@ -442,8 +437,7 @@ class Block(object): @staticmethod def couple_metadata(block_dict, metadatal): - """ - Given a block_dict with no metadata (as returned from a database call) + """Given a block_dict with no metadata (as returned from a database call) and a list of metadata, reconstruct the original block by putting the metadata of each transaction back into its original transaction. @@ -470,8 +464,7 @@ class Block(object): @staticmethod def get_asset_ids(block_dict): - """ - Given a block_dict return all the asset_ids for that block (the txid + """Given a block_dict return all the asset_ids for that block (the txid of CREATE transactions). Useful to know which assets to retrieve from the database to reconstruct the block. @@ -493,8 +486,7 @@ class Block(object): @staticmethod def get_txn_ids(block_dict): - """ - Given a block_dict return all the transaction ids. + """Given a block_dict return all the transaction ids. Args: block_dict (:obj:`dict`): The block dict as returned from a @@ -515,13 +507,13 @@ class Block(object): class FastTransaction: - """ - A minimal wrapper around a transaction dictionary. This is useful for + """A minimal wrapper around a transaction dictionary. This is useful for when validation is not required but a routine expects something that looks like a transaction, for example during block creation. Note: immutability could also be provided """ + def __init__(self, tx_dict): self.data = tx_dict diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index bf4af2e4..4d482255 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -145,7 +145,7 @@ class BlockPipeline: def tx_collector(): - """ A helper to deduplicate transactions """ + """A helper to deduplicate transactions""" def snowflake(): txids = set() @@ -167,7 +167,8 @@ def tx_collector(): def create_pipeline(): """Create and return the pipeline of operations to be distributed - on different processes.""" + on different processes. + """ block_pipeline = BlockPipeline() diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py index 47d5c248..55894b52 100644 --- a/bigchaindb/pipelines/election.py +++ b/bigchaindb/pipelines/election.py @@ -28,8 +28,7 @@ class Election: self.events_queue = events_queue def check_for_quorum(self, next_vote): - """ - Checks if block has enough invalid votes to make a decision + """Checks if block has enough invalid votes to make a decision Args: next_vote: The next vote. @@ -60,8 +59,7 @@ class Election: }) def requeue_transactions(self, invalid_block): - """ - Liquidates transactions from invalid blocks so they can be processed again + """Liquidates transactions from invalid blocks so they can be processed again """ logger.info('Rewriting %s transactions from invalid block %s', len(invalid_block.transactions), diff --git a/bigchaindb/pipelines/stale.py b/bigchaindb/pipelines/stale.py index 010bb8e1..afaad6f3 100644 --- a/bigchaindb/pipelines/stale.py +++ b/bigchaindb/pipelines/stale.py @@ -56,7 +56,8 @@ class StaleTransactionMonitor: def create_pipeline(timeout=5, backlog_reassign_delay=5): """Create and return the pipeline of operations to be distributed - on different processes.""" + on different processes. + """ stm = StaleTransactionMonitor(timeout=timeout, backlog_reassign_delay=backlog_reassign_delay) diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index 366d9575..f6959da0 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -155,7 +155,8 @@ class Vote: def create_pipeline(): """Create and return the pipeline of operations to be distributed - on different processes.""" + on different processes. + """ voter = Vote() @@ -170,7 +171,8 @@ def create_pipeline(): def get_changefeed(): """Create and return ordered changefeed of blocks starting from - last voted block""" + last voted block + """ b = Bigchain() last_block_id = b.get_last_voted_block().id feed = backend.query.get_new_blocks_feed(b.connection, last_block_id) diff --git a/bigchaindb/tendermint/core.py b/bigchaindb/tendermint/core.py index c3692aa5..ea08dfdd 100644 --- a/bigchaindb/tendermint/core.py +++ b/bigchaindb/tendermint/core.py @@ -62,8 +62,12 @@ class App(BaseApplication): logger.debug('check_tx: INVALID') return Result.error() - def begin_block(self, block_hash, header): - """Initialize list of transaction.""" + def begin_block(self, req_begin_block): + """Initialize list of transaction. + Args: + req_begin_block: block object which contains block header + and block hash. + """ self.block_txn_ids = [] diff --git a/bigchaindb/tendermint/fastquery.py b/bigchaindb/tendermint/fastquery.py new file mode 100644 index 00000000..7da15b99 --- /dev/null +++ b/bigchaindb/tendermint/fastquery.py @@ -0,0 +1,48 @@ +from bigchaindb.utils import condition_details_has_owner +from bigchaindb.backend import query +from bigchaindb.common.transaction import TransactionLink + + +class FastQuery(): + """ + Database queries that join on block results from a single node. + """ + + def get_outputs_by_public_key(self, public_key): + """ + Get outputs for a public key + """ + txs = list(query.get_owned_ids(self.connection, public_key)) + return [TransactionLink(tx['id'], index) + for tx in txs + for index, output in enumerate(tx['outputs']) + if condition_details_has_owner(output['condition']['details'], + public_key)] + + def filter_spent_outputs(self, outputs): + """ + Remove outputs that have been spent + + Args: + outputs: list of TransactionLink + """ + links = [o.to_dict() for o in outputs] + txs = list(query.get_spending_transactions(self.connection, links)) + spends = {TransactionLink.from_dict(input_['fulfills']) + for tx in txs + for input_ in tx['inputs']} + return [ff for ff in outputs if ff not in spends] + + def filter_unspent_outputs(self, outputs): + """ + Remove outputs that have not been spent + + Args: + outputs: list of TransactionLink + """ + links = [o.to_dict() for o in outputs] + txs = list(query.get_spending_transactions(self.connection, links)) + spends = {TransactionLink.from_dict(input_['fulfills']) + for tx in txs + for input_ in tx['inputs']} + return [ff for ff in outputs if ff in spends] diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 5b0f1bdb..67f899e4 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -11,6 +11,7 @@ from bigchaindb import Bigchain from bigchaindb.models import Transaction from bigchaindb.common.exceptions import SchemaValidationError, ValidationError from bigchaindb.tendermint.utils import encode_transaction +from bigchaindb.tendermint import fastquery logger = logging.getLogger(__name__) @@ -117,5 +118,9 @@ class BigchainDB(Bigchain): return False return transaction + @property + def fastquery(self): + return fastquery.FastQuery(self.connection, self.me) + Block = namedtuple('Block', ('app_hash', 'height')) diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 03651f8e..cf0bf625 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -83,9 +83,7 @@ def pool(builder, size, timeout=None): # TODO: Rename this function, it's handling fulfillments not conditions def condition_details_has_owner(condition_details, owner): - """ - - Check if the public_key of owner is in the condition details + """Check if the public_key of owner is in the condition details as an Ed25519Fulfillment.public_key Args: diff --git a/bigchaindb/voting.py b/bigchaindb/voting.py index cc20944d..971e9c30 100644 --- a/bigchaindb/voting.py +++ b/bigchaindb/voting.py @@ -12,8 +12,7 @@ UNDECIDED = 'undecided' class Voting: - """ - Everything to do with verifying and counting votes for block election. + """Everything to do with verifying and counting votes for block election. All functions in this class should be referentially transparent, that is, they always give the same output for a given input. This makes it easier @@ -27,9 +26,7 @@ class Voting: @classmethod def block_election(cls, block, votes, keyring): - """ - Calculate the election status of a block. - """ + """Calculate the election status of a block.""" eligible_voters = set(block['block']['voters']) & set(keyring) n_voters = len(eligible_voters) eligible_votes, ineligible_votes = \ @@ -43,8 +40,7 @@ class Voting: @classmethod def partition_eligible_votes(cls, votes, eligible_voters): - """ - Filter votes from unknown nodes or nodes that are not listed on + """Filter votes from unknown nodes or nodes that are not listed on block. This is the primary Sybill protection. """ eligible, ineligible = ([], []) @@ -63,8 +59,7 @@ class Voting: @classmethod def dedupe_by_voter(cls, eligible_votes): - """ - Throw a critical error if there is a duplicate vote + """Throw a critical error if there is a duplicate vote """ by_voter = {} for vote in eligible_votes: @@ -76,8 +71,7 @@ class Voting: @classmethod def count_votes(cls, by_voter): - """ - Given a list of eligible votes, (votes from known nodes that are listed + """Given a list of eligible votes, (votes from known nodes that are listed as voters), produce the number that say valid and the number that say invalid. Votes must agree on previous block, otherwise they become invalid. """ @@ -111,8 +105,7 @@ class Voting: @classmethod def decide_votes(cls, n_voters, n_valid, n_invalid): - """ - Decide on votes. + """Decide on votes. To return VALID there must be a clear majority that say VALID and also agree on the previous block. @@ -127,8 +120,7 @@ class Voting: @classmethod def verify_vote_signature(cls, vote): - """ - Verify the signature of a vote + """Verify the signature of a vote """ signature = vote.get('signature') pk_base58 = vote.get('node_pubkey') diff --git a/bigchaindb/web/routes.py b/bigchaindb/web/routes.py index e74c58cd..b5fd8eb3 100644 --- a/bigchaindb/web/routes.py +++ b/bigchaindb/web/routes.py @@ -1,4 +1,4 @@ -""" API routes definition """ +"""API routes definition""" from flask_restful import Api from bigchaindb.web.views import ( assets, @@ -13,7 +13,7 @@ from bigchaindb.web.views import ( def add_routes(app): - """ Add the routes to an app """ + """Add the routes to an app""" for (prefix, routes) in API_SECTIONS: api = Api(app, prefix=prefix) for ((pattern, resource, *args), kwargs) in routes: diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index fa10d25b..2fdb17a5 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -25,13 +25,13 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication): """ def __init__(self, app, *, options=None): - '''Initialize a new standalone application. + """Initialize a new standalone application. Args: app: A wsgi Python application. options (dict): the configuration. - ''' + """ self.options = options or {} self.application = app super().__init__() diff --git a/bigchaindb/web/views/base.py b/bigchaindb/web/views/base.py index 4e7cf8fc..9d804d89 100644 --- a/bigchaindb/web/views/base.py +++ b/bigchaindb/web/views/base.py @@ -1,5 +1,4 @@ -""" -Common classes and methods for API handlers +"""Common classes and methods for API handlers """ import logging diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index 810eb9c5..d6162a34 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -1,4 +1,4 @@ -""" API Index endpoint """ +"""API Index endpoint""" import flask from flask_restful import Resource @@ -33,8 +33,7 @@ class ApiV1Index(Resource): def get_api_v1_info(api_prefix): - """ - Return a dict with all the information specific for the v1 of the + """Return a dict with all the information specific for the v1 of the api. """ websocket_root = base_ws_uri() + EVENTS_ENDPOINT diff --git a/docker-compose.network.yml b/docker-compose.network.yml index 311b8035..38a4b5c4 100644 --- a/docker-compose.network.yml +++ b/docker-compose.network.yml @@ -30,7 +30,7 @@ services: TENDERMINT_HOST: tendermint-one ports: - "9984" - command: bigchaindb -l DEBUG start --init + command: bigchaindb -l DEBUG start tendermint-one: image: tendermint/tendermint volumes: @@ -67,7 +67,7 @@ services: TENDERMINT_HOST: tendermint-two ports: - "9984" - command: bigchaindb -l DEBUG start --init + command: bigchaindb -l DEBUG start tendermint-two: image: tendermint/tendermint volumes: @@ -104,7 +104,7 @@ services: TENDERMINT_HOST: tendermint-three ports: - "9984" - command: bigchaindb -l DEBUG start --init + command: bigchaindb -l DEBUG start tendermint-three: image: tendermint/tendermint volumes: @@ -141,7 +141,7 @@ services: TENDERMINT_HOST: tendermint-four ports: - "9984" - command: bigchaindb -l DEBUG start --init + command: bigchaindb -l DEBUG start tendermint-four: image: tendermint/tendermint volumes: diff --git a/docker-compose.tendermint.yml b/docker-compose.tendermint.yml index 966368c4..2c43b36a 100644 --- a/docker-compose.tendermint.yml +++ b/docker-compose.tendermint.yml @@ -29,7 +29,7 @@ services: TENDERMINT_PORT: 46657 ports: - "9984" - command: bigchaindb -l DEBUG start --init + command: bigchaindb -l DEBUG start tendermint: image: tendermint/tendermint volumes: diff --git a/docs/root/source/assets.rst b/docs/root/source/assets.rst index f08deaf0..a0aa5a31 100644 --- a/docs/root/source/assets.rst +++ b/docs/root/source/assets.rst @@ -6,19 +6,11 @@ BigchainDB can store data of any kind (within reason), but it's designed to be p * The fundamental thing that one sends to a BigchainDB cluster, to be checked and stored (if valid), is a *transaction*, and there are two kinds: CREATE transactions and TRANSFER transactions. * A CREATE transaction can be use to register any kind of asset (divisible or indivisible), along with arbitrary metadata. * An asset can have zero, one, or several owners. -* The owners of an asset can specify (crypto-)conditions which must be satisfied by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a transfer transaction. -* BigchainDB verifies that the conditions have been satisfied as part of checking the validity of transfer transactions. (Moreover, anyone can check that they were satisfied.) +* The owners of an asset can specify (crypto-)conditions which must be satisfied by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a TRANSFER transaction. +* BigchainDB verifies that the conditions have been satisfied as part of checking the validity of TRANSFER transactions. (Moreover, anyone can check that they were satisfied.) * BigchainDB prevents double-spending of an asset. -* Validated transactions are strongly tamper-resistant; see :doc:`the page about immutability `. - - -BigchainDB Integration with Other Blockchains ---------------------------------------------- - -BigchainDB works with the `Interledger protocol `_, enabling the transfer of assets between BigchainDB and other blockchains, ledgers, and payment systems. - -We’re actively exploring ways that BigchainDB can be used with other blockchains and platforms. +* Validated transactions are :doc:`"immutable" `. .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See BigchainDB Server `issue #626 `_. + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec `_. diff --git a/docs/root/source/bft.md b/docs/root/source/bft.md index fce8ca3d..919a0df9 100644 --- a/docs/root/source/bft.md +++ b/docs/root/source/bft.md @@ -1,8 +1,6 @@ # BigchainDB and Byzantine Fault Tolerance -While BigchainDB is not currently [Byzantine fault tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance), we plan to offer it as an option. -Update Nov 2017: we're actively working on this, the next release or two will likely have support. More details to come in blog form and github issues - -Related issue: [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). We anticipate that turning on BFT will cause a dropoff in performance (for a gain in security). - -In the meantime, there are practical things that one can do to increase security (e.g. firewalls, key management, and access controls). +[BigchainDB Server](https://docs.bigchaindb.com/projects/server/en/latest/index.html) +uses [Tendermint](https://tendermint.com/) +for consensus and transaction replication, +and Tendermint is [Byzantine Fault Tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance). diff --git a/docs/root/source/decentralized.md b/docs/root/source/decentralized.md index c0c213cf..755a84a2 100644 --- a/docs/root/source/decentralized.md +++ b/docs/root/source/decentralized.md @@ -14,8 +14,6 @@ A consortium can increase its decentralization (and its resilience) by increasin There’s no node that has a long-term special position in the cluster. All nodes run the same software and perform the same duties. -MongoDB and RethinkDB have an “admin” user which can’t be deleted and which can make big changes to the database, such as dropping a table. Right now, that’s a big security vulnerability, but we have plans to mitigate it by: -1. Locking down the admin user as much as possible. -2. Having all nodes inspect admin-type requests before acting on them. Requests can be checked against an evolving whitelist of allowed actions. Nodes requesting non-allowed requests can be removed from the list of cluster nodes. +If someone has (or gets) admin access to a node, they can mess with that node (e.g. change or delete data stored on that node), but those changes should remain isolated to that node. The BigchainDB cluster can only be compromised if more than one third of the nodes get compromised. See the [Tendermint documentation](https://tendermint.readthedocs.io/projects/tools/en/master/introduction.html) for more details. -It’s worth noting that the admin user can’t transfer assets, even today. The only way to create a valid transfer transaction is to fulfill the current (crypto) conditions on the asset, and the admin user can’t do that because the admin user doesn’t have the necessary private keys (or preimages, in the case of hashlock conditions). They’re not stored in the database. +It’s worth noting that not even the admin or superuser of a node can transfer assets. The only way to create a valid transfer transaction is to fulfill the current crypto-conditions on the asset, and the admin/superuser can’t do that because the admin user doesn’t have the necessary information (e.g. private keys). diff --git a/docs/root/source/diversity.md b/docs/root/source/diversity.md index 20c9afb5..767962e1 100644 --- a/docs/root/source/diversity.md +++ b/docs/root/source/diversity.md @@ -1,11 +1,10 @@ # Kinds of Node Diversity -Steps should be taken to make it difficult for any one actor or event to control or damage “enough” of the nodes. (“Enough” is usually a quorum.) There are many kinds of diversity to consider, listed below. It may be quite difficult to have high diversity of all kinds. +Steps should be taken to make it difficult for any one actor or event to control or damage “enough” of the nodes. (Because BigchainDB Server uses Tendermint, "enough" is ⅓.) There are many kinds of diversity to consider, listed below. It may be quite difficult to have high diversity of all kinds. 1. **Jurisdictional diversity.** The nodes should be controlled by entities within multiple legal jurisdictions, so that it becomes difficult to use legal means to compel enough of them to do something. -2. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems. -3. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes. -4. **Operating system diversity.** The servers should use a variety of operating systems, so that a security bug in one OS can’t be used to exploit enough of the nodes. -5. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a consortium. For example, it provides the consortium with a source of various ideas for addressing challenges. +1. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems. +1. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes. +1. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a consortium. For example, it provides the consortium with a source of various ideas for addressing challenges. -Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a consortium could also have a diversity of server implementations. +Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a consortium could also have a diversity of server implementations. Similar remarks can be made about the operating system. diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst index 71fdd022..be2f5f1b 100644 --- a/docs/root/source/index.rst +++ b/docs/root/source/index.rst @@ -2,13 +2,12 @@ BigchainDB Documentation ======================== `BigchainDB `_ is a scalable blockchain database. -That is, it's a "big data" database with some blockchain characteristics added, including `decentralization `_, +It has some database characteristics and some blockchain characteristics, +including `decentralization `_, `immutability `_ -and -`native support for assets `_. -You can read about the motivations, goals and high-level architecture in the `BigchainDB whitepaper `_. +and `native support for assets `_. -At a high level, one can communicate with a BigchainDB cluster (set of nodes) using the BigchainDB Client-Server HTTP API, or a wrapper for that API, such as the BigchainDB Python Driver. Each BigchainDB node runs BigchainDB Server and various other software. The `terminology page `_ explains some of those terms in more detail. +At a high level, one can communicate with a BigchainDB cluster (set of nodes) using the BigchainDB HTTP API, or a wrapper for that API, such as the BigchainDB Python Driver. Each BigchainDB node runs BigchainDB Server and various other software. The `terminology page `_ explains some of those terms in more detail. .. raw:: html @@ -79,7 +78,6 @@ More About BigchainDB :maxdepth: 1 BigchainDB Docs Home - production-ready terminology decentralized diversity @@ -89,5 +87,4 @@ More About BigchainDB smart-contracts transaction-concepts permissions - timestamps Data Models diff --git a/docs/root/source/permissions.rst b/docs/root/source/permissions.rst index d5972ac0..b934effd 100644 --- a/docs/root/source/permissions.rst +++ b/docs/root/source/permissions.rst @@ -15,7 +15,8 @@ To spend/transfer an unspent output, a user (or group of users) must fulfill the - "…three of these four people must sign." - "…either Bob must sign, or both Tom and Sylvia must sign." -For details, see `the documentation about conditions in BigchainDB `_. +For details, see +`the documentation about conditions in the IPDB Transaction Spec `_. Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output. diff --git a/docs/root/source/production-ready.md b/docs/root/source/production-ready.md deleted file mode 100644 index 21358c37..00000000 --- a/docs/root/source/production-ready.md +++ /dev/null @@ -1,14 +0,0 @@ -# Production-Ready? - -BigchainDB is not production-ready. You can use it to build a prototype or proof-of-concept (POC); many people are already doing that. -Once BigchainDB is production-ready, we'll make an announcement. - -BigchainDB version numbers follow the conventions of *Semantic Versioning* as documented at [semver.org](http://semver.org/). (For Python stuff, we use [Python's version of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme).) This means, among other things: - -* Before version 1.0, breaking API changes could happen in any new version, even in a change from version 0.Y.4 to 0.Y.5. - -* Starting with version 1.0.0, breaking API changes will only happen when the MAJOR version changes (e.g. from 1.7.4 to 2.0.0, or from 4.9.3 to 5.0.0). - -To review the release history of some particular BigchainDB software, go to the GitHub repository of that software and click on "Releases". For example, the release history of BigchainDB Server can be found at [https://github.com/bigchaindb/bigchaindb/releases](https://github.com/bigchaindb/bigchaindb/releases). - -[The BigchainDB Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) will give you a sense of the things we intend to do with BigchainDB in the near term and the long term. \ No newline at end of file diff --git a/docs/root/source/smart-contracts.rst b/docs/root/source/smart-contracts.rst index 04c0c9bb..143d6dfe 100644 --- a/docs/root/source/smart-contracts.rst +++ b/docs/root/source/smart-contracts.rst @@ -15,5 +15,4 @@ Crypto-conditions can be quite complex. They can't include loops or recursion an .. note:: - We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See BigchainDB Server `issue #626 `_. - \ No newline at end of file + We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec `_. diff --git a/docs/root/source/terminology.md b/docs/root/source/terminology.md index a9ece75d..ef05436e 100644 --- a/docs/root/source/terminology.md +++ b/docs/root/source/terminology.md @@ -2,16 +2,13 @@ There is some specialized terminology associated with BigchainDB. To get started, you should at least know the following: - ## BigchainDB Node -A **BigchainDB node** is a machine or set of closely-linked machines running MongoDB Server (or RethinkDB Server), BigchainDB Server, and related software. Each node is controlled by one person or organization. - +A **BigchainDB node** is a machine (or logical machine) running [BigchainDB Server](https://docs.bigchaindb.com/projects/server/en/latest/introduction.html) and related software. Each node is controlled by one person or organization. ## BigchainDB Cluster -A set of BigchainDB nodes can connect to each other to form a **BigchainDB cluster**. Each node in the cluster runs the same software. A cluster contains one logical MongoDB/RethinkDB datastore. A cluster may have additional machines to do things such as cluster monitoring. - +A set of BigchainDB nodes can connect to each other to form a **BigchainDB cluster**. Each node in the cluster runs the same software. A cluster may have additional machines to do things such as cluster monitoring. ## BigchainDB Consortium diff --git a/docs/root/source/timestamps.md b/docs/root/source/timestamps.md deleted file mode 100644 index 79d5e700..00000000 --- a/docs/root/source/timestamps.md +++ /dev/null @@ -1,84 +0,0 @@ -# Timestamps in BigchainDB - -Each block and vote has an associated timestamp. Interpreting those timestamps is tricky, hence the need for this section. - - -## Timestamp Sources & Accuracy - -Timestamps in BigchainDB are provided by the node which created the block and the node that created the vote. - -When a BigchainDB node needs a timestamp, it calls a BigchainDB utility function named `timestamp()`. There's a detailed explanation of how that function works below, but the short version is that it gets the [Unix time](https://en.wikipedia.org/wiki/Unix_time) from its system clock, rounded to the nearest second. - -We advise BigchainDB nodes to run special software (an "NTP daemon") to keep their system clock in sync with standard time servers. (NTP stands for [Network Time Protocol](https://en.wikipedia.org/wiki/Network_Time_Protocol).) - - -## Converting Timestamps to UTC - -To convert a BigchainDB timestamp (a Unix time) to UTC, you need to know how the node providing the timestamp was set up. That's because different setups will report a different "Unix time" value around leap seconds! There's [a nice Red Hat Developer Blog post about the various setup options](https://developers.redhat.com/blog/2015/06/01/five-different-ways-handle-leap-seconds-ntp/). If you want more details, see [David Mills' pages about leap seconds, NTP, etc.](https://www.eecis.udel.edu/~mills/leap.html) (David Mills designed NTP.) - -We advise BigchainDB nodes to run an NTP daemon with particular settings so that their timestamps are consistent. - -If a timestamp comes from a node that's set up as we advise, it can be converted to UTC as follows: - -1. Use a standard "Unix time to UTC" converter to get a UTC timestamp. -2. Is the UTC timestamp a leap second, or the second before/after a leap second? There's [a list of all the leap seconds on Wikipedia](https://en.wikipedia.org/wiki/Leap_second). -3. If no, then you are done. -4. If yes, then it might not be possible to convert it to a single UTC timestamp. Even if it can't be converted to a single UTC timestamp, it _can_ be converted to a list of two possible UTC timestamps. -Showing how to do that is beyond the scope of this documentation. -In all likelihood, you will never have to worry about leap seconds because they are very rare. -(There were only 26 between 1972 and the end of 2015.) - - -## Calculating Elapsed Time Between Two Timestamps - -There's another gotcha with (Unix time) timestamps: you can't calculate the real-world elapsed time between two timestamps (correctly) by subtracting the smaller timestamp from the larger one. The result won't include any of the leap seconds that occured between the two timestamps. You could look up how many leap seconds happened between the two timestamps and add that to the result. There are many library functions for working with timestamps; those are beyond the scope of this documentation. - - -## Interpreting Sets of Timestamps - -You can look at many timestamps to get a statistical sense of when something happened. For example, a transaction in a decided-valid block has many associated timestamps: - -* the timestamp of the block -* the timestamps of all the votes on the block - - -## How BigchainDB Uses Timestamps - -BigchainDB _doesn't_ use timestamps to determine the order of transactions or blocks. In particular, the order of blocks is determined by MongoDB's oplog (or RethinkDB's changefeed) on the bigchain table. - -BigchainDB does use timestamps for some things. When a Transaction is written to the backlog, a timestamp is assigned called the `assignment_timestamp`, to determine if it has been waiting in the backlog for too long (i.e. because the node assigned to it hasn't handled it yet). - - -## Including Trusted Timestamps - -If you want to create a transaction payload with a trusted timestamp, you can. - -One way to do that would be to send a payload to a trusted timestamping service. They will send back a timestamp, a signature, and their public key. They should also explain how you can verify the signature. You can then include the original payload, the timestamp, the signature, and the service's public key in your transaction metadata. That way, anyone with the verification instructions can verify that the original payload was signed by the trusted timestamping service. - - -## How the timestamp() Function Works - -BigchainDB has a utility function named `timestamp()` which amounts to: -```python -timestamp() = str(round(time.time())) -``` - -In other words, it calls the `time()` function in Python's `time` module, [rounds](https://docs.python.org/3/library/functions.html#round) that to the nearest integer, and converts the result to a string. - -It rounds the output of `time.time()` to the nearest second because, according to [the Python documentation for `time.time()`](https://docs.python.org/3.4/library/time.html#time.time), "...not all systems provide time with a better precision than 1 second." - -How does `time.time()` work? If you look in the C source code, it calls `floattime()` and `floattime()` calls [clock_gettime()](https://www.cs.rutgers.edu/~pxk/416/notes/c-tutorials/gettime.html), if it's available. -```text -ret = clock_gettime(CLOCK_REALTIME, &tp); -``` - -With `CLOCK_REALTIME` as the first argument, it returns the "Unix time." ("Unix time" is in quotes because its value around leap seconds depends on how the system is set up; see above.) - - -## Why Not Use UTC, TAI or Some Other Time that Has Unambiguous Timestamps for Leap Seconds? - -It would be nice to use UTC or TAI timestamps, but unfortunately there's no commonly-available, standard way to get always-accurate UTC or TAI timestamps from the operating system on typical computers today (i.e. accurate around leap seconds). - -There _are_ commonly-available, standard ways to get the "Unix time," such as clock_gettime() function available in C. That's what we use (indirectly via Python). ("Unix time" is in quotes because its value around leap seconds depends on how the system is set up; see above.) - -The Unix-time-based timestamps we use are only ambiguous circa leap seconds, and those are very rare. Even for those timestamps, the extra uncertainty is only one second, and that's not bad considering that we only report timestamps to a precision of one second in the first place. All other timestamps can be converted to UTC with no ambiguity. diff --git a/docs/root/source/transaction-concepts.md b/docs/root/source/transaction-concepts.md index 66c4ec0f..02820ac5 100644 --- a/docs/root/source/transaction-concepts.md +++ b/docs/root/source/transaction-concepts.md @@ -6,7 +6,6 @@ things (e.g. assets). Transactions are the most basic kind of record stored by BigchainDB. There are two kinds: CREATE transactions and TRANSFER transactions. - ## CREATE Transactions A CREATE transaction can be used to register, issue, create or otherwise @@ -31,20 +30,19 @@ Each output also has an associated condition: the condition that must be met BigchainDB supports a variety of conditions, a subset of the [Interledger Protocol (ILP)](https://interledger.org/) crypto-conditions. For details, see -[the documentation about Inputs and Outputs](https://docs.bigchaindb.com/projects/server/en/latest/data-models/inputs-outputs.html). +[the documentation about conditions in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-components/conditions.html). Each output also has a list of all the public keys associated with the conditions on that output. Loosely speaking, that list might be interpreted as the list of "owners." A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. -See BigchainDB Server [issue #626](https://github.com/bigchaindb/bigchaindb/issues/626). +See the [note about "owners" in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/ownership.html). A CREATE transaction must be signed by all the owners. (If you're looking for that signature, it's in the one "fulfillment" of the one input, albeit encoded.) - ## TRANSFER Transactions A TRANSFER transaction can transfer/spend one or more outputs @@ -82,7 +80,6 @@ transferred if both Jack and Kelly sign. Note how the sum of the incoming paperclips must equal the sum of the outgoing paperclips (100). - ## Transaction Validity When a node is asked to check if a transaction is valid, it checks several @@ -90,6 +87,7 @@ things. We documented those things in a post on *The BigchainDB Blog*: ["What is a Valid Transaction in BigchainDB?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598) (Note: That post was about BigchainDB Server v1.0.0.) +The [IPDB Transaction Spec documents the conditions for a transaction to be valid](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-validation.html). ## Example Transactions diff --git a/docs/server/source/appendices/azure-quickstart-template.md b/docs/server/source/appendices/azure-quickstart-template.md deleted file mode 100644 index 59f52fd3..00000000 --- a/docs/server/source/appendices/azure-quickstart-template.md +++ /dev/null @@ -1,68 +0,0 @@ -# Azure Quickstart Template - -This page outlines how to run a single BigchainDB node on the Microsoft Azure public cloud, with RethinkDB as the database backend. It uses an Azure Quickstart Template. That template is dated because we now recommend using MongoDB instead of RethinkDB. That's why we moved this page to the Appendices. - -Note: There was an Azure quickstart template in the `blockchain` directory of Microsoft's `Azure/azure-quickstart-templates` repository on GitHub. It's gone now; it was replaced by the one described here. - -One can deploy a BigchainDB node on Azure using the template in the `bigchaindb-on-ubuntu` directory of Microsoft's `Azure/azure-quickstart-templates` repository on GitHub. Here's how: - -1. Go to [that directory on GitHub](https://github.com/Azure/azure-quickstart-templates/tree/master/bigchaindb-on-ubuntu). - -1. Click the button labelled **Deploy to Azure**. - -1. If you're not already logged in to Microsoft Azure, then you'll be prompted to login. If you don't have an account, then you'll have to create one. - -1. Once you are logged in to the Microsoft Azure Portal, you should be taken to a form titled **BigchainDB**. Some notes to help with filling in that form are available [below](azure-quickstart-template.html#notes-on-the-blockchain-template-form-fields). - -1. Deployment takes a few minutes. You can follow the notifications by clicking the bell icon at the top of the screen. At the time of writing, the final deployment operation (running the `init.sh` script) was failing, but a pull request ([#2884](https://github.com/Azure/azure-quickstart-templates/pull/2884)) has been made to fix that and these instructions say what you can do before that pull request gets merged... - -1. Find out the public IP address of the virtual machine in the Azure Portal. Example: `40.69.87.250` - -1. ssh in to the virtual machine at that IP address, i.e. do `ssh @` where `` is the admin username you entered into the form and `` is the virtual machine IP address determined in the last step. Example: `ssh bcdbadmin@40.69.87.250` - -1. You should be prompted for a password. Give the `` you entered into the form. - -1. Configure BigchainDB Server by doing: -```text -bigchaindb configure rethinkdb -``` -It will ask you several questions. You can press `Enter` (or `Return`) to accept the default for all of them *except for one*. When it asks **API Server bind? (default \`localhost:9984\`):**, you should answer: -```text -API Server bind? (default `localhost:9984`): 0.0.0.0:9984 -``` - -Finally, run BigchainDB Server by doing: -```text -bigchaindb start -``` - -BigchainDB Server should now be running on the Azure virtual machine. - -Remember to shut everything down when you're done (via the Azure Portal), because it generally costs money to run stuff on Azure. - - -## Notes on the Blockchain Template Form Fields - -### BASICS - -**Resource group** - You can use an existing resource group (if you have one) or create a new one named whatever you like, but avoid using fancy characters in the name because Azure might have problems if you do. - -**Location** is the Microsoft Azure data center where you want the BigchainDB node to run. Pick one close to where you are located. - -### SETTINGS - -You can use whatever **Admin\_username** and **Admin\_password** you like (provided you don't get too fancy). It will complain if your password is too simple. You'll need these later to `ssh` into the virtual machine. - -**Dns\_label\_prefix** - Once your virtual machine is deployed, it will have a public IP address and a DNS name (hostname) something like `.northeurope.cloudapp.azure.com`. The `` will be whatever you enter into this field. - -**Virtual\_machine\_size** - This should be one of Azure's standard virtual machine sizes, such as `Standard_D1_v2`. There's a [list of virtual machine sizes in the Azure docs](https://docs.microsoft.com/en-us/azure/virtual-machines/virtual-machines-windows-sizes?toc=%2fazure%2fvirtual-machines%2fwindows%2ftoc.json). - -**\_artifacts Location** - Leave this alone. - -**\_artifacts Location Sas Token** - Leave this alone (blank). - -### TERMS AND CONDITIONS - -Read the terms and conditions. If you agree to them, then check the checkbox. - -Finally, click the button labelled **Purchase**. (Generally speaking, it costs money to run stuff on Azure.) diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index a4ff150a..588f7ecd 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -1,6 +1,3 @@ -.. You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - Appendices ========== @@ -8,23 +5,17 @@ Appendices :maxdepth: 1 install-os-level-deps - install-latest-pip run-with-docker json-serialization cryptography the-Bigchain-class - pipelines backend commands aws-setup - azure-quickstart-template generate-key-pair-for-ssh firewall-notes ntp-notes - rethinkdb-reqs - rethinkdb-backup licenses - install-with-lxd run-with-vagrant run-with-ansible vote-yaml diff --git a/docs/server/source/appendices/install-latest-pip.md b/docs/server/source/appendices/install-latest-pip.md deleted file mode 100644 index fc992fee..00000000 --- a/docs/server/source/appendices/install-latest-pip.md +++ /dev/null @@ -1,20 +0,0 @@ -# How to Install the Latest pip and setuptools - -You can check the version of `pip` you're using (in your current virtualenv) by doing: -```text -pip -V -``` - -If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.5, then you must install a `pip` version associated with Python 3.5+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). - -On Ubuntu 16.04, we found that this works: -```text -sudo apt-get install python3-pip -``` - -That should install a Python 3 version of `pip` named `pip3`. If that didn't work, then another way to get `pip3` is to do `sudo apt-get install python3-setuptools` followed by `sudo easy_install3 pip`. - -You can upgrade `pip` (`pip3`) and `setuptools` to the latest versions using: -```text -pip3 install --upgrade pip setuptools -``` diff --git a/docs/server/source/appendices/install-with-lxd.md b/docs/server/source/appendices/install-with-lxd.md deleted file mode 100644 index 969f6841..00000000 --- a/docs/server/source/appendices/install-with-lxd.md +++ /dev/null @@ -1,43 +0,0 @@ -# Installing BigchainDB on LXC containers using LXD - -**Note: This page was contributed by an external contributor and is not actively maintained. We include it in case someone is interested.** - -You can visit this link to install LXD (instructions here): [LXD Install](https://linuxcontainers.org/lxd/getting-started-cli/) - -(assumption is that you are using Ubuntu 14.04 for host/container) - -Let us create an LXC container (via LXD) with the following command: - -`lxc launch ubuntu:14.04 bigchaindb` - -(ubuntu:14.04 - this is the remote server the command fetches the image from) -(bigchaindb - is the name of the container) - -Below is the `install.sh` script you will need to install BigchainDB within your container. - -Here is my `install.sh`: - -``` -#!/bin/bash -set -ex -export DEBIAN_FRONTEND=noninteractive -apt-get install -y wget -source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list -wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - -apt-get update -apt-get install -y rethinkdb python3-pip -pip3 install --upgrade pip wheel setuptools -pip install ptpython bigchaindb -``` - -Copy/Paste the above `install.sh` into the directory/path you are going to execute your LXD commands from (ie. the host). - -Make sure your container is running by typing: - -`lxc list` - -Now, from the host (and the correct directory) where you saved `install.sh`, run this command: - -`cat install.sh | lxc exec bigchaindb /bin/bash` - -If you followed the commands correctly, you will have successfully created an LXC container (using LXD) that can get you up and running with BigchainDB in <5 minutes (depending on how long it takes to download all the packages). diff --git a/docs/server/source/appendices/pipelines.rst b/docs/server/source/appendices/pipelines.rst deleted file mode 100644 index 8016e8cb..00000000 --- a/docs/server/source/appendices/pipelines.rst +++ /dev/null @@ -1,26 +0,0 @@ -######### -Pipelines -######### - -Block Creation -============== - -.. automodule:: bigchaindb.pipelines.block - - -Block Voting -============ - -.. automodule:: bigchaindb.pipelines.vote - - -Block Status -============ - -.. automodule:: bigchaindb.pipelines.election - - -Stale Transaction Monitoring -============================ - -.. automodule:: bigchaindb.pipelines.stale diff --git a/docs/server/source/appendices/rethinkdb-backup.md b/docs/server/source/appendices/rethinkdb-backup.md deleted file mode 100644 index dc33231e..00000000 --- a/docs/server/source/appendices/rethinkdb-backup.md +++ /dev/null @@ -1,124 +0,0 @@ -# Backing Up and Restoring Data - -This page was written when BigchainDB only worked with RethinkDB, so its focus is on RethinkDB-based backup. BigchainDB now supports MongoDB as a backend database and we recommend that you use MongoDB in production. Nevertheless, some of the following backup ideas are still relevant regardless of the backend database being used, so we moved this page to the Appendices. - - -## RethinkDB's Replication as a form of Backup - -RethinkDB already has internal replication: every document is stored on _R_ different nodes, where _R_ is the replication factor (set using `bigchaindb set-replicas R`). Those replicas can be thought of as "live backups" because if one node goes down, the cluster will continue to work and no data will be lost. - -At this point, there should be someone saying, "But replication isn't backup!" - -It's true. Replication alone isn't enough, because something bad might happen _inside_ the database, and that could affect the replicas. For example, what if someone logged in as a RethinkDB admin and did a "drop table"? We currently plan for each node to be protected by a next-generation firewall (or something similar) to prevent such things from getting very far. For example, see [issue #240](https://github.com/bigchaindb/bigchaindb/issues/240). - -Nevertheless, you should still consider having normal, "cold" backups, because bad things can still happen. - - -## Live Replication of RethinkDB Data Files - -Each BigchainDB node stores its subset of the RethinkDB data in one directory. You could set up the node's file system so that directory lives on its own hard drive. Furthermore, you could make that hard drive part of a [RAID](https://en.wikipedia.org/wiki/RAID) array, so that a second hard drive would always have a copy of the original. If the original hard drive fails, then the second hard drive could take its place and the node would continue to function. Meanwhile, the original hard drive could be replaced. - -That's just one possible way of setting up the file system so as to provide extra reliability. - -Another way to get similar reliability would be to mount the RethinkDB data directory on an [Amazon EBS](https://aws.amazon.com/ebs/) volume. Each Amazon EBS volume is, "automatically replicated within its Availability Zone to protect you from component failure, offering high availability and durability." - -As with shard replication, live file-system replication protects against many failure modes, but it doesn't protect against them all. You should still consider having normal, "cold" backups. - - -## rethinkdb dump (to a File) - -RethinkDB can create an archive of all data in the cluster (or all data in specified tables), as a compressed file. According to [the RethinkDB blog post when that functionality became available](https://rethinkdb.com/blog/1.7-release/): - -> Since the backup process is using client drivers, it automatically takes advantage of the MVCC [multiversion concurrency control] functionality built into RethinkDB. It will use some cluster resources, but will not lock out any of the clients, so you can safely run it on a live cluster. - -To back up all the data in a BigchainDB cluster, the RethinkDB admin user must run a command like the following on one of the nodes: -```text -rethinkdb dump -e bigchain.bigchain -e bigchain.votes -``` - -That should write a file named `rethinkdb_dump__