Mirroristas/bigchaindb
2
0
Fork 0
mirror of https://github.com/bigchaindb/bigchaindb.git synced 2024-10-13 13:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'tendermint' into rm-aws-deployment-tools-and-docs

Browse Source
This commit is contained in:
Troy McConaghy 2017-12-06 14:14:04 +01:00 committed by GitHub
commit e0b454953c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
78 changed files with 328 additions and 705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,20 @@
repos:
- repo: git://github.com/pre-commit/pre-commit-hooks
sha: v1.1.1
hooks:
- id: trailing-whitespace
args: ['--no-markdown-linebreak-ext']
- id: check-merge-conflict
- id: debug-statements
- id: check-added-large-files
- id: flake8
- repo: git://github.com/chewse/pre-commit-mirrors-pydocstyle
sha: v2.1.1
hooks:
- id: pydocstyle
# list of error codes to check, see: http://www.pydocstyle.org/en/latest/error_codes.html
args: ['--select=D204,D201,D209,D210,D212,D300,D403']
# negate the exclude to only apply the hooks to 'bigchaindb' and 'tests' folder
exclude: '^(?!bigchaindb/)(?!tests/)'

5
CONTRIBUTING.md
View File

@ -115,6 +115,9 @@ git add new-or-changed-file-1
git add new-or-changed-file-2
git commit -m "Short description of new or changed things"
```
We use [pre-commit](http://pre-commit.com/) which should be triggered with every commit. Some hooks will change files but others will give errors that needs to be fixed. Every time a hook is failing you need to add the changed files again.
The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file.
You will want to merge changes from upstream (i.e. the original repository) into your new branch from time to time, using something like:
```text
@ -133,7 +136,7 @@ Make sure you've commited all the additions or changes you want to include in yo
git push origin new-branch-name
```
### Step 10 - Create a Pull Request
### Step 10 - Create a Pull Request
Go to the GitHub website and to _your_ remote bigchaindb repository (i.e. something like https://github.com/your-user-name/bigchaindb).

3
PYTHON_STYLE_GUIDE.md
View File

@ -8,6 +8,9 @@ Our starting point is [PEP8](https://www.python.org/dev/peps/pep-0008/), the sta
BigchainDB uses Python 3.5+, so you can ignore all PEP8 guidelines specific to Python 2.
We use [pre-commit](http://pre-commit.com/) to check some of the rules below before every commit but not everything is realized yet.
The hooks we use can be found in the [.pre-commit-config.yaml](https://github.com/bigchaindb/bigchaindb/blob/master/.pre-commit-config.yaml) file.
### Python Docstrings
PEP8 says some things about docstrings, but not what to put in them or how to structure them. [PEP257](https://www.python.org/dev/peps/pep-0257/) was one proposal for docstring conventions, but we prefer [Google-style docstrings](https://google.github.io/styleguide/pyguide.html?showone=Comments#Comments) instead: they're easier to read and the [napoleon extension](http://www.sphinx-doc.org/en/stable/ext/napoleon.html) for Sphinx lets us turn them into nice-looking documentation. Here are some references on Google-style docstrings:

24
bigchaindb/backend/localmongodb/query.py
View File

@ -111,3 +111,27 @@ def get_txids_filtered(conn, asset_id, operation=None):
@register_query(LocalMongoDBConnection)
def text_search(*args, **kwargs):
return mongodb.query.text_search(*args, **kwargs)
@register_query(LocalMongoDBConnection)
def get_owned_ids(conn, owner):
cursor = conn.run(
conn.collection('transactions').aggregate([
{'$match': {'outputs.public_keys': owner}},
{'$project': {'_id': False}}
]))
return cursor
@register_query(LocalMongoDBConnection)
def get_spending_transactions(conn, inputs):
cursor = conn.run(
conn.collection('transactions').aggregate([
{'$match': {
'inputs.fulfills': {
'$in': inputs,
},
}},
{'$project': {'_id': False}}
]))
return cursor

1
bigchaindb/backend/mongodb/changefeed.py
View File

@ -20,6 +20,7 @@ class MongoDBChangeFeed(ChangeFeed):
We emulate the behaviour of the RethinkDB changefeed by using a tailable
cursor that listens for events on the oplog.
"""
def run_forever(self):
for element in self.prefeed:
self.outqueue.put(element)

1
bigchaindb/backend/mongodb/connection.py
View File

@ -20,7 +20,6 @@ class MongoDBConnection(Connection):
def __init__(self, replicaset=None, ssl=None, login=None, password=None,
ca_cert=None, certfile=None, keyfile=None,
keyfile_passphrase=None, crlfile=None, **kwargs):
"""Create a new Connection instance.
Args:

6
bigchaindb/backend/query.py
View File

@ -408,8 +408,7 @@ def get_last_voted_block_id(connection, node_pubkey):
@singledispatch
def get_txids_filtered(connection, asset_id, operation=None):
"""
Return all transactions for a particular asset id and optional operation.
"""Return all transactions for a particular asset id and optional operation.
Args:
asset_id (str): ID of transaction that defined the asset
@ -421,8 +420,7 @@ def get_txids_filtered(connection, asset_id, operation=None):
@singledispatch
def get_new_blocks_feed(connection, start_block_id):
"""
Return a generator that yields change events of the blocks feed
"""Return a generator that yields change events of the blocks feed
Args:
start_block_id (str): ID of block to resume from

3
bigchaindb/backend/rethinkdb/connection.py
View File

@ -4,8 +4,7 @@ from bigchaindb.backend.exceptions import ConnectionError, OperationError
class RethinkDBConnection(Connection):
"""
This class is a proxy to run queries against the database, it is:
"""This class is a proxy to run queries against the database, it is:
- lazy, since it creates a connection only when needed
- resilient, because before raising exceptions it tries

2
bigchaindb/backend/rethinkdb/query.py
View File

@ -293,7 +293,7 @@ def get_votes_for_blocks_by_voter(connection, block_ids, node_pubkey):
def unwind_block_transactions(block):
""" Yield a block for each transaction in given block """
"""Yield a block for each transaction in given block"""
return block['block']['transactions'].map(lambda tx: block.merge({'tx': tx}))

3
bigchaindb/backend/utils.py
View File

@ -1,6 +1,7 @@
class ModuleDispatchRegistrationError(Exception):
"""Raised when there is a problem registering dispatched functions for a
module"""
module
"""
def module_dispatch_registrar(module):

12
bigchaindb/common/exceptions.py
View File

@ -29,7 +29,8 @@ class KeypairNotFoundException(BigchainDBError):
class KeypairMismatchException(BigchainDBError):
"""Raised if the private key(s) provided for signing don't match any of the
current owner(s)"""
current owner(s)
"""
class OperationError(BigchainDBError):
@ -55,7 +56,8 @@ class DoubleSpend(ValidationError):
class InvalidHash(ValidationError):
"""Raised if there was an error checking the hash for a particular
operation"""
operation
"""
class SchemaValidationError(ValidationError):
@ -64,7 +66,8 @@ class SchemaValidationError(ValidationError):
class InvalidSignature(ValidationError):
"""Raised if there was an error checking the signature for a particular
operation"""
operation
"""
class ImproperVoteError(ValidationError):
@ -77,7 +80,8 @@ class MultipleVotesError(ValidationError):
class TransactionNotInValidBlock(ValidationError):
"""Raised when a transfer transaction is attempting to fulfill the
outputs of a transaction that is in an invalid or undecided block"""
outputs of a transaction that is in an invalid or undecided block
"""
class AssetIdMismatch(ValidationError):

11
bigchaindb/common/schema/__init__.py
View File

@ -1,4 +1,4 @@
""" Schema validation related functions and data """
"""Schema validation related functions and data"""
import os.path
import logging
@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
def _load_schema(name):
""" Load a schema from disk """
"""Load a schema from disk"""
path = os.path.join(os.path.dirname(__file__), name + '.yaml')
with open(path) as handle:
schema = yaml.safe_load(handle)
@ -34,7 +34,7 @@ VOTE_SCHEMA_PATH, VOTE_SCHEMA = _load_schema('vote')
def _validate_schema(schema, body):
""" Validate data against a schema """
"""Validate data against a schema"""
# Note
#
@ -59,8 +59,7 @@ def _validate_schema(schema, body):
def validate_transaction_schema(tx):
"""
Validate a transaction dict.
"""Validate a transaction dict.
TX_SCHEMA_COMMON contains properties that are common to all types of
transaction. TX_SCHEMA_[TRANSFER|CREATE] add additional constraints on top.
@ -73,5 +72,5 @@ def validate_transaction_schema(tx):
def validate_vote_schema(vote):
""" Validate a vote dict """
"""Validate a vote dict"""
_validate_schema(VOTE_SCHEMA, vote)

9
bigchaindb/common/transaction.py
View File

@ -128,8 +128,7 @@ class Input(object):
def _fulfillment_to_details(fulfillment):
"""
Encode a fulfillment as a details dictionary
"""Encode a fulfillment as a details dictionary
Args:
fulfillment: Crypto-conditions Fulfillment object
@ -156,8 +155,7 @@ def _fulfillment_to_details(fulfillment):
def _fulfillment_from_details(data, _depth=0):
"""
Load a fulfillment for a signing spec dictionary
"""Load a fulfillment for a signing spec dictionary
Args:
data: tx.output[].condition.details dictionary
@ -471,6 +469,7 @@ class Transaction(object):
Metadata to be stored along with the Transaction.
version (string): Defines the version number of a Transaction.
"""
CREATE = 'CREATE'
TRANSFER = 'TRANSFER'
GENESIS = 'GENESIS'
@ -911,7 +910,7 @@ class Transaction(object):
tx_serialized = Transaction._to_str(tx_dict)
def validate(i, output_condition_uri=None):
""" Validate input against output condition URI """
"""Validate input against output condition URI"""
return self._input_valid(self.inputs[i], self.operation,
tx_serialized, output_condition_uri)

6
bigchaindb/config_utils.py
View File

@ -135,7 +135,8 @@ def env_config(config):
def update_types(config, reference, list_sep=':'):
"""Return a new configuration where all the values types
are aligned with the ones in the default configuration"""
are aligned with the ones in the default configuration
"""
def _coerce(current, value):
# Coerce a value to the `current` type.
@ -226,7 +227,8 @@ def is_configured():
def autoconfigure(filename=None, config=None, force=False):
"""Run ``file_config`` and ``env_config`` if the module has not
been initialized."""
been initialized.
"""
if not force and is_configured():
logger.debug('System already configured, skipping autoconfiguration')
return

4
bigchaindb/consensus.py
View File

@ -9,12 +9,14 @@ class BaseConsensusRules():
All methods listed below must be implemented.
"""
voting = Voting
@staticmethod
def validate_transaction(bigchain, transaction):
"""See :meth:`bigchaindb.models.Transaction.validate`
for documentation."""
for documentation.
"""
return transaction.validate(bigchain)
@staticmethod

27
bigchaindb/core.py
View File

@ -159,8 +159,7 @@ class Bigchain(object):
return self.consensus.validate_transaction(self, transaction)
def is_new_transaction(self, txid, exclude_block_id=None):
"""
Return True if the transaction does not exist in any
"""Return True if the transaction does not exist in any
VALID or UNDECIDED block. Return False otherwise.
Args:
@ -421,8 +420,7 @@ class Bigchain(object):
return fastquery.FastQuery(self.connection, self.me)
def get_outputs_filtered(self, owner, spent=None):
"""
Get a list of output links filtered on some criteria
"""Get a list of output links filtered on some criteria
Args:
owner (str): base58 encoded public_key.
@ -443,8 +441,7 @@ class Bigchain(object):
return self.fastquery.filter_spent_outputs(outputs)
def get_transactions_filtered(self, asset_id, operation=None):
"""
Get a list of transactions filtered on some criteria
"""Get a list of transactions filtered on some criteria
"""
txids = backend.query.get_txids_filtered(self.connection, asset_id,
operation)
@ -617,12 +614,12 @@ class Bigchain(object):
def block_election_status(self, block):
"""Tally the votes on a block, and return the status:
valid, invalid, or undecided."""
valid, invalid, or undecided.
"""
return self.block_election(block)['status']
def get_assets(self, asset_ids):
"""
Return a list of assets that match the asset_ids
"""Return a list of assets that match the asset_ids
Args:
asset_ids (:obj:`list` of :obj:`str`): A list of asset_ids to
@ -634,8 +631,7 @@ class Bigchain(object):
return backend.query.get_assets(self.connection, asset_ids)
def get_metadata(self, txn_ids):
"""
Return a list of metadata that match the transaction ids (txn_ids)
"""Return a list of metadata that match the transaction ids (txn_ids)
Args:
txn_ids (:obj:`list` of :obj:`str`): A list of txn_ids to
@ -647,8 +643,7 @@ class Bigchain(object):
return backend.query.get_metadata(self.connection, txn_ids)
def write_assets(self, assets):
"""
Writes a list of assets into the database.
"""Writes a list of assets into the database.
Args:
assets (:obj:`list` of :obj:`dict`): A list of assets to write to
@ -657,8 +652,7 @@ class Bigchain(object):
return backend.query.write_assets(self.connection, assets)
def write_metadata(self, metadata):
"""
Writes a list of metadata into the database.
"""Writes a list of metadata into the database.
Args:
metadata (:obj:`list` of :obj:`dict`): A list of metadata to write to
@ -667,8 +661,7 @@ class Bigchain(object):
return backend.query.write_metadata(self.connection, metadata)
def text_search(self, search, *, limit=0, table='assets'):
"""
Return an iterator of assets that match the text search
"""Return an iterator of assets that match the text search
Args:
search (str): Text search string to query the text index

3
bigchaindb/events.py
View File

@ -7,7 +7,8 @@ POISON_PILL = 'POISON_PILL'
class EventTypes:
"""Container class that holds all the possible
events BigchainDB manages."""
events BigchainDB manages.
"""
# If you add a new Event Type, make sure to add it
# to the docs in docs/server/source/event-plugin-api.rst

21
bigchaindb/fastquery.py
View File

@ -4,8 +4,7 @@ from bigchaindb.common.transaction import TransactionLink
class FastQuery:
"""
Database queries that join on block results from a single node.
"""Database queries that join on block results from a single node.
* Votes are not validated for security (security is a replication concern)
* Votes come from only one node, and as such, non-byzantine fault tolerance
@ -18,14 +17,13 @@ class FastQuery:
query multiple nodes to insure against getting an incorrect response from
a byzantine node.
"""
def __init__(self, connection, me):
self.connection = connection
self.me = me
def filter_valid_block_ids(self, block_ids, include_undecided=False):
"""
Given block ids, return only the ones that are valid.
"""
"""Given block ids, return only the ones that are valid."""
block_ids = list(set(block_ids))
votes = query.get_votes_for_blocks_by_voter(
self.connection, block_ids, self.me)
@ -35,8 +33,7 @@ class FastQuery:
if votes.get(block_id, include_undecided)]
def filter_valid_items(self, items, block_id_key=lambda b: b[0]):
"""
Given items with block ids, return only the ones that are valid or undecided.
"""Given items with block ids, return only the ones that are valid or undecided.
"""
items = list(items)
block_ids = map(block_id_key, items)
@ -44,9 +41,7 @@ class FastQuery:
return [b for b in items if block_id_key(b) in valid_block_ids]
def get_outputs_by_public_key(self, public_key):
"""
Get outputs for a public key
"""
"""Get outputs for a public key"""
res = list(query.get_owned_ids(self.connection, public_key))
txs = [tx for _, tx in self.filter_valid_items(res)]
return [TransactionLink(tx['id'], index)
@ -56,8 +51,7 @@ class FastQuery:
public_key)]
def filter_spent_outputs(self, outputs):
"""
Remove outputs that have been spent
"""Remove outputs that have been spent
Args:
outputs: list of TransactionLink
@ -71,8 +65,7 @@ class FastQuery:
return [ff for ff in outputs if ff not in spends]
def filter_unspent_outputs(self, outputs):
"""
Remove outputs that have not been spent
"""Remove outputs that have not been spent
Args:
outputs: list of TransactionLink

1
bigchaindb/log/loggers.py
View File

@ -12,6 +12,7 @@ class HttpServerLogger(Logger):
configuration setting on gunicorn.
"""
def setup(self, cfg):
"""Setup the gunicorn access and error loggers. This overrides
the parent method. Its main goal is to simply pipe all the logs to

7
bigchaindb/log/setup.py
View File

@ -116,8 +116,7 @@ class LogRecordStreamHandler(StreamRequestHandler):
"""
def handle(self):
"""
Handle multiple requests - each expected to be a 4-byte length,
"""Handle multiple requests - each expected to be a 4-byte length,
followed by the LogRecord in pickle format. Logs the record
according to whatever policy is configured locally.
"""
@ -152,10 +151,10 @@ class LogRecordStreamHandler(StreamRequestHandler):
class LogRecordSocketServer(ThreadingTCPServer):
"""
Simple TCP socket-based logging server.
"""Simple TCP socket-based logging server.
"""
allow_reuse_address = True
def __init__(self,

28
bigchaindb/models.py
View File

@ -96,8 +96,7 @@ class Transaction(Transaction):
@classmethod
def from_db(cls, bigchain, tx_dict):
"""
Helper method that reconstructs a transaction dict that was returned
"""Helper method that reconstructs a transaction dict that was returned
from the database. It checks what asset_id to retrieve, retrieves the
asset from the asset table and reconstructs the transaction.
@ -346,8 +345,7 @@ class Block(object):
@classmethod
def from_db(cls, bigchain, block_dict, from_dict_kwargs=None):
"""
Helper method that reconstructs a block_dict that was returned from
"""Helper method that reconstructs a block_dict that was returned from
the database. It checks what asset_ids to retrieve, retrieves the
assets from the assets table and reconstructs the block.
@ -373,8 +371,7 @@ class Block(object):
return cls.from_dict(block_dict, **kwargs)
def decouple_assets(self, block_dict=None):
"""
Extracts the assets from the ``CREATE`` transactions in the block.
"""Extracts the assets from the ``CREATE`` transactions in the block.
Returns:
tuple: (assets, block) with the assets being a list of dicts and
@ -395,8 +392,7 @@ class Block(object):
return (assets, block_dict)
def decouple_metadata(self, block_dict=None):
"""
Extracts the metadata from transactions in the block.
"""Extracts the metadata from transactions in the block.
Returns:
tuple: (metadatas, block) with the metadatas being a list of dict/null and
@ -417,8 +413,7 @@ class Block(object):
@staticmethod
def couple_assets(block_dict, assets):
"""
Given a block_dict with no assets (as returned from a database call)
"""Given a block_dict with no assets (as returned from a database call)
and a list of assets, reconstruct the original block by putting the
assets back into the ``CREATE`` transactions in the block.
@ -442,8 +437,7 @@ class Block(object):
@staticmethod
def couple_metadata(block_dict, metadatal):
"""
Given a block_dict with no metadata (as returned from a database call)
"""Given a block_dict with no metadata (as returned from a database call)
and a list of metadata, reconstruct the original block by putting the
metadata of each transaction back into its original transaction.
@ -470,8 +464,7 @@ class Block(object):
@staticmethod
def get_asset_ids(block_dict):
"""
Given a block_dict return all the asset_ids for that block (the txid
"""Given a block_dict return all the asset_ids for that block (the txid
of CREATE transactions). Useful to know which assets to retrieve
from the database to reconstruct the block.
@ -493,8 +486,7 @@ class Block(object):
@staticmethod
def get_txn_ids(block_dict):
"""
Given a block_dict return all the transaction ids.
"""Given a block_dict return all the transaction ids.
Args:
block_dict (:obj:`dict`): The block dict as returned from a
@ -515,13 +507,13 @@ class Block(object):
class FastTransaction:
"""
A minimal wrapper around a transaction dictionary. This is useful for
"""A minimal wrapper around a transaction dictionary. This is useful for
when validation is not required but a routine expects something that looks
like a transaction, for example during block creation.
Note: immutability could also be provided
"""
def __init__(self, tx_dict):
self.data = tx_dict

5
bigchaindb/pipelines/block.py
View File

@ -145,7 +145,7 @@ class BlockPipeline:
def tx_collector():
""" A helper to deduplicate transactions """
"""A helper to deduplicate transactions"""
def snowflake():
txids = set()
@ -167,7 +167,8 @@ def tx_collector():
def create_pipeline():
"""Create and return the pipeline of operations to be distributed
on different processes."""
on different processes.
"""
block_pipeline = BlockPipeline()

6
bigchaindb/pipelines/election.py
View File

@ -28,8 +28,7 @@ class Election:
self.events_queue = events_queue
def check_for_quorum(self, next_vote):
"""
Checks if block has enough invalid votes to make a decision
"""Checks if block has enough invalid votes to make a decision
Args:
next_vote: The next vote.
@ -60,8 +59,7 @@ class Election:
})
def requeue_transactions(self, invalid_block):
"""
Liquidates transactions from invalid blocks so they can be processed again
"""Liquidates transactions from invalid blocks so they can be processed again
"""
logger.info('Rewriting %s transactions from invalid block %s',
len(invalid_block.transactions),

3
bigchaindb/pipelines/stale.py
View File

@ -56,7 +56,8 @@ class StaleTransactionMonitor:
def create_pipeline(timeout=5, backlog_reassign_delay=5):
"""Create and return the pipeline of operations to be distributed
on different processes."""
on different processes.
"""
stm = StaleTransactionMonitor(timeout=timeout,
backlog_reassign_delay=backlog_reassign_delay)

6
bigchaindb/pipelines/vote.py
View File

@ -155,7 +155,8 @@ class Vote:
def create_pipeline():
"""Create and return the pipeline of operations to be distributed
on different processes."""
on different processes.
"""
voter = Vote()
@ -170,7 +171,8 @@ def create_pipeline():
def get_changefeed():
"""Create and return ordered changefeed of blocks starting from
last voted block"""
last voted block
"""
b = Bigchain()
last_block_id = b.get_last_voted_block().id
feed = backend.query.get_new_blocks_feed(b.connection, last_block_id)

8
bigchaindb/tendermint/core.py
View File

@ -62,8 +62,12 @@ class App(BaseApplication):
logger.debug('check_tx: INVALID')
return Result.error()
def begin_block(self, block_hash, header):
"""Initialize list of transaction."""
def begin_block(self, req_begin_block):
"""Initialize list of transaction.
Args:
req_begin_block: block object which contains block header
and block hash.
"""
self.block_txn_ids = []

48
bigchaindb/tendermint/fastquery.py Normal file
View File

@ -0,0 +1,48 @@
from bigchaindb.utils import condition_details_has_owner
from bigchaindb.backend import query
from bigchaindb.common.transaction import TransactionLink
class FastQuery():
"""
Database queries that join on block results from a single node.
"""
def get_outputs_by_public_key(self, public_key):
"""
Get outputs for a public key
"""
txs = list(query.get_owned_ids(self.connection, public_key))
return [TransactionLink(tx['id'], index)
for tx in txs
for index, output in enumerate(tx['outputs'])
if condition_details_has_owner(output['condition']['details'],
public_key)]
def filter_spent_outputs(self, outputs):
"""
Remove outputs that have been spent
Args:
outputs: list of TransactionLink
"""
links = [o.to_dict() for o in outputs]
txs = list(query.get_spending_transactions(self.connection, links))
spends = {TransactionLink.from_dict(input_['fulfills'])
for tx in txs
for input_ in tx['inputs']}
return [ff for ff in outputs if ff not in spends]
def filter_unspent_outputs(self, outputs):
"""
Remove outputs that have not been spent
Args:
outputs: list of TransactionLink
"""
links = [o.to_dict() for o in outputs]
txs = list(query.get_spending_transactions(self.connection, links))
spends = {TransactionLink.from_dict(input_['fulfills'])
for tx in txs
for input_ in tx['inputs']}
return [ff for ff in outputs if ff in spends]

5
bigchaindb/tendermint/lib.py
View File

@ -11,6 +11,7 @@ from bigchaindb import Bigchain
from bigchaindb.models import Transaction
from bigchaindb.common.exceptions import SchemaValidationError, ValidationError
from bigchaindb.tendermint.utils import encode_transaction
from bigchaindb.tendermint import fastquery
logger = logging.getLogger(__name__)
@ -117,5 +118,9 @@ class BigchainDB(Bigchain):
return False
return transaction
@property
def fastquery(self):
return fastquery.FastQuery(self.connection, self.me)
Block = namedtuple('Block', ('app_hash', 'height'))

4
bigchaindb/utils.py
View File

@ -83,9 +83,7 @@ def pool(builder, size, timeout=None):
# TODO: Rename this function, it's handling fulfillments not conditions
def condition_details_has_owner(condition_details, owner):
"""
Check if the public_key of owner is in the condition details
"""Check if the public_key of owner is in the condition details
as an Ed25519Fulfillment.public_key
Args:

22
bigchaindb/voting.py
View File

@ -12,8 +12,7 @@ UNDECIDED = 'undecided'
class Voting:
"""
Everything to do with verifying and counting votes for block election.
"""Everything to do with verifying and counting votes for block election.
All functions in this class should be referentially transparent, that is,
they always give the same output for a given input. This makes it easier
@ -27,9 +26,7 @@ class Voting:
@classmethod
def block_election(cls, block, votes, keyring):
"""
Calculate the election status of a block.
"""
"""Calculate the election status of a block."""
eligible_voters = set(block['block']['voters']) & set(keyring)
n_voters = len(eligible_voters)
eligible_votes, ineligible_votes = \
@ -43,8 +40,7 @@ class Voting:
@classmethod
def partition_eligible_votes(cls, votes, eligible_voters):
"""
Filter votes from unknown nodes or nodes that are not listed on
"""Filter votes from unknown nodes or nodes that are not listed on
block. This is the primary Sybill protection.
"""
eligible, ineligible = ([], [])
@ -63,8 +59,7 @@ class Voting:
@classmethod
def dedupe_by_voter(cls, eligible_votes):
"""
Throw a critical error if there is a duplicate vote
"""Throw a critical error if there is a duplicate vote
"""
by_voter = {}
for vote in eligible_votes:
@ -76,8 +71,7 @@ class Voting:
@classmethod
def count_votes(cls, by_voter):
"""
Given a list of eligible votes, (votes from known nodes that are listed
"""Given a list of eligible votes, (votes from known nodes that are listed
as voters), produce the number that say valid and the number that say
invalid. Votes must agree on previous block, otherwise they become invalid.
"""
@ -111,8 +105,7 @@ class Voting:
@classmethod
def decide_votes(cls, n_voters, n_valid, n_invalid):
"""
Decide on votes.
"""Decide on votes.
To return VALID there must be a clear majority that say VALID
and also agree on the previous block.
@ -127,8 +120,7 @@ class Voting:
@classmethod
def verify_vote_signature(cls, vote):
"""
Verify the signature of a vote
"""Verify the signature of a vote
"""
signature = vote.get('signature')
pk_base58 = vote.get('node_pubkey')

4
bigchaindb/web/routes.py
View File

@ -1,4 +1,4 @@
""" API routes definition """
"""API routes definition"""
from flask_restful import Api
from bigchaindb.web.views import (
assets,
@ -13,7 +13,7 @@ from bigchaindb.web.views import (
def add_routes(app):
""" Add the routes to an app """
"""Add the routes to an app"""
for (prefix, routes) in API_SECTIONS:
api = Api(app, prefix=prefix)
for ((pattern, resource, *args), kwargs) in routes:

4
bigchaindb/web/server.py
View File

@ -25,13 +25,13 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication):
"""
def __init__(self, app, *, options=None):
'''Initialize a new standalone application.
"""Initialize a new standalone application.
Args:
app: A wsgi Python application.
options (dict): the configuration.
'''
"""
self.options = options or {}
self.application = app
super().__init__()

3
bigchaindb/web/views/base.py
View File

@ -1,5 +1,4 @@
"""
Common classes and methods for API handlers
"""Common classes and methods for API handlers
"""
import logging

5
bigchaindb/web/views/info.py
View File

@ -1,4 +1,4 @@
""" API Index endpoint """
"""API Index endpoint"""
import flask
from flask_restful import Resource
@ -33,8 +33,7 @@ class ApiV1Index(Resource):
def get_api_v1_info(api_prefix):
"""
Return a dict with all the information specific for the v1 of the
"""Return a dict with all the information specific for the v1 of the
api.
"""
websocket_root = base_ws_uri() + EVENTS_ENDPOINT

8
docker-compose.network.yml
View File

@ -30,7 +30,7 @@ services:
TENDERMINT_HOST: tendermint-one
ports:
- "9984"
command: bigchaindb -l DEBUG start --init
command: bigchaindb -l DEBUG start
tendermint-one:
image: tendermint/tendermint
volumes:
@ -67,7 +67,7 @@ services:
TENDERMINT_HOST: tendermint-two
ports:
- "9984"
command: bigchaindb -l DEBUG start --init
command: bigchaindb -l DEBUG start
tendermint-two:
image: tendermint/tendermint
volumes:
@ -104,7 +104,7 @@ services:
TENDERMINT_HOST: tendermint-three
ports:
- "9984"
command: bigchaindb -l DEBUG start --init
command: bigchaindb -l DEBUG start
tendermint-three:
image: tendermint/tendermint
volumes:
@ -141,7 +141,7 @@ services:
TENDERMINT_HOST: tendermint-four
ports:
- "9984"
command: bigchaindb -l DEBUG start --init
command: bigchaindb -l DEBUG start
tendermint-four:
image: tendermint/tendermint
volumes:

2
docker-compose.tendermint.yml
View File

@ -29,7 +29,7 @@ services:
TENDERMINT_PORT: 46657
ports:
- "9984"
command: bigchaindb -l DEBUG start --init
command: bigchaindb -l DEBUG start
tendermint:
image: tendermint/tendermint
volumes:

16
docs/root/source/assets.rst
View File

@ -6,19 +6,11 @@ BigchainDB can store data of any kind (within reason), but it's designed to be p
* The fundamental thing that one sends to a BigchainDB cluster, to be checked and stored (if valid), is a *transaction*, and there are two kinds: CREATE transactions and TRANSFER transactions.
* A CREATE transaction can be use to register any kind of asset (divisible or indivisible), along with arbitrary metadata.
* An asset can have zero, one, or several owners.
* The owners of an asset can specify (crypto-)conditions which must be satisfied by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a transfer transaction.
* BigchainDB verifies that the conditions have been satisfied as part of checking the validity of transfer transactions. (Moreover, anyone can check that they were satisfied.)
* The owners of an asset can specify (crypto-)conditions which must be satisfied by anyone wishing transfer the asset to new owners. For example, a condition might be that at least 3 of the 5 current owners must cryptographically sign a TRANSFER transaction.
* BigchainDB verifies that the conditions have been satisfied as part of checking the validity of TRANSFER transactions. (Moreover, anyone can check that they were satisfied.)
* BigchainDB prevents double-spending of an asset.
* Validated transactions are strongly tamper-resistant; see :doc:`the page about immutability <immutable>`.
BigchainDB Integration with Other Blockchains
---------------------------------------------
BigchainDB works with the `Interledger protocol <https://interledger.org/>`_, enabling the transfer of assets between BigchainDB and other blockchains, ledgers, and payment systems.
We’re actively exploring ways that BigchainDB can be used with other blockchains and platforms.
* Validated transactions are :doc:`"immutable" <immutable>`.
.. note::
We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See BigchainDB Server `issue #626 <https://github.com/bigchaindb/bigchaindb/issues/626>`_.
We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec <https://the-ipdb-transaction-spec.readthedocs.io/en/latest/ownership.html>`_.

10
docs/root/source/bft.md
View File

@ -1,8 +1,6 @@
# BigchainDB and Byzantine Fault Tolerance
While BigchainDB is not currently [Byzantine fault tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance), we plan to offer it as an option.
Update Nov 2017: we're actively working on this, the next release or two will likely have support. More details to come in blog form and github issues
Related issue: [Issue #293](https://github.com/bigchaindb/bigchaindb/issues/293). We anticipate that turning on BFT will cause a dropoff in performance (for a gain in security).
In the meantime, there are practical things that one can do to increase security (e.g. firewalls, key management, and access controls).
[BigchainDB Server](https://docs.bigchaindb.com/projects/server/en/latest/index.html)
uses [Tendermint](https://tendermint.com/)
for consensus and transaction replication,
and Tendermint is [Byzantine Fault Tolerant (BFT)](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance).

6
docs/root/source/decentralized.md
View File

@ -14,8 +14,6 @@ A consortium can increase its decentralization (and its resilience) by increasin
There’s no node that has a long-term special position in the cluster. All nodes run the same software and perform the same duties.
MongoDB and RethinkDB have an “admin” user which can’t be deleted and which can make big changes to the database, such as dropping a table. Right now, that’s a big security vulnerability, but we have plans to mitigate it by:
1. Locking down the admin user as much as possible.
2. Having all nodes inspect admin-type requests before acting on them. Requests can be checked against an evolving whitelist of allowed actions. Nodes requesting non-allowed requests can be removed from the list of cluster nodes.
If someone has (or gets) admin access to a node, they can mess with that node (e.g. change or delete data stored on that node), but those changes should remain isolated to that node. The BigchainDB cluster can only be compromised if more than one third of the nodes get compromised. See the [Tendermint documentation](https://tendermint.readthedocs.io/projects/tools/en/master/introduction.html) for more details.
It’s worth noting that the admin user can’t transfer assets, even today. The only way to create a valid transfer transaction is to fulfill the current (crypto) conditions on the asset, and the admin user can’t do that because the admin user doesn’t have the necessary private keys (or preimages, in the case of hashlock conditions). They’re not stored in the database.
It’s worth noting that not even the admin or superuser of a node can transfer assets. The only way to create a valid transfer transaction is to fulfill the current crypto-conditions on the asset, and the admin/superuser can’t do that because the admin user doesn’t have the necessary information (e.g. private keys).

11
docs/root/source/diversity.md
View File

@ -1,11 +1,10 @@
# Kinds of Node Diversity
Steps should be taken to make it difficult for any one actor or event to control or damage “enough” of the nodes. (“Enough” is usually a quorum.) There are many kinds of diversity to consider, listed below. It may be quite difficult to have high diversity of all kinds.
Steps should be taken to make it difficult for any one actor or event to control or damage “enough” of the nodes. (Because BigchainDB Server uses Tendermint, "enough" is ⅓.) There are many kinds of diversity to consider, listed below. It may be quite difficult to have high diversity of all kinds.
1. **Jurisdictional diversity.** The nodes should be controlled by entities within multiple legal jurisdictions, so that it becomes difficult to use legal means to compel enough of them to do something.
2. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems.
3. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes.
4. **Operating system diversity.** The servers should use a variety of operating systems, so that a security bug in one OS can’t be used to exploit enough of the nodes.
5. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a consortium. For example, it provides the consortium with a source of various ideas for addressing challenges.
1. **Geographic diversity.** The servers should be physically located at multiple geographic locations, so that it becomes difficult for a natural disaster (such as a flood or earthquake) to damage enough of them to cause problems.
1. **Hosting diversity.** The servers should be hosted by multiple hosting providers (e.g. Amazon Web Services, Microsoft Azure, Digital Ocean, Rackspace), so that it becomes difficult for one hosting provider to influence enough of the nodes.
1. **Diversity in general.** In general, membership diversity (of all kinds) confers many advantages on a consortium. For example, it provides the consortium with a source of various ideas for addressing challenges.
Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a consortium could also have a diversity of server implementations.
Note: If all the nodes are running the same code, i.e. the same implementation of BigchainDB, then a bug in that code could be used to compromise all of the nodes. Ideally, there would be several different, well-maintained implementations of BigchainDB Server (e.g. one in Python, one in Go, etc.), so that a consortium could also have a diversity of server implementations. Similar remarks can be made about the operating system.

11
docs/root/source/index.rst
View File

@ -2,13 +2,12 @@ BigchainDB Documentation
========================
`BigchainDB <https://www.bigchaindb.com/>`_ is a scalable blockchain database.
That is, it's a "big data" database with some blockchain characteristics added, including `decentralization <decentralized.html>`_,
It has some database characteristics and some blockchain characteristics,
including `decentralization <decentralized.html>`_,
`immutability <immutable.html>`_
and
`native support for assets <assets.html>`_.
You can read about the motivations, goals and high-level architecture in the `BigchainDB whitepaper <https://www.bigchaindb.com/whitepaper/>`_.
and `native support for assets <assets.html>`_.
At a high level, one can communicate with a BigchainDB cluster (set of nodes) using the BigchainDB Client-Server HTTP API, or a wrapper for that API, such as the BigchainDB Python Driver. Each BigchainDB node runs BigchainDB Server and various other software. The `terminology page <terminology.html>`_ explains some of those terms in more detail.
At a high level, one can communicate with a BigchainDB cluster (set of nodes) using the BigchainDB HTTP API, or a wrapper for that API, such as the BigchainDB Python Driver. Each BigchainDB node runs BigchainDB Server and various other software. The `terminology page <terminology.html>`_ explains some of those terms in more detail.
.. raw:: html
@ -79,7 +78,6 @@ More About BigchainDB
:maxdepth: 1
BigchainDB Docs Home <self>
production-ready
terminology
decentralized
diversity
@ -89,5 +87,4 @@ More About BigchainDB
smart-contracts
transaction-concepts
permissions
timestamps
Data Models <https://docs.bigchaindb.com/projects/server/en/latest/data-models/index.html>

3
docs/root/source/permissions.rst
View File

@ -15,7 +15,8 @@ To spend/transfer an unspent output, a user (or group of users) must fulfill the
- "…three of these four people must sign."
- "…either Bob must sign, or both Tom and Sylvia must sign."
For details, see `the documentation about conditions in BigchainDB <https://docs.bigchaindb.com/projects/server/en/latest/data-models/conditions.html>`_.
For details, see
`the documentation about conditions in the IPDB Transaction Spec <https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-components/conditions.html>`_.
Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output.

14
docs/root/source/production-ready.md
View File

@ -1,14 +0,0 @@
# Production-Ready?
BigchainDB is not production-ready. You can use it to build a prototype or proof-of-concept (POC); many people are already doing that.
Once BigchainDB is production-ready, we'll make an announcement.
BigchainDB version numbers follow the conventions of *Semantic Versioning* as documented at [semver.org](http://semver.org/). (For Python stuff, we use [Python's version of Semantic Versioning](https://packaging.python.org/tutorials/distributing-packages/#choosing-a-versioning-scheme).) This means, among other things:
* Before version 1.0, breaking API changes could happen in any new version, even in a change from version 0.Y.4 to 0.Y.5.
* Starting with version 1.0.0, breaking API changes will only happen when the MAJOR version changes (e.g. from 1.7.4 to 2.0.0, or from 4.9.3 to 5.0.0).
To review the release history of some particular BigchainDB software, go to the GitHub repository of that software and click on "Releases". For example, the release history of BigchainDB Server can be found at [https://github.com/bigchaindb/bigchaindb/releases](https://github.com/bigchaindb/bigchaindb/releases).
[The BigchainDB Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) will give you a sense of the things we intend to do with BigchainDB in the near term and the long term.

3
docs/root/source/smart-contracts.rst
View File

@ -15,5 +15,4 @@ Crypto-conditions can be quite complex. They can't include loops or recursion an
.. note::
We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See BigchainDB Server `issue #626 <https://github.com/bigchaindb/bigchaindb/issues/626>`_.
We used the word "owners" somewhat loosely above. A more accurate word might be fulfillers, signers, controllers, or transfer-enablers. See the `note about "owners" in the IPDB Transaction Spec <https://the-ipdb-transaction-spec.readthedocs.io/en/latest/ownership.html>`_.

7
docs/root/source/terminology.md
View File

@ -2,16 +2,13 @@
There is some specialized terminology associated with BigchainDB. To get started, you should at least know the following:
## BigchainDB Node
A **BigchainDB node** is a machine or set of closely-linked machines running MongoDB Server (or RethinkDB Server), BigchainDB Server, and related software. Each node is controlled by one person or organization.
A **BigchainDB node** is a machine (or logical machine) running [BigchainDB Server](https://docs.bigchaindb.com/projects/server/en/latest/introduction.html) and related software. Each node is controlled by one person or organization.
## BigchainDB Cluster
A set of BigchainDB nodes can connect to each other to form a **BigchainDB cluster**. Each node in the cluster runs the same software. A cluster contains one logical MongoDB/RethinkDB datastore. A cluster may have additional machines to do things such as cluster monitoring.
A set of BigchainDB nodes can connect to each other to form a **BigchainDB cluster**. Each node in the cluster runs the same software. A cluster may have additional machines to do things such as cluster monitoring.
## BigchainDB Consortium

84
docs/root/source/timestamps.md
View File

@ -1,84 +0,0 @@
# Timestamps in BigchainDB
Each block and vote has an associated timestamp. Interpreting those timestamps is tricky, hence the need for this section.
## Timestamp Sources & Accuracy
Timestamps in BigchainDB are provided by the node which created the block and the node that created the vote.
When a BigchainDB node needs a timestamp, it calls a BigchainDB utility function named `timestamp()`. There's a detailed explanation of how that function works below, but the short version is that it gets the [Unix time](https://en.wikipedia.org/wiki/Unix_time) from its system clock, rounded to the nearest second.
We advise BigchainDB nodes to run special software (an "NTP daemon") to keep their system clock in sync with standard time servers. (NTP stands for [Network Time Protocol](https://en.wikipedia.org/wiki/Network_Time_Protocol).)
## Converting Timestamps to UTC
To convert a BigchainDB timestamp (a Unix time) to UTC, you need to know how the node providing the timestamp was set up. That's because different setups will report a different "Unix time" value around leap seconds! There's [a nice Red Hat Developer Blog post about the various setup options](https://developers.redhat.com/blog/2015/06/01/five-different-ways-handle-leap-seconds-ntp/). If you want more details, see [David Mills' pages about leap seconds, NTP, etc.](https://www.eecis.udel.edu/~mills/leap.html) (David Mills designed NTP.)
We advise BigchainDB nodes to run an NTP daemon with particular settings so that their timestamps are consistent.
If a timestamp comes from a node that's set up as we advise, it can be converted to UTC as follows:
1. Use a standard "Unix time to UTC" converter to get a UTC timestamp.
2. Is the UTC timestamp a leap second, or the second before/after a leap second? There's [a list of all the leap seconds on Wikipedia](https://en.wikipedia.org/wiki/Leap_second).
3. If no, then you are done.
4. If yes, then it might not be possible to convert it to a single UTC timestamp. Even if it can't be converted to a single UTC timestamp, it _can_ be converted to a list of two possible UTC timestamps.
Showing how to do that is beyond the scope of this documentation.
In all likelihood, you will never have to worry about leap seconds because they are very rare.
(There were only 26 between 1972 and the end of 2015.)
## Calculating Elapsed Time Between Two Timestamps
There's another gotcha with (Unix time) timestamps: you can't calculate the real-world elapsed time between two timestamps (correctly) by subtracting the smaller timestamp from the larger one. The result won't include any of the leap seconds that occured between the two timestamps. You could look up how many leap seconds happened between the two timestamps and add that to the result. There are many library functions for working with timestamps; those are beyond the scope of this documentation.
## Interpreting Sets of Timestamps
You can look at many timestamps to get a statistical sense of when something happened. For example, a transaction in a decided-valid block has many associated timestamps:
* the timestamp of the block
* the timestamps of all the votes on the block
## How BigchainDB Uses Timestamps
BigchainDB _doesn't_ use timestamps to determine the order of transactions or blocks. In particular, the order of blocks is determined by MongoDB's oplog (or RethinkDB's changefeed) on the bigchain table.
BigchainDB does use timestamps for some things. When a Transaction is written to the backlog, a timestamp is assigned called the `assignment_timestamp`, to determine if it has been waiting in the backlog for too long (i.e. because the node assigned to it hasn't handled it yet).
## Including Trusted Timestamps
If you want to create a transaction payload with a trusted timestamp, you can.
One way to do that would be to send a payload to a trusted timestamping service. They will send back a timestamp, a signature, and their public key. They should also explain how you can verify the signature. You can then include the original payload, the timestamp, the signature, and the service's public key in your transaction metadata. That way, anyone with the verification instructions can verify that the original payload was signed by the trusted timestamping service.
## How the timestamp() Function Works
BigchainDB has a utility function named `timestamp()` which amounts to:
```python
timestamp() = str(round(time.time()))
```
In other words, it calls the `time()` function in Python's `time` module, [rounds](https://docs.python.org/3/library/functions.html#round) that to the nearest integer, and converts the result to a string.
It rounds the output of `time.time()` to the nearest second because, according to [the Python documentation for `time.time()`](https://docs.python.org/3.4/library/time.html#time.time), "...not all systems provide time with a better precision than 1 second."
How does `time.time()` work? If you look in the C source code, it calls `floattime()` and `floattime()` calls [clock_gettime()](https://www.cs.rutgers.edu/~pxk/416/notes/c-tutorials/gettime.html), if it's available.
```text
ret = clock_gettime(CLOCK_REALTIME, &tp);
```
With `CLOCK_REALTIME` as the first argument, it returns the "Unix time." ("Unix time" is in quotes because its value around leap seconds depends on how the system is set up; see above.)
## Why Not Use UTC, TAI or Some Other Time that Has Unambiguous Timestamps for Leap Seconds?
It would be nice to use UTC or TAI timestamps, but unfortunately there's no commonly-available, standard way to get always-accurate UTC or TAI timestamps from the operating system on typical computers today (i.e. accurate around leap seconds).
There _are_ commonly-available, standard ways to get the "Unix time," such as clock_gettime() function available in C. That's what we use (indirectly via Python). ("Unix time" is in quotes because its value around leap seconds depends on how the system is set up; see above.)
The Unix-time-based timestamps we use are only ambiguous circa leap seconds, and those are very rare. Even for those timestamps, the extra uncertainty is only one second, and that's not bad considering that we only report timestamps to a precision of one second in the first place. All other timestamps can be converted to UTC with no ambiguity.

8
docs/root/source/transaction-concepts.md
View File

@ -6,7 +6,6 @@ things (e.g. assets).
Transactions are the most basic kind of record stored by BigchainDB. There are
two kinds: CREATE transactions and TRANSFER transactions.
## CREATE Transactions
A CREATE transaction can be used to register, issue, create or otherwise
@ -31,20 +30,19 @@ Each output also has an associated condition: the condition that must be met
BigchainDB supports a variety of conditions,
a subset of the [Interledger Protocol (ILP)](https://interledger.org/)
crypto-conditions. For details, see
[the documentation about Inputs and Outputs](https://docs.bigchaindb.com/projects/server/en/latest/data-models/inputs-outputs.html).
[the documentation about conditions in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-components/conditions.html).
Each output also has a list of all the public keys associated
with the conditions on that output.
Loosely speaking, that list might be interpreted as the list of "owners."
A more accurate word might be fulfillers, signers, controllers,
or transfer-enablers.
See BigchainDB Server [issue #626](https://github.com/bigchaindb/bigchaindb/issues/626).
See the [note about "owners" in the IPDB Transaction Spec](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/ownership.html).
A CREATE transaction must be signed by all the owners.
(If you're looking for that signature,
it's in the one "fulfillment" of the one input, albeit encoded.)
## TRANSFER Transactions
A TRANSFER transaction can transfer/spend one or more outputs
@ -82,7 +80,6 @@ transferred if both Jack and Kelly sign.
Note how the sum of the incoming paperclips must equal the sum
of the outgoing paperclips (100).
## Transaction Validity
When a node is asked to check if a transaction is valid, it checks several
@ -90,6 +87,7 @@ things. We documented those things in a post on *The BigchainDB Blog*:
["What is a Valid Transaction in BigchainDB?"](https://blog.bigchaindb.com/what-is-a-valid-transaction-in-bigchaindb-9a1a075a9598)
(Note: That post was about BigchainDB Server v1.0.0.)
The [IPDB Transaction Spec documents the conditions for a transaction to be valid](https://the-ipdb-transaction-spec.readthedocs.io/en/latest/transaction-validation.html).
## Example Transactions

68
docs/server/source/appendices/azure-quickstart-template.md
View File

@ -1,68 +0,0 @@
# Azure Quickstart Template
This page outlines how to run a single BigchainDB node on the Microsoft Azure public cloud, with RethinkDB as the database backend. It uses an Azure Quickstart Template. That template is dated because we now recommend using MongoDB instead of RethinkDB. That's why we moved this page to the Appendices.
Note: There was an Azure quickstart template in the `blockchain` directory of Microsoft's `Azure/azure-quickstart-templates` repository on GitHub. It's gone now; it was replaced by the one described here.
One can deploy a BigchainDB node on Azure using the template in the `bigchaindb-on-ubuntu` directory of Microsoft's `Azure/azure-quickstart-templates` repository on GitHub. Here's how:
1. Go to [that directory on GitHub](https://github.com/Azure/azure-quickstart-templates/tree/master/bigchaindb-on-ubuntu).
1. Click the button labelled **Deploy to Azure**.
1. If you're not already logged in to Microsoft Azure, then you'll be prompted to login. If you don't have an account, then you'll have to create one.
1. Once you are logged in to the Microsoft Azure Portal, you should be taken to a form titled **BigchainDB**. Some notes to help with filling in that form are available [below](azure-quickstart-template.html#notes-on-the-blockchain-template-form-fields).
1. Deployment takes a few minutes. You can follow the notifications by clicking the bell icon at the top of the screen. At the time of writing, the final deployment operation (running the `init.sh` script) was failing, but a pull request ([#2884](https://github.com/Azure/azure-quickstart-templates/pull/2884)) has been made to fix that and these instructions say what you can do before that pull request gets merged...
1. Find out the public IP address of the virtual machine in the Azure Portal. Example: `40.69.87.250`
1. ssh in to the virtual machine at that IP address, i.e. do `ssh <Admin_username>@<machine-ip>` where `<Admin_username>` is the admin username you entered into the form and `<machine-ip>` is the virtual machine IP address determined in the last step. Example: `ssh bcdbadmin@40.69.87.250`
1. You should be prompted for a password. Give the `<Admin_password>` you entered into the form.
1. Configure BigchainDB Server by doing:
```text
bigchaindb configure rethinkdb
```
It will ask you several questions. You can press `Enter` (or `Return`) to accept the default for all of them *except for one*. When it asks **API Server bind? (default \`localhost:9984\`):**, you should answer:
```text
API Server bind? (default `localhost:9984`): 0.0.0.0:9984
```
Finally, run BigchainDB Server by doing:
```text
bigchaindb start
```
BigchainDB Server should now be running on the Azure virtual machine.
Remember to shut everything down when you're done (via the Azure Portal), because it generally costs money to run stuff on Azure.
## Notes on the Blockchain Template Form Fields
### BASICS
**Resource group** - You can use an existing resource group (if you have one) or create a new one named whatever you like, but avoid using fancy characters in the name because Azure might have problems if you do.
**Location** is the Microsoft Azure data center where you want the BigchainDB node to run. Pick one close to where you are located.
### SETTINGS
You can use whatever **Admin\_username** and **Admin\_password** you like (provided you don't get too fancy). It will complain if your password is too simple. You'll need these later to `ssh` into the virtual machine.
**Dns\_label\_prefix** - Once your virtual machine is deployed, it will have a public IP address and a DNS name (hostname) something like `<DNSprefix>.northeurope.cloudapp.azure.com`. The `<DNSprefix>` will be whatever you enter into this field.
**Virtual\_machine\_size** - This should be one of Azure's standard virtual machine sizes, such as `Standard_D1_v2`. There's a [list of virtual machine sizes in the Azure docs](https://docs.microsoft.com/en-us/azure/virtual-machines/virtual-machines-windows-sizes?toc=%2fazure%2fvirtual-machines%2fwindows%2ftoc.json).
**\_artifacts Location** - Leave this alone.
**\_artifacts Location Sas Token** - Leave this alone (blank).
### TERMS AND CONDITIONS
Read the terms and conditions. If you agree to them, then check the checkbox.
Finally, click the button labelled **Purchase**. (Generally speaking, it costs money to run stuff on Azure.)

9
docs/server/source/appendices/index.rst
View File

@ -1,6 +1,3 @@
.. You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Appendices
==========
@ -8,23 +5,17 @@ Appendices
:maxdepth: 1
install-os-level-deps
install-latest-pip
run-with-docker
json-serialization
cryptography
the-Bigchain-class
pipelines
backend
commands
aws-setup
azure-quickstart-template
generate-key-pair-for-ssh
firewall-notes
ntp-notes
rethinkdb-reqs
rethinkdb-backup
licenses
install-with-lxd
run-with-vagrant
run-with-ansible
vote-yaml

20
docs/server/source/appendices/install-latest-pip.md
View File

@ -1,20 +0,0 @@
# How to Install the Latest pip and setuptools
You can check the version of `pip` you're using (in your current virtualenv) by doing:
```text
pip -V
```
If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.5, then you must install a `pip` version associated with Python 3.5+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/).
On Ubuntu 16.04, we found that this works:
```text
sudo apt-get install python3-pip
```
That should install a Python 3 version of `pip` named `pip3`. If that didn't work, then another way to get `pip3` is to do `sudo apt-get install python3-setuptools` followed by `sudo easy_install3 pip`.
You can upgrade `pip` (`pip3`) and `setuptools` to the latest versions using:
```text
pip3 install --upgrade pip setuptools
```

43
docs/server/source/appendices/install-with-lxd.md
View File

@ -1,43 +0,0 @@
# Installing BigchainDB on LXC containers using LXD
**Note: This page was contributed by an external contributor and is not actively maintained. We include it in case someone is interested.**
You can visit this link to install LXD (instructions here): [LXD Install](https://linuxcontainers.org/lxd/getting-started-cli/)
(assumption is that you are using Ubuntu 14.04 for host/container)
Let us create an LXC container (via LXD) with the following command:
`lxc launch ubuntu:14.04 bigchaindb`
(ubuntu:14.04 - this is the remote server the command fetches the image from)
(bigchaindb - is the name of the container)
Below is the `install.sh` script you will need to install BigchainDB within your container.
Here is my `install.sh`:
```
#!/bin/bash
set -ex
export DEBIAN_FRONTEND=noninteractive
apt-get install -y wget
source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -
apt-get update
apt-get install -y rethinkdb python3-pip
pip3 install --upgrade pip wheel setuptools
pip install ptpython bigchaindb
```
Copy/Paste the above `install.sh` into the directory/path you are going to execute your LXD commands from (ie. the host).
Make sure your container is running by typing:
`lxc list`
Now, from the host (and the correct directory) where you saved `install.sh`, run this command:
`cat install.sh | lxc exec bigchaindb /bin/bash`
If you followed the commands correctly, you will have successfully created an LXC container (using LXD) that can get you up and running with BigchainDB in <5 minutes (depending on how long it takes to download all the packages).

26
docs/server/source/appendices/pipelines.rst
View File

@ -1,26 +0,0 @@
#########
Pipelines
#########
Block Creation
==============
.. automodule:: bigchaindb.pipelines.block
Block Voting
============
.. automodule:: bigchaindb.pipelines.vote
Block Status
============
.. automodule:: bigchaindb.pipelines.election
Stale Transaction Monitoring
============================
.. automodule:: bigchaindb.pipelines.stale

124
docs/server/source/appendices/rethinkdb-backup.md
View File

@ -1,124 +0,0 @@
# Backing Up and Restoring Data
This page was written when BigchainDB only worked with RethinkDB, so its focus is on RethinkDB-based backup. BigchainDB now supports MongoDB as a backend database and we recommend that you use MongoDB in production. Nevertheless, some of the following backup ideas are still relevant regardless of the backend database being used, so we moved this page to the Appendices.
## RethinkDB's Replication as a form of Backup
RethinkDB already has internal replication: every document is stored on _R_ different nodes, where _R_ is the replication factor (set using `bigchaindb set-replicas R`). Those replicas can be thought of as "live backups" because if one node goes down, the cluster will continue to work and no data will be lost.
At this point, there should be someone saying, "But replication isn't backup!"
It's true. Replication alone isn't enough, because something bad might happen _inside_ the database, and that could affect the replicas. For example, what if someone logged in as a RethinkDB admin and did a "drop table"? We currently plan for each node to be protected by a next-generation firewall (or something similar) to prevent such things from getting very far. For example, see [issue #240](https://github.com/bigchaindb/bigchaindb/issues/240).
Nevertheless, you should still consider having normal, "cold" backups, because bad things can still happen.
## Live Replication of RethinkDB Data Files
Each BigchainDB node stores its subset of the RethinkDB data in one directory. You could set up the node's file system so that directory lives on its own hard drive. Furthermore, you could make that hard drive part of a [RAID](https://en.wikipedia.org/wiki/RAID) array, so that a second hard drive would always have a copy of the original. If the original hard drive fails, then the second hard drive could take its place and the node would continue to function. Meanwhile, the original hard drive could be replaced.
That's just one possible way of setting up the file system so as to provide extra reliability.
Another way to get similar reliability would be to mount the RethinkDB data directory on an [Amazon EBS](https://aws.amazon.com/ebs/) volume. Each Amazon EBS volume is, "automatically replicated within its Availability Zone to protect you from component failure, offering high availability and durability."
As with shard replication, live file-system replication protects against many failure modes, but it doesn't protect against them all. You should still consider having normal, "cold" backups.
## rethinkdb dump (to a File)
RethinkDB can create an archive of all data in the cluster (or all data in specified tables), as a compressed file. According to [the RethinkDB blog post when that functionality became available](https://rethinkdb.com/blog/1.7-release/):
> Since the backup process is using client drivers, it automatically takes advantage of the MVCC [multiversion concurrency control] functionality built into RethinkDB. It will use some cluster resources, but will not lock out any of the clients, so you can safely run it on a live cluster.
To back up all the data in a BigchainDB cluster, the RethinkDB admin user must run a command like the following on one of the nodes:
```text
rethinkdb dump -e bigchain.bigchain -e bigchain.votes
```
That should write a file named `rethinkdb_dump_<date>_<time>.tar.gz`. The `-e` option is used to specify which tables should be exported. You probably don't need to export the backlog table, but you definitely need to export the bigchain and votes tables.
`bigchain.votes` means the `votes` table in the RethinkDB database named `bigchain`. It's possible that your database has a different name: [the database name is a BigchainDB configuration setting](../server-reference/configuration.html#database-host-database-port-database-name). The default name is `bigchain`. (Tip: you can see the values of all configuration settings using the `bigchaindb show-config` command.)
There's [more information about the `rethinkdb dump` command in the RethinkDB documentation](https://www.rethinkdb.com/docs/backup/). It also explains how to restore data to a cluster from an archive file.
**Notes**
* If the `rethinkdb dump` subcommand fails and the last line of the Traceback says "NameError: name 'file' is not defined", then you need to update your RethinkDB Python driver; do a `pip install --upgrade rethinkdb`
* It might take a long time to backup data this way. The more data, the longer it will take.
* You need enough free disk space to store the backup file.
* If a document changes after the backup starts but before it ends, then the changed document may not be in the final backup. This shouldn't be a problem for BigchainDB, because blocks and votes can't change anyway.
* `rethinkdb dump` saves data and secondary indexes, but does *not* save cluster metadata. You will need to recreate your cluster setup yourself after you run `rethinkdb restore`.
* RethinkDB also has [subcommands to import/export](https://gist.github.com/coffeemug/5894257) collections of JSON or CSV files. While one could use those for backup/restore, it wouldn't be very practical.
## Client-Side Backup
In the future, it will be possible for clients to query for the blocks containing the transactions they care about, and for the votes on those blocks. They could save a local copy of those blocks and votes.
**How could we be sure blocks and votes from a client are valid?**
All blocks and votes are signed by cluster nodes (owned and operated by consortium members). Only cluster nodes can produce valid signatures because only cluster nodes have the necessary private keys. A client can't produce a valid signature for a block or vote.
**Could we restore an entire BigchainDB database using client-saved blocks and votes?**
Yes, in principle, but it would be difficult to know if you've recovered every block and vote. Votes link to the block they're voting on and to the previous block, so one could detect some missing blocks. It would be difficult to know if you've recovered all the votes.
## Backup by Copying RethinkDB Data Files
It's _possible_ to back up a BigchainDB database by creating a point-in-time copy of the RethinkDB data files (on all nodes, at roughly the same time). It's not a very practical approach to backup: the resulting set of files will be much larger (collectively) than what one would get using `rethinkdb dump`, and there are no guarantees on how consistent that data will be, especially for recently-written data.
If you're curious about what's involved, see the [MongoDB documentation about "Backup by Copying Underlying Data Files"](https://docs.mongodb.com/manual/core/backups/#backup-with-file-copies). (Yes, that's documentation for MongoDB, but the principles are the same.)
See the last subsection of this page for a better way to use this idea.
## Incremental or Continuous Backup
**Incremental backup** is where backup happens on a regular basis (e.g. daily), and each one only records the changes since the last backup.
**Continuous backup** might mean incremental backup on a very regular basis (e.g. every ten minutes), or it might mean backup of every database operation as it happens. The latter is also called transaction logging or continuous archiving.
At the time of writing, RethinkDB didn't have a built-in incremental or continuous backup capability, but the idea was raised in RethinkDB issues [#89](https://github.com/rethinkdb/rethinkdb/issues/89) and [#5890](https://github.com/rethinkdb/rethinkdb/issues/5890). On July 5, 2016, Daniel Mewes (of RethinkDB) wrote the following comment on issue #5890: "We would like to add this feature [continuous backup], but haven't started working on it yet."
To get a sense of what continuous backup might look like for RethinkDB, one can look at the continuous backup options available for MongoDB. MongoDB, the company, offers continuous backup with [Ops Manager](https://www.mongodb.com/products/ops-manager) (self-hosted) or [Cloud Manager](https://www.mongodb.com/cloud) (fully managed). Features include:
* It "continuously maintains backups, so if your MongoDB deployment experiences a failure, the most recent backup is only moments behind..."
* It "offers point-in-time backups of replica sets and cluster-wide snapshots of sharded clusters. You can restore to precisely the moment you need, quickly and safely."
* "You can rebuild entire running clusters, just from your backups."
* It enables, "fast and seamless provisioning of new dev and test environments."
The MongoDB documentation has more [details about how Ops Manager Backup works](https://docs.opsmanager.mongodb.com/current/application/#backup).
Considerations for BigchainDB:
* We'd like the cost of backup to be low. To get a sense of the cost, MongoDB Cloud Manager backup [costed $30 / GB / year prepaid](https://www.mongodb.com/blog/post/lower-mms-backup-prices-backing-mongodb-now-easier-and-more-affordable). One thousand gigabytes backed up (i.e. about a terabyte) would cost 30 thousand US dollars per year. (That's just for the backup; there's also a cost per server per year.)
* We'd like the backup to be decentralized, with no single point of control or single point of failure. (Note: some file systems have a single point of failure. For example, HDFS has one Namenode.)
* We only care to back up blocks and votes, and once written, those never change. There are no updates or deletes, just new blocks and votes.
## Combining RethinkDB Replication with Storage Snapshots
Although it's not advertised as such, RethinkDB's built-in replication feature is similar to continous backup, except the "backup" (i.e. the set of replica shards) is spread across all the nodes. One could take that idea a bit farther by creating a set of backup-only servers with one full backup:
* Give all the original BigchainDB nodes (RethinkDB nodes) the server tag `original`.
* Set up a group of servers running RethinkDB only, and give them the server tag `backup`. The `backup` servers could be geographically separated from all the `original` nodes (or not; it's up to the consortium to decide).
* Clients shouldn't be able to read from or write to servers in the `backup` set.
* Send a RethinkDB reconfigure command to the RethinkDB cluster to make it so that the `original` set has the same number of replicas as before (or maybe one less), and the `backup` set has one replica. Also, make sure the `primary_replica_tag='original'` so that all primary shards live on the `original` nodes.
The [RethinkDB documentation on sharding and replication](https://www.rethinkdb.com/docs/sharding-and-replication/) has the details of how to set server tags and do RethinkDB reconfiguration.
Once you've set up a set of backup-only RethinkDB servers, you could make a point-in-time snapshot of their storage devices, as a form of backup.
You might want to disconnect the `backup` set from the `original` set first, and then wait for reads and writes in the `backup` set to stop. (The `backup` set should have only one copy of each shard, so there's no opportunity for inconsistency between shards of the `backup` set.)
You will want to re-connect the `backup` set to the `original` set as soon as possible, so it's able to catch up.
If something bad happens to the entire original BigchainDB cluster (including the `backup` set) and you need to restore it from a snapshot, you can, but before you make BigchainDB live, you should 1) delete all entries in the backlog table, 2) delete all blocks after the last voted-valid block, 3) delete all votes on the blocks deleted in part 2, and 4) rebuild the RethinkDB indexes.
**NOTE:** Sometimes snapshots are _incremental_. For example, [Amazon EBS snapshots](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSSnapshots.html) are incremental, meaning "only the blocks on the device that have changed after your most recent snapshot are saved. **This minimizes the time required to create the snapshot and saves on storage costs.**" [Emphasis added]

61
docs/server/source/appendices/rethinkdb-reqs.md
View File

@ -1,61 +0,0 @@
# RethinkDB Requirements
[The RethinkDB documentation](https://rethinkdb.com/docs/) should be your first source of information about its requirements. This page serves mostly to document some of its more obscure requirements.
RethinkDB Server [will run on any modern OS](https://www.rethinkdb.com/docs/install/). Note that the Fedora package isn't officially supported. Also, official support for Windows is fairly recent ([April 2016](https://rethinkdb.com/blog/2.3-release/)).
## Storage Requirements
When it comes to storage for RethinkDB, there are many things that are nice to have (e.g. SSDs, high-speed input/output [IOPS], replication, reliability, scalability, pay-for-what-you-use), but there are few _requirements_ other than:
1. have enough storage to store all your data (and its replicas), and
2. make sure your storage solution (hardware and interconnects) can handle your expected read & write rates.
For RethinkDB's failover mechanisms to work, [every RethinkDB table must have at least three replicas](https://rethinkdb.com/docs/failover/) (i.e. a primary replica and two others). For example, if you want to store 10 GB of unique data, then you need at least 30 GB of storage. (Indexes and internal metadata are stored in RAM.)
As for the read & write rates, what do you expect those to be for your situation? It's not enough for the storage system alone to handle those rates: the interconnects between the nodes must also be able to handle them.
**Storage Notes Specific to RethinkDB**
* The RethinkDB storage engine has a number of SSD optimizations, so you _can_ benefit from using SSDs. ([source](https://www.rethinkdb.com/docs/architecture/))
* If you have an N-node RethinkDB cluster and 1) you want to use it to store an amount of data D (unique records, before replication), 2) you want the replication factor to be R (all tables), and 3) you want N shards (all tables), then each BigchainDB node must have storage space of at least R×D/N.
* RethinkDB tables can have [at most 64 shards](https://rethinkdb.com/limitations/). What does that imply? Suppose you only have one table, with 64 shards. How big could that table be? It depends on how much data can be stored in each node. If the maximum amount of data that a node can store is d, then the biggest-possible shard is d, and the biggest-possible table size is 64 times that. (All shard replicas would have to be stored on other nodes beyond the initial 64.) If there are two tables, the second table could also have 64 shards, stored on 64 other maxed-out nodes, so the total amount of unique data in the database would be (64 shards/table)×(2 tables)×d. In general, if you have T tables, the maximum amount of unique data that can be stored in the database (i.e. the amount of data before replication) is 64×T×d.
* When you set up storage for your RethinkDB data, you may have to select a filesystem. (Sometimes, the filesystem is already decided by the choice of storage.) We recommend using a filesystem that supports direct I/O (Input/Output). Many compressed or encrypted file systems don't support direct I/O. The ext4 filesystem supports direct I/O (but be careful: if you enable the data=journal mode, then direct I/O support will be disabled; the default is data=ordered). If your chosen filesystem supports direct I/O and you're using Linux, then you don't need to do anything to request or enable direct I/O. RethinkDB does that.
<p style="background-color: lightgrey;">What is direct I/O? It allows RethinkDB to write directly to the storage device (or use its own in-memory caching mechanisms), rather than relying on the operating system's file read and write caching mechanisms. (If you're using Linux, a write-to-file normally writes to the in-memory Page Cache first; only later does that Page Cache get flushed to disk. The Page Cache is also used when reading files.)</p>
* RethinkDB stores its data in a specific directory. You can tell RethinkDB _which_ directory using the RethinkDB config file, as explained below. In this documentation, we assume the directory is `/data`. If you set up a separate device (partition, RAID array, or logical volume) to store the RethinkDB data, then mount that device on `/data`.
## Memory (RAM) Requirements
In their [FAQ](https://rethinkdb.com/faq/), RethinkDB recommends that, "RethinkDB servers have at least 2GB of RAM..." ([source](https://rethinkdb.com/faq/))
In particular: "RethinkDB requires data structures in RAM on each server proportional to the size of the data on that server’s disk, usually around 1% of the size of the total data set." ([source](https://rethinkdb.com/limitations/)) We asked what they meant by "total data set" and [they said](https://github.com/rethinkdb/rethinkdb/issues/5902#issuecomment-230860607) it's "referring to only the data stored on the particular server."
Also, "The storage engine is used in conjunction with a custom, B-Tree-aware caching engine which allows file sizes many orders of magnitude greater than the amount of available memory. RethinkDB can operate on a terabyte of data with about ten gigabytes of free RAM." ([source](https://www.rethinkdb.com/docs/architecture/)) (In this case, it's the _cluster_ which has a total of one terabyte of data, and it's the _cluster_ which has a total of ten gigabytes of RAM. That is, if you add up the RethinkDB RAM on all the servers, it's ten gigabytes.)
In reponse to our questions about RAM requirements, @danielmewes (of RethinkDB) [wrote](https://github.com/rethinkdb/rethinkdb/issues/5902#issuecomment-230860607):
> ... If you replicate the data, the amount of data per server increases accordingly, because multiple copies of the same data will be held by different servers in the cluster.
For example, if you increase the data replication factor from 1 to 2 (i.e. the primary plus one copy), then that will double the RAM needed for metadata. Also from @danielmewes:
> **For reasonable performance, you should probably aim at something closer to 5-10% of the data size.** [Emphasis added] The 1% is the bare minimum and doesn't include any caching. If you want to run near the minimum, you'll also need to manually lower RethinkDB's cache size through the `--cache-size` parameter to free up enough RAM for the metadata overhead...
RethinkDB has [documentation about its memory requirements](https://rethinkdb.com/docs/memory-usage/). You can use that page to get a better estimate of how much memory you'll need. In particular, note that RethinkDB automatically configures the cache size limit to be about half the available memory, but it can be no lower than 100 MB. As @danielmewes noted, you can manually change the cache size limit (e.g. to free up RAM for queries, metadata, or other things).
If a RethinkDB process (on a server) runs out of RAM, the operating system will start swapping RAM out to disk, slowing everything down. According to @danielmewes:
> Going into swap is usually pretty bad for RethinkDB, and RethinkDB servers that have gone into swap often become so slow that other nodes in the cluster consider them unavailable and terminate the connection to them. I recommend adjusting RethinkDB's cache size conservatively to avoid this scenario. RethinkDB will still make use of additional RAM through the operating system's block cache (though less efficiently than when it can keep data in its own cache).
## Filesystem Requirements
RethinkDB "supports most commonly used file systems" ([source](https://www.rethinkdb.com/docs/architecture/)) but it has [issues with BTRFS](https://github.com/rethinkdb/rethinkdb/issues/2781) (B-tree file system).
It's best to use a filesystem that supports direct I/O, because that will improve RethinkDB performance (if you tell RethinkDB to use direct I/O). Many compressed or encrypted filesystems don't support direct I/O.

6
docs/server/source/introduction.md
View File

@ -19,12 +19,6 @@ Note that there are a few kinds of nodes:
* [Set up a local BigchainDB node for development, experimenting and testing](dev-and-test/index.html)
* [Set up and run a BigchainDB cluster](clusters.html)
There are some old RethinkDB-based deployment instructions as well:
* [Deploy a bare-bones RethinkDB-based node on Azure](appendices/azure-quickstart-template.html)
Instructions for setting up a client will be provided once there's a public test net.
## Can I Help?

3
setup.py
View File

@ -32,6 +32,7 @@ dev_require = [
'ipython',
'watchdog',
'logging_tree',
'pre-commit'
]
docs_require = [
@ -83,7 +84,7 @@ install_requires = [
'aiohttp~=2.0',
'python-rapidjson-schema==0.1.1',
'statsd==3.2.1',
'abci~=0.2.0',
'abci~=0.3.0',
]
setup(

39
tests/backend/localmongodb/test_queries.py
View File

@ -3,10 +3,9 @@ from copy import deepcopy
import pytest
import pymongo
pytestmark = [pytest.mark.tendermint, pytest.mark.localmongodb]
pytestmark = [pytest.mark.tendermint, pytest.mark.localmongodb, pytest.mark.bdb]
@pytest.mark.bdb
def test_get_txids_filtered(signed_create_tx, signed_transfer_tx):
from bigchaindb.backend import connect, query
from bigchaindb.models import Transaction
@ -32,7 +31,6 @@ def test_get_txids_filtered(signed_create_tx, signed_transfer_tx):
assert txids == {signed_transfer_tx.id}
@pytest.mark.bdb
def test_write_assets():
from bigchaindb.backend import connect, query
conn = connect()
@ -57,7 +55,6 @@ def test_write_assets():
assert list(cursor) == assets[:-1]
@pytest.mark.bdb
def test_get_assets():
from bigchaindb.backend import connect, query
conn = connect()
@ -74,8 +71,40 @@ def test_get_assets():
assert query.get_asset(conn, asset['id'])
@pytest.mark.bdb
def test_text_search():
from ..mongodb.test_queries import test_text_search
test_text_search('assets')
def test_get_owned_ids(signed_create_tx, user_pk):
from bigchaindb.backend import connect, query
conn = connect()
# insert a transaction
conn.db.transactions.insert_one(signed_create_tx.to_dict())
txns = list(query.get_owned_ids(conn, user_pk))
assert txns[0] == signed_create_tx.to_dict()
def test_get_spending_transactions(user_pk):
from bigchaindb.backend import connect, query
from bigchaindb.models import Transaction
conn = connect()
out = [([user_pk], 1)]
tx1 = Transaction.create([user_pk], out * 3)
inputs = tx1.to_inputs()
tx2 = Transaction.transfer([inputs[0]], out, tx1.id)
tx3 = Transaction.transfer([inputs[1]], out, tx1.id)
tx4 = Transaction.transfer([inputs[2]], out, tx1.id)
txns = [tx.to_dict() for tx in [tx1, tx2, tx3, tx4]]
conn.db.transactions.insert_many(txns)
links = [inputs[0].fulfills.to_dict(), inputs[2].fulfills.to_dict()]
txns = list(query.get_spending_transactions(conn, links))
# tx3 not a member because input 1 not asked for
assert txns == [tx2.to_dict(), tx4.to_dict()]

10
tests/backend/mongodb-ssl/certs/test_bdb_ssl.crt
View File

@ -49,18 +49,18 @@ Certificate:
1a:b2:61
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Subject Key Identifier:
X509v3 Subject Key Identifier:
B7:F5:E7:0E:F8:D8:FE:A1:56:5B:EA:80:2F:18:71:C2:44:0C:91:D0
X509v3 Authority Key Identifier:
X509v3 Authority Key Identifier:
keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98
DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com
serial:E8:06:B2:C9:2A:9C:2E:FC
X509v3 Extended Key Usage:
X509v3 Extended Key Usage:
TLS Web Client Authentication
X509v3 Key Usage:
X509v3 Key Usage:
Digital Signature
Signature Algorithm: sha256WithRSAEncryption
78:44:00:be:10:3b:f3:40:e1:5e:e4:3a:64:99:13:71:1d:91:

10
tests/backend/mongodb-ssl/certs/test_mdb_bak_ssl_cert_and_key.pem
View File

@ -49,18 +49,18 @@ Certificate:
a6:7f:b9
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Subject Key Identifier:
X509v3 Subject Key Identifier:
95:F3:A7:FB:99:C6:9D:91:36:06:51:31:39:EC:37:42:89:07:AB:31
X509v3 Authority Key Identifier:
X509v3 Authority Key Identifier:
keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98
DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com
serial:E8:06:B2:C9:2A:9C:2E:FC
X509v3 Extended Key Usage:
X509v3 Extended Key Usage:
TLS Web Client Authentication
X509v3 Key Usage:
X509v3 Key Usage:
Digital Signature
Signature Algorithm: sha256WithRSAEncryption
5b:42:f5:e9:cc:2a:40:8a:53:29:d9:67:2b:5d:df:25:b8:08:

10
tests/backend/mongodb-ssl/certs/test_mdb_mon_ssl_cert_and_key.pem
View File

@ -49,18 +49,18 @@ Certificate:
f8:f2:1d
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Subject Key Identifier:
X509v3 Subject Key Identifier:
31:63:2C:98:2F:9F:6C:44:82:A9:B8:D3:06:15:95:84:D9:52:98:71
X509v3 Authority Key Identifier:
X509v3 Authority Key Identifier:
keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98
DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com
serial:E8:06:B2:C9:2A:9C:2E:FC
X509v3 Extended Key Usage:
X509v3 Extended Key Usage:
TLS Web Client Authentication
X509v3 Key Usage:
X509v3 Key Usage:
Digital Signature
Signature Algorithm: sha256WithRSAEncryption
1e:16:02:5b:35:f6:36:0a:54:bc:48:11:51:39:a1:b1:e5:39:

12
tests/backend/mongodb-ssl/certs/test_mdb_ssl_cert_and_key.pem
View File

@ -49,20 +49,20 @@ Certificate:
99:1f:23
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Subject Key Identifier:
X509v3 Subject Key Identifier:
0B:53:E2:76:40:AD:73:C4:12:6C:85:CF:36:5B:5F:FB:6E:E8:03:A7
X509v3 Authority Key Identifier:
X509v3 Authority Key Identifier:
keyid:69:C7:85:80:64:E6:90:40:E9:30:68:88:23:D2:4D:BB:EC:DF:98:98
DirName:/C=DE/ST=Berlin/L=Berlin/O=BigchainDB GmbH/OU=ROOT-CA/CN=Test Infra Root CA/emailAddress=dev@bigchaindb.com
serial:E8:06:B2:C9:2A:9C:2E:FC
X509v3 Extended Key Usage:
X509v3 Extended Key Usage:
TLS Web Server Authentication, TLS Web Client Authentication
X509v3 Key Usage:
X509v3 Key Usage:
Digital Signature, Key Encipherment
X509v3 Subject Alternative Name:
X509v3 Subject Alternative Name:
DNS:localhost, DNS:test-mdb-ssl
Signature Algorithm: sha256WithRSAEncryption
4c:14:3e:6b:af:f8:e8:69:11:2e:13:12:b7:9b:91:c7:68:01:

2
tests/backend/mongodb/test_queries.py
View File

@ -529,7 +529,7 @@ def test_get_assets():
assert list(cursor.sort('id', pymongo.ASCENDING)) == assets[::2]
@pytest.mark.parametrize("table", ['assets', 'metadata'])
@pytest.mark.parametrize('table', ['assets', 'metadata'])
def test_text_search(table):
from bigchaindb.backend import connect, query
conn = connect()

6
tests/common/test_schema.py
View File

@ -1,5 +1,4 @@
"""
This module is tests related to schema checking, but _not_ of granular schematic
"""This module is tests related to schema checking, but _not_ of granular schematic
properties related to validation.
"""
@ -23,8 +22,7 @@ UNSUPPORTED_CRYPTOCONDITION_TYPES = (
# Test of schema utils
def _test_additionalproperties(node, path=''):
"""
Validate that each object node has additionalProperties set, so that
"""Validate that each object node has additionalProperties set, so that
objects with junk keys do not pass as valid.
"""
if isinstance(node, list):

3
tests/common/test_transaction.py
View File

@ -1,5 +1,4 @@
"""
These are tests of the API of the Transaction class and associated classes.
"""These are tests of the API of the Transaction class and associated classes.
Tests for transaction validation are separate.
"""
from copy import deepcopy

3
tests/conftest.py
View File

@ -1,5 +1,4 @@
"""
Fixtures and setup / teardown functions
"""Fixtures and setup / teardown functions
Tasks:
1. setup test database before starting the tests

3
tests/db/test_bigchain_api.py
View File

@ -1242,8 +1242,7 @@ def test_get_outputs_filtered(filter_spent, filter_unspent):
@pytest.mark.bdb
def test_cant_spend_same_input_twice_in_tx(b, genesis_block):
"""
Recreate duplicated fulfillments bug
"""Recreate duplicated fulfillments bug
https://github.com/bigchaindb/bigchaindb/issues/1099
"""
from bigchaindb.models import Transaction

13
tests/integration/test_federation.py
View File

@ -15,8 +15,7 @@ from tests.pipelines.stepping import create_stepper
@contextmanager
def federation(n):
"""
Return a list of Bigchain objects and pipeline steppers to represent
"""Return a list of Bigchain objects and pipeline steppers to represent
a BigchainDB federation
"""
keys = [generate_key_pair() for _ in range(n)]
@ -24,8 +23,7 @@ def federation(n):
@contextmanager
def make_nodes(i):
"""
make_nodes is a recursive context manager. Essentially it is doing:
"""make_nodes is a recursive context manager. Essentially it is doing:
with f(a[0]) as b0:
with f(a[1]) as b1:
@ -159,8 +157,7 @@ def test_elect_sybill(federation_3):
@pytest.mark.bdb
@pytest.mark.genesis
def test_elect_dos(federation_3):
"""
https://github.com/bigchaindb/bigchaindb/issues/1314
"""https://github.com/bigchaindb/bigchaindb/issues/1314
Test that a node cannot block another node's opportunity to vote
on a block by writing an incorrectly signed vote
"""
@ -171,9 +168,7 @@ def test_elect_dos(federation_3):
@pytest.mark.bdb
@pytest.mark.genesis
def test_elect_bad_block_voters_list(federation_3):
"""
See https://github.com/bigchaindb/bigchaindb/issues/1224
"""
"""See https://github.com/bigchaindb/bigchaindb/issues/1224"""
[bx, (s0, s1, s2)] = federation_3
b = s0.block.bigchain
# First remove other nodes from node 0 so that it self assigns the tx

2
tests/integration/test_integration.py
View File

@ -29,7 +29,7 @@ def test_double_create(b, user_pk):
@pytest.mark.dspend
@pytest.mark.usefixtures('inputs')
def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk):
""" Test for #633 https://github.com/bigchaindb/bigchaindb/issues/633 """
"""Test for #633 https://github.com/bigchaindb/bigchaindb/issues/633"""
from bigchaindb.common.exceptions import DoubleSpend
from bigchaindb.models import Transaction
input_valid = b.get_owned_ids(user_pk).pop()

18
tests/pipelines/stepping.py
View File

@ -1,5 +1,4 @@
"""
Pipeline stepping is a way to advance the asynchronous data pipeline
"""Pipeline stepping is a way to advance the asynchronous data pipeline
deterministically by exposing each step separately and advancing the states
manually.
@ -53,7 +52,7 @@ class MultipipesStepper:
self.processes = []
def add_input(self, prefix, node, next):
""" Add an input task; Reads from the outqueue of the Node """
"""Add an input task; Reads from the outqueue of the Node"""
name = '%s_%s' % (prefix, node.name)
next_name = '%s_%s' % (prefix, next.name)
@ -77,8 +76,7 @@ class MultipipesStepper:
self.input_tasks.add(name)
def add_stage(self, prefix, node, next):
"""
Add a stage task, popping from own queue and appending to the queue
"""Add a stage task, popping from own queue and appending to the queue
of the next node
"""
f = node.target
@ -96,7 +94,7 @@ class MultipipesStepper:
self.tasks[name] = task
def _enqueue(self, name, item):
""" internal function; add item(s) to queue) """
"""Internal function; add item(s) to queue)"""
queue = self.queues.setdefault(name, [])
if isinstance(item, types.GeneratorType):
items = list(item)
@ -108,7 +106,7 @@ class MultipipesStepper:
queue.append(list(item))
def step(self, name, **kwargs):
""" Advance pipeline stage. Throws Empty if no data to consume. """
"""Advance pipeline stage. Throws Empty if no data to consume."""
logging.debug('Stepping %s', name)
task = self.tasks[name]
if name in self.input_tasks:
@ -122,7 +120,7 @@ class MultipipesStepper:
@property
def counts(self):
""" Get sizes of non empty queues """
"""Get sizes of non empty queues"""
counts = {}
for name in self.queues:
n = len(self.queues[name])
@ -131,12 +129,12 @@ class MultipipesStepper:
return counts
def __getattr__(self, name):
""" Shortcut to get a queue """
"""Shortcut to get a queue"""
return lambda **kwargs: self.step(name, **kwargs)
@contextmanager
def start(self):
""" Start async inputs; changefeeds etc """
"""Start async inputs; changefeeds etc"""
for p in self.processes:
p.start()
# It would be nice to have a better way to wait for changefeeds here.

3
tests/pipelines/test_block_creation.py
View File

@ -40,8 +40,7 @@ def test_validate_transaction(b, create_tx):
def test_validate_transaction_handles_exceptions(b, signed_create_tx):
"""
This test makes sure that `BlockPipeline.validate_tx` handles possible
"""This test makes sure that `BlockPipeline.validate_tx` handles possible
exceptions from `Transaction.from_dict`.
"""
from bigchaindb.pipelines.block import BlockPipeline

2
tests/tendermint/test_core.py
View File

@ -92,7 +92,7 @@ def test_deliver_transfer_tx__double_spend_fails(b):
carly = generate_key_pair()
asset = {
"msg": "live long and prosper"
'msg': 'live long and prosper'
}
tx = Transaction.create([alice.public_key],

3
tests/test_txlist.py
View File

@ -1,5 +1,4 @@
"""
Test getting a list of transactions from the backend.
"""Test getting a list of transactions from the backend.
This test module defines it's own fixture which is used by all the tests.
"""

3
tests/test_voting.py
View File

@ -122,8 +122,7 @@ def test_decide_votes_invalid(kwargs):
@pytest.mark.parametrize('n_voters', range(8))
def test_vote_actions(n_voters):
"""
* Legal transitions are UNDECIDED -> [VALID|INVALID] only
"""* Legal transitions are UNDECIDED -> [VALID|INVALID] only
* Block is never left UNDECIDED after voting
* Accomodates rogues on previous block / invalid schema
"""

3
tests/validation/test_transaction_structure.py
View File

@ -1,5 +1,4 @@
"""
All tests of transaction structure. The concern here is that transaction
"""All tests of transaction structure. The concern here is that transaction
structural / schematic issues are caught when reading a transaction
(ie going from dict -> transaction).
"""

3
tests/web/conftest.py
View File

@ -3,11 +3,10 @@ import pytest
@pytest.fixture
def app(request):
from bigchaindb import config
from bigchaindb.web import server
from bigchaindb.tendermint.lib import BigchainDB
if config['database']['backend'] == 'localmongodb':
if request.config.getoption('--database-backend') == 'localmongodb':
app = server.create_app(debug=True, bigchaindb_factory=BigchainDB)
else:
app = server.create_app(debug=True)

4
tests/web/test_info.py
View File

@ -18,7 +18,7 @@ def test_api_root_endpoint(client, wsserver_base_url):
'outputs': '/api/v1/outputs/',
'streams': '{}/api/v1/streams/valid_transactions'.format(
wsserver_base_url),
"metadata": "/api/v1/metadata/"
'metadata': '/api/v1/metadata/',
}
},
'docs': 'https://docs.bigchaindb.com/projects/server/en/vtsttst/',
@ -42,7 +42,7 @@ def test_api_v1_endpoint(client, wsserver_base_url):
'outputs': '/outputs/',
'streams': '{}/api/v1/streams/valid_transactions'.format(
wsserver_base_url),
"metadata": "/metadata/"
'metadata': '/metadata/',
}
res = client.get('/api/v1')
assert res.json == api_v1_info

12
tests/web/test_outputs.py
View File

@ -6,11 +6,12 @@ pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('inputs')]
OUTPUTS_ENDPOINT = '/api/v1/outputs/'
@pytest.mark.tendermint
def test_get_outputs_endpoint(client, user_pk):
m = MagicMock()
m.txid = 'a'
m.output = 0
with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof:
with patch('bigchaindb.tendermint.lib.BigchainDB.get_outputs_filtered') as gof:
gof.return_value = [m, m]
res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk))
assert res.json == [
@ -21,11 +22,12 @@ def test_get_outputs_endpoint(client, user_pk):
gof.assert_called_once_with(user_pk, None)
@pytest.mark.tendermint
def test_get_outputs_endpoint_unspent(client, user_pk):
m = MagicMock()
m.txid = 'a'
m.output = 0
with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof:
with patch('bigchaindb.tendermint.lib.BigchainDB.get_outputs_filtered') as gof:
gof.return_value = [m]
params = '?spent=False&public_key={}'.format(user_pk)
res = client.get(OUTPUTS_ENDPOINT + params)
@ -34,11 +36,12 @@ def test_get_outputs_endpoint_unspent(client, user_pk):
gof.assert_called_once_with(user_pk, False)
@pytest.mark.tendermint
def test_get_outputs_endpoint_spent(client, user_pk):
m = MagicMock()
m.txid = 'a'
m.output = 0
with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof:
with patch('bigchaindb.tendermint.lib.BigchainDB.get_outputs_filtered') as gof:
gof.return_value = [m]
params = '?spent=true&public_key={}'.format(user_pk)
res = client.get(OUTPUTS_ENDPOINT + params)
@ -47,11 +50,13 @@ def test_get_outputs_endpoint_spent(client, user_pk):
gof.assert_called_once_with(user_pk, True)
@pytest.mark.tendermint
def test_get_outputs_endpoint_without_public_key(client):
res = client.get(OUTPUTS_ENDPOINT)
assert res.status_code == 400
@pytest.mark.tendermint
def test_get_outputs_endpoint_with_invalid_public_key(client):
expected = {'message': {'public_key': 'Invalid base58 ed25519 key'}}
res = client.get(OUTPUTS_ENDPOINT + '?public_key=abc')
@ -59,6 +64,7 @@ def test_get_outputs_endpoint_with_invalid_public_key(client):
assert res.status_code == 400
@pytest.mark.tendermint
def test_get_outputs_endpoint_with_invalid_spent(client, user_pk):
expected = {'message': {'spent': 'Boolean value must be "true" or "false" (lowercase)'}}
params = '?spent=tru&public_key={}'.format(user_pk)

6
tests/web/test_transactions.py
View File

@ -47,8 +47,8 @@ def test_post_create_transaction_endpoint(b, client):
assert res.json['outputs'][0]['public_keys'][0] == user_pub
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize("language,expected_status_code", [
@pytest.mark.parametrize('nested', [False, True])
@pytest.mark.parametrize('language,expected_status_code', [
('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202),
('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202),
('norwegian', 202), ('portuguese', 202), ('romanian', 202), ('none', 202),
@ -322,7 +322,7 @@ def test_transactions_get_list_good(client):
from functools import partial
def get_txs_patched(conn, **args):
""" Patch `get_transactions_filtered` so that rather than return an array
"""Patch `get_transactions_filtered` so that rather than return an array
of transactions it returns an array of shims with a to_dict() method
that reports one of the arguments passed to `get_transactions_filtered`.
"""