From e253ca015f884f85fa3cf8f445152b977c4b7b22 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sun, 22 Jan 2017 13:45:17 +0100 Subject: [PATCH 001/219] Line-length changes, many suggested by @amirelemam in #762 --- benchmarking-tests/benchmark_utils.py | 55 ++++++++++++++++++--------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/benchmarking-tests/benchmark_utils.py b/benchmarking-tests/benchmark_utils.py index 807146e8..b7cc5eb2 100644 --- a/benchmarking-tests/benchmark_utils.py +++ b/benchmarking-tests/benchmark_utils.py @@ -31,8 +31,9 @@ def create_write_transaction(tx_left, payload_filler): if payload_filler: payload_dict['filler'] = payload_filler while tx_left > 0: - # Include a random uuid string in the payload to prevent duplicate - # transactions (i.e. transactions with the same hash) + # Include a random uuid string in the payload + # to prevent duplicate transactions + # (i.e. transactions with the same hash) payload_dict['msg'] = str(uuid.uuid4()) tx = Transaction.create([b.me], [b.me], payload=payload_dict) tx = tx.sign([b.me_private]) @@ -69,22 +70,31 @@ def run_gather_metrics(args): num_transactions = r.table('backlog').count().run(conn) num_transactions_received = 0 initial_time = None - logger.info('Starting gathering metrics. {} transasctions in the backlog'.format(num_transactions)) + logger.info('Starting gathering metrics.') + logger.info('{} transasctions in the backlog'.format(num_transactions)) logger.info('This process should exit automatically. ' - 'If this does not happen you can exit at any time using Ctrl-C' - ' saving all the metrics gathered up to this point.') + 'If this does not happen you can exit at any time using Ctrl-C ' + 'saving all the metrics gathered up to this point.') - logger.info('\t{:<20} {:<20} {:<20} {:<20}'.format('timestamp', 'tx in block', - 'tx/s', '% complete')) + logger.info('\t{:<20} {:<20} {:<20} {:<20}'.format( + 'timestamp', + 'tx in block', + 'tx/s', + '% complete' + )) # listen to the changefeed try: for change in r.table('bigchain').changes().run(conn): # check only for new blocks if change['old_val'] is None: - block_num_transactions = len(change['new_val']['block']['transactions']) + block_num_transactions = len( + change['new_val']['block']['transactions'] + ) time_now = time.time() - csv_writer.writerow([str(time_now), str(block_num_transactions)]) + csv_writer.writerow( + [str(time_now), str(block_num_transactions)] + ) # log statistics if initial_time is None: @@ -92,15 +102,23 @@ def run_gather_metrics(args): num_transactions_received += block_num_transactions elapsed_time = time_now - initial_time - percent_complete = round((num_transactions_received / num_transactions) * 100) + percent_complete = round( + (num_transactions_received / num_transactions) * 100 + ) if elapsed_time != 0: - transactions_per_second = round(num_transactions_received / elapsed_time) + transactions_per_second = round( + num_transactions_received / elapsed_time + ) else: transactions_per_second = float('nan') - logger.info('\t{:<20} {:<20} {:<20} {:<20}'.format(time_now, block_num_transactions, - transactions_per_second, percent_complete)) + logger.info('\t{:<20} {:<20} {:<20} {:<20}'.format( + time_now, + block_num_transactions, + transactions_per_second, + percent_complete + )) if (num_transactions - num_transactions_received) == 0: break @@ -118,7 +136,8 @@ def main(): # add transactions to backlog backlog_parser = subparsers.add_parser('add-backlog', help='Add transactions to the backlog') - backlog_parser.add_argument('num_transactions', metavar='num_transactions', + backlog_parser.add_argument('num_transactions', + metavar='num_transactions', type=int, default=0, help='Number of transactions to add to the backlog') backlog_parser.add_argument('-s', '--payload-size', @@ -129,7 +148,9 @@ def main(): # set statsd host statsd_parser = subparsers.add_parser('set-statsd-host', help='Set statsd host') - statsd_parser.add_argument('statsd_host', metavar='statsd_host', default='localhost', + statsd_parser.add_argument('statsd_host', + metavar='statsd_host', + default='localhost', help='Hostname of the statsd server') # metrics @@ -138,7 +159,8 @@ def main(): metrics_parser.add_argument('-b', '--bigchaindb-host', required=True, - help='Bigchaindb node hostname to connect to gather cluster metrics') + help=('Bigchaindb node hostname to connect ' + 'to gather cluster metrics')) metrics_parser.add_argument('-c', '--csvfile', required=True, @@ -149,4 +171,3 @@ def main(): if __name__ == '__main__': main() - From b10b03e3ca7d1450390421272a3f2ff2fc71a88b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sun, 22 Jan 2017 13:47:48 +0100 Subject: [PATCH 002/219] Changed 'len(validated_transactions) == 0' to 'not validated_transactions' as suggested by @amirelemam in #762 --- bigchaindb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 3c62e65d..8ca25df8 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -455,7 +455,7 @@ class Bigchain(object): Block: created block. """ # Prevent the creation of empty blocks - if len(validated_transactions) == 0: + if not validated_transactions: raise exceptions.OperationError('Empty block creation is not ' 'allowed') From ccdbb91c1ca58e78d07958d2776f2e43a1ab42c2 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 23 Jan 2017 15:16:15 +0100 Subject: [PATCH 003/219] short form 0.9, 0.10 etc tx version with no '.dev' suffix --- bigchaindb/common/transaction.py | 2 +- tests/common/test_transaction.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index bda62663..1bc0b1a5 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -410,7 +410,7 @@ class Transaction(object): TRANSFER = 'TRANSFER' GENESIS = 'GENESIS' ALLOWED_OPERATIONS = (CREATE, TRANSFER, GENESIS) - VERSION = bigchaindb.version.__version__ + VERSION = bigchaindb.version.__short_version__[:-4] # 0.9, 0.10 etc def __init__(self, operation, asset, inputs=None, outputs=None, metadata=None, version=None): diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index b4ef427c..da0fba17 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -966,11 +966,13 @@ def test_cant_add_empty_input(): def test_validate_version(utx): + import re import bigchaindb.version from .utils import validate_transaction_model from bigchaindb.common.exceptions import SchemaValidationError - assert utx.version == bigchaindb.version.__version__ + short_ver = bigchaindb.version.__short_version__ + assert utx.version == re.match(r'^(.*\d)', short_ver).group(1) validate_transaction_model(utx) From 4bb64fa0b83d337e44ba2b961ac9d8966c2fd8c6 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 19 Jan 2017 13:02:58 +0100 Subject: [PATCH 004/219] generalise get_owned_ids to get_outputs and get_owned_ids --- bigchaindb/common/transaction.py | 2 +- bigchaindb/core.py | 27 ++++++++++++++++++++------- tests/common/test_transaction.py | 9 +++++++++ tests/db/test_bigchain_api.py | 13 +++++++++++++ 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index bda62663..563638ce 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -159,7 +159,7 @@ class TransactionLink(object): def __eq__(self, other): # TODO: If `other !== TransactionLink` return `False` - return self.to_dict() == self.to_dict() + return self.to_dict() == other.to_dict() @classmethod def from_dict(cls, link): diff --git a/bigchaindb/core.py b/bigchaindb/core.py index f9c96bed..459c4908 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -373,8 +373,9 @@ class Bigchain(object): else: return None - def get_owned_ids(self, owner): - """Retrieve a list of ``txid`` s that can be used as inputs. + def get_outputs(self, owner): + """Retrieve a list of links to transaction outputs for a given public + key. Args: owner (str): base58 encoded public key. @@ -383,10 +384,9 @@ class Bigchain(object): :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s pointing to another transaction's condition """ - # get all transactions in which owner is in the `owners_after` list response = backend.query.get_owned_ids(self.connection, owner) - owned = [] + links = [] for tx in response: # disregard transactions from invalid blocks @@ -411,10 +411,23 @@ class Bigchain(object): # subfulfillment for `owner` if utils.condition_details_has_owner(output['condition']['details'], owner): tx_link = TransactionLink(tx['id'], index) - # check if input was already spent - if not self.get_spent(tx_link.txid, tx_link.output): - owned.append(tx_link) + links.append(tx_link) + return links + def get_owned_ids(self, owner): + """Retrieve a list of ``txid`` s that can be used as inputs. + + Args: + owner (str): base58 encoded public key. + + Returns: + :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s + pointing to another transaction's condition + """ + owned = [] + for tx_link in self.get_outputs(owner): + if not self.get_spent(tx_link.txid, tx_link.output): + owned.append(tx_link) return owned def get_transactions_filtered(self, asset_id, operation=None): diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index b4ef427c..56f81262 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -436,6 +436,15 @@ def test_cast_transaction_link_to_boolean(): assert bool(TransactionLink(False, False)) is True +def test_transaction_link_eq(): + from bigchaindb.common.transaction import TransactionLink + + assert TransactionLink(1, 2) == TransactionLink(1, 2) + assert TransactionLink(2, 2) != TransactionLink(1, 2) + assert TransactionLink(1, 1) != TransactionLink(1, 2) + assert TransactionLink(2, 1) != TransactionLink(1, 2) + + def test_add_input_to_tx(user_input, asset_definition): from bigchaindb.common.transaction import Transaction diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 1bf028b0..cfc2e93f 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1,6 +1,7 @@ from time import sleep import pytest +from unittest.mock import patch pytestmark = pytest.mark.bdb @@ -1156,3 +1157,15 @@ class TestMultipleInputs(object): # check that the other remain marked as unspent for unspent in transactions[1:]: assert b.get_spent(unspent.id, 0) is None + + +def test_get_owned_ids_calls(): + from bigchaindb.common.transaction import TransactionLink as TL + from bigchaindb.core import Bigchain + with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: + get_outputs.return_value = [TL('a', 1), TL('b', 2)] + with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: + get_spent.side_effect = [True, False] + out = Bigchain().get_owned_ids('abc') + assert get_outputs.called_once_with('abc') + assert out == [TL('b', 2)] From 897ffe81bca5dc5c4856e4f81a67d69e3a2ea091 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 19 Jan 2017 15:49:30 +0100 Subject: [PATCH 005/219] outputs endpoint with unspent filter parameter --- bigchaindb/core.py | 16 +++++++---- bigchaindb/web/routes.py | 4 +-- bigchaindb/web/views/outputs.py | 28 ++++++++++++++++++ bigchaindb/web/views/unspents.py | 23 --------------- tests/db/test_bigchain_api.py | 27 ++++++++++++++++-- tests/web/test_outputs.py | 49 ++++++++++++++++++++++++++++++++ tests/web/test_unspents.py | 24 ---------------- 7 files changed, 114 insertions(+), 57 deletions(-) create mode 100644 bigchaindb/web/views/outputs.py delete mode 100644 bigchaindb/web/views/unspents.py create mode 100644 tests/web/test_outputs.py delete mode 100644 tests/web/test_unspents.py diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 459c4908..871c3707 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -424,11 +424,17 @@ class Bigchain(object): :obj:`list` of TransactionLink: list of ``txid`` s and ``output`` s pointing to another transaction's condition """ - owned = [] - for tx_link in self.get_outputs(owner): - if not self.get_spent(tx_link.txid, tx_link.output): - owned.append(tx_link) - return owned + return self.get_outputs_filtered(owner, include_spent=False) + + def get_outputs_filtered(self, owner, include_spent=True): + """ + Get a list of output links filtered on some criteria + """ + outputs = self.get_outputs(owner) + if not include_spent: + outputs = [o for o in outputs + if not self.get_spent(o.txid, o.output)] + return outputs def get_transactions_filtered(self, asset_id, operation=None): """ diff --git a/bigchaindb/web/routes.py b/bigchaindb/web/routes.py index 18133b3e..b20f8d40 100644 --- a/bigchaindb/web/routes.py +++ b/bigchaindb/web/routes.py @@ -5,7 +5,7 @@ from bigchaindb.web.views import ( info, statuses, transactions as tx, - unspents, + outputs, votes, ) @@ -30,7 +30,7 @@ ROUTES_API_V1 = [ r('statuses/', statuses.StatusApi), r('transactions/', tx.TransactionApi), r('transactions', tx.TransactionListApi), - r('unspents/', unspents.UnspentListApi), + r('outputs/', outputs.OutputListApi), r('votes/', votes.VotesApi), ] diff --git a/bigchaindb/web/views/outputs.py b/bigchaindb/web/views/outputs.py new file mode 100644 index 00000000..735a428f --- /dev/null +++ b/bigchaindb/web/views/outputs.py @@ -0,0 +1,28 @@ +from flask import current_app +from flask_restful import reqparse, Resource + +from bigchaindb.web.views import parameters + + +class OutputListApi(Resource): + def get(self): + """API endpoint to retrieve a list of links to transaction + outputs. + + Returns: + A :obj:`list` of :cls:`str` of links to outputs. + """ + parser = reqparse.RequestParser() + parser.add_argument('public_key', type=parameters.valid_ed25519, + required=True) + parser.add_argument('unspent', type=parameters.valid_bool) + args = parser.parse_args() + + pool = current_app.config['bigchain_pool'] + include_spent = not args['unspent'] + + with pool() as bigchain: + outputs = bigchain.get_outputs_filtered(args['public_key'], + include_spent) + # NOTE: We pass '..' as a path to create a valid relative URI + return [u.to_uri('..') for u in outputs] diff --git a/bigchaindb/web/views/unspents.py b/bigchaindb/web/views/unspents.py deleted file mode 100644 index 8cca995f..00000000 --- a/bigchaindb/web/views/unspents.py +++ /dev/null @@ -1,23 +0,0 @@ -from flask import current_app -from flask_restful import reqparse, Resource - - -class UnspentListApi(Resource): - def get(self): - """API endpoint to retrieve a list of links to transactions's - conditions that have not been used in any previous transaction. - - Returns: - A :obj:`list` of :cls:`str` of links to unfulfilled conditions. - """ - parser = reqparse.RequestParser() - parser.add_argument('public_key', type=str, location='args', - required=True) - args = parser.parse_args() - - pool = current_app.config['bigchain_pool'] - - with pool() as bigchain: - unspents = bigchain.get_owned_ids(args['public_key']) - # NOTE: We pass '..' as a path to create a valid relative URI - return [u.to_uri('..') for u in unspents] diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index cfc2e93f..78c14a28 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1159,13 +1159,34 @@ class TestMultipleInputs(object): assert b.get_spent(unspent.id, 0) is None -def test_get_owned_ids_calls(): +def test_get_owned_ids_calls_get_outputs_filtered(): + from bigchaindb.core import Bigchain + with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: + b = Bigchain() + res = b.get_owned_ids("abc") + gof.assert_called_once_with("abc", include_spent=False) + assert res == gof() + + +def test_get_outputs_filtered_only_unspent(): from bigchaindb.common.transaction import TransactionLink as TL from bigchaindb.core import Bigchain with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: get_outputs.return_value = [TL('a', 1), TL('b', 2)] with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: get_spent.side_effect = [True, False] - out = Bigchain().get_owned_ids('abc') - assert get_outputs.called_once_with('abc') + out = Bigchain().get_outputs_filtered('abc', include_spent=False) + get_outputs.assert_called_once_with('abc') assert out == [TL('b', 2)] + + +def test_get_outputs_filtered(): + from bigchaindb.common.transaction import TransactionLink as TL + from bigchaindb.core import Bigchain + with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: + get_outputs.return_value = [TL('a', 1), TL('b', 2)] + with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: + out = Bigchain().get_outputs_filtered('abc') + get_outputs.assert_called_once_with('abc') + get_spent.assert_not_called() + assert out == get_outputs.return_value diff --git a/tests/web/test_outputs.py b/tests/web/test_outputs.py new file mode 100644 index 00000000..8fb418ea --- /dev/null +++ b/tests/web/test_outputs.py @@ -0,0 +1,49 @@ +import pytest +from unittest.mock import MagicMock, patch + +pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('inputs')] + +UNSPENTS_ENDPOINT = '/api/v1/outputs/' + + +def test_get_outputs_endpoint(client, user_pk): + m = MagicMock() + m.to_uri.side_effect = lambda s: s + with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: + gof.return_value = [m, m] + res = client.get(UNSPENTS_ENDPOINT + '?public_key={}'.format(user_pk)) + assert res.json == ["..", ".."] + assert res.status_code == 200 + gof.assert_called_once_with(user_pk, True) + + +def test_get_outputs_endpoint_unspent(client, user_pk): + m = MagicMock() + m.to_uri.side_effect = lambda s: s + with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: + gof.return_value = [m] + params = '?unspent=true&public_key={}'.format(user_pk) + res = client.get(UNSPENTS_ENDPOINT + params) + assert res.json == [".."] + assert res.status_code == 200 + gof.assert_called_once_with(user_pk, False) + + +def test_get_outputs_endpoint_without_public_key(client): + res = client.get(UNSPENTS_ENDPOINT) + assert res.status_code == 400 + + +def test_get_outputs_endpoint_with_invalid_public_key(client): + expected = {'message': {'public_key': 'Invalid base58 ed25519 key'}} + res = client.get(UNSPENTS_ENDPOINT + '?public_key=abc') + assert expected == res.json + assert res.status_code == 400 + + +def test_get_outputs_endpoint_with_invalid_unspent(client, user_pk): + expected = {'message': {'unspent': 'Boolean value must be "true" or "false" (lowercase)'}} + params = '?unspent=tru&public_key={}'.format(user_pk) + res = client.get(UNSPENTS_ENDPOINT + params) + assert expected == res.json + assert res.status_code == 400 diff --git a/tests/web/test_unspents.py b/tests/web/test_unspents.py deleted file mode 100644 index 9539c664..00000000 --- a/tests/web/test_unspents.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('inputs')] - -UNSPENTS_ENDPOINT = '/api/v1/unspents/' - - -def test_get_unspents_endpoint(b, client, user_pk): - expected = [u.to_uri('..') for u in b.get_owned_ids(user_pk)] - res = client.get(UNSPENTS_ENDPOINT + '?public_key={}'.format(user_pk)) - assert expected == res.json - assert res.status_code == 200 - - -def test_get_unspents_endpoint_without_public_key(client): - res = client.get(UNSPENTS_ENDPOINT) - assert res.status_code == 400 - - -def test_get_unspents_endpoint_with_unused_public_key(client): - expected = [] - res = client.get(UNSPENTS_ENDPOINT + '?public_key=abc') - assert expected == res.json - assert res.status_code == 200 From f12264773c33df20ec55e3f61c6f54f329fa2d96 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 23 Jan 2017 16:49:59 +0100 Subject: [PATCH 006/219] bigchaindb configure now requires a positional backend argument. Created and fixed tests. Updated `b` fixture --- bigchaindb/__init__.py | 22 +++++++++++++------ bigchaindb/commands/bigchain.py | 20 ++++++++++++++--- tests/backend/test_generics.py | 29 +++++++++++++----------- tests/commands/test_commands.py | 39 +++++++++++++++++++++++++++++---- tests/conftest.py | 14 +++++------- tests/test_config_utils.py | 17 ++++++++++++-- 6 files changed, 104 insertions(+), 37 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index c50f4810..315774e5 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,6 +5,20 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), + 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), + 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), +} + +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), + 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), + 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), + 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), +} config = { 'server': { @@ -14,13 +28,7 @@ config = { 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, - 'database': { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - }, + 'database': _database_rethinkdb, 'keypair': { 'public': None, 'private': None, diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 6661e902..2fc8df70 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -86,6 +86,16 @@ def run_configure(args, skip_if_exists=False): conf['keypair']['private'], conf['keypair']['public'] = \ crypto.generate_key_pair() + # select the correct config defaults based on the backend + print('Generating default configuration for backend {}' + .format(args.backend)) + database = {} + if args.backend == 'rethinkdb': + database = bigchaindb._database_rethinkdb + elif args.backend == 'mongodb': + database = bigchaindb._database_mongodb + conf['database'] = database + if not args.yes: for key in ('bind', ): val = conf['server'][key] @@ -282,9 +292,13 @@ def create_parser(): dest='command') # parser for writing a config file - subparsers.add_parser('configure', - help='Prepare the config file ' - 'and create the node keypair') + config_parser = subparsers.add_parser('configure', + help='Prepare the config file ' + 'and create the node keypair') + config_parser.add_argument('backend', + choices=['rethinkdb', 'mongodb'], + help='The backend to use. It can be either ' + 'rethinkdb or mongodb.') # parsers for showing/exporting config values subparsers.add_parser('show-config', diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 2ab33a7c..2049d72b 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -1,4 +1,3 @@ -from importlib import import_module from unittest.mock import patch from pytest import mark, raises @@ -69,10 +68,6 @@ def test_changefeed_class(changefeed_class_func_name, args_qty): changefeed_class_func(None, *range(args_qty)) -@mark.parametrize('db,conn_cls', ( - ('mongodb', 'MongoDBConnection'), - ('rethinkdb', 'RethinkDBConnection'), -)) @patch('bigchaindb.backend.schema.create_indexes', autospec=True, return_value=None) @patch('bigchaindb.backend.schema.create_tables', @@ -80,16 +75,24 @@ def test_changefeed_class(changefeed_class_func_name, args_qty): @patch('bigchaindb.backend.schema.create_database', autospec=True, return_value=None) def test_init_database(mock_create_database, mock_create_tables, - mock_create_indexes, db, conn_cls): + mock_create_indexes): from bigchaindb.backend.schema import init_database - conn = getattr( - import_module('bigchaindb.backend.{}.connection'.format(db)), - conn_cls, - )('host', 'port', 'dbname') + from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + # rethinkdb + conn = RethinkDBConnection('host', 'port', 'dbname') init_database(connection=conn, dbname='mickeymouse') - mock_create_database.assert_called_once_with(conn, 'mickeymouse') - mock_create_tables.assert_called_once_with(conn, 'mickeymouse') - mock_create_indexes.assert_called_once_with(conn, 'mickeymouse') + mock_create_database.assert_called_with(conn, 'mickeymouse') + mock_create_tables.assert_called_with(conn, 'mickeymouse') + mock_create_indexes.assert_called_with(conn, 'mickeymouse') + + # mongodb + conn = MongoDBConnection('host', 'port', 'dbname', replicaset='rs') + init_database(connection=conn, dbname='mickeymouse') + mock_create_database.assert_called_with(conn, 'mickeymouse') + mock_create_tables.assert_called_with(conn, 'mickeymouse') + mock_create_indexes.assert_called_with(conn, 'mickeymouse') @mark.parametrize('admin_func_name,kwargs', ( diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a2e485da..1a1291e3 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -12,7 +12,8 @@ def test_make_sure_we_dont_remove_any_command(): parser = create_parser() - assert parser.parse_args(['configure']).command + assert parser.parse_args(['configure', 'rethinkdb']).command + assert parser.parse_args(['configure', 'mongodb']).command assert parser.parse_args(['show-config']).command assert parser.parse_args(['export-my-pubkey']).command assert parser.parse_args(['init']).command @@ -31,8 +32,8 @@ def test_start_raises_if_command_not_implemented(): with pytest.raises(NotImplementedError): # Will raise because `scope`, the third parameter, - # doesn't contain the function `run_configure` - utils.start(parser, ['configure'], {}) + # doesn't contain the function `run_start` + utils.start(parser, ['start'], {}) def test_start_raises_if_no_arguments_given(): @@ -204,7 +205,7 @@ def test_run_configure_when_config_does_not_exist(monkeypatch, from bigchaindb.commands.bigchain import run_configure monkeypatch.setattr('os.path.exists', lambda path: False) monkeypatch.setattr('builtins.input', lambda: '\n') - args = Namespace(config='foo', yes=True) + args = Namespace(config='foo', backend='rethinkdb', yes=True) return_value = run_configure(args) assert return_value is None @@ -228,6 +229,36 @@ def test_run_configure_when_config_does_exist(monkeypatch, assert value == {} +@pytest.mark.parametrize('backend', ( + 'rethinkdb', + 'mongodb', +)) +def test_run_configure_with_backend(backend, monkeypatch, mock_write_config): + import bigchaindb + from bigchaindb.commands.bigchain import run_configure + + value = {} + + def mock_write_config(new_config, filename=None): + value['return'] = new_config + + monkeypatch.setattr('os.path.exists', lambda path: False) + monkeypatch.setattr('builtins.input', lambda: '\n') + monkeypatch.setattr('bigchaindb.config_utils.write_config', + mock_write_config) + + args = Namespace(config='foo', backend=backend, yes=True) + expected_config = bigchaindb.config + run_configure(args) + + # update the expected config with the correct backend and keypair + backend_conf = getattr(bigchaindb, '_database_' + backend) + expected_config.update({'database': backend_conf, + 'keypair': value['return']['keypair']}) + + assert value['return'] == expected_config + + @patch('bigchaindb.common.crypto.generate_key_pair', return_value=('private_key', 'public_key')) @pytest.mark.usefixtures('ignore_local_config_file') diff --git a/tests/conftest.py b/tests/conftest.py index a69564bc..c3177c14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,26 +109,24 @@ def _restore_dbs(request): @pytest.fixture(scope='session') def _configure_bigchaindb(request): + import bigchaindb from bigchaindb import config_utils test_db_name = TEST_DB_NAME # Put a suffix like _gw0, _gw1 etc on xdist processes xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid') if xdist_suffix: test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) + + backend = request.config.getoption('--database-backend') + backend_conf = getattr(bigchaindb, '_database_' + backend) config = { - 'database': { - 'name': test_db_name, - 'backend': request.config.getoption('--database-backend'), - }, + 'database': backend_conf, 'keypair': { 'private': '31Lb1ZGKTyHnmVK3LUMrAUrPNfd4sE2YyBt3UA4A25aA', 'public': '4XYfCbabAWVUCbjTmRTFEu2sc3dFEdkse4r6X498B1s8', } } - # FIXME - if config['database']['backend'] == 'mongodb': - # not a great way to do this - config['database']['port'] = 27017 + config['database']['name'] = test_db_name config_utils.set_config(config) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 6328d28c..4cb8bd45 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -130,7 +130,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'host': 'test-host', 'port': 4242, 'name': 'test-dbname', - 'replicaset': 'bigchain-rs' }, 'keypair': { 'public': None, @@ -215,7 +214,6 @@ def test_write_config(): ('BIGCHAINDB_DATABASE_HOST', 'test-host', 'host'), ('BIGCHAINDB_DATABASE_PORT', 4242, 'port'), ('BIGCHAINDB_DATABASE_NAME', 'test-db', 'name'), - ('BIGCHAINDB_DATABASE_REPLICASET', 'test-replicaset', 'replicaset') )) def test_database_envs(env_name, env_value, config_key, monkeypatch): import bigchaindb @@ -227,3 +225,18 @@ def test_database_envs(env_name, env_value, config_key, monkeypatch): expected_config['database'][config_key] = env_value assert bigchaindb.config == expected_config + + +def test_database_envs_replicaset(monkeypatch): + # the replica set env is only used if the backend is mongodb + import bigchaindb + + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_REPLICASET': + 'test-replicaset'}) + bigchaindb.config['database'] = bigchaindb._database_mongodb + bigchaindb.config_utils.autoconfigure() + + expected_config = copy.deepcopy(bigchaindb.config) + expected_config['database']['replicaset'] = 'test-replicaset' + + assert bigchaindb.config == expected_config From 927e57beba0a51d78c07313cca50f3b6926ffefa Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 23 Jan 2017 17:29:42 +0100 Subject: [PATCH 007/219] Changed `backend.connect` to handle backend specific kwargs. Fixed tests --- bigchaindb/backend/connection.py | 8 +++++++- bigchaindb/backend/mongodb/connection.py | 2 +- bigchaindb/backend/rethinkdb/connection.py | 2 +- tests/test_config_utils.py | 17 +++++++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 0fda4078..629b6d8b 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -40,6 +40,12 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None): host = host or bigchaindb.config['database']['host'] port = port or bigchaindb.config['database']['port'] dbname = name or bigchaindb.config['database']['name'] + # Not sure how to handle this here. This setting is only relevant for + # mongodb. + # I added **kwargs for both RethinkDBConnection and MongoDBConnection + # to handle these these additional args. In case of RethinkDBConnection + # it just does not do anything with it. + replicaset = replicaset or bigchaindb.config['database'].get('replicaset') try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -51,7 +57,7 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None): raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc logger.debug('Connection: {}'.format(Class)) - return Class(host, port, dbname) + return Class(host, port, dbname, replicaset=replicaset) class Connection: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 19731161..43e92dd0 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): def __init__(self, host=None, port=None, dbname=None, max_tries=3, - replicaset=None): + replicaset=None, **kwargs): """Create a new Connection instance. Args: diff --git a/bigchaindb/backend/rethinkdb/connection.py b/bigchaindb/backend/rethinkdb/connection.py index 173cdc7b..c0506f1f 100644 --- a/bigchaindb/backend/rethinkdb/connection.py +++ b/bigchaindb/backend/rethinkdb/connection.py @@ -17,7 +17,7 @@ class RethinkDBConnection(Connection): more times to run the query or open a connection. """ - def __init__(self, host, port, dbname, max_tries=3): + def __init__(self, host, port, dbname, max_tries=3, **kwargs): """Create a new :class:`~.RethinkDBConnection` instance. See :meth:`.Connection.__init__` for diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 4cb8bd45..c1f63742 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -14,20 +14,28 @@ def clean_config(monkeypatch): monkeypatch.setattr('bigchaindb.config', copy.deepcopy(ORIGINAL_CONFIG)) -def test_bigchain_instance_is_initialized_when_conf_provided(): +def test_bigchain_instance_is_initialized_when_conf_provided(request): + import bigchaindb from bigchaindb import config_utils assert 'CONFIGURED' not in bigchaindb.config config_utils.set_config({'keypair': {'public': 'a', 'private': 'b'}}) assert bigchaindb.config['CONFIGURED'] is True + + # set the current backend so that Bigchain can create a connection + backend = request.config.getoption('--database-backend') + backend_conf = getattr(bigchaindb, '_database_' + backend) + bigchaindb.config['database'] = backend_conf + b = bigchaindb.Bigchain() assert b.me assert b.me_private -def test_bigchain_instance_raises_when_not_configured(monkeypatch): +def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): + import bigchaindb from bigchaindb import config_utils from bigchaindb.common import exceptions assert 'CONFIGURED' not in bigchaindb.config @@ -36,6 +44,11 @@ def test_bigchain_instance_raises_when_not_configured(monkeypatch): # from existing configurations monkeypatch.setattr(config_utils, 'autoconfigure', lambda: 0) + # set the current backend so that Bigchain can create a connection + backend = request.config.getoption('--database-backend') + backend_conf = getattr(bigchaindb, '_database_' + backend) + bigchaindb.config['database'] = backend_conf + with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() From bd048a311590fdb9c7e0b86e7048cde955dd585b Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 23 Jan 2017 17:30:37 +0100 Subject: [PATCH 008/219] add flake8-quotes checker and cleanup double quotes --- bigchaindb/backend/utils.py | 6 ++-- bigchaindb/common/transaction.py | 2 +- bigchaindb/consensus.py | 4 +-- bigchaindb/web/views/info.py | 10 +++---- bigchaindb/web/views/statuses.py | 6 ++-- docs/server/generate_schema_documentation.py | 6 ++-- setup.py | 1 + tests/backend/rethinkdb/test_admin.py | 12 ++++---- tests/common/schema/test_schema.py | 4 +-- tests/web/test_blocks.py | 30 ++++++++++---------- tests/web/test_statuses.py | 18 ++++++------ tests/web/test_transactions.py | 10 +++---- tests/web/test_votes.py | 12 ++++---- 13 files changed, 61 insertions(+), 60 deletions(-) diff --git a/bigchaindb/backend/utils.py b/bigchaindb/backend/utils.py index 23b3e2d9..4a0c4095 100644 --- a/bigchaindb/backend/utils.py +++ b/bigchaindb/backend/utils.py @@ -12,9 +12,9 @@ def module_dispatch_registrar(module): return dispatch_registrar.register(obj_type)(func) except AttributeError as ex: raise ModuleDispatchRegistrationError( - ("`{module}` does not contain a single-dispatchable " - "function named `{func}`. The module being registered " - "was not implemented correctly!").format( + ('`{module}` does not contain a single-dispatchable ' + 'function named `{func}`. The module being registered ' + 'was not implemented correctly!').format( func=func_name, module=module.__name__)) from ex return wrapper return dispatch_wrapper diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index bda62663..e88ecabe 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -118,7 +118,7 @@ class Input(object): fulfillment = Fulfillment.from_uri(data['fulfillment']) except ValueError: # TODO FOR CC: Throw an `InvalidSignature` error in this case. - raise InvalidSignature("Fulfillment URI couldn't been parsed") + raise InvalidSignature('Fulfillment URI couldn\'t been parsed') except TypeError: # NOTE: See comment about this special case in # `Input.to_dict` diff --git a/bigchaindb/consensus.py b/bigchaindb/consensus.py index 9e7a11a1..1cc6e9ec 100644 --- a/bigchaindb/consensus.py +++ b/bigchaindb/consensus.py @@ -39,6 +39,6 @@ class BaseConsensusRules(): except SchemaValidationError as exc: logger.warning(exc) else: - logger.warning("Vote failed signature verification: " - "%s with voters: %s", signed_vote, voters) + logger.warning('Vote failed signature verification: ' + '%s with voters: %s', signed_vote, voters) return False diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index 98e061b6..cbe32acb 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -36,10 +36,10 @@ class ApiV1Index(Resource): '/drivers-clients/http-client-server-api.html', ] return { - "_links": { - "docs": ''.join(docs_url), - "self": api_root, - "statuses": api_root + "statuses/", - "transactions": api_root + "transactions/", + '_links': { + 'docs': ''.join(docs_url), + 'self': api_root, + 'statuses': api_root + 'statuses/', + 'transactions': api_root + 'transactions/', }, } diff --git a/bigchaindb/web/views/statuses.py b/bigchaindb/web/views/statuses.py index 06b768e6..39f880b1 100644 --- a/bigchaindb/web/views/statuses.py +++ b/bigchaindb/web/views/statuses.py @@ -28,7 +28,7 @@ class StatusApi(Resource): # logical xor - exactly one query argument required if bool(tx_id) == bool(block_id): - return make_error(400, "Provide exactly one query parameter. Choices are: block_id, tx_id") + return make_error(400, 'Provide exactly one query parameter. Choices are: block_id, tx_id') pool = current_app.config['bigchain_pool'] status, links = None, None @@ -37,7 +37,7 @@ class StatusApi(Resource): if tx_id: status = bigchain.get_status(tx_id) links = { - "tx": "/transactions/{}".format(tx_id) + 'tx': '/transactions/{}'.format(tx_id) } elif block_id: @@ -56,7 +56,7 @@ class StatusApi(Resource): if links: response.update({ - "_links": links + '_links': links }) return response diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 9b6c1407..c94fe3a9 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -189,7 +189,7 @@ def render_section(section_name, obj): 'type': property_type(prop), }] except Exception as exc: - raise ValueError("Error rendering property: %s" % name, exc) + raise ValueError('Error rendering property: %s' % name, exc) return '\n\n'.join(out + ['']) @@ -201,7 +201,7 @@ def property_description(prop): return property_description(resolve_ref(prop['$ref'])) if 'anyOf' in prop: return property_description(prop['anyOf'][0]) - raise KeyError("description") + raise KeyError('description') def property_type(prop): @@ -214,7 +214,7 @@ def property_type(prop): return ' or '.join(property_type(p) for p in prop['anyOf']) if '$ref' in prop: return property_type(resolve_ref(prop['$ref'])) - raise ValueError("Could not resolve property type") + raise ValueError('Could not resolve property type') DEFINITION_BASE_PATH = '#/definitions/' diff --git a/setup.py b/setup.py index 9dd050bc..f7085218 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ tests_require = [ 'coverage', 'pep8', 'flake8', + 'flake8-quotes==0.8.1', 'pylint', 'pytest>=3.0.0', 'pytest-catchlog>=1.2.2', diff --git a/tests/backend/rethinkdb/test_admin.py b/tests/backend/rethinkdb/test_admin.py index f489f5f5..8c4f0528 100644 --- a/tests/backend/rethinkdb/test_admin.py +++ b/tests/backend/rethinkdb/test_admin.py @@ -57,8 +57,8 @@ def test_set_shards_dry_run(rdb_conn, db_name, db_conn): @pytest.mark.bdb @pytest.mark.skipif( _count_rethinkdb_servers() < 2, - reason=("Requires at least two servers. It's impossible to have" - "more replicas of the data than there are servers.") + reason=('Requires at least two servers. It\'s impossible to have' + 'more replicas of the data than there are servers.') ) def test_set_replicas(rdb_conn, db_name, db_conn): from bigchaindb.backend.schema import TABLES @@ -85,8 +85,8 @@ def test_set_replicas(rdb_conn, db_name, db_conn): @pytest.mark.bdb @pytest.mark.skipif( _count_rethinkdb_servers() < 2, - reason=("Requires at least two servers. It's impossible to have" - "more replicas of the data than there are servers.") + reason=('Requires at least two servers. It\'s impossible to have' + 'more replicas of the data than there are servers.') ) def test_set_replicas_dry_run(rdb_conn, db_name, db_conn): from bigchaindb.backend.schema import TABLES @@ -109,8 +109,8 @@ def test_set_replicas_dry_run(rdb_conn, db_name, db_conn): @pytest.mark.bdb @pytest.mark.skipif( _count_rethinkdb_servers() < 2, - reason=("Requires at least two servers. It's impossible to have" - "more replicas of the data than there are servers.") + reason=('Requires at least two servers. It\'s impossible to have' + 'more replicas of the data than there are servers.') ) def test_reconfigure(rdb_conn, db_name, db_conn): from bigchaindb.backend.rethinkdb.admin import reconfigure diff --git a/tests/common/schema/test_schema.py b/tests/common/schema/test_schema.py index a7cc6891..02a00ee2 100644 --- a/tests/common/schema/test_schema.py +++ b/tests/common/schema/test_schema.py @@ -13,7 +13,7 @@ def _test_additionalproperties(node, path=''): if isinstance(node, dict): if node.get('type') == 'object': assert 'additionalProperties' in node, \ - ("additionalProperties not set at path:" + path) + ('additionalProperties not set at path:' + path) for name, val in node.items(): _test_additionalproperties(val, path + name + '.') @@ -47,7 +47,7 @@ def test_drop_descriptions(): }, 'definitions': { 'wat': { - 'description': "go" + 'description': 'go' } } } diff --git a/tests/web/test_blocks.py b/tests/web/test_blocks.py index b0581061..01c17d71 100644 --- a/tests/web/test_blocks.py +++ b/tests/web/test_blocks.py @@ -41,7 +41,7 @@ def test_get_blocks_by_txid_endpoint(b, client): block_invalid = b.create_block([tx]) b.write_block(block_invalid) - res = client.get(BLOCKS_ENDPOINT + "?tx_id=" + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) # test if block is retrieved as undecided assert res.status_code == 200 assert block_invalid.id in res.json @@ -51,7 +51,7 @@ def test_get_blocks_by_txid_endpoint(b, client): vote = b.vote(block_invalid.id, b.get_last_voted_block().id, False) b.write_vote(vote) - res = client.get(BLOCKS_ENDPOINT + "?tx_id=" + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) # test if block is retrieved as invalid assert res.status_code == 200 assert block_invalid.id in res.json @@ -61,7 +61,7 @@ def test_get_blocks_by_txid_endpoint(b, client): block_valid = b.create_block([tx, tx2]) b.write_block(block_valid) - res = client.get(BLOCKS_ENDPOINT + "?tx_id=" + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) # test if block is retrieved as undecided assert res.status_code == 200 assert block_valid.id in res.json @@ -71,7 +71,7 @@ def test_get_blocks_by_txid_endpoint(b, client): vote = b.vote(block_valid.id, block_invalid.id, True) b.write_vote(vote) - res = client.get(BLOCKS_ENDPOINT + "?tx_id=" + tx.id) + res = client.get(BLOCKS_ENDPOINT + '?tx_id=' + tx.id) # test if block is retrieved as valid assert res.status_code == 200 assert block_valid.id in res.json @@ -96,19 +96,19 @@ def test_get_blocks_by_txid_and_status_endpoint(b, client): block_valid = b.create_block([tx, tx2]) b.write_block(block_valid) - res = client.get("{}?tx_id={}&status={}".format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) + res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) # test if no blocks are retrieved as invalid assert res.status_code == 200 assert len(res.json) == 0 - res = client.get("{}?tx_id={}&status={}".format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) + res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) # test if both blocks are retrieved as undecided assert res.status_code == 200 assert block_valid.id in res.json assert block_invalid.id in res.json assert len(res.json) == 2 - res = client.get("{}?tx_id={}&status={}".format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) + res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) # test if no blocks are retrieved as valid assert res.status_code == 200 assert len(res.json) == 0 @@ -121,18 +121,18 @@ def test_get_blocks_by_txid_and_status_endpoint(b, client): vote = b.vote(block_valid.id, block_invalid.id, True) b.write_vote(vote) - res = client.get("{}?tx_id={}&status={}".format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) + res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_INVALID)) # test if the invalid block is retrieved as invalid assert res.status_code == 200 assert block_invalid.id in res.json assert len(res.json) == 1 - res = client.get("{}?tx_id={}&status={}".format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) + res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_UNDECIDED)) # test if no blocks are retrieved as undecided assert res.status_code == 200 assert len(res.json) == 0 - res = client.get("{}?tx_id={}&status={}".format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) + res = client.get('{}?tx_id={}&status={}'.format(BLOCKS_ENDPOINT, tx.id, Bigchain.BLOCK_VALID)) # test if the valid block is retrieved as valid assert res.status_code == 200 assert block_valid.id in res.json @@ -141,11 +141,11 @@ def test_get_blocks_by_txid_and_status_endpoint(b, client): @pytest.mark.bdb def test_get_blocks_by_txid_endpoint_returns_empty_list_not_found(client): - res = client.get(BLOCKS_ENDPOINT + "?tx_id=") + res = client.get(BLOCKS_ENDPOINT + '?tx_id=') assert res.status_code == 200 assert len(res.json) == 0 - res = client.get(BLOCKS_ENDPOINT + "?tx_id=123") + res = client.get(BLOCKS_ENDPOINT + '?tx_id=123') assert res.status_code == 200 assert len(res.json) == 0 @@ -155,7 +155,7 @@ def test_get_blocks_by_txid_endpoint_returns_400_bad_query_params(client): res = client.get(BLOCKS_ENDPOINT) assert res.status_code == 400 - res = client.get(BLOCKS_ENDPOINT + "?ts_id=123") + res = client.get(BLOCKS_ENDPOINT + '?ts_id=123') assert res.status_code == 400 assert res.json == { 'message': { @@ -163,13 +163,13 @@ def test_get_blocks_by_txid_endpoint_returns_400_bad_query_params(client): } } - res = client.get(BLOCKS_ENDPOINT + "?tx_id=123&foo=123") + res = client.get(BLOCKS_ENDPOINT + '?tx_id=123&foo=123') assert res.status_code == 400 assert res.json == { 'message': 'Unknown arguments: foo' } - res = client.get(BLOCKS_ENDPOINT + "?tx_id=123&status=123") + res = client.get(BLOCKS_ENDPOINT + '?tx_id=123&status=123') assert res.status_code == 400 assert res.json == { 'message': { diff --git a/tests/web/test_statuses.py b/tests/web/test_statuses.py index 4c65bec4..af9d09d3 100644 --- a/tests/web/test_statuses.py +++ b/tests/web/test_statuses.py @@ -10,15 +10,15 @@ STATUSES_ENDPOINT = '/api/v1/statuses' def test_get_transaction_status_endpoint(b, client, user_pk): input_tx = b.get_owned_ids(user_pk).pop() tx, status = b.get_transaction(input_tx.txid, include_status=True) - res = client.get(STATUSES_ENDPOINT + "?tx_id=" + input_tx.txid) + res = client.get(STATUSES_ENDPOINT + '?tx_id=' + input_tx.txid) assert status == res.json['status'] - assert res.json['_links']['tx'] == "/transactions/{}".format(input_tx.txid) + assert res.json['_links']['tx'] == '/transactions/{}'.format(input_tx.txid) assert res.status_code == 200 @pytest.mark.bdb def test_get_transaction_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + "?tx_id=123") + res = client.get(STATUSES_ENDPOINT + '?tx_id=123') assert res.status_code == 404 @@ -32,7 +32,7 @@ def test_get_block_status_endpoint_undecided(b, client): status = b.block_election_status(block.id, block.voters) - res = client.get(STATUSES_ENDPOINT + "?block_id=" + block.id) + res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) assert status == res.json['status'] assert '_links' not in res.json assert res.status_code == 200 @@ -53,7 +53,7 @@ def test_get_block_status_endpoint_valid(b, client): status = b.block_election_status(block.id, block.voters) - res = client.get(STATUSES_ENDPOINT + "?block_id=" + block.id) + res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) assert status == res.json['status'] assert '_links' not in res.json assert res.status_code == 200 @@ -74,7 +74,7 @@ def test_get_block_status_endpoint_invalid(b, client): status = b.block_election_status(block.id, block.voters) - res = client.get(STATUSES_ENDPOINT + "?block_id=" + block.id) + res = client.get(STATUSES_ENDPOINT + '?block_id=' + block.id) assert status == res.json['status'] assert '_links' not in res.json assert res.status_code == 200 @@ -82,7 +82,7 @@ def test_get_block_status_endpoint_invalid(b, client): @pytest.mark.bdb def test_get_block_status_endpoint_returns_404_if_not_found(client): - res = client.get(STATUSES_ENDPOINT + "?block_id=123") + res = client.get(STATUSES_ENDPOINT + '?block_id=123') assert res.status_code == 404 @@ -91,8 +91,8 @@ def test_get_status_endpoint_returns_400_bad_query_params(client): res = client.get(STATUSES_ENDPOINT) assert res.status_code == 400 - res = client.get(STATUSES_ENDPOINT + "?ts_id=123") + res = client.get(STATUSES_ENDPOINT + '?ts_id=123') assert res.status_code == 400 - res = client.get(STATUSES_ENDPOINT + "?tx_id=123&block_id=123") + res = client.get(STATUSES_ENDPOINT + '?tx_id=123&block_id=123') assert res.status_code == 400 diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 9b7fac3f..bc951739 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -53,8 +53,8 @@ def test_post_create_transaction_with_invalid_id(b, client, caplog): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) expected_status_code = 400 expected_error_message = ( - "Invalid transaction ({}): The transaction's id '{}' isn't equal to " - "the hash of its body, i.e. it's not valid." + 'Invalid transaction ({}): The transaction\'s id \'{}\' isn\'t equal to ' + 'the hash of its body, i.e. it\'s not valid.' ).format(InvalidHash.__name__, tx['id']) assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message @@ -74,8 +74,8 @@ def test_post_create_transaction_with_invalid_signature(b, client, caplog): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) expected_status_code = 400 expected_error_message = ( - "Invalid transaction ({}): Fulfillment URI " - "couldn't been parsed" + 'Invalid transaction ({}): Fulfillment URI ' + 'couldn\'t been parsed' ).format(InvalidSignature.__name__) assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message @@ -97,7 +97,7 @@ def test_post_create_transaction_with_invalid_schema(client, caplog): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) expected_status_code = 400 expected_error_message = ( - "Invalid transaction schema: 'version' is a required property") + 'Invalid transaction schema: \'version\' is a required property') assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message assert caplog.records[0].args['status'] == expected_status_code diff --git a/tests/web/test_votes.py b/tests/web/test_votes.py index 0f788fc4..bae31b9a 100644 --- a/tests/web/test_votes.py +++ b/tests/web/test_votes.py @@ -18,7 +18,7 @@ def test_get_votes_endpoint(b, client): vote = b.vote(block.id, b.get_last_voted_block().id, True) b.write_vote(vote) - res = client.get(VOTES_ENDPOINT + "?block_id=" + block.id) + res = client.get(VOTES_ENDPOINT + '?block_id=' + block.id) assert vote == res.json[0] assert len(res.json) == 1 assert res.status_code == 200 @@ -41,18 +41,18 @@ def test_get_votes_endpoint_multiple_votes(b, client): vote_invalid = b.vote(block.id, last_block, False) b.write_vote(vote_invalid) - res = client.get(VOTES_ENDPOINT + "?block_id=" + block.id) + res = client.get(VOTES_ENDPOINT + '?block_id=' + block.id) assert len(res.json) == 2 assert res.status_code == 200 @pytest.mark.bdb def test_get_votes_endpoint_returns_empty_list_not_found(client): - res = client.get(VOTES_ENDPOINT + "?block_id=") + res = client.get(VOTES_ENDPOINT + '?block_id=') assert [] == res.json assert res.status_code == 200 - res = client.get(VOTES_ENDPOINT + "?block_id=123") + res = client.get(VOTES_ENDPOINT + '?block_id=123') assert [] == res.json assert res.status_code == 200 @@ -62,8 +62,8 @@ def test_get_votes_endpoint_returns_400_bad_query_params(client): res = client.get(VOTES_ENDPOINT) assert res.status_code == 400 - res = client.get(VOTES_ENDPOINT + "?ts_id=123") + res = client.get(VOTES_ENDPOINT + '?ts_id=123') assert res.status_code == 400 - res = client.get(VOTES_ENDPOINT + "?tx_id=123&block_id=123") + res = client.get(VOTES_ENDPOINT + '?tx_id=123&block_id=123') assert res.status_code == 400 From fe5d966dcad8d983875edb1c5734fde9ba5c7f5d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 24 Jan 2017 10:22:32 +0100 Subject: [PATCH 009/219] Put back some strings with wrapped single quotes just to double check that flake8-quotes does indeed tolerate it --- bigchaindb/common/transaction.py | 2 +- tests/web/test_transactions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index e88ecabe..bda62663 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -118,7 +118,7 @@ class Input(object): fulfillment = Fulfillment.from_uri(data['fulfillment']) except ValueError: # TODO FOR CC: Throw an `InvalidSignature` error in this case. - raise InvalidSignature('Fulfillment URI couldn\'t been parsed') + raise InvalidSignature("Fulfillment URI couldn't been parsed") except TypeError: # NOTE: See comment about this special case in # `Input.to_dict` diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index bc951739..17f976e6 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -97,7 +97,7 @@ def test_post_create_transaction_with_invalid_schema(client, caplog): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) expected_status_code = 400 expected_error_message = ( - 'Invalid transaction schema: \'version\' is a required property') + "Invalid transaction schema: 'version' is a required property") assert res.status_code == expected_status_code assert res.json['message'] == expected_error_message assert caplog.records[0].args['status'] == expected_status_code From 7207f578793134d0703433af5970290eaa26d2a0 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 24 Jan 2017 10:24:16 +0100 Subject: [PATCH 010/219] Change to single quotes --- bigchaindb/common/transaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index bda62663..5e9216da 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1039,7 +1039,7 @@ class Transaction(object): if tx_body.get('operation') == Transaction.CREATE: if proposed_tx_id != tx_body['asset'].get('id'): - raise InvalidHash("CREATE tx has wrong asset_id") + raise InvalidHash('CREATE tx has wrong asset_id') @classmethod def from_dict(cls, tx): From af23ff5b65a008752a832f824d65dcf7baeb18e2 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 24 Jan 2017 12:08:55 +0100 Subject: [PATCH 011/219] clean up use of double quotes, rename UNSPENTS_ENDPOINT, clarify test --- tests/db/test_bigchain_api.py | 4 ++-- tests/web/test_outputs.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 78c14a28..bd0508de 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1163,8 +1163,8 @@ def test_get_owned_ids_calls_get_outputs_filtered(): from bigchaindb.core import Bigchain with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: b = Bigchain() - res = b.get_owned_ids("abc") - gof.assert_called_once_with("abc", include_spent=False) + res = b.get_owned_ids('abc') + gof.assert_called_once_with('abc', include_spent=False) assert res == gof() diff --git a/tests/web/test_outputs.py b/tests/web/test_outputs.py index 8fb418ea..fd17d46d 100644 --- a/tests/web/test_outputs.py +++ b/tests/web/test_outputs.py @@ -3,40 +3,40 @@ from unittest.mock import MagicMock, patch pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('inputs')] -UNSPENTS_ENDPOINT = '/api/v1/outputs/' +OUTPUTS_ENDPOINT = '/api/v1/outputs/' def test_get_outputs_endpoint(client, user_pk): m = MagicMock() - m.to_uri.side_effect = lambda s: s + m.to_uri.side_effect = lambda s: 'a%sb' % s with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: gof.return_value = [m, m] - res = client.get(UNSPENTS_ENDPOINT + '?public_key={}'.format(user_pk)) - assert res.json == ["..", ".."] + res = client.get(OUTPUTS_ENDPOINT + '?public_key={}'.format(user_pk)) + assert res.json == ['a..b', 'a..b'] assert res.status_code == 200 gof.assert_called_once_with(user_pk, True) def test_get_outputs_endpoint_unspent(client, user_pk): m = MagicMock() - m.to_uri.side_effect = lambda s: s + m.to_uri.side_effect = lambda s: 'a%sb' % s with patch('bigchaindb.core.Bigchain.get_outputs_filtered') as gof: gof.return_value = [m] params = '?unspent=true&public_key={}'.format(user_pk) - res = client.get(UNSPENTS_ENDPOINT + params) - assert res.json == [".."] + res = client.get(OUTPUTS_ENDPOINT + params) + assert res.json == ['a..b'] assert res.status_code == 200 gof.assert_called_once_with(user_pk, False) def test_get_outputs_endpoint_without_public_key(client): - res = client.get(UNSPENTS_ENDPOINT) + res = client.get(OUTPUTS_ENDPOINT) assert res.status_code == 400 def test_get_outputs_endpoint_with_invalid_public_key(client): expected = {'message': {'public_key': 'Invalid base58 ed25519 key'}} - res = client.get(UNSPENTS_ENDPOINT + '?public_key=abc') + res = client.get(OUTPUTS_ENDPOINT + '?public_key=abc') assert expected == res.json assert res.status_code == 400 @@ -44,6 +44,6 @@ def test_get_outputs_endpoint_with_invalid_public_key(client): def test_get_outputs_endpoint_with_invalid_unspent(client, user_pk): expected = {'message': {'unspent': 'Boolean value must be "true" or "false" (lowercase)'}} params = '?unspent=tru&public_key={}'.format(user_pk) - res = client.get(UNSPENTS_ENDPOINT + params) + res = client.get(OUTPUTS_ENDPOINT + params) assert expected == res.json assert res.status_code == 400 From e3317b370bbeecb9177ce142703dccdd9717b95b Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 24 Jan 2017 12:11:21 +0100 Subject: [PATCH 012/219] don't rename TransactionLink to TL --- tests/db/test_bigchain_api.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index bd0508de..8a2040e8 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1169,22 +1169,24 @@ def test_get_owned_ids_calls_get_outputs_filtered(): def test_get_outputs_filtered_only_unspent(): - from bigchaindb.common.transaction import TransactionLink as TL + from bigchaindb.common.transaction import TransactionLink from bigchaindb.core import Bigchain with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: - get_outputs.return_value = [TL('a', 1), TL('b', 2)] + get_outputs.return_value = [TransactionLink('a', 1), + TransactionLink('b', 2)] with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: get_spent.side_effect = [True, False] out = Bigchain().get_outputs_filtered('abc', include_spent=False) get_outputs.assert_called_once_with('abc') - assert out == [TL('b', 2)] + assert out == [TransactionLink('b', 2)] def test_get_outputs_filtered(): - from bigchaindb.common.transaction import TransactionLink as TL + from bigchaindb.common.transaction import TransactionLink from bigchaindb.core import Bigchain with patch('bigchaindb.core.Bigchain.get_outputs') as get_outputs: - get_outputs.return_value = [TL('a', 1), TL('b', 2)] + get_outputs.return_value = [TransactionLink('a', 1), + TransactionLink('b', 2)] with patch('bigchaindb.core.Bigchain.get_spent') as get_spent: out = Bigchain().get_outputs_filtered('abc') get_outputs.assert_called_once_with('abc') From 5683ed5163367cd4bc49fbf7069566f77f762eb4 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 24 Jan 2017 15:59:02 +0100 Subject: [PATCH 013/219] Added mongodb admin commands to add and remove members from the replicaset --- bigchaindb/backend/admin.py | 10 +++++ bigchaindb/backend/mongodb/admin.py | 59 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 bigchaindb/backend/mongodb/admin.py diff --git a/bigchaindb/backend/admin.py b/bigchaindb/backend/admin.py index 057b5995..1a57c8a4 100644 --- a/bigchaindb/backend/admin.py +++ b/bigchaindb/backend/admin.py @@ -20,3 +20,13 @@ def set_shards(connection, *, shards): @singledispatch def set_replicas(connection, *, replicas): raise NotImplementedError + + +@singledispatch +def add_replicas(connection, *, replicas): + raise NotImplementedError + + +@singledispatch +def remove_replicas(connection, *, replicas): + raise NotImplementedError diff --git a/bigchaindb/backend/mongodb/admin.py b/bigchaindb/backend/mongodb/admin.py new file mode 100644 index 00000000..b41021e9 --- /dev/null +++ b/bigchaindb/backend/mongodb/admin.py @@ -0,0 +1,59 @@ +"""Database configuration functions.""" +import logging + +from bigchaindb.backend import admin +from bigchaindb.backend.utils import module_dispatch_registrar +from bigchaindb.backend.mongodb.connection import MongoDBConnection + +logger = logging.getLogger(__name__) + +register_admin = module_dispatch_registrar(admin) + + +@register_admin(MongoDBConnection) +def add_replicas(connection, replicas): + """Add a set of replicas to the replicaset + + Args: + replicas list of strings: of the form "hostname:port". + """ + # get current configuration + conf = connection.conn.admin.command('replSetGetConfig') + + # MongoDB does not automatically add and id for the members so we need + # to chose one that does not exists yet. The safest way is to use + # incrementing ids, so we first check what is the highest id already in + # the set and continue from there. + cur_id = max([member['_id'] for member in conf['config']['members']]) + + # add the nodes to the members list of the replica set + for replica in replicas: + cur_id += 1 + conf['config']['members'].append({'_id': cur_id, 'host': replica}) + + # increase the configuration version number + conf['config']['version'] += 1 + + # apply new configuration + return connection.conn.admin.command('replSetReconfig', conf['config']) + + +@register_admin(MongoDBConnection) +def remove_replicas(connection, replicas): + """Remove a set of replicas from the replicaset + + """ + # get the current configuration + conf = connection.conn.admin.command('replSetGetConfig') + + # remove the nodes from the members list in the replica set + conf['config']['members'] = list( + filter(lambda member: member['host'] not in replicas, + conf['config']['members']) + ) + + # increase the configuration version number + conf['config']['version'] += 1 + + # apply new configuration + return connection.conn.admin.command('replSetReconfig', conf['config']) From 69505a366baf76db045ac9e61f9723b1a95ae714 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 24 Jan 2017 17:55:06 +0100 Subject: [PATCH 014/219] Added bigchaindb commands to add and remove nodes from replicaset --- bigchaindb/backend/admin.py | 4 +- bigchaindb/backend/mongodb/__init__.py | 2 +- bigchaindb/backend/mongodb/admin.py | 13 +++++- bigchaindb/commands/bigchain.py | 55 +++++++++++++++++++++++++- bigchaindb/commands/utils.py | 31 ++++++++++++++- 5 files changed, 98 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/admin.py b/bigchaindb/backend/admin.py index 1a57c8a4..da54397b 100644 --- a/bigchaindb/backend/admin.py +++ b/bigchaindb/backend/admin.py @@ -23,10 +23,10 @@ def set_replicas(connection, *, replicas): @singledispatch -def add_replicas(connection, *, replicas): +def add_replicas(connection, replicas): raise NotImplementedError @singledispatch -def remove_replicas(connection, *, replicas): +def remove_replicas(connection, replicas): raise NotImplementedError diff --git a/bigchaindb/backend/mongodb/__init__.py b/bigchaindb/backend/mongodb/__init__.py index af5293ac..e3746fa3 100644 --- a/bigchaindb/backend/mongodb/__init__.py +++ b/bigchaindb/backend/mongodb/__init__.py @@ -16,7 +16,7 @@ generic backend interfaces to the implementations in this module. """ # Register the single dispatched modules on import. -from bigchaindb.backend.mongodb import schema, query, changefeed # noqa +from bigchaindb.backend.mongodb import admin, schema, query, changefeed # noqa # MongoDBConnection should always be accessed via # ``bigchaindb.backend.connect()``. diff --git a/bigchaindb/backend/mongodb/admin.py b/bigchaindb/backend/mongodb/admin.py index b41021e9..3c2001d5 100644 --- a/bigchaindb/backend/mongodb/admin.py +++ b/bigchaindb/backend/mongodb/admin.py @@ -1,8 +1,11 @@ """Database configuration functions.""" import logging +from pymongo.errors import OperationFailure + from bigchaindb.backend import admin from bigchaindb.backend.utils import module_dispatch_registrar +from bigchaindb.backend.exceptions import DatabaseOpFailedError from bigchaindb.backend.mongodb.connection import MongoDBConnection logger = logging.getLogger(__name__) @@ -35,7 +38,10 @@ def add_replicas(connection, replicas): conf['config']['version'] += 1 # apply new configuration - return connection.conn.admin.command('replSetReconfig', conf['config']) + try: + return connection.conn.admin.command('replSetReconfig', conf['config']) + except OperationFailure as exc: + raise DatabaseOpFailedError(exc.details['errmsg']) @register_admin(MongoDBConnection) @@ -56,4 +62,7 @@ def remove_replicas(connection, replicas): conf['config']['version'] += 1 # apply new configuration - return connection.conn.admin.command('replSetReconfig', conf['config']) + try: + return connection.conn.admin.command('replSetReconfig', conf['config']) + except OperationFailure as exc: + raise DatabaseOpFailedError(exc.details['errmsg']) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 2fc8df70..78b3b745 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -22,7 +22,8 @@ from bigchaindb.models import Transaction from bigchaindb.utils import ProcessGroup from bigchaindb import backend from bigchaindb.backend import schema -from bigchaindb.backend.admin import set_replicas, set_shards +from bigchaindb.backend.admin import (set_replicas, set_shards, add_replicas, + remove_replicas) from bigchaindb.backend.exceptions import DatabaseOpFailedError from bigchaindb.commands import utils from bigchaindb import processes @@ -269,6 +270,32 @@ def run_set_replicas(args): logger.warn(e) +def run_add_replicas(args): + # Note: This command is specific to MongoDB + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + conn = backend.connect() + + try: + add_replicas(conn, args.replicas) + except DatabaseOpFailedError as e: + logger.warn(e) + else: + logger.info('Added {} to the replicaset.'.format(args.replicas)) + + +def run_remove_replicas(args): + # Note: This command is specific to MongoDB + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + conn = backend.connect() + + try: + remove_replicas(conn, args.replicas) + except DatabaseOpFailedError as e: + logger.warn(e) + else: + logger.info('Removed {} from the replicaset.'.format(args.replicas)) + + def create_parser(): parser = argparse.ArgumentParser( description='Control your BigchainDB node.', @@ -334,6 +361,32 @@ def create_parser(): type=int, default=1, help='Number of replicas (i.e. the replication factor)') + # parser for adding nodes to the replica set + add_replicas_parser = subparsers.add_parser('add-replicas', + help='Add a set of nodes to the ' + 'replica set. This command ' + 'is specific to the MongoDB' + ' backend.') + + add_replicas_parser.add_argument('replicas', nargs='+', + type=utils.mongodb_host, + help='A list of space separated hosts to ' + 'add to the replicaset. Each host ' + 'should be in the form `host:port`.') + + # parser for removing nodes from the replica set + rm_replicas_parser = subparsers.add_parser('remove-replicas', + help='Remove a set of nodes from the ' + 'replica set. This command ' + 'is specific to the MongoDB' + ' backend.') + + rm_replicas_parser.add_argument('replicas', nargs='+', + type=utils.mongodb_host, + help='A list of space separated hosts to ' + 'remove from the replicaset. Each host ' + 'should be in the form `host:port`.') + load_parser = subparsers.add_parser('load', help='Write transactions to the backlog') diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 510eb2f6..7b662308 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -3,14 +3,15 @@ for ``argparse.ArgumentParser``. """ import argparse -from bigchaindb.common.exceptions import StartupError import multiprocessing as mp import subprocess import rethinkdb as r +from pymongo import uri_parser import bigchaindb from bigchaindb import backend +from bigchaindb.common.exceptions import StartupError from bigchaindb.version import __version__ @@ -95,6 +96,34 @@ def start(parser, argv, scope): return func(args) +def mongodb_host(host): + """Utility function that works as a type for mongodb ``host`` args. + + This function validates the ``host`` args provided by to the + ``add-replicas`` and ``remove-replicas`` commands and checks if each arg + is in the form "host:port" + + Args: + host (str): A string containing hostname and port (e.g. "host:port") + + Raises: + ArgumentTypeError: if it fails to parse the argument + """ + # check if mongodb can parse the host + try: + hostname, port = uri_parser.parse_host(host, default_port=None) + except ValueError as exc: + raise argparse.ArgumentTypeError(exc.args[0]) + + # we do require the port to be provided. + if port is None: + raise argparse.ArgumentTypeError('expected host in the form ' + '`host:port`. Got `{}` instead.' + .format(host)) + + return host + + base_parser = argparse.ArgumentParser(add_help=False, prog='bigchaindb') base_parser.add_argument('-c', '--config', From f15a7f7e8b593c39c40922e9d90b4fa265da90dc Mon Sep 17 00:00:00 2001 From: tim Date: Tue, 15 Nov 2016 14:43:30 +0100 Subject: [PATCH 015/219] Document conditions endpoint --- .../http-client-server-api.rst | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 969d912b..2d8fe0fd 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -123,6 +123,55 @@ GET /transactions/{tx_id} :statuscode 404: A transaction with that ID was not found. +GET /transactions/{tx_id}/conditions/{cid} +------------------------- + +.. http:get:: /transactions/{tx_id}/conditions/{cid} + + Returns the condition with index ``cid`` from a transaction with ID + ``txid``. + + If either a transaction with ID ``txid`` isn't found or the condition + requested at the index ``cid`` is not found, this endpoint will return a + ``400 Bad Request``. + + :param tx_id: transaction ID + :type tx_id: hex string + + :param cid: A condition's index in the transaction + :type cid: integer + + **Example request**: + + .. sourcecode:: http + + GET /transactions/2d431...0b4b0e/conditions/0 HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + } + } + } + + :statuscode 200: A condition with ``cid`` was found in a transaction with ID ``tx_id``. + :statuscode 400: Either a transaction with ``tx_id`` or a condition with ``cid`` wasn't found. + + GET /unspents/ ------------------------- From 1086c3a5c4b46e8576b45849900fc373cbdf406a Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 12:01:57 +0100 Subject: [PATCH 016/219] POST /transactions status code to 202 According to https://www.ietf.org/rfc/rfc2616.txt a 201 Created status code MUST only be returned when: "The origin server MUST create the resource before returning the 201 status code." hence, a 202 Accepted's definition is more appropriate: "The request has been accepted for processing, but the processing has not been completed. The entity returned with this response SHOULD include an indication of the request's current status and either a pointer to a status monitor or some estimate of when the user can expect the request to be fulfilled." --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 2d8fe0fd..590d788d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -64,7 +64,7 @@ POST /transactions/ .. literalinclude:: samples/post-tx-response.http :language: http - :statuscode 201: A new transaction was created. + :statuscode 202: The pushed transaction was accepted, but the processing has not been completed. :statuscode 400: The transaction was invalid and not created. From 71d3c70fdae1a471a9ea43ecefb351b54146dff7 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 12:08:06 +0100 Subject: [PATCH 017/219] Status --> Statuses Usage of singular resource names is discouraged in REST: http://www.vinaysahni.com/best-practices-for-a-pragmatic-restful-api The plural of status is statuses: http://english.stackexchange.com/questions/877/what-is-the-plural-form-of-status --- .../source/drivers-clients/http-client-server-api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 590d788d..5affaa44 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -68,12 +68,12 @@ POST /transactions/ :statuscode 400: The transaction was invalid and not created. -GET /transactions/{tx_id}/status +GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e -------------------------------- -.. http:get:: /transactions/{tx_id}/status +.. http:get:: /statuses/{tx_id}/ - Get the status of the transaction with the ID ``tx_id``, if a transaction + Get the status of a transaction with the ID ``tx_id``, if a transaction with that ``tx_id`` exists. The possible status values are ``backlog``, ``undecided``, ``valid`` or From 5789a37664746bd634fb8e5cc1fa3f716fdc7527 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 12:16:27 +0100 Subject: [PATCH 018/219] Allow /statuses to return a 303 See Other response. According to: https://www.ietf.org/rfc/rfc2616.txt a 303 See Other can be returned to indicate that the resource the user is looking for can be found under a new path. In the case of a transaction including the `status == 'valid'`, we return 303 See Other, as well as a Location header to the /transactions endpoint. "The response to the request can be found under a different URI and SHOULD be retrieved using a GET method on that resource." --- .../source/drivers-clients/http-client-server-api.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 5affaa44..d39eedfa 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -71,7 +71,7 @@ POST /transactions/ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e -------------------------------- -.. http:get:: /statuses/{tx_id}/ +.. http:get:: /statuses/{tx_id} Get the status of a transaction with the ID ``tx_id``, if a transaction with that ``tx_id`` exists. @@ -79,6 +79,10 @@ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. + If a transaction is persisted to the chain and it's status is set to + ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, as + well as a URL to the resource in the location header. + :param tx_id: transaction ID :type tx_id: hex string @@ -92,7 +96,8 @@ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e .. literalinclude:: samples/get-tx-status-response.http :language: http - :statuscode 200: A transaction with that ID was found and the status is returned. + :statuscode 200: A transaction with that ID was found. The status is either ``backlog``, ``invalid``. + :statuscode 303: A transaction with that ID was found and persisted to the chain. A location header to the resource is provided. :statuscode 404: A transaction with that ID was not found. From b4889973535c10584e766fcd0f860ba4174f8554 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 12:23:54 +0100 Subject: [PATCH 019/219] tx_id --> txid --- .../http-client-server-api.rst | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index d39eedfa..3f100463 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -71,10 +71,10 @@ POST /transactions/ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e -------------------------------- -.. http:get:: /statuses/{tx_id} +.. http:get:: /statuses/{txid} - Get the status of a transaction with the ID ``tx_id``, if a transaction - with that ``tx_id`` exists. + Get the status of a transaction with the ID ``txid``, if a transaction + with that ``txid`` exists. The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. @@ -83,8 +83,8 @@ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, as well as a URL to the resource in the location header. - :param tx_id: transaction ID - :type tx_id: hex string + :param txid: transaction ID + :type txid: hex string **Example request**: @@ -101,18 +101,18 @@ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e :statuscode 404: A transaction with that ID was not found. -GET /transactions/{tx_id} +GET /transactions/{txid} ------------------------- -.. http:get:: /transactions/{tx_id} +.. http:get:: /transactions/{txid} - Get the transaction with the ID ``tx_id``. + Get the transaction with the ID ``txid``. This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` block on ``bigchain``, if exists. - :param tx_id: transaction ID - :type tx_id: hex string + :param txid: transaction ID + :type txid: hex string **Example request**: @@ -128,10 +128,10 @@ GET /transactions/{tx_id} :statuscode 404: A transaction with that ID was not found. -GET /transactions/{tx_id}/conditions/{cid} +GET /transactions/{txid}/conditions/{cid} ------------------------- -.. http:get:: /transactions/{tx_id}/conditions/{cid} +.. http:get:: /transactions/{txid}/conditions/{cid} Returns the condition with index ``cid`` from a transaction with ID ``txid``. @@ -140,8 +140,8 @@ GET /transactions/{tx_id}/conditions/{cid} requested at the index ``cid`` is not found, this endpoint will return a ``400 Bad Request``. - :param tx_id: transaction ID - :type tx_id: hex string + :param txid: transaction ID + :type txid: hex string :param cid: A condition's index in the transaction :type cid: integer @@ -173,8 +173,8 @@ GET /transactions/{tx_id}/conditions/{cid} } } - :statuscode 200: A condition with ``cid`` was found in a transaction with ID ``tx_id``. - :statuscode 400: Either a transaction with ``tx_id`` or a condition with ``cid`` wasn't found. + :statuscode 200: A condition with ``cid`` was found in a transaction with ID ``txid``. + :statuscode 400: Either a transaction with ``txid`` or a condition with ``cid`` wasn't found. GET /unspents/ From 0dc9b46ea735fd1e56b05a3e215e9940a6069f56 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 13:20:54 +0100 Subject: [PATCH 020/219] Structural changes to the document - Remove /unspents/ and replace with endpoint under /transactions - Remove /transactions/txid/conditions/cid endpoint --- .../http-client-server-api.rst | 202 ++++++------------ 1 file changed, 69 insertions(+), 133 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 3f100463..d5a7a61f 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -101,6 +101,75 @@ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e :statuscode 404: A transaction with that ID was not found. +GET /transactions +------------------------- + +.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_after={owner_after} + + Get a list of transactions with unfulfilled conditions (conditions that have + not been used yet in a persisted transaction. + + If the querystring ``fulfilled`` is set to ``false`` and all conditions for + ``owner_after`` happen to be fulfilled already, this endpoint will return + an empty list. + + + .. note:: + + This endpoint will return a ``HTTP 400 Bad Request`` if the querystring + ``owner_after`` happens to not be defined in the request. + + :param fields: The fields to be included in a transaction. + :type fields: string + + :param fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. + :type fields: boolean + + :param owner_after: A public key, able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :type owner_after: base58 encoded string + + **Example request**: + + .. sourcecode:: http + + GET /transactions?fields=id,conditions&fulfilled=false&owner_after=1AAAbbb...ccc HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [{ + "transaction": { + "conditions": [ + { + "cid": 0, + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "1AAAbbb...ccc" + } + }, + "amount": 1, + "owners_after": [ + "1AAAbbb...ccc" + ] + } + ], + "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + }, ...] + + + :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. the ``owner_after`` querystring was not included in the request. + GET /transactions/{txid} ------------------------- @@ -126,136 +195,3 @@ GET /transactions/{txid} :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. - - -GET /transactions/{txid}/conditions/{cid} -------------------------- - -.. http:get:: /transactions/{txid}/conditions/{cid} - - Returns the condition with index ``cid`` from a transaction with ID - ``txid``. - - If either a transaction with ID ``txid`` isn't found or the condition - requested at the index ``cid`` is not found, this endpoint will return a - ``400 Bad Request``. - - :param txid: transaction ID - :type txid: hex string - - :param cid: A condition's index in the transaction - :type cid: integer - - **Example request**: - - .. sourcecode:: http - - GET /transactions/2d431...0b4b0e/conditions/0 HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - } - } - - :statuscode 200: A condition with ``cid`` was found in a transaction with ID ``txid``. - :statuscode 400: Either a transaction with ``txid`` or a condition with ``cid`` wasn't found. - - -GET /unspents/ -------------------------- - -.. note:: - - This endpoint (unspents) is not yet implemented. We published it here for preview and comment. - - -.. http:get:: /unspents?owner_after={owner_after} - - Get a list of links to transactions' outputs that have not been used in - a previous transaction and could hence be called unspent outputs - (or simply: unspents). - - This endpoint will return a ``HTTP 400 Bad Request`` if the querystring - ``owner_after`` happens to not be defined in the request. - - Note that if unspents for a certain ``public_key`` have not been found by - the server, this will result in the server returning a 200 OK HTTP status - code and an empty list in the response's body. - - :param owner_after: A public key, able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :type owner_after: base58 encoded string - - **Example request**: - - .. sourcecode:: http - - GET /unspents?owner_after=1AAAbbb...ccc HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - [ - "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/0", - "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/1" - ] - - :statuscode 200: A list of outputs were found and returned in the body of the response. - :statuscode 400: The request wasn't understood by the server, e.g. the ``owner_after`` querystring was not included in the request. - - -Determining the API Root URL ----------------------------- - -When you start BigchainDB Server using ``bigchaindb start``, -an HTTP API is exposed at some address. The default is: - -`http://localhost:9984/api/v1/ `_ - -It's bound to ``localhost``, -so you can access it from the same machine, -but it won't be directly accessible from the outside world. -(The outside world could connect via a SOCKS proxy or whatnot.) - -The documentation about BigchainDB Server :any:`Configuration Settings` -has a section about how to set ``server.bind`` so as to make -the HTTP API publicly accessible. - -If the API endpoint is publicly accessible, -then the public API Root URL is determined as follows: - -- The public IP address (like 12.34.56.78) - is the public IP address of the machine exposing - the HTTP API to the public internet (e.g. either the machine hosting - Gunicorn or the machine running the reverse proxy such as Nginx). - It's determined by AWS, Azure, Rackspace, or whoever is hosting the machine. - -- The DNS hostname (like apihosting4u.net) is determined by DNS records, - such as an "A Record" associating apihosting4u.net with 12.34.56.78 - -- The port (like 9984) is determined by the ``server.bind`` setting - if Gunicorn is exposed directly to the public Internet. - If a reverse proxy (like Nginx) is exposed directly to the public Internet - instead, then it could expose the HTTP API on whatever port it wants to. - (It should expose the HTTP API on port 9984, but it's not bound to do - that by anything other than convention.) From e243a1be9b556eda9d8cec69f956b23c40d430ce Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 13:47:51 +0100 Subject: [PATCH 021/219] Use sphinx note for note in document --- .../drivers-clients/http-client-server-api.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index d5a7a61f..b82e0169 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -47,12 +47,13 @@ POST /transactions/ Push a new transaction. - Note: The posted transaction should be a valid and signed :doc:`transaction <../data-models/transaction-model>`. - The steps to build a valid transaction are beyond the scope of this page. - One would normally use a driver such as the `BigchainDB Python Driver - `_ to - build a valid transaction. The exact contents of a valid transaction depend - on the associated public/private keypairs. + .. note:: + The posted transaction should be valid `transaction + `_. + The steps to build a valid transaction are beyond the scope of this page. + One would normally use a driver such as the `BigchainDB Python Driver + `_ + to build a valid transaction. **Example request**: From 85d9553a1e6f0ee6c9cdbe59e15a5e16236e0e43 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 13:48:14 +0100 Subject: [PATCH 022/219] owner_after --> owners_after Querystring keywords should be in line with data model. --- .../http-client-server-api.rst | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index b82e0169..9fd4fdca 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -69,7 +69,7 @@ POST /transactions/ :statuscode 400: The transaction was invalid and not created. -GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e +GET /statuses -------------------------------- .. http:get:: /statuses/{txid} @@ -105,35 +105,32 @@ GET /statuses/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e GET /transactions ------------------------- -.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_after={owner_after} +.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_afters={owners_after} Get a list of transactions with unfulfilled conditions (conditions that have not been used yet in a persisted transaction. If the querystring ``fulfilled`` is set to ``false`` and all conditions for - ``owner_after`` happen to be fulfilled already, this endpoint will return + ``owners_after`` happen to be fulfilled already, this endpoint will return an empty list. - - .. note:: - - This endpoint will return a ``HTTP 400 Bad Request`` if the querystring - ``owner_after`` happens to not be defined in the request. + This endpoint will return a ``HTTP 400 Bad Request`` if the querystring + ``owners_after`` happens to not be defined in the request. :param fields: The fields to be included in a transaction. :type fields: string :param fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :type fields: boolean + :type fulfilled: boolean - :param owner_after: A public key, able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :type owner_after: base58 encoded string + :param owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :type owners_after: base58 encoded string **Example request**: .. sourcecode:: http - GET /transactions?fields=id,conditions&fulfilled=false&owner_after=1AAAbbb...ccc HTTP/1.1 + GET /transactions?fields=id,conditions&fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 Host: example.com **Example response**: @@ -165,11 +162,10 @@ GET /transactions } ], "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - }, ...] - + }] :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. the ``owner_after`` querystring was not included in the request. + :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. GET /transactions/{txid} ------------------------- From 9766332b8bb58eacab89b789c58ff10cca9b043c Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 14:05:54 +0100 Subject: [PATCH 023/219] Restructure doc: Endpoints as roots --- .../http-client-server-api.rst | 118 +++++++++++++----- 1 file changed, 84 insertions(+), 34 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 9fd4fdca..a5d4ab30 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -39,10 +39,86 @@ with something like the following in the body: "version": "0.6.0" } - -POST /transactions/ +Transactions ------------------- +.. http:get:: /transactions/{txid} + + Get the transaction with the ID ``txid``. + + This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` + block on ``bigchain``, if exists. + + :param txid: transaction ID + :type txid: hex string + + **Example request**: + + .. sourcecode:: http + + GET /transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "transaction": { + "conditions": [ + { + "cid": 0, + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + } + }, + "amount": 1, + "owners_after": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ], + "operation": "CREATE", + "asset": { + "divisible": false, + "updatable": false, + "data": null, + "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", + "refillable": false + }, + "metadata": null, + "timestamp": "1477578978", + "fulfillments": [ + { + "fid": 0, + "input": null, + "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "owners_before": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ] + }, + "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "version": 1 + } + + :statuscode 200: A transaction with that ID was found. + :statuscode 404: A transaction with that ID was not found. + +.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_afters={owners_after} + + is an `alias for retrieving unfulfilled conditions for a set of public keys. <#get--conditions?fulfilled=false&owner_afters=owners_after>`_ + .. http:post:: /transactions/ Push a new transaction. @@ -69,7 +145,7 @@ POST /transactions/ :statuscode 400: The transaction was invalid and not created. -GET /statuses +Statuses -------------------------------- .. http:get:: /statuses/{txid} @@ -102,10 +178,10 @@ GET /statuses :statuscode 404: A transaction with that ID was not found. -GET /transactions +Conditions ------------------------- -.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_afters={owners_after} +.. http:get:: /conditions?fulfilled=false&owner_afters={owners_after} Get a list of transactions with unfulfilled conditions (conditions that have not been used yet in a persisted transaction. @@ -117,8 +193,8 @@ GET /transactions This endpoint will return a ``HTTP 400 Bad Request`` if the querystring ``owners_after`` happens to not be defined in the request. - :param fields: The fields to be included in a transaction. - :type fields: string + This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` + block on ``bigchain``, if exists. :param fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. :type fulfilled: boolean @@ -130,7 +206,7 @@ GET /transactions .. sourcecode:: http - GET /transactions?fields=id,conditions&fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 + GET /conditions?fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 Host: example.com **Example response**: @@ -166,29 +242,3 @@ GET /transactions :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. - -GET /transactions/{txid} -------------------------- - -.. http:get:: /transactions/{txid} - - Get the transaction with the ID ``txid``. - - This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` - block on ``bigchain``, if exists. - - :param txid: transaction ID - :type txid: hex string - - **Example request**: - - .. literalinclude:: samples/get-tx-request.http - :language: http - - **Example response**: - - .. literalinclude:: samples/get-tx-response.http - :language: http - - :statuscode 200: A transaction with that ID was found. - :statuscode 404: A transaction with that ID was not found. From 156bf4fb2144e81956d60709aa5653f460630444 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 14:26:30 +0100 Subject: [PATCH 024/219] txid --> id KISS: A transaction is a resource as every other. Let's not give it a special id (like 'txid'), but simply a regular id. --- .../drivers-clients/http-client-server-api.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index a5d4ab30..323a0889 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -42,15 +42,15 @@ with something like the following in the body: Transactions ------------------- -.. http:get:: /transactions/{txid} +.. http:get:: /transactions/{id} - Get the transaction with the ID ``txid``. + Get the transaction with the ID ``id``. This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` block on ``bigchain``, if exists. - :param txid: transaction ID - :type txid: hex string + :param id: transaction ID + :type id: hex string **Example request**: @@ -148,10 +148,10 @@ Transactions Statuses -------------------------------- -.. http:get:: /statuses/{txid} +.. http:get:: /statuses/{id} - Get the status of a transaction with the ID ``txid``, if a transaction - with that ``txid`` exists. + Get the status of a transaction with the ID ``id``, if a transaction + with that ``id`` exists. The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. @@ -160,8 +160,8 @@ Statuses ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, as well as a URL to the resource in the location header. - :param txid: transaction ID - :type txid: hex string + :param id: transaction ID + :type id: hex string **Example request**: From 90aff0e202f6c609a8ab4cdc8ba2f0d12ffa802c Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 14:45:29 +0100 Subject: [PATCH 025/219] Give up /conditions endpoint A transaction contains: - conditions - fulfillments - assets - meta data While: - assets; and - meta data could be viewed as their own "tables" or resources, conditions and fulfillments cannot. Why? Because in comparison they do not contain a primary key, allowing them to be queried by it. --- .../http-client-server-api.rst | 129 +++++++++--------- 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 323a0889..3bfddb4a 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -117,7 +117,68 @@ Transactions .. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_afters={owners_after} - is an `alias for retrieving unfulfilled conditions for a set of public keys. <#get--conditions?fulfilled=false&owner_afters=owners_after>`_ + Get a list of transactions with unfulfilled conditions. + + If the querystring ``fulfilled`` is set to ``false`` and all conditions for + ``owners_after`` happen to be fulfilled already, this endpoint will return + an empty list. + + This endpoint will return a ``HTTP 400 Bad Request`` if the querystring + ``owners_after`` happens to not be defined in the request. + + This endpoint returns conditions only if the transaction they're in are + included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + + :param fields: A comma separated string to expand properties on the transaction object to be returned. + :type fields: string + + :param fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. + :type fulfilled: boolean + + :param owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :type owners_after: base58 encoded string + + **Example request**: + + .. sourcecode:: http + + GET /transactions?fields=id,conditions&fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [{ + "transaction": { + "conditions": [ + { + "cid": 0, + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "1AAAbbb...ccc" + } + }, + "amount": 1, + "owners_after": [ + "1AAAbbb...ccc" + ] + } + ], + "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + }] + + :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. + .. http:post:: /transactions/ @@ -176,69 +237,3 @@ Statuses :statuscode 200: A transaction with that ID was found. The status is either ``backlog``, ``invalid``. :statuscode 303: A transaction with that ID was found and persisted to the chain. A location header to the resource is provided. :statuscode 404: A transaction with that ID was not found. - - -Conditions -------------------------- - -.. http:get:: /conditions?fulfilled=false&owner_afters={owners_after} - - Get a list of transactions with unfulfilled conditions (conditions that have - not been used yet in a persisted transaction. - - If the querystring ``fulfilled`` is set to ``false`` and all conditions for - ``owners_after`` happen to be fulfilled already, this endpoint will return - an empty list. - - This endpoint will return a ``HTTP 400 Bad Request`` if the querystring - ``owners_after`` happens to not be defined in the request. - - This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` - block on ``bigchain``, if exists. - - :param fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :type fulfilled: boolean - - :param owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :type owners_after: base58 encoded string - - **Example request**: - - .. sourcecode:: http - - GET /conditions?fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - [{ - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "1AAAbbb...ccc" - } - }, - "amount": 1, - "owners_after": [ - "1AAAbbb...ccc" - ] - } - ], - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - }] - - :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. From 58d0b771cb66663e21ee6d1c0ac094a6bfcb5b92 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 14:58:44 +0100 Subject: [PATCH 026/219] id --> resource_id Inevitably, some resources will not allow to filter by the exact keyword that is included in a resources body. Take for example asset and metadata. They both have a property called 'id', hence requests of a form: /transactions&fields=x,y&property_name=z might now be allowed to be resolved as the keyword 'id' in this case could reference both 'metadata.id' and 'asset.id'. This problem cannot be structurally resolved with URL paths. Hence it was decided to emphasize on a few resources that implement 'id' as a sort-of primary key. --- .../drivers-clients/http-client-server-api.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 3bfddb4a..c403d1ec 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -42,15 +42,15 @@ with something like the following in the body: Transactions ------------------- -.. http:get:: /transactions/{id} +.. http:get:: /transactions/{tx_id} - Get the transaction with the ID ``id``. + Get the transaction with the ID ``tx_id``. This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` block on ``bigchain``, if exists. - :param id: transaction ID - :type id: hex string + :param tx_id: transaction ID + :type tx_id: hex string **Example request**: @@ -209,10 +209,10 @@ Transactions Statuses -------------------------------- -.. http:get:: /statuses/{id} +.. http:get:: /statuses/{tx_id} - Get the status of a transaction with the ID ``id``, if a transaction - with that ``id`` exists. + Get the status of a transaction with the ID ``tx_id``, if a transaction + with that ``tx_id`` exists. The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. @@ -221,8 +221,8 @@ Statuses ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, as well as a URL to the resource in the location header. - :param id: transaction ID - :type id: hex string + :param tx_id: transaction ID + :type tx_id: hex string **Example request**: From 748e15537856df93179a6555a1d31f24674d3cef Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 15:41:26 +0100 Subject: [PATCH 027/219] Get block status using /statuses --- .../http-client-server-api.rst | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index c403d1ec..69b8b7a3 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -209,21 +209,25 @@ Transactions Statuses -------------------------------- -.. http:get:: /statuses/{tx_id} +.. http:get:: /statuses/{tx_id | block_id} - Get the status of a transaction with the ID ``tx_id``, if a transaction - with that ``tx_id`` exists. + Get the status of an asynchronously written resource by their id. + Supports the retrieval of a status for a transaction using ``tx_id`` or the + retrieval of a status for a block using ``block_id``. The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. - If a transaction is persisted to the chain and it's status is set to + If a transaction or block is persisted to the chain and it's status is set to ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, as - well as a URL to the resource in the location header. + well as an URL to the resource in the location header. :param tx_id: transaction ID :type tx_id: hex string + :param block_id: block ID + :type block_id: hex string + **Example request**: .. literalinclude:: samples/get-tx-status-request.http @@ -234,6 +238,6 @@ Statuses .. literalinclude:: samples/get-tx-status-response.http :language: http - :statuscode 200: A transaction with that ID was found. The status is either ``backlog``, ``invalid``. - :statuscode 303: A transaction with that ID was found and persisted to the chain. A location header to the resource is provided. - :statuscode 404: A transaction with that ID was not found. + :statuscode 200: A transaction or block with that ID was found. The status is either ``backlog``, ``invalid``. + :statuscode 303: A transaction or block with that ID was found and persisted to the chain. A location header to the resource is provided. + :statuscode 404: A transaction or block with that ID was not found. From ee904d78f4df97e46c3aba5bc9fe87be2ade57c5 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:01:16 +0100 Subject: [PATCH 028/219] Add transactions by asset id endpoint --- .../http-client-server-api.rst | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 69b8b7a3..abdff476 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -179,8 +179,58 @@ Transactions :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. +.. http:get:: /transactions?fields=id,asset,operation&operation={CREATE|TRANSFER}&asset_id={asset_id} -.. http:post:: /transactions/ + Get a list of transactions that use an asset with the ID ``asset_id``. + + This endpoint will return a ``HTTP 400 Bad Request`` if the querystring + ``asset_id`` happens to not be defined in the request. + + ``operation`` can either be ``GENESIS``, ``CREATE`` or ``TRANSFER``. + + This endpoint returns assets only if the transaction they're in are + included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + + :param fields: A comma separated string to expand properties on the transaction object to be returned. + :type fields: string + + :param operation: One of the three supported operations of a transaction. + :type operation: string + + :param asset_id: asset ID. + :type asset_id: uuidv4 + + **Example request**: + + .. sourcecode:: http + + GET /transactions?fields=id,asset,operation&operation=CREATE&asset_id=1AAAbbb...ccc HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [{ + "transaction": { + "asset": { + "divisible": false, + "updatable": false, + "data": null, + "id": "1AAAbbb...ccc", + "refillable": false + }, + "operation": "CREATE", + "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + }] + + :statuscode 200: A list of transaction's containing an asset with ID ``asset_id`` was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. + +.. http:post:: /transactions Push a new transaction. From 0a06547c1b34a6e50dd7162656f0eab2c2879367 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:19:53 +0100 Subject: [PATCH 029/219] Add transactions by metadata id endpoint --- .../http-client-server-api.rst | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index abdff476..eb906f27 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -230,6 +230,49 @@ Transactions :statuscode 200: A list of transaction's containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. +.. http:get:: /transactions?fields=id,metadata&metadata_id={metadata_id} + + Get a list of transactions that use metadata with the ID ``metadata_id``. + + This endpoint will return a ``HTTP 400 Bad Request`` if the querystring + ``metadata_id`` happens to not be defined in the request. + + This endpoint returns assets only if the transaction they're in are + included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + + :param fields: A comma separated string to expand properties on the transaction object to be returned. + :type fields: string + + :param metadata_id: metadata ID. + :type metadata_id: uuidv4 + + **Example request**: + + .. sourcecode:: http + + GET /transactions?fields=id,metadata&metadata_id=1AAAbbb...ccc HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [{ + "transaction": { + "metadata": { + "id": "1AAAbbb...ccc", + "data": { + "hello": "world" + }, + "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + }] + + :statuscode 200: A list of transaction's containing metadata with ID ``metadata_id`` was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. the ``metadata_id`` querystring was not included in the request. + .. http:post:: /transactions Push a new transaction. From f18f3cb8d20541fd16f07371cf2e805e8e3b38ac Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:26:40 +0100 Subject: [PATCH 030/219] Add sections to be done --- .../http-client-server-api.rst | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index eb906f27..1a300b55 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -334,3 +334,33 @@ Statuses :statuscode 200: A transaction or block with that ID was found. The status is either ``backlog``, ``invalid``. :statuscode 303: A transaction or block with that ID was found and persisted to the chain. A location header to the resource is provided. :statuscode 404: A transaction or block with that ID was not found. + + +Assets +-------------------------------- + +.. http:get:: /assets/{asset_id} + + Descriptions: TODO + + +Metadata +-------------------------------- + +.. http:get:: /metadata/{metadata_id} + + +Blocks +-------------------------------- + +.. http:get:: /blocks/{block_id} + + Descriptions: TODO + + +Votes +-------------------------------- + +.. http:get:: /votes?block_id={block_id} + + Descriptions: TODO From eda8cdbba192d261d45968750400c2df76bbe452 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:40:08 +0100 Subject: [PATCH 031/219] Add docs about /transactions endpoint --- .../http-client-server-api.rst | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 1a300b55..c8970926 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -115,6 +115,46 @@ Transactions :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. +.. http:get:: /transactions + + The current ``/transactions`` endpoint returns a ``404 Not Found`` HTTP + status code. Eventually, this functionality will get implemented. + We believe a PUSH rather than a PULL pattern is more appropriate, as the + items returned in the collection would change by the second. + + There are however requests that might come of use, given the endpoint is + queried correctly. Some of them include retrieving a list of transactions + that include: + + * `Unfulfilled conditions <#get--transactions?fields=id,conditions&fulfilled=false&owner_afters=owners_after>`_ + * `A specific asset <#get--transactions?fields=id,asset,operation&operation=CREATE|TRANSFER&asset_id=asset_id>`_ + * `Specific metadata <#get--transactions?fields=id,metadata&metadata_id=metadata_id>`_ + + In this section, we've listed those particular requests, as they will likely + to be very handy when implementing your application on top of BigchainDB. + A generalization of those parameters can follows: + + :query fields: A comma separated string to expand properties on the transaction object to be returned. + :type fields: string + + :query fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. + :type fulfilled: boolean + + :query owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :type owners_after: base58 encoded string + + :query operation: One of the three supported operations of a transaction. + :type operation: string + + :query asset_id: asset ID. + :type asset_id: uuidv4 + + :query metadata_id: metadata ID. + :type metadata_id: uuidv4 + + :statuscode 404: BigchainDB does not expose this endpoint. + + .. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_afters={owners_after} Get a list of transactions with unfulfilled conditions. From 152f151f23f79723aff35b0670e9a68fab006954 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:42:09 +0100 Subject: [PATCH 032/219] :param _: --> :query _: --- .../drivers-clients/http-client-server-api.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index c8970926..9a7a5c74 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -169,13 +169,13 @@ Transactions This endpoint returns conditions only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :param fields: A comma separated string to expand properties on the transaction object to be returned. + :query fields: A comma separated string to expand properties on the transaction object to be returned. :type fields: string - :param fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. + :query fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. :type fulfilled: boolean - :param owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :query owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. :type owners_after: base58 encoded string **Example request**: @@ -231,13 +231,13 @@ Transactions This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :param fields: A comma separated string to expand properties on the transaction object to be returned. + :query fields: A comma separated string to expand properties on the transaction object to be returned. :type fields: string - :param operation: One of the three supported operations of a transaction. + :query operation: One of the three supported operations of a transaction. :type operation: string - :param asset_id: asset ID. + :query asset_id: asset ID. :type asset_id: uuidv4 **Example request**: @@ -280,10 +280,10 @@ Transactions This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :param fields: A comma separated string to expand properties on the transaction object to be returned. + :query fields: A comma separated string to expand properties on the transaction object to be returned. :type fields: string - :param metadata_id: metadata ID. + :query metadata_id: metadata ID. :type metadata_id: uuidv4 **Example request**: From 9a1b1364eab84dd323a123db380701b9561a1413 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:47:22 +0100 Subject: [PATCH 033/219] Add response headers --- .../source/drivers-clients/http-client-server-api.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 9a7a5c74..df8c093c 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -112,6 +112,8 @@ Transactions "version": 1 } + :resheader Content-Type: ``application/json`` + :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. @@ -216,6 +218,8 @@ Transactions "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", }] + :resheader Content-Type: ``application/json`` + :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. @@ -267,6 +271,8 @@ Transactions "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", }] + :resheader Content-Type: ``application/json`` + :statuscode 200: A list of transaction's containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. @@ -310,6 +316,8 @@ Transactions "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", }] + :resheader Content-Type: ``application/json`` + :statuscode 200: A list of transaction's containing metadata with ID ``metadata_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``metadata_id`` querystring was not included in the request. @@ -371,6 +379,9 @@ Statuses .. literalinclude:: samples/get-tx-status-response.http :language: http + :resheader Content-Type: ``application/json`` + :resheader Location: Once the transaction has been persisted, this header will link to the actual resource. + :statuscode 200: A transaction or block with that ID was found. The status is either ``backlog``, ``invalid``. :statuscode 303: A transaction or block with that ID was found and persisted to the chain. A location header to the resource is provided. :statuscode 404: A transaction or block with that ID was not found. From d748a1dc18eeca81932811bb5ca5f29dd1416081 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 16:53:31 +0100 Subject: [PATCH 034/219] Minor corrections of redundant infos --- .../drivers-clients/http-client-server-api.rst | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index df8c093c..e55f65c5 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -223,22 +223,17 @@ Transactions :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. -.. http:get:: /transactions?fields=id,asset,operation&operation={CREATE|TRANSFER}&asset_id={asset_id} +.. http:get:: /transactions?fields=id,asset,operation&operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} Get a list of transactions that use an asset with the ID ``asset_id``. - This endpoint will return a ``HTTP 400 Bad Request`` if the querystring - ``asset_id`` happens to not be defined in the request. - - ``operation`` can either be ``GENESIS``, ``CREATE`` or ``TRANSFER``. - This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. :query fields: A comma separated string to expand properties on the transaction object to be returned. :type fields: string - :query operation: One of the three supported operations of a transaction. + :query operation: One of the three supported operations of a transaction (``GENESIS``, ``CREATE``, ``TRANSFER``). :type operation: string :query asset_id: asset ID. @@ -280,9 +275,6 @@ Transactions Get a list of transactions that use metadata with the ID ``metadata_id``. - This endpoint will return a ``HTTP 400 Bad Request`` if the querystring - ``metadata_id`` happens to not be defined in the request. - This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. @@ -353,6 +345,7 @@ Statuses .. http:get:: /statuses/{tx_id | block_id} Get the status of an asynchronously written resource by their id. + Supports the retrieval of a status for a transaction using ``tx_id`` or the retrieval of a status for a block using ``block_id``. From 2997a5e994e93c8846d2b9ec7848f32eb6509a71 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 16 Nov 2016 18:09:06 +0100 Subject: [PATCH 035/219] Minor corrections to endpoints --- .../drivers-clients/http-client-server-api.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index e55f65c5..32b9d6bc 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -165,9 +165,6 @@ Transactions ``owners_after`` happen to be fulfilled already, this endpoint will return an empty list. - This endpoint will return a ``HTTP 400 Bad Request`` if the querystring - ``owners_after`` happens to not be defined in the request. - This endpoint returns conditions only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. @@ -401,10 +398,22 @@ Blocks Descriptions: TODO +.. http:get:: /blocks?tx_id={tx_id} + + Descriptions: TODO + Votes -------------------------------- +.. http:get:: /votes/{vote_id} + + Descriptions: TODO + .. http:get:: /votes?block_id={block_id} Descriptions: TODO + +.. http:get:: /votes?block_id={block_id}&voter={voter} + + Descriptions: TODO From af6b799c4fa5a9fcaa63a102c090a57f6fe275f1 Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 17 Nov 2016 14:11:27 +0100 Subject: [PATCH 036/219] Adjustments according to feedback - owner_afters --> owners_after - List choices for operation query string - List choices for fields query string - Use proper sphinx-way to define type of query string --- .../http-client-server-api.rst | 46 +++++++------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 32b9d6bc..d7a2902c 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -128,7 +128,7 @@ Transactions queried correctly. Some of them include retrieving a list of transactions that include: - * `Unfulfilled conditions <#get--transactions?fields=id,conditions&fulfilled=false&owner_afters=owners_after>`_ + * `Unfulfilled conditions <#get--transactions?fields=id,conditions&fulfilled=false&owners_after=owners_after>`_ * `A specific asset <#get--transactions?fields=id,asset,operation&operation=CREATE|TRANSFER&asset_id=asset_id>`_ * `Specific metadata <#get--transactions?fields=id,metadata&metadata_id=metadata_id>`_ @@ -136,28 +136,22 @@ Transactions to be very handy when implementing your application on top of BigchainDB. A generalization of those parameters can follows: - :query fields: A comma separated string to expand properties on the transaction object to be returned. - :type fields: string + :query string fields: Comma separated list, allowed values are: ``asset``, ``conditions``, ``fulfillments``, ``id``, ``metadata``, ``operation``, ``owners_after``, ``version``. - :query fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :type fulfilled: boolean + :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :query owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :type owners_after: base58 encoded string + :query string owners_after: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :query operation: One of the three supported operations of a transaction. - :type operation: string + :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. - :query asset_id: asset ID. - :type asset_id: uuidv4 + :query string asset_id: asset ID. - :query metadata_id: metadata ID. - :type metadata_id: uuidv4 + :query string metadata_id: metadata ID. :statuscode 404: BigchainDB does not expose this endpoint. -.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owner_afters={owners_after} +.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owners_after={owners_after} Get a list of transactions with unfulfilled conditions. @@ -168,14 +162,11 @@ Transactions This endpoint returns conditions only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query fields: A comma separated string to expand properties on the transaction object to be returned. - :type fields: string + :query string fields: A comma separated string to expand properties on the transaction object to be returned. - :query fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :type fulfilled: boolean + :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :query owners_after: Public keys able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :type owners_after: base58 encoded string + :query string owners_after: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. **Example request**: @@ -227,14 +218,11 @@ Transactions This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query fields: A comma separated string to expand properties on the transaction object to be returned. - :type fields: string + :query string fields: A comma separated string to expand properties on the transaction object to be returned. - :query operation: One of the three supported operations of a transaction (``GENESIS``, ``CREATE``, ``TRANSFER``). - :type operation: string + :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. - :query asset_id: asset ID. - :type asset_id: uuidv4 + :query string asset_id: asset ID. **Example request**: @@ -275,11 +263,9 @@ Transactions This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query fields: A comma separated string to expand properties on the transaction object to be returned. - :type fields: string + :query string fields: A comma separated string to expand properties on the transaction object to be returned. - :query metadata_id: metadata ID. - :type metadata_id: uuidv4 + :query string metadata_id: metadata ID. **Example request**: From 26cb00ab9c64c9878616fd5ac6bb98618619ca38 Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 17 Nov 2016 17:38:49 +0100 Subject: [PATCH 037/219] Document /block/block_id endpoint --- .../http-client-server-api.rst | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index d7a2902c..f6fc498d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -382,7 +382,48 @@ Blocks .. http:get:: /blocks/{block_id} - Descriptions: TODO + Get the block with the ID ``block_id``. + + A block is only returned if it was labeled ``VALID`` or ``UNDECIDED`` and + exists in the table ``bigchain``. + + :param block_id: block ID + :type block_id: hex string + + **Example request**: + + .. sourcecode:: http + + GET /blocks/6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202 HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "block":{ + "node_pubkey":"ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt", + "timestamp":"1479389911", + "transactions":[ + '', + '' + ], + "voters":[ + "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" + ] + }, + "id":"6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202", + "signature":"53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" + } + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A block with that ID was found. Its status is either ``VALID`` or ``UNDECIDED``. + :statuscode 404: A block with that ID was not found. .. http:get:: /blocks?tx_id={tx_id} From ea16a8731e0d8c778233261b2bcf821ae541bf6b Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 17 Nov 2016 17:48:24 +0100 Subject: [PATCH 038/219] Remove ?fields query string --- .../http-client-server-api.rst | 132 +++++++++++++++--- 1 file changed, 110 insertions(+), 22 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index f6fc498d..e08d1ab0 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -128,16 +128,14 @@ Transactions queried correctly. Some of them include retrieving a list of transactions that include: - * `Unfulfilled conditions <#get--transactions?fields=id,conditions&fulfilled=false&owners_after=owners_after>`_ - * `A specific asset <#get--transactions?fields=id,asset,operation&operation=CREATE|TRANSFER&asset_id=asset_id>`_ - * `Specific metadata <#get--transactions?fields=id,metadata&metadata_id=metadata_id>`_ + * `Unfulfilled conditions <#get--transactions?fulfilled=false&owners_after=owners_after>`_ + * `A specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ + * `Specific metadata <#get--transactions?&metadata_id=metadata_id>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. A generalization of those parameters can follows: - :query string fields: Comma separated list, allowed values are: ``asset``, ``conditions``, ``fulfillments``, ``id``, ``metadata``, ``operation``, ``owners_after``, ``version``. - :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. :query string owners_after: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. @@ -151,7 +149,7 @@ Transactions :statuscode 404: BigchainDB does not expose this endpoint. -.. http:get:: /transactions?fields=id,conditions&fulfilled=false&owners_after={owners_after} +.. http:get:: /transactions?fulfilled=false&owners_after={owners_after} Get a list of transactions with unfulfilled conditions. @@ -162,8 +160,6 @@ Transactions This endpoint returns conditions only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query string fields: A comma separated string to expand properties on the transaction object to be returned. - :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. :query string owners_after: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. @@ -172,7 +168,7 @@ Transactions .. sourcecode:: http - GET /transactions?fields=id,conditions&fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 + GET /transactions?fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 Host: example.com **Example response**: @@ -203,7 +199,29 @@ Transactions ] } ], + "operation": "CREATE", + "asset": { + "divisible": false, + "updatable": false, + "data": null, + "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", + "refillable": false + }, + "metadata": null, + "timestamp": "1477578978", + "fulfillments": [ + { + "fid": 0, + "input": null, + "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "owners_before": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ] + }, "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "version": 1 }] :resheader Content-Type: ``application/json`` @@ -211,15 +229,13 @@ Transactions :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. -.. http:get:: /transactions?fields=id,asset,operation&operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} +.. http:get:: /transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} Get a list of transactions that use an asset with the ID ``asset_id``. This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query string fields: A comma separated string to expand properties on the transaction object to be returned. - :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. :query string asset_id: asset ID. @@ -228,7 +244,7 @@ Transactions .. sourcecode:: http - GET /transactions?fields=id,asset,operation&operation=CREATE&asset_id=1AAAbbb...ccc HTTP/1.1 + GET /transactions?operation=CREATE&asset_id=1AAAbbb...ccc HTTP/1.1 Host: example.com **Example response**: @@ -240,6 +256,26 @@ Transactions [{ "transaction": { + "conditions": [ + { + "cid": 0, + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + } + }, + "amount": 1, + "owners_after": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ], + "operation": "CREATE", "asset": { "divisible": false, "updatable": false, @@ -247,8 +283,21 @@ Transactions "id": "1AAAbbb...ccc", "refillable": false }, - "operation": "CREATE", + "metadata": null, + "timestamp": "1477578978", + "fulfillments": [ + { + "fid": 0, + "input": null, + "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "owners_before": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ] + }, "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "version": 1 }] :resheader Content-Type: ``application/json`` @@ -256,22 +305,20 @@ Transactions :statuscode 200: A list of transaction's containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. -.. http:get:: /transactions?fields=id,metadata&metadata_id={metadata_id} +.. http:get:: /transactions?metadata_id={metadata_id} Get a list of transactions that use metadata with the ID ``metadata_id``. This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query string fields: A comma separated string to expand properties on the transaction object to be returned. - :query string metadata_id: metadata ID. **Example request**: .. sourcecode:: http - GET /transactions?fields=id,metadata&metadata_id=1AAAbbb...ccc HTTP/1.1 + GET /transactions?metadata_id=1AAAbbb...ccc HTTP/1.1 Host: example.com **Example response**: @@ -283,12 +330,53 @@ Transactions [{ "transaction": { - "metadata": { - "id": "1AAAbbb...ccc", - "data": { - "hello": "world" + "conditions": [ + { + "cid": 0, + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + } + }, + "amount": 1, + "owners_after": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ], + "operation": "CREATE", + "asset": { + "divisible": false, + "updatable": false, + "data": null, + "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", + "refillable": false }, + "metadata": { + "id": "1AAAbbb...ccc", + "data": { + "hello": "world" + }, + }, + "timestamp": "1477578978", + "fulfillments": [ + { + "fid": 0, + "input": null, + "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "owners_before": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ] + }, "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "version": 1 }] :resheader Content-Type: ``application/json`` From 9cd4c18fb474d9559fc094f7ed327ce6cd23f4b6 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 23 Nov 2016 10:25:07 +0100 Subject: [PATCH 039/219] Remove uuid based endpoints --- .../drivers-clients/http-client-server-api.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index e08d1ab0..9f189511 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -450,21 +450,6 @@ Statuses :statuscode 303: A transaction or block with that ID was found and persisted to the chain. A location header to the resource is provided. :statuscode 404: A transaction or block with that ID was not found. - -Assets --------------------------------- - -.. http:get:: /assets/{asset_id} - - Descriptions: TODO - - -Metadata --------------------------------- - -.. http:get:: /metadata/{metadata_id} - - Blocks -------------------------------- From 5ac7eb9b5a121940842d58618c8d3fa979865d13 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 23 Nov 2016 11:04:28 +0100 Subject: [PATCH 040/219] Add notes about endpoint specialities For details, see the following comment: https://github.com/bigchaindb/bigchaindb/pull/830#issuecomment-262468005 --- .../http-client-server-api.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 9f189511..545aaf3e 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -236,6 +236,17 @@ Transactions This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + .. note:: + The BigchainDB API currently doesn't expose an + ``/assets/{asset_id}`` endpoint, as there wouldn't be any way for a + client to verify that what was received is consistent with what was + persisted in the database. + However, BigchainDB's consensus ensures that any ``asset_id`` is + a unique key identifying an asset, meaning that when calling + ``/transactions?operation=CREATE&asset_id={asset_id}``, there will in + any case only be one transaction returned (in a list though, since + ``/transactions`` is a list-returning endpoint). + :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. :query string asset_id: asset ID. @@ -312,6 +323,17 @@ Transactions This endpoint returns assets only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + .. note:: + The BigchainDB API currently doesn't expose an + ``/metadata/{metadata_id}`` endpoint, as there wouldn't be any way for a + client to verify that what was received is consistent with what was + persisted in the database. + However, BigchainDB's consensus ensures that any ``metadata_id`` is + a unique key identifying metadata, meaning that when calling + ``/transactions?metadata_id={metadata_id}``, there will in any case only + be one transaction returned (in a list though, since ``/transactions`` + is a list-returning endpoint). + :query string metadata_id: metadata ID. **Example request**: From 5048846b0f941ef318a1b53b27de7305c9bf85eb Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 23 Nov 2016 13:27:01 +0100 Subject: [PATCH 041/219] Minor corrections on /statuses --- .../source/drivers-clients/http-client-server-api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 545aaf3e..51456f53 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -435,7 +435,7 @@ Transactions Statuses -------------------------------- -.. http:get:: /statuses/{tx_id | block_id} +.. http:get:: /statuses/{tx_id|block_id} Get the status of an asynchronously written resource by their id. @@ -445,9 +445,9 @@ Statuses The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. - If a transaction or block is persisted to the chain and it's status is set to - ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, as - well as an URL to the resource in the location header. + If a transaction or block is persisted to the chain and it's status is set + to ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, + as well as an URL to the resource in the location header. :param tx_id: transaction ID :type tx_id: hex string From 91a115125544cda8b4856f4483d5e693118427f4 Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 24 Nov 2016 10:17:00 +0100 Subject: [PATCH 042/219] Add endpoint to get blocks by transaction --- .../http-client-server-api.rst | 102 +++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 51456f53..8206a5fa 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -520,9 +520,107 @@ Blocks :statuscode 200: A block with that ID was found. Its status is either ``VALID`` or ``UNDECIDED``. :statuscode 404: A block with that ID was not found. -.. http:get:: /blocks?tx_id={tx_id} +.. http:get:: /blocks - Descriptions: TODO + The current ``/blocks`` endpoint returns a ``404 Not Found`` HTTP status + code. Eventually, this functionality will get implemented. + We believe a PUSH rather than a PULL pattern is more appropriate, as the + items returned in the collection would change by the second. + + :statuscode 404: BigchainDB does not expose this endpoint. + + +.. http:get:: /blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID} + + Retrieve a list of blocks that contain a transaction with the ID ``tx_id``. + + Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be + returned. To filter blocks by their status, use the optional ``status`` + querystring. + + .. note:: + In case no block was found, an empty list and an HTTP status code + ``200 OK`` is returned, as the request was still successful. + + :query string tx_id: transaction ID + :query string status: Filter blocks by their status. One of ``VALID``, ``UNDECIDED`` or ``INVALID``. + + **Example request**: + + .. sourcecode:: http + + GET /blocks?tx_id=2d431...0b4b0e HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "block":{ + "node_pubkey":"ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt", + "timestamp":"1479389911", + "transactions":[ + { + "transaction": { + "conditions": [ + { + "cid": 0, + "condition": { + "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "details": { + "signature": null, + "type": "fulfillment", + "type_id": 4, + "bitmask": 32, + "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + } + }, + "amount": 1, + "owners_after": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ], + "operation": "CREATE", + "asset": { + "divisible": false, + "updatable": false, + "data": null, + "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", + "refillable": false + }, + "metadata": null, + "timestamp": "1477578978", + "fulfillments": [ + { + "fid": 0, + "input": null, + "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "owners_before": [ + "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + ] + } + ] + }, + "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "version": 1 + }], + "voters":[ + "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" + ] + }, + "id":"6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202", + "signature":"53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" + } + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A list of blocks containing a transaction with ID ``tx_id`` was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``tx_id``. Votes From 8f7816d32501f16d06d51cc6efdad590ba06e790 Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 24 Nov 2016 11:14:41 +0100 Subject: [PATCH 043/219] Bigchaindb --> BigchainDB --- .../server/source/drivers-clients/http-client-server-api.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 8206a5fa..cebe9566 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -7,11 +7,6 @@ The HTTP Client-Server API there is no ability to do complex queries using the HTTP API. We plan to add more querying capabilities in the future. -This page assumes you already know an API Root URL -for a BigchainDB node or reverse proxy. -It should be something like ``http://apihosting4u.net:9984`` -or ``http://12.34.56.78:9984``. - If you set up a BigchainDB node or reverse proxy yourself, and you're not sure what the API Root URL is, then see the last section of this page for help. From 8e91e14e86b68bbe68e9afaeec05b55c97678fb7 Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 24 Nov 2016 11:37:58 +0100 Subject: [PATCH 044/219] Add /blocks/{block_id} safety querystring It didn't feel good letting users retrieve also invalid or undecided blocks in the /blocks/{block_id} endpoint. Hence, now a block can be evaluated by it's status. If block_id and status do not match, a 404 Not Found HTTP status code is returned. Per default, status is set to valid only. --- .../drivers-clients/http-client-server-api.rst | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index cebe9566..c57bb9c1 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -470,16 +470,23 @@ Statuses Blocks -------------------------------- -.. http:get:: /blocks/{block_id} +.. http:get:: /blocks/{block_id}?status={VALID|UNDECIDED|INVALID} Get the block with the ID ``block_id``. - A block is only returned if it was labeled ``VALID`` or ``UNDECIDED`` and - exists in the table ``bigchain``. + .. note:: + As ``status``'s default value is set to ``VALID``, only ``VALID`` blocks + will be returned by this endpoint. In case ``status=VALID``, but a block + that was labeled ``UNDECIDED`` or ``INVALID`` is requested by + ``block_id``, this endpoint will return a ``404 Not Found`` status code + to warn the user. To check a block's status independently, use the + `Statuses endpoint <#get--statuses-tx_id|block_id>`_. :param block_id: block ID :type block_id: hex string + :query string status: Per default set to ``VALID``. One of ``VALID``, ``UNDECIDED`` or ``INVALID``. + **Example request**: .. sourcecode:: http @@ -512,8 +519,9 @@ Blocks :resheader Content-Type: ``application/json`` - :statuscode 200: A block with that ID was found. Its status is either ``VALID`` or ``UNDECIDED``. - :statuscode 404: A block with that ID was not found. + :statuscode 200: A block with that ID was found. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. + :statuscode 404: A block with that ID and a certain ``status`` was not found. .. http:get:: /blocks From 0c651f6b112fb6c1de3aecff15ce958086597f1d Mon Sep 17 00:00:00 2001 From: tim Date: Thu, 24 Nov 2016 18:03:46 +0100 Subject: [PATCH 045/219] Timestamp of block as header: /transactions/tx_id For reasoning, see: https://github.com/bigchaindb/bigchaindb/pull/830#issuecomment-262774345 --- .../source/drivers-clients/http-client-server-api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index c57bb9c1..d84c0991 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -44,6 +44,11 @@ Transactions This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` block on ``bigchain``, if exists. + A transaction itself doesn't include a ``timestamp`` property. Only the + block a transaction was included in has a unix ``timestamp`` property. It is + returned by this endpoint in a HTTP custom entity header called + ``X-BigchainDB-Timestamp``. + :param tx_id: transaction ID :type tx_id: hex string @@ -60,6 +65,7 @@ Transactions HTTP/1.1 200 OK Content-Type: application/json + X-BigchainDB-Timestamp: 1234567890 { "transaction": { @@ -107,6 +113,7 @@ Transactions "version": 1 } + :resheader X-BigchainDB-Timestamp: A unix timestamp describing when a transaction was included into a valid block. The timestamp provided is taken from the block the transaction was included in. :resheader Content-Type: ``application/json`` :statuscode 200: A transaction with that ID was found. From df3ded315cd2e7fced36671b503ea5f4f2c76530 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 25 Nov 2016 14:13:46 +0100 Subject: [PATCH 046/219] Remove retrieve vote by vote_id After talking to @r-marques and @libscott: - A vote's id is currently generated by RethinkDB - To verify a vote, the signature and the pubkey should be used - Vote's id will be removed from the "external" vote model in the future - Nobody would want to retrieve a vote, but rather a vote by block In case of the HTTP API, this means that a /votes/vote_id endpoint is not feasible to implement. --- .../source/drivers-clients/http-client-server-api.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index d84c0991..98df59ed 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -636,14 +636,8 @@ Blocks Votes -------------------------------- -.. http:get:: /votes/{vote_id} - - Descriptions: TODO - .. http:get:: /votes?block_id={block_id} Descriptions: TODO .. http:get:: /votes?block_id={block_id}&voter={voter} - - Descriptions: TODO From 6e25ef45989f8190ef1a529f363960b9f598d533 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 25 Nov 2016 15:51:17 +0100 Subject: [PATCH 047/219] Add retrieve votes by block --- .../http-client-server-api.rst | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 98df59ed..fa3178b4 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -638,6 +638,48 @@ Votes .. http:get:: /votes?block_id={block_id} - Descriptions: TODO + Retrieve a list of votes for a certain block with ID ``block_id``. + To check for the validity of a vote, a user of this endpoint needs to + perform the `following steps: `_ + + 1. Check if the vote's ``node_pubkey`` is allowed to vote. + 2. Verify the vote's signature against the vote's body (``vote.vote``) and + ``node_pubkey``. + + :query string block_id: The block ID to filter the votes. + + **Example request**: + + .. sourcecode:: http + + GET /votes?block_id=6152f...e7202 HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [{ + "node_pubkey": "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" , + "signature": "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA", + "vote": { + "invalid_reason": null , + "is_block_valid": true , + "previous_block": "6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202" , + "timestamp": "1480082692" , + "voting_for_block": "6152f...e7202" + } + }] + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A list of votes voting for a block with ID ``block_id`` was found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/votes``, without defining ``block_id``. + .. http:get:: /votes?block_id={block_id}&voter={voter} + + Description: TODO From 56b4da02c9d2032b2186b6bf322ae334a0f152ce Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 26 Dec 2016 16:18:09 +0100 Subject: [PATCH 048/219] (fix) : minor fixes to server docs for building --- .../source/drivers-clients/http-client-server-api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index fa3178b4..6fe56a6e 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -513,8 +513,8 @@ Blocks "node_pubkey":"ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt", "timestamp":"1479389911", "transactions":[ - '', - '' + "", + "" ], "voters":[ "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" @@ -643,8 +643,8 @@ Votes perform the `following steps: `_ 1. Check if the vote's ``node_pubkey`` is allowed to vote. - 2. Verify the vote's signature against the vote's body (``vote.vote``) and - ``node_pubkey``. + 2. Verify the vote's signature against the vote's body (``vote.vote``) and ``node_pubkey``. + :query string block_id: The block ID to filter the votes. From a8f8c7f4a98cd39bd2fe8f48e2c99ed5180d5273 Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 26 Dec 2016 16:36:21 +0100 Subject: [PATCH 049/219] GET transactions/id examples --> samples --- .../generate_http_server_api_documentation.py | 1 + .../http-client-server-api.rst | 59 ++----------------- 2 files changed, 5 insertions(+), 55 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 7c7c4b97..df685315 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -53,6 +53,7 @@ Host: example.com TPLS['get-tx-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json +X-BigchainDB-Timestamp: 1482766245 %(tx)s """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 6fe56a6e..c51dec65 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -54,64 +54,13 @@ Transactions **Example request**: - .. sourcecode:: http - - GET /transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - X-BigchainDB-Timestamp: 1234567890 - - { - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "amount": 1, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", - "refillable": false - }, - "metadata": null, - "timestamp": "1477578978", - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - } + .. literalinclude:: samples/get-tx-response.http + :language: http :resheader X-BigchainDB-Timestamp: A unix timestamp describing when a transaction was included into a valid block. The timestamp provided is taken from the block the transaction was included in. :resheader Content-Type: ``application/json`` From d09a93f1a7d95e17652de9dc050626e1589929fc Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 26 Dec 2016 17:23:22 +0100 Subject: [PATCH 050/219] update GET transactions?unfulfilled - owners_after -> public_keys - examples -> samples --- .../generate_http_server_api_documentation.py | 51 ++++++++----- .../http-client-server-api.rst | 72 ++++--------------- 2 files changed, 47 insertions(+), 76 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index df685315..b960d078 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -9,6 +9,38 @@ from bigchaindb.common.transaction import Transaction TPLS = {} + +TPLS['get-tx-id-request'] = """\ +GET /transactions/%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-id-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json +X-BigchainDB-Timestamp: 1482766245 + +%(tx)s +""" + + +TPLS['get-tx-unfulfilled-request'] = """\ +GET /transactions?fulfilled=false&public_keys=%(public_keys)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-unfulfilled-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +[%(tx)s] +""" + + TPLS['post-tx-request'] = """\ POST /transactions/ HTTP/1.1 Host: example.com @@ -32,7 +64,6 @@ Host: example.com """ - TPLS['get-tx-status-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json @@ -43,20 +74,6 @@ Content-Type: application/json """ -TPLS['get-tx-request'] = """\ -GET /transactions/%(txid)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-tx-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json -X-BigchainDB-Timestamp: 1482766245 - -%(tx)s -""" def main(): @@ -75,7 +92,9 @@ def main(): for name, tpl in TPLS.items(): path = os.path.join(base_path, name + '.http') - code = tpl % {'tx': tx_json, 'txid': tx.id} + code = tpl % {'tx': tx_json, + 'txid': tx.id, + 'public_keys': tx.outputs[0].public_keys[0]} with open(path, 'w') as handle: handle.write(code) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index c51dec65..284625f9 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -54,12 +54,12 @@ Transactions **Example request**: - .. literalinclude:: samples/get-tx-request.http + .. literalinclude:: samples/get-tx-id-request.http :language: http **Example response**: - .. literalinclude:: samples/get-tx-response.http + .. literalinclude:: samples/get-tx-id-response.http :language: http :resheader X-BigchainDB-Timestamp: A unix timestamp describing when a transaction was included into a valid block. The timestamp provided is taken from the block the transaction was included in. @@ -79,7 +79,7 @@ Transactions queried correctly. Some of them include retrieving a list of transactions that include: - * `Unfulfilled conditions <#get--transactions?fulfilled=false&owners_after=owners_after>`_ + * `Unfulfilled conditions <#get--transactions?fulfilled=false&public_keys=public_keys>`_ * `A specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ * `Specific metadata <#get--transactions?&metadata_id=metadata_id>`_ @@ -89,7 +89,7 @@ Transactions :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :query string owners_after: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. @@ -100,12 +100,12 @@ Transactions :statuscode 404: BigchainDB does not expose this endpoint. -.. http:get:: /transactions?fulfilled=false&owners_after={owners_after} +.. http:get:: /transactions?fulfilled=false&public_keys={public_keys} Get a list of transactions with unfulfilled conditions. If the querystring ``fulfilled`` is set to ``false`` and all conditions for - ``owners_after`` happen to be fulfilled already, this endpoint will return + ``public_keys`` happen to be fulfilled already, this endpoint will return an empty list. This endpoint returns conditions only if the transaction they're in are @@ -113,67 +113,19 @@ Transactions :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. - :query string owners_after: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. **Example request**: - .. sourcecode:: http - GET /transactions?fulfilled=false&owners_after=1AAAbbb...ccc HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-unfulfilled-request.http + :language: http + **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - [{ - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "1AAAbbb...ccc" - } - }, - "amount": 1, - "owners_after": [ - "1AAAbbb...ccc" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", - "refillable": false - }, - "metadata": null, - "timestamp": "1477578978", - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - }] + .. literalinclude:: samples/get-tx-unfulfilled-response.http + :language: http :resheader Content-Type: ``application/json`` From a37b505c8779ddb5d9f821dff8661540321cc2a3 Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 26 Dec 2016 17:28:06 +0100 Subject: [PATCH 051/219] (fix): forgot one owners_after -> public_keys --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 284625f9..b6c4f3ef 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -130,7 +130,7 @@ Transactions :resheader Content-Type: ``application/json`` :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. the ``owners_after`` querystring was not included in the request. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_keys`` querystring was not included in the request. .. http:get:: /transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} From b947cf7c729447ef4766ecd13dc196acac73dbdd Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 28 Dec 2016 15:43:54 +0100 Subject: [PATCH 052/219] GET transactions?operation&asset_id examples --> samples Updated example to include more than one TRANSFER --- .../generate_http_server_api_documentation.py | 56 ++++++++++++++++-- .../http-client-server-api.rst | 58 ++----------------- 2 files changed, 54 insertions(+), 60 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index b960d078..2d4d0fba 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -4,8 +4,7 @@ import json import os import os.path -from bigchaindb.common.transaction import Transaction - +from bigchaindb.common.transaction import Transaction, Input, TransactionLink TPLS = {} @@ -27,7 +26,7 @@ X-BigchainDB-Timestamp: 1482766245 TPLS['get-tx-unfulfilled-request'] = """\ -GET /transactions?fulfilled=false&public_keys=%(public_keys)s HTTP/1.1 +GET /transactions?fulfilled=false&public_keys=%(public_keys_transfer_last)s HTTP/1.1 Host: example.com """ @@ -37,7 +36,23 @@ TPLS['get-tx-unfulfilled-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json -[%(tx)s] +[%(tx_transfer_last)s] +""" + + +TPLS['get-tx-by-asset-request'] = """\ +GET /transactions?operation=transfer&asset_id=%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-by-asset-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +[%(tx_transfer)s, +%(tx_transfer_last)s] """ @@ -80,10 +95,33 @@ def main(): """ Main function """ privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' - tx = Transaction.create([pubkey], [([pubkey], 1)]) + asset = {'msg': 'Hello BigchainDB!'} + tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset) tx = tx.sign([privkey]) tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) + privkey_transfer = '3AeWpPdhEZzWLYfkfYHBfMFC2r1f8HEaGS9NtbbKssya' + pubkey_transfer = '3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9' + + cid = 0 + input_ = Input(fulfillment=tx.outputs[cid].fulfillment, + fulfills=TransactionLink(txid=tx.id, output=cid), + owners_before=tx.outputs[cid].public_keys) + tx_transfer = Transaction.transfer([input_], [([pubkey_transfer], 1)], asset_id=tx.id) + tx_transfer = tx_transfer.sign([privkey]) + tx_transfer_json = json.dumps(tx_transfer.to_dict(), indent=2, sort_keys=True) + + privkey_transfer_last = 'sG3jWDtdTXUidBJK53ucSTrosktG616U3tQHBk81eQe' + pubkey_transfer_last = '3Af3fhhjU6d9WecEM9Uw5hfom9kNEwE7YuDWdqAUssqm' + + cid = 0 + input_ = Input(fulfillment=tx_transfer.outputs[cid].fulfillment, + fulfills=TransactionLink(txid=tx_transfer.id, output=cid), + owners_before=tx_transfer.outputs[cid].public_keys) + tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)], asset_id=tx.id) + tx_transfer_last = tx_transfer_last.sign([privkey_transfer]) + tx_transfer_last_json = json.dumps(tx_transfer_last.to_dict(), indent=2, sort_keys=True) + base_path = os.path.join(os.path.dirname(__file__), 'source/drivers-clients/samples') @@ -94,7 +132,13 @@ def main(): path = os.path.join(base_path, name + '.http') code = tpl % {'tx': tx_json, 'txid': tx.id, - 'public_keys': tx.outputs[0].public_keys[0]} + 'tx_transfer': tx_transfer_json, + 'tx_transfer_id': tx_transfer.id, + 'tx_transfer_last': tx_transfer_last_json, + 'tx_transfer_last_id': tx_transfer_last.id, + 'public_keys': tx.outputs[0].public_keys[0], + 'public_keys_transfer': tx_transfer.outputs[0].public_keys[0], + 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0]} with open(path, 'w') as handle: handle.write(code) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index b6c4f3ef..e774528d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -156,63 +156,13 @@ Transactions **Example request**: - .. sourcecode:: http - - GET /transactions?operation=CREATE&asset_id=1AAAbbb...ccc HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-by-asset-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - [{ - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "amount": 1, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "1AAAbbb...ccc", - "refillable": false - }, - "metadata": null, - "timestamp": "1477578978", - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - }] + .. literalinclude:: samples/get-tx-by-asset-response.http + :language: http :resheader Content-Type: ``application/json`` From 35a7d11fb324b80b36bb03fc041483ae54110d48 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 28 Dec 2016 16:24:37 +0100 Subject: [PATCH 053/219] remove metadata query from endpoint -> see #856 --- .../http-client-server-api.rst | 104 ++---------------- 1 file changed, 9 insertions(+), 95 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index e774528d..08e0d5e3 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -79,13 +79,18 @@ Transactions queried correctly. Some of them include retrieving a list of transactions that include: - * `Unfulfilled conditions <#get--transactions?fulfilled=false&public_keys=public_keys>`_ - * `A specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ - * `Specific metadata <#get--transactions?&metadata_id=metadata_id>`_ + * `Unfulfilled outputs <#get--transactions?fulfilled=false&public_keys=public_keys>`_ + * `Transactions related to a specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. - A generalization of those parameters can follows: + + .. note:: + Looking up transactions with a specific ``metadata`` field is currently not supported. + This functionality requires something like custom indexing per client or read-only followers, + which is not yet on the roadmap. + + A generalization of those parameters follows: :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. @@ -95,8 +100,6 @@ Transactions :query string asset_id: asset ID. - :query string metadata_id: metadata ID. - :statuscode 404: BigchainDB does not expose this endpoint. @@ -169,95 +172,6 @@ Transactions :statuscode 200: A list of transaction's containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. -.. http:get:: /transactions?metadata_id={metadata_id} - - Get a list of transactions that use metadata with the ID ``metadata_id``. - - This endpoint returns assets only if the transaction they're in are - included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - - .. note:: - The BigchainDB API currently doesn't expose an - ``/metadata/{metadata_id}`` endpoint, as there wouldn't be any way for a - client to verify that what was received is consistent with what was - persisted in the database. - However, BigchainDB's consensus ensures that any ``metadata_id`` is - a unique key identifying metadata, meaning that when calling - ``/transactions?metadata_id={metadata_id}``, there will in any case only - be one transaction returned (in a list though, since ``/transactions`` - is a list-returning endpoint). - - :query string metadata_id: metadata ID. - - **Example request**: - - .. sourcecode:: http - - GET /transactions?metadata_id=1AAAbbb...ccc HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - [{ - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "amount": 1, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", - "refillable": false - }, - "metadata": { - "id": "1AAAbbb...ccc", - "data": { - "hello": "world" - }, - }, - "timestamp": "1477578978", - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - }] - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A list of transaction's containing metadata with ID ``metadata_id`` was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. the ``metadata_id`` querystring was not included in the request. .. http:post:: /transactions From 5c106027548bd66cd95c5bedd17c67b7d2965158 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 28 Dec 2016 17:55:26 +0100 Subject: [PATCH 054/219] examples -> samples POST /transaction and /statuses --- .../generate_http_server_api_documentation.py | 24 ++++++++++++------- .../http-client-server-api.rst | 9 +++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 2d4d0fba..6cd35d90 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -66,31 +66,39 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ -HTTP/1.1 201 Created +HTTP/1.1 202 Accepted Content-Type: application/json - -%(tx)s +Location: ../statuses/%(txid)s """ -TPLS['get-tx-status-request'] = """\ -GET /transactions/%(txid)s/status HTTP/1.1 +TPLS['get-statuses-tx-request'] = """\ +GET /statuses/%(txid)s HTTP/1.1 Host: example.com """ -TPLS['get-tx-status-response'] = """\ +TPLS['get-statuses-tx-invalid-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json +{ + "status": "invalid" +} +""" + + +TPLS['get-statuses-tx-valid-response'] = """\ +HTTP/1.1 303 See Other +Content-Type: application/json +Location: ../transactions/%(txid)s + { "status": "valid" } """ - - def main(): """ Main function """ privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 08e0d5e3..fca6069a 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -224,12 +224,17 @@ Statuses **Example request**: - .. literalinclude:: samples/get-tx-status-request.http + .. literalinclude:: samples/get-statuses-tx-request.http :language: http **Example response**: - .. literalinclude:: samples/get-tx-status-response.http + .. literalinclude:: samples/get-statuses-tx-invalid-response.http + :language: http + + **Example response**: + + .. literalinclude:: samples/get-statuses-tx-valid-response.http :language: http :resheader Content-Type: ``application/json`` From 204c8cce9c7d7b33648ea3fe178e439811e5d627 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 29 Dec 2016 16:23:58 +0100 Subject: [PATCH 055/219] GET blocks/id examples -> samples --- .../generate_http_server_api_documentation.py | 27 ++++++++++++++++++- .../http-client-server-api.rst | 27 +++---------------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 6cd35d90..350a286a 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -5,6 +5,8 @@ import os import os.path from bigchaindb.common.transaction import Transaction, Input, TransactionLink +from bigchaindb.models import Block + TPLS = {} @@ -99,6 +101,20 @@ Location: ../transactions/%(txid)s """ +TPLS['get-block-request'] = """\ +GET /blocks/%(blockid)s HTTP/1.1 +Host: example.com + +""" + +TPLS['get-block-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(block)s +""" + + def main(): """ Main function """ privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' @@ -130,6 +146,11 @@ def main(): tx_transfer_last = tx_transfer_last.sign([privkey_transfer]) tx_transfer_last_json = json.dumps(tx_transfer_last.to_dict(), indent=2, sort_keys=True) + node = "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" + signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" + block = Block(transactions=[tx], node_pubkey=node, voters=[node], signature=signature) + block_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) + base_path = os.path.join(os.path.dirname(__file__), 'source/drivers-clients/samples') @@ -146,7 +167,9 @@ def main(): 'tx_transfer_last_id': tx_transfer_last.id, 'public_keys': tx.outputs[0].public_keys[0], 'public_keys_transfer': tx_transfer.outputs[0].public_keys[0], - 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0]} + 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0], + 'block': block_json, + 'blockid': block.id} with open(path, 'w') as handle: handle.write(code) @@ -158,3 +181,5 @@ def setup(*_): if __name__ == '__main__': main() + + diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index fca6069a..56ebfc86 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -266,33 +266,14 @@ Blocks **Example request**: - .. sourcecode:: http - - GET /blocks/6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202 HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-block-request.http + :language: http **Example response**: - .. sourcecode:: http + .. literalinclude:: samples/get-block-response.http + :language: http - HTTP/1.1 200 OK - Content-Type: application/json - - { - "block":{ - "node_pubkey":"ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt", - "timestamp":"1479389911", - "transactions":[ - "", - "" - ], - "voters":[ - "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" - ] - }, - "id":"6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202", - "signature":"53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" - } :resheader Content-Type: ``application/json`` From 21ffe225166034367eec92d3292eb6c229d18269 Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 2 Jan 2017 15:24:14 +0100 Subject: [PATCH 056/219] API root URL vs BigchainDB root URL --- .../http-client-server-api.rst | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 56ebfc86..126a7903 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -12,8 +12,8 @@ and you're not sure what the API Root URL is, then see the last section of this page for help. -API Root URL ------------- +BigchainDB Root URL +------------------- If you send an HTTP GET request to the API Root URL e.g. ``http://localhost:9984`` @@ -31,7 +31,28 @@ with something like the following in the body: ], "public_key": "AiygKSRhZWTxxYT4AfgKoTG4TZAoPsWoEt6C6bLq4jJR", "software": "BigchainDB", - "version": "0.6.0" + "version": "0.9.0" + } + + +API Root URL +------------------- + +If you send an HTTP GET request to the API Root URL +e.g. ``http://localhost:9984/api/v1/`` +or ``http://apihosting4u.net:9984/api/v1/``, +then you should get an HTTP response +that allows you to discover the BigchainDB endpoints: + +.. code-block:: json + + { + "_links": { + "self": { "href": "/" }, + "transactions": { "href": "/transactions" }, + "statuses": { "href": "/statuses" }, + "blocks": { "href": "/blocks" }, + "votes": { "href": "/votes" } } Transactions From e54d50a58f57aa0985013130debae049343f80d6 Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 2 Jan 2017 15:43:45 +0100 Subject: [PATCH 057/219] Add docs links to root endpoints --- .../http-client-server-api.rst | 61 ++++++++++++++++--- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 126a7903..95f08bd7 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -25,34 +25,40 @@ with something like the following in the body: .. code-block:: json { + "_links": { + "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/" } + } "keyring": [ "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" ], "public_key": "AiygKSRhZWTxxYT4AfgKoTG4TZAoPsWoEt6C6bLq4jJR", "software": "BigchainDB", - "version": "0.9.0" + "version": "0.9.0", } -API Root URL +API Root Endpoint ------------------- -If you send an HTTP GET request to the API Root URL -e.g. ``http://localhost:9984/api/v1/`` -or ``http://apihosting4u.net:9984/api/v1/``, +If you send an HTTP GET request to the API Root Endpoint +e.g. ``http://localhost:9984/api/v0.9/`` +or ``http://apihosting4u.net:9984/api/v0.9/``, then you should get an HTTP response -that allows you to discover the BigchainDB endpoints: +that allows you to discover the BigchainDB API endpoints: .. code-block:: json { "_links": { - "self": { "href": "/" }, - "transactions": { "href": "/transactions" }, - "statuses": { "href": "/statuses" }, "blocks": { "href": "/blocks" }, + "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, + "self": { "href": "/" }, + "statuses": { "href": "/statuses" }, + "transactions": { "href": "/transactions" }, "votes": { "href": "/votes" } + }, + "version" : "0.9.0" } Transactions @@ -455,3 +461,40 @@ Votes .. http:get:: /votes?block_id={block_id}&voter={voter} Description: TODO + + +Determining the API Root URL +---------------------------- + +When you start BigchainDB Server using ``bigchaindb start``, +an HTTP API is exposed at some address. The default is: + +`http://localhost:9984/api/v0.9/ `_ + +It's bound to ``localhost``, +so you can access it from the same machine, +but it won't be directly accessible from the outside world. +(The outside world could connect via a SOCKS proxy or whatnot.) + +The documentation about BigchainDB Server :any:`Configuration Settings` +has a section about how to set ``server.bind`` so as to make +the HTTP API publicly accessible. + +If the API endpoint is publicly accessible, +then the public API Root URL is determined as follows: + +- The public IP address (like 12.34.56.78) + is the public IP address of the machine exposing + the HTTP API to the public internet (e.g. either the machine hosting + Gunicorn or the machine running the reverse proxy such as Nginx). + It's determined by AWS, Azure, Rackspace, or whoever is hosting the machine. + +- The DNS hostname (like apihosting4u.net) is determined by DNS records, + such as an "A Record" associating apihosting4u.net with 12.34.56.78 + +- The port (like 9984) is determined by the ``server.bind`` setting + if Gunicorn is exposed directly to the public Internet. + If a reverse proxy (like Nginx) is exposed directly to the public Internet + instead, then it could expose the HTTP API on whatever port it wants to. + (It should expose the HTTP API on port 9984, but it's not bound to do + that by anything other than convention.) From e79a76512f1dd70f87918c0f6753c3f77b0f8827 Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 2 Jan 2017 16:45:00 +0100 Subject: [PATCH 058/219] remove rudimentary note and typo's --- .../drivers-clients/http-client-server-api.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 95f08bd7..0e3cb8a4 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -1,11 +1,10 @@ The HTTP Client-Server API ========================== -.. note:: - - The HTTP client-server API is currently quite rudimentary. For example, - there is no ability to do complex queries using the HTTP API. We plan to add - more querying capabilities in the future. +This page assumes you already know an API Root URL +for a BigchainDB node or reverse proxy. +It should be something like ``http://apihosting4u.net:9984`` +or ``http://12.34.56.78:9984``. If you set up a BigchainDB node or reverse proxy yourself, and you're not sure what the API Root URL is, @@ -119,7 +118,7 @@ Transactions A generalization of those parameters follows: - :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. + :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. @@ -141,7 +140,7 @@ Transactions This endpoint returns conditions only if the transaction they're in are included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query boolean fulfilled: A flag to indicate if transaction's with fulfilled conditions should be returned. + :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. @@ -159,7 +158,7 @@ Transactions :resheader Content-Type: ``application/json`` - :statuscode 200: A list of transaction's containing unfulfilled conditions was found and returned. + :statuscode 200: A list of transactions containing unfulfilled conditions was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``public_keys`` querystring was not included in the request. .. http:get:: /transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} @@ -196,7 +195,7 @@ Transactions :resheader Content-Type: ``application/json`` - :statuscode 200: A list of transaction's containing an asset with ID ``asset_id`` was found and returned. + :statuscode 200: A list of transactions containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. From be5bc9fafd1ac330feb1394591a7812bc354831c Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 2 Jan 2017 17:10:11 +0100 Subject: [PATCH 059/219] GET blocks?txid examples -> samples --- .../generate_http_server_api_documentation.py | 13 ++++ .../http-client-server-api.rst | 70 ++----------------- 2 files changed, 17 insertions(+), 66 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 350a286a..0716107a 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -114,6 +114,19 @@ Content-Type: application/json %(block)s """ +TPLS['get-block-txid-request'] = """\ +GET /blocks?tx_id=%(txid)s HTTP/1.1 +Host: example.com + +""" + +TPLS['get-block-txid-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +[%(block)s] +""" + def main(): """ Main function """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 0e3cb8a4..b11566c2 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -334,75 +334,13 @@ Blocks **Example request**: - .. sourcecode:: http - - GET /blocks?tx_id=2d431...0b4b0e HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-block-txid-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "block":{ - "node_pubkey":"ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt", - "timestamp":"1479389911", - "transactions":[ - { - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "amount": 1, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", - "refillable": false - }, - "metadata": null, - "timestamp": "1477578978", - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - }], - "voters":[ - "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" - ] - }, - "id":"6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202", - "signature":"53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" - } + .. literalinclude:: samples/get-block-txid-response.http + :language: http :resheader Content-Type: ``application/json`` From b252c4079358d514653a073cdf924fc0ab91b7ba Mon Sep 17 00:00:00 2001 From: diminator Date: Tue, 3 Jan 2017 17:20:16 +0100 Subject: [PATCH 060/219] GET /votes examples -> samples --- .../generate_http_server_api_documentation.py | 34 ++++++++++++++++++- .../http-client-server-api.rst | 31 +++-------------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 0716107a..c6c9b63f 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -5,9 +5,11 @@ import os import os.path from bigchaindb.common.transaction import Transaction, Input, TransactionLink +from bigchaindb.core import Bigchain from bigchaindb.models import Block + TPLS = {} @@ -80,6 +82,7 @@ Host: example.com """ + TPLS['get-statuses-tx-invalid-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json @@ -107,6 +110,7 @@ Host: example.com """ + TPLS['get-block-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json @@ -114,12 +118,14 @@ Content-Type: application/json %(block)s """ + TPLS['get-block-txid-request'] = """\ GET /blocks?tx_id=%(txid)s HTTP/1.1 Host: example.com """ + TPLS['get-block-txid-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json @@ -128,8 +134,25 @@ Content-Type: application/json """ +TPLS['get-vote-request'] = """\ +GET /votes?block_id=%(blockid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-vote-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +[%(vote)s] +""" + + def main(): """ Main function """ + + # tx create privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' asset = {'msg': 'Hello BigchainDB!'} @@ -137,6 +160,7 @@ def main(): tx = tx.sign([privkey]) tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) + # tx transfer privkey_transfer = '3AeWpPdhEZzWLYfkfYHBfMFC2r1f8HEaGS9NtbbKssya' pubkey_transfer = '3yfQPHeWAa1MxTX9Zf9176QqcpcnWcanVZZbaHb8B3h9' @@ -159,6 +183,7 @@ def main(): tx_transfer_last = tx_transfer_last.sign([privkey_transfer]) tx_transfer_last_json = json.dumps(tx_transfer_last.to_dict(), indent=2, sort_keys=True) + # block node = "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" block = Block(transactions=[tx], node_pubkey=node, voters=[node], signature=signature) @@ -167,6 +192,12 @@ def main(): base_path = os.path.join(os.path.dirname(__file__), 'source/drivers-clients/samples') + # vote + DUMMY_SHA3 = '0123456789abcdef' * 4 + b = Bigchain(public_key=node) + vote = b.vote(block.id, DUMMY_SHA3, True) + vote_json = json.dumps(vote, indent=2, sort_keys=True) + if not os.path.exists(base_path): os.makedirs(base_path) @@ -182,7 +213,8 @@ def main(): 'public_keys_transfer': tx_transfer.outputs[0].public_keys[0], 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0], 'block': block_json, - 'blockid': block.id} + 'blockid': block.id, + 'vote': vote_json} with open(path, 'w') as handle: handle.write(code) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index b11566c2..90ccc58c 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -26,7 +26,7 @@ with something like the following in the body: { "_links": { "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/" } - } + }, "keyring": [ "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" @@ -365,29 +365,13 @@ Votes **Example request**: - .. sourcecode:: http - - GET /votes?block_id=6152f...e7202 HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-vote-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - [{ - "node_pubkey": "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" , - "signature": "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA", - "vote": { - "invalid_reason": null , - "is_block_valid": true , - "previous_block": "6152fbc7e0f7686512ed6b92c01e8c73ea1e3f51a7b037ac5cc8c860215e7202" , - "timestamp": "1480082692" , - "voting_for_block": "6152f...e7202" - } - }] + .. literalinclude:: samples/get-vote-response.http + :language: http :resheader Content-Type: ``application/json`` @@ -395,11 +379,6 @@ Votes :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/votes``, without defining ``block_id``. -.. http:get:: /votes?block_id={block_id}&voter={voter} - - Description: TODO - - Determining the API Root URL ---------------------------- From 14f22cb8afe834935bab2756797dc74eda7a741d Mon Sep 17 00:00:00 2001 From: diminator Date: Tue, 3 Jan 2017 17:25:40 +0100 Subject: [PATCH 061/219] status with block and transaction context --- .../generate_http_server_api_documentation.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index c6c9b63f..d51050d3 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -88,7 +88,10 @@ HTTP/1.1 200 OK Content-Type: application/json { - "status": "invalid" + "status": "invalid", + "_links" : { + "block": "/blocks/%(blockid)s" + } } """ @@ -99,7 +102,11 @@ Content-Type: application/json Location: ../transactions/%(txid)s { - "status": "valid" + "status": "valid", + "_links" : { + "tx" : "/transactions/%(txid)s", + "block": "/blocks/%(blockid)s" + } } """ @@ -189,15 +196,14 @@ def main(): block = Block(transactions=[tx], node_pubkey=node, voters=[node], signature=signature) block_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) - base_path = os.path.join(os.path.dirname(__file__), - 'source/drivers-clients/samples') - # vote DUMMY_SHA3 = '0123456789abcdef' * 4 b = Bigchain(public_key=node) vote = b.vote(block.id, DUMMY_SHA3, True) vote_json = json.dumps(vote, indent=2, sort_keys=True) + base_path = os.path.join(os.path.dirname(__file__), + 'source/drivers-clients/samples') if not os.path.exists(base_path): os.makedirs(base_path) From 6aca7e2605da97706b59892830fc7fb9ba10b2fa Mon Sep 17 00:00:00 2001 From: diminator Date: Tue, 3 Jan 2017 17:50:20 +0100 Subject: [PATCH 062/219] list assets recipe and sample --- .../generate_http_server_api_documentation.py | 14 +++++++ .../http-client-server-api.rst | 37 ++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index d51050d3..6376ae1c 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -59,6 +59,20 @@ Content-Type: application/json %(tx_transfer_last)s] """ +TPLS['get-assets-request'] = """\ +GET /transactions?operation=CREATE&is_asset=true&public_keys=%(public_keys)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-assets-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +[%(tx)s] +""" + TPLS['post-tx-request'] = """\ POST /transactions/ HTTP/1.1 diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 90ccc58c..45821afc 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -106,7 +106,8 @@ Transactions that include: * `Unfulfilled outputs <#get--transactions?fulfilled=false&public_keys=public_keys>`_ - * `Transactions related to a specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ + * `Transactions related to a specific asset <#get--transactions?operation=GENESIS|CREATE|TRANSFER&asset_id=asset_id>`_ + * `Listing of assets <#get--transactions?operation=CREATE&is_asset=true&public_keys=public_keys>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. @@ -120,12 +121,15 @@ Transactions :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. + :query boolean is_asset: A flag to indicate if the ``asset`` field of the transaction is ``null`` or not. + :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. :query string asset_id: asset ID. + :statuscode 404: BigchainDB does not expose this endpoint. @@ -198,6 +202,37 @@ Transactions :statuscode 200: A list of transactions containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. +.. http:get:: /transactions?operation=CREATE&is_asset=true&public_keys={public_keys} + + Get a list of ``CREATE`` transactions that have the asset field defined. + This can serve as a recipe for retrieving your list of assets. + Currently, filtering on specific fields in the ``asset`` or ``metadata`` is assumed to be done clientside. + + This endpoint returns assets only if the transaction they're in are + included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + + :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. + + :query boolean is_asset: A flag to indicate if the ``asset`` field of the transaction is ``null`` or not. + + :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. + + + **Example request**: + + .. literalinclude:: samples/get-assets-request.http + :language: http + + **Example response**: + + .. literalinclude:: samples/get-assets-response.http + :language: http + + + :resheader Content-Type: ``application/json`` + + :statuscode 200: A list of transactions containing an asset and ``public_keys`` found and returned. + :statuscode 400: The request wasn't understood by the server, e.g. the ``is_asset`` querystring was not included in the request. .. http:post:: /transactions From 0ce9cc4f551d8e985b2fe2861e518bc2de4ef00a Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 4 Jan 2017 10:55:51 +0100 Subject: [PATCH 063/219] Return transaction if it's in backlog also --- .../source/drivers-clients/http-client-server-api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 45821afc..e5a552a4 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -67,7 +67,7 @@ Transactions Get the transaction with the ID ``tx_id``. - This endpoint returns only a transaction from a ``VALID`` or ``UNDECIDED`` + This endpoint returns only a transaction from the ``BACKLOG`` or a ``VALID`` or ``UNDECIDED`` block on ``bigchain``, if exists. A transaction itself doesn't include a ``timestamp`` property. Only the @@ -142,7 +142,7 @@ Transactions an empty list. This endpoint returns conditions only if the transaction they're in are - included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. @@ -170,7 +170,7 @@ Transactions Get a list of transactions that use an asset with the ID ``asset_id``. This endpoint returns assets only if the transaction they're in are - included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. .. note:: The BigchainDB API currently doesn't expose an @@ -209,7 +209,7 @@ Transactions Currently, filtering on specific fields in the ``asset`` or ``metadata`` is assumed to be done clientside. This endpoint returns assets only if the transaction they're in are - included in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. + included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. From 8c46cc3fb4fcda21bf9e9bc3d539403b3421a4cb Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 4 Jan 2017 10:59:33 +0100 Subject: [PATCH 064/219] (fix): travis error on keypair --- docs/server/generate_http_server_api_documentation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 6376ae1c..7d8c0ad9 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -205,14 +205,15 @@ def main(): tx_transfer_last_json = json.dumps(tx_transfer_last.to_dict(), indent=2, sort_keys=True) # block - node = "ErEeVZt8AfLbMJub25tjNxbpzzTNp3mGidL3GxGdd9bt" + node_private = "5G2kE1zJAgTajkVSbPAQWo4c2izvtwqaNHYsaNpbbvxX" + node_public = "DngBurxfeNVKZWCEcDnLj1eMPAS7focUZTE5FndFGuHT" signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" - block = Block(transactions=[tx], node_pubkey=node, voters=[node], signature=signature) + block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) block_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) # vote DUMMY_SHA3 = '0123456789abcdef' * 4 - b = Bigchain(public_key=node) + b = Bigchain(public_key=node_public) vote = b.vote(block.id, DUMMY_SHA3, True) vote_json = json.dumps(vote, indent=2, sort_keys=True) From 940a0cd4dff141535c60291fede373d7ed6d5dcd Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 4 Jan 2017 11:00:08 +0100 Subject: [PATCH 065/219] (fix): add private_key of node to bigchain instance --- docs/server/generate_http_server_api_documentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 7d8c0ad9..64899535 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -213,7 +213,7 @@ def main(): # vote DUMMY_SHA3 = '0123456789abcdef' * 4 - b = Bigchain(public_key=node_public) + b = Bigchain(public_key=node_public, private_key=node_private) vote = b.vote(block.id, DUMMY_SHA3, True) vote_json = json.dumps(vote, indent=2, sort_keys=True) From 305adba3a419b39a85b7e1de8b5bcd004d580f2d Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 4 Jan 2017 11:02:50 +0100 Subject: [PATCH 066/219] include metadata in samples --- docs/server/generate_http_server_api_documentation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 64899535..aa9b5ccc 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -177,7 +177,7 @@ def main(): privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' asset = {'msg': 'Hello BigchainDB!'} - tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset) + tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset, metadata={'sequence': 0}) tx = tx.sign([privkey]) tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) @@ -189,7 +189,7 @@ def main(): input_ = Input(fulfillment=tx.outputs[cid].fulfillment, fulfills=TransactionLink(txid=tx.id, output=cid), owners_before=tx.outputs[cid].public_keys) - tx_transfer = Transaction.transfer([input_], [([pubkey_transfer], 1)], asset_id=tx.id) + tx_transfer = Transaction.transfer([input_], [([pubkey_transfer], 1)], asset_id=tx.id, metadata={'sequence': 1}) tx_transfer = tx_transfer.sign([privkey]) tx_transfer_json = json.dumps(tx_transfer.to_dict(), indent=2, sort_keys=True) @@ -200,7 +200,7 @@ def main(): input_ = Input(fulfillment=tx_transfer.outputs[cid].fulfillment, fulfills=TransactionLink(txid=tx_transfer.id, output=cid), owners_before=tx_transfer.outputs[cid].public_keys) - tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)], asset_id=tx.id) + tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)], asset_id=tx.id, metadata={'sequence': 2}) tx_transfer_last = tx_transfer_last.sign([privkey_transfer]) tx_transfer_last_json = json.dumps(tx_transfer_last.to_dict(), indent=2, sort_keys=True) From 5fa3cddbb7fadd12b4de7750e5b1ac70f4b47412 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 4 Jan 2017 11:29:42 +0100 Subject: [PATCH 067/219] transactions and blocks root endpoints --- .../http-client-server-api.rst | 98 ++++++++++++++----- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index e5a552a4..3eef71fb 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -3,8 +3,8 @@ The HTTP Client-Server API This page assumes you already know an API Root URL for a BigchainDB node or reverse proxy. -It should be something like ``http://apihosting4u.net:9984`` -or ``http://12.34.56.78:9984``. +It should be something like ``https://example.com:9984`` +or ``https://12.34.56.78:9984``. If you set up a BigchainDB node or reverse proxy yourself, and you're not sure what the API Root URL is, @@ -14,11 +14,11 @@ then see the last section of this page for help. BigchainDB Root URL ------------------- -If you send an HTTP GET request to the API Root URL -e.g. ``http://localhost:9984`` -or ``http://apihosting4u.net:9984`` -(with no ``/api/v1/`` on the end), -then you should get an HTTP response +If you send an HTTP GET request to the BigchainDB Root URL +e.g. ``http://localhost:9984`` +or ``https://example.com:9984`` +(with no ``/api/v1/`` on the end), +then you should get an HTTP response with something like the following in the body: .. code-block:: json @@ -42,7 +42,7 @@ API Root Endpoint If you send an HTTP GET request to the API Root Endpoint e.g. ``http://localhost:9984/api/v0.9/`` -or ``http://apihosting4u.net:9984/api/v0.9/``, +or ``https://example.com:9984/api/v0.9/``, then you should get an HTTP response that allows you to discover the BigchainDB API endpoints: @@ -50,12 +50,12 @@ that allows you to discover the BigchainDB API endpoints: { "_links": { - "blocks": { "href": "/blocks" }, + "blocks": { "href": "https://example.com:9984/api/v0.9/blocks" }, "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "self": { "href": "/" }, - "statuses": { "href": "/statuses" }, - "transactions": { "href": "/transactions" }, - "votes": { "href": "/votes" } + "self": { "href": "https://example.com:9984/api/v0.9" }, + "statuses": { "href": "https://example.com:9984/api/v0.9/statuses" }, + "transactions": { "href": "https://example.com:9984/api/v0.9/transactions" }, + "votes": { "href": "https://example.com:9984/api/v0.9/votes" } }, "version" : "0.9.0" } @@ -96,12 +96,40 @@ Transactions .. http:get:: /transactions - The current ``/transactions`` endpoint returns a ``404 Not Found`` HTTP - status code. Eventually, this functionality will get implemented. + The unfiltered ``/transactions`` endpoint without any query parameters + returns a list of available transaction usages and relevant endpoints. We believe a PUSH rather than a PULL pattern is more appropriate, as the items returned in the collection would change by the second. - There are however requests that might come of use, given the endpoint is + **Example request**: + + .. sourcecode:: http + + GET /transactions HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "_links": { + "asset_history": { "href": "https://example.com:9984/api/v0.9/transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id}" }, + "asset_list": { "href": "https://example.com:9984/api/v0.9/transactions?operation=CREATE&is_asset=true&public_keys={public_keys}" }, + "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, + "item": { "href": "https://example.com:9984/api/v0.9/transactions/{tx_id}" }, + "self": { "href": "https://example.com:9984/api/v0.9/transactions" }, + "unfulfilled": { "href": "https://example.com:9984/api/v0.9/transactions?fulfilled=false&public_keys={public_keys}" } + }, + "version" : "0.9.0" + } + + :statuscode 200: BigchainDB transactions root endpoint. + + There are however filtered requests that might come of use, given the endpoint is queried correctly. Some of them include retrieving a list of transactions that include: @@ -130,9 +158,6 @@ Transactions :query string asset_id: asset ID. - :statuscode 404: BigchainDB does not expose this endpoint. - - .. http:get:: /transactions?fulfilled=false&public_keys={public_keys} Get a list of transactions with unfulfilled conditions. @@ -344,12 +369,37 @@ Blocks .. http:get:: /blocks - The current ``/blocks`` endpoint returns a ``404 Not Found`` HTTP status - code. Eventually, this functionality will get implemented. + The unfiltered ``/blocks`` endpoint without any query parameters + returns a list of available block usages and relevant endpoints. We believe a PUSH rather than a PULL pattern is more appropriate, as the items returned in the collection would change by the second. - :statuscode 404: BigchainDB does not expose this endpoint. + + **Example request**: + + .. sourcecode:: http + + GET /blocks HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "_links": { + "blocks": { "href": "https://example.com:9984/api/v0.9/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}" }, + "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, + "item": { "href": "https://example.com:9984/api/v0.9/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}" }, + "self": { "href": "https://example.com:9984/api/v0.9/blocks" } + }, + "version" : "0.9.0" + } + + :statuscode 200: BigchainDB blocks root endpoint. .. http:get:: /blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID} @@ -440,8 +490,8 @@ then the public API Root URL is determined as follows: Gunicorn or the machine running the reverse proxy such as Nginx). It's determined by AWS, Azure, Rackspace, or whoever is hosting the machine. -- The DNS hostname (like apihosting4u.net) is determined by DNS records, - such as an "A Record" associating apihosting4u.net with 12.34.56.78 +- The DNS hostname (like example.com) is determined by DNS records, + such as an "A Record" associating example.com with 12.34.56.78 - The port (like 9984) is determined by the ``server.bind`` setting if Gunicorn is exposed directly to the public Internet. From 9e5623f881a952512431ef0c4981b994faa8f842 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 10:36:34 +0100 Subject: [PATCH 068/219] Removed X-bigchaindb-timestamp --- docs/server/generate_http_server_api_documentation.py | 1 - .../source/drivers-clients/http-client-server-api.rst | 6 ------ 2 files changed, 7 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index aa9b5ccc..6f5ad04c 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -23,7 +23,6 @@ Host: example.com TPLS['get-tx-id-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json -X-BigchainDB-Timestamp: 1482766245 %(tx)s """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 3eef71fb..c2e9cd60 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -70,11 +70,6 @@ Transactions This endpoint returns only a transaction from the ``BACKLOG`` or a ``VALID`` or ``UNDECIDED`` block on ``bigchain``, if exists. - A transaction itself doesn't include a ``timestamp`` property. Only the - block a transaction was included in has a unix ``timestamp`` property. It is - returned by this endpoint in a HTTP custom entity header called - ``X-BigchainDB-Timestamp``. - :param tx_id: transaction ID :type tx_id: hex string @@ -88,7 +83,6 @@ Transactions .. literalinclude:: samples/get-tx-id-response.http :language: http - :resheader X-BigchainDB-Timestamp: A unix timestamp describing when a transaction was included into a valid block. The timestamp provided is taken from the block the transaction was included in. :resheader Content-Type: ``application/json`` :statuscode 200: A transaction with that ID was found. From f17a7226343cbfb55026924ea8c0cd66ab679426 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 10:52:22 +0100 Subject: [PATCH 069/219] remove is_asset --- .../http-client-server-api.rst | 37 +------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index c2e9cd60..7b91c69d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -111,8 +111,7 @@ Transactions { "_links": { - "asset_history": { "href": "https://example.com:9984/api/v0.9/transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id}" }, - "asset_list": { "href": "https://example.com:9984/api/v0.9/transactions?operation=CREATE&is_asset=true&public_keys={public_keys}" }, + "assets": { "href": "https://example.com:9984/api/v0.9/transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id}" }, "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, "item": { "href": "https://example.com:9984/api/v0.9/transactions/{tx_id}" }, "self": { "href": "https://example.com:9984/api/v0.9/transactions" }, @@ -129,7 +128,6 @@ Transactions * `Unfulfilled outputs <#get--transactions?fulfilled=false&public_keys=public_keys>`_ * `Transactions related to a specific asset <#get--transactions?operation=GENESIS|CREATE|TRANSFER&asset_id=asset_id>`_ - * `Listing of assets <#get--transactions?operation=CREATE&is_asset=true&public_keys=public_keys>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. @@ -143,8 +141,6 @@ Transactions :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. - :query boolean is_asset: A flag to indicate if the ``asset`` field of the transaction is ``null`` or not. - :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. @@ -221,37 +217,6 @@ Transactions :statuscode 200: A list of transactions containing an asset with ID ``asset_id`` was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. -.. http:get:: /transactions?operation=CREATE&is_asset=true&public_keys={public_keys} - - Get a list of ``CREATE`` transactions that have the asset field defined. - This can serve as a recipe for retrieving your list of assets. - Currently, filtering on specific fields in the ``asset`` or ``metadata`` is assumed to be done clientside. - - This endpoint returns assets only if the transaction they're in are - included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - - :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. - - :query boolean is_asset: A flag to indicate if the ``asset`` field of the transaction is ``null`` or not. - - :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - - - **Example request**: - - .. literalinclude:: samples/get-assets-request.http - :language: http - - **Example response**: - - .. literalinclude:: samples/get-assets-response.http - :language: http - - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A list of transactions containing an asset and ``public_keys`` found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. the ``is_asset`` querystring was not included in the request. .. http:post:: /transactions From 6ea7b8a411339901155ae65aae11dcce10ccd0c2 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 11:08:21 +0100 Subject: [PATCH 070/219] remove is_assets sample --- .../generate_http_server_api_documentation.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 6f5ad04c..40204c8b 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -58,21 +58,6 @@ Content-Type: application/json %(tx_transfer_last)s] """ -TPLS['get-assets-request'] = """\ -GET /transactions?operation=CREATE&is_asset=true&public_keys=%(public_keys)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-assets-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -[%(tx)s] -""" - - TPLS['post-tx-request'] = """\ POST /transactions/ HTTP/1.1 Host: example.com From df94427ccaf2cc72f15bc79f0def33fe3162e77d Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 12:17:15 +0100 Subject: [PATCH 071/219] fulfilled conditions -> unspent outputs --- .../generate_http_server_api_documentation.py | 6 ++--- .../http-client-server-api.rst | 27 ++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 40204c8b..37d5facc 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -28,14 +28,14 @@ Content-Type: application/json """ -TPLS['get-tx-unfulfilled-request'] = """\ -GET /transactions?fulfilled=false&public_keys=%(public_keys_transfer_last)s HTTP/1.1 +TPLS['get-tx-unspent-request'] = """\ +GET /transactions?unspent=true&public_keys=%(public_keys_transfer_last)s HTTP/1.1 Host: example.com """ -TPLS['get-tx-unfulfilled-response'] = """\ +TPLS['get-tx-unspent-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 7b91c69d..76e3e74a 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -115,7 +115,7 @@ Transactions "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, "item": { "href": "https://example.com:9984/api/v0.9/transactions/{tx_id}" }, "self": { "href": "https://example.com:9984/api/v0.9/transactions" }, - "unfulfilled": { "href": "https://example.com:9984/api/v0.9/transactions?fulfilled=false&public_keys={public_keys}" } + "unspent": { "href": "https://example.com:9984/api/v0.9/transactions?unspent=true&public_keys={public_keys}" } }, "version" : "0.9.0" } @@ -126,7 +126,7 @@ Transactions queried correctly. Some of them include retrieving a list of transactions that include: - * `Unfulfilled outputs <#get--transactions?fulfilled=false&public_keys=public_keys>`_ + * `Unspent outputs <#get--transactions?unspent=true&public_keys=public_keys>`_ * `Transactions related to a specific asset <#get--transactions?operation=GENESIS|CREATE|TRANSFER&asset_id=asset_id>`_ In this section, we've listed those particular requests, as they will likely @@ -139,7 +139,7 @@ Transactions A generalization of those parameters follows: - :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. + :query boolean unspent: A flag to indicate whether only transactions with unspent outputs should be returned. :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. @@ -148,36 +148,37 @@ Transactions :query string asset_id: asset ID. -.. http:get:: /transactions?fulfilled=false&public_keys={public_keys} +.. http:get:: /transactions?unspent=true&public_keys={public_keys} - Get a list of transactions with unfulfilled conditions. + Get a list of transactions with unspent outputs. - If the querystring ``fulfilled`` is set to ``false`` and all conditions for - ``public_keys`` happen to be fulfilled already, this endpoint will return - an empty list. + If the querystring ``unspent`` is set to ``false`` and all outputs for + ``public_keys`` happen to be spent already, this endpoint will return + an empty list. Transactions with multiple outputs that have not all been spent + will be included in the response. - This endpoint returns conditions only if the transaction they're in are + This endpoint returns transactions only if they are included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - :query boolean fulfilled: A flag to indicate if transactions with fulfilled conditions should be returned. + :query boolean unspent: A flag to indicate if transactions with unspent outputs should be returned. :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. **Example request**: - .. literalinclude:: samples/get-tx-unfulfilled-request.http + .. literalinclude:: samples/get-tx-unspent-request.http :language: http **Example response**: - .. literalinclude:: samples/get-tx-unfulfilled-response.http + .. literalinclude:: samples/get-tx-unspent-response.http :language: http :resheader Content-Type: ``application/json`` - :statuscode 200: A list of transactions containing unfulfilled conditions was found and returned. + :statuscode 200: A list of transactions containing unspent outputs was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``public_keys`` querystring was not included in the request. .. http:get:: /transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} From 7576210c82dfb7ec6c6adfbfacca45c5ed3aa00c Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 13:39:38 +0100 Subject: [PATCH 072/219] remove GENESIS from operations --- .../source/drivers-clients/http-client-server-api.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 76e3e74a..8437cfc9 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -111,7 +111,7 @@ Transactions { "_links": { - "assets": { "href": "https://example.com:9984/api/v0.9/transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id}" }, + "assets": { "href": "https://example.com:9984/api/v0.9/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id}" }, "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, "item": { "href": "https://example.com:9984/api/v0.9/transactions/{tx_id}" }, "self": { "href": "https://example.com:9984/api/v0.9/transactions" }, @@ -127,7 +127,7 @@ Transactions that include: * `Unspent outputs <#get--transactions?unspent=true&public_keys=public_keys>`_ - * `Transactions related to a specific asset <#get--transactions?operation=GENESIS|CREATE|TRANSFER&asset_id=asset_id>`_ + * `Transactions related to a specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. @@ -143,7 +143,7 @@ Transactions :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. + :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. :query string asset_id: asset ID. @@ -181,7 +181,7 @@ Transactions :statuscode 200: A list of transactions containing unspent outputs was found and returned. :statuscode 400: The request wasn't understood by the server, e.g. the ``public_keys`` querystring was not included in the request. -.. http:get:: /transactions?operation={GENESIS|CREATE|TRANSFER}&asset_id={asset_id} +.. http:get:: /transactions?operation={CREATE|TRANSFER}&asset_id={asset_id} Get a list of transactions that use an asset with the ID ``asset_id``. @@ -199,7 +199,7 @@ Transactions any case only be one transaction returned (in a list though, since ``/transactions`` is a list-returning endpoint). - :query string operation: One of the three supported operations of a transaction: ``GENESIS``, ``CREATE``, ``TRANSFER``. + :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. :query string asset_id: asset ID. From 9d6ca861a7939e74bd4940610a9c7d1dca325667 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 13:52:03 +0100 Subject: [PATCH 073/219] Note on retrieving the list of assets --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 8437cfc9..a17486d0 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -198,6 +198,8 @@ Transactions ``/transactions?operation=CREATE&asset_id={asset_id}``, there will in any case only be one transaction returned (in a list though, since ``/transactions`` is a list-returning endpoint). + Leaving out the ``asset_id`` query and calling + ``/transactions?operation=CREATE`` returns the list of assets. :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. From 1ef2da7c32248c54d1dbd377ea2736506f69e714 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 14:22:37 +0100 Subject: [PATCH 074/219] mention history and provenance --- .../server/source/drivers-clients/http-client-server-api.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index a17486d0..3a3bdd71 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -184,8 +184,11 @@ Transactions .. http:get:: /transactions?operation={CREATE|TRANSFER}&asset_id={asset_id} Get a list of transactions that use an asset with the ID ``asset_id``. + Every ``TRANSFER`` transaction that originates from a ``CREATE`` transaction + with ``asset_id`` will be included. This allows users to query the entire history or + provenance of an asset. - This endpoint returns assets only if the transaction they're in are + This endpoint returns transactions only if they are included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. .. note:: From 005a164e6b86174719a6cf0a2fa31c937f9a1072 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 15:15:17 +0100 Subject: [PATCH 075/219] additional info on INVALID block --- .../server/source/drivers-clients/http-client-server-api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 3a3bdd71..f60c5f0e 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -303,12 +303,13 @@ Blocks Get the block with the ID ``block_id``. .. note:: - As ``status``'s default value is set to ``VALID``, only ``VALID`` blocks + As ``status``'s default value is set to ``VALID``, hence only ``VALID`` blocks will be returned by this endpoint. In case ``status=VALID``, but a block that was labeled ``UNDECIDED`` or ``INVALID`` is requested by ``block_id``, this endpoint will return a ``404 Not Found`` status code to warn the user. To check a block's status independently, use the - `Statuses endpoint <#get--statuses-tx_id|block_id>`_. + `Statuses endpoint <#get--statuses-tx_id|block_id>`_. The ``INVALID`` status + can be handy to figure out why the block was rejected. :param block_id: block ID :type block_id: hex string From fde3d21ba71776fd55fbcc0c07c69f3a43477028 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 15:22:22 +0100 Subject: [PATCH 076/219] POST /transactions returns statuses payload --- docs/server/generate_http_server_api_documentation.py | 4 ++++ docs/server/source/drivers-clients/http-client-server-api.rst | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 37d5facc..65ce61c0 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -71,6 +71,10 @@ TPLS['post-tx-response'] = """\ HTTP/1.1 202 Accepted Content-Type: application/json Location: ../statuses/%(txid)s + +{ + "status": "/statuses/%(txid)s" +} """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index f60c5f0e..5d59b85d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -226,7 +226,8 @@ Transactions .. http:post:: /transactions - Push a new transaction. + Push a new transaction. The endpoint will return a ``statuses`` endpoint to track + the status of the transaction. .. note:: The posted transaction should be valid `transaction From 6cf39542eae6fec3a9b1bb41242cf0596e47af67 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 5 Jan 2017 16:06:27 +0100 Subject: [PATCH 077/219] improve wording on post transactions status codes --- .../source/drivers-clients/http-client-server-api.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 5d59b85d..24e354c7 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -247,8 +247,11 @@ Transactions .. literalinclude:: samples/post-tx-response.http :language: http - :statuscode 202: The pushed transaction was accepted, but the processing has not been completed. - :statuscode 400: The transaction was invalid and not created. + :resheader Content-Type: ``application/json`` + :resheader Location: As the transaction will be persisted asynchronously, an endpoint to monitor its status is provided in this header. + + :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. + :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. Statuses From 8bbaa0e40ed210b2c956c55e94d294ad3bbf64e3 Mon Sep 17 00:00:00 2001 From: diminator Date: Fri, 6 Jan 2017 11:58:34 +0100 Subject: [PATCH 078/219] status endpoint query params and remove links --- .../generate_http_server_api_documentation.py | 17 +++++------------ .../drivers-clients/http-client-server-api.rst | 7 ++++--- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 65ce61c0..ed76ed15 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -70,16 +70,16 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ HTTP/1.1 202 Accepted Content-Type: application/json -Location: ../statuses/%(txid)s +Location: ../statuses?tx_id=%(txid)s { - "status": "/statuses/%(txid)s" + "status": "/statuses?tx_id=%(txid)s" } """ TPLS['get-statuses-tx-request'] = """\ -GET /statuses/%(txid)s HTTP/1.1 +GET /statuses?tx_id=%(txid)s HTTP/1.1 Host: example.com """ @@ -90,10 +90,7 @@ HTTP/1.1 200 OK Content-Type: application/json { - "status": "invalid", - "_links" : { - "block": "/blocks/%(blockid)s" - } + "status": "invalid" } """ @@ -104,11 +101,7 @@ Content-Type: application/json Location: ../transactions/%(txid)s { - "status": "valid", - "_links" : { - "tx" : "/transactions/%(txid)s", - "block": "/blocks/%(blockid)s" - } + "status": "valid" } """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 24e354c7..5cbb7bed 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -257,12 +257,13 @@ Transactions Statuses -------------------------------- -.. http:get:: /statuses/{tx_id|block_id} +.. http:get:: /statuses?tx_id={tx_id}|block_id={block_id} Get the status of an asynchronously written resource by their id. Supports the retrieval of a status for a transaction using ``tx_id`` or the - retrieval of a status for a block using ``block_id``. + retrieval of a status for a block using ``block_id``. Only use exactly one of both + queries, as they are required but mutually exclusive. The possible status values are ``backlog``, ``undecided``, ``valid`` or ``invalid``. @@ -312,7 +313,7 @@ Blocks that was labeled ``UNDECIDED`` or ``INVALID`` is requested by ``block_id``, this endpoint will return a ``404 Not Found`` status code to warn the user. To check a block's status independently, use the - `Statuses endpoint <#get--statuses-tx_id|block_id>`_. The ``INVALID`` status + `Statuses endpoint <#get--statuses?tx_id=tx_id|block_id=block_id>`_. The ``INVALID`` status can be handy to figure out why the block was rejected. :param block_id: block ID From cb4207fb32ab324942451a2411ae14e590e6315e Mon Sep 17 00:00:00 2001 From: diminator Date: Fri, 6 Jan 2017 13:55:17 +0100 Subject: [PATCH 079/219] version 0.9 to api version 1 --- .../http-client-server-api.rst | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 5cbb7bed..5bde1207 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -41,8 +41,8 @@ API Root Endpoint ------------------- If you send an HTTP GET request to the API Root Endpoint -e.g. ``http://localhost:9984/api/v0.9/`` -or ``https://example.com:9984/api/v0.9/``, +e.g. ``http://localhost:9984/api/v1/`` +or ``https://example.com:9984/api/v1/``, then you should get an HTTP response that allows you to discover the BigchainDB API endpoints: @@ -50,12 +50,12 @@ that allows you to discover the BigchainDB API endpoints: { "_links": { - "blocks": { "href": "https://example.com:9984/api/v0.9/blocks" }, + "blocks": { "href": "https://example.com:9984/api/v1/blocks" }, "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "self": { "href": "https://example.com:9984/api/v0.9" }, - "statuses": { "href": "https://example.com:9984/api/v0.9/statuses" }, - "transactions": { "href": "https://example.com:9984/api/v0.9/transactions" }, - "votes": { "href": "https://example.com:9984/api/v0.9/votes" } + "self": { "href": "https://example.com:9984/api/v1" }, + "statuses": { "href": "https://example.com:9984/api/v1/statuses" }, + "transactions": { "href": "https://example.com:9984/api/v1/transactions" }, + "votes": { "href": "https://example.com:9984/api/v1/votes" } }, "version" : "0.9.0" } @@ -111,11 +111,11 @@ Transactions { "_links": { - "assets": { "href": "https://example.com:9984/api/v0.9/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id}" }, + "assets": { "href": "https://example.com:9984/api/v1/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id}" }, "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "item": { "href": "https://example.com:9984/api/v0.9/transactions/{tx_id}" }, - "self": { "href": "https://example.com:9984/api/v0.9/transactions" }, - "unspent": { "href": "https://example.com:9984/api/v0.9/transactions?unspent=true&public_keys={public_keys}" } + "item": { "href": "https://example.com:9984/api/v1/transactions/{tx_id}" }, + "self": { "href": "https://example.com:9984/api/v1/transactions" }, + "unspent": { "href": "https://example.com:9984/api/v1/transactions?unspent=true&public_keys={public_keys}" } }, "version" : "0.9.0" } @@ -362,10 +362,10 @@ Blocks { "_links": { - "blocks": { "href": "https://example.com:9984/api/v0.9/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}" }, + "blocks": { "href": "https://example.com:9984/api/v1/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}" }, "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "item": { "href": "https://example.com:9984/api/v0.9/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}" }, - "self": { "href": "https://example.com:9984/api/v0.9/blocks" } + "item": { "href": "https://example.com:9984/api/v1/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}" }, + "self": { "href": "https://example.com:9984/api/v1/blocks" } }, "version" : "0.9.0" } @@ -441,7 +441,7 @@ Determining the API Root URL When you start BigchainDB Server using ``bigchaindb start``, an HTTP API is exposed at some address. The default is: -`http://localhost:9984/api/v0.9/ `_ +`http://localhost:9984/api/v1/ `_ It's bound to ``localhost``, so you can access it from the same machine, From 7d7a05c7064c5782e595952ff9d3f63ff7f11335 Mon Sep 17 00:00:00 2001 From: Dimitri De Jonghe Date: Fri, 6 Jan 2017 14:45:15 +0100 Subject: [PATCH 080/219] remove href from links --- .../http-client-server-api.rst | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 5bde1207..15cf8a1d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -25,7 +25,7 @@ with something like the following in the body: { "_links": { - "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/" } + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/" }, "keyring": [ "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", @@ -50,12 +50,12 @@ that allows you to discover the BigchainDB API endpoints: { "_links": { - "blocks": { "href": "https://example.com:9984/api/v1/blocks" }, - "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "self": { "href": "https://example.com:9984/api/v1" }, - "statuses": { "href": "https://example.com:9984/api/v1/statuses" }, - "transactions": { "href": "https://example.com:9984/api/v1/transactions" }, - "votes": { "href": "https://example.com:9984/api/v1/votes" } + "blocks": "https://example.com:9984/api/v1/blocks", + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", + "self": "https://example.com:9984/api/v1", + "statuses": "https://example.com:9984/api/v1/statuses", + "transactions": "https://example.com:9984/api/v1/transactions", + "votes": "https://example.com:9984/api/v1/votes" }, "version" : "0.9.0" } @@ -111,11 +111,11 @@ Transactions { "_links": { - "assets": { "href": "https://example.com:9984/api/v1/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id}" }, - "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "item": { "href": "https://example.com:9984/api/v1/transactions/{tx_id}" }, - "self": { "href": "https://example.com:9984/api/v1/transactions" }, - "unspent": { "href": "https://example.com:9984/api/v1/transactions?unspent=true&public_keys={public_keys}" } + "assets": "https://example.com:9984/api/v1/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id}", + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", + "item": "https://example.com:9984/api/v1/transactions/{tx_id}", + "self": "https://example.com:9984/api/v1/transactions", + "unspent": "https://example.com:9984/api/v1/transactions?unspent=true&public_keys={public_keys}" }, "version" : "0.9.0" } @@ -362,10 +362,10 @@ Blocks { "_links": { - "blocks": { "href": "https://example.com:9984/api/v1/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}" }, - "docs": { "href": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html" }, - "item": { "href": "https://example.com:9984/api/v1/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}" }, - "self": { "href": "https://example.com:9984/api/v1/blocks" } + "blocks": "https://example.com:9984/api/v1/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}", + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", + "item": "https://example.com:9984/api/v1/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}", + "self": "https://example.com:9984/api/v1/blocks" }, "version" : "0.9.0" } From 9e163ed2e5402373461b49c5e9cb201b36610849 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 6 Jan 2017 15:19:42 +0100 Subject: [PATCH 081/219] update api root informational endpoint docs --- docs/server/source/drivers-clients/http-client-server-api.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 15cf8a1d..f580ef5b 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -26,6 +26,7 @@ with something like the following in the body: { "_links": { "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/" + "api_v1": "http://example.com:9984/api/v1/" }, "keyring": [ "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", @@ -50,12 +51,10 @@ that allows you to discover the BigchainDB API endpoints: { "_links": { - "blocks": "https://example.com:9984/api/v1/blocks", "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", "self": "https://example.com:9984/api/v1", "statuses": "https://example.com:9984/api/v1/statuses", "transactions": "https://example.com:9984/api/v1/transactions", - "votes": "https://example.com:9984/api/v1/votes" }, "version" : "0.9.0" } From 60b21fd24c738d7b1aaeffa44b7f2a78c07dac78 Mon Sep 17 00:00:00 2001 From: Dimitri De Jonghe Date: Mon, 9 Jan 2017 16:55:16 +0100 Subject: [PATCH 082/219] add comma --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index f580ef5b..97c8f017 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -25,7 +25,7 @@ with something like the following in the body: { "_links": { - "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/" + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/", "api_v1": "http://example.com:9984/api/v1/" }, "keyring": [ From 4631c93dbf09fb543e8c0d97e578b9a978ee13b9 Mon Sep 17 00:00:00 2001 From: diminator Date: Mon, 9 Jan 2017 17:00:36 +0100 Subject: [PATCH 083/219] statuses remove 303 and location, links in payload --- docs/server/generate_http_server_api_documentation.py | 8 +++++--- .../source/drivers-clients/http-client-server-api.rst | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index ed76ed15..1581a5fe 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -96,12 +96,14 @@ Content-Type: application/json TPLS['get-statuses-tx-valid-response'] = """\ -HTTP/1.1 303 See Other +HTTP/1.1 200 OK Content-Type: application/json -Location: ../transactions/%(txid)s { - "status": "valid" + "status": "valid", + "_links": { + "tx": "/transactions/%(txid)s" + } } """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 97c8f017..18dc0dbd 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -268,8 +268,8 @@ Statuses ``invalid``. If a transaction or block is persisted to the chain and it's status is set - to ``valid`` or ``undecided``, a ``303 See Other`` status code is returned, - as well as an URL to the resource in the location header. + to ``valid`` or ``undecided``, a ``200`` status code is returned, + as well as an URL to the resource. :param tx_id: transaction ID :type tx_id: hex string @@ -295,8 +295,7 @@ Statuses :resheader Content-Type: ``application/json`` :resheader Location: Once the transaction has been persisted, this header will link to the actual resource. - :statuscode 200: A transaction or block with that ID was found. The status is either ``backlog``, ``invalid``. - :statuscode 303: A transaction or block with that ID was found and persisted to the chain. A location header to the resource is provided. + :statuscode 200: A transaction or block with that ID was found. :statuscode 404: A transaction or block with that ID was not found. Blocks From 86be0e5983b5016b148350c05942127bad357cb1 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 Jan 2017 10:48:52 +0100 Subject: [PATCH 084/219] move blocks and votes to advanced with intro --- .../generate_http_server_api_documentation.py | 13 ++- .../http-client-server-api.rst | 95 +++++++++++-------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 1581a5fe..131b6a0a 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -134,7 +134,7 @@ TPLS['get-block-txid-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json -[%(block)s] +%(block_status)s """ @@ -194,12 +194,22 @@ def main(): block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) block_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) + block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, voters=[node_public], signature=signature) + block_transfer_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) + # vote DUMMY_SHA3 = '0123456789abcdef' * 4 b = Bigchain(public_key=node_public, private_key=node_private) vote = b.vote(block.id, DUMMY_SHA3, True) vote_json = json.dumps(vote, indent=2, sort_keys=True) + # block status + block_status = { + block_transfer.id: 'invalid', + block.id: 'valid' + } + block_status_json = json.dumps(block_status, indent=2, sort_keys=True) + base_path = os.path.join(os.path.dirname(__file__), 'source/drivers-clients/samples') if not os.path.exists(base_path): @@ -218,6 +228,7 @@ def main(): 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0], 'block': block_json, 'blockid': block.id, + 'block_status': block_status_json, 'vote': vote_json} with open(path, 'w') as handle: handle.write(code) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 18dc0dbd..b9b1e59c 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -298,9 +298,58 @@ Statuses :statuscode 200: A transaction or block with that ID was found. :statuscode 404: A transaction or block with that ID was not found. -Blocks +Advanced Usage -------------------------------- +The following endpoints are more advanced and meant for debugging and transparency purposes. + +More precisely, the `blocks endpoint <#blocks>`_ allows you to retrieve a block by ``block_id`` as well the list of blocks that +a certain transaction with ``tx_id`` occured in (a transaction can occur in multiple ``invalid`` blocks until it +either gets rejected or validated by the system). This endpoint gives the ability to drill down on the lifecycle of a +transaction + +The `votes endpoint <#votes>`_ contains all the voting information for a specific block. So after retrieving the +``block_id`` for a given ``tx_id``, one can now simply inspect the votes that happened at a specific time on that block. + + + +Blocks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. http:get:: /blocks + + The unfiltered ``/blocks`` endpoint without any query parameters + returns a list of available block usages and relevant endpoints. + We believe a PUSH rather than a PULL pattern is more appropriate, as the + items returned in the collection would change by the second. + + + **Example request**: + + .. sourcecode:: http + + GET /blocks HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "_links": { + "blocks": "https://example.com:9984/api/v1/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}", + "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", + "item": "https://example.com:9984/api/v1/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}", + "self": "https://example.com:9984/api/v1/blocks" + }, + "version" : "0.9.0" + } + + :statuscode 200: BigchainDB blocks root endpoint. + .. http:get:: /blocks/{block_id}?status={VALID|UNDECIDED|INVALID} Get the block with the ID ``block_id``. @@ -336,48 +385,12 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. :statuscode 404: A block with that ID and a certain ``status`` was not found. -.. http:get:: /blocks +.. http:get:: /blocks?tx_id={tx_id} - The unfiltered ``/blocks`` endpoint without any query parameters - returns a list of available block usages and relevant endpoints. - We believe a PUSH rather than a PULL pattern is more appropriate, as the - items returned in the collection would change by the second. - - - **Example request**: - - .. sourcecode:: http - - GET /blocks HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "_links": { - "blocks": "https://example.com:9984/api/v1/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}", - "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", - "item": "https://example.com:9984/api/v1/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}", - "self": "https://example.com:9984/api/v1/blocks" - }, - "version" : "0.9.0" - } - - :statuscode 200: BigchainDB blocks root endpoint. - - -.. http:get:: /blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID} - - Retrieve a list of blocks that contain a transaction with the ID ``tx_id``. + Retrieve a list of ``block_id`` with their corresponding status that contain a transaction with the ID ``tx_id``. Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be - returned. To filter blocks by their status, use the optional ``status`` - querystring. + returned. .. note:: In case no block was found, an empty list and an HTTP status code @@ -403,7 +416,7 @@ Blocks Votes --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. http:get:: /votes?block_id={block_id} From a6facc8ada03b622c22c26a67a3dae99555ee8d9 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 Jan 2017 11:33:25 +0100 Subject: [PATCH 085/219] reformat blocks + blocks endpoint as list --- .../generate_http_server_api_documentation.py | 14 +-- .../http-client-server-api.rst | 87 ++++++++----------- 2 files changed, 42 insertions(+), 59 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 131b6a0a..975e6550 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -134,7 +134,7 @@ TPLS['get-block-txid-response'] = """\ HTTP/1.1 200 OK Content-Type: application/json -%(block_status)s +%(block_list)s """ @@ -204,11 +204,11 @@ def main(): vote_json = json.dumps(vote, indent=2, sort_keys=True) # block status - block_status = { - block_transfer.id: 'invalid', - block.id: 'valid' - } - block_status_json = json.dumps(block_status, indent=2, sort_keys=True) + block_list = [ + block_transfer.id, + block.id + ] + block_list_json = json.dumps(block_list, indent=2, sort_keys=True) base_path = os.path.join(os.path.dirname(__file__), 'source/drivers-clients/samples') @@ -228,7 +228,7 @@ def main(): 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0], 'block': block_json, 'blockid': block.id, - 'block_status': block_status_json, + 'block_list': block_list_json, 'vote': vote_json} with open(path, 'w') as handle: handle.write(code) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index b9b1e59c..4115ed64 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -316,58 +316,15 @@ The `votes endpoint <#votes>`_ contains all the voting information for a specifi Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. http:get:: /blocks +.. http:get:: /blocks/{block_id} - The unfiltered ``/blocks`` endpoint without any query parameters - returns a list of available block usages and relevant endpoints. - We believe a PUSH rather than a PULL pattern is more appropriate, as the - items returned in the collection would change by the second. - - - **Example request**: - - .. sourcecode:: http - - GET /blocks HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "_links": { - "blocks": "https://example.com:9984/api/v1/blocks?tx_id={tx_id}&status={VALID|UNDECIDED|INVALID}", - "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", - "item": "https://example.com:9984/api/v1/blocks/{block_id}?status={VALID|UNDECIDED|INVALID}", - "self": "https://example.com:9984/api/v1/blocks" - }, - "version" : "0.9.0" - } - - :statuscode 200: BigchainDB blocks root endpoint. - -.. http:get:: /blocks/{block_id}?status={VALID|UNDECIDED|INVALID} - - Get the block with the ID ``block_id``. - - .. note:: - As ``status``'s default value is set to ``VALID``, hence only ``VALID`` blocks - will be returned by this endpoint. In case ``status=VALID``, but a block - that was labeled ``UNDECIDED`` or ``INVALID`` is requested by - ``block_id``, this endpoint will return a ``404 Not Found`` status code - to warn the user. To check a block's status independently, use the - `Statuses endpoint <#get--statuses?tx_id=tx_id|block_id=block_id>`_. The ``INVALID`` status - can be handy to figure out why the block was rejected. + Get the block with the ID ``block_id``. Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be + returned. To check a block's status independently, use the `Statuses endpoint <#get--statuses?tx_id=tx_id|block_id=block_id>`_. + To check the votes on a block, have a look at the `votes endpoint <#votes>`_. :param block_id: block ID :type block_id: hex string - :query string status: Per default set to ``VALID``. One of ``VALID``, ``UNDECIDED`` or ``INVALID``. - **Example request**: .. literalinclude:: samples/get-block-request.http @@ -383,20 +340,44 @@ Blocks :statuscode 200: A block with that ID was found. :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. - :statuscode 404: A block with that ID and a certain ``status`` was not found. + :statuscode 404: A block with that ID was not found. -.. http:get:: /blocks?tx_id={tx_id} + +.. http:get:: /blocks + + The unfiltered ``/blocks`` endpoint without any query parameters returns a `400` status code. + The list endpoint should be filtered with a ``tx_id`` query parameter, + see the ``/blocks?tx_id={tx_id}&status=UNDECIDED|VALID|INVALID`` + `endpoint <#get--blocks?tx_id=tx_id&status=UNDECIDED|VALID|INVALID>`_. + + + **Example request**: + + .. sourcecode:: http + + GET /blocks HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 400 OK + + :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. + +.. http:get:: /blocks?tx_id={tx_id}&status={UNDECIDED|VALID|INVALID} Retrieve a list of ``block_id`` with their corresponding status that contain a transaction with the ID ``tx_id``. - Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be - returned. + Any blocks, be they ``UNDECIDED``, ``VALID`` or ``INVALID`` will be + returned if no status filter is provided. .. note:: In case no block was found, an empty list and an HTTP status code ``200 OK`` is returned, as the request was still successful. - :query string tx_id: transaction ID + :query string tx_id: transaction ID *(required)* :query string status: Filter blocks by their status. One of ``VALID``, ``UNDECIDED`` or ``INVALID``. **Example request**: @@ -415,6 +396,8 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``tx_id``. + + Votes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From deba454eb0c48f63c9235c6c7a8576692115e006 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 Jan 2017 11:34:35 +0100 Subject: [PATCH 086/219] 400 OK -> bad request --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 4115ed64..bc572e5b 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -362,7 +362,7 @@ Blocks .. sourcecode:: http - HTTP/1.1 400 OK + HTTP/1.1 400 Bad Request :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. From 4dea64f3beba01aa9f5962f2e77734fcf9ea4bdd Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 Jan 2017 11:38:57 +0100 Subject: [PATCH 087/219] /transactions root endpoint returns 400 --- .../http-client-server-api.rst | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index bc572e5b..2990af44 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -90,9 +90,7 @@ Transactions .. http:get:: /transactions The unfiltered ``/transactions`` endpoint without any query parameters - returns a list of available transaction usages and relevant endpoints. - We believe a PUSH rather than a PULL pattern is more appropriate, as the - items returned in the collection would change by the second. + returns a status code `400`. For valid filters, see the sections below. **Example request**: @@ -105,21 +103,9 @@ Transactions .. sourcecode:: http - HTTP/1.1 200 OK - Content-Type: application/json + HTTP/1.1 400 Bad Request - { - "_links": { - "assets": "https://example.com:9984/api/v1/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id}", - "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", - "item": "https://example.com:9984/api/v1/transactions/{tx_id}", - "self": "https://example.com:9984/api/v1/transactions", - "unspent": "https://example.com:9984/api/v1/transactions?unspent=true&public_keys={public_keys}" - }, - "version" : "0.9.0" - } - - :statuscode 200: BigchainDB transactions root endpoint. + :statuscode 400: The request wasn't understood by the server, a mandatory querystring was not included in the request. There are however filtered requests that might come of use, given the endpoint is queried correctly. Some of them include retrieving a list of transactions From ec5000bd47fd0520bf96300a0d3af07f15a41d49 Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 11 Jan 2017 12:03:28 +0100 Subject: [PATCH 088/219] add curly brackets around url --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 2990af44..552f4d39 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -333,7 +333,7 @@ Blocks The unfiltered ``/blocks`` endpoint without any query parameters returns a `400` status code. The list endpoint should be filtered with a ``tx_id`` query parameter, - see the ``/blocks?tx_id={tx_id}&status=UNDECIDED|VALID|INVALID`` + see the ``/blocks?tx_id={tx_id}&status={UNDECIDED|VALID|INVALID}`` `endpoint <#get--blocks?tx_id=tx_id&status=UNDECIDED|VALID|INVALID>`_. From 7d5d79a50cdecaf21337e25e37e3c06006907049 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 12 Jan 2017 16:19:49 +0100 Subject: [PATCH 089/219] remove location header from status --- docs/server/generate_http_server_api_documentation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 975e6550..1c6301aa 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -70,7 +70,6 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ HTTP/1.1 202 Accepted Content-Type: application/json -Location: ../statuses?tx_id=%(txid)s { "status": "/statuses?tx_id=%(txid)s" From d68d70ebc310ab4eb1dd25ab13b6f5a642e77da6 Mon Sep 17 00:00:00 2001 From: diminator Date: Thu, 12 Jan 2017 16:48:58 +0100 Subject: [PATCH 090/219] improved documentation on statuses links and values --- .../drivers-clients/http-client-server-api.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 552f4d39..e6ef58bb 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -248,14 +248,12 @@ Statuses Supports the retrieval of a status for a transaction using ``tx_id`` or the retrieval of a status for a block using ``block_id``. Only use exactly one of both - queries, as they are required but mutually exclusive. + queries, as they are required but mutually exclusive. A URL to the resource is also + provided under ``_links``. - The possible status values are ``backlog``, ``undecided``, ``valid`` or - ``invalid``. - - If a transaction or block is persisted to the chain and it's status is set - to ``valid`` or ``undecided``, a ``200`` status code is returned, - as well as an URL to the resource. + The possible status values are ``undecided``, ``valid`` or + ``invalid`` for both blocks and transactions. An additional state ``backlog`` is provided + for transactions. :param tx_id: transaction ID :type tx_id: hex string From 787512014a6b6221c8f02425edba9de909b6690d Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Fri, 13 Jan 2017 16:03:18 +0100 Subject: [PATCH 091/219] Fix small typos in API docs --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index e6ef58bb..36084b86 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -34,7 +34,7 @@ with something like the following in the body: ], "public_key": "AiygKSRhZWTxxYT4AfgKoTG4TZAoPsWoEt6C6bLq4jJR", "software": "BigchainDB", - "version": "0.9.0", + "version": "0.9.0" } From 2d1699e79063f78133a517fa1113879ef9c4329c Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Fri, 13 Jan 2017 17:59:40 +0100 Subject: [PATCH 092/219] Split block and transaction status endpoint descriptions --- .../generate_http_server_api_documentation.py | 30 +++++++++ .../http-client-server-api.rst | 66 ++++++++++++++----- 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 1c6301aa..b32bda02 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -107,6 +107,36 @@ Content-Type: application/json """ +TPLS['get-statuses-block-request'] = """\ +GET /statuses?block_id=%(blockid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-statuses-block-invalid-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "status": "invalid" +} +""" + + +TPLS['get-statuses-block-valid-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "status": "valid", + "_links": { + "block": "/blocks/%(blockid)s" + } +} +""" + + TPLS['get-block-request'] = """\ GET /blocks/%(blockid)s HTTP/1.1 Host: example.com diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 36084b86..75dbffe0 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -242,24 +242,31 @@ Transactions Statuses -------------------------------- -.. http:get:: /statuses?tx_id={tx_id}|block_id={block_id} +.. http:get:: /statuses - Get the status of an asynchronously written resource by their id. + Get the status of an asynchronously written transaction or block by their id. - Supports the retrieval of a status for a transaction using ``tx_id`` or the - retrieval of a status for a block using ``block_id``. Only use exactly one of both - queries, as they are required but mutually exclusive. A URL to the resource is also - provided under ``_links``. - - The possible status values are ``undecided``, ``valid`` or - ``invalid`` for both blocks and transactions. An additional state ``backlog`` is provided + The possible status values are ``undecided``, ``valid`` or ``invalid`` for + both blocks and transactions. An additional state ``backlog`` is provided for transactions. - :param tx_id: transaction ID - :type tx_id: hex string + A link to the resource is also provided in the returned payload under + ``_links``. - :param block_id: block ID - :type block_id: hex string + :query string tx_id: transaction ID + :query string block_id: block ID + + .. note:: + + Exactly one of the ``tx_id`` or ``block_id`` query parameters must be + used together with this endpoint (see below for getting `transaction + statuses <#get--statuses?tx_id=tx_id>`_ and `block statuses + <#get--statuses?block_id=block_id>`_). + + +.. http:get:: /statuses?tx_id={tx_id} + + Get the status of a transaction. **Example request**: @@ -279,8 +286,35 @@ Statuses :resheader Content-Type: ``application/json`` :resheader Location: Once the transaction has been persisted, this header will link to the actual resource. - :statuscode 200: A transaction or block with that ID was found. - :statuscode 404: A transaction or block with that ID was not found. + :statuscode 200: A transaction with that ID was found. + :statuscode 404: A transaction with that ID was not found. + + +.. http:get:: /statuses?block_id={block_id} + + Get the status of a block. + + **Example request**: + + .. literalinclude:: samples/get-statuses-block-request.http + :language: http + + **Example response**: + + .. literalinclude:: samples/get-statuses-block-invalid-response.http + :language: http + + **Example response**: + + .. literalinclude:: samples/get-statuses-block-valid-response.http + :language: http + + :resheader Content-Type: ``application/json`` + :resheader Location: Once the block has been persisted, this header will link to the actual resource. + + :statuscode 200: A block with that ID was found. + :statuscode 404: A block with that ID was not found. + Advanced Usage -------------------------------- @@ -303,7 +337,7 @@ Blocks .. http:get:: /blocks/{block_id} Get the block with the ID ``block_id``. Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be - returned. To check a block's status independently, use the `Statuses endpoint <#get--statuses?tx_id=tx_id|block_id=block_id>`_. + returned. To check a block's status independently, use the `Statuses endpoint <#status>`_. To check the votes on a block, have a look at the `votes endpoint <#votes>`_. :param block_id: block ID From d932184b8842456f1eba6efcdd50977c10fb2fa7 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 23 Jan 2017 14:30:29 +0100 Subject: [PATCH 093/219] update http docs to reflect changes implemented for 0.9 --- .../generate_http_server_api_documentation.py | 118 ++++++----- .../http-client-server-api.rst | 188 +++++++----------- 2 files changed, 136 insertions(+), 170 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index b32bda02..4407cda4 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -7,14 +7,28 @@ import os.path from bigchaindb.common.transaction import Transaction, Input, TransactionLink from bigchaindb.core import Bigchain from bigchaindb.models import Block - +from bigchaindb.web import server TPLS = {} +TPLS['index-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(index)s +""" + +TPLS['api-index-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(api_index)s +""" + TPLS['get-tx-id-request'] = """\ -GET /transactions/%(txid)s HTTP/1.1 +GET /api/v1/transactions/%(txid)s HTTP/1.1 Host: example.com """ @@ -28,23 +42,8 @@ Content-Type: application/json """ -TPLS['get-tx-unspent-request'] = """\ -GET /transactions?unspent=true&public_keys=%(public_keys_transfer_last)s HTTP/1.1 -Host: example.com - -""" - - -TPLS['get-tx-unspent-response'] = """\ -HTTP/1.1 200 OK -Content-Type: application/json - -[%(tx_transfer_last)s] -""" - - TPLS['get-tx-by-asset-request'] = """\ -GET /transactions?operation=transfer&asset_id=%(txid)s HTTP/1.1 +GET /api/v1/transactions?operation=TRANSFER&asset_id=%(txid)s HTTP/1.1 Host: example.com """ @@ -59,7 +58,7 @@ Content-Type: application/json """ TPLS['post-tx-request'] = """\ -POST /transactions/ HTTP/1.1 +POST /api/v1/transactions/ HTTP/1.1 Host: example.com Content-Type: application/json @@ -68,12 +67,10 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ -HTTP/1.1 202 Accepted +HTTP/1.1 200 OK Content-Type: application/json -{ - "status": "/statuses?tx_id=%(txid)s" -} +%(tx)s """ @@ -108,7 +105,7 @@ Content-Type: application/json TPLS['get-statuses-block-request'] = """\ -GET /statuses?block_id=%(blockid)s HTTP/1.1 +GET /api/v1/statuses?block_id=%(blockid)s HTTP/1.1 Host: example.com """ @@ -138,7 +135,7 @@ Content-Type: application/json TPLS['get-block-request'] = """\ -GET /blocks/%(blockid)s HTTP/1.1 +GET /api/v1/blocks/%(blockid)s HTTP/1.1 Host: example.com """ @@ -153,7 +150,7 @@ Content-Type: application/json TPLS['get-block-txid-request'] = """\ -GET /blocks?tx_id=%(txid)s HTTP/1.1 +GET /api/v1/blocks?tx_id=%(txid)s HTTP/1.1 Host: example.com """ @@ -168,7 +165,7 @@ Content-Type: application/json TPLS['get-vote-request'] = """\ -GET /votes?block_id=%(blockid)s HTTP/1.1 +GET /api/v1/votes?block_id=%(blockid)s HTTP/1.1 Host: example.com """ @@ -185,13 +182,37 @@ Content-Type: application/json def main(): """ Main function """ + ctx = {} + + def pretty_json(data): + return json.dumps(data, indent=2, sort_keys=True) + + client = server.create_app().test_client() + + host = 'example.com:9984' + + # HTTP Index + res = client.get('/', environ_overrides={'HTTP_HOST': host}) + res_data = json.loads(res.data.decode()) + res_data['keyring'] = [ + "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", + "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" + ] + ctx['index'] = pretty_json(res_data) + + # API index + res = client.get('/api/v1/', environ_overrides={'HTTP_HOST': host}) + ctx['api_index'] = pretty_json(json.loads(res.data.decode())) + # tx create privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' asset = {'msg': 'Hello BigchainDB!'} tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset, metadata={'sequence': 0}) tx = tx.sign([privkey]) - tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) + ctx['tx'] = pretty_json(tx.to_dict()) + ctx['public_keys'] = tx.outputs[0].public_keys[0] + ctx['txid'] = tx.id # tx transfer privkey_transfer = '3AeWpPdhEZzWLYfkfYHBfMFC2r1f8HEaGS9NtbbKssya' @@ -203,41 +224,48 @@ def main(): owners_before=tx.outputs[cid].public_keys) tx_transfer = Transaction.transfer([input_], [([pubkey_transfer], 1)], asset_id=tx.id, metadata={'sequence': 1}) tx_transfer = tx_transfer.sign([privkey]) - tx_transfer_json = json.dumps(tx_transfer.to_dict(), indent=2, sort_keys=True) + ctx['tx_transfer'] = pretty_json(tx_transfer.to_dict()) + ctx['public_keys_transfer'] = tx_transfer.outputs[0].public_keys[0] + ctx['tx_transfer_id'] = tx_transfer.id - privkey_transfer_last = 'sG3jWDtdTXUidBJK53ucSTrosktG616U3tQHBk81eQe' + # privkey_transfer_last = 'sG3jWDtdTXUidBJK53ucSTrosktG616U3tQHBk81eQe' pubkey_transfer_last = '3Af3fhhjU6d9WecEM9Uw5hfom9kNEwE7YuDWdqAUssqm' cid = 0 input_ = Input(fulfillment=tx_transfer.outputs[cid].fulfillment, fulfills=TransactionLink(txid=tx_transfer.id, output=cid), owners_before=tx_transfer.outputs[cid].public_keys) - tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)], asset_id=tx.id, metadata={'sequence': 2}) + tx_transfer_last = Transaction.transfer([input_], [([pubkey_transfer_last], 1)], + asset_id=tx.id, metadata={'sequence': 2}) tx_transfer_last = tx_transfer_last.sign([privkey_transfer]) - tx_transfer_last_json = json.dumps(tx_transfer_last.to_dict(), indent=2, sort_keys=True) + ctx['tx_transfer_last'] = pretty_json(tx_transfer_last.to_dict()) + ctx['tx_transfer_last_id'] = tx_transfer_last.id + ctx['public_keys_transfer_last'] = tx_transfer_last.outputs[0].public_keys[0] # block node_private = "5G2kE1zJAgTajkVSbPAQWo4c2izvtwqaNHYsaNpbbvxX" node_public = "DngBurxfeNVKZWCEcDnLj1eMPAS7focUZTE5FndFGuHT" signature = "53wxrEQDYk1dXzmvNSytbCfmNVnPqPkDQaTnAe8Jf43s6ssejPxezkCvUnGTnduNUmaLjhaan1iRLi3peu6s5DzA" block = Block(transactions=[tx], node_pubkey=node_public, voters=[node_public], signature=signature) - block_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) + ctx['block'] = pretty_json(block.to_dict()) + ctx['blockid'] = block.id - block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, voters=[node_public], signature=signature) - block_transfer_json = json.dumps(block.to_dict(), indent=2, sort_keys=True) + block_transfer = Block(transactions=[tx_transfer], node_pubkey=node_public, + voters=[node_public], signature=signature) + ctx['block_transfer'] = pretty_json(block.to_dict()) # vote DUMMY_SHA3 = '0123456789abcdef' * 4 b = Bigchain(public_key=node_public, private_key=node_private) vote = b.vote(block.id, DUMMY_SHA3, True) - vote_json = json.dumps(vote, indent=2, sort_keys=True) + ctx['vote'] = pretty_json(vote) # block status block_list = [ block_transfer.id, block.id ] - block_list_json = json.dumps(block_list, indent=2, sort_keys=True) + ctx['block_list'] = pretty_json(block_list) base_path = os.path.join(os.path.dirname(__file__), 'source/drivers-clients/samples') @@ -246,19 +274,7 @@ def main(): for name, tpl in TPLS.items(): path = os.path.join(base_path, name + '.http') - code = tpl % {'tx': tx_json, - 'txid': tx.id, - 'tx_transfer': tx_transfer_json, - 'tx_transfer_id': tx_transfer.id, - 'tx_transfer_last': tx_transfer_last_json, - 'tx_transfer_last_id': tx_transfer_last.id, - 'public_keys': tx.outputs[0].public_keys[0], - 'public_keys_transfer': tx_transfer.outputs[0].public_keys[0], - 'public_keys_transfer_last': tx_transfer_last.outputs[0].public_keys[0], - 'block': block_json, - 'blockid': block.id, - 'block_list': block_list_json, - 'vote': vote_json} + code = tpl % ctx with open(path, 'w') as handle: handle.write(code) @@ -270,5 +286,3 @@ def setup(*_): if __name__ == '__main__': main() - - diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 75dbffe0..72c22870 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -21,21 +21,8 @@ or ``https://example.com:9984`` then you should get an HTTP response with something like the following in the body: -.. code-block:: json - - { - "_links": { - "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/", - "api_v1": "http://example.com:9984/api/v1/" - }, - "keyring": [ - "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", - "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" - ], - "public_key": "AiygKSRhZWTxxYT4AfgKoTG4TZAoPsWoEt6C6bLq4jJR", - "software": "BigchainDB", - "version": "0.9.0" - } +.. literalinclude:: samples/index-response.http + :language: http API Root Endpoint @@ -47,22 +34,13 @@ or ``https://example.com:9984/api/v1/``, then you should get an HTTP response that allows you to discover the BigchainDB API endpoints: -.. code-block:: json - - { - "_links": { - "docs": "https://docs.bigchaindb.com/projects/server/en/v0.9.0/drivers-clients/http-client-server-api.html", - "self": "https://example.com:9984/api/v1", - "statuses": "https://example.com:9984/api/v1/statuses", - "transactions": "https://example.com:9984/api/v1/transactions", - }, - "version" : "0.9.0" - } +.. literalinclude:: samples/api-index-response.http + :language: http Transactions ------------------- -.. http:get:: /transactions/{tx_id} +.. http:get:: /api/v1/transactions/{tx_id} Get the transaction with the ID ``tx_id``. @@ -87,107 +65,38 @@ Transactions :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. -.. http:get:: /transactions +.. http:get:: /api/v1/transactions - The unfiltered ``/transactions`` endpoint without any query parameters + The unfiltered ``/api/v1/transactions`` endpoint without any query parameters returns a status code `400`. For valid filters, see the sections below. - **Example request**: - - .. sourcecode:: http - - GET /transactions HTTP/1.1 - Host: example.com - - **Example response**: - - .. sourcecode:: http - - HTTP/1.1 400 Bad Request - - :statuscode 400: The request wasn't understood by the server, a mandatory querystring was not included in the request. - There are however filtered requests that might come of use, given the endpoint is queried correctly. Some of them include retrieving a list of transactions that include: - * `Unspent outputs <#get--transactions?unspent=true&public_keys=public_keys>`_ * `Transactions related to a specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. .. note:: - Looking up transactions with a specific ``metadata`` field is currently not supported. - This functionality requires something like custom indexing per client or read-only followers, - which is not yet on the roadmap. + Looking up transactions with a specific ``metadata`` field is currently not supported, + however, providing a way to query based on ``metadata`` data is on our roadmap. A generalization of those parameters follows: - :query boolean unspent: A flag to indicate whether only transactions with unspent outputs should be returned. - - :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. - :query string asset_id: asset ID. + :query string asset_id: The ID of the asset. - -.. http:get:: /transactions?unspent=true&public_keys={public_keys} - - Get a list of transactions with unspent outputs. - - If the querystring ``unspent`` is set to ``false`` and all outputs for - ``public_keys`` happen to be spent already, this endpoint will return - an empty list. Transactions with multiple outputs that have not all been spent - will be included in the response. - - This endpoint returns transactions only if they are - included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - - :query boolean unspent: A flag to indicate if transactions with unspent outputs should be returned. - - :query string public_keys: Public key able to validly spend an output of a transaction, assuming the user also has the corresponding private key. - - **Example request**: - - - .. literalinclude:: samples/get-tx-unspent-request.http - :language: http - - - **Example response**: - - .. literalinclude:: samples/get-tx-unspent-response.http - :language: http - - :resheader Content-Type: ``application/json`` - - :statuscode 200: A list of transactions containing unspent outputs was found and returned. - :statuscode 400: The request wasn't understood by the server, e.g. the ``public_keys`` querystring was not included in the request. - -.. http:get:: /transactions?operation={CREATE|TRANSFER}&asset_id={asset_id} +.. http:get:: /api/v1/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id} Get a list of transactions that use an asset with the ID ``asset_id``. Every ``TRANSFER`` transaction that originates from a ``CREATE`` transaction with ``asset_id`` will be included. This allows users to query the entire history or provenance of an asset. - This endpoint returns transactions only if they are - included in the ``BACKLOG`` or in a ``VALID`` or ``UNDECIDED`` block on ``bigchain``. - - .. note:: - The BigchainDB API currently doesn't expose an - ``/assets/{asset_id}`` endpoint, as there wouldn't be any way for a - client to verify that what was received is consistent with what was - persisted in the database. - However, BigchainDB's consensus ensures that any ``asset_id`` is - a unique key identifying an asset, meaning that when calling - ``/transactions?operation=CREATE&asset_id={asset_id}``, there will in - any case only be one transaction returned (in a list though, since - ``/transactions`` is a list-returning endpoint). - Leaving out the ``asset_id`` query and calling - ``/transactions?operation=CREATE`` returns the list of assets. + This endpoint returns transactions only if they are decided ``VALID`` by the server. :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. @@ -209,14 +118,14 @@ Transactions :statuscode 400: The request wasn't understood by the server, e.g. the ``asset_id`` querystring was not included in the request. -.. http:post:: /transactions +.. http:post:: /api/v1/transactions - Push a new transaction. The endpoint will return a ``statuses`` endpoint to track - the status of the transaction. + Push a new transaction. .. note:: - The posted transaction should be valid `transaction - `_. + The posted `transaction + `_ + should be structurally valid and not spending an already spent output. The steps to build a valid transaction are beyond the scope of this page. One would normally use a driver such as the `BigchainDB Python Driver `_ @@ -233,16 +142,59 @@ Transactions :language: http :resheader Content-Type: ``application/json`` - :resheader Location: As the transaction will be persisted asynchronously, an endpoint to monitor its status is provided in this header. - :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. + :statuscode 200: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. +Transaction Outputs +------------------- + +The ``/api/v1/outputs`` endpoint returns transactions outputs filtered by a +given public key, and optionally filtered to only include outputs that have +not already been spent. + + +.. http:get:: /api/v1/outputs?public_key={public_key} + + Get transaction outputs by public key. The `public_key` parameter must be + a base58 encoded ed25519 public key associated with transaction output + ownership. + + Returns a list of links to transaction outputs. + + :param public_key: Base58 encoded public key associated with output ownership. This parameter is mandatory and without it the endpoint will return a ``400`` response code. + :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend. + + + **Example request**: + + .. sourcecode:: http + + GET /api/v1/outputs?public_key=1AAAbbb...ccc HTTP/1.1 + Host: example.com + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/0", + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/1" + ] + + :statuscode 200: A list of outputs were found and returned in the body of the response. + :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. + + + Statuses -------------------------------- -.. http:get:: /statuses +.. http:get:: /api/v1/statuses Get the status of an asynchronously written transaction or block by their id. @@ -264,7 +216,7 @@ Statuses <#get--statuses?block_id=block_id>`_). -.. http:get:: /statuses?tx_id={tx_id} +.. http:get:: /api/v1/statuses?tx_id={tx_id} Get the status of a transaction. @@ -290,7 +242,7 @@ Statuses :statuscode 404: A transaction with that ID was not found. -.. http:get:: /statuses?block_id={block_id} +.. http:get:: /api/v1/statuses?block_id={block_id} Get the status of a block. @@ -334,7 +286,7 @@ The `votes endpoint <#votes>`_ contains all the voting information for a specifi Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. http:get:: /blocks/{block_id} +.. http:get:: /api/v1/blocks/{block_id} Get the block with the ID ``block_id``. Any blocks, be they ``VALID``, ``UNDECIDED`` or ``INVALID`` will be returned. To check a block's status independently, use the `Statuses endpoint <#status>`_. @@ -361,7 +313,7 @@ Blocks :statuscode 404: A block with that ID was not found. -.. http:get:: /blocks +.. http:get:: /api/v1/blocks The unfiltered ``/blocks`` endpoint without any query parameters returns a `400` status code. The list endpoint should be filtered with a ``tx_id`` query parameter, @@ -373,7 +325,7 @@ Blocks .. sourcecode:: http - GET /blocks HTTP/1.1 + GET /api/v1/blocks HTTP/1.1 Host: example.com **Example response**: @@ -384,7 +336,7 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks`` without the ``block_id``. -.. http:get:: /blocks?tx_id={tx_id}&status={UNDECIDED|VALID|INVALID} +.. http:get:: /api/v1/blocks?tx_id={tx_id}&status={UNDECIDED|VALID|INVALID} Retrieve a list of ``block_id`` with their corresponding status that contain a transaction with the ID ``tx_id``. @@ -419,7 +371,7 @@ Blocks Votes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. http:get:: /votes?block_id={block_id} +.. http:get:: /api/v1/votes?block_id={block_id} Retrieve a list of votes for a certain block with ID ``block_id``. To check for the validity of a vote, a user of this endpoint needs to From 88b99d7bdc98de20694490365b0185f675d39b7e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 23 Jan 2017 16:37:50 +0100 Subject: [PATCH 094/219] document optional operation parameter in transaction --- .../source/drivers-clients/http-client-server-api.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 72c22870..6c67bfda 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -74,7 +74,7 @@ Transactions queried correctly. Some of them include retrieving a list of transactions that include: - * `Transactions related to a specific asset <#get--transactions?operation=CREATE|TRANSFER&asset_id=asset_id>`_ + * `Transactions related to a specific asset <#get--transactions?asset_id=asset_id&operation=CREATE|TRANSFER>`_ In this section, we've listed those particular requests, as they will likely to be very handy when implementing your application on top of BigchainDB. @@ -85,11 +85,11 @@ Transactions A generalization of those parameters follows: - :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. - :query string asset_id: The ID of the asset. -.. http:get:: /api/v1/transactions?operation={CREATE|TRANSFER}&asset_id={asset_id} + :query string operation: (Optional) One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. + +.. http:get:: /api/v1/transactions?asset_id={asset_id}&operation={CREATE|TRANSFER} Get a list of transactions that use an asset with the ID ``asset_id``. Every ``TRANSFER`` transaction that originates from a ``CREATE`` transaction @@ -98,7 +98,7 @@ Transactions This endpoint returns transactions only if they are decided ``VALID`` by the server. - :query string operation: One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. + :query string operation: (Optional) One of the two supported operations of a transaction: ``CREATE``, ``TRANSFER``. :query string asset_id: asset ID. From 7a71e386f33b38e7f279d45074792755eec5f7db Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 25 Jan 2017 10:47:38 +0100 Subject: [PATCH 095/219] Fix whitespace --- .../http-client-server-api.rst | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 6c67bfda..b1b4686d 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -37,6 +37,7 @@ that allows you to discover the BigchainDB API endpoints: .. literalinclude:: samples/api-index-response.http :language: http + Transactions ------------------- @@ -152,7 +153,7 @@ Transaction Outputs The ``/api/v1/outputs`` endpoint returns transactions outputs filtered by a given public key, and optionally filtered to only include outputs that have -not already been spent. +not already been spent. .. http:get:: /api/v1/outputs?public_key={public_key} @@ -162,20 +163,20 @@ not already been spent. ownership. Returns a list of links to transaction outputs. - + :param public_key: Base58 encoded public key associated with output ownership. This parameter is mandatory and without it the endpoint will return a ``400`` response code. :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend. - - + + **Example request**: - + .. sourcecode:: http - + GET /api/v1/outputs?public_key=1AAAbbb...ccc HTTP/1.1 Host: example.com **Example response**: - + .. sourcecode:: http HTTP/1.1 200 OK @@ -185,12 +186,11 @@ not already been spent. "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/0", "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/outputs/1" ] - + :statuscode 200: A list of outputs were found and returned in the body of the response. :statuscode 400: The request wasn't understood by the server, e.g. the ``public_key`` querystring was not included in the request. - Statuses -------------------------------- @@ -282,7 +282,6 @@ The `votes endpoint <#votes>`_ contains all the voting information for a specifi ``block_id`` for a given ``tx_id``, one can now simply inspect the votes that happened at a specific time on that block. - Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -366,8 +365,6 @@ Blocks :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/blocks``, without defining ``tx_id``. - - Votes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 5b7dd672b8d8f490045d353466044196a057e01f Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 25 Jan 2017 11:30:40 +0100 Subject: [PATCH 096/219] Remove controversial endpoints for now --- .../generate_http_server_api_documentation.py | 2 +- .../http-client-server-api.rst | 21 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 4407cda4..c36b10b5 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -67,7 +67,7 @@ Content-Type: application/json TPLS['post-tx-response'] = """\ -HTTP/1.1 200 OK +HTTP/1.1 202 Accepted Content-Type: application/json %(tx)s diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index b1b4686d..9e66f603 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -45,8 +45,8 @@ Transactions Get the transaction with the ID ``tx_id``. - This endpoint returns only a transaction from the ``BACKLOG`` or a ``VALID`` or ``UNDECIDED`` - block on ``bigchain``, if exists. + This endpoint returns a transaction only if a ``VALID`` block on + ``bigchain`` exists. :param tx_id: transaction ID :type tx_id: hex string @@ -144,7 +144,7 @@ Transactions :resheader Content-Type: ``application/json`` - :statuscode 200: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. + :statuscode 202: The pushed transaction was accepted in the ``BACKLOG``, but the processing has not been completed. :statuscode 400: The transaction was malformed and not accepted in the ``BACKLOG``. @@ -198,10 +198,6 @@ Statuses Get the status of an asynchronously written transaction or block by their id. - The possible status values are ``undecided``, ``valid`` or ``invalid`` for - both blocks and transactions. An additional state ``backlog`` is provided - for transactions. - A link to the resource is also provided in the returned payload under ``_links``. @@ -220,6 +216,10 @@ Statuses Get the status of a transaction. + The possible status values are ``undecided``, ``valid`` or ``backlog``. + If a transaction in neither of those states is found, a ``404 Not Found`` + HTTP status code is returned. `We're currently looking into ways to unambigously let the user know about a transaction's status that was included in an invalid block. `_ + **Example request**: .. literalinclude:: samples/get-statuses-tx-request.http @@ -227,11 +227,6 @@ Statuses **Example response**: - .. literalinclude:: samples/get-statuses-tx-invalid-response.http - :language: http - - **Example response**: - .. literalinclude:: samples/get-statuses-tx-valid-response.http :language: http @@ -246,6 +241,8 @@ Statuses Get the status of a block. + The possible status values are ``undecided``, ``valid`` or ``invalid``. + **Example request**: .. literalinclude:: samples/get-statuses-block-request.http From 7af997508141d7281ae6247a4b3a12ace688bbd5 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 25 Jan 2017 12:22:34 +0100 Subject: [PATCH 097/219] return 202 on successful transaction POST --- bigchaindb/web/views/transactions.py | 2 +- tests/web/test_transactions.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index 3059b34f..a4a983ef 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -113,4 +113,4 @@ class TransactionListApi(Resource): with monitor.timer('write_transaction', rate=rate): bigchain.write_transaction(tx_obj) - return tx + return tx, 202 diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 28e0fab0..6970c725 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -38,6 +38,9 @@ def test_post_create_transaction_endpoint(b, client): tx = tx.sign([user_priv]) res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + + assert res.status_code == 202 + assert res.json['inputs'][0]['owners_before'][0] == user_pub assert res.json['outputs'][0]['public_keys'][0] == user_pub @@ -157,6 +160,8 @@ def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx.to_dict())) + assert res.status_code == 202 + assert res.json['inputs'][0]['owners_before'][0] == user_pk assert res.json['outputs'][0]['public_keys'][0] == user_pub From 391da2cf604287acf217051acd2743f65a8b94a8 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 25 Jan 2017 12:36:08 +0100 Subject: [PATCH 098/219] Added tests --- bigchaindb/backend/admin.py | 6 +- bigchaindb/commands/bigchain.py | 4 +- tests/backend/mongodb/test_admin.py | 118 ++++++++++++++++++++++++++++ tests/backend/test_generics.py | 2 + tests/commands/test_commands.py | 54 ++++++++++++- 5 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 tests/backend/mongodb/test_admin.py diff --git a/bigchaindb/backend/admin.py b/bigchaindb/backend/admin.py index da54397b..f0ea62fd 100644 --- a/bigchaindb/backend/admin.py +++ b/bigchaindb/backend/admin.py @@ -24,9 +24,11 @@ def set_replicas(connection, *, replicas): @singledispatch def add_replicas(connection, replicas): - raise NotImplementedError + raise NotImplementedError('This command is specific to the ' + 'MongoDB backend.') @singledispatch def remove_replicas(connection, replicas): - raise NotImplementedError + raise NotImplementedError('This command is specific to the ' + 'MongoDB backend.') diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 78b3b745..98e9d81c 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -277,7 +277,7 @@ def run_add_replicas(args): try: add_replicas(conn, args.replicas) - except DatabaseOpFailedError as e: + except (DatabaseOpFailedError, NotImplementedError) as e: logger.warn(e) else: logger.info('Added {} to the replicaset.'.format(args.replicas)) @@ -290,7 +290,7 @@ def run_remove_replicas(args): try: remove_replicas(conn, args.replicas) - except DatabaseOpFailedError as e: + except (DatabaseOpFailedError, NotImplementedError) as e: logger.warn(e) else: logger.info('Removed {} from the replicaset.'.format(args.replicas)) diff --git a/tests/backend/mongodb/test_admin.py b/tests/backend/mongodb/test_admin.py new file mode 100644 index 00000000..138bc616 --- /dev/null +++ b/tests/backend/mongodb/test_admin.py @@ -0,0 +1,118 @@ +"""Tests for the :mod:`bigchaindb.backend.mongodb.admin` module.""" +import copy +from unittest import mock + +import pytest +from pymongo.database import Database +from pymongo.errors import OperationFailure + + +@pytest.fixture +def mock_replicaset_config(): + return { + 'config': { + '_id': 'bigchain-rs', + 'members': [ + { + '_id': 0, + 'arbiterOnly': False, + 'buildIndexes': True, + 'hidden': False, + 'host': 'localhost:27017', + 'priority': 1.0, + 'slaveDelay': 0, + 'tags': {}, + 'votes': 1 + } + ], + 'version': 1 + } + } + + +def test_add_replicas(mock_replicaset_config): + from bigchaindb.backend import connect + from bigchaindb.backend.admin import add_replicas + + connection = connect() + # force the connection object to setup a connection to the database + # before we mock `Database.command` + connection.conn + + expected_config = copy.deepcopy(mock_replicaset_config) + expected_config['config']['members'] += [ + {'_id': 1, 'host': 'localhost:27018'}, + {'_id': 2, 'host': 'localhost:27019'} + ] + expected_config['config']['version'] += 1 + + with mock.patch.object(Database, 'command') as mock_command: + mock_command.return_value = mock_replicaset_config + add_replicas(connection, ['localhost:27018', 'localhost:27019']) + + mock_command.assert_called_with('replSetReconfig', + expected_config['config']) + + +def test_add_replicas_raises(mock_replicaset_config): + from bigchaindb.backend import connect + from bigchaindb.backend.admin import add_replicas + from bigchaindb.backend.exceptions import DatabaseOpFailedError + + connection = connect() + # force the connection object to setup a connection to the database + # before we mock `Database.command` + connection.conn + + with mock.patch.object(Database, 'command') as mock_command: + mock_command.side_effect = [ + mock_replicaset_config, + OperationFailure(error=1, details={'errmsg': ''}) + ] + with pytest.raises(DatabaseOpFailedError): + add_replicas(connection, ['localhost:27018']) + + +def test_remove_replicas(mock_replicaset_config): + from bigchaindb.backend import connect + from bigchaindb.backend.admin import remove_replicas + + connection = connect() + # force the connection object to setup a connection to the database + # before we mock `Database.command` + connection.conn + + expected_config = copy.deepcopy(mock_replicaset_config) + expected_config['config']['version'] += 1 + + # add some hosts to the configuration to remove + mock_replicaset_config['config']['members'] += [ + {'_id': 1, 'host': 'localhost:27018'}, + {'_id': 2, 'host': 'localhost:27019'} + ] + + with mock.patch.object(Database, 'command') as mock_command: + mock_command.return_value = mock_replicaset_config + remove_replicas(connection, ['localhost:27018', 'localhost:27019']) + + mock_command.assert_called_with('replSetReconfig', + expected_config['config']) + + +def test_remove_replicas_raises(mock_replicaset_config): + from bigchaindb.backend import connect + from bigchaindb.backend.admin import remove_replicas + from bigchaindb.backend.exceptions import DatabaseOpFailedError + + connection = connect() + # force the connection object to setup a connection to the database + # before we mock `Database.command` + connection.conn + + with mock.patch.object(Database, 'command') as mock_command: + mock_command.side_effect = [ + mock_replicaset_config, + OperationFailure(error=1, details={'errmsg': ''}) + ] + with pytest.raises(DatabaseOpFailedError): + remove_replicas(connection, ['localhost:27018']) diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 2049d72b..0dd2637f 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -100,6 +100,8 @@ def test_init_database(mock_create_database, mock_create_tables, ('reconfigure', {'table': None, 'shards': None, 'replicas': None}), ('set_shards', {'shards': None}), ('set_replicas', {'replicas': None}), + ('add_replicas', {'replicas': None}), + ('remove_replicas', {'replicas': None}), )) def test_admin(admin_func_name, kwargs): from bigchaindb.backend import admin diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1a1291e3..f61c4c6f 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,6 +1,6 @@ import json from unittest.mock import Mock, patch -from argparse import Namespace +from argparse import Namespace, ArgumentTypeError import copy import pytest @@ -376,3 +376,55 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, 'distributed equally to all ' 'the processes') assert start_mock.called is True + + +@patch('bigchaindb.backend.admin.add_replicas') +def test_run_add_replicas(mock_add_replicas): + from bigchaindb.commands.bigchain import run_add_replicas + from bigchaindb.backend.exceptions import DatabaseOpFailedError + + args = Namespace(config=None, replicas=['localhost:27017']) + + # test add_replicas no raises + mock_add_replicas.return_value = None + assert run_add_replicas(args) is None + + # test add_replicas with `DatabaseOpFailedError` + mock_add_replicas.side_effect = DatabaseOpFailedError() + assert run_add_replicas(args) is None + + # test add_replicas with `NotImplementedError` + mock_add_replicas.side_effect = NotImplementedError() + assert run_add_replicas(args) is None + + +@patch('bigchaindb.backend.admin.remove_replicas') +def test_run_remove_replicas(mock_remove_replicas): + from bigchaindb.commands.bigchain import run_remove_replicas + from bigchaindb.backend.exceptions import DatabaseOpFailedError + + args = Namespace(config=None, replicas=['localhost:27017']) + + # test add_replicas no raises + mock_remove_replicas.return_value = None + assert run_remove_replicas(args) is None + + # test add_replicas with `DatabaseOpFailedError` + mock_remove_replicas.side_effect = DatabaseOpFailedError() + assert run_remove_replicas(args) is None + + # test add_replicas with `NotImplementedError` + mock_remove_replicas.side_effect = NotImplementedError() + assert run_remove_replicas(args) is None + + +def test_mongodb_host_type(): + from bigchaindb.commands.utils import mongodb_host + + # bad port provided + with pytest.raises(ArgumentTypeError): + mongodb_host('localhost:11111111111') + + # no port information provided + with pytest.raises(ArgumentTypeError): + mongodb_host('localhost') From 8e8a60a5399a43804c153a2209f4e5431d069335 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 25 Jan 2017 13:58:35 +0100 Subject: [PATCH 099/219] Get the correct configuration if backend is set by envs --- bigchaindb/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 315774e5..79118e23 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,6 +5,12 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 + +def _get_database_from_env(): + return globals()['_database_' + os.environ.get( + 'BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb')] + + _database_rethinkdb = { 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), @@ -28,7 +34,7 @@ config = { 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, - 'database': _database_rethinkdb, + 'database': _get_database_from_env(), 'keypair': { 'public': None, 'private': None, From baeae199516009e14584dbdff16f82f096c8afb9 Mon Sep 17 00:00:00 2001 From: vrde Date: Wed, 25 Jan 2017 19:05:48 +0100 Subject: [PATCH 100/219] Add experimental run interface --- bigchaindb/backend/mongodb/connection.py | 8 + bigchaindb/backend/mongodb/query.py | 235 +++++++++++++---------- bigchaindb/utils.py | 28 +++ tests/test_utils.py | 8 + 4 files changed, 180 insertions(+), 99 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 19731161..eaa9852e 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -5,6 +5,7 @@ from pymongo import MongoClient from pymongo import errors import bigchaindb +from bigchaindb.utils import Lazy from bigchaindb.common import exceptions from bigchaindb.backend.connection import Connection @@ -43,6 +44,9 @@ class MongoDBConnection(Connection): def db(self): return self.conn[self.dbname] + def run(self, query): + return query.run(self.db) + def _connect(self): # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the @@ -60,6 +64,10 @@ class MongoDBConnection(Connection): time.sleep(2**i) +def table(name): + return Lazy()[name] + + def initialize_replica_set(): """Initialize a replica set. If already initialized skip.""" diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index c4e3cdc8..ee7562c1 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -5,10 +5,11 @@ from time import time from pymongo import ReturnDocument from pymongo import errors + from bigchaindb import backend from bigchaindb.common.exceptions import CyclicBlockchainError from bigchaindb.backend.utils import module_dispatch_registrar -from bigchaindb.backend.mongodb.connection import MongoDBConnection +from bigchaindb.backend.mongodb.connection import MongoDBConnection, table register_query = module_dispatch_registrar(backend.query) @@ -17,7 +18,9 @@ register_query = module_dispatch_registrar(backend.query) @register_query(MongoDBConnection) def write_transaction(conn, signed_transaction): try: - return conn.db['backlog'].insert_one(signed_transaction) + return conn.run( + table('backlog') + .insert_one(signed_transaction)) except errors.DuplicateKeyError: return @@ -26,40 +29,49 @@ def write_transaction(conn, signed_transaction): def update_transaction(conn, transaction_id, doc): # with mongodb we need to add update operators to the doc doc = {'$set': doc} - return conn.db['backlog']\ - .find_one_and_update({'id': transaction_id}, - doc, - return_document=ReturnDocument.AFTER) + return conn.run( + table('backlog') + .find_one_and_update( + {'id': transaction_id}, + doc, + return_document=ReturnDocument.AFTER)) @register_query(MongoDBConnection) def delete_transaction(conn, *transaction_id): - return conn.db['backlog'].delete_many({'id': {'$in': transaction_id}}) + return conn.run( + table('backlog') + .delete_many({'id': {'$in': transaction_id}})) @register_query(MongoDBConnection) def get_stale_transactions(conn, reassign_delay): - return conn.db['backlog']\ - .find({'assignment_timestamp': {'$lt': time() - reassign_delay}}, - projection={'_id': False}) + return conn.run( + table('backlog') + .find({'assignment_timestamp': {'$lt': time() - reassign_delay}}, + projection={'_id': False})) @register_query(MongoDBConnection) def get_transaction_from_block(conn, transaction_id, block_id): try: - return conn.db['bigchain'].aggregate([ - {'$match': {'id': block_id}}, - {'$project': { - 'block.transactions': { - '$filter': { - 'input': '$block.transactions', - 'as': 'transaction', - 'cond': { - '$eq': ['$$transaction.id', transaction_id] + return conn.run( + table('bigchain') + .aggregate([ + {'$match': {'id': block_id}}, + {'$project': { + 'block.transactions': { + '$filter': { + 'input': '$block.transactions', + 'as': 'transaction', + 'cond': { + '$eq': ['$$transaction.id', transaction_id] + } + } } - } - } - }}]).next()['block']['transactions'].pop() + }}]) + .next()['block']['transactions'] + .pop()) except (StopIteration, IndexError): # StopIteration is raised if the block was not found # IndexError is returned if the block is found but no transactions @@ -69,33 +81,38 @@ def get_transaction_from_block(conn, transaction_id, block_id): @register_query(MongoDBConnection) def get_transaction_from_backlog(conn, transaction_id): - return conn.db['backlog']\ - .find_one({'id': transaction_id}, - projection={'_id': False, 'assignee': False, - 'assignment_timestamp': False}) + return conn.run( + table('backlog') + .find_one({'id': transaction_id}, + projection={'_id': False, + 'assignee': False, + 'assignment_timestamp': False})) @register_query(MongoDBConnection) def get_blocks_status_from_transaction(conn, transaction_id): - return conn.db['bigchain']\ - .find({'block.transactions.id': transaction_id}, - projection=['id', 'block.voters']) + return conn.run( + table('bigchain') + .find({'block.transactions.id': transaction_id}, + projection=['id', 'block.voters'])) @register_query(MongoDBConnection) def get_asset_by_id(conn, asset_id): - cursor = conn.db['bigchain'].aggregate([ - {'$match': { - 'block.transactions.id': asset_id, - 'block.transactions.operation': 'CREATE' - }}, - {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.id': asset_id, - 'block.transactions.operation': 'CREATE' - }}, - {'$project': {'block.transactions.asset': True}} - ]) + cursor = conn.run( + table('bigchain') + .aggregate([ + {'$match': { + 'block.transactions.id': asset_id, + 'block.transactions.operation': 'CREATE' + }}, + {'$unwind': '$block.transactions'}, + {'$match': { + 'block.transactions.id': asset_id, + 'block.transactions.operation': 'CREATE' + }}, + {'$project': {'block.transactions.asset': True}} + ])) # we need to access some nested fields before returning so lets use a # generator to avoid having to read all records on the cursor at this point return (elem['block']['transactions'] for elem in cursor) @@ -103,13 +120,14 @@ def get_asset_by_id(conn, asset_id): @register_query(MongoDBConnection) def get_spent(conn, transaction_id, output): - cursor = conn.db['bigchain'].aggregate([ - {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.inputs.fulfills.txid': transaction_id, - 'block.transactions.inputs.fulfills.output': output - }} - ]) + cursor = conn.run( + table('bigchain').aggregate([ + {'$unwind': '$block.transactions'}, + {'$match': { + 'block.transactions.inputs.fulfills.txid': transaction_id, + 'block.transactions.inputs.fulfills.output': output + }} + ])) # we need to access some nested fields before returning so lets use a # generator to avoid having to read all records on the cursor at this point return (elem['block']['transactions'] for elem in cursor) @@ -117,14 +135,16 @@ def get_spent(conn, transaction_id, output): @register_query(MongoDBConnection) def get_owned_ids(conn, owner): - cursor = conn.db['bigchain'].aggregate([ - {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.outputs.public_keys': { - '$elemMatch': {'$eq': owner} - } - }} - ]) + cursor = conn.run( + table('bigchain') + .aggregate([ + {'$unwind': '$block.transactions'}, + {'$match': { + 'block.transactions.outputs.public_keys': { + '$elemMatch': {'$eq': owner} + } + }} + ])) # we need to access some nested fields before returning so lets use a # generator to avoid having to read all records on the cursor at this point return (elem['block']['transactions'] for elem in cursor) @@ -132,66 +152,80 @@ def get_owned_ids(conn, owner): @register_query(MongoDBConnection) def get_votes_by_block_id(conn, block_id): - return conn.db['votes']\ - .find({'vote.voting_for_block': block_id}, - projection={'_id': False}) + return conn.run( + table('votes') + .find({'vote.voting_for_block': block_id}, + projection={'_id': False})) @register_query(MongoDBConnection) def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): - return conn.db['votes']\ - .find({'vote.voting_for_block': block_id, - 'node_pubkey': node_pubkey}, - projection={'_id': False}) + return conn.run( + table('votes') + .find({'vote.voting_for_block': block_id, + 'node_pubkey': node_pubkey}, + projection={'_id': False})) @register_query(MongoDBConnection) def write_block(conn, block): - return conn.db['bigchain'].insert_one(block.to_dict()) + return conn.run( + table('bigchain') + .insert_one(block.to_dict())) @register_query(MongoDBConnection) def get_block(conn, block_id): - return conn.db['bigchain'].find_one({'id': block_id}, - projection={'_id': False}) + return conn.run( + table('bigchain') + .find_one({'id': block_id}, + projection={'_id': False})) @register_query(MongoDBConnection) def has_transaction(conn, transaction_id): - return bool(conn.db['bigchain'] - .find_one({'block.transactions.id': transaction_id})) + return bool(conn.run( + table('bigchain') + .find_one({'block.transactions.id': transaction_id}))) @register_query(MongoDBConnection) def count_blocks(conn): - return conn.db['bigchain'].count() + return conn.run( + table('bigchain') + .count()) @register_query(MongoDBConnection) def count_backlog(conn): - return conn.db['backlog'].count() + return conn.run( + table('backlog') + .count()) @register_query(MongoDBConnection) def write_vote(conn, vote): - conn.db['votes'].insert_one(vote) + conn.run(table('votes').insert_one(vote)) vote.pop('_id') return vote @register_query(MongoDBConnection) def get_genesis_block(conn): - return conn.db['bigchain'].find_one( - {'block.transactions.0.operation': 'GENESIS'}, - {'_id': False} - ) + return conn.run( + table('bigchain') + .find_one( + {'block.transactions.0.operation': 'GENESIS'}, + {'_id': False} + )) @register_query(MongoDBConnection) def get_last_voted_block(conn, node_pubkey): - last_voted = conn.db['votes']\ - .find({'node_pubkey': node_pubkey}, - sort=[('vote.timestamp', -1)]) + last_voted = conn.run( + table('votes') + .find({'node_pubkey': node_pubkey}, + sort=[('vote.timestamp', -1)])) # pymongo seems to return a cursor even if there are no results # so we actually need to check the count @@ -219,21 +253,22 @@ def get_last_voted_block(conn, node_pubkey): @register_query(MongoDBConnection) def get_unvoted_blocks(conn, node_pubkey): - return conn.db['bigchain'].aggregate([ - {'$lookup': { - 'from': 'votes', - 'localField': 'id', - 'foreignField': 'vote.voting_for_block', - 'as': 'votes' - }}, - {'$match': { - 'votes.node_pubkey': {'$ne': node_pubkey}, - 'block.transactions.operation': {'$ne': 'GENESIS'} - }}, - {'$project': { - 'votes': False, '_id': False - }} - ]) + return conn.run( + table('bigchain').aggregate([ + {'$lookup': { + 'from': 'votes', + 'localField': 'id', + 'foreignField': 'vote.voting_for_block', + 'as': 'votes' + }}, + {'$match': { + 'votes.node_pubkey': {'$ne': node_pubkey}, + 'block.transactions.operation': {'$ne': 'GENESIS'} + }}, + {'$project': { + 'votes': False, '_id': False + }} + ])) @register_query(MongoDBConnection) @@ -243,10 +278,12 @@ def get_txids_filtered(conn, asset_id, operation=None): if operation: match['block.transactions.operation'] = operation - cursor = conn.db['bigchain'].aggregate([ - {'$match': match}, - {'$unwind': '$block.transactions'}, - {'$match': match}, - {'$project': {'block.transactions.id': True}} - ]) + cursor = conn.run( + table('bigchain') + .aggregate([ + {'$match': match}, + {'$unwind': '$block.transactions'}, + {'$match': match}, + {'$project': {'block.transactions.id': True}} + ])) return (r['block']['transactions']['id'] for r in cursor) diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 434c6376..e89c3b6f 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -157,3 +157,31 @@ def is_genesis_block(block): return block.transactions[0].operation == 'GENESIS' except AttributeError: return block['block']['transactions'][0]['operation'] == 'GENESIS' + + +class Lazy: + + def __init__(self): + self.stack = [] + + def __getattr__(self, name): + self.stack.append(name) + return self + + def __call__(self, *args, **kwargs): + self.stack.append((args, kwargs)) + return self + + def __getitem__(self, key): + self.stack.append('__getitem__') + self.stack.append(([key], {})) + return self + + def run(self, instance): + last = instance + + for method, (args, kwargs) in zip(self.stack[::2], self.stack[1::2]): + last = getattr(last, method)(*args, **kwargs) + + self.stack = [] + return last diff --git a/tests/test_utils.py b/tests/test_utils.py index f76ba6d9..47343ace 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -137,3 +137,11 @@ def test_is_genesis_block_returns_true_if_genesis(b): from bigchaindb.utils import is_genesis_block genesis_block = b.prepare_genesis_block() assert is_genesis_block(genesis_block) + + +def test_lazy_execution(): + from bigchaindb.utils import Lazy + l = Lazy() + l.split(',')[1].split(' ').pop(1).strip() + result = l.run('Like humans, cats tend to favor one paw over another') + assert result == 'cats' From 83afab4958bcfbebd84818376467246ac277b69a Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 26 Jan 2017 10:22:43 +0100 Subject: [PATCH 101/219] Fix indentation problem --- bigchaindb/backend/mongodb/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index ee7562c1..f1acf601 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -214,7 +214,7 @@ def write_vote(conn, vote): def get_genesis_block(conn): return conn.run( table('bigchain') - .find_one( + .find_one( {'block.transactions.0.operation': 'GENESIS'}, {'_id': False} )) From 05fdcef670c403fcb2c4dafc6fb2db859fc2799c Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 25 Jan 2017 18:54:03 +0100 Subject: [PATCH 102/219] Document default value for GET /outputs?unspent= --- docs/server/source/drivers-clients/http-client-server-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 9e66f603..5444be8f 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -165,7 +165,7 @@ not already been spent. Returns a list of links to transaction outputs. :param public_key: Base58 encoded public key associated with output ownership. This parameter is mandatory and without it the endpoint will return a ``400`` response code. - :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend. + :param unspent: Boolean value ("true" or "false") indicating if the result set should be limited to outputs that are available to spend. Defaults to "false". **Example request**: From ca49718d7e9f14ae997c4e00a66098c9b19581cb Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 26 Jan 2017 11:23:50 +0100 Subject: [PATCH 103/219] s/table/collection/g --- bigchaindb/backend/mongodb/connection.py | 2 +- bigchaindb/backend/mongodb/query.py | 46 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index eaa9852e..61af4469 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -64,7 +64,7 @@ class MongoDBConnection(Connection): time.sleep(2**i) -def table(name): +def collection(name): return Lazy()[name] diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index f1acf601..14c1ae21 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -9,7 +9,7 @@ from pymongo import errors from bigchaindb import backend from bigchaindb.common.exceptions import CyclicBlockchainError from bigchaindb.backend.utils import module_dispatch_registrar -from bigchaindb.backend.mongodb.connection import MongoDBConnection, table +from bigchaindb.backend.mongodb.connection import MongoDBConnection, collection register_query = module_dispatch_registrar(backend.query) @@ -19,7 +19,7 @@ register_query = module_dispatch_registrar(backend.query) def write_transaction(conn, signed_transaction): try: return conn.run( - table('backlog') + collection('backlog') .insert_one(signed_transaction)) except errors.DuplicateKeyError: return @@ -30,7 +30,7 @@ def update_transaction(conn, transaction_id, doc): # with mongodb we need to add update operators to the doc doc = {'$set': doc} return conn.run( - table('backlog') + collection('backlog') .find_one_and_update( {'id': transaction_id}, doc, @@ -40,14 +40,14 @@ def update_transaction(conn, transaction_id, doc): @register_query(MongoDBConnection) def delete_transaction(conn, *transaction_id): return conn.run( - table('backlog') + collection('backlog') .delete_many({'id': {'$in': transaction_id}})) @register_query(MongoDBConnection) def get_stale_transactions(conn, reassign_delay): return conn.run( - table('backlog') + collection('backlog') .find({'assignment_timestamp': {'$lt': time() - reassign_delay}}, projection={'_id': False})) @@ -56,7 +56,7 @@ def get_stale_transactions(conn, reassign_delay): def get_transaction_from_block(conn, transaction_id, block_id): try: return conn.run( - table('bigchain') + collection('bigchain') .aggregate([ {'$match': {'id': block_id}}, {'$project': { @@ -82,7 +82,7 @@ def get_transaction_from_block(conn, transaction_id, block_id): @register_query(MongoDBConnection) def get_transaction_from_backlog(conn, transaction_id): return conn.run( - table('backlog') + collection('backlog') .find_one({'id': transaction_id}, projection={'_id': False, 'assignee': False, @@ -92,7 +92,7 @@ def get_transaction_from_backlog(conn, transaction_id): @register_query(MongoDBConnection) def get_blocks_status_from_transaction(conn, transaction_id): return conn.run( - table('bigchain') + collection('bigchain') .find({'block.transactions.id': transaction_id}, projection=['id', 'block.voters'])) @@ -100,7 +100,7 @@ def get_blocks_status_from_transaction(conn, transaction_id): @register_query(MongoDBConnection) def get_asset_by_id(conn, asset_id): cursor = conn.run( - table('bigchain') + collection('bigchain') .aggregate([ {'$match': { 'block.transactions.id': asset_id, @@ -121,7 +121,7 @@ def get_asset_by_id(conn, asset_id): @register_query(MongoDBConnection) def get_spent(conn, transaction_id, output): cursor = conn.run( - table('bigchain').aggregate([ + collection('bigchain').aggregate([ {'$unwind': '$block.transactions'}, {'$match': { 'block.transactions.inputs.fulfills.txid': transaction_id, @@ -136,7 +136,7 @@ def get_spent(conn, transaction_id, output): @register_query(MongoDBConnection) def get_owned_ids(conn, owner): cursor = conn.run( - table('bigchain') + collection('bigchain') .aggregate([ {'$unwind': '$block.transactions'}, {'$match': { @@ -153,7 +153,7 @@ def get_owned_ids(conn, owner): @register_query(MongoDBConnection) def get_votes_by_block_id(conn, block_id): return conn.run( - table('votes') + collection('votes') .find({'vote.voting_for_block': block_id}, projection={'_id': False})) @@ -161,7 +161,7 @@ def get_votes_by_block_id(conn, block_id): @register_query(MongoDBConnection) def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): return conn.run( - table('votes') + collection('votes') .find({'vote.voting_for_block': block_id, 'node_pubkey': node_pubkey}, projection={'_id': False})) @@ -170,14 +170,14 @@ def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): @register_query(MongoDBConnection) def write_block(conn, block): return conn.run( - table('bigchain') + collection('bigchain') .insert_one(block.to_dict())) @register_query(MongoDBConnection) def get_block(conn, block_id): return conn.run( - table('bigchain') + collection('bigchain') .find_one({'id': block_id}, projection={'_id': False})) @@ -185,27 +185,27 @@ def get_block(conn, block_id): @register_query(MongoDBConnection) def has_transaction(conn, transaction_id): return bool(conn.run( - table('bigchain') + collection('bigchain') .find_one({'block.transactions.id': transaction_id}))) @register_query(MongoDBConnection) def count_blocks(conn): return conn.run( - table('bigchain') + collection('bigchain') .count()) @register_query(MongoDBConnection) def count_backlog(conn): return conn.run( - table('backlog') + collection('backlog') .count()) @register_query(MongoDBConnection) def write_vote(conn, vote): - conn.run(table('votes').insert_one(vote)) + conn.run(collection('votes').insert_one(vote)) vote.pop('_id') return vote @@ -213,7 +213,7 @@ def write_vote(conn, vote): @register_query(MongoDBConnection) def get_genesis_block(conn): return conn.run( - table('bigchain') + collection('bigchain') .find_one( {'block.transactions.0.operation': 'GENESIS'}, {'_id': False} @@ -223,7 +223,7 @@ def get_genesis_block(conn): @register_query(MongoDBConnection) def get_last_voted_block(conn, node_pubkey): last_voted = conn.run( - table('votes') + collection('votes') .find({'node_pubkey': node_pubkey}, sort=[('vote.timestamp', -1)])) @@ -254,7 +254,7 @@ def get_last_voted_block(conn, node_pubkey): @register_query(MongoDBConnection) def get_unvoted_blocks(conn, node_pubkey): return conn.run( - table('bigchain').aggregate([ + collection('bigchain').aggregate([ {'$lookup': { 'from': 'votes', 'localField': 'id', @@ -279,7 +279,7 @@ def get_txids_filtered(conn, asset_id, operation=None): match['block.transactions.operation'] = operation cursor = conn.run( - table('bigchain') + collection('bigchain') .aggregate([ {'$match': match}, {'$unwind': '$block.transactions'}, From 9762b4b96854cd5896f4b71c7fbfcc82d6f24f5c Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 26 Jan 2017 13:39:06 +0100 Subject: [PATCH 104/219] fix spend input twice bug (https://github.com/bigchaindb/bigchaindb/issues/1099) --- bigchaindb/models.py | 5 +++++ tests/db/test_bigchain_api.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 159e9f49..c6e81956 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -88,6 +88,11 @@ class Transaction(Transaction): if output.amount < 1: raise AmountError('`amount` needs to be greater than zero') + # Validate that all inputs are distinct + links = [i.fulfills.to_uri() for i in self.inputs] + if len(links) != len(set(links)): + raise DoubleSpend('tx "{}" spends inputs twice'.format(self.id)) + # validate asset id asset_id = Transaction.get_asset_id(input_txs) if asset_id != self.asset['id']: diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 8a2040e8..4d9314a1 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1192,3 +1192,33 @@ def test_get_outputs_filtered(): get_outputs.assert_called_once_with('abc') get_spent.assert_not_called() assert out == get_outputs.return_value + + +@pytest.mark.bdb +def test_cant_spend_same_input_twice_in_tx(b, genesis_block): + """ + Recreate duplicated fulfillments bug + https://github.com/bigchaindb/bigchaindb/issues/1099 + """ + from bigchaindb.models import Transaction + from bigchaindb.common.exceptions import DoubleSpend + + # create a divisible asset + tx_create = Transaction.create([b.me], [([b.me], 100)]) + tx_create_signed = tx_create.sign([b.me_private]) + assert b.validate_transaction(tx_create_signed) == tx_create_signed + + # create a block and valid vote + block = b.create_block([tx_create_signed]) + b.write_block(block) + vote = b.vote(block.id, genesis_block.id, True) + b.write_vote(vote) + + # Create a transfer transaction with duplicated fulfillments + dup_inputs = tx_create.to_inputs() + tx_create.to_inputs() + tx_transfer = Transaction.transfer(dup_inputs, [([b.me], 200)], + asset_id=tx_create.id) + tx_transfer_signed = tx_transfer.sign([b.me_private]) + assert b.is_valid_transaction(tx_transfer_signed) is False + with pytest.raises(DoubleSpend): + tx_transfer_signed.validate(b) From adb579ac0a29ac857eed5886cabfed17b1121994 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 26 Jan 2017 13:52:09 +0100 Subject: [PATCH 105/219] Revert "duplicate asset ID" and apply "get_txids_filtered" interface. --- bigchaindb/backend/mongodb/query.py | 55 +++++++++++++------ bigchaindb/backend/rethinkdb/query.py | 43 +++++++++------ bigchaindb/common/schema/transaction.yaml | 4 +- bigchaindb/common/transaction.py | 16 +----- docs/server/source/data-models/asset-model.md | 5 +- tests/common/test_transaction.py | 28 +--------- 6 files changed, 73 insertions(+), 78 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index c4e3cdc8..d7ee6afc 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -1,12 +1,14 @@ """Query implementation for MongoDB""" from time import time +from itertools import chain from pymongo import ReturnDocument from pymongo import errors from bigchaindb import backend from bigchaindb.common.exceptions import CyclicBlockchainError +from bigchaindb.common.transaction import Transaction from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.mongodb.connection import MongoDBConnection @@ -82,6 +84,43 @@ def get_blocks_status_from_transaction(conn, transaction_id): projection=['id', 'block.voters']) +@register_query(MongoDBConnection) +def get_txids_filtered(conn, asset_id, operation=None): + parts = [] + + if operation in (Transaction.CREATE, None): + # get the txid of the create transaction for asset_id + cursor = conn.db['bigchain'].aggregate([ + {'$match': { + 'block.transactions.id': asset_id, + 'block.transactions.operation': 'CREATE' + }}, + {'$unwind': '$block.transactions'}, + {'$match': { + 'block.transactions.id': asset_id, + 'block.transactions.operation': 'CREATE' + }}, + {'$project': {'block.transactions.id': True}} + ]) + parts.append(elem['block']['transactions']['id'] for elem in cursor) + + if operation in (Transaction.TRANSFER, None): + # get txids of transfer transaction with asset_id + cursor = conn.db['bigchain'].aggregate([ + {'$match': { + 'block.transactions.asset.id': asset_id + }}, + {'$unwind': '$block.transactions'}, + {'$match': { + 'block.transactions.asset.id': asset_id + }}, + {'$project': {'block.transactions.id': True}} + ]) + parts.append(elem['block']['transactions']['id'] for elem in cursor) + + return chain(*parts) + + @register_query(MongoDBConnection) def get_asset_by_id(conn, asset_id): cursor = conn.db['bigchain'].aggregate([ @@ -234,19 +273,3 @@ def get_unvoted_blocks(conn, node_pubkey): 'votes': False, '_id': False }} ]) - - -@register_query(MongoDBConnection) -def get_txids_filtered(conn, asset_id, operation=None): - match = {'block.transactions.asset.id': asset_id} - - if operation: - match['block.transactions.operation'] = operation - - cursor = conn.db['bigchain'].aggregate([ - {'$match': match}, - {'$unwind': '$block.transactions'}, - {'$match': match}, - {'$project': {'block.transactions.id': True}} - ]) - return (r['block']['transactions']['id'] for r in cursor) diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index fd0bdcb3..aa7c3be6 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -1,9 +1,11 @@ +from itertools import chain from time import time import rethinkdb as r from bigchaindb import backend, utils from bigchaindb.common import exceptions +from bigchaindb.common.transaction import Transaction from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection @@ -71,6 +73,30 @@ def get_blocks_status_from_transaction(connection, transaction_id): .pluck('votes', 'id', {'block': ['voters']})) +@register_query(RethinkDBConnection) +def get_txids_filtered(connection, asset_id, operation=None): + # here we only want to return the transaction ids since later on when + # we are going to retrieve the transaction with status validation + + parts = [] + + if operation in (Transaction.CREATE, None): + # First find the asset's CREATE transaction + parts.append(connection.run( + _get_asset_create_tx_query(asset_id).get_field('id'))) + + if operation in (Transaction.TRANSFER, None): + # Then find any TRANSFER transactions related to the asset + parts.append(connection.run( + r.table('bigchain') + .get_all(asset_id, index='asset_id') + .concat_map(lambda block: block['block']['transactions']) + .filter(lambda transaction: transaction['asset']['id'] == asset_id) + .get_field('id'))) + + return chain(*parts) + + @register_query(RethinkDBConnection) def get_asset_by_id(connection, asset_id): return connection.run(_get_asset_create_tx_query(asset_id).pluck('asset')) @@ -233,20 +259,3 @@ def get_unvoted_blocks(connection, node_pubkey): # database level. Solving issue #444 can help untangling the situation unvoted_blocks = filter(lambda block: not utils.is_genesis_block(block), unvoted) return unvoted_blocks - - -@register_query(RethinkDBConnection) -def get_txids_filtered(connection, asset_id, operation=None): - # here we only want to return the transaction ids since later on when - # we are going to retrieve the transaction with status validation - - tx_filter = r.row['asset']['id'] == asset_id - if operation: - tx_filter &= r.row['operation'] == operation - - return connection.run( - r.table('bigchain') - .get_all(asset_id, index='asset_id') - .concat_map(lambda block: block['block']['transactions']) - .filter(tx_filter) - .get_field('id')) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index a0edd1e3..86e5947b 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -103,8 +103,8 @@ definitions: description: | Description of the asset being transacted. In the case of a ``TRANSFER`` transaction, this field contains only the ID of asset. In the case - of a ``CREATE`` transaction, this field contains the user-defined - payload and the asset ID (duplicated from the Transaction ID). + of a ``CREATE`` transaction, this field contains only the user-defined + payload. additionalProperties: false properties: id: diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 0e6d84f2..65b12eed 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -444,7 +444,6 @@ class Transaction(object): asset is not None and not (isinstance(asset, dict) and 'data' in asset)): raise TypeError(('`asset` must be None or a dict holding a `data` ' " property instance for '{}' Transactions".format(operation))) - asset.pop('id', None) # Remove duplicated asset ID if there is one elif (operation == Transaction.TRANSFER and not (isinstance(asset, dict) and 'id' in asset)): raise TypeError(('`asset` must be a dict holding an `id` property ' @@ -927,11 +926,9 @@ class Transaction(object): tx_no_signatures = Transaction._remove_signatures(tx) tx_serialized = Transaction._to_str(tx_no_signatures) - tx['id'] = Transaction._to_hash(tx_serialized) - if self.operation == Transaction.CREATE: - # Duplicate asset into asset for consistency with TRANSFER - # transactions - tx['asset']['id'] = tx['id'] + tx_id = Transaction._to_hash(tx_serialized) + + tx['id'] = tx_id return tx @staticmethod @@ -955,9 +952,6 @@ class Transaction(object): # case could yield incorrect signatures. This is why we only # set it to `None` if it's set in the dict. input_['fulfillment'] = None - # Pop duplicated asset_id from CREATE tx - if tx_dict['operation'] == Transaction.CREATE: - tx_dict['asset'].pop('id', None) return tx_dict @staticmethod @@ -1037,10 +1031,6 @@ class Transaction(object): "the hash of its body, i.e. it's not valid.") raise InvalidHash(err_msg.format(proposed_tx_id)) - if tx_body.get('operation') == Transaction.CREATE: - if proposed_tx_id != tx_body['asset'].get('id'): - raise InvalidHash('CREATE tx has wrong asset_id') - @classmethod def from_dict(cls, tx): """Transforms a Python dictionary to a Transaction object. diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index 16188400..312c6765 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -1,11 +1,10 @@ # The Digital Asset Model -The asset ID is the same as the ID of the CREATE transaction that defined the asset. +To avoid redundant data in transactions, the digital asset model is different for `CREATE` and `TRANSFER` transactions. -In the case of a CREATE transaction, the transaction ID is duplicated into the asset object for clarity and consistency in the database. The CREATE transaction also contains a user definable payload to describe the asset: +A digital asset's properties are defined in a `CREATE` transaction with the following model: ```json { - "id": "", "data": "" } ``` diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 7e70939d..a2782583 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -300,7 +300,6 @@ def test_transaction_serialization(user_input, user_output, data): 'operation': Transaction.CREATE, 'metadata': None, 'asset': { - 'id': tx_id, 'data': data, } } @@ -308,7 +307,7 @@ def test_transaction_serialization(user_input, user_output, data): tx = Transaction(Transaction.CREATE, {'data': data}, [user_input], [user_output]) tx_dict = tx.to_dict() - tx_dict['id'] = tx_dict['asset']['id'] = tx_id + tx_dict['id'] = tx_id assert tx_dict == expected @@ -335,7 +334,6 @@ def test_transaction_deserialization(user_input, user_output, data): } tx_no_signatures = Transaction._remove_signatures(tx) tx['id'] = Transaction._to_hash(Transaction._to_str(tx_no_signatures)) - tx['asset']['id'] = tx['id'] tx = Transaction.from_dict(tx) assert tx == expected @@ -691,7 +689,6 @@ def test_create_create_transaction_single_io(user_output, user_pub, data): tx_dict = tx.to_dict() tx_dict['inputs'][0]['fulfillment'] = None tx_dict.pop('id') - tx_dict['asset'].pop('id') assert tx_dict == expected @@ -775,7 +772,6 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, metadata=data, asset=data) tx_dict = tx.to_dict() tx_dict.pop('id') - tx_dict['asset'].pop('id') tx_dict['inputs'][0]['fulfillment'] = None assert tx_dict == expected @@ -989,25 +985,3 @@ def test_validate_version(utx): utx.version = '1.0.0' with raises(SchemaValidationError): validate_transaction_model(utx) - - -def test_create_tx_has_asset_id(tx): - tx = tx.to_dict() - assert tx['id'] == tx['asset']['id'] - - -def test_create_tx_validates_asset_id(tx): - from bigchaindb.common.transaction import Transaction - from bigchaindb.common.exceptions import InvalidHash - - tx = tx.to_dict() - - # Test fails with wrong asset_id - tx['asset']['id'] = tx['asset']['id'][::-1] - with raises(InvalidHash): - Transaction.from_dict(tx) - - # Test fails with no asset_id - tx['asset'].pop('id') - with raises(InvalidHash): - Transaction.from_dict(tx) From 1243322aad661bf62efa16cf1865c8aff658f1d2 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 26 Jan 2017 13:59:52 +0100 Subject: [PATCH 106/219] Case insensitive "unspent" and "operation" parameters" --- bigchaindb/web/views/parameters.py | 2 ++ tests/web/test_parameters.py | 16 ++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/bigchaindb/web/views/parameters.py b/bigchaindb/web/views/parameters.py index 222adb97..9759563e 100644 --- a/bigchaindb/web/views/parameters.py +++ b/bigchaindb/web/views/parameters.py @@ -8,6 +8,7 @@ def valid_txid(txid): def valid_bool(val): + val = val.lower() if val == 'true': return True if val == 'false': @@ -23,6 +24,7 @@ def valid_ed25519(key): def valid_operation(op): + op = op.upper() if op == 'CREATE': return 'CREATE' if op == 'TRANSFER': diff --git a/tests/web/test_parameters.py b/tests/web/test_parameters.py index d39c6f38..7da2b739 100644 --- a/tests/web/test_parameters.py +++ b/tests/web/test_parameters.py @@ -24,11 +24,9 @@ def test_valid_bool(): assert valid_bool('true') is True assert valid_bool('false') is False + assert valid_bool('tRUE') is True + assert valid_bool('fALSE') is False - with pytest.raises(ValueError): - valid_bool('TRUE') - with pytest.raises(ValueError): - valid_bool('FALSE') with pytest.raises(ValueError): valid_bool('0') with pytest.raises(ValueError): @@ -64,13 +62,11 @@ def test_valid_ed25519(): def test_valid_operation(): from bigchaindb.web.views.parameters import valid_operation - assert valid_operation('CREATE') == 'CREATE' - assert valid_operation('TRANSFER') == 'TRANSFER' + assert valid_operation('create') == 'CREATE' + assert valid_operation('transfer') == 'TRANSFER' + assert valid_operation('CREATe') == 'CREATE' + assert valid_operation('TRANSFEr') == 'TRANSFER' - with pytest.raises(ValueError): - valid_operation('create') - with pytest.raises(ValueError): - valid_operation('transfer') with pytest.raises(ValueError): valid_operation('GENESIS') with pytest.raises(ValueError): From 9d03aeb72a8debea42e7775e8b77983620efa4d3 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 26 Jan 2017 15:02:48 +0100 Subject: [PATCH 107/219] fixed tests --- tests/commands/test_commands.py | 2 ++ tests/test_config_utils.py | 27 +++++++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index f61c4c6f..16b615eb 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -378,6 +378,7 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, assert start_mock.called is True +@pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.backend.admin.add_replicas') def test_run_add_replicas(mock_add_replicas): from bigchaindb.commands.bigchain import run_add_replicas @@ -398,6 +399,7 @@ def test_run_add_replicas(mock_add_replicas): assert run_add_replicas(args) is None +@pytest.mark.usefixtures('ignore_local_config_file') @patch('bigchaindb.backend.admin.remove_replicas') def test_run_remove_replicas(mock_remove_replicas): from bigchaindb.commands.bigchain import run_remove_replicas diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index c1f63742..7ebe5579 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -131,6 +131,26 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): from bigchaindb import config_utils config_utils.autoconfigure() + backend = request.config.getoption('--database-backend') + database_rethinkdb = { + 'backend': 'rethinkdb', + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + } + database_mongodb = { + 'backend': 'mongodb', + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + 'replicaset': 'bigchain-rs', + } + + # default + database = database_rethinkdb + if backend == 'mongodb': + database = database_mongodb + assert bigchaindb.config == { 'CONFIGURED': True, 'server': { @@ -138,12 +158,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'workers': None, 'threads': None, }, - 'database': { - 'backend': request.config.getoption('--database-backend'), - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', - }, + 'database': database, 'keypair': { 'public': None, 'private': None, From 3c5511563627f4311e99f91bac11738cdf2399d5 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 26 Jan 2017 15:53:51 +0100 Subject: [PATCH 108/219] Add changes for 0.8.1 release to changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fdfe8c2..b6fd28ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,19 @@ For reference, the possible headings are: * **Notes** +## [0.8.1] - 2017-01-16 +Tag name: v0.8.1 += commit: +committed: + +### Changed +- Upgrade pysha3 to 1.0.0 (supports official NIST standard). + +### Fixed +- Workaround for rapidjson problem with package metadata extraction + (https://github.com/kenrobbins/python-rapidjson/pull/52). + + ## [0.8.0] - 2016-11-29 Tag name: v0.8.0 = commit: From 54544f66a050b2333ff61d0b161f9e28dfd0cb61 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 26 Jan 2017 17:12:35 +0100 Subject: [PATCH 109/219] Simplify run function --- bigchaindb/utils.py | 7 +++++-- tests/test_utils.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index e89c3b6f..6ceace6b 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -180,8 +180,11 @@ class Lazy: def run(self, instance): last = instance - for method, (args, kwargs) in zip(self.stack[::2], self.stack[1::2]): - last = getattr(last, method)(*args, **kwargs) + for item in self.stack: + if isinstance(item, str): + last = getattr(last, item) + else: + last = last(*item[0], **item[1]) self.stack = [] return last diff --git a/tests/test_utils.py b/tests/test_utils.py index 47343ace..fbf5d65d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -141,7 +141,19 @@ def test_is_genesis_block_returns_true_if_genesis(b): def test_lazy_execution(): from bigchaindb.utils import Lazy + l = Lazy() l.split(',')[1].split(' ').pop(1).strip() result = l.run('Like humans, cats tend to favor one paw over another') assert result == 'cats' + + class Cat: + def __init__(self, name): + self.name = name + + cat = Cat('Shmui') + + l = Lazy() + l.name.upper() + result = l.run(cat) + assert result == 'SHMUI' From 509b590b32c1e19b1c8970cfdfd64bf582811721 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 27 Jan 2017 11:40:41 +0100 Subject: [PATCH 110/219] pull changelog from 0.8.2 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6fd28ea..312c589f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ For reference, the possible headings are: * **Notes** +## [0.8.2] - 2017-01-27 +Tag name: v0.8.2 + +### Fixed +- Fix spend input twice in same transaction + (https://github.com/bigchaindb/bigchaindb/issues/1099). + + ## [0.8.1] - 2017-01-16 Tag name: v0.8.1 = commit: From 2c87f1c28acd81ebc25e5688d70307ac6b034938 Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 27 Jan 2017 14:35:37 +0100 Subject: [PATCH 111/219] Update docs --- bigchaindb/backend/mongodb/connection.py | 5 +++++ bigchaindb/utils.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 61af4469..2be39194 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -65,6 +65,11 @@ class MongoDBConnection(Connection): def collection(name): + """Return a lazy object that can be used to compose a query. + + Args: + name (str): the name of the collection to query. + """ return Lazy()[name] diff --git a/bigchaindb/utils.py b/bigchaindb/utils.py index 6ceace6b..1860dd3e 100644 --- a/bigchaindb/utils.py +++ b/bigchaindb/utils.py @@ -160,8 +160,17 @@ def is_genesis_block(block): class Lazy: + """Lazy objects are useful to create chains of methods to + execute later. + + A lazy object records the methods that has been called, and + replay them when the :py:meth:`run` method is called. Note that + :py:meth:`run` needs an object `instance` to replay all the + methods that have been recorded. + """ def __init__(self): + """Instantiate a new Lazy object.""" self.stack = [] def __getattr__(self, name): @@ -178,6 +187,12 @@ class Lazy: return self def run(self, instance): + """Run the recorded chain of methods on `instance`. + + Args: + instance: an object. + """ + last = instance for item in self.stack: From aa3a525dc1c496ca2a4b958d8046be50bd8018ea Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 27 Jan 2017 15:27:44 +0100 Subject: [PATCH 112/219] Release process changes for Minor and Patch version --- Release_Process.md | 55 +++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index 16dc800b..b24398af 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -1,23 +1,48 @@ # Our Release Process -This is a summary of the steps we go through to release a new version of BigchainDB Server. +The release process for BigchainDB server differs slightly depending on whether it's a minor or a patch release. -1. Update the `CHANGELOG.md` file -1. Update the version numbers in `bigchaindb/version.py`. Note that we try to use [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH) -1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) - and click the "Draft a new release" button -1. Name the tag something like v0.7.0 -1. The target should be a specific commit: the one when the update of `bigchaindb/version.py` got merged into master -1. The release title should be something like v0.7.0 -1. The description should be copied from the `CHANGELOG.md` file updated above -1. Generate and send the latest `bigchaindb` package to PyPI. Dimi and Sylvain can do this, maybe others -1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. - Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is - "Active" and "Public" +BigchainDB follows [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH), taking into account +that [major version 0 does not export a stable API](http://semver.org/#spec-item-4). -After the release: +## Minor release -1. Update `bigchaindb/version.py` again, to be something like 0.8.0.dev (with a dev on the end). +A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server. + +1. Update the `CHANGELOG.md` file in master +1. Create and checkout a new branch for the release, named after the minor version, without preceeding 'v', ie: `git checkout -b 0.9` +1. Commit changes and push new branch to Github +1. Follow steps outlined in [Common Steps](#common-steps) +1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release ie: `0.10.0.dev'. This is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available). + +Congratulations, you have released BigchainDB! + +## Patch release + +A patch release is similar to a minor release, but piggybacks on an existing minor release branch: + +1. Check out the minor release branch +1. Apply the changes you want, ie using `git cherry-pick`. +1. Update the `CHANGELOG.md` file +1. Increment the patch version in `bigchaindb/version.py`, ie: "0.9.1" +1. Follow steps outlined in [Common Steps](#common-steps) + +## Common steps + +These steps are common between minor and patch releases: + +1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) + and click the "Draft a new release" button +1. Fill in the details: + - Tag version: version number preceeded by 'v', ie: "v0.9.1" + - Target: the release branch that was just pushed + - Title: Same as tag name + - Description: The body of the changelog entry (Added, Changed etc) +1. Publish the release on Github +1. Generate the release tarball with `python setup.py sdist`. Upload the release to Pypi. +1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. + Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is + "Active" and "Public" From f6a6b72a9c965360dad113a2d327cae3076c5f38 Mon Sep 17 00:00:00 2001 From: libscott Date: Fri, 27 Jan 2017 17:48:11 +0100 Subject: [PATCH 113/219] Update Release_Process.md --- Release_Process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Release_Process.md b/Release_Process.md index b24398af..bf267d34 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -3,7 +3,7 @@ The release process for BigchainDB server differs slightly depending on whether it's a minor or a patch release. BigchainDB follows [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH), taking into account -that [major version 0 does not export a stable API](http://semver.org/#spec-item-4). +that [major version 0.x does not export a stable API](http://semver.org/#spec-item-4). ## Minor release From 5b084edaf405bccc697497e89970ce0309ea1093 Mon Sep 17 00:00:00 2001 From: vrde Date: Mon, 30 Jan 2017 10:56:46 +0100 Subject: [PATCH 114/219] Break out of the loop once a connection is established closes #1068 --- bigchaindb/backend/rethinkdb/connection.py | 2 ++ tests/backend/rethinkdb/test_connection.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/bigchaindb/backend/rethinkdb/connection.py b/bigchaindb/backend/rethinkdb/connection.py index 173cdc7b..601125f2 100644 --- a/bigchaindb/backend/rethinkdb/connection.py +++ b/bigchaindb/backend/rethinkdb/connection.py @@ -77,3 +77,5 @@ class RethinkDBConnection(Connection): wait_time = 2**i logging.debug('Error connecting to database, waiting %ss', wait_time) time.sleep(wait_time) + else: + break diff --git a/tests/backend/rethinkdb/test_connection.py b/tests/backend/rethinkdb/test_connection.py index 65c665af..073fecee 100644 --- a/tests/backend/rethinkdb/test_connection.py +++ b/tests/backend/rethinkdb/test_connection.py @@ -1,6 +1,7 @@ import time import multiprocessing as mp from threading import Thread +from unittest.mock import patch import pytest import rethinkdb as r @@ -118,3 +119,15 @@ def test_changefeed_reconnects_when_connection_lost(monkeypatch): fact = changefeed.outqueue.get()['fact'] assert fact == 'Cats sleep 70% of their lives.' + + +@patch('rethinkdb.connect') +def test_connection_happens_one_time_if_successful(mock_connect): + from bigchaindb.backend import connect + + query = r.expr('1') + conn = connect('rethinkdb', 'localhost', 1337, 'whatev') + conn.run(query) + mock_connect.assert_called_once_with(host='localhost', + port=1337, + db='whatev') From 736ecb0bc8d7d1a57981a5a01671afd5e32b77c6 Mon Sep 17 00:00:00 2001 From: libscott Date: Mon, 30 Jan 2017 12:04:07 +0100 Subject: [PATCH 115/219] Update Release_Process.md --- Release_Process.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Release_Process.md b/Release_Process.md index bf267d34..0f89c509 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -45,4 +45,8 @@ These steps are common between minor and patch releases: 1. Generate the release tarball with `python setup.py sdist`. Upload the release to Pypi. 1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is - "Active" and "Public" + "Active" and "Public", and make sure the new version's branch + (without the 'v' in front) is _not_ active +1. Also in readthedocs.org, go to Admin --> Advanced Settings + and make sure that "Default branch:" (i.e. what "latest" points to) + is set to the new release's tag, e.g. `v0.9.1`. (Don't miss the 'v' in front.) From 49464484d7add4481f09caf174849235ad1df54a Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 30 Jan 2017 14:04:32 +0100 Subject: [PATCH 116/219] set public_key manually in bigchaindb root url docs example --- docs/server/generate_http_server_api_documentation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index c36b10b5..ba082ba3 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -198,6 +198,7 @@ def main(): "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" ] + res_data['public_key'] = 'NC8c8rYcAhyKVpx1PCV65CBmyq4YUbLysy3Rqrg8L8mz' ctx['index'] = pretty_json(res_data) # API index From 2a07362bad9c0d6bc0c52bbd42b109cda85bc80e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 30 Jan 2017 16:11:44 +0100 Subject: [PATCH 117/219] test transaction supports unicode --- tests/db/test_bigchain_api.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 4d9314a1..aa8488c4 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1222,3 +1222,15 @@ def test_cant_spend_same_input_twice_in_tx(b, genesis_block): assert b.is_valid_transaction(tx_transfer_signed) is False with pytest.raises(DoubleSpend): tx_transfer_signed.validate(b) + + +@pytest.mark.bdb +def test_transaction_unicode(b): + import json + from bigchaindb.models import Transaction + tx = (Transaction.create([b.me], [([b.me], 100)], + {'beer': '\N{BEER MUG}'}) + ).sign([b.me_private]) + block = b.create_block([tx]) + assert block.validate(b) == block + assert '{"beer": "\\ud83c\\udf7a"}' in json.dumps(block.to_dict()) From a1aa64aa61adda32985d996669308ddcc626b4d2 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Tue, 24 Jan 2017 19:03:59 +0100 Subject: [PATCH 118/219] Fix Makefile for new docs structure --- Makefile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 95b18f0d..d8ef2bf4 100644 --- a/Makefile +++ b/Makefile @@ -65,12 +65,11 @@ coverage: ## check code coverage quickly with the default Python $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs - rm -f docs/bigchaindb.rst - rm -f docs/modules.rst - sphinx-apidoc -o docs/ bigchaindb - $(MAKE) -C docs clean - $(MAKE) -C docs html - $(BROWSER) docs/_build/html/index.html + $(MAKE) -C docs/root clean + $(MAKE) -C docs/root html + $(MAKE) -C docs/server clean + $(MAKE) -C docs/server html + $(BROWSER) docs/root/_build/html/index.html servedocs: docs ## compile the docs watching for changes watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . From 17a07a80f1db317094ce3778d0eef114aa47464b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 30 Jan 2017 18:11:29 +0100 Subject: [PATCH 119/219] Removed 'Example Apps' page from the docs --- docs/server/source/drivers-clients/example-apps.rst | 10 ---------- docs/server/source/drivers-clients/index.rst | 1 - 2 files changed, 11 deletions(-) delete mode 100644 docs/server/source/drivers-clients/example-apps.rst diff --git a/docs/server/source/drivers-clients/example-apps.rst b/docs/server/source/drivers-clients/example-apps.rst deleted file mode 100644 index 0aab953e..00000000 --- a/docs/server/source/drivers-clients/example-apps.rst +++ /dev/null @@ -1,10 +0,0 @@ -Example Apps -============ - -.. warning:: - - There are some example BigchainDB apps (i.e. apps which use BigchainDB) in the GitHub repository named `bigchaindb-examples `_. They were created before there was much of an HTTP API, so instead of communicating with a BigchainDB node via the HTTP API, they communicate directly with the node using the BigchainDB Python server API and the RethinkDB Python Driver. That's not how a real production app would work. The HTTP API is getting better, and we recommend using it to communicate with BigchainDB nodes. - - Moreover, because of changes to the BigchainDB Server code, some of the examples in the bigchaindb-examples repo might not work anymore, or they might not work as expected. - - In the future, we hope to create a set of examples using the HTTP API (or wrappers of it, such as the Python Driver API). diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 9eb81f6c..dd33e18b 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -14,4 +14,3 @@ your choice, and then use the HTTP API directly to post transactions. http-client-server-api The Python Driver Transaction CLI - example-apps From a8bbc87c1ce0a4ceb6127368a71d23c7eef1e4ac Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 31 Jan 2017 01:27:55 +0100 Subject: [PATCH 120/219] Major improvs for MongoDBConnection class --- bigchaindb/backend/mongodb/connection.py | 73 +++++++++++++++++------- tests/backend/mongodb/test_connection.py | 4 +- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 2be39194..c641d726 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -1,5 +1,6 @@ import time import logging +from itertools import repeat from pymongo import MongoClient from pymongo import errors @@ -12,9 +13,17 @@ from bigchaindb.backend.connection import Connection logger = logging.getLogger(__name__) +# TODO: waiting for #1082 to be merged +# to move this constants in the configuration. + +CONNECTION_TIMEOUT = 4000 # in milliseconds +MAX_RETRIES = 3 # number of tries before giving up, if 0 then try forever + + class MongoDBConnection(Connection): - def __init__(self, host=None, port=None, dbname=None, max_tries=3, + def __init__(self, host=None, port=None, dbname=None, + connection_timeout=None, max_tries=None, replicaset=None): """Create a new Connection instance. @@ -22,7 +31,10 @@ class MongoDBConnection(Connection): host (str, optional): the host to connect to. port (int, optional): the port to connect to. dbname (str, optional): the database to use. - max_tries (int, optional): how many tries before giving up. + connection_timeout (int, optional): the milliseconds to wait + until timing out the database connection attempt. + max_tries (int, optional): how many tries before giving up, + if 0 then try forever. replicaset (str, optional): the name of the replica set to connect to. """ @@ -31,13 +43,15 @@ class MongoDBConnection(Connection): self.port = port or bigchaindb.config['database']['port'] self.replicaset = replicaset or bigchaindb.config['database']['replicaset'] self.dbname = dbname or bigchaindb.config['database']['name'] - self.max_tries = max_tries + self.connection_timeout = connection_timeout if connection_timeout is not None else CONNECTION_TIMEOUT + self.max_tries = max_tries if max_tries is not None else MAX_RETRIES + self.max_tries_counter = range(self.max_tries) if self.max_tries != 0 else repeat(0) self.connection = None @property def conn(self): if self.connection is None: - self._connect() + self.connection = self._connect() return self.connection @property @@ -45,23 +59,41 @@ class MongoDBConnection(Connection): return self.conn[self.dbname] def run(self, query): - return query.run(self.db) + attempt = 0 + for i in self.max_tries_counter: + attempt += 1 + try: + return query.run(self.conn[self.dbname]) + except errors.AutoReconnect: + if attempt == self.max_tries: + raise + self._connect() def _connect(self): - # we should only return a connection if the replica set is - # initialized. initialize_replica_set will check if the - # replica set is initialized else it will initialize it. - initialize_replica_set() + attempt = 0 + for i in self.max_tries_counter: + attempt += 1 - for i in range(self.max_tries): try: - self.connection = MongoClient(self.host, self.port, - replicaset=self.replicaset) - except errors.ConnectionFailure: - if i + 1 == self.max_tries: - raise - else: - time.sleep(2**i) + # FYI: this might raise a `ServerSelectionTimeoutError`, + # that is a subclass of `ConnectionFailure`. + connection = MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout) + + # we should only return a connection if the replica set is + # initialized. initialize_replica_set will check if the + # replica set is initialized else it will initialize it. + initialize_replica_set(self.host, self.port, self.connection_timeout) + return connection + except (errors.ConnectionFailure, errors.AutoReconnect) as exc: + logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.', + attempt, self.max_tries if self.max_tries != 0 else '∞', + self.host, self.port, self.connection_timeout) + if attempt == self.max_tries: + logger.exception('Cannot connect to the Database. Giving up.') + raise errors.ConnectionFailure() from exc def collection(name): @@ -73,15 +105,16 @@ def collection(name): return Lazy()[name] -def initialize_replica_set(): +def initialize_replica_set(host, port, connection_timeout): """Initialize a replica set. If already initialized skip.""" # Setup a MongoDB connection # The reason we do this instead of `backend.connect` is that # `backend.connect` will connect you to a replica set but this fails if # you try to connect to a replica set that is not yet initialized - conn = MongoClient(host=bigchaindb.config['database']['host'], - port=bigchaindb.config['database']['port']) + conn = MongoClient(host=host, + port=port, + serverselectiontimeoutms=connection_timeout) _check_replica_set(conn) host = '{}:{}'.format(bigchaindb.config['database']['host'], bigchaindb.config['database']['port']) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index 4c2919fe..49ab0604 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -138,7 +138,7 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] # check that it returns - assert initialize_replica_set() is None + assert initialize_replica_set('host', 1337, 1000) is None # test it raises OperationError if anything wrong with mock.patch.object(Database, 'command') as mock_command: @@ -148,4 +148,4 @@ def test_initialize_replica_set(mock_cmd_line_opts): ] with pytest.raises(OperationFailure): - initialize_replica_set() + initialize_replica_set('host', 1337, 1000) From 1557645e946bb554a0877a267b0837308db9b590 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 31 Jan 2017 02:07:36 +0100 Subject: [PATCH 121/219] Better exception handling --- bigchaindb/backend/exceptions.py | 3 +++ bigchaindb/backend/mongodb/changefeed.py | 6 ++++-- bigchaindb/backend/mongodb/connection.py | 18 ++++++++++-------- bigchaindb/commands/bigchain.py | 7 ++++++- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/bigchaindb/backend/exceptions.py b/bigchaindb/backend/exceptions.py index 1d55bc52..1f7c57a7 100644 --- a/bigchaindb/backend/exceptions.py +++ b/bigchaindb/backend/exceptions.py @@ -1,5 +1,8 @@ from bigchaindb.exceptions import BigchainDBError +class ConnectionError(BigchainDBError): + """Exception raised when the connection to the DataBase fails.""" + class DatabaseOpFailedError(BigchainDBError): """Exception for database operation errors.""" diff --git a/bigchaindb/backend/mongodb/changefeed.py b/bigchaindb/backend/mongodb/changefeed.py index c54bd5da..ecf36bcc 100644 --- a/bigchaindb/backend/mongodb/changefeed.py +++ b/bigchaindb/backend/mongodb/changefeed.py @@ -8,7 +8,8 @@ from bigchaindb import backend from bigchaindb.backend.changefeed import ChangeFeed from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.mongodb.connection import MongoDBConnection - +from bigchaindb.backend.exceptions import (DatabaseOpFailedError, + ConnectionError) logger = logging.getLogger(__name__) register_changefeed = module_dispatch_registrar(backend.changefeed) @@ -31,7 +32,8 @@ class MongoDBChangeFeed(ChangeFeed): break except (errors.ConnectionFailure, errors.OperationFailure, errors.AutoReconnect, - errors.ServerSelectionTimeoutError) as exc: + errors.ServerSelectionTimeoutError, + DatabaseOpFailedError, ConnectionError) as exc: logger.exception(exc) time.sleep(1) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index c641d726..61cc430d 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -8,6 +8,7 @@ from pymongo import errors import bigchaindb from bigchaindb.utils import Lazy from bigchaindb.common import exceptions +from bigchaindb.backend import exceptions as backend_exceptions from bigchaindb.backend.connection import Connection logger = logging.getLogger(__name__) @@ -51,7 +52,7 @@ class MongoDBConnection(Connection): @property def conn(self): if self.connection is None: - self.connection = self._connect() + self._connect() return self.connection @property @@ -77,23 +78,24 @@ class MongoDBConnection(Connection): try: # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. - connection = MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout) + self.connection = MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout) # we should only return a connection if the replica set is # initialized. initialize_replica_set will check if the # replica set is initialized else it will initialize it. initialize_replica_set(self.host, self.port, self.connection_timeout) - return connection except (errors.ConnectionFailure, errors.AutoReconnect) as exc: logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.', attempt, self.max_tries if self.max_tries != 0 else '∞', self.host, self.port, self.connection_timeout) if attempt == self.max_tries: - logger.exception('Cannot connect to the Database. Giving up.') - raise errors.ConnectionFailure() from exc + logger.critical('Cannot connect to the Database. Giving up.') + raise backend_exceptions.ConnectionError() from exc + else: + break def collection(name): diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 6661e902..4e18543d 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -23,7 +23,8 @@ from bigchaindb.utils import ProcessGroup from bigchaindb import backend from bigchaindb.backend import schema from bigchaindb.backend.admin import set_replicas, set_shards -from bigchaindb.backend.exceptions import DatabaseOpFailedError +from bigchaindb.backend.exceptions import (DatabaseOpFailedError, + ConnectionError) from bigchaindb.commands import utils from bigchaindb import processes @@ -157,6 +158,8 @@ def run_init(args): except DatabaseAlreadyExists: print('The database already exists.', file=sys.stderr) print('If you wish to re-initialize it, first drop it.', file=sys.stderr) + except ConnectionError: + print('Cannot connect to the database.', file=sys.stderr) def run_drop(args): @@ -201,6 +204,8 @@ def run_start(args): _run_init() except DatabaseAlreadyExists: pass + except ConnectionError: + print('Cannot connect to the database.', file=sys.stderr) except KeypairNotFoundException: sys.exit("Can't start BigchainDB, no keypair found. " 'Did you run `bigchaindb configure`?') From 86542fd74583902cc0a8d2826d0adff9b6b8719a Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 09:58:20 +0100 Subject: [PATCH 122/219] remove unnecessary test --- tests/backend/test_generics.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 2049d72b..956b698f 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -68,33 +68,6 @@ def test_changefeed_class(changefeed_class_func_name, args_qty): changefeed_class_func(None, *range(args_qty)) -@patch('bigchaindb.backend.schema.create_indexes', - autospec=True, return_value=None) -@patch('bigchaindb.backend.schema.create_tables', - autospec=True, return_value=None) -@patch('bigchaindb.backend.schema.create_database', - autospec=True, return_value=None) -def test_init_database(mock_create_database, mock_create_tables, - mock_create_indexes): - from bigchaindb.backend.schema import init_database - from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection - from bigchaindb.backend.mongodb.connection import MongoDBConnection - - # rethinkdb - conn = RethinkDBConnection('host', 'port', 'dbname') - init_database(connection=conn, dbname='mickeymouse') - mock_create_database.assert_called_with(conn, 'mickeymouse') - mock_create_tables.assert_called_with(conn, 'mickeymouse') - mock_create_indexes.assert_called_with(conn, 'mickeymouse') - - # mongodb - conn = MongoDBConnection('host', 'port', 'dbname', replicaset='rs') - init_database(connection=conn, dbname='mickeymouse') - mock_create_database.assert_called_with(conn, 'mickeymouse') - mock_create_tables.assert_called_with(conn, 'mickeymouse') - mock_create_indexes.assert_called_with(conn, 'mickeymouse') - - @mark.parametrize('admin_func_name,kwargs', ( ('get_config', {'table': None}), ('reconfigure', {'table': None, 'shards': None, 'replicas': None}), From 555745abbf69753acdce82d83c0ec3b611a93f6f Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 10:04:45 +0100 Subject: [PATCH 123/219] fixed pep8 issue --- tests/backend/test_generics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 956b698f..946b694f 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -1,5 +1,3 @@ -from unittest.mock import patch - from pytest import mark, raises From fafdac252308314ade77e4ec175aef3869a74347 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 10:34:45 +0100 Subject: [PATCH 124/219] Retrieve default backend from env if set. Fixed tests. --- bigchaindb/__init__.py | 9 ++++++++- tests/test_config_utils.py | 28 ++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 315774e5..072c7b6b 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -20,6 +20,11 @@ _database_mongodb = { 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), } +_database_map = { + 'mongodb': _database_mongodb, + 'rethinkdb': _database_rethinkdb +} + config = { 'server': { # Note: this section supports all the Gunicorn settings: @@ -28,7 +33,9 @@ config = { 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, - 'database': _database_rethinkdb, + 'database': _database_map[ + os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') + ], 'keypair': { 'public': None, 'private': None, diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index c1f63742..ebf630a9 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -131,6 +131,27 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): from bigchaindb import config_utils config_utils.autoconfigure() + backend = request.config.getoption('--database-backend') + database_rethinkdb = { + 'backend': 'rethinkdb', + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + } + database_mongodb = { + 'backend': 'mongodb', + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + 'replicaset': 'bigchain-rs', + } + + database = {} + if backend == 'mongodb': + database = database_mongodb + elif backend == 'rethinkdb': + database = database_rethinkdb + assert bigchaindb.config == { 'CONFIGURED': True, 'server': { @@ -138,12 +159,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'workers': None, 'threads': None, }, - 'database': { - 'backend': request.config.getoption('--database-backend'), - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', - }, + 'database': database, 'keypair': { 'public': None, 'private': None, From 4740150f6d772bc3b8cf9ccf2a3fc56d9909cee9 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 10:57:58 +0100 Subject: [PATCH 125/219] Updated config fixture Simplified tests. --- tests/conftest.py | 3 +-- tests/test_config_utils.py | 10 ---------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c3177c14..9612f38b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -118,9 +118,8 @@ def _configure_bigchaindb(request): test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) backend = request.config.getoption('--database-backend') - backend_conf = getattr(bigchaindb, '_database_' + backend) config = { - 'database': backend_conf, + 'database': bigchaindb._database_map[backend], 'keypair': { 'private': '31Lb1ZGKTyHnmVK3LUMrAUrPNfd4sE2YyBt3UA4A25aA', 'public': '4XYfCbabAWVUCbjTmRTFEu2sc3dFEdkse4r6X498B1s8', diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index ebf630a9..7feec4c9 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -23,11 +23,6 @@ def test_bigchain_instance_is_initialized_when_conf_provided(request): assert bigchaindb.config['CONFIGURED'] is True - # set the current backend so that Bigchain can create a connection - backend = request.config.getoption('--database-backend') - backend_conf = getattr(bigchaindb, '_database_' + backend) - bigchaindb.config['database'] = backend_conf - b = bigchaindb.Bigchain() assert b.me @@ -44,11 +39,6 @@ def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): # from existing configurations monkeypatch.setattr(config_utils, 'autoconfigure', lambda: 0) - # set the current backend so that Bigchain can create a connection - backend = request.config.getoption('--database-backend') - backend_conf = getattr(bigchaindb, '_database_' + backend) - bigchaindb.config['database'] = backend_conf - with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() From 002c567bdfd26aa2e234fc25da700b43dff52ac4 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 31 Jan 2017 11:13:45 +0100 Subject: [PATCH 126/219] Updated the AMIs to Ubuntu 16.04 images in amis.tf --- ntools/one-m/aws/amis.tf | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ntools/one-m/aws/amis.tf b/ntools/one-m/aws/amis.tf index 1e3910cc..7e15ed5b 100644 --- a/ntools/one-m/aws/amis.tf +++ b/ntools/one-m/aws/amis.tf @@ -2,19 +2,20 @@ # even though the contents are the same. # This file has the mapping from region --> AMI name. # -# These are all Ubuntu 14.04 LTS AMIs +# These are all Ubuntu 16.04 LTS AMIs # with Arch = amd64, Instance Type = hvm:ebs-ssd # from https://cloud-images.ubuntu.com/locator/ec2/ +# as of Jan. 31, 2017 variable "amis" { type = "map" default = { - eu-west-1 = "ami-55452e26" - eu-central-1 = "ami-b1cf39de" - us-east-1 = "ami-8e0b9499" - us-west-2 = "ami-547b3834" - ap-northeast-1 = "ami-49d31328" - ap-southeast-1 = "ami-5e429c3d" - ap-southeast-2 = "ami-25f3c746" - sa-east-1 = "ami-97980efb" + eu-west-1 = "ami-d8f4deab" + eu-central-1 = "ami-5aee2235" + us-east-1 = "ami-6edd3078" + us-west-2 = "ami-7c803d1c" + ap-northeast-1 = "ami-eb49358c" + ap-southeast-1 = "ami-b1943fd2" + ap-southeast-2 = "ami-fe71759d" + sa-east-1 = "ami-7379e31f" } } From 375873b6ede2b8f77c7ba840ea54353ba91c0b57 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 31 Jan 2017 11:24:46 +0100 Subject: [PATCH 127/219] Updated docs page about Terraform: Ubuntu 14.04 --> 16.04 --- .../template-terraform-aws.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md b/docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md index 5d1292d3..85e4cf9d 100644 --- a/docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md +++ b/docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md @@ -2,7 +2,7 @@ If you didn't read the introduction to the [cloud deployment starter templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. -This page explains a way to use [Terraform](https://www.terraform.io/) to provision an Ubuntu machine (i.e. an EC2 instance with Ubuntu 14.04) and other resources on [AWS](https://aws.amazon.com/). That machine can then be used to host a one-machine BigchainDB node. +This page explains a way to use [Terraform](https://www.terraform.io/) to provision an Ubuntu machine (i.e. an EC2 instance with Ubuntu 16.04) and other resources on [AWS](https://aws.amazon.com/). That machine can then be used to host a one-machine BigchainDB node. ## Install Terraform @@ -65,7 +65,7 @@ terraform apply Terraform will report its progress as it provisions all the resources. Once it's done, you can go to the Amazon EC2 web console and see the instance, its security group, its elastic IP, and its attached storage volumes (one for the root directory and one for RethinkDB storage). -At this point, there is no software installed on the instance except for Ubuntu 14.04 and whatever else came with the Amazon Machine Image (AMI) specified in the Terraform configuration (files). +At this point, there is no software installed on the instance except for Ubuntu 16.04 and whatever else came with the Amazon Machine Image (AMI) specified in the Terraform configuration (files). The next step is to install, configure and run all the necessary software for a BigchainDB node. You could use [our example Ansible playbook](template-ansible.html) to do that. From 2c26468cea61404d59329be5d055fef80e47dfd6 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 13:51:49 +0100 Subject: [PATCH 128/219] Added some constants to simplify test --- tests/test_config_utils.py | 50 +++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 7feec4c9..af78585e 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -10,8 +10,13 @@ ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config) @pytest.fixture(scope='function', autouse=True) -def clean_config(monkeypatch): - monkeypatch.setattr('bigchaindb.config', copy.deepcopy(ORIGINAL_CONFIG)) +def clean_config(monkeypatch, request): + + import bigchaindb + original_config = copy.deepcopy(ORIGINAL_CONFIG) + backend = request.config.getoption('--database-backend') + original_config['database'] = bigchaindb._database_map[backend] + monkeypatch.setattr('bigchaindb.config', original_config) def test_bigchain_instance_is_initialized_when_conf_provided(request): @@ -104,48 +109,55 @@ def test_env_config(monkeypatch): def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): + # constants + DATABASE_HOST = 'test-host' + DATABASE_NAME = 'test-dbname' + DATABASE_PORT = 4242 + DATABASE_BACKEND = request.config.getoption('--database-backend') + SERVER_BIND = '1.2.3.4:56' + KEYRING = 'pubkey_0:pubkey_1:pubkey_2' + file_config = { 'database': { - 'host': 'test-host', - 'backend': request.config.getoption('--database-backend') + 'host': DATABASE_HOST }, 'backlog_reassign_delay': 5 } monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) - monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname', - 'BIGCHAINDB_DATABASE_PORT': '4242', - 'BIGCHAINDB_SERVER_BIND': '1.2.3.4:56', - 'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'}) + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': DATABASE_NAME, + 'BIGCHAINDB_DATABASE_PORT': str(DATABASE_PORT), + 'BIGCHAINDB_DATABASE_BACKEND': DATABASE_BACKEND, + 'BIGCHAINDB_SERVER_BIND': SERVER_BIND, + 'BIGCHAINDB_KEYRING': KEYRING}) import bigchaindb from bigchaindb import config_utils config_utils.autoconfigure() - backend = request.config.getoption('--database-backend') database_rethinkdb = { 'backend': 'rethinkdb', - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', + 'host': DATABASE_HOST, + 'port': DATABASE_PORT, + 'name': DATABASE_NAME, } database_mongodb = { 'backend': 'mongodb', - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', + 'host': DATABASE_HOST, + 'port': DATABASE_PORT, + 'name': DATABASE_NAME, 'replicaset': 'bigchain-rs', } database = {} - if backend == 'mongodb': + if DATABASE_BACKEND == 'mongodb': database = database_mongodb - elif backend == 'rethinkdb': + elif DATABASE_BACKEND == 'rethinkdb': database = database_rethinkdb assert bigchaindb.config == { 'CONFIGURED': True, 'server': { - 'bind': '1.2.3.4:56', + 'bind': SERVER_BIND, 'workers': None, 'threads': None, }, @@ -154,7 +166,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'public': None, 'private': None, }, - 'keyring': ['pubkey_0', 'pubkey_1', 'pubkey_2'], + 'keyring': KEYRING.split(':'), 'statsd': { 'host': 'localhost', 'port': 8125, From 69068fc919279699dbc02da714dea0166a38c336 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 13:53:36 +0100 Subject: [PATCH 129/219] Document how to run BigchainDB with MongoDB (#1116) * Document changes in the configure command. Document new add/remove replicas commands. * updated quickstart with mongodb instructions * Docs on how to setup mongodb dev node with and without docker. Update replSet option in docker-compose * Fixed typo. More explicit on how to run the tests. * Fixed typo in mongodb docker instructions. More explicit about requiring mongodb 3.4+ --- docker-compose.yml | 2 +- .../source/dev-and-test/setup-run-node.md | 88 ++++++++++++++++--- docs/server/source/quickstart.md | 43 ++++++--- .../source/server-reference/bigchaindb-cli.md | 32 ++++++- 4 files changed, 140 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index db8abd4f..f5dbcdc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: mongo:3.4.1 ports: - "27017" - command: mongod --replSet=rs0 + command: mongod --replSet=bigchain-rs rdb: image: rethinkdb diff --git a/docs/server/source/dev-and-test/setup-run-node.md b/docs/server/source/dev-and-test/setup-run-node.md index 0cf5334c..bb7285b4 100644 --- a/docs/server/source/dev-and-test/setup-run-node.md +++ b/docs/server/source/dev-and-test/setup-run-node.md @@ -7,25 +7,27 @@ The BigchainDB core dev team develops BigchainDB on recent Ubuntu and Fedora dis ## Option A: Using a Local Dev Machine -First, read through the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to setup a machine for developing and testing BigchainDB. +Read through the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to setup a machine for developing and testing BigchainDB. -Next, create a default BigchainDB config file (in `$HOME/.bigchaindb`): +### With RethinkDB + +Create a default BigchainDB config file (in `$HOME/.bigchaindb`): ```text -bigchaindb -y configure +$ bigchaindb -y configure rethinkdb ``` Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) Start RethinkDB using: ```text -rethinkdb +$ rethinkdb ``` You can verify that RethinkDB is running by opening the RethinkDB web interface in your web browser. It should be at [http://localhost:8080/](http://localhost:8080/). To run BigchainDB Server, do: ```text -bigchaindb start +$ bigchaindb start ``` You can [run all the unit tests](running-unit-tests.html) to test your installation. @@ -33,13 +35,37 @@ You can [run all the unit tests](running-unit-tests.html) to test your installat The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md) has more details about how to contribute. -## Option B: Using a Dev Machine on Cloud9 +### With MongoDB -Ian Worrall of [Encrypted Labs](http://www.encryptedlabs.com/) wrote a document (PDF) explaining how to set up a BigchainDB (Server) dev machine on Cloud9: +Create a default BigchainDB config file (in `$HOME/.bigchaindb`): +```text +$ bigchaindb -y configure mongodb +``` -[Download that document from GitHub](https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/docs/server/source/_static/cloud9.pdf) +Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) -## Option C: Using a Local Dev Machine and Docker +Start MongoDB __3.4+__ using: +```text +$ mongod --replSet=bigchain-rs +``` + +You can verify that MongoDB is running correctly by checking the output of the +previous command for the line: +```text +waiting for connections on port 27017 +``` + +To run BigchainDB Server, do: +```text +$ bigchaindb start +``` + +You can [run all the unit tests](running-unit-tests.html) to test your installation. + +The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md) has more details about how to contribute. + + +## Option B: Using a Local Dev Machine and Docker You need to have recent versions of [Docker Engine](https://docs.docker.com/engine/installation/) and (Docker) [Compose](https://docs.docker.com/compose/install/). @@ -50,6 +76,8 @@ Build the images: docker-compose build ``` +### Docker with RethinkDB + **Note**: If you're upgrading BigchainDB and have previously already built the images, you may need to rebuild them after the upgrade to install any new dependencies. @@ -62,7 +90,7 @@ docker-compose up -d rdb The RethinkDB web interface should be accessible at . Depending on which platform, and/or how you are running docker, you may need to change `localhost` for the `ip` of the machine that is running docker. As a -dummy example, if the `ip` of that machine was `0.0.0.0`, you would accees the +dummy example, if the `ip` of that machine was `0.0.0.0`, you would access the web interface at: . Start a BigchainDB node: @@ -83,6 +111,40 @@ If you wish to run the tests: docker-compose run --rm bdb py.test -v -n auto ``` +### Docker with MongoDB + +Start MongoDB: + +```bash +docker-compose up -d mdb +``` + +MongoDB should now be up and running. You can check the port binding for the +MongoDB driver port using: +```bash +$ docker-compose port mdb 27017 +``` + +Start a BigchainDB node: + +```bash +docker-compose up -d bdb-mdb +``` + +You can monitor the logs: + +```bash +docker-compose logs -f bdb-mdb +``` + +If you wish to run the tests: + +```bash +docker-compose run --rm bdb-mdb py.test -v --database-backend=mongodb +``` + +### Accessing the HTTP API + A quick check to make sure that the BigchainDB server API is operational: ```bash @@ -123,3 +185,9 @@ root: ```bash curl 0.0.0.0:32772 ``` + +## Option C: Using a Dev Machine on Cloud9 + +Ian Worrall of [Encrypted Labs](http://www.encryptedlabs.com/) wrote a document (PDF) explaining how to set up a BigchainDB (Server) dev machine on Cloud9: + +[Download that document from GitHub](https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/docs/server/source/_static/cloud9.pdf) diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index dfe485c5..3c6a78f3 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -2,34 +2,55 @@ This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. If you're not using Linux, then you might try [running BigchainDB with Docker](appendices/run-with-docker.html). -A. [Install RethinkDB Server](https://rethinkdb.com/docs/install/ubuntu/) +A. Install the database backend. -B. Open a Terminal and run RethinkDB Server with the command: +[Install RethinkDB Server](https://rethinkdb.com/docs/install/ubuntu/) or +[Install MongoDB Server 3.4+](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) + +B. Run the database backend. Open a Terminal and run the command: + +with RethinkDB ```text -rethinkdb +$ rethinkdb +``` + +with MongoDB __3.4+__ +```text +$ mongod --replSet=bigchain-rs ``` C. Ubuntu 16.04 already has Python 3.5, so you don't need to install it, but you do need to install some other things: ```text -sudo apt-get update -sudo apt-get install g++ python3-dev libffi-dev +$ sudo apt-get update +$ sudo apt-get install g++ python3-dev libffi-dev ``` D. Get the latest version of pip and setuptools: ```text -sudo apt-get install python3-pip -sudo pip3 install --upgrade pip setuptools +$ sudo apt-get install python3-pip +$ sudo pip3 install --upgrade pip setuptools ``` E. Install the `bigchaindb` Python package from PyPI: ```text -sudo pip3 install bigchaindb +$ sudo pip3 install bigchaindb ``` -F. Configure and run BigchainDB Server: +F. Configure the BigchainDB Server: and run BigchainDB Server: + +with RethinkDB ```text -bigchaindb -y configure -bigchaindb start +$ bigchaindb -y configure rethinkdb +``` + +with MongoDB +```text +$ bigchaindb -y configure mongodb +``` + +G. Run the BigchainDB Server: +```text +$ bigchaindb start ``` That's it! diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 869ef804..f647b4bf 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -15,18 +15,22 @@ Show the version number. `bigchaindb -v` does the same thing. ## bigchaindb configure -Generate a local config file (which can be used to set some or all [BigchainDB node configuration settings](configuration.html)). It will auto-generate a public-private keypair and then ask you for the values of other configuration settings. If you press Enter for a value, it will use the default value. +Generate a local configuration file (which can be used to set some or all [BigchainDB node configuration settings](configuration.html)). It will auto-generate a public-private keypair and then ask you for the values of other configuration settings. If you press Enter for a value, it will use the default value. + +Since BigchainDB supports multiple databases you need to always specify the +database backend that you want to use. At this point only two database backends +are supported: `rethinkdb` and `mongodb`. If you use the `-c` command-line option, it will generate the file at the specified path: ```text -bigchaindb -c path/to/new_config.json configure +bigchaindb -c path/to/new_config.json configure rethinkdb ``` If you don't use the `-c` command-line option, the file will be written to `$HOME/.bigchaindb` (the default location where BigchainDB looks for a config file, if one isn't specified). If you use the `-y` command-line option, then there won't be any interactive prompts: it will just generate a keypair and use the default values for all the other configuration settings. ```text -bigchaindb -y configure +bigchaindb -y configure rethinkdb ``` @@ -83,3 +87,25 @@ Set the number of replicas (of each shard) in the underlying datastore. For exam ```text $ bigchaindb set-replicas 3 ``` + +## bigchaindb add-replicas + +This command is specific to MongoDB so it will only run if BigchainDB is +configured with `mongodb` as the backend. + +This command is used to add nodes to a BigchainDB cluster. It accepts a list of +space separated hosts in the form _hostname:port_: +```text +$ bigchaindb add-replicas server1.com:27017 server2.com:27017 server3.com:27017 +``` + +## bigchaindb remove-replicas + +This command is specific to MongoDB so it will only run if BigchainDB is +configured with `mongodb` as the backend. + +This command is used to remove nodes from a BigchainDB cluster. It accepts a +list of space separated hosts in the form _hostname:port_: +```text +$ bigchaindb remove-replicas server1.com:27017 server2.com:27017 server3.com:27017 +``` From f0e298bcd7d436ffd42a9562ac1495952a64dc8e Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 14:54:36 +0100 Subject: [PATCH 130/219] Added docstrings. Removed unnecessary returns. Created fixture to simplify the tests. Better comments. --- bigchaindb/backend/mongodb/admin.py | 28 ++++++++++++++---- tests/backend/mongodb/test_admin.py | 44 +++++++++++------------------ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/bigchaindb/backend/mongodb/admin.py b/bigchaindb/backend/mongodb/admin.py index 3c2001d5..afe909ac 100644 --- a/bigchaindb/backend/mongodb/admin.py +++ b/bigchaindb/backend/mongodb/admin.py @@ -18,13 +18,20 @@ def add_replicas(connection, replicas): """Add a set of replicas to the replicaset Args: - replicas list of strings: of the form "hostname:port". + connection (:class:`~bigchaindb.backend.connection.Connection`): + A connection to the database. + replicas (:obj:`list` of :obj:`str`): replica addresses in the + form "hostname:port". + + Raises: + DatabaseOpFailedError: If the reconfiguration fails due to a MongoDB + :exc:`OperationFailure` """ # get current configuration conf = connection.conn.admin.command('replSetGetConfig') - # MongoDB does not automatically add and id for the members so we need - # to chose one that does not exists yet. The safest way is to use + # MongoDB does not automatically add an id for the members so we need + # to choose one that does not exists yet. The safest way is to use # incrementing ids, so we first check what is the highest id already in # the set and continue from there. cur_id = max([member['_id'] for member in conf['config']['members']]) @@ -35,11 +42,13 @@ def add_replicas(connection, replicas): conf['config']['members'].append({'_id': cur_id, 'host': replica}) # increase the configuration version number + # when reconfiguring, mongodb expects a version number higher than the one + # it currently has conf['config']['version'] += 1 # apply new configuration try: - return connection.conn.admin.command('replSetReconfig', conf['config']) + connection.conn.admin.command('replSetReconfig', conf['config']) except OperationFailure as exc: raise DatabaseOpFailedError(exc.details['errmsg']) @@ -48,6 +57,15 @@ def add_replicas(connection, replicas): def remove_replicas(connection, replicas): """Remove a set of replicas from the replicaset + Args: + connection (:class:`~bigchaindb.backend.connection.Connection`): + A connection to the database. + replicas (:obj:`list` of :obj:`str`): replica addresses in the + form "hostname:port". + + Raises: + DatabaseOpFailedError: If the reconfiguration fails due to a MongoDB + :exc:`OperationFailure` """ # get the current configuration conf = connection.conn.admin.command('replSetGetConfig') @@ -63,6 +81,6 @@ def remove_replicas(connection, replicas): # apply new configuration try: - return connection.conn.admin.command('replSetReconfig', conf['config']) + connection.conn.admin.command('replSetReconfig', conf['config']) except OperationFailure as exc: raise DatabaseOpFailedError(exc.details['errmsg']) diff --git a/tests/backend/mongodb/test_admin.py b/tests/backend/mongodb/test_admin.py index 138bc616..a7784369 100644 --- a/tests/backend/mongodb/test_admin.py +++ b/tests/backend/mongodb/test_admin.py @@ -30,14 +30,22 @@ def mock_replicaset_config(): } -def test_add_replicas(mock_replicaset_config): +@pytest.fixture +def connection(): from bigchaindb.backend import connect - from bigchaindb.backend.admin import add_replicas - connection = connect() - # force the connection object to setup a connection to the database - # before we mock `Database.command` - connection.conn + # connection is a lazy object. It only actually creates a connection to + # the database when its first used. + # During the setup of a MongoDBConnection some `Database.command` are + # executed to make sure that the replica set is correctly initialized. + # Here we force the the connection setup so that all required + # `Database.command` are executed before we mock them it in the tests. + connection._connect() + return connection + + +def test_add_replicas(mock_replicaset_config, connection): + from bigchaindb.backend.admin import add_replicas expected_config = copy.deepcopy(mock_replicaset_config) expected_config['config']['members'] += [ @@ -54,16 +62,10 @@ def test_add_replicas(mock_replicaset_config): expected_config['config']) -def test_add_replicas_raises(mock_replicaset_config): - from bigchaindb.backend import connect +def test_add_replicas_raises(mock_replicaset_config, connection): from bigchaindb.backend.admin import add_replicas from bigchaindb.backend.exceptions import DatabaseOpFailedError - connection = connect() - # force the connection object to setup a connection to the database - # before we mock `Database.command` - connection.conn - with mock.patch.object(Database, 'command') as mock_command: mock_command.side_effect = [ mock_replicaset_config, @@ -73,15 +75,9 @@ def test_add_replicas_raises(mock_replicaset_config): add_replicas(connection, ['localhost:27018']) -def test_remove_replicas(mock_replicaset_config): - from bigchaindb.backend import connect +def test_remove_replicas(mock_replicaset_config, connection): from bigchaindb.backend.admin import remove_replicas - connection = connect() - # force the connection object to setup a connection to the database - # before we mock `Database.command` - connection.conn - expected_config = copy.deepcopy(mock_replicaset_config) expected_config['config']['version'] += 1 @@ -99,16 +95,10 @@ def test_remove_replicas(mock_replicaset_config): expected_config['config']) -def test_remove_replicas_raises(mock_replicaset_config): - from bigchaindb.backend import connect +def test_remove_replicas_raises(mock_replicaset_config, connection): from bigchaindb.backend.admin import remove_replicas from bigchaindb.backend.exceptions import DatabaseOpFailedError - connection = connect() - # force the connection object to setup a connection to the database - # before we mock `Database.command` - connection.conn - with mock.patch.object(Database, 'command') as mock_command: mock_command.side_effect = [ mock_replicaset_config, From b68557507f2ab7ad0720e0745180ab6be076eabc Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 31 Jan 2017 15:40:39 +0100 Subject: [PATCH 131/219] Updated Ansible playbooks & docs for Ubuntu 16.04 deployment --- .../template-ansible.md | 22 ++++++++++++------- ntools/one-m/ansible/install-python2.yml | 15 +++++++++++++ .../ansible/roles/bigchaindb/tasks/main.yml | 18 ++++++++------- .../ansible/roles/db_storage/tasks/main.yml | 4 +++- .../ansible/roles/rethinkdb/tasks/main.yml | 9 ++++---- 5 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 ntools/one-m/ansible/install-python2.yml diff --git a/docs/server/source/cloud-deployment-starter-templates/template-ansible.md b/docs/server/source/cloud-deployment-starter-templates/template-ansible.md index 1fd55950..e71d4cc1 100644 --- a/docs/server/source/cloud-deployment-starter-templates/template-ansible.md +++ b/docs/server/source/cloud-deployment-starter-templates/template-ansible.md @@ -2,12 +2,12 @@ If you didn't read the introduction to the [cloud deployment starter templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. -This page explains how to use [Ansible](https://www.ansible.com/) to install, configure and run all the software needed to run a one-machine BigchainDB node on a server running Ubuntu 14.04. +This page explains how to use [Ansible](https://www.ansible.com/) to install, configure and run all the software needed to run a one-machine BigchainDB node on a server running Ubuntu 16.04. ## Install Ansible -The Ansible documentation has [installation instructions](https://docs.ansible.com/ansible/intro_installation.html). Note the control machine requirements: at the time of writing, Ansible required Python 2.6 or 2.7. (Support for Python 3 [is a goal of Ansible 2.2](https://github.com/ansible/ansible/issues/15976#issuecomment-221264089).) +The Ansible documentation has [installation instructions](https://docs.ansible.com/ansible/intro_installation.html). Note the control machine requirements: at the time of writing, Ansible required Python 2.6 or 2.7. ([Python 3 support is coming](https://docs.ansible.com/ansible/python_3_support.html): "Ansible 2.2 features a tech preview of Python 3 support." and the latest version, as of January 31, 2017, was 2.2.1.0. For now, it's probably best to use it with Python 2.) For example, you could create a special Python 2.x virtualenv named `ansenv` and then install Ansible in it: ```text @@ -19,9 +19,9 @@ pip install ansible ## About Our Example Ansible Playbook -Our example Ansible playbook installs, configures and runs a basic BigchainDB node on an Ubuntu 14.04 machine. That playbook is in `.../bigchaindb/ntools/one-m/ansible/one-m-node.yml`. +Our example Ansible playbook installs, configures and runs a basic BigchainDB node on an Ubuntu 16.04 machine. That playbook is in `.../bigchaindb/ntools/one-m/ansible/one-m-node.yml`. -When you run the playbook (as per the instructions below), it ensures all the necessary software is installed, configured and running. It can be used to get a BigchainDB node set up on a bare Ubuntu 14.04 machine, but it can also be used to ensure that everything is okay on a running BigchainDB node. (If you run the playbook against a host where everything is okay, then it won't change anything on that host.) +When you run the playbook (as per the instructions below), it ensures all the necessary software is installed, configured and running. It can be used to get a BigchainDB node set up on a bare Ubuntu 16.04 machine, but it can also be used to ensure that everything is okay on a running BigchainDB node. (If you run the playbook against a host where everything is okay, then it won't change anything on that host.) ## Create an Ansible Inventory File @@ -39,7 +39,15 @@ echo "192.0.2.128" > hosts but replace `192.0.2.128` with the IP address of the host. -## Run the Ansible Playbook +## Run the Ansible Playbook(s) + +The latest Ubuntu 16.04 AMIs from Canonical don't include Python 2 (which is required by Ansible), so the first step is to run a small Ansible playbook to install Python 2 on the managed node: +```text +# cd to the directory .../bigchaindb/ntools/one-m/ansible +ansible-playbook -i hosts --private-key ~/.ssh/ install-python2.yml +``` + +where `` should be replaced by the name of the SSH private key you created earlier (for SSHing to the host machine at your cloud hosting provider). The next step is to run the Ansible playbook named `one-m-node.yml`: ```text @@ -47,14 +55,12 @@ The next step is to run the Ansible playbook named `one-m-node.yml`: ansible-playbook -i hosts --private-key ~/.ssh/ one-m-node.yml ``` -where `` should be replaced by the name of the SSH private key you created earlier (for SSHing to the host machine at your cloud hosting provider). - What did you just do? Running that playbook ensures all the software necessary for a one-machine BigchainDB node is installed, configured, and running properly. You can run that playbook on a regular schedule to ensure that the system stays properly configured. If something is okay, it does nothing; it only takes action when something is not as-desired. ## Some Notes on the One-Machine Node You Just Got Running -* It ensures that the installed version of RethinkDB is `2.3.4~0trusty`. You can change that by changing the installation task. +* It ensures that the installed version of RethinkDB is the latest. You can change that by changing the installation task. * It uses a very basic RethinkDB configuration file based on `bigchaindb/ntools/one-m/ansible/roles/rethinkdb/templates/rethinkdb.conf.j2`. * If you edit the RethinkDB configuration file, then running the Ansible playbook will **not** restart RethinkDB for you. You must do that manually. (You can stop RethinkDB using `sudo /etc/init.d/rethinkdb stop`; run the playbook to get RethinkDB started again. This assumes you're using init.d, which is what the Ansible playbook assumes. If you want to use systemd, you'll have to edit the playbook accordingly, and stop RethinkDB using `sudo systemctl stop rethinkdb@`.) * It generates and uses a default BigchainDB configuration file, which it stores in `~/.bigchaindb` (the default location). diff --git a/ntools/one-m/ansible/install-python2.yml b/ntools/one-m/ansible/install-python2.yml new file mode 100644 index 00000000..54dd7d0a --- /dev/null +++ b/ntools/one-m/ansible/install-python2.yml @@ -0,0 +1,15 @@ +--- +# This playbook ensures Python 2 is installed on the managed node. +# This is inspired by https://gist.github.com/gwillem/4ba393dceb55e5ae276a87300f6b8e6f + +- hosts: all + gather_facts: false + remote_user: ubuntu + + pre_tasks: + - name: Install Python 2 + raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) + become: true + + # action: setup will gather facts after python2 has been installed + - action: setup diff --git a/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml b/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml index 7ed7e992..5632bf9e 100644 --- a/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml +++ b/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml @@ -10,22 +10,24 @@ apt: name={{item}} state=latest update_cache=yes become: true with_items: + - make - git - g++ - - python3-dev - libffi-dev - - python3-setuptools # mainly for easy_install3, which is used to get latest pip3 - -# This should make both pip and pip3 be pip version >=8.1.2 (python 3.4). -# See the comments about this below. -- name: Ensure the latest pip/pip3 is installed, using easy_install3 - easy_install: executable=easy_install3 name=pip state=latest - become: true + - python3-dev + - python3-pip + - python3-setuptools - name: Ensure the latest setuptools (Python package) is installed pip: executable=pip3 name=setuptools state=latest become: true +# This should make both pip and pip3 be pip version >=8.1.2 (python 3.4). +# See the comments about this below. +#- name: Ensure the latest pip/pip3 is installed, using easy_install3 +# easy_install: executable=easy_install3 name=pip state=latest +# become: true + - name: Install BigchainDB from PyPI using sudo pip3 install bigchaindb pip: executable=pip3 name=bigchaindb state=latest become: true diff --git a/ntools/one-m/ansible/roles/db_storage/tasks/main.yml b/ntools/one-m/ansible/roles/db_storage/tasks/main.yml index 618a154f..0cb93555 100644 --- a/ntools/one-m/ansible/roles/db_storage/tasks/main.yml +++ b/ntools/one-m/ansible/roles/db_storage/tasks/main.yml @@ -12,12 +12,14 @@ # To better understand the /etc/fstab fields/columns, see: # http://man7.org/linux/man-pages/man5/fstab.5.html # https://tinyurl.com/jmmsyon = the soure code of the mount module +# Note: It seems the "nobootwait" option is gone in Ubuntu 16.04. See +# https://askubuntu.com/questions/786928/ubuntu-16-04-fstab-fails-with-nobootwait - name: Ensure /data dir exists and is mounted + update /etc/fstab mount: name=/data src=/dev/xvdp fstype=ext4 - opts="defaults,nofail,nobootwait" + opts="defaults,nofail" dump=0 passno=2 state=mounted diff --git a/ntools/one-m/ansible/roles/rethinkdb/tasks/main.yml b/ntools/one-m/ansible/roles/rethinkdb/tasks/main.yml index 61f3fd52..994a7d4f 100644 --- a/ntools/one-m/ansible/roles/rethinkdb/tasks/main.yml +++ b/ntools/one-m/ansible/roles/rethinkdb/tasks/main.yml @@ -2,11 +2,12 @@ # ansible/roles/rethinkdb/tasks/main.yml # Note: the .list extension will be added to the rethinkdb filename automatically +# Note: xenial is the $DISTRIB_CODENAME for Ubuntu 16.04 - name: > - Ensure RethinkDB's APT repository for Ubuntu trusty is present + Ensure RethinkDB's APT repository for Ubuntu xenial is present in /etc/apt/sources.list.d/rethinkdb.list apt_repository: - repo='deb http://download.rethinkdb.com/apt trusty main' + repo='deb http://download.rethinkdb.com/apt xenial main' filename=rethinkdb state=present become: true @@ -15,8 +16,8 @@ apt_key: url=http://download.rethinkdb.com/apt/pubkey.gpg state=present become: true -- name: Ensure the Ubuntu package rethinkdb 2.3.4~0trusty is installed - apt: name=rethinkdb=2.3.4~0trusty state=present update_cache=yes +- name: Ensure the latest rethinkdb package is installed + apt: name=rethinkdb state=latest update_cache=yes become: true - name: Ensure the /data directory's owner and group are both 'rethinkdb' From 248e89a666be55c32f4dd206b5b2994da7b5734e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 31 Jan 2017 15:43:46 +0100 Subject: [PATCH 132/219] unicode tests uses serialize() and includes info about unicode symbol --- tests/db/test_bigchain_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index aa8488c4..129b77c4 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1226,11 +1226,15 @@ def test_cant_spend_same_input_twice_in_tx(b, genesis_block): @pytest.mark.bdb def test_transaction_unicode(b): - import json + from bigchaindb.common.utils import serialize from bigchaindb.models import Transaction - tx = (Transaction.create([b.me], [([b.me], 100)], - {'beer': '\N{BEER MUG}'}) + + # http://www.fileformat.info/info/unicode/char/1f37a/index.htm + beer_python = {'beer': '\N{BEER MUG}'} + beer_json = '{"beer":"\N{BEER MUG}"}' + + tx = (Transaction.create([b.me], [([b.me], 100)], beer_python) ).sign([b.me_private]) block = b.create_block([tx]) assert block.validate(b) == block - assert '{"beer": "\\ud83c\\udf7a"}' in json.dumps(block.to_dict()) + assert beer_json in serialize(block.to_dict()) From 2af8fcb91863317794a2a8f9c772a394832f718e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 31 Jan 2017 15:48:34 +0100 Subject: [PATCH 133/219] test unicode write block to disk --- tests/db/test_bigchain_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 129b77c4..31abe176 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -1236,5 +1236,7 @@ def test_transaction_unicode(b): tx = (Transaction.create([b.me], [([b.me], 100)], beer_python) ).sign([b.me_private]) block = b.create_block([tx]) + b.write_block(block) + assert b.get_block(block.id) == block.to_dict() assert block.validate(b) == block assert beer_json in serialize(block.to_dict()) From 70e9a7a33cdd1bab80235544b6f2777606eaa7c5 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 31 Jan 2017 16:03:57 +0100 Subject: [PATCH 134/219] Updated many docs pages from Ubuntu 14.04 to 16.04 --- docs/server/source/appendices/install-latest-pip.md | 2 +- .../source/appendices/install-os-level-deps.md | 4 ++-- docs/server/source/appendices/install-with-lxd.md | 2 ++ docs/server/source/appendices/ntp-notes.md | 4 ++-- .../source/clusters-feds/aws-testing-cluster.md | 6 +++--- docs/server/source/nodes/node-requirements.md | 2 +- docs/server/source/nodes/setup-run-node.md | 12 ++---------- 7 files changed, 13 insertions(+), 19 deletions(-) diff --git a/docs/server/source/appendices/install-latest-pip.md b/docs/server/source/appendices/install-latest-pip.md index fac7dbed..97405882 100644 --- a/docs/server/source/appendices/install-latest-pip.md +++ b/docs/server/source/appendices/install-latest-pip.md @@ -7,7 +7,7 @@ pip -V If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.4, then you must install a `pip` version associated with Python 3.4+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). -On Ubuntu 14.04, we found that this works: +On Ubuntu 16.04, we found that this works: ```text sudo apt-get install python3-pip ``` diff --git a/docs/server/source/appendices/install-os-level-deps.md b/docs/server/source/appendices/install-os-level-deps.md index aa0df363..f1c4da99 100644 --- a/docs/server/source/appendices/install-os-level-deps.md +++ b/docs/server/source/appendices/install-os-level-deps.md @@ -2,13 +2,13 @@ BigchainDB Server has some OS-level dependencies that must be installed. -On Ubuntu 14.04 and 16.04, we found that the following was enough: +On Ubuntu 16.04, we found that the following was enough: ```text sudo apt-get update sudo apt-get install g++ python3-dev libffi-dev ``` -On Fedora 23 and 24, we found that the following was enough: +On Fedora 23–25, we found that the following was enough: ```text sudo dnf update sudo dnf install gcc-c++ redhat-rpm-config python3-devel libffi-devel diff --git a/docs/server/source/appendices/install-with-lxd.md b/docs/server/source/appendices/install-with-lxd.md index cb12b4ff..969f6841 100644 --- a/docs/server/source/appendices/install-with-lxd.md +++ b/docs/server/source/appendices/install-with-lxd.md @@ -1,5 +1,7 @@ # Installing BigchainDB on LXC containers using LXD +**Note: This page was contributed by an external contributor and is not actively maintained. We include it in case someone is interested.** + You can visit this link to install LXD (instructions here): [LXD Install](https://linuxcontainers.org/lxd/getting-started-cli/) (assumption is that you are using Ubuntu 14.04 for host/container) diff --git a/docs/server/source/appendices/ntp-notes.md b/docs/server/source/appendices/ntp-notes.md index 08861cb1..c1a2b261 100644 --- a/docs/server/source/appendices/ntp-notes.md +++ b/docs/server/source/appendices/ntp-notes.md @@ -23,9 +23,9 @@ If your BigchainDB node is running on an Amazon Linux instance (i.e. a Linux ins That said, you should check _which_ NTP daemon is installed. Is it recent? Is it configured securely? -## Ubuntu's ntp Package +## The Ubuntu ntp Packages -The [Ubuntu 14.04 (Trusty Tahr) package `ntp`](https://launchpad.net/ubuntu/trusty/+source/ntp) is based on the reference implementation of an NTP daemon (i.e. `ntpd`). +The [Ubuntu `ntp` packages](https://launchpad.net/ubuntu/+source/ntp) are based on the reference implementation of NTP. The following commands will uninstall the `ntp` and `ntpdate` packages, install the latest `ntp` package (which _might not be based on the latest ntpd code_), and start the NTP daemon (a local NTP server). (`ntpdate` is not reinstalled because it's [deprecated](https://askubuntu.com/questions/297560/ntpd-vs-ntpdate-pros-and-cons) and you shouldn't use it.) ```text diff --git a/docs/server/source/clusters-feds/aws-testing-cluster.md b/docs/server/source/clusters-feds/aws-testing-cluster.md index 2df15917..fbc623f3 100644 --- a/docs/server/source/clusters-feds/aws-testing-cluster.md +++ b/docs/server/source/clusters-feds/aws-testing-cluster.md @@ -14,11 +14,11 @@ We use some Bash and Python scripts to launch several instances (virtual servers ## Python Setup -The instructions that follow have been tested on Ubuntu 14.04, but may also work on similar distros or operating systems. +The instructions that follow have been tested on Ubuntu 16.04. Similar instructions should work on similar Linux distros. **Note: Our Python scripts for deploying to AWS use Python 2 because Fabric doesn't work with Python 3.** -You must install the Python package named `fabric`, but it depends on the `cryptography` package, and that depends on some OS-level packages. On Ubuntu 14.04, you can install those OS-level packages using: +You must install the Python package named `fabric`, but it depends on the `cryptography` package, and that depends on some OS-level packages. On Ubuntu 16.04, you can install those OS-level packages using: ```text sudo apt-get install build-essential libssl-dev libffi-dev python-dev ``` @@ -72,7 +72,7 @@ One way to monitor a BigchainDB cluster is to use the monitoring setup described You can deploy a monitoring server on AWS. To do that, go to the AWS EC2 Console and launch an instance: -1. Choose an AMI: select Ubuntu Server 14.04 LTS. +1. Choose an AMI: select Ubuntu Server 16.04 LTS. 2. Choose an Instance Type: a t2.micro will suffice. 3. Configure Instance Details: you can accept the defaults, but feel free to change them. 4. Add Storage: A "Root" volume type should already be included. You _could_ store monitoring data there (e.g. in a folder named `/influxdb-data`) but we will attach another volume and store the monitoring data there instead. Select "Add New Volume" and an EBS volume type. diff --git a/docs/server/source/nodes/node-requirements.md b/docs/server/source/nodes/node-requirements.md index bd72b9f4..56d52f13 100644 --- a/docs/server/source/nodes/node-requirements.md +++ b/docs/server/source/nodes/node-requirements.md @@ -9,7 +9,7 @@ Note: This section will be broken apart into several pages, e.g. NTP requirement * BigchainDB Server requires Python 3.4+ and Python 3.4+ [will run on any modern OS](https://docs.python.org/3.4/using/index.html). * BigchaindB Server uses the Python `multiprocessing` package and [some functionality in the `multiprocessing` package doesn't work on OS X](https://docs.python.org/3.4/library/multiprocessing.html#multiprocessing.Queue.qsize). You can still use Mac OS X if you use Docker or a virtual machine. -The BigchainDB core dev team uses Ubuntu 14.04, Ubuntu 16.04, Fedora 23, and Fedora 24. +The BigchainDB core dev team uses recent LTS versions of Ubuntu and recent versions of Fedora. We don't test BigchainDB on Windows or Mac OS X, but you can try. diff --git a/docs/server/source/nodes/setup-run-node.md b/docs/server/source/nodes/setup-run-node.md index b8de7340..2feb800d 100644 --- a/docs/server/source/nodes/setup-run-node.md +++ b/docs/server/source/nodes/setup-run-node.md @@ -96,20 +96,12 @@ If you're testing or developing BigchainDB on a stand-alone node, then you shoul BigchainDB Server has some OS-level dependencies that must be installed. -On Ubuntu 14.04, we found that the following was enough: +On Ubuntu 16.04, we found that the following was enough: ```text sudo apt-get update sudo apt-get install g++ python3-dev libffi-dev ``` -On Fedora 23, we found that the following was enough (tested in February 2015): -```text -sudo dnf update -sudo dnf install gcc-c++ redhat-rpm-config python3-devel libffi-devel -``` - -(If you're using a version of Fedora before version 22, you may have to use `yum` instead of `dnf`.) - With OS-level dependencies installed, you can install BigchainDB Server with `pip` or from source. @@ -122,7 +114,7 @@ pip -V If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.4, then you must install a `pip` version associated with Python 3.4+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). -On Ubuntu 14.04, we found that this works: +On Ubuntu 16.04, we found that this works: ```text sudo apt-get install python3-pip ``` From 9913929b9d3c12af8a073445016a4ec3e0512940 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 16:14:18 +0100 Subject: [PATCH 135/219] simplify run_configure --- bigchaindb/commands/bigchain.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 2fc8df70..272f8107 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -89,12 +89,7 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend)) - database = {} - if args.backend == 'rethinkdb': - database = bigchaindb._database_rethinkdb - elif args.backend == 'mongodb': - database = bigchaindb._database_mongodb - conf['database'] = database + conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: for key in ('bind', ): From 88ea6c6564aac020a83468f7ce56c83494383536 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 31 Jan 2017 16:18:20 +0100 Subject: [PATCH 136/219] In setup-run-node.md, link to page in Appendices re/ installing OS-level deps --- docs/server/source/nodes/setup-run-node.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/server/source/nodes/setup-run-node.md b/docs/server/source/nodes/setup-run-node.md index 2feb800d..9d0a4892 100644 --- a/docs/server/source/nodes/setup-run-node.md +++ b/docs/server/source/nodes/setup-run-node.md @@ -94,13 +94,7 @@ If you're testing or developing BigchainDB on a stand-alone node, then you shoul ## Install BigchainDB Server -BigchainDB Server has some OS-level dependencies that must be installed. - -On Ubuntu 16.04, we found that the following was enough: -```text -sudo apt-get update -sudo apt-get install g++ python3-dev libffi-dev -``` +First, [install the OS-level dependencies of BigchainDB Server (link)](../appendices/install-os-level-deps.html). With OS-level dependencies installed, you can install BigchainDB Server with `pip` or from source. From 84626b6e327369c3e3ee521638ecac6fff8b7598 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 16:23:09 +0100 Subject: [PATCH 137/219] Improved tests Fixed typo Add extra validation to hostnames to make sure host is not empty --- bigchaindb/backend/mongodb/admin.py | 2 +- bigchaindb/commands/utils.py | 2 +- tests/commands/test_commands.py | 22 ++++++++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/mongodb/admin.py b/bigchaindb/backend/mongodb/admin.py index afe909ac..7d72c3a4 100644 --- a/bigchaindb/backend/mongodb/admin.py +++ b/bigchaindb/backend/mongodb/admin.py @@ -31,7 +31,7 @@ def add_replicas(connection, replicas): conf = connection.conn.admin.command('replSetGetConfig') # MongoDB does not automatically add an id for the members so we need - # to choose one that does not exists yet. The safest way is to use + # to choose one that does not exist yet. The safest way is to use # incrementing ids, so we first check what is the highest id already in # the set and continue from there. cur_id = max([member['_id'] for member in conf['config']['members']]) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 7b662308..80ee7a6b 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -116,7 +116,7 @@ def mongodb_host(host): raise argparse.ArgumentTypeError(exc.args[0]) # we do require the port to be provided. - if port is None: + if port is None or hostname == '': raise argparse.ArgumentTypeError('expected host in the form ' '`host:port`. Got `{}` instead.' .format(host)) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 16b615eb..95bb0db7 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -22,6 +22,8 @@ def test_make_sure_we_dont_remove_any_command(): assert parser.parse_args(['set-shards', '1']).command assert parser.parse_args(['set-replicas', '1']).command assert parser.parse_args(['load']).command + assert parser.parse_args(['add-replicas', 'localhost:27017']).command + assert parser.parse_args(['remove-replicas', 'localhost:27017']).command def test_start_raises_if_command_not_implemented(): @@ -379,7 +381,7 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock, @pytest.mark.usefixtures('ignore_local_config_file') -@patch('bigchaindb.backend.admin.add_replicas') +@patch('bigchaindb.commands.bigchain.add_replicas') def test_run_add_replicas(mock_add_replicas): from bigchaindb.commands.bigchain import run_add_replicas from bigchaindb.backend.exceptions import DatabaseOpFailedError @@ -389,18 +391,24 @@ def test_run_add_replicas(mock_add_replicas): # test add_replicas no raises mock_add_replicas.return_value = None assert run_add_replicas(args) is None + assert mock_add_replicas.call_count == 1 + mock_add_replicas.reset_mock() # test add_replicas with `DatabaseOpFailedError` mock_add_replicas.side_effect = DatabaseOpFailedError() assert run_add_replicas(args) is None + assert mock_add_replicas.call_count == 1 + mock_add_replicas.reset_mock() # test add_replicas with `NotImplementedError` mock_add_replicas.side_effect = NotImplementedError() assert run_add_replicas(args) is None + assert mock_add_replicas.call_count == 1 + mock_add_replicas.reset_mock() @pytest.mark.usefixtures('ignore_local_config_file') -@patch('bigchaindb.backend.admin.remove_replicas') +@patch('bigchaindb.commands.bigchain.remove_replicas') def test_run_remove_replicas(mock_remove_replicas): from bigchaindb.commands.bigchain import run_remove_replicas from bigchaindb.backend.exceptions import DatabaseOpFailedError @@ -410,14 +418,20 @@ def test_run_remove_replicas(mock_remove_replicas): # test add_replicas no raises mock_remove_replicas.return_value = None assert run_remove_replicas(args) is None + assert mock_remove_replicas.call_count == 1 + mock_remove_replicas.reset_mock() # test add_replicas with `DatabaseOpFailedError` mock_remove_replicas.side_effect = DatabaseOpFailedError() assert run_remove_replicas(args) is None + assert mock_remove_replicas.call_count == 1 + mock_remove_replicas.reset_mock() # test add_replicas with `NotImplementedError` mock_remove_replicas.side_effect = NotImplementedError() assert run_remove_replicas(args) is None + assert mock_remove_replicas.call_count == 1 + mock_remove_replicas.reset_mock() def test_mongodb_host_type(): @@ -430,3 +444,7 @@ def test_mongodb_host_type(): # no port information provided with pytest.raises(ArgumentTypeError): mongodb_host('localhost') + + # bad host provided + with pytest.raises(ArgumentTypeError): + mongodb_host(':27017') From 157db3e01fd08b0f0ba6770ed420e1b7931fb55f Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 31 Jan 2017 16:59:43 +0100 Subject: [PATCH 138/219] Fix exception in test --- tests/backend/mongodb/test_connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index 49ab0604..60181f25 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -56,13 +56,14 @@ def test_get_connection_returns_the_correct_instance(): @mock.patch('time.sleep') def test_connection_error(mock_sleep, mock_client, mock_init_repl_set): from bigchaindb.backend import connect + from bigchaindb.backend.exceptions import ConnectionError # force the driver to trow ConnectionFailure # the mock on time.sleep is to prevent the actual sleep when running # the tests mock_client.side_effect = ConnectionFailure() - with pytest.raises(ConnectionFailure): + with pytest.raises(ConnectionError): conn = connect() conn.db From 857cdb9b341568300fd8012149c70a747ca6c20f Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 31 Jan 2017 17:01:22 +0100 Subject: [PATCH 139/219] Checking for replica set is now within try..except --- bigchaindb/backend/mongodb/connection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 61cc430d..79a085fb 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -76,17 +76,17 @@ class MongoDBConnection(Connection): attempt += 1 try: + # we should only return a connection if the replica set is + # initialized. initialize_replica_set will check if the + # replica set is initialized else it will initialize it. + initialize_replica_set(self.host, self.port, self.connection_timeout) + # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. self.connection = MongoClient(self.host, self.port, replicaset=self.replicaset, serverselectiontimeoutms=self.connection_timeout) - - # we should only return a connection if the replica set is - # initialized. initialize_replica_set will check if the - # replica set is initialized else it will initialize it. - initialize_replica_set(self.host, self.port, self.connection_timeout) except (errors.ConnectionFailure, errors.AutoReconnect) as exc: logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.', attempt, self.max_tries if self.max_tries != 0 else '∞', From 1588681c5b7edabf9e6b02206053a557e3233c73 Mon Sep 17 00:00:00 2001 From: vrde Date: Tue, 31 Jan 2017 17:13:26 +0100 Subject: [PATCH 140/219] Fix pep8 error --- bigchaindb/backend/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigchaindb/backend/exceptions.py b/bigchaindb/backend/exceptions.py index 1f7c57a7..41bac0c6 100644 --- a/bigchaindb/backend/exceptions.py +++ b/bigchaindb/backend/exceptions.py @@ -4,5 +4,6 @@ from bigchaindb.exceptions import BigchainDBError class ConnectionError(BigchainDBError): """Exception raised when the connection to the DataBase fails.""" + class DatabaseOpFailedError(BigchainDBError): """Exception for database operation errors.""" From aae60ea467b085ed6234b5594eae2bd3ef592c28 Mon Sep 17 00:00:00 2001 From: "krish7919 (Krish)" Date: Tue, 31 Jan 2017 17:03:02 +0100 Subject: [PATCH 141/219] Solves #1105. The `apt-get update` command executed with the install instructions should not use a locally cached storage layer. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c181625a..54fdc41b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,9 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ ENV LANG en_US.UTF-8 -RUN apt-get -y install python3 python3-pip libffi-dev -RUN pip3 install --upgrade pip -RUN pip3 install --upgrade setuptools +RUN apt-get update && apt-get -y install python3 python3-pip libffi-dev \ + && pip3 install --upgrade pip \ + && pip3 install --upgrade setuptools RUN mkdir -p /usr/src/app From b318d62f7ec76f308f4d06e19c3951b0bee6cf5b Mon Sep 17 00:00:00 2001 From: "krish7919 (Krish)" Date: Tue, 31 Jan 2017 17:34:25 +0100 Subject: [PATCH 142/219] Update Dockerfile with comments on force refresh --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 54fdc41b..f5e4b3b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,9 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ ENV LANG en_US.UTF-8 +# The `apt-get update` command executed with the install instructions should +# not use a locally cached storage layer. Force update the cache again. +# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#run RUN apt-get update && apt-get -y install python3 python3-pip libffi-dev \ && pip3 install --upgrade pip \ && pip3 install --upgrade setuptools From 2de652ad5c1a9855bf08ebf22fa006e56c286eeb Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 8 Dec 2016 11:51:23 +0100 Subject: [PATCH 143/219] add replicating test --- tests/integration/test_integration.py | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 70781096..78de03d3 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -44,3 +44,35 @@ def test_double_create(b, user_pk): last_voted_block = b.get_last_voted_block() assert len(last_voted_block.transactions) == 1 assert count_blocks(b.connection) == 2 + + +@pytest.mark.usefixtures('processes', 'inputs') +def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): + """See issue 633.""" + from bigchaindb.models import Transaction + input_valid = b.get_owned_ids(user_pk).pop() + input_valid = b.get_transaction(input_valid.txid) + tx_valid = Transaction.transfer(input_valid.to_inputs(), + [([user_pk], 1)], + input_valid.asset).sign([user_sk]) + + # write the valid tx and wait for voting/block to catch up + b.write_transaction(tx_valid) + time.sleep(2) + + # doesn't throw an exception + b.get_owned_ids(user_pk) + + # create another transaction with the same input + tx_double_spend = Transaction.transfer(input_valid.to_inputs(), + [([user_pk], 1)], + input_valid.asset) \ + .sign([user_sk]) + + # write the double spend tx + b.write_transaction(tx_double_spend) + time.sleep(2) + + # still doesn't throw an exception + b.get_owned_ids(user_pk) + assert b.is_valid_transaction(tx_double_spend) is False From d1b3a206ca5c54e52d6fb7f673a2ac267ca9709e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 31 Jan 2017 16:52:23 +0100 Subject: [PATCH 144/219] get integration test working issue 633 --- tests/integration/test_integration.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 78de03d3..ceb18b04 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -46,15 +46,16 @@ def test_double_create(b, user_pk): assert count_blocks(b.connection) == 2 -@pytest.mark.usefixtures('processes', 'inputs') +@pytest.mark.usefixtures('inputs') def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): - """See issue 633.""" + """ Test for #633 https://github.com/bigchaindb/bigchaindb/issues/633 """ from bigchaindb.models import Transaction input_valid = b.get_owned_ids(user_pk).pop() input_valid = b.get_transaction(input_valid.txid) tx_valid = Transaction.transfer(input_valid.to_inputs(), [([user_pk], 1)], - input_valid.asset).sign([user_sk]) + input_valid.id, + {'1': 1}).sign([user_sk]) # write the valid tx and wait for voting/block to catch up b.write_transaction(tx_valid) @@ -66,8 +67,8 @@ def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): # create another transaction with the same input tx_double_spend = Transaction.transfer(input_valid.to_inputs(), [([user_pk], 1)], - input_valid.asset) \ - .sign([user_sk]) + input_valid.id, + {'2': 2}).sign([user_sk]) # write the double spend tx b.write_transaction(tx_double_spend) From 7f5318cba43a1cbffe63280fe04ee1b9840fe816 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 1 Feb 2017 14:33:53 +0100 Subject: [PATCH 145/219] check validate_transaction raises DoubleSpend in integration test --- tests/integration/test_integration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index ceb18b04..6597a0e7 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -49,6 +49,7 @@ def test_double_create(b, user_pk): @pytest.mark.usefixtures('inputs') def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): """ Test for #633 https://github.com/bigchaindb/bigchaindb/issues/633 """ + from bigchaindb.common.exceptions import DoubleSpend from bigchaindb.models import Transaction input_valid = b.get_owned_ids(user_pk).pop() input_valid = b.get_transaction(input_valid.txid) @@ -76,4 +77,5 @@ def test_get_owned_ids_works_after_double_spend(b, user_pk, user_sk): # still doesn't throw an exception b.get_owned_ids(user_pk) - assert b.is_valid_transaction(tx_double_spend) is False + with pytest.raises(DoubleSpend): + b.validate_transaction(tx_double_spend) From c572464be656f829a3cb935d2bc71dd680f7d645 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 1 Feb 2017 14:47:19 +0100 Subject: [PATCH 146/219] fix asset_id index in mongodb --- bigchaindb/backend/mongodb/schema.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 2c526e7c..f6aac93d 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -60,8 +60,7 @@ def create_bigchain_secondary_index(conn, dbname): # secondary index for asset uuid, this field is unique conn.conn[dbname]['bigchain']\ - .create_index('block.transactions.transaction.asset.id', - name='asset_id') + .create_index('block.transactions.asset.id', name='asset_id') def create_backlog_secondary_index(conn, dbname): From b7f70befe6cd6d8647aa2eaa3e9505348be8cf0e Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 1 Feb 2017 15:10:17 +0100 Subject: [PATCH 147/219] fix wrong logic in validate_block --- bigchaindb/models.py | 5 ----- tests/test_models.py | 13 ------------- 2 files changed, 18 deletions(-) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c6e81956..c3683a03 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -202,11 +202,6 @@ class Block(object): OperationError: If a non-federation node signed the Block. InvalidSignature: If a Block's signature is invalid. """ - - # First, make sure this node hasn't already voted on this block - if bigchain.has_previous_vote(self.id, self.voters): - return self - # Check if the block was created by a federation node possible_voters = (bigchain.nodes_except_me + [bigchain.me]) if self.node_pubkey not in possible_voters: diff --git a/tests/test_models.py b/tests/test_models.py index 7ab97e9e..58aa64fd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -163,16 +163,3 @@ class TestBlockModel(object): public_key = PublicKey(b.me) assert public_key.verify(expected_block_serialized, block.signature) - - def test_validate_already_voted_on_block(self, b, monkeypatch): - from unittest.mock import Mock - from bigchaindb.models import Transaction - - tx = Transaction.create([b.me], [([b.me], 1)]) - block = b.create_block([tx]) - - has_previous_vote = Mock() - has_previous_vote.return_value = True - monkeypatch.setattr(b, 'has_previous_vote', has_previous_vote) - assert block == block.validate(b) - assert has_previous_vote.called is True From d49b06933a0237ecbedde9c0cba033d4d7af12ca Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Wed, 1 Feb 2017 16:24:34 +0100 Subject: [PATCH 148/219] Fix docstring of `recipient` argument to be a list of tuples (#1091) --- bigchaindb/common/transaction.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 65b12eed..e4bd642f 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -482,8 +482,8 @@ class Transaction(object): Args: tx_signers (:obj:`list` of :obj:`str`): A list of keys that represent the signers of the CREATE Transaction. - recipients (:obj:`list` of :obj:`str`): A list of keys that - represent the recipients of the outputs of this + recipients (:obj:`list` of :obj:`tuple`): A list of + ([keys],amount) that represent the recipients of this Transaction. metadata (dict): The metadata to be stored along with the Transaction. @@ -549,7 +549,7 @@ class Transaction(object): inputs (:obj:`list` of :class:`~bigchaindb.common.transaction. Input`): Converted `Output`s, intended to be used as inputs in the transfer to generate. - recipients (:obj:`list` of :obj:`str`): A list of + recipients (:obj:`list` of :obj:`tuple`): A list of ([keys],amount) that represent the recipients of this Transaction. asset_id (str): The asset ID of the asset to be transferred in From 7206eb35c45289f132e025019e399ed4bfbb1e50 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 1 Feb 2017 17:44:11 +0100 Subject: [PATCH 149/219] Updated CHANGELOG.md for v0.9.0 --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 312c589f..e993968a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,39 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [0.9.0] - 2017-02-06 +Tag name: v0.9.0 + +It has been more than two months since the v0.8.0 release, so there have been _many_ changes. We decided to describe them in broad strokes, with links to more details elsewhere. + +### Added +- Support for MongoDB as a backend database. +- Some configuration settings and `bigchaindb` command-line commands were added. In particular, one can specify the database backend (`rethinkdb` or `mongodb`). For MongoDB, one can specify the name of the replicaset. Also for MongoDB, there are new command-line commands to add and remove hosts from the replicaset. See [the Settings & CLI docs](https://docs.bigchaindb.com/projects/server/en/v0.9.0/server-reference/index.html). +- Transaction schema validation. The transaction schema is also used to auto-generate some docs. [Pull Request #880](https://github.com/bigchaindb/bigchaindb/pull/880) +- Vote schema validation. The vote schema is also used to auto-generate some docs. [Pull Request #865](https://github.com/bigchaindb/bigchaindb/pull/865) +- New `ENABLE_WEB_ADMIN` setting in the AWS deployment configuration file. [Pull Request #1015](https://github.com/bigchaindb/bigchaindb/pull/1015) + +### Changed +- The transaction model has changed substantially. @libscott wrote a blog post about the changes and it will be published soon on [the BigchainDB Blog](https://blog.bigchaindb.com/). Also, see [the docs about the transaction model](https://docs.bigchaindb.com/projects/server/en/v0.9.0/data-models/transaction-model.html). +- The HTTP API has changed substantially. @diminator wrote a blog post about the changes and it will be published soon on [the BigchainDB Blog](https://blog.bigchaindb.com/). Also, see [the docs about the vote model](https://docs.bigchaindb.com/projects/server/en/v0.9.0/data-models/vote-model.html). +- All RethinkDB-specific database calls were replaced with abstract calls to a backend database. +- Some improvements to the Dockerfile, e.g. Pull Requests [#1011](https://github.com/bigchaindb/bigchaindb/pull/1011) and [#1121](https://github.com/bigchaindb/bigchaindb/pull/1121) +- Many improvements to the tests +- We standardized on supporting Ubuntu 16.04 for now (but BigchainDB Server also works on similar Linux distros). + +### Removed +- `api_endpoint` was removed from the BigchainDB configuration settings. (It was never used anyway.) [Pull Request #821](https://github.com/bigchaindb/bigchaindb/pull/821) + +### Fixed +- Fixed a memory (RAM) overflow problem when under heavy load by bounding the size of the queue at the entrance to the block pipeline. [Pull Request #908](https://github.com/bigchaindb/bigchaindb/pull/908) + +### External Contributors +- @utarl - [Pull Request #1019](https://github.com/bigchaindb/bigchaindb/pull/1019) + +### Notes +- There were many additions and changes to the documentation. Fun fact: The JSON in the HTTP API docs is now auto-generated to be consistent with the current code. +- There's a draft spec for a BigchainDB Event Stream API and we welcome your feedback. See [Pull Request #1086](https://github.com/bigchaindb/bigchaindb/pull/1086) + ## [0.8.2] - 2017-01-27 Tag name: v0.8.2 @@ -26,8 +59,6 @@ Tag name: v0.8.2 ## [0.8.1] - 2017-01-16 Tag name: v0.8.1 -= commit: -committed: ### Changed - Upgrade pysha3 to 1.0.0 (supports official NIST standard). @@ -39,8 +70,6 @@ committed: ## [0.8.0] - 2016-11-29 Tag name: v0.8.0 -= commit: -committed: ### Added - The big new thing in version 0.8.0 is support for divisible assets, i.e. assets like carrots or thumbtacks, where the initial CREATE transaction can register/create some amount (e.g. 542 carrots), the first TRANSFER transaction can split that amount across multiple owners, and so on. [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) From 6fd8c7a20bdcf9295b0ec63ef14ed3cb67f49868 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 2 Feb 2017 09:45:13 +0100 Subject: [PATCH 150/219] Feat/105/secondary indexes inputs outputs (#1125) * Added inputs/outputs secondary indexes for rethinkdb Added tests. * Added inputs/outputs secondary indexes for mongodb Fixed tests. * fixed comment --- bigchaindb/backend/mongodb/query.py | 11 ++++++----- bigchaindb/backend/mongodb/schema.py | 12 ++++++++++++ bigchaindb/backend/rethinkdb/query.py | 15 ++++++++------- bigchaindb/backend/rethinkdb/schema.py | 25 +++++++++++++++++++++++++ tests/backend/mongodb/test_schema.py | 8 ++++---- tests/backend/rethinkdb/test_schema.py | 4 ++++ 6 files changed, 59 insertions(+), 16 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index d7ee6afc..e3b71315 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -143,6 +143,10 @@ def get_asset_by_id(conn, asset_id): @register_query(MongoDBConnection) def get_spent(conn, transaction_id, output): cursor = conn.db['bigchain'].aggregate([ + {'$match': { + 'block.transactions.inputs.fulfills.txid': transaction_id, + 'block.transactions.inputs.fulfills.output': output + }}, {'$unwind': '$block.transactions'}, {'$match': { 'block.transactions.inputs.fulfills.txid': transaction_id, @@ -157,12 +161,9 @@ def get_spent(conn, transaction_id, output): @register_query(MongoDBConnection) def get_owned_ids(conn, owner): cursor = conn.db['bigchain'].aggregate([ + {'$match': {'block.transactions.outputs.public_keys': owner}}, {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.outputs.public_keys': { - '$elemMatch': {'$eq': owner} - } - }} + {'$match': {'block.transactions.outputs.public_keys': owner}} ]) # we need to access some nested fields before returning so lets use a # generator to avoid having to read all records on the cursor at this point diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 2c526e7c..95c2d02a 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -63,6 +63,18 @@ def create_bigchain_secondary_index(conn, dbname): .create_index('block.transactions.transaction.asset.id', name='asset_id') + # secondary index on the public keys of outputs + conn.conn[dbname]['bigchain']\ + .create_index('block.transactions.outputs.public_keys', + name='outputs') + + # secondary index on inputs/transaction links (txid, output) + conn.conn[dbname]['bigchain']\ + .create_index([ + ('block.transactions.inputs.fulfills.txid', ASCENDING), + ('block.transactions.inputs.fulfills.output', ASCENDING), + ], name='inputs') + def create_backlog_secondary_index(conn, dbname): logger.info('Create `backlog` secondary index.') diff --git a/bigchaindb/backend/rethinkdb/query.py b/bigchaindb/backend/rethinkdb/query.py index aa7c3be6..99346984 100644 --- a/bigchaindb/backend/rethinkdb/query.py +++ b/bigchaindb/backend/rethinkdb/query.py @@ -111,21 +111,22 @@ def _get_asset_create_tx_query(asset_id): @register_query(RethinkDBConnection) def get_spent(connection, transaction_id, output): - # TODO: use index! return connection.run( r.table('bigchain', read_mode=READ_MODE) - .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda transaction: transaction['inputs'].contains( - lambda input: input['fulfills'] == {'txid': transaction_id, 'output': output}))) + .get_all([transaction_id, output], index='inputs') + .concat_map(lambda doc: doc['block']['transactions']) + .filter(lambda transaction: transaction['inputs'].contains( + lambda input_: input_['fulfills'] == {'txid': transaction_id, 'output': output}))) @register_query(RethinkDBConnection) def get_owned_ids(connection, owner): - # TODO: use index! return connection.run( r.table('bigchain', read_mode=READ_MODE) - .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda tx: tx['outputs'].contains( + .get_all(owner, index='outputs') + .distinct() + .concat_map(lambda doc: doc['block']['transactions']) + .filter(lambda tx: tx['outputs'].contains( lambda c: c['public_keys'].contains(owner)))) diff --git a/bigchaindb/backend/rethinkdb/schema.py b/bigchaindb/backend/rethinkdb/schema.py index 4a76a06b..997ec5fc 100644 --- a/bigchaindb/backend/rethinkdb/schema.py +++ b/bigchaindb/backend/rethinkdb/schema.py @@ -66,6 +66,31 @@ def create_bigchain_secondary_index(connection, dbname): .table('bigchain') .index_create('asset_id', r.row['block']['transactions']['asset']['id'], multi=True)) + # secondary index on the public keys of outputs + # the last reduce operation is to return a flatten list of public_keys + # without it we would need to match exactly the public_keys list. + # For instance querying for `pk1` would not match documents with + # `public_keys: [pk1, pk2, pk3]` + connection.run( + r.db(dbname) + .table('bigchain') + .index_create('outputs', + r.row['block']['transactions'] + .concat_map(lambda tx: tx['outputs']['public_keys']) + .reduce(lambda l, r: l + r), multi=True)) + + # secondary index on inputs/transaction links (txid, output) + connection.run( + r.db(dbname) + .table('bigchain') + .index_create('inputs', + r.row['block']['transactions'] + .concat_map(lambda tx: tx['inputs']['fulfills']) + .with_fields('txid', 'output') + .map(lambda fulfills: [fulfills['txid'], + fulfills['output']]), + multi=True)) + # wait for rethinkdb to finish creating secondary indexes connection.run( r.db(dbname) diff --git a/tests/backend/mongodb/test_schema.py b/tests/backend/mongodb/test_schema.py index 34b6edf9..71eac7ff 100644 --- a/tests/backend/mongodb/test_schema.py +++ b/tests/backend/mongodb/test_schema.py @@ -21,8 +21,8 @@ def test_init_creates_db_tables_and_indexes(): assert sorted(collection_names) == ['backlog', 'bigchain', 'votes'] indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', - 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', + 'outputs', 'transaction_id'] indexes = conn.conn[dbname]['backlog'].index_information().keys() assert sorted(indexes) == ['_id_', 'assignee__transaction_timestamp', @@ -81,8 +81,8 @@ def test_create_secondary_indexes(): # Bigchain table indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', - 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', + 'outputs', 'transaction_id'] # Backlog table indexes = conn.conn[dbname]['backlog'].index_information().keys() diff --git a/tests/backend/rethinkdb/test_schema.py b/tests/backend/rethinkdb/test_schema.py index 1447e80f..e19dfdc2 100644 --- a/tests/backend/rethinkdb/test_schema.py +++ b/tests/backend/rethinkdb/test_schema.py @@ -85,6 +85,10 @@ def test_create_secondary_indexes(): 'transaction_id')) is True assert conn.run(r.db(dbname).table('bigchain').index_list().contains( 'asset_id')) is True + assert conn.run(r.db(dbname).table('bigchain').index_list().contains( + 'inputs')) is True + assert conn.run(r.db(dbname).table('bigchain').index_list().contains( + 'outputs')) is True # Backlog table assert conn.run(r.db(dbname).table('backlog').index_list().contains( From 6b073f089845fc28f10e653518563d3f93f09736 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 2 Feb 2017 13:35:20 +0100 Subject: [PATCH 151/219] Removed duplicated transaction validation in votes pipeline. Separated block validation from transaction validation. Refactored block signature validation. --- bigchaindb/models.py | 39 +++++++++++++----------------------- bigchaindb/pipelines/vote.py | 8 +++----- tests/test_models.py | 4 ++-- 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c6e81956..c5efffa7 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -203,6 +203,12 @@ class Block(object): InvalidSignature: If a Block's signature is invalid. """ + self._validate_block(bigchain) + self._validate_block_transactions(bigchain) + + return self + + def _validate_block(self, bigchain): # First, make sure this node hasn't already voted on this block if bigchain.has_previous_vote(self.id, self.voters): return self @@ -212,18 +218,16 @@ class Block(object): if self.node_pubkey not in possible_voters: raise OperationError('Only federation nodes can create blocks') + # Check that the signature is valid if not self.is_signature_valid(): - raise InvalidSignature('Block signature invalid') + raise InvalidSignature('Invalid block signature') - # Finally: Tentative assumption that every blockchain will want to - # validate all transactions in each block + def _validate_block_transactions(self, bigchain): for tx in self.transactions: - # NOTE: If a transaction is not valid, `is_valid` will throw an - # an exception and block validation will be canceled. + # If a transaction is not valid, `validate_transactions` will + # throw an an exception and block validation will be canceled. bigchain.validate_transaction(tx) - return self - def sign(self, private_key): """Create a signature for the Block and overwrite `self.signature`. @@ -273,34 +277,19 @@ class Block(object): InvalidSignature: If the block's signature is not corresponding to it's data or `node_pubkey`. """ - # TODO: Reuse `is_signature_valid` method here. + # Validate block id block = block_body['block'] block_serialized = serialize(block) block_id = hash_data(block_serialized) - public_key = PublicKey(block['node_pubkey']) - - try: - signature = block_body['signature'] - except KeyError: - signature = None if block_id != block_body['id']: raise InvalidHash() - if signature is not None: - # NOTE: CC throws a `ValueError` on some wrong signatures - # https://github.com/bigchaindb/cryptoconditions/issues/27 - try: - signature_valid = public_key\ - .verify(block_serialized.encode(), signature) - except ValueError: - signature_valid = False - if signature_valid is False: - raise InvalidSignature('Invalid block signature') - transactions = [Transaction.from_dict(tx) for tx in block['transactions']] + signature = block_body.get('signature') + return cls(transactions, block['node_pubkey'], block['timestamp'], block['voters'], signature) diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py index 2a9b0ee5..5f385f53 100644 --- a/bigchaindb/pipelines/vote.py +++ b/bigchaindb/pipelines/vote.py @@ -53,7 +53,7 @@ class Vote: block['block']['voters']): try: block = Block.from_dict(block) - except (exceptions.InvalidHash, exceptions.InvalidSignature): + except (exceptions.InvalidHash): # XXX: if a block is invalid we should skip the `validate_tx` # step, but since we are in a pipeline we cannot just jump to # another function. Hackish solution: generate an invalid @@ -61,10 +61,8 @@ class Vote: # pipeline. return block['id'], [self.invalid_dummy_tx] try: - self.consensus.validate_block(self.bigchain, block) - except (exceptions.InvalidHash, - exceptions.OperationError, - exceptions.InvalidSignature): + block._validate_block(self.bigchain) + except (exceptions.OperationError, exceptions.InvalidSignature): # XXX: if a block is invalid we should skip the `validate_tx` # step, but since we are in a pipeline we cannot just jump to # another function. Hackish solution: generate an invalid diff --git a/tests/test_models.py b/tests/test_models.py index 7ab97e9e..9811fcae 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -107,7 +107,7 @@ class TestBlockModel(object): with raises(InvalidHash): Block.from_dict(block) - def test_block_invalid_signature_deserialization(self, b): + def test_block_invalid_signature(self, b): from bigchaindb.common.crypto import hash_data from bigchaindb.common.exceptions import InvalidSignature from bigchaindb.common.utils import gen_timestamp, serialize @@ -131,7 +131,7 @@ class TestBlockModel(object): } with raises(InvalidSignature): - Block.from_dict(block_body) + Block.from_dict(block_body).validate(b) def test_compare_blocks(self, b): from bigchaindb.models import Block, Transaction From 0e6422f6ccd219d9f02b88e54b1f356dfd2606d0 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 2 Feb 2017 15:24:04 +0100 Subject: [PATCH 152/219] Added line re: PR #1130 to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e993968a..32da0036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ It has been more than two months since the v0.8.0 release, so there have been _m ### Fixed - Fixed a memory (RAM) overflow problem when under heavy load by bounding the size of the queue at the entrance to the block pipeline. [Pull Request #908](https://github.com/bigchaindb/bigchaindb/pull/908) +- Fixed some logic in block validation. [Pull Request #1130](https://github.com/bigchaindb/bigchaindb/pull/1130) ### External Contributors - @utarl - [Pull Request #1019](https://github.com/bigchaindb/bigchaindb/pull/1019) From 516c7539108c2d69dd40a826d2ac4d5801369c54 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 2 Feb 2017 16:06:50 +0100 Subject: [PATCH 153/219] Add param 'rethinkdb' in docs for configure cmd. --- docs/server/source/appendices/run-with-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index 455331ed..d6e33a70 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -21,7 +21,7 @@ be stored in a file on your host machine at `~/bigchaindb_docker/.bigchaindb`: ```text docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ - bigchaindb/bigchaindb -y configure + bigchaindb/bigchaindb -y configure rethinkdb Generating keypair Configuration written to /data/.bigchaindb Ready to go! From 16571b539f7e9034fbcbc8ca63d53b6b799f6022 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 2 Feb 2017 19:18:47 +0100 Subject: [PATCH 154/219] Normalize exceptions --- bigchaindb/backend/connection.py | 8 +++ bigchaindb/backend/exceptions.py | 16 +++-- bigchaindb/backend/mongodb/changefeed.py | 21 ++++--- bigchaindb/backend/mongodb/connection.py | 74 ++++++++++++++---------- bigchaindb/backend/mongodb/query.py | 45 +++++++------- bigchaindb/backend/rethinkdb/admin.py | 6 +- bigchaindb/commands/bigchain.py | 6 +- tests/backend/mongodb/test_changefeed.py | 6 +- tests/backend/mongodb/test_connection.py | 2 +- tests/backend/rethinkdb/test_admin.py | 8 +-- 10 files changed, 109 insertions(+), 83 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 0fda4078..bb4688da 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -32,6 +32,7 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None): based on the given (or defaulted) :attr:`backend`. Raises: + :exc:`~ConnectionError`: If the connection to the database fails. :exc:`~ConfigurationError`: If the given (or defaulted) :attr:`backend` is not supported or could not be loaded. """ @@ -77,6 +78,13 @@ class Connection: Args: query: the query to run + Raises: + :exc:`~DuplicateKeyError`: If the query fails because of a + duplicate key constraint. + :exc:`~OperationFailure`: If the query fails for any other + reason. + :exc:`~ConnectionError`: If the connection to the database + fails. """ raise NotImplementedError() diff --git a/bigchaindb/backend/exceptions.py b/bigchaindb/backend/exceptions.py index 41bac0c6..017e19e4 100644 --- a/bigchaindb/backend/exceptions.py +++ b/bigchaindb/backend/exceptions.py @@ -1,9 +1,17 @@ from bigchaindb.exceptions import BigchainDBError -class ConnectionError(BigchainDBError): - """Exception raised when the connection to the DataBase fails.""" +class BackendError(BigchainDBError): + """Top level exception for any backend exception.""" -class DatabaseOpFailedError(BigchainDBError): - """Exception for database operation errors.""" +class ConnectionError(BackendError): + """Exception raised when the connection to the backend fails.""" + + +class OperationError(BackendError): + """Exception raised when a backend operation fails.""" + + +class DuplicateKeyError(OperationError): + """Exception raised when an insert fails because the key is not unique""" diff --git a/bigchaindb/backend/mongodb/changefeed.py b/bigchaindb/backend/mongodb/changefeed.py index ecf36bcc..0cf37943 100644 --- a/bigchaindb/backend/mongodb/changefeed.py +++ b/bigchaindb/backend/mongodb/changefeed.py @@ -1,15 +1,15 @@ +import os import logging import time import pymongo -from pymongo import errors from bigchaindb import backend from bigchaindb.backend.changefeed import ChangeFeed from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.mongodb.connection import MongoDBConnection -from bigchaindb.backend.exceptions import (DatabaseOpFailedError, - ConnectionError) +from bigchaindb.backend.exceptions import BackendError + logger = logging.getLogger(__name__) register_changefeed = module_dispatch_registrar(backend.changefeed) @@ -30,11 +30,8 @@ class MongoDBChangeFeed(ChangeFeed): try: self.run_changefeed() break - except (errors.ConnectionFailure, errors.OperationFailure, - errors.AutoReconnect, - errors.ServerSelectionTimeoutError, - DatabaseOpFailedError, ConnectionError) as exc: - logger.exception(exc) + except BackendError: + logger.exception('Error connecting to the database, retrying') time.sleep(1) def run_changefeed(self): @@ -43,9 +40,10 @@ class MongoDBChangeFeed(ChangeFeed): namespace = '{}.{}'.format(dbname, table) # last timestamp in the oplog. We only care for operations happening # in the future. - last_ts = self.connection.conn.local.oplog.rs.find()\ - .sort('$natural', pymongo.DESCENDING).limit(1)\ - .next()['ts'] + last_ts = self.connection.run( + self.connection.query().local.oplog.rs.find() + .sort('$natural', pymongo.DESCENDING).limit(1) + .next()['ts']) # tailable cursor. A tailable cursor will remain open even after the # last result was returned. ``TAILABLE_AWAIT`` will block for some # timeout after the last result was returned. If no result is received @@ -56,6 +54,7 @@ class MongoDBChangeFeed(ChangeFeed): ) while cursor.alive: + print(os.getpid(), 'alive') try: record = cursor.next() except StopIteration: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 79a085fb..5bd0d017 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -2,13 +2,14 @@ import time import logging from itertools import repeat -from pymongo import MongoClient -from pymongo import errors +import pymongo import bigchaindb from bigchaindb.utils import Lazy -from bigchaindb.common import exceptions -from bigchaindb.backend import exceptions as backend_exceptions +from bigchaindb.common.exceptions import ConfigurationError +from bigchaindb.backend.exceptions import (DuplicateKeyError, + OperationError, + ConnectionError) from bigchaindb.backend.connection import Connection logger = logging.getLogger(__name__) @@ -59,18 +60,33 @@ class MongoDBConnection(Connection): def db(self): return self.conn[self.dbname] + def query(self): + return Lazy() + + def collection(self, name): + """Return a lazy object that can be used to compose a query. + + Args: + name (str): the name of the collection to query. + """ + return self.query()[self.dbname][name] + def run(self, query): - attempt = 0 - for i in self.max_tries_counter: - attempt += 1 - try: - return query.run(self.conn[self.dbname]) - except errors.AutoReconnect: - if attempt == self.max_tries: - raise - self._connect() + try: + return query.run(self.conn) + except pymongo.errors.DuplicateKeyError as exc: + raise DuplicateKeyError from exc + except pymongo.errors.OperationFailure as exc: + raise OperationError from exc def _connect(self): + """Try to connect to the database. + + Raises: + :exc:`~ConnectionError`: If the connection to the database + fails. + """ + attempt = 0 for i in self.max_tries_counter: attempt += 1 @@ -83,30 +99,24 @@ class MongoDBConnection(Connection): # FYI: this might raise a `ServerSelectionTimeoutError`, # that is a subclass of `ConnectionFailure`. - self.connection = MongoClient(self.host, - self.port, - replicaset=self.replicaset, - serverselectiontimeoutms=self.connection_timeout) - except (errors.ConnectionFailure, errors.AutoReconnect) as exc: + self.connection = pymongo.MongoClient(self.host, + self.port, + replicaset=self.replicaset, + serverselectiontimeoutms=self.connection_timeout) + + # `initialize_replica_set` might raise `ConnectionFailure` or `OperationFailure`. + except (pymongo.errors.ConnectionFailure, + pymongo.errors.OperationFailure) as exc: logger.warning('Attempt %s/%s. Connection to %s:%s failed after %sms.', attempt, self.max_tries if self.max_tries != 0 else '∞', self.host, self.port, self.connection_timeout) if attempt == self.max_tries: logger.critical('Cannot connect to the Database. Giving up.') - raise backend_exceptions.ConnectionError() from exc + raise ConnectionError() from exc else: break -def collection(name): - """Return a lazy object that can be used to compose a query. - - Args: - name (str): the name of the collection to query. - """ - return Lazy()[name] - - def initialize_replica_set(host, port, connection_timeout): """Initialize a replica set. If already initialized skip.""" @@ -114,7 +124,7 @@ def initialize_replica_set(host, port, connection_timeout): # The reason we do this instead of `backend.connect` is that # `backend.connect` will connect you to a replica set but this fails if # you try to connect to a replica set that is not yet initialized - conn = MongoClient(host=host, + conn = pymongo.MongoClient(host=host, port=port, serverselectiontimeoutms=connection_timeout) _check_replica_set(conn) @@ -125,7 +135,7 @@ def initialize_replica_set(host, port, connection_timeout): try: conn.admin.command('replSetInitiate', config) - except errors.OperationFailure as exc_info: + except pymongo.errors.OperationFailure as exc_info: if exc_info.details['codeName'] == 'AlreadyInitialized': return raise @@ -153,12 +163,12 @@ def _check_replica_set(conn): repl_opts = options['parsed']['replication'] repl_set_name = repl_opts.get('replSetName', None) or repl_opts['replSet'] except KeyError: - raise exceptions.ConfigurationError('mongod was not started with' + raise ConfigurationError('mongod was not started with' ' the replSet option.') bdb_repl_set_name = bigchaindb.config['database']['replicaset'] if repl_set_name != bdb_repl_set_name: - raise exceptions.ConfigurationError('The replicaset configuration of ' + raise ConfigurationError('The replicaset configuration of ' 'bigchaindb (`{}`) needs to match ' 'the replica set name from MongoDB' ' (`{}`)' diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index b4739dc2..e68608d7 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -10,8 +10,9 @@ from pymongo import errors from bigchaindb import backend from bigchaindb.common.exceptions import CyclicBlockchainError from bigchaindb.common.transaction import Transaction +from bigchaindb.backend.exceptions import DuplicateKeyError from bigchaindb.backend.utils import module_dispatch_registrar -from bigchaindb.backend.mongodb.connection import MongoDBConnection, collection +from bigchaindb.backend.mongodb.connection import MongoDBConnection register_query = module_dispatch_registrar(backend.query) @@ -21,9 +22,9 @@ register_query = module_dispatch_registrar(backend.query) def write_transaction(conn, signed_transaction): try: return conn.run( - collection('backlog') + conn.collection('backlog') .insert_one(signed_transaction)) - except errors.DuplicateKeyError: + except DuplicateKeyError: return @@ -32,7 +33,7 @@ def update_transaction(conn, transaction_id, doc): # with mongodb we need to add update operators to the doc doc = {'$set': doc} return conn.run( - collection('backlog') + conn.collection('backlog') .find_one_and_update( {'id': transaction_id}, doc, @@ -42,14 +43,14 @@ def update_transaction(conn, transaction_id, doc): @register_query(MongoDBConnection) def delete_transaction(conn, *transaction_id): return conn.run( - collection('backlog') + conn.collection('backlog') .delete_many({'id': {'$in': transaction_id}})) @register_query(MongoDBConnection) def get_stale_transactions(conn, reassign_delay): return conn.run( - collection('backlog') + conn.collection('backlog') .find({'assignment_timestamp': {'$lt': time() - reassign_delay}}, projection={'_id': False})) @@ -58,7 +59,7 @@ def get_stale_transactions(conn, reassign_delay): def get_transaction_from_block(conn, transaction_id, block_id): try: return conn.run( - collection('bigchain') + conn.collection('bigchain') .aggregate([ {'$match': {'id': block_id}}, {'$project': { @@ -84,7 +85,7 @@ def get_transaction_from_block(conn, transaction_id, block_id): @register_query(MongoDBConnection) def get_transaction_from_backlog(conn, transaction_id): return conn.run( - collection('backlog') + conn.collection('backlog') .find_one({'id': transaction_id}, projection={'_id': False, 'assignee': False, @@ -94,7 +95,7 @@ def get_transaction_from_backlog(conn, transaction_id): @register_query(MongoDBConnection) def get_blocks_status_from_transaction(conn, transaction_id): return conn.run( - collection('bigchain') + conn.collection('bigchain') .find({'block.transactions.id': transaction_id}, projection=['id', 'block.voters'])) @@ -139,7 +140,7 @@ def get_txids_filtered(conn, asset_id, operation=None): @register_query(MongoDBConnection) def get_asset_by_id(conn, asset_id): cursor = conn.run( - collection('bigchain') + conn.collection('bigchain') .aggregate([ {'$match': { 'block.transactions.id': asset_id, @@ -160,7 +161,7 @@ def get_asset_by_id(conn, asset_id): @register_query(MongoDBConnection) def get_spent(conn, transaction_id, output): cursor = conn.run( - collection('bigchain').aggregate([ + conn.collection('bigchain').aggregate([ {'$unwind': '$block.transactions'}, {'$match': { 'block.transactions.inputs.fulfills.txid': transaction_id, @@ -175,7 +176,7 @@ def get_spent(conn, transaction_id, output): @register_query(MongoDBConnection) def get_owned_ids(conn, owner): cursor = conn.run( - collection('bigchain') + conn.collection('bigchain') .aggregate([ {'$unwind': '$block.transactions'}, {'$match': { @@ -192,7 +193,7 @@ def get_owned_ids(conn, owner): @register_query(MongoDBConnection) def get_votes_by_block_id(conn, block_id): return conn.run( - collection('votes') + conn.collection('votes') .find({'vote.voting_for_block': block_id}, projection={'_id': False})) @@ -200,7 +201,7 @@ def get_votes_by_block_id(conn, block_id): @register_query(MongoDBConnection) def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): return conn.run( - collection('votes') + conn.collection('votes') .find({'vote.voting_for_block': block_id, 'node_pubkey': node_pubkey}, projection={'_id': False})) @@ -209,14 +210,14 @@ def get_votes_by_block_id_and_voter(conn, block_id, node_pubkey): @register_query(MongoDBConnection) def write_block(conn, block): return conn.run( - collection('bigchain') + conn.collection('bigchain') .insert_one(block.to_dict())) @register_query(MongoDBConnection) def get_block(conn, block_id): return conn.run( - collection('bigchain') + conn.collection('bigchain') .find_one({'id': block_id}, projection={'_id': False})) @@ -224,27 +225,27 @@ def get_block(conn, block_id): @register_query(MongoDBConnection) def has_transaction(conn, transaction_id): return bool(conn.run( - collection('bigchain') + conn.collection('bigchain') .find_one({'block.transactions.id': transaction_id}))) @register_query(MongoDBConnection) def count_blocks(conn): return conn.run( - collection('bigchain') + conn.collection('bigchain') .count()) @register_query(MongoDBConnection) def count_backlog(conn): return conn.run( - collection('backlog') + conn.collection('backlog') .count()) @register_query(MongoDBConnection) def write_vote(conn, vote): - conn.run(collection('votes').insert_one(vote)) + conn.run(conn.collection('votes').insert_one(vote)) vote.pop('_id') return vote @@ -252,7 +253,7 @@ def write_vote(conn, vote): @register_query(MongoDBConnection) def get_genesis_block(conn): return conn.run( - collection('bigchain') + conn.collection('bigchain') .find_one( {'block.transactions.0.operation': 'GENESIS'}, {'_id': False} @@ -262,7 +263,7 @@ def get_genesis_block(conn): @register_query(MongoDBConnection) def get_last_voted_block(conn, node_pubkey): last_voted = conn.run( - collection('votes') + conn.collection('votes') .find({'node_pubkey': node_pubkey}, sort=[('vote.timestamp', -1)])) diff --git a/bigchaindb/backend/rethinkdb/admin.py b/bigchaindb/backend/rethinkdb/admin.py index 63548f87..23b55048 100644 --- a/bigchaindb/backend/rethinkdb/admin.py +++ b/bigchaindb/backend/rethinkdb/admin.py @@ -5,7 +5,7 @@ import rethinkdb as r from bigchaindb.backend import admin from bigchaindb.backend.schema import TABLES -from bigchaindb.backend.exceptions import DatabaseOpFailedError +from bigchaindb.backend.exceptions import OperationError from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection @@ -78,7 +78,7 @@ def reconfigure(connection, *, table, shards, replicas, `_. Raises: - DatabaseOpFailedError: If the reconfiguration fails due to a + OperationError: If the reconfiguration fails due to a RethinkDB :exc:`ReqlOpFailedError` or :exc:`ReqlQueryLogicError`. @@ -96,7 +96,7 @@ def reconfigure(connection, *, table, shards, replicas, try: return connection.run(r.table(table).reconfigure(**params)) except (r.ReqlOpFailedError, r.ReqlQueryLogicError) as e: - raise DatabaseOpFailedError from e + raise OperationError from e @register_admin(RethinkDBConnection) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 4e18543d..89b84dba 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -23,7 +23,7 @@ from bigchaindb.utils import ProcessGroup from bigchaindb import backend from bigchaindb.backend import schema from bigchaindb.backend.admin import set_replicas, set_shards -from bigchaindb.backend.exceptions import (DatabaseOpFailedError, +from bigchaindb.backend.exceptions import (OperationError, ConnectionError) from bigchaindb.commands import utils from bigchaindb import processes @@ -252,7 +252,7 @@ def run_set_shards(args): conn = backend.connect() try: set_shards(conn, shards=args.num_shards) - except DatabaseOpFailedError as e: + except OperationError as e: logger.warn(e) @@ -260,7 +260,7 @@ def run_set_replicas(args): conn = backend.connect() try: set_replicas(conn, replicas=args.num_replicas) - except DatabaseOpFailedError as e: + except OperationError as e: logger.warn(e) diff --git a/tests/backend/mongodb/test_changefeed.py b/tests/backend/mongodb/test_changefeed.py index e7581b34..de07613c 100644 --- a/tests/backend/mongodb/test_changefeed.py +++ b/tests/backend/mongodb/test_changefeed.py @@ -1,7 +1,6 @@ from unittest import mock import pytest -from pymongo.errors import ConnectionFailure from multipipes import Pipe @@ -152,14 +151,15 @@ def test_changefeed_prefeed(mock_cursor_next, mock_cursor_alive, @pytest.mark.bdb @mock.patch('pymongo.cursor.Cursor.alive', new_callable=mock.PropertyMock) -@mock.patch('bigchaindb.backend.mongodb.changefeed.MongoDBChangeFeed.run_changefeed') # noqa +@mock.patch('bigchaindb.backend.mongodb.connection.MongoDBConnection.run') # noqa def test_connection_failure(mock_run_changefeed, mock_cursor_alive): from bigchaindb.backend import get_changefeed, connect + from bigchaindb.backend.exceptions import ConnectionError from bigchaindb.backend.changefeed import ChangeFeed conn = connect() mock_cursor_alive.return_value = False - mock_run_changefeed.side_effect = [ConnectionFailure(), mock.DEFAULT] + mock_run_changefeed.side_effect = [ConnectionError(), mock.DEFAULT] changefeed = get_changefeed(conn, 'backlog', ChangeFeed.INSERT) changefeed.run_forever() diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index 60181f25..feda77ed 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -58,7 +58,7 @@ def test_connection_error(mock_sleep, mock_client, mock_init_repl_set): from bigchaindb.backend import connect from bigchaindb.backend.exceptions import ConnectionError - # force the driver to trow ConnectionFailure + # force the driver to throw ConnectionFailure # the mock on time.sleep is to prevent the actual sleep when running # the tests mock_client.side_effect = ConnectionFailure() diff --git a/tests/backend/rethinkdb/test_admin.py b/tests/backend/rethinkdb/test_admin.py index 8c4f0528..0f71ca21 100644 --- a/tests/backend/rethinkdb/test_admin.py +++ b/tests/backend/rethinkdb/test_admin.py @@ -177,8 +177,8 @@ def test_reconfigure_replicas_without_nonvoting_replica_tags(rdb_conn, db_name, db_conn): from bigchaindb.backend.rethinkdb.admin import reconfigure - from bigchaindb.backend.exceptions import DatabaseOpFailedError - with pytest.raises(DatabaseOpFailedError) as exc: + from bigchaindb.backend.exceptions import OperationError + with pytest.raises(OperationError) as exc: reconfigure(db_conn, table='backlog', shards=1, replicas={'default': 1}, primary_replica_tag='default') assert isinstance(exc.value.__cause__, r.ReqlQueryLogicError) @@ -187,8 +187,8 @@ def test_reconfigure_replicas_without_nonvoting_replica_tags(rdb_conn, @pytest.mark.bdb def test_reconfigure_too_many_replicas(rdb_conn, db_name, db_conn): from bigchaindb.backend.rethinkdb.admin import reconfigure - from bigchaindb.backend.exceptions import DatabaseOpFailedError + from bigchaindb.backend.exceptions import OperationError replicas = _count_rethinkdb_servers() + 1 - with pytest.raises(DatabaseOpFailedError) as exc: + with pytest.raises(OperationError) as exc: reconfigure(db_conn, table='backlog', shards=1, replicas=replicas) assert isinstance(exc.value.__cause__, r.ReqlOpFailedError) From e3a6d3d3433e99441b4fd3f5eadf27fa2e7ae2ab Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 2 Feb 2017 19:58:00 +0100 Subject: [PATCH 155/219] Add tests for connection.run --- bigchaindb/backend/mongodb/connection.py | 2 ++ tests/backend/mongodb/test_connection.py | 32 +++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index ce0c6113..4bd822b4 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -76,6 +76,8 @@ class MongoDBConnection(Connection): return query.run(self.conn) except pymongo.errors.DuplicateKeyError as exc: raise DuplicateKeyError from exc + except pymongo.errors.AutoReconnect as exc: + raise ConnectionError from exc except pymongo.errors.OperationFailure as exc: raise OperationError from exc diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index feda77ed..be660d0a 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -1,9 +1,9 @@ from unittest import mock import pytest +import pymongo from pymongo import MongoClient from pymongo.database import Database -from pymongo.errors import ConnectionFailure, OperationFailure pytestmark = pytest.mark.bdb @@ -61,7 +61,7 @@ def test_connection_error(mock_sleep, mock_client, mock_init_repl_set): # force the driver to throw ConnectionFailure # the mock on time.sleep is to prevent the actual sleep when running # the tests - mock_client.side_effect = ConnectionFailure() + mock_client.side_effect = pymongo.errors.ConnectionFailure() with pytest.raises(ConnectionError): conn = connect() @@ -70,6 +70,30 @@ def test_connection_error(mock_sleep, mock_client, mock_init_repl_set): assert mock_client.call_count == 3 +@mock.patch('bigchaindb.backend.mongodb.connection.initialize_replica_set') +@mock.patch('pymongo.MongoClient') +def test_connection_run_errors(mock_client, mock_init_repl_set): + from bigchaindb.backend import connect + from bigchaindb.backend.exceptions import (DuplicateKeyError, + OperationError, + ConnectionError) + + conn = connect() + query = mock.Mock() + + query.run.side_effect = pymongo.errors.AutoReconnect('foo') + with pytest.raises(ConnectionError): + conn.run(query) + + query.run.side_effect = pymongo.errors.DuplicateKeyError('foo') + with pytest.raises(DuplicateKeyError): + conn.run(query) + + query.run.side_effect = pymongo.errors.OperationFailure('foo') + with pytest.raises(OperationError): + conn.run(query) + + def test_check_replica_set_not_enabled(mongodb_connection): from bigchaindb.backend.mongodb.connection import _check_replica_set from bigchaindb.common.exceptions import ConfigurationError @@ -145,8 +169,8 @@ def test_initialize_replica_set(mock_cmd_line_opts): with mock.patch.object(Database, 'command') as mock_command: mock_command.side_effect = [ mock_cmd_line_opts, - OperationFailure(None, details={'codeName': ''}) + pymongo.errors.OperationFailure(None, details={'codeName': ''}) ] - with pytest.raises(OperationFailure): + with pytest.raises(pymongo.errors.OperationFailure): initialize_replica_set('host', 1337, 1000) From 6d3c04169c3160daedef9421462052f4e2a9792a Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 2 Feb 2017 20:11:31 +0100 Subject: [PATCH 156/219] Remove except for connection error in commands It's out of scoper for this PR --- bigchaindb/commands/bigchain.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 1743b851..c8a5ba72 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -24,8 +24,7 @@ from bigchaindb import backend from bigchaindb.backend import schema from bigchaindb.backend.admin import (set_replicas, set_shards, add_replicas, remove_replicas) -from bigchaindb.backend.exceptions import (OperationError, - ConnectionError) +from bigchaindb.backend.exceptions import OperationError from bigchaindb.commands import utils from bigchaindb import processes @@ -164,8 +163,6 @@ def run_init(args): except DatabaseAlreadyExists: print('The database already exists.', file=sys.stderr) print('If you wish to re-initialize it, first drop it.', file=sys.stderr) - except ConnectionError: - print('Cannot connect to the database.', file=sys.stderr) def run_drop(args): @@ -210,8 +207,6 @@ def run_start(args): _run_init() except DatabaseAlreadyExists: pass - except ConnectionError: - print('Cannot connect to the database.', file=sys.stderr) except KeypairNotFoundException: sys.exit("Can't start BigchainDB, no keypair found. " 'Did you run `bigchaindb configure`?') From 98135c7df9e43367b19da711c84977e5e15c3cc5 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 2 Feb 2017 20:59:17 +0100 Subject: [PATCH 157/219] Add hack to handle reconnection to changefeed --- bigchaindb/backend/mongodb/changefeed.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bigchaindb/backend/mongodb/changefeed.py b/bigchaindb/backend/mongodb/changefeed.py index 0cf37943..8bdaca92 100644 --- a/bigchaindb/backend/mongodb/changefeed.py +++ b/bigchaindb/backend/mongodb/changefeed.py @@ -1,4 +1,3 @@ -import os import logging import time @@ -28,9 +27,13 @@ class MongoDBChangeFeed(ChangeFeed): while True: try: + # XXX: hack to force reconnection, + # the correct way to fix this is to manage errors + # for cursors.conn + self.connection.connection = None self.run_changefeed() break - except BackendError: + except (BackendError, pymongo.errors.ConnectionFailure): logger.exception('Error connecting to the database, retrying') time.sleep(1) @@ -48,13 +51,13 @@ class MongoDBChangeFeed(ChangeFeed): # last result was returned. ``TAILABLE_AWAIT`` will block for some # timeout after the last result was returned. If no result is received # in the meantime it will raise a StopIteration excetiption. - cursor = self.connection.conn.local.oplog.rs.find( - {'ns': namespace, 'ts': {'$gt': last_ts}}, - cursor_type=pymongo.CursorType.TAILABLE_AWAIT - ) + cursor = self.connection.run( + self.connection.query().local.oplog.rs.find( + {'ns': namespace, 'ts': {'$gt': last_ts}}, + cursor_type=pymongo.CursorType.TAILABLE_AWAIT + )) while cursor.alive: - print(os.getpid(), 'alive') try: record = cursor.next() except StopIteration: From e98a16180515390151e4c02227f1bb47d8bf1b04 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 3 Feb 2017 09:57:10 +0100 Subject: [PATCH 158/219] set -euo pipefail in make_confiles.sh --- deploy-cluster-aws/make_confiles.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deploy-cluster-aws/make_confiles.sh b/deploy-cluster-aws/make_confiles.sh index 72735cb3..35f1f0e6 100755 --- a/deploy-cluster-aws/make_confiles.sh +++ b/deploy-cluster-aws/make_confiles.sh @@ -1,8 +1,6 @@ #! /bin/bash -# The set -e option instructs bash to immediately exit -# if any command has a non-zero exit status -set -e +set -euo pipefail function printErr() { From 5750027cd4e25ccfc439269443a7ab2a9f90f2f1 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 3 Feb 2017 10:01:36 +0100 Subject: [PATCH 159/219] Include db backend (rethinkdb) when call bigchaindb configure --- deploy-cluster-aws/fabfile.py | 2 +- deploy-cluster-aws/make_confiles.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy-cluster-aws/fabfile.py b/deploy-cluster-aws/fabfile.py index 77d0e558..9ef24edd 100644 --- a/deploy-cluster-aws/fabfile.py +++ b/deploy-cluster-aws/fabfile.py @@ -221,7 +221,7 @@ def install_bigchaindb_from_git_archive(): @task @parallel def configure_bigchaindb(): - run('bigchaindb -y configure', pty=False) + run('bigchaindb -y configure rethinkdb', pty=False) # Send the specified configuration file to diff --git a/deploy-cluster-aws/make_confiles.sh b/deploy-cluster-aws/make_confiles.sh index 35f1f0e6..052ecaf0 100755 --- a/deploy-cluster-aws/make_confiles.sh +++ b/deploy-cluster-aws/make_confiles.sh @@ -34,5 +34,5 @@ mkdir $CONFDIR for (( i=0; i<$NUMFILES; i++ )); do CONPATH=$CONFDIR"/bcdb_conf"$i echo "Writing "$CONPATH - bigchaindb -y -c $CONPATH configure + bigchaindb -y -c $CONPATH configure rethinkdb done From b9db0de6574fb34a5475abe6883514fa7aef34f8 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 3 Feb 2017 10:12:42 +0100 Subject: [PATCH 160/219] Minor changes in .gitattributes --- .gitattributes | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index c82148e4..cd945c78 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,7 @@ benchmarking-tests export-ignore deploy-cluster-aws export-ignore -docs export-ignore export-ignore +docs export-ignore +ntools export-ignore speed-tests export-ignore tests export-ignore .gitattributes export-ignore From 2fe9c47b630e44f38748366d09bd4e6185ce0bd7 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 3 Feb 2017 10:28:28 +0100 Subject: [PATCH 161/219] Feat/990/cleanup monitoring code (#1138) * remove statsd dependencie * remove monitoring related code * removed statsd configuration * fixed tests * Removed monitoring docker compose file. Remove statsd settings from test fixture. Removed statsd related code from benchmarking tests * removed monitoring related documentation * remove unused import --- benchmarking-tests/benchmark_utils.py | 18 ---- benchmarking-tests/fabfile.py | 8 -- benchmarking-tests/test1/README.md | 3 +- bigchaindb/README.md | 4 - bigchaindb/__init__.py | 5 -- bigchaindb/commands/bigchain.py | 6 -- bigchaindb/monitor.py | 32 ------- bigchaindb/web/server.py | 3 - bigchaindb/web/views/transactions.py | 6 +- deploy-cluster-aws/fabfile-monitor.py | 89 ------------------- docker-compose-monitor.yml | 28 ------ .../source/appendices/firewall-notes.md | 10 --- .../clusters-feds/aws-testing-cluster.md | 44 --------- docs/server/source/clusters-feds/index.rst | 3 +- .../server/source/clusters-feds/monitoring.md | 40 --------- .../source/server-reference/configuration.md | 20 ----- ntools/one-m/aws/security_group.tf | 8 -- setup.py | 1 - tests/commands/conftest.py | 1 - tests/test_config_utils.py | 5 -- tests/test_monitor.py | 14 --- 21 files changed, 3 insertions(+), 345 deletions(-) delete mode 100644 bigchaindb/monitor.py delete mode 100644 deploy-cluster-aws/fabfile-monitor.py delete mode 100644 docker-compose-monitor.yml delete mode 100644 docs/server/source/clusters-feds/monitoring.md delete mode 100644 tests/test_monitor.py diff --git a/benchmarking-tests/benchmark_utils.py b/benchmarking-tests/benchmark_utils.py index 807146e8..510eae41 100644 --- a/benchmarking-tests/benchmark_utils.py +++ b/benchmarking-tests/benchmark_utils.py @@ -1,13 +1,11 @@ import multiprocessing as mp import uuid -import json import argparse import csv import time import logging import rethinkdb as r -from os.path import expanduser from bigchaindb.common.transaction import Transaction from bigchaindb import Bigchain @@ -48,15 +46,6 @@ def run_add_backlog(args): workers.start() -def run_set_statsd_host(args): - with open(expanduser('~') + '/.bigchaindb', 'r') as f: - conf = json.load(f) - - conf['statsd']['host'] = args.statsd_host - with open(expanduser('~') + '/.bigchaindb', 'w') as f: - json.dump(conf, f) - - def run_gather_metrics(args): # setup a rethinkdb connection conn = r.connect(args.bigchaindb_host, 28015, 'bigchain') @@ -126,12 +115,6 @@ def main(): default='minimal', help='Payload size') - # set statsd host - statsd_parser = subparsers.add_parser('set-statsd-host', - help='Set statsd host') - statsd_parser.add_argument('statsd_host', metavar='statsd_host', default='localhost', - help='Hostname of the statsd server') - # metrics metrics_parser = subparsers.add_parser('gather-metrics', help='Gather metrics to a csv file') @@ -149,4 +132,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py index 44a31888..0dd4e964 100644 --- a/benchmarking-tests/fabfile.py +++ b/benchmarking-tests/fabfile.py @@ -28,14 +28,6 @@ def put_benchmark_utils(): put('benchmark_utils.py') -@task -@parallel -def set_statsd_host(statsd_host='localhost'): - run('python3 benchmark_utils.py set-statsd-host {}'.format(statsd_host)) - print('update configuration') - run('bigchaindb show-config') - - @task @parallel def prepare_backlog(num_transactions=10000): diff --git a/benchmarking-tests/test1/README.md b/benchmarking-tests/test1/README.md index aadccfdc..38a4569b 100644 --- a/benchmarking-tests/test1/README.md +++ b/benchmarking-tests/test1/README.md @@ -15,7 +15,6 @@ Then: ```bash fab put_benchmark_utils -fab set_statsd_host: fab prepare_backlog: # wait for process to finish fab start_bigchaindb -``` \ No newline at end of file +``` diff --git a/bigchaindb/README.md b/bigchaindb/README.md index 3bad7331..dbb59a1e 100644 --- a/bigchaindb/README.md +++ b/bigchaindb/README.md @@ -26,10 +26,6 @@ Entry point for the BigchainDB process, after initialization. All subprocesses Methods for managing the configuration, including loading configuration files, automatically generating the configuration, and keeping the configuration consistent across BigchainDB instances. -### [`monitor.py`](./monitor.py) - -Code for monitoring speed of various processes in BigchainDB via `statsd` and Grafana. [See documentation.](https://docs.bigchaindb.com/projects/server/en/latest/clusters-feds/monitoring.html) - ## Folders ### [`pipelines`](./pipelines) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 072c7b6b..10e9e6ce 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -41,11 +41,6 @@ config = { 'private': None, }, 'keyring': [], - 'statsd': { - 'host': 'localhost', - 'port': 8125, - 'rate': 0.01, - }, 'backlog_reassign_delay': 120 } diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 4e8de28b..70836f3c 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -105,12 +105,6 @@ def run_configure(args, skip_if_exists=False): input_on_stderr('Database {}? (default `{}`): '.format(key, val)) \ or val - for key in ('host', 'port', 'rate'): - val = conf['statsd'][key] - conf['statsd'][key] = \ - input_on_stderr('Statsd {}? (default `{}`): '.format(key, val)) \ - or val - val = conf['backlog_reassign_delay'] conf['backlog_reassign_delay'] = \ input_on_stderr(('Stale transaction reassignment delay (in ' diff --git a/bigchaindb/monitor.py b/bigchaindb/monitor.py deleted file mode 100644 index 9d2bc527..00000000 --- a/bigchaindb/monitor.py +++ /dev/null @@ -1,32 +0,0 @@ -from platform import node - -import statsd - -import bigchaindb -from bigchaindb import config_utils - - -class Monitor(statsd.StatsClient): - """Set up statsd monitoring.""" - - def __init__(self, *args, **kwargs): - """Overrides statsd client, fixing prefix to messages and loading configuration - - Args: - *args: arguments (identical to Statsclient) - **kwargs: keyword arguments (identical to Statsclient) - """ - - config_utils.autoconfigure() - - if not kwargs: - kwargs = {} - - # set prefix, parameters from configuration file - if 'prefix' not in kwargs: - kwargs['prefix'] = '{hostname}.'.format(hostname=node()) - if 'host' not in kwargs: - kwargs['host'] = bigchaindb.config['statsd']['host'] - if 'port' not in kwargs: - kwargs['port'] = bigchaindb.config['statsd']['port'] - super().__init__(*args, **kwargs) diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index d8ce2e0b..bcd44d11 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -13,8 +13,6 @@ from bigchaindb import utils from bigchaindb import Bigchain from bigchaindb.web.routes import add_routes -from bigchaindb.monitor import Monitor - # TODO: Figure out if we do we need all this boilerplate. class StandaloneApplication(gunicorn.app.base.BaseApplication): @@ -65,7 +63,6 @@ def create_app(*, debug=False, threads=4): app.debug = debug app.config['bigchain_pool'] = utils.pool(Bigchain, size=threads) - app.config['monitor'] = Monitor() add_routes(app) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index a4a983ef..7acaa279 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -23,7 +23,6 @@ from bigchaindb.common.exceptions import ( ValidationError, ) -import bigchaindb from bigchaindb.models import Transaction from bigchaindb.web.views.base import make_error from bigchaindb.web.views import parameters @@ -72,7 +71,6 @@ class TransactionListApi(Resource): A ``dict`` containing the data about the transaction. """ pool = current_app.config['bigchain_pool'] - monitor = current_app.config['monitor'] # `force` will try to format the body of the POST request even if the # `content-type` header is not set to `application/json` @@ -109,8 +107,6 @@ class TransactionListApi(Resource): 'Invalid transaction ({}): {}'.format(type(e).__name__, e) ) else: - rate = bigchaindb.config['statsd']['rate'] - with monitor.timer('write_transaction', rate=rate): - bigchain.write_transaction(tx_obj) + bigchain.write_transaction(tx_obj) return tx, 202 diff --git a/deploy-cluster-aws/fabfile-monitor.py b/deploy-cluster-aws/fabfile-monitor.py deleted file mode 100644 index 8d2d282c..00000000 --- a/deploy-cluster-aws/fabfile-monitor.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -"""A Fabric fabfile with functionality to install Docker, -install Docker Compose, and run a BigchainDB monitoring server -(using the docker-compose-monitor.yml file) -""" - -from __future__ import with_statement, unicode_literals - -from fabric.api import sudo, env -from fabric.api import task -from fabric.operations import put, run - -from ssh_key import ssh_key_path - -# Ignore known_hosts -# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts -env.disable_known_hosts = True - -env.user = 'ubuntu' -env.key_filename = ssh_key_path - - -@task -def install_docker_engine(): - """Install Docker on an EC2 Ubuntu 14.04 instance - - Example: - fab --fabfile=fabfile-monitor.py \ - --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ - install_docker_engine - """ - - # install prerequisites - sudo('apt-get update') - sudo('apt-get -y install apt-transport-https ca-certificates linux-image-extra-$(uname -r) apparmor') - - # install docker repositories - sudo('apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 \ - --recv-keys 58118E89F3A912897C070ADBF76221572C52609D') - sudo("echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' | \ - sudo tee /etc/apt/sources.list.d/docker.list") - - # install docker engine - sudo('apt-get update') - sudo('apt-get -y install docker-engine') - - # add ubuntu user to the docker group - sudo('usermod -aG docker ubuntu') - - -@task -def install_docker_compose(): - """Install Docker Compose on an EC2 Ubuntu 14.04 instance - - Example: - fab --fabfile=fabfile-monitor.py \ - --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ - install_docker_compose - """ - sudo('curl -L https://github.com/docker/compose/releases/download/1.7.0/docker-compose-`uname \ - -s`-`uname -m` > /usr/local/bin/docker-compose') - sudo('chmod +x /usr/local/bin/docker-compose') - - -@task -def install_docker(): - """Install Docker and Docker Compose on an EC2 Ubuntu 14.04 instance - - Example: - fab --fabfile=fabfile-monitor.py \ - --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ - install_docker - """ - install_docker_engine() - install_docker_compose() - - -@task -def run_monitor(): - """Run BigchainDB monitor on an EC2 Ubuntu 14.04 instance - - Example: - fab --fabfile=fabfile-monitor.py \ - --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ - run_monitor - """ - # copy docker-compose-monitor to the ec2 instance - put('../docker-compose-monitor.yml') - run('INFLUXDB_DATA=/influxdb-data docker-compose -f docker-compose-monitor.yml up -d') diff --git a/docker-compose-monitor.yml b/docker-compose-monitor.yml deleted file mode 100644 index e695387c..00000000 --- a/docker-compose-monitor.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: '2' -services: - influxdb: - image: tutum/influxdb - ports: - - "8083:8083" - - "8086:8086" - - "8090" - - "8099" - environment: - PRE_CREATE_DB: "telegraf" - volumes: - - $INFLUXDB_DATA:/data - - grafana: - image: bigchaindb/grafana-bigchaindb-docker - tty: true - ports: - - "3000:3000" - environment: - INFLUXDB_HOST: "influxdb" - - statsd: - image: bigchaindb/docker-telegraf-statsd - ports: - - "8125:8125/udp" - environment: - INFLUXDB_HOST: "influxdb" \ No newline at end of file diff --git a/docs/server/source/appendices/firewall-notes.md b/docs/server/source/appendices/firewall-notes.md index bad09b05..19d7c234 100644 --- a/docs/server/source/appendices/firewall-notes.md +++ b/docs/server/source/appendices/firewall-notes.md @@ -44,11 +44,6 @@ Port 161 is the default SNMP port (usually UDP, sometimes TCP). SNMP is used, fo Port 443 is the default HTTPS port (TCP). You may need to open it up for outbound requests (and inbound responses) temporarily because some RethinkDB installation instructions use wget over HTTPS to get the RethinkDB GPG key. Package managers might also get some packages using HTTPS. -## Port 8125 - -If you set up a [cluster-monitoring server](../clusters-feds/monitoring.html), then StatsD will send UDP packets to Telegraf (on the monitoring server) via port 8125. - - ## Port 8080 Port 8080 is the default port used by RethinkDB for its adminstrative web (HTTP) interface (TCP). While you _can_, you shouldn't allow traffic arbitrary external sources. You can still use the RethinkDB web interface by binding it to localhost and then accessing it via a SOCKS proxy or reverse proxy; see "Binding the web interface port" on [the RethinkDB page about securing your cluster](https://rethinkdb.com/docs/security/). @@ -76,8 +71,3 @@ Port 29015 is the default port for RethinkDB intracluster connections (TCP). It ## Other Ports On Linux, you can use commands such as `netstat -tunlp` or `lsof -i` to get a sense of currently open/listening ports and connections, and the associated processes. - - -## Cluster-Monitoring Server - -If you set up a [cluster-monitoring server](../clusters-feds/monitoring.html) (running Telegraf, InfluxDB & Grafana), Telegraf will listen on port 8125 for UDP packets from StatsD, and the Grafana web dashboard will use port 3000. (Those are the default ports.) diff --git a/docs/server/source/clusters-feds/aws-testing-cluster.md b/docs/server/source/clusters-feds/aws-testing-cluster.md index fbc623f3..ac1deff1 100644 --- a/docs/server/source/clusters-feds/aws-testing-cluster.md +++ b/docs/server/source/clusters-feds/aws-testing-cluster.md @@ -64,50 +64,6 @@ For a super lax, somewhat risky, anything-can-enter security group, add these ru If you want to set up a more secure security group, see the [Notes for Firewall Setup](../appendices/firewall-notes.html). -## Deploy a BigchainDB Monitor - -This step is optional. - -One way to monitor a BigchainDB cluster is to use the monitoring setup described in the [Monitoring](monitoring.html) section of this documentation. If you want to do that, then you may want to deploy the monitoring server first, so you can tell your BigchainDB nodes where to send their monitoring data. - -You can deploy a monitoring server on AWS. To do that, go to the AWS EC2 Console and launch an instance: - -1. Choose an AMI: select Ubuntu Server 16.04 LTS. -2. Choose an Instance Type: a t2.micro will suffice. -3. Configure Instance Details: you can accept the defaults, but feel free to change them. -4. Add Storage: A "Root" volume type should already be included. You _could_ store monitoring data there (e.g. in a folder named `/influxdb-data`) but we will attach another volume and store the monitoring data there instead. Select "Add New Volume" and an EBS volume type. -5. Tag Instance: give your instance a memorable name. -6. Configure Security Group: choose your bigchaindb security group. -7. Review and launch your instance. - -When it asks, choose an existing key pair: the one you created earlier (named `bigchaindb`). - -Give your instance some time to launch and become able to accept SSH connections. You can see its current status in the AWS EC2 Console (in the "Instances" section). SSH into your instance using something like: -```text -cd deploy-cluster-aws -ssh -i pem/bigchaindb.pem ubuntu@ec2-52-58-157-229.eu-central-1.compute.amazonaws.com -``` - -where `ec2-52-58-157-229.eu-central-1.compute.amazonaws.com` should be replaced by your new instance's EC2 hostname. (To get that, go to the AWS EC2 Console, select Instances, click on your newly-launched instance, and copy its "Public DNS" name.) - -Next, create a file system on the attached volume, make a directory named `/influxdb-data`, and set the attached volume's mount point to be `/influxdb-data`. For detailed instructions on how to do that, see the AWS documentation for [Making an Amazon EBS Volume Available for Use](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-using-volumes.html). - -Then install Docker and Docker Compose: -```text -# in a Python 2.5-2.7 virtual environment where fabric, boto3, etc. are installed -fab --fabfile=fabfile-monitor.py --hosts= install_docker -``` - -After Docker is installed, we can run the monitor with: -```text -fab --fabfile=fabfile-monitor.py --hosts= run_monitor -``` - -For more information about monitoring (e.g. how to view the Grafana dashboard in your web browser), see the [Monitoring](monitoring.html) section of this documentation. - -To configure a BigchainDB node to send monitoring data to the monitoring server, change the statsd host in the configuration of the BigchainDB node. The section on [Configuring a BigchainDB Node](../server-reference/configuration.html) explains how you can do that. (For example, you can change the statsd host in `$HOME/.bigchaindb`.) - - ## Deploy a BigchainDB Cluster ### Step 1 diff --git a/docs/server/source/clusters-feds/index.rst b/docs/server/source/clusters-feds/index.rst index e55867fa..d13221ce 100644 --- a/docs/server/source/clusters-feds/index.rst +++ b/docs/server/source/clusters-feds/index.rst @@ -7,5 +7,4 @@ Clusters & Federations set-up-a-federation backup aws-testing-cluster - monitoring - \ No newline at end of file + diff --git a/docs/server/source/clusters-feds/monitoring.md b/docs/server/source/clusters-feds/monitoring.md deleted file mode 100644 index 4a5de698..00000000 --- a/docs/server/source/clusters-feds/monitoring.md +++ /dev/null @@ -1,40 +0,0 @@ -# Cluster Monitoring - -BigchainDB uses [StatsD](https://github.com/etsy/statsd) for cluster monitoring. We require some additional infrastructure to take full advantage of its functionality: - -* an agent to listen for metrics: [Telegraf](https://github.com/influxdata/telegraf), -* a time-series database: [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/), and -* a frontend to display analytics: [Grafana](http://grafana.org/). - -We put each of those inside its own Docker container. The whole system is illustrated below. - -![BigchainDB monitoring system diagram: Application metrics flow from servers running BigchainDB to Telegraf to InfluxDB to Grafana](../_static/monitoring_system_diagram.png) - -For ease of use, we've created a Docker [_Compose file_](https://docs.docker.com/compose/compose-file/) (named `docker-compose-monitor.yml`) to define the monitoring system setup. To use it, just go to to the top `bigchaindb` directory and run: -```text -$ docker-compose -f docker-compose-monitor.yml build -$ docker-compose -f docker-compose-monitor.yml up -``` - -It is also possible to mount a host directory as a data volume for InfluxDB -by setting the `INFLUXDB_DATA` environment variable: -```text -$ INFLUXDB_DATA=/data docker-compose -f docker-compose-monitor.yml up -``` - -You can view the Grafana dashboard in your web browser at: - -[http://localhost:3000/dashboard/script/bigchaindb_dashboard.js](http://localhost:3000/dashboard/script/bigchaindb_dashboard.js) - -(You may want to replace `localhost` with another hostname in that URL, e.g. the hostname of a remote monitoring server.) - -The login and password are `admin` by default. If BigchainDB is running and processing transactions, you should see analytics—if not, [start BigchainDB](../dev-and-test/setup-run-node.html#run-bigchaindb) and load some test transactions: -```text -$ bigchaindb load -``` - -then refresh the page after a few seconds. - -If you're not interested in monitoring, don't worry: BigchainDB will function just fine without any monitoring setup. - -Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking! diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 783591ca..b7842d6a 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -19,9 +19,6 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_SERVER_BIND`
`BIGCHAINDB_SERVER_WORKERS`
`BIGCHAINDB_SERVER_THREADS`
-`BIGCHAINDB_STATSD_HOST`
-`BIGCHAINDB_STATSD_PORT`
-`BIGCHAINDB_STATSD_RATE`
`BIGCHAINDB_CONFIG_PATH`
`BIGCHAINDB_BACKLOG_REASSIGN_DELAY`
@@ -151,23 +148,6 @@ export BIGCHAINDB_SERVER_THREADS=5 } ``` - -## statsd.host, statsd.port & statsd.rate - -These settings are used to configure where, and how often, [StatsD](https://github.com/etsy/statsd) should send data for [cluster monitoring](../clusters-feds/monitoring.html) purposes. `statsd.host` is the hostname of the monitoring server, where StatsD should send its data. `stats.port` is the port. `statsd.rate` is the fraction of transaction operations that should be sampled. It's a float between 0.0 and 1.0. - -**Example using environment variables** -```text -export BIGCHAINDB_STATSD_HOST="http://monitor.monitors-r-us.io" -export BIGCHAINDB_STATSD_PORT=8125 -export BIGCHAINDB_STATSD_RATE=0.01 -``` - -**Example config file snippet: the default** -```js -"statsd": {"host": "localhost", "port": 8125, "rate": 0.01} -``` - ## backlog_reassign_delay Specifies how long, in seconds, transactions can remain in the backlog before being reassigned. Long-waiting transactions must be reassigned because the assigned node may no longer be responsive. The default duration is 120 seconds. diff --git a/ntools/one-m/aws/security_group.tf b/ntools/one-m/aws/security_group.tf index f8fa3e1d..64037ff6 100644 --- a/ntools/one-m/aws/security_group.tf +++ b/ntools/one-m/aws/security_group.tf @@ -62,14 +62,6 @@ resource "aws_security_group" "node_sg1" { cidr_blocks = ["0.0.0.0/0"] } - # StatsD - ingress { - from_port = 8125 - to_port = 8125 - protocol = "udp" - cidr_blocks = ["0.0.0.0/0"] - } - # Future: Don't allow port 8080 for the RethinkDB web interface. # Use a SOCKS proxy or reverse proxy instead. diff --git a/setup.py b/setup.py index f7085218..d1a22279 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,6 @@ install_requires = [ 'pymongo~=3.4', 'pysha3==1.0.0', 'cryptoconditions>=0.5.0', - 'statsd>=3.2.1', 'python-rapidjson>=0.0.8', 'logstats>=0.2.1', 'flask>=0.10.1', diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index d3ecdadc..1cffbc2f 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -35,7 +35,6 @@ def mock_bigchaindb_backup_config(monkeypatch): config = { 'keypair': {}, 'database': {'host': 'host', 'port': 12345, 'name': 'adbname'}, - 'statsd': {'host': 'host', 'port': 12345, 'rate': 0.1}, 'backlog_reassign_delay': 5 } monkeypatch.setattr('bigchaindb._config', config) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index af78585e..d69b789a 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -167,11 +167,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'private': None, }, 'keyring': KEYRING.split(':'), - 'statsd': { - 'host': 'localhost', - 'port': 8125, - 'rate': 0.01, - }, 'backlog_reassign_delay': 5 } diff --git a/tests/test_monitor.py b/tests/test_monitor.py deleted file mode 100644 index a138b9b1..00000000 --- a/tests/test_monitor.py +++ /dev/null @@ -1,14 +0,0 @@ -from platform import node - - -def test_monitor_class_init_defaults(): - import bigchaindb - from bigchaindb.monitor import Monitor - monitor = Monitor() - assert monitor - assert len(monitor._addr) == 2 - # TODO get value from config - # assert monitor._addr[0] == bigchaindb.config['statsd']['host'] - assert monitor._addr[0] == '127.0.0.1' - assert monitor._addr[1] == bigchaindb.config['statsd']['port'] - assert monitor._prefix == node() + '.' From d3e394e7edf1facb05260b97833c2221b08f8491 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 3 Feb 2017 10:44:06 +0100 Subject: [PATCH 162/219] refactor get_txids_filtered query to be more efficient and add test to check that appropriate indexes are used --- bigchaindb/backend/mongodb/query.py | 54 +++++++++++---------------- tests/backend/mongodb/test_indexes.py | 23 ++++++++++++ 2 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 tests/backend/mongodb/test_indexes.py diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index d7ee6afc..2bbfa08f 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -1,7 +1,6 @@ """Query implementation for MongoDB""" from time import time -from itertools import chain from pymongo import ReturnDocument from pymongo import errors @@ -86,39 +85,30 @@ def get_blocks_status_from_transaction(conn, transaction_id): @register_query(MongoDBConnection) def get_txids_filtered(conn, asset_id, operation=None): - parts = [] + match_create = { + 'block.transactions.operation': 'CREATE', + 'block.transactions.id': asset_id + } + match_transfer = { + 'block.transactions.operation': 'TRANSFER', + 'block.transactions.asset.id': asset_id + } - if operation in (Transaction.CREATE, None): - # get the txid of the create transaction for asset_id - cursor = conn.db['bigchain'].aggregate([ - {'$match': { - 'block.transactions.id': asset_id, - 'block.transactions.operation': 'CREATE' - }}, - {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.id': asset_id, - 'block.transactions.operation': 'CREATE' - }}, - {'$project': {'block.transactions.id': True}} - ]) - parts.append(elem['block']['transactions']['id'] for elem in cursor) + if operation == Transaction.CREATE: + match = match_create + elif operation == Transaction.TRANSFER: + match = match_transfer + else: + match = {'$or': [match_create, match_transfer]} - if operation in (Transaction.TRANSFER, None): - # get txids of transfer transaction with asset_id - cursor = conn.db['bigchain'].aggregate([ - {'$match': { - 'block.transactions.asset.id': asset_id - }}, - {'$unwind': '$block.transactions'}, - {'$match': { - 'block.transactions.asset.id': asset_id - }}, - {'$project': {'block.transactions.id': True}} - ]) - parts.append(elem['block']['transactions']['id'] for elem in cursor) - - return chain(*parts) + pipeline = [ + {'$match': match}, + {'$unwind': '$block.transactions'}, + {'$match': match}, + {'$project': {'block.transactions.id': True}} + ] + cursor = conn.db['bigchain'].aggregate(pipeline) + return (elem['block']['transactions']['id'] for elem in cursor) @register_query(MongoDBConnection) diff --git a/tests/backend/mongodb/test_indexes.py b/tests/backend/mongodb/test_indexes.py new file mode 100644 index 00000000..ba6afae1 --- /dev/null +++ b/tests/backend/mongodb/test_indexes.py @@ -0,0 +1,23 @@ +import pytest +from unittest.mock import MagicMock + +pytestmark = pytest.mark.bdb + + +def test_asset_id_index(): + from bigchaindb.backend.mongodb.query import get_txids_filtered + from bigchaindb.backend import connect + + # Passes a mock in place of a connection to get the query params from the + # query function, then gets the explain plan from MongoDB to test that + # it's using certain indexes. + + m = MagicMock() + get_txids_filtered(m, '') + pipeline = m.db['bigchain'].aggregate.call_args[0][0] + run = connect().db.command + res = run('aggregate', 'bigchain', pipeline=pipeline, explain=True) + stages = (res['stages'][0]['$cursor']['queryPlanner']['winningPlan'] + ['inputStage']['inputStages']) + indexes = [s['inputStage']['indexName'] for s in stages] + assert set(indexes) == {'asset_id', 'transaction_id'} From e7d729a40d6082abe8e94defb293f8e52814100b Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 3 Feb 2017 10:56:51 +0100 Subject: [PATCH 163/219] Improved docstrings --- bigchaindb/models.py | 47 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 3c316355..ee7efe8f 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -193,6 +193,12 @@ class Block(object): bigchain (:class:`~bigchaindb.Bigchain`): An instantiated Bigchain object. + Note: + The hash of the block (`id`) is validated on the `self.from_dict` + method. This is because the `from_dict` is the only method in + which we have the original json payload. The `id` provided by + this class is a mutable property that is generated on the fly. + Returns: :class:`~.Block`: If valid, return a `Block` object. Else an appropriate exception describing the reason of invalidity is @@ -200,7 +206,17 @@ class Block(object): Raises: OperationError: If a non-federation node signed the Block. - InvalidSignature: If a Block's signature is invalid. + InvalidSignature: If a Block's signature is invalid or if the + block contains a transaction with an invalid signature. + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not + found + TransactionNotInValidBlock: if the input of the transaction is not + in a valid block + TransactionOwnerError: if the new transaction is using an input it + doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong """ self._validate_block(bigchain) @@ -209,6 +225,16 @@ class Block(object): return self def _validate_block(self, bigchain): + """Validate the Block without validating the transactions. + + Args: + bigchain (:class:`~bigchaindb.Bigchain`): An instantiated Bigchain + object. + + Raises: + OperationError: If a non-federation node signed the Block. + InvalidSignature: If a Block's signature is invalid. + """ # Check if the block was created by a federation node possible_voters = (bigchain.nodes_except_me + [bigchain.me]) if self.node_pubkey not in possible_voters: @@ -219,6 +245,23 @@ class Block(object): raise InvalidSignature('Invalid block signature') def _validate_block_transactions(self, bigchain): + """Validate Block transactions. + + Args: + bigchain (Bigchain): an instantiated bigchaindb.Bigchain object. + + Raises: + OperationError: if the transaction operation is not supported + TransactionDoesNotExist: if the input of the transaction is not + found + TransactionNotInValidBlock: if the input of the transaction is not + in a valid block + TransactionOwnerError: if the new transaction is using an input it + doesn't own + DoubleSpend: if the transaction is a double spend + InvalidHash: if the hash of the transaction is wrong + InvalidSignature: if the signature of the transaction is wrong + """ for tx in self.transactions: # If a transaction is not valid, `validate_transactions` will # throw an an exception and block validation will be canceled. @@ -270,8 +313,6 @@ class Block(object): Raises: InvalidHash: If the block's id is not corresponding to its data. - InvalidSignature: If the block's signature is not corresponding - to it's data or `node_pubkey`. """ # Validate block id block = block_body['block'] From 105d0172da5cdca8270e038f95b3b648e007f1d7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 3 Feb 2017 11:31:24 +0100 Subject: [PATCH 164/219] Minor fixes to the bigchaindb CLI docs --- .../source/server-reference/bigchaindb-cli.md | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index f647b4bf..5fdf8fdf 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -46,14 +46,19 @@ Write the node's public key (i.e. one of its configuration values) to standard o ## bigchaindb init -Create a RethinkDB database, all RethinkDB database tables, various RethinkDB database indexes, and the genesis block. +Create a backend database (RethinkDB or MongoDB), +all database tables/collections, +various backend database indexes, +and the genesis block. -Note: The `bigchaindb start` command (see below) always starts by trying a `bigchaindb init` first. If it sees that the RethinkDB database already exists, then it doesn't re-initialize the database. One doesn't have to do `bigchaindb init` before `bigchaindb start`. `bigchaindb init` is useful if you only want to initialize (but not start). +Note: The `bigchaindb start` command (see below) always starts by trying a `bigchaindb init` first. If it sees that the backend database already exists, then it doesn't re-initialize the database. One doesn't have to do `bigchaindb init` before `bigchaindb start`. `bigchaindb init` is useful if you only want to initialize (but not start). ## bigchaindb drop -Drop (erase) the RethinkDB database. You will be prompted to make sure. If you want to force-drop the database (i.e. skipping the yes/no prompt), then use `bigchaindb -y drop` +Drop (erase) the backend database (a RethinkDB or MongoDB database). +You will be prompted to make sure. +If you want to force-drop the database (i.e. skipping the yes/no prompt), then use `bigchaindb -y drop` ## bigchaindb start @@ -76,18 +81,31 @@ Note: This command uses the Python Server API to write transactions to the datab ## bigchaindb set-shards -Set the number of shards in the underlying datastore. For example, the following command will set the number of shards to four: +This command is specific to RethinkDB so it will only run if BigchainDB is +configured with `rethinkdb` as the backend. + +If RethinkDB is the backend database, then: ```text $ bigchaindb set-shards 4 ``` +will set the number of shards (in all RethinkDB tables) to 4. + + ## bigchaindb set-replicas -Set the number of replicas (of each shard) in the underlying datastore. For example, the following command will set the number of replicas to three (i.e. it will set the replication factor to three): +This command is specific to RethinkDB so it will only run if BigchainDB is +configured with `rethinkdb` as the backend. + +If RethinkDB is the backend database, then: ```text $ bigchaindb set-replicas 3 ``` +will set the number of replicas (of each shard) to 3 +(i.e. it will set the replication factor to 3). + + ## bigchaindb add-replicas This command is specific to MongoDB so it will only run if BigchainDB is From 7f50d76d9ec5ac3d3f8a2f3d50cea6b6314e07f0 Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 3 Feb 2017 11:58:18 +0100 Subject: [PATCH 165/219] Fix test on changefeed reconnection --- tests/backend/mongodb/test_changefeed.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/backend/mongodb/test_changefeed.py b/tests/backend/mongodb/test_changefeed.py index de07613c..67b54cd8 100644 --- a/tests/backend/mongodb/test_changefeed.py +++ b/tests/backend/mongodb/test_changefeed.py @@ -150,16 +150,15 @@ def test_changefeed_prefeed(mock_cursor_next, mock_cursor_alive, @pytest.mark.bdb -@mock.patch('pymongo.cursor.Cursor.alive', new_callable=mock.PropertyMock) -@mock.patch('bigchaindb.backend.mongodb.connection.MongoDBConnection.run') # noqa -def test_connection_failure(mock_run_changefeed, mock_cursor_alive): +@mock.patch('bigchaindb.backend.mongodb.changefeed.MongoDBChangeFeed.run_changefeed') # noqa +def test_connection_failure(mock_run_changefeed): from bigchaindb.backend import get_changefeed, connect from bigchaindb.backend.exceptions import ConnectionError from bigchaindb.backend.changefeed import ChangeFeed conn = connect() - mock_cursor_alive.return_value = False - mock_run_changefeed.side_effect = [ConnectionError(), mock.DEFAULT] + mock_run_changefeed.side_effect = [ConnectionError(), + mock.DEFAULT] changefeed = get_changefeed(conn, 'backlog', ChangeFeed.INSERT) changefeed.run_forever() From c174271ae67968e278e0d8e00611b3675bb5d4f7 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 3 Feb 2017 13:05:04 +0100 Subject: [PATCH 166/219] Make sure BlockPipeline.validate_tx handles all exceptions --- bigchaindb/pipelines/block.py | 9 +++++++- tests/pipelines/test_block_creation.py | 31 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index d5c81741..5f372384 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -13,6 +13,8 @@ import bigchaindb from bigchaindb import backend from bigchaindb.backend.changefeed import ChangeFeed from bigchaindb.models import Transaction +from bigchaindb.common.exceptions import (SchemaValidationError, InvalidHash, + InvalidSignature, AmountError) from bigchaindb import Bigchain @@ -59,7 +61,12 @@ class BlockPipeline: :class:`~bigchaindb.models.Transaction`: The transaction if valid, ``None`` otherwise. """ - tx = Transaction.from_dict(tx) + try: + tx = Transaction.from_dict(tx) + except (SchemaValidationError, InvalidHash, InvalidSignature, + AmountError): + return None + if self.bigchain.transaction_exists(tx.id): # if the transaction already exists, we must check whether # it's in a valid or undecided block diff --git a/tests/pipelines/test_block_creation.py b/tests/pipelines/test_block_creation.py index 18d291ea..2991f3cf 100644 --- a/tests/pipelines/test_block_creation.py +++ b/tests/pipelines/test_block_creation.py @@ -39,6 +39,37 @@ def test_validate_transaction(b, create_tx): assert block_maker.validate_tx(valid_tx.to_dict()) == valid_tx +def test_validate_transaction_handles_exceptions(b, signed_create_tx): + """ + This test makes sure that `BlockPipeline.validate_tx` handles possible + exceptions from `Transaction.from_dict`. + """ + from bigchaindb.pipelines.block import BlockPipeline + block_maker = BlockPipeline() + + # Test SchemaValidationError + tx_dict = signed_create_tx.to_dict() + tx_dict['invalid_key'] = 'schema validation gonna getcha!' + assert block_maker.validate_tx(tx_dict) is None + + # Test InvalidHash + tx_dict = signed_create_tx.to_dict() + tx_dict['id'] = 'a' * 64 + assert block_maker.validate_tx(tx_dict) is None + + # Test InvalidSignature when we pass a bad fulfillment + tx_dict = signed_create_tx.to_dict() + tx_dict['inputs'][0]['fulfillment'] = 'cf:0:aaaaaaaaaaaaaaaaaaaaaaaaa' + assert block_maker.validate_tx(tx_dict) is None + + # Test AmountError + signed_create_tx.outputs[0].amount = 0 + tx_dict = signed_create_tx.to_dict() + # set the correct value back so that we can continue using it + signed_create_tx.outputs[0].amount = 1 + assert block_maker.validate_tx(tx_dict) is None + + def test_create_block(b, user_pk): from bigchaindb.models import Transaction from bigchaindb.pipelines.block import BlockPipeline From 818dfc85874740d2f06bdcb0538c6abd33c61fc4 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 3 Feb 2017 14:33:22 +0100 Subject: [PATCH 167/219] Add @amirelemam to list of external contributors --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32da0036..d144bdb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ It has been more than two months since the v0.8.0 release, so there have been _m - Fixed some logic in block validation. [Pull Request #1130](https://github.com/bigchaindb/bigchaindb/pull/1130) ### External Contributors +- @amirelemam - [Pull Request #762](https://github.com/bigchaindb/bigchaindb/pull/762) (closed but used as the basis for [Pull Request #1074](https://github.com/bigchaindb/bigchaindb/pull/1074)) - @utarl - [Pull Request #1019](https://github.com/bigchaindb/bigchaindb/pull/1019) ### Notes From f9556f4c3f02f59d67d9150366847a6c2a4c3f69 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 3 Feb 2017 14:41:17 +0100 Subject: [PATCH 168/219] Changelog: Noted removal of all StatsD monitoring stuff --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d144bdb1..2ff7908e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ It has been more than two months since the v0.8.0 release, so there have been _m ### Removed - `api_endpoint` was removed from the BigchainDB configuration settings. (It was never used anyway.) [Pull Request #821](https://github.com/bigchaindb/bigchaindb/pull/821) +- Removed all remaining StatsD monitoring code, configuration settings, docs, etc. (We'll add another monitoring solution in the future.) [Pull Request #1138](https://github.com/bigchaindb/bigchaindb/pull/1138) ### Fixed - Fixed a memory (RAM) overflow problem when under heavy load by bounding the size of the queue at the entrance to the block pipeline. [Pull Request #908](https://github.com/bigchaindb/bigchaindb/pull/908) From 3c45de70d020a796ec8cfb00da23993c594c704e Mon Sep 17 00:00:00 2001 From: vrde Date: Mon, 6 Feb 2017 14:43:40 +0100 Subject: [PATCH 169/219] Wrap queries with connection object --- bigchaindb/backend/mongodb/query.py | 36 ++++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/bigchaindb/backend/mongodb/query.py b/bigchaindb/backend/mongodb/query.py index 5347df9f..6a241a78 100644 --- a/bigchaindb/backend/mongodb/query.py +++ b/bigchaindb/backend/mongodb/query.py @@ -121,7 +121,9 @@ def get_txids_filtered(conn, asset_id, operation=None): {'$match': match}, {'$project': {'block.transactions.id': True}} ] - cursor = conn.db['bigchain'].aggregate(pipeline) + cursor = conn.run( + conn.collection('bigchain') + .aggregate(pipeline)) return (elem['block']['transactions']['id'] for elem in cursor) @@ -281,18 +283,20 @@ def get_last_voted_block(conn, node_pubkey): @register_query(MongoDBConnection) def get_unvoted_blocks(conn, node_pubkey): - return conn.db['bigchain'].aggregate([ - {'$lookup': { - 'from': 'votes', - 'localField': 'id', - 'foreignField': 'vote.voting_for_block', - 'as': 'votes' - }}, - {'$match': { - 'votes.node_pubkey': {'$ne': node_pubkey}, - 'block.transactions.operation': {'$ne': 'GENESIS'} - }}, - {'$project': { - 'votes': False, '_id': False - }} - ]) + return conn.run( + conn.collection('bigchain') + .aggregate([ + {'$lookup': { + 'from': 'votes', + 'localField': 'id', + 'foreignField': 'vote.voting_for_block', + 'as': 'votes' + }}, + {'$match': { + 'votes.node_pubkey': {'$ne': node_pubkey}, + 'block.transactions.operation': {'$ne': 'GENESIS'} + }}, + {'$project': { + 'votes': False, '_id': False + }} + ])) From 5604e32d572d3a375318679ed7ddf3882e4bf7cc Mon Sep 17 00:00:00 2001 From: vrde Date: Mon, 6 Feb 2017 16:14:14 +0100 Subject: [PATCH 170/219] Try once if AutoReconnect is raised --- bigchaindb/backend/mongodb/connection.py | 11 ++++++++--- tests/backend/mongodb/test_connection.py | 7 ++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 4bd822b4..d01d5861 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -73,11 +73,16 @@ class MongoDBConnection(Connection): def run(self, query): try: - return query.run(self.conn) - except pymongo.errors.DuplicateKeyError as exc: - raise DuplicateKeyError from exc + try: + return query.run(self.conn) + except pymongo.errors.AutoReconnect as exc: + logger.warning('Lost connection to the database, ' + 'retrying query.') + return query.run(self.conn) except pymongo.errors.AutoReconnect as exc: raise ConnectionError from exc + except pymongo.errors.DuplicateKeyError as exc: + raise DuplicateKeyError from exc except pymongo.errors.OperationFailure as exc: raise OperationError from exc diff --git a/tests/backend/mongodb/test_connection.py b/tests/backend/mongodb/test_connection.py index be660d0a..786b7d7b 100644 --- a/tests/backend/mongodb/test_connection.py +++ b/tests/backend/mongodb/test_connection.py @@ -79,19 +79,24 @@ def test_connection_run_errors(mock_client, mock_init_repl_set): ConnectionError) conn = connect() - query = mock.Mock() + query = mock.Mock() query.run.side_effect = pymongo.errors.AutoReconnect('foo') with pytest.raises(ConnectionError): conn.run(query) + assert query.run.call_count == 2 + query = mock.Mock() query.run.side_effect = pymongo.errors.DuplicateKeyError('foo') with pytest.raises(DuplicateKeyError): conn.run(query) + assert query.run.call_count == 1 + query = mock.Mock() query.run.side_effect = pymongo.errors.OperationFailure('foo') with pytest.raises(OperationError): conn.run(query) + assert query.run.call_count == 1 def test_check_replica_set_not_enabled(mongodb_connection): From 553db6e34293b74a3e59e3f37846d686d02a4a14 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 16:21:16 +0100 Subject: [PATCH 171/219] Update version.py step in Release_Process.md --- Release_Process.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index 0f89c509..0b4db504 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -10,10 +10,11 @@ that [major version 0.x does not export a stable API](http://semver.org/#spec-it A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server. 1. Update the `CHANGELOG.md` file in master -1. Create and checkout a new branch for the release, named after the minor version, without preceeding 'v', ie: `git checkout -b 0.9` +1. Create and checkout a new branch for the release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` +1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) 1. Commit changes and push new branch to Github 1. Follow steps outlined in [Common Steps](#common-steps) -1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release ie: `0.10.0.dev'. +1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev'. This is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available). @@ -27,7 +28,7 @@ A patch release is similar to a minor release, but piggybacks on an existing min 1. Check out the minor release branch 1. Apply the changes you want, ie using `git cherry-pick`. 1. Update the `CHANGELOG.md` file -1. Increment the patch version in `bigchaindb/version.py`, ie: "0.9.1" +1. Increment the patch version in `bigchaindb/version.py`, e.g. "0.9.1" 1. Follow steps outlined in [Common Steps](#common-steps) ## Common steps @@ -37,7 +38,7 @@ These steps are common between minor and patch releases: 1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) and click the "Draft a new release" button 1. Fill in the details: - - Tag version: version number preceeded by 'v', ie: "v0.9.1" + - Tag version: version number preceeded by 'v', e.g. "v0.9.1" - Target: the release branch that was just pushed - Title: Same as tag name - Description: The body of the changelog entry (Added, Changed etc) From 0c8927dbbe80471f59241b556b24e31dc4396c3f Mon Sep 17 00:00:00 2001 From: vrde Date: Mon, 6 Feb 2017 16:21:22 +0100 Subject: [PATCH 172/219] Add comment to hackish solution --- bigchaindb/backend/mongodb/changefeed.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bigchaindb/backend/mongodb/changefeed.py b/bigchaindb/backend/mongodb/changefeed.py index 8bdaca92..4a5a5b7e 100644 --- a/bigchaindb/backend/mongodb/changefeed.py +++ b/bigchaindb/backend/mongodb/changefeed.py @@ -27,9 +27,11 @@ class MongoDBChangeFeed(ChangeFeed): while True: try: - # XXX: hack to force reconnection, - # the correct way to fix this is to manage errors - # for cursors.conn + # XXX: hack to force reconnection. Why? Because the cursor + # in `run_changefeed` does not run in the context of a + # Connection object, so if the connection is lost we need + # to manually reset the connection to None. + # See #1154 self.connection.connection = None self.run_changefeed() break From eded8e50a459d534c618be8ac90de370fee2b4b0 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 17:53:22 +0100 Subject: [PATCH 173/219] append 'rethinkdb' to 'bigchaindb -y configure' in Ansible role --- ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml b/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml index 5632bf9e..07ef1fef 100644 --- a/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml +++ b/ntools/one-m/ansible/roles/bigchaindb/tasks/main.yml @@ -37,7 +37,7 @@ register: home_bigchaindb_config_file - name: If ~/.bigchaindb doesn't exist, generate a default BigchainDB config file there - shell: bigchaindb -y configure + shell: bigchaindb -y configure rethinkdb when: not home_bigchaindb_config_file.stat.exists - name: Look up all processes with 'bigchaindb' in their name From 758b65ba0ae2d766c1b85937715fd73379d21d04 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Mon, 6 Feb 2017 12:08:48 +0100 Subject: [PATCH 174/219] Upgrade pysha3 to 1.0.2 to fix usage on Python 3.6 on macOS --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d1a22279..7e8c3441 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ install_requires = [ # TODO Consider not installing the db drivers, or putting them in extras. 'rethinkdb~=2.3', # i.e. a version between 2.3 and 3.0 'pymongo~=3.4', - 'pysha3==1.0.0', + 'pysha3~=1.0.2', 'cryptoconditions>=0.5.0', 'python-rapidjson>=0.0.8', 'logstats>=0.2.1', From 85128a0634dc4b354e928e54b2996398871d0a83 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 18:01:38 +0100 Subject: [PATCH 175/219] append 'rethinkdb' to 'bigchaindb -y configure' in some docs --- .../azure-quickstart-template.md | 2 +- docs/server/source/nodes/setup-run-node.md | 2 +- docs/server/source/server-reference/configuration.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md b/docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md index 9f8a8d6e..7603319f 100644 --- a/docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md +++ b/docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md @@ -24,7 +24,7 @@ One can deploy a BigchainDB node on Azure using the template in the `bigchaindb- 1. Configure BigchainDB Server by doing: ```text -bigchaindb configure +bigchaindb configure rethinkdb ``` It will ask you several questions. You can press `Enter` (or `Return`) to accept the default for all of them *except for one*. When it asks **API Server bind? (default \`localhost:9984\`):**, you should answer: ```text diff --git a/docs/server/source/nodes/setup-run-node.md b/docs/server/source/nodes/setup-run-node.md index 9d0a4892..41a9cdd1 100644 --- a/docs/server/source/nodes/setup-run-node.md +++ b/docs/server/source/nodes/setup-run-node.md @@ -144,7 +144,7 @@ python setup.py install Start by creating a default BigchainDB config file: ```text -bigchaindb -y configure +bigchaindb -y configure rethinkdb ``` (There's documentation for the `bigchaindb` command is in the section on [the BigchainDB Command Line Interface (CLI)](bigchaindb-cli.html).) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index b7842d6a..fb8d8a6a 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -28,7 +28,7 @@ Note that the `-c` command line option will always take precedence if both the ` You can read the current default values in the file [bigchaindb/\_\_init\_\_.py](https://github.com/bigchaindb/bigchaindb/blob/master/bigchaindb/__init__.py). (The link is to the latest version.) -Running `bigchaindb -y configure` will generate a local config file in `$HOME/.bigchaindb` with all the default values, with two exceptions: It will generate a valid private/public keypair, rather than using the default keypair (`None` and `None`). +Running `bigchaindb -y configure rethinkdb` will generate a local config file in `$HOME/.bigchaindb` with all the default values, with two exceptions: It will generate a valid private/public keypair, rather than using the default keypair (`None` and `None`). ## keypair.public & keypair.private @@ -49,7 +49,7 @@ export BIGCHAINDB_KEYPAIR_PRIVATE=5C5Cknco7YxBRP9AgB1cbUVTL4FAcooxErLygw1DeG2D } ``` -Internally (i.e. in the Python code), both keys have a default value of `None`, but that's not a valid key. Therefore you can't rely on the defaults for the keypair. If you want to run BigchainDB, you must provide a valid keypair, either in the environment variables or in the local config file. You can generate a local config file with a valid keypair (and default everything else) using `bigchaindb -y configure`. +Internally (i.e. in the Python code), both keys have a default value of `None`, but that's not a valid key. Therefore you can't rely on the defaults for the keypair. If you want to run BigchainDB, you must provide a valid keypair, either in the environment variables or in the local config file. You can generate a local config file with a valid keypair (and default everything else) using `bigchaindb -y configure rethinkdb`. ## keyring From c5333594243211f8e4cca35aaabdd352a5acdab8 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Mon, 6 Feb 2017 17:37:56 +0100 Subject: [PATCH 176/219] Default Dockerfile-dev to configure BigchainDB for RethinkDB --- Dockerfile-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index c201fd0b..2ae4e2ba 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -10,4 +10,4 @@ RUN pip install --upgrade pip COPY . /usr/src/app/ RUN pip install --no-cache-dir -e .[dev] -RUN bigchaindb -y configure +RUN bigchaindb -y configure rethinkdb From 2962b4a27d45fb87fb8a7c2cc345bcb4727ffb12 Mon Sep 17 00:00:00 2001 From: vrde Date: Mon, 6 Feb 2017 18:01:21 +0100 Subject: [PATCH 177/219] Skip test --- tests/backend/mongodb/test_indexes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/backend/mongodb/test_indexes.py b/tests/backend/mongodb/test_indexes.py index ba6afae1..d11d124c 100644 --- a/tests/backend/mongodb/test_indexes.py +++ b/tests/backend/mongodb/test_indexes.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock pytestmark = pytest.mark.bdb +@pytest.mark.skipif(reason='Will be handled by #1126') def test_asset_id_index(): from bigchaindb.backend.mongodb.query import get_txids_filtered from bigchaindb.backend import connect From 1e7adabc355cbeff094dfafa9e5a8b1bba953c12 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 18:43:34 +0100 Subject: [PATCH 178/219] Branch names should have the form X.Y.Z --- Release_Process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Release_Process.md b/Release_Process.md index 0b4db504..40a82f12 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -10,7 +10,7 @@ that [major version 0.x does not export a stable API](http://semver.org/#spec-it A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server. 1. Update the `CHANGELOG.md` file in master -1. Create and checkout a new branch for the release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` +1. Create and checkout a new branch for the release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9.4` 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) 1. Commit changes and push new branch to Github 1. Follow steps outlined in [Common Steps](#common-steps) From 5dead8d3d2ea938406e2c8c2d528f8b6fdb03251 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 18:46:00 +0100 Subject: [PATCH 179/219] version.py update for 0.9.0 release --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 060e76d3..a0ea3097 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.9.0.dev' -__short_version__ = '0.9.dev' +__version__ = '0.9.0' +__short_version__ = '0.9' From 88d08a551c5f5f9465c2c89967dece495da2e3b6 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 19:05:11 +0100 Subject: [PATCH 180/219] Changed version.py in master to 0.10.0.dev --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 060e76d3..c59a3cbe 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.9.0.dev' -__short_version__ = '0.9.dev' +__version__ = '0.10.0.dev' +__short_version__ = '0.10.dev' From 232ddc00a00eb7f67b0209bf4fff3441b7e376c5 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 19:54:53 +0100 Subject: [PATCH 181/219] Two changes to Release_Process.md 1. Merge the new version's branch into the master branch 2. Added a helper line about how to use twine to upload to PyPI --- Release_Process.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index 40a82f12..c11f1a63 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -12,7 +12,7 @@ A minor release is preceeded by a feature freeze and created from the 'master' b 1. Update the `CHANGELOG.md` file in master 1. Create and checkout a new branch for the release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9.4` 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) -1. Commit changes and push new branch to Github +1. Commit changes, push new branch to Github, and merge it into master 1. Follow steps outlined in [Common Steps](#common-steps) 1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev'. This is so people reading the latest docs will know that they're for the latest (master branch) @@ -43,7 +43,7 @@ These steps are common between minor and patch releases: - Title: Same as tag name - Description: The body of the changelog entry (Added, Changed etc) 1. Publish the release on Github -1. Generate the release tarball with `python setup.py sdist`. Upload the release to Pypi. +1. Generate the release tarball with `python setup.py sdist`. Upload the release to PyPI, e.g. `twine upload dist/BigchainDB-0.9.0.tar.gz` (requires you to have a `~/.pypirc` file) 1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is "Active" and "Public", and make sure the new version's branch From f7b6fdc6e894d6152fed50f7000c06122859ef9d Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 20:03:54 +0100 Subject: [PATCH 182/219] In master, bump version.py to 0.10.0.dev again --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index a0ea3097..c59a3cbe 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.9.0' -__short_version__ = '0.9' +__version__ = '0.10.0.dev' +__short_version__ = '0.10.dev' From a049081b0772adee8164c2590d82eb9cfb570f9e Mon Sep 17 00:00:00 2001 From: tim Date: Mon, 6 Feb 2017 20:09:08 +0100 Subject: [PATCH 183/219] Fix tx model versioning --- bigchaindb/common/transaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index e4bd642f..9da2421a 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -410,7 +410,7 @@ class Transaction(object): TRANSFER = 'TRANSFER' GENESIS = 'GENESIS' ALLOWED_OPERATIONS = (CREATE, TRANSFER, GENESIS) - VERSION = bigchaindb.version.__short_version__[:-4] # 0.9, 0.10 etc + VERSION = '.'.join(bigchaindb.version.__short_version__.split('.')[:2]) def __init__(self, operation, asset, inputs=None, outputs=None, metadata=None, version=None): From 55670effe30ad60e479ba22a344cfc535f7a8378 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Feb 2017 21:40:17 +0100 Subject: [PATCH 184/219] Updated CHANGELOG.md for v0.9.1 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ff7908e..f1f8d6b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ For reference, the possible headings are: * **External Contributors** to list contributors outside of BigchainDB GmbH. * **Notes** +## [0.9.1] - 2017-02-06 +Tag name: v0.9.1 + +### Fixed +* Fixed bug in how the transaction `VERSION` string was calculated from the BigchainDB Server `__short_version__` string. [Pull Request #1160](https://github.com/bigchaindb/bigchaindb/pull/1160) + + ## [0.9.0] - 2017-02-06 Tag name: v0.9.0 From a55ae0e44d8e10a5823008e2f06db1f1460b41fa Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Feb 2017 11:46:25 +0100 Subject: [PATCH 185/219] Various modifications to Release_Process.md --- Release_Process.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index c11f1a63..856ed2bc 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -10,11 +10,11 @@ that [major version 0.x does not export a stable API](http://semver.org/#spec-it A minor release is preceeded by a feature freeze and created from the 'master' branch. This is a summary of the steps we go through to release a new minor version of BigchainDB Server. 1. Update the `CHANGELOG.md` file in master -1. Create and checkout a new branch for the release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9.4` +1. Create and checkout a new branch for the minor release, named after the minor version, without a preceeding 'v', e.g. `git checkout -b 0.9` (*not* 0.9.0, this new branch will be for e.g. 0.9.0, 0.9.1, 0.9.2, etc. each of which will be identified by a tagged commit) 1. In `bigchaindb/version.py`, update `__version__` and `__short_version__`, e.g. to `0.9` and `0.9.0` (with no `.dev` on the end) -1. Commit changes, push new branch to Github, and merge it into master +1. Commit that change, and push the new branch to GitHub 1. Follow steps outlined in [Common Steps](#common-steps) -1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev'. +1. In 'master' branch, Edit `bigchaindb/version.py`, increment the minor version to the next planned release, e.g. `0.10.0.dev' This is so people reading the latest docs will know that they're for the latest (master branch) version of BigchainDB Server, not the docs at the time of the most recent release (which are also available). @@ -25,11 +25,12 @@ Congratulations, you have released BigchainDB! A patch release is similar to a minor release, but piggybacks on an existing minor release branch: -1. Check out the minor release branch -1. Apply the changes you want, ie using `git cherry-pick`. +1. Check out the minor release branch, e.g. `0.9` +1. Apply the changes you want, e.g. using `git cherry-pick`. 1. Update the `CHANGELOG.md` file 1. Increment the patch version in `bigchaindb/version.py`, e.g. "0.9.1" 1. Follow steps outlined in [Common Steps](#common-steps) +1. Cherry-pick the `CHANGELOG.md` update commit (made above) to the `master` branch ## Common steps @@ -42,7 +43,8 @@ These steps are common between minor and patch releases: - Target: the release branch that was just pushed - Title: Same as tag name - Description: The body of the changelog entry (Added, Changed etc) -1. Publish the release on Github +1. Publish the release on GitHub +1. (Optional) Do a `make clean` in the repository root directory (`bigchaindb/`) 1. Generate the release tarball with `python setup.py sdist`. Upload the release to PyPI, e.g. `twine upload dist/BigchainDB-0.9.0.tar.gz` (requires you to have a `~/.pypirc` file) 1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is From 0d0c93cab05d02b80ec6438aa6cd65f2a48cf4ed Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Feb 2017 15:18:45 +0100 Subject: [PATCH 186/219] Made changes suggested by @sbellem in #1164 + some clarifications --- Release_Process.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Release_Process.md b/Release_Process.md index 856ed2bc..a4e3d427 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -41,11 +41,12 @@ These steps are common between minor and patch releases: 1. Fill in the details: - Tag version: version number preceeded by 'v', e.g. "v0.9.1" - Target: the release branch that was just pushed - - Title: Same as tag name - - Description: The body of the changelog entry (Added, Changed etc) -1. Publish the release on GitHub -1. (Optional) Do a `make clean` in the repository root directory (`bigchaindb/`) -1. Generate the release tarball with `python setup.py sdist`. Upload the release to PyPI, e.g. `twine upload dist/BigchainDB-0.9.0.tar.gz` (requires you to have a `~/.pypirc` file) + - Title: Same as tag name above, e.g "v0.9.1" + - Description: The body of the changelog entry (Added, Changed, etc.) +1. Click "Publish release" to publish the release on GitHub +1. Make sure your local Git is in the same state as the release: e.g. `git fetch ` and `git checkout v0.9.1` +1. Make sure you have a `~/.pypirc` file containing credentials for PyPI +1. Do a `make release` to build and publish the new `bigchaindb` package on PyPI 1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is "Active" and "Public", and make sure the new version's branch From 1d0396212b35d27ac5a04535b029de31be39395f Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 7 Feb 2017 14:44:45 +0100 Subject: [PATCH 187/219] Modify Makefile for releases --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d8ef2bf4..77c6b9ec 100644 --- a/Makefile +++ b/Makefile @@ -74,13 +74,12 @@ docs: ## generate Sphinx HTML documentation, including API docs servedocs: docs ## compile the docs watching for changes watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . -release: clean ## package and upload a release - python setup.py sdist upload - python setup.py bdist_wheel upload +release: clean ## package and upload a release (requires a `~/.pypirc` file) + twine upload dist/* -dist: clean ## builds source and wheel package +dist: clean ## builds source (and not for now, wheel package) python setup.py sdist - python setup.py bdist_wheel + # python setup.py bdist_wheel ls -l dist install: clean ## install the package to the active Python's site-packages From 5e479d831b80824c36affe5a854bb232bb746243 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 7 Feb 2017 18:05:01 +0100 Subject: [PATCH 188/219] Remove superfluous comment --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 77c6b9ec..7fc9c1c0 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ docs: ## generate Sphinx HTML documentation, including API docs servedocs: docs ## compile the docs watching for changes watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . -release: clean ## package and upload a release (requires a `~/.pypirc` file) +release: clean ## package and upload a release twine upload dist/* dist: clean ## builds source (and not for now, wheel package) From aae8c57902765201704c12a4a4bc0019d5cebc84 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 8 Feb 2017 13:56:55 +0100 Subject: [PATCH 189/219] Fix production-ready-note --- docs/root/source/production-ready.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/source/production-ready.md b/docs/root/source/production-ready.md index eb24c5b3..deb87714 100644 --- a/docs/root/source/production-ready.md +++ b/docs/root/source/production-ready.md @@ -2,6 +2,6 @@ BigchainDB is not production-ready. You can use it to build a prototype or proof-of-concept (POC); many people are already doing that. -BigchainDB Server is currently in version 0.X. ([The Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) has the exact version number.) Once we believe that BigchainDB Server is production-ready, we'll release version 1.0. +BigchainDB Server is currently in version 0.X. ([The Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) has the exact version number.) Once BigchainDB Server is production-ready, we'll issue an announcement. [The BigchainDB Roadmap](https://github.com/bigchaindb/org/blob/master/ROADMAP.md) will give you a sense of the things we intend to do with BigchainDB in the near term and the long term. \ No newline at end of file From f66c44689a486f06d3546a595616f583dd05977c Mon Sep 17 00:00:00 2001 From: "krish7919 (Krish)" Date: Wed, 8 Feb 2017 17:10:45 +0100 Subject: [PATCH 190/219] Decouple backend DB and BigchainDB --- Dockerfile | 6 +- .../source/appendices/run-with-docker.md | 96 ++++++++++++++----- 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5e4b3b1..ed0ca462 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rethinkdb:2.3 +FROM ubuntu:xenial # From http://stackoverflow.com/a/38553499 @@ -34,8 +34,6 @@ ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 # but maybe our Docker or Docker Compose stuff does? # ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1 -ENTRYPOINT ["bigchaindb", "--dev-start-rethinkdb", "--dev-allow-temp-keypair"] +ENTRYPOINT ["bigchaindb"] CMD ["start"] - -EXPOSE 8080 9984 28015 29015 diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index d6e33a70..6c401102 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -15,13 +15,21 @@ In a terminal shell, pull the latest version of the BigchainDB Docker image usin docker pull bigchaindb/bigchaindb ``` -then do a one-time configuration step to create the config file; we will use +### Configuration +A one-time configuration step is required to create the config file; we will use the `-y` option to accept all the default values. The configuration file will be stored in a file on your host machine at `~/bigchaindb_docker/.bigchaindb`: ```text -docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ - bigchaindb/bigchaindb -y configure rethinkdb +docker run \ + --interactive \ + --rm \ + --tty \ + --volume "$HOME/bigchaindb_docker:/data" \ + bigchaindb/bigchaindb \ + -y configure --dev-allow-temp-keypair \ + [mongodb|rethinkdb] + Generating keypair Configuration written to /data/.bigchaindb Ready to go! @@ -30,35 +38,77 @@ Ready to go! Let's analyze that command: * `docker run` tells Docker to run some image +* `--interactive` keep STDIN open even if not attached * `--rm` remove the container once we are done -* `-v "$HOME/bigchaindb_docker:/data"` map the host directory +* `--tty` allocate a pseudo-TTY +* `--volume "$HOME/bigchaindb_docker:/data"` map the host directory `$HOME/bigchaindb_docker` to the container directory `/data`; this allows us to have the data persisted on the host machine, you can read more in the [official Docker documentation](https://docs.docker.com/engine/tutorials/dockervolumes/#/mount-a-host-directory-as-a-data-volume) -* `-t` allocate a pseudo-TTY -* `-i` keep STDIN open even if not attached -* `bigchaindb/bigchaindb` the image to use -* `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values +* `bigchaindb/bigchaindb` the image to use. All the options after the container name are passed on to the entrypoint inside the container. +* `-y configure` execute the `configure` sub-command (of the `bigchaindb` + command) inside the container, with the `-y` option to automatically use all the default config values +* `--dev-allow-temp-keypair` specifies that this is a dev environment and + enables creation of temporary key pairs to be written to the configuration + file +* `mongodb` or `rethinkdb` specifies the database backend to use with bigchaindb -After configuring the system, you can run BigchainDB with the following command: +### Run the backend database +From v0.9 onwards, you can run either RethinkDB or MongoDB. + +We use the virtual interface created by the docker daemon to allow +communication between the BigchainDB and database containers. +It has an IP address of 172.17.0.1 by default. + +You can also use docker host networking or bind to your primary (eth) + interface, if needed. + +#### For RethinkDB ```text -docker run -v "$HOME/bigchaindb_docker:/data" -d \ - --name bigchaindb \ - -p "58080:8080" -p "59984:9984" \ - bigchaindb/bigchaindb start +docker run \ + --detach \ + --name=rethinkdb \ + --publish=172.17.0.1:28015:28015 \ + --publish=172.17.0.1:58080:8080 \ + rethinkdb:2.3 +``` + + +You can also access the RethinkDB dashboard at +[http://localhost:58080/](http://localhost:58080/) + + +#### For MongoDB + +```text +docker run \ + --detach \ + --name=mongodb \ + --publish=172.17.0.1:27017:27017 \ + mongo:3.4.1 --replSet=bigchain-rs +``` + +### Run BigchainDB + +```text +docker run \ + --detach \ + --name=bigchaindb \ + --publish=59984:9984 \ + --volume=$HOME/bigchaindb_docker:/data \ + bigchaindb/bigchaindb \ + start ``` The command is slightly different from the previous one, the differences are: -* `-d` run the container in the background +* `--detach` run the container in the background * `--name bigchaindb` give a nice name to the container so it's easier to refer to it later -* `-p "58080:8080"` map the host port `58080` to the container port `8080` - (the RethinkDB admin interface) -* `-p "59984:9984"` map the host port `59984` to the container port `9984` +* `--publish "59984:9984"` map the host port `59984` to the container port `9984` (the BigchainDB API server) * `start` start the BigchainDB service @@ -66,9 +116,6 @@ Another way to publish the ports exposed by the container is to use the `-P` (or `--publish-all`) option. This will publish all exposed ports to random ports. You can always run `docker ps` to check the random mapping. -You can also access the RethinkDB dashboard at -[http://localhost:58080/](http://localhost:58080/) - If that doesn't work, then replace `localhost` with the IP or hostname of the machine running the Docker engine. If you are running docker-machine (e.g. on Mac OS X) this will be the IP of the Docker machine (`docker-machine ip @@ -89,10 +136,13 @@ You should see a container named `bigchaindb` in the list. You can load test the BigchainDB running in that container by running the `bigchaindb load` command in a second container: ```text -docker run --rm -v "$HOME/bigchaindb_docker:/data" \ - -e BIGCHAINDB_DATABASE_HOST=bigchaindb \ +docker run \ + --env BIGCHAINDB_DATABASE_HOST=bigchaindb \ --link bigchaindb \ - bigchaindb/bigchaindb load + --rm \ + --volume "$HOME/bigchaindb_docker:/data" \ + bigchaindb/bigchaindb \ + load ``` Note the `--link` option to link to the first container (named `bigchaindb`). From 5ffa6568c4ad4868e41639c3d7c6cb5d9ed5f620 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 8 Feb 2017 12:13:54 +0100 Subject: [PATCH 191/219] Add a section on community-driven libs --- docs/server/source/drivers-clients/index.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index dd33e18b..f6515a4f 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -7,6 +7,9 @@ We also provide the Transaction CLI to be able to script the building of transactions. You may be able to wrap this tool inside the language of your choice, and then use the HTTP API directly to post transactions. +If you use a language other than Python, you may want to look at the current +community projects listed below. + .. toctree:: :maxdepth: 1 @@ -14,3 +17,12 @@ your choice, and then use the HTTP API directly to post transactions. http-client-server-api The Python Driver Transaction CLI + + +Community Driven Libraries and Tools +------------------------------------ + +* `Javascript transaction builder <(https://github.com/sohkai/js-bigchaindb-quickstart>`_ +* `Haskell transaction builder `_ +* `Go driver `_ + From 1be05d3ffc24fc56ea641d76b577dd1719340341 Mon Sep 17 00:00:00 2001 From: "krish7919 (Krish)" Date: Thu, 9 Feb 2017 09:55:53 +0100 Subject: [PATCH 192/219] Changes as requested by @ttmc --- docs/server/source/appendices/run-with-docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index 6c401102..7c6f9777 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -27,7 +27,7 @@ docker run \ --tty \ --volume "$HOME/bigchaindb_docker:/data" \ bigchaindb/bigchaindb \ - -y configure --dev-allow-temp-keypair \ + -y configure \ [mongodb|rethinkdb] Generating keypair @@ -58,7 +58,7 @@ Let's analyze that command: ### Run the backend database From v0.9 onwards, you can run either RethinkDB or MongoDB. -We use the virtual interface created by the docker daemon to allow +We use the virtual interface created by the Docker daemon to allow communication between the BigchainDB and database containers. It has an IP address of 172.17.0.1 by default. From f2a05c9e3b3a649e526e7caa9bad29d9e20b23ac Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 9 Feb 2017 11:10:58 +0100 Subject: [PATCH 193/219] Fixed the docs URL in HTTP API responses --- bigchaindb/web/views/info.py | 8 ++++---- tests/web/test_info.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index cbe32acb..04a15749 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -11,8 +11,8 @@ from bigchaindb import version class RootIndex(Resource): def get(self): docs_url = [ - 'https://docs.bigchaindb.com/projects/server/en/', - version.__short_version__ + '/' + 'https://docs.bigchaindb.com/projects/server/en/v', + version.__version__ + '/' ] api_v1_url = base_url() + 'api/v1/' return flask.jsonify({ @@ -31,8 +31,8 @@ class ApiV1Index(Resource): def get(self): api_root = base_url() + 'api/v1/' docs_url = [ - 'https://docs.bigchaindb.com/projects/server/en/', - version.__short_version__, + 'https://docs.bigchaindb.com/projects/server/en/v', + version.__version__, '/drivers-clients/http-client-server-api.html', ] return { diff --git a/tests/web/test_info.py b/tests/web/test_info.py index 93b39390..c55f467f 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -8,7 +8,7 @@ def test_api_root_endpoint(client): res = client.get('/') assert res.json == { '_links': { - 'docs': 'https://docs.bigchaindb.com/projects/server/en/tst/', + 'docs': 'https://docs.bigchaindb.com/projects/server/en/vtsttst/', 'api_v1': 'http://localhost/api/v1/', }, 'version': 'tsttst', @@ -19,10 +19,10 @@ def test_api_root_endpoint(client): @mock.patch('bigchaindb.version.__short_version__', 'tst') +@mock.patch('bigchaindb.version.__version__', 'tsttst') def test_api_v1_endpoint(client): res = client.get('/api/v1') - docs_url = ['https://docs.bigchaindb.com/projects/server/en/', - 'tst', + docs_url = ['https://docs.bigchaindb.com/projects/server/en/vtsttst', '/drivers-clients/http-client-server-api.html', ] assert res.json == { From 8da47243ac183707543d893226009caecc397b0d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Fri, 10 Feb 2017 17:36:19 +0100 Subject: [PATCH 194/219] Fix link --- docs/server/source/drivers-clients/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index f6515a4f..d7654337 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -22,7 +22,7 @@ community projects listed below. Community Driven Libraries and Tools ------------------------------------ -* `Javascript transaction builder <(https://github.com/sohkai/js-bigchaindb-quickstart>`_ +* `Javascript transaction builder `_ * `Haskell transaction builder `_ * `Go driver `_ From edb773695e2f658f1b912eff0d417070b5fda92e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sat, 11 Feb 2017 10:11:52 +0100 Subject: [PATCH 195/219] Added some MongoDB ports to docs Notes for Firewall Setup --- docs/server/source/appendices/firewall-notes.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/server/source/appendices/firewall-notes.md b/docs/server/source/appendices/firewall-notes.md index 19d7c234..14d85937 100644 --- a/docs/server/source/appendices/firewall-notes.md +++ b/docs/server/source/appendices/firewall-notes.md @@ -9,7 +9,8 @@ Assuming you aren't exposing the RethinkDB web interface on port 8080 (or any ot 1. **Port 22** can expect inbound SSH (TCP) traffic from the node administrator (i.e. a small set of IP addresses). 2. **Port 9984** can expect inbound HTTP (TCP) traffic from BigchainDB clients sending transactions to the BigchainDB HTTP API. -3. **Port 29015** can expect inbound TCP traffic from other RethinkDB nodes in the RethinkDB cluster (for RethinkDB intracluster communications). +3. If you're using RethinkDB, **Port 29015** can expect inbound TCP traffic from other RethinkDB nodes in the RethinkDB cluster (for RethinkDB intracluster communications). +4. If you're using MongoDB, **Port 27017** can expect inbound TCP traffic from other nodes. Also, see Port 28017 below. All other ports should only get inbound traffic in response to specific requests from inside the node. @@ -63,6 +64,11 @@ You may want to have Gunicorn and the reverse proxy running on different servers Port 28015 is the default port used by RethinkDB client driver connections (TCP). If your BigchainDB node is just one server, then Port 28015 only needs to listen on localhost, because all the client drivers will be running on localhost. Port 28015 doesn't need to accept inbound traffic from the outside world. +## Port 28017 + +Port 28017 is the default port used by MongoDB for its web status page. + + ## Port 29015 Port 29015 is the default port for RethinkDB intracluster connections (TCP). It should only accept incoming traffic from other RethinkDB servers in the cluster (a list of IP addresses that you should be able to find out). From e8b8b595b71a4939ddcfcb87f3c23da0ce1a1edc Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sat, 11 Feb 2017 10:29:56 +0100 Subject: [PATCH 196/219] Added terraform.tfstate* files to .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 1a22ad18..efa00db2 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,8 @@ ntools/one-m/ansible/ansible.cfg # Just in time documentation docs/server/source/schema docs/server/source/drivers-clients/samples + +# Terraform state files +# See https://stackoverflow.com/a/41482391 +terraform.tfstate +terraform.tfstate.backup From 832a0e9e413852df7047410f6b9d4c500a734524 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Sat, 11 Feb 2017 14:59:55 +0100 Subject: [PATCH 197/219] Add Java driver link to docs --- docs/server/source/drivers-clients/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index d7654337..18894f60 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -21,8 +21,10 @@ community projects listed below. Community Driven Libraries and Tools ------------------------------------ +Please note that some of these projects may be work in progress, but may +nevertheless be very useful. * `Javascript transaction builder `_ * `Haskell transaction builder `_ * `Go driver `_ - +* `Java driver `_ From 8128caa86df7c9a940c7bd9fafcf7d1469c80191 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 13 Feb 2017 15:09:48 +0100 Subject: [PATCH 198/219] Removed notes about port 28017 --- docs/server/source/appendices/firewall-notes.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/server/source/appendices/firewall-notes.md b/docs/server/source/appendices/firewall-notes.md index 14d85937..cd440774 100644 --- a/docs/server/source/appendices/firewall-notes.md +++ b/docs/server/source/appendices/firewall-notes.md @@ -10,7 +10,7 @@ Assuming you aren't exposing the RethinkDB web interface on port 8080 (or any ot 1. **Port 22** can expect inbound SSH (TCP) traffic from the node administrator (i.e. a small set of IP addresses). 2. **Port 9984** can expect inbound HTTP (TCP) traffic from BigchainDB clients sending transactions to the BigchainDB HTTP API. 3. If you're using RethinkDB, **Port 29015** can expect inbound TCP traffic from other RethinkDB nodes in the RethinkDB cluster (for RethinkDB intracluster communications). -4. If you're using MongoDB, **Port 27017** can expect inbound TCP traffic from other nodes. Also, see Port 28017 below. +4. If you're using MongoDB, **Port 27017** can expect inbound TCP traffic from other nodes. All other ports should only get inbound traffic in response to specific requests from inside the node. @@ -64,11 +64,6 @@ You may want to have Gunicorn and the reverse proxy running on different servers Port 28015 is the default port used by RethinkDB client driver connections (TCP). If your BigchainDB node is just one server, then Port 28015 only needs to listen on localhost, because all the client drivers will be running on localhost. Port 28015 doesn't need to accept inbound traffic from the outside world. -## Port 28017 - -Port 28017 is the default port used by MongoDB for its web status page. - - ## Port 29015 Port 29015 is the default port for RethinkDB intracluster connections (TCP). It should only accept incoming traffic from other RethinkDB servers in the cluster (a list of IP addresses that you should be able to find out). From 2780b9406a7c5be97736e5e9f110b063ad1d2171 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 14 Feb 2017 18:55:46 +0100 Subject: [PATCH 199/219] docs: renamed Starter Templates --> Templates --- .../cloud-deployment-starter-templates/index.rst | 15 --------------- .../azure-quickstart-template.md | 2 +- .../source/cloud-deployment-templates/index.rst | 16 ++++++++++++++++ .../template-ansible.md | 2 +- .../template-terraform-aws.md | 2 +- docs/server/source/index.rst | 2 +- docs/server/source/introduction.md | 4 ++-- docs/server/source/nodes/node-assumptions.md | 2 +- 8 files changed, 23 insertions(+), 22 deletions(-) delete mode 100644 docs/server/source/cloud-deployment-starter-templates/index.rst rename docs/server/source/{cloud-deployment-starter-templates => cloud-deployment-templates}/azure-quickstart-template.md (94%) create mode 100644 docs/server/source/cloud-deployment-templates/index.rst rename docs/server/source/{cloud-deployment-starter-templates => cloud-deployment-templates}/template-ansible.md (96%) rename docs/server/source/{cloud-deployment-starter-templates => cloud-deployment-templates}/template-terraform-aws.md (95%) diff --git a/docs/server/source/cloud-deployment-starter-templates/index.rst b/docs/server/source/cloud-deployment-starter-templates/index.rst deleted file mode 100644 index bb71561e..00000000 --- a/docs/server/source/cloud-deployment-starter-templates/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -Cloud Deployment Starter Templates -================================== - -We have some "starter templates" to deploy a basic, working, but bare-bones BigchainDB node on various cloud providers. They should *not* be used as-is to deploy a node for production. They can be used as a starting point. A full production node should meet the requirements outlined in the section on :doc:`production node assumptions, components and requirements <../nodes/index>`. - -You don't have to use the tools we use in the starter templates (e.g. Terraform and Ansible). You can use whatever tools you prefer. - -If you find the cloud deployment starter templates for nodes helpful, then you may also be interested in our scripts for :doc:`deploying a testing cluster on AWS <../clusters-feds/aws-testing-cluster>` (documented in the Clusters & Federations section). - -.. toctree:: - :maxdepth: 1 - - template-terraform-aws - template-ansible - azure-quickstart-template diff --git a/docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md b/docs/server/source/cloud-deployment-templates/azure-quickstart-template.md similarity index 94% rename from docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md rename to docs/server/source/cloud-deployment-templates/azure-quickstart-template.md index 7603319f..1bf35a31 100644 --- a/docs/server/source/cloud-deployment-starter-templates/azure-quickstart-template.md +++ b/docs/server/source/cloud-deployment-templates/azure-quickstart-template.md @@ -1,6 +1,6 @@ # Azure Quickstart Template -If you didn't read the introduction to the [cloud deployment starter templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. +If you didn't read the introduction to the [cloud deployment templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. Note: There was an Azure quickstart template in the `blockchain` directory of Microsoft's `Azure/azure-quickstart-templates` repository on GitHub. It's gone now; it was replaced by the one described here. diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst new file mode 100644 index 00000000..960abaa5 --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -0,0 +1,16 @@ +Cloud Deployment Templates +========================== + +We have some "templates" to deploy a basic, working, but bare-bones BigchainDB node on various cloud providers. They should *not* be used as-is to deploy a node for production. They can be used as a starting point. + +You don't have to use the tools we use in the templates. You can use whatever tools you prefer. + +If you find the cloud deployment templates for nodes helpful, then you may also be interested in our scripts for :doc:`deploying a testing cluster on AWS <../clusters-feds/aws-testing-cluster>` (documented in the Clusters & Federations section). + +.. toctree:: + :maxdepth: 1 + + template-terraform-aws + template-ansible + azure-quickstart-template + advanced-node-on-azure diff --git a/docs/server/source/cloud-deployment-starter-templates/template-ansible.md b/docs/server/source/cloud-deployment-templates/template-ansible.md similarity index 96% rename from docs/server/source/cloud-deployment-starter-templates/template-ansible.md rename to docs/server/source/cloud-deployment-templates/template-ansible.md index e71d4cc1..666ad790 100644 --- a/docs/server/source/cloud-deployment-starter-templates/template-ansible.md +++ b/docs/server/source/cloud-deployment-templates/template-ansible.md @@ -1,6 +1,6 @@ # Template: Ansible Playbook to Run a BigchainDB Node on an Ubuntu Machine -If you didn't read the introduction to the [cloud deployment starter templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. +If you didn't read the introduction to the [cloud deployment templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. This page explains how to use [Ansible](https://www.ansible.com/) to install, configure and run all the software needed to run a one-machine BigchainDB node on a server running Ubuntu 16.04. diff --git a/docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md b/docs/server/source/cloud-deployment-templates/template-terraform-aws.md similarity index 95% rename from docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md rename to docs/server/source/cloud-deployment-templates/template-terraform-aws.md index 85e4cf9d..d4a22e83 100644 --- a/docs/server/source/cloud-deployment-starter-templates/template-terraform-aws.md +++ b/docs/server/source/cloud-deployment-templates/template-terraform-aws.md @@ -1,6 +1,6 @@ # Template: Using Terraform to Provision an Ubuntu Machine on AWS -If you didn't read the introduction to the [cloud deployment starter templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. +If you didn't read the introduction to the [cloud deployment templates](index.html), please do that now. The main point is that they're not for deploying a production node; they can be used as a starting point. This page explains a way to use [Terraform](https://www.terraform.io/) to provision an Ubuntu machine (i.e. an EC2 instance with Ubuntu 16.04) and other resources on [AWS](https://aws.amazon.com/). That machine can then be used to host a one-machine BigchainDB node. diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index 932f1951..7f85a228 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -7,7 +7,7 @@ BigchainDB Server Documentation ← Back to All BigchainDB Docs introduction quickstart - cloud-deployment-starter-templates/index + cloud-deployment-templates/index nodes/index dev-and-test/index server-reference/index diff --git a/docs/server/source/introduction.md b/docs/server/source/introduction.md index 2b02d964..b9e6bf0a 100644 --- a/docs/server/source/introduction.md +++ b/docs/server/source/introduction.md @@ -8,7 +8,7 @@ Note that there are a few kinds of nodes: - A **dev/test node** is a node created by a developer working on BigchainDB Server, e.g. for testing new or changed code. A dev/test node is typically run on the developer's local machine. -- A **bare-bones node** is a node deployed in the cloud, either as part of a testing cluster or as a starting point before upgrading the node to be production-ready. Our cloud deployment starter templates deploy a bare-bones node, as do our scripts for deploying a testing cluster on AWS. +- A **bare-bones node** is a node deployed in the cloud, either as part of a testing cluster or as a starting point before upgrading the node to be production-ready. Our cloud deployment templates deploy a bare-bones node, as do our scripts for deploying a testing cluster on AWS. - A **production node** is a node that is part of a federation's BigchainDB cluster. A production node has the most components and requirements. @@ -16,7 +16,7 @@ Note that there are a few kinds of nodes: ## Setup Instructions for Various Cases * [Set up a local stand-alone BigchainDB node for learning and experimenting: Quickstart](quickstart.html) -* [Set up and run a bare-bones node in the cloud](cloud-deployment-starter-templates/index.html) +* [Set up and run a bare-bones node in the cloud](cloud-deployment-templates/index.html) * [Set up and run a local dev/test node for developing and testing BigchainDB Server](dev-and-test/setup-run-node.html) * [Deploy a testing cluster on AWS](clusters-feds/aws-testing-cluster.html) * [Set up and run a federation (including production nodes)](clusters-feds/set-up-a-federation.html) diff --git a/docs/server/source/nodes/node-assumptions.md b/docs/server/source/nodes/node-assumptions.md index 46d45ca7..f7e8379f 100644 --- a/docs/server/source/nodes/node-assumptions.md +++ b/docs/server/source/nodes/node-assumptions.md @@ -8,6 +8,6 @@ We make some assumptions about production nodes: 2. Each production node in a federation's cluster is managed by a different person or team. -Because of the first assumption, we don't provide a detailed cookbook explaining how to secure a server, or other things that a sysadmin should know. (We do provide some [starter templates](../cloud-deployment-starter-templates/index.html), but those are just a starting point.) +Because of the first assumption, we don't provide a detailed cookbook explaining how to secure a server, or other things that a sysadmin should know. (We do provide some [templates](../cloud-deployment-templates/index.html), but those are just a starting point.) From 52344787de1a88c444718d66b29bea04a4b2671b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 14 Feb 2017 18:56:43 +0100 Subject: [PATCH 200/219] docs: initial steps to deploy advanced node on Azure --- .../advanced-node-on-azure.rst | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst diff --git a/docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst b/docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst new file mode 100644 index 00000000..75c92281 --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst @@ -0,0 +1,106 @@ +Advanced Node on Azure +====================== + +.. note:: + + This page is a work in progress. + +This page describes how to deploy an "advanced BigchainDB node" +on Microsoft Azure; advanced because it uses Docker containers, +multiple virtual machines, and `Kubernetes `_ +for container orchestration. + + +Step 1: Get a Pay-As-You-Go Azure Subscription +---------------------------------------------- + +Microsoft Azure has a Free Trial subscription (at the time of writing), +but it's too limited to run an advanced BigchainDB node. +Sign up for a Pay-As-You-Go Azure subscription +via `the Azure website `_. + +You may find that you have to sign up for a Free Trial subscription first. +That's okay: you can have many subscriptions. + + +Step 2: Deploy an Azure Container Service (ACS) +----------------------------------------------- + +It's *possible* to deploy an Azure Container Service (ACS) +from the `Azure Portal `_ +(i.e. online in your web browser) +but it's actually easier to do it using the Azure +Command-Line Interface (CLI). +(The Azure Portal will ask you for a public SSH key +and a "service principal," and you'll have to create those +first if they don't exist. The CLI will create them +for you if necessary.) + +Microsoft has `instructions to install the Azure CLI 2.0 +on most common operating systems +`_. +Do that. + +Next, login to your account using: + +.. code:: bash + + $ az login + +It will tell you to open a web page and to copy a code to that page. + +If the login is a success, you will see some information +about all your subscriptions, including the one that is currently +enabled (``"state": "Enabled"``). If the wrong one is enabled, +you can switch to the right one using: + +.. code:: bash + + $ az account set --subscription + +Next, you will have to pick the Azure data center location +where you'd like to deploy your cluster. +You can get a list of all available locations using: + +.. code:: bash + + $ az account list-locations + +Next, create an Azure "resource group" to contain all the +resources (virtual machines, subnets, etc.) associated +with your soon-to-be-deployed cluster. You can name it +whatever you like but avoid fancy characters because they may +confuse some software. + +.. code:: bash + + $ az group create --name --location + +Example location names are ``koreacentral`` and ``westeurope``. + +Finally, you can deploy an ACS using something like: + +.. code:: bash + + $ az acs create --name \ + --resource-group \ + --agent-count 3 \ + --agent-vm-size Standard_D2_v2 \ + --dns-prefix \ + --generate-ssh-keys \ + --location \ + --orchestrator-type kubernetes + +There are more options. For help understanding all the options, use the built-in help: + +.. code:: bash + + $ az acs create --help + +It takes a few minutes for all the resources to deploy. +You can watch the progress in the `Azure Portal +`_: +go to **Resource groups** (with the blue cube icon) +and click on the one you created +to see all the resources in it. + From 2a9042f39cb081de62a64adc904a1389095e0a6b Mon Sep 17 00:00:00 2001 From: "krish7919 (Krish)" Date: Wed, 15 Feb 2017 11:20:41 +0100 Subject: [PATCH 201/219] Typo: Removing redundant line from docs --- docs/server/source/appendices/run-with-docker.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index 7c6f9777..9b727268 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -49,9 +49,6 @@ Let's analyze that command: * `bigchaindb/bigchaindb` the image to use. All the options after the container name are passed on to the entrypoint inside the container. * `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values -* `--dev-allow-temp-keypair` specifies that this is a dev environment and - enables creation of temporary key pairs to be written to the configuration - file * `mongodb` or `rethinkdb` specifies the database backend to use with bigchaindb From 9dba09144a0b09d51d9de5ecc651be59fbdcf44e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 15 Feb 2017 11:34:40 +0100 Subject: [PATCH 202/219] Started new page: Run a BigchainDB Node in a Kubernetes Cluster --- .../cloud-deployment-templates/index.rst | 4 +++- .../node-on-kubernetes.rst | 20 +++++++++++++++++++ ...zure.rst => template-kubernetes-azure.rst} | 17 +++++++--------- 3 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst rename docs/server/source/cloud-deployment-templates/{advanced-node-on-azure.rst => template-kubernetes-azure.rst} (89%) diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 960abaa5..67a2ace4 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -13,4 +13,6 @@ If you find the cloud deployment templates for nodes helpful, then you may also template-terraform-aws template-ansible azure-quickstart-template - advanced-node-on-azure + template-kubernetes-azure + node-on-kubernetes + \ No newline at end of file diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst new file mode 100644 index 00000000..7d9a3c1d --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -0,0 +1,20 @@ +Run a BigchainDB Node in a Kubernetes Cluster +============================================= + +Assuming you already have a `Kubernetes `_ +cluster up and running, this page describes how to run a +BigchainDB node on it. + +Step 1: Install and Configure kubectl +------------------------------------- + +kubectl is the Kubernetes CLI. +See the `Kubernetes docs to install and configure kubectl +`_. + + + + +Note: The BigchainDB Dashboard can be deployed +as a Docker container, like everything else. + diff --git a/docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst b/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst similarity index 89% rename from docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst rename to docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst index 75c92281..ad4a8b04 100644 --- a/docs/server/source/cloud-deployment-templates/advanced-node-on-azure.rst +++ b/docs/server/source/cloud-deployment-templates/template-kubernetes-azure.rst @@ -1,14 +1,9 @@ -Advanced Node on Azure -====================== +Template: Deploy a Kubernetes Cluster on Azure +============================================== -.. note:: - - This page is a work in progress. - -This page describes how to deploy an "advanced BigchainDB node" -on Microsoft Azure; advanced because it uses Docker containers, -multiple virtual machines, and `Kubernetes `_ -for container orchestration. +A BigchainDB node can be run inside a `Kubernetes `_ +cluster. +This page describes one way to deploy a Kubernetes cluster on Azure. Step 1: Get a Pay-As-You-Go Azure Subscription @@ -104,3 +99,5 @@ go to **Resource groups** (with the blue cube icon) and click on the one you created to see all the resources in it. +Next, you can :doc:`run a BigchainDB node on your new +Kubernetes cluster `. \ No newline at end of file From 2b235b21aafb514855763bfd1bc40efb3ea4d23b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 15 Feb 2017 14:57:42 +0100 Subject: [PATCH 203/219] add docs on kubectl install & setup --- .../node-on-kubernetes.rst | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst index 7d9a3c1d..03ffb2fe 100644 --- a/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst +++ b/docs/server/source/cloud-deployment-templates/node-on-kubernetes.rst @@ -3,18 +3,44 @@ Run a BigchainDB Node in a Kubernetes Cluster Assuming you already have a `Kubernetes `_ cluster up and running, this page describes how to run a -BigchainDB node on it. +BigchainDB node in it. -Step 1: Install and Configure kubectl -------------------------------------- + +Step 1: Install kubectl +----------------------- kubectl is the Kubernetes CLI. -See the `Kubernetes docs to install and configure kubectl +If you don't already have it installed, +then see the `Kubernetes docs to install it `_. +Step 2: Configure kubectl +------------------------- + +The default location of the kubectl configuration file is ``~/.kube/config``. +If you don't have that file, then you need to get it. + +If you deployed your Kubernetes cluster on Azure +using the Azure CLI 2.0 (as per :doc:`our template `), +then you can get the ``~/.kube/config`` file using: + +.. code:: bash + + $ az acs kubernetes get-credentials \ + --resource-group \ + --name + + +Step 3: Run a MongoDB Container +------------------------------- + +To start a MongoDB Docker container in a pod on one of the cluster nodes: + +.. code:: bash + + $ kubectl ????? Note: The BigchainDB Dashboard can be deployed as a Docker container, like everything else. - From 955b018be8995743015f3efdfa78b55332c3fd26 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 15 Feb 2017 14:58:16 +0100 Subject: [PATCH 204/219] add stepping pipeline and fix issue #1191 --- bigchaindb/backend/mongodb/changefeed.py | 4 +- bigchaindb/core.py | 4 +- tests/pipelines/conftest.py | 10 ++ tests/pipelines/stepping.py | 138 +++++++++++++++++++++++ tests/pipelines/test_steps.py | 24 ++++ 5 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 tests/pipelines/conftest.py create mode 100644 tests/pipelines/stepping.py create mode 100644 tests/pipelines/test_steps.py diff --git a/bigchaindb/backend/mongodb/changefeed.py b/bigchaindb/backend/mongodb/changefeed.py index 4a5a5b7e..3abcbeda 100644 --- a/bigchaindb/backend/mongodb/changefeed.py +++ b/bigchaindb/backend/mongodb/changefeed.py @@ -85,11 +85,13 @@ class MongoDBChangeFeed(ChangeFeed): # document itself. So here we first read the document # and then return it. doc = self.connection.conn[dbname][table].find_one( - {'_id': record['o2']}, + {'_id': record['o2']['_id']}, {'_id': False} ) self.outqueue.put(doc) + logger.debug('Record in changefeed: %s:%s', table, record['op']) + @register_changefeed(MongoDBConnection) def get_changefeed(connection, table, operation, *, prefeed=None): diff --git a/bigchaindb/core.py b/bigchaindb/core.py index e4dddb6f..a7ed93f0 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -55,7 +55,9 @@ class Bigchain(object): self.me = public_key or bigchaindb.config['keypair']['public'] self.me_private = private_key or bigchaindb.config['keypair']['private'] self.nodes_except_me = keyring or bigchaindb.config['keyring'] - self.backlog_reassign_delay = backlog_reassign_delay or bigchaindb.config['backlog_reassign_delay'] + if backlog_reassign_delay is None: + backlog_reassign_delay = bigchaindb.config['backlog_reassign_delay'] + self.backlog_reassign_delay = backlog_reassign_delay self.consensus = BaseConsensusRules self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) if not self.me or not self.me_private: diff --git a/tests/pipelines/conftest.py b/tests/pipelines/conftest.py new file mode 100644 index 00000000..5b66f048 --- /dev/null +++ b/tests/pipelines/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from .stepping import create_stepper + + +@pytest.fixture +def steps(): + stepper = create_stepper() + with stepper.start(): + yield stepper diff --git a/tests/pipelines/stepping.py b/tests/pipelines/stepping.py new file mode 100644 index 00000000..567dc846 --- /dev/null +++ b/tests/pipelines/stepping.py @@ -0,0 +1,138 @@ +""" +Pipeline stepping is a way to advance the asynchronous data pipeline +deterministically by exposing each step separately and advancing the states +manually. +""" + + +import functools +import time +import types +import logging +from contextlib import contextmanager +from unittest.mock import patch + +import bigchaindb.core +from bigchaindb.backend.changefeed import ChangeFeed +import bigchaindb.pipelines.block +import bigchaindb.pipelines.stale + + +class MultipipesStepper: + def __init__(self): + self.queues = {} + self.tasks = {} + self.input_tasks = set() + self.processes = [] + + def add_input(self, prefix, node, next): + name = '%s_%s' % (prefix, node.name) + next_name = '%s_%s' % (prefix, next.name) + + if isinstance(node, ChangeFeed): + self.processes.append(node) + + def f(*args, **kwargs): + _kwargs = {'timeout': 0.1} + _kwargs.update(kwargs) + return node.outqueue.get(*args, **kwargs) + else: + f = node.target + + def inner(**kwargs): + r = f(**kwargs) + if r is not None: + self.enqueue(next_name, r) + + self.tasks[name] = functools.wraps(f)(inner) + self.input_tasks.add(name) + + def add_stage(self, prefix, node, next): + """ + Convert pipeline stage into task. + """ + f = node.target + name = '%s_%s' % (prefix, node.name) + if next: + next_name = '%s_%s' % (prefix, next.name) + + def inner(*args): + out = f(*args) + if out is not None and next: + self.enqueue(next_name, out) + + task = functools.wraps(f)(inner) + self.tasks[name] = task + + def enqueue(self, name, item): + queue = self.queues.setdefault(name, []) + if isinstance(item, types.GeneratorType): + queue.extend(list(item)) + else: + queue.append(item) + + def step(self, name, **kwargs): + logging.debug('Stepping %s', name) + task = self.tasks[name] + if name in self.input_tasks: + task(**kwargs) + else: + queue = self.queues.get(name, []) + if not queue: + raise Empty(name) + task(queue.pop(0), **kwargs) + logging.debug('Stepped %s', name) + + def get_counts(self): + counts = {} + for name in self.queues: + n = len(self.queues[name]) + if n: + counts[name] = n + return counts + + def __getattr__(self, name): + return lambda **kwargs: self.step(name, **kwargs) + + @contextmanager + def start(self): + for p in self.processes: + p.start() + # It would be nice to have a better way to wait for changefeeds here. + # We have to wait some amount of time because the feed setup is + # happening in a different process and won't include any writes we + # perform before it is ready. + time.sleep(0.2) + try: + yield + finally: + for p in self.processes: + p.terminate() + + +class Empty(Exception): + pass + + +def update_stepper(stepper, prefix, pipeline): + nodes = pipeline.nodes + for i in range(len(nodes)): + n0 = nodes[i] + n1 = (nodes + [None])[i+1] + f = stepper.add_input if i == 0 else stepper.add_stage + f(prefix, n0, n1) + + +def create_stepper(): + stepper = MultipipesStepper() + + with patch('bigchaindb.pipelines.block.Pipeline.start'): + pipeline = bigchaindb.pipelines.block.start() + update_stepper(stepper, 'block', pipeline) + + with patch('bigchaindb.pipelines.stale.Pipeline.start'): + pipeline = bigchaindb.pipelines.stale.start( + timeout=0, backlog_reassign_delay=0) + update_stepper(stepper, 'stale', pipeline) + + return stepper diff --git a/tests/pipelines/test_steps.py b/tests/pipelines/test_steps.py new file mode 100644 index 00000000..76862bd9 --- /dev/null +++ b/tests/pipelines/test_steps.py @@ -0,0 +1,24 @@ +import pytest +import random + + +@pytest.mark.bdb +def test_stepping_changefeed_produces_update(b, steps): + input_single_create(b) + steps.block_changefeed() + steps.block_filter_tx() + + # timeouts are 0 so will reassign immediately + steps.stale_check_transactions() + steps.stale_reassign_transactions() + + # We expect 2 changefeed events + steps.block_changefeed() + steps.block_filter_tx() + + +def input_single_create(b): + from bigchaindb.common.transaction import Transaction + metadata = {'r': random.random()} + tx = Transaction.create([b.me], [([b.me], 1)], metadata) + b.write_transaction(tx) From 7469f60d17dbc68b61bcb663645ef3a328c2f90a Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 15 Feb 2017 15:15:20 +0100 Subject: [PATCH 205/219] more assertions in test --- tests/pipelines/test_steps.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/pipelines/test_steps.py b/tests/pipelines/test_steps.py index 76862bd9..287a17c4 100644 --- a/tests/pipelines/test_steps.py +++ b/tests/pipelines/test_steps.py @@ -4,9 +4,7 @@ import random @pytest.mark.bdb def test_stepping_changefeed_produces_update(b, steps): - input_single_create(b) - steps.block_changefeed() - steps.block_filter_tx() + tx = input_single_create(b) # timeouts are 0 so will reassign immediately steps.stale_check_transactions() @@ -14,7 +12,11 @@ def test_stepping_changefeed_produces_update(b, steps): # We expect 2 changefeed events steps.block_changefeed() - steps.block_filter_tx() + steps.block_changefeed() + + assert steps.get_counts() == {'block_filter_tx': 2} + assert ([tx['id'] for tx in steps.queues['block_filter_tx']] == + [tx.id, tx.id]) def input_single_create(b): @@ -22,3 +24,4 @@ def input_single_create(b): metadata = {'r': random.random()} tx = Transaction.create([b.me], [([b.me], 1)], metadata) b.write_transaction(tx) + return tx From fc04cd7bcd94a3b34801a8433c57655bf85c4f71 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 15 Feb 2017 17:31:36 +0100 Subject: [PATCH 206/219] update changefeed test for update --- tests/backend/mongodb/test_changefeed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/backend/mongodb/test_changefeed.py b/tests/backend/mongodb/test_changefeed.py index 67b54cd8..00dcaca5 100644 --- a/tests/backend/mongodb/test_changefeed.py +++ b/tests/backend/mongodb/test_changefeed.py @@ -19,7 +19,7 @@ def mock_changefeed_data(): { 'op': 'u', 'o': {'msg': 'seems like we have an update here'}, - 'o2': 'some-id' + 'o2': {'_id': 'some-id'} }, ] From f6ebba8389a40f9458a27f8b5a1a9ef8d47f1524 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Thu, 16 Feb 2017 09:52:00 +0100 Subject: [PATCH 207/219] docs: clarified how block id & signature are calculated --- docs/server/source/data-models/block-model.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index e2e3b418..889dc50f 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -17,7 +17,7 @@ A block has the following structure: } -- ``id``: The hash of the serialized ``block`` (i.e. the ``timestamp``, ``transactions``, ``node_pubkey``, and ``voters``). This is also a database primary key; that's how we ensure that all blocks are unique. +- ``id``: The :ref:`hash ` of the serialized inner ``block`` (i.e. the ``timestamp``, ``transactions``, ``node_pubkey``, and ``voters``). It's used as a unique index in the database backend (e.g. RethinkDB or MongoDB). - ``block``: - ``timestamp``: The Unix time when the block was created. It's provided by the node that created the block. @@ -27,7 +27,7 @@ A block has the following structure: It's the list of federation nodes which can cast a vote on this block. This list can change from block to block, as nodes join and leave the federation. -- ``signature``: Cryptographic signature of the block by the node that created the block. (To create the signature, the node serializes the block contents and signs it with its private key.) +- ``signature``: :ref:`Cryptographic signature ` of the block by the node that created the block. To generate the signature, the node builds a dict including the ``id``, the inner ``block`` & the ``signature`` (with a value of ``None``), it serializes that dict, and then signs *that* with its private key. Working with Blocks From 00785e5f02d21d8e2eb226d21f08d108c4f94392 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Tue, 24 Jan 2017 19:02:37 +0100 Subject: [PATCH 208/219] Add WebSocket Event Stream API spec --- .../http-client-server-api.rst | 3 + docs/server/source/drivers-clients/index.rst | 1 + .../websocket-event-stream-api.rst | 114 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 docs/server/source/drivers-clients/websocket-event-stream-api.rst diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 5444be8f..26ccd2f5 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -10,6 +10,7 @@ If you set up a BigchainDB node or reverse proxy yourself, and you're not sure what the API Root URL is, then see the last section of this page for help. +.. _bigchaindb-root-url: BigchainDB Root URL ------------------- @@ -393,6 +394,8 @@ Votes :statuscode 400: The request wasn't understood by the server, e.g. just requesting ``/votes``, without defining ``block_id``. +.. _determining-the-api-root-url: + Determining the API Root URL ---------------------------- diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 18894f60..704832c0 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -15,6 +15,7 @@ community projects listed below. :maxdepth: 1 http-client-server-api + websocket-event-stream-api The Python Driver Transaction CLI diff --git a/docs/server/source/drivers-clients/websocket-event-stream-api.rst b/docs/server/source/drivers-clients/websocket-event-stream-api.rst new file mode 100644 index 00000000..f10575a9 --- /dev/null +++ b/docs/server/source/drivers-clients/websocket-event-stream-api.rst @@ -0,0 +1,114 @@ +The WebSocket Event Stream API +============================== + +.. important:: + This is currently scheduled to be implemented in BigchainDB Server 0.10. + +BigchainDB provides real-time event streams over the WebSocket protocol with +the Event Stream API. + +Connecting to an event stream from your application enables a BigchainDB node +to notify you as events are processed, such as new `validated transactions <#valid-transactions>`_. + + +Demoing the API +--------------- + +You may be interested in demoing the Event Stream API with the `WebSocket echo test `_ +to familiarize yourself before attempting an integration. + + +Determining Support for the Event Stream API +-------------------------------------------- + +In practice, it's a good idea to make sure that the node you're connecting with +has advertised support for the Event Stream API. To do so, send a HTTP GET +request to the node's :ref:`Root URL ` and check that the +response contains a ``streams_`` property in ``_links``:: + + { + "_links": { + "streams_v1": "ws://example.com:9984/api/v1/streams/" + } + } + + +Connection Keep Alive +~~~~~~~~~~~~~~~~~~~~~ + +The Event Stream API requires clients to signal that they'd like their +connection to stay open by sending "pings" across the open connection. +BigchainDB nodes will automatically close any connections that haven't sent a +ping in the last three minutes. + +.. note:: + + While three minutes is the limit before a BigchainDB server will terminate + a connection, we suggest sending a ping every 30 seconds for better + reliability. + +A "ping" consists of a message containing only the string ``"ping"``, for example +in JavaScript: + +.. code-block:: javascript + + new WebSocket("...").send("ping") + +If the BigchainDB node received the ping, it'll respond back with a message +containing only the string ``"pong"``. + + +Streams +------- + +Each stream is meant as a unidirectional communication channel, where the +BigchainDB node is the only party sending messages (except for `keep-alive +pings <#connection-keep-alive>`_). Any messages sent to the BigchainDB node +(except the keep-alive pings) will be ignored. + +Streams will always be under the WebSocket protocol (so ``ws://`` or +``wss://``) and accessible as extensions to the ``/api/v/streams/`` +API root URL (for example, `validated transactions <#valid-transactions>`_ +would be accessible under ``/api/v1/streams/valid_tx``). If you're running your +own BigchainDB instance and need help determining its root URL, you can find +more :ref:`here `. + +All messages sent in a stream are in the JSON format. + +.. note:: + + For simplicity, BigchainDB initially only provides a stream for all + validated transactions. In the future, we may provide streams for other + information, such as new blocks, new votes, or invalid transactions. We may + also provide the ability to filter the stream for specific qualities, such + as a specific ``output``'s ``public_key``. + + If you have specific use cases that you think would fit as part of this + API, feel free to reach out via `gitter `_ + or `email `_. + +Valid Transactions +~~~~~~~~~~~~~~~~~~ + +``/valid_tx`` + +Streams an event for any newly validated transactions. Message bodies contain +the transaction's ID, associated asset ID, and containing block's ID. + +Example message:: + + { + "txid": "", + "assetid": "", + "blockid": "" + } + + +.. note:: + + Transactions in BigchainDB are validated in batches ("blocks") and will, + therefore, be streamed in batches. Each block can contain up to a 1000 + transactions, ordered by the time at which they were included in the block. + The ``/valid_tx`` stream will send these transactions in the same order + that the block stored them in, but this does **NOT** guarantee that you + will recieve the events in that same order. From 9307c21baa1677ee0a06a2ba3d283fde83bcd3c3 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Thu, 16 Feb 2017 15:26:17 +0100 Subject: [PATCH 209/219] Remove connect keep alive protocol from Event Stream API --- .../websocket-event-stream-api.rst | 30 ++++--------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/docs/server/source/drivers-clients/websocket-event-stream-api.rst b/docs/server/source/drivers-clients/websocket-event-stream-api.rst index f10575a9..22effbc1 100644 --- a/docs/server/source/drivers-clients/websocket-event-stream-api.rst +++ b/docs/server/source/drivers-clients/websocket-event-stream-api.rst @@ -36,35 +36,17 @@ response contains a ``streams_`` property in ``_links``:: Connection Keep Alive ~~~~~~~~~~~~~~~~~~~~~ -The Event Stream API requires clients to signal that they'd like their -connection to stay open by sending "pings" across the open connection. -BigchainDB nodes will automatically close any connections that haven't sent a -ping in the last three minutes. - -.. note:: - - While three minutes is the limit before a BigchainDB server will terminate - a connection, we suggest sending a ping every 30 seconds for better - reliability. - -A "ping" consists of a message containing only the string ``"ping"``, for example -in JavaScript: - -.. code-block:: javascript - - new WebSocket("...").send("ping") - -If the BigchainDB node received the ping, it'll respond back with a message -containing only the string ``"pong"``. - +The Event Stream API initially does not provide any mechanisms for connection +keep alive other than enabling TCP keepalive on each open WebSocket connection. +In the future, we may add additional functionality to handle ping/pong frames +or payloads designed for keep alive. Streams ------- Each stream is meant as a unidirectional communication channel, where the -BigchainDB node is the only party sending messages (except for `keep-alive -pings <#connection-keep-alive>`_). Any messages sent to the BigchainDB node -(except the keep-alive pings) will be ignored. +BigchainDB node is the only party sending messages. Any messages sent to the +BigchainDB node will be ignored. Streams will always be under the WebSocket protocol (so ``ws://`` or ``wss://``) and accessible as extensions to the ``/api/v/streams/`` From 8ec2c6bc3423e29c22318cd728241e9aea28cf3d Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 17 Feb 2017 12:15:50 +0100 Subject: [PATCH 210/219] Docs: fixed errors in some documented config settings defaults --- .../source/server-reference/configuration.md | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index fb8d8a6a..95db54dd 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -88,32 +88,31 @@ export BIGCHAINDB_DATABASE_NAME=bigchain export BIGCHAINDB_DATABASE_REPLICASET=bigchain-rs ``` -**Example config file snippet** +Note: If the backend database is RethinkDB, you *can* set `BIGCHAINDB_DATABASE_REPLICASET` but it won't be used for anything. + +**Default values** + +If (no environment variables were set and there's no local config file), or you used `bigchaindb -y configure rethinkdb` to create a default local config file for a RethinkDB backend, then the defaults will be: ```js "database": { "backend": "rethinkdb", "host": "localhost", - "port": 28015, "name": "bigchain", + "port": 28015 +} +``` + +If you used `bigchaindb -y configure mongodb` to create a default local config file for a MongoDB backend, then the defaults will be: +```js +"database": { + "backend": "mongodb", + "host": "localhost", + "name": "bigchain", + "port": 27017, "replicaset": "bigchain-rs" } ``` -**Default values (a snippet from `bigchaindb/__init__.py`)** -```python -'database': { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs') -} -``` - -**Note**: We are currently adding support for MongoDB. The `replicaset` and -`BIGCHAINDB_DATABASE_REPLICASET` option is only used if the `backend` or -`BIGCHAINDB_DATABASE_BACKEND` is set to `"mongodb"`. - ## server.bind, server.workers & server.threads From be72875330fbd10b99b1c7d68c9dd1578f961626 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sun, 19 Feb 2017 15:27:21 +0100 Subject: [PATCH 211/219] docs: added some clarification to the replica set setting --- docs/server/source/server-reference/configuration.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 95db54dd..29431e82 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -77,19 +77,17 @@ Note how the keys in the list are separated by colons. ## database.backend, database.host, database.port, database.name & database.replicaset -The database backend to use (e.g. RethinkDB) and its hostname, port and name. +The database backend to use (`rethinkdb` or `mongodb`) and its hostname, port and name. If the database backend is `mongodb`, then there's a fifth setting: the name of the replica set. If the database backend is `rethinkdb`, you *can* set the name of the replica set, but it won't be used for anything. **Example using environment variables** ```text -export BIGCHAINDB_DATABASE_BACKEND=rethinkdb +export BIGCHAINDB_DATABASE_BACKEND=mongodb export BIGCHAINDB_DATABASE_HOST=localhost -export BIGCHAINDB_DATABASE_PORT=28015 +export BIGCHAINDB_DATABASE_PORT=27017 export BIGCHAINDB_DATABASE_NAME=bigchain export BIGCHAINDB_DATABASE_REPLICASET=bigchain-rs ``` -Note: If the backend database is RethinkDB, you *can* set `BIGCHAINDB_DATABASE_REPLICASET` but it won't be used for anything. - **Default values** If (no environment variables were set and there's no local config file), or you used `bigchaindb -y configure rethinkdb` to create a default local config file for a RethinkDB backend, then the defaults will be: From 96e760046f88e24025a3bb628abfaf439c3d5361 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 20 Feb 2017 14:15:18 +0100 Subject: [PATCH 212/219] docs: better links at end of Quickstart page --- docs/server/source/quickstart.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 3c6a78f3..0a1938a2 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -55,4 +55,22 @@ $ bigchaindb start That's it! -Next Steps: You could build valid transactions and push them to your running BigchainDB Server using the [BigchaindB Python Driver](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html). +Next Steps: You could... + +* [install the BigchainDB Python Driver](https://docs.bigchaindb.com/projects/py-driver/en/latest/quickstart.html) and +* [use the BigchainDB Python Driver to build a valid transaction, and post that transaction to your running server](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html). + +
+ +
+
+
+
+
+
+
+
+
+
+
+
From 5e8cec65cdee82c2b491eb38a13b0d9b113191c7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 20 Feb 2017 17:52:48 +0100 Subject: [PATCH 213/219] docs: Simplified Quickstart page even more --- docs/server/source/quickstart.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 0a1938a2..9ef6702d 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -4,7 +4,6 @@ This page has instructions to set up a single stand-alone BigchainDB node for le A. Install the database backend. -[Install RethinkDB Server](https://rethinkdb.com/docs/install/ubuntu/) or [Install MongoDB Server 3.4+](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) B. Run the database backend. Open a Terminal and run the command: @@ -53,12 +52,10 @@ G. Run the BigchainDB Server: $ bigchaindb start ``` -That's it! +You now have a running BigchainDB Server and can post transactions to it. +One way to do that is to use the BigchainDB Python Driver. -Next Steps: You could... - -* [install the BigchainDB Python Driver](https://docs.bigchaindb.com/projects/py-driver/en/latest/quickstart.html) and -* [use the BigchainDB Python Driver to build a valid transaction, and post that transaction to your running server](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html). +[Install the BigchainDB Python Driver (link)](https://docs.bigchaindb.com/projects/py-driver/en/latest/quickstart.html)
From 34286467b7cc0f8a7a684ca70ac9023be75e2904 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 20 Feb 2017 18:04:47 +0100 Subject: [PATCH 214/219] docs: removed all RethinkDB stuff from Quickstart page --- docs/server/source/quickstart.md | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 9ef6702d..2eae07f0 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -2,18 +2,11 @@ This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. If you're not using Linux, then you might try [running BigchainDB with Docker](appendices/run-with-docker.html). -A. Install the database backend. +A. Install MongoDB as the database backend. (There are other options but you can ignore them for now.) [Install MongoDB Server 3.4+](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) -B. Run the database backend. Open a Terminal and run the command: - -with RethinkDB -```text -$ rethinkdb -``` - -with MongoDB __3.4+__ +B. Run MongoDB. Open a Terminal and run the command: ```text $ mongod --replSet=bigchain-rs ``` @@ -35,19 +28,12 @@ E. Install the `bigchaindb` Python package from PyPI: $ sudo pip3 install bigchaindb ``` -F. Configure the BigchainDB Server: and run BigchainDB Server: - -with RethinkDB -```text -$ bigchaindb -y configure rethinkdb -``` - -with MongoDB +F. Configure BigchainDB Server: ```text $ bigchaindb -y configure mongodb ``` -G. Run the BigchainDB Server: +G. Run BigchainDB Server: ```text $ bigchaindb start ``` From 6110693ae847d83c0f1306d6fa9484e118a10f26 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 21 Feb 2017 13:04:43 +0100 Subject: [PATCH 215/219] provide more documentation for MultipipesStepper --- tests/pipelines/stepping.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/pipelines/stepping.py b/tests/pipelines/stepping.py index 567dc846..bb70add0 100644 --- a/tests/pipelines/stepping.py +++ b/tests/pipelines/stepping.py @@ -2,6 +2,33 @@ Pipeline stepping is a way to advance the asynchronous data pipeline deterministically by exposing each step separately and advancing the states manually. + +The multipipes.Pipeline class implements a pipeline that advanced +asynchronously and concurrently. This module provides an interface to the +BigchainDB pipelines that is static, ie, does not advance without prompting. + +Rather than having a pipeline that is in an all or nothing running / not running +state, one can do the following: + + +steps = create_stepper() + +with steps.start(): + tx = my_create_and_write_tx() + steps.block_changefeed(timeout=1) + steps.block_filter_tx() + steps.block_validate_tx() + steps.block_create(timeout=True) + +assert steps.counts == {'block_write': 1} + +Pending items are held in the `.queues` attribute, and every task has it's own +queue (as in multipipes.Pipeline). Queues are just lists though so they can +be easily inspected. + +As a shortcut, the `.counts` attribute is provided which returns the number of +pending items for each task. This is useful to assert the expected status of +the queues after performing some sequence. """ From a23a741253d74f07f51b38ebc5d25e38e0eca9d1 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 21 Feb 2017 13:13:40 +0100 Subject: [PATCH 216/219] document MultipipesStepper --- tests/pipelines/stepping.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/pipelines/stepping.py b/tests/pipelines/stepping.py index bb70add0..5f099750 100644 --- a/tests/pipelines/stepping.py +++ b/tests/pipelines/stepping.py @@ -53,6 +53,7 @@ class MultipipesStepper: self.processes = [] def add_input(self, prefix, node, next): + """ Add an input task; Reads from the outqueue of the Node """ name = '%s_%s' % (prefix, node.name) next_name = '%s_%s' % (prefix, next.name) @@ -76,7 +77,8 @@ class MultipipesStepper: def add_stage(self, prefix, node, next): """ - Convert pipeline stage into task. + Add a stage task, popping from own queue and appending to the queue + of the next node """ f = node.target name = '%s_%s' % (prefix, node.name) @@ -91,7 +93,8 @@ class MultipipesStepper: task = functools.wraps(f)(inner) self.tasks[name] = task - def enqueue(self, name, item): + def _enqueue(self, name, item): + """ internal function; add item(s) to queue) """ queue = self.queues.setdefault(name, []) if isinstance(item, types.GeneratorType): queue.extend(list(item)) @@ -99,6 +102,7 @@ class MultipipesStepper: queue.append(item) def step(self, name, **kwargs): + """ Advance pipeline stage. Throws Empty if no data to consume. """ logging.debug('Stepping %s', name) task = self.tasks[name] if name in self.input_tasks: @@ -111,6 +115,7 @@ class MultipipesStepper: logging.debug('Stepped %s', name) def get_counts(self): + """ Get sizes of non empty queues """ counts = {} for name in self.queues: n = len(self.queues[name]) @@ -119,10 +124,12 @@ class MultipipesStepper: return counts def __getattr__(self, name): + """ Shortcut to get a queue """ return lambda **kwargs: self.step(name, **kwargs) @contextmanager def start(self): + """ Start async inputs; changefeeds etc """ for p in self.processes: p.start() # It would be nice to have a better way to wait for changefeeds here. @@ -141,7 +148,7 @@ class Empty(Exception): pass -def update_stepper(stepper, prefix, pipeline): +def _update_stepper(stepper, prefix, pipeline): nodes = pipeline.nodes for i in range(len(nodes)): n0 = nodes[i] @@ -155,11 +162,11 @@ def create_stepper(): with patch('bigchaindb.pipelines.block.Pipeline.start'): pipeline = bigchaindb.pipelines.block.start() - update_stepper(stepper, 'block', pipeline) + _update_stepper(stepper, 'block', pipeline) with patch('bigchaindb.pipelines.stale.Pipeline.start'): pipeline = bigchaindb.pipelines.stale.start( timeout=0, backlog_reassign_delay=0) - update_stepper(stepper, 'stale', pipeline) + _update_stepper(stepper, 'stale', pipeline) return stepper From 37f5298962fcc7ade4893489921f92400cf77db0 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 21 Feb 2017 13:51:09 +0100 Subject: [PATCH 217/219] pull stepper changes from no-double-inclusion --- tests/pipelines/stepping.py | 24 +++++++++++++++++------- tests/pipelines/test_steps.py | 5 +++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/pipelines/stepping.py b/tests/pipelines/stepping.py index 5f099750..0e286829 100644 --- a/tests/pipelines/stepping.py +++ b/tests/pipelines/stepping.py @@ -43,6 +43,7 @@ import bigchaindb.core from bigchaindb.backend.changefeed import ChangeFeed import bigchaindb.pipelines.block import bigchaindb.pipelines.stale +import bigchaindb.pipelines.vote class MultipipesStepper: @@ -70,7 +71,7 @@ class MultipipesStepper: def inner(**kwargs): r = f(**kwargs) if r is not None: - self.enqueue(next_name, r) + self._enqueue(next_name, r) self.tasks[name] = functools.wraps(f)(inner) self.input_tasks.add(name) @@ -85,10 +86,10 @@ class MultipipesStepper: if next: next_name = '%s_%s' % (prefix, next.name) - def inner(*args): - out = f(*args) + def inner(*args, **kwargs): + out = f(*args, **kwargs) if out is not None and next: - self.enqueue(next_name, out) + self._enqueue(next_name, out) task = functools.wraps(f)(inner) self.tasks[name] = task @@ -97,8 +98,12 @@ class MultipipesStepper: """ internal function; add item(s) to queue) """ queue = self.queues.setdefault(name, []) if isinstance(item, types.GeneratorType): - queue.extend(list(item)) + items = list(item) else: + items = [item] + for item in items: + if type(item) != tuple: + item = (item,) queue.append(item) def step(self, name, **kwargs): @@ -111,10 +116,11 @@ class MultipipesStepper: queue = self.queues.get(name, []) if not queue: raise Empty(name) - task(queue.pop(0), **kwargs) + task(*queue.pop(0), **kwargs) logging.debug('Stepped %s', name) - def get_counts(self): + @property + def counts(self): """ Get sizes of non empty queues """ counts = {} for name in self.queues: @@ -169,4 +175,8 @@ def create_stepper(): timeout=0, backlog_reassign_delay=0) _update_stepper(stepper, 'stale', pipeline) + with patch('bigchaindb.pipelines.vote.Pipeline.start'): + pipeline = bigchaindb.pipelines.vote.start() + _update_stepper(stepper, 'vote', pipeline) + return stepper diff --git a/tests/pipelines/test_steps.py b/tests/pipelines/test_steps.py index 287a17c4..c63a673a 100644 --- a/tests/pipelines/test_steps.py +++ b/tests/pipelines/test_steps.py @@ -3,6 +3,7 @@ import random @pytest.mark.bdb +@pytest.mark.genesis def test_stepping_changefeed_produces_update(b, steps): tx = input_single_create(b) @@ -14,8 +15,8 @@ def test_stepping_changefeed_produces_update(b, steps): steps.block_changefeed() steps.block_changefeed() - assert steps.get_counts() == {'block_filter_tx': 2} - assert ([tx['id'] for tx in steps.queues['block_filter_tx']] == + assert steps.counts == {'block_filter_tx': 2} + assert ([tx['id'] for (tx,) in steps.queues['block_filter_tx']] == [tx.id, tx.id]) From 87f677d76e2e21b1295d83a6c7ece276938dfa0b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 21 Feb 2017 14:20:14 +0100 Subject: [PATCH 218/219] docs: correction to explanation of block signature calc. --- docs/server/source/data-models/block-model.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/source/data-models/block-model.rst b/docs/server/source/data-models/block-model.rst index 889dc50f..3c94fca1 100644 --- a/docs/server/source/data-models/block-model.rst +++ b/docs/server/source/data-models/block-model.rst @@ -27,7 +27,7 @@ A block has the following structure: It's the list of federation nodes which can cast a vote on this block. This list can change from block to block, as nodes join and leave the federation. -- ``signature``: :ref:`Cryptographic signature ` of the block by the node that created the block. To generate the signature, the node builds a dict including the ``id``, the inner ``block`` & the ``signature`` (with a value of ``None``), it serializes that dict, and then signs *that* with its private key. +- ``signature``: :ref:`Cryptographic signature ` of the block by the node that created the block (i.e. the node with public key ``node_pubkey``). To generate the signature, the node signs the serialized inner ``block`` (the same thing that was hashed to determine the ``id``) using the private key corresponding to ``node_pubkey``. Working with Blocks From 306053af9165bf5bd572717e29948a3050933c46 Mon Sep 17 00:00:00 2001 From: Krish Date: Wed, 22 Feb 2017 10:56:29 +0100 Subject: [PATCH 219/219] Adding a line to modify the configuration file (#1205) * Adding a line to modify the configuration file * Changing the access URL for rethinkdb * Typo fix and with clarification * Changes as per @ttmc's comments --- docs/server/source/appendices/run-with-docker.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md index 9b727268..6c1d2ce0 100644 --- a/docs/server/source/appendices/run-with-docker.md +++ b/docs/server/source/appendices/run-with-docker.md @@ -51,6 +51,11 @@ Let's analyze that command: command) inside the container, with the `-y` option to automatically use all the default config values * `mongodb` or `rethinkdb` specifies the database backend to use with bigchaindb +To ensure that BigchainDB connects to the backend database bound to the virtual +interface `172.17.0.1`, you must edit the BigchainDB configuration file +(`~/bigchaindb_docker/.bigchaindb`) and change database.host from `localhost` +to `172.17.0.1`. + ### Run the backend database From v0.9 onwards, you can run either RethinkDB or MongoDB. @@ -75,7 +80,7 @@ docker run \ You can also access the RethinkDB dashboard at -[http://localhost:58080/](http://localhost:58080/) +[http://172.17.0.1:58080/](http://172.17.0.1:58080/) #### For MongoDB