From 0a428442de0b8f062c45bcb49698796cdcf635d5 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 7 Apr 2016 11:22:44 +0200 Subject: [PATCH 01/30] Added script to release allocated-but-unassociated elastic IPs on AWS --- deploy-cluster-aws/release_eips.py | 51 ++++++++++++++++++++++++++++++ docs/source/deploy-on-aws.md | 12 +++++-- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 deploy-cluster-aws/release_eips.py diff --git a/deploy-cluster-aws/release_eips.py b/deploy-cluster-aws/release_eips.py new file mode 100644 index 00000000..a936ba6f --- /dev/null +++ b/deploy-cluster-aws/release_eips.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +"""Release all allocated but non-associated elastic IP addresses +(EIPs). Why? From the AWS docs: + +``To ensure efficient use of Elastic IP addresses, we impose a small +hourly charge if an Elastic IP address is not associated with a +running instance, or if it is associated with a stopped instance or +an unattached network interface. While your instance is running, +you are not charged for one Elastic IP address associated with the +instance, but you are charged for any additional Elastic IP +addresses associated with the instance. For more information, see +Amazon EC2 Pricing.'' + +Source: http://tinyurl.com/ozhxatx +""" + +from __future__ import unicode_literals +import botocore +import boto3 +from awscommon import get_naeips + +# Get an AWS EC2 "resource" +# See http://boto3.readthedocs.org/en/latest/guide/resources.html +ec2 = boto3.resource(service_name='ec2') + +# Create a client from the EC2 resource +# See http://boto3.readthedocs.org/en/latest/guide/clients.html +client = ec2.meta.client + +non_associated_eips = get_naeips(client) + +print('You have {} allocated elactic IPs which are ' + 'not associated with instances'. + format(len(non_associated_eips))) + +for i, eip in enumerate(non_associated_eips): + public_ip = eip['PublicIp'] + print('{}: Releasing {}'.format(i, public_ip)) + domain = eip['Domain'] + print('(It has Domain = {}.)'.format(domain)) + try: + if domain == 'vpc': + client.release_address(AllocationId=eip['AllocationId']) + else: + client.release_address(PublicIp=public_ip) + except botocore.exceptions.ClientError as e: + print('A boto error occurred:') + raise + except: + print('An unexpected error occurred:') + raise diff --git a/docs/source/deploy-on-aws.md b/docs/source/deploy-on-aws.md index 2e4a18e2..ebd73b4f 100644 --- a/docs/source/deploy-on-aws.md +++ b/docs/source/deploy-on-aws.md @@ -122,9 +122,17 @@ bigchaindb --help bigchaindb show-config ``` -There are fees associated with running instances on EC2, so if you're not using them, you should terminate them. You can do that from the AWS EC2 Console. +There are fees associated with running instances on EC2, so if you're not using them, you should terminate them. You can do that using the AWS EC2 Console. -The same is true of your allocated elastic IP addresses. There's a small fee to keep them allocated if they're not associated with a running instance. You can release them from the AWS EC2 Console. +The same is true of your allocated elastic IP addresses. There's a small fee to keep them allocated if they're not associated with a running instance. You can release them using the AWS EC2 Console, or by using a handy little script named `release_eips.py`. For example: +```text +$ python release_eips.py +You have 2 allocated elactic IPs which are not associated with instances +0: Releasing 52.58.110.110 +(It has Domain = vpc.) +1: Releasing 52.58.107.211 +(It has Domain = vpc.) +``` ## Known Deployment Issues From c168c5a5a15a42de969598f8f260d768e725ade9 Mon Sep 17 00:00:00 2001 From: troymc Date: Wed, 13 Apr 2016 11:04:31 +0200 Subject: [PATCH 02/30] default config file is now .bigchaindb --- bigchaindb/config_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 53eb7954..694f375f 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -1,8 +1,8 @@ -"""Utils to configure Bigchain. +"""Utils to configure BigchainDB. By calling `file_config`, the global configuration (stored in -`bigchain.config`) will be updated with the values contained in the -configuration file. +`$HOME/.bigchaindb`) will be updated with the values contained +in the configuration file. Note that there is a precedence in reading configuration values: - local config file; From 5fb573f0ca42a7af6747785295a14db5f7fbb57b Mon Sep 17 00:00:00 2001 From: troymc Date: Wed, 13 Apr 2016 16:12:31 +0200 Subject: [PATCH 03/30] Removed old try..except from release_eips.py --- deploy-cluster-aws/release_eips.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/deploy-cluster-aws/release_eips.py b/deploy-cluster-aws/release_eips.py index a936ba6f..a3e1f855 100644 --- a/deploy-cluster-aws/release_eips.py +++ b/deploy-cluster-aws/release_eips.py @@ -15,7 +15,6 @@ Source: http://tinyurl.com/ozhxatx """ from __future__ import unicode_literals -import botocore import boto3 from awscommon import get_naeips @@ -38,14 +37,7 @@ for i, eip in enumerate(non_associated_eips): print('{}: Releasing {}'.format(i, public_ip)) domain = eip['Domain'] print('(It has Domain = {}.)'.format(domain)) - try: - if domain == 'vpc': - client.release_address(AllocationId=eip['AllocationId']) - else: - client.release_address(PublicIp=public_ip) - except botocore.exceptions.ClientError as e: - print('A boto error occurred:') - raise - except: - print('An unexpected error occurred:') - raise + if domain == 'vpc': + client.release_address(AllocationId=eip['AllocationId']) + else: + client.release_address(PublicIp=public_ip) From 29f3327271fd25938830df1c07c88d44b3bd57b5 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 14 Apr 2016 09:25:23 +0200 Subject: [PATCH 04/30] Add docstring to update() in config_utils.py --- bigchaindb/config_utils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 694f375f..e7769dfa 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -54,7 +54,20 @@ def map_leafs(func, mapping): # Thanks Alex <3 # http://stackoverflow.com/a/3233356/597097 def update(d, u): - """Recursively update a mapping.""" + """Recursively update a mapping (i.e. a dict, list, set, or tuple). + + Conceptually, d and u are two sets trees (with nodes and edges). + This function goes through all the nodes of u. For each node in u, + if d doesn't have that node yet, then this function adds the node from u, + otherwise this function overwrites the node already in d with u's node. + + Args: + d (mapping): The mapping to overwrite and add to. + u (mapping): The mapping to read for changes. + + Returns: + mapping: An updated version of d (updated by u). + """ for k, v in u.items(): if isinstance(v, collections.Mapping): r = update(d.get(k, {}), v) From d0770da51d032efd6d26d686aec587d6ff62a755 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 14 Apr 2016 09:56:59 +0200 Subject: [PATCH 05/30] Edited the dict_config() docstring --- bigchaindb/config_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index e7769dfa..d5c19ea6 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -159,16 +159,20 @@ def update_types(config, reference, list_sep=':'): def dict_config(config): - """Merge the provided configuration with the default one. + """Set bigchaindb.config equal to the default config dict, + then update that with whatever is in the provided config dict, + and then set bigchaindb.config['CONFIGURED'] = True Args: - newconfig (dict): a dictionary with the configuration to load. + config (dict): the config dict to read for changes + to the default config Note: - The function merges ``newconfig`` with the **default configuration**, so any - update made to ``bigchaindb.config`` will be lost. + Any previous changes made to ``bigchaindb.config`` will be lost. """ + # Deep copy the default config into bigchaindb.config bigchaindb.config = copy.deepcopy(bigchaindb._config) + # Update the default config with whatever is in the passed config update(bigchaindb.config, update_types(config, bigchaindb.config)) bigchaindb.config['CONFIGURED'] = True From 53196c5b8c677a348e1b493a710be50913e68ac2 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 14 Apr 2016 10:55:07 +0200 Subject: [PATCH 06/30] Renamed dict_config() to set_config() --- bigchaindb/config_utils.py | 5 ++--- tests/conftest.py | 2 +- tests/db/conftest.py | 2 +- tests/test_commands.py | 2 +- tests/utils/test_config_utils.py | 2 +- tests/web/conftest.py | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index d5c19ea6..cd24081e 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -158,7 +158,7 @@ def update_types(config, reference, list_sep=':'): return map_leafs(_update_type, config) -def dict_config(config): +def set_config(config): """Set bigchaindb.config equal to the default config dict, then update that with whatever is in the provided config dict, and then set bigchaindb.config['CONFIGURED'] = True @@ -210,8 +210,7 @@ def autoconfigure(filename=None, config=None, force=False): if config: newconfig = update(newconfig, config) - dict_config(newconfig) - return newconfig + return set_config(newconfig) def load_consensus_plugin(name=None): diff --git a/tests/conftest.py b/tests/conftest.py index 44de6316..325b155b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ def ignore_local_config_file(monkeypatch): @pytest.fixture(scope='function', autouse=True) def restore_config(request, node_config): from bigchaindb import config_utils - config_utils.dict_config(node_config) + config_utils.set_config(node_config) @pytest.fixture(scope='module') diff --git a/tests/db/conftest.py b/tests/db/conftest.py index d79ee846..2233d569 100644 --- a/tests/db/conftest.py +++ b/tests/db/conftest.py @@ -17,7 +17,7 @@ from bigchaindb.db import get_conn @pytest.fixture(autouse=True) def restore_config(request, node_config): from bigchaindb import config_utils - config_utils.dict_config(node_config) + config_utils.set_config(node_config) @pytest.fixture(scope='module', autouse=True) diff --git a/tests/test_commands.py b/tests/test_commands.py index c289f279..8f02c541 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -82,7 +82,7 @@ def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_p value['return'] = newconfig monkeypatch.setattr(config_utils, 'write_config', mock_write_config) - monkeypatch.setattr(config_utils, 'file_config', lambda x: config_utils.dict_config(expected_config)) + monkeypatch.setattr(config_utils, 'file_config', lambda x: config_utils.set_config(expected_config)) monkeypatch.setattr('os.path.exists', lambda path: False) args = Namespace(config=None, yes=True) diff --git a/tests/utils/test_config_utils.py b/tests/utils/test_config_utils.py index cec7743f..cb5ed2ee 100644 --- a/tests/utils/test_config_utils.py +++ b/tests/utils/test_config_utils.py @@ -18,7 +18,7 @@ def test_bigchain_instance_is_initialized_when_conf_provided(): from bigchaindb import config_utils assert 'CONFIGURED' not in bigchaindb.config - config_utils.dict_config({'keypair': {'public': 'a', 'private': 'b'}}) + config_utils.set_config({'keypair': {'public': 'a', 'private': 'b'}}) assert bigchaindb.config['CONFIGURED'] is True b = bigchaindb.Bigchain() diff --git a/tests/web/conftest.py b/tests/web/conftest.py index 099f2fd3..fbfc7d50 100644 --- a/tests/web/conftest.py +++ b/tests/web/conftest.py @@ -5,7 +5,7 @@ from ..db import conftest @pytest.fixture(autouse=True) def restore_config(request, node_config): from bigchaindb import config_utils - config_utils.dict_config(node_config) + config_utils.set_config(node_config) @pytest.fixture(scope='module', autouse=True) From 3d504df4de3232ad4d63e4d542695cd520d7abe0 Mon Sep 17 00:00:00 2001 From: vrde Date: Thu, 14 Apr 2016 18:55:09 +0200 Subject: [PATCH 07/30] WIP for connection pool --- bigchaindb/util.py | 28 ++++++++++++++++++++++++++++ tests/test_util.py | 14 +++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index d8ddc9ce..4c76fa50 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -1,6 +1,9 @@ import json import time +import contextlib +import threading +import queue import multiprocessing as mp from datetime import datetime @@ -31,6 +34,31 @@ class ProcessGroup(object): self.processes.append(proc) +# Inspired by: +# - http://stackoverflow.com/a/24741694/597097 +def pool(builder, limit=None): + lock = threading.Lock() + local_pool = queue.Queue() + size = 0 + + @contextlib.contextmanager + def pooled(): + nonlocal size + if size == limit: + instance = local_pool.get() + else: + with lock: + if size == limit: + instance = local_pool.get() + else: + size += 1 + instance = builder() + yield instance + local_pool.put(instance) + + return pooled + + def serialize(data): """Serialize a dict into a JSON formatted string. diff --git a/tests/test_util.py b/tests/test_util.py index f4708b59..c15eb922 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,7 +1,9 @@ -from bigchaindb import util +import pytest def test_transform_create(b, user_private_key, user_public_key): + from bigchaindb import util + tx = util.create_tx(user_public_key, user_public_key, None, 'CREATE') tx = util.transform_create(tx) tx = util.sign_tx(tx, b.me_private) @@ -10,3 +12,13 @@ def test_transform_create(b, user_private_key, user_public_key): assert tx['transaction']['new_owner'] == user_public_key assert util.verify_signature(tx) +@pytest.mark.skipif(reason='asdf') +def test_pool(): + from bigchaindb import util + + pool = util.pool(lambda: 'hello', limit=4) + + assert pool().__enter__() == 'hello' + assert pool().__enter__() == 'hello' + assert pool().__enter__() == 'hello' + From 3f69f208a1234c4847fba26bc4ef5755ada8f56d Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 12:29:01 +0200 Subject: [PATCH 08/30] Add more tests for pool --- tests/test_util.py | 81 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index c15eb922..0694e4dd 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,6 +1,24 @@ import pytest +@pytest.fixture +def mock_queue(monkeypatch): + + class MockQueue: + items = [] + + def get(self): + return self.items.pop() + + def put(self, item): + self.items.append(item) + + mockqueue = MockQueue() + + monkeypatch.setattr('queue.Queue', lambda: mockqueue) + return mockqueue + + def test_transform_create(b, user_private_key, user_public_key): from bigchaindb import util @@ -12,13 +30,66 @@ def test_transform_create(b, user_private_key, user_public_key): assert tx['transaction']['new_owner'] == user_public_key assert util.verify_signature(tx) -@pytest.mark.skipif(reason='asdf') -def test_pool(): + +def test_empty_pool_is_populated_with_instances(mock_queue): from bigchaindb import util pool = util.pool(lambda: 'hello', limit=4) - assert pool().__enter__() == 'hello' - assert pool().__enter__() == 'hello' - assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 1 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 2 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 3 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 4 + + + +def test_pool_blocks_if_no_instances_available(mock_queue): + from bigchaindb import util + + pool = util.pool(lambda: 'hello', limit=4) + + assert len(mock_queue.items) == 0 + + # We need to manually trigger the `__enter__` method so the context + # manager will "hang" and not return the resource to the pool + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + # We need to keep a reference of the last context manager so we can + # manually release the resource + last = pool() + assert last.__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + # This would block using `queue.Queue` but since we mocked it it will + # just raise a IndexError because it's trying to pop from an empty list. + with pytest.raises(IndexError): + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 + + # Release the last resource + last.__exit__(None, None, None) + assert len(mock_queue.items) == 1 + + assert pool().__enter__() == 'hello' + assert len(mock_queue.items) == 0 From 99ac8f99e074e061b3e0db4e928b6dbe7deef6a9 Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 12:34:27 +0200 Subject: [PATCH 09/30] Add new check for queue size --- tests/test_util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 0694e4dd..45d67f3a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -54,6 +54,10 @@ def test_empty_pool_is_populated_with_instances(mock_queue): assert instance == 'hello' assert len(mock_queue.items) == 4 + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 4 + def test_pool_blocks_if_no_instances_available(mock_queue): From f7fc829662572a7947f81574a6fb0139e45c6895 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 15 Apr 2016 12:47:14 +0200 Subject: [PATCH 10/30] Fleshed out the docs on the CLI --- docs/source/bigchaindb-cli.md | 62 +++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/docs/source/bigchaindb-cli.md b/docs/source/bigchaindb-cli.md index 4186426e..ae70b341 100644 --- a/docs/source/bigchaindb-cli.md +++ b/docs/source/bigchaindb-cli.md @@ -1,27 +1,55 @@ -# The BigchainDB Command Line Interfaces (CLIs) +# The BigchainDB Command Line Interface (CLI) -BigchainDB has some Command Line Interfaces (CLIs). One of them is the `bigchaindb` command which we already saw when we first started BigchainDB using: +There are some command-line commands for working with BigchainDB: `bigchaindb` and `bigchaindb-benchmark`. This section provides an overview of those commands. + +## bigchaindb + +### bigchaindb --help + +One can get basic help with the `bigchaindb` command using `bigchaindb --help` or `bigchaindb -h`. + +### bigchaindb configure + +This command generates a public/private keypair for the node, and writes a BigchainDB configuration file to the node's file system. It's documented in the section [Configuring a BigchainDB Node](configuration.html). + +If you want to force-generate a new configuration file regardless of whether one already exists (i.e. skipping the yes/no prompt), then use `bigchaindb -y configure`. + +### bigchaindb add-to-keyring KEY + +This command is used to add a single public key (KEY) to the node's keyring (a list of public keys of _other_ nodes in the federation). For example, this command: ```text -$ bigchaindb configure -$ bigchaindb start +bigchaindb add-to-keyring F9C2vsnEkiaeUTrDRnJrmtV1AJxWjud9eTvMU5LLqa1C ``` -When you run `bigchaindb configure`, it creates a default configuration file in `$HOME/.bigchaindb`. You can check that configuration using: -```text -$ bigchaindb show-config -``` +adds the public key `F9C2vsnEkiaeUTrDRnJrmtV1AJxWjud9eTvMU5LLqa1C` to the node's keyring. If you attempt to add the node's own public key, or you attempt to add a key that's already in the keyring, then no key will be added to the keyring. -To find out what else you can do with the `bigchain` command, use: -```text -$ bigchaindb -h -``` +### bigchaindb show-config -There's another command named `bigchaindb-benchmark`. It's used to run benchmarking tests. You can learn more about it using: +This command shows the values of the configuration settings, which can come from a variety of sources. See [the section on configuring BigchainDB](configuration.html) for more details and examples. + +### bigchaindb export-my-pubkey + +This command writes the node's public key (i.e. one of its configuration values) to standard output (stdout). + +### bigchaindb init + +This command creates a RethinkDB database, two RethinkDB database tables (backlog and bigchain), various RethinkDB database indexes, and the genesis block. + +Note: The `bigchaindb start` command (see below) always starts by trying a `bigchaindb init` first. If it sees that the RethinkDB database already exists, then it doesn't re-initialize the database. One doesn't have to do `bigchaindb init` before `bigchaindb start`. `bigchaindb init` is useful if you only want to initialize (but not start). + +### bigchaindb drop + +This command drops (erases) the RethinkDB database. You will be prompted to make sure. If you want to force-drop the database (i.e. skipping the yes/no prompt), then use `bigchaindb -y drop` + +### bigchaindb start + +This command starts BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`. + + +## bigchaindb-benchmark + +The `bigchaindb-benchmark` command is used to run benchmarking tests. You can learn more about it using: ```text $ bigchaindb-benchmark -h $ bigchaindb-benchmark load -h ``` - -Note that you can always start `bigchaindb` using a different config file using the `-c` option. -For more information check the help with `bigchaindb -h`. - From 8d2d68fec43004f77541313ec927299e119fb375 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 15 Apr 2016 12:48:33 +0200 Subject: [PATCH 11/30] install docs: bigchaindb init not needed before bigchaindb start --- docs/source/installing-server.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/installing-server.md b/docs/source/installing-server.md index 0d12a875..2c8f0c32 100644 --- a/docs/source/installing-server.md +++ b/docs/source/installing-server.md @@ -95,16 +95,17 @@ $ rethinkdb Then open a different terminal and run: ```text $ bigchaindb -y configure -$ bigchaindb init ``` -That creates a configuration file in `$HOME/.bigchaindb` (documented in [the section on configuration](configuration.html)), initializes the database, creates the tables, creates the indexes, and generates the genesis block. +That creates a configuration file in `$HOME/.bigchaindb` (documented in [the section on configuration](configuration.html)). More documentation about the `bigchaindb` command is in the section on [the BigchainDB Command Line Interface (CLI)](bigchaindb-cli.html). You can start BigchainDB Server using: ```text $ bigchaindb start ``` +If it's the first time you've run `bigchaindb start`, then it creates the database (a RethinkDB database), the tables, the indexes, and the genesis block. It then starts BigchainDB. If you're run `bigchaindb start` or `bigchaindb init` before (and you haven't dropped the database), then `bigchaindb start` just starts BigchainDB. + ## Run BigchainDB with Docker From 57ffd361a6464c83c0bf0e08662303a883a349e7 Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 12:57:53 +0200 Subject: [PATCH 12/30] Add timeout parameter to pool context manager --- bigchaindb/util.py | 26 ++++++++++++++++++-------- tests/test_util.py | 34 +++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 4c76fa50..44db89f0 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -36,22 +36,32 @@ class ProcessGroup(object): # Inspired by: # - http://stackoverflow.com/a/24741694/597097 -def pool(builder, limit=None): +def pool(builder, size, timeout=None): + """Create a pool that imposes a limit on the number of stored + instances. + + Args: + builder: a function to build an instance. + size: the size of the pool. + + Returns: + A context manager that can be used with ``with``. + """ lock = threading.Lock() local_pool = queue.Queue() - size = 0 + current_size = 0 @contextlib.contextmanager def pooled(): - nonlocal size - if size == limit: - instance = local_pool.get() + nonlocal current_size + if current_size == size: + instance = local_pool.get(timeout=timeout) else: with lock: - if size == limit: - instance = local_pool.get() + if current_size == size: + instance = local_pool.get(timeout=timeout) else: - size += 1 + current_size += 1 instance = builder() yield instance local_pool.put(instance) diff --git a/tests/test_util.py b/tests/test_util.py index 45d67f3a..22cc0c22 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,5 @@ import pytest +import queue @pytest.fixture @@ -7,8 +8,13 @@ def mock_queue(monkeypatch): class MockQueue: items = [] - def get(self): - return self.items.pop() + def get(self, timeout=None): + try: + return self.items.pop() + except IndexError: + if timeout: + raise queue.Empty() + raise def put(self, item): self.items.append(item) @@ -34,7 +40,7 @@ def test_transform_create(b, user_private_key, user_public_key): def test_empty_pool_is_populated_with_instances(mock_queue): from bigchaindb import util - pool = util.pool(lambda: 'hello', limit=4) + pool = util.pool(lambda: 'hello', 4) assert len(mock_queue.items) == 0 @@ -59,11 +65,10 @@ def test_empty_pool_is_populated_with_instances(mock_queue): assert len(mock_queue.items) == 4 - def test_pool_blocks_if_no_instances_available(mock_queue): from bigchaindb import util - pool = util.pool(lambda: 'hello', limit=4) + pool = util.pool(lambda: 'hello', 4) assert len(mock_queue.items) == 0 @@ -97,3 +102,22 @@ def test_pool_blocks_if_no_instances_available(mock_queue): assert pool().__enter__() == 'hello' assert len(mock_queue.items) == 0 + +def test_pool_raises_empty_exception_when_timeout(mock_queue): + from bigchaindb import util + + pool = util.pool(lambda: 'hello', 1, timeout=1) + + assert len(mock_queue.items) == 0 + + with pool() as instance: + assert instance == 'hello' + assert len(mock_queue.items) == 1 + + # take the only resource available + assert pool().__enter__() == 'hello' + + with pytest.raises(queue.Empty): + with pool() as instance: + assert instance == 'hello' + From 40896baa69125429269564055e582fab58cf5b35 Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 15:50:47 +0200 Subject: [PATCH 13/30] Reorganize the code for the context manager There was probably a deadlock in the previous version. --- bigchaindb/util.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 44db89f0..822345b8 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -45,8 +45,10 @@ def pool(builder, size, timeout=None): size: the size of the pool. Returns: - A context manager that can be used with ``with``. + A context manager that can be used with the ``with`` + statement. """ + lock = threading.Lock() local_pool = queue.Queue() current_size = 0 @@ -54,16 +56,26 @@ def pool(builder, size, timeout=None): @contextlib.contextmanager def pooled(): nonlocal current_size - if current_size == size: - instance = local_pool.get(timeout=timeout) - else: + instance = None + + # If we still have free slots, then we have room to create new + # instances. + if current_size < size: with lock: - if current_size == size: - instance = local_pool.get(timeout=timeout) - else: + # We need to check again if we have slots available, since + # the situation might be different after acquiring the lock + if current_size < size: current_size += 1 instance = builder() + + # Watchout: current_size can be equal to size if the previous part of + # the function has been executed, that's why we need to check if the + # instance is None. + if instance is None and current_size == size: + instance = local_pool.get(timeout=timeout) + yield instance + local_pool.put(instance) return pooled From 667b30e5e8cf01628ce0d8242eaa5864f7c987ea Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 16:08:08 +0200 Subject: [PATCH 14/30] Add connection pool to web views --- bigchaindb/web/server.py | 11 ++++++----- bigchaindb/web/views.py | 29 +++++++++++++++-------------- tests/web/conftest.py | 2 +- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 021ceab4..3001cb21 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -8,6 +8,7 @@ import multiprocessing from flask import Flask +from bigchaindb import util from bigchaindb import Bigchain from bigchaindb.web import views import gunicorn.app.base @@ -45,7 +46,7 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication): return self.application -def create_app(debug=False): +def create_app(settings): """Return an instance of the Flask application. Args: @@ -54,8 +55,8 @@ def create_app(debug=False): """ app = Flask(__name__) - app.debug = debug - app.config['bigchain'] = Bigchain() + app.debug = settings.get('debug', False) + app.config['bigchain_pool'] = util.pool(Bigchain, size=settings.get('threads', 4)) app.register_blueprint(views.basic_views, url_prefix='/api/v1') return app @@ -79,8 +80,8 @@ def create_server(settings): if not settings.get('threads'): settings['threads'] = (multiprocessing.cpu_count() * 2) + 1 - debug = settings.pop('debug', False) - app = create_app(debug) + app = create_app(settings) + settings.pop('debug', False) standalone = StandaloneApplication(app, settings) return standalone diff --git a/bigchaindb/web/views.py b/bigchaindb/web/views.py index 53db6f66..cf618fff 100644 --- a/bigchaindb/web/views.py +++ b/bigchaindb/web/views.py @@ -15,12 +15,11 @@ basic_views = Blueprint('basic_views', __name__) @basic_views.record def get_bigchain(state): - bigchain = state.app.config.get('bigchain') + bigchain_pool = state.app.config.get('bigchain_pool') - if bigchain is None: + if bigchain_pool is None: raise Exception('This blueprint expects you to provide ' - 'database access through `bigchain`') - + 'a pool of Bigchain instances called `bigchain_pool`') @basic_views.route('/transactions/') @@ -34,9 +33,11 @@ def get_transaction(tx_id): A JSON string containing the data about the transaction. """ - bigchain = current_app.config['bigchain'] + pool = current_app.config['bigchain_pool'] + + with pool() as bigchain: + tx = bigchain.get_transaction(tx_id) - tx = bigchain.get_transaction(tx_id) return flask.jsonify(**tx) @@ -47,7 +48,7 @@ def create_transaction(): Return: A JSON string containing the data about the transaction. """ - bigchain = current_app.config['bigchain'] + pool = current_app.config['bigchain_pool'] val = {} @@ -55,15 +56,15 @@ def create_transaction(): # set to `application/json` tx = request.get_json(force=True) - if tx['transaction']['operation'] == 'CREATE': - tx = util.transform_create(tx) - tx = bigchain.consensus.sign_transaction( - tx, private_key=bigchain.me_private) + with pool() as bigchain: + if tx['transaction']['operation'] == 'CREATE': + tx = util.transform_create(tx) + tx = bigchain.consensus.sign_transaction(tx, private_key=bigchain.me_private) - if not bigchain.consensus.verify_signature(tx): - val['error'] = 'Invalid transaction signature' + if not bigchain.consensus.verify_signature(tx): + val['error'] = 'Invalid transaction signature' - val = bigchain.write_transaction(tx) + val = bigchain.write_transaction(tx) return flask.jsonify(**tx) diff --git a/tests/web/conftest.py b/tests/web/conftest.py index 099f2fd3..bf1dd697 100644 --- a/tests/web/conftest.py +++ b/tests/web/conftest.py @@ -25,7 +25,7 @@ def app(request, node_config): restore_config(request, node_config) from bigchaindb.web import server - app = server.create_app(debug=True) + app = server.create_app({'debug': True}) return app From 6844a1a212a2e998be119aadfb6a83653cd62398 Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 16:24:38 +0200 Subject: [PATCH 15/30] Bind to localhost, safer --- bigchaindb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 6a462ab7..3c94ca38 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -9,7 +9,7 @@ config = { 'server': { # Note: this section supports all the Gunicorn settings: # - http://docs.gunicorn.org/en/stable/settings.html - 'bind': '0.0.0.0:9984', + 'bind': 'localhost:9984', 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, From 7cd26cbd3a59abfa4c80ff9044dc5ff7e905268d Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 16:33:26 +0200 Subject: [PATCH 16/30] Update documentation about default bind address --- docs/source/configuration.md | 8 ++++---- docs/source/http-client-server-api.md | 6 ++++-- docs/source/installing-server.md | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/configuration.md b/docs/source/configuration.md index b5d538e1..86da3172 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -65,7 +65,7 @@ environment variables available are: - `BIGCHAINDB_STATSD_RATE` is a float between `0` and `1` that defines the fraction of transaction operations sampled. - `BIGCHAINDB_API_ENDPOINT` defines the API endpoint to use (e.g. `http://localhost:9984/api/v1`). - `BIGCHAINDB_CONSENSUS_PLUGIN` defines the name of the [consensus plugin](consensus.html) to use. -- `BIGCHAINDB_SERVER_BIND` defines where to bind the server socket, the format is `addr:port` (e.g. `0.0.0.0:9984`). +- `BIGCHAINDB_SERVER_BIND` defines where to bind the server socket, the format is `addr:port` (e.g. `localhost:9984`). - `BIGCHAINDB_SERVER_WORKERS` defines the [number of workers](http://docs.gunicorn.org/en/stable/settings.html#workers) to start for the server API. - `BIGCHAINDB_SERVER_THREADS` defines the [number of threads](http://docs.gunicorn.org/en/stable/settings.html#threads) @@ -129,7 +129,7 @@ you will get the following values for all the configuration settings: "pubkey1" ], "server": { - "bind": "0.0.0.0:9984", + "bind": "localhost:9984", "threads": null, "workers": null }, @@ -164,7 +164,7 @@ WARNING:bigchaindb.config_utils:Cannot find config file `/home/vrde/.bigchaindb` }, "keyring": [], "server": { - "bind": "0.0.0.0:9984", + "bind": "localhost:9984", "threads": null, "workers": null }, @@ -213,7 +213,7 @@ WARNING:bigchaindb.config_utils:Cannot find config file `/home/vrde/.bigchaindb` }, "keyring": [], "server": { - "bind": "0.0.0.0:9984", + "bind": "localhost:9984", "threads": null, "workers": null }, diff --git a/docs/source/http-client-server-api.md b/docs/source/http-client-server-api.md index da045fa3..9160082e 100644 --- a/docs/source/http-client-server-api.md +++ b/docs/source/http-client-server-api.md @@ -5,8 +5,10 @@ When you start Bigchaindb using `bigchaindb start`, an HTTP API is exposed at: - [http://localhost:9984/api/v1/](http://localhost:9984/api/v1/) -Please note that by default the server binds to `0.0.0.0:9984`, hence the API -is exposed to the world. +Please note that for security reasons the server binds to `localhost:9984`. +If you want to bind the server to `0.0.0.0` we recommend you to read +[Deploying Gunicorn](http://docs.gunicorn.org/en/stable/deploy.html) and +follow the instructions to deploy it in production. The HTTP API currently exposes two endpoints, one to get information about a specific transaction id, and one to push a transaction to the BigchainDB diff --git a/docs/source/installing-server.md b/docs/source/installing-server.md index 0d12a875..86629a54 100644 --- a/docs/source/installing-server.md +++ b/docs/source/installing-server.md @@ -130,7 +130,7 @@ stored on your host machine under ` ~/.bigchaindb_docker/config`: $ docker-compose run --rm bigchaindb bigchaindb configure Starting bigchaindb_rethinkdb-data_1 Generating keypair -API Server bind? (default `0.0.0.0:9984`): +API Server bind? (default `localhost:9984`): Database host? (default `localhost`): rethinkdb Database port? (default `28015`): Database name? (default `bigchain`): From bbc08a58690ebe8498e74500de0d66a52f73c54e Mon Sep 17 00:00:00 2001 From: vrde Date: Fri, 15 Apr 2016 16:35:30 +0200 Subject: [PATCH 17/30] Fix tests --- tests/utils/test_config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_config_utils.py b/tests/utils/test_config_utils.py index cec7743f..0a80fec7 100644 --- a/tests/utils/test_config_utils.py +++ b/tests/utils/test_config_utils.py @@ -142,7 +142,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): assert bigchaindb.config == { 'CONFIGURED': True, 'server': { - 'bind': '0.0.0.0:9984', + 'bind': 'localhost:9984', 'workers': None, 'threads': None, }, From 16d888d571232a142c2eb4656e1a571337ee29a7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Sun, 17 Apr 2016 13:43:21 +0200 Subject: [PATCH 18/30] Minor change to codecov.yml --- codecov.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 27507adb..d3956b6c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -28,5 +28,7 @@ coverage: - "tests/*" comment: - layout: "header, diff, changes, sunburst, suggestions" + # @stevepeak (from codecov.io) suggested we change 'suggestions' to 'uncovered' + # in the following line. Thanks Steve! + layout: "header, diff, changes, sunburst, uncovered" behavior: default From 8476154369522afe262985f088759e31d41ee387 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 08:55:28 +0200 Subject: [PATCH 19/30] bigchain-benchmark -> bigchaindb-benchmark in opening docstring --- bigchaindb/commands/bigchain_benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigchaindb/commands/bigchain_benchmark.py b/bigchaindb/commands/bigchain_benchmark.py index 2bac4c08..177f0934 100644 --- a/bigchaindb/commands/bigchain_benchmark.py +++ b/bigchaindb/commands/bigchain_benchmark.py @@ -1,4 +1,5 @@ -'''Command line interface for the `bigchain-benchmark` command.''' +"""Command line interface for the `bigchaindb-benchmark` command.""" + import logging import argparse From 36a072948cb0e33bacf4741f652b46ad7273b3d3 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 08:56:57 +0200 Subject: [PATCH 20/30] Minor formatting changes in commands/utils.py --- bigchaindb/commands/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index 4714de5a..31d4a46d 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -1,4 +1,6 @@ -"""Utility functions and basic common arguments for ``argparse.ArgumentParser``.""" +"""Utility functions and basic common arguments +for ``argparse.ArgumentParser``. +""" import argparse import multiprocessing as mp @@ -7,7 +9,8 @@ import multiprocessing as mp def start(parser, scope): """Utility function to execute a subcommand. - The function will look up in the ``scope`` if there is a function called ``run_`` + The function will look up in the ``scope`` + if there is a function called ``run_`` and will run it using ``parser.args`` as first positional argument. Args: @@ -15,7 +18,8 @@ def start(parser, scope): scope (dict): map containing (eventually) the functions to be called. Raises: - NotImplementedError: if ``scope`` doesn't contain a function called ``run_``. + NotImplementedError: if ``scope`` doesn't contain a function called + ``run_``. """ args = parser.parse_args() @@ -29,7 +33,8 @@ def start(parser, scope): # if no command has been found, raise a `NotImplementedError` if not func: - raise NotImplementedError('Command `{}` not yet implemented'.format(args.command)) + raise NotImplementedError('Command `{}` not yet implemented'. + format(args.command)) args.multiprocess = getattr(args, 'multiprocess', False) From 548293486886e0ecd38316065ac89311d80f413e Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 11:06:09 +0200 Subject: [PATCH 21/30] More tests for bigchain export-my-pubkey --- tests/test_commands.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 8f02c541..b8b81c50 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -108,6 +108,43 @@ def test_bigchain_show_config(capsys): assert output_config == config +def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): + from bigchaindb import config + from bigchaindb.commands.bigchain import run_export_my_pubkey + + args = Namespace(config='dummy') + # so in run_export_my_pubkey(args) below, + # filename=args.config='dummy' is passed to autoconfigure(). + # We just assume autoconfigure() works and sets + # config['keypair']['public'] correctly (tested elsewhere). + # We force-set config['keypair']['public'] using monkeypatch. + monkeypatch.setitem(config['keypair'], 'public', 'Charlie_Bucket') + _, _ = capsys.readouterr() # has the effect of clearing capsys + run_export_my_pubkey(args) + out, err = capsys.readouterr() + assert out == config['keypair']['public'] + '\n' + assert out == 'Charlie_Bucket\n' + assert err == '' + + +def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): + from bigchaindb import config + from bigchaindb.commands.bigchain import run_export_my_pubkey + + args = Namespace(config='dummy') + monkeypatch.setitem(config['keypair'], 'public', None) + # assert that run_export_my_pubkey(args) raises SystemExit: + with pytest.raises(SystemExit) as exc_info: + run_export_my_pubkey(args) + # exc_info is an object of class ExceptionInfo + # https://pytest.org/latest/builtin.html#_pytest._code.ExceptionInfo + assert exc_info.type == SystemExit + # exc_info.value is an object of class SystemExit + # https://docs.python.org/3/library/exceptions.html#SystemExit + assert exc_info.value.code == \ + "This node's public key wasn't set anywhere so it can't be exported" + + def test_bigchain_run_init_when_db_exists(mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_init args = Namespace(config=None) @@ -159,4 +196,3 @@ def test_run_configure_when_config_does_exist(monkeypatch, args = Namespace(config='foo', yes=None) run_configure(args) assert value == {} - From 21d234a1ebc7b741a59b33c8838909b9df5df221 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 11:07:53 +0200 Subject: [PATCH 22/30] Improved docstrings + minor changes in config_utils.py --- bigchaindb/config_utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index cd24081e..57792778 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -22,6 +22,7 @@ from pkg_resources import iter_entry_points, ResolutionError import bigchaindb from bigchaindb.consensus import AbstractConsensusRules +# logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) CONFIG_DEFAULT_PATH = os.environ.setdefault( @@ -78,19 +79,21 @@ def update(d, u): def file_config(filename=None): - """Returns the values found in a configuration file. + """Returns the config values found in a configuration file. Args: - filename (str): the JSON file with the configuration. Defaults to ``None``. - If ``None``, the HOME of the current user and the string ``.bigchaindb`` will be used. + filename (str): the JSON file with the configuration values. + If ``None``, CONFIG_DEFAULT_PATH will be used. - Note: - The function merges the values in ``filename`` with the **default configuration**, - so any update made to ``bigchaindb.config`` will be lost. + Returns: + dict: The config values in the specified config file (or the + file at CONFIG_DEFAULT_PATH, if filename == None) """ + logger.debug('On entry into file_config(), filename = {}'.format(filename)) if not filename: filename = CONFIG_DEFAULT_PATH + logger.debug('file_config() will try to open `{}`'.format(filename)) with open(filename) as f: config = json.load(f) @@ -210,7 +213,7 @@ def autoconfigure(filename=None, config=None, force=False): if config: newconfig = update(newconfig, config) - return set_config(newconfig) + set_config(newconfig) # sets bigchaindb.config def load_consensus_plugin(name=None): From 3fccd1ef1d17a5bf6ac2ad96f1aaef08c6749505 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 11:11:59 +0200 Subject: [PATCH 23/30] Added bigchaindb export-my-pubkey command --- bigchaindb/commands/bigchain.py | 78 ++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 15092db5..fe0ee105 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -1,5 +1,7 @@ -"""Command line interface for the `bigchain` command.""" - +"""Implementation of the `bigchaindb` command, +which is one of the commands in the BigchainDB +command-line interface. +""" import os import sys @@ -11,7 +13,10 @@ import json import bigchaindb import bigchaindb.config_utils from bigchaindb import db -from bigchaindb.exceptions import DatabaseAlreadyExists, KeypairNotFoundException +from bigchaindb.exceptions import ( + DatabaseAlreadyExists, + KeypairNotFoundException +) from bigchaindb.commands.utils import base_parser, start from bigchaindb.processes import Processes from bigchaindb import crypto @@ -38,7 +43,8 @@ def run_configure(args, skip_if_exists=False): """Run a script to configure the current node. Args: - skip_if_exists (bool): skip the function if a conf file already exists + skip_if_exists (bool): skip the function if a config file already + exists """ config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH config_file_exists = os.path.exists(config_path) @@ -48,7 +54,7 @@ def run_configure(args, skip_if_exists=False): if config_file_exists and not args.yes: want = input('Config file `{}` exists, do you want to override it? ' - '(cannot be undone) [y/n]: '.format(config_path)) + '(cannot be undone) [y/N]: '.format(config_path)) if want != 'y': return @@ -56,25 +62,49 @@ def run_configure(args, skip_if_exists=False): conf = copy.deepcopy(bigchaindb._config) print('Generating keypair') - conf['keypair']['private'], conf['keypair']['public'] = crypto.generate_key_pair() + conf['keypair']['private'], conf['keypair']['public'] = \ + crypto.generate_key_pair() if not args.yes: for key in ('bind', ): val = conf['server'][key] - conf['server'][key] = input('API Server {}? (default `{}`): '.format(key, val)) or val + conf['server'][key] = \ + input('API Server {}? (default `{}`): '.format(key, val)) \ + or val for key in ('host', 'port', 'name'): val = conf['database'][key] - conf['database'][key] = input('Database {}? (default `{}`): '.format(key, val)) or val + conf['database'][key] = \ + input('Database {}? (default `{}`): '.format(key, val)) \ + or val for key in ('host', 'port', 'rate'): val = conf['statsd'][key] - conf['statsd'][key] = input('Statsd {}? (default `{}`): '.format(key, val)) or val + conf['statsd'][key] = \ + input('Statsd {}? (default `{}`): '.format(key, val)) \ + or val bigchaindb.config_utils.write_config(conf, config_path) + print('Configuration written to {}'.format(config_path)) print('Ready to go!') +def run_export_my_pubkey(args): + """Export this node's public key to standard output + """ + logger.debug('bigchaindb args = {}'.format(args)) + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + pubkey = bigchaindb.config['keypair']['public'] + if pubkey is not None: + print(pubkey) + else: + sys.exit("This node's public key wasn't set anywhere " + "so it can't be exported") + # raises SystemExit exception + # message is sent to stderr + # exits with exit code 1 (signals tha an error happened) + + def run_init(args): """Initialize the database""" bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) @@ -103,16 +133,18 @@ def run_start(args): except DatabaseAlreadyExists: pass except KeypairNotFoundException: - sys.exit('Cannot start BigchainDB, no keypair found. Did you run `bigchaindb configure`?') + sys.exit("Can't start BigchainDB, no keypair found. " + 'Did you run `bigchaindb configure`?') processes = Processes() - logger.info('Start bigchaindb main process') + logger.info('Starting BigchainDB main process') processes.start() def main(): - parser = argparse.ArgumentParser(description='Control your bigchain node.', - parents=[base_parser]) + parser = argparse.ArgumentParser( + description='Control your BigchainDB node.', + parents=[base_parser]) # all the commands are contained in the subparsers object, # the command selected by the user will be stored in `args.command` @@ -121,22 +153,28 @@ def main(): subparsers = parser.add_subparsers(title='Commands', dest='command') + # parser for writing a config file subparsers.add_parser('configure', - help='Prepare the config file and create the node keypair') + help='Prepare the config file ' + 'and create the node keypair') - # parser for database level commands + # parsers for showing/exporting config values + subparsers.add_parser('show-config', + help='Show the current configuration') + + subparsers.add_parser('export-my-pubkey', + help="Export this node's public key") + + # parser for database-level commands subparsers.add_parser('init', help='Init the database') subparsers.add_parser('drop', help='Drop the database') - # TODO how about just config, or info? - subparsers.add_parser('show-config', - help='Show the current configuration') - + # parser for starting BigchainDB subparsers.add_parser('start', - help='Start bigchain') + help='Start BigchainDB') start(parser, globals()) From 1c483cd8a92b911da2b66003090c1427cc984a05 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 11:17:57 +0200 Subject: [PATCH 24/30] Remove docs for bigchaindb add-to-keyring KEY --- docs/source/bigchaindb-cli.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/source/bigchaindb-cli.md b/docs/source/bigchaindb-cli.md index ae70b341..37822717 100644 --- a/docs/source/bigchaindb-cli.md +++ b/docs/source/bigchaindb-cli.md @@ -14,15 +14,6 @@ This command generates a public/private keypair for the node, and writes a Bigch If you want to force-generate a new configuration file regardless of whether one already exists (i.e. skipping the yes/no prompt), then use `bigchaindb -y configure`. -### bigchaindb add-to-keyring KEY - -This command is used to add a single public key (KEY) to the node's keyring (a list of public keys of _other_ nodes in the federation). For example, this command: -```text -bigchaindb add-to-keyring F9C2vsnEkiaeUTrDRnJrmtV1AJxWjud9eTvMU5LLqa1C -``` - -adds the public key `F9C2vsnEkiaeUTrDRnJrmtV1AJxWjud9eTvMU5LLqa1C` to the node's keyring. If you attempt to add the node's own public key, or you attempt to add a key that's already in the keyring, then no key will be added to the keyring. - ### bigchaindb show-config This command shows the values of the configuration settings, which can come from a variety of sources. See [the section on configuring BigchainDB](configuration.html) for more details and examples. From 0870985049ad9d099dc93def8c0f03575d56830f Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 11:49:41 +0200 Subject: [PATCH 25/30] Nevermind checking err when export pubkey and pubkey set --- tests/test_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index b8b81c50..b7814206 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -124,7 +124,6 @@ def test_bigchain_export_my_pubkey_when_pubkey_set(capsys, monkeypatch): out, err = capsys.readouterr() assert out == config['keypair']['public'] + '\n' assert out == 'Charlie_Bucket\n' - assert err == '' def test_bigchain_export_my_pubkey_when_pubkey_not_set(monkeypatch): From 6619036f3de0e6436ad40ce05a3266fb683259fb Mon Sep 17 00:00:00 2001 From: vrde Date: Mon, 18 Apr 2016 12:02:05 +0200 Subject: [PATCH 26/30] Add more docs and remove useless code --- bigchaindb/util.py | 10 +++++++++- bigchaindb/web/server.py | 1 - docs/source/configuration.md | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 2da25f73..165c2de7 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -43,10 +43,18 @@ def pool(builder, size, timeout=None): Args: builder: a function to build an instance. size: the size of the pool. + timeout(Optional[float]): the seconds to wait before raising + a ``queue.Empty`` exception if no instances are available + within that time. + Raises: + If ``timeout`` is defined but the request is taking longer + than the specified time, the context manager will raise + a ``queue.Empty`` exception. Returns: A context manager that can be used with the ``with`` statement. + """ lock = threading.Lock() @@ -71,7 +79,7 @@ def pool(builder, size, timeout=None): # Watchout: current_size can be equal to size if the previous part of # the function has been executed, that's why we need to check if the # instance is None. - if instance is None and current_size == size: + if instance is None: instance = local_pool.get(timeout=timeout) yield instance diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 3001cb21..3e26a007 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -81,7 +81,6 @@ def create_server(settings): settings['threads'] = (multiprocessing.cpu_count() * 2) + 1 app = create_app(settings) - settings.pop('debug', False) standalone = StandaloneApplication(app, settings) return standalone diff --git a/docs/source/configuration.md b/docs/source/configuration.md index 86da3172..2dda7ca0 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -72,6 +72,13 @@ environment variables available are: to start for the server API. +## Configuring the API Server +The API Server is powered by [Gunicorn](http://gunicorn.org/), a Python WSGI HTTP Server for UNIX. +If you need to tweak some settings for the API server you can manually edit your `.bigchaindb` config file: +the `server` section accepts all the options specified in the +[Gunicorn settings](http://docs.gunicorn.org/en/stable/settings.html) documentation. + + ## Order of Precedence in Determining Configuration Values All configuration values start with their default values (defined in `bigchaindb.__init__`), but a default value can be overriden by an environment variable, and a value set by an environment variable can be overriden by a value in a local configuration file (`$HOME/.bigchaindb` or the location specified by the `-c` command-line option). From 4bfad0bbbc100694761435fda2e91dbe0d0c7a2c Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 16:08:25 +0200 Subject: [PATCH 27/30] Minor formatting changes suggested by @r-marques --- bigchaindb/commands/bigchain.py | 8 ++------ bigchaindb/config_utils.py | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index fe0ee105..1907c185 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -13,10 +13,7 @@ import json import bigchaindb import bigchaindb.config_utils from bigchaindb import db -from bigchaindb.exceptions import ( - DatabaseAlreadyExists, - KeypairNotFoundException -) +from bigchaindb.exceptions import DatabaseAlreadyExists, KeypairNotFoundException from bigchaindb.commands.utils import base_parser, start from bigchaindb.processes import Processes from bigchaindb import crypto @@ -43,8 +40,7 @@ def run_configure(args, skip_if_exists=False): """Run a script to configure the current node. Args: - skip_if_exists (bool): skip the function if a config file already - exists + skip_if_exists (bool): skip the function if a config file already exists """ config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH config_file_exists = os.path.exists(config_path) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 57792778..98be7fc9 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -22,7 +22,6 @@ from pkg_resources import iter_entry_points, ResolutionError import bigchaindb from bigchaindb.consensus import AbstractConsensusRules -# logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) CONFIG_DEFAULT_PATH = os.environ.setdefault( From b6fe85526f66ec083d5aa0b0a15dd010689835bc Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 16:48:39 +0200 Subject: [PATCH 28/30] Revised the Cryptography section of the docs --- docs/source/cryptography.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/source/cryptography.md b/docs/source/cryptography.md index 03fe22c3..48b2d88e 100644 --- a/docs/source/cryptography.md +++ b/docs/source/cryptography.md @@ -17,11 +17,8 @@ data = "message" tx_hash = hashlib.sha3_256(data).hexdigest() ``` -## Signature algorithm and keys +## Signature Algorithm and Keys -The signature algorithm used by BigchainDB is [ED25519](https://tools.ietf.org/html/draft-irtf-cfrg-eddsa-04) -using the python [ed25519](https://github.com/warner/python-ed25519) module, overloaded by the [cryptoconditions library](https://github.com/bigchaindb/cryptoconditions). +BigchainDB uses the [Ed25519](https://ed25519.cr.yp.to/) public-key signature system for generating its public/private key pairs (also called verifying/signing keys). Ed25519 is an instance of the [Edwards-curve Digital Signature Algorithm (EdDSA)](https://en.wikipedia.org/wiki/EdDSA). As of April 2016, EdDSA was in ["Internet-Draft" status with the IETF](https://tools.ietf.org/html/draft-irtf-cfrg-eddsa-05) but was [already widely used](https://ianix.com/pub/ed25519-deployment.html). -The private key is the base58 encoded hexadecimal representation of private number. -The public key is the base58 encoded hexadecimal representation of the -compressed public numbers. +BigchainDB uses the the [ed25519](https://github.com/warner/python-ed25519) Python package, overloaded by the [cryptoconditions library](https://github.com/bigchaindb/cryptoconditions). From 9638a0b733c4c7123abe72cb62500f953f166ec3 Mon Sep 17 00:00:00 2001 From: troymc Date: Mon, 18 Apr 2016 17:36:11 +0200 Subject: [PATCH 29/30] docs: Added sentence about how crypto keys are represented --- docs/source/cryptography.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/cryptography.md b/docs/source/cryptography.md index 48b2d88e..0fde8371 100644 --- a/docs/source/cryptography.md +++ b/docs/source/cryptography.md @@ -22,3 +22,5 @@ tx_hash = hashlib.sha3_256(data).hexdigest() BigchainDB uses the [Ed25519](https://ed25519.cr.yp.to/) public-key signature system for generating its public/private key pairs (also called verifying/signing keys). Ed25519 is an instance of the [Edwards-curve Digital Signature Algorithm (EdDSA)](https://en.wikipedia.org/wiki/EdDSA). As of April 2016, EdDSA was in ["Internet-Draft" status with the IETF](https://tools.ietf.org/html/draft-irtf-cfrg-eddsa-05) but was [already widely used](https://ianix.com/pub/ed25519-deployment.html). BigchainDB uses the the [ed25519](https://github.com/warner/python-ed25519) Python package, overloaded by the [cryptoconditions library](https://github.com/bigchaindb/cryptoconditions). + +All keys are represented with the base58 encoding by default. \ No newline at end of file From 9f62cddbaf44167692cfee71db707bce93e3395f Mon Sep 17 00:00:00 2001 From: diminator Date: Wed, 20 Apr 2016 15:31:52 +0200 Subject: [PATCH 30/30] bugfix and bump version to 0.1.5 --- docs/source/conf.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3651eb79..240961e1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -68,7 +68,7 @@ author = 'BigchainDB Contributors' # The short X.Y version. version = '0.1' # The full version, including alpha/beta/rc tags. -release = '0.1.4' +release = '0.1.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3fd8dae4..cd7f5475 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ docs_require = [ setup( name='BigchainDB', - version='0.1.4', + version='0.1.5', description='BigchainDB: A Scalable Blockchain Database', long_description=__doc__, url='https://github.com/BigchainDB/bigchaindb/',