diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index e22063a3..6a462ab7 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -1,49 +1,35 @@ -import os import copy - -def e(key, default=None, conv=None): - """Get the environment variable `key`, fallback to `default` - if nothing is found. - - Keyword arguments: - key -- the key to look for in the environment - default -- the default value if nothing is found (default: None) - conv -- a callable used to convert the value (default: use the type of the - default value) - """ - - val = os.environ.get(key, default) - - if conv or default is not None: - conv = conv or type(default) - return conv(val) - - return val +# from functools import reduce +# PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 +# basically, the port number is 9984 config = { 'server': { - 'bind': e('BIGCHAIN_SERVER_BIND', default='0.0.0.0:5000'), + # Note: this section supports all the Gunicorn settings: + # - http://docs.gunicorn.org/en/stable/settings.html + 'bind': '0.0.0.0:9984', + 'workers': None, # if none, the value will be cpu_count * 2 + 1 + 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, 'database': { - 'host': e('BIGCHAIN_DATABASE_HOST', default='localhost'), - 'port': e('BIGCHAIN_DATABASE_PORT', default=28015), - 'name': e('BIGCHAIN_DATABASE_NAME', default='bigchain') + 'host': 'localhost', + 'port': 28015, + 'name': 'bigchain', }, 'keypair': { - 'public': e('BIGCHAIN_KEYPAIR_PUBLIC'), - 'private': e('BIGCHAIN_KEYPAIR_PRIVATE') + 'public': None, + 'private': None, }, - 'keyring': [ - ], + 'keyring': [], 'statsd': { - 'host': e('BIGCHAIN_STATSD_HOST', default='localhost'), - 'port': e('BIGCHAIN_STATSD_PORT', default=8125), - 'rate': e('BIGCHAIN_STATSD_SAMPLERATE', default=0.01) + 'host': 'localhost', + 'port': 8125, + 'rate': 0.01, }, - 'api_endpoint': 'http://localhost:8008/api/v1', - 'consensus_plugin': e('BIGCHAIN_CONSENSUS_PLUGIN', default='default') + 'api_endpoint': 'http://localhost:9984/api/v1', + 'consensus_plugin': 'default', } # We need to maintain a backup copy of the original config dict in case diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index c1be6761..15092db5 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -2,14 +2,16 @@ import os +import sys import logging import argparse import copy +import json import bigchaindb import bigchaindb.config_utils from bigchaindb import db -from bigchaindb.exceptions import DatabaseAlreadyExists +from bigchaindb.exceptions import DatabaseAlreadyExists, KeypairNotFoundException from bigchaindb.commands.utils import base_parser, start from bigchaindb.processes import Processes from bigchaindb import crypto @@ -21,13 +23,15 @@ logger = logging.getLogger(__name__) def run_show_config(args): """Show the current configuration""" - from pprint import pprint - # TODO Proposal: remove the "hidden" configuration. Only show config. If # the system needs to be configured, then display information on how to # configure the system. - bigchaindb.config_utils.file_config(args.config) - pprint(bigchaindb.config) + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + config = copy.deepcopy(bigchaindb.config) + del config['CONFIGURED'] + private_key = config['keypair']['private'] + config['keypair']['private'] = 'x' * 45 if private_key else None + print(json.dumps(config, indent=4, sort_keys=True)) def run_configure(args, skip_if_exists=False): @@ -55,7 +59,7 @@ def run_configure(args, skip_if_exists=False): conf['keypair']['private'], conf['keypair']['public'] = crypto.generate_key_pair() if not args.yes: - for key in ('host', 'port'): + for key in ('bind', ): val = conf['server'][key] conf['server'][key] = input('API Server {}? (default `{}`): '.format(key, val)) or val @@ -73,7 +77,7 @@ def run_configure(args, skip_if_exists=False): def run_init(args): """Initialize the database""" - bigchaindb.config_utils.file_config(args.config) + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) # TODO Provide mechanism to: # 1. prompt the user to inquire whether they wish to drop the db # 2. force the init, (e.g., via -f flag) @@ -86,18 +90,21 @@ def run_init(args): def run_drop(args): """Drop the database""" - bigchaindb.config_utils.file_config(args.config) + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) db.drop(assume_yes=args.yes) def run_start(args): """Start the processes to run the node""" - run_configure(args, skip_if_exists=True) - bigchaindb.config_utils.file_config(args.config) + # run_configure(args, skip_if_exists=True) + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) try: db.init() except DatabaseAlreadyExists: pass + except KeypairNotFoundException: + sys.exit('Cannot start BigchainDB, no keypair found. Did you run `bigchaindb configure`?') + processes = Processes() logger.info('Start bigchaindb main process') processes.start() diff --git a/bigchaindb/commands/bigchain_benchmark.py b/bigchaindb/commands/bigchain_benchmark.py index 96ba0a5b..2bac4c08 100644 --- a/bigchaindb/commands/bigchain_benchmark.py +++ b/bigchaindb/commands/bigchain_benchmark.py @@ -34,7 +34,7 @@ def _run_load(tx_left, stats): def run_load(args): - bigchaindb.config_utils.file_config(args.config) + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) logger.info('Starting %s processes', args.multiprocess) stats = logstats.Logstats() logstats.thread.start(stats) diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 645780eb..53eb7954 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -5,12 +5,12 @@ By calling `file_config`, the global configuration (stored in configuration file. Note that there is a precedence in reading configuration values: - - [not yet] command line; - local config file; - environment vars; - - default config file (contained in `bigchain.__init__`). + - default config file (contained in ``bigchaindb.__init__``). """ + import os import copy import json @@ -23,11 +23,33 @@ import bigchaindb from bigchaindb.consensus import AbstractConsensusRules logger = logging.getLogger(__name__) + CONFIG_DEFAULT_PATH = os.environ.setdefault( 'BIGCHAINDB_CONFIG_PATH', os.path.join(os.path.expanduser('~'), '.bigchaindb'), ) +CONFIG_PREFIX = 'BIGCHAINDB' +CONFIG_SEP = '_' + + +def map_leafs(func, mapping): + """Map a function to the leafs of a mapping.""" + + def _inner(mapping, path=None): + if path is None: + path = [] + + for key, val in mapping.items(): + if isinstance(val, collections.Mapping): + _inner(val, path + [key]) + else: + mapping[key] = func(val, path=path+[key]) + + return mapping + + return _inner(copy.deepcopy(mapping)) + # Thanks Alex <3 # http://stackoverflow.com/a/3233356/597097 @@ -43,7 +65,7 @@ def update(d, u): def file_config(filename=None): - """Read a configuration file and merge it with the default configuration. + """Returns the values found in a configuration file. Args: filename (str): the JSON file with the configuration. Defaults to ``None``. @@ -57,13 +79,73 @@ def file_config(filename=None): filename = CONFIG_DEFAULT_PATH with open(filename) as f: - newconfig = json.load(f) + config = json.load(f) - dict_config(newconfig) logger.info('Configuration loaded from `{}`'.format(filename)) + return config -def dict_config(newconfig): + +def env_config(config): + """Return a new configuration with the values found in the environment. + + The function recursively iterates over the config, checking if there is + a matching env variable. If an env variable is found, the func updates + the configuration with that value. + + The name of the env variable is built combining a prefix (``BIGCHAINDB``) + with the path to the value. If the ``config`` in input is: + ``{'database': {'host': 'localhost'}}`` + this function will try to read the env variable ``BIGCHAINDB_DATABASE_HOST``. + """ + + def load_from_env(value, path): + var_name = CONFIG_SEP.join([CONFIG_PREFIX] + list(map(lambda s: s.upper(), path))) + return os.environ.get(var_name, value) + + return map_leafs(load_from_env, config) + + +def update_types(config, reference, list_sep=':'): + """Return a new configuration where all the values types + are aligned with the ones in the default configuration""" + + def _coerce(current, value): + # Coerce a value to the `current` type. + try: + # First we try to apply current to the value, since it + # might be a function + return current(value) + except TypeError: + # Then we check if current is a list AND if the value + # is a string. + if isinstance(current, list) and isinstance(value, str): + # If so, we use the colon as the separator + return value.split(list_sep) + + try: + # If we are here, we should try to apply the type + # of `current` to the value + return type(current)(value) + except TypeError: + # Worst case scenario we return the value itself. + return value + + def _update_type(value, path): + current = reference + + for elem in path: + try: + current = current[elem] + except KeyError: + return value + + return _coerce(current, value) + + return map_leafs(_update_type, config) + + +def dict_config(config): """Merge the provided configuration with the default one. Args: @@ -74,11 +156,11 @@ def dict_config(newconfig): update made to ``bigchaindb.config`` will be lost. """ bigchaindb.config = copy.deepcopy(bigchaindb._config) - update(bigchaindb.config, newconfig) + update(bigchaindb.config, update_types(config, bigchaindb.config)) bigchaindb.config['CONFIGURED'] = True -def write_config(newconfig, filename=None): +def write_config(config, filename=None): """Write the provided configuration to a specific location. Args: @@ -90,18 +172,29 @@ def write_config(newconfig, filename=None): filename = CONFIG_DEFAULT_PATH with open(filename, 'w') as f: - json.dump(newconfig, f) + json.dump(config, f) -def autoconfigure(): - """Run ``file_config`` if the module has not been initialized. - """ - if bigchaindb.config.get('CONFIGURED'): +def autoconfigure(filename=None, config=None, force=False): + """Run ``file_config`` and ``env_config`` if the module has not + been initialized.""" + + if not force and bigchaindb.config.get('CONFIGURED'): + logger.debug('System already configured, skipping autoconfiguration') return + + newconfig = env_config(bigchaindb.config) + try: - file_config() - except FileNotFoundError: - logger.warning('Cannot find your config file. Run `bigchaindb configure` to create one') + newconfig = update(newconfig, file_config(filename=filename)) + except FileNotFoundError as e: + logger.warning('Cannot find config file `%s`.' % e.filename) + + if config: + newconfig = update(newconfig, config) + + dict_config(newconfig) + return newconfig def load_consensus_plugin(name=None): diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 76252a89..ea4b2576 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,6 +1,5 @@ import rethinkdb as r import random -import json import rapidjson diff --git a/bigchaindb/db/utils.py b/bigchaindb/db/utils.py index d5551833..b0a44d6a 100644 --- a/bigchaindb/db/utils.py +++ b/bigchaindb/db/utils.py @@ -19,6 +19,9 @@ def get_conn(): def init(): + # Try to access the keypair, throws an exception if it does not exist + b = bigchaindb.Bigchain() + conn = get_conn() dbname = bigchaindb.config['database']['name'] @@ -59,7 +62,6 @@ def init(): r.db(dbname).table('bigchain').index_wait().run(conn) logger.info(' - genesis block') - b = bigchaindb.Bigchain() b.create_genesis_block() logger.info('Done, have fun!') diff --git a/bigchaindb/web/server.py b/bigchaindb/web/server.py index 208f2458..021ceab4 100644 --- a/bigchaindb/web/server.py +++ b/bigchaindb/web/server.py @@ -76,6 +76,9 @@ def create_server(settings): if not settings.get('workers'): settings['workers'] = (multiprocessing.cpu_count() * 2) + 1 + if not settings.get('threads'): + settings['threads'] = (multiprocessing.cpu_count() * 2) + 1 + debug = settings.pop('debug', False) app = create_app(debug) standalone = StandaloneApplication(app, settings) diff --git a/docs/source/bigchaindb-cli.md b/docs/source/bigchaindb-cli.md index 108e5846..4186426e 100644 --- a/docs/source/bigchaindb-cli.md +++ b/docs/source/bigchaindb-cli.md @@ -2,10 +2,11 @@ BigchainDB has some Command Line Interfaces (CLIs). One of them is the `bigchaindb` command which we already saw when we first started BigchainDB using: ```text +$ bigchaindb configure $ bigchaindb start ``` -The fist time you run `bigchaindb start`, it creates a default configuration file in `$HOME/.bigchaindb`. You can check that configuration using: +When you run `bigchaindb configure`, it creates a default configuration file in `$HOME/.bigchaindb`. You can check that configuration using: ```text $ bigchaindb show-config ``` @@ -20,3 +21,7 @@ There's another command named `bigchaindb-benchmark`. It's used to run benchmark $ bigchaindb-benchmark -h $ bigchaindb-benchmark load -h ``` + +Note that you can always start `bigchaindb` using a different config file using the `-c` option. +For more information check the help with `bigchaindb -h`. + diff --git a/docs/source/configuration.md b/docs/source/configuration.md new file mode 100644 index 00000000..b5d538e1 --- /dev/null +++ b/docs/source/configuration.md @@ -0,0 +1,226 @@ +# Configuring a BigchainDB Node + +The BigchainDB configuration settings for a particular node are stored on that node in a configuration file at `$HOME/.bigchaindb`. That file doesn't exist by default. (It's not created when installing BigchainDB.) One could create it using a text editor, but it's easiest to use the `bigchaindb configure` command: + +```text +$ bigchaindb configure +``` + +It will ask some questions and generate a new keypair (i.e. a private key and corresponding public key for the node). See below for some additional explanation of the settings and their meanings. To accept a suggested default value, press Enter or Return. If you want to accept all the default values, use the `-y` option when running the command, that is: + +```text +$ bigchaindb -y configure +``` + +## Using a Different Path for the Configuration File + +By default, the configuration settings are stored in `$HOME/.bigchaindb`. If you want to +specify a different path for your configuration file, you can use the `-c` parameter. +This works for every subcommand under the `bigchaindb` executable. + +For example, if you want to **generate** a new configuration file under a +specific path, you can run: + +``` +$ bigchaindb -c local.json configure +$ bigchaindb -c test.json configure +``` + +This will create two new files named `local.json` and `test.json` in your +current working directory. + +From now on, you can refer to those configuration files using the `-c` +parameter; for example: + +``` +$ bigchaindb -c local.json show-config +``` + +will show the configuration for `local.json`. + +If you want to **start** BigchainDB with the `test.json` configuration file, you can use: + +``` +$ bigchaindb -c test.json start +``` + + +## Using Environment Variables to Configure the Node + +Sometimes it's more convenient to use environment variables to configure the +system, for example when using Docker or Heroku. In +that case you can configure the system using environment variables. + +Every configuration parameter can be mapped to an environment variable. The +environment variables available are: + +- `BIGCHAINDB_DATABASE_HOST` defines the RethinkDB database hostname to connect to. +- `BIGCHAINDB_DATABASE_PORT` defines the RethinkDB database port to connect to. +- `BIGCHAINDB_DATABASE_NAME` defines the RethinkDB database name to use. +- `BIGCHAINDB_KEYPAIR_PUBLIC` defines the public key of the BigchainDB node. +- `BIGCHAINDB_KEYPAIR_PRIVATE` defines the private key of the BigchainDB node. +- `BIGCHAINDB_KEYRING` is a colon-separated list of the public keys of all _other_ nodes in the cluster. +- `BIGCHAINDB_STATSD_HOST` defines the hostname of the statsd server for [monitoring](monitoring.html). +- `BIGCHAINDB_STATSD_PORT` defines the port of the statsd server for monitoring. +- `BIGCHAINDB_STATSD_RATE` is a float between `0` and `1` that defines the fraction of transaction operations sampled. +- `BIGCHAINDB_API_ENDPOINT` defines the API endpoint to use (e.g. `http://localhost:9984/api/v1`). +- `BIGCHAINDB_CONSENSUS_PLUGIN` defines the name of the [consensus plugin](consensus.html) to use. +- `BIGCHAINDB_SERVER_BIND` defines where to bind the server socket, the format is `addr:port` (e.g. `0.0.0.0:9984`). +- `BIGCHAINDB_SERVER_WORKERS` defines the [number of workers](http://docs.gunicorn.org/en/stable/settings.html#workers) + to start for the server API. +- `BIGCHAINDB_SERVER_THREADS` defines the [number of threads](http://docs.gunicorn.org/en/stable/settings.html#threads) + to start for the server API. + + +## Order of Precedence in Determining Configuration Values + +All configuration values start with their default values (defined in `bigchaindb.__init__`), but a default value can be overriden by an environment variable, and a value set by an environment variable can be overriden by a value in a local configuration file (`$HOME/.bigchaindb` or the location specified by the `-c` command-line option). + +In summary, there is an order of precedence in reading configuration values: +1. local configuration file +2. environment variables +3. default configuration file (defined in ``bigchaindb.__init__``) + +This means that if the default configuration contains: + +```json +{ + "database": { + "host": "localhost", + "port": 28015 + } +} +``` + +while the local file `local.json` contains: +```json +{ + "database": { + "host": "ec2-xx-xx-xxx-xxx.eu-central-1.compute.amazonaws.com" + } +} + +``` + +and you run this command: +``` +$ BIGCHAINDB_DATABASE_HOST=anotherhost.com \ + BIGCHAINDB_DATABASE_PORT=4242 \ + BIGCHAINDB_KEYRING=pubkey0:pubkey1 \ + bigchaindb -c local.json show-config +``` + +you will get the following values for all the configuration settings: +```json +{ + "api_endpoint": "http://localhost:8008/api/v1", + "consensus_plugin": "default", + "database": { + "host": "ec2-xx-xx-xxx-xxx.eu-central-1.compute.amazonaws.com", + "name": "bigchain", + "port": 4242 + }, + "keypair": { + "private": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "public": "nJq6EmdUkvFjQRB5hFvDmvZtv1deb3W3RgmiAq6dyygC" + }, + "keyring": [ + "pubkey0", + "pubkey1" + ], + "server": { + "bind": "0.0.0.0:9984", + "threads": null, + "workers": null + }, + "statsd": { + "host": "localhost", + "port": 8125, + "rate": 0.01 + } +} +``` + + +## Another Example + +As another example, let's assume we **don't** have any configuration file stored in +`$HOME/.bigchaindb`. As you can see, `show-config` displays the default configuration (and a +warning): +``` +$ bigchaindb show-config +WARNING:bigchaindb.config_utils:Cannot find config file `/home/vrde/.bigchaindb`. +{ + "api_endpoint": "http://localhost:9984/api/v1", + "consensus_plugin": "default", + "database": { + "host": "localhost", + "name": "bigchain", + "port": 28015 + }, + "keypair": { + "private": null, + "public": null + }, + "keyring": [], + "server": { + "bind": "0.0.0.0:9984", + "threads": null, + "workers": null + }, + "statsd": { + "host": "localhost", + "port": 8125, + "rate": 0.01 + } +} +``` + +If we try to run the node, the command will fail: + +``` +$ bigchaindb start +WARNING:bigchaindb.config_utils:Cannot find config file `/home/vrde/.bigchaindb`. +Cannot start BigchainDB, no keypair found. Did you run `bigchaindb configure`? +``` + +This is failing as expected: a BigchainDB node needs at least a key pair to work. +We can pass the key pair using environment variables: +``` +$ BIGCHAINDB_KEYPAIR_PUBLIC=26y9EuyGP44JXxqcvF8GbCJGqkiqFXddZzxVjLU3rWbHp \ + BIGCHAINDB_KEYPAIR_PRIVATE=9PkLfHbzXnSSNnb1sSBL73C2MydzKLs5fAHoA4Q7otrG \ + bigchaindb start +``` + +We can also run `show-config` to see how the configuration looks: +``` +$ BIGCHAINDB_KEYPAIR_PUBLIC=26y9EuyGP44JXxqcvF8GbCJGqkiqFXddZzxVjLU3rWbHp \ + BIGCHAINDB_KEYPAIR_PRIVATE=9PkLfHbzXnSSNnb1sSBL73C2MydzKLs5fAHoA4Q7otrG \ + bigchaindb show-config + +WARNING:bigchaindb.config_utils:Cannot find config file `/home/vrde/.bigchaindb`. +{ + "api_endpoint": "http://localhost:9984/api/v1", + "consensus_plugin": "default", + "database": { + "host": "localhost", + "name": "bigchain", + "port": 28015 + }, + "keypair": { + "private": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "public": "26y9EuyGP44JXxqcvF8GbCJGqkiqFXddZzxVjLU3rWbHp" + }, + "keyring": [], + "server": { + "bind": "0.0.0.0:9984", + "threads": null, + "workers": null + }, + "statsd": { + "host": "localhost", + "port": 8125, + "rate": 0.01 + } +} +``` diff --git a/docs/source/http-client-server-api.md b/docs/source/http-client-server-api.md index 1520b046..da045fa3 100644 --- a/docs/source/http-client-server-api.md +++ b/docs/source/http-client-server-api.md @@ -2,10 +2,15 @@ When you start Bigchaindb using `bigchaindb start`, an HTTP API is exposed at: -[http://localhost:5000/api/v1/](http://localhost:5000/api/v1/) +- [http://localhost:9984/api/v1/](http://localhost:9984/api/v1/) -Right now, that API can only be accessed from localhost (i.e. not remotely). In the future, we'll enable remote access and explain how that works. See [Issue #149](https://github.com/bigchaindb/bigchaindb/issues/149) on GitHub. -The HTTP API currently exposes two endpoints, one to get information about a specific transaction id, and one to push a transaction to the BigchainDB cluster. Those endpoints are documented at: +Please note that by default the server binds to `0.0.0.0:9984`, hence the API +is exposed to the world. + +The HTTP API currently exposes two endpoints, one to get information about a +specific transaction id, and one to push a transaction to the BigchainDB +cluster. Those endpoints are documented at: + +- [http://docs.bigchaindb.apiary.io/](http://docs.bigchaindb.apiary.io/) -[http://docs.bigchaindb.apiary.io/](http://docs.bigchaindb.apiary.io/) diff --git a/docs/source/index.rst b/docs/source/index.rst index 8a75833f..ec10c228 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,6 +15,7 @@ Table of Contents introduction installing-server running-unit-tests + configuration python-server-api-examples bigchaindb-cli http-client-server-api diff --git a/docs/source/installing-server.md b/docs/source/installing-server.md index c168f62d..7da2b1c4 100644 --- a/docs/source/installing-server.md +++ b/docs/source/installing-server.md @@ -80,6 +80,11 @@ Install from the source: $ python setup.py install ``` +If you want to update BigchainDB to reflect the latest local source code changes, you can use: +```text +$ pip install -e . +``` + ### How to Install BigchainDB on a VM with Vagrant One of our community members ([@Mec-Is](https://github.com/Mec-iS)) wrote [a page about how to install BigchainDB on a VM with Vagrant](https://gist.github.com/Mec-iS/b84758397f1b21f21700). @@ -94,10 +99,16 @@ $ rethinkdb Then open a different terminal and run: ```text -$ bigchaindb start +$ bigchaindb -y configure +$ bigchaindb init ``` -During its first run, BigchainDB Server takes care of configuring a single node environment. +That creates a configuration file in `$HOME/.bigchaindb` (documented in [the section on configuration](configuration.html)), initializes the database, creates the tables, creates the indexes, and generates the genesis block. + +You can start BigchainDB Server using: +```text +$ bigchaindb start +``` ## Run BigchainDB with Docker @@ -122,8 +133,22 @@ then do a one-time configuration step to create the config file; it will be stored on your host machine under ` ~/.bigchaindb_docker/config`: ```text $ docker-compose run --rm bigchaindb bigchaindb configure +Starting bigchaindb_rethinkdb-data_1 +Generating keypair +API Server bind? (default `0.0.0.0:9984`): +Database host? (default `localhost`): rethinkdb +Database port? (default `28015`): +Database name? (default `bigchain`): +Statsd host? (default `localhost`): statsd +Statsd port? (default `8125`): +Statsd rate? (default `0.01`): +Ready to go! ``` +As shown above, make sure that you set the database and statsd hosts to their +corresponding service names (`rethinkdb`, `statsd`), defined in`docker-compose.yml` +and `docker-compose-monitor.yml`. + You can then start it up (in the background, as a daemon) using: ```text $ docker-compose up -d diff --git a/docs/source/python-server-api-examples.md b/docs/source/python-server-api-examples.md index 25fc5b19..2b2bfd86 100644 --- a/docs/source/python-server-api-examples.md +++ b/docs/source/python-server-api-examples.md @@ -11,6 +11,8 @@ We create a digital asset, sign it, write it to a BigchainDB Server instance, re First, make sure you have RethinkDB and BigchainDB _installed and running_, i.e. you [installed them](installing-server.html) and you ran: ```text $ rethinkdb +$ bigchaindb configure +$ bigchaindb init $ bigchaindb start ``` diff --git a/tests/conftest.py b/tests/conftest.py index 1116b52f..44de6316 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,20 @@ USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie' USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' +# We need this function to avoid loading an existing +# conf file located in the home of the user running +# the tests. If it's too aggressive we can change it +# later. +@pytest.fixture(scope='function', autouse=True) +def ignore_local_config_file(monkeypatch): + def mock_file_config(filename=None): + raise FileNotFoundError() + + monkeypatch.setattr('bigchaindb.config_utils.file_config', mock_file_config) + + @pytest.fixture +@pytest.fixture(scope='function', autouse=True) def restore_config(request, node_config): from bigchaindb import config_utils config_utils.dict_config(node_config) diff --git a/tests/test_commands.py b/tests/test_commands.py index 6e731c13..c289f279 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import json from argparse import Namespace from pprint import pprint import copy @@ -11,12 +12,6 @@ def mock_run_configure(monkeypatch): monkeypatch.setattr(bigchain, 'run_configure', lambda *args, **kwargs: None) -@pytest.fixture -def mock_file_config(monkeypatch): - from bigchaindb import config_utils - monkeypatch.setattr(config_utils, 'file_config', lambda *args: None) - - @pytest.fixture def mock_write_config(monkeypatch): from bigchaindb import config_utils @@ -65,13 +60,13 @@ def mock_bigchaindb_backup_config(monkeypatch): monkeypatch.setattr('bigchaindb._config', config) -def test_bigchain_run_start(mock_run_configure, mock_file_config, - mock_processes_start, mock_db_init_with_existing_db): +def test_bigchain_run_start(mock_run_configure, mock_processes_start, mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_start args = Namespace(config=None, yes=True) run_start(args) +@pytest.mark.skipif(reason="BigchainDB doesn't support the automatic creation of a config file anymore") def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_processes_start, mock_generate_key_pair, mock_db_init_with_existing_db): import bigchaindb @@ -99,25 +94,27 @@ def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_p # TODO Please beware, that if debugging, the "-s" switch for pytest will # interfere with capsys. # See related issue: https://github.com/pytest-dev/pytest/issues/128 -def test_bigchain_show_config(capsys, mock_file_config): +@pytest.mark.usefixtures('restore_config') +def test_bigchain_show_config(capsys): from bigchaindb import config from bigchaindb.commands.bigchain import run_show_config + args = Namespace(config=None) _, _ = capsys.readouterr() run_show_config(args) - output_config, _ = capsys.readouterr() - pprint(config) - expected_outout_config, _ = capsys.readouterr() - assert output_config == expected_outout_config + output_config = json.loads(capsys.readouterr()[0]) + del config['CONFIGURED'] + config['keypair']['private'] = 'x' * 45 + assert output_config == config -def test_bigchain_run_init_when_db_exists(mock_file_config, mock_db_init_with_existing_db): +def test_bigchain_run_init_when_db_exists(mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_init args = Namespace(config=None) run_init(args) -def test_drop_existing_db(mock_file_config, mock_rethink_db_drop): +def test_drop_existing_db(mock_rethink_db_drop): from bigchaindb.commands.bigchain import run_drop args = Namespace(config=None, yes=True) run_drop(args) diff --git a/tests/utils/test_config_utils.py b/tests/utils/test_config_utils.py index fa63158d..cec7743f 100644 --- a/tests/utils/test_config_utils.py +++ b/tests/utils/test_config_utils.py @@ -60,9 +60,108 @@ def test_load_consensus_plugin_raises_with_invalid_subclass(monkeypatch): from bigchaindb import config_utils monkeypatch.setattr(config_utils, 'iter_entry_points', - lambda *args: [ type('entry_point', - (object), - {'load': lambda: object}) ]) + lambda *args: [type('entry_point', (object), {'load': lambda: object})]) with pytest.raises(TypeError): config_utils.load_consensus_plugin() + + +def test_map_leafs_iterator(): + from bigchaindb import config_utils + + mapping = { + 'a': {'b': {'c': 1}, + 'd': {'z': 44}}, + 'b': {'d': 2}, + 'c': 3 + } + + result = config_utils.map_leafs(lambda x, path: x * 2, mapping) + assert result == { + 'a': {'b': {'c': 2}, + 'd': {'z': 88}}, + 'b': {'d': 4}, + 'c': 6 + } + + result = config_utils.map_leafs(lambda x, path: path, mapping) + assert result == { + 'a': {'b': {'c': ['a', 'b', 'c']}, + 'd': {'z': ['a', 'd', 'z']}}, + 'b': {'d': ['b', 'd']}, + 'c': ['c'] + } + + +def test_update_types(): + from bigchaindb import config_utils + + raw = { + 'a_string': 'test', + 'an_int': '42', + 'a_float': '3.14', + 'a_list': 'a:b:c', + } + + reference = { + 'a_string': 'test', + 'an_int': 42, + 'a_float': 3.14, + 'a_list': ['a', 'b', 'c'], + } + + result = config_utils.update_types(raw, reference) + assert result == reference + + +def test_env_config(monkeypatch): + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_HOST': 'test-host', + 'BIGCHAINDB_DATABASE_PORT': 'test-port'}) + + from bigchaindb import config_utils + + result = config_utils.env_config({'database': {'host': None, 'port': None}}) + expected = {'database': {'host': 'test-host', 'port': 'test-port'}} + + assert result == expected + + +def test_autoconfigure_read_both_from_file_and_env(monkeypatch): + file_config = { + 'database': {'host': 'test-host'} + } + monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname', + 'BIGCHAINDB_DATABASE_PORT': '4242', + 'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'}) + + import bigchaindb + from bigchaindb import config_utils + config_utils.autoconfigure() + + assert bigchaindb.config == { + 'CONFIGURED': True, + 'server': { + 'bind': '0.0.0.0:9984', + 'workers': None, + 'threads': None, + }, + 'database': { + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + }, + 'keypair': { + 'public': None, + 'private': None, + }, + 'keyring': ['pubkey_0', 'pubkey_1', 'pubkey_2'], + 'statsd': { + 'host': 'localhost', + 'port': 8125, + 'rate': 0.01, + }, + 'api_endpoint': 'http://localhost:9984/api/v1', + 'consensus_plugin': 'default', + } +