diff --git a/Dockerfile b/Dockerfile index a955dd5d..0fcac07f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,26 @@ -FROM python:3.5 +FROM rethinkdb:2.3 RUN apt-get update +RUN apt-get -y install python3 python3-pip +RUN pip3 install --upgrade pip +RUN pip3 install --upgrade setuptools RUN mkdir -p /usr/src/app -WORKDIR /usr/src/app - -RUN pip install --upgrade pip COPY . /usr/src/app/ -RUN pip install --no-cache-dir -e .[dev] +WORKDIR /usr/src/app + +RUN pip3 install --no-cache-dir -e . + +WORKDIR /data + +ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb +ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 +ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1 + +ENTRYPOINT ["bigchaindb", "--experimental-start-rethinkdb"] + +CMD ["start"] + +EXPOSE 8080 9984 28015 29015 diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 1907c185..b37a3812 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -9,12 +9,20 @@ import logging import argparse import copy import json +import builtins + +import logstats + import bigchaindb import bigchaindb.config_utils +from bigchaindb.util import ProcessGroup +from bigchaindb.client import temp_client from bigchaindb import db -from bigchaindb.exceptions import DatabaseAlreadyExists, KeypairNotFoundException -from bigchaindb.commands.utils import base_parser, start +from bigchaindb.exceptions import (StartupError, + DatabaseAlreadyExists, + KeypairNotFoundException) +from bigchaindb.commands import utils from bigchaindb.processes import Processes from bigchaindb import crypto @@ -23,6 +31,14 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +# We need this because `input` always prints on stdout, while it should print +# to stderr. It's a very old bug, check it out here: +# - https://bugs.python.org/issue1927 +def input(prompt): + print(prompt, end='', file=sys.stderr) + return builtins.input() + + def run_show_config(args): """Show the current configuration""" # TODO Proposal: remove the "hidden" configuration. Only show config. If @@ -43,7 +59,11 @@ def run_configure(args, skip_if_exists=False): skip_if_exists (bool): skip the function if a config file already exists """ config_path = args.config or bigchaindb.config_utils.CONFIG_DEFAULT_PATH - config_file_exists = os.path.exists(config_path) + + config_file_exists = False + # if the config path is `-` then it's stdout + if config_path != '-': + config_file_exists = os.path.exists(config_path) if config_file_exists and skip_if_exists: return @@ -54,10 +74,15 @@ def run_configure(args, skip_if_exists=False): if want != 'y': return - # Patch the default configuration with the new values - conf = copy.deepcopy(bigchaindb._config) + conf = copy.deepcopy(bigchaindb.config) - print('Generating keypair') + # Patch the default configuration with the new values + conf = bigchaindb.config_utils.update( + conf, + bigchaindb.config_utils.env_config(bigchaindb.config)) + + + print('Generating keypair', file=sys.stderr) conf['keypair']['private'], conf['keypair']['public'] = \ crypto.generate_key_pair() @@ -80,9 +105,12 @@ def run_configure(args, skip_if_exists=False): input('Statsd {}? (default `{}`): '.format(key, val)) \ or val - bigchaindb.config_utils.write_config(conf, config_path) - print('Configuration written to {}'.format(config_path)) - print('Ready to go!') + if config_path != '-': + bigchaindb.config_utils.write_config(conf, config_path) + else: + print(json.dumps(conf, indent=4, sort_keys=True)) + print('Configuration written to {}'.format(config_path), file=sys.stderr) + print('Ready to go!', file=sys.stderr) def run_export_my_pubkey(args): @@ -110,8 +138,8 @@ def run_init(args): try: db.init() except DatabaseAlreadyExists: - print('The database already exists.') - print('If you wish to re-initialize it, first drop it.') + print('The database already exists.', file=sys.stderr) + print('If you wish to re-initialize it, first drop it.', file=sys.stderr) def run_drop(args): @@ -122,8 +150,15 @@ def run_drop(args): def run_start(args): """Start the processes to run the node""" - # run_configure(args, skip_if_exists=True) bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + + if args.start_rethinkdb: + try: + proc = utils.start_rethinkdb() + except StartupError as e: + sys.exit('Error starting RethinkDB, reason is: {}'.format(e)) + logger.info('RethinkDB started with PID %s' % proc.pid) + try: db.init() except DatabaseAlreadyExists: @@ -137,10 +172,46 @@ def run_start(args): processes.start() +def _run_load(tx_left, stats): + logstats.thread.start(stats) + client = temp_client() + + while True: + tx = client.create() + + stats['transactions'] += 1 + + if tx_left is not None: + tx_left -= 1 + if tx_left == 0: + break + + +def run_load(args): + bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) + logger.info('Starting %s processes', args.multiprocess) + stats = logstats.Logstats() + logstats.thread.start(stats) + + tx_left = None + if args.count > 0: + tx_left = int(args.count / args.multiprocess) + + workers = ProcessGroup(concurrency=args.multiprocess, + target=_run_load, + args=(tx_left, stats.get_child())) + workers.start() + + def main(): parser = argparse.ArgumentParser( description='Control your BigchainDB node.', - parents=[base_parser]) + parents=[utils.base_parser]) + + parser.add_argument('--experimental-start-rethinkdb', + dest='start_rethinkdb', + action='store_true', + help='Run RethinkDB on start') # all the commands are contained in the subparsers object, # the command selected by the user will be stored in `args.command` @@ -172,7 +243,25 @@ def main(): subparsers.add_parser('start', help='Start BigchainDB') - start(parser, globals()) + load_parser = subparsers.add_parser('load', + help='Write transactions to the backlog') + + load_parser.add_argument('-m', '--multiprocess', + nargs='?', + type=int, + default=False, + help='Spawn multiple processes to run the command, ' + 'if no value is provided, the number of processes ' + 'is equal to the number of cores of the host machine') + + load_parser.add_argument('-c', '--count', + default=0, + type=int, + help='Number of transactions to push. If the parameter -m ' + 'is set, the count is distributed equally to all the ' + 'processes') + + utils.start(parser, globals()) if __name__ == '__main__': diff --git a/bigchaindb/commands/bigchain_benchmark.py b/bigchaindb/commands/bigchain_benchmark.py deleted file mode 100644 index 9c0edc63..00000000 --- a/bigchaindb/commands/bigchain_benchmark.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Command line interface for the `bigchaindb-benchmark` command.""" - -import logging -import argparse - -import logstats - -import bigchaindb -import bigchaindb.config_utils -from bigchaindb.util import ProcessGroup -from bigchaindb.client import temp_client -from bigchaindb.commands.utils import base_parser, start - - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def _run_load(tx_left, stats): - logstats.thread.start(stats) - client = temp_client() - # b = bigchaindb.Bigchain() - - while True: - tx = client.create() - - stats['transactions'] += 1 - - if tx_left is not None: - tx_left -= 1 - if tx_left == 0: - break - - -def run_load(args): - bigchaindb.config_utils.autoconfigure(filename=args.config, force=True) - logger.info('Starting %s processes', args.multiprocess) - stats = logstats.Logstats() - logstats.thread.start(stats) - - tx_left = None - if args.count > 0: - tx_left = int(args.count / args.multiprocess) - - workers = ProcessGroup(concurrency=args.multiprocess, - target=_run_load, - args=(tx_left, stats.get_child())) - workers.start() - - -def main(): - parser = argparse.ArgumentParser(description='Benchmark your bigchain federation.', - parents=[base_parser]) - - # all the commands are contained in the subparsers object, - # the command selected by the user will be stored in `args.command` - # that is used by the `main` function to select which other - # function to call. - subparsers = parser.add_subparsers(title='Commands', - dest='command') - - # parser for database level commands - load_parser = subparsers.add_parser('load', - help='Write transactions to the backlog') - - load_parser.add_argument('-m', '--multiprocess', - nargs='?', - type=int, - default=False, - help='Spawn multiple processes to run the command, ' - 'if no value is provided, the number of processes ' - 'is equal to the number of cores of the host machine') - - load_parser.add_argument('-c', '--count', - default=0, - type=int, - help='Number of transactions to push. If the parameter -m ' - 'is set, the count is distributed equally to all the ' - 'processes') - - start(parser, globals()) - -if __name__ == '__main__': - main() diff --git a/bigchaindb/commands/utils.py b/bigchaindb/commands/utils.py index c23b25ec..dc035de6 100644 --- a/bigchaindb/commands/utils.py +++ b/bigchaindb/commands/utils.py @@ -4,10 +4,57 @@ for ``argparse.ArgumentParser``. import argparse import multiprocessing as mp +import subprocess +import rethinkdb as r + +import bigchaindb +from bigchaindb.exceptions import StartupError +from bigchaindb import db from bigchaindb.version import __version__ +def start_rethinkdb(): + """Start RethinkDB as a child process and wait for it to be + available. + + Raises: + ``bigchaindb.exceptions.StartupError`` if RethinkDB cannot + be started. + """ + + proc = subprocess.Popen(['rethinkdb', '--bind', 'all'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + + dbname = bigchaindb.config['database']['name'] + line = '' + + for line in proc.stdout: + if line.startswith('Server ready'): + # FIXME: seems like tables are not ready when the server is ready, + # that's why we need to query RethinkDB to know the state + # of the database. This code assumes the tables are ready + # when the database is ready. This seems a valid assumption. + + try: + conn = db.get_conn() + # Before checking if the db is ready, we need to query + # the server to check if it contains that db + if r.db_list().contains(dbname).run(conn): + r.db(dbname).wait().run(conn) + except (r.ReqlOpFailedError, r.ReqlDriverError) as exc: + raise StartupError('Error waiting for the database `{}` ' + 'to be ready'.format(dbname)) from exc + + return proc + + # We are here when we exhaust the stdout of the process. + # The last `line` contains info about the error. + raise StartupError(line) + + def start(parser, scope): """Utility function to execute a subcommand. @@ -51,7 +98,8 @@ def start(parser, scope): base_parser = argparse.ArgumentParser(add_help=False, prog='bigchaindb') base_parser.add_argument('-c', '--config', - help='Specify the location of the configuration file') + help='Specify the location of the configuration file ' + '(use "-" for stdout)') base_parser.add_argument('-y', '--yes', '--yes-please', action='store_true', diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 3ec17656..dc396522 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -91,7 +91,8 @@ def file_config(filename=None): file at CONFIG_DEFAULT_PATH, if filename == None) """ logger.debug('On entry into file_config(), filename = {}'.format(filename)) - if not filename: + + if filename is None: filename = CONFIG_DEFAULT_PATH logger.debug('file_config() will try to open `{}`'.format(filename)) diff --git a/bigchaindb/exceptions.py b/bigchaindb/exceptions.py index 0baa4ad2..8abfaaf7 100644 --- a/bigchaindb/exceptions.py +++ b/bigchaindb/exceptions.py @@ -28,4 +28,5 @@ class DatabaseDoesNotExist(Exception): class KeypairNotFoundException(Exception): """Raised if operation cannot proceed because the keypair was not given""" - +class StartupError(Exception): + """Raised when there is an error starting up the system""" diff --git a/docs/source/bigchaindb-cli.md b/docs/source/bigchaindb-cli.md index 37822717..229d5b4c 100644 --- a/docs/source/bigchaindb-cli.md +++ b/docs/source/bigchaindb-cli.md @@ -1,6 +1,6 @@ # The BigchainDB Command Line Interface (CLI) -There are some command-line commands for working with BigchainDB: `bigchaindb` and `bigchaindb-benchmark`. This section provides an overview of those commands. +The command to interact with BigchainDB is `bigchaindb`. This section provides an overview of the command. ## bigchaindb @@ -37,10 +37,9 @@ This command drops (erases) the RethinkDB database. You will be prompted to make This command starts BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`. -## bigchaindb-benchmark +### bigchaindb load -The `bigchaindb-benchmark` command is used to run benchmarking tests. You can learn more about it using: +This command is used to run benchmarking tests. You can learn more about it using: ```text -$ bigchaindb-benchmark -h -$ bigchaindb-benchmark load -h +$ bigchaindb load -h ``` diff --git a/docs/source/installing-server.md b/docs/source/installing-server.md index 2fbf77aa..1b830811 100644 --- a/docs/source/installing-server.md +++ b/docs/source/installing-server.md @@ -111,9 +111,110 @@ If it's the first time you've run `bigchaindb start`, then it creates the databa **NOT for Production Use** -For those who like using Docker and wish to experiment with BigchainDB in non-production environments, we currently maintain a `dockerfile` that can be used to build an image for `bigchaindb`, along with a `docker-compose.yml` file to manage a "standalone node", consisting mainly of two containers: one for RethinkDB, and another for BigchainDB. +For those who like using Docker and wish to experiment with BigchainDB in +non-production environments, we currently maintain a Docker image and a +`Dockerfile` that can be used to build an image for `bigchaindb`. -Assuming you have `docker` and `docker-compose` installed, you would proceed as follows. +### Pull and Run the Image from Docker Hub + +Assuming you have Docker installed, you would proceed as follows. + +In a terminal shell, pull the latest version of the BigchainDB Docker image using: +```text +docker pull bigchaindb/bigchaindb:latest +``` + +then do a one-time configuration step to create the config file; we will use +the `-y` option to accept all the default values. The configuration file will +be stored in a file on your host machine at `~/bigchaindb_docker/.bigchaindb`: + +```text +$ docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ + bigchaindb/bigchaindb:latest -y configure +Generating keypair +Configuration written to /data/.bigchaindb +Ready to go! +``` + +Let's analyze that command: + +* `docker run` tells Docker to run some image +* `--rm` remove the container once we are done +* `-v "$HOME/bigchaindb_docker:/data"` map the host directory + `$HOME/bigchaindb_docker` to the container directory `/data`; + this allows us to have the data persisted on the host machine, + you can read more in the [official Docker + documentation](https://docs.docker.com/engine/userguide/containers/dockervolumes/#mount-a-host-directory-as-a-data-volume) +* `-t` allocate a pseudo-TTY +* `-i` keep STDIN open even if not attached +* `bigchaindb/bigchaindb:latest` the image to use +* `-y configure` execute the `configure` sub-command (of the `bigchaindb` command) inside the container, with the `-y` option to automatically use all the default config values + + +After configuring the system, you can run BigchainDB with the following command: + +```text +$ docker run -v "$HOME/bigchaindb_docker:/data" -d \ + --name bigchaindb \ + -p "58080:8080" -p "59984:9984" \ + bigchaindb/bigchaindb:latest start +``` + +The command is slightly different from the previous one, the differences are: + +* `-d` run the container in the background +* `--name bigchaindb` give a nice name to the container so it's easier to + refer to it later +* `-p "58080:8080"` map the host port `58080` to the container port `8080` + (the RethinkDB admin interface) +* `-p "59984:9984"` map the host port `59984` to the container port `9984` + (the BigchainDB API server) +* `start` start the BigchainDB service + +Another way to publish the ports exposed by the container is to use the `-P` (or +`--publish-all`) option. This will publish all exposed ports to random ports. You can +always run `docker ps` to check the random mapping. + +You can also access the RethinkDB dashboard at +[http://localhost:58080/](http://localhost:58080/) + +If that doesn't work, then replace `localhost` with the IP or hostname of the +machine running the Docker engine. If you are running docker-machine (e.g. on +Mac OS X) this will be the IP of the Docker machine (`docker-machine ip +machine_name`). + +#### Load Testing with Docker + +Now that we have BigchainDB running in the Docker container named `bigchaindb`, we can +start another BigchainDB container to generate a load test for it. + +First, make sure the container named `bigchaindb` is still running. You can check that using: +```text +docker ps +``` + +You should see a container named `bigchaindb` in the list. + +You can load test the BigchainDB running in that container by running the `bigchaindb load` command in a second container: + +```text +$ docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ + --link bigchaindb \ + bigchaindb/bigchaindb:latest load +``` + +Note the `--link` option to link to the first container (named `bigchaindb`). + +Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](bigchaindb-cli.html). + +If you look at the RethinkDB dashboard (in your web browser), you should see the effects of the load test. You can also see some effects in the Docker logs using: +```text +$ docker logs -f bigchaindb +``` + +### Building Your Own Image + +Assuming you have Docker installed, you would proceed as follows. In a terminal shell: ```text @@ -122,41 +223,7 @@ $ git clone git@github.com:bigchaindb/bigchaindb.git Build the Docker image: ```text -$ docker-compose build +$ docker build --tag local-bigchaindb . ``` -then do a one-time configuration step to create the config file; it will be -stored on your host machine under ` ~/.bigchaindb_docker/config`: -```text -$ docker-compose run --rm bigchaindb bigchaindb configure -Starting bigchaindb_rethinkdb-data_1 -Generating keypair -API Server bind? (default `localhost:9984`): -Database host? (default `localhost`): rethinkdb -Database port? (default `28015`): -Database name? (default `bigchain`): -Statsd host? (default `localhost`): -Statsd port? (default `8125`): -Statsd rate? (default `0.01`): -Ready to go! -``` - -As shown above, make sure that you set the database and statsd hosts to their -corresponding service names (`rethinkdb`, `statsd`), defined in`docker-compose.yml` -and `docker-compose-monitor.yml`. - -You can then start it up (in the background, as a daemon) using: -```text -$ docker-compose up -d -``` - -then you can load test transactions via: -```text -$ docker exec -it docker-bigchaindb bigchaindb-benchmark load -m -``` - -If you're on Linux, you can probably view the RethinkDB dashboard at: - -[http://localhost:58080/](http://localhost:58080/) - -If that doesn't work, then replace `localhost` with the IP or hostname of the machine running the Docker engine. If you are running docker-machine (e.g.: on Mac OS X) this will be the IP of the Docker machine (`docker-machine ip machine_name`). +Now you can use your own image to run BigchainDB containers. diff --git a/docs/source/monitoring.md b/docs/source/monitoring.md index b5b9ca58..4fb53072 100644 --- a/docs/source/monitoring.md +++ b/docs/source/monitoring.md @@ -22,11 +22,11 @@ then point a browser tab to: The login and password are `admin` by default. If BigchainDB is running and processing transactions, you should see analytics—if not, [start BigchainDB](installing-server.html#run-bigchaindb) and load some test transactions: ```text -$ bigchaindb-benchmark load +$ bigchaindb load ``` then refresh the page after a few seconds. If you're not interested in monitoring, don't worry: BigchainDB will function just fine without any monitoring setup. -Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking! \ No newline at end of file +Feel free to modify the [custom Grafana dashboard](https://github.com/rhsimplex/grafana-bigchaindb-docker/blob/master/bigchaindb_dashboard.js) to your liking! diff --git a/docs/source/running-unit-tests.md b/docs/source/running-unit-tests.md index e20a0fc2..9b0ea505 100644 --- a/docs/source/running-unit-tests.md +++ b/docs/source/running-unit-tests.md @@ -18,16 +18,3 @@ $ python setup.py test (Aside: How does the above command work? The documentation for [pytest-runner](https://pypi.python.org/pypi/pytest-runner) explains. We use [pytest](http://pytest.org/latest/) to write all unit tests.) -### Using docker-compose to Run the Tests - -You can also use `docker-compose` to run the unit tests. (You don't have to start RethinkDB first: `docker-compose` does that on its own, when it reads the `docker-compose.yml` file.) - -First, build the images (~once), using: -```text -$ docker-compose build -``` - -then run the unit tests using: -```text -$ docker-compose run --rm bigchaindb py.test -v -``` diff --git a/setup.py b/setup.py index 77e3a02d..a4294176 100644 --- a/setup.py +++ b/setup.py @@ -65,15 +65,14 @@ setup( entry_points={ 'console_scripts': [ - 'bigchaindb=bigchaindb.commands.bigchain:main', - 'bigchaindb-benchmark=bigchaindb.commands.bigchain_benchmark:main' + 'bigchaindb=bigchaindb.commands.bigchain:main' ], 'bigchaindb.consensus': [ 'default=bigchaindb.consensus:BaseConsensusRules' ] }, install_requires=[ - 'rethinkdb==2.2.0.post4', + 'rethinkdb==2.3.0', 'pysha3==0.3', 'pytz==2015.7', 'cryptoconditions==0.1.6', diff --git a/tests/test_commands.py b/tests/test_commands.py index b7814206..99a4f466 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,4 +1,5 @@ import json +from unittest.mock import Mock, patch from argparse import Namespace from pprint import pprint import copy @@ -62,10 +63,22 @@ def mock_bigchaindb_backup_config(monkeypatch): def test_bigchain_run_start(mock_run_configure, mock_processes_start, mock_db_init_with_existing_db): from bigchaindb.commands.bigchain import run_start - args = Namespace(config=None, yes=True) + args = Namespace(start_rethinkdb=False, config=None, yes=True) run_start(args) +@patch('bigchaindb.commands.utils.start_rethinkdb') +def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, + mock_run_configure, + mock_processes_start, + mock_db_init_with_existing_db): + from bigchaindb.commands.bigchain import run_start + args = Namespace(start_rethinkdb=True, config=None, yes=True) + run_start(args) + + mock_start_rethinkdb.assert_called_with() + + @pytest.mark.skipif(reason="BigchainDB doesn't support the automatic creation of a config file anymore") def test_bigchain_run_start_assume_yes_create_default_config(monkeypatch, mock_processes_start, mock_generate_key_pair, mock_db_init_with_existing_db): @@ -173,7 +186,7 @@ def test_run_configure_when_config_does_not_exist(monkeypatch, mock_bigchaindb_backup_config): from bigchaindb.commands.bigchain import run_configure monkeypatch.setattr('os.path.exists', lambda path: False) - monkeypatch.setattr('builtins.input', lambda question: '\n') + monkeypatch.setattr('builtins.input', lambda: '\n') args = Namespace(config='foo', yes=True) return_value = run_configure(args) assert return_value is None @@ -189,9 +202,26 @@ def test_run_configure_when_config_does_exist(monkeypatch, from bigchaindb.commands.bigchain import run_configure monkeypatch.setattr('os.path.exists', lambda path: True) - monkeypatch.setattr('builtins.input', lambda question: '\n') + monkeypatch.setattr('builtins.input', lambda: '\n') monkeypatch.setattr('bigchaindb.config_utils.write_config', mock_write_config) args = Namespace(config='foo', yes=None) run_configure(args) assert value == {} + + +@patch('subprocess.Popen') +def test_start_rethinkdb_returns_a_process_when_successful(mock_popen): + from bigchaindb.commands import utils + mock_popen.return_value = Mock(stdout=['Server ready']) + assert utils.start_rethinkdb() is mock_popen.return_value + + +@patch('subprocess.Popen') +def test_start_rethinkdb_exits_when_cannot_start(mock_popen): + from bigchaindb import exceptions + from bigchaindb.commands import utils + mock_popen.return_value = Mock(stdout=['Nopety nope']) + with pytest.raises(exceptions.StartupError): + utils.start_rethinkdb() +