From f12264773c33df20ec55e3f61c6f54f329fa2d96 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 23 Jan 2017 16:49:59 +0100 Subject: [PATCH 01/10] bigchaindb configure now requires a positional backend argument. Created and fixed tests. Updated `b` fixture --- bigchaindb/__init__.py | 22 +++++++++++++------ bigchaindb/commands/bigchain.py | 20 ++++++++++++++--- tests/backend/test_generics.py | 29 +++++++++++++----------- tests/commands/test_commands.py | 39 +++++++++++++++++++++++++++++---- tests/conftest.py | 14 +++++------- tests/test_config_utils.py | 17 ++++++++++++-- 6 files changed, 104 insertions(+), 37 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index c50f4810..315774e5 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -5,6 +5,20 @@ import os # PORT_NUMBER = reduce(lambda x, y: x * y, map(ord, 'BigchainDB')) % 2**16 # basically, the port number is 9984 +_database_rethinkdb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), + 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), + 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), + 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), +} + +_database_mongodb = { + 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'mongodb'), + 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), + 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 27017)), + 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), + 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), +} config = { 'server': { @@ -14,13 +28,7 @@ config = { 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, - 'database': { - 'backend': os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb'), - 'host': os.environ.get('BIGCHAINDB_DATABASE_HOST', 'localhost'), - 'port': int(os.environ.get('BIGCHAINDB_DATABASE_PORT', 28015)), - 'name': os.environ.get('BIGCHAINDB_DATABASE_NAME', 'bigchain'), - 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), - }, + 'database': _database_rethinkdb, 'keypair': { 'public': None, 'private': None, diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 6661e902..2fc8df70 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -86,6 +86,16 @@ def run_configure(args, skip_if_exists=False): conf['keypair']['private'], conf['keypair']['public'] = \ crypto.generate_key_pair() + # select the correct config defaults based on the backend + print('Generating default configuration for backend {}' + .format(args.backend)) + database = {} + if args.backend == 'rethinkdb': + database = bigchaindb._database_rethinkdb + elif args.backend == 'mongodb': + database = bigchaindb._database_mongodb + conf['database'] = database + if not args.yes: for key in ('bind', ): val = conf['server'][key] @@ -282,9 +292,13 @@ def create_parser(): dest='command') # parser for writing a config file - subparsers.add_parser('configure', - help='Prepare the config file ' - 'and create the node keypair') + config_parser = subparsers.add_parser('configure', + help='Prepare the config file ' + 'and create the node keypair') + config_parser.add_argument('backend', + choices=['rethinkdb', 'mongodb'], + help='The backend to use. It can be either ' + 'rethinkdb or mongodb.') # parsers for showing/exporting config values subparsers.add_parser('show-config', diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 2ab33a7c..2049d72b 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -1,4 +1,3 @@ -from importlib import import_module from unittest.mock import patch from pytest import mark, raises @@ -69,10 +68,6 @@ def test_changefeed_class(changefeed_class_func_name, args_qty): changefeed_class_func(None, *range(args_qty)) -@mark.parametrize('db,conn_cls', ( - ('mongodb', 'MongoDBConnection'), - ('rethinkdb', 'RethinkDBConnection'), -)) @patch('bigchaindb.backend.schema.create_indexes', autospec=True, return_value=None) @patch('bigchaindb.backend.schema.create_tables', @@ -80,16 +75,24 @@ def test_changefeed_class(changefeed_class_func_name, args_qty): @patch('bigchaindb.backend.schema.create_database', autospec=True, return_value=None) def test_init_database(mock_create_database, mock_create_tables, - mock_create_indexes, db, conn_cls): + mock_create_indexes): from bigchaindb.backend.schema import init_database - conn = getattr( - import_module('bigchaindb.backend.{}.connection'.format(db)), - conn_cls, - )('host', 'port', 'dbname') + from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + # rethinkdb + conn = RethinkDBConnection('host', 'port', 'dbname') init_database(connection=conn, dbname='mickeymouse') - mock_create_database.assert_called_once_with(conn, 'mickeymouse') - mock_create_tables.assert_called_once_with(conn, 'mickeymouse') - mock_create_indexes.assert_called_once_with(conn, 'mickeymouse') + mock_create_database.assert_called_with(conn, 'mickeymouse') + mock_create_tables.assert_called_with(conn, 'mickeymouse') + mock_create_indexes.assert_called_with(conn, 'mickeymouse') + + # mongodb + conn = MongoDBConnection('host', 'port', 'dbname', replicaset='rs') + init_database(connection=conn, dbname='mickeymouse') + mock_create_database.assert_called_with(conn, 'mickeymouse') + mock_create_tables.assert_called_with(conn, 'mickeymouse') + mock_create_indexes.assert_called_with(conn, 'mickeymouse') @mark.parametrize('admin_func_name,kwargs', ( diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index a2e485da..1a1291e3 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -12,7 +12,8 @@ def test_make_sure_we_dont_remove_any_command(): parser = create_parser() - assert parser.parse_args(['configure']).command + assert parser.parse_args(['configure', 'rethinkdb']).command + assert parser.parse_args(['configure', 'mongodb']).command assert parser.parse_args(['show-config']).command assert parser.parse_args(['export-my-pubkey']).command assert parser.parse_args(['init']).command @@ -31,8 +32,8 @@ def test_start_raises_if_command_not_implemented(): with pytest.raises(NotImplementedError): # Will raise because `scope`, the third parameter, - # doesn't contain the function `run_configure` - utils.start(parser, ['configure'], {}) + # doesn't contain the function `run_start` + utils.start(parser, ['start'], {}) def test_start_raises_if_no_arguments_given(): @@ -204,7 +205,7 @@ def test_run_configure_when_config_does_not_exist(monkeypatch, from bigchaindb.commands.bigchain import run_configure monkeypatch.setattr('os.path.exists', lambda path: False) monkeypatch.setattr('builtins.input', lambda: '\n') - args = Namespace(config='foo', yes=True) + args = Namespace(config='foo', backend='rethinkdb', yes=True) return_value = run_configure(args) assert return_value is None @@ -228,6 +229,36 @@ def test_run_configure_when_config_does_exist(monkeypatch, assert value == {} +@pytest.mark.parametrize('backend', ( + 'rethinkdb', + 'mongodb', +)) +def test_run_configure_with_backend(backend, monkeypatch, mock_write_config): + import bigchaindb + from bigchaindb.commands.bigchain import run_configure + + value = {} + + def mock_write_config(new_config, filename=None): + value['return'] = new_config + + monkeypatch.setattr('os.path.exists', lambda path: False) + monkeypatch.setattr('builtins.input', lambda: '\n') + monkeypatch.setattr('bigchaindb.config_utils.write_config', + mock_write_config) + + args = Namespace(config='foo', backend=backend, yes=True) + expected_config = bigchaindb.config + run_configure(args) + + # update the expected config with the correct backend and keypair + backend_conf = getattr(bigchaindb, '_database_' + backend) + expected_config.update({'database': backend_conf, + 'keypair': value['return']['keypair']}) + + assert value['return'] == expected_config + + @patch('bigchaindb.common.crypto.generate_key_pair', return_value=('private_key', 'public_key')) @pytest.mark.usefixtures('ignore_local_config_file') diff --git a/tests/conftest.py b/tests/conftest.py index a69564bc..c3177c14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,26 +109,24 @@ def _restore_dbs(request): @pytest.fixture(scope='session') def _configure_bigchaindb(request): + import bigchaindb from bigchaindb import config_utils test_db_name = TEST_DB_NAME # Put a suffix like _gw0, _gw1 etc on xdist processes xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid') if xdist_suffix: test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) + + backend = request.config.getoption('--database-backend') + backend_conf = getattr(bigchaindb, '_database_' + backend) config = { - 'database': { - 'name': test_db_name, - 'backend': request.config.getoption('--database-backend'), - }, + 'database': backend_conf, 'keypair': { 'private': '31Lb1ZGKTyHnmVK3LUMrAUrPNfd4sE2YyBt3UA4A25aA', 'public': '4XYfCbabAWVUCbjTmRTFEu2sc3dFEdkse4r6X498B1s8', } } - # FIXME - if config['database']['backend'] == 'mongodb': - # not a great way to do this - config['database']['port'] = 27017 + config['database']['name'] = test_db_name config_utils.set_config(config) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 6328d28c..4cb8bd45 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -130,7 +130,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'host': 'test-host', 'port': 4242, 'name': 'test-dbname', - 'replicaset': 'bigchain-rs' }, 'keypair': { 'public': None, @@ -215,7 +214,6 @@ def test_write_config(): ('BIGCHAINDB_DATABASE_HOST', 'test-host', 'host'), ('BIGCHAINDB_DATABASE_PORT', 4242, 'port'), ('BIGCHAINDB_DATABASE_NAME', 'test-db', 'name'), - ('BIGCHAINDB_DATABASE_REPLICASET', 'test-replicaset', 'replicaset') )) def test_database_envs(env_name, env_value, config_key, monkeypatch): import bigchaindb @@ -227,3 +225,18 @@ def test_database_envs(env_name, env_value, config_key, monkeypatch): expected_config['database'][config_key] = env_value assert bigchaindb.config == expected_config + + +def test_database_envs_replicaset(monkeypatch): + # the replica set env is only used if the backend is mongodb + import bigchaindb + + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_REPLICASET': + 'test-replicaset'}) + bigchaindb.config['database'] = bigchaindb._database_mongodb + bigchaindb.config_utils.autoconfigure() + + expected_config = copy.deepcopy(bigchaindb.config) + expected_config['database']['replicaset'] = 'test-replicaset' + + assert bigchaindb.config == expected_config From 927e57beba0a51d78c07313cca50f3b6926ffefa Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 23 Jan 2017 17:29:42 +0100 Subject: [PATCH 02/10] Changed `backend.connect` to handle backend specific kwargs. Fixed tests --- bigchaindb/backend/connection.py | 8 +++++++- bigchaindb/backend/mongodb/connection.py | 2 +- bigchaindb/backend/rethinkdb/connection.py | 2 +- tests/test_config_utils.py | 17 +++++++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/bigchaindb/backend/connection.py b/bigchaindb/backend/connection.py index 0fda4078..629b6d8b 100644 --- a/bigchaindb/backend/connection.py +++ b/bigchaindb/backend/connection.py @@ -40,6 +40,12 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None): host = host or bigchaindb.config['database']['host'] port = port or bigchaindb.config['database']['port'] dbname = name or bigchaindb.config['database']['name'] + # Not sure how to handle this here. This setting is only relevant for + # mongodb. + # I added **kwargs for both RethinkDBConnection and MongoDBConnection + # to handle these these additional args. In case of RethinkDBConnection + # it just does not do anything with it. + replicaset = replicaset or bigchaindb.config['database'].get('replicaset') try: module_name, _, class_name = BACKENDS[backend].rpartition('.') @@ -51,7 +57,7 @@ def connect(backend=None, host=None, port=None, name=None, replicaset=None): raise ConfigurationError('Error loading backend `{}`'.format(backend)) from exc logger.debug('Connection: {}'.format(Class)) - return Class(host, port, dbname) + return Class(host, port, dbname, replicaset=replicaset) class Connection: diff --git a/bigchaindb/backend/mongodb/connection.py b/bigchaindb/backend/mongodb/connection.py index 19731161..43e92dd0 100644 --- a/bigchaindb/backend/mongodb/connection.py +++ b/bigchaindb/backend/mongodb/connection.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) class MongoDBConnection(Connection): def __init__(self, host=None, port=None, dbname=None, max_tries=3, - replicaset=None): + replicaset=None, **kwargs): """Create a new Connection instance. Args: diff --git a/bigchaindb/backend/rethinkdb/connection.py b/bigchaindb/backend/rethinkdb/connection.py index 173cdc7b..c0506f1f 100644 --- a/bigchaindb/backend/rethinkdb/connection.py +++ b/bigchaindb/backend/rethinkdb/connection.py @@ -17,7 +17,7 @@ class RethinkDBConnection(Connection): more times to run the query or open a connection. """ - def __init__(self, host, port, dbname, max_tries=3): + def __init__(self, host, port, dbname, max_tries=3, **kwargs): """Create a new :class:`~.RethinkDBConnection` instance. See :meth:`.Connection.__init__` for diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 4cb8bd45..c1f63742 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -14,20 +14,28 @@ def clean_config(monkeypatch): monkeypatch.setattr('bigchaindb.config', copy.deepcopy(ORIGINAL_CONFIG)) -def test_bigchain_instance_is_initialized_when_conf_provided(): +def test_bigchain_instance_is_initialized_when_conf_provided(request): + import bigchaindb from bigchaindb import config_utils assert 'CONFIGURED' not in bigchaindb.config config_utils.set_config({'keypair': {'public': 'a', 'private': 'b'}}) assert bigchaindb.config['CONFIGURED'] is True + + # set the current backend so that Bigchain can create a connection + backend = request.config.getoption('--database-backend') + backend_conf = getattr(bigchaindb, '_database_' + backend) + bigchaindb.config['database'] = backend_conf + b = bigchaindb.Bigchain() assert b.me assert b.me_private -def test_bigchain_instance_raises_when_not_configured(monkeypatch): +def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): + import bigchaindb from bigchaindb import config_utils from bigchaindb.common import exceptions assert 'CONFIGURED' not in bigchaindb.config @@ -36,6 +44,11 @@ def test_bigchain_instance_raises_when_not_configured(monkeypatch): # from existing configurations monkeypatch.setattr(config_utils, 'autoconfigure', lambda: 0) + # set the current backend so that Bigchain can create a connection + backend = request.config.getoption('--database-backend') + backend_conf = getattr(bigchaindb, '_database_' + backend) + bigchaindb.config['database'] = backend_conf + with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() From 17a07a80f1db317094ce3778d0eef114aa47464b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 30 Jan 2017 18:11:29 +0100 Subject: [PATCH 03/10] Removed 'Example Apps' page from the docs --- docs/server/source/drivers-clients/example-apps.rst | 10 ---------- docs/server/source/drivers-clients/index.rst | 1 - 2 files changed, 11 deletions(-) delete mode 100644 docs/server/source/drivers-clients/example-apps.rst diff --git a/docs/server/source/drivers-clients/example-apps.rst b/docs/server/source/drivers-clients/example-apps.rst deleted file mode 100644 index 0aab953e..00000000 --- a/docs/server/source/drivers-clients/example-apps.rst +++ /dev/null @@ -1,10 +0,0 @@ -Example Apps -============ - -.. warning:: - - There are some example BigchainDB apps (i.e. apps which use BigchainDB) in the GitHub repository named `bigchaindb-examples `_. They were created before there was much of an HTTP API, so instead of communicating with a BigchainDB node via the HTTP API, they communicate directly with the node using the BigchainDB Python server API and the RethinkDB Python Driver. That's not how a real production app would work. The HTTP API is getting better, and we recommend using it to communicate with BigchainDB nodes. - - Moreover, because of changes to the BigchainDB Server code, some of the examples in the bigchaindb-examples repo might not work anymore, or they might not work as expected. - - In the future, we hope to create a set of examples using the HTTP API (or wrappers of it, such as the Python Driver API). diff --git a/docs/server/source/drivers-clients/index.rst b/docs/server/source/drivers-clients/index.rst index 9eb81f6c..dd33e18b 100644 --- a/docs/server/source/drivers-clients/index.rst +++ b/docs/server/source/drivers-clients/index.rst @@ -14,4 +14,3 @@ your choice, and then use the HTTP API directly to post transactions. http-client-server-api The Python Driver Transaction CLI - example-apps From 86542fd74583902cc0a8d2826d0adff9b6b8719a Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 09:58:20 +0100 Subject: [PATCH 04/10] remove unnecessary test --- tests/backend/test_generics.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 2049d72b..956b698f 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -68,33 +68,6 @@ def test_changefeed_class(changefeed_class_func_name, args_qty): changefeed_class_func(None, *range(args_qty)) -@patch('bigchaindb.backend.schema.create_indexes', - autospec=True, return_value=None) -@patch('bigchaindb.backend.schema.create_tables', - autospec=True, return_value=None) -@patch('bigchaindb.backend.schema.create_database', - autospec=True, return_value=None) -def test_init_database(mock_create_database, mock_create_tables, - mock_create_indexes): - from bigchaindb.backend.schema import init_database - from bigchaindb.backend.rethinkdb.connection import RethinkDBConnection - from bigchaindb.backend.mongodb.connection import MongoDBConnection - - # rethinkdb - conn = RethinkDBConnection('host', 'port', 'dbname') - init_database(connection=conn, dbname='mickeymouse') - mock_create_database.assert_called_with(conn, 'mickeymouse') - mock_create_tables.assert_called_with(conn, 'mickeymouse') - mock_create_indexes.assert_called_with(conn, 'mickeymouse') - - # mongodb - conn = MongoDBConnection('host', 'port', 'dbname', replicaset='rs') - init_database(connection=conn, dbname='mickeymouse') - mock_create_database.assert_called_with(conn, 'mickeymouse') - mock_create_tables.assert_called_with(conn, 'mickeymouse') - mock_create_indexes.assert_called_with(conn, 'mickeymouse') - - @mark.parametrize('admin_func_name,kwargs', ( ('get_config', {'table': None}), ('reconfigure', {'table': None, 'shards': None, 'replicas': None}), From 555745abbf69753acdce82d83c0ec3b611a93f6f Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 10:04:45 +0100 Subject: [PATCH 05/10] fixed pep8 issue --- tests/backend/test_generics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/backend/test_generics.py b/tests/backend/test_generics.py index 956b698f..946b694f 100644 --- a/tests/backend/test_generics.py +++ b/tests/backend/test_generics.py @@ -1,5 +1,3 @@ -from unittest.mock import patch - from pytest import mark, raises From fafdac252308314ade77e4ec175aef3869a74347 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 10:34:45 +0100 Subject: [PATCH 06/10] Retrieve default backend from env if set. Fixed tests. --- bigchaindb/__init__.py | 9 ++++++++- tests/test_config_utils.py | 28 ++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 315774e5..072c7b6b 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -20,6 +20,11 @@ _database_mongodb = { 'replicaset': os.environ.get('BIGCHAINDB_DATABASE_REPLICASET', 'bigchain-rs'), } +_database_map = { + 'mongodb': _database_mongodb, + 'rethinkdb': _database_rethinkdb +} + config = { 'server': { # Note: this section supports all the Gunicorn settings: @@ -28,7 +33,9 @@ config = { 'workers': None, # if none, the value will be cpu_count * 2 + 1 'threads': None, # if none, the value will be cpu_count * 2 + 1 }, - 'database': _database_rethinkdb, + 'database': _database_map[ + os.environ.get('BIGCHAINDB_DATABASE_BACKEND', 'rethinkdb') + ], 'keypair': { 'public': None, 'private': None, diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index c1f63742..ebf630a9 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -131,6 +131,27 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): from bigchaindb import config_utils config_utils.autoconfigure() + backend = request.config.getoption('--database-backend') + database_rethinkdb = { + 'backend': 'rethinkdb', + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + } + database_mongodb = { + 'backend': 'mongodb', + 'host': 'test-host', + 'port': 4242, + 'name': 'test-dbname', + 'replicaset': 'bigchain-rs', + } + + database = {} + if backend == 'mongodb': + database = database_mongodb + elif backend == 'rethinkdb': + database = database_rethinkdb + assert bigchaindb.config == { 'CONFIGURED': True, 'server': { @@ -138,12 +159,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'workers': None, 'threads': None, }, - 'database': { - 'backend': request.config.getoption('--database-backend'), - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', - }, + 'database': database, 'keypair': { 'public': None, 'private': None, From 4740150f6d772bc3b8cf9ccf2a3fc56d9909cee9 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 10:57:58 +0100 Subject: [PATCH 07/10] Updated config fixture Simplified tests. --- tests/conftest.py | 3 +-- tests/test_config_utils.py | 10 ---------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c3177c14..9612f38b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -118,9 +118,8 @@ def _configure_bigchaindb(request): test_db_name = '{}_{}'.format(TEST_DB_NAME, xdist_suffix) backend = request.config.getoption('--database-backend') - backend_conf = getattr(bigchaindb, '_database_' + backend) config = { - 'database': backend_conf, + 'database': bigchaindb._database_map[backend], 'keypair': { 'private': '31Lb1ZGKTyHnmVK3LUMrAUrPNfd4sE2YyBt3UA4A25aA', 'public': '4XYfCbabAWVUCbjTmRTFEu2sc3dFEdkse4r6X498B1s8', diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index ebf630a9..7feec4c9 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -23,11 +23,6 @@ def test_bigchain_instance_is_initialized_when_conf_provided(request): assert bigchaindb.config['CONFIGURED'] is True - # set the current backend so that Bigchain can create a connection - backend = request.config.getoption('--database-backend') - backend_conf = getattr(bigchaindb, '_database_' + backend) - bigchaindb.config['database'] = backend_conf - b = bigchaindb.Bigchain() assert b.me @@ -44,11 +39,6 @@ def test_bigchain_instance_raises_when_not_configured(request, monkeypatch): # from existing configurations monkeypatch.setattr(config_utils, 'autoconfigure', lambda: 0) - # set the current backend so that Bigchain can create a connection - backend = request.config.getoption('--database-backend') - backend_conf = getattr(bigchaindb, '_database_' + backend) - bigchaindb.config['database'] = backend_conf - with pytest.raises(exceptions.KeypairNotFoundException): bigchaindb.Bigchain() From 2c26468cea61404d59329be5d055fef80e47dfd6 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 13:51:49 +0100 Subject: [PATCH 08/10] Added some constants to simplify test --- tests/test_config_utils.py | 50 +++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index 7feec4c9..af78585e 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -10,8 +10,13 @@ ORIGINAL_CONFIG = copy.deepcopy(bigchaindb._config) @pytest.fixture(scope='function', autouse=True) -def clean_config(monkeypatch): - monkeypatch.setattr('bigchaindb.config', copy.deepcopy(ORIGINAL_CONFIG)) +def clean_config(monkeypatch, request): + + import bigchaindb + original_config = copy.deepcopy(ORIGINAL_CONFIG) + backend = request.config.getoption('--database-backend') + original_config['database'] = bigchaindb._database_map[backend] + monkeypatch.setattr('bigchaindb.config', original_config) def test_bigchain_instance_is_initialized_when_conf_provided(request): @@ -104,48 +109,55 @@ def test_env_config(monkeypatch): def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): + # constants + DATABASE_HOST = 'test-host' + DATABASE_NAME = 'test-dbname' + DATABASE_PORT = 4242 + DATABASE_BACKEND = request.config.getoption('--database-backend') + SERVER_BIND = '1.2.3.4:56' + KEYRING = 'pubkey_0:pubkey_1:pubkey_2' + file_config = { 'database': { - 'host': 'test-host', - 'backend': request.config.getoption('--database-backend') + 'host': DATABASE_HOST }, 'backlog_reassign_delay': 5 } monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) - monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname', - 'BIGCHAINDB_DATABASE_PORT': '4242', - 'BIGCHAINDB_SERVER_BIND': '1.2.3.4:56', - 'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'}) + monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': DATABASE_NAME, + 'BIGCHAINDB_DATABASE_PORT': str(DATABASE_PORT), + 'BIGCHAINDB_DATABASE_BACKEND': DATABASE_BACKEND, + 'BIGCHAINDB_SERVER_BIND': SERVER_BIND, + 'BIGCHAINDB_KEYRING': KEYRING}) import bigchaindb from bigchaindb import config_utils config_utils.autoconfigure() - backend = request.config.getoption('--database-backend') database_rethinkdb = { 'backend': 'rethinkdb', - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', + 'host': DATABASE_HOST, + 'port': DATABASE_PORT, + 'name': DATABASE_NAME, } database_mongodb = { 'backend': 'mongodb', - 'host': 'test-host', - 'port': 4242, - 'name': 'test-dbname', + 'host': DATABASE_HOST, + 'port': DATABASE_PORT, + 'name': DATABASE_NAME, 'replicaset': 'bigchain-rs', } database = {} - if backend == 'mongodb': + if DATABASE_BACKEND == 'mongodb': database = database_mongodb - elif backend == 'rethinkdb': + elif DATABASE_BACKEND == 'rethinkdb': database = database_rethinkdb assert bigchaindb.config == { 'CONFIGURED': True, 'server': { - 'bind': '1.2.3.4:56', + 'bind': SERVER_BIND, 'workers': None, 'threads': None, }, @@ -154,7 +166,7 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch, request): 'public': None, 'private': None, }, - 'keyring': ['pubkey_0', 'pubkey_1', 'pubkey_2'], + 'keyring': KEYRING.split(':'), 'statsd': { 'host': 'localhost', 'port': 8125, From 69068fc919279699dbc02da714dea0166a38c336 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 13:53:36 +0100 Subject: [PATCH 09/10] Document how to run BigchainDB with MongoDB (#1116) * Document changes in the configure command. Document new add/remove replicas commands. * updated quickstart with mongodb instructions * Docs on how to setup mongodb dev node with and without docker. Update replSet option in docker-compose * Fixed typo. More explicit on how to run the tests. * Fixed typo in mongodb docker instructions. More explicit about requiring mongodb 3.4+ --- docker-compose.yml | 2 +- .../source/dev-and-test/setup-run-node.md | 88 ++++++++++++++++--- docs/server/source/quickstart.md | 43 ++++++--- .../source/server-reference/bigchaindb-cli.md | 32 ++++++- 4 files changed, 140 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index db8abd4f..f5dbcdc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: mongo:3.4.1 ports: - "27017" - command: mongod --replSet=rs0 + command: mongod --replSet=bigchain-rs rdb: image: rethinkdb diff --git a/docs/server/source/dev-and-test/setup-run-node.md b/docs/server/source/dev-and-test/setup-run-node.md index 0cf5334c..bb7285b4 100644 --- a/docs/server/source/dev-and-test/setup-run-node.md +++ b/docs/server/source/dev-and-test/setup-run-node.md @@ -7,25 +7,27 @@ The BigchainDB core dev team develops BigchainDB on recent Ubuntu and Fedora dis ## Option A: Using a Local Dev Machine -First, read through the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to setup a machine for developing and testing BigchainDB. +Read through the BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md). It outlines the steps to setup a machine for developing and testing BigchainDB. -Next, create a default BigchainDB config file (in `$HOME/.bigchaindb`): +### With RethinkDB + +Create a default BigchainDB config file (in `$HOME/.bigchaindb`): ```text -bigchaindb -y configure +$ bigchaindb -y configure rethinkdb ``` Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) Start RethinkDB using: ```text -rethinkdb +$ rethinkdb ``` You can verify that RethinkDB is running by opening the RethinkDB web interface in your web browser. It should be at [http://localhost:8080/](http://localhost:8080/). To run BigchainDB Server, do: ```text -bigchaindb start +$ bigchaindb start ``` You can [run all the unit tests](running-unit-tests.html) to test your installation. @@ -33,13 +35,37 @@ You can [run all the unit tests](running-unit-tests.html) to test your installat The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md) has more details about how to contribute. -## Option B: Using a Dev Machine on Cloud9 +### With MongoDB -Ian Worrall of [Encrypted Labs](http://www.encryptedlabs.com/) wrote a document (PDF) explaining how to set up a BigchainDB (Server) dev machine on Cloud9: +Create a default BigchainDB config file (in `$HOME/.bigchaindb`): +```text +$ bigchaindb -y configure mongodb +``` -[Download that document from GitHub](https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/docs/server/source/_static/cloud9.pdf) +Note: [The BigchainDB CLI](../server-reference/bigchaindb-cli.html) and the [BigchainDB Configuration Settings](../server-reference/configuration.html) are documented elsewhere. (Click the links.) -## Option C: Using a Local Dev Machine and Docker +Start MongoDB __3.4+__ using: +```text +$ mongod --replSet=bigchain-rs +``` + +You can verify that MongoDB is running correctly by checking the output of the +previous command for the line: +```text +waiting for connections on port 27017 +``` + +To run BigchainDB Server, do: +```text +$ bigchaindb start +``` + +You can [run all the unit tests](running-unit-tests.html) to test your installation. + +The BigchainDB [CONTRIBUTING.md file](https://github.com/bigchaindb/bigchaindb/blob/master/CONTRIBUTING.md) has more details about how to contribute. + + +## Option B: Using a Local Dev Machine and Docker You need to have recent versions of [Docker Engine](https://docs.docker.com/engine/installation/) and (Docker) [Compose](https://docs.docker.com/compose/install/). @@ -50,6 +76,8 @@ Build the images: docker-compose build ``` +### Docker with RethinkDB + **Note**: If you're upgrading BigchainDB and have previously already built the images, you may need to rebuild them after the upgrade to install any new dependencies. @@ -62,7 +90,7 @@ docker-compose up -d rdb The RethinkDB web interface should be accessible at . Depending on which platform, and/or how you are running docker, you may need to change `localhost` for the `ip` of the machine that is running docker. As a -dummy example, if the `ip` of that machine was `0.0.0.0`, you would accees the +dummy example, if the `ip` of that machine was `0.0.0.0`, you would access the web interface at: . Start a BigchainDB node: @@ -83,6 +111,40 @@ If you wish to run the tests: docker-compose run --rm bdb py.test -v -n auto ``` +### Docker with MongoDB + +Start MongoDB: + +```bash +docker-compose up -d mdb +``` + +MongoDB should now be up and running. You can check the port binding for the +MongoDB driver port using: +```bash +$ docker-compose port mdb 27017 +``` + +Start a BigchainDB node: + +```bash +docker-compose up -d bdb-mdb +``` + +You can monitor the logs: + +```bash +docker-compose logs -f bdb-mdb +``` + +If you wish to run the tests: + +```bash +docker-compose run --rm bdb-mdb py.test -v --database-backend=mongodb +``` + +### Accessing the HTTP API + A quick check to make sure that the BigchainDB server API is operational: ```bash @@ -123,3 +185,9 @@ root: ```bash curl 0.0.0.0:32772 ``` + +## Option C: Using a Dev Machine on Cloud9 + +Ian Worrall of [Encrypted Labs](http://www.encryptedlabs.com/) wrote a document (PDF) explaining how to set up a BigchainDB (Server) dev machine on Cloud9: + +[Download that document from GitHub](https://raw.githubusercontent.com/bigchaindb/bigchaindb/master/docs/server/source/_static/cloud9.pdf) diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index dfe485c5..3c6a78f3 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -2,34 +2,55 @@ This page has instructions to set up a single stand-alone BigchainDB node for learning or experimenting. Instructions for other cases are [elsewhere](introduction.html). We will assume you're using Ubuntu 16.04 or similar. If you're not using Linux, then you might try [running BigchainDB with Docker](appendices/run-with-docker.html). -A. [Install RethinkDB Server](https://rethinkdb.com/docs/install/ubuntu/) +A. Install the database backend. -B. Open a Terminal and run RethinkDB Server with the command: +[Install RethinkDB Server](https://rethinkdb.com/docs/install/ubuntu/) or +[Install MongoDB Server 3.4+](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/) + +B. Run the database backend. Open a Terminal and run the command: + +with RethinkDB ```text -rethinkdb +$ rethinkdb +``` + +with MongoDB __3.4+__ +```text +$ mongod --replSet=bigchain-rs ``` C. Ubuntu 16.04 already has Python 3.5, so you don't need to install it, but you do need to install some other things: ```text -sudo apt-get update -sudo apt-get install g++ python3-dev libffi-dev +$ sudo apt-get update +$ sudo apt-get install g++ python3-dev libffi-dev ``` D. Get the latest version of pip and setuptools: ```text -sudo apt-get install python3-pip -sudo pip3 install --upgrade pip setuptools +$ sudo apt-get install python3-pip +$ sudo pip3 install --upgrade pip setuptools ``` E. Install the `bigchaindb` Python package from PyPI: ```text -sudo pip3 install bigchaindb +$ sudo pip3 install bigchaindb ``` -F. Configure and run BigchainDB Server: +F. Configure the BigchainDB Server: and run BigchainDB Server: + +with RethinkDB ```text -bigchaindb -y configure -bigchaindb start +$ bigchaindb -y configure rethinkdb +``` + +with MongoDB +```text +$ bigchaindb -y configure mongodb +``` + +G. Run the BigchainDB Server: +```text +$ bigchaindb start ``` That's it! diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index 869ef804..f647b4bf 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -15,18 +15,22 @@ Show the version number. `bigchaindb -v` does the same thing. ## bigchaindb configure -Generate a local config file (which can be used to set some or all [BigchainDB node configuration settings](configuration.html)). It will auto-generate a public-private keypair and then ask you for the values of other configuration settings. If you press Enter for a value, it will use the default value. +Generate a local configuration file (which can be used to set some or all [BigchainDB node configuration settings](configuration.html)). It will auto-generate a public-private keypair and then ask you for the values of other configuration settings. If you press Enter for a value, it will use the default value. + +Since BigchainDB supports multiple databases you need to always specify the +database backend that you want to use. At this point only two database backends +are supported: `rethinkdb` and `mongodb`. If you use the `-c` command-line option, it will generate the file at the specified path: ```text -bigchaindb -c path/to/new_config.json configure +bigchaindb -c path/to/new_config.json configure rethinkdb ``` If you don't use the `-c` command-line option, the file will be written to `$HOME/.bigchaindb` (the default location where BigchainDB looks for a config file, if one isn't specified). If you use the `-y` command-line option, then there won't be any interactive prompts: it will just generate a keypair and use the default values for all the other configuration settings. ```text -bigchaindb -y configure +bigchaindb -y configure rethinkdb ``` @@ -83,3 +87,25 @@ Set the number of replicas (of each shard) in the underlying datastore. For exam ```text $ bigchaindb set-replicas 3 ``` + +## bigchaindb add-replicas + +This command is specific to MongoDB so it will only run if BigchainDB is +configured with `mongodb` as the backend. + +This command is used to add nodes to a BigchainDB cluster. It accepts a list of +space separated hosts in the form _hostname:port_: +```text +$ bigchaindb add-replicas server1.com:27017 server2.com:27017 server3.com:27017 +``` + +## bigchaindb remove-replicas + +This command is specific to MongoDB so it will only run if BigchainDB is +configured with `mongodb` as the backend. + +This command is used to remove nodes from a BigchainDB cluster. It accepts a +list of space separated hosts in the form _hostname:port_: +```text +$ bigchaindb remove-replicas server1.com:27017 server2.com:27017 server3.com:27017 +``` From 9913929b9d3c12af8a073445016a4ec3e0512940 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Tue, 31 Jan 2017 16:14:18 +0100 Subject: [PATCH 10/10] simplify run_configure --- bigchaindb/commands/bigchain.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 2fc8df70..272f8107 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -89,12 +89,7 @@ def run_configure(args, skip_if_exists=False): # select the correct config defaults based on the backend print('Generating default configuration for backend {}' .format(args.backend)) - database = {} - if args.backend == 'rethinkdb': - database = bigchaindb._database_rethinkdb - elif args.backend == 'mongodb': - database = bigchaindb._database_mongodb - conf['database'] = database + conf['database'] = bigchaindb._database_map[args.backend] if not args.yes: for key in ('bind', ):