From cc6129103f8d69648ad89a622e45cfac540764de Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 16 Jun 2016 14:44:57 +0200 Subject: [PATCH 1/3] WIP: added bigchaindb set-replicas cmd w/ non-working test --- bigchaindb/commands/bigchain.py | 27 ++++++++++++++++++++++++--- docs/source/servers/bigchaindb-cli.md | 9 ++++++++- tests/test_commands.py | 16 ++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index 8a9c6888..bbdffedb 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -207,8 +207,20 @@ def run_load(args): def run_set_shards(args): b = bigchaindb.Bigchain() - r.table('bigchain').reconfigure(shards=args.num_shards, replicas=1).run(b.conn) - r.table('backlog').reconfigure(shards=args.num_shards, replicas=1).run(b.conn) + for table in ['bigchain', 'backlog']: + # See https://www.rethinkdb.com/api/python/config/ + table_config = r.table(table).config().run(b.conn) + num_replicas = len(table_config['shards'][0]['replicas']) + r.table(table).reconfigure(shards=args.num_shards, replicas=num_replicas).run(b.conn) + + +def run_set_replicas(args): + b = bigchaindb.Bigchain() + for table in ['bigchain', 'backlog']: + # See https://www.rethinkdb.com/api/python/config/ + table_config = r.table(table).config().run(b.conn) + num_shards = len(table_config['shards']) + r.table(table).reconfigure(shards=num_shards, replicas=args.num_replicas).run(b.conn) def main(): @@ -255,9 +267,18 @@ def main(): sharding_parser = subparsers.add_parser('set-shards', help='Configure number of shards') - sharding_parser.add_argument('num_shards', metavar='num_shards', type=int, default=1, + sharding_parser.add_argument('num_shards', metavar='num_shards', + type=int, default=1, help='Number of shards') + # parser for configuring the number of replicas + replicas_parser = subparsers.add_parser('set-replicas', + help='Configure number of replicas') + + replicas_parser.add_argument('num_replicas', metavar='num_replicas', + type=int, default=1, + help='Number of replicas (i.e. the replication factor)') + load_parser = subparsers.add_parser('load', help='Write transactions to the backlog') diff --git a/docs/source/servers/bigchaindb-cli.md b/docs/source/servers/bigchaindb-cli.md index ad9fd5a0..6ca22328 100644 --- a/docs/source/servers/bigchaindb-cli.md +++ b/docs/source/servers/bigchaindb-cli.md @@ -49,4 +49,11 @@ $ bigchaindb load -h This command is used to set the number of shards in the underlying datastore. For example, the following command will set the number of shards to four: ```text $ bigchaindb set-shards 4 -``` \ No newline at end of file +``` + +### bigchaindb set-replicas + +This command is used to set the number of replicas (of each shard) in the underlying datastore. For example, the following command will set the number of replicas to three (i.e. it will set the replication factor to three): +```text +$ bigchaindb set-replicas 3 +``` diff --git a/tests/test_commands.py b/tests/test_commands.py index 12c1350e..37af2b0e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -245,3 +245,19 @@ def test_set_shards(b): for table in table_config: if table['name'] in ['backlog', 'bigchain']: assert len(table['shards']) == 3 + + +def test_set_replicas(b): + import rethindb as r + from bigchaindb.commands.bigchain import run_set_replicas + + # set the number of replicas + args = Namespace(num_replicas=2) + run_set_replicas(args) + + # check that the replication factor got set to 2 in all tables + for table in ['backlog', 'bigchain']: + # See https://www.rethinkdb.com/api/python/config/ + table_config = r.table(table).config().run(b.conn) + num_replicas = len(table_config['shards'][0]['replicas']) + assert num_replicas == 2 From 8d204a50ebad9a42cd5987dbad47d0c4ad5b83c6 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 17 Jun 2016 15:51:45 +0200 Subject: [PATCH 2/3] Catch rethinkdb exceptions when configuring shards and replicas --- bigchaindb/commands/bigchain.py | 10 ++++++++-- tests/test_commands.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index bbdffedb..b495b13a 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -211,7 +211,10 @@ def run_set_shards(args): # See https://www.rethinkdb.com/api/python/config/ table_config = r.table(table).config().run(b.conn) num_replicas = len(table_config['shards'][0]['replicas']) - r.table(table).reconfigure(shards=args.num_shards, replicas=num_replicas).run(b.conn) + try: + r.table(table).reconfigure(shards=args.num_shards, replicas=num_replicas).run(b.conn) + except r.ReqlOpFailedError as e: + logger.warn(e) def run_set_replicas(args): @@ -220,7 +223,10 @@ def run_set_replicas(args): # See https://www.rethinkdb.com/api/python/config/ table_config = r.table(table).config().run(b.conn) num_shards = len(table_config['shards']) - r.table(table).reconfigure(shards=num_shards, replicas=args.num_replicas).run(b.conn) + try: + r.table(table).reconfigure(shards=num_shards, replicas=args.num_replicas).run(b.conn) + except r.ReqlOpFailedError as e: + logger.warn(e) def main(): diff --git a/tests/test_commands.py b/tests/test_commands.py index 37af2b0e..562df51c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -248,7 +248,7 @@ def test_set_shards(b): def test_set_replicas(b): - import rethindb as r + import rethinkdb as r from bigchaindb.commands.bigchain import run_set_replicas # set the number of replicas From 6208305ae48a678a44aba85df625d4f663cf8d0a Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 20 Jun 2016 13:24:14 +0200 Subject: [PATCH 3/3] added tests --- tests/test_commands.py | 99 ++++++++++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 24 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 562df51c..e803f7c9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -4,8 +4,7 @@ from argparse import Namespace import copy import pytest - -from tests.db.conftest import setup_database +import rethinkdb @pytest.fixture @@ -227,37 +226,89 @@ def test_start_rethinkdb_exits_when_cannot_start(mock_popen): utils.start_rethinkdb() -def test_set_shards(b): - import rethinkdb as r +@patch('rethinkdb.ast.Table.reconfigure') +def test_set_shards(mock_reconfigure, monkeypatch, b): from bigchaindb.commands.bigchain import run_set_shards - # set the number of shards + # this will mock the call to retrieve the database config + # we will set it to return one replica + def mockreturn_one_replica(self, conn): + return {'shards': [{'replicas': [1]}]} + + monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_one_replica) + args = Namespace(num_shards=3) + run_set_shards(args) + mock_reconfigure.assert_called_with(replicas=1, shards=3) + + # this will mock the call to retrieve the database config + # we will set it to return three replica + def mockreturn_three_replicas(self, conn): + return {'shards': [{'replicas': [1, 2, 3]}]} + + monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_replicas) + run_set_shards(args) + mock_reconfigure.assert_called_with(replicas=3, shards=3) + + +@patch('logging.Logger.warn') +def test_set_shards_raises_exception(mock_log, monkeypatch, b): + from bigchaindb.commands.bigchain import run_set_shards + + # test that we are correctly catching the exception + def mock_raise(*args, **kwargs): + raise rethinkdb.ReqlOpFailedError('') + + def mockreturn_one_replica(self, conn): + return {'shards': [{'replicas': [1]}]} + + monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_one_replica) + monkeypatch.setattr(rethinkdb.ast.Table, 'reconfigure', mock_raise) + args = Namespace(num_shards=3) run_set_shards(args) - # retrieve table configuration - table_config = list(r.db('rethinkdb') - .table('table_config') - .filter(r.row['db'] == b.dbname) - .run(b.conn)) - - # check that the number of shards got set to the correct value - for table in table_config: - if table['name'] in ['backlog', 'bigchain']: - assert len(table['shards']) == 3 + assert mock_log.called -def test_set_replicas(b): - import rethinkdb as r +@patch('rethinkdb.ast.Table.reconfigure') +def test_set_replicas(mock_reconfigure, monkeypatch, b): from bigchaindb.commands.bigchain import run_set_replicas - # set the number of replicas + # this will mock the call to retrieve the database config + # we will set it to return two shards + def mockreturn_two_shards(self, conn): + return {'shards': [1, 2]} + + monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_two_shards) + args = Namespace(num_replicas=2) + run_set_replicas(args) + mock_reconfigure.assert_called_with(replicas=2, shards=2) + + # this will mock the call to retrieve the database config + # we will set it to return three shards + def mockreturn_three_shards(self, conn): + return {'shards': [1, 2, 3]} + + monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_three_shards) + run_set_replicas(args) + mock_reconfigure.assert_called_with(replicas=2, shards=3) + + +@patch('logging.Logger.warn') +def test_set_replicas_raises_exception(mock_log, monkeypatch, b): + from bigchaindb.commands.bigchain import run_set_replicas + + # test that we are correctly catching the exception + def mock_raise(*args, **kwargs): + raise rethinkdb.ReqlOpFailedError('') + + def mockreturn_two_shards(self, conn): + return {'shards': [1, 2]} + + monkeypatch.setattr(rethinkdb.RqlQuery, 'run', mockreturn_two_shards) + monkeypatch.setattr(rethinkdb.ast.Table, 'reconfigure', mock_raise) + args = Namespace(num_replicas=2) run_set_replicas(args) - # check that the replication factor got set to 2 in all tables - for table in ['backlog', 'bigchain']: - # See https://www.rethinkdb.com/api/python/config/ - table_config = r.table(table).config().run(b.conn) - num_replicas = len(table_config['shards'][0]['replicas']) - assert num_replicas == 2 + assert mock_log.called