Merge pull request #258 from bigchaindb/automatic-sharding

Add command to configure number of shards.

bigchaindb/commands/bigchain.py

@@ -13,6 +13,7 @@ import builtins
 
 import logstats
 
+import rethinkdb as r
 
 import bigchaindb
 import bigchaindb.config_utils
@@ -203,6 +204,12 @@ def run_load(args):
     workers.start()
 
 
+def run_set_shards(args):
+    b = bigchaindb.Bigchain()
+    r.table('bigchain').reconfigure(shards=args.num_shards, replicas=1).run(b.conn)
+    r.table('backlog').reconfigure(shards=args.num_shards, replicas=1).run(b.conn)
+
+
 def main():
     parser = argparse.ArgumentParser(
         description='Control your BigchainDB node.',
@@ -243,6 +250,13 @@ def main():
     subparsers.add_parser('start',
                           help='Start BigchainDB')
 
+    # parser for configuring the number of shards
+    sharding_parser = subparsers.add_parser('set-shards',
+                                            help='Configure number of shards')
+
+    sharding_parser.add_argument('num_shards', metavar='num_shards', type=int, default=1,
+                                 help='Number of shards')
+
     load_parser = subparsers.add_parser('load',
                                         help='Write transactions to the backlog')
 

deploy-cluster-aws/awsdeploy.sh

@@ -131,6 +131,7 @@ if [ "$WHAT_TO_DEPLOY" == "servers" ]; then
     # this will only be sent to one of the nodes, see the
     # definition of init_bigchaindb() in fabfile.py to see why.
     fab init_bigchaindb
+    fab configure_sharding:$NUM_NODES
 else
     # Deploying clients
     # The only thing to configure on clients is the api_endpoint

deploy-cluster-aws/fabfile.py

@@ -166,6 +166,12 @@ def init_bigchaindb():
     run('bigchaindb init', pty=False)
 
 
+@task
+@hosts(public_dns_names[0])
+def configure_sharding(num_shards):
+    run('bigchaindb sharding {}'.format(num_shards))
+
+
 # Start BigchainDB using screen
 @task
 @parallel

docs/source/bigchaindb-cli.md

@@ -43,3 +43,10 @@ This command is used to run benchmarking tests. You can learn more about it usin
 ```text
 $ bigchaindb load -h
 ```
+
+### bigchaindb sharding
+
+This command is used to configure the number of shards in the underlying datastore, for example:
+```text
+$ bigchaindb sharding 3
+```

tests/test_commands.py

@@ -1,11 +1,12 @@
 import json
 from unittest.mock import Mock, patch
 from argparse import Namespace
-from pprint import pprint
 import copy
 
 import pytest
 
+from tests.db.conftest import setup_database
+
 
 @pytest.fixture
 def mock_run_configure(monkeypatch):
@@ -225,3 +226,22 @@ def test_start_rethinkdb_exits_when_cannot_start(mock_popen):
     with pytest.raises(exceptions.StartupError):
         utils.start_rethinkdb()
 
+
+def test_configure_sharding(b):
+    import rethinkdb as r
+    from bigchaindb.commands.bigchain import run_set_shards
+
+    # change number of shards
+    args = Namespace(num_shards=3)
+    run_set_shards(args)
+
+    # retrieve table configuration
+    table_config = list(r.db('rethinkdb')
+                        .table('table_config')
+                        .filter(r.row['db'] == b.dbname)
+                        .run(b.conn))
+
+    # check shard configuration
+    for table in table_config:
+        if table['name'] in ['backlog', 'bigchain']:
+            assert len(table['shards']) == 3