Merge remote-tracking branch 'remotes/origin/develop' into feat/127/crypto-conditions-ilp-sha256-fulfillments

bigchaindb/block.py

@@ -9,7 +9,6 @@ from bigchaindb import Bigchain
 from bigchaindb.monitor import Monitor
 
 
-
 logger = logging.getLogger(__name__)
 
 monitor = Monitor()
@@ -27,6 +26,7 @@ class Block(object):
         self.q_tx_validated = mp.Queue()
         self.q_tx_delete = mp.Queue()
         self.q_block = mp.Queue()
+        self.initialized = mp.Event()
 
     def filter_by_assignee(self):
         """
@@ -190,6 +190,10 @@ class Block(object):
 
         logger.info('starting block module...')
         self.q_new_transaction = self._q_new_transaction
+
+        # signal initialization complete
+        self.initialized.set()
+
         self._start()
         logger.info('exiting block module...')
 

bigchaindb/processes.py

@@ -62,9 +62,16 @@ class Processes(object):
                 pass
 
     def start(self):
+        logger.info('Initializing BigchainDB...')
+
         # instantiate block and voter
         block = Block(self.q_new_transaction)
 
+        # start the web api
+        webapi = server.create_app()
+        p_webapi = mp.Process(name='webapi', target=webapi.run, kwargs={'host': 'localhost'})
+        p_webapi.start()
+
         # initialize the processes
         p_map_bigchain = mp.Process(name='bigchain_mapper', target=self.map_bigchain)
         p_map_backlog = mp.Process(name='backlog_mapper', target=self.map_backlog)
@@ -82,8 +89,8 @@ class Processes(object):
         logger.info('starting voter')
         p_voter.start()
 
-        # start the web api
+        # start message
-        webapi = server.create_app()
+        block.initialized.wait()
-        p_webapi = mp.Process(name='webapi', target=webapi.run, kwargs={'host': 'localhost'})
+        p_voter.initialized.wait()
-        p_webapi.start()
+        logger.info('Initialization complete. BigchainDB ready and waiting for events.')
-
+        logger.info('You can send events through the API documented at http://docs.bigchaindb.apiary.io/')

bigchaindb/voter.py

@@ -62,6 +62,7 @@ class Voter(object):
         self.q_voted_block = mp.Queue()
         self.v_previous_block_id = mp.Value(ctypes.c_char_p)
         self.v_previous_block_number = mp.Value(ctypes.c_uint64)
+        self.initialized = mp.Event()
 
     def feed_blocks(self):
         """
@@ -88,6 +89,9 @@ class Voter(object):
         b = Bigchain()
 
         logger.info('voter waiting for new blocks')
+        # signal initialization complete
+        self.initialized.set()
+
         while True:
             new_block = self.q_blocks_to_validate.get()
 

bigchaindb/web/server.py

@@ -18,4 +18,3 @@ def create_app(debug=False):
     app.config['bigchain'] = Bigchain()
     app.register_blueprint(views.basic_views, url_prefix='/api/v1')
     return app
-

docs/source/admin.md

@@ -1,62 +0,0 @@
-# Server/Cluster Deployment and Administration
-This section covers everything which might concern a BigchainDB server/cluster administrator:
-* deployment
-* security
-* monitoring
-* troubleshooting
-
-
-
-## Deploying a local cluster
-One of the advantages of RethinkDB as the storage backend is the easy installation. Developers like to have everything locally, so let's install a local storage backend cluster from scratch.
-Here is an example to run a cluster assuming rethinkdb is already [installed](installing-server.html#installing-and-running-rethinkdb-server-on-ubuntu-12-04) in
-your system:
-
-    # preparing two additional nodes
-    # remember, that the user who starts rethinkdb must have write access to the paths
-    mkdir -p /path/to/node2
-    mkdir -p /path/to/node3
-
-    # then start your additional nodes
-    rethinkdb --port-offset 1 --directory /path/to/node2 --join localhost:29015
-    rethinkdb --port-offset 2 --directory /path/to/node3 --join localhost:29015
-
-That's all, folks! Cluster is up and running. Check it out in your browser at http://localhost:8080, which opens the console.
-
-Now you can install BigchainDB and run it against the storage backend!
-
-## Securing the storage backend
-We have turned on the bind=all option for connecting other nodes and making RethinkDB accessible from outside the server. Unfortunately this is insecure. So, we will need to block RethinkDB off from the Internet. But we need to allow access to its services from authorized computers.
-
-For the cluster port, we will use a firewall to enclose our cluster. For the web management console and the driver port, we will use SSH tunnels to access them from outside the server. SSH tunnels redirect requests on a client computer to a remote computer over SSH, giving the client access to all of the services only available on the remote server's localhost name space.
-
-Repeat these steps on all your RethinkDB servers.
-
-First, block all outside connections:
-
-    # the web management console
-    sudo iptables -A INPUT -i eth0 -p tcp --dport 8080 -j DROP
-    sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 8080 -j ACCEPT
-
-    # the driver port
-    sudo iptables -A INPUT -i eth0 -p tcp --dport 28015 -j DROP
-    sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 28015 -j ACCEPT
-
-    # the communication port
-    sudo iptables -A INPUT -i eth0 -p tcp --dport 29015 -j DROP
-    sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 29015 -j ACCEPT
-
-Save the iptables config:
-
-    sudo apt-get update
-    sudo apt-get install iptables-persistent
-
-More about iptables you can find in the [man pages](http://linux.die.net/man/8/iptables).
-
-## Monitoring the storage backend
-Monitoring is pretty simple. You can do this via the [monitoring url](http://localhost:8080). Here you see the complete behaviour of all nodes.
-One annotation: if you play around with  replication the number of transaction will increase. So for the real throughput you should devide the number of transactions by the number of replicas.
-
-## Troubleshooting
-Since every software may have some minor issues we are not responsible for the storage backend.
-If your nodes in a sharded and replicated cluster are not in sync, it may help if you delete the data of the affected node and restart the node. If there are no other problems your node will come back and sync within a couple of minutes. You can verify this by monitoring the cluster via the [monitoring url](http://localhost:8080).

docs/source/index.rst

@@ -19,11 +19,12 @@ Table of Contents
    bigchaindb-cli
    http-client-server-api
    python-driver-api-examples
-   admin
+   local-rethinkdb-cluster
    cryptography
    models
    json-serialization
    developer-interface
+   consensus
    monitoring
    licenses
    contributing

docs/source/installing-server.md

@@ -4,9 +4,9 @@ We're developing BigchainDB Server on Ubuntu 14.04, but it should work on any OS
 
 BigchainDB Server is intended to be run on each server in a large distributed cluster of servers; it's not very useful running by itself on a single computer. That's _why_ we're developing it on Ubuntu 14.04: it's one of the more common server operating systems.
 
-Mac OS X users may get the best results [running BigchainDB Server with Docker](https://bigchaindb.readthedocs.org/en/develop/installing-server.html#run-bigchaindb-with-docker).
+Mac OS X users may get the best results [running BigchainDB Server with Docker](installing-server.html#run-bigchaindb-with-docker).
 
-Windows users may get the best results [running BigchainDB Server in a VM with Vagrant](https://bigchaindb.readthedocs.org/en/develop/installing-server.html#how-to-install-bigchaindb-on-a-vm-with-vagrant).
+Windows users may get the best results [running BigchainDB Server in a VM with Vagrant](installing-server.html#how-to-install-bigchaindb-on-a-vm-with-vagrant).
 
 (BigchainDB clients should run on a much larger array of operating systems.)
 

docs/source/local-rethinkdb-cluster.md

@@ -0,0 +1,54 @@
+# Deploying a Local Multi-Node RethinkDB Cluster
+
+This section explains one way to deploy a multi-node RethinkDB cluster on one machine. You could use such a cluster as a storage backend for a BigchaindB process running on that machine.
+
+## Launching More RethinkDB Nodes
+
+Assuming you've already [installed RethinkDB](installing-server.html#install-and-run-rethinkdb-server) and have one RethinkDB node running, here's how you can launch two more nodes on the same machine. First, prepare two additional nodes. Note that the user who starts RethinkDB must have write access to the created directories:
+
+    mkdir -p /path/to/node2
+    mkdir -p /path/to/node3
+
+then launch two more nodes:
+
+    rethinkdb --port-offset 1 --directory /path/to/node2 --join localhost:29015
+    rethinkdb --port-offset 2 --directory /path/to/node3 --join localhost:29015
+
+You should now have a three-node RethinkDB cluster up and running. You should be able to monitor it at [http://localhost:8080](http://localhost:8080).
+
+Note: if you play around with replication, the number of transactions will increase. For the real throughput, you should divide the number of transactions by the number of replicas.
+
+## Securing the Cluster
+
+We have turned on the `bind=all` option for connecting other nodes and making RethinkDB accessible from outside the server. Unfortunately, that is insecure, so we need to block access to the RethinkDB cluster from the Internet. At the same time, we need to allow access to its services from authorized computers.
+
+For the cluster port, we will use a firewall to enclose our cluster. For the web management console and the driver port, we will use SSH tunnels to access them from outside the server. SSH tunnels redirect requests on a client computer to a remote computer over SSH, giving the client access to all of the services only available on the remote server's localhost name space.
+
+Repeat the following steps on all your RethinkDB servers.
+
+First, block all outside connections:
+
+    # the web management console
+    sudo iptables -A INPUT -i eth0 -p tcp --dport 8080 -j DROP
+    sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 8080 -j ACCEPT
+
+    # the driver port
+    sudo iptables -A INPUT -i eth0 -p tcp --dport 28015 -j DROP
+    sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 28015 -j ACCEPT
+
+    # the communication port
+    sudo iptables -A INPUT -i eth0 -p tcp --dport 29015 -j DROP
+    sudo iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 29015 -j ACCEPT
+
+Save the `iptables` config:
+
+    sudo apt-get update
+    sudo apt-get install iptables-persistent
+
+You can find out more about `iptables` in the [man pages](http://linux.die.net/man/8/iptables).
+
+## Troubleshooting
+
+You can get help with RethinkDB from [RethinkDB experts](https://rethinkdb.com/services/) and [the RethinkDB community](https://rethinkdb.com/community/).
+
+If your nodes in a sharded and replicated cluster are not in sync, it may help if you delete the data of the affected node and restart the node. If there are no other problems, your node will come back and sync within a couple of minutes. You can verify this by monitoring the cluster via the monitoring url: [http://localhost:8080](http://localhost:8080).

docs/source/running-unit-tests.md

@@ -2,7 +2,7 @@
 
 Once you've installed BigchainDB Server, you may want to run all the unit tests. This section explains how.
 
-First of all, if you installed BigchainDB Server using `pip` (i.e. by getting the package from PyPI), then you didn't install the tests. Before you can run all the unit tests, you must [install BigchainDB from source](http://bigchaindb.readthedocs.org/en/develop/installing-server.html#how-to-install-bigchaindb-from-source).
+First of all, if you installed BigchainDB Server using `pip` (i.e. by getting the package from PyPI), then you didn't install the tests. Before you can run all the unit tests, you must [install BigchainDB from source](installing-server.html#how-to-install-bigchaindb-from-source).
 
 To run all the unit tests, first make sure you have RethinkDB running:
 ```text

setup.py

@@ -4,7 +4,7 @@ BigchainDB: A Scalable Blockchain Database
 For full docs visit https://bigchaindb.readthedocs.org
 
 """
-from setuptools import setup
+from setuptools import setup, find_packages
 
 tests_require = [
     'pytest',
@@ -56,12 +56,7 @@ setup(
         'Operating System :: POSIX :: Linux',
     ],
 
-    packages=[
+    packages=find_packages(exclude=['tests*']),
-        'bigchaindb',
-        'bigchaindb.commands',
-        'bigchaindb.db',
-        'bigchaindb.web'
-    ],
 
     entry_points={
         'console_scripts': [