diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6ea6a87..5dc7002a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,9 +38,9 @@ Familiarize yourself with how we do coding and documentation in the BigchainDB p ### Step 2 - Install some Dependencies * [Install RethinkDB Server](https://rethinkdb.com/docs/install/) -* Make sure you have Python 3.4+ (maybe in a virtualenv) -* [Install BigchaindB Server's OS-level dependencies](http://bigchaindb.readthedocs.io/en/latest/nodes/setup-run-node.html#install-bigchaindb-server) -* [Make sure you have the latest version of pip](http://bigchaindb.readthedocs.io/en/latest/nodes/setup-run-node.html#how-to-install-bigchaindb-with-pip) +* Make sure you have Python 3.4+ (preferably in a virtualenv) +* [Install BigchaindB Server's OS-level dependencies](http://bigchaindb.readthedocs.io/en/latest/appendices/install-os-level-deps.html) +* [Make sure you have the latest Python 3 version of pip and setuptools](http://bigchaindb.readthedocs.io/en/latest/appendices/install-latest-pip.html) ### Step 3 - Fork bigchaindb on GitHub diff --git a/README.md b/README.md index 04fe222f..dc3e0036 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ We're hiring! [Learn more](https://github.com/bigchaindb/org/blob/master/engjob. ## Get Started ### [Quickstart](http://bigchaindb.readthedocs.io/en/latest/quickstart.html) -### [Set Up and Run a BigchainDB Node](http://bigchaindb.readthedocs.io/en/latest/nodes/setup-run-node.html) -### [Run BigchainDB with Docker](http://bigchaindb.readthedocs.io/en/latest/nodes/run-with-docker.html) +### [Set Up & Run a Dev/Test Node](http://bigchaindb.readthedocs.io/en/latest/dev-and-test/setup-run-node.html) +### [Run BigchainDB with Docker](http://bigchaindb.readthedocs.io/en/latest/appendices/run-with-docker.html) ## Links for Everyone * [BigchainDB.com](https://www.bigchaindb.com/) - the main BigchainDB website, including newsletter signup diff --git a/bigchaindb/block.py b/bigchaindb/block.py deleted file mode 100644 index 2267ef0e..00000000 --- a/bigchaindb/block.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -import multiprocessing as mp -import queue - -import rethinkdb as r - -import bigchaindb -from bigchaindb import Bigchain -from bigchaindb.monitor import Monitor -from bigchaindb.util import ProcessGroup - - -logger = logging.getLogger(__name__) - - -class BlockDeleteRevert(object): - - def __init__(self, q_delete_to_revert): - self.q_delete_to_revert = q_delete_to_revert - - def write_blocks(self): - """ - Write blocks to the bigchain - """ - - # create bigchain instance - b = Bigchain() - - # Write blocks - while True: - block = self.q_delete_to_revert.get() - - # poison pill - if block == 'stop': - return - - b.write_block(block) - - def kill(self): - for i in range(mp.cpu_count()): - self.q_delete_to_revert.put('stop') - - def start(self): - """ - Initialize, spawn, and start the processes - """ - - # initialize the processes - p_write = ProcessGroup(name='write_blocks', target=self.write_blocks) - - # start the processes - p_write.start() diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index dbdfeddf..1943167b 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -24,7 +24,7 @@ from bigchaindb.exceptions import (StartupError, DatabaseAlreadyExists, KeypairNotFoundException) from bigchaindb.commands import utils -from bigchaindb.processes import Processes +from bigchaindb import processes from bigchaindb import crypto @@ -169,7 +169,6 @@ def run_start(args): sys.exit("Can't start BigchainDB, no keypair found. " 'Did you run `bigchaindb configure`?') - processes = Processes() logger.info('Starting BigchainDB main process') processes.start() diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 7d469504..81f8ed46 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -209,7 +209,7 @@ def write_config(config, filename=None): filename = CONFIG_DEFAULT_PATH with open(filename, 'w') as f: - json.dump(config, f) + json.dump(config, f, indent=4) def autoconfigure(filename=None, config=None, force=False): diff --git a/bigchaindb/pipelines/election.py b/bigchaindb/pipelines/election.py new file mode 100644 index 00000000..538a5453 --- /dev/null +++ b/bigchaindb/pipelines/election.py @@ -0,0 +1,65 @@ +"""This module takes care of all the logic related to block status. + +Specifically, what happens when a block becomes invalid. The logic is +encapsulated in the ``Election`` class, while the sequence of actions +is specified in ``create_pipeline``. +""" +import logging + +import rethinkdb as r +from multipipes import Pipeline, Node + +from bigchaindb.pipelines.utils import ChangeFeed +from bigchaindb import Bigchain + + +logger = logging.getLogger(__name__) + + +class Election: + + def __init__(self): + self.bigchain = Bigchain() + + def check_for_quorum(self, next_vote): + """ + Checks if block has enough invalid votes to make a decision + """ + next_block = r.table('bigchain')\ + .get(next_vote['vote']['voting_for_block'])\ + .run(self.bigchain.conn) + if self.bigchain.block_election_status(next_block) == self.bigchain.BLOCK_INVALID: + return next_block + + def requeue_transactions(self, invalid_block): + """ + Liquidates transactions from invalid blocks so they can be processed again + """ + logger.info('Rewriting %s transactions from invalid block %s', + len(invalid_block['block']['transactions']), + invalid_block['id']) + for tx in invalid_block['block']['transactions']: + self.bigchain.write_transaction(tx) + return invalid_block + + +def get_changefeed(): + return ChangeFeed(table='votes', operation='insert') + + +def create_pipeline(): + election = Election() + + election_pipeline = Pipeline([ + Node(election.check_for_quorum), + Node(election.requeue_transactions) + ]) + + return election_pipeline + + +def start(): + pipeline = create_pipeline() + pipeline.setup(indata=get_changefeed()) + pipeline.start() + return pipeline diff --git a/bigchaindb/processes.py b/bigchaindb/processes.py index 80ed43ad..4b8aa0eb 100644 --- a/bigchaindb/processes.py +++ b/bigchaindb/processes.py @@ -1,13 +1,8 @@ import logging import multiprocessing as mp -import rethinkdb as r - import bigchaindb -from bigchaindb.pipelines import block, vote -from bigchaindb import Bigchain -from bigchaindb.voter import Election -from bigchaindb.block import BlockDeleteRevert +from bigchaindb.pipelines import vote, block, election from bigchaindb.web import server @@ -26,56 +21,23 @@ BANNER = """ """ -class Processes(object): +def start(): + logger.info('Initializing BigchainDB...') - def __init__(self): - # initialize the class - self.q_block_new_vote = mp.Queue() - self.q_revert_delete = mp.Queue() + # start the processes + logger.info('Starting block') + block.start() - def map_bigchain(self): - # listen to changes on the bigchain and redirect the changes - # to the correct queues + logger.info('Starting voter') + vote.start() - # create a bigchain instance - b = Bigchain() + logger.info('Starting election') + election.start() - for change in r.table('bigchain').changes().run(b.conn): + # start the web api + app_server = server.create_server(bigchaindb.config['server']) + p_webapi = mp.Process(name='webapi', target=app_server.run) + p_webapi.start() - # delete - if change['new_val'] is None: - # this should never happen in regular operation - self.q_revert_delete.put(change['old_val']) - - # update (new vote) - elif change['new_val'] is not None and change['old_val'] is not None: - self.q_block_new_vote.put(change['new_val']) - - def start(self): - logger.info('Initializing BigchainDB...') - - delete_reverter = BlockDeleteRevert(self.q_revert_delete) - - # start the web api - app_server = server.create_server(bigchaindb.config['server']) - p_webapi = mp.Process(name='webapi', target=app_server.run) - p_webapi.start() - - # initialize the processes - p_map_bigchain = mp.Process(name='bigchain_mapper', target=self.map_bigchain) - p_block_delete_revert = mp.Process(name='block_delete_revert', target=delete_reverter.start) - p_election = Election(self.q_block_new_vote) - # start the processes - logger.info('starting bigchain mapper') - p_map_bigchain.start() - logger.info('starting block') - block.start() - p_block_delete_revert.start() - - logger.info('starting voter') - vote.start() - logger.info('starting election') - p_election.start() - - # start message - logger.info(BANNER.format(bigchaindb.config['server']['bind'])) + # start message + logger.info(BANNER.format(bigchaindb.config['server']['bind'])) diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 8492cc81..67853ed2 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -169,8 +169,8 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): Reference: { "id": "", - "version": "transaction version number", "transaction": { + "version": "transaction version number", "fulfillments": [ { "current_owners": ["list of "], @@ -278,6 +278,7 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): }) tx = { + 'version': 1, 'fulfillments': fulfillments, 'conditions': conditions, 'operation': operation, @@ -291,7 +292,6 @@ def create_tx(current_owners, new_owners, inputs, operation, payload=None): # create the transaction transaction = { 'id': tx_hash, - 'version': 1, 'transaction': tx } @@ -479,7 +479,7 @@ def get_fulfillment_message(transaction, fulfillment, serialized=False): 'operation': transaction['transaction']['operation'], 'timestamp': transaction['transaction']['timestamp'], 'data': transaction['transaction']['data'], - 'version': transaction['version'], + 'version': transaction['transaction']['version'], 'id': transaction['id'] } # and the condition which needs to be retrieved from the output of a previous transaction diff --git a/bigchaindb/voter.py b/bigchaindb/voter.py deleted file mode 100644 index 2cef51bf..00000000 --- a/bigchaindb/voter.py +++ /dev/null @@ -1,72 +0,0 @@ -import logging -import multiprocessing as mp - -from bigchaindb import Bigchain - - -logger = logging.getLogger(__name__) - - -class Election(object): - - def __init__(self, q_block_new_vote): - """ - Initialize the class with the needed queues. - - Initialize a queue where blocks with new votes will be held - """ - self.q_block_new_vote = q_block_new_vote - self.q_invalid_blocks = mp.Queue() - - def check_for_quorum(self): - """ - Checks if block has enough invalid votes to make a decision - """ - b = Bigchain() - - while True: - next_block = self.q_block_new_vote.get() - - # poison pill - if next_block == 'stop': - self.q_invalid_blocks.put('stop') - logger.info('clean exit') - return - - if b.block_election_status(next_block) == 'invalid': - self.q_invalid_blocks.put(next_block) - - def requeue_transactions(self): - """ - Liquidates transactions from invalid blocks so they can be processed again - """ - while True: - invalid_block = self.q_invalid_blocks.get() - - # poison pill - if invalid_block == 'stop': - logger.info('clean exit') - return - - b = Bigchain() - for tx in invalid_block['block']['transactions']: - b.write_transaction(tx) - - def kill(self): - """ - Terminate processes - """ - self.q_block_new_vote.put('stop') - - def start(self): - """ - Initialize, spawn, and start the processes - """ - - # initialize the processes - p_quorum_check = mp.Process(name='check_for_quorum', target=self.check_for_quorum) - p_requeue_tx = mp.Process(name='requeue_tx', target=self.requeue_transactions) - - # start the processes - p_quorum_check.start() - p_requeue_tx.start() diff --git a/docs/source/appendices/example-rethinkdb-storage-setups.md b/docs/source/appendices/example-rethinkdb-storage-setups.md index 2c863034..0fc4c273 100755 --- a/docs/source/appendices/example-rethinkdb-storage-setups.md +++ b/docs/source/appendices/example-rethinkdb-storage-setups.md @@ -2,7 +2,7 @@ ## Example Amazon EC2 Setups -We have some scripts for [deploying a _test_ BigchainDB cluster on AWS](../clusters-feds/deploy-on-aws.html). Those scripts include command sequences to set up storage for RethinkDB. +We have some scripts for [deploying a _test_ BigchainDB cluster on AWS](../clusters-feds/aws-testing-cluster.html). Those scripts include command sequences to set up storage for RethinkDB. In particular, look in the file [/deploy-cluster-aws/fabfile.py](https://github.com/bigchaindb/bigchaindb/blob/master/deploy-cluster-aws/fabfile.py), under `def prep_rethinkdb_storage(USING_EBS)`. Note that there are two cases: 1. **Using EBS ([Amazon Elastic Block Store](https://aws.amazon.com/ebs/)).** This is always an option, and for some instance types ("EBS-only"), it's the only option. diff --git a/docs/source/appendices/firewall-notes.md b/docs/source/appendices/firewall-notes.md index 4f1e780c..dca89ac6 100644 --- a/docs/source/appendices/firewall-notes.md +++ b/docs/source/appendices/firewall-notes.md @@ -47,7 +47,7 @@ Port 8080 is the default port used by RethinkDB for its adminstrative web (HTTP) Port 9984 is the default port for the BigchainDB client-server HTTP API (TCP), which is served by Gunicorn HTTP Server. It's _possible_ allow port 9984 to accept inbound traffic from anyone, but we recommend against doing that. Instead, set up a reverse proxy server (e.g. using Nginx) and only allow traffic from there. Information about how to do that can be found [in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) -If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [BigchainDB Configuration Settings](../nodes/configuration.html), where PORT is whatever port you chose (e.g. 9983). +If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [BigchainDB Configuration Settings](../server-reference/configuration.html), where PORT is whatever port you chose (e.g. 9983). You may want to have Gunicorn and the reverse proxy running on different servers, so that both can listen on port 9984. That would also help isolate the effects of a denial-of-service attack. diff --git a/docs/source/appendices/index.rst b/docs/source/appendices/index.rst index 52b9038e..5e9133ae 100755 --- a/docs/source/appendices/index.rst +++ b/docs/source/appendices/index.rst @@ -7,6 +7,9 @@ Appendices .. toctree:: :maxdepth: 1 + install-os-level-deps + install-latest-pip + run-with-docker json-serialization cryptography the-Bigchain-class @@ -15,4 +18,5 @@ Appendices firewall-notes ntp-notes example-rethinkdb-storage-setups - licenses \ No newline at end of file + licenses + install-with-lxd \ No newline at end of file diff --git a/docs/source/appendices/install-latest-pip.md b/docs/source/appendices/install-latest-pip.md new file mode 100644 index 00000000..fac7dbed --- /dev/null +++ b/docs/source/appendices/install-latest-pip.md @@ -0,0 +1,20 @@ +# How to Install the Latest pip and setuptools + +You can check the version of `pip` you're using (in your current virtualenv) by doing: +```text +pip -V +``` + +If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.4, then you must install a `pip` version associated with Python 3.4+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). + +On Ubuntu 14.04, we found that this works: +```text +sudo apt-get install python3-pip +``` + +That should install a Python 3 version of `pip` named `pip3`. If that didn't work, then another way to get `pip3` is to do `sudo apt-get install python3-setuptools` followed by `sudo easy_install3 pip`. + +You can upgrade `pip` (`pip3`) and `setuptools` to the latest versions using: +```text +pip3 install --upgrade pip setuptools +``` \ No newline at end of file diff --git a/docs/source/appendices/install-os-level-deps.md b/docs/source/appendices/install-os-level-deps.md new file mode 100644 index 00000000..3f9f8f57 --- /dev/null +++ b/docs/source/appendices/install-os-level-deps.md @@ -0,0 +1,17 @@ +# How to Install OS-Level Dependencies + +BigchainDB Server has some OS-level dependencies that must be installed. + +On Ubuntu 14.04 and 16.04, we found that the following was enough: +```text +sudo apt-get update +sudo apt-get install g++ python3-dev +``` + +On Fedora 23 and 24, we found that the following was enough: +```text +sudo dnf update +sudo dnf install gcc-c++ redhat-rpm-config python3-devel +``` + +(If you're using a version of Fedora before version 22, you may have to use `yum` instead of `dnf`.) diff --git a/docs/source/appendices/install-with-lxd.md b/docs/source/appendices/install-with-lxd.md new file mode 100644 index 00000000..7cf5cf81 --- /dev/null +++ b/docs/source/appendices/install-with-lxd.md @@ -0,0 +1,43 @@ +# Installing BigchainDB on LXC containers using LXD + +You can visit this link to install LXD (instructions here): [LXD Install](https://linuxcontainers.org/lxd/getting-started-cli/) + +(assumption is that you are using Ubuntu 14.04 for host/container) + +Let us create an LXC container (via LXD) with the following command: + +`lxc launch ubuntu:14.04 bigchaindb` + +(ubuntu:14.04 - this is the remote server the command fetches the image from) +(bigchaindb - is the name of the container) + +Below is the `install.sh` script you will need to install BigchainDB within your container. + +Here is my `install.sh`: + +``` +#!/bin/bash +set -ex +export DEBIAN_FRONTEND=noninteractive +apt-get install -y wget +source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list +wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - +apt-get update +apt-get install -y rethinkdb python3-pip +pip3 install --upgrade pip wheel setuptools +pip install ptpython bigchaindb +``` + +Copy/Paste the above `install.sh` into the directory/path you are going to execute your LXD commands from (ie. the host). + +Make sure your container is running by typing: + +`lxc list` + +Now, from the host (and the correct directory) where you saved `install.sh`, run this command: + +`cat install.sh | lxc exec bigchaindb /bin/bash` + +If you followed the commands correctly, you will have successfully created an LXC container (using LXD) that can get you up and running with BigchainDB in <5 minutes (depending on how long it takes to download all the packages). + +From this point onwards, you can follow the [Python Example](https://bigchaindb.readthedocs.io/en/latest/drivers-clients/python-server-api-examples.html) . diff --git a/docs/source/nodes/run-with-docker.md b/docs/source/appendices/run-with-docker.md similarity index 98% rename from docs/source/nodes/run-with-docker.md rename to docs/source/appendices/run-with-docker.md index e39766d1..d763e98a 100644 --- a/docs/source/nodes/run-with-docker.md +++ b/docs/source/appendices/run-with-docker.md @@ -96,7 +96,7 @@ docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ Note the `--link` option to link to the first container (named `bigchaindb`). -Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](bigchaindb-cli.html). +Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](../server-reference/bigchaindb-cli.html). If you look at the RethinkDB dashboard (in your web browser), you should see the effects of the load test. You can also see some effects in the Docker logs using: ```text diff --git a/docs/source/clusters-feds/deploy-on-aws.md b/docs/source/clusters-feds/aws-testing-cluster.md similarity index 94% rename from docs/source/clusters-feds/deploy-on-aws.md rename to docs/source/clusters-feds/aws-testing-cluster.md index c870591d..bfc4ddca 100644 --- a/docs/source/clusters-feds/deploy-on-aws.md +++ b/docs/source/clusters-feds/aws-testing-cluster.md @@ -1,12 +1,16 @@ -# Deploy a Cluster on AWS +# Deploy a Testing Cluster on AWS -This section explains a way to deploy a cluster of BigchainDB nodes on Amazon Web Services (AWS). We use some Bash and Python scripts to launch several instances (virtual servers) on Amazon Elastic Compute Cloud (EC2). Then we use Fabric to install RethinkDB and BigchainDB on all those instances. +This section explains a way to deploy a cluster of BigchainDB nodes on Amazon Web Services (AWS) for testing purposes. ## Why? -You might ask why one would want to deploy a centrally-controlled BigchainDB cluster. Isn't BigchainDB supposed to be decentralized, where each node is controlled by a different person or organization? +Why would anyone want to deploy a centrally-controlled BigchainDB cluster? Isn't BigchainDB supposed to be decentralized, where each node is controlled by a different person or organization? -Yes! These scripts are for deploying _test_ clusters, not production clusters. +Yes! These scripts are for deploying a testing cluster, not a production cluster. + +## How? + +We use some Bash and Python scripts to launch several instances (virtual servers) on Amazon Elastic Compute Cloud (EC2). Then we use Fabric to install RethinkDB and BigchainDB on all those instances. ## Python Setup @@ -74,7 +78,7 @@ fab --fabfile=fabfile-monitor.py --hosts= run_monitor For more information about monitoring (e.g. how to view the Grafana dashboard in your web browser), see the [Monitoring](monitoring.html) section of this documentation. -To configure a BigchainDB node to send monitoring data to the monitoring server, change the statsd host in the configuration of the BigchainDB node. The section on [Configuring a BigchainDB Node](../nodes/configuration.html) explains how you can do that. (For example, you can change the statsd host in `$HOME/.bigchaindb`.) +To configure a BigchainDB node to send monitoring data to the monitoring server, change the statsd host in the configuration of the BigchainDB node. The section on [Configuring a BigchainDB Node](../server-reference/configuration.html) explains how you can do that. (For example, you can change the statsd host in `$HOME/.bigchaindb`.) ## Deploy a BigchainDB Cluster diff --git a/docs/source/clusters-feds/backup.md b/docs/source/clusters-feds/backup.md index ff45ef47..93fd9aac 100644 --- a/docs/source/clusters-feds/backup.md +++ b/docs/source/clusters-feds/backup.md @@ -22,7 +22,7 @@ That's just one possible way of setting up the file system so as to provide extr Another way to get similar reliability would be to mount the RethinkDB data directory on an [Amazon EBS](https://aws.amazon.com/ebs/) volume. Each Amazon EBS volume is, "automatically replicated within its Availability Zone to protect you from component failure, offering high availability and durability." -See [the section on setting up storage for RethinkDB](../nodes/setup-run-node.html#set-up-storage-for-rethinkdb-data) for more details. +See [the section on setting up storage for RethinkDB](../dev-and-test/setup-run-node.html#set-up-storage-for-rethinkdb-data) for more details. As with shard replication, live file-system replication protects against many failure modes, but it doesn't protect against them all. You should still consider having normal, "cold" backups. @@ -39,7 +39,7 @@ rethinkdb dump -e bigchain.bigchain -e bigchain.votes ``` That should write a file named `rethinkdb_dump__