diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6ea6a87..5dc7002a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,9 +38,9 @@ Familiarize yourself with how we do coding and documentation in the BigchainDB p ### Step 2 - Install some Dependencies * [Install RethinkDB Server](https://rethinkdb.com/docs/install/) -* Make sure you have Python 3.4+ (maybe in a virtualenv) -* [Install BigchaindB Server's OS-level dependencies](http://bigchaindb.readthedocs.io/en/latest/nodes/setup-run-node.html#install-bigchaindb-server) -* [Make sure you have the latest version of pip](http://bigchaindb.readthedocs.io/en/latest/nodes/setup-run-node.html#how-to-install-bigchaindb-with-pip) +* Make sure you have Python 3.4+ (preferably in a virtualenv) +* [Install BigchaindB Server's OS-level dependencies](http://bigchaindb.readthedocs.io/en/latest/appendices/install-os-level-deps.html) +* [Make sure you have the latest Python 3 version of pip and setuptools](http://bigchaindb.readthedocs.io/en/latest/appendices/install-latest-pip.html) ### Step 3 - Fork bigchaindb on GitHub diff --git a/README.md b/README.md index 04fe222f..dc3e0036 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ We're hiring! [Learn more](https://github.com/bigchaindb/org/blob/master/engjob. ## Get Started ### [Quickstart](http://bigchaindb.readthedocs.io/en/latest/quickstart.html) -### [Set Up and Run a BigchainDB Node](http://bigchaindb.readthedocs.io/en/latest/nodes/setup-run-node.html) -### [Run BigchainDB with Docker](http://bigchaindb.readthedocs.io/en/latest/nodes/run-with-docker.html) +### [Set Up & Run a Dev/Test Node](http://bigchaindb.readthedocs.io/en/latest/dev-and-test/setup-run-node.html) +### [Run BigchainDB with Docker](http://bigchaindb.readthedocs.io/en/latest/appendices/run-with-docker.html) ## Links for Everyone * [BigchainDB.com](https://www.bigchaindb.com/) - the main BigchainDB website, including newsletter signup diff --git a/bigchaindb/block.py b/bigchaindb/block.py deleted file mode 100644 index 2267ef0e..00000000 --- a/bigchaindb/block.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -import multiprocessing as mp -import queue - -import rethinkdb as r - -import bigchaindb -from bigchaindb import Bigchain -from bigchaindb.monitor import Monitor -from bigchaindb.util import ProcessGroup - - -logger = logging.getLogger(__name__) - - -class BlockDeleteRevert(object): - - def __init__(self, q_delete_to_revert): - self.q_delete_to_revert = q_delete_to_revert - - def write_blocks(self): - """ - Write blocks to the bigchain - """ - - # create bigchain instance - b = Bigchain() - - # Write blocks - while True: - block = self.q_delete_to_revert.get() - - # poison pill - if block == 'stop': - return - - b.write_block(block) - - def kill(self): - for i in range(mp.cpu_count()): - self.q_delete_to_revert.put('stop') - - def start(self): - """ - Initialize, spawn, and start the processes - """ - - # initialize the processes - p_write = ProcessGroup(name='write_blocks', target=self.write_blocks) - - # start the processes - p_write.start() diff --git a/bigchaindb/commands/bigchain.py b/bigchaindb/commands/bigchain.py index dbdfeddf..1943167b 100644 --- a/bigchaindb/commands/bigchain.py +++ b/bigchaindb/commands/bigchain.py @@ -24,7 +24,7 @@ from bigchaindb.exceptions import (StartupError, DatabaseAlreadyExists, KeypairNotFoundException) from bigchaindb.commands import utils -from bigchaindb.processes import Processes +from bigchaindb import processes from bigchaindb import crypto @@ -169,7 +169,6 @@ def run_start(args): sys.exit("Can't start BigchainDB, no keypair found. " 'Did you run `bigchaindb configure`?') - processes = Processes() logger.info('Starting BigchainDB main process') processes.start() diff --git a/bigchaindb/core.py b/bigchaindb/core.py index a221a177..6b15d489 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -565,13 +565,9 @@ class Bigchain(object): return vote_signed - def write_vote(self, block, vote): + def write_vote(self, vote): """Write the vote to the database.""" - # First, make sure this block doesn't contain a vote from this node - if self.has_previous_vote(block): - return None - r.table('votes') \ .insert(vote) \ .run(self.conn) diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index a73054a7..59375c57 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -152,6 +152,7 @@ def create_pipeline(): def start(): """Create, start, and return the block pipeline.""" + pipeline = create_pipeline() pipeline.setup(indata=get_changefeed()) pipeline.start() diff --git a/bigchaindb/pipelines/vote.py b/bigchaindb/pipelines/vote.py new file mode 100644 index 00000000..2dc259ed --- /dev/null +++ b/bigchaindb/pipelines/vote.py @@ -0,0 +1,173 @@ +"""This module takes care of all the logic related to block voting. + +The logic is encapsulated in the ``Vote`` class, while the sequence +of actions to do on transactions is specified in the ``create_pipeline`` +function. +""" + +from collections import Counter + +from multipipes import Pipeline, Node + +from bigchaindb import config_utils, exceptions +from bigchaindb.pipelines.utils import ChangeFeed +from bigchaindb import Bigchain + + +def create_invalid_tx(): + """Create and return an invalid transaction. + + The transaction is invalid because it's missing the signature.""" + + b = Bigchain() + tx = b.create_transaction(b.me, b.me, None, 'CREATE') + return tx + + +class Vote: + """This class encapsulates the logic to vote on blocks. + + Note: + Methods of this class will be executed in different processes. + """ + + def __init__(self): + """Initialize the Block voter.""" + + # Since cannot share a connection to RethinkDB using multiprocessing, + # we need to create a temporary instance of BigchainDB that we use + # only to query RethinkDB + last_voted = Bigchain().get_last_voted_block() + self.consensus = config_utils.load_consensus_plugin() + + # This is the Bigchain instance that will be "shared" (aka: copied) + # by all the subprocesses + self.bigchain = Bigchain() + self.last_voted_id = last_voted['id'] + + self.counters = Counter() + self.validity = {} + + self.invalid_dummy_tx = create_invalid_tx() + + def validate_block(self, block): + if not self.bigchain.has_previous_vote(block): + try: + self.consensus.validate_block(self.bigchain, block) + valid = True + except (exceptions.InvalidHash, + exceptions.OperationError, + exceptions.InvalidSignature) as e: + valid = False + return block, valid + + def ungroup(self, block, valid): + """Given a block, ungroup the transactions in it. + + Args: + block (dict): the block to process + + Returns: + ``None`` if the block has been already voted, an iterator that + yields a transaction, block id, and the total number of + transactions contained in the block otherwise. + """ + + # XXX: if a block is invalid we should skip the `validate_tx` step, + # but since we are in a pipeline we cannot just jump to another + # function. Hackish solution: generate an invalid transaction + # and propagate it to the next steps of the pipeline + if valid: + num_tx = len(block['block']['transactions']) + for tx in block['block']['transactions']: + yield tx, block['id'], num_tx + else: + yield self.invalid_dummy_tx, block['id'], 1 + + def validate_tx(self, tx, block_id, num_tx): + """Validate a transaction. + + Args: + tx (dict): the transaction to validate + block_id (str): the id of block containing the transaction + num_tx (int): the total number of transactions to process + + Returns: + Three values are returned, the validity of the transaction, + ``block_id``, ``num_tx``. + """ + return bool(self.bigchain.is_valid_transaction(tx)), block_id, num_tx + + def vote(self, tx_validity, block_id, num_tx): + """Collect the validity of transactions and cast a vote when ready. + + Args: + tx_validity (bool): the validity of the transaction + block_id (str): the id of block containing the transaction + num_tx (int): the total number of transactions to process + + Returns: + None, or a vote if a decision has been reached. + """ + + self.counters[block_id] += 1 + self.validity[block_id] = tx_validity and self.validity.get(block_id, + True) + + if self.counters[block_id] == num_tx: + vote = self.bigchain.vote(block_id, + self.last_voted_id, + self.validity[block_id]) + self.last_voted_id = block_id + del self.counters[block_id] + del self.validity[block_id] + return vote + + def write_vote(self, vote): + """Write vote to the database. + + Args: + vote: the vote to write. + """ + + self.bigchain.write_vote(vote) + return vote + + +def initial(): + """Return unvoted blocks.""" + b = Bigchain() + rs = b.get_unvoted_blocks() + return rs + + +def get_changefeed(): + """Create and return the changefeed for the bigchain table.""" + + return ChangeFeed('bigchain', 'insert', prefeed=initial()) + + +def create_pipeline(): + """Create and return the pipeline of operations to be distributed + on different processes.""" + + voter = Vote() + + vote_pipeline = Pipeline([ + Node(voter.validate_block), + Node(voter.ungroup), + Node(voter.validate_tx, fraction_of_cores=1), + Node(voter.vote), + Node(voter.write_vote) + ]) + + return vote_pipeline + + +def start(): + """Create, start, and return the block pipeline.""" + + pipeline = create_pipeline() + pipeline.setup(indata=get_changefeed()) + pipeline.start() + return pipeline diff --git a/bigchaindb/processes.py b/bigchaindb/processes.py index 5c7edcc3..4b8aa0eb 100644 --- a/bigchaindb/processes.py +++ b/bigchaindb/processes.py @@ -1,13 +1,8 @@ import logging import multiprocessing as mp -import rethinkdb as r - import bigchaindb -from bigchaindb.pipelines import block, election -from bigchaindb import Bigchain -from bigchaindb.voter import Voter -from bigchaindb.block import BlockDeleteRevert +from bigchaindb.pipelines import vote, block, election from bigchaindb.web import server @@ -26,57 +21,23 @@ BANNER = """ """ -class Processes(object): +def start(): + logger.info('Initializing BigchainDB...') - def __init__(self): - # initialize the class - self.q_new_block = mp.Queue() - self.q_revert_delete = mp.Queue() + # start the processes + logger.info('Starting block') + block.start() - def map_bigchain(self): - # listen to changes on the bigchain and redirect the changes - # to the correct queues + logger.info('Starting voter') + vote.start() - # create a bigchain instance - b = Bigchain() + logger.info('Starting election') + election.start() - for change in r.table('bigchain').changes().run(b.conn): + # start the web api + app_server = server.create_server(bigchaindb.config['server']) + p_webapi = mp.Process(name='webapi', target=app_server.run) + p_webapi.start() - # insert - if change['old_val'] is None: - self.q_new_block.put(change['new_val']) - - # delete - elif change['new_val'] is None: - # this should never happen in regular operation - self.q_revert_delete.put(change['old_val']) - - def start(self): - logger.info('Initializing BigchainDB...') - - delete_reverter = BlockDeleteRevert(self.q_revert_delete) - - # start the web api - app_server = server.create_server(bigchaindb.config['server']) - p_webapi = mp.Process(name='webapi', target=app_server.run) - p_webapi.start() - - # initialize the processes - p_map_bigchain = mp.Process(name='bigchain_mapper', target=self.map_bigchain) - p_block_delete_revert = mp.Process(name='block_delete_revert', target=delete_reverter.start) - p_voter = Voter(self.q_new_block) - # start the processes - logger.info('starting bigchain mapper') - p_map_bigchain.start() - logger.info('starting block') - block.start() - p_block_delete_revert.start() - - logger.info('starting voter') - p_voter.start() - election.start() - logger.info('starting election') - - # start message - p_voter.initialized.wait() - logger.info(BANNER.format(bigchaindb.config['server']['bind'])) + # start message + logger.info(BANNER.format(bigchaindb.config['server']['bind'])) diff --git a/bigchaindb/voter.py b/bigchaindb/voter.py deleted file mode 100644 index 4d60ecd5..00000000 --- a/bigchaindb/voter.py +++ /dev/null @@ -1,199 +0,0 @@ -import logging -import multiprocessing as mp -import ctypes - -from bigchaindb import Bigchain -from bigchaindb.monitor import Monitor - - -logger = logging.getLogger(__name__) - - -class BlockStream(object): - """ - Combine the stream of new blocks coming from the changefeed with the list of unvoted blocks. - - This is a utility class that abstracts the source of data for the `Voter`. - """ - - def __init__(self, new_blocks): - """ - Create a new BlockStream instance. - - Args: - new_block (queue): a queue of new blocks - """ - - b = Bigchain() - self.new_blocks = new_blocks - # TODO: there might be duplicate blocks since we *first* get the changefeed and only *then* we query the - # database to get the old blocks. - - # TODO how about a one liner, something like: - # self.unvoted_blocks = b.get_unvoted_blocks() if not b.nodes_except_me else [] - self.unvoted_blocks = [] - if not b.nodes_except_me: - self.unvoted_blocks = b.get_unvoted_blocks() - - def get(self): - """ - Return the next block to be processed. - """ - try: - # FIXME: apparently RethinkDB returns a list instead of a cursor when using `order_by`. - # We might change the `pop` in the future, when the driver will return a cursor. - # We have a test for this, so if the driver implementation changes we will get a failure: - # - tests/test_voter.py::TestBlockStream::test_if_old_blocks_get_should_return_old_block_first - return self.unvoted_blocks.pop(0) - except IndexError: - return self.new_blocks.get() - - -class Voter(object): - - def __init__(self, q_new_block): - """ - Initialize the class with the needed queues. - - Initialize with a queue where new blocks added to the bigchain will be put - """ - - self.monitor = Monitor() - - self.q_new_block = q_new_block - self.q_blocks_to_validate = mp.Queue() - self.q_validated_block = mp.Queue() - self.q_voted_block = mp.Queue() - self.v_previous_block_id = mp.Value(ctypes.c_char_p) - self.initialized = mp.Event() - - def feed_blocks(self): - """ - Prepare the queue with blocks to validate - """ - - block_stream = BlockStream(self.q_new_block) - while True: - # poison pill - block = block_stream.get() - if block == 'stop': - self.q_blocks_to_validate.put('stop') - return - - self.q_blocks_to_validate.put(block) - - def validate(self): - """ - Checks if incoming blocks are valid or not - """ - - # create a bigchain instance. All processes should create their own bigchcain instance so that they all - # have their own connection to the database - b = Bigchain() - - logger.info('voter waiting for new blocks') - # signal initialization complete - self.initialized.set() - - while True: - new_block = self.q_blocks_to_validate.get() - - # poison pill - if new_block == 'stop': - self.q_validated_block.put('stop') - return - - logger.info('new_block arrived to voter') - - with self.monitor.timer('validate_block'): - # FIXME: the following check is done also in `is_valid_block`, - # but validity can be true even if the block has already - # a vote. - if b.has_previous_vote(new_block): - continue - validity = b.is_valid_block(new_block) - - self.q_validated_block.put((new_block, - self.v_previous_block_id.value.decode(), - validity)) - - self.v_previous_block_id.value = new_block['id'].encode() - - def vote(self): - """ - Votes on the block based on the decision of the validation - """ - - # create a bigchain instance - b = Bigchain() - - while True: - elem = self.q_validated_block.get() - - # poison pill - if elem == 'stop': - self.q_voted_block.put('stop') - return - - validated_block, previous_block_id, decision = elem - vote = b.vote(validated_block['id'], previous_block_id, decision) - self.q_voted_block.put((validated_block, vote)) - - def update_block(self): - """ - Appends the vote in the bigchain table - """ - - # create a bigchain instance - b = Bigchain() - - while True: - elem = self.q_voted_block.get() - - # poison pill - if elem == 'stop': - logger.info('clean exit') - return - - block, vote = elem - pretty_vote = 'valid' if vote['vote']['is_block_valid'] else 'invalid' - logger.info('voting %s for block %s', pretty_vote, block['id']) - b.write_vote(block, vote) - - def bootstrap(self): - """ - Before starting handling the new blocks received by the changefeed we need to handle unvoted blocks - added to the bigchain while the process was down - - We also need to set the previous_block_id. - """ - - b = Bigchain() - last_voted = b.get_last_voted_block() - - self.v_previous_block_id.value = last_voted['id'].encode() - - def kill(self): - """ - Terminate processes - """ - self.q_new_block.put('stop') - - def start(self): - """ - Initialize, spawn, and start the processes - """ - - self.bootstrap() - - # initialize the processes - p_feed_blocks = mp.Process(name='block_feeder', target=self.feed_blocks) - p_validate = mp.Process(name='block_validator', target=self.validate) - p_vote = mp.Process(name='block_voter', target=self.vote) - p_update = mp.Process(name='block_updater', target=self.update_block) - - # start the processes - p_feed_blocks.start() - p_validate.start() - p_vote.start() - p_update.start() diff --git a/docs/source/appendices/aws-setup.md b/docs/source/appendices/aws-setup.md index caae7339..8b7dcd6c 100644 --- a/docs/source/appendices/aws-setup.md +++ b/docs/source/appendices/aws-setup.md @@ -1,6 +1,6 @@ -# AWS Setup +# Basic AWS Setup -Before you can deploy a BigchainDB node or cluster on AWS, you must do a few things. +Before you can deploy anything on AWS, you must do a few things. ## Get an AWS Account @@ -36,37 +36,3 @@ Default output format [None]: [Press Enter] ``` This writes two files: `~/.aws/credentials` and `~/.aws/config`. AWS tools and packages look for those files. - - -## Get Enough Amazon Elastic IP Addresses - -You can skip this if you're deploying a single node. - -Our AWS cluster deployment scripts use elastic IP addresses (although that may change in the future). By default, AWS accounts get five elastic IP addresses. If you want to deploy a cluster with more than five nodes, then you will need more than five elastic IP addresses; you may have to apply for those; see [the AWS documentation on elastic IP addresses](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html). - - -## Create an Amazon EC2 Key Pair - -Go to the AWS EC2 Console and select "Key Pairs" in the left sidebar. Click the "Create Key Pair" button. Give it the name `bigchaindb`. You should be prompted to save a file named `bigchaindb.pem`. That file contains the RSA private key. (You can get the public key from the private key, so there's no need to send it separately.) - -If you're deploying a cluster, save the file in `bigchaindb/deploy-cluster-aws/pem/bigchaindb.pem`. - -If you're deploying a single node, save the file in `bigchaindb/deploy-node-aws/pem/bigchaindb.pem`. - -**You should not share your private key.** - - -## Create an Amazon EC2 Security Group - -Go to the AWS EC2 Console and select "Security Groups" in the left sidebar. Click the "Create Security Group" button. If you're deploying a cluster, give it the name `bigchaindb`, otherwise you can name it whatever you like. The description probably doesn't matter but we also put `bigchaindb` for that. - -If you're deploying a test cluster, then add these rules for Inbound traffic: - -* Type = All TCP, Protocol = TCP, Port Range = 0-65535, Source = 0.0.0.0/0 -* Type = SSH, Protocol = SSH, Port Range = 22, Source = 0.0.0.0/0 -* Type = All UDP, Protocol = UDP, Port Range = 0-65535, Source = 0.0.0.0/0 -* Type = All ICMP, Protocol = ICMP, Port Range = 0-65535, Source = 0.0.0.0/0 - -**Note: These rules are extremely lax! They're meant to make testing easy.** For example, Source = 0.0.0.0/0 is [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) for "allow this traffic to come from _any_ IP address." - -If you're deploying a single node, then see [the BigchainDB Notes for Firewall Setup](firewall-notes.html) and [the AWS documentation about security groups](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html). diff --git a/docs/source/appendices/example-rethinkdb-storage-setups.md b/docs/source/appendices/example-rethinkdb-storage-setups.md index 2c863034..0fc4c273 100755 --- a/docs/source/appendices/example-rethinkdb-storage-setups.md +++ b/docs/source/appendices/example-rethinkdb-storage-setups.md @@ -2,7 +2,7 @@ ## Example Amazon EC2 Setups -We have some scripts for [deploying a _test_ BigchainDB cluster on AWS](../clusters-feds/deploy-on-aws.html). Those scripts include command sequences to set up storage for RethinkDB. +We have some scripts for [deploying a _test_ BigchainDB cluster on AWS](../clusters-feds/aws-testing-cluster.html). Those scripts include command sequences to set up storage for RethinkDB. In particular, look in the file [/deploy-cluster-aws/fabfile.py](https://github.com/bigchaindb/bigchaindb/blob/master/deploy-cluster-aws/fabfile.py), under `def prep_rethinkdb_storage(USING_EBS)`. Note that there are two cases: 1. **Using EBS ([Amazon Elastic Block Store](https://aws.amazon.com/ebs/)).** This is always an option, and for some instance types ("EBS-only"), it's the only option. diff --git a/docs/source/appendices/firewall-notes.md b/docs/source/appendices/firewall-notes.md index 4f1e780c..dca89ac6 100644 --- a/docs/source/appendices/firewall-notes.md +++ b/docs/source/appendices/firewall-notes.md @@ -47,7 +47,7 @@ Port 8080 is the default port used by RethinkDB for its adminstrative web (HTTP) Port 9984 is the default port for the BigchainDB client-server HTTP API (TCP), which is served by Gunicorn HTTP Server. It's _possible_ allow port 9984 to accept inbound traffic from anyone, but we recommend against doing that. Instead, set up a reverse proxy server (e.g. using Nginx) and only allow traffic from there. Information about how to do that can be found [in the Gunicorn documentation](http://docs.gunicorn.org/en/stable/deploy.html). (They call it a proxy.) -If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [BigchainDB Configuration Settings](../nodes/configuration.html), where PORT is whatever port you chose (e.g. 9983). +If Gunicorn and the reverse proxy are running on the same server, then you'll have to tell Gunicorn to listen on some port other than 9984 (so that the reverse proxy can listen on port 9984). You can do that by setting `server.bind` to 'localhost:PORT' in the [BigchainDB Configuration Settings](../server-reference/configuration.html), where PORT is whatever port you chose (e.g. 9983). You may want to have Gunicorn and the reverse proxy running on different servers, so that both can listen on port 9984. That would also help isolate the effects of a denial-of-service attack. diff --git a/docs/source/appendices/index.rst b/docs/source/appendices/index.rst index 52b9038e..5e9133ae 100755 --- a/docs/source/appendices/index.rst +++ b/docs/source/appendices/index.rst @@ -7,6 +7,9 @@ Appendices .. toctree:: :maxdepth: 1 + install-os-level-deps + install-latest-pip + run-with-docker json-serialization cryptography the-Bigchain-class @@ -15,4 +18,5 @@ Appendices firewall-notes ntp-notes example-rethinkdb-storage-setups - licenses \ No newline at end of file + licenses + install-with-lxd \ No newline at end of file diff --git a/docs/source/appendices/install-latest-pip.md b/docs/source/appendices/install-latest-pip.md new file mode 100644 index 00000000..fac7dbed --- /dev/null +++ b/docs/source/appendices/install-latest-pip.md @@ -0,0 +1,20 @@ +# How to Install the Latest pip and setuptools + +You can check the version of `pip` you're using (in your current virtualenv) by doing: +```text +pip -V +``` + +If it says that `pip` isn't installed, or it says `pip` is associated with a Python version less than 3.4, then you must install a `pip` version associated with Python 3.4+. In the following instructions, we call it `pip3` but you may be able to use `pip` if that refers to the same thing. See [the `pip` installation instructions](https://pip.pypa.io/en/stable/installing/). + +On Ubuntu 14.04, we found that this works: +```text +sudo apt-get install python3-pip +``` + +That should install a Python 3 version of `pip` named `pip3`. If that didn't work, then another way to get `pip3` is to do `sudo apt-get install python3-setuptools` followed by `sudo easy_install3 pip`. + +You can upgrade `pip` (`pip3`) and `setuptools` to the latest versions using: +```text +pip3 install --upgrade pip setuptools +``` \ No newline at end of file diff --git a/docs/source/appendices/install-os-level-deps.md b/docs/source/appendices/install-os-level-deps.md new file mode 100644 index 00000000..3f9f8f57 --- /dev/null +++ b/docs/source/appendices/install-os-level-deps.md @@ -0,0 +1,17 @@ +# How to Install OS-Level Dependencies + +BigchainDB Server has some OS-level dependencies that must be installed. + +On Ubuntu 14.04 and 16.04, we found that the following was enough: +```text +sudo apt-get update +sudo apt-get install g++ python3-dev +``` + +On Fedora 23 and 24, we found that the following was enough: +```text +sudo dnf update +sudo dnf install gcc-c++ redhat-rpm-config python3-devel +``` + +(If you're using a version of Fedora before version 22, you may have to use `yum` instead of `dnf`.) diff --git a/docs/source/appendices/install-with-lxd.md b/docs/source/appendices/install-with-lxd.md new file mode 100644 index 00000000..7cf5cf81 --- /dev/null +++ b/docs/source/appendices/install-with-lxd.md @@ -0,0 +1,43 @@ +# Installing BigchainDB on LXC containers using LXD + +You can visit this link to install LXD (instructions here): [LXD Install](https://linuxcontainers.org/lxd/getting-started-cli/) + +(assumption is that you are using Ubuntu 14.04 for host/container) + +Let us create an LXC container (via LXD) with the following command: + +`lxc launch ubuntu:14.04 bigchaindb` + +(ubuntu:14.04 - this is the remote server the command fetches the image from) +(bigchaindb - is the name of the container) + +Below is the `install.sh` script you will need to install BigchainDB within your container. + +Here is my `install.sh`: + +``` +#!/bin/bash +set -ex +export DEBIAN_FRONTEND=noninteractive +apt-get install -y wget +source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list +wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - +apt-get update +apt-get install -y rethinkdb python3-pip +pip3 install --upgrade pip wheel setuptools +pip install ptpython bigchaindb +``` + +Copy/Paste the above `install.sh` into the directory/path you are going to execute your LXD commands from (ie. the host). + +Make sure your container is running by typing: + +`lxc list` + +Now, from the host (and the correct directory) where you saved `install.sh`, run this command: + +`cat install.sh | lxc exec bigchaindb /bin/bash` + +If you followed the commands correctly, you will have successfully created an LXC container (using LXD) that can get you up and running with BigchainDB in <5 minutes (depending on how long it takes to download all the packages). + +From this point onwards, you can follow the [Python Example](https://bigchaindb.readthedocs.io/en/latest/drivers-clients/python-server-api-examples.html) . diff --git a/docs/source/nodes/run-with-docker.md b/docs/source/appendices/run-with-docker.md similarity index 98% rename from docs/source/nodes/run-with-docker.md rename to docs/source/appendices/run-with-docker.md index e39766d1..d763e98a 100644 --- a/docs/source/nodes/run-with-docker.md +++ b/docs/source/appendices/run-with-docker.md @@ -96,7 +96,7 @@ docker run --rm -v "$HOME/bigchaindb_docker:/data" -ti \ Note the `--link` option to link to the first container (named `bigchaindb`). -Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](bigchaindb-cli.html). +Aside: The `bigchaindb load` command has several options (e.g. `-m`). You can read more about it in [the documentation about the BigchainDB command line interface](../server-reference/bigchaindb-cli.html). If you look at the RethinkDB dashboard (in your web browser), you should see the effects of the load test. You can also see some effects in the Docker logs using: ```text diff --git a/docs/source/clusters-feds/deploy-on-aws.md b/docs/source/clusters-feds/aws-testing-cluster.md similarity index 81% rename from docs/source/clusters-feds/deploy-on-aws.md rename to docs/source/clusters-feds/aws-testing-cluster.md index c870591d..59277ac4 100644 --- a/docs/source/clusters-feds/deploy-on-aws.md +++ b/docs/source/clusters-feds/aws-testing-cluster.md @@ -1,12 +1,16 @@ -# Deploy a Cluster on AWS +# Deploy a Testing Cluster on AWS -This section explains a way to deploy a cluster of BigchainDB nodes on Amazon Web Services (AWS). We use some Bash and Python scripts to launch several instances (virtual servers) on Amazon Elastic Compute Cloud (EC2). Then we use Fabric to install RethinkDB and BigchainDB on all those instances. +This section explains a way to deploy a cluster of BigchainDB nodes on Amazon Web Services (AWS) for testing purposes. ## Why? -You might ask why one would want to deploy a centrally-controlled BigchainDB cluster. Isn't BigchainDB supposed to be decentralized, where each node is controlled by a different person or organization? +Why would anyone want to deploy a centrally-controlled BigchainDB cluster? Isn't BigchainDB supposed to be decentralized, where each node is controlled by a different person or organization? -Yes! These scripts are for deploying _test_ clusters, not production clusters. +Yes! These scripts are for deploying a testing cluster, not a production cluster. + +## How? + +We use some Bash and Python scripts to launch several instances (virtual servers) on Amazon Elastic Compute Cloud (EC2). Then we use Fabric to install RethinkDB and BigchainDB on all those instances. ## Python Setup @@ -28,9 +32,37 @@ What did you just install? * [The aws-cli package](https://pypi.python.org/pypi/awscli), which is an AWS Command Line Interface (CLI). -## AWS Setup +## Basic AWS Setup -See the page about [AWS Setup](../appendices/aws-setup.html) in the Appendices. +See the page about [basic AWS Setup](../appendices/aws-setup.html) in the Appendices. + + +## Get Enough Amazon Elastic IP Addresses + +The AWS cluster deployment scripts use elastic IP addresses (although that may change in the future). By default, AWS accounts get five elastic IP addresses. If you want to deploy a cluster with more than five nodes, then you will need more than five elastic IP addresses; you may have to apply for those; see [the AWS documentation on elastic IP addresses](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html). + + +## Create an Amazon EC2 Key Pair + +Go to the AWS EC2 Console and select "Key Pairs" in the left sidebar. Click the "Create Key Pair" button. Give it the name `bigchaindb`. You should be prompted to save a file named `bigchaindb.pem`. That file contains the RSA private key. (You can get the public key from the private key, so there's no need to send it separately.) + +Save the file in `bigchaindb/deploy-cluster-aws/pem/bigchaindb.pem`. + +**You should not share your private key.** + + +## Create an Amazon EC2 Security Group + +Go to the AWS EC2 Console and select "Security Groups" in the left sidebar. Click the "Create Security Group" button. Name it `bigchaindb`. The description probably doesn't matter; you can also put `bigchaindb` for that. + +Add these rules for Inbound traffic: + +* Type = All TCP, Protocol = TCP, Port Range = 0-65535, Source = 0.0.0.0/0 +* Type = SSH, Protocol = SSH, Port Range = 22, Source = 0.0.0.0/0 +* Type = All UDP, Protocol = UDP, Port Range = 0-65535, Source = 0.0.0.0/0 +* Type = All ICMP, Protocol = ICMP, Port Range = 0-65535, Source = 0.0.0.0/0 + +**Note: These rules are extremely lax! They're meant to make testing easy.** For example, Source = 0.0.0.0/0 is [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) for "allow this traffic to come from _any_ IP address." ## Deploy a BigchainDB Monitor @@ -74,7 +106,7 @@ fab --fabfile=fabfile-monitor.py --hosts= run_monitor For more information about monitoring (e.g. how to view the Grafana dashboard in your web browser), see the [Monitoring](monitoring.html) section of this documentation. -To configure a BigchainDB node to send monitoring data to the monitoring server, change the statsd host in the configuration of the BigchainDB node. The section on [Configuring a BigchainDB Node](../nodes/configuration.html) explains how you can do that. (For example, you can change the statsd host in `$HOME/.bigchaindb`.) +To configure a BigchainDB node to send monitoring data to the monitoring server, change the statsd host in the configuration of the BigchainDB node. The section on [Configuring a BigchainDB Node](../server-reference/configuration.html) explains how you can do that. (For example, you can change the statsd host in `$HOME/.bigchaindb`.) ## Deploy a BigchainDB Cluster diff --git a/docs/source/clusters-feds/backup.md b/docs/source/clusters-feds/backup.md index ff45ef47..93fd9aac 100644 --- a/docs/source/clusters-feds/backup.md +++ b/docs/source/clusters-feds/backup.md @@ -22,7 +22,7 @@ That's just one possible way of setting up the file system so as to provide extr Another way to get similar reliability would be to mount the RethinkDB data directory on an [Amazon EBS](https://aws.amazon.com/ebs/) volume. Each Amazon EBS volume is, "automatically replicated within its Availability Zone to protect you from component failure, offering high availability and durability." -See [the section on setting up storage for RethinkDB](../nodes/setup-run-node.html#set-up-storage-for-rethinkdb-data) for more details. +See [the section on setting up storage for RethinkDB](../dev-and-test/setup-run-node.html#set-up-storage-for-rethinkdb-data) for more details. As with shard replication, live file-system replication protects against many failure modes, but it doesn't protect against them all. You should still consider having normal, "cold" backups. @@ -39,7 +39,7 @@ rethinkdb dump -e bigchain.bigchain -e bigchain.votes ``` That should write a file named `rethinkdb_dump__