mirror of
https://github.com/bigchaindb/bigchaindb.git
synced 2024-10-13 13:34:05 +00:00
Merge remote-tracking branch 'origin/master' into feat/540/provision-prod-node-aws
This commit is contained in:
troymc
commit
d1143024d6
@ -1,52 +0,0 @@
|
||||
import logging
|
||||
import multiprocessing as mp
|
||||
import queue
|
||||
|
||||
import rethinkdb as r
|
||||
|
||||
import bigchaindb
|
||||
from bigchaindb import Bigchain
|
||||
from bigchaindb.monitor import Monitor
|
||||
from bigchaindb.util import ProcessGroup
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlockDeleteRevert(object):
|
||||
|
||||
def __init__(self, q_delete_to_revert):
|
||||
self.q_delete_to_revert = q_delete_to_revert
|
||||
|
||||
def write_blocks(self):
|
||||
"""
|
||||
Write blocks to the bigchain
|
||||
"""
|
||||
|
||||
# create bigchain instance
|
||||
b = Bigchain()
|
||||
|
||||
# Write blocks
|
||||
while True:
|
||||
block = self.q_delete_to_revert.get()
|
||||
|
||||
# poison pill
|
||||
if block == 'stop':
|
||||
return
|
||||
|
||||
b.write_block(block)
|
||||
|
||||
def kill(self):
|
||||
for i in range(mp.cpu_count()):
|
||||
self.q_delete_to_revert.put('stop')
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
Initialize, spawn, and start the processes
|
||||
"""
|
||||
|
||||
# initialize the processes
|
||||
p_write = ProcessGroup(name='write_blocks', target=self.write_blocks)
|
||||
|
||||
# start the processes
|
||||
p_write.start()
|
||||
@ -7,7 +7,6 @@ import bigchaindb
|
||||
from bigchaindb.pipelines import block, election
|
||||
from bigchaindb import Bigchain
|
||||
from bigchaindb.voter import Voter
|
||||
from bigchaindb.block import BlockDeleteRevert
|
||||
from bigchaindb.web import server
|
||||
|
||||
|
||||
@ -54,8 +53,6 @@ class Processes(object):
|
||||
def start(self):
|
||||
logger.info('Initializing BigchainDB...')
|
||||
|
||||
delete_reverter = BlockDeleteRevert(self.q_revert_delete)
|
||||
|
||||
# start the web api
|
||||
app_server = server.create_server(bigchaindb.config['server'])
|
||||
p_webapi = mp.Process(name='webapi', target=app_server.run)
|
||||
@ -63,14 +60,12 @@ class Processes(object):
|
||||
|
||||
# initialize the processes
|
||||
p_map_bigchain = mp.Process(name='bigchain_mapper', target=self.map_bigchain)
|
||||
p_block_delete_revert = mp.Process(name='block_delete_revert', target=delete_reverter.start)
|
||||
p_voter = Voter(self.q_new_block)
|
||||
# start the processes
|
||||
logger.info('starting bigchain mapper')
|
||||
p_map_bigchain.start()
|
||||
logger.info('starting block')
|
||||
block.start()
|
||||
p_block_delete_revert.start()
|
||||
|
||||
logger.info('starting voter')
|
||||
p_voter.start()
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# AWS Setup
|
||||
# Basic AWS Setup
|
||||
|
||||
Before you can deploy a BigchainDB node or cluster on AWS, you must do a few things.
|
||||
Before you can deploy anything on AWS, you must do a few things.
|
||||
|
||||
|
||||
## Get an AWS Account
|
||||
@ -36,37 +36,3 @@ Default output format [None]: [Press Enter]
|
||||
```
|
||||
|
||||
This writes two files: `~/.aws/credentials` and `~/.aws/config`. AWS tools and packages look for those files.
|
||||
|
||||
|
||||
## Get Enough Amazon Elastic IP Addresses
|
||||
|
||||
You can skip this if you're deploying a single node.
|
||||
|
||||
Our AWS cluster deployment scripts use elastic IP addresses (although that may change in the future). By default, AWS accounts get five elastic IP addresses. If you want to deploy a cluster with more than five nodes, then you will need more than five elastic IP addresses; you may have to apply for those; see [the AWS documentation on elastic IP addresses](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html).
|
||||
|
||||
|
||||
## Create an Amazon EC2 Key Pair
|
||||
|
||||
Go to the AWS EC2 Console and select "Key Pairs" in the left sidebar. Click the "Create Key Pair" button. Give it the name `bigchaindb`. You should be prompted to save a file named `bigchaindb.pem`. That file contains the RSA private key. (You can get the public key from the private key, so there's no need to send it separately.)
|
||||
|
||||
If you're deploying a cluster, save the file in `bigchaindb/deploy-cluster-aws/pem/bigchaindb.pem`.
|
||||
|
||||
If you're deploying a single node, save the file in `bigchaindb/deploy-node-aws/pem/bigchaindb.pem`.
|
||||
|
||||
**You should not share your private key.**
|
||||
|
||||
|
||||
## Create an Amazon EC2 Security Group
|
||||
|
||||
Go to the AWS EC2 Console and select "Security Groups" in the left sidebar. Click the "Create Security Group" button. If you're deploying a cluster, give it the name `bigchaindb`, otherwise you can name it whatever you like. The description probably doesn't matter but we also put `bigchaindb` for that.
|
||||
|
||||
If you're deploying a test cluster, then add these rules for Inbound traffic:
|
||||
|
||||
* Type = All TCP, Protocol = TCP, Port Range = 0-65535, Source = 0.0.0.0/0
|
||||
* Type = SSH, Protocol = SSH, Port Range = 22, Source = 0.0.0.0/0
|
||||
* Type = All UDP, Protocol = UDP, Port Range = 0-65535, Source = 0.0.0.0/0
|
||||
* Type = All ICMP, Protocol = ICMP, Port Range = 0-65535, Source = 0.0.0.0/0
|
||||
|
||||
**Note: These rules are extremely lax! They're meant to make testing easy.** For example, Source = 0.0.0.0/0 is [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) for "allow this traffic to come from _any_ IP address."
|
||||
|
||||
If you're deploying a single node, then see [the BigchainDB Notes for Firewall Setup](firewall-notes.html) and [the AWS documentation about security groups](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html).
|
||||
|
||||
@ -32,9 +32,37 @@ What did you just install?
|
||||
* [The aws-cli package](https://pypi.python.org/pypi/awscli), which is an AWS Command Line Interface (CLI).
|
||||
|
||||
|
||||
## AWS Setup
|
||||
## Basic AWS Setup
|
||||
|
||||
See the page about [AWS Setup](../appendices/aws-setup.html) in the Appendices.
|
||||
See the page about [basic AWS Setup](../appendices/aws-setup.html) in the Appendices.
|
||||
|
||||
|
||||
## Get Enough Amazon Elastic IP Addresses
|
||||
|
||||
The AWS cluster deployment scripts use elastic IP addresses (although that may change in the future). By default, AWS accounts get five elastic IP addresses. If you want to deploy a cluster with more than five nodes, then you will need more than five elastic IP addresses; you may have to apply for those; see [the AWS documentation on elastic IP addresses](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html).
|
||||
|
||||
|
||||
## Create an Amazon EC2 Key Pair
|
||||
|
||||
Go to the AWS EC2 Console and select "Key Pairs" in the left sidebar. Click the "Create Key Pair" button. Give it the name `bigchaindb`. You should be prompted to save a file named `bigchaindb.pem`. That file contains the RSA private key. (You can get the public key from the private key, so there's no need to send it separately.)
|
||||
|
||||
Save the file in `bigchaindb/deploy-cluster-aws/pem/bigchaindb.pem`.
|
||||
|
||||
**You should not share your private key.**
|
||||
|
||||
|
||||
## Create an Amazon EC2 Security Group
|
||||
|
||||
Go to the AWS EC2 Console and select "Security Groups" in the left sidebar. Click the "Create Security Group" button. Name it `bigchaindb`. The description probably doesn't matter; you can also put `bigchaindb` for that.
|
||||
|
||||
Add these rules for Inbound traffic:
|
||||
|
||||
* Type = All TCP, Protocol = TCP, Port Range = 0-65535, Source = 0.0.0.0/0
|
||||
* Type = SSH, Protocol = SSH, Port Range = 22, Source = 0.0.0.0/0
|
||||
* Type = All UDP, Protocol = UDP, Port Range = 0-65535, Source = 0.0.0.0/0
|
||||
* Type = All ICMP, Protocol = ICMP, Port Range = 0-65535, Source = 0.0.0.0/0
|
||||
|
||||
**Note: These rules are extremely lax! They're meant to make testing easy.** For example, Source = 0.0.0.0/0 is [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) for "allow this traffic to come from _any_ IP address."
|
||||
|
||||
|
||||
## Deploy a BigchainDB Monitor
|
||||
|
||||
@ -7,9 +7,9 @@ When you start Bigchaindb using `bigchaindb start`, an HTTP API is exposed at th
|
||||
|
||||
`http://localhost:9984/api/v1/ <http://localhost:9984/api/v1/>`_
|
||||
|
||||
but that address can be changed by changing the "API endpoint" configuration setting (e.g. in a local config file). There's more information about setting the API endpoint in :doc:`the section about Configuring a BigchainDB Node <../nodes/configuration>`.
|
||||
but that address can be changed by changing the "API endpoint" configuration setting (e.g. in a local config file). There's more information about setting the API endpoint in :doc:`the section about BigchainDB Configuration Settings <../server-reference/configuration>`.
|
||||
|
||||
There are other configuration settings related to the web server (serving the HTTP API). In particular, the default is for the web server socket to bind to `localhost:9984` but that can be changed (e.g. to `0.0.0.0:9984`). For more details, see the "server" settings ("bind", "workers" and "threads") in :doc:`the section about Configuring a BigchainDB Node <../nodes/configuration>`.
|
||||
There are other configuration settings related to the web server (serving the HTTP API). In particular, the default is for the web server socket to bind to `localhost:9984` but that can be changed (e.g. to `0.0.0.0:9984`). For more details, see the "server" settings ("bind", "workers" and "threads") in :doc:`the section about BigchainDB Configuration Settings <../server-reference/configuration>`.
|
||||
|
||||
The HTTP API currently exposes two endpoints, one to get information about a specific transaction, and one to push a new transaction to the BigchainDB cluster.
|
||||
|
||||
|
||||
@ -12,6 +12,6 @@ In a production environment, a BigchainDB node can have several other components
|
||||
* nginx or similar, as a reverse proxy and/or load balancer for the web server
|
||||
* An NTP daemon running on all machines running BigchainDB code, and possibly other machines
|
||||
* A RethinkDB proxy server
|
||||
* Scalable storage for RethinkDB (e.g. using RAID)
|
||||
* Monitoring software, to monitor all the machines in the node
|
||||
* Maybe more, e.g. a configuration management server and agents on all machines
|
||||
|
||||
* Maybe a configuration management (CM) server and CM agents on all machines
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
.. You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Production Node Setup & Management
|
||||
==================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
install-chef-dk
|
||||
overview
|
||||
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
# Install Chef Dev Kit
|
||||
|
||||
TODO
|
||||
|
||||
|
||||
12
docs/source/prod-node-setup-mgmt/overview.md
Normal file
12
docs/source/prod-node-setup-mgmt/overview.md
Normal file
@ -0,0 +1,12 @@
|
||||
# Overview
|
||||
|
||||
Deploying and managing a production BigchainDB node is much more involved than working with a dev/test node:
|
||||
|
||||
* There are more components in a production node; see [the page about node components](../nodes/node-components.html)
|
||||
* Production nodes need more security
|
||||
* Production nodes need monitoring
|
||||
* Production nodes need maintenance, e.g. software upgrades, scaling
|
||||
|
||||
Thankfully, there are tools to help!
|
||||
|
||||
This section explains how to use various tools to deploy and manage a production node.
|
||||
@ -10,7 +10,6 @@ import cryptoconditions as cc
|
||||
import bigchaindb
|
||||
from bigchaindb import crypto, exceptions, util
|
||||
from bigchaindb.voter import Voter
|
||||
from bigchaindb.block import BlockDeleteRevert
|
||||
|
||||
|
||||
@pytest.mark.skipif(reason='Some tests throw a ResourceWarning that might result in some weird '
|
||||
@ -610,40 +609,6 @@ class TestBlockValidation(object):
|
||||
|
||||
|
||||
class TestBigchainBlock(object):
|
||||
|
||||
def test_revert_delete_block(self, b):
|
||||
b.create_genesis_block()
|
||||
|
||||
block_1 = dummy_block()
|
||||
block_2 = dummy_block()
|
||||
block_3 = dummy_block()
|
||||
|
||||
b.write_block(block_1, durability='hard')
|
||||
b.write_block(block_2, durability='hard')
|
||||
b.write_block(block_3, durability='hard')
|
||||
|
||||
b.write_vote(block_1, b.vote(block_1['id'], b.get_last_voted_block()['id'], True))
|
||||
b.write_vote(block_2, b.vote(block_2['id'], b.get_last_voted_block()['id'], True))
|
||||
b.write_vote(block_3, b.vote(block_3['id'], b.get_last_voted_block()['id'], True))
|
||||
|
||||
q_revert_delete = mp.Queue()
|
||||
|
||||
reverter = BlockDeleteRevert(q_revert_delete)
|
||||
|
||||
# simulate changefeed
|
||||
r.table('bigchain').get(block_2['id']).delete().run(b.conn)
|
||||
q_revert_delete.put(block_2)
|
||||
|
||||
assert r.table('bigchain').get(block_2['id']).run(b.conn) is None
|
||||
|
||||
reverter.start()
|
||||
time.sleep(1)
|
||||
reverter.kill()
|
||||
|
||||
reverted_block_2 = r.table('bigchain').get(block_2['id']).run(b.conn)
|
||||
|
||||
assert reverted_block_2 == block_2
|
||||
|
||||
def test_duplicated_transactions(self):
|
||||
pytest.skip('We may have duplicates in the initial_results and changefeed')
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user