Make config filename the 1st arg of awsdeploy.sh

This commit is contained in:
troymc 2016-05-16 16:58:37 +02:00
parent 4e89d491b1
commit 1a2f0e749d
5 changed files with 104 additions and 64 deletions

View File

@ -4,13 +4,25 @@
# if any command has a non-zero exit status
set -e
# Validate the values in deploy_conf.py
python validate_deploy_conf.py
# Check the (optional) command-line argument
# (the name of the AWS deployment config file)
if [ -z "$1" ]; then
# no first argument was provided
DEPLOY_CONF_FILE=default_deploy_conf.py
else
DEPLOY_CONF_FILE=$1
fi
# Read deploy_conf.py
# Check to make sure DEPLOY_CONF_FILE exists
if [ ! -f $DEPLOY_CONF_FILE ]; then
echo "AWS deployment configuration file not found: "$DEPLOY_CONF_FILE
exit 1
fi
# Read DEPLOY_CONF_FILE
# to set environment variables related to AWS deployment
echo "Reading deploy_conf.py"
source deploy_conf.py
echo "Reading "$DEPLOY_CONF_FILE
source $DEPLOY_CONF_FILE
echo "NUM_NODES = "$NUM_NODES
echo "BRANCH = "$BRANCH
echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY
@ -31,6 +43,15 @@ if [ ! -d "confiles" ]; then
exit 1
fi
# Check if the number of files in confiles directory == NUM_NODES
CONFILES_COUNT=`ls confiles | wc -l`
if [ $CONFILES_COUNT != $NUM_NODES ]; then
echo "ERROR: CONFILES_COUNT = "$CONFILES_COUNT
echo "but NUM_NODES = "$NUM_NODES
echo "so there should be "$NUM_NODES" files in the confiles directory"
exit 1
fi
# Auto-generate the tag to apply to all nodes in the cluster
TAG="BDB-"$WHAT_TO_DEPLOY"-"`date +%m-%d@%H:%M`
echo "TAG = "$TAG
@ -49,7 +70,7 @@ chmod 0400 pem/bigchaindb.pem
# 5. writes the shellscript add2known_hosts.sh
# 6. (over)writes a file named hostlist.py
# containing a list of all public DNS names.
python launch_ec2_nodes.py --tag $TAG
python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG
# Make add2known_hosts.sh executable then execute it.
# This adds remote keys to ~/.ssh/known_hosts

View File

@ -1,16 +1,19 @@
# AWS deployment config file
# To use in a Bash shell script:
# source deploy_conf.py
# echo $EXAMPLEVAR
# source default_deploy_conf.py
# # $EXAMPLEVAR now has a value
# To use in a Python script:
# from deploy_conf import *
# # EXAMPLEVAR now has a value
# from default_deploy_conf import *
# or
# import importlib
# cf = importlib.import_module('default_deploy_conf')
# # cf.EXAMPLEVAR now has a value
# DON'T PUT SPACES AROUND THE =
# because that would confuse Bash.
# Values can be strings in double quotes, or integers like 23
# Example values: "string in double quotes", 32, True, False
# NUM_NODES is the number of nodes to deploy
NUM_NODES=3

View File

@ -16,16 +16,13 @@ import sys
import time
import socket
import argparse
import importlib
import botocore
import boto3
from awscommon import get_naeips
from deploy_conf import *
# Make sure NUM_NODES is an int
assert isinstance(NUM_NODES, int)
# Ensure they're using Python 2.5-2.7
pyver = sys.version_info
major = pyver[0]
@ -41,8 +38,52 @@ parser = argparse.ArgumentParser()
parser.add_argument("--tag",
help="tag to add to all launched instances on AWS",
required=True)
parser.add_argument("--deploy-conf-file",
help="AWS deployment configuration file",
required=True)
args = parser.parse_args()
tag = args.tag
deploy_conf_file = args.deploy_conf_file
# Import all the variables set in the AWS deployment configuration file
# (Remove the '.py' from the end of deploy_conf_file.)
cf = importlib.import_module(deploy_conf_file[:-3])
try:
NUM_NODES = cf.NUM_NODES
BRANCH = cf.BRANCH
WHAT_TO_DEPLOY = cf.WHAT_TO_DEPLOY
USE_KEYPAIRS_FILE = cf.USE_KEYPAIRS_FILE
IMAGE_ID = cf.IMAGE_ID
INSTANCE_TYPE = cf.INSTANCE_TYPE
except AttributeError as e:
print('One of the AWS deployment configuration settings was '
'not set in the AWS deployment configuration file ' +
'{}'.format(deploy_conf_file))
print('Read this traceback to find out which one (in ALL_CAPS):')
raise
# Validate the variables set in the AWS deployment configuration file
try:
assert isinstance(NUM_NODES, int)
assert isinstance(BRANCH, str)
assert isinstance(WHAT_TO_DEPLOY, str)
assert isinstance(USE_KEYPAIRS_FILE, bool)
assert isinstance(IMAGE_ID, str)
assert isinstance(INSTANCE_TYPE, str)
except AssertionError as e:
print('One of the AWS deployment settings has a value of the wrong type.')
print('Read this traceback to find out which one (in ALL_CAPS):')
raise
if NUM_NODES > 64:
raise ValueError('NUM_NODES should be less than or equal to 64. '
'The AWS deployment configuration file sets it to {}'.
format(NUM_NODES))
if WHAT_TO_DEPLOY not in ['servers', 'clients']:
raise ValueError('WHAT_TO_DEPLOY should be either "servers" or "clients". '
'The AWS deployment configuration file sets it to {}'.
format(WHAT_TO_DEPLOY))
# Get an AWS EC2 "resource"
# See http://boto3.readthedocs.org/en/latest/guide/resources.html

View File

@ -1,28 +0,0 @@
# -*- coding: utf-8 -*-
"""This script validates the values in deploy_conf.py
"""
from __future__ import unicode_literals
import sys
from deploy_conf import *
try:
assert isinstance(NUM_NODES, int)
assert isinstance(BRANCH, str)
assert isinstance(WHAT_TO_DEPLOY, str)
assert isinstance(USE_KEYPAIRS_FILE, bool)
assert isinstance(IMAGE_ID, str)
assert isinstance(INSTANCE_TYPE, str)
except NameError as e:
sys.exit('A variable with {} '.format(e.args[0]) + 'in deploy_conf.py')
if NUM_NODES > 64:
raise ValueError('NUM_NODES should be less than or equal to 64. '
'The deploy_conf.py file sets it to {}'.format(NUM_NODES))
if WHAT_TO_DEPLOY not in ['servers', 'clients']:
raise ValueError('WHAT_TO_DEPLOY should be either "servers" or "clients". '
'The deploy_conf.py file sets it to {}'.
format(WHAT_TO_DEPLOY))

View File

@ -131,7 +131,7 @@ To configure a BigchainDB node to send monitoring data to the monitoring server,
### Step 1
Suppose _N_ is the number of nodes you want in your BigchainDB cluster. If you already have a set of _N_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory, then you can jump to step 2. To create such a set, you can do something like:
Suppose _N_ is the number of nodes you want in your BigchainDB cluster. If you already have a set of _N_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory, then you can jump to the next step. To create such a set, you can do something like:
```text
# in a Python 3 virtual environment where bigchaindb is installed
cd bigchaindb
@ -141,25 +141,11 @@ cd deploy-cluster-aws
That will create three (3) _default_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory (which will be created if it doesn't already exist). The three files will be named `bcdb_conf0`, `bcdb_conf1`, and `bcdb_conf2`.
You can look inside those files if you're curious. In step 2, they'll be modified. For example, the default keyring is an empty list. In step 2, the deployment script automatically changes the keyring of each node to be a list of the public keys of all other nodes. Other changes are also made.
**An Aside on Using a Standard Set of Keypairs**
It's possible to deploy BigchainDB servers with a known set of keypairs. You can generate a set of keypairs in a file named `keypairs.py` using the `write_keypairs_file.py` script. For example:
```text
# in a Python 3 virtual environment where bigchaindb is installed
cd bigchaindb
cd deploy-cluster-aws
python3 write_keypairs_file.py 100
```
The above command generates a file with 100 keypairs. (You can generate more keypairs than you need, so you can use the same list over and over again, for different numbers of servers.) To make the `awsdeploy.sh` script read all keys from `keypairs.py`, just set `USE_KEYPAIRS_FILE=True` in `deploy_conf.py`.
You can look inside those files if you're curious. For example, the default keyring is an empty list. Later, the deployment script automatically changes the keyring of each node to be a list of the public keys of all other nodes. Other changes are also made. That is, the configuration files generated in this step are _not_ what will be sent to the deployed nodes; they're just a starting point.
### Step 2
Step 2 is to launch the nodes ("instances") on AWS, to install all the necessary software on them, configure the software, run the software, and more.
First, edit the AWS deployment configuration file, `deploy_conf.py`, in the `bigchaindb/deploy-cluster-aws` directory. It comes with comments explaining each of the configuration settings. You may want to make a copy of `deploy_conf.py` before editing it. The defaults are (or should be):
Step 2 is to make an AWS deployment configuration file, if necessary. There's a default AWS configuration file named `default_deploy_conf.py`. It has many comments explaining each setting. The default AWS deployment settings are (or should be):
```text
NUM_NODES=3
BRANCH="master"
@ -169,19 +155,36 @@ IMAGE_ID="ami-accff2b1"
INSTANCE_TYPE="m3.2xlarge"
```
Once you've edited `deploy_conf.py` to your liking:
If you're happy with those settings, then you can skip to the next step. Otherwise, you could make a copy of `default_deploy_conf.py` (e.g. `cp default_deploy_conf.py my_deploy_conf.py`) and then edit the new file using a text editor.
If you want your nodes to have a predictable set of pre-generated keypairs, then you should 1) set `USE_KEYPAIRS_FILE=True` in the AWS deployment configuration file, and 2) provide a `keypairs.py` file containing enough keypairs for all of your nodes. You can generate a `keypairs.py` file using the `write_keypairs_file.py` script. For example:
```text
# in a Python 3 virtual environment where bigchaindb is installed
cd bigchaindb
cd deploy-cluster-aws
python3 write_keypairs_file.py 100
```
The above command generates a `keypairs.py` file with 100 keypairs. You can generate more keypairs than you need, so you can use the same list over and over again, for different numbers of servers. The deployment scripts will only use the first NUM_NODES keypairs.
### Step 3
Step 3 is to launch the nodes ("instances") on AWS, to install all the necessary software on them, configure the software, run the software, and more. Here's how you'd do that:
```text
# in a Python 2.5-2.7 virtual environment where fabric, boto3, etc. are installed
cd bigchaindb
cd deploy-cluster-aws
./awsdeploy.sh
./awsdeploy.sh my_deploy_conf.py
# Only if you want to start BigchainDB on all the nodes:
fab start_bigchaindb
```
`awsdeploy.sh` is a Bash script which calls some Python and Fabric scripts. If you're curious what it does, [the source code](https://github.com/bigchaindb/bigchaindb/blob/master/deploy-cluster-aws/awsdeploy.sh) has lots of explanatory comments, so it's quite easy to read.
`awsdeploy.sh` is a Bash script which calls some Python and Fabric scripts. If you're curious what it does, [the source code](https://github.com/bigchaindb/bigchaindb/blob/master/deploy-cluster-aws/awsdeploy.sh) has many explanatory comments.
It should take a few minutes for the deployment to finish. If you run into problems, see the section on Known Deployment Issues below.
If you don't specify the name of your AWS deployment configuration file (e.g. `my_deploy_conf.py` above), then `default_deploy_conf.py` will be used by default.
It should take a few minutes for the deployment to finish. If you run into problems, see the section on **Known Deployment Issues** below.
The EC2 Console has a section where you can see all the instances you have running on EC2. You can `ssh` into a running instance using a command like:
```text