Make config filename the 1st arg of awsdeploy.sh

deploy-cluster-aws/awsdeploy.sh

@@ -4,13 +4,25 @@
 # if any command has a non-zero exit status
 set -e
 
-# Validate the values in deploy_conf.py
+# Check the (optional) command-line argument
-python validate_deploy_conf.py
+# (the name of the AWS deployment config file)
+if [ -z "$1" ]; then
+    # no first argument was provided
+    DEPLOY_CONF_FILE=default_deploy_conf.py
+else
+    DEPLOY_CONF_FILE=$1
+fi
 
-# Read deploy_conf.py
+# Check to make sure DEPLOY_CONF_FILE exists
+if [ ! -f $DEPLOY_CONF_FILE ]; then
+    echo "AWS deployment configuration file not found: "$DEPLOY_CONF_FILE
+    exit 1
+fi
+
+# Read DEPLOY_CONF_FILE
 # to set environment variables related to AWS deployment
-echo "Reading deploy_conf.py"
+echo "Reading "$DEPLOY_CONF_FILE
-source deploy_conf.py
+source $DEPLOY_CONF_FILE
 echo "NUM_NODES = "$NUM_NODES
 echo "BRANCH = "$BRANCH
 echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY
@@ -31,6 +43,15 @@ if [ ! -d "confiles" ]; then
     exit 1
 fi
 
+# Check if the number of files in confiles directory == NUM_NODES
+CONFILES_COUNT=`ls confiles | wc -l`
+if [ $CONFILES_COUNT != $NUM_NODES ]; then
+    echo "ERROR: CONFILES_COUNT = "$CONFILES_COUNT
+    echo "but NUM_NODES = "$NUM_NODES
+    echo "so there should be "$NUM_NODES" files in the confiles directory" 
+    exit 1
+fi
+
 # Auto-generate the tag to apply to all nodes in the cluster
 TAG="BDB-"$WHAT_TO_DEPLOY"-"`date +%m-%d@%H:%M`
 echo "TAG = "$TAG
@@ -49,7 +70,7 @@ chmod 0400 pem/bigchaindb.pem
 # 5. writes the shellscript add2known_hosts.sh
 # 6. (over)writes a file named hostlist.py
 #    containing a list of all public DNS names.
-python launch_ec2_nodes.py --tag $TAG
+python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG
 
 # Make add2known_hosts.sh executable then execute it.
 # This adds remote keys to ~/.ssh/known_hosts

deploy-cluster-aws/default_deploy_conf.py

@@ -1,16 +1,19 @@
 # AWS deployment config file
 
 # To use in a Bash shell script:
-# source deploy_conf.py
+# source default_deploy_conf.py
-# echo $EXAMPLEVAR
+# # $EXAMPLEVAR now has a value
 
 # To use in a Python script:
-# from deploy_conf import *
+# from default_deploy_conf import *
-# # EXAMPLEVAR now has a value
+# or
+# import importlib
+# cf = importlib.import_module('default_deploy_conf')
+# # cf.EXAMPLEVAR now has a value
 
 # DON'T PUT SPACES AROUND THE =
 # because that would confuse Bash.
-# Values can be strings in double quotes, or integers like 23
+# Example values: "string in double quotes", 32, True, False
 
 # NUM_NODES is the number of nodes to deploy
 NUM_NODES=3

deploy-cluster-aws/launch_ec2_nodes.py

@@ -16,16 +16,13 @@ import sys
 import time
 import socket
 import argparse
+import importlib
 import botocore
 import boto3
 
 from awscommon import get_naeips
-from deploy_conf import *
 
 
-# Make sure NUM_NODES is an int
-assert isinstance(NUM_NODES, int)
-
 # Ensure they're using Python 2.5-2.7
 pyver = sys.version_info
 major = pyver[0]
@@ -41,8 +38,52 @@ parser = argparse.ArgumentParser()
 parser.add_argument("--tag",
                     help="tag to add to all launched instances on AWS",
                     required=True)
+parser.add_argument("--deploy-conf-file",
+                    help="AWS deployment configuration file",
+                    required=True)
 args = parser.parse_args()
 tag = args.tag
+deploy_conf_file = args.deploy_conf_file
+
+# Import all the variables set in the AWS deployment configuration file
+# (Remove the '.py' from the end of deploy_conf_file.)
+cf = importlib.import_module(deploy_conf_file[:-3])
+try:
+    NUM_NODES = cf.NUM_NODES
+    BRANCH = cf.BRANCH
+    WHAT_TO_DEPLOY = cf.WHAT_TO_DEPLOY
+    USE_KEYPAIRS_FILE = cf.USE_KEYPAIRS_FILE
+    IMAGE_ID = cf.IMAGE_ID
+    INSTANCE_TYPE = cf.INSTANCE_TYPE
+except AttributeError as e:
+    print('One of the AWS deployment configuration settings was '
+          'not set in the AWS deployment configuration file ' +
+          '{}'.format(deploy_conf_file))
+    print('Read this traceback to find out which one (in ALL_CAPS):')
+    raise
+
+# Validate the variables set in the AWS deployment configuration file
+try:
+    assert isinstance(NUM_NODES, int)
+    assert isinstance(BRANCH, str)
+    assert isinstance(WHAT_TO_DEPLOY, str)
+    assert isinstance(USE_KEYPAIRS_FILE, bool)
+    assert isinstance(IMAGE_ID, str)
+    assert isinstance(INSTANCE_TYPE, str)
+except AssertionError as e:
+    print('One of the AWS deployment settings has a value of the wrong type.')
+    print('Read this traceback to find out which one (in ALL_CAPS):')
+    raise
+
+if NUM_NODES > 64:
+    raise ValueError('NUM_NODES should be less than or equal to 64. '
+                     'The AWS deployment configuration file sets it to {}'.
+                     format(NUM_NODES))
+
+if WHAT_TO_DEPLOY not in ['servers', 'clients']:
+    raise ValueError('WHAT_TO_DEPLOY should be either "servers" or "clients". '
+                     'The AWS deployment configuration file sets it to {}'.
+                     format(WHAT_TO_DEPLOY))
 
 # Get an AWS EC2 "resource"
 # See http://boto3.readthedocs.org/en/latest/guide/resources.html

deploy-cluster-aws/validate_deploy_conf.py

@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-"""This script validates the values in deploy_conf.py
-"""
-
-from __future__ import unicode_literals
-
-import sys
-
-from deploy_conf import *
-
-try:
-    assert isinstance(NUM_NODES, int)
-    assert isinstance(BRANCH, str)
-    assert isinstance(WHAT_TO_DEPLOY, str)
-    assert isinstance(USE_KEYPAIRS_FILE, bool)
-    assert isinstance(IMAGE_ID, str)
-    assert isinstance(INSTANCE_TYPE, str)
-except NameError as e:
-    sys.exit('A variable with {} '.format(e.args[0]) + 'in deploy_conf.py')
-
-if NUM_NODES > 64:
-    raise ValueError('NUM_NODES should be less than or equal to 64. '
-                     'The deploy_conf.py file sets it to {}'.format(NUM_NODES))
-
-if WHAT_TO_DEPLOY not in ['servers', 'clients']:
-    raise ValueError('WHAT_TO_DEPLOY should be either "servers" or "clients". '
-                     'The deploy_conf.py file sets it to {}'.
-                     format(WHAT_TO_DEPLOY))

docs/source/deploy-on-aws.md

@@ -131,7 +131,7 @@ To configure a BigchainDB node to send monitoring data to the monitoring server,
 
 ### Step 1
 
-Suppose _N_ is the number of nodes you want in your BigchainDB cluster. If you already have a set of _N_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory, then you can jump to step 2. To create such a set, you can do something like:
+Suppose _N_ is the number of nodes you want in your BigchainDB cluster. If you already have a set of _N_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory, then you can jump to the next step. To create such a set, you can do something like:
 ```text
 # in a Python 3 virtual environment where bigchaindb is installed
 cd bigchaindb
@@ -141,25 +141,11 @@ cd deploy-cluster-aws
 
 That will create three (3) _default_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory (which will be created if it doesn't already exist). The three files will be named `bcdb_conf0`, `bcdb_conf1`, and `bcdb_conf2`.
 
-You can look inside those files if you're curious. In step 2, they'll be modified. For example, the default keyring is an empty list. In step 2, the deployment script automatically changes the keyring of each node to be a list of the public keys of all other nodes. Other changes are also made.
+You can look inside those files if you're curious. For example, the default keyring is an empty list. Later, the deployment script automatically changes the keyring of each node to be a list of the public keys of all other nodes. Other changes are also made. That is, the configuration files generated in this step are _not_ what will be sent to the deployed nodes; they're just a starting point.
-
-**An Aside on Using a Standard Set of Keypairs**
-
-It's possible to deploy BigchainDB servers with a known set of keypairs. You can generate a set of keypairs in a file named `keypairs.py` using the `write_keypairs_file.py` script. For example:
-```text
-# in a Python 3 virtual environment where bigchaindb is installed
-cd bigchaindb
-cd deploy-cluster-aws
-python3 write_keypairs_file.py 100
-```
-
-The above command generates a file with 100 keypairs. (You can generate more keypairs than you need, so you can use the same list over and over again, for different numbers of servers.) To make the `awsdeploy.sh` script read all keys from `keypairs.py`, just set `USE_KEYPAIRS_FILE=True` in `deploy_conf.py`.
 
 ### Step 2
 
-Step 2 is to launch the nodes ("instances") on AWS, to install all the necessary software on them, configure the software, run the software, and more.
+Step 2 is to make an AWS deployment configuration file, if necessary. There's a default AWS configuration file named `default_deploy_conf.py`. It has many comments explaining each setting. The default AWS deployment settings are (or should be):
-
-First, edit the AWS deployment configuration file, `deploy_conf.py`, in the `bigchaindb/deploy-cluster-aws` directory. It comes with comments explaining each of the configuration settings. You may want to make a copy of `deploy_conf.py` before editing it. The defaults are (or should be):
 ```text
 NUM_NODES=3
 BRANCH="master"
@@ -169,19 +155,36 @@ IMAGE_ID="ami-accff2b1"
 INSTANCE_TYPE="m3.2xlarge"
 ```
 
-Once you've edited `deploy_conf.py` to your liking:
+If you're happy with those settings, then you can skip to the next step. Otherwise, you could make a copy of `default_deploy_conf.py` (e.g. `cp default_deploy_conf.py my_deploy_conf.py`) and then edit the new file using a text editor.
+
+If you want your nodes to have a predictable set of pre-generated keypairs, then you should 1) set `USE_KEYPAIRS_FILE=True` in the AWS deployment configuration file, and 2) provide a `keypairs.py` file containing enough keypairs for all of your nodes. You can generate a `keypairs.py` file using the `write_keypairs_file.py` script. For example:
+```text
+# in a Python 3 virtual environment where bigchaindb is installed
+cd bigchaindb
+cd deploy-cluster-aws
+python3 write_keypairs_file.py 100
+```
+
+The above command generates a `keypairs.py` file with 100 keypairs. You can generate more keypairs than you need, so you can use the same list over and over again, for different numbers of servers. The deployment scripts will only use the first NUM_NODES keypairs.
+
+### Step 3
+
+Step 3 is to launch the nodes ("instances") on AWS, to install all the necessary software on them, configure the software, run the software, and more. Here's how you'd do that:
+
 ```text
 # in a Python 2.5-2.7 virtual environment where fabric, boto3, etc. are installed
 cd bigchaindb
 cd deploy-cluster-aws
-./awsdeploy.sh
+./awsdeploy.sh my_deploy_conf.py
 # Only if you want to start BigchainDB on all the nodes:
 fab start_bigchaindb
 ```
 
-`awsdeploy.sh` is a Bash script which calls some Python and Fabric scripts. If you're curious what it does, [the source code](https://github.com/bigchaindb/bigchaindb/blob/master/deploy-cluster-aws/awsdeploy.sh) has lots of explanatory comments, so it's quite easy to read.
+`awsdeploy.sh` is a Bash script which calls some Python and Fabric scripts. If you're curious what it does, [the source code](https://github.com/bigchaindb/bigchaindb/blob/master/deploy-cluster-aws/awsdeploy.sh) has many explanatory comments.
 
-It should take a few minutes for the deployment to finish. If you run into problems, see the section on Known Deployment Issues below.
+If you don't specify the name of your AWS deployment configuration file (e.g. `my_deploy_conf.py` above), then `default_deploy_conf.py` will be used by default.
+
+It should take a few minutes for the deployment to finish. If you run into problems, see the section on **Known Deployment Issues** below.
 
 The EC2 Console has a section where you can see all the instances you have running on EC2. You can `ssh` into a running instance using a command like:
 ```text