bigchaindb/deploy-cluster-aws/awsdeploy.sh

#!/bin/bash

set -euo pipefail
# -e Abort at the first failed line (i.e. if exit status is not 0)
# -u Abort when undefined variable is used
# -o pipefail (Bash-only) Piped commands return the status
#    of the last failed command, rather than the status of the last command

# Check for the first command-line argument
# (the name of the AWS deployment config file)
if [ -z "$1" ]; then
    # no first argument was provided
    echo "awsdeploy: missing file operand"
    echo "Usage: awsdeploy DEPLOY_CONF_FILE"
    echo "Deploy BigchainDB on AWS using the specified AWS deployment configuration file"
    exit 1
fi

DEPLOY_CONF_FILE=$1

# Check to make sure DEPLOY_CONF_FILE exists
if [ ! -f "$DEPLOY_CONF_FILE" ]; then
    echo "AWS deployment configuration file not found: "$DEPLOY_CONF_FILE
    exit 1
fi

# Read DEPLOY_CONF_FILE
# to set environment variables related to AWS deployment
echo "Reading "$DEPLOY_CONF_FILE
source $DEPLOY_CONF_FILE

# Check if SSH_KEY_NAME got set
if [ "$SSH_KEY_NAME" == "not-set-yet" ] || \
   [ "$SSH_KEY_NAME" == "" ] || \
   [ -z ${SSH_KEY_NAME+x} ]; then
    echo "SSH_KEY_NAME was not set in that file"
    exit 1
fi

echo "NUM_NODES = "$NUM_NODES
echo "BRANCH = "$BRANCH
echo "SSH_KEY_NAME" = $SSH_KEY_NAME
echo "USE_KEYPAIRS_FILE = "$USE_KEYPAIRS_FILE
echo "IMAGE_ID = "$IMAGE_ID
echo "INSTANCE_TYPE = "$INSTANCE_TYPE
echo "SECURITY_GROUP = "$SECURITY_GROUP
echo "USING_EBS = "$USING_EBS
# Treat booleans as strings which must be either "True" or "False"
if [ "$USING_EBS" == "True" ]; then
    echo "EBS_VOLUME_SIZE = "$EBS_VOLUME_SIZE
    echo "EBS_OPTIMIZED = "$EBS_OPTIMIZED
fi
echo "ENABLE_WEB_ADMIN = "$ENABLE_WEB_ADMIN
if [ "$ENABLE_WEB_ADMIN" == "True" ]; then
    echo "BIND_HTTP_TO_LOCALHOST = "$BIND_HTTP_TO_LOCALHOST
fi

# Check for the SSH private key file
if [ ! -f "$HOME/.ssh/$SSH_KEY_NAME" ]; then
    echo "The SSH private key file "$HOME"/.ssh/"$SSH_KEY_NAME" is missing"
    exit 1
fi

# Check for the confiles directory
if [ ! -d "confiles" ]; then
    echo "Directory confiles is needed but does not exist"
    echo "See make_confiles.sh to find out how to make it"
    exit 1
fi

# Check if NUM_NODES got set
if [ -z "$NUM_NODES" ]; then
    echo "NUM_NODES is not set in the AWS deployment configuration file "$DEPLOY_CONF_FILE
    exit 1
fi

# Check if the number of files in confiles directory == NUM_NODES
CONFILES_COUNT=`ls confiles | wc -l`
if [[ $CONFILES_COUNT != $NUM_NODES ]]; then
    echo "ERROR: CONFILES_COUNT = "$CONFILES_COUNT
    echo "but NUM_NODES = "$NUM_NODES
    echo "so there should be "$NUM_NODES" files in the confiles directory" 
    exit 1
fi

# Auto-generate the tag to apply to all nodes in the cluster
TAG="BDB-Server-"`date +%m-%d@%H:%M`
echo "TAG = "$TAG

# Change the file permissions on the SSH private key file
# so that the owner can read it, but that's all
chmod 0400 $HOME/.ssh/$SSH_KEY_NAME

# The following Python script does these things:
# 0. allocates more elastic IP addresses if necessary,
# 1. launches the specified number of nodes (instances) on Amazon EC2,
# 2. tags them with the specified tag,
# 3. waits until those instances exist and are running,
# 4. for each instance, it associates an elastic IP address
#    with that instance,
# 5. writes the shellscript add2known_hosts.sh
# 6. (over)writes a file named hostlist.py
#    containing a list of all public DNS names.
# 7. (over)writes a file named ssh_key.py
#    containing the location of the private SSH key file.
python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG

# Make add2known_hosts.sh executable then execute it.
# This adds remote keys to ~/.ssh/known_hosts
chmod +x add2known_hosts.sh
./add2known_hosts.sh

# Test an SSH connection to one of the hosts
# and prompt the user for their SSH password if necessary
fab set_host:0 test_ssh

# Rollout base packages (dependencies) needed before
# storage backend (RethinkDB) and BigchainDB can be rolled out
fab install_base_software
fab get_pip3
fab upgrade_setuptools

# (Re)create the RethinkDB configuration file conf/rethinkdb.conf
if [ "$ENABLE_WEB_ADMIN" == "True" ]; then
    if [ "$BIND_HTTP_TO_LOCALHOST" == "True" ]; then
        python create_rethinkdb_conf.py --enable-web-admin --bind-http-to-localhost
    else
        python create_rethinkdb_conf.py --enable-web-admin
    fi
else
    python create_rethinkdb_conf.py
fi

# Rollout RethinkDB and start it
fab prep_rethinkdb_storage:$USING_EBS
fab install_rethinkdb
fab configure_rethinkdb
fab delete_rethinkdb_data
fab start_rethinkdb

# Rollout BigchainDB (but don't start it yet)
if [ "$BRANCH" == "pypi" ]; then
    fab install_bigchaindb_from_pypi
else
    cd ..
    rm -f bigchaindb-archive.tar.gz
    git archive $BRANCH --format=tar --output=bigchaindb-archive.tar
    gzip bigchaindb-archive.tar
    mv bigchaindb-archive.tar.gz deploy-cluster-aws
    cd deploy-cluster-aws
    fab install_bigchaindb_from_git_archive
    rm bigchaindb-archive.tar.gz
fi

# Configure BigchainDB on all nodes

# The idea is to send a bunch of locally-created configuration
# files out to each of the instances / nodes.

# Assume a set of $NUM_NODES BigchaindB config files
# already exists in the confiles directory.
# One can create a set using a command like
# ./make_confiles.sh confiles $NUM_NODES
# (We can't do that here now because this virtual environment
# is a Python 2 environment that may not even have
# bigchaindb installed, so bigchaindb configure can't be called)

# Transform the config files in the confiles directory
# to have proper keyrings etc.
if [ "$USE_KEYPAIRS_FILE" == "True" ]; then
    python clusterize_confiles.py -k confiles $NUM_NODES
else
    python clusterize_confiles.py confiles $NUM_NODES
fi

# Send one of the config files to each instance
for (( HOST=0 ; HOST<$NUM_NODES ; HOST++ )); do
    CONFILE="bcdb_conf"$HOST
    echo "Sending "$CONFILE
    fab set_host:$HOST send_confile:$CONFILE
done

# Initialize BigchainDB (i.e. Create the RethinkDB database,
# the tables, the indexes, and genesis glock). Note that
# this will only be sent to one of the nodes, see the
# definition of init_bigchaindb() in fabfile.py to see why.
fab init_bigchaindb
fab set_shards:$NUM_NODES
echo "To set the replication factor to 3, do: fab set_replicas:3"
echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb"

# cleanup
rm add2known_hosts.sh