several AWS deployment improvements

deploy-cluster-aws/create_rethinkdb_conf.py

@@ -8,7 +8,7 @@ from __future__ import unicode_literals
 import os
 import os.path
 import shutil
-from hostlist import hosts_dev
+from hostlist import public_dns_names
 
 # cwd = current working directory
 old_cwd = os.getcwd()
@@ -22,7 +22,7 @@ shutil.copy2('rethinkdb.conf.template', 'rethinkdb.conf')
 # Append additional lines to rethinkdb.conf
 with open('rethinkdb.conf', 'a') as f:
     f.write('## The host:port of a node that RethinkDB will connect to\n')
-    for public_dns_name in hosts_dev:
+    for public_dns_name in public_dns_names:
         f.write('join=' + public_dns_name + ':29015\n')
 
 os.chdir(old_cwd)

deploy-cluster-aws/fab_prepare_chain.py

@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-
-""" Generating genesis block
-"""
-
-from __future__ import with_statement, unicode_literals
-
-from fabric import colors as c
-from fabric.api import *
-from fabric.api import local, puts, settings, hide, abort, lcd, prefix
-from fabric.api import run, sudo, cd, get, local, lcd, env, hide
-from fabric.api import task, parallel
-from fabric.contrib import files
-from fabric.contrib.files import append, exists
-from fabric.contrib.console import confirm
-from fabric.contrib.project import rsync_project
-from fabric.operations import run, put
-from fabric.context_managers import settings
-from fabric.decorators import roles
-from fabtools import *
-
-env.user = 'ubuntu'
-env.key_filename = 'pem/bigchaindb.pem'
-
-@task
-def init_bigchaindb():
-    run('bigchaindb -y start &', pty = False)

deploy-cluster-aws/fabfile.py

@@ -1,47 +1,58 @@
 # -*- coding: utf-8 -*-
-
+"""A Fabric fabfile with functionality to prepare, install, and configure
-"""A fabfile with functionality to prepare, install, and configure
+BigchainDB, including its storage backend (RethinkDB).
-bigchaindb, including its storage backend.
 """
 
 from __future__ import with_statement, unicode_literals
 
-import requests
+from fabric.api import sudo, env
-from time import *
-import os
-from datetime import datetime, timedelta
-import json
-from pprint import pprint
-
-from fabric import colors as c
-from fabric.api import *
-from fabric.api import local, puts, settings, hide, abort, lcd, prefix
-from fabric.api import run, sudo, cd, get, local, lcd, env, hide
 from fabric.api import task, parallel
-from fabric.contrib import files
+from fabric.contrib.files import sed
-from fabric.contrib.files import append, exists
-from fabric.contrib.console import confirm
-from fabric.contrib.project import rsync_project
 from fabric.operations import run, put
 from fabric.context_managers import settings
-from fabric.decorators import roles
-from fabtools import *
 
-from hostlist import hosts_dev
+from hostlist import public_dns_names
 
-env.hosts = hosts_dev
+# Ignore known_hosts
-env.roledefs = {
+# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts
-    "role1": hosts_dev,
+env.disable_known_hosts = True
-    "role2": [hosts_dev[0]],
+
-    }
+# What remote servers should Fabric connect to? With what usernames?
-env.roles = ["role1"]
 env.user = 'ubuntu'
+env.hosts = public_dns_names
+
+# SSH key files to try when connecting:
+# http://docs.fabfile.org/en/1.10/usage/env.html#key-filename
 env.key_filename = 'pem/bigchaindb.pem'
 
+newrelic_license_key = 'you_need_a_real_license_key'
+
 
 ######################################################################
 
-# base software rollout
+# DON'T PUT @parallel
+@task
+def set_hosts(hosts):
+    """A helper function to change env.hosts from the
+    command line.
+
+    Args:
+        hosts (str): 'one_node' or 'two_nodes'
+
+    Example:
+        fab set_hosts:one_node init_bigchaindb
+    """
+    if hosts == 'one_node':
+        env.hosts = public_dns_names[0]
+    elif hosts == 'two_nodes':
+        env.hosts = public_dns_names[0:1]
+    else:
+        raise ValueError('Invalid input to set_hosts.'
+                         ' Expected one_node or two_nodes.'
+                         ' Got {}'.format(hosts))
+
+
+# Install base software
 @task
 @parallel
 def install_base_software():
@@ -59,7 +70,7 @@ def install_base_software():
                      python3-pip ipython3 sysstat s3cmd')
 
 
-# RethinkDB
+# Install RethinkDB
 @task
 @parallel
 def install_rethinkdb():
@@ -67,7 +78,7 @@ def install_rethinkdb():
     with settings(warn_only=True):
         # preparing filesystem
         sudo("mkdir -p /data")
-        # Locally mounted storage (m3.2xlarge, aber auch c3.xxx)
+        # Locally mounted storage (m3.2xlarge, but also c3.xxx)
         try:
             sudo("umount /mnt")
             sudo("mkfs -t ext4 /dev/xvdb")
@@ -91,27 +102,48 @@ def install_rethinkdb():
         sudo('chown -R rethinkdb:rethinkdb /data')
         # copy config file to target system
         put('conf/rethinkdb.conf',
-            '/etc/rethinkdb/instances.d/instance1.conf', mode=0600, use_sudo=True)
+            '/etc/rethinkdb/instances.d/instance1.conf',
+            mode=0600,
+            use_sudo=True)
         # initialize data-dir
         sudo('rm -rf /data/*')
         # finally restart instance
         sudo('/etc/init.d/rethinkdb restart')
 
 
-# bigchaindb deployment
+# Install BigchainDB (from PyPI)
 @task
 @parallel
 def install_bigchaindb():
     sudo('python3 -m pip install bigchaindb')
 
 
-# startup all nodes of bigchaindb in cluster
+# Configure BigchainDB
 @task
 @parallel
-def start_bigchaindb_nodes():
+def configure_bigchaindb():
+    run('bigchaindb -y configure', pty=False)
+
+
+# Initialize BigchainDB
+# i.e. create the database, the tables,
+# the indexes, and the genesis block.
+# (This only needs to be run on one node.)
+# Call using:
+#     fab set_hosts:one_node init_bigchaindb
+@task
+def init_bigchaindb():
+    run('bigchaindb init', pty=False)
+
+
+# Start BigchainDB using screen
+@task
+@parallel
+def start_bigchaindb():
     sudo('screen -d -m bigchaindb -y start &', pty=False)
 
 
+# Install and run New Relic
 @task
 def install_newrelic():
     with settings(warn_only=True):
@@ -119,18 +151,18 @@ def install_newrelic():
         # sudo('apt-key adv --keyserver hkp://subkeys.pgp.net --recv-keys 548C16BF')
         sudo('apt-get update')
         sudo('apt-get -y --force-yes install newrelic-sysmond')
-        sudo('nrsysmond-config --set license_key=c88af00c813983f8ee12e9b455aa13fde1cddaa8')
+        sudo('nrsysmond-config --set license_key=' + newrelic_license_key)
         sudo('/etc/init.d/newrelic-sysmond restart')
 
 
-###############################
+###########################
-# Security / FirewallStuff next
+# Security / Firewall Stuff
-###############################
+###########################
 
 @task
 def harden_sshd():
-    """Security harden sshd."""
+    """Security harden sshd.
-
+    """
     # Disable password authentication
     sed('/etc/ssh/sshd_config',
         '#PasswordAuthentication yes',
@@ -147,7 +179,8 @@ def harden_sshd():
 def disable_root_login():
     """Disable `root` login for even more security. Access to `root` account
     is now possible by first connecting with your dedicated maintenance
-    account and then running ``sudo su -``."""
+    account and then running ``sudo su -``.
+    """
     sudo('passwd --lock root')
 
 
@@ -172,7 +205,7 @@ def set_fw():
 
 
 #########################################################
-# some helper-functions to handle bad behavior of cluster
+# Some helper-functions to handle bad behavior of cluster
 #########################################################
 
 # rebuild indexes

deploy-cluster-aws/launch_ec2_nodes.py

@@ -166,26 +166,31 @@ for i, instance in enumerate(instances_with_tag):
           format(instance.instance_id))
 
 # Get a list of the pubic DNS names of the instances_with_tag
-hosts_dev = []
+public_dns_names = []
 for instance in instances_with_tag:
     public_dns_name = getattr(instance, 'public_dns_name', None)
     if public_dns_name is not None:
-        hosts_dev.append(public_dns_name)
+        public_dns_names.append(public_dns_name)
 
 # Write a shellscript to add remote keys to ~/.ssh/known_hosts
 print('Preparing shellscript to add remote keys to known_hosts')
 with open('add2known_hosts.sh', 'w') as f:
     f.write('#!/bin/bash\n')
-    for public_dns_name in hosts_dev:
+    for public_dns_name in public_dns_names:
         f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n')
 
-# Create a file named hostlist.py containing hosts_dev.
+# Create a file named hostlist.py containing public_dns_names.
 # If a hostlist.py already exists, it will be overwritten.
 print('Writing hostlist.py')
 with open('hostlist.py', 'w') as f:
     f.write('# -*- coding: utf-8 -*-\n')
+    f.write('"""A list of the public DNS names of all the nodes in this\n')
+    f.write('BigchainDB cluster/federation.\n')
+    f.write('"""\n')
+    f.write('\n')
     f.write('from __future__ import unicode_literals\n')
-    f.write('hosts_dev = {}\n'.format(hosts_dev))
+    f.write('\n')
+    f.write('public_dns_names = {}\n'.format(public_dns_names))
 
 # Wait
 wait_time = 45

deploy-cluster-aws/startup.sh

@@ -55,27 +55,33 @@ chmod +x add2known_hosts.sh
 # (Re)create the RethinkDB configuration file conf/rethinkdb.conf
 python create_rethinkdb_conf.py
 
-# rollout base packages (dependencies) needed before
+# Rollout base packages (dependencies) needed before
-# storage backend (rethinkdb) and bigchaindb can be rolled out
+# storage backend (RethinkDB) and BigchainDB can be rolled out
 fab install_base_software
 
-# rollout storage backend (rethinkdb)
+# Rollout storage backend (RethinkDB) and start it
 fab install_rethinkdb
 
-# rollout bigchaindb
+# Rollout BigchainDB (but don't start it yet)
 fab install_bigchaindb
 
-# generate genesis block
+# Configure BigchainDB on all nodes
-# HORST is the last public_dns_name listed in conf/rethinkdb.conf
+fab configure_bigchaindb
-# For example:
-# ec2-52-58-86-145.eu-central-1.compute.amazonaws.com
-HORST=`tail -1 conf/rethinkdb.conf|cut -d: -f1|cut -d= -f2`
-fab -H $HORST -f fab_prepare_chain.py init_bigchaindb
 
-# initiate sharding
+# TODO Get public keys from all nodes
-fab start_bigchaindb_nodes
+# using e.g. bigchaindb export-pubkey
+
+# TODO Add list of public keys to keyring of all nodes
+# using e.g. bigchaindb import-pubkey
+
+# Send a "bigchaindb init" command to one node
+# to initialize the BigchainDB database 
+# i.e. create the database, the tables,
+# the indexes, and the genesis block.
+fab set_hosts:one_node init_bigchaindb
+
+# Start BigchainDB on all the nodes using "screen"
+fab start_bigchaindb
 
 # cleanup
 rm add2known_hosts.sh
-
-# DONE