diff --git a/.gitignore b/.gitignore index b1d9188c..8005b32c 100644 --- a/.gitignore +++ b/.gitignore @@ -65,12 +65,11 @@ target/ # pyenv .python-version -# Private key files from AWS -*.pem - # Some files created when deploying a cluster on AWS deploy-cluster-aws/conf/rethinkdb.conf -deploy-cluster-aws/hostlist.py deploy-cluster-aws/confiles/ deploy-cluster-aws/client_confile +deploy-cluster-aws/hostlist.py +deploy-cluster-aws/ssh_key.py benchmarking-tests/hostlist.py +benchmarking-tests/ssh_key.py diff --git a/benchmarking-tests/fabfile.py b/benchmarking-tests/fabfile.py index ddfb36dd..44a31888 100644 --- a/benchmarking-tests/fabfile.py +++ b/benchmarking-tests/fabfile.py @@ -7,6 +7,7 @@ from fabric.operations import run, put from fabric.context_managers import settings from hostlist import public_dns_names +from ssh_key import ssh_key_path # Ignore known_hosts # http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts @@ -18,7 +19,7 @@ env.hosts = public_dns_names # SSH key files to try when connecting: # http://docs.fabfile.org/en/1.10/usage/env.html#key-filename -env.key_filename = 'pem/bigchaindb.pem' +env.key_filename = ssh_key_path @task diff --git a/deploy-cluster-aws/awsdeploy.sh b/deploy-cluster-aws/awsdeploy.sh index d8daa1f8..e2154155 100755 --- a/deploy-cluster-aws/awsdeploy.sh +++ b/deploy-cluster-aws/awsdeploy.sh @@ -26,9 +26,19 @@ fi # to set environment variables related to AWS deployment echo "Reading "$DEPLOY_CONF_FILE source $DEPLOY_CONF_FILE + +# Check if SSH_KEY_NAME got set +if [ "$SSH_KEY_NAME" == "not-set-yet" ] || \ + [ "$SSH_KEY_NAME" == "" ] || \ + [ -z ${SSH_KEY_NAME+x} ]; then + echo "SSH_KEY_NAME was not set in that file" + exit 1 +fi + echo "NUM_NODES = "$NUM_NODES echo "BRANCH = "$BRANCH echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY +echo "SSH_KEY_NAME" = $SSH_KEY_NAME echo "USE_KEYPAIRS_FILE = "$USE_KEYPAIRS_FILE echo "IMAGE_ID = "$IMAGE_ID echo "INSTANCE_TYPE = "$INSTANCE_TYPE @@ -38,9 +48,9 @@ if [ "$USING_EBS" = True ]; then echo "EBS_OPTIMIZED = "$EBS_OPTIMIZED fi -# Check for AWS private key file (.pem file) -if [ ! -f "pem/bigchaindb.pem" ]; then - echo "File pem/bigchaindb.pem (AWS private key) is missing" +# Check for the SSH private key file +if [ ! -f "$HOME/.ssh/$SSH_KEY_NAME" ]; then + echo "The SSH private key file "$HOME"/.ssh/"$SSH_KEY_NAME" is missing" exit 1 fi @@ -70,9 +80,9 @@ fi TAG="BDB-"$WHAT_TO_DEPLOY"-"`date +%m-%d@%H:%M` echo "TAG = "$TAG -# Change the file permissions on pem/bigchaindb.pem +# Change the file permissions on the SSH private key file # so that the owner can read it, but that's all -chmod 0400 pem/bigchaindb.pem +chmod 0400 $HOME/.ssh/$SSH_KEY_NAME # The following Python script does these things: # 0. allocates more elastic IP addresses if necessary, @@ -84,6 +94,8 @@ chmod 0400 pem/bigchaindb.pem # 5. writes the shellscript add2known_hosts.sh # 6. (over)writes a file named hostlist.py # containing a list of all public DNS names. +# 7. (over)writes a file named ssh_key.py +# containing the location of the private SSH key file. python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG # Make add2known_hosts.sh executable then execute it. @@ -91,6 +103,10 @@ python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG chmod +x add2known_hosts.sh ./add2known_hosts.sh +# Test an SSH connection to one of the hosts +# and prompt the user for their SSH password if necessary +fab set_host:0 test_ssh + # Rollout base packages (dependencies) needed before # storage backend (RethinkDB) and BigchainDB can be rolled out fab install_base_software diff --git a/deploy-cluster-aws/example_deploy_conf.py b/deploy-cluster-aws/example_deploy_conf.py index f1da94e9..ed755a0d 100644 --- a/deploy-cluster-aws/example_deploy_conf.py +++ b/deploy-cluster-aws/example_deploy_conf.py @@ -27,6 +27,11 @@ BRANCH="master" # What do you want to deploy? WHAT_TO_DEPLOY="servers" +# SSH_KEY_NAME is the name of the SSH private key file +# in $HOME/.ssh/ +# It is used for SSH communications with AWS instances. +SSH_KEY_NAME="not-set-yet" + # USE_KEYPAIRS_FILE is either True or False # Should node keypairs be read from keypairs.py? # (If False, then the keypairs will be whatever is in the the diff --git a/deploy-cluster-aws/fabfile.py b/deploy-cluster-aws/fabfile.py index 3186811f..4898490e 100644 --- a/deploy-cluster-aws/fabfile.py +++ b/deploy-cluster-aws/fabfile.py @@ -15,6 +15,7 @@ from fabric.operations import run, put from fabric.context_managers import settings from hostlist import public_dns_names +from ssh_key import ssh_key_path # Ignore known_hosts # http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts @@ -26,7 +27,7 @@ env.hosts = public_dns_names # SSH key files to try when connecting: # http://docs.fabfile.org/en/1.10/usage/env.html#key-filename -env.key_filename = 'pem/bigchaindb.pem' +env.key_filename = ssh_key_path ###################################################################### @@ -48,6 +49,11 @@ def set_host(host_index): env.hosts = [public_dns_names[int(host_index)]] +@task +def test_ssh(): + run('echo "If you see this, then SSH to a remote host worked."') + + # Install base software @task @parallel diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py index 8e0c2d9b..339b3519 100644 --- a/deploy-cluster-aws/launch_ec2_nodes.py +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -9,9 +9,12 @@ 5. writes the shellscript add2known_hosts.sh 6. (over)writes a file named hostlist.py containing a list of all public DNS names. +7. (over)writes a file named ssh_key.py + containing the location of the private SSH key file. """ from __future__ import unicode_literals +from os.path import expanduser import sys import time import socket @@ -23,9 +26,9 @@ import boto3 from awscommon import get_naeips -SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'USE_KEYPAIRS_FILE', - 'IMAGE_ID', 'INSTANCE_TYPE', 'USING_EBS', 'EBS_VOLUME_SIZE', - 'EBS_OPTIMIZED'] +SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'SSH_KEY_NAME', + 'USE_KEYPAIRS_FILE', 'IMAGE_ID', 'INSTANCE_TYPE', 'USING_EBS', + 'EBS_VOLUME_SIZE', 'EBS_OPTIMIZED'] class SettingsTypeError(TypeError): @@ -76,6 +79,9 @@ if not isinstance(BRANCH, str): if not isinstance(WHAT_TO_DEPLOY, str): raise SettingsTypeError('WHAT_TO_DEPLOY should be a string') +if not isinstance(SSH_KEY_NAME, str): + raise SettingsTypeError('SSH_KEY_NAME should be a string') + if not isinstance(USE_KEYPAIRS_FILE, bool): msg = 'USE_KEYPAIRS_FILE should be a boolean (True or False)' raise SettingsTypeError(msg) @@ -105,6 +111,11 @@ if WHAT_TO_DEPLOY not in ['servers', 'clients']: 'The AWS deployment configuration file sets it to {}'. format(WHAT_TO_DEPLOY)) +if SSH_KEY_NAME in ['not-set-yet', '', None]: + raise ValueError('SSH_KEY_NAME should be set. ' + 'The AWS deployment configuration file sets it to {}'. + format(SSH_KEY_NAME)) + # Since we assume 'gp2' volumes (for now), the possible range is 1 to 16384 if EBS_VOLUME_SIZE > 16384: raise ValueError('EBS_VOLUME_SIZE should be <= 16384. ' @@ -193,7 +204,7 @@ for _ in range(NUM_NODES): ImageId=IMAGE_ID, MinCount=1, MaxCount=1, - KeyName='bigchaindb', + KeyName=SSH_KEY_NAME, InstanceType=INSTANCE_TYPE, SecurityGroupIds=['bigchaindb'], BlockDeviceMappings=[dm], @@ -204,7 +215,7 @@ for _ in range(NUM_NODES): ImageId=IMAGE_ID, MinCount=1, MaxCount=1, - KeyName='bigchaindb', + KeyName=SSH_KEY_NAME, InstanceType=INSTANCE_TYPE, SecurityGroupIds=['bigchaindb'] ) @@ -281,6 +292,20 @@ with open('hostlist.py', 'w') as f: f.write('\n') f.write('public_dns_names = {}\n'.format(public_dns_names)) +# Create a file named ssh_key.py +# containing the location of the private SSH key file. +# If a ssh_key.py already exists, it will be overwritten. +print('Writing ssh_key.py') +with open('ssh_key.py', 'w') as f: + f.write('# -*- coding: utf-8 -*-\n') + f.write('"""This file exists as a convenient way for Fabric to get\n') + f.write('the location of the private SSH key file.') + f.write('"""\n') + f.write('\n') + f.write('from __future__ import unicode_literals\n') + f.write('\n') + home = expanduser('~') + f.write('ssh_key_path = "{}/.ssh/{}"\n'.format(home, SSH_KEY_NAME)) # For each node in the cluster, check port 22 (ssh) until it's reachable for instance in instances_with_tag: diff --git a/docs/source/appendices/aws-setup.md b/docs/source/appendices/aws-setup.md index 8b7dcd6c..f57997c5 100644 --- a/docs/source/appendices/aws-setup.md +++ b/docs/source/appendices/aws-setup.md @@ -36,3 +36,37 @@ Default output format [None]: [Press Enter] ``` This writes two files: `~/.aws/credentials` and `~/.aws/config`. AWS tools and packages look for those files. + + +## Generate an RSA Key Pair for SSH + +Eventually, you'll have one or more instances (virtual machines) running on AWS and you'll want to SSH to them. To do that, you need a public/private key pair. The public key will be sent to AWS, and you can tell AWS to put it in any instances you provision there. You'll keep the private key on your local workstation. + +First you need to make up a key name. Some ideas: + +* `bcdb-troy-1` +* `bigchaindb-7` +* `bcdb-jupiter` + +If you already have key pairs on AWS (Amazon EC2), you have to pick a name that's not already being used. +Below, replace every instance of `` with your actual key name. +To generate a public/private RSA key pair with that name: +```text +ssh-keygen -t rsa -C "" -f ~/.ssh/ +``` + +It will ask you for a passphrase. You can use whatever passphrase you like, but don't lose it. Two keys (files) will be created in `~/.ssh/`: + +1. `~/.ssh/.pub` is the public key +2. `~/.ssh/` is the private key + +To send the public key to AWS, use the AWS Command-Line Interface: +```text +aws ec2 import-key-pair \ +--key-name "" \ +--public-key-material file://~/.ssh/.pub +``` + +If you're curious why there's a `file://` in front of the path to the public key, see issue [aws/aws-cli#41 on GitHub](https://github.com/aws/aws-cli/issues/41). + +If you want to verify that your key pair was imported by AWS, go to the Amazon EC2 console at [https://console.aws.amazon.com/ec2/](https://console.aws.amazon.com/ec2/), select the region you gave above when you did `aws configure` (e.g. eu-central-1), click on **Key Pairs** in the left sidebar, and check that `` is listed. diff --git a/docs/source/clusters-feds/aws-testing-cluster.md b/docs/source/clusters-feds/aws-testing-cluster.md index 59277ac4..650ba1d2 100644 --- a/docs/source/clusters-feds/aws-testing-cluster.md +++ b/docs/source/clusters-feds/aws-testing-cluster.md @@ -41,16 +41,6 @@ See the page about [basic AWS Setup](../appendices/aws-setup.html) in the Append The AWS cluster deployment scripts use elastic IP addresses (although that may change in the future). By default, AWS accounts get five elastic IP addresses. If you want to deploy a cluster with more than five nodes, then you will need more than five elastic IP addresses; you may have to apply for those; see [the AWS documentation on elastic IP addresses](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html). - -## Create an Amazon EC2 Key Pair - -Go to the AWS EC2 Console and select "Key Pairs" in the left sidebar. Click the "Create Key Pair" button. Give it the name `bigchaindb`. You should be prompted to save a file named `bigchaindb.pem`. That file contains the RSA private key. (You can get the public key from the private key, so there's no need to send it separately.) - -Save the file in `bigchaindb/deploy-cluster-aws/pem/bigchaindb.pem`. - -**You should not share your private key.** - - ## Create an Amazon EC2 Security Group Go to the AWS EC2 Console and select "Security Groups" in the left sidebar. Click the "Create Security Group" button. Name it `bigchaindb`. The description probably doesn't matter; you can also put `bigchaindb` for that. @@ -132,6 +122,7 @@ Step 2 is to make an AWS deployment configuration file, if necessary. There's an NUM_NODES=3 BRANCH="master" WHAT_TO_DEPLOY="servers" +SSH_KEY_NAME="not-set-yet" USE_KEYPAIRS_FILE=False IMAGE_ID="ami-accff2b1" INSTANCE_TYPE="m3.2xlarge" @@ -140,7 +131,7 @@ EBS_VOLUME_SIZE=30 EBS_OPTIMIZED=False ``` -If you're happy with those settings, then you can skip to the next step. Otherwise, you could make a copy of `example_deploy_conf.py` (e.g. `cp example_deploy_conf.py my_deploy_conf.py`) and then edit the copy using a text editor. +Make a copy of that file and call it whatever you like (e.g. `cp example_deploy_conf.py my_deploy_conf.py`). You can leave most of the settings at their default values, but you must change the value of `SSH_KEY_NAME` to the name of your private SSH key. You can do that with a text editor. Set `SSH_KEY_NAME` to the name you used for `` when you generated an RSA key pair for SSH (in basic AWS setup). If you want your nodes to have a predictable set of pre-generated keypairs, then you should 1) set `USE_KEYPAIRS_FILE=True` in the AWS deployment configuration file, and 2) provide a `keypairs.py` file containing enough keypairs for all of your nodes. You can generate a `keypairs.py` file using the `write_keypairs_file.py` script. For example: ```text