Mirroristas/bigchaindb
2
0
Fork 0
mirror of https://github.com/bigchaindb/bigchaindb.git synced 2024-10-13 13:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #567 from bigchaindb/feat/556/bring-your-own-ssh-keypair-for-aws

Browse Source 
AWS users now generate their own SSH key pair
This commit is contained in:
Troy McConaghy 2016-08-18 13:53:51 +02:00 committed by GitHub
commit 025b5cc15d
8 changed files with 104 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -65,12 +65,11 @@ target/
# pyenv
.python-version
# Private key files from AWS
*.pem
# Some files created when deploying a cluster on AWS
deploy-cluster-aws/conf/rethinkdb.conf
deploy-cluster-aws/hostlist.py
deploy-cluster-aws/confiles/
deploy-cluster-aws/client_confile
deploy-cluster-aws/hostlist.py
deploy-cluster-aws/ssh_key.py
benchmarking-tests/hostlist.py
benchmarking-tests/ssh_key.py

3
benchmarking-tests/fabfile.py vendored
View File

@ -7,6 +7,7 @@ from fabric.operations import run, put
from fabric.context_managers import settings
from hostlist import public_dns_names
from ssh_key import ssh_key_path
# Ignore known_hosts
# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts
@ -18,7 +19,7 @@ env.hosts = public_dns_names
# SSH key files to try when connecting:
# http://docs.fabfile.org/en/1.10/usage/env.html#key-filename
env.key_filename = 'pem/bigchaindb.pem'
env.key_filename = ssh_key_path
@task

26
deploy-cluster-aws/awsdeploy.sh
View File

@ -26,9 +26,19 @@ fi
# to set environment variables related to AWS deployment
echo "Reading "$DEPLOY_CONF_FILE
source $DEPLOY_CONF_FILE
# Check if SSH_KEY_NAME got set
if [ "$SSH_KEY_NAME" == "not-set-yet" ] || \
[ "$SSH_KEY_NAME" == "" ] || \
[ -z ${SSH_KEY_NAME+x} ]; then
echo "SSH_KEY_NAME was not set in that file"
exit 1
fi
echo "NUM_NODES = "$NUM_NODES
echo "BRANCH = "$BRANCH
echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY
echo "SSH_KEY_NAME" = $SSH_KEY_NAME
echo "USE_KEYPAIRS_FILE = "$USE_KEYPAIRS_FILE
echo "IMAGE_ID = "$IMAGE_ID
echo "INSTANCE_TYPE = "$INSTANCE_TYPE
@ -38,9 +48,9 @@ if [ "$USING_EBS" = True ]; then
echo "EBS_OPTIMIZED = "$EBS_OPTIMIZED
fi
# Check for AWS private key file (.pem file)
if [ ! -f "pem/bigchaindb.pem" ]; then
echo "File pem/bigchaindb.pem (AWS private key) is missing"
# Check for the SSH private key file
if [ ! -f "$HOME/.ssh/$SSH_KEY_NAME" ]; then
echo "The SSH private key file "$HOME"/.ssh/"$SSH_KEY_NAME" is missing"
exit 1
fi
@ -70,9 +80,9 @@ fi
TAG="BDB-"$WHAT_TO_DEPLOY"-"`date +%m-%d@%H:%M`
echo "TAG = "$TAG
# Change the file permissions on pem/bigchaindb.pem
# Change the file permissions on the SSH private key file
# so that the owner can read it, but that's all
chmod 0400 pem/bigchaindb.pem
chmod 0400 $HOME/.ssh/$SSH_KEY_NAME
# The following Python script does these things:
# 0. allocates more elastic IP addresses if necessary,
@ -84,6 +94,8 @@ chmod 0400 pem/bigchaindb.pem
# 5. writes the shellscript add2known_hosts.sh
# 6. (over)writes a file named hostlist.py
# containing a list of all public DNS names.
# 7. (over)writes a file named ssh_key.py
# containing the location of the private SSH key file.
python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG
# Make add2known_hosts.sh executable then execute it.
@ -91,6 +103,10 @@ python launch_ec2_nodes.py --deploy-conf-file $DEPLOY_CONF_FILE --tag $TAG
chmod +x add2known_hosts.sh
./add2known_hosts.sh
# Test an SSH connection to one of the hosts
# and prompt the user for their SSH password if necessary
fab set_host:0 test_ssh
# Rollout base packages (dependencies) needed before
# storage backend (RethinkDB) and BigchainDB can be rolled out
fab install_base_software

5
deploy-cluster-aws/example_deploy_conf.py
View File

@ -27,6 +27,11 @@ BRANCH="master"
# What do you want to deploy?
WHAT_TO_DEPLOY="servers"
# SSH_KEY_NAME is the name of the SSH private key file
# in $HOME/.ssh/
# It is used for SSH communications with AWS instances.
SSH_KEY_NAME="not-set-yet"
# USE_KEYPAIRS_FILE is either True or False
# Should node keypairs be read from keypairs.py?
# (If False, then the keypairs will be whatever is in the the

8
deploy-cluster-aws/fabfile.py vendored
View File

@ -15,6 +15,7 @@ from fabric.operations import run, put
from fabric.context_managers import settings
from hostlist import public_dns_names
from ssh_key import ssh_key_path
# Ignore known_hosts
# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts
@ -26,7 +27,7 @@ env.hosts = public_dns_names
# SSH key files to try when connecting:
# http://docs.fabfile.org/en/1.10/usage/env.html#key-filename
env.key_filename = 'pem/bigchaindb.pem'
env.key_filename = ssh_key_path
######################################################################
@ -48,6 +49,11 @@ def set_host(host_index):
env.hosts = [public_dns_names[int(host_index)]]
@task
def test_ssh():
run('echo "If you see this, then SSH to a remote host worked."')
# Install base software
@task
@parallel

35
deploy-cluster-aws/launch_ec2_nodes.py
View File

@ -9,9 +9,12 @@
5. writes the shellscript add2known_hosts.sh
6. (over)writes a file named hostlist.py
containing a list of all public DNS names.
7. (over)writes a file named ssh_key.py
containing the location of the private SSH key file.
"""
from __future__ import unicode_literals
from os.path import expanduser
import sys
import time
import socket
@ -23,9 +26,9 @@ import boto3
from awscommon import get_naeips
SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'USE_KEYPAIRS_FILE',
'IMAGE_ID', 'INSTANCE_TYPE', 'USING_EBS', 'EBS_VOLUME_SIZE',
'EBS_OPTIMIZED']
SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'SSH_KEY_NAME',
'USE_KEYPAIRS_FILE', 'IMAGE_ID', 'INSTANCE_TYPE', 'USING_EBS',
'EBS_VOLUME_SIZE', 'EBS_OPTIMIZED']
class SettingsTypeError(TypeError):
@ -76,6 +79,9 @@ if not isinstance(BRANCH, str):
if not isinstance(WHAT_TO_DEPLOY, str):
raise SettingsTypeError('WHAT_TO_DEPLOY should be a string')
if not isinstance(SSH_KEY_NAME, str):
raise SettingsTypeError('SSH_KEY_NAME should be a string')
if not isinstance(USE_KEYPAIRS_FILE, bool):
msg = 'USE_KEYPAIRS_FILE should be a boolean (True or False)'
raise SettingsTypeError(msg)
@ -105,6 +111,11 @@ if WHAT_TO_DEPLOY not in ['servers', 'clients']:
'The AWS deployment configuration file sets it to {}'.
format(WHAT_TO_DEPLOY))
if SSH_KEY_NAME in ['not-set-yet', '', None]:
raise ValueError('SSH_KEY_NAME should be set. '
'The AWS deployment configuration file sets it to {}'.
format(SSH_KEY_NAME))
# Since we assume 'gp2' volumes (for now), the possible range is 1 to 16384
if EBS_VOLUME_SIZE > 16384:
raise ValueError('EBS_VOLUME_SIZE should be <= 16384. '
@ -193,7 +204,7 @@ for _ in range(NUM_NODES):
ImageId=IMAGE_ID,
MinCount=1,
MaxCount=1,
KeyName='bigchaindb',
KeyName=SSH_KEY_NAME,
InstanceType=INSTANCE_TYPE,
SecurityGroupIds=['bigchaindb'],
BlockDeviceMappings=[dm],
@ -204,7 +215,7 @@ for _ in range(NUM_NODES):
ImageId=IMAGE_ID,
MinCount=1,
MaxCount=1,
KeyName='bigchaindb',
KeyName=SSH_KEY_NAME,
InstanceType=INSTANCE_TYPE,
SecurityGroupIds=['bigchaindb']
)
@ -281,6 +292,20 @@ with open('hostlist.py', 'w') as f:
f.write('\n')
f.write('public_dns_names = {}\n'.format(public_dns_names))
# Create a file named ssh_key.py
# containing the location of the private SSH key file.
# If a ssh_key.py already exists, it will be overwritten.
print('Writing ssh_key.py')
with open('ssh_key.py', 'w') as f:
f.write('# -*- coding: utf-8 -*-\n')
f.write('"""This file exists as a convenient way for Fabric to get\n')
f.write('the location of the private SSH key file.')
f.write('"""\n')
f.write('\n')
f.write('from __future__ import unicode_literals\n')
f.write('\n')
home = expanduser('~')
f.write('ssh_key_path = "{}/.ssh/{}"\n'.format(home, SSH_KEY_NAME))
# For each node in the cluster, check port 22 (ssh) until it's reachable
for instance in instances_with_tag:

34
docs/source/appendices/aws-setup.md
View File

@ -36,3 +36,37 @@ Default output format [None]: [Press Enter]
```
This writes two files: `~/.aws/credentials` and `~/.aws/config`. AWS tools and packages look for those files.
## Generate an RSA Key Pair for SSH
Eventually, you'll have one or more instances (virtual machines) running on AWS and you'll want to SSH to them. To do that, you need a public/private key pair. The public key will be sent to AWS, and you can tell AWS to put it in any instances you provision there. You'll keep the private key on your local workstation.
First you need to make up a key name. Some ideas:
* `bcdb-troy-1`
* `bigchaindb-7`
* `bcdb-jupiter`
If you already have key pairs on AWS (Amazon EC2), you have to pick a name that's not already being used.
Below, replace every instance of `<key-name>` with your actual key name.
To generate a public/private RSA key pair with that name:
```text
ssh-keygen -t rsa -C "<key-name>" -f ~/.ssh/<key-name>
```
It will ask you for a passphrase. You can use whatever passphrase you like, but don't lose it. Two keys (files) will be created in `~/.ssh/`:
1. `~/.ssh/<key-name>.pub` is the public key
2. `~/.ssh/<key-name>` is the private key
To send the public key to AWS, use the AWS Command-Line Interface:
```text
aws ec2 import-key-pair \
--key-name "<key-name>" \
--public-key-material file://~/.ssh/<key-name>.pub
```
If you're curious why there's a `file://` in front of the path to the public key, see issue [aws/aws-cli#41 on GitHub](https://github.com/aws/aws-cli/issues/41).
If you want to verify that your key pair was imported by AWS, go to the Amazon EC2 console at [https://console.aws.amazon.com/ec2/](https://console.aws.amazon.com/ec2/), select the region you gave above when you did `aws configure` (e.g. eu-central-1), click on **Key Pairs** in the left sidebar, and check that `<key-name>` is listed.

13
docs/source/clusters-feds/aws-testing-cluster.md
View File

@ -41,16 +41,6 @@ See the page about [basic AWS Setup](../appendices/aws-setup.html) in the Append
The AWS cluster deployment scripts use elastic IP addresses (although that may change in the future). By default, AWS accounts get five elastic IP addresses. If you want to deploy a cluster with more than five nodes, then you will need more than five elastic IP addresses; you may have to apply for those; see [the AWS documentation on elastic IP addresses](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html).
## Create an Amazon EC2 Key Pair
Go to the AWS EC2 Console and select "Key Pairs" in the left sidebar. Click the "Create Key Pair" button. Give it the name `bigchaindb`. You should be prompted to save a file named `bigchaindb.pem`. That file contains the RSA private key. (You can get the public key from the private key, so there's no need to send it separately.)
Save the file in `bigchaindb/deploy-cluster-aws/pem/bigchaindb.pem`.
**You should not share your private key.**
## Create an Amazon EC2 Security Group
Go to the AWS EC2 Console and select "Security Groups" in the left sidebar. Click the "Create Security Group" button. Name it `bigchaindb`. The description probably doesn't matter; you can also put `bigchaindb` for that.
@ -132,6 +122,7 @@ Step 2 is to make an AWS deployment configuration file, if necessary. There's an
NUM_NODES=3
BRANCH="master"
WHAT_TO_DEPLOY="servers"
SSH_KEY_NAME="not-set-yet"
USE_KEYPAIRS_FILE=False
IMAGE_ID="ami-accff2b1"
INSTANCE_TYPE="m3.2xlarge"
@ -140,7 +131,7 @@ EBS_VOLUME_SIZE=30
EBS_OPTIMIZED=False
```
If you're happy with those settings, then you can skip to the next step. Otherwise, you could make a copy of `example_deploy_conf.py` (e.g. `cp example_deploy_conf.py my_deploy_conf.py`) and then edit the copy using a text editor.
Make a copy of that file and call it whatever you like (e.g. `cp example_deploy_conf.py my_deploy_conf.py`). You can leave most of the settings at their default values, but you must change the value of `SSH_KEY_NAME` to the name of your private SSH key. You can do that with a text editor. Set `SSH_KEY_NAME` to the name you used for `<key-name>` when you generated an RSA key pair for SSH (in basic AWS setup).
If you want your nodes to have a predictable set of pre-generated keypairs, then you should 1) set `USE_KEYPAIRS_FILE=True` in the AWS deployment configuration file, and 2) provide a `keypairs.py` file containing enough keypairs for all of your nodes. You can generate a `keypairs.py` file using the `write_keypairs_file.py` script. For example:
```text