From 03c2d0f1f2b5ed585bc0d32ef686fefeddc8f84a Mon Sep 17 00:00:00 2001 From: troymc Date: Wed, 30 Mar 2016 10:16:02 +0200 Subject: [PATCH] AWS deploy: merge a bunch of py scripts into one --- deploy-cluster-aws/create_hostlist.py | 67 --------- deploy-cluster-aws/get_elastic_ips.py | 65 --------- deploy-cluster-aws/launch_ec2_nodes.py | 145 +++++++++++++++++++ deploy-cluster-aws/run_and_tag.py | 57 -------- deploy-cluster-aws/startup.sh | 24 ++- deploy-cluster-aws/wait_until_all_running.py | 46 ------ 6 files changed, 156 insertions(+), 248 deletions(-) delete mode 100644 deploy-cluster-aws/create_hostlist.py delete mode 100644 deploy-cluster-aws/get_elastic_ips.py create mode 100644 deploy-cluster-aws/launch_ec2_nodes.py delete mode 100644 deploy-cluster-aws/run_and_tag.py delete mode 100644 deploy-cluster-aws/wait_until_all_running.py diff --git a/deploy-cluster-aws/create_hostlist.py b/deploy-cluster-aws/create_hostlist.py deleted file mode 100644 index 4610c750..00000000 --- a/deploy-cluster-aws/create_hostlist.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- - -# from __future__ import unicode_literals -import argparse -import boto3 -import os - -AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] -AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -AWS_REGION = os.environ['AWS_REGION'] - -parser = argparse.ArgumentParser() -parser.add_argument("--tag", - help="tag instances in aws", - required=True) -args = parser.parse_args() - -tag = args.tag - -# Connect to Amazon EC2 -ec2 = boto3.resource(service_name='ec2', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY) - -# Get a list of all instances with the specified tag. -# (Technically, instances_with_tag is an ec2.instancesCollection.) -instances_with_tag = ec2.instances.filter( - Filters=[{'Name': 'tag:Name', 'Values': [tag]}] - ) - -publist = [] -for instance in instances_with_tag: - public_dns_name = getattr(instance, 'public_dns_name', None) - if public_dns_name is not None: - publist.append(public_dns_name) - -# Create shellscript add2known_hosts.sh for adding remote keys to known_hosts -with open('add2known_hosts.sh', 'w') as f: - f.write('#! /bin/bash\n') - for public_dns_name in publist: - f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n') - -# Create a file named add2dbconf, overwriting one if it already exists -with open('add2dbconf', 'w') as f: - f.write('## The host:port of a node that RethinkDB will connect to\n') - for public_dns_name in publist: - f.write('join=' + public_dns_name + ':29015\n') - -# Note: The original code by Andreas wrote a file with lines of the form -# join=public_dns_name_0:29015 -# join=public_dns_name_1:29015 -# but it stopped about halfway through the list of public_dns_names -# (publist). In principle, it's only strictly necessary to -# have one join= line. -# Maybe Andreas thought that more is better, but all is too much? -# Below is Andreas' original code. -Troy -# localFile = open('add2dbconf', 'w') -# before = 'join=' -# after = ':29015' -# localFile.write('## The host:port of a node that rethinkdb will connect to\n') -# for entry in range(0,int(len(publist)/2)): -# localFile.write(before + publist[entry] + after + '\n') - -# Create a file named hostlist.py, overwriting one if it already exists -with open('hostlist.py', 'w') as f: - f.write('hosts_dev = {}'.format(publist)) diff --git a/deploy-cluster-aws/get_elastic_ips.py b/deploy-cluster-aws/get_elastic_ips.py deleted file mode 100644 index 9976eb27..00000000 --- a/deploy-cluster-aws/get_elastic_ips.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- - -""" Create Elastic IPs and assign them to instances if needed. -""" - -from __future__ import unicode_literals -import os -import boto3 -import argparse -import time - -AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] -AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -AWS_REGION = os.environ['AWS_REGION'] - -parser = argparse.ArgumentParser() -parser.add_argument("--tag", help="tag instances in aws") -args = parser.parse_args() - -if args.tag: - tag = args.tag -else: - # reading credentials from config for remote connection - print('usage: python get_elastic_ips.py --tag ') - print('reason: tag missing!!!') - exit(1) - -# Connect to Amazon EC2 -ec2 = boto3.resource(service_name='ec2', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY) - - -# Get a list of all instances with the specified tag. -# (Technically, instances_with_tag is an ec2.instancesCollection.) -instances_with_tag = ec2.instances.filter( - Filters=[{'Name': 'tag:Name', 'Values': [tag]}] - ) - -print('Allocating elastic IP addresses and assigning them to the instances...') - -for instance in instances_with_tag: - # Create a client from the ec2 resource - # See http://boto3.readthedocs.org/en/latest/guide/clients.html - client = ec2.meta.client - - # Acquire an Elastic IP address - # response is a dict. See http://tinyurl.com/z2n7u9k - response = client.allocate_address(DryRun=False, Domain='standard') - public_ip = response['PublicIp'] - print('public_ip = {}'.format(public_ip)) - - # Associate that Elastic IP address with an instance - response2 = client.associate_address( - DryRun=False, - InstanceId=instance.instance_id, - PublicIp=public_ip - ) - print('was associated with the instance with id {}'. - format(instance.instance_id)) - -# Make sure all IP addresses are assigned... -print('Waiting 30 seconds to make sure all IP addresses are assigned...') -time.sleep(30) diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py new file mode 100644 index 00000000..5412a037 --- /dev/null +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +"""This script: +1. Launches the specified number of nodes (instances) on Amazon EC2, +2. tags them with the specified tag, +3. waits until those instances exist and are running, +4. for each instance, allocates an elastic IP address + and associates it with that instance, and +5. creates three files: + * add2known_hosts.sh + * add2dbconf + * hostlist.py +""" + +from __future__ import unicode_literals +import os +import time +import argparse +import boto3 + +AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] +AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] +AWS_REGION = os.environ['AWS_REGION'] + +parser = argparse.ArgumentParser() +parser.add_argument("--tag", + help="tag to add to all launched instances on AWS", + required=True) +parser.add_argument("--nodes", + help="number of nodes in the cluster", + required=True, + type=int) +args = parser.parse_args() + +tag = args.tag +num_nodes = int(args.nodes) + +# Connect to Amazon EC2 +ec2 = boto3.resource(service_name='ec2', + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY) + +print('Commencing launch of {} instances on Amazon EC2...'. + format(num_nodes)) + +for _ in range(num_nodes): + # Request the launch of one instance at a time + # (so list_of_instances should contain only one item) + list_of_instances = ec2.create_instances( + ImageId='ami-accff2b1', # ubuntu-image + # 'ami-596b7235', # ubuntu w/ iops storage + MinCount=1, + MaxCount=1, + KeyName='bigchaindb', + InstanceType='m3.2xlarge', + # 'c3.8xlarge', + # 'c4.8xlarge', + SecurityGroupIds=['bigchaindb'] + ) + + # Tag the just-launched instances (should be just one) + for instance in list_of_instances: + time.sleep(5) + instance.create_tags(Tags=[{'Key': 'Name', 'Value': tag}]) + +# Get a list of all instances with the specified tag. +# (Technically, instances_with_tag is an ec2.instancesCollection.) +filters = [{'Name': 'tag:Name', 'Values': [tag]}] +instances_with_tag = ec2.instances.filter(Filters=filters) +print('The launched instances will have these ids:'.format(tag)) +for instance in instances_with_tag: + print(instance.id) + +print('Waiting until all those instances exist...') +for instance in instances_with_tag: + instance.wait_until_exists() + +print('Waiting until all those instances are running...') +for instance in instances_with_tag: + instance.wait_until_running() + +print('Allocating elastic IP addresses and assigning them to the instances...') + +for instance in instances_with_tag: + # Create a client from the ec2 resource + # See http://boto3.readthedocs.org/en/latest/guide/clients.html + client = ec2.meta.client + + # Acquire an Elastic IP address + # response is a dict. See http://tinyurl.com/z2n7u9k + response = client.allocate_address(DryRun=False, Domain='standard') + public_ip = response['PublicIp'] + print('The public IP address {}'.format(public_ip)) + + # Associate that Elastic IP address with an instance + response2 = client.associate_address( + DryRun=False, + InstanceId=instance.instance_id, + PublicIp=public_ip + ) + print('was associated with the instance with id {}'. + format(instance.instance_id)) + +wait_time = 45 +print('Waiting {} seconds to make sure all instances are ready...'. + format(wait_time)) +time.sleep(wait_time) + +# Get a list of the pubic DNS names of the instances_with_tag +publist = [] +for instance in instances_with_tag: + public_dns_name = getattr(instance, 'public_dns_name', None) + if public_dns_name is not None: + publist.append(public_dns_name) + +# Create shellscript add2known_hosts.sh for adding remote keys to known_hosts +with open('add2known_hosts.sh', 'w') as f: + f.write('#! /bin/bash\n') + for public_dns_name in publist: + f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n') + +# Create a file named add2dbconf, overwriting one if it already exists +with open('add2dbconf', 'w') as f: + f.write('## The host:port of a node that RethinkDB will connect to\n') + for public_dns_name in publist: + f.write('join=' + public_dns_name + ':29015\n') + +# Note: The original code by Andreas wrote a file with lines of the form +# join=public_dns_name_0:29015 +# join=public_dns_name_1:29015 +# but it stopped about halfway through the list of public_dns_names +# (publist). In principle, it's only strictly necessary to +# have one join= line. +# Maybe Andreas thought that more is better, but all is too much? +# Below is Andreas' original code. -Troy +# localFile = open('add2dbconf', 'w') +# before = 'join=' +# after = ':29015' +# localFile.write('## The host:port of a node that rethinkdb will connect to\n') +# for entry in range(0,int(len(publist)/2)): +# localFile.write(before + publist[entry] + after + '\n') + +# Create a file named hostlist.py, overwriting one if it already exists +with open('hostlist.py', 'w') as f: + f.write('hosts_dev = {}'.format(publist)) diff --git a/deploy-cluster-aws/run_and_tag.py b/deploy-cluster-aws/run_and_tag.py deleted file mode 100644 index 10cde20f..00000000 --- a/deploy-cluster-aws/run_and_tag.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -"""Launch the specified number of instances on Amazon EC2 -and tag them with the specified tag. -""" - -from __future__ import unicode_literals -import boto3 -import time -import argparse -import os - -AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] -AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -AWS_REGION = os.environ['AWS_REGION'] - -parser = argparse.ArgumentParser() -parser.add_argument("--tag", - help="tag to add to all launched instances on AWS", - required=True) -parser.add_argument("--nodes", - help="number of nodes in the cluster", - required=True, - type=int) -args = parser.parse_args() - -tag = args.tag -num_nodes = int(args.nodes) - -# Connect to Amazon EC2 -ec2 = boto3.resource(service_name='ec2', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY) - - -print('Starting {} instances on Amazon EC2 and tagging them with {}...'. - format(num_nodes, tag)) - -for _ in range(num_nodes): - # Request the launch of one instance at a time - # (so list_of_instances should contain only one item) - list_of_instances = ec2.create_instances( - ImageId='ami-accff2b1', # ubuntu-image - # 'ami-596b7235', # ubuntu w/ iops storage - MinCount=1, - MaxCount=1, - KeyName='bigchaindb', - InstanceType='m3.2xlarge', - # 'c3.8xlarge', - # 'c4.8xlarge', - SecurityGroupIds=['bigchaindb'] - ) - - # Tag the just-launched instances (should be just one) - for instance in list_of_instances: - time.sleep(5) - instance.create_tags(Tags=[{'Key': 'Name', 'Value': tag}]) diff --git a/deploy-cluster-aws/startup.sh b/deploy-cluster-aws/startup.sh index d526769e..3d97a4c1 100755 --- a/deploy-cluster-aws/startup.sh +++ b/deploy-cluster-aws/startup.sh @@ -49,19 +49,17 @@ fi # so that the owner can read it, but that's all chmod 0400 pem/bigchaindb.pem -# Start the specified number of nodes on Amazon EC2 -# and tag them with the specified tag -python run_and_tag.py --tag $TAG --nodes $NODES - -# Wait until all those instances are running -python wait_until_all_running.py --tag $TAG - -# Allocate elastic IP addresses and assign them to the instances -python get_elastic_ips.py --tag $TAG - -# Create three files: -# add2known_hosts.sh, add2dbconf and hostlist.py -python create_hostlist.py --tag $TAG +# The following Python script does these things: +# 1. Launches the specified number of nodes (instances) on Amazon EC2, +# 2. tags them with the specified tag, +# 3. waits until those instances exist and are running, +# 4. for each instance, allocates an elastic IP address +# and associates it with that instance, and +# 5. creates three files: +# * add2known_hosts.sh +# * add2dbconf +# * hostlist.py +python launch_ec2_nodes.py --tag $TAG --nodes $NODES # Make add2known_hosts.sh executable and execute it chmod +x add2known_hosts.sh diff --git a/deploy-cluster-aws/wait_until_all_running.py b/deploy-cluster-aws/wait_until_all_running.py deleted file mode 100644 index 732ce617..00000000 --- a/deploy-cluster-aws/wait_until_all_running.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import unicode_literals -import os -import boto3 -import argparse - - -AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] -AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -AWS_REGION = os.environ['AWS_REGION'] - -parser = argparse.ArgumentParser() -parser.add_argument("--tag", help="tag instances in aws") -args = parser.parse_args() - -if args.tag: - tag = args.tag -else: - # reading credentials from config for remote connection - print('usage: python get_instance_status.py --tag ') - print('reason: tag missing!!!') - exit(1) - -# Connect to Amazon EC2 -ec2 = boto3.resource(service_name='ec2', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY) - -# Get a list of all instances with the specified tag. -# (Technically, instances_with_tag is an ec2.instancesCollection.) -instances_with_tag = ec2.instances.filter( - Filters=[{'Name': 'tag:Name', 'Values': [tag]}] - ) -print('The instances with tag {} have these ids:'.format(tag)) -for instance in instances_with_tag: - print(instance.id) - -print('Waiting until all those instances exist...') -for instance in instances_with_tag: - instance.wait_until_exists() - -print('Waiting until all those instances are running...') -for instance in instances_with_tag: - instance.wait_until_running()