mirror of
https://github.com/bigchaindb/bigchaindb.git
synced 2024-10-13 13:34:05 +00:00
AWS deploy: merge a bunch of py scripts into one
This commit is contained in:
troymc
parent
1852192c75
commit
03c2d0f1f2
@ -1,67 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# from __future__ import unicode_literals
|
||||
import argparse
|
||||
import boto3
|
||||
import os
|
||||
|
||||
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
|
||||
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
|
||||
AWS_REGION = os.environ['AWS_REGION']
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag",
|
||||
help="tag instances in aws",
|
||||
required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
tag = args.tag
|
||||
|
||||
# Connect to Amazon EC2
|
||||
ec2 = boto3.resource(service_name='ec2',
|
||||
region_name=AWS_REGION,
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
|
||||
|
||||
# Get a list of all instances with the specified tag.
|
||||
# (Technically, instances_with_tag is an ec2.instancesCollection.)
|
||||
instances_with_tag = ec2.instances.filter(
|
||||
Filters=[{'Name': 'tag:Name', 'Values': [tag]}]
|
||||
)
|
||||
|
||||
publist = []
|
||||
for instance in instances_with_tag:
|
||||
public_dns_name = getattr(instance, 'public_dns_name', None)
|
||||
if public_dns_name is not None:
|
||||
publist.append(public_dns_name)
|
||||
|
||||
# Create shellscript add2known_hosts.sh for adding remote keys to known_hosts
|
||||
with open('add2known_hosts.sh', 'w') as f:
|
||||
f.write('#! /bin/bash\n')
|
||||
for public_dns_name in publist:
|
||||
f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n')
|
||||
|
||||
# Create a file named add2dbconf, overwriting one if it already exists
|
||||
with open('add2dbconf', 'w') as f:
|
||||
f.write('## The host:port of a node that RethinkDB will connect to\n')
|
||||
for public_dns_name in publist:
|
||||
f.write('join=' + public_dns_name + ':29015\n')
|
||||
|
||||
# Note: The original code by Andreas wrote a file with lines of the form
|
||||
# join=public_dns_name_0:29015
|
||||
# join=public_dns_name_1:29015
|
||||
# but it stopped about halfway through the list of public_dns_names
|
||||
# (publist). In principle, it's only strictly necessary to
|
||||
# have one join= line.
|
||||
# Maybe Andreas thought that more is better, but all is too much?
|
||||
# Below is Andreas' original code. -Troy
|
||||
# localFile = open('add2dbconf', 'w')
|
||||
# before = 'join='
|
||||
# after = ':29015'
|
||||
# localFile.write('## The host:port of a node that rethinkdb will connect to\n')
|
||||
# for entry in range(0,int(len(publist)/2)):
|
||||
# localFile.write(before + publist[entry] + after + '\n')
|
||||
|
||||
# Create a file named hostlist.py, overwriting one if it already exists
|
||||
with open('hostlist.py', 'w') as f:
|
||||
f.write('hosts_dev = {}'.format(publist))
|
||||
@ -1,65 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
""" Create Elastic IPs and assign them to instances if needed.
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import os
|
||||
import boto3
|
||||
import argparse
|
||||
import time
|
||||
|
||||
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
|
||||
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
|
||||
AWS_REGION = os.environ['AWS_REGION']
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag", help="tag instances in aws")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.tag:
|
||||
tag = args.tag
|
||||
else:
|
||||
# reading credentials from config for remote connection
|
||||
print('usage: python get_elastic_ips.py --tag <tag>')
|
||||
print('reason: tag missing!!!')
|
||||
exit(1)
|
||||
|
||||
# Connect to Amazon EC2
|
||||
ec2 = boto3.resource(service_name='ec2',
|
||||
region_name=AWS_REGION,
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
|
||||
|
||||
|
||||
# Get a list of all instances with the specified tag.
|
||||
# (Technically, instances_with_tag is an ec2.instancesCollection.)
|
||||
instances_with_tag = ec2.instances.filter(
|
||||
Filters=[{'Name': 'tag:Name', 'Values': [tag]}]
|
||||
)
|
||||
|
||||
print('Allocating elastic IP addresses and assigning them to the instances...')
|
||||
|
||||
for instance in instances_with_tag:
|
||||
# Create a client from the ec2 resource
|
||||
# See http://boto3.readthedocs.org/en/latest/guide/clients.html
|
||||
client = ec2.meta.client
|
||||
|
||||
# Acquire an Elastic IP address
|
||||
# response is a dict. See http://tinyurl.com/z2n7u9k
|
||||
response = client.allocate_address(DryRun=False, Domain='standard')
|
||||
public_ip = response['PublicIp']
|
||||
print('public_ip = {}'.format(public_ip))
|
||||
|
||||
# Associate that Elastic IP address with an instance
|
||||
response2 = client.associate_address(
|
||||
DryRun=False,
|
||||
InstanceId=instance.instance_id,
|
||||
PublicIp=public_ip
|
||||
)
|
||||
print('was associated with the instance with id {}'.
|
||||
format(instance.instance_id))
|
||||
|
||||
# Make sure all IP addresses are assigned...
|
||||
print('Waiting 30 seconds to make sure all IP addresses are assigned...')
|
||||
time.sleep(30)
|
||||
145
deploy-cluster-aws/launch_ec2_nodes.py
Normal file
145
deploy-cluster-aws/launch_ec2_nodes.py
Normal file
@ -0,0 +1,145 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""This script:
|
||||
1. Launches the specified number of nodes (instances) on Amazon EC2,
|
||||
2. tags them with the specified tag,
|
||||
3. waits until those instances exist and are running,
|
||||
4. for each instance, allocates an elastic IP address
|
||||
and associates it with that instance, and
|
||||
5. creates three files:
|
||||
* add2known_hosts.sh
|
||||
* add2dbconf
|
||||
* hostlist.py
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import os
|
||||
import time
|
||||
import argparse
|
||||
import boto3
|
||||
|
||||
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
|
||||
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
|
||||
AWS_REGION = os.environ['AWS_REGION']
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag",
|
||||
help="tag to add to all launched instances on AWS",
|
||||
required=True)
|
||||
parser.add_argument("--nodes",
|
||||
help="number of nodes in the cluster",
|
||||
required=True,
|
||||
type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
tag = args.tag
|
||||
num_nodes = int(args.nodes)
|
||||
|
||||
# Connect to Amazon EC2
|
||||
ec2 = boto3.resource(service_name='ec2',
|
||||
region_name=AWS_REGION,
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
|
||||
|
||||
print('Commencing launch of {} instances on Amazon EC2...'.
|
||||
format(num_nodes))
|
||||
|
||||
for _ in range(num_nodes):
|
||||
# Request the launch of one instance at a time
|
||||
# (so list_of_instances should contain only one item)
|
||||
list_of_instances = ec2.create_instances(
|
||||
ImageId='ami-accff2b1', # ubuntu-image
|
||||
# 'ami-596b7235', # ubuntu w/ iops storage
|
||||
MinCount=1,
|
||||
MaxCount=1,
|
||||
KeyName='bigchaindb',
|
||||
InstanceType='m3.2xlarge',
|
||||
# 'c3.8xlarge',
|
||||
# 'c4.8xlarge',
|
||||
SecurityGroupIds=['bigchaindb']
|
||||
)
|
||||
|
||||
# Tag the just-launched instances (should be just one)
|
||||
for instance in list_of_instances:
|
||||
time.sleep(5)
|
||||
instance.create_tags(Tags=[{'Key': 'Name', 'Value': tag}])
|
||||
|
||||
# Get a list of all instances with the specified tag.
|
||||
# (Technically, instances_with_tag is an ec2.instancesCollection.)
|
||||
filters = [{'Name': 'tag:Name', 'Values': [tag]}]
|
||||
instances_with_tag = ec2.instances.filter(Filters=filters)
|
||||
print('The launched instances will have these ids:'.format(tag))
|
||||
for instance in instances_with_tag:
|
||||
print(instance.id)
|
||||
|
||||
print('Waiting until all those instances exist...')
|
||||
for instance in instances_with_tag:
|
||||
instance.wait_until_exists()
|
||||
|
||||
print('Waiting until all those instances are running...')
|
||||
for instance in instances_with_tag:
|
||||
instance.wait_until_running()
|
||||
|
||||
print('Allocating elastic IP addresses and assigning them to the instances...')
|
||||
|
||||
for instance in instances_with_tag:
|
||||
# Create a client from the ec2 resource
|
||||
# See http://boto3.readthedocs.org/en/latest/guide/clients.html
|
||||
client = ec2.meta.client
|
||||
|
||||
# Acquire an Elastic IP address
|
||||
# response is a dict. See http://tinyurl.com/z2n7u9k
|
||||
response = client.allocate_address(DryRun=False, Domain='standard')
|
||||
public_ip = response['PublicIp']
|
||||
print('The public IP address {}'.format(public_ip))
|
||||
|
||||
# Associate that Elastic IP address with an instance
|
||||
response2 = client.associate_address(
|
||||
DryRun=False,
|
||||
InstanceId=instance.instance_id,
|
||||
PublicIp=public_ip
|
||||
)
|
||||
print('was associated with the instance with id {}'.
|
||||
format(instance.instance_id))
|
||||
|
||||
wait_time = 45
|
||||
print('Waiting {} seconds to make sure all instances are ready...'.
|
||||
format(wait_time))
|
||||
time.sleep(wait_time)
|
||||
|
||||
# Get a list of the pubic DNS names of the instances_with_tag
|
||||
publist = []
|
||||
for instance in instances_with_tag:
|
||||
public_dns_name = getattr(instance, 'public_dns_name', None)
|
||||
if public_dns_name is not None:
|
||||
publist.append(public_dns_name)
|
||||
|
||||
# Create shellscript add2known_hosts.sh for adding remote keys to known_hosts
|
||||
with open('add2known_hosts.sh', 'w') as f:
|
||||
f.write('#! /bin/bash\n')
|
||||
for public_dns_name in publist:
|
||||
f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n')
|
||||
|
||||
# Create a file named add2dbconf, overwriting one if it already exists
|
||||
with open('add2dbconf', 'w') as f:
|
||||
f.write('## The host:port of a node that RethinkDB will connect to\n')
|
||||
for public_dns_name in publist:
|
||||
f.write('join=' + public_dns_name + ':29015\n')
|
||||
|
||||
# Note: The original code by Andreas wrote a file with lines of the form
|
||||
# join=public_dns_name_0:29015
|
||||
# join=public_dns_name_1:29015
|
||||
# but it stopped about halfway through the list of public_dns_names
|
||||
# (publist). In principle, it's only strictly necessary to
|
||||
# have one join= line.
|
||||
# Maybe Andreas thought that more is better, but all is too much?
|
||||
# Below is Andreas' original code. -Troy
|
||||
# localFile = open('add2dbconf', 'w')
|
||||
# before = 'join='
|
||||
# after = ':29015'
|
||||
# localFile.write('## The host:port of a node that rethinkdb will connect to\n')
|
||||
# for entry in range(0,int(len(publist)/2)):
|
||||
# localFile.write(before + publist[entry] + after + '\n')
|
||||
|
||||
# Create a file named hostlist.py, overwriting one if it already exists
|
||||
with open('hostlist.py', 'w') as f:
|
||||
f.write('hosts_dev = {}'.format(publist))
|
||||
@ -1,57 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Launch the specified number of instances on Amazon EC2
|
||||
and tag them with the specified tag.
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import boto3
|
||||
import time
|
||||
import argparse
|
||||
import os
|
||||
|
||||
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
|
||||
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
|
||||
AWS_REGION = os.environ['AWS_REGION']
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag",
|
||||
help="tag to add to all launched instances on AWS",
|
||||
required=True)
|
||||
parser.add_argument("--nodes",
|
||||
help="number of nodes in the cluster",
|
||||
required=True,
|
||||
type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
tag = args.tag
|
||||
num_nodes = int(args.nodes)
|
||||
|
||||
# Connect to Amazon EC2
|
||||
ec2 = boto3.resource(service_name='ec2',
|
||||
region_name=AWS_REGION,
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
|
||||
|
||||
|
||||
print('Starting {} instances on Amazon EC2 and tagging them with {}...'.
|
||||
format(num_nodes, tag))
|
||||
|
||||
for _ in range(num_nodes):
|
||||
# Request the launch of one instance at a time
|
||||
# (so list_of_instances should contain only one item)
|
||||
list_of_instances = ec2.create_instances(
|
||||
ImageId='ami-accff2b1', # ubuntu-image
|
||||
# 'ami-596b7235', # ubuntu w/ iops storage
|
||||
MinCount=1,
|
||||
MaxCount=1,
|
||||
KeyName='bigchaindb',
|
||||
InstanceType='m3.2xlarge',
|
||||
# 'c3.8xlarge',
|
||||
# 'c4.8xlarge',
|
||||
SecurityGroupIds=['bigchaindb']
|
||||
)
|
||||
|
||||
# Tag the just-launched instances (should be just one)
|
||||
for instance in list_of_instances:
|
||||
time.sleep(5)
|
||||
instance.create_tags(Tags=[{'Key': 'Name', 'Value': tag}])
|
||||
@ -49,19 +49,17 @@ fi
|
||||
# so that the owner can read it, but that's all
|
||||
chmod 0400 pem/bigchaindb.pem
|
||||
|
||||
# Start the specified number of nodes on Amazon EC2
|
||||
# and tag them with the specified tag
|
||||
python run_and_tag.py --tag $TAG --nodes $NODES
|
||||
|
||||
# Wait until all those instances are running
|
||||
python wait_until_all_running.py --tag $TAG
|
||||
|
||||
# Allocate elastic IP addresses and assign them to the instances
|
||||
python get_elastic_ips.py --tag $TAG
|
||||
|
||||
# Create three files:
|
||||
# add2known_hosts.sh, add2dbconf and hostlist.py
|
||||
python create_hostlist.py --tag $TAG
|
||||
# The following Python script does these things:
|
||||
# 1. Launches the specified number of nodes (instances) on Amazon EC2,
|
||||
# 2. tags them with the specified tag,
|
||||
# 3. waits until those instances exist and are running,
|
||||
# 4. for each instance, allocates an elastic IP address
|
||||
# and associates it with that instance, and
|
||||
# 5. creates three files:
|
||||
# * add2known_hosts.sh
|
||||
# * add2dbconf
|
||||
# * hostlist.py
|
||||
python launch_ec2_nodes.py --tag $TAG --nodes $NODES
|
||||
|
||||
# Make add2known_hosts.sh executable and execute it
|
||||
chmod +x add2known_hosts.sh
|
||||
|
||||
@ -1,46 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import os
|
||||
import boto3
|
||||
import argparse
|
||||
|
||||
|
||||
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
|
||||
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
|
||||
AWS_REGION = os.environ['AWS_REGION']
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag", help="tag instances in aws")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.tag:
|
||||
tag = args.tag
|
||||
else:
|
||||
# reading credentials from config for remote connection
|
||||
print('usage: python get_instance_status.py --tag <tag>')
|
||||
print('reason: tag missing!!!')
|
||||
exit(1)
|
||||
|
||||
# Connect to Amazon EC2
|
||||
ec2 = boto3.resource(service_name='ec2',
|
||||
region_name=AWS_REGION,
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
|
||||
|
||||
# Get a list of all instances with the specified tag.
|
||||
# (Technically, instances_with_tag is an ec2.instancesCollection.)
|
||||
instances_with_tag = ec2.instances.filter(
|
||||
Filters=[{'Name': 'tag:Name', 'Values': [tag]}]
|
||||
)
|
||||
print('The instances with tag {} have these ids:'.format(tag))
|
||||
for instance in instances_with_tag:
|
||||
print(instance.id)
|
||||
|
||||
print('Waiting until all those instances exist...')
|
||||
for instance in instances_with_tag:
|
||||
instance.wait_until_exists()
|
||||
|
||||
print('Waiting until all those instances are running...')
|
||||
for instance in instances_with_tag:
|
||||
instance.wait_until_running()
|
||||
Loading…
x
Reference in New Issue
Block a user