From 531a60e8aa8a0ba3d02fef5e4c9050b77a3541c4 Mon Sep 17 00:00:00 2001 From: troymc Date: Thu, 31 Mar 2016 14:38:11 +0200 Subject: [PATCH] Refactor how bigchaindb.conf gets written --- deploy-cluster-aws/create_bigchaindb_conf.py | 43 ++++++++++++ deploy-cluster-aws/launch_ec2_nodes.py | 70 ++++++++------------ deploy-cluster-aws/startup.sh | 19 +++--- 3 files changed, 79 insertions(+), 53 deletions(-) create mode 100644 deploy-cluster-aws/create_bigchaindb_conf.py diff --git a/deploy-cluster-aws/create_bigchaindb_conf.py b/deploy-cluster-aws/create_bigchaindb_conf.py new file mode 100644 index 00000000..4ac5a73e --- /dev/null +++ b/deploy-cluster-aws/create_bigchaindb_conf.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +"""(Re)create the RethinkDB configuration file conf/bigchaindb.conf. +Start with conf/bigchaindb.conf.template +then append additional configuration settings (lines). +""" + +from __future__ import unicode_literals +import os +import os.path +import shutil +from hostlist import hosts_dev + +# cwd = current working directory +old_cwd = os.getcwd() +os.chdir('conf') +if os.path.isfile('bigchaindb.conf'): + os.remove('bigchaindb.conf') + +# Create the initial bigchaindb.conf using bigchaindb.conf.template +shutil.copy2('bigchaindb.conf.template', 'bigchaindb.conf') + +# Append additional lines to bigchaindb.conf +with open('bigchaindb.conf', 'a') as f: + f.write('## The host:port of a node that RethinkDB will connect to\n') + for public_dns_name in hosts_dev: + f.write('join=' + public_dns_name + ':29015\n') + +os.chdir(old_cwd) + +# Note: The original code by Andreas wrote a file with lines of the form +# join=public_dns_name_0:29015 +# join=public_dns_name_1:29015 +# but it stopped about halfway through the list of public_dns_names +# (publist). In principle, it's only strictly necessary to +# have one join= line. +# Maybe Andreas thought that more is better, but all is too much? +# Below is Andreas' original code. -Troy +# lfile = open('add2dbconf', 'w') +# before = 'join=' +# after = ':29015' +# lfile.write('## The host:port of a node that rethinkdb will connect to\n') +# for entry in range(0,int(len(publist)/2)): +# lfile.write(before + publist[entry] + after + '\n') diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py index 2c4fe34b..578093ff 100644 --- a/deploy-cluster-aws/launch_ec2_nodes.py +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -4,11 +4,10 @@ 2. tags them with the specified tag, 3. waits until those instances exist and are running, 4. for each instance, allocates an elastic IP address - and associates it with that instance, and -5. creates three files: - * add2known_hosts.sh - * add2dbconf - * hostlist.py + and associates it with that instance, +5. writes the shellscript add2known_hosts.sh +6. (over)writes a file named hostlist.py + containing a list of all public DNS names. """ from __future__ import unicode_literals @@ -147,45 +146,30 @@ for i, instance in enumerate(instances_with_tag): print('was associated with the instance with id {}'. format(instance.instance_id)) +# Get a list of the pubic DNS names of the instances_with_tag +hosts_dev = [] +for instance in instances_with_tag: + public_dns_name = getattr(instance, 'public_dns_name', None) + if public_dns_name is not None: + hosts_dev.append(public_dns_name) + +# Write a shellscript to add remote keys to ~/.ssh/known_hosts +print('Preparing shellscript to add remote keys to known_hosts') +with open('add2known_hosts.sh', 'w') as f: + f.write('#!/bin/bash\n') + for public_dns_name in hosts_dev: + f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n') + +# Create a file named hostlist.py containing hosts_dev. +# If a hostlist.py already exists, it will be overwritten. +print('Writing hostlist.py') +with open('hostlist.py', 'w') as f: + f.write('# -*- coding: utf-8 -*-\n') + f.write('from __future__ import unicode_literals\n') + f.write('hosts_dev = {}\n'.format(hosts_dev)) + +# Wait wait_time = 45 print('Waiting {} seconds to make sure all instances are ready...'. format(wait_time)) time.sleep(wait_time) - -# Get a list of the pubic DNS names of the instances_with_tag -publist = [] -for instance in instances_with_tag: - public_dns_name = getattr(instance, 'public_dns_name', None) - if public_dns_name is not None: - publist.append(public_dns_name) - -# Create shellscript add2known_hosts.sh for adding remote keys to known_hosts -with open('add2known_hosts.sh', 'w') as f: - f.write('#! /bin/bash\n') - for public_dns_name in publist: - f.write('ssh-keyscan ' + public_dns_name + ' >> ~/.ssh/known_hosts\n') - -# Create a file named add2dbconf, overwriting one if it already exists -with open('add2dbconf', 'w') as f: - f.write('## The host:port of a node that RethinkDB will connect to\n') - for public_dns_name in publist: - f.write('join=' + public_dns_name + ':29015\n') - -# Note: The original code by Andreas wrote a file with lines of the form -# join=public_dns_name_0:29015 -# join=public_dns_name_1:29015 -# but it stopped about halfway through the list of public_dns_names -# (publist). In principle, it's only strictly necessary to -# have one join= line. -# Maybe Andreas thought that more is better, but all is too much? -# Below is Andreas' original code. -Troy -# localFile = open('add2dbconf', 'w') -# before = 'join=' -# after = ':29015' -# localFile.write('## The host:port of a node that rethinkdb will connect to\n') -# for entry in range(0,int(len(publist)/2)): -# localFile.write(before + publist[entry] + after + '\n') - -# Create a file named hostlist.py, overwriting one if it already exists -with open('hostlist.py', 'w') as f: - f.write('hosts_dev = {}'.format(publist)) diff --git a/deploy-cluster-aws/startup.sh b/deploy-cluster-aws/startup.sh index 3d97a4c1..3374052e 100755 --- a/deploy-cluster-aws/startup.sh +++ b/deploy-cluster-aws/startup.sh @@ -54,20 +54,19 @@ chmod 0400 pem/bigchaindb.pem # 2. tags them with the specified tag, # 3. waits until those instances exist and are running, # 4. for each instance, allocates an elastic IP address -# and associates it with that instance, and -# 5. creates three files: -# * add2known_hosts.sh -# * add2dbconf -# * hostlist.py +# and associates it with that instance, +# 5. writes the shellscript add2known_hosts.sh +# 6. (over)writes a file named hostlist.py +# containing a list of all public DNS names. python launch_ec2_nodes.py --tag $TAG --nodes $NODES -# Make add2known_hosts.sh executable and execute it +# Make add2known_hosts.sh executable then execute it. +# This adds remote keys to ~/.ssh/known_hosts chmod +x add2known_hosts.sh ./add2known_hosts.sh -# Reset the RethinkDB configuration file and add the nodes to join -cp conf/bigchaindb.conf.template conf/bigchaindb.conf -cat add2dbconf >> conf/bigchaindb.conf +# (Re)create the RethinkDB configuration file conf/bigchaindb.conf +python create_bigchaindb_conf.py # rollout base packages (dependencies) needed before # storage backend (rethinkdb) and bigchaindb can be rolled out @@ -90,6 +89,6 @@ fab -H $HORST -f fab_prepare_chain.py init_bigchaindb fab start_bigchaindb_nodes # cleanup -rm add2known_hosts.sh add2dbconf +rm add2known_hosts.sh # DONE