diff --git a/deploy-cluster-aws/awsdeploy.sh b/deploy-cluster-aws/awsdeploy.sh index 5a9fc8ed..5c9de41e 100755 --- a/deploy-cluster-aws/awsdeploy.sh +++ b/deploy-cluster-aws/awsdeploy.sh @@ -32,6 +32,9 @@ echo "WHAT_TO_DEPLOY = "$WHAT_TO_DEPLOY echo "USE_KEYPAIRS_FILE = "$USE_KEYPAIRS_FILE echo "IMAGE_ID = "$IMAGE_ID echo "INSTANCE_TYPE = "$INSTANCE_TYPE +echo "USING_EBS = "$USING_EBS +echo "EBS_VOLUME_SIZE = "$EBS_VOLUME_SIZE +echo "EBS_OPTIMIZED = "$EBS_OPTIMIZED # Check for AWS private key file (.pem file) if [ ! -f "pem/bigchaindb.pem" ]; then @@ -146,6 +149,7 @@ if [ "$WHAT_TO_DEPLOY" == "servers" ]; then # definition of init_bigchaindb() in fabfile.py to see why. fab init_bigchaindb fab set_shards:$NUM_NODES + echo "To set the replication factor to 3, do: fab set_replicas:3" echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" else # Deploying clients diff --git a/deploy-cluster-aws/example_deploy_conf.py b/deploy-cluster-aws/example_deploy_conf.py index 3e1ee2c9..f1da94e9 100644 --- a/deploy-cluster-aws/example_deploy_conf.py +++ b/deploy-cluster-aws/example_deploy_conf.py @@ -49,3 +49,19 @@ IMAGE_ID="ami-accff2b1" # Examples: "m3.2xlarge", "c3.8xlarge", "c4.8xlarge" # For all options, see https://aws.amazon.com/ec2/instance-types/ INSTANCE_TYPE="m3.2xlarge" + +# USING_EBS is True if you want to attach an Amazon EBS volume +USING_EBS=False + +# EBS_VOLUME_SIZE is the size of the EBS volume to attach, in GiB +# Since we assume 'gp2' volumes (for now), the possible range is 1 to 16384 +# If USING_EBS=False, EBS_VOLUME_SIZE is irrelevant and not used +EBS_VOLUME_SIZE=30 + +# EBS_OPTIMIZED is True or False, depending on whether you want +# EBS-optimized instances. See: +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSOptimized.html +# Not all instance types support EBS optimization. +# Setting EBS_OPTIMIZED=True may cost more, but not always. +# If USING_EBS=False, EBS_OPTIMIZED is irrelevant and not used +EBS_OPTIMIZED=False diff --git a/deploy-cluster-aws/launch_ec2_nodes.py b/deploy-cluster-aws/launch_ec2_nodes.py index c5c6c5f1..8e0c2d9b 100644 --- a/deploy-cluster-aws/launch_ec2_nodes.py +++ b/deploy-cluster-aws/launch_ec2_nodes.py @@ -24,7 +24,8 @@ from awscommon import get_naeips SETTINGS = ['NUM_NODES', 'BRANCH', 'WHAT_TO_DEPLOY', 'USE_KEYPAIRS_FILE', - 'IMAGE_ID', 'INSTANCE_TYPE'] + 'IMAGE_ID', 'INSTANCE_TYPE', 'USING_EBS', 'EBS_VOLUME_SIZE', + 'EBS_OPTIMIZED'] class SettingsTypeError(TypeError): @@ -76,7 +77,7 @@ if not isinstance(WHAT_TO_DEPLOY, str): raise SettingsTypeError('WHAT_TO_DEPLOY should be a string') if not isinstance(USE_KEYPAIRS_FILE, bool): - msg = 'USE_KEYPAIRS_FILE should a boolean (True or False)' + msg = 'USE_KEYPAIRS_FILE should be a boolean (True or False)' raise SettingsTypeError(msg) if not isinstance(IMAGE_ID, str): @@ -85,6 +86,15 @@ if not isinstance(IMAGE_ID, str): if not isinstance(INSTANCE_TYPE, str): raise SettingsTypeError('INSTANCE_TYPE should be a string') +if not isinstance(USING_EBS, bool): + raise SettingsTypeError('USING_EBS should be a boolean (True or False)') + +if not isinstance(EBS_VOLUME_SIZE, int): + raise SettingsTypeError('EBS_VOLUME_SIZE should be an int') + +if not isinstance(EBS_OPTIMIZED, bool): + raise SettingsTypeError('EBS_OPTIMIZED should be a boolean (True or False)') + if NUM_NODES > 64: raise ValueError('NUM_NODES should be less than or equal to 64. ' 'The AWS deployment configuration file sets it to {}'. @@ -95,6 +105,12 @@ if WHAT_TO_DEPLOY not in ['servers', 'clients']: 'The AWS deployment configuration file sets it to {}'. format(WHAT_TO_DEPLOY)) +# Since we assume 'gp2' volumes (for now), the possible range is 1 to 16384 +if EBS_VOLUME_SIZE > 16384: + raise ValueError('EBS_VOLUME_SIZE should be <= 16384. ' + 'The AWS deployment configuration file sets it to {}'. + format(EBS_VOLUME_SIZE)) + # Get an AWS EC2 "resource" # See http://boto3.readthedocs.org/en/latest/guide/resources.html ec2 = boto3.resource(service_name='ec2') @@ -158,14 +174,40 @@ print('Commencing launch of {} instances on Amazon EC2...'. for _ in range(NUM_NODES): # Request the launch of one instance at a time # (so list_of_instances should contain only one item) - list_of_instances = ec2.create_instances( - ImageId=IMAGE_ID, - MinCount=1, - MaxCount=1, - KeyName='bigchaindb', - InstanceType=INSTANCE_TYPE, - SecurityGroupIds=['bigchaindb'] - ) + # See https://tinyurl.com/hbjewbb + if USING_EBS: + dm = { + 'DeviceName': '/dev/sdp', + # Why /dev/sdp? See https://tinyurl.com/z2zqm6n + 'Ebs': { + 'VolumeSize': EBS_VOLUME_SIZE, # GiB + 'DeleteOnTermination': False, + 'VolumeType': 'gp2', + 'Encrypted': False + }, + # 'NoDevice': 'device' + # Suppresses the specified device included + # in the block device mapping of the AMI. + } + list_of_instances = ec2.create_instances( + ImageId=IMAGE_ID, + MinCount=1, + MaxCount=1, + KeyName='bigchaindb', + InstanceType=INSTANCE_TYPE, + SecurityGroupIds=['bigchaindb'], + BlockDeviceMappings=[dm], + EbsOptimized=EBS_OPTIMIZED + ) + else: # not USING_EBS + list_of_instances = ec2.create_instances( + ImageId=IMAGE_ID, + MinCount=1, + MaxCount=1, + KeyName='bigchaindb', + InstanceType=INSTANCE_TYPE, + SecurityGroupIds=['bigchaindb'] + ) # Tag the just-launched instances (should be just one) for instance in list_of_instances: diff --git a/docs/source/clusters-feds/deploy-on-aws.md b/docs/source/clusters-feds/deploy-on-aws.md index da60c337..c870591d 100644 --- a/docs/source/clusters-feds/deploy-on-aws.md +++ b/docs/source/clusters-feds/deploy-on-aws.md @@ -103,6 +103,9 @@ WHAT_TO_DEPLOY="servers" USE_KEYPAIRS_FILE=False IMAGE_ID="ami-accff2b1" INSTANCE_TYPE="m3.2xlarge" +USING_EBS=False +EBS_VOLUME_SIZE=30 +EBS_OPTIMIZED=False ``` If you're happy with those settings, then you can skip to the next step. Otherwise, you could make a copy of `example_deploy_conf.py` (e.g. `cp example_deploy_conf.py my_deploy_conf.py`) and then edit the copy using a text editor. @@ -126,6 +129,8 @@ Step 3 is to launch the nodes ("instances") on AWS, to install all the necessary cd bigchaindb cd deploy-cluster-aws ./awsdeploy.sh my_deploy_conf.py +# Only if you want to set the replication factor to 3 +fab set_replicas:3 # Only if you want to start BigchainDB on all the nodes: fab start_bigchaindb ```