Ansible role to set up DB storage. Also ensure latest pip3 & setuptools

ntools/one-m/ansible/one-m-node.yml

@@ -8,4 +8,4 @@
   roles:
     - bcdb_base
     - ntp
-    # TODO: upgrade pip and setuptools, see https://github.com/bobbyrenwick/ansible-pip 
+    - db_storage

ntools/one-m/ansible/roles/bcdb_base/tasks/main.yml

@@ -4,22 +4,44 @@
 # Note: "become: true" basically means "become root user for this task" i.e. sudo <cmd>
 # See https://docs.ansible.com/ansible/become.html
 
-- name: Do the equivalent of "sudo apt-get update"
+# Note: update_cache=yes means it will do the equivalent of
-  apt: update_cache=yes
+# sudo apt-get update before the operation.
-  become: true
+- name: Ensure the latest BigchainDB-required Ubuntu packages are installed
-
+  apt: name={{item}} state=latest update_cache=yes
-- name: Configure all unpacked but unconfigured packages
-  shell: /usr/bin/dpkg --configure -a
-  become: true
-
-- name: Attempt to correct a system with broken dependencies in place
-  shell: /usr/bin/apt-get -y -f install
-  become: true
-
-- name: Ensure the LATEST git g++ python3-dev are installed
-  apt: name={{item}} state=latest
   become: true
   with_items:
     - git
     - g++
     - python3-dev
+    - python3-setuptools  # mainly for easy_install3, which is used to get latest pip3
+
+# This should make both pip and pip3 be pip version >=8.1.2 (python 3.4).
+# See the comments about this below.
+- name: Ensure the latest pip/pip3 is installed, using easy_install3
+  easy_install: executable=easy_install3 name=pip state=latest
+  become: true
+
+- name: Ensure the latest setuptools (Python package) is installed
+  pip: executable=pip3 name=setuptools state=latest
+  become: true
+
+# Notes about getting the latest version of pip3:
+#
+# The first way I tried to get the latest pip3 (commented-out below) didn't work.
+# The first task works, but then the attempt to do
+# the equivalent of "pip install -U pip" fails. "Found existing installation"
+# and it didn't want to uninstall it
+
+# Installing the python3-pip package installs a Python 3 version of pip named pip3
+#- name: Ensure the latest python-pip and python3-pip Ubuntu packages are installed
+#  apt: name={{item}} state=latest update_cache=yes
+#  become: true
+#  with_items:
+#    - python-pip
+#    - python3-pip
+#
+#- name: Ensure pip is the latest version
+#  pip: executable=pip name=pip state=latest
+#
+#- name: Ensure pip3 is the latest version
+#  pip: executable=pip3 name=pip state=latest

ntools/one-m/ansible/roles/db_storage/tasks/edit_etc_fstab.py

@@ -0,0 +1,23 @@
+"""
+This Python 3 script reads the /etc/fstab file line by line
+and writes a new file /tmp/fstab line by line.
+If a line contains the string '/dev/xvdp', it replaces that
+line with:
+/dev/xvdp /data ext4 defaults,nofail,nobootwait 0 2
+It then:
+- moves /etc/fstab to /etc/old_fstab
+- moves /tmp/fstab to /etc/fstab
+"""
+
+import shutil
+
+with open('/tmp/fstab', 'a') as tmp_fstab:
+    with open('/etc/fstab') as f:
+        for line in f:
+            if '/dev/xvdp' in line:
+                tmp_fstab.write('/dev/xvdp /data ext4 defaults,nofail,nobootwait 0 2\n')
+            else:
+                tmp_fstab.write(line)
+
+shutil.move('/etc/fstab', '/etc/old_fstab')
+shutil.move('/tmp/fstab', '/etc/fstab')

ntools/one-m/ansible/roles/db_storage/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+# ansible/roles/db_storage/tasks/main.yml
+
+#- name: Ensure the /data directory (for DB storage) exists
+#  file: path=/data state=directory
+
+- name: Format the block storage device at /dev/xvdp with an ext4 file system
+  filesystem: fstype=ext4 dev=/dev/xvdp
+  become: true
+
+# Note that this also modifies /etc/fstab so the mount will persist through a crash
+- name: Ensure /data dir exists and is mounted on /dev/xvdp + update /etc/fstab
+  mount: name=/data src=/dev/xvdp fstype=ext4 state=mounted
+  become: true
+
+# After allowing the above to proceed,
+# I did "cat /etc/fstab" and got:
+# LABEL=cloudimg-rootfs	/	 ext4	defaults,discard	0 0
+# /dev/xvdp /data ext4 defaults 0 0
+
+# Let's change "defaults 0 0" to "defaults,nofail,nobootwait 0 2"
+
+- name: Ensure any old edit_etc_fstab.py file is deleted
+  file: name=/tmp/edit_etc_fstab.py state=absent
+
+- name: Copy local Python script edit_etc_fstab.py to the remote host
+  copy: src={{role_path}}/tasks/edit_etc_fstab.py dest=/tmp/edit_etc_fstab.py
+
+- name: Run edit_etc_fstab.py using Python 3
+  shell: /usr/bin/python3 /tmp/edit_etc_fstab.py
+  become: true
+
+- name: Ensure /tmp/edit_etc_fstab.py is deleted
+  file: name=/tmp/edit_etc_fstab.py state=absent
+
+# Modify the I/O scheduler? Is that even a good idea?
+# Must do this in /sys/block/xvdp/queue/scheduler
+# and also with grub (so the I/O scheduler stays changed on reboot)
+# Example: https://gist.github.com/keithchambers/80b60559ad83cebf1672
+

ntools/one-m/ansible/roles/ntp/tasks/main.yml

@@ -5,7 +5,7 @@
   apt: name=ntpdate state=absent
   become: true
 
-- name: Ensure the LATEST ntp is installed and do "sudo apt-get update" 
+- name: Ensure the latest ntp is installed
   apt: name=ntp state=latest update_cache=yes
   become: true
 
@@ -13,6 +13,8 @@
   stat: path=/etc/ntp.conf
   register: ntp_conf_file
 
+- debug: var=ntp_conf_file.stat.exists
+
 - name: Fail when /etc/ntp.conf doesn't exist
   fail: msg="The NTP config file /etc/ntp.conf doesn't exist'"
   when: ntp_conf_file.stat.exists == False

ntools/one-m/aws/variables.tf

@@ -3,7 +3,7 @@ variable "aws_region" {
 }
 
 variable "aws_instance_type" {
-  default = "m4.xlarge"
+  default = "m4.large"
 }
 
 variable "root_storage_in_GiB" {