Problem: No automated way to set up a dev/test network (#2300)

* Problem: No automated way to set up a dev/test network * Problem: docs not updated for stack, ansible and docker based dev environments * Problem: Using apt triggers java runtime installation on MacOS - Update pre_tasks in ansible - Update bigchaindb-stop playbook to handle pre_tasks * Update Tendermint version to 0.19.3 from 0.19.2 * Update tendermint version to 0.19.3 everywhere else * Problem: Sphinx warns about duplicate section labels Solution: Don't use sphinx.ext.autosectionlabel * Problem: Sphinx complains that run-node-with-docker not in any TOC Solution: Add run-node-with-docker to a TOC (i.e. to an index.rst file) * Tendermint has not tagged `0.19.3` container * Problem: Internal hyperlinks to new pages not working Solution: Add .html to the ends of the filename strings * Problem: Invalid script imports * Problem: Invalid comparison for supported OS version * Addressing comments I * Problem: No way to configure dev/forked repo for developers * Problem: Docs not updated with STACK_REPO parameter * Addressing comments II
2024-10-13 13:34:05 +00:00 · 2018-05-31 16:56:45 +02:00
parent 0f56e4372d
commit 10e55aa4e7
86 changed files with 2235 additions and 1508 deletions
--- a/pkg/scripts/stack.sh
+++ b/pkg/scripts/stack.sh
@@ -1,124 +1,242 @@
 #!/usr/bin/env bash

-# ``stack.sh`` is an opinionated BigchainDB developer installation.  It
-# installs and configures **BigchainDb Server**, **Tendermint Server**,
-# **MongoDB**
-
-# Print the commands being run so that we can see the command that triggers
-# an error.  It is also useful for following along as the install occurs.
-set -o xtrace
+set -o nounset

 # Make sure umask is sane
 umask 022

-# Keep track of the stack.sh directory
-TOP_DIR=$(cd $(dirname "$0") && pwd)
-BASE_DIR=${TOP_DIR}/../..
+# defaults
+stack_branch=${STACK_BRANCH:="master"}
+stack_repo=${STACK_REPO:="bigchaindb/bigchaindb"}
+stack_size=${STACK_SIZE:=4}
+stack_type=${STACK_TYPE:="docker"}
+stack_type_provider=${STACK_TYPE_PROVIDER:=""}
+tm_version=${TM_VERSION:="0.19.2"}
+mongo_version=${MONGO_VERSION:="3.6"}
+stack_vm_memory=${STACK_VM_MEMORY:=2048}
+stack_vm_cpus=${STACK_VM_CPUS:=2}
+stack_box_name=${STACK_BOX_NAME:="ubuntu/xenial64"}
+azure_subscription_id=${AZURE_SUBSCRIPTION_ID:=""}
+azure_tenant_id=${AZURE_TENANT_ID:=""}
+azure_client_secret=${AZURE_CLIENT_SECRET:=""}
+azure_client_id=${AZURE_CLIENT_ID:=""}
+azure_region=${AZURE_REGION:="westeurope"}
+azure_image_urn=${AZURE_IMAGE_URN:="Canonical:UbuntuServer:16.04-LTS:latest"}
+azure_resource_group=${AZURE_RESOURCE_GROUP:="bdb-vagrant-rg-$(date '+%Y-%m-%d')"}
+azure_dns_prefix=${AZURE_DNS_PREFIX:="bdb-instance-$(date '+%Y-%m-%d')"}
+azure_admin_username=${AZURE_ADMIN_USERNAME:="vagrant"}
+azure_vm_size=${AZURE_VM_SIZE:="Standard_D2_v2"}
+ssh_private_key_path=${SSH_PRIVATE_KEY_PATH:=""}

-# Check for uninitialized variables, a big cause of bugs
+
+# Check for uninitialized variables
 NOUNSET=${NOUNSET:-}
 if [[ -n "$NOUNSET" ]]; then
-    set -o nounset
+	set -o nounset
 fi

-# Set default MongoDB version
-if [[ "$MONGO_VERSION" = "" ]]; then
-    MONGO_VERSION="3.4"
-fi
+TOP_DIR=$(cd $(dirname "$0") && pwd)
+SCRIPTS_DIR=$TOP_DIR/bigchaindb/pkg/scripts
+CONF_DIR=$TOP_DIR/bigchaindb/pkg/configuration

-# Set default tendermint version
-if [[ "$TM_VERSION" = "" ]]; then
-    TM_VERSION="0.12.1"
-fi

-# Configuration
-# =============
+function usage() {
+	cat <<EOM

-# Source utility functions
-source ${TOP_DIR}/functions-common
+    Usage: $ bash ${0##*/} [-h]

-# Configure Distro Repositories
-# -----------------------------
+    Deploys the BigchainDB network.

-# For Debian/Ubuntu make apt attempt to retry network ops on it's own and mongodb pub key
-# source repo
-if is_ubuntu; then
-    echo 'APT::Acquire::Retries "20";' | sudo tee /etc/apt/apt.conf.d/80retry  >/dev/null
-    sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
-    echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/${MONGO_VERSION} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list
-fi
+    ENV[STACK_SIZE]
+        Set STACK_SIZE environment variable to the size of the network you desire.
+        Network mimics a production network environment with single or multiple BDB
+        nodes. (default: ${stack_size}).

-# Ensure required packages are installed
-# --------------------------------------
+    ENV[STACK_TYPE]
+        Set STACK_TYPE environment variable to the type of deployment you desire.
+        You can set it one of the following: ["docker", "local", "cloud"].
+        (default: ${stack_type})

-is_package_installed python3 || install_package python3
-is_package_installed python3-pip || install_package python3-pip
-is_package_installed libffi-dev || install_package libffi-dev
-is_package_installed libssl-dev || install_package libssl-dev
-is_package_installed tmux || install_package tmux
-is_package_installed mongodb-org || install_package mongodb-org
-is_package_installed unzip || install_package unzip
-install_tendermint_bin
+    ENV[STACK_TYPE_PROVIDER]
+        Set only when STACK_TYPE="cloud". Only "azure" is supported.
+        (default: ${stack_type_provider})

-# Clean system if re-running the script
-OIFS=$IFS
-IFS=':'
-session_str=$(tmux ls | grep -w bdb-dev)
-if [[ $session_str = "" ]]; then
-    continue
-else
-    session=($session_str)
-    tmux kill-session -t ${session[0]}
-fi
+    ENV[STACK_VM_MEMORY]
+        (Optional) Set only when STACK_TYPE="local". This sets the memory
+        of the instance(s) spawned. (default: ${stack_vm_memory})

-# Stop bigchaindb service
-if is_running "bigchaindb"; then
-    sudo pkill bigchaindb
-fi
+    ENV[STACK_VM_CPUS]
+        (Optional) Set only when STACK_TYPE="local". This sets the number of VCPUs
+        of the instance(s) spawned. (default: ${stack_vm_cpus})

-# Stop tendermint service
-if is_running "tendermint"; then
-    sudo pkill tendermint
-fi
+    ENV[STACK_BOX_NAME]
+        (Optional) Set only when STACK_TYPE="local". This sets the box Vagrant box name
+        of the instance(s) spawned. (default: ${stack_box_name})

-# Stop mongodb service
-if is_running "monogod"; then
-    sudo pkill mongod
-fi
+    ENV[STACK_REPO]
+        (Optional) To configure bigchaindb repo to use, set STACK_REPO environment
+        variable. (default: ${stack_repo})

-sleep 5
+    ENV[STACK_BRANCH]
+        (Optional) To configure bigchaindb repo branch to use set STACK_BRANCH environment
+        variable. (default: ${stack_branch})

-# Create data dir for mongod
-if [[ ! -d /data/db ]]; then
-    sudo mkdir -p /data/db
-fi
-sudo chmod -R 700 /data/db
+    ENV[TM_VERSION]
+        (Optional) Tendermint version to use for the setup. (default: ${tm_version})

-# Configure tendermint
-tendermint init
+    ENV[MONGO_VERSION]
+        (Optional) MongoDB version to use with the setup. (default: ${mongo_version})

-# Configure tmux
-cd ${BASE_DIR}
-tmux new-session -s bdb-dev -n bdb -d
-tmux new-window -n mdb
-tmux new-window -n tendermint
+    ENV[AZURE_CLIENT_ID]
+        Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
+        https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application

-# Start MongoDB
-tmux send-keys -t bdb-dev:mdb 'sudo mongod --replSet=bigchain-rs' C-m
+    ENV[AZURE_TENANT_ID]
+        Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
+        https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application

-# Start BigchainDB
-tmux send-keys -t bdb-dev:bdb 'sudo python3 setup.py install && bigchaindb -y configure mongodb && bigchaindb -l DEBUG start' C-m
+    ENV[AZURE_SUBSCRIPTION_ID]
+        Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
+        https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application

-while ! is_running "bigchaindb"; do
-    echo "Waiting bigchaindb service to start"
-    sleep 5
+    ENV[AZURE_CLIENT_SECRET]
+        Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
+        https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
+
+    ENV[AZURE_REGION]
+        (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
+        Azure region for the BigchainDB instance. Get list of regions using Azure CLI.
+        e.g. az account list-locations. (default: ${azure_region})
+
+    ENV[AZURE_IMAGE_URN]
+        (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
+        Azure image to use. Get list of available images using Azure CLI.
+        e.g. az vm image list --output table. (default: ${azure_image_urn})
+
+    ENV[AZURE_RESOURCE_GROUP]
+        (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
+        Name of Azure resource group for the instance.
+        (default: ${azure_resource_group})
+
+    ENV[AZURE_DNS_PREFIX]
+        (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
+        DNS Prefix of the instance. (default: ${azure_dns_prefix})
+
+    ENV[AZURE_ADMIN_USERNAME]
+        (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
+        Admin username of the the instance. (default: ${azure_admin_username})
+
+    ENV[AZURE_VM_SIZE]
+        (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
+        Azure VM size. (default: ${azure_vm_size})
+
+    ENV[SSH_PRIVATE_KEY_PATH]
+        Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Absolute path of
+        SSH keypair required to log into the Azure instance.
+
+    -h
+        Show this help and exit.
+
+EOM
+}
+
+while getopts "h" opt; do
+	case "$opt" in
+	h)
+		usage
+		exit
+		;;
+	*)
+		usage
+		exit 1
+		;;
+	esac
 done

-# Start tendermint service
-tmux send-key -t bdb-dev:tendermint 'tendermint init && tendermint unsafe_reset_all && tendermint node' C-m
+mkdir -p logs
+log_file=logs/install-$(date +%Y%m%d-%H%M%S).log
+exec > >(tee $log_file) 2>&1
+echo "Capturing output to $log_file"
+echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"

-# Configure Error Traps
-# ---------------------
+function finish() {
+	echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
+}
+trap finish EXIT
+
+export STACK_REPO=$stack_repo
+export STACK_BRANCH=$stack_branch
+echo "Using bigchaindb repo: '$STACK_REPO'"
+echo "Using bigchaindb branch '$STACK_BRANCH'"
+
+git clone https://github.com/${stack_repo}.git -b $stack_branch || true
+
+# Source utility functions
+source ${SCRIPTS_DIR}/functions-common
+
+if [[ $stack_type == "local" ]]; then
+	mongo_version=$(echo "$mongo_version" | cut -d. -f-2)
+fi
+
+# configure stack-config.yml
+cat >$TOP_DIR/bigchaindb/pkg/configuration/vars/stack-config.yml <<EOF
+---
+stack_type: "${stack_type}"
+stack_size: "${stack_size}"
+stack_type_provider: "${stack_type_provider}"
+stack_box_name: "${stack_box_name}"
+stack_vm_memory: "${stack_vm_memory}"
+stack_vm_cpus: "${stack_vm_cpus}"
+tm_version: "${tm_version}"
+mongo_version: "${mongo_version}"
+azure_region: "${azure_region}"
+azure_image_urn: "${azure_image_urn}"
+azure_resource_group: "${azure_resource_group}"
+azure_dns_prefix: "${azure_dns_prefix}"
+azure_admin_username: "${azure_admin_username}"
+azure_vm_size: "${azure_vm_size}"
+ssh_private_key_path: "${ssh_private_key_path}"
+EOF
+
+curl -fOL# https://raw.githubusercontent.com/${stack_repo}/${stack_branch}/pkg/scripts/Vagrantfile
+
+#Convert to lowercase
+stack_type="$(echo $stack_type | tr '[A-Z]' '[a-z]')"
+stack_type_provider="$(echo $stack_type_provider | tr '[A-Z]' '[a-z]')"
+
+if [[ $stack_type == "local" ]]; then
+	echo "Configuring setup locally!"
+	vagrant up --provider virtualbox --provision
+	ansible-playbook $CONF_DIR/bigchaindb-start.yml \
+		-i $CONF_DIR/hosts/all \
+		--extra-vars "operation=start home_path=${TOP_DIR}"
+elif [[ $stack_type == "cloud" && $stack_type_provider == "azure" ]]; then
+    echo ${azure_tenant_id:?AZURE_TENANT_ID not set! Exiting. $(exit 1)}
+    echo ${azure_client_secret:?AZURE_CLIENT_SECRET not set! Exiting. $(exit 1)}
+    echo ${azure_client_id:?AZURE_CLIENT_ID not set! Exiting. $(exit 1)}
+    echo ${azure_subscription_id:?AZURE_SUBSCRIPTION_ID not set! Exiting. $(exit 1)}
+    echo ${ssh_private_key_path:?SSH_PRIVATE_KEY_PATH not set! $(exit 1)}
+    echo "Configuring Setup on Azure!"
+    # Dummy box does not really do anything because we are relying on Azure VM images
+    vagrant box add azure-dummy https://github.com/azure/vagrant-azure/raw/v2.0/dummy.box \
+    --provider azure --force
+	vagrant up --provider azure --provision
+	ansible-playbook $CONF_DIR/bigchaindb-start.yml \
+		-i $CONF_DIR/hosts/all \
+		--extra-vars "operation=start home_path=/bigchaindb"
+elif [[ $stack_type == "docker" ]]; then
+	echo "Configuring Dockers locally!"
+	source $SCRIPTS_DIR/bootstrap.sh --operation install
+	cat >$CONF_DIR/hosts/all <<EOF
+  $(hostname)  ansible_connection=local
+EOF
+	ansible-playbook $CONF_DIR/bigchaindb-start.yml \
+    -i $CONF_DIR/hosts/all \
+	--extra-vars "operation=start home_path=${TOP_DIR}"
+else
+	echo "Invalid Stack Type OR Provider"
+	exit 1
+fi

 # Kill background processes on exit
 trap exit_trap EXIT
@@ -134,5 +252,5 @@ function err_trap {
    exit $?
 }

-# Begin trapping error exit codes
+echo -e "Finished stacking!"
 set -o errexit