Scripts to bootstrap devstack setup for bigchaindb with tendermint (#1887)

bash scripts to setup dev environment with bigchaindb+tendermint+mongodb

pkg/scripts/Vagrantfile

@@ -0,0 +1,61 @@
+Vagrant.require_version ">= 1.8.7"
+unless Vagrant.has_plugin?("vagrant-vbguest")
+  raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
+end
+
+unless Vagrant.has_plugin?("vagrant-cachier")
+  raise "Please install the vagrant-cachier plugin by running `vagrant plugin install vagrant-cachier`"
+end
+
+VAGRANTFILE_API_VERSION = "2"
+
+MEMORY = 4096
+CPU_COUNT = 2
+
+MOUNT_DIRS = {
+  :bigchaindb => {:repo => "bigchaindb", :local => "/opt/stack/bigchaindb", :owner => "edxapp"},
+}
+
+boxname = ENV['BOXNAME'] || "ubuntu/xenial64"
+tm_version = ENV['TM_VERSION']
+
+$script = <<SCRIPT
+if [ ! -d /opt/stack/bigchaindb/pkg/scripts ]; then
+    echo "Error: Base box is missing provisioning scripts." 1>&2
+    exit 1
+fi
+
+bash /opt/stack/bigchaindb/pkg/scripts/stack.sh
+
+SCRIPT
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  config.vm.box = "ubuntu/xenial64"
+  config.vm.box_check_update = false
+
+  config.vm.network :private_network, ip: "192.168.33.10"
+
+
+  config.vm.network :forwarded_port, guest: 9984, host: 9984  # BDB
+
+  config.ssh.insert_key = true
+
+  config.vm.synced_folder  "bigchaindb", "/opt/stack/bigchaindb"
+
+
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
+    vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
+  end
+
+  # Use vagrant-vbguest plugin to make sure Guest Additions are in sync
+  config.vbguest.auto_reboot = true
+  config.vbguest.auto_update = true
+
+  config.vm.provision "shell", inline: $script,
+    privileged: false,
+    env: {
+      :TM_VERSION => ENV['TM_VERSION'], 
+      :MONGO_VERSION => ENV['MONGO_VERSION']
+    }
+end

pkg/scripts/functions-common

@@ -0,0 +1,390 @@
+#!/bin/bash
+
+# Save trace setting
+_XTRACE_FUNCTIONS_COMMON=$(set +o | grep xtrace)
+set +o xtrace
+
+# Distro Functions
+# ================
+
+# Determine OS Vendor, Release and Update
+
+#
+# NOTE : For portability, you almost certainly do not want to use
+# these variables directly!  The "is_*" functions defined below this
+# bundle up compatible platforms under larger umbrellas that we have
+# determinted are compatible enough (e.g. is_ubuntu covers Ubuntu &
+# Debian, is_fedora covers RPM-based distros).  Higher-level functions
+# such as "install_package" further abstract things in better ways.
+#
+# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
+# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
+# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
+# ``os_CODENAME`` - vendor's codename for release: ``xenial``
+
+declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME
+
+# Make a *best effort* attempt to install lsb_release packages for the
+# user if not available.  Note can't use generic install_package*
+# because they depend on this!
+function _ensure_lsb_release {
+    if [[ -x $(command -v lsb_release 2>/dev/null) ]]; then
+        return
+    fi
+
+    if [[ -x $(command -v apt-get 2>/dev/null) ]]; then
+        sudo apt-get install -y lsb-release
+    elif [[ -x $(command -v zypper 2>/dev/null) ]]; then
+        sudo zypper -n install lsb-release
+    elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
+        sudo dnf install -y redhat-lsb-core
+    elif [[ -x $(command -v yum 2>/dev/null) ]]; then
+        # all rh patforms (fedora, centos, rhel) have this pkg
+        sudo yum install -y redhat-lsb-core
+    else
+        die $LINENO "Unable to find or auto-install lsb_release"
+    fi
+}
+
+# GetOSVersion
+#  Set the following variables:
+#  - os_RELEASE
+#  - os_CODENAME
+#  - os_VENDOR
+#  - os_PACKAGE
+function GetOSVersion {
+    # We only support distros that provide a sane lsb_release
+    _ensure_lsb_release
+
+    os_RELEASE=$(lsb_release -r -s)
+    os_CODENAME=$(lsb_release -c -s)
+    os_VENDOR=$(lsb_release -i -s)
+
+    if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
+        os_PACKAGE="deb"
+    else
+        os_PACKAGE="rpm"
+    fi
+
+    typeset -xr os_VENDOR
+    typeset -xr os_RELEASE
+    typeset -xr os_PACKAGE
+    typeset -xr os_CODENAME
+}
+
+# Translate the OS version values into common nomenclature
+# Sets global ``DISTRO`` from the ``os_*`` values
+declare -g DISTRO
+
+function GetDistro {
+    GetOSVersion
+    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
+            "$os_VENDOR" =~ (LinuxMint) ]]; then
+        # 'Everyone' refers to Ubuntu / Debian / Mint releases by
+        # the code name adjective
+        DISTRO=$os_CODENAME
+    elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
+        # For Fedora, just use 'f' and the release
+        DISTRO="f$os_RELEASE"
+    elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
+        DISTRO="opensuse-$os_RELEASE"
+    elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
+        # just use major release
+        DISTRO="sle${os_RELEASE%.*}"
+    elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
+        "$os_VENDOR" =~ (CentOS) || \
+        "$os_VENDOR" =~ (Scientific) || \
+        "$os_VENDOR" =~ (OracleServer) || \
+        "$os_VENDOR" =~ (Virtuozzo) ]]; then
+        # Drop the . release as we assume it's compatible
+        # XXX re-evaluate when we get RHEL10
+        DISTRO="rhel${os_RELEASE::1}"
+    elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
+        DISTRO="xs${os_RELEASE%.*}"
+    elif [[ "$os_VENDOR" =~ (kvmibm) ]]; then
+        DISTRO="${os_VENDOR}${os_RELEASE::1}"
+    else
+        # We can't make a good choice here.  Setting a sensible DISTRO
+        # is part of the problem, but not the major issue -- we really
+        # only use DISTRO in the code as a fine-filter.
+        #
+        # The bigger problem is categorising the system into one of
+        # our two big categories as Ubuntu/Debian-ish or
+        # Fedora/CentOS-ish.
+        #
+        # The setting of os_PACKAGE above is only set to "deb" based
+        # on a hard-coded list of vendor names ... thus we will
+        # default to thinking unknown distros are RPM based
+        # (ie. is_ubuntu does not match).  But the platform will then
+        # also not match in is_fedora, because that also has a list of
+        # names.
+        #
+        # So, if you are reading this, getting your distro supported
+        # is really about making sure it matches correctly in these
+        # functions.  Then you can choose a sensible way to construct
+        # DISTRO based on your distros release approach.
+        die $LINENO "Unable to determine DISTRO, can not continue."
+    fi
+    typeset -xr DISTRO
+}
+
+# Utility function for checking machine architecture
+# is_arch arch-type
+function is_arch {
+    [[ "$(uname -m)" == "$1" ]]
+}
+
+# Determine if current distribution is an Oracle distribution
+# is_oraclelinux
+function is_oraclelinux {
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    [ "$os_VENDOR" = "OracleServer" ]
+}
+
+
+# Determine if current distribution is a Fedora-based distribution
+# (Fedora, RHEL, CentOS, etc).
+# is_fedora
+function is_fedora {
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
+        [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
+        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \
+        [ "$os_VENDOR" = "Virtuozzo" ] || [ "$os_VENDOR" = "kvmibm" ]
+}
+
+
+# Determine if current distribution is a SUSE-based distribution
+# (openSUSE, SLE).
+# is_suse
+function is_suse {
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    [[ "$os_VENDOR" =~ (openSUSE) || "$os_VENDOR" == "SUSE LINUX" ]]
+}
+
+
+# Determine if current distribution is an Ubuntu-based distribution
+# It will also detect non-Ubuntu but Debian-based distros
+# is_ubuntu
+function is_ubuntu {
+    if [[ -z "$os_PACKAGE" ]]; then
+        GetOSVersion
+    fi
+    [ "$os_PACKAGE" = "deb" ]
+}
+
+# Package Functions
+# =================
+
+# Wrapper for ``apt-get update`` to try multiple times on the update
+# to address bad package mirrors (which happen all the time).
+function apt_get_update {
+    # only do this once per run
+    if [[ "$REPOS_UPDATED" == "True" && "$RETRY_UPDATE" != "True" ]]; then
+        return
+    fi
+
+    # bail if we are offline
+    [[ "$OFFLINE" = "True" ]] && return
+
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+
+    # time all the apt operations
+    time_start "apt-get-update"
+
+    local proxies="http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} "
+    local update_cmd="$sudo $proxies apt-get update"
+    if ! timeout 300 sh -c "while ! $update_cmd; do sleep 30; done"; then
+        die $LINENO "Failed to update apt repos, we're dead now"
+    fi
+
+    REPOS_UPDATED=True
+    # stop the clock
+    time_stop "apt-get-update"
+}
+
+# Wrapper for ``apt-get`` to set cache and proxy environment variables
+# Uses globals ``OFFLINE``, ``*_proxy``
+# apt_get operation package [package ...]
+function apt_get {
+    local xtrace result
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    [[ "$OFFLINE" = "True" || -z "$@" ]] && return
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+
+    # time all the apt operations
+    time_start "apt-get"
+
+    $xtrace
+
+    $sudo DEBIAN_FRONTEND=noninteractive \
+        http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \
+        no_proxy=${no_proxy:-} \
+        apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@" < /dev/null
+    result=$?
+
+    # stop the clock
+    time_stop "apt-get"
+    return $result
+}
+
+
+# Distro-agnostic package installer
+# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
+# install_package package [package ...]
+function update_package_repo {
+    NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False}
+    REPOS_UPDATED=${REPOS_UPDATED:-False}
+    RETRY_UPDATE=${RETRY_UPDATE:-False}
+
+    if [[ "$NO_UPDATE_REPOS" = "True" ]]; then
+        return 0
+    fi
+
+    if is_ubuntu; then
+        apt_get_update
+    fi
+}
+
+function real_install_package {
+    if is_ubuntu; then
+        apt_get install "$@"
+    elif is_fedora; then
+        yum_install "$@"
+    elif is_suse; then
+        zypper_install "$@"
+    else
+        exit_distro_not_supported "installing packages"
+    fi
+}
+
+# Distro-agnostic package installer
+# install_package package [package ...]
+function install_package {
+    update_package_repo
+    if ! real_install_package "$@"; then
+        RETRY_UPDATE=True update_package_repo && real_install_package "$@"
+    fi
+}
+
+# Distro-agnostic function to tell if a package is installed
+# is_package_installed package [package ...]
+function is_package_installed {
+    if [[ -z "$@" ]]; then
+        return 1
+    fi
+
+    if [[ -z "$os_PACKAGE" ]]; then
+        GetOSVersion
+    fi
+
+    if [[ "$os_PACKAGE" = "deb" ]]; then
+        dpkg -s "$@" > /dev/null 2> /dev/null
+    elif [[ "$os_PACKAGE" = "rpm" ]]; then
+        rpm --quiet -q "$@"
+    else
+        exit_distro_not_supported "finding if a package is installed"
+    fi
+}
+
+# Distro-agnostic package uninstaller
+# uninstall_package package [package ...]
+function uninstall_package {
+    if is_ubuntu; then
+        apt_get purge "$@"
+    elif is_fedora; then
+        sudo ${YUM:-yum} remove -y "$@" ||:
+    elif is_suse; then
+        sudo zypper remove -y "$@" ||:
+    else
+        exit_distro_not_supported "uninstalling packages"
+    fi
+}
+
+# Wrapper for ``yum`` to set proxy environment variables
+# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM``
+# yum_install package [package ...]
+function yum_install {
+    local result parse_yum_result
+    time_start "yum_install"
+
+    # This is a bit tricky, because yum -y assumes missing or failed
+    # packages are OK (see [1]).  We want devstack to stop if we are
+    # installing missing packages.
+    #
+    # Thus we manually match on the output (stack.sh runs in a fixed
+    # locale, so lang shouldn't change).
+    #
+    # If yum returns !0, we echo the result as "YUM_FAILED" and return
+    # that from the awk (we're subverting -e with this trick).
+    # Otherwise we use awk to look for failure strings and return "2"
+    # to indicate a terminal failure.
+    #
+    # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567
+    parse_yum_result='              \
+        BEGIN { result=0 }          \
+        /^YUM_FAILED/ { result=$2 } \
+        /^No package/ { result=2 }  \
+        /^Failed:/    { result=2 }  \
+        //{ print }                 \
+        END { exit result }'
+    (sudo_with_proxies "${YUM:-yum}" install -y "$@" 2>&1 || echo YUM_FAILED $?) \
+        | awk "$parse_yum_result" && result=$? || result=$?
+
+    time_stop "yum_install"
+
+    # if we return 1, then the wrapper functions will run an update
+    # and try installing the package again as a defense against bad
+    # mirrors.  This can hide failures, especially when we have
+    # packages that are in the "Failed:" section because their rpm
+    # install scripts failed to run correctly (in this case, the
+    # package looks installed, so when the retry happens we just think
+    # the package is OK, and incorrectly continue on).
+    if [ "$result" == 2 ]; then
+        die "Detected fatal package install failure"
+    fi
+
+    return "$result"
+}
+
+# zypper wrapper to set arguments correctly
+# Uses globals ``OFFLINE``, ``*_proxy``
+# zypper_install package [package ...]
+function zypper_install {
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+    $sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
+        no_proxy="${no_proxy:-}" \
+        zypper --non-interactive install --auto-agree-with-licenses "$@"
+}
+
+function install_tendermint_bin {
+    wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip
+    unzip tendermint_${TM_VERSION}_linux_amd64.zip
+    sudo mv tendermint /usr/local/bin
+}
+
+# Find out if a process exists by partial name.
+# is_running name
+function is_running {
+    local name=$1
+    ps auxw | grep -v grep | grep ${name} > /dev/null
+    local exitcode=$?
+    return $exitcode
+}
+
+# Restore xtrace
+$_XTRACE_FUNCTIONS_COMMON

pkg/scripts/install_stack.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+function usage
+{
+    cat << EOM
+
+    Usage: $ bash ${0##*/} [-v] [-h]
+
+    Installs the BigchainDB devstack or network.
+
+    ENV[STACK]
+        Set STACK environment variable to Either 'devstack' or 'network'.
+        Network mimics a production network environment with multiple BDB
+        nodes, whereas devstack is useful if you plan on modifying the
+        bigchaindb code.
+
+    ENV[GIT_BRANCH]
+        To configure bigchaindb repo branch to use set GIT_BRANCH environment
+        variable
+
+    ENV[TM_VERSION]
+        Tendermint version to use for the devstack setup
+
+    ENV[MONGO_VERSION]
+        MongoDB version to use with the devstack setup
+
+    -v
+        Verbose output from ansible playbooks.
+
+    -h
+        Show this help and exit.
+
+EOM
+}
+
+# GIT_BRANCH
+git_branch=$GIT_BRANCH
+
+while getopts "h" opt; do
+    case "$opt" in
+        h)
+            usage
+            exit
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+if [[ ! $git_branch ]]; then
+    echo "You must specify GIT_BRANCH before running."
+    echo
+    echo usage
+    exit 1
+fi
+
+mkdir -p logs
+log_file=logs/install-$(date +%Y%m%d-%H%M%S).log
+exec > >(tee $log_file) 2>&1
+echo "Capturing output to $log_file"
+echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
+
+function finish {
+    echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
+}
+trap finish EXIT
+
+export GIT_BRANCH=$git_branch
+echo "Using bigchaindb branch '$GIT_BRANCH'"
+
+git clone https://github.com/bigchaindb/bigchaindb.git -b $GIT_BRANCH || true
+curl -fOL# https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/Vagrantfile
+vagrant up --provider virtualbox
+
+echo -e "Finished installing! You may now log in using 'vagrant ssh'"
+echo -e "Once inside the VM do 'tmux attach' to attach to tmux session running all services"

pkg/scripts/stack.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+# ``stack.sh`` is an opinionated BigchainDB developer installation.  It
+# installs and configures **BigchainDb Server**, **Tendermint Server**,
+# **MongoDB**
+
+# Print the commands being run so that we can see the command that triggers
+# an error.  It is also useful for following along as the install occurs.
+set -o xtrace
+
+# Make sure umask is sane
+umask 022
+
+# Keep track of the stack.sh directory
+TOP_DIR=$(cd $(dirname "$0") && pwd)
+BASE_DIR=${TOP_DIR}/../..
+
+# Check for uninitialized variables, a big cause of bugs
+NOUNSET=${NOUNSET:-}
+if [[ -n "$NOUNSET" ]]; then
+    set -o nounset
+fi
+
+# Set default MongoDB version
+if [[ "$MONGO_VERSION" = "" ]]; then
+    MONGO_VERSION="3.4"
+fi
+
+# Set default tendermint version
+if [[ "$TM_VERSION" = "" ]]; then
+    TM_VERSION="0.12.1"
+fi
+
+# Configuration
+# =============
+
+# Source utility functions
+source ${TOP_DIR}/functions-common
+
+# Configure Distro Repositories
+# -----------------------------
+
+# For Debian/Ubuntu make apt attempt to retry network ops on it's own and mongodb pub key
+# source repo
+if is_ubuntu; then
+    echo 'APT::Acquire::Retries "20";' | sudo tee /etc/apt/apt.conf.d/80retry  >/dev/null
+    sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
+    echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/${MONGO_VERSION} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list
+fi
+
+# Ensure required packages are installed
+# --------------------------------------
+
+is_package_installed python3 || install_package python3
+is_package_installed python3-pip || install_package python3-pip
+is_package_installed libffi-dev || install_package libffi-dev
+is_package_installed libssl-dev || install_package libssl-dev
+is_package_installed tmux || install_package tmux
+is_package_installed mongodb-org || install_package mongodb-org
+is_package_installed unzip || install_package unzip
+install_tendermint_bin
+
+# Clean system if re-running the script
+OIFS=$IFS
+IFS=':'
+session_str=$(tmux ls | grep -w bdb-dev)
+if [[ $session_str = "" ]]; then
+    continue
+else
+    session=($session_str)
+    tmux kill-session -t ${session[0]}
+fi
+
+# Stop bigchaindb service
+if is_running "bigchaindb"; then
+    sudo pkill bigchaindb
+fi
+
+# Stop tendermint service
+if is_running "tendermint"; then
+    sudo pkill tendermint
+fi
+
+# Stop mongodb service
+if is_running "monogod"; then
+    sudo pkill mongod
+fi
+
+sleep 5
+
+# Create data dir for mongod
+if [[ ! -d /data/db ]]; then
+    sudo mkdir -p /data/db
+fi
+sudo chmod -R 700 /data/db
+
+# Configure tendermint
+tendermint init
+
+# Configure tmux
+cd ${BASE_DIR}
+tmux new-session -s bdb-dev -n bdb -d
+tmux new-window -n mdb
+tmux new-window -n tendermint
+
+# Start MongoDB
+tmux send-keys -t bdb-dev:mdb 'sudo mongod --replSet=bigchain-rs' C-m
+
+# Start BigchainDB
+tmux send-keys -t bdb-dev:bdb 'sudo python3 setup.py install && bigchaindb -y configure mongodb && bigchaindb -l DEBUG start' C-m
+
+while ! is_running "bigchaindb"; do
+    echo "Waiting bigchaindb service to start"
+    sleep 5
+done
+
+# Start tendermint service
+tmux send-key -t bdb-dev:tendermint 'tendermint init && tendermint unsafe_reset_all && tendermint node' C-m
+
+# Configure Error Traps
+# ---------------------
+
+# Kill background processes on exit
+trap exit_trap EXIT
+function exit_trap {
+    exit $?
+}
+# Exit on any errors so that errors don't compound and kill if any services already started
+trap err_trap ERR
+function err_trap {
+    local r=$?
+    tmux kill-session bdb-dev
+    set +o xtrace
+    exit $?
+}
+
+# Begin trapping error exit codes
+set -o errexit