Scripts to bootstrap devstack setup for bigchaindb with tendermint (#1887)

bash scripts to setup dev environment with bigchaindb+tendermint+mongodb
This commit is contained in:
Shahbaz Nazir 2017-12-11 17:03:32 +01:00 committed by GitHub
parent 8c0e28ee46
commit 6ddf37dc66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 670 additions and 0 deletions

61
pkg/scripts/Vagrantfile vendored Normal file
View File

@ -0,0 +1,61 @@
Vagrant.require_version ">= 1.8.7"
unless Vagrant.has_plugin?("vagrant-vbguest")
raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
end
unless Vagrant.has_plugin?("vagrant-cachier")
raise "Please install the vagrant-cachier plugin by running `vagrant plugin install vagrant-cachier`"
end
VAGRANTFILE_API_VERSION = "2"
MEMORY = 4096
CPU_COUNT = 2
MOUNT_DIRS = {
:bigchaindb => {:repo => "bigchaindb", :local => "/opt/stack/bigchaindb", :owner => "edxapp"},
}
boxname = ENV['BOXNAME'] || "ubuntu/xenial64"
tm_version = ENV['TM_VERSION']
$script = <<SCRIPT
if [ ! -d /opt/stack/bigchaindb/pkg/scripts ]; then
echo "Error: Base box is missing provisioning scripts." 1>&2
exit 1
fi
bash /opt/stack/bigchaindb/pkg/scripts/stack.sh
SCRIPT
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "ubuntu/xenial64"
config.vm.box_check_update = false
config.vm.network :private_network, ip: "192.168.33.10"
config.vm.network :forwarded_port, guest: 9984, host: 9984 # BDB
config.ssh.insert_key = true
config.vm.synced_folder "bigchaindb", "/opt/stack/bigchaindb"
config.vm.provider :virtualbox do |vb|
vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
end
# Use vagrant-vbguest plugin to make sure Guest Additions are in sync
config.vbguest.auto_reboot = true
config.vbguest.auto_update = true
config.vm.provision "shell", inline: $script,
privileged: false,
env: {
:TM_VERSION => ENV['TM_VERSION'],
:MONGO_VERSION => ENV['MONGO_VERSION']
}
end

View File

@ -0,0 +1,390 @@
#!/bin/bash
# Save trace setting
_XTRACE_FUNCTIONS_COMMON=$(set +o | grep xtrace)
set +o xtrace
# Distro Functions
# ================
# Determine OS Vendor, Release and Update
#
# NOTE : For portability, you almost certainly do not want to use
# these variables directly! The "is_*" functions defined below this
# bundle up compatible platforms under larger umbrellas that we have
# determinted are compatible enough (e.g. is_ubuntu covers Ubuntu &
# Debian, is_fedora covers RPM-based distros). Higher-level functions
# such as "install_package" further abstract things in better ways.
#
# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
# ``os_CODENAME`` - vendor's codename for release: ``xenial``
declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME
# Make a *best effort* attempt to install lsb_release packages for the
# user if not available. Note can't use generic install_package*
# because they depend on this!
function _ensure_lsb_release {
if [[ -x $(command -v lsb_release 2>/dev/null) ]]; then
return
fi
if [[ -x $(command -v apt-get 2>/dev/null) ]]; then
sudo apt-get install -y lsb-release
elif [[ -x $(command -v zypper 2>/dev/null) ]]; then
sudo zypper -n install lsb-release
elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
sudo dnf install -y redhat-lsb-core
elif [[ -x $(command -v yum 2>/dev/null) ]]; then
# all rh patforms (fedora, centos, rhel) have this pkg
sudo yum install -y redhat-lsb-core
else
die $LINENO "Unable to find or auto-install lsb_release"
fi
}
# GetOSVersion
# Set the following variables:
# - os_RELEASE
# - os_CODENAME
# - os_VENDOR
# - os_PACKAGE
function GetOSVersion {
# We only support distros that provide a sane lsb_release
_ensure_lsb_release
os_RELEASE=$(lsb_release -r -s)
os_CODENAME=$(lsb_release -c -s)
os_VENDOR=$(lsb_release -i -s)
if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
os_PACKAGE="deb"
else
os_PACKAGE="rpm"
fi
typeset -xr os_VENDOR
typeset -xr os_RELEASE
typeset -xr os_PACKAGE
typeset -xr os_CODENAME
}
# Translate the OS version values into common nomenclature
# Sets global ``DISTRO`` from the ``os_*`` values
declare -g DISTRO
function GetDistro {
GetOSVersion
if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
"$os_VENDOR" =~ (LinuxMint) ]]; then
# 'Everyone' refers to Ubuntu / Debian / Mint releases by
# the code name adjective
DISTRO=$os_CODENAME
elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
# For Fedora, just use 'f' and the release
DISTRO="f$os_RELEASE"
elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
DISTRO="opensuse-$os_RELEASE"
elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
# just use major release
DISTRO="sle${os_RELEASE%.*}"
elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
"$os_VENDOR" =~ (CentOS) || \
"$os_VENDOR" =~ (Scientific) || \
"$os_VENDOR" =~ (OracleServer) || \
"$os_VENDOR" =~ (Virtuozzo) ]]; then
# Drop the . release as we assume it's compatible
# XXX re-evaluate when we get RHEL10
DISTRO="rhel${os_RELEASE::1}"
elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
DISTRO="xs${os_RELEASE%.*}"
elif [[ "$os_VENDOR" =~ (kvmibm) ]]; then
DISTRO="${os_VENDOR}${os_RELEASE::1}"
else
# We can't make a good choice here. Setting a sensible DISTRO
# is part of the problem, but not the major issue -- we really
# only use DISTRO in the code as a fine-filter.
#
# The bigger problem is categorising the system into one of
# our two big categories as Ubuntu/Debian-ish or
# Fedora/CentOS-ish.
#
# The setting of os_PACKAGE above is only set to "deb" based
# on a hard-coded list of vendor names ... thus we will
# default to thinking unknown distros are RPM based
# (ie. is_ubuntu does not match). But the platform will then
# also not match in is_fedora, because that also has a list of
# names.
#
# So, if you are reading this, getting your distro supported
# is really about making sure it matches correctly in these
# functions. Then you can choose a sensible way to construct
# DISTRO based on your distros release approach.
die $LINENO "Unable to determine DISTRO, can not continue."
fi
typeset -xr DISTRO
}
# Utility function for checking machine architecture
# is_arch arch-type
function is_arch {
[[ "$(uname -m)" == "$1" ]]
}
# Determine if current distribution is an Oracle distribution
# is_oraclelinux
function is_oraclelinux {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "OracleServer" ]
}
# Determine if current distribution is a Fedora-based distribution
# (Fedora, RHEL, CentOS, etc).
# is_fedora
function is_fedora {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
[ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
[ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \
[ "$os_VENDOR" = "Virtuozzo" ] || [ "$os_VENDOR" = "kvmibm" ]
}
# Determine if current distribution is a SUSE-based distribution
# (openSUSE, SLE).
# is_suse
function is_suse {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[[ "$os_VENDOR" =~ (openSUSE) || "$os_VENDOR" == "SUSE LINUX" ]]
}
# Determine if current distribution is an Ubuntu-based distribution
# It will also detect non-Ubuntu but Debian-based distros
# is_ubuntu
function is_ubuntu {
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
[ "$os_PACKAGE" = "deb" ]
}
# Package Functions
# =================
# Wrapper for ``apt-get update`` to try multiple times on the update
# to address bad package mirrors (which happen all the time).
function apt_get_update {
# only do this once per run
if [[ "$REPOS_UPDATED" == "True" && "$RETRY_UPDATE" != "True" ]]; then
return
fi
# bail if we are offline
[[ "$OFFLINE" = "True" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
# time all the apt operations
time_start "apt-get-update"
local proxies="http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} "
local update_cmd="$sudo $proxies apt-get update"
if ! timeout 300 sh -c "while ! $update_cmd; do sleep 30; done"; then
die $LINENO "Failed to update apt repos, we're dead now"
fi
REPOS_UPDATED=True
# stop the clock
time_stop "apt-get-update"
}
# Wrapper for ``apt-get`` to set cache and proxy environment variables
# Uses globals ``OFFLINE``, ``*_proxy``
# apt_get operation package [package ...]
function apt_get {
local xtrace result
xtrace=$(set +o | grep xtrace)
set +o xtrace
[[ "$OFFLINE" = "True" || -z "$@" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
# time all the apt operations
time_start "apt-get"
$xtrace
$sudo DEBIAN_FRONTEND=noninteractive \
http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \
no_proxy=${no_proxy:-} \
apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@" < /dev/null
result=$?
# stop the clock
time_stop "apt-get"
return $result
}
# Distro-agnostic package installer
# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
# install_package package [package ...]
function update_package_repo {
NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False}
REPOS_UPDATED=${REPOS_UPDATED:-False}
RETRY_UPDATE=${RETRY_UPDATE:-False}
if [[ "$NO_UPDATE_REPOS" = "True" ]]; then
return 0
fi
if is_ubuntu; then
apt_get_update
fi
}
function real_install_package {
if is_ubuntu; then
apt_get install "$@"
elif is_fedora; then
yum_install "$@"
elif is_suse; then
zypper_install "$@"
else
exit_distro_not_supported "installing packages"
fi
}
# Distro-agnostic package installer
# install_package package [package ...]
function install_package {
update_package_repo
if ! real_install_package "$@"; then
RETRY_UPDATE=True update_package_repo && real_install_package "$@"
fi
}
# Distro-agnostic function to tell if a package is installed
# is_package_installed package [package ...]
function is_package_installed {
if [[ -z "$@" ]]; then
return 1
fi
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
if [[ "$os_PACKAGE" = "deb" ]]; then
dpkg -s "$@" > /dev/null 2> /dev/null
elif [[ "$os_PACKAGE" = "rpm" ]]; then
rpm --quiet -q "$@"
else
exit_distro_not_supported "finding if a package is installed"
fi
}
# Distro-agnostic package uninstaller
# uninstall_package package [package ...]
function uninstall_package {
if is_ubuntu; then
apt_get purge "$@"
elif is_fedora; then
sudo ${YUM:-yum} remove -y "$@" ||:
elif is_suse; then
sudo zypper remove -y "$@" ||:
else
exit_distro_not_supported "uninstalling packages"
fi
}
# Wrapper for ``yum`` to set proxy environment variables
# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM``
# yum_install package [package ...]
function yum_install {
local result parse_yum_result
time_start "yum_install"
# This is a bit tricky, because yum -y assumes missing or failed
# packages are OK (see [1]). We want devstack to stop if we are
# installing missing packages.
#
# Thus we manually match on the output (stack.sh runs in a fixed
# locale, so lang shouldn't change).
#
# If yum returns !0, we echo the result as "YUM_FAILED" and return
# that from the awk (we're subverting -e with this trick).
# Otherwise we use awk to look for failure strings and return "2"
# to indicate a terminal failure.
#
# [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567
parse_yum_result=' \
BEGIN { result=0 } \
/^YUM_FAILED/ { result=$2 } \
/^No package/ { result=2 } \
/^Failed:/ { result=2 } \
//{ print } \
END { exit result }'
(sudo_with_proxies "${YUM:-yum}" install -y "$@" 2>&1 || echo YUM_FAILED $?) \
| awk "$parse_yum_result" && result=$? || result=$?
time_stop "yum_install"
# if we return 1, then the wrapper functions will run an update
# and try installing the package again as a defense against bad
# mirrors. This can hide failures, especially when we have
# packages that are in the "Failed:" section because their rpm
# install scripts failed to run correctly (in this case, the
# package looks installed, so when the retry happens we just think
# the package is OK, and incorrectly continue on).
if [ "$result" == 2 ]; then
die "Detected fatal package install failure"
fi
return "$result"
}
# zypper wrapper to set arguments correctly
# Uses globals ``OFFLINE``, ``*_proxy``
# zypper_install package [package ...]
function zypper_install {
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
$sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
zypper --non-interactive install --auto-agree-with-licenses "$@"
}
function install_tendermint_bin {
wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip
unzip tendermint_${TM_VERSION}_linux_amd64.zip
sudo mv tendermint /usr/local/bin
}
# Find out if a process exists by partial name.
# is_running name
function is_running {
local name=$1
ps auxw | grep -v grep | grep ${name} > /dev/null
local exitcode=$?
return $exitcode
}
# Restore xtrace
$_XTRACE_FUNCTIONS_COMMON

View File

@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -o nounset
set -o errexit
function usage
{
cat << EOM
Usage: $ bash ${0##*/} [-v] [-h]
Installs the BigchainDB devstack or network.
ENV[STACK]
Set STACK environment variable to Either 'devstack' or 'network'.
Network mimics a production network environment with multiple BDB
nodes, whereas devstack is useful if you plan on modifying the
bigchaindb code.
ENV[GIT_BRANCH]
To configure bigchaindb repo branch to use set GIT_BRANCH environment
variable
ENV[TM_VERSION]
Tendermint version to use for the devstack setup
ENV[MONGO_VERSION]
MongoDB version to use with the devstack setup
-v
Verbose output from ansible playbooks.
-h
Show this help and exit.
EOM
}
# GIT_BRANCH
git_branch=$GIT_BRANCH
while getopts "h" opt; do
case "$opt" in
h)
usage
exit
;;
*)
usage
exit 1
;;
esac
done
if [[ ! $git_branch ]]; then
echo "You must specify GIT_BRANCH before running."
echo
echo usage
exit 1
fi
mkdir -p logs
log_file=logs/install-$(date +%Y%m%d-%H%M%S).log
exec > >(tee $log_file) 2>&1
echo "Capturing output to $log_file"
echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
function finish {
echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
}
trap finish EXIT
export GIT_BRANCH=$git_branch
echo "Using bigchaindb branch '$GIT_BRANCH'"
git clone https://github.com/bigchaindb/bigchaindb.git -b $GIT_BRANCH || true
curl -fOL# https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/Vagrantfile
vagrant up --provider virtualbox
echo -e "Finished installing! You may now log in using 'vagrant ssh'"
echo -e "Once inside the VM do 'tmux attach' to attach to tmux session running all services"

138
pkg/scripts/stack.sh Normal file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env bash
# ``stack.sh`` is an opinionated BigchainDB developer installation. It
# installs and configures **BigchainDb Server**, **Tendermint Server**,
# **MongoDB**
# Print the commands being run so that we can see the command that triggers
# an error. It is also useful for following along as the install occurs.
set -o xtrace
# Make sure umask is sane
umask 022
# Keep track of the stack.sh directory
TOP_DIR=$(cd $(dirname "$0") && pwd)
BASE_DIR=${TOP_DIR}/../..
# Check for uninitialized variables, a big cause of bugs
NOUNSET=${NOUNSET:-}
if [[ -n "$NOUNSET" ]]; then
set -o nounset
fi
# Set default MongoDB version
if [[ "$MONGO_VERSION" = "" ]]; then
MONGO_VERSION="3.4"
fi
# Set default tendermint version
if [[ "$TM_VERSION" = "" ]]; then
TM_VERSION="0.12.1"
fi
# Configuration
# =============
# Source utility functions
source ${TOP_DIR}/functions-common
# Configure Distro Repositories
# -----------------------------
# For Debian/Ubuntu make apt attempt to retry network ops on it's own and mongodb pub key
# source repo
if is_ubuntu; then
echo 'APT::Acquire::Retries "20";' | sudo tee /etc/apt/apt.conf.d/80retry >/dev/null
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/${MONGO_VERSION} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list
fi
# Ensure required packages are installed
# --------------------------------------
is_package_installed python3 || install_package python3
is_package_installed python3-pip || install_package python3-pip
is_package_installed libffi-dev || install_package libffi-dev
is_package_installed libssl-dev || install_package libssl-dev
is_package_installed tmux || install_package tmux
is_package_installed mongodb-org || install_package mongodb-org
is_package_installed unzip || install_package unzip
install_tendermint_bin
# Clean system if re-running the script
OIFS=$IFS
IFS=':'
session_str=$(tmux ls | grep -w bdb-dev)
if [[ $session_str = "" ]]; then
continue
else
session=($session_str)
tmux kill-session -t ${session[0]}
fi
# Stop bigchaindb service
if is_running "bigchaindb"; then
sudo pkill bigchaindb
fi
# Stop tendermint service
if is_running "tendermint"; then
sudo pkill tendermint
fi
# Stop mongodb service
if is_running "monogod"; then
sudo pkill mongod
fi
sleep 5
# Create data dir for mongod
if [[ ! -d /data/db ]]; then
sudo mkdir -p /data/db
fi
sudo chmod -R 700 /data/db
# Configure tendermint
tendermint init
# Configure tmux
cd ${BASE_DIR}
tmux new-session -s bdb-dev -n bdb -d
tmux new-window -n mdb
tmux new-window -n tendermint
# Start MongoDB
tmux send-keys -t bdb-dev:mdb 'sudo mongod --replSet=bigchain-rs' C-m
# Start BigchainDB
tmux send-keys -t bdb-dev:bdb 'sudo python3 setup.py install && bigchaindb -y configure mongodb && bigchaindb -l DEBUG start' C-m
while ! is_running "bigchaindb"; do
echo "Waiting bigchaindb service to start"
sleep 5
done
# Start tendermint service
tmux send-key -t bdb-dev:tendermint 'tendermint init && tendermint unsafe_reset_all && tendermint node' C-m
# Configure Error Traps
# ---------------------
# Kill background processes on exit
trap exit_trap EXIT
function exit_trap {
exit $?
}
# Exit on any errors so that errors don't compound and kill if any services already started
trap err_trap ERR
function err_trap {
local r=$?
tmux kill-session bdb-dev
set +o xtrace
exit $?
}
# Begin trapping error exit codes
set -o errexit