Mirroristas/bigchaindb
2
0
Fork 0
mirror of https://github.com/bigchaindb/bigchaindb.git synced 2024-10-13 13:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Problem: No automated way to set up a dev/test network (#2300)

Browse Source 
* Problem: No automated way to set up a dev/test network

* Problem: docs not updated for stack, ansible and docker based dev environments

* Problem: Using apt triggers java runtime installation on MacOS

- Update pre_tasks in ansible
- Update bigchaindb-stop playbook to handle pre_tasks

* Update Tendermint version to 0.19.3 from 0.19.2

* Update tendermint version to 0.19.3 everywhere else

* Problem: Sphinx warns about duplicate section labels

Solution: Don't use sphinx.ext.autosectionlabel

* Problem: Sphinx complains that run-node-with-docker not in any TOC

Solution: Add run-node-with-docker to a TOC (i.e. to an index.rst file)

* Tendermint has not tagged `0.19.3` container

* Problem: Internal hyperlinks to new pages not working

Solution: Add .html to the ends of the filename strings

* Problem: Invalid script imports

* Problem: Invalid comparison for supported OS version

* Addressing comments I

* Problem: No way to configure dev/forked repo for developers

* Problem: Docs not updated with STACK_REPO parameter

* Addressing comments II
This commit is contained in:
Muawia Khan
2018-05-31 16:56:45 +02:00
committed by GitHub
parent 0f56e4372d
commit 10e55aa4e7
86 changed files with 2235 additions and 1508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
pkg/scripts/Vagrantfile vendored
View File

@@ -1,61 +1,102 @@
Vagrant.require_version ">= 1.8.7"
unless Vagrant.has_plugin?("vagrant-vbguest")
raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
end
unless Vagrant.has_plugin?("vagrant-cachier")
raise "Please install the vagrant-cachier plugin by running `vagrant plugin install vagrant-cachier`"
end
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Required modules
require 'yaml'
VAGRANTFILE_API_VERSION = "2"
MEMORY = 4096
CPU_COUNT = 2
Vagrant.require_version ">= 1.8.7"
# Validate if all the required plugins are present
# vagrant-hostmanager replaced
required_plugins = ["vagrant-cachier", "vagrant-vbguest", "vagrant-hosts", "vagrant-azure"]
required_plugins.each do |plugin|
if not Vagrant.has_plugin?(plugin)
raise "Required vagrant plugin #{plugin} not found. Please run `vagrant plugin install #{plugin}`"
end
end
MOUNT_DIRS = {
:bigchaindb => {:repo => "bigchaindb", :local => "/opt/stack/bigchaindb", :owner => "edxapp"},
}
# Configuration files
CONFIGURATION_FILE = 'bigchaindb/pkg/configuration/vars/stack-config.yml'
HOSTS_FILE = 'bigchaindb/pkg/configuration/hosts/all'
HOST_VARS_PATH = 'bigchaindb/pkg/configuration/host_vars'
boxname = ENV['BOXNAME'] || "ubuntu/xenial64"
tm_version = ENV['TM_VERSION']
# Read configuration file(s)
instances_config = YAML.load_file(File.join(File.dirname(__FILE__), CONFIGURATION_FILE))
hosts_config = File.open(HOSTS_FILE, 'w+')
$script = <<SCRIPT
if [ ! -d /opt/stack/bigchaindb/pkg/scripts ]; then
echo "Error: Base box is missing provisioning scripts." 1>&2
exit 1
fi
# Vars needed for VM configuration
if (instances_config["stack_type"] == "cloud" && instances_config["stack_type_provider"] == "azure")
box_name = "azure-dummy"
else
box_name = instances_config['stack_box_name']
end
bash /opt/stack/bigchaindb/pkg/scripts/stack.sh
# configure instance names and private ip addresses
SCRIPT
instances_arr = Array.new
private_ipam_arr = Array.new
if instances_config['stack_type'] == "local"
for i in 1..Integer(instances_config['stack_size'])
instance_name = "bdb-node-#{i}"
instance_ip_address = "10.20.30.#{i+10}"
instances_arr.push instance_name
private_ipam_arr.push instance_ip_address
hosts_config.puts("#{instance_ip_address} ansible_user=vagrant")
File.open("#{HOST_VARS_PATH}/#{instance_ip_address}", "w+") {|f| \
f.write("ansible_ssh_private_key_file: .vagrant/machines/#{instance_name}/virtualbox/private_key") }
end
elsif
instance_name = instances_config["azure_dns_prefix"] + "." + instances_config["azure_region"] + ".cloudapp.azure.com"
instances_arr.push(instances_config["azure_dns_prefix"])
hosts_config.puts("#{instance_name} ansible_user=#{instances_config["azure_admin_username"]}")
File.open("#{HOST_VARS_PATH}/#{instance_name}", "w+") {|f| \
f.write("ansible_ssh_private_key_file: #{instances_config["ssh_private_key_path"]}") }
end
hosts_config.close
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "ubuntu/xenial64"
config.vm.box_check_update = false
config.vm.network :private_network, ip: "192.168.33.10"
config.vm.network :forwarded_port, guest: 9984, host: 9984 # BDB
config.ssh.insert_key = true
config.vm.synced_folder "bigchaindb", "/opt/stack/bigchaindb"
config.vm.provider :virtualbox do |vb|
vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
instances_arr.each_with_index do |instance, index|
config.vm.define "#{instance}" do |node|
node.vm.box = box_name
if instances_config["stack_type"] == "cloud"
node.ssh.private_key_path= instances_config["ssh_private_key_path"]
end
node.vm.box_check_update = false
# Workaround until vagrant cachier plugin supports dnf
if !(box_name.include? "fedora")
if Vagrant.has_plugin?("vagrant-cachier")
node.cache.scope = :box
end
elsif box_name == "ubuntu/xenial64"
if Vagrant.has_plugin?("vagrant-vbguest")
node.vbguest.auto_update = true
node.vbguest.auto_reboot = true
config.vbguest.no_install = true
config.vbguest.no_remote = true
end
end
node.vm.synced_folder "bigchaindb", "/opt/stack/bigchaindb"
node.vm.hostname = instance
node.vm.provision :hosts, :sync_hosts => true
node.ssh.insert_key = true
node.vm.network :private_network, ip: private_ipam_arr[index]
node.vm.provider :virtualbox do |vb, override|
vb.customize ["modifyvm", :id, "--memory", instances_config['stack_vm_memory'].to_s]
vb.customize ["modifyvm", :id, "--cpus", instances_config['stack_vm_cpus'].to_s]
end
node.vm.provider :azure do |azure, override|
azure.tenant_id = ENV['AZURE_TENANT_ID']
azure.client_id = ENV['AZURE_CLIENT_ID']
azure.client_secret = ENV['AZURE_CLIENT_SECRET']
azure.subscription_id = ENV['AZURE_SUBSCRIPTION_ID']
azure.admin_username = instances_config["azure_admin_username"]
azure.dns_name = instances_config["azure_dns_prefix"]
azure.vm_name = instances_config["azure_dns_prefix"]
azure.vm_size = instances_config["azure_vm_size"]
azure.vm_image_urn = instances_config["azure_image_urn"]
azure.resource_group_name = instances_config["azure_resource_group"]
azure.location = instances_config["azure_region"]
end
end
end
# Use vagrant-vbguest plugin to make sure Guest Additions are in sync
config.vbguest.auto_reboot = true
config.vbguest.auto_update = true
config.vm.provision "shell", inline: $script,
privileged: false,
env: {
:TM_VERSION => ENV['TM_VERSION'],
:MONGO_VERSION => ENV['MONGO_VERSION']
}
end
end

74
pkg/scripts/bootstrap.sh
View File

@@ -1,25 +1,24 @@
#!/bin/bash
set -e
#!/usr/bin/env bash
. ./bootstrap_constants.sh
. ./bootstrap_helper.sh
BASEDIR="${BASH_SOURCE%/*}"
if [[ ! -d "$BASEDIR" ]]; then BASEDIR="$PWD"; fi
. "$BASEDIR/bootstrap_constants.sh"
. "$BASEDIR/bootstrap_helper.sh"
# OS ID(ubuntu, centos, fedora)
# OS ID(centos, debian, fedora, ubuntu)
OS=""
# OS Version(16.04, 7, 24)
# OS Version(7, 9, 24, 16.04)
VER=""
# OP (install, uninstall)
OPERATION=${OPERATION:=""}
# Parsing arguments
while [[ $# -gt 1 ]]; do
arg="$1"
case $arg in
--os)
OS="$2"
shift
;;
--os-version)
VER="$2"
shift
--operation)
OPERATION="$2"
shift
;;
*)
echo "Unknown option: $1"
@@ -29,6 +28,11 @@ while [[ $# -gt 1 ]]; do
shift
done
# sanity checks
if [[ -z "${OPERATION:?Missing '--operation' [install,uninstall])}" ]] ; then
exit 1
fi
validate_os_configuration(){
valid_os=1
if [ -f $1 ]; then
@@ -38,25 +42,55 @@ validate_os_configuration(){
elif type lsb_release >/dev/null 2>&1; then
OS=$(lsb_release -si)
VER=$(lsb_release -sr)
elif [ "$(uname -s)" == "Darwin" ]; then
echo "Using macOS"
OS="macOS"
VER="None"
valid_os=True
return
else
echo "Cannot find $OS_CONF. Pass arguments to your OS configurations: NAME, VERSION_ID.
Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]."
exit 1
fi
for os in "${SUPPORTED_OS[@]}"; do
if [[ $os = $2 ]]; then
if [[ $os = $OS ]]; then
valid_os=true
break
fi
done
}
validate_os_configuration $OS_CONF $OS $VER
validate_os_configuration $OS_CONF
echo "Operation Sytem: $OS"
echo "Version: $VER"
install_deps=$(validate_os_version_and_deps true $OS $VER)
if [[ $install_deps -eq 1 ]]; then
install_dependencies $OS
# Installing dependencies
if [ "$OPERATION" = "install" ]; then
install_deps=$(validate_os_version_and_deps true $OS $VER)
if [[ $install_deps -eq 1 ]]; then
for dep in "${OS_DEPENDENCIES[@]}"
do
install_"$dep" $OS
done
elif [[ $install_deps -eq 2 ]]; then
echo "Unsupported $OS Version: $VER"
else
echo "Dependencies already installed:[ ${OS_DEPENDENCIES[*]} ]"
fi
# Uninstalling dependencies
elif [ "$OPERATION" = "uninstall" ]; then
uninstall_deps=$(validate_os_version_and_deps true $OS $VER)
if [[ $install_deps -eq 1 ]]; then
echo "Dependencies already uninstalled:[ ${OS_DEPENDENCIES[*]} ]"
elif [[ $install_deps -eq 2 ]]; then
echo "Unsupported $OS Version: $VER"
else
for dep in "${OS_DEPENDENCIES[@]}"
do
uninstall_"$dep" $OS
done
fi
else
echo "Dependencies already installed:[ ${OS_DEPENDENCIES[*]} ]"
fi
echo "Invalid Operation specified. Only [install, uninstall] are supported."
exit 1
fi

7
pkg/scripts/bootstrap_constants.sh
View File

@@ -1,7 +1,8 @@
#!/bin/bash
#!/usr/bin/env bash
OS_CONF=/etc/os-release
declare -a SUPPORTED_OS=('ubuntu' 'centos' 'fedora')
declare -a SUPPORTED_OS=('centos' 'fedora' 'ubuntu' 'debian' 'macOS')
declare -a OS_DEPENDENCIES=('ansible')
MINIMUM_UBUNTU_VERSION=16.04
MINIUMUM_CENTOS_VERSION=7
MINIMIUM_FEDORA_VERSION=24
MINIMIUM_FEDORA_VERSION=24
MINIMUM_DEBIAN_VERSION=8

161
pkg/scripts/bootstrap_helper.sh
View File

@@ -1,43 +1,52 @@
#!/bin/bash
#!/usr/bin/env bash
. ./bootstrap_constants.sh
BASEDIR="${BASH_SOURCE%/*}"
if [[ ! -d "$BASEDIR" ]]; then BASEDIR="$PWD"; fi
. "$BASEDIR/bootstrap_constants.sh"
validate_os_version_and_deps(){
if $1; then
case $2 in
ubuntu)
apt-get install bc -y > /dev/null 2>&1
if [[ ($(echo $3 | bc) > $MINIMUM_UBUNTU_VERSION)
|| ($(echo $3 | bc) == $MINIMUM_UBUNTU_VERSION)]]; then
centos)
if [[ ($(version_compare_gt $3 $MINIMUM_CENTOS_VERSION) == 0)
|| ($(version_compare_eq $3 $MINIMUM_CENTOS_VERSION) == 0) ]]; then
rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1
echo $?
else
echo 2
fi
;;
debian)
if [[ ($(version_compare_gt $3 $MINIMUM_DEBIAN_VERSION) == 0)
|| ($(version_compare_eq $3 $MINIMUM_DEBIAN_VERSION) == 0) ]]; then
dpkg -s "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1
echo $?
else
echo "Supported $2 Versions: >= $MINIMUM_UBUNTU_VERSION"
exit 1
fi
;;
centos)
yum install bc -y > /dev/null 2>&1
if [[ ($(echo $3 | bc) > $MINIMUM_CENTOS_VERSION)
|| ($(echo $3 | bc) == $MINIMUM_CENTOS_VERSION) ]]; then
rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1
echo $?
else
echo "Supported $2 Versions: >= $MINIMUM_CENTOS_VERSION"
exit 1
echo 2
fi
;;
fedora)
dnf install bc python2-dnf libselinux-python -y > /dev/null 2>&1
if [[ ($(echo $3 | bc) > $MINIMUM_FEDORA_VERSION)
|| ($(echo $3 | bc) == $MINIMUM_FEDORA_VERSION) ]]; then
if [[ ($(version_compare_gt $3 $MINIMUM_FEDORA_VERSION) == 0)
|| ($(version_compare_eq $3 $MINIMUM_FEDORA_VERSION) == 0) ]]; then
rpm -q "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1
echo $?
else
echo "Supported $2 Versions: >= $MINIMUM_FEDORA_VERSION"
exit 1
echo 2
fi
;;
ubuntu)
if [[ ($(version_compare_gt $3 $MINIMUM_UBUNTU_VERSION) == 0)
|| ($(version_compare_eq $3 $MINIMUM_UBUNTU_VERSION) == 0) ]]; then
dpkg -s "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1
echo $?
else
echo 2
fi
;;
macOS)
pip show "${OS_DEPENDENCIES[@]}" > /dev/null 2>&1
echo $?
;;
*)
echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]."
exit 1
@@ -45,48 +54,78 @@ validate_os_version_and_deps(){
esac
else
echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]."
exit 1
fi
}
install_dependencies() {
version_compare_gt(){
test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
echo $?
}
version_compare_eq(){
test "$(printf '%s\n' "$@" | sort -V | head -n 1)" == "$2"
echo $?
}
install_ansible() {
echo "Installing Ansible..."
case $1 in
ubuntu)
install_deps_deb
;;
centos)
install_deps_centos
;;
fedora)
install_deps_fedora
;;
*)
echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]."
exit 1
;;
centos)
yum install epel-release -y
yum install -y https://centos7.iuscommunity.org/ius-release.rpm
yum install ansible -y
;;
debian)
apt-get update -y && apt-get install --fix-missing
apt-get install lsb-release software-properties-common gnupg -y
echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/ansible-debian.list
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
apt-get update
apt-get install -y ansible
echo 'localhost' > /etc/ansible/hosts
;;
fedora)
export LC_ALL=C
dnf makecache
dnf -y install ansible
;;
macOS)
easy_install pip
pip install ansible
;;
ubuntu)
apt-get update -y
apt-get install -y software-properties-common
apt-add-repository ppa:ansible/ansible -y
apt-get update -y
apt-get install -y ansible
;;
*)
echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]."
esac
}
#TODO: muawiakh(Currently only ansible is required. Make it generic for
# multiple dependencies)
install_deps_deb() {
echo "Installing Dependencies..."
apt-get update -y
apt-get install -y software-properties-common
apt-add-repository ppa:ansible/ansible
apt-get update -y
apt-get install -y "${OS_DEPENDENCIES[@]}"
}
install_deps_centos() {
echo "Installing Dependencies..."
yum install epel-release -y
yum install -y https://centos7.iuscommunity.org/ius-release.rpm
yum install "${OS_DEPENDENCIES[@]}" -y
}
install_deps_fedora() {
echo "Installing Dependencies..."
export LC_ALL=C
dnf makecache
echo "${OS_DEPENDENCIES[@]}"
dnf -y install "${OS_DEPENDENCIES[@]}"
uninstall_ansible() {
echo "Uninstalling Ansible..."
case $1 in
centos)
yum remove ansible -y
;;
debian)
apt-get purge ansible -y
;;
fedora)
export LC_ALL=C
dnf remove ansible -y
;;
macOS)
pip uninstall ansible -y
;;
ubuntu)
apt-get purge ansible -y
;;
*)
echo "Supported OS(s) are: [ ${SUPPORTED_OS[*]} ]."
esac
}

38
pkg/scripts/functions-common Normal file → Executable file
View File

@@ -22,7 +22,7 @@ set +o xtrace
# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
# ``os_CODENAME`` - vendor's codename for release: ``xenial``
declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME
#declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME
# Make a *best effort* attempt to install lsb_release packages for the
# user if not available. Note can't use generic install_package*
@@ -74,7 +74,7 @@ function GetOSVersion {
# Translate the OS version values into common nomenclature
# Sets global ``DISTRO`` from the ``os_*`` values
declare -g DISTRO
#declare -g DISTRO
function GetDistro {
GetOSVersion
@@ -104,25 +104,6 @@ function GetDistro {
elif [[ "$os_VENDOR" =~ (kvmibm) ]]; then
DISTRO="${os_VENDOR}${os_RELEASE::1}"
else
# We can't make a good choice here. Setting a sensible DISTRO
# is part of the problem, but not the major issue -- we really
# only use DISTRO in the code as a fine-filter.
#
# The bigger problem is categorising the system into one of
# our two big categories as Ubuntu/Debian-ish or
# Fedora/CentOS-ish.
#
# The setting of os_PACKAGE above is only set to "deb" based
# on a hard-coded list of vendor names ... thus we will
# default to thinking unknown distros are RPM based
# (ie. is_ubuntu does not match). But the platform will then
# also not match in is_fedora, because that also has a list of
# names.
#
# So, if you are reading this, getting your distro supported
# is really about making sure it matches correctly in these
# functions. Then you can choose a sensible way to construct
# DISTRO based on your distros release approach.
die $LINENO "Unable to determine DISTRO, can not continue."
fi
typeset -xr DISTRO
@@ -134,16 +115,6 @@ function is_arch {
[[ "$(uname -m)" == "$1" ]]
}
# Determine if current distribution is an Oracle distribution
# is_oraclelinux
function is_oraclelinux {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "OracleServer" ]
}
# Determine if current distribution is a Fedora-based distribution
# (Fedora, RHEL, CentOS, etc).
@@ -155,8 +126,7 @@ function is_fedora {
[ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
[ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
[ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \
[ "$os_VENDOR" = "Virtuozzo" ] || [ "$os_VENDOR" = "kvmibm" ]
[ "$os_VENDOR" = "CentOS" ]
}
@@ -387,4 +357,4 @@ function is_running {
}
# Restore xtrace
$_XTRACE_FUNCTIONS_COMMON
$_XTRACE_FUNCTIONS_COMMON

81
pkg/scripts/install_stack.sh
View File

@@ -1,81 +0,0 @@
#!/usr/bin/env bash
set -o nounset
set -o errexit
function usage
{
cat << EOM
Usage: $ bash ${0##*/} [-v] [-h]
Installs the BigchainDB devstack or network.
ENV[STACK]
Set STACK environment variable to Either 'devstack' or 'network'.
Network mimics a production network environment with multiple BDB
nodes, whereas devstack is useful if you plan on modifying the
bigchaindb code.
ENV[GIT_BRANCH]
To configure bigchaindb repo branch to use set GIT_BRANCH environment
variable
ENV[TM_VERSION]
Tendermint version to use for the devstack setup
ENV[MONGO_VERSION]
MongoDB version to use with the devstack setup
-v
Verbose output from ansible playbooks.
-h
Show this help and exit.
EOM
}
# GIT_BRANCH
git_branch=$GIT_BRANCH
while getopts "h" opt; do
case "$opt" in
h)
usage
exit
;;
*)
usage
exit 1
;;
esac
done
if [[ ! $git_branch ]]; then
echo "You must specify GIT_BRANCH before running."
echo
echo usage
exit 1
fi
mkdir -p logs
log_file=logs/install-$(date +%Y%m%d-%H%M%S).log
exec > >(tee $log_file) 2>&1
echo "Capturing output to $log_file"
echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
function finish {
echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
}
trap finish EXIT
export GIT_BRANCH=$git_branch
echo "Using bigchaindb branch '$GIT_BRANCH'"
git clone https://github.com/bigchaindb/bigchaindb.git -b $GIT_BRANCH || true
curl -fOL# https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/Vagrantfile
vagrant up --provider virtualbox
echo -e "Finished installing! You may now log in using 'vagrant ssh'"
echo -e "Once inside the VM do 'tmux attach' to attach to tmux session running all services"

298
pkg/scripts/stack.sh Normal file → Executable file
View File

@@ -1,124 +1,242 @@
#!/usr/bin/env bash
# ``stack.sh`` is an opinionated BigchainDB developer installation. It
# installs and configures **BigchainDb Server**, **Tendermint Server**,
# **MongoDB**
# Print the commands being run so that we can see the command that triggers
# an error. It is also useful for following along as the install occurs.
set -o xtrace
set -o nounset
# Make sure umask is sane
umask 022
# Keep track of the stack.sh directory
TOP_DIR=$(cd $(dirname "$0") && pwd)
BASE_DIR=${TOP_DIR}/../..
# defaults
stack_branch=${STACK_BRANCH:="master"}
stack_repo=${STACK_REPO:="bigchaindb/bigchaindb"}
stack_size=${STACK_SIZE:=4}
stack_type=${STACK_TYPE:="docker"}
stack_type_provider=${STACK_TYPE_PROVIDER:=""}
tm_version=${TM_VERSION:="0.19.2"}
mongo_version=${MONGO_VERSION:="3.6"}
stack_vm_memory=${STACK_VM_MEMORY:=2048}
stack_vm_cpus=${STACK_VM_CPUS:=2}
stack_box_name=${STACK_BOX_NAME:="ubuntu/xenial64"}
azure_subscription_id=${AZURE_SUBSCRIPTION_ID:=""}
azure_tenant_id=${AZURE_TENANT_ID:=""}
azure_client_secret=${AZURE_CLIENT_SECRET:=""}
azure_client_id=${AZURE_CLIENT_ID:=""}
azure_region=${AZURE_REGION:="westeurope"}
azure_image_urn=${AZURE_IMAGE_URN:="Canonical:UbuntuServer:16.04-LTS:latest"}
azure_resource_group=${AZURE_RESOURCE_GROUP:="bdb-vagrant-rg-$(date '+%Y-%m-%d')"}
azure_dns_prefix=${AZURE_DNS_PREFIX:="bdb-instance-$(date '+%Y-%m-%d')"}
azure_admin_username=${AZURE_ADMIN_USERNAME:="vagrant"}
azure_vm_size=${AZURE_VM_SIZE:="Standard_D2_v2"}
ssh_private_key_path=${SSH_PRIVATE_KEY_PATH:=""}
# Check for uninitialized variables, a big cause of bugs
# Check for uninitialized variables
NOUNSET=${NOUNSET:-}
if [[ -n "$NOUNSET" ]]; then
set -o nounset
set -o nounset
fi
# Set default MongoDB version
if [[ "$MONGO_VERSION" = "" ]]; then
MONGO_VERSION="3.4"
fi
TOP_DIR=$(cd $(dirname "$0") && pwd)
SCRIPTS_DIR=$TOP_DIR/bigchaindb/pkg/scripts
CONF_DIR=$TOP_DIR/bigchaindb/pkg/configuration
# Set default tendermint version
if [[ "$TM_VERSION" = "" ]]; then
TM_VERSION="0.12.1"
fi
# Configuration
# =============
function usage() {
cat <<EOM
# Source utility functions
source ${TOP_DIR}/functions-common
Usage: $ bash ${0##*/} [-h]
# Configure Distro Repositories
# -----------------------------
Deploys the BigchainDB network.
# For Debian/Ubuntu make apt attempt to retry network ops on it's own and mongodb pub key
# source repo
if is_ubuntu; then
echo 'APT::Acquire::Retries "20";' | sudo tee /etc/apt/apt.conf.d/80retry >/dev/null
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/${MONGO_VERSION} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list
fi
ENV[STACK_SIZE]
Set STACK_SIZE environment variable to the size of the network you desire.
Network mimics a production network environment with single or multiple BDB
nodes. (default: ${stack_size}).
# Ensure required packages are installed
# --------------------------------------
ENV[STACK_TYPE]
Set STACK_TYPE environment variable to the type of deployment you desire.
You can set it one of the following: ["docker", "local", "cloud"].
(default: ${stack_type})
is_package_installed python3 || install_package python3
is_package_installed python3-pip || install_package python3-pip
is_package_installed libffi-dev || install_package libffi-dev
is_package_installed libssl-dev || install_package libssl-dev
is_package_installed tmux || install_package tmux
is_package_installed mongodb-org || install_package mongodb-org
is_package_installed unzip || install_package unzip
install_tendermint_bin
ENV[STACK_TYPE_PROVIDER]
Set only when STACK_TYPE="cloud". Only "azure" is supported.
(default: ${stack_type_provider})
# Clean system if re-running the script
OIFS=$IFS
IFS=':'
session_str=$(tmux ls | grep -w bdb-dev)
if [[ $session_str = "" ]]; then
continue
else
session=($session_str)
tmux kill-session -t ${session[0]}
fi
ENV[STACK_VM_MEMORY]
(Optional) Set only when STACK_TYPE="local". This sets the memory
of the instance(s) spawned. (default: ${stack_vm_memory})
# Stop bigchaindb service
if is_running "bigchaindb"; then
sudo pkill bigchaindb
fi
ENV[STACK_VM_CPUS]
(Optional) Set only when STACK_TYPE="local". This sets the number of VCPUs
of the instance(s) spawned. (default: ${stack_vm_cpus})
# Stop tendermint service
if is_running "tendermint"; then
sudo pkill tendermint
fi
ENV[STACK_BOX_NAME]
(Optional) Set only when STACK_TYPE="local". This sets the box Vagrant box name
of the instance(s) spawned. (default: ${stack_box_name})
# Stop mongodb service
if is_running "monogod"; then
sudo pkill mongod
fi
ENV[STACK_REPO]
(Optional) To configure bigchaindb repo to use, set STACK_REPO environment
variable. (default: ${stack_repo})
sleep 5
ENV[STACK_BRANCH]
(Optional) To configure bigchaindb repo branch to use set STACK_BRANCH environment
variable. (default: ${stack_branch})
# Create data dir for mongod
if [[ ! -d /data/db ]]; then
sudo mkdir -p /data/db
fi
sudo chmod -R 700 /data/db
ENV[TM_VERSION]
(Optional) Tendermint version to use for the setup. (default: ${tm_version})
# Configure tendermint
tendermint init
ENV[MONGO_VERSION]
(Optional) MongoDB version to use with the setup. (default: ${mongo_version})
# Configure tmux
cd ${BASE_DIR}
tmux new-session -s bdb-dev -n bdb -d
tmux new-window -n mdb
tmux new-window -n tendermint
ENV[AZURE_CLIENT_ID]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
# Start MongoDB
tmux send-keys -t bdb-dev:mdb 'sudo mongod --replSet=bigchain-rs' C-m
ENV[AZURE_TENANT_ID]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
# Start BigchainDB
tmux send-keys -t bdb-dev:bdb 'sudo python3 setup.py install && bigchaindb -y configure mongodb && bigchaindb -l DEBUG start' C-m
ENV[AZURE_SUBSCRIPTION_ID]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
while ! is_running "bigchaindb"; do
echo "Waiting bigchaindb service to start"
sleep 5
ENV[AZURE_CLIENT_SECRET]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
ENV[AZURE_REGION]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Azure region for the BigchainDB instance. Get list of regions using Azure CLI.
e.g. az account list-locations. (default: ${azure_region})
ENV[AZURE_IMAGE_URN]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Azure image to use. Get list of available images using Azure CLI.
e.g. az vm image list --output table. (default: ${azure_image_urn})
ENV[AZURE_RESOURCE_GROUP]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Name of Azure resource group for the instance.
(default: ${azure_resource_group})
ENV[AZURE_DNS_PREFIX]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
DNS Prefix of the instance. (default: ${azure_dns_prefix})
ENV[AZURE_ADMIN_USERNAME]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Admin username of the the instance. (default: ${azure_admin_username})
ENV[AZURE_VM_SIZE]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Azure VM size. (default: ${azure_vm_size})
ENV[SSH_PRIVATE_KEY_PATH]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Absolute path of
SSH keypair required to log into the Azure instance.
-h
Show this help and exit.
EOM
}
while getopts "h" opt; do
case "$opt" in
h)
usage
exit
;;
*)
usage
exit 1
;;
esac
done
# Start tendermint service
tmux send-key -t bdb-dev:tendermint 'tendermint init && tendermint unsafe_reset_all && tendermint node' C-m
mkdir -p logs
log_file=logs/install-$(date +%Y%m%d-%H%M%S).log
exec > >(tee $log_file) 2>&1
echo "Capturing output to $log_file"
echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
# Configure Error Traps
# ---------------------
function finish() {
echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
}
trap finish EXIT
export STACK_REPO=$stack_repo
export STACK_BRANCH=$stack_branch
echo "Using bigchaindb repo: '$STACK_REPO'"
echo "Using bigchaindb branch '$STACK_BRANCH'"
git clone https://github.com/${stack_repo}.git -b $stack_branch || true
# Source utility functions
source ${SCRIPTS_DIR}/functions-common
if [[ $stack_type == "local" ]]; then
mongo_version=$(echo "$mongo_version" | cut -d. -f-2)
fi
# configure stack-config.yml
cat >$TOP_DIR/bigchaindb/pkg/configuration/vars/stack-config.yml <<EOF
---
stack_type: "${stack_type}"
stack_size: "${stack_size}"
stack_type_provider: "${stack_type_provider}"
stack_box_name: "${stack_box_name}"
stack_vm_memory: "${stack_vm_memory}"
stack_vm_cpus: "${stack_vm_cpus}"
tm_version: "${tm_version}"
mongo_version: "${mongo_version}"
azure_region: "${azure_region}"
azure_image_urn: "${azure_image_urn}"
azure_resource_group: "${azure_resource_group}"
azure_dns_prefix: "${azure_dns_prefix}"
azure_admin_username: "${azure_admin_username}"
azure_vm_size: "${azure_vm_size}"
ssh_private_key_path: "${ssh_private_key_path}"
EOF
curl -fOL# https://raw.githubusercontent.com/${stack_repo}/${stack_branch}/pkg/scripts/Vagrantfile
#Convert to lowercase
stack_type="$(echo $stack_type | tr '[A-Z]' '[a-z]')"
stack_type_provider="$(echo $stack_type_provider | tr '[A-Z]' '[a-z]')"
if [[ $stack_type == "local" ]]; then
echo "Configuring setup locally!"
vagrant up --provider virtualbox --provision
ansible-playbook $CONF_DIR/bigchaindb-start.yml \
-i $CONF_DIR/hosts/all \
--extra-vars "operation=start home_path=${TOP_DIR}"
elif [[ $stack_type == "cloud" && $stack_type_provider == "azure" ]]; then
echo ${azure_tenant_id:?AZURE_TENANT_ID not set! Exiting. $(exit 1)}
echo ${azure_client_secret:?AZURE_CLIENT_SECRET not set! Exiting. $(exit 1)}
echo ${azure_client_id:?AZURE_CLIENT_ID not set! Exiting. $(exit 1)}
echo ${azure_subscription_id:?AZURE_SUBSCRIPTION_ID not set! Exiting. $(exit 1)}
echo ${ssh_private_key_path:?SSH_PRIVATE_KEY_PATH not set! $(exit 1)}
echo "Configuring Setup on Azure!"
# Dummy box does not really do anything because we are relying on Azure VM images
vagrant box add azure-dummy https://github.com/azure/vagrant-azure/raw/v2.0/dummy.box \
--provider azure --force
vagrant up --provider azure --provision
ansible-playbook $CONF_DIR/bigchaindb-start.yml \
-i $CONF_DIR/hosts/all \
--extra-vars "operation=start home_path=/bigchaindb"
elif [[ $stack_type == "docker" ]]; then
echo "Configuring Dockers locally!"
source $SCRIPTS_DIR/bootstrap.sh --operation install
cat >$CONF_DIR/hosts/all <<EOF
$(hostname) ansible_connection=local
EOF
ansible-playbook $CONF_DIR/bigchaindb-start.yml \
-i $CONF_DIR/hosts/all \
--extra-vars "operation=start home_path=${TOP_DIR}"
else
echo "Invalid Stack Type OR Provider"
exit 1
fi
# Kill background processes on exit
trap exit_trap EXIT
@@ -134,5 +252,5 @@ function err_trap {
exit $?
}
# Begin trapping error exit codes
echo -e "Finished stacking!"
set -o errexit

260
pkg/scripts/unstack.sh Executable file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env bash
set -o nounset
# Make sure umask is sane
umask 022
# defaults
stack_branch=${STACK_BRANCH:="master"}
stack_repo=${STACK_REPO:="bigchaindb/bigchaindb"}
stack_size=${STACK_SIZE:=4}
stack_type=${STACK_TYPE:="docker"}
stack_type_provider=${STACK_TYPE_PROVIDER:=""}
tm_version=${TM_VERSION:="0.19.2"}
mongo_version=${MONGO_VERSION:="3.6"}
stack_vm_memory=${STACK_VM_MEMORY:=2048}
stack_vm_cpus=${STACK_VM_CPUS:=2}
stack_box_name=${STACK_BOX_NAME:="ubuntu/xenial64"}
azure_subscription_id=${AZURE_SUBSCRIPTION_ID:=""}
azure_tenant_id=${AZURE_TENANT_ID:=""}
azure_client_secret=${AZURE_CLIENT_SECRET:=""}
azure_client_id=${AZURE_CLIENT_ID:=""}
azure_region=${AZURE_REGION:="westeurope"}
azure_image_urn=${AZURE_IMAGE_URN:="Canonical:UbuntuServer:16.04-LTS:latest"}
azure_resource_group=${AZURE_RESOURCE_GROUP:="bdb-vagrant-rg-$(date '+%Y-%m-%d')"}
azure_dns_prefix=${AZURE_DNS_PREFIX:="bdb-instance-$(date '+%Y-%m-%d')"}
azure_admin_username=${AZURE_ADMIN_USERNAME:="vagrant"}
azure_vm_size=${AZURE_VM_SIZE:="Standard_D2_v2"}
ssh_private_key_path=${SSH_PRIVATE_KEY_PATH:=""}
unstack_type=${UNSTACK_TYPE:="hard"}
# Check for uninitialized variables
NOUNSET=${NOUNSET:-}
if [[ -n "$NOUNSET" ]]; then
set -o nounset
fi
TOP_DIR=$(cd $(dirname "$0") && pwd)
SCRIPTS_DIR=$TOP_DIR/bigchaindb/pkg/scripts
CONF_DIR=$TOP_DIR/bigchaindb/pkg/configuration
function usage() {
cat <<EOM
Usage: $ bash ${0##*/} [-h]
Deploys the BigchainDB network.
ENV[STACK_SIZE]
Set STACK_SIZE environment variable to the size of the network you desire.
Network mimics a production network environment with single or multiple BDB
nodes. (default: ${stack_size}).
ENV[STACK_TYPE]
Set STACK_TYPE environment variable to the type of deployment you desire.
You can set it one of the following: ["docker", "local", "cloud"].
(default: ${stack_type})
ENV[STACK_TYPE_PROVIDER]
Set only when STACK_TYPE="cloud". Only "azure" is supported.
(default: ${stack_type_provider})
ENV[STACK_VM_MEMORY]
(Optional) Set only when STACK_TYPE="local". This sets the memory
of the instance(s) spawned. (default: ${stack_vm_memory})
ENV[STACK_VM_CPUS]
(Optional) Set only when STACK_TYPE="local". This sets the number of VCPUs
of the instance(s) spawned. (default: ${stack_vm_cpus})
ENV[STACK_BOX_NAME]
(Optional) Set only when STACK_TYPE="local". This sets the box Vagrant box name
of the instance(s) spawned. (default: ${stack_box_name})
ENV[STACK_REPO]
(Optional) To configure bigchaindb repo to use, set STACK_REPO environment
variable. (default: ${stack_repo})
ENV[STACK_BRANCH]
(Optional) To configure bigchaindb repo branch to use set STACK_BRANCH environment
variable. (default: ${stack_branch})
ENV[TM_VERSION]
(Optional) Tendermint version to use for the setup. (default: ${tm_version})
ENV[MONGO_VERSION]
(Optional) MongoDB version to use with the setup. (default: ${mongo_version})
ENV[AZURE_CLIENT_ID]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
ENV[AZURE_TENANT_ID]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
ENV[AZURE_SUBSCRIPTION_ID]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
ENV[AZURE_CLIENT_SECRET]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate:
https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application
ENV[AZURE_REGION]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Azure region for the BigchainDB instance. Get list of regions using Azure CLI.
e.g. az account list-locations. (default: ${azure_region})
ENV[AZURE_IMAGE_URN]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Azure image to use. Get list of available images using Azure CLI.
e.g. az vm image list --output table. (default: ${azure_image_urn})
ENV[AZURE_RESOURCE_GROUP]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Name of Azure resource group for the instance.
(default: ${azure_resource_group})
ENV[AZURE_DNS_PREFIX]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
DNS Prefix of the instance. (default: ${azure_dns_prefix})
ENV[AZURE_ADMIN_USERNAME]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Admin username of the the instance. (default: ${azure_admin_username})
ENV[AZURE_VM_SIZE]
(Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure".
Azure VM size. (default: ${azure_vm_size})
ENV[SSH_PRIVATE_KEY_PATH]
Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Absolute path of
SSH keypair required to log into the Azure instance.
ENV[UNSTACK_TYPE]
Hard or Soft unstack. (default ${unstack_type})
-s
Soft unstack, only stop the processes.
-h
Show this help and exit.
EOM
}
while getopts ":h:s" opt; do
case "$opt" in
h)
usage
exit
;;
s)
unstack_type="soft"
;;
*)
usage
exit 1
;;
esac
done
mkdir -p logs
log_file=logs/install-$(date +%Y%m%d-%H%M%S).log
exec > >(tee $log_file) 2>&1
echo "Capturing output to $log_file"
echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
function finish() {
echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
}
trap finish EXIT
export STACK_REPO=$stack_repo
export STACK_BRANCH=$stack_branch
echo "Using bigchaindb repo: '$STACK_REPO'"
echo "Using bigchaindb branch '$STACK_BRANCH'"
git clone https://github.com/${stack_repo}.git -b ${stack_branch} || true
# Source utility functions
source ${SCRIPTS_DIR}/functions-common
if [[ $stack_type == "local" ]]; then
mongo_version=$(echo "$mongo_version" | cut -d. -f-2)
fi
# configure stack-config.yml
cat >$TOP_DIR/bigchaindb/pkg/configuration/vars/stack-config.yml <<EOF
---
stack_type: "${stack_type}"
stack_size: "${stack_size}"
stack_type_provider: "${stack_type_provider}"
stack_box_name: "${stack_box_name}"
stack_vm_memory: "${stack_vm_memory}"
stack_vm_cpus: "${stack_vm_cpus}"
tm_version: "${tm_version}"
mongo_version: "${mongo_version}"
azure_region: "${azure_region}"
azure_image_urn: "${azure_image_urn}"
azure_resource_group: "${azure_resource_group}"
azure_dns_prefix: "${azure_dns_prefix}"
azure_admin_username: "${azure_admin_username}"
azure_vm_size: "${azure_vm_size}"
ssh_private_key_path: "${ssh_private_key_path}"
EOF
curl -fOL# https://raw.githubusercontent.com/${stack_repo}/${stack_branch}/pkg/scripts/Vagrantfile
#Convert to lowercase
stack_type="$(echo $stack_type | tr '[A-Z]' '[a-z]')"
stack_type_provider="$(echo $stack_type_provider | tr '[A-Z]' '[a-z]')"
if [[ $stack_type == "local" ]]; then
if [[ $unstack_type == "hard" ]]; then
vagrant destroy -f
elif [[ $unstack_type == "soft" ]]; then
ansible-playbook $CONF_DIR/bigchaindb-stop.yml -i $CONF_DIR/hosts/all \
--extra-vars "operation=stop home_path=${TOP_DIR}"
fi
elif [[ $stack_type == "cloud" && $stack_type_provider == "azure" ]]; then
echo "Configuring Setup on Azure!"
if [[ $unstack_type == "hard" ]]; then
vagrant destroy -f
elif [[ $unstack_type == "soft" ]]; then
ansible-playbook $CONF_DIR/bigchaindb-stop.yml -i $CONF_DIR/hosts/all \
--extra-vars "operation=stop home_path=${TOP_DIR}"
fi
elif [[ $stack_type == "docker" ]]; then
echo "Configuring Dockers locally!"
source $SCRIPTS_DIR/bootstrap.sh --operation install
cat > $CONF_DIR/hosts/all << EOF
$(hostname) ansible_connection=local
EOF
ansible-playbook $CONF_DIR/bigchaindb-stop.yml -i $CONF_DIR/hosts/all \
--extra-vars "operation=stop home_path=${TOP_DIR}"
else
echo "Invalid Stack Type OR Provider"
exit 1
fi
# Kill background processes on exit
trap exit_trap EXIT
function exit_trap {
exit $?
}
# Exit on any errors so that errors don't compound and kill if any services already started
trap err_trap ERR
function err_trap {
local r=$?
tmux kill-session bdb-dev
set +o xtrace
exit $?
}
echo -e "Finished unstacking!!"
set -o errexit