From 10e55aa4e70e05cf62bdd1f831a4dafd541df330 Mon Sep 17 00:00:00 2001 From: Muawia Khan Date: Thu, 31 May 2018 16:56:45 +0200 Subject: [PATCH] Problem: No automated way to set up a dev/test network (#2300) * Problem: No automated way to set up a dev/test network * Problem: docs not updated for stack, ansible and docker based dev environments * Problem: Using apt triggers java runtime installation on MacOS - Update pre_tasks in ansible - Update bigchaindb-stop playbook to handle pre_tasks * Update Tendermint version to 0.19.3 from 0.19.2 * Update tendermint version to 0.19.3 everywhere else * Problem: Sphinx warns about duplicate section labels Solution: Don't use sphinx.ext.autosectionlabel * Problem: Sphinx complains that run-node-with-docker not in any TOC Solution: Add run-node-with-docker to a TOC (i.e. to an index.rst file) * Tendermint has not tagged `0.19.3` container * Problem: Internal hyperlinks to new pages not working Solution: Add .html to the ends of the filename strings * Problem: Invalid script imports * Problem: Invalid comparison for supported OS version * Addressing comments I * Problem: No way to configure dev/forked repo for developers * Problem: Docs not updated with STACK_REPO parameter * Addressing comments II --- Dockerfile-alpine | 30 ++ docs/contributing/source/conf.py | 2 +- .../index.rst | 2 + .../run-dev-network-ansible.md} | 98 +++--- .../run-dev-network-stack.md | 317 ++++++++++++++++++ .../write-code.rst | 6 +- docs/server/source/appendices/index.rst | 3 - .../source/appendices/run-with-docker.md | 157 --------- .../source/appendices/run-with-vagrant.md | 170 ---------- pkg/Vagrantfile | 103 ------ pkg/configuration/bdb-deploy.yml | 12 - pkg/configuration/bigchaindb-start.yml | 26 ++ pkg/configuration/bigchaindb-stop.yml | 15 + pkg/configuration/multi_node.yml | 5 - pkg/configuration/pre_req.yml | 8 - .../roles/bigchaindb-driver/defaults/main.yml | 31 -- .../roles/bigchaindb-driver/tasks/centos.yml | 12 - .../roles/bigchaindb-driver/tasks/common.yml | 14 - .../roles/bigchaindb-driver/tasks/debian.yml | 8 - .../roles/bigchaindb-driver/tasks/fedora.yml | 7 - .../roles/bigchaindb-driver/tasks/main.yml | 12 - .../roles/bigchaindb/defaults/main.yml | 26 +- .../roles/bigchaindb/tasks/centos.yml | 7 +- .../roles/bigchaindb/tasks/common.yml | 64 ++-- .../roles/bigchaindb/tasks/debian.yml | 2 +- .../roles/bigchaindb/tasks/deploy_docker.yml | 48 --- .../roles/bigchaindb/tasks/fedora.yml | 2 +- .../roles/bigchaindb/tasks/main.yml | 27 +- .../roles/bigchaindb/tasks/start.yml | 57 ++++ .../roles/bigchaindb/tasks/stop.yml | 33 ++ .../roles/docker-compose/defaults/main.yml | 7 - .../roles/docker-compose/tasks/main.yml | 9 - .../roles/docker/defaults/main.yml | 5 +- .../roles/docker/tasks/debian.yml | 6 +- .../roles/docker/tasks/macos.yml | 24 ++ pkg/configuration/roles/docker/tasks/main.yml | 40 ++- .../roles/key-exchange/defaults/main.yml | 12 - .../roles/key-exchange/tasks/main.yml | 8 - .../tasks/pub_key_exchange_docker.yml | 29 -- .../tasks/pub_key_exchange_host.yml | 28 -- .../templates/exchange_keyring_docker.j2 | 18 - .../templates/exchange_keyring_host.j2 | 21 -- .../roles/mongodb/defaults/main.yml | 26 +- .../roles/mongodb/files/mongod.conf | 101 ------ .../roles/mongodb/tasks/centos.yml | 13 +- .../roles/mongodb/tasks/common.yml | 25 -- .../roles/mongodb/tasks/debian.yml | 30 +- .../roles/mongodb/tasks/deploy_docker.yml | 33 -- .../roles/mongodb/tasks/fedora.yml | 6 +- .../roles/mongodb/tasks/initiate_repl_set.yml | 6 - .../tasks/initiate_repl_set_docker.yml | 13 - .../mongodb/tasks/initiate_repl_set_host.yml | 20 -- .../roles/mongodb/tasks/main.yml | 38 +-- .../roles/mongodb/tasks/start.yml | 56 ++++ .../roles/mongodb/tasks/stop.yml | 41 +++ .../mongodb/templates/replSet_init_docker.j2 | 30 -- .../mongodb/templates/replSet_init_host.j2 | 7 - pkg/configuration/roles/py36/tasks/centos.yml | 21 ++ pkg/configuration/roles/py36/tasks/debian.yml | 25 ++ pkg/configuration/roles/py36/tasks/fedora.yml | 37 ++ pkg/configuration/roles/py36/tasks/main.yml | 11 + .../roles/py36/templates/install_pip36.j2 | 8 + .../roles/py36/templates/install_py36.j2 | 34 ++ .../roles/tendermint/defaults/main.yml | 27 ++ .../roles/tendermint/files/Dockerfile | 7 + .../tendermint/files/access_pub_key.conf | 10 + .../roles/tendermint/files/config.toml | 169 ++++++++++ .../roles/tendermint/files/genesis.json | 6 + .../roles/tendermint/tasks/centos.yml | 11 + .../roles/tendermint/tasks/common.yml | 51 +++ .../roles/tendermint/tasks/debian.yml | 12 + .../roles/tendermint/tasks/fedora.yml | 21 ++ .../roles/tendermint/tasks/main.yml | 19 ++ .../roles/tendermint/tasks/start.yml | 90 +++++ .../roles/tendermint/tasks/stop.yml | 57 ++++ .../tendermint/templates/start_tendermint.j2 | 98 ++++++ pkg/configuration/vars/bdb-config.yml | 13 - pkg/configuration/vars/stack-config.yml | 0 pkg/scripts/Vagrantfile | 141 +++++--- pkg/scripts/bootstrap.sh | 74 ++-- pkg/scripts/bootstrap_constants.sh | 7 +- pkg/scripts/bootstrap_helper.sh | 161 +++++---- pkg/scripts/functions-common | 38 +-- pkg/scripts/install_stack.sh | 81 ----- pkg/scripts/stack.sh | 298 +++++++++++----- pkg/scripts/unstack.sh | 260 ++++++++++++++ 86 files changed, 2235 insertions(+), 1508 deletions(-) create mode 100644 Dockerfile-alpine rename docs/{server/source/appendices/run-with-ansible.md => contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md} (56%) create mode 100644 docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-stack.md delete mode 100644 docs/server/source/appendices/run-with-docker.md delete mode 100644 docs/server/source/appendices/run-with-vagrant.md delete mode 100644 pkg/Vagrantfile delete mode 100644 pkg/configuration/bdb-deploy.yml create mode 100644 pkg/configuration/bigchaindb-start.yml create mode 100644 pkg/configuration/bigchaindb-stop.yml delete mode 100644 pkg/configuration/multi_node.yml delete mode 100644 pkg/configuration/pre_req.yml delete mode 100644 pkg/configuration/roles/bigchaindb-driver/defaults/main.yml delete mode 100644 pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml delete mode 100644 pkg/configuration/roles/bigchaindb-driver/tasks/common.yml delete mode 100644 pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml delete mode 100644 pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml delete mode 100644 pkg/configuration/roles/bigchaindb-driver/tasks/main.yml delete mode 100644 pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml create mode 100644 pkg/configuration/roles/bigchaindb/tasks/start.yml create mode 100644 pkg/configuration/roles/bigchaindb/tasks/stop.yml delete mode 100644 pkg/configuration/roles/docker-compose/defaults/main.yml delete mode 100644 pkg/configuration/roles/docker-compose/tasks/main.yml create mode 100644 pkg/configuration/roles/docker/tasks/macos.yml delete mode 100644 pkg/configuration/roles/key-exchange/defaults/main.yml delete mode 100644 pkg/configuration/roles/key-exchange/tasks/main.yml delete mode 100644 pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml delete mode 100644 pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml delete mode 100644 pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 delete mode 100644 pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 delete mode 100644 pkg/configuration/roles/mongodb/files/mongod.conf delete mode 100644 pkg/configuration/roles/mongodb/tasks/common.yml delete mode 100644 pkg/configuration/roles/mongodb/tasks/deploy_docker.yml delete mode 100644 pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml delete mode 100644 pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml delete mode 100644 pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml create mode 100644 pkg/configuration/roles/mongodb/tasks/start.yml create mode 100644 pkg/configuration/roles/mongodb/tasks/stop.yml delete mode 100644 pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 delete mode 100644 pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 create mode 100644 pkg/configuration/roles/py36/tasks/centos.yml create mode 100644 pkg/configuration/roles/py36/tasks/debian.yml create mode 100644 pkg/configuration/roles/py36/tasks/fedora.yml create mode 100644 pkg/configuration/roles/py36/tasks/main.yml create mode 100644 pkg/configuration/roles/py36/templates/install_pip36.j2 create mode 100644 pkg/configuration/roles/py36/templates/install_py36.j2 create mode 100644 pkg/configuration/roles/tendermint/defaults/main.yml create mode 100644 pkg/configuration/roles/tendermint/files/Dockerfile create mode 100644 pkg/configuration/roles/tendermint/files/access_pub_key.conf create mode 100644 pkg/configuration/roles/tendermint/files/config.toml create mode 100644 pkg/configuration/roles/tendermint/files/genesis.json create mode 100644 pkg/configuration/roles/tendermint/tasks/centos.yml create mode 100644 pkg/configuration/roles/tendermint/tasks/common.yml create mode 100644 pkg/configuration/roles/tendermint/tasks/debian.yml create mode 100644 pkg/configuration/roles/tendermint/tasks/fedora.yml create mode 100644 pkg/configuration/roles/tendermint/tasks/main.yml create mode 100644 pkg/configuration/roles/tendermint/tasks/start.yml create mode 100644 pkg/configuration/roles/tendermint/tasks/stop.yml create mode 100644 pkg/configuration/roles/tendermint/templates/start_tendermint.j2 delete mode 100644 pkg/configuration/vars/bdb-config.yml create mode 100644 pkg/configuration/vars/stack-config.yml mode change 100644 => 100755 pkg/scripts/functions-common delete mode 100644 pkg/scripts/install_stack.sh mode change 100644 => 100755 pkg/scripts/stack.sh create mode 100755 pkg/scripts/unstack.sh diff --git a/Dockerfile-alpine b/Dockerfile-alpine new file mode 100644 index 00000000..66c95f73 --- /dev/null +++ b/Dockerfile-alpine @@ -0,0 +1,30 @@ +FROM alpine:latest +LABEL maintainer "dev@bigchaindb.com" +RUN mkdir -p /usr/src/app +COPY . /usr/src/app/ +WORKDIR /usr/src/app +RUN apk --update add sudo \ + && apk --update add python3 openssl ca-certificates git\ + && apk --update add --virtual build-dependencies python3-dev \ + libffi-dev openssl-dev build-base \ + && apk add --no-cache libstdc++ \ + && pip3 install --upgrade pip cffi \ + && pip install --no-cache-dir --process-dependency-links -e .[dev] \ + && apk del build-dependencies \ + && rm -f /var/cache/apk/* +# When developing with Python in a docker container, we are using PYTHONBUFFERED +# to force stdin, stdout and stderr to be totally unbuffered and to capture logs/outputs +ENV PYTHONUNBUFFERED 0 + +ENV BIGCHAINDB_DATABASE_PORT 27017 +ENV BIGCHAINDB_DATABASE_BACKEND $backend +ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 +ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_SCHEME ws + +ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0 +ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws + +ENV BIGCHAINDB_TENDERMINT_PORT 46657 +ARG backend +RUN bigchaindb -y configure "$backend" \ No newline at end of file diff --git a/docs/contributing/source/conf.py b/docs/contributing/source/conf.py index f0ce5b2a..4e0dbed9 100644 --- a/docs/contributing/source/conf.py +++ b/docs/contributing/source/conf.py @@ -38,7 +38,7 @@ from recommonmark.parser import CommonMarkParser import sphinx_rtd_theme extensions = [ - 'sphinx.ext.autosectionlabel', +# 'sphinx.ext.autosectionlabel', ] # Get remote files to here. diff --git a/docs/contributing/source/dev-setup-coding-and-contribution-process/index.rst b/docs/contributing/source/dev-setup-coding-and-contribution-process/index.rst index 787753a9..9c96200b 100644 --- a/docs/contributing/source/dev-setup-coding-and-contribution-process/index.rst +++ b/docs/contributing/source/dev-setup-coding-and-contribution-process/index.rst @@ -7,3 +7,5 @@ Developer Setup, Coding & Contribution Process write-code run-node-with-docker-compose run-node-as-processes + run-dev-network-stack + run-dev-network-ansible diff --git a/docs/server/source/appendices/run-with-ansible.md b/docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md similarity index 56% rename from docs/server/source/appendices/run-with-ansible.md rename to docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md index b1ec6769..3bb9dc47 100644 --- a/docs/server/source/appendices/run-with-ansible.md +++ b/docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-ansible.md @@ -1,50 +1,50 @@ -# Run BigchainDB with Ansible +# Run a BigchainDB network with Ansible **NOT for Production Use** You can use the following instructions to deploy a single or multi node -BigchainDB setup for dev/test using Ansible. Ansible will setup BigchainDB node(s) along with -[Docker](https://www.docker.com/), [Docker Compose](https://docs.docker.com/compose/), -[MongoDB](https://www.mongodb.com/), [BigchainDB Python driver](https://docs.bigchaindb.com/projects/py-driver/en/latest/). +BigchainDB network for dev/test using Ansible. Ansible will configure the BigchainDB node(s). Currently, this workflow is only supported for the following distributions: - Ubuntu >= 16.04 - CentOS >= 7 - Fedora >= 24 +- MacOSX -## Minimum Requirements | Ansible +## Minimum Requirements Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: - Memory >= 512MB - VCPUs >= 1 -## Clone the BigchainDB repository | Ansible + +## Clone the BigchainDB repository ```text $ git clone https://github.com/bigchaindb/bigchaindb.git ``` -## Install dependencies | Ansible +## Install dependencies - [Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) You can also install `ansible` and other dependencies, if any, using the `boostrap.sh` script inside the BigchainDB repository. -Navigate to `bigchaindb/pkg/scripts` and run the `bootstrap.sh` script to install the dependecies +Navigate to `bigchaindb/pkg/scripts` and run the `bootstrap.sh` script to install the dependencies for your OS. The script also checks if the OS you are running is compatible with the supported versions. -**Note**: `bootstrap.sh` only supports Ubuntu >= 16.04, CentOS >= 7 and Fedora >=24. +**Note**: `bootstrap.sh` only supports Ubuntu >= 16.04, CentOS >= 7 and Fedora >=24 and MacOSX. ```text $ cd bigchaindb/pkg/scripts/ -$ sudo ./bootstrap.sh +$ bash bootstrap.sh --operation install ``` -### BigchainDB Setup Configuration(s) | Ansible -#### Local Setup | Ansible -You can run the Ansible playbook `bdb-deploy.yml` on your local dev machine and set up the BigchainDB node where +### BigchainDB Setup Configuration(s) +#### Local Setup +You can run the Ansible playbook `bigchaindb-start.yml` on your local dev machine and set up the BigchainDB node where BigchainDB can be run as a process or inside a Docker container(s) depending on your configuration. -Before, running the playbook locally, you need to update the `hosts` and `bdb-config.yml` configuration, which will notify Ansible that we need to run the play locally. +Before, running the playbook locally, you need to update the `hosts` and `stack-config.yml` configuration, which will notify Ansible that we need to run the play locally. -##### Update Hosts | Local +##### Update Hosts Navigate to `bigchaindb/pkg/configuration/hosts` inside the BigchainDB repository. ```text $ cd bigchaindb/pkg/configuration/hosts @@ -56,37 +56,33 @@ Edit `all` configuration file: # Hostname of dev machine ansible_connection=local ``` -##### Update Configuration | Local +##### Update Configuration Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. ```text -$ cd bigchaindb/pkg/configuration/vars/bdb-config.yml +$ cd bigchaindb/pkg/configuration/vars/stack-config.yml ``` Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): ```text --- -deploy_docker: false #[true, false] -docker_cluster_size: 1 # Only needed if `deploy_docker` is true -bdb_hosts: - - name: "" # Hostname of dev machine -``` -**Note**: You can also orchestrate a multi-node BigchainDB cluster on a local dev host using Docker containers. -Here is a sample `bdb-config.yml` -```text +stack_type: "docker" +stack_size: "4" + + +OR + --- -deploy_docker: true #[true, false] -docker_cluster_size: 3 -bdb_hosts: - - name: "" +stack_type: "local" +stack_type: "1" ``` -### BigchainDB Setup | Ansible -Now, You can safely run the `bdb-deploy.yml` playbook and everything will be taken care of by `Ansible`. To run the playbook please navigate to the `bigchaindb/pkg/configuration` directory inside the BigchainDB repository and run the `bdb-deploy.yml` playbook. +### BigchainDB Setup +Now, You can safely run the `bigchaindb-start.yml` playbook and everything will be taken care of by `Ansible`. To run the playbook please navigate to the `bigchaindb/pkg/configuration` directory inside the BigchainDB repository and run the `bigchaindb-start.yml` playbook. ```text $ cd bigchaindb/pkg/configuration/ -$ sudo ansible-playbook bdb-deploy.yml -i hosts/all +$ ansible-playbook bigchaindb-start.yml -i hosts/all --extra-vars "operation=start home_path=$(pwd)" ``` After successful execution of the playbook, you can verify that BigchainDB docker(s)/process(es) is(are) running. @@ -103,13 +99,10 @@ Verify BigchainDB Docker(s): $ docker ps | grep bigchaindb ``` -The playbook also installs the BigchainDB Python Driver, -so you can use it to make transactions -and verify the functionality of your BigchainDB node. +You can now send transactions and verify the functionality of your BigchainDB node. See the [BigchainDB Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) for details on how to use it. - **Note**: The `bdb_root_url` can be be one of the following: ```text # BigchainDB is running as a process @@ -124,14 +117,13 @@ bdb_root_url = http://: **Note**: BigchainDB has [other drivers as well](../drivers-clients/index.html). ### Experimental: Running Ansible a Remote Dev/Host -#### Remote Setup | Ansible -You can also run the Ansible playbook `bdb-deploy.yml` on remote machine(s) and set up the BigchainDB node where +#### Remote Setup +You can also run the Ansible playbook `bigchaindb-start.yml` on remote machine(s) and set up the BigchainDB node where BigchainDB can run as a process or inside a Docker container(s) depending on your configuration. -Before, running the playbook on a remote host, you need to update the `hosts` and `bdb-config.yml` configuration, which will notify Ansible that we need to -run the play on a remote host. +Before, running the playbook on a remote host, you need to update the `hosts` and `stack-config.yml` configuration, which will notify Ansible that we need to run the play on a remote host. -##### Update Hosts | Remote +##### Update Hosts Navigate to `bigchaindb/pkg/configuration/hosts` inside the BigchainDB repository. ```text $ cd bigchaindb/pkg/configuration/hosts @@ -140,28 +132,32 @@ $ cd bigchaindb/pkg/configuration/hosts Edit `all` configuration file: ```text # Delete any existing configuration in this file and insert - ansible_ssh_user= ansible_sudo_pass= + ansible_ssh_user= ansible_sudo_pass= ``` -**Note**: You can add multiple hosts to the `all` configuration file. Root password is needed because ansible -will run some tasks that require root permissions. +**Note**: You can add multiple hosts to the `all` configuration file. Non-root user with sudo enabled password is needed because ansible will run some tasks that require those permissions. **Note**: You can also use other methods to get inside the remote machines instead of password based SSH. For other methods please consult [Ansible Documentation](http://docs.ansible.com/ansible/latest/intro_getting_started.html). -##### Update Configuration | Remote +##### Update Configuration Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. ```text -$ cd bigchaindb/pkg/configuration/vars/bdb-config.yml +$ cd bigchaindb/pkg/configuration/vars/stack-config.yml ``` -Edit `bdb-config.yml` configuration file as per your requirements, sample configuration file(s): +Edit `stack-config.yml` configuration file as per your requirements, sample configuration file(s): ```text --- -deploy_docker: false #[true, false] -docker_cluster_size: 1 # Only needed if `deploy_docker` is true -bdb_hosts: - - name: "" +stack_type: "docker" +stack_size: "4" + + +OR + +--- +stack_type: "local" +stack_type: "1" ``` After, the configuration of remote hosts, [run the Ansible playbook and verify your deployment](#bigchaindb-setup-ansible). diff --git a/docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-stack.md b/docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-stack.md new file mode 100644 index 00000000..6296e22d --- /dev/null +++ b/docs/contributing/source/dev-setup-coding-and-contribution-process/run-dev-network-stack.md @@ -0,0 +1,317 @@ +# Run a BigchainDB network + +**NOT for Production Use** + +You can use the following instructions to deploy a single or multi node +BigchainDB network for dev/test using the extensible `stack` script(s). + +Currently, this workflow is only supported for the following Operating systems: +- Ubuntu >= 16.04 +- CentOS >= 7 +- Fedora >= 24 +- MacOSX + +## Minimum Requirements +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 + +## Download the scripts +> **Note**: If you're working on BigchainDB Server code, on a branch based on +> recent code, then you already have local recent versions of *stack.sh* and +> *unstack.sh* in your bigchaindb/pkg/scripts/ directory. Otherwise you can +> get them using: + +```text +$ export GIT_BRANCH=master +$ curl -fOL https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/stack.sh + +# Optional +$ curl -fOL https://raw.githubusercontent.com/bigchaindb/bigchaindb/${GIT_BRANCH}/pkg/scripts/unstack.sh +``` + +## Quick Start +If you run `stack.sh` out of the box i.e. without any configuration changes, you will be able to deploy a 4 node +BigchainDB network with Docker containers, created from `master` branch of `bigchaindb/bigchaindb` repo and Tendermint version `0.19.2`. + +**Note**: Run `stack.sh` with either root or non-root user with sudo enabled. + +```text +$ bash stack.sh +...Logs.. +......... +......... +Finished stacking! +``` + +## Configure the BigchainDB network + +The `stack.sh` script has multiple deployment methods and parameters and they can be explored using: `bash stack.sh -h` + +```text +$ bash stack.sh -h + + Usage: $ bash stack.sh [-h] + + Deploys the BigchainDB network. + + ENV[STACK_SIZE] + Set STACK_SIZE environment variable to the size of the network you desire. + Network mimics a production network environment with single or multiple BDB + nodes. (default: 4). + + ENV[STACK_TYPE] + Set STACK_TYPE environment variable to the type of deployment you desire. + You can set it one of the following: ["docker", "local", "cloud"]. + (default: docker) + + ENV[STACK_TYPE_PROVIDER] + Set only when STACK_TYPE="cloud". Only "azure" is supported. + (default: ) + + ENV[STACK_VM_MEMORY] + (Optional) Set only when STACK_TYPE="local". This sets the memory + of the instance(s) spawned. (default: 2048) + + ENV[STACK_VM_CPUS] + (Optional) Set only when STACK_TYPE="local". This sets the number of VCPUs + of the instance(s) spawned. (default: 2) + + ENV[STACK_BOX_NAME] + (Optional) Set only when STACK_TYPE="local". This sets the box Vagrant box name + of the instance(s) spawned. (default: ubuntu/xenial64) + + ENV[STACK_REPO] + (Optional) To configure bigchaindb repo to use, set STACK_REPO environment + variable. (default: bigchaindb/bigchaindb) + + ENV[STACK_BRANCH] + (Optional) To configure bigchaindb repo branch to use set STACK_BRANCH environment + variable. (default: master) + + ENV[TM_VERSION] + (Optional) Tendermint version to use for the setup. (default: 0.19.2) + + ENV[MONGO_VERSION] + (Optional) MongoDB version to use with the setup. (default: 3.6) + + ENV[AZURE_CLIENT_ID] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_TENANT_ID] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_SUBSCRIPTION_ID] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_CLIENT_SECRET] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Steps to generate: + https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application + + ENV[AZURE_REGION] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Azure region for the BigchainDB instance. Get list of regions using Azure CLI. + e.g. az account list-locations. (default: westeurope) + + ENV[AZURE_IMAGE_URN] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Azure image to use. Get list of available images using Azure CLI. + e.g. az vm image list --output table. (default: Canonical:UbuntuServer:16.04-LTS:latest) + + ENV[AZURE_RESOURCE_GROUP] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Name of Azure resource group for the instance. + (default: bdb-vagrant-rg-2018-05-30) + + ENV[AZURE_DNS_PREFIX] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + DNS Prefix of the instance. (default: bdb-instance-2018-05-30) + + ENV[AZURE_ADMIN_USERNAME] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Admin username of the the instance. (default: vagrant) + + ENV[AZURE_VM_SIZE] + (Optional) Only applicable, when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". + Azure VM size. (default: Standard_D2_v2) + + ENV[SSH_PRIVATE_KEY_PATH] + Only required when STACK_TYPE="cloud" and STACK_TYPE_PROVIDER="azure". Absolute path of + SSH keypair required to log into the Azure instance. + + -h + Show this help and exit. +``` + + +The parameter that differentiates between the deployment type is `STACK_TYPE` which currently, supports +an opinionated deployment of BigchainDB on `docker`, `local` and `cloud`. + +### STACK_TYPE: docker +This configuration deploys a docker based BigchainDB network on the dev/test machine that you are running `stack.sh` on. This is also the default `STACK_TYPE` config for `stack.sh`. + +#### Example +Deploy a 4 node docker based BigchainDB network on your host. + +```text +#Optional, since 4 is the default size. +$ export STACK_SIZE=4 + +#Optional, since docker is the default type. +$ export STACK_TYPE=docker + +#Optional, repo to use for the network deployment +# Default: bigchaindb/bigchaindb +$ export STACK_REPO=bigchaindb/bigchaindb + +#Optional, codebase to use for the network deployment +# Default: master +$ export STACK_BRANCH=master + +#Optional, since 0.19.2 is the default tendermint version. +$ export TM_VERSION=0.19.2 + +#Optional, since 3.6 is the default MongoDB version. +$ export MONGO_VERSION=3.6 + +$ bash stack.sh +``` + +**Note**: For MacOSX users, the script will not install `Docker for Mac`, it only detects if `docker` is installed on the system, if not make sure to install [Docker for Mac](https://docs.docker.com/docker-for-mac/). Also make sure Docker API Version > 1.25. To check Docker API Version: + +`docker version --format '{{.Server.APIVersion}}'` + +### STACK_TYPE: local +This configuration deploys a VM based BigchainDB network on your host/dev. All the services are running as processes on the VMs. For `local` deployments the following dependencies must be installed i.e. + +- Vagrant + - Vagrant plugins. + - vagrant-cachier + - vagrant-vbguest + - vagrant-hosts + - vagrant-azure + - `vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts vagrant-azure` +- [Virtualbox](https://www.virtualbox.org/wiki/Downloads) + +#### Example +Deploy a 4 node VM based BigchainDB network. + +```text +$ export STACK_TYPE=local + +# Optional, since 4 is the default size. +$ export STACK_SIZE=4 + +# Optional, default is 2048 +$ export STACK_VM_MEMORY=2048 + +#Optional, default is 1 +$ export STACK_VM_CPUS=1 + +#Optional, default is ubuntu/xenial64. Supported/tested images: bento/centos-7, fedora/25-cloud-base +$ export STACK_BOX_NAME=ubuntu/xenial64 + +#Optional, repo to use for the network deployment +# Default: bigchaindb/bigchaindb +$ export STACK_REPO=bigchaindb/bigchaindb + +#Optional, codebase to use for the network deployment +# Default: master +$ export STACK_BRANCH=master + +#Optional, since 0.19.2 is the default tendermint version +$ export TM_VERSION=0.19.2 + +#Optional, since 3.6 is the default MongoDB version. +$ export MONGO_VERSION=3.6 + +$ bash stack.sh +``` + +### STACK_TYPE: cloud + +This configuration deploys a docker based BigchainDB network on a cloud instance. Currently, only Azure is supported. +For `cloud` deployments the following dependencies must be installed i.e. + +- Vagrant + - Vagrant plugins. + - vagrant-cachier + - vagrant-vbguest + - vagrant-hosts + - vagrant-azure + - `vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts vagrant-azure` + +#### Example: stack +Deploy a 4 node docker based BigchainDB network on an Azure instance. + +- [Create an Azure Active Directory(AAD) Application](https://github.com/Azure/vagrant-azure#create-an-azure-active-directory-aad-application) + +- Generate or specify an SSH keypair to login to the Azure instance. + + - **Example:** + ```text + $ ssh-keygen -t rsa -C "" -f /path/to/key/ + ``` + +- Configure parameters for `stack.sh` +```text + +# After creating the AAD application with access to Azure Resource +# Group Manager for your subscription, it will return a JSON object + +$ export AZURE_CLIENT_ID= + +$ export AZURE_TENANT_ID= + +# Can be retrieved via +# az account list --query "[?isDefault].id" -o tsv +$ export AZURE_SUBSCRIPTION_ID= + +$ export AZURE_CLIENT_SECRET= + +$ export STACK_TYPE=cloud + +# Currently on azure is supported +$ export STACK_TYPE_PROVIDER=azure + +$ export SSH_PRIVATE_KEY_PATH= + +# Optional, Azure region of the instance. Default: westeurope +$ export AZURE_REGION=westeurope + +# Optional, Azure image urn of the instance. Default: Canonical:UbuntuServer:16.04-LTS:latest +$ export AZURE_IMAGE_URN=Canonical:UbuntuServer:16.04-LTS:latest + +# Optional, Azure resource group. Default: bdb-vagrant-rg-yyyy-mm-dd(current date) +$ export AZURE_RESOURCE_GROUP=bdb-vagrant-rg-2018-01-01 + +# Optional, DNS prefix of the Azure instance. Default: bdb-instance-yyyy-mm-dd(current date) +$ export AZURE_DNS_PREFIX=bdb-instance-2018-01-01 + +# Optional, Admin username of the Azure instance. Default: vagrant +$ export AZURE_ADMIN_USERNAME=vagrant + +# Optional, Azure instance size. Default: Standard_D2_v2 +$ export AZURE_VM_SIZE=Standard_D2_v2 + +$ bash stack.sh +``` + +## Delete/Unstack a BigchainDB network + +Export all the variables exported for the corresponding `stack.sh` script and +run `unstack.sh` to delete/remove/unstack the BigchainDB network/stack. + +```text +$ bash unstack.sh + +OR + +# -s implies soft unstack. i.e. Only applicable for local and cloud based +# networks. Only deletes/stops the docker(s)/process(es) and does not +# delete the instances created via Vagrant or on Cloud. Default: hard +$ bash unstack.sh -s +``` diff --git a/docs/contributing/source/dev-setup-coding-and-contribution-process/write-code.rst b/docs/contributing/source/dev-setup-coding-and-contribution-process/write-code.rst index 9a1663df..60526086 100644 --- a/docs/contributing/source/dev-setup-coding-and-contribution-process/write-code.rst +++ b/docs/contributing/source/dev-setup-coding-and-contribution-process/write-code.rst @@ -97,13 +97,15 @@ framework to write Python tests. Read all about it. Most tests are in the ``tests/`` folder. Take a look around. -Running a Local Node for Dev and Test -------------------------------------- +Running a Local Node/Network for Dev and Test +--------------------------------------------- This is tricky and personal. Different people do it different ways. We documented some of those on separate pages: - `Dev node setup and running all tests with Docker Compose `_ - `Dev node setup and running all tests as processes `_ +- `Dev network setup with stack.sh `_ +- `Dev network setup with Ansible `_ - More to come? diff --git a/docs/server/source/appendices/index.rst b/docs/server/source/appendices/index.rst index ab37aae0..25cca246 100755 --- a/docs/server/source/appendices/index.rst +++ b/docs/server/source/appendices/index.rst @@ -5,7 +5,6 @@ Appendices :maxdepth: 1 install-os-level-deps - run-with-docker json-serialization cryptography the-Bigchain-class @@ -17,5 +16,3 @@ Appendices firewall-notes ntp-notes licenses - run-with-vagrant - run-with-ansible diff --git a/docs/server/source/appendices/run-with-docker.md b/docs/server/source/appendices/run-with-docker.md deleted file mode 100644 index b75c9300..00000000 --- a/docs/server/source/appendices/run-with-docker.md +++ /dev/null @@ -1,157 +0,0 @@ -# Run BigchainDB with Docker - -**NOT for Production Use** - -For those who like using Docker and wish to experiment with BigchainDB in -non-production environments, we currently maintain a Docker image and a -`Dockerfile` that can be used to build an image for `bigchaindb`. - -## Prerequisite(s) -- [Docker](https://docs.docker.com/engine/installation/) - -## Pull and Run the Image from Docker Hub - -With Docker installed, you can proceed as follows. - -In a terminal shell, pull the latest version of the BigchainDB Docker image using: -```text -docker pull bigchaindb/bigchaindb -``` - -### Configuration -A one-time configuration step is required to create the config file; we will use -the `-y` option to accept all the default values. The configuration file will -be stored in a file on your host machine at `~/bigchaindb_docker/.bigchaindb`: - -```text -docker run \ - --interactive \ - --rm \ - --tty \ - --volume $HOME/bigchaindb_docker:/data \ - --env BIGCHAINDB_DATABASE_HOST=172.17.0.1 \ - bigchaindb/bigchaindb \ - -y configure \ - [mongodb|rethinkdb] - -Generating keypair -Configuration written to /data/.bigchaindb -Ready to go! -``` - -Let's analyze that command: - -* `docker run` tells Docker to run some image -* `--interactive` keep STDIN open even if not attached -* `--rm` remove the container once we are done -* `--tty` allocate a pseudo-TTY -* `--volume "$HOME/bigchaindb_docker:/data"` map the host directory - `$HOME/bigchaindb_docker` to the container directory `/data`; - this allows us to have the data persisted on the host machine, - you can read more in the [official Docker - documentation](https://docs.docker.com/engine/tutorials/dockervolumes) -* `--env BIGCHAINDB_DATABASE_HOST=172.17.0.1`, `172.17.0.1` is the default `docker0` bridge -IP address, for fresh Docker installations. It is used for the communication between BigchainDB and database -containers. -* `bigchaindb/bigchaindb` the image to use. All the options after the container name are passed on to the entrypoint inside the container. -* `-y configure` execute the `configure` sub-command (of the `bigchaindb` - command) inside the container, with the `-y` option to automatically use all the default config values -* `mongodb` or `rethinkdb` specifies the database backend to use with bigchaindb - - -### Run the backend database -From v0.9 onwards, you can run either RethinkDB or MongoDB. - -You can also use docker host networking or bind to your primary (eth) - interface, if needed. - -#### For RethinkDB - -```text -docker run \ - --detach \ - --name=rethinkdb \ - --publish=28015:28015 \ - --publish=58080:8080 \ - --restart=always \ - --volume $HOME/bigchaindb_docker:/data \ - rethinkdb:2.3 -``` - - - -You can also access the RethinkDB dashboard at http://172.17.0.1:58080/ - - -#### For MongoDB - -Note: MongoDB runs as user `mongodb` which had the UID `999` and GID `999` -inside the container. For the volume to be mounted properly, as user `mongodb` -in your host, you should have a `mongodb` user with UID and GID `999`. -If you have another user on the host with UID `999`, the mapped files will -be owned by this user in the host. -If there is no owner with UID 999, you can create the corresponding user and -group. - -`useradd -r --uid 999 mongodb` OR `groupadd -r --gid 999 mongodb && useradd -r --uid 999 -g mongodb mongodb` should work. - - -```text -docker run \ - --detach \ - --name=mongodb \ - --publish=27017:27017 \ - --restart=always \ - --volume=$HOME/mongodb_docker/db:/data/db \ - --volume=$HOME/mongodb_docker/configdb:/data/configdb \ - mongo:3.4.9 --replSet=bigchain-rs -``` - -### Run BigchainDB - -```text -docker run \ - --detach \ - --name=bigchaindb \ - --publish=59984:9984 \ - --restart=always \ - --volume=$HOME/bigchaindb_docker:/data \ - bigchaindb/bigchaindb \ - start -``` - -The command is slightly different from the previous one, the differences are: - -* `--detach` run the container in the background -* `--name bigchaindb` give a nice name to the container so it's easier to - refer to it later -* `--publish "59984:9984"` map the host port `59984` to the container port `9984` - (the BigchainDB API server) -* `start` start the BigchainDB service - -Another way to publish the ports exposed by the container is to use the `-P` (or -`--publish-all`) option. This will publish all exposed ports to random ports. You can -always run `docker ps` to check the random mapping. - -If that doesn't work, then replace `localhost` with the IP or hostname of the -machine running the Docker engine. If you are running docker-machine (e.g. on -Mac OS X) this will be the IP of the Docker machine (`docker-machine ip -machine_name`). - - -## Building Your Own Image - -Assuming you have Docker installed, you would proceed as follows. - -In a terminal shell: -```text -git clone git@github.com:bigchaindb/bigchaindb.git -``` - -Build the Docker image: -```text -docker build --tag local-bigchaindb . -``` - -Now you can use your own image to run BigchainDB containers. - diff --git a/docs/server/source/appendices/run-with-vagrant.md b/docs/server/source/appendices/run-with-vagrant.md deleted file mode 100644 index 8bde0a38..00000000 --- a/docs/server/source/appendices/run-with-vagrant.md +++ /dev/null @@ -1,170 +0,0 @@ -# Run BigchainDB with Vagrant - -**NOT for Production Use** - -You can use the following instructions to deploy a single or multi node -BigchainDB setup for dev/test using Vagrant. Vagrant will set up the BigchainDB node(s) -with all the dependencies along with MongoDB and BigchainDB Python driver. You -can also tweak the following configurations for the BigchainDB node(s). -- Vagrant Box - - Currently, we support the following boxes: - - `ubuntu/xenial64 # >=16.04` - - `centos/7 # >=7` - - `fedora/24 # >=24` - - **NOTE** : You can choose any other vagrant box of your choice but these are - the minimum versioning requirements. -- Resources and specs for your box. - - RAM - - VCPUs - - Network Type - - Currently, only `private_network` is supported. - - IP Address -- Deploy node with Docker - - Deploy all the services in Docker containers or as processes. -- Number of BigchainDB nodes - - If you want to deploy the services inside Docker containers, you - can specify number of member(s) in the BigchainDB cluster. -- Upstart Script -- Vagrant Provider - - Virtualbox - - VMware - -## Minimum Requirements | Vagrant -Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: -- Memory >= 512MB -- VCPUs >= 1 - -## Install dependencies | Vagrant -1. [VirtualBox](https://www.virtualbox.org/wiki/Downloads) >= 5.0.0 -2. [Vagrant](https://www.vagrantup.com/downloads.html) >= 1.16.0 - -## Clone the BigchainDB repository | Vagrant -```text -$ git clone https://github.com/bigchaindb/bigchaindb.git -``` - -## Configuration | Vagrant -Navigate to `bigchaindb/pkg/configuration/vars/` inside the BigchainDB repository. -```text -$ cd bigchaindb/pkg/configuration/vars/ -``` - -Edit `bdb-config.yml` as per your requirements. Sample `bdb-config.yml`: - -```text ---- -deploy_docker: false #[true, false] -docker_cluster_size: 1 -upstart: "/bigchaindb/scripts/bootstrap.sh" -bdb_hosts: - - name: "bdb-node-01" - box: - name: "ubuntu/xenial64" - ram: "2048" - vcpus: "2" - network: - ip: "10.20.30.40" - type: "private_network" -``` - -**Note**: You can spawn multiple instances to orchestrate a multi-node BigchainDB cluster. -Here is a sample `bdb-config.yml`: -```text ---- -deploy_docker: false #[true, false] -docker_cluster_size: 1 -upstart: "/bigchaindb/scripts/bootstrap.sh" -bdb_hosts: - - name: "bdb-node-01" - box: - name: "ubuntu/xenial64" - ram: "2048" - vcpus: "2" - network: - ip: "10.20.30.40" - type: "private_network" - - name: "bdb-node-02" - box: - name: "ubuntu/xenial64" - ram: "2048" - vcpus: "2" - network: - ip: "10.20.30.50" - type: "private_network" -``` -**Note**: You can also orchestrate a multi-node BigchainDB cluster on a single dev host using Docker containers. -Here is a sample `bdb-config.yml` -```text ---- -deploy_docker: true #[true, false] -docker_cluster_size: 3 -upstart: "/bigchaindb/scripts/bootstrap.sh" -bdb_hosts: - - name: "bdb-node-01" - box: - name: "ubuntu/xenial64" - ram: "8192" - vcpus: "4" - network: - ip: "10.20.30.40" - type: "private_network" -``` -The above mentioned configuration will deploy a 3 node BigchainDB cluster with Docker containers -on your specified host. - -## BigchainDB Setup | Vagrant - -**Note**: There are some vagrant plugins required for the installation, -user will be prompted to install them if they are not present. To install -the required plugins, run the following command: - -```text -$ vagrant plugin install vagrant-cachier vagrant-vbguest vagrant-hosts -``` - -To bring up the BigchainDB node(s), run the following command: - -```text -$ vagrant up -``` - -After successful execution of Vagrant, you can log in to your fresh BigchainDB node. - -```text -$ vagrant ssh -``` - -## Make your first transaction -Once you are inside the BigchainDB node, you can verify that BigchainDB -docker(s)/process(es) is(are) running. - -Verify BigchainDB process(es): -```text -$ ps -ef | grep bigchaindb -``` - -OR - -Verify BigchainDB Docker(s): -```text -$ docker ps | grep bigchaindb -``` - -The BigchainDB Python Driver is pre-installed in the instance, -so you can use it to make transactions -and verify the functionality of your BigchainDB node. -See the [BigchainDB Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) -for details on how to use it. - -Note 1: The `bdb_root_url` can be one of the following: -```text -# BigchainDB is running as a process -bdb_root_url = http://:9984 - -OR - -# BigchainDB is running inside a docker container -bdb_root_url = http://: -``` - -Note 2: BigchainDB has [other drivers as well](../drivers-clients/index.html). diff --git a/pkg/Vagrantfile b/pkg/Vagrantfile deleted file mode 100644 index 4031df07..00000000 --- a/pkg/Vagrantfile +++ /dev/null @@ -1,103 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Required modules -require 'yaml' - -# Minimum Requirements -Vagrant.require_version '>= 1.6.0' -VAGRANTFILE_API_VERSION = '2' - -# Configuration files -CONFIGURATION_FILE = 'configuration/vars/bdb-config.yml' -HOSTS_FILE = 'configuration/hosts/all' -HOST_VARS_PATH = 'configuration/host_vars' - -# Validate if all the required plugins are present -required_plugins = ["vagrant-cachier", "vagrant-vbguest", "vagrant-hosts"] -required_plugins.each do |plugin| - if not Vagrant.has_plugin?(plugin) - raise "Required vagrant plugin #{plugin} not found. Please run `vagrant plugin install #{plugin}`" - end -end - -# Read configuration file(s) -instances_config = YAML.load_file(File.join(File.dirname(__FILE__), CONFIGURATION_FILE)) -hosts_config = File.open(HOSTS_FILE, 'w+') -# TODO: (muawiakh) Add support for Docker, AWS, Azure -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - instances_config["bdb_hosts"].each do |instance| - # Workaround till canonical fixes https://bugs.launchpad.net/cloud-images/+bug/1569237 - # using -u ubuntu as remote user, conventionally vagrant boxes use `vagrant` user - if instance["box"]["name"] == "ubuntu/xenial64" - hosts_config.puts("#{instance["name"]} ansible_user=ubuntu") - else - hosts_config.puts("#{instance["name"]} ansible_user=vagrant") - end - config.vm.define instance['name'] do |bdb| - # Workaround until vagrant cachier plugin supports dnf - if !(instance["box"]["name"].include? "fedora") - if Vagrant.has_plugin?("vagrant-cachier") - bdb.cache.scope = :box - end - elsif instance["box"]["name"] == "ubuntu/xenial64" - if Vagrant.has_plugin?("vagrant-vbguest") - bdb.vbguest.auto_update = false - bdb.vbguest.no_install = true - bdb.vbguest.no_remote = true - end - end - bdb.vm.hostname = instance["name"] - if instance["network"]["type"] == "private_network" - bdb.vm.network instance["network"]["type"], ip: instance["network"]["ip"] - elsif instance["network"]["type"] == "public_network" - bdb.vm.network instance["network"]["type"], use_dhcp_assigned_default_route: true, bridge: instance["network"]["bridge"] - else - raise "Invalid network type: Please specify one of the following: [private_network, public_network]" - end - bdb.vm.provision :hosts, :sync_hosts => true - bdb.vm.box = instance["box"]["name"] - bdb.vm.synced_folder ".", "/bigchaindb" - File.open("#{HOST_VARS_PATH}/#{instance["name"]}", "w+") {|f| \ - f.write("ansible_ssh_private_key_file: /bigchaindb/.vagrant/machines/#{instance["name"]}/virtualbox/private_key") } - bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" - bdb.vm.provider 'vmware_fusion' do |vmwf, override| - vmwf.vmx['memsize'] = instance["ram"] - vmwf.vmx['numvcpus'] = instance['vcpus'] - end - - bdb.vm.provider 'virtualbox' do |vb, override| - vb.memory = instance["ram"] - vb.cpus = instance['vcpus'] - end - end - end - hosts_config.close - config.vm.define "config-node" do |bdb| - bdb.vm.box = "ubuntu/xenial64" - bdb.vm.hostname = "config-node" - bdb.vm.provision :hosts, :sync_hosts => true - bdb.vm.synced_folder ".", "/bigchaindb" - bdb.vm.network "private_network", ip: "192.168.100.200" - bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" - bdb.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /bigchaindb/configuration/bdb-deploy.yml \ - -i /bigchaindb/configuration/hosts/all" - bdb.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] - vb.memory = 2048 - vb.cpus = 2 - end - bdb.vm.provider 'vmware_fusion' do |vmwf| - vmwf.vmx['memsize'] = 2048 - vmwf.vmx['numvcpus'] = 2 - end - if Vagrant.has_plugin?("vagrant-vbguest") - config.vbguest.auto_update = false - config.vbguest.no_install = true - config.vbguest.no_remote = true - end - if Vagrant.has_plugin?("vagrant-cachier") - config.cache.scope = :box - end - end -end diff --git a/pkg/configuration/bdb-deploy.yml b/pkg/configuration/bdb-deploy.yml deleted file mode 100644 index 8dd8c7ea..00000000 --- a/pkg/configuration/bdb-deploy.yml +++ /dev/null @@ -1,12 +0,0 @@ -- import_playbook: pre_req.yml - -- hosts: all - vars_files: - - vars/bdb-config.yml - serial: 1 - roles: - - bigchaindb - - bigchaindb-driver - -- import_playbook: multi_node.yml - when: (bdb_hosts|length > 1) or docker_cluster_size|int > 1 diff --git a/pkg/configuration/bigchaindb-start.yml b/pkg/configuration/bigchaindb-start.yml new file mode 100644 index 00000000..80e3091e --- /dev/null +++ b/pkg/configuration/bigchaindb-start.yml @@ -0,0 +1,26 @@ +- hosts: all + gather_facts: false + vars_files: + - vars/stack-config.yml + vars: + action: "{{ operation }}" + home_dir: "{{ home_path }}" + serial: 1 + pre_tasks: + - raw: "apt-get -qqy update && apt-get install -qqy python-minimal python-pip || dnf -y install python27 || which python" + - name: Gathering Facts + setup: #aka gather_facts + roles: + - { role: py36, when: stack_type|lower == "local" } + - { role: docker, when: stack_type|lower == "docker" or stack_type|lower == "cloud" } + - mongodb + +- hosts: all + vars_files: + - vars/stack-config.yml + vars: + action: "{{ operation }}" + home_dir: "{{ home_path }}" + roles: + - tendermint + - bigchaindb diff --git a/pkg/configuration/bigchaindb-stop.yml b/pkg/configuration/bigchaindb-stop.yml new file mode 100644 index 00000000..98f888b5 --- /dev/null +++ b/pkg/configuration/bigchaindb-stop.yml @@ -0,0 +1,15 @@ +- hosts: all + vars_files: + - vars/stack-config.yml + vars: + action: "{{ operation }}" + home_dir: "{{ home_path }}" + pre_tasks: + - raw: "apt-get -qqy update && apt-get install -qqy python-minimal python-pip || dnf -y install python27 || which python" + - name: Gathering Facts + setup: #aka gather_facts + roles: + - tendermint + - bigchaindb + - mongodb + - { role: docker, when: stack_type|lower == "docker" or stack_type|lower == "cloud" } diff --git a/pkg/configuration/multi_node.yml b/pkg/configuration/multi_node.yml deleted file mode 100644 index aa293ad9..00000000 --- a/pkg/configuration/multi_node.yml +++ /dev/null @@ -1,5 +0,0 @@ -- hosts: all - vars_files: - - vars/bdb-config.yml - roles: - - key-exchange \ No newline at end of file diff --git a/pkg/configuration/pre_req.yml b/pkg/configuration/pre_req.yml deleted file mode 100644 index b7e54581..00000000 --- a/pkg/configuration/pre_req.yml +++ /dev/null @@ -1,8 +0,0 @@ -- hosts: all - vars_files: - - vars/bdb-config.yml - serial: 1 - roles: - - { role: docker, when: deploy_docker|bool } - - { role: docker-compose, when: deploy_docker|bool } - - mongodb \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml b/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml deleted file mode 100644 index 28c99323..00000000 --- a/pkg/configuration/roles/bigchaindb-driver/defaults/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -dependencies_deb: - - python3-dev - - libffi-dev - - libssl-dev - - python3-pip - -dependencies_yum: - - gcc-c++ - - "@Development Tools" - - python35u-devel - - libffi-devel - - openssl-devel - - python35u-setuptools - -dependencies_dnf: - - gcc-c++ - - redhat-rpm-config - - "@Development Tools" - - python3-devel - - libffi-devel - - openssl-devel - - python3-pip - -python_pip_upgrade: true -python_setuptools_upgrade: true - -# Host configuration -distribution_name: "{{ ansible_distribution|lower }}" -distribution_codename: "{{ ansible_distribution_release|lower }}" -distribution_major: "{{ ansible_distribution_major_version }}" \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml deleted file mode 100644 index 0eb69356..00000000 --- a/pkg/configuration/roles/bigchaindb-driver/tasks/centos.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Install dependencies - yum: - name: "{{ item }}" - state: present - update_cache: yes - with_items: "{{ dependencies_yum }}" - tags: [bigchaindb-driver] - -- name: Install pip - shell: "easy_install-3.5 pip" - tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml deleted file mode 100644 index 55641bb4..00000000 --- a/pkg/configuration/roles/bigchaindb-driver/tasks/common.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Upgrade pip - shell: "pip3 install --upgrade pip" - when: python_pip_upgrade - tags: [bigchaindb-driver] - -- name: Upgade setuptools - shell: "pip3 install --upgrade setuptools" - when: python_setuptools_upgrade - tags: [bigchaindb-driver] - -- name: Install BigchainDB Driver - shell: "pip3 install bigchaindb-driver" - tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml deleted file mode 100644 index e00a7061..00000000 --- a/pkg/configuration/roles/bigchaindb-driver/tasks/debian.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Install dependencies - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: "{{ dependencies_deb }}" - tags: [bigchaindb] diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml deleted file mode 100644 index bd8b701a..00000000 --- a/pkg/configuration/roles/bigchaindb-driver/tasks/fedora.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Install dependencies - dnf: - name: "{{ item }}" - state: present - with_items: "{{ dependencies_dnf }}" - tags: [bigchaindb-driver] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml b/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml deleted file mode 100644 index f70ef304..00000000 --- a/pkg/configuration/roles/bigchaindb-driver/tasks/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -- import_tasks: debian.yml - when: distribution_name == "debian" or distribution_name == "ubuntu" - -- import_tasks: centos.yml - when: distribution_name == "centos" or distribution_name == "red hat enterprise linux" - -- import_tasks: fedora.yml - when: distribution_name == "fedora" - -- import_tasks: common.yml \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/defaults/main.yml b/pkg/configuration/roles/bigchaindb/defaults/main.yml index 594deb20..37afca6e 100644 --- a/pkg/configuration/roles/bigchaindb/defaults/main.yml +++ b/pkg/configuration/roles/bigchaindb/defaults/main.yml @@ -1,31 +1,22 @@ --- dependencies_deb: - g++ - - python3-dev - libffi-dev - build-essential - libssl-dev - - python3-pip dependencies_yum: - gcc-c++ - "@Development Tools" - - python35u-devel - libffi-devel - openssl-devel - - python35u-setuptools dependencies_dnf: - gcc-c++ - redhat-rpm-config - "@Development Tools" - - python3-devel - libffi-devel - openssl-devel - - python3-pip - -python_pip_upgrade: true -python_setuptools_upgrade: true # Host configuration distribution_name: "{{ ansible_distribution|lower }}" @@ -35,17 +26,18 @@ distribution_major: "{{ ansible_distribution_major_version }}" directories: - /data -backend_db: mongodb #[mongodb] -bigchaindb_config_path: /data/.bigchaindb +backend_db: localmongodb #[localmongodb] bigchaindb_server_bind: "0.0.0.0:9984" -bigchaindb_log_file: "{{ ansible_env.HOME }}/bigchaindb.log" # Docker configuration bigchaindb_image_name: "bigchaindb/bigchaindb" +bigchaindb_image_tag: "develop" bigchaindb_docker_name: "bigchaindb" -bigchaindb_default_port: 9984 -bigchandb_host_port: 59984 -bigchaindb_host_mount_dir: "{{ ansible_env.HOME }}/bigchaindb_docker" -# Default IP of docker0 bridge -bigchaindb_default_host: "172.17.0.1" +mongodb_docker_name: "mongodb" +tendermint_docker_name: "tendermint" +bigchaindb_default_server_port: 9984 +bigchaindb_default_ws_port: 9985 +bigchaindb_tendermint_port: 46657 +tendermint_abci_port: 45558 +bigchaindb_docker_net: "bigchaindb_docker_net" \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/centos.yml b/pkg/configuration/roles/bigchaindb/tasks/centos.yml index 912d0f74..97c37a75 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/centos.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/centos.yml @@ -1,5 +1,5 @@ --- -- name: Creating directories | CentOS +- name: Creating directories | yum file: path: "{{ item }}" state: directory @@ -7,7 +7,7 @@ with_items: "{{ directories }}" tags: [bigchaindb] -- name: Install dependencies | CentOS +- name: Install dependencies | yum yum: name: "{{ item }}" state: present @@ -16,5 +16,4 @@ tags: [bigchaindb] - name: Install pip - shell: "easy_install-3.5 pip" - tags: [bigchaindb-driver] \ No newline at end of file + shell: "easy_install-3.5 pip" \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/common.yml b/pkg/configuration/roles/bigchaindb/tasks/common.yml index d29cc3ef..3fc4da50 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/common.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/common.yml @@ -1,30 +1,14 @@ --- -- name: Upgrade pip - shell: "pip3 install --upgrade pip" - when: python_pip_upgrade - tags: [bigchaindb] - -- name: Upgade setuptools - shell: "pip3 install --upgrade setuptools" - when: python_setuptools_upgrade +- name: Install pymongo + pip: + name: pymongo + state: present tags: [bigchaindb] - name: Install BigchainDB - shell: "pip3 install bigchaindb" - tags: [bigchaindb] - -- name: Check if BigchainDB node is already configured - stat: - path: "{{ bigchaindb_config_path }}" - register: stat_result - -- name: Configure BigchainDB - shell: "bigchaindb -y configure {{ backend_db }}" - environment: - BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" - BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path }}" - BIGCHAINDB_DATABASE_HOST: "{{ ansible_hostname }}" - when: stat_result.stat.exists == False + shell: "python3.6 -m pip install --no-cache-dir --process-dependency-links -e /opt/stack/bigchaindb/.[dev] --ignore-installed pyyaml" + register: install_bdb + failed_when: "'FAILED' in install_bdb.stderr or install_bdb.rc != 0" tags: [bigchaindb] - name: MongoDB Process Check @@ -32,28 +16,26 @@ register: mdb_pchk tags: [bigchaindb] +- name: Tendermint Process Check + shell: pgrep tendermint | wc -l + register: tm_pchk + tags: [bigchaindb] + - name: BigchainDB Process Check shell: pgrep bigchaindb | wc -l register: bdb_pchk tags: [bigchaindb] - name: Start BigchainDB - shell: "bigchaindb start > {{ bigchaindb_log_file }} 2>&1 &" + shell: nohup bigchaindb -l DEBUG start > /tmp/bigchaindb_log_$(date +%Y%m%d_%H%M%S) 2>&1 & environment: - BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path }}" - when: mdb_pchk.stdout| int >= 1 and bdb_pchk.stdout| int == 0 - async: 10 - poll: 0 - tags: [bigchaindb] - -- name: Get BigchainDB node public key - shell: "cat {{ bigchaindb_config_path }}" - register: bdb_node_config - tags: [bigchaindb] - -- name: Set Facts BigchainDB - set_fact: - pub_key="{{ ( bdb_node_config.stdout|from_json).keypair.public }}" - hostname="{{ ansible_hostname }}" - bdb_config="{{ bigchaindb_config_path }}" - tags: [bigchaindb] + BIGCHAINDB_DATABASE_BACKEND: localmongodb + BIGCHAINDB_DATABASE_HOST: 127.0.0.1 + BIGCHAINDB_DATABASE_PORT: 27017 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 + BIGCHAINDB_WSSERVER_PORT: 9985 + BIGCHAINDB_TENDERMINT_HOST: 127.0.0.1 + BIGCHAINDB_TENDERMINT_PORT: 46657 + when: mdb_pchk.stdout| int != 0 and bdb_pchk.stdout| int == 0 and tm_pchk.stdout| int != 0 + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/debian.yml b/pkg/configuration/roles/bigchaindb/tasks/debian.yml index 0686848a..08fb27cb 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/debian.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/debian.yml @@ -1,5 +1,5 @@ --- -- name: Install dependencies +- name: Install dependencies | apt apt: name: "{{ item }}" state: present diff --git a/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml deleted file mode 100644 index 98fd90d5..00000000 --- a/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Check if BigchainDB Dockers are already configured - stat: - path: "{{ bigchaindb_host_mount_dir }}{{ item|string }}/.bigchaindb" - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - register: stat_result - tags: [bigchaindb] - -- name: Configuring BigchainDB Docker - docker_container: - name: "{{ bigchaindb_docker_name }}{{ item }}" - hostname: "{{ bigchaindb_docker_name }}{{ item }}" - image: "{{ bigchaindb_image_name }}" - volumes: - - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" - env: - BIGCHAINDB_SERVER_BIND: "{{ bigchaindb_server_bind }}" - BIGCHAINDB_DATABASE_HOST: "{{ hostvars[ansible_hostname]['mongodb' + item|string] }}" - entrypoint: "bigchaindb -y configure mongodb" - when: stat_result.results[item|int].stat.exists == False - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [bigchaindb] - -- name: Start BigchainDB Docker - docker_container: - name: "{{ bigchaindb_docker_name }}{{ item }}" - image: "{{ bigchaindb_image_name }}" - detach: true - published_ports: - - "{{ bigchandb_host_port|int + item|int }}:{{ bigchaindb_default_port }}" - restart_policy: always - volumes: - - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" - state: started - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [bigchaindb] - -- name: Get BigchainDB node public key - shell: "cat {{ bigchaindb_host_mount_dir + item|string }}/.bigchaindb" - register: bdb_node_config - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [bigchaindb] - -- name: Set facts for BigchainDB containers - set_fact: - pub_key_{{ bigchaindb_docker_name }}{{ item }}="{{ (bdb_node_config.results[item|int].stdout|from_json).keypair.public }}" - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [bigchaindb] diff --git a/pkg/configuration/roles/bigchaindb/tasks/fedora.yml b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml index 4f542d3c..e6372ca0 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/fedora.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/fedora.yml @@ -1,5 +1,5 @@ --- -- name: Install dependencies +- name: Install dependencies | dnf dnf: name: "{{ item }}" state: present diff --git a/pkg/configuration/roles/bigchaindb/tasks/main.yml b/pkg/configuration/roles/bigchaindb/tasks/main.yml index 3b1de267..15a79441 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/main.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/main.yml @@ -1,20 +1,21 @@ --- -- import_tasks: deploy_docker.yml - when: deploy_docker|bool +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + when: stack_type|lower == "docker" or stack_type|lower == "cloud" tags: [bigchaindb] -- import_tasks: debian.yml - when: not deploy_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: (stack_type|lower == "docker" or stack_type|lower == "cloud") and info_result is failed tags: [bigchaindb] -- import_tasks: centos.yml - when: not deploy_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") +- import_tasks: start.yml + when: action|lower == 'start' tags: [bigchaindb] -- import_tasks: fedora.yml - when: not deploy_docker|bool and (distribution_name == "fedora") - tags: [bigchaindb] - -- import_tasks: common.yml - when: not deploy_docker|bool - tags: [bigchaindb] +- import_tasks: stop.yml + when: action|lower == 'stop' + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/start.yml b/pkg/configuration/roles/bigchaindb/tasks/start.yml new file mode 100644 index 00000000..7810917d --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/start.yml @@ -0,0 +1,57 @@ +--- +- name: Building BigchainDB Docker + docker_image: + name: "{{ bigchaindb_image_name }}" + state: build + tag: "{{bigchaindb_image_tag }}" + dockerfile: Dockerfile-alpine + path: "{{ home_dir }}/bigchaindb" + nocache: yes + buildargs: + backend: localmongodb + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- name: Start BigchainDB Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}:{{ bigchaindb_image_tag }}" + hostname: "{{ bigchaindb_docker_name }}{{ item }}" + detach: true + network_mode: bridge + networks: + - name: "{{ bigchaindb_docker_net }}" + env: + BIGCHAINDB_DATABASE_BACKEND: localmongodb + BIGCHAINDB_DATABASE_HOST: "{{ mongodb_docker_name }}{{ item }}" + BIGCHAINDB_DATABASE_PORT: 27017 + BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 + BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 + BIGCHAINDB_TENDERMINT_HOST: "{{ tendermint_docker_name }}{{ item }}" + BIGCHAINDB_TENDERMINT_PORT: "{{ bigchaindb_tendermint_port }}" + published_ports: + - "{{ bigchaindb_default_server_port }}" + - "{{ bigchaindb_default_ws_port }}" + - "{{ tendermint_abci_port }}" + restart_policy: always + state: started + command: bigchaindb -l DEBUG start + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- import_tasks: debian.yml + when: stack_type == "local" and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [bigchaindb] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [bigchaindb] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (distribution_name == "fedora") + tags: [bigchaindb] + +- import_tasks: common.yml + when: stack_type|lower == "local" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/bigchaindb/tasks/stop.yml b/pkg/configuration/roles/bigchaindb/tasks/stop.yml new file mode 100644 index 00000000..016a06fe --- /dev/null +++ b/pkg/configuration/roles/bigchaindb/tasks/stop.yml @@ -0,0 +1,33 @@ +--- +- name: Stopping BigchainDB Docker + docker_container: + name: "{{ bigchaindb_docker_name }}{{ item }}" + image: "{{ bigchaindb_image_name }}:{{ bigchaindb_image_tag }}" + keep_volumes: no + state: absent + force_kill: yes + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [bigchaindb] + +- name: Get Running BigchainDB Process(es) + shell: "ps aux | grep \"[b]igchaindb\" | awk '{print $2}'" + register: bdb_ps + when: stack_type|lower == "local" + tags: [bigchaindb] + +- name: Kill BigchainDB Process(es) + shell: "kill -9 {{ item }}" + with_items: "{{ bdb_ps.stdout_lines }}" + when: stack_type|lower == "local" and bdb_ps.stdout_lines|length > 0 + tags: [bigchaindb] + +- name: Kill auxiliary BigchainDB Process(es) + shell: "pkill {{ item }}" + with_items: + - logging_server + - ws + - gunicorn + ignore_errors: yes + when: stack_type|lower == "local" + tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/docker-compose/defaults/main.yml b/pkg/configuration/roles/docker-compose/defaults/main.yml deleted file mode 100644 index e50ee45c..00000000 --- a/pkg/configuration/roles/docker-compose/defaults/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# TODO: (muawiakh) Install docker-compose using pip -# docker_compose_pip_install: false #[true, false] -docker_compose_install: true #[true, false] -docker_compose_version: "1.15.0" -docker_compose_binary: /usr/local/bin/docker-compose -docker_compose_base_url: "https://github.com/docker/compose/releases/download/" \ No newline at end of file diff --git a/pkg/configuration/roles/docker-compose/tasks/main.yml b/pkg/configuration/roles/docker-compose/tasks/main.yml deleted file mode 100644 index 2862061e..00000000 --- a/pkg/configuration/roles/docker-compose/tasks/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Get docker-compose - become: yes - get_url: - url: "{{ docker_compose_base_url }}{{ docker_compose_version }}/docker-compose-{{ ansible_system }}-{{ ansible_machine}}" - dest: "{{docker_compose_binary}}" - mode: 0755 - when: docker_compose_install - tags: [docker-compose] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/defaults/main.yml b/pkg/configuration/roles/docker/defaults/main.yml index b1eda858..1ffa2b63 100644 --- a/pkg/configuration/roles/docker/defaults/main.yml +++ b/pkg/configuration/roles/docker/defaults/main.yml @@ -15,4 +15,7 @@ docker_apt_repo: "deb [arch={{ server_arch }}] https://download.docker.com/linux apt_key_fingerprint: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88" apt_key_url: "https://download.docker.com/linux/{{ distribution_name }}/gpg" docker_yum_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ distribution_major }}/$basearch/{{ docker_update_channel }}" -docker_dnf_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ docker_pkg }}.repo" \ No newline at end of file +docker_dnf_repo: "https://download.docker.com/linux/{{ distribution_name }}/{{ docker_pkg }}.repo" + +# Docker configuration +bigchaindb_docker_net: "bigchaindb_docker_net" \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/debian.yml b/pkg/configuration/roles/docker/tasks/debian.yml index 483b7ac4..eda15ea9 100644 --- a/pkg/configuration/roles/docker/tasks/debian.yml +++ b/pkg/configuration/roles/docker/tasks/debian.yml @@ -1,4 +1,4 @@ ---- +--- - name: Uninstall older versions of Docker | Debian apt: name: "{{ item }}" @@ -19,7 +19,7 @@ - ca-certificates - curl - software-properties-common - - python-pip + - python3-pip tags: [docker] - name: Add APT Key | Debian @@ -35,7 +35,7 @@ shell: "curl -sSl {{ apt_key_url }} | sudo apt-key add -" args: warn: no - when: add_repository_key|failed + when: add_repository_key is failed tags: [docker] - name: Add Docker repo and update cache | Debian diff --git a/pkg/configuration/roles/docker/tasks/macos.yml b/pkg/configuration/roles/docker/tasks/macos.yml new file mode 100644 index 00000000..e61b8437 --- /dev/null +++ b/pkg/configuration/roles/docker/tasks/macos.yml @@ -0,0 +1,24 @@ +--- +- name: Check if Docker is Present + command: docker + register: result + ignore_errors: True + tags: [docker] + +- name: Error! Docker not found + fail: + msg: "Docker not installed on MacOSX, please visit: https://docs.docker.com/docker-for-mac/install/" + when: result is failed + tags: [docker] + +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + tags: [docker] + +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: info_result is failed + tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/docker/tasks/main.yml b/pkg/configuration/roles/docker/tasks/main.yml index 5676e153..4de9da78 100644 --- a/pkg/configuration/roles/docker/tasks/main.yml +++ b/pkg/configuration/roles/docker/tasks/main.yml @@ -8,6 +8,9 @@ - import_tasks: fedora.yml when: distribution_name == "fedora" +- import_tasks: macos.yml + when: distribution_name == 'macosx' + - name: Create Docker group group: name: docker @@ -22,19 +25,50 @@ state: present group: docker with_items: - - vagrant + - "{{ ansible_hostname }}" - "{{ distribution_name }}" + - "vagrant" tags: [docker] -- name: Start docker service +- name: Start docker service | systemd systemd: name: docker enabled: yes state: started + when: (distribution_name != 'macosx') and distribution_name != 'debian' + tags: [docker] + +- name: Start docker service | init + service: + name: docker + state: started + enabled: yes + when: distribution_name == 'debian' tags: [docker] - name: Install docker-py pip: name: docker-py - state: present + state: latest + tags: [docker] + +- name: Check if BigchainDB Docker network exists + shell: docker inspect "{{ bigchaindb_docker_net }}" + register: network_ls + ignore_errors: True + tags: [docker] + + +- name: Create BigchainDB Docker network + shell: docker network create "{{ bigchaindb_docker_net }}" --attachable + register: docker_net_create + when: (stack_type|lower == "docker" or stack_type == "cloud") and network_ls.rc != 0 and action|lower == 'start' + failed_when: "'FAILED' in docker_net_create.stderr or docker_net_create.rc != 0" + tags: [docker] + +- name: Delete BigchainDB Docker network + shell: docker network rm "{{ bigchaindb_docker_net }}" + register: docker_net_delete + failed_when: "'FAILED' in docker_net_delete.stderr or docker_net_delete.rc != 0" + when: (stack_type|lower == "docker" or stack_type == "cloud") and network_ls.rc != 0 and action|lower == 'stop' tags: [docker] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/defaults/main.yml b/pkg/configuration/roles/key-exchange/defaults/main.yml deleted file mode 100644 index 6bfe9035..00000000 --- a/pkg/configuration/roles/key-exchange/defaults/main.yml +++ /dev/null @@ -1,12 +0,0 @@ -keyring_script_host: /tmp/keyring.py -bigchaindb_log_file_host: "{{ ansible_env.HOME }}/bigchaindb.log" -bigchaindb_config_path_host: /data/.bigchaindb - -# Docker configuration -keyring_script_docker: "{{ ansible_env.HOME }}/config/keyring.py" -bigchaindb_config_path_docker: "{{ ansible_env.HOME }}/bigchaindb_docker" -bigchaindb_docker_name: bigchaindb -bigchaindb_default_port: 9984 -bigchandb_host_port: 59984 -bigchaindb_host_mount_dir: "{{ ansible_env.HOME }}/bigchaindb_docker" -bigchaindb_image_name: "bigchaindb/bigchaindb" diff --git a/pkg/configuration/roles/key-exchange/tasks/main.yml b/pkg/configuration/roles/key-exchange/tasks/main.yml deleted file mode 100644 index 031bf16b..00000000 --- a/pkg/configuration/roles/key-exchange/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- include_tasks: pub_key_exchange_host.yml - when: not deploy_docker|bool - tags: [bigchaindb] - -- include_tasks: pub_key_exchange_docker.yml - when: deploy_docker|bool - tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml deleted file mode 100644 index eaf0eb14..00000000 --- a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Creating files for key exchange - template: src=exchange_keyring_docker.j2 dest="{{ keyring_script_docker }}" - tags: [keyex] - -- name: Setting permissions - file: - path: "{{ keyring_script_docker }}" - mode: "0777" - tags: [keyex] - -- name: Update Keyring Configuration - shell: "python {{ keyring_script_docker }}" - tags: [keyex] - -- name: Restart BigchainDB Docker after keyring update - docker_container: - name: "{{ bigchaindb_docker_name }}{{ item }}" - image: "{{ bigchaindb_image_name }}" - detach: true - published_ports: - - "{{ bigchandb_host_port|int + item|int }}:{{ bigchaindb_default_port }}" - restart_policy: always - volumes: - - "{{ bigchaindb_host_mount_dir }}{{ item|string }}:/data" - state: started - restart: true - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [keyex] diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml deleted file mode 100644 index f3d5c4d6..00000000 --- a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_host.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Creating files for key exchange - template: src=exchange_keyring_host.j2 dest="{{ keyring_script_host }}" - tags: [keyex] - -- name: Setting permissions - file: - path: "{{ keyring_script_host }}" - mode: "0777" - tags: [keyex] - -- name: Update Keyring Configuration - shell: "python {{ keyring_script_host }}" - tags: [keyex] - -- name: Stop BigchainDB - shell: pkill bigchaindb - register: bdb_stop - tags: [keyex] - -- name: Start BigchainDB - shell: "bigchaindb start > {{ bigchaindb_log_file_host }} 2>&1 &" - environment: - BIGCHAINDB_CONFIG_PATH: "{{ bigchaindb_config_path_host }}" - async: 10 - poll: 0 - when: bdb_stop|succeeded - tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 deleted file mode 100644 index 31aed4fb..00000000 --- a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/python -import json -{% set keyring = {} %} -{% for docker in range(0, docker_cluster_size|int, 1) %} - {{- keyring.update({'pub_key_' + bigchaindb_docker_name + docker|string: hostvars[ansible_hostname]['pub_key_' + bigchaindb_docker_name + docker|string]}) -}} -{%- endfor -%} -{% for docker in range(0, docker_cluster_size|int, 1) %} -keyring = {{ keyring }} -keyring.pop('{{ 'pub_key_' + bigchaindb_docker_name + docker|string }}', None) -with open('{{ bigchaindb_config_path_docker + docker|string }}/.bigchaindb', 'r+') as f: - data = json.load(f) - del data['keyring'][:] - for key, value in keyring.iteritems(): - data['keyring'].append(value) - f.seek(0) - json.dump(data, f, indent=4) - f.truncate() -{% endfor %} \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 deleted file mode 100644 index f02a208c..00000000 --- a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_host.j2 +++ /dev/null @@ -1,21 +0,0 @@ -{%- set keyring = [] -%} -{%- set bdb_config_path = {'path': ''} -%} -{%- for host in bdb_hosts -%} - {%- if host["name"] != ansible_hostname -%} - {{- keyring.append(hostvars[host["name"]]["pub_key"]) -}} - {%- else -%} - {%- if bdb_config_path.update({'path': hostvars[host["name"]]["bdb_config"]}) -%} - {%- endif -%} - {%- endif -%} -{%- endfor -%} -{%- if keyring|length != 0 -%} -#!/usr/bin/python -import json -with open('{{ bdb_config_path['path'] }}', 'r+') as f: - data = json.load(f) - del data['keyring'][:] - data['keyring'] = {{ keyring }} - f.seek(0) - json.dump(data, f, indent=4) - f.truncate() -{%- endif -%} \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/defaults/main.yml b/pkg/configuration/roles/mongodb/defaults/main.yml index 7088a61b..3e209bbd 100644 --- a/pkg/configuration/roles/mongodb/defaults/main.yml +++ b/pkg/configuration/roles/mongodb/defaults/main.yml @@ -1,17 +1,16 @@ --- -mongodb_version: "3.4" mongodb_package: "mongodb-org" -apt_key_fingerprint: "0C49F3730359A14518585931BC711F9BA15703C6" -apt_keyserver: "keyserver.ubuntu.com" -distribution_name: "{{ansible_distribution|lower }}" +apt_key_fingerprint: "2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5" +apt_keyserver: "hkp://keyserver.ubuntu.com:80" +distribution_name: "{{ ansible_distribution|lower }}" distribution_codename: "{{ ansible_distribution_release|lower }}" distribution_major: "{{ ansible_distribution_major_version }}" server_arch: "amd64,arm64" # MongoDB Repos -mongodb_apt_repo: "deb [arch={{ server_arch }}] http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongodb_version }} {{'main' if ansible_distribution == 'debian' else 'multiverse'}}" -mongodb_yum_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/$releasever/{{ mongodb_package }}/{{ mongodb_version }}/{{ ansible_architecture }}" -mongodb_dnf_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/7/{{ mongodb_package }}/{{ mongodb_version }}/{{ ansible_architecture }}" +mongodb_apt_repo: "deb [arch={{ server_arch }}] http://repo.mongodb.org/apt/{{ distribution_name }} {{ distribution_codename }}/{{ mongodb_package }}/{{ mongo_version }} {{'main' if ansible_distribution == 'debian' else 'multiverse'}}" +mongodb_yum_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/$releasever/{{ mongodb_package }}/{{ mongo_version }}/{{ ansible_architecture }}" +mongodb_dnf_base_url: "https://repo.mongodb.org/yum/{{ ansible_os_family|lower }}/7/{{ mongodb_package }}/{{ mongo_version }}/{{ ansible_architecture }}" # MongoDB running config mongodb_storage_path: /data/db/main @@ -22,18 +21,9 @@ directories: - "{{ mongodb_log_path }}" - "{{ mongodb_config_path }}" -mongodb_conf_file: /etc/mongod.conf -mongodb_conf_files: [ - { src: "mongod.conf", dest: "{{ mongodb_conf_file }}"} -] mongodb_port: 27017 -mongodb_admin_user: "adminUser" -mongodb_admin_password: "superstrongpassword" - -replica_set_name: bigchain-rs # Docker configuration -mongodb_docker_image: "mongo:3.4.9" +bigchaindb_docker_net: "bigchaindb_docker_net" mongodb_docker_name: "mongodb" -mongodb_host_mount_dir: "{{ ansible_env.HOME }}/mongodb_docker" -mongodb_host_config: "{{ ansible_env.HOME }}/config" +mongodb_host_mount_dir: "{{ home_dir }}/mongodb_docker" \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/files/mongod.conf b/pkg/configuration/roles/mongodb/files/mongod.conf deleted file mode 100644 index ed961801..00000000 --- a/pkg/configuration/roles/mongodb/files/mongod.conf +++ /dev/null @@ -1,101 +0,0 @@ -# mongod.conf - -# for documentation of all options, see: -# http://docs.mongodb.org/manual/reference/configuration-options/ - -# where to write logging data. -systemLog: - verbosity: 0 - # traceAllExceptions: true - timeStampFormat: iso8601-utc - component: - accessControl: - verbosity: 0 - command: - verbosity: 0 - control: - verbosity: 0 - ftdc: - verbosity: 0 - geo: - verbosity: 0 - index: - verbosity: 0 - network: - verbosity: 0 - query: - verbosity: 0 - replication: - verbosity: 0 - sharding: - verbosity: 0 - storage: - verbosity: 0 - journal: - verbosity: 0 - write: - verbosity: 0 - -processManagement: - fork: false - pidFilePath: /tmp/mongod.pid - -net: - port: 27017 - bindIp: 0.0.0.0 - maxIncomingConnections: 8192 - wireObjectCheck: false - unixDomainSocket: - enabled: false - pathPrefix: /tmp - filePermissions: 0700 - http: - enabled: false - compression: - compressors: snappy -# ssl: -# mode: requireSSL -# PEMKeyFile: MONGODB_KEY_FILE_PATH -# PEMKeyPassword: MONGODB_KEY_FILE_PASSWORD -# CAFile: MONGODB_CA_FILE_PATH -# CRLFile: MONGODB_CRL_FILE_PATH - -# allowConnectionsWithoutCertificates: false -# allowInvalidHostnames: false -# weakCertificateValidation: false -# allowInvalidCertificates: false - -#security: -# authorization: enabled -# clusterAuthMode: x509 - -#setParameter: -# enableLocalhostAuthBypass: true -# #notablescan: 1 -# logUserIds: 1 -# authenticationMechanisms: MONGODB-X509,SCRAM-SHA-1 - -storage: - dbPath: /data/db/main - indexBuildRetry: true - journal: - enabled: true - commitIntervalMs: 100 - directoryPerDB: true - engine: wiredTiger - wiredTiger: - engineConfig: - journalCompressor: snappy -# configString: cache_size=STORAGE_ENGINE_CACHE_SIZE - collectionConfig: - blockCompressor: snappy - indexConfig: - prefixCompression: true # TODO false may affect performance? - -operationProfiling: - mode: slowOp - slowOpThresholdMs: 100 - -replication: - replSetName: bigchain-rs - enableMajorityReadConcern: true diff --git a/pkg/configuration/roles/mongodb/tasks/centos.yml b/pkg/configuration/roles/mongodb/tasks/centos.yml index 62e8faa0..64bca086 100644 --- a/pkg/configuration/roles/mongodb/tasks/centos.yml +++ b/pkg/configuration/roles/mongodb/tasks/centos.yml @@ -1,25 +1,26 @@ --- -- name: Add MongoDB Repo | CentOS +- name: Add MongoDB Repo | yum yum_repository: - name: "{{ mongodb_package }}" + name: "{{ mongodb_package }}-{{ mongo_version }}" gpgcheck: yes - gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongodb_version }}.asc + gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongo_version }}.asc baseurl: "{{ mongodb_yum_base_url }}" file: "{{ mongodb_package }}" description: "MongoDB Repo" enabled: yes tags: [mongodb] -- name: Install MongoDB | CentOS +- name: Install MongoDB | yum yum: name: "{{ mongodb_package }}" state: present update_cache: yes tags: [mongodb] -- name: Install pip | CentOS +- name: Install pip | yum yum: name: python-pip state: present update_cache: yes - tags: [mongodb] \ No newline at end of file + tags: [mongodb] + \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/common.yml b/pkg/configuration/roles/mongodb/tasks/common.yml deleted file mode 100644 index 40369191..00000000 --- a/pkg/configuration/roles/mongodb/tasks/common.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: MongoDB config files are copied - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: "{{ mongodb_conf_files }}" - tags: [mongodb] - -- name: MongoDB Process Check - shell: pgrep mongod | wc -l - register: command_result - tags: [mongodb] - -- name: Install pymongo - pip: - name: pymongo - state: present - tags: [mongodb] - -- name: Run MongoDB - shell: "mongod --config {{ mongodb_conf_file }} 2>&1 &" - when: command_result.stdout| int != 1 - async: 5 - poll: 0 - tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/debian.yml b/pkg/configuration/roles/mongodb/tasks/debian.yml index aac606fa..86dab821 100644 --- a/pkg/configuration/roles/mongodb/tasks/debian.yml +++ b/pkg/configuration/roles/mongodb/tasks/debian.yml @@ -1,22 +1,34 @@ --- -- name: Add APT Key | Debian +- name: Removing apt cache lists + file: + path: /var/lib/apt/lists/* + state: absent + force: yes + tags: [mongodb] + +- name: Update Cache + apt: update_cache=yes + tags: [mongodb] + +- name: Add APT Key | apt apt_key: keyserver: "{{ apt_keyserver }}" id: "{{ apt_key_fingerprint }}" - tags: [mongodb] - -- name: Add MongoDB repo and update cache | Debian - apt_repository: - repo: "{{ mongodb_apt_repo }}" - update_cache: yes state: present tags: [mongodb] -- name: Install MongoDB | Debian +- name: Add MongoDB repo and update cache | apt + apt_repository: + repo: "{{ mongodb_apt_repo }}" + state: present + update_cache: no + tags: [mongodb] + +- name: Install MongoDB | apt apt: name: "{{ item }}" state: present + update_cache: yes with_items: - "{{ mongodb_package }}" - - python-pip tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml deleted file mode 100644 index ecadea49..00000000 --- a/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Check Docker Service - systemd: - name: docker - enabled: yes - state: started - tags: [mongodb] - -- name: Running MongoDB Docker - docker_container: - name: "{{ mongodb_docker_name }}{{ item }}" - hostname: "{{ mongodb_docker_name }}{{ item }}" - image: "{{ mongodb_docker_image }}" - detach: true - published_ports: - - "{{ (mongodb_port|int + item|int)|string }}:{{ mongodb_port }}" - restart_policy: always - volumes: - - "{{ mongodb_host_mount_dir }}{{ item|string }}/db:{{ mongodb_storage_path }}" - - "{{ mongodb_host_mount_dir }}{{ item|string }}/configdb:{{ mongodb_config_path }}" - - "{{ mongodb_host_config }}:/bdb_config" - state: started - keep_volumes: true - entrypoint: /entrypoint.sh --replSet=bigchain-rs - register: mongo_container_info - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [mongodb] - -- name: Set facts for MongoDB containers - set_fact: - mongodb{{ item }}={{ mongo_container_info.results[item|int].ansible_facts.docker_container.NetworkSettings.IPAddress }} - with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 - tags: [mongodb] diff --git a/pkg/configuration/roles/mongodb/tasks/fedora.yml b/pkg/configuration/roles/mongodb/tasks/fedora.yml index c2f61110..4825933b 100644 --- a/pkg/configuration/roles/mongodb/tasks/fedora.yml +++ b/pkg/configuration/roles/mongodb/tasks/fedora.yml @@ -1,16 +1,16 @@ --- -- name: Add MongoDB Repo | Fedora +- name: Add MongoDB Repo | dnf yum_repository: name: "{{ mongodb_package }}" gpgcheck: yes - gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongodb_version }}.asc + gpgkey: https://www.mongodb.org/static/pgp/server-{{ mongo_version }}.asc baseurl: "{{ mongodb_dnf_base_url }}" file: "{{ mongodb_package }}" description: "MongoDB Repo" enabled: yes tags: [mongodb] -- name: Install MongoDB | Fedora +- name: Install MongoDB | dnf dnf: name: "{{ item }}" state: present diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml deleted file mode 100644 index ecbdc625..00000000 --- a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- import_tasks: initiate_repl_set_host.yml - when: (ansible_hostname == bdb_hosts[bdb_hosts|length-1]['name']) and not deploy_docker|bool - -- import_tasks: initiate_repl_set_docker.yml - when: deploy_docker|bool and docker_cluster_size|int > 1 diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml deleted file mode 100644 index 5cd341fa..00000000 --- a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Creating files to initialize MongoDB Replica Set | Docker - template: src=replSet_init_docker.j2 dest="{{ mongodb_host_config }}/replSet_init.js" - tags: [mongodb] - -- name: Initializing Replica Set and Adding AdminUser | Docker - run_once: true - shell: - cmd: - "docker exec {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} bash -l -c - '/usr/bin/mongo --host {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} - --port {{ mongodb_port }} < /bdb_config/replSet_init.js'" - tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml deleted file mode 100644 index 8d398bcb..00000000 --- a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_host.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Creating files to initialize MongoDB Replica Set - template: src=replSet_init_host.j2 dest=/tmp/replSet_init.js - tags: [mongodb] - -- name: Initializing Replica Set - shell: "/usr/bin/mongo --host {{ ansible_hostname }} --port {{ mongodb_port }} < /tmp/replSet_init.js" - tags: [mongodb] - -- name: Adding AdminUser to MongoDB - run_once: true - mongodb_user: - database: admin - login_host: "{{ ansible_hostname }}" - login_port: "{{ mongodb_port }}" - name: "{{ mongodb_admin_user }}" - password: "{{ mongodb_admin_password }}" - roles: readWriteAnyDatabase,clusterManager - state: present - tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/main.yml b/pkg/configuration/roles/mongodb/tasks/main.yml index 8daeffc6..d08fb417 100644 --- a/pkg/configuration/roles/mongodb/tasks/main.yml +++ b/pkg/configuration/roles/mongodb/tasks/main.yml @@ -1,31 +1,19 @@ --- -- name: Creating directories - file: - path: "{{ item }}" - state: directory - mode: 0700 - with_items: "{{ directories }}" - when: not deploy_docker|bool +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + when: stack_type|lower == "docker" or stack_type|lower == "cloud" tags: [mongodb] -- import_tasks: deploy_docker.yml - when: deploy_docker|bool +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: (stack_type|lower == "docker" or stack_type|lower == "cloud") and info_result is failed tags: [mongodb] -- import_tasks: debian.yml - when: not deploy_docker|bool and (distribution_name == "debian" or distribution_name == "ubuntu") - tags: [mongodb] +- import_tasks: start.yml + when: action|lower == 'start' -- import_tasks: centos.yml - when: not deploy_docker|bool and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") - tags: [mongodb] - -- import_tasks: fedora.yml - when: not deploy_docker|bool and (distribution_name == "fedora") - tags: [mongodb] - -- import_tasks: common.yml - when: not deploy_docker|bool - tags: [mongodb] - -- import_tasks: initiate_repl_set.yml +- import_tasks: stop.yml + when: action|lower == 'stop' \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/start.yml b/pkg/configuration/roles/mongodb/tasks/start.yml new file mode 100644 index 00000000..ef79be95 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/start.yml @@ -0,0 +1,56 @@ +--- +- name: Running MongoDB Docker + docker_container: + name: "{{ mongodb_docker_name }}{{ item }}" + hostname: "{{ mongodb_docker_name }}{{ item }}" + image: "mongo:{{ mongo_version }}" + detach: true + network_mode: bridge + networks: + - name: "{{ bigchaindb_docker_net }}" + published_ports: + - "{{ mongodb_port }}" + restart_policy: unless-stopped + volumes: + - "{{ mongodb_host_mount_dir }}{{ item|string }}/db:{{ mongodb_storage_path }}" + - "{{ mongodb_host_mount_dir }}{{ item|string }}/configdb:{{ mongodb_config_path }}" + state: started + keep_volumes: true + command: mongod + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Creating directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [mongodb] + +- import_tasks: debian.yml + when: stack_type|lower == "local" and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [mongodb] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [mongodb] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (distribution_name == "fedora") + tags: [mongodb] + +- name: MongoDB Process Check + shell: pgrep mongod | wc -l + register: command_result + when: stack_type|lower == "local" + tags: [mongodb] + +- name: Run MongoDB + shell: nohup mongod > /tmp/mongodb_log_$(date +%Y%m%d_%H%M%S) 2>&1 & + register: start_mdb + failed_when: "'FAILED' in start_mdb.stderr or start_mdb.rc != 0" + when: stack_type|lower == "local" and command_result.stdout| int == 0 + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/stop.yml b/pkg/configuration/roles/mongodb/tasks/stop.yml new file mode 100644 index 00000000..9d7c5125 --- /dev/null +++ b/pkg/configuration/roles/mongodb/tasks/stop.yml @@ -0,0 +1,41 @@ +--- +- name: Stopping MongoDB Docker + docker_container: + name: "{{ mongodb_docker_name }}{{ item }}" + image: "mongo:{{ mongo_version }}" + state: absent + keep_volumes: no + force_kill: yes + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Removing MongoDB Directories + file: + path: "{{ mongodb_host_mount_dir }}{{ item }}" + state: absent + force: yes + with_sequence: start=0 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Get Running MongoDB Process + shell: "ps aux | grep \"[m]ongo\" | awk 'NR == 1 {print $2}'" + register: mongo_ps + when: stack_type|lower == "local" + tags: [mongodb] + +- name: Kill MongoDB Process + shell: "kill {{ item }}" + with_items: "{{ mongo_ps.stdout_lines }}" + when: stack_type|lower == "local" and mongo_ps.stdout_lines|length > 0 + tags: [mongodb] + +- name: Removing MongoDB Directories + file: + path: "{{ item }}" + state: absent + force: yes + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 deleted file mode 100644 index ed8c8fa4..00000000 --- a/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 +++ /dev/null @@ -1,30 +0,0 @@ -rs.initiate({ - "_id": "{{ replica_set_name }}", - "members": [ - { - "_id": 0, - "host": "{{ hostvars[ansible_hostname][mongodb_docker_name + (docker_cluster_size|int - 1)|string] }}:{{ mongodb_port }}" - } - ] -}); -sleep(5000); -{% for docker in range(0, docker_cluster_size|int, 1) %} -{%- if docker != (docker_cluster_size|int - 1) -%} -rs.add("{{ hostvars[ansible_hostname][mongodb_docker_name + docker|string] }}:{{ mongodb_port }}"); -{% endif %} -{%- endfor -%} -use admin; -db.createUser(db.createUser({ - "user": "{{ mongodb_admin_user }}", - "pwd": "{{ mongodb_admin_password }}", - "roles": [ - { - "role": "userAdminAnyDatabase", - "db": "admin" - }, - { - "role": "clusterManager", - "db": "admin" - } - ] -}); diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 deleted file mode 100644 index 42bca2a3..00000000 --- a/pkg/configuration/roles/mongodb/templates/replSet_init_host.j2 +++ /dev/null @@ -1,7 +0,0 @@ -rs.initiate( { _id : "{{ replica_set_name }}", members: [ { _id : 0, host :"{{ bdb_hosts[bdb_hosts|length-1]['name'] }}:{{ mongodb_port }}" } ] } ) -sleep(5000); -{% for host in bdb_hosts %} -{%- if ansible_hostname != host["name"] -%} -rs.add("{{ host["name"] }}:{{ mongodb_port }}"); -{% endif %} -{%- endfor -%} \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/centos.yml b/pkg/configuration/roles/py36/tasks/centos.yml new file mode 100644 index 00000000..636101f9 --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/centos.yml @@ -0,0 +1,21 @@ +--- +- name: Install dependencies py36 | yum + yum: + name: "{{ item }}" + update_cache: yes + state: present + with_items: + - "yum-utils" + - "https://centos7.iuscommunity.org/ius-release.rpm" + tags: [py36] + +- name: Install py36 | yum + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "python36u" + - "python36u-pip" + - "python36u-devel" + tags: [py36] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/debian.yml b/pkg/configuration/roles/py36/tasks/debian.yml new file mode 100644 index 00000000..802f829e --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/debian.yml @@ -0,0 +1,25 @@ +--- +- name: Check if python3 already installed + shell: which python3 + register: chk_py36 + ignore_errors: yes + tags: [py36] + +- name: Check version of python3 + shell: "python3 -c 'import platform; print(platform.python_version())' | cut -d. -f-2" + when: chk_py36.rc == 0 + register: py36_ver + tags: [py36] + +- name: Creating files for python 3.6 installation + template: src=install_py36.j2 dest=/home/vagrant/install_py36.bash + mode: 0755 + when: chk_py36.rc == 0 and (py36_ver.stdout | float < 3.6) + tags: [py36] + +- name: Install py36 + shell: "bash /home/vagrant/install_py36.bash > install_py36.txt" + register: install_py36 + failed_when: "'FAILED' in install_py36.stderr or install_py36.rc != 0" + when: chk_py36.rc == 0 and (py36_ver.stdout | float < 3.6) + tags: [py36] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/fedora.yml b/pkg/configuration/roles/py36/tasks/fedora.yml new file mode 100644 index 00000000..99912835 --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/fedora.yml @@ -0,0 +1,37 @@ +--- +- name: Check version of python3 + shell: "python3 -c 'import platform; print(platform.python_version())' | cut -d. -f-2" + register: py36_ver + tags: [py36] + +- name: Install py36 | dnf + dnf: + name: "{{ item }}" + state: present + with_items: + - "python36" + - libselinux-python + - policycoreutils-python + - wget + when: py36_ver.stdout | float < 3.6 + tags: [py36] + +- name: Update fedora + shell: dnf -y update + register: update_fed + failed_when: "'FAILED' in update_fed.stderr or update_fed.rc != 0" + warn: no + tags: [py36] + +- name: Creating files for python 3.6 installation + template: src=install_pip36.j2 dest=/home/vagrant/install_pip36.bash + mode: 0755 + when: py36_ver.stdout | float < 3.6 + tags: [py36] + +- name: Install pip36 + shell: "bash /home/vagrant/install_pip36.bash > install_pip36.txt" + register: install_pip36 + failed_when: "'FAILED' in install_pip36.stderr or install_pip36.rc != 0" + when: py36_ver.stdout | float < 3.6 + tags: [py36] \ No newline at end of file diff --git a/pkg/configuration/roles/py36/tasks/main.yml b/pkg/configuration/roles/py36/tasks/main.yml new file mode 100644 index 00000000..9678709b --- /dev/null +++ b/pkg/configuration/roles/py36/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- import_tasks: debian.yml + when: stack_type|lower == "local" and (ansible_distribution|lower == "debian" or ansible_distribution|lower == "ubuntu") + tags: [py36] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (ansible_distribution|lower == "centos" or ansible_distribution|lower == "red hat enterprise linux") + tags: [py36] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (ansible_distribution|lower == "fedora") \ No newline at end of file diff --git a/pkg/configuration/roles/py36/templates/install_pip36.j2 b/pkg/configuration/roles/py36/templates/install_pip36.j2 new file mode 100644 index 00000000..30cdc0e1 --- /dev/null +++ b/pkg/configuration/roles/py36/templates/install_pip36.j2 @@ -0,0 +1,8 @@ +{% raw %} +#!/bin/bash +set -ex +export PYTHON_PIP_VERSION=10.0.1 +wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py' +python3.6 get-pip.py --disable-pip-version-check --no-cache-dir +rm -f get-pip.py +{% endraw %} \ No newline at end of file diff --git a/pkg/configuration/roles/py36/templates/install_py36.j2 b/pkg/configuration/roles/py36/templates/install_py36.j2 new file mode 100644 index 00000000..3563e9a7 --- /dev/null +++ b/pkg/configuration/roles/py36/templates/install_py36.j2 @@ -0,0 +1,34 @@ +{% raw %} +#!/bin/bash +set -ex +export PATH=/usr/local/bin:$PATH +export LANG=C.UTF-8 +export GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D +export PYTHON_VERSION=3.6.5 +rm -rf /var/lib/apt/lists/* && \ +apt-get update && apt-get install -y --no-install-recommends tcl tk libssl-dev && rm -rf /var/lib/apt/lists/* && \ +apt-get update && apt-get install -y dpkg-dev tcl-dev tk-dev --no-install-recommends && rm -rf /var/lib/apt/lists/* && \ +wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" && \ +wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" && \ +export GNUPGHOME="$(mktemp -d)" && \ +gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" && \ +gpg --batch --verify python.tar.xz.asc python.tar.xz && \ +rm -rf "$GNUPGHOME" python.tar.xz.asc && \ +mkdir -p /usr/src/python && \ +tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz && \ +rm python.tar.xz && \ +cd /usr/src/python/ && \ +gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" && \ +./configure --build="$gnuArch" --enable-loadable-sqlite-extensions --enable-shared --with-system-expat --with-system-ffi && \ +make -j "$(nproc)" && \ +make install && \ +apt-get purge -y --auto-remove dpkg-dev tcl-dev tk-dev && \ +find /usr/local -depth \( \( -type d -a \( -name test -o -name tests \) \) -o \( -type f -a \( -name "*.pyc" -o -name "*.pyo" \) \) \) -exec rm -rf '{}' + && \ +cd $HOME && \ +rm -rf /usr/src/python +export PYTHON_PIP_VERSION=10.0.1 +wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py' +python3 get-pip.py --disable-pip-version-check --no-cache-dir "pip==$PYTHON_PIP_VERSION" +find /usr/local -depth \( \( -type d -a \( -name test -o -name tests \) \) -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \) -exec rm -rf '{}' +; +rm -f get-pip.py +{% endraw %} \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/defaults/main.yml b/pkg/configuration/roles/tendermint/defaults/main.yml new file mode 100644 index 00000000..2fff2d58 --- /dev/null +++ b/pkg/configuration/roles/tendermint/defaults/main.yml @@ -0,0 +1,27 @@ +dependencies_key_exchange: + - nginx + - jq + +tendermint_binary_url: "https://github.com/tendermint/tendermint/releases/download/v{{ tm_version }}/tendermint_{{ tm_version }}_linux_amd64.zip" + +# Docker configuration +tendermint_image_name: "bigchaindb/tendermint" +tendermint_image_tag: "develop" +tendermint_docker_name: "tendermint" +bigchaindb_docker_name: "bigchaindb" +bigchaindb_docker_net: "bigchaindb_docker_net" +tendermint_host_mount_dir: "{{ home_dir }}/tendermint_docker" +tendermint_host_mount_config_dir: "{{ home_dir }}/tendermint_config" +tendermint_home: /tendermint/config +tendermint_p2p_port: 46656 +tendermint_rpc_port: 46657 +tendermint_abci_port: 46658 + +directories: + - "{{ tendermint_home }}" + +tendermint_conf_files: [ + { src: "genesis.json", dest: "{{ tendermint_home }}/genesis.json"}, + { src: "config.toml", dest: "{{ tendermint_home}}/config.toml"}, + { src: "access_pub_key.conf", dest: "/etc/nginx/conf.d/access_pub_key.conf"} +] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/files/Dockerfile b/pkg/configuration/roles/tendermint/files/Dockerfile new file mode 100644 index 00000000..70a0a8ee --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/Dockerfile @@ -0,0 +1,7 @@ +ARG tm_version=0.19.3 +FROM tendermint/tendermint:${tm_version} +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +USER root +RUN apk --update add bash +ENTRYPOINT ["/usr/bin/tendermint"] diff --git a/pkg/configuration/roles/tendermint/files/access_pub_key.conf b/pkg/configuration/roles/tendermint/files/access_pub_key.conf new file mode 100644 index 00000000..5de7bb46 --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/access_pub_key.conf @@ -0,0 +1,10 @@ +# Serve the public key for a tendermint instance + +server { + listen 9986 default_server; + listen [::]:9986 default_server ipv6only=on; + location / { + root /usr/share/nginx/; + autoindex on; + } +} \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/files/config.toml b/pkg/configuration/roles/tendermint/files/config.toml new file mode 100644 index 00000000..bc6a1fe3 --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/config.toml @@ -0,0 +1,169 @@ +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +##### main base config options ##### + +# TCP or UNIX socket address of the ABCI application, +# or the name of an ABCI application compiled in with the Tendermint binary +proxy_app = "tcp://127.0.0.1:46658" + +# A custom human readable name for this node +moniker = "d16137710ef8" + +# If this node is many blocks behind the tip of the chain, FastSync +# allows them to catchup quickly by downloading blocks in parallel +# and verifying their commits +fast_sync = true + +# Database backend: leveldb | memdb +db_backend = "leveldb" + +# Database directory +db_path = "data" + +# Output level for logging, including package level options +log_level = "main:info,state:info,*:error" + +##### additional base config options ##### + +# Path to the JSON file containing the initial validator set and other meta data +genesis_file = "config/genesis.json" + +# Path to the JSON file containing the private key to use as a validator in the consensus protocol +priv_validator_file = "config/priv_validator.json" + +# Path to the JSON file containing the private key to use for node authentication in the p2p protocol +node_key_file = "config/node_key.json" + +# Mechanism to connect to the ABCI application: socket | grpc +abci = "socket" + +# TCP or UNIX socket address for the profiling server to listen on +prof_laddr = "" + +# If true, query the ABCI app on connecting to a new peer +# so the app can decide if we should keep the connection or not +filter_peers = false + +##### advanced configuration options ##### + +##### rpc server configuration options ##### +[rpc] + +# TCP or UNIX socket address for the RPC server to listen on +laddr = "tcp://0.0.0.0:46657" + +# TCP or UNIX socket address for the gRPC server to listen on +# NOTE: This server only supports /broadcast_tx_commit +grpc_laddr = "" + +# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool +unsafe = false + +##### peer to peer configuration options ##### +[p2p] + +# Address to listen for incoming connections +laddr = "tcp://0.0.0.0:46656" + +# Comma separated list of seed nodes to connect to +seeds = "" + +# Comma separated list of nodes to keep persistent connections to +# Do not add private peers to this list if you don't want them advertised +persistent_peers = "" + +# Path to address book +addr_book_file = "config/addrbook.json" + +# Set true for strict address routability rules +addr_book_strict = true + +# Time to wait before flushing messages out on the connection, in ms +flush_throttle_timeout = 100 + +# Maximum number of peers to connect to +max_num_peers = 50 + +# Maximum size of a message packet payload, in bytes +max_packet_msg_payload_size = 1024 + +# Rate at which packets can be sent, in bytes/second +send_rate = 512000 + +# Rate at which packets can be received, in bytes/second +recv_rate = 512000 + +# Set true to enable the peer-exchange reactor +pex = true + +# Seed mode, in which node constantly crawls the network and looks for +# peers. If another node asks it for addresses, it responds and disconnects. +# +# Does not work if the peer-exchange reactor is disabled. +seed_mode = false + +# Authenticated encryption +auth_enc = true + +# Comma separated list of peer IDs to keep private (will not be gossiped to other peers) +private_peer_ids = "" + +##### mempool configuration options ##### +[mempool] + +recheck = true +recheck_empty = true +broadcast = true +wal_dir = "data/mempool.wal" + +##### consensus configuration options ##### +[consensus] + +wal_file = "data/cs.wal/wal" + +# All timeouts are in milliseconds +timeout_propose = 3000 +timeout_propose_delta = 500 +timeout_prevote = 1000 +timeout_prevote_delta = 500 +timeout_precommit = 1000 +timeout_precommit_delta = 500 +timeout_commit = 1000 + +# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) +skip_timeout_commit = false + +# BlockSize +max_block_size_txs = 10000 +max_block_size_bytes = 1 + +# EmptyBlocks mode and possible interval between empty blocks in seconds +create_empty_blocks = true +create_empty_blocks_interval = 0 + +# Reactor sleep duration parameters are in milliseconds +peer_gossip_sleep_duration = 100 +peer_query_maj23_sleep_duration = 2000 + +##### transactions indexer configuration options ##### +[tx_index] + +# What indexer to use for transactions +# +# Options: +# 1) "null" (default) +# 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). +indexer = "kv" + +# Comma-separated list of tags to index (by default the only tag is tx hash) +# +# It's recommended to index only a subset of tags due to possible memory +# bloat. This is, of course, depends on the indexer's DB and the volume of +# transactions. +index_tags = "" + +# When set to true, tells indexer to index all tags. Note this may be not +# desirable (see the comment above). IndexTags has a precedence over +# IndexAllTags (i.e. when given both, IndexTags will be indexed). +index_all_tags = false \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/files/genesis.json b/pkg/configuration/roles/tendermint/files/genesis.json new file mode 100644 index 00000000..9b7ea8f6 --- /dev/null +++ b/pkg/configuration/roles/tendermint/files/genesis.json @@ -0,0 +1,6 @@ +{ + "genesis_time": "0001-01-01T00:00:00Z", + "chain_id": "test-chain-KPI1Ud", + "validators": [], + "app_hash": "" +} diff --git a/pkg/configuration/roles/tendermint/tasks/centos.yml b/pkg/configuration/roles/tendermint/tasks/centos.yml new file mode 100644 index 00000000..0485c92b --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/centos.yml @@ -0,0 +1,11 @@ +--- +- name: Install dependencies | yum + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - unzip + - nginx + - jq + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/common.yml b/pkg/configuration/roles/tendermint/tasks/common.yml new file mode 100644 index 00000000..575558cb --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/common.yml @@ -0,0 +1,51 @@ +- name: Download Tendermint Binary + get_url: + url: "{{ tendermint_binary_url }}" + dest: "{{ ansible_env.HOME }}/tendermint_binary.zip" + mode: 0755 + tags: [tendermint] + +- name: Extract Tendermint Binary + unarchive: + src: "{{ ansible_env.HOME }}/tendermint_binary.zip" + dest: /usr/local/bin + remote_src: yes + tags: [tendermint] + +- name: Creating Tendermint config directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + recurse: yes + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [tendermint] + +- name: Copying Tendermint config files + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: "{{ tendermint_conf_files }}" + tags: [tendermint] + +- name: Creating files for key exchange + template: src=start_tendermint.j2 dest=/home/vagrant/test.sh + mode: 0755 + tags: [tendermint] + +- name: Start nginx to host public_key + systemd: name=nginx state=restarted + tags: [tendermint] + +- name: Tendermint Process Check + shell: pgrep tendermint | wc -l + register: tm_pchk + tags: [bigchaindb] + +- name: Run tendermint + shell: nohup /bin/bash /home/vagrant/test.sh > /tmp/tendermint_log_$(date +%Y%m%d_%H%M%S) 2>&1 & + register: run_tendermint + when: tm_pchk.stdout| int == 0 + failed_when: "'FAILED' in run_tendermint.stderr or run_tendermint.rc != 0" + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/debian.yml b/pkg/configuration/roles/tendermint/tasks/debian.yml new file mode 100644 index 00000000..4715b3f5 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/debian.yml @@ -0,0 +1,12 @@ +--- +- name: Install dependencies for Tendermint | apt + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - wget + - unzip + - nginx + - jq + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/fedora.yml b/pkg/configuration/roles/tendermint/tasks/fedora.yml new file mode 100644 index 00000000..d91b531f --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/fedora.yml @@ -0,0 +1,21 @@ +--- +- name: Install dependencies | dnf + dnf: + name: "{{ item }}" + state: present + with_items: + - wget + - unzip + - libselinux-python + - policycoreutils-python + - nginx + - jq + tags: [tendermint] + +- name: Allow nginx to host public key + seport: + ports: 9986 + proto: tcp + setype: http_port_t + state: present + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/main.yml b/pkg/configuration/roles/tendermint/tasks/main.yml new file mode 100644 index 00000000..5c3d9859 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Check if Docker is running + command: docker info + register: info_result + ignore_errors: True + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Error! Docker not running + fail: + msg: "Docker not running." + when: (stack_type|lower == "docker" or stack_type|lower == "cloud") and info_result is failed + tags: [tendermint] + +- import_tasks: start.yml + when: action|lower == 'start' + +- import_tasks: stop.yml + when: action|lower == 'stop' \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/start.yml b/pkg/configuration/roles/tendermint/tasks/start.yml new file mode 100644 index 00000000..e9128cf2 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/start.yml @@ -0,0 +1,90 @@ +--- +- name: Building Tendermint Docker + docker_image: + name: "{{ tendermint_image_name }}" + state: build + tag: "{{tendermint_image_tag }}" + dockerfile: Dockerfile + path: "{{ home_dir }}/bigchaindb/pkg/configuration/roles/tendermint/files" + buildargs: + tm_version: "{{ tm_version }}" + nocache: yes + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Configuring Tendermint Containers + docker_container: + name: "tm_config_gen" + image: "{{ tendermint_image_name }}:{{ tendermint_image_tag }}" + detach: true + volumes: + - "{{ tendermint_host_mount_config_dir }}{{ tendermint_home }}:{{ tendermint_home }}" + entrypoint: '' + command: | + bash -c 'tendermint init && + jq ".validators=[]" /tendermint/config/genesis.json > /tendermint/config/genesis.tmp && + mv /tendermint/config/genesis.tmp /tendermint/config/genesis.json && rm /tendermint/config/node_key.json && + for i in $( seq {{ stack_size }} );do + tendermint gen_validator > /tendermint/config/priv_validator$i.json; + tendermint gen_node_key > /tendermint/config/node_id$i; mv /tendermint/config/node_key.json /tendermint/config/node_key$i.json; + cat tendermint/config/priv_validator$i.json | jq ".pub_key" | jq ". as \$k | {pub_key: \$k, power: 10, + name: \"{{ tendermint_docker_name }}$i\"}" > pub_validator$i.json; + cat /tendermint/config/genesis.json | jq ".validators |= .+ [$(cat pub_validator$i.json)]" > tmpgenesis; + mv tmpgenesis /tendermint/config/genesis.json; + done' + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Starting Tendermint Containers + docker_container: + name: "{{ tendermint_docker_name }}{{ item }}" + hostname: "{{ tendermint_docker_name }}{{ item }}" + image: "{{ tendermint_image_name }}:{{ tendermint_image_tag }}" + detach: true + network_mode: bridge + networks: + - name: "{{ bigchaindb_docker_net }}" + published_ports: + - "{{ tendermint_p2p_port }}" + - "{{ tendermint_rpc_port }}" + volumes: + - "{{ tendermint_host_mount_dir }}{{ item|string }}{{ tendermint_home }}:{{ tendermint_home }}" + - "{{ tendermint_host_mount_config_dir }}{{ tendermint_home }}:/tendermint_config" + entrypoint: '' + command: bash -c 'cp /tendermint_config/genesis.json /tendermint/config/genesis.json && + mv /tendermint_config/priv_validator"{{ item|string }}".json /tendermint/config/priv_validator.json && + mv /tendermint_config/node_key"{{ item|string }}".json /tendermint/config/node_key.json && + peers=() && for i in $( seq {{ stack_size }} );do peers+=($(cat /tendermint_config/node_id$i)@"{{ tendermint_docker_name }}$i:{{ tendermint_p2p_port }}");done && + peers=$(IFS=","; echo "${peers[*]}") && echo $peers && + tendermint node --p2p.persistent_peers="$peers" --p2p.laddr "tcp://"{{ tendermint_docker_name }}{{ item|string }}":46656" + --proxy_app="tcp://"{{ bigchaindb_docker_name }}{{ item|string }}":46658" --consensus.create_empty_blocks=false --p2p.pex=false' + state: started + keep_volumes: true + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Creating directories + file: + path: "{{ item }}" + state: directory + mode: 0700 + with_items: "{{ directories }}" + when: stack_type|lower == "local" + tags: [tendermint] + +- import_tasks: debian.yml + when: stack_type|lower == "local" and (distribution_name == "debian" or distribution_name == "ubuntu") + tags: [tendermint] + +- import_tasks: centos.yml + when: stack_type|lower == "local" and (distribution_name == "centos" or distribution_name == "red hat enterprise linux") + tags: [tendermint] + +- import_tasks: fedora.yml + when: stack_type|lower == "local" and (distribution_name == "fedora") + tags: [tendermint] + +- import_tasks: common.yml + when: stack_type|lower == "local" + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/tasks/stop.yml b/pkg/configuration/roles/tendermint/tasks/stop.yml new file mode 100644 index 00000000..edaffc37 --- /dev/null +++ b/pkg/configuration/roles/tendermint/tasks/stop.yml @@ -0,0 +1,57 @@ +--- +- name: Stopping Tendermint Containers I + docker_container: + name: "{{ tendermint_docker_name }}{{ item }}" + image: "tendermint/tendermint:{{ tm_version }}" + state: absent + keep_volumes: no + force_kill: yes + with_sequence: start=1 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Configuring Tendermint Containers II + docker_container: + name: "tm_config_gen" + image: "tendermint/tendermint:{{ tm_version }}" + detach: true + force_kill: yes + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [tendermint] + +- name: Removing Tendermint Directories I + file: + path: "{{ tendermint_host_mount_dir }}{{ item }}" + state: absent + force: yes + with_sequence: start=0 end="{{ stack_size|int }}" stride=1 + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Removing Tendermint Directories II + file: + path: "{{ tendermint_host_mount_config_dir }}" + state: absent + force: yes + when: stack_type|lower == "docker" or stack_type|lower == "cloud" + tags: [mongodb] + +- name: Get Running Tendermint Process + shell: "ps aux | grep \"[t]endermint\" | awk '{print $2}'" + register: tendermint_ps + when: stack_type|lower == "local" + tags: [tendermint] + +- name: Kill Tendermint Process + shell: "kill {{ item }}" + with_items: "{{ tendermint_ps.stdout_lines }}" + when: stack_type|lower == "local" and tendermint_ps.stdout_lines|length > 0 + tags: [tendermint] + +- name: Removing Tendermint Directories + file: + path: "{{ tendermint_home }}" + state: absent + force: yes + when: stack_type|lower == "local" + tags: [tendermint] \ No newline at end of file diff --git a/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 b/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 new file mode 100644 index 00000000..15aeb619 --- /dev/null +++ b/pkg/configuration/roles/tendermint/templates/start_tendermint.j2 @@ -0,0 +1,98 @@ +#!/bin/bash +set -euo pipefail + +{%- set peers = [] -%} +{%- set validator_power = [] -%} +{% for host in hostvars %} +{{- peers.append(host) -}} +{{- validator_power.append(10) -}} +{% endfor %} + +tm_persistent_peers="{{ peers| reverse | join(',') }}" +tm_validators="{{ peers | reverse | join(',') }}" +tm_validator_power="{{ validator_power | join(',') }}" +tm_pub_key_access_port="9986" +tm_p2p_port="{{ tendermint_p2p_port|string }}" +tm_proxy_app="{{ ansible_hostname|string }}" +tm_abci_port="{{ tendermint_abci_port|string }}" +tm_instance_name="{{ ansible_hostname }}" + +{% raw %} +# Container vars +RETRIES=0 +CANNOT_INITIATLIZE_INSTANCE='Cannot start instance, if initial validator(s) are unreachable.' + +TM_GENESIS_FILE=/tendermint/config/genesis.json +TM_PUB_KEY_DIR=/usr/share/nginx + +if [ ! -f /tendermint/config/priv_validator.json ]; then + /usr/local/bin/tendermint gen_validator > /tendermint/config/priv_validator.json + # pub_key.json will be served by the nginx container + cat /tendermint/config/priv_validator.json + cat /tendermint/config/priv_validator.json | jq ".pub_key" > "$TM_PUB_KEY_DIR"/pub_key.json +fi + +if [ ! -f /tendermint/config/node_key.json ]; then + /usr/local/bin/tendermint --home "/tendermint" gen_node_key > "$TM_PUB_KEY_DIR"/address +fi + +# fill genesis file with validators +IFS=',' read -ra VALS_ARR <<< "$tm_validators" +IFS=',' read -ra VAL_POWERS_ARR <<< "$tm_validator_power" +if [ ${#VALS_ARR[@]} -ne ${#VAL_POWERS_ARR[@]} ]; then + echo "Invalid configuration of Validator(s) and Validator Power(s)" + exit 1 +fi +for i in "${!VALS_ARR[@]}"; do + # wait until validator generates priv/pub key pair + set +e + echo Validator: "${VALS_ARR[$i]}" + echo Validator Power: "${VAL_POWERS_ARR[$i]}" + echo "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + RETRIES=$((RETRIES+1)) + if [ $RETRIES -eq 10 ]; then + echo "${CANNOT_INITIATLIZE_INSTANCE}" + exit 1 + fi + # 300(30 * 10(retries)) second timeout before container dies if it cannot find initial peers + sleep 30 + curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null + ERR=$? + echo "Cannot connect to Tendermint instance: ${VALS_ARR[$i]}" + done + set -e + # add validator to genesis file along with its pub_key + curl -s "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" | jq ". as \$k | {pub_key: \$k, power: ${VAL_POWERS_ARR[$i]}, name: \"${VALS_ARR[$i]}\"}" > pub_validator.json + cat /tendermint/config/genesis.json | jq ".validators |= .+ [$(cat pub_validator.json)]" > tmpgenesis && mv tmpgenesis /tendermint/config/genesis.json + rm pub_validator.json + done +# construct persistent peers +IFS=',' read -ra PEERS_ARR <<< "$tm_persistent_peers" +peers=() +for s in "${PEERS_ARR[@]}"; do + echo "http://$s:$tm_pub_key_access_port/address" + curl -s --fail "http://$s:$tm_pub_key_access_port/address" > /dev/null + ERR=$? + while [ "$ERR" != 0 ]; do + RETRIES=$((RETRIES+1)) + if [ $RETRIES -eq 10 ]; then + echo "${CANNOT_INITIATLIZE_INSTANCE}" + exit 1 + fi + # 300(30 * 10(retries)) second timeout before container dies if it cannot find initial peers + sleep 30 + curl -s --fail "http://$s:$tm_pub_key_access_port/address" > /dev/null + ERR=$? + echo "Cannot get address for Tendermint instance: ${s}" + done + peer_addr=$(curl -s "http://$s:$tm_pub_key_access_port/address") + peers+=("$peer_addr@$s:$tm_p2p_port") +done +peers=$(IFS=','; echo "${peers[*]}") + +echo "INFO: starting tendermint..." +/usr/local/bin/tendermint node --home "/tendermint" --p2p.persistent_peers="$peers" --proxy_app="tcp://0.0.0.0:46658" --consensus.create_empty_blocks=false --p2p.pex=false +{% endraw %} \ No newline at end of file diff --git a/pkg/configuration/vars/bdb-config.yml b/pkg/configuration/vars/bdb-config.yml deleted file mode 100644 index f34cd7c7..00000000 --- a/pkg/configuration/vars/bdb-config.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -deploy_docker: false #[true, false] -docker_cluster_size: 1 -upstart: "/bigchaindb/scripts/bootstrap.sh" -bdb_hosts: - - name: "bdb-node-01" - box: - name: "ubuntu/xenial64" - ram: "2048" - vcpus: "2" - network: - ip: "10.20.30.20" - type: "private_network" diff --git a/pkg/configuration/vars/stack-config.yml b/pkg/configuration/vars/stack-config.yml new file mode 100644 index 00000000..e69de29b diff --git a/pkg/scripts/Vagrantfile b/pkg/scripts/Vagrantfile index 01bf6a11..a0962f06 100644 --- a/pkg/scripts/Vagrantfile +++ b/pkg/scripts/Vagrantfile @@ -1,61 +1,102 @@ -Vagrant.require_version ">= 1.8.7" -unless Vagrant.has_plugin?("vagrant-vbguest") - raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`" -end - -unless Vagrant.has_plugin?("vagrant-cachier") - raise "Please install the vagrant-cachier plugin by running `vagrant plugin install vagrant-cachier`" -end +# -*- mode: ruby -*- +# vi: set ft=ruby : +# Required modules +require 'yaml' VAGRANTFILE_API_VERSION = "2" -MEMORY = 4096 -CPU_COUNT = 2 +Vagrant.require_version ">= 1.8.7" +# Validate if all the required plugins are present +# vagrant-hostmanager replaced +required_plugins = ["vagrant-cachier", "vagrant-vbguest", "vagrant-hosts", "vagrant-azure"] +required_plugins.each do |plugin| + if not Vagrant.has_plugin?(plugin) + raise "Required vagrant plugin #{plugin} not found. Please run `vagrant plugin install #{plugin}`" + end +end -MOUNT_DIRS = { - :bigchaindb => {:repo => "bigchaindb", :local => "/opt/stack/bigchaindb", :owner => "edxapp"}, -} +# Configuration files +CONFIGURATION_FILE = 'bigchaindb/pkg/configuration/vars/stack-config.yml' +HOSTS_FILE = 'bigchaindb/pkg/configuration/hosts/all' +HOST_VARS_PATH = 'bigchaindb/pkg/configuration/host_vars' -boxname = ENV['BOXNAME'] || "ubuntu/xenial64" -tm_version = ENV['TM_VERSION'] +# Read configuration file(s) +instances_config = YAML.load_file(File.join(File.dirname(__FILE__), CONFIGURATION_FILE)) +hosts_config = File.open(HOSTS_FILE, 'w+') -$script = <