From 0f68c41b890dd62973cc347b42d2476a1ece2fd5 Mon Sep 17 00:00:00 2001 From: muawiakh Date: Fri, 27 Oct 2017 17:54:19 +0200 Subject: [PATCH] Addressing comments and updating some documentation --- .../source/appendices/run-with-ansible.md | 19 +++++++++------ .../source/appendices/run-with-vagrant.md | 17 ++++++++----- pkg/Vagrantfile | 15 ++++++------ pkg/configuration/bdb-deploy.yml | 2 +- pkg/configuration/host_vars/bdb-node-01 | 2 +- pkg/configuration/hosts/all | 4 ++-- .../roles/bigchaindb/tasks/deploy_docker.yml | 10 ++++---- .../tasks/pub_key_exchange_docker.yml | 2 +- .../templates/exchange_keyring_docker.j2 | 4 ++-- .../roles/mongodb/tasks/deploy_docker.yml | 4 ++-- .../tasks/initiate_repl_set_docker.yml | 4 ++-- .../mongodb/templates/replSet_init_docker.j2 | 6 ++--- pkg/configuration/vars/bdb-config.yml | 24 +++++++++++++++---- 13 files changed, 70 insertions(+), 43 deletions(-) diff --git a/docs/server/source/appendices/run-with-ansible.md b/docs/server/source/appendices/run-with-ansible.md index b3bcdfba..9917841b 100644 --- a/docs/server/source/appendices/run-with-ansible.md +++ b/docs/server/source/appendices/run-with-ansible.md @@ -12,6 +12,10 @@ Currently, this workflow is only supported for the following distributions: - CentOS >= 7 - Fedora >= 24 +## Minimum Requirements +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 ## Clone the BigchainDB repository | Ansible ```text $ git clone https://github.com/bigchaindb/bigchaindb.git @@ -49,7 +53,8 @@ $ cd bigchaindb/pkg/configuration/hosts Edit `all` configuration file: ```text # Delete any existing configuration in this file and insert -localhost ansible_connection=local +# Hostname of dev machine + ansible_connection=local ``` ##### Update Configuration | Local Navigate to `bigchaindb/pkg/configuration/vars` inside the BigchainDB repository. @@ -61,16 +66,16 @@ Edit `bdb-config.yml` configuration file as per your requirements, sample config ```text --- deploy_docker: false #[true, false] -docker_replset_size: 1 # Only needed if `deploy_docker` is true +docker_cluster_size: 1 # Only needed if `deploy_docker` is true bdb_hosts: - - name: "" + - name: "HOSTNAME>" # Hostname of dev machine ``` **Note**: You can also orchestrate a multi-node BigchainDB cluster on a local dev host using Docker containers. Here is a sample `bdb-config.yml` ```text --- deploy_docker: true #[true, false] -docker_replset_size: 3 +docker_cluster_size: 3 bdb_hosts: - name: "" ``` @@ -109,7 +114,7 @@ Edit `bdb-config.yml` configuration file as per your requirements, sample config ```text --- deploy_docker: false #[true, false] -docker_replset_size: 1 # Only needed if `deploy_docker` is true +docker_cluster_size: 1 # Only needed if `deploy_docker` is true bdb_hosts: - name: "" ``` @@ -120,7 +125,7 @@ Now, You can safely run the `bdb-deploy.yml` playbook and everything will be tak ```text $ cd bigchaindb/pkg/configuration/ -$ sudo ansible-playbook bdb-deploy.yml -i /bigchaindb/configuration/hosts/all +$ sudo ansible-playbook bdb-deploy.yml -i hosts/all ``` After successfull execution of the playbook, you can verify that BigchainDB docker(s)/process(es) is(are) running. @@ -137,7 +142,7 @@ Verify BigchainDB Docker(s): $ docker ps | grep bigchaindb ``` -The playbook also installs the BigchainDB Python Driver, +The playbook also installs the BigchainDB Python Driver, so you can use it to make transactions and verify the functionality of your BigchainDB node. See the [BigchainDB Python Driver documentation](https://docs.bigchaindb.com/projects/py-driver/en/latest/index.html) diff --git a/docs/server/source/appendices/run-with-vagrant.md b/docs/server/source/appendices/run-with-vagrant.md index e70dd715..4c61ad9c 100644 --- a/docs/server/source/appendices/run-with-vagrant.md +++ b/docs/server/source/appendices/run-with-vagrant.md @@ -29,6 +29,11 @@ can also tweak the following configurations for the BigchainDB node(s). - Virtualbox - VMware +## Minimum Requirements +Minimum resource requirements for a single node BigchainDB dev setup. **The more the better**: +- Memory >= 512MB +- VCPUs >= 1 + ## Install dependencies | Vagrant 1. [VirtualBox](https://www.virtualbox.org/wiki/Downloads) >= 5.0.0 2. [Vagrant](https://www.vagrantup.com/downloads.html) >= 1.16.0 @@ -39,9 +44,9 @@ $ git clone https://github.com/bigchaindb/bigchaindb.git ``` ## Configuration | Vagrant -Navigate to `bigchaindb/pkg/config/` inside the BigchainDB repository. +Navigate to `bigchaindb/pkg/configuration/vars/` inside the BigchainDB repository. ```text -$ cd bigchaindb/pkg/config/ +$ cd bigchaindb/pkg/configuration/vars/ ``` Edit `bdb-config.yml` as per your requirements. Sample `bdb-config.yml`: @@ -49,7 +54,7 @@ Edit `bdb-config.yml` as per your requirements. Sample `bdb-config.yml`: ```text --- deploy_docker: false #[true, false] -docker_replset_size: 1 +docker_cluster_size: 1 upstart: "/bigchaindb/scripts/bootstrap.sh" bdb_hosts: - name: "bdb-node-01" @@ -67,7 +72,7 @@ Here is a sample `bdb-config.yml`: ```text --- deploy_docker: false #[true, false] -docker_replset_size: 1 +docker_cluster_size: 1 upstart: "/bigchaindb/scripts/bootstrap.sh" bdb_hosts: - name: "bdb-node-01" @@ -92,7 +97,7 @@ Here is a sample `bdb-config.yml` ```text --- deploy_docker: true #[true, false] -docker_replset_size: 3 +docker_cluster_size: 3 upstart: "/bigchaindb/scripts/bootstrap.sh" bdb_hosts: - name: "bdb-node-01" @@ -115,7 +120,7 @@ $ vagrant up ``` **Note**: There are some vagrant plugins required for the installation, -user will be prompted to install them if they are not present. Instructions +user will be prompted to install them if they are not present. Instructions to install the plugins can be extracted from the message. ```text diff --git a/pkg/Vagrantfile b/pkg/Vagrantfile index 8bfebe94..4031df07 100644 --- a/pkg/Vagrantfile +++ b/pkg/Vagrantfile @@ -31,11 +31,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # using -u ubuntu as remote user, conventionally vagrant boxes use `vagrant` user if instance["box"]["name"] == "ubuntu/xenial64" hosts_config.puts("#{instance["name"]} ansible_user=ubuntu") - if Vagrant.has_plugin?("vagrant-vbguest") - config.vbguest.auto_update = false - config.vbguest.no_install = true - config.vbguest.no_remote = true - end else hosts_config.puts("#{instance["name"]} ansible_user=vagrant") end @@ -43,7 +38,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Workaround until vagrant cachier plugin supports dnf if !(instance["box"]["name"].include? "fedora") if Vagrant.has_plugin?("vagrant-cachier") - config.cache.scope = :box + bdb.cache.scope = :box + end + elsif instance["box"]["name"] == "ubuntu/xenial64" + if Vagrant.has_plugin?("vagrant-vbguest") + bdb.vbguest.auto_update = false + bdb.vbguest.no_install = true + bdb.vbguest.no_remote = true end end bdb.vm.hostname = instance["name"] @@ -80,7 +81,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| bdb.vm.network "private_network", ip: "192.168.100.200" bdb.vm.provision :shell, inline: "cd /bigchaindb/scripts;/bin/bash #{instances_config["upstart"]}" bdb.vm.provision :shell, inline: "PYTHONUNBUFFERED=1 ansible-playbook /bigchaindb/configuration/bdb-deploy.yml \ - -c /bigchaindb/configuration/hosts/all" + -i /bigchaindb/configuration/hosts/all" bdb.vm.provider "virtualbox" do |vb| vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] vb.memory = 2048 diff --git a/pkg/configuration/bdb-deploy.yml b/pkg/configuration/bdb-deploy.yml index 9bd93d98..5c4b5786 100644 --- a/pkg/configuration/bdb-deploy.yml +++ b/pkg/configuration/bdb-deploy.yml @@ -9,4 +9,4 @@ - bigchaindb-driver - import_playbook: multi_node.yml - when: (bdb_hosts|length > 1) or docker_replset_size|int > 1 \ No newline at end of file + when: (bdb_hosts|length > 1) or docker_cluster_size|int > 1 \ No newline at end of file diff --git a/pkg/configuration/host_vars/bdb-node-01 b/pkg/configuration/host_vars/bdb-node-01 index e20d655a..fef41e4a 100644 --- a/pkg/configuration/host_vars/bdb-node-01 +++ b/pkg/configuration/host_vars/bdb-node-01 @@ -2,4 +2,4 @@ # populates this dynamically. # Only needed for logging into remote hosts and adding host specific variables e.g. -#ansible_ssh_private_key_file: "/path/to/private/key" \ No newline at end of file +#ansible_ssh_private_key_file: "/path/to/private/key" diff --git a/pkg/configuration/hosts/all b/pkg/configuration/hosts/all index 1f7800f2..f84cc374 100644 --- a/pkg/configuration/hosts/all +++ b/pkg/configuration/hosts/all @@ -2,7 +2,7 @@ # populates this dynamically. # For local host -#localhost ansible_connection=local +# ansible_connection=local # For remote host(s) -# ansible_ssh_user= ansible_sudo_pass= \ No newline at end of file +# ansible_ssh_user= ansible_sudo_pass= diff --git a/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml index 82ca060e..e8ebbb63 100644 --- a/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml +++ b/pkg/configuration/roles/bigchaindb/tasks/deploy_docker.yml @@ -2,7 +2,7 @@ - name: Check if BigchainDB Dockers are already configured stat: path: "{{ bigchaindb_host_mount_dir }}{{ item|string }}/.bigchaindb" - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 register: stat_result - name: Configuring BigchainDB Docker @@ -19,7 +19,7 @@ networks: - name: "{{ bdb_docker_net_name }}" when: stat_result.results[item|int].stat.exists == False - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [bigchaindb] - name: Start BigchainDB Docker @@ -35,17 +35,17 @@ networks: - name: "{{ bdb_docker_net_name }}" state: started - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [bigchaindb] - name: Get BigchainDB node public key shell: "cat {{ bigchaindb_host_mount_dir + item|string }}/.bigchaindb" register: bdb_node_config - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [bigchaindb] - name: Set facts for BigchainDB containers set_fact: pub_key_{{ bigchaindb_docker_name }}{{ item }}="{{ (bdb_node_config.results[item|int].stdout|from_json).keypair.public }}" - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml index 87772617..bd5be749 100644 --- a/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml +++ b/pkg/configuration/roles/key-exchange/tasks/pub_key_exchange_docker.yml @@ -27,5 +27,5 @@ restart: true networks: - name: "{{ bdb_docker_net_name }}" - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [bigchaindb] \ No newline at end of file diff --git a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 index 04c30716..31aed4fb 100644 --- a/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 +++ b/pkg/configuration/roles/key-exchange/templates/exchange_keyring_docker.j2 @@ -1,10 +1,10 @@ #!/usr/bin/python import json {% set keyring = {} %} -{% for docker in range(0, docker_replset_size|int, 1) %} +{% for docker in range(0, docker_cluster_size|int, 1) %} {{- keyring.update({'pub_key_' + bigchaindb_docker_name + docker|string: hostvars[ansible_hostname]['pub_key_' + bigchaindb_docker_name + docker|string]}) -}} {%- endfor -%} -{% for docker in range(0, docker_replset_size|int, 1) %} +{% for docker in range(0, docker_cluster_size|int, 1) %} keyring = {{ keyring }} keyring.pop('{{ 'pub_key_' + bigchaindb_docker_name + docker|string }}', None) with open('{{ bigchaindb_config_path_docker + docker|string }}/.bigchaindb', 'r+') as f: diff --git a/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml index 52ec3fde..295f2105 100644 --- a/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml +++ b/pkg/configuration/roles/mongodb/tasks/deploy_docker.yml @@ -38,11 +38,11 @@ networks: - name: "{{ bdb_docker_net_name }}" register: mongo_container_info - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [mongodb] - name: Set facts for MongoDB containers set_fact: mongodb{{ item }}={{ mongo_container_info.results[item|int].ansible_facts.docker_container.NetworkSettings.IPAddress }} - with_sequence: start=0 end="{{ docker_replset_size|int - 1 }}" stride=1 + with_sequence: start=0 end="{{ docker_cluster_size|int - 1 }}" stride=1 tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml index af5b89ca..5cd341fa 100644 --- a/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml +++ b/pkg/configuration/roles/mongodb/tasks/initiate_repl_set_docker.yml @@ -7,7 +7,7 @@ run_once: true shell: cmd: - "docker exec {{ mongodb_docker_name }}{{ docker_replset_size|int - 1 }} bash -l -c - '/usr/bin/mongo --host {{ mongodb_docker_name }}{{ docker_replset_size|int - 1 }} + "docker exec {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} bash -l -c + '/usr/bin/mongo --host {{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }} --port {{ mongodb_port }} < /bdb_config/replSet_init.js'" tags: [mongodb] \ No newline at end of file diff --git a/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 index a0121b19..e1764993 100644 --- a/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 +++ b/pkg/configuration/roles/mongodb/templates/replSet_init_docker.j2 @@ -3,13 +3,13 @@ rs.initiate({ "members": [ { "_id": 0, - "host": "{{ mongodb_docker_name }}{{ docker_replset_size|int - 1 }}:{{ mongodb_port }}" + "host": "{{ mongodb_docker_name }}{{ docker_cluster_size|int - 1 }}:{{ mongodb_port }}" } ] }); sleep(5000); -{% for docker in range(0, docker_replset_size|int, 1) %} -{%- if docker != (docker_replset_size|int - 1) -%} +{% for docker in range(0, docker_cluster_size|int, 1) %} +{%- if docker != (docker_cluster_size|int - 1) -%} rs.add("{{ mongodb_docker_name }}{{ docker }}:{{ mongodb_port }}"); {% endif %} {%- endfor -%} diff --git a/pkg/configuration/vars/bdb-config.yml b/pkg/configuration/vars/bdb-config.yml index 952738b2..6dc79b30 100644 --- a/pkg/configuration/vars/bdb-config.yml +++ b/pkg/configuration/vars/bdb-config.yml @@ -1,13 +1,29 @@ --- deploy_docker: false #[true, false] -docker_replset_size: 1 +docker_cluster_size: 1 upstart: "/bigchaindb/scripts/bootstrap.sh" bdb_hosts: - - name: "config-node" + - name: "bdb-node-01" box: name: "ubuntu/xenial64" ram: "2048" - vcpus: "2" + vcpus: "1" + network: + ip: "10.20.30.20" + type: "private_network" + - name: "bdb-node-02" + box: + name: "centos/7" + ram: "2048" + vcpus: "1" + network: + ip: "10.20.30.30" + type: "private_network" + - name: "bdb-node-03" + box: + name: "bento/fedora-24" + ram: "2048" + vcpus: "1" network: ip: "10.20.30.40" - type: "private_network" \ No newline at end of file + type: "private_network"