diff --git a/deploy-cluster-aws/fabfile-monitor.py b/deploy-cluster-aws/fabfile-monitor.py new file mode 100644 index 00000000..4d26097b --- /dev/null +++ b/deploy-cluster-aws/fabfile-monitor.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""A Fabric fabfile with functionality to install Docker, +install Docker Compose, and run a BigchainDB monitoring server +(using the docker-compose-monitor.yml file) +""" + +from __future__ import with_statement, unicode_literals + +from fabric.api import sudo, env +from fabric.api import task +from fabric.operations import put, run + +# Ignore known_hosts +# http://docs.fabfile.org/en/1.10/usage/env.html#disable-known-hosts +env.disable_known_hosts = True + +env.user = 'ubuntu' +env.key_filename = 'pem/bigchaindb.pem' + + +@task +def install_docker_engine(): + """Install Docker on an EC2 Ubuntu 14.04 instance + + Example: + fab --fabfile=fabfile-monitor.py \ + --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ + install_docker_engine + """ + + # install prerequisites + sudo('apt-get update') + sudo('apt-get -y install apt-transport-https ca-certificates linux-image-extra-$(uname -r) apparmor') + + # install docker repositories + sudo('apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 \ + --recv-keys 58118E89F3A912897C070ADBF76221572C52609D') + sudo("echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' | \ + sudo tee /etc/apt/sources.list.d/docker.list") + + # install docker engine + sudo('apt-get update') + sudo('apt-get -y install docker-engine') + + # add ubuntu user to the docker group + sudo('usermod -aG docker ubuntu') + + +@task +def install_docker_compose(): + """Install Docker Compose on an EC2 Ubuntu 14.04 instance + + Example: + fab --fabfile=fabfile-monitor.py \ + --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ + install_docker_compose + """ + sudo('curl -L https://github.com/docker/compose/releases/download/1.7.0/docker-compose-`uname \ + -s`-`uname -m` > /usr/local/bin/docker-compose') + sudo('chmod +x /usr/local/bin/docker-compose') + + +@task +def install_docker(): + """Install Docker and Docker Compose on an EC2 Ubuntu 14.04 instance + + Example: + fab --fabfile=fabfile-monitor.py \ + --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ + install_docker + """ + install_docker_engine() + install_docker_compose() + + +@task +def run_monitor(): + """Run BigchainDB monitor on an EC2 Ubuntu 14.04 instance + + Example: + fab --fabfile=fabfile-monitor.py \ + --hosts=ec2-52-58-106-17.eu-central-1.compute.amazonaws.com \ + run_monitor + """ + # copy docker-compose-monitor to the ec2 instance + put('../docker-compose-monitor.yml') + run('INFLUXDB_DATA=/influxdb-data docker-compose -f docker-compose-monitor.yml up -d') diff --git a/docker-compose-monitor.yml b/docker-compose-monitor.yml index 6865931d..e695387c 100644 --- a/docker-compose-monitor.yml +++ b/docker-compose-monitor.yml @@ -1,25 +1,28 @@ -influxdb: - image: tutum/influxdb - ports: - - "8083:8083" - - "8086:8086" - expose: - - "8090" - - "8099" - environment: - PRE_CREATE_DB: "telegraf" +version: '2' +services: + influxdb: + image: tutum/influxdb + ports: + - "8083:8083" + - "8086:8086" + - "8090" + - "8099" + environment: + PRE_CREATE_DB: "telegraf" + volumes: + - $INFLUXDB_DATA:/data -grafana: - image: bigchaindb/grafana-bigchaindb-docker - tty: true - ports: - - "3000:3000" - links: - - influxdb:localhost + grafana: + image: bigchaindb/grafana-bigchaindb-docker + tty: true + ports: + - "3000:3000" + environment: + INFLUXDB_HOST: "influxdb" -statsd: - image: bigchaindb/docker-telegraf-statsd - ports: - - "8125:8125/udp" - links: - - influxdb:localhost \ No newline at end of file + statsd: + image: bigchaindb/docker-telegraf-statsd + ports: + - "8125:8125/udp" + environment: + INFLUXDB_HOST: "influxdb" \ No newline at end of file diff --git a/docs/source/deploy-on-aws.md b/docs/source/deploy-on-aws.md index 49c83021..a27e686a 100644 --- a/docs/source/deploy-on-aws.md +++ b/docs/source/deploy-on-aws.md @@ -83,9 +83,53 @@ Add some rules for Inbound traffic: **Note: These rules are extremely lax! They're meant to make testing easy.** You'll want to tighten them up if you intend to have a secure cluster. For example, Source = 0.0.0.0/0 is [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) for "allow this traffic to come from _any_ IP address." -## AWS Deployment +## Deploy a BigchainDB Monitor -### AWS Deployment Step 1 +This step is optional. + +One way to monitor a BigchainDB cluster is to use the monitoring setup described in the [Monitoring](monitoring.html) section of this documentation. If you want to do that, then you may want to deploy the monitoring server first, so you can tell your BigchainDB nodes where to send their monitoring data. + +You can deploy a monitoring server on AWS. To do that, go to the AWS EC2 Console and launch an instance: + +1. Choose an AMI: select Ubuntu Server 14.04 LTS. +2. Choose an Instance Type: a t2.micro will suffice. +3. Configure Instance Details: you can accept the defaults, but feel free to change them. +4. Add Storage: A "Root" volume type should already be included. You _could_ store monitoring data there (e.g. in a folder named `/influxdb-data`) but we will attach another volume and store the monitoring data there instead. Select "Add New Volume" and an EBS volume type. +5. Tag Instance: give your instance a memorable name. +6. Configure Security Group: choose your bigchaindb security group. +7. Review and launch your instance. + +When it asks, choose an existing key pair: the one you created earlier (named `bigchaindb`). + +Give your instance some time to launch and become able to accept SSH connections. You can see its current status in the AWS EC2 Console (in the "Instances" section). SSH into your instance using something like: +```text +cd deploy-cluster-aws +ssh -i pem/bigchaindb.pem ubuntu@ec2-52-58-157-229.eu-central-1.compute.amazonaws.com +``` + +where `ec2-52-58-157-229.eu-central-1.compute.amazonaws.com` should be replaced by your new instance's EC2 hostname. (To get that, go to the AWS EC2 Console, select Instances, click on your newly-launched instance, and copy its "Public DNS" name.) + +Next, create a file system on the attached volume, make a directory named `/influxdb-data`, and set the attached volume's mount point to be `/influxdb-data`. For detailed instructions on how to do that, see the AWS documentation for [Making an Amazon EBS Volume Available for Use](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-using-volumes.html). + +Then install Docker and Docker Compose: +```text +# in a Python 2.5-2.7 virtual environment where fabric, boto3, etc. are installed +fab --fabfile=fabfile-monitor.py --hosts= install_docker +``` + +After Docker is installed, we can run the monitor with: +```text +fab --fabfile=fabfile-monitor.py --hosts= run_monitor +``` + +For more information about monitoring (e.g. how to view the Grafana dashboard in your web browser), see the [Monitoring](monitoring.html) section of this documentation. + +To configure a BigchainDB node to send monitoring data to the monitoring server, change the statsd host in the configuration of the BigchainDB node. The section on [Configuring a BigchainDB Node](configuration.html) explains how you can do that. (For example, you can change the statsd host in `$HOME/.bigchaindb`.) + + +## Deploy a BigchainDB Cluster + +### Step 1 Suppose _N_ is the number of nodes you want in your BigchainDB cluster. If you already have a set of _N_ BigchainDB configuration files in the `deploy-cluster-aws/confiles` directory, then you can jump to step 2. To create such a set, you can do something like: ```text @@ -99,7 +143,7 @@ That will create three (3) _default_ BigchainDB configuration files in the `depl You can look inside those files if you're curious. In step 2, they'll be modified. For example, the default keyring is an empty list. In step 2, the deployment script automatically changes the keyring of each node to be a list of the public keys of all other nodes. Other changes are also made. -### AWS Deployment Step 2 +### Step 2 Step 2 is to launch the nodes ("instances") on AWS, to install all the necessary software on them, configure the software, run the software, and more. diff --git a/docs/source/monitoring.md b/docs/source/monitoring.md index 4fb53072..acae3fea 100644 --- a/docs/source/monitoring.md +++ b/docs/source/monitoring.md @@ -16,10 +16,18 @@ $ docker-compose -f docker-compose-monitor.yml build $ docker-compose -f docker-compose-monitor.yml up ``` -then point a browser tab to: +It is also possible to mount a host directory as a data volume for InfluxDB +by setting the `INFLUXDB_DATA` environment variable: +```text +$ INFLUXDB_DATA=/data docker-compose -f docker-compose-monitor.yml up +``` + +You can view the Grafana dashboard in your web browser at: [http://localhost:3000/dashboard/script/bigchaindb_dashboard.js](http://localhost:3000/dashboard/script/bigchaindb_dashboard.js) +(You may want to replace `localhost` with another hostname in that URL, e.g. the hostname of a remote monitoring server.) + The login and password are `admin` by default. If BigchainDB is running and processing transactions, you should see analytics—if not, [start BigchainDB](installing-server.html#run-bigchaindb) and load some test transactions: ```text $ bigchaindb load