Deployment of Tendermint + BigchainDB cluster

- Update existing docker-containers to support tendermint integration, nginx,
  mongodb, bigchaindb
- Add tendermint configuration files for statefulset, pvc, pv and service.
- Update some READMEs.

k8s/nginx-https/container/Dockerfile

@@ -7,5 +7,5 @@ RUN apt-get update \
     && apt-get clean
 COPY nginx.conf.template /etc/nginx/nginx.conf
 COPY nginx_entrypoint.bash /
-EXPOSE 80 443 27017
+EXPOSE 80 443 27017 9986 46656
 ENTRYPOINT ["/nginx_entrypoint.bash"]

k8s/nginx-https/container/README.md

@@ -9,7 +9,7 @@ reflect any changes made to the container.
 
 
 ### Note about testing Websocket connections:
-You can test the WebSocket server by using 
+You can test the WebSocket server by using
 [wsc](https://www.npmjs.com/package/wsc) tool with a command like:
 
 `wsc -er wss://localhost:9985/api/v1/streams/valid_transactions`.

k8s/nginx-https/container/docker_build_and_push.bash

@@ -3,3 +3,7 @@
 docker build -t bigchaindb/nginx_https:1.1 .
 
 docker push bigchaindb/nginx_https:1.1
+
+# For tendermint deployments
+# docker build -t bigchaindb/nginx_https:unstable-tmt .
+# docker push bigchaindb/nginx_https:unstable-tmt

k8s/nginx-https/container/nginx.conf.template

@@ -152,16 +152,16 @@ http {
   }
 }
 
-# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP
-# connection.
+# NGINX stream block for TCP and UDP proxies. Used to proxy MongoDB and Tendermint TCP
+# connections.
 stream {
-  log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
+  log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
                      '$proxy_protocol_addr $proxy_protocol_port '
                      '$protocol $status $session_time $bytes_sent '
                      '$bytes_received "$upstream_addr" "$upstream_bytes_sent" '
                      '"$upstream_bytes_received" "$upstream_connect_time" ';
 
-  access_log /dev/stdout mdb_log buffer=16k flush=5s;
+  access_log /dev/stdout bdb_log buffer=16k flush=5s;
 
   # Define a zone 'two' of size 10 megabytes to store the counters
   # that hold number of TCP connections from a specific IP address.
@@ -186,6 +186,12 @@ stream {
     default MONGODB_BACKEND_HOST;
   }
 
+  # The following map block enables lazy-binding to the backend at runtime,
+  # rather than binding as soon as NGINX starts.
+  map $remote_addr $tm_backend {
+    default TM_BACKEND_HOST;
+  }
+
   # Frontend server to forward connections to MDB instance.
   server {
     listen MONGODB_FRONTEND_PORT so_keepalive=3m:1m:5;
@@ -193,5 +199,20 @@ stream {
     tcp_nodelay on;
     proxy_pass $mdb_backend:MONGODB_BACKEND_PORT;
   }
+
+  # Server to forward connection to nginx instance hosting
+  # tendermint node public key.
+  server {
+    listen TM_PUB_KEY_ACCESS_PORT;
+    proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT;
+  }
+
+  # Server to forward p2p connections to Tendermint instance.
+  server {
+    listen TM_P2P_PORT so_keepalive=3m:1m:5;
+    preread_timeout 60s;
+    tcp_nodelay on;
+    proxy_pass $tm_backend:TM_P2P_PORT;
+  }
 }
 

k8s/nginx-https/container/nginx_entrypoint.bash

@@ -27,6 +27,10 @@ bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST`
 bdb_api_port=`printenv BIGCHAINDB_API_PORT`
 bdb_ws_port=`printenv BIGCHAINDB_WS_PORT`
 
+# Tendermint vars
+tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT`
+tm_backend_host=`printenv TM_BACKEND_HOST`
+tm_p2p_port=`printenv TM_P2P_PORT`
 
 # sanity check
 if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!}" || \
@@ -40,7 +44,11 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!
       -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \
       -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \
       -z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \
-      -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" ]]; then
+      -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" || \
+      -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \
+      -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \
+      -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then
+  echo "Missing required environment variables. Exiting!"
   exit 1
 else
   echo CLUSTER_FQDN="$cluster_fqdn"
@@ -55,6 +63,9 @@ else
   echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host"
   echo BIGCHAINDB_API_PORT="$bdb_api_port"
   echo BIGCHAINDB_WS_PORT="$bdb_ws_port"
+  echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port"
+  echo TM_BACKEND_HOST="$tm_backend_host"
+  echo TM_P2P_PORT="$tm_p2p_port"
 fi
 
 NGINX_CONF_FILE=/etc/nginx/nginx.conf
@@ -72,8 +83,10 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE}
 sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE}
 sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE}
 sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE}
+sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE}
 
 # start nginx
 echo "INFO: starting nginx..."
 exec nginx -c /etc/nginx/nginx.conf
-

k8s/nginx-https/nginx-https-dep.yaml

@@ -75,6 +75,21 @@ spec:
             configMapKeyRef:
               name: vars
               key: bigchaindb-ws-port
+        - name: TM_PUB_KEY_ACCESS_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: tendermint-config
+              key: tm-pub-key-access
+        - name: TM_BACKEND_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: tendermint-config
+              key: ngx-tendermint-instance-name
+        - name: TM_P2P_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: tendermint-config
+              key: tm-p2p-port
         ports:
         # return a pretty error message on port 80, since we are expecting
         # HTTPS traffic.
@@ -87,6 +102,12 @@ spec:
         - containerPort: "<cluster-health-check-port from ConfigMap>"
           protocol: TCP
           name: ngx-port
+        - containerPort: "<tm-pub-key-access from ConfigMap>"
+          protocol: TCP
+          name: tm-pub-key
+        - containerPort: "<tm-p2p-port from ConfigMap>"
+          protocol: TCP
+          name: tm-p2p-port
         livenessProbe:
           httpGet:
             path: /health

k8s/nginx-https/nginx-https-svc.yaml

@@ -25,4 +25,12 @@ spec:
     targetPort: 80
     name: public-insecure-cluster-port
     protocol: TCP
+  - port: "<tm-pub-key-access from ConfigMap>"
+    targetPort: "<tm-pub-key-access from ConfigMap>"
+    name: tm-pub-key-access
+    protocol: TCP
+  - port: "<tm-p2p-port from ConfigMap>"
+    targetPort: "<tm-p2p-port from ConfigMap>"
+    protocol: TCP
+    name: tm-p2p-port
   type: LoadBalancer