Secure WebSocket Support (#1619)

* Remove support for whitelist

* Rename nginx to nginx-api

* Remove websocket support from nginx-api

* Change nginx to nginx-api service
The nginx-api service will proxy requests to the BigchainDB HTTP API.

* Rename ngx-instance-0 to ngx-api-instance-0 in nginx_3scale

* Update nginx-api service base docker image and README

* Add nginx-ws service to support Websocket

* Add config files for simple dev setup

* WS support with split NGINX

* NGINX module as single entrypoint into the cluster.

* Tested HTTP and WS with latest configs

* Openresty as separate service

* Remove upstream api port as configuration parameter

* Changes while testing

* Adding READMEs for nginx-http and nginx-https modules

* Documentation update

* Change 'Openresty' to 'OpenResty'.

k8s/nginx-https/container/Dockerfile

@@ -0,0 +1,11 @@
+FROM nginx:1.13.1
+LABEL maintainer "dev@bigchaindb.com"
+WORKDIR /
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get autoremove \
+    && apt-get clean
+COPY nginx.conf.template /etc/nginx/nginx.conf
+COPY nginx_entrypoint.bash /
+EXPOSE 80 443 27017
+ENTRYPOINT ["/nginx_entrypoint.bash"]

k8s/nginx-https/container/README.md

@@ -0,0 +1,15 @@
+## Nginx container for Secure WebSocket Support
+
+
+### Step 1: Build and Push the Latest Container
+Use the `docker_build_and_push.bash` script to build the latest docker image
+and upload it to Docker Hub.
+Ensure that the image tag is updated to a new version number to properly
+reflect any changes made to the container.
+
+
+### Note about testing Websocket connections:
+You can test the WebSocket server by using 
+[wsc](https://www.npmjs.com/package/wsc) tool with a command like:
+
+`wsc -er wss://localhost:9985/api/v1/streams/valid_transactions`.

k8s/nginx-https/container/docker_build_and_push.bash

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker build -t bigchaindb/nginx_https:1.0 .
+
+docker push bigchaindb/nginx_https:1.0

k8s/nginx-https/container/nginx.conf.template

@@ -0,0 +1,193 @@
+# Frontend API server that:
+# 1. Acts as the HTTPS termination point.
+# 2. Forwards BDB HTTP requests to OpenResty backend.
+# 3. Forwards BDB WS requests to BDB backend.
+# 4. Forwards MDB TCP connections to MDB backend.
+# 5. Forwards requests from 3scale to OpenResty backend.
+# 6. Does health check with LB.
+
+worker_processes 2;
+daemon off;
+user nobody nogroup;
+pid /tmp/nginx.pid;
+error_log /dev/stderr;
+
+events {
+  # Each worker handles up to 512 connections. Increase this for heavy
+  # workloads.
+  worker_connections 512;
+  accept_mutex on;
+  use epoll;
+}
+
+http {
+  access_log /dev/stdout combined buffer=16k flush=5s;
+
+  # Allow 10 req/sec from the same IP address, and store the counters in a
+  # `zone` or shared memory location tagged as 'one'.
+  limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s;
+
+  # Enable logging when requests are being throttled.
+  limit_req_log_level notice;
+
+  # HTTP status code that is returned to the client; 429 is for TooManyRequests,
+  # ref. RFC 6585
+  limit_req_status 429;
+
+  # Limit requests from the same client, allow `burst` to 20 r/s,
+  # `nodelay` or drop connection immediately in case it exceeds this
+  # threshold.
+  limit_req zone=one burst=20 nodelay;
+
+  # `slowloris` attack mitigation settings.
+  client_body_timeout 10s;
+  client_header_timeout 10s;
+
+  # DNS resolver to use for all the backend names specified in this configuration.
+  resolver DNS_SERVER valid=30s ipv6=off;
+
+  keepalive_timeout 60s;
+
+  # The following map blocks enable lazy-binding to the backend at runtime,
+  # rather than binding as soon as NGINX starts.
+  map $remote_addr $bdb_backend {
+    default BIGCHAINDB_BACKEND_HOST;
+  }
+  map $remote_addr $openresty_backend {
+    default OPENRESTY_BACKEND_HOST;
+  }
+
+  # Frontend server for the external clients; acts as HTTPS termination point.
+  server {
+    listen CLUSTER_FRONTEND_PORT ssl;
+    server_name "CLUSTER_FQDN";
+    ssl_certificate        /etc/nginx/ssl/cert.pem;
+    ssl_certificate_key    /etc/nginx/ssl/cert.key;
+    ssl_protocols          TLSv1 TLSv1.1 TLSv1.2;
+    ssl_ciphers            HIGH:!aNULL:!MD5;
+
+    underscores_in_headers on;
+
+    # Forward websockets directly to backend BDB.
+    location /api/v1/streams/valid_transactions {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      proxy_pass http://$bdb_backend:BIGCHAINDB_WS_PORT;
+      proxy_read_timeout 600s;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+    }
+
+    # Forward other URL paths as per business logic/use case to BDB or
+    # OpenResty instance.
+    location / {
+      proxy_ignore_client_abort on;
+      proxy_set_header X-Real-IP  $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      # max client request body size: avg transaction size.
+      client_max_body_size 15k;
+
+      # No auth for GETs, forward directly to BDB.
+      if ($request_method = GET) {
+        proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT;
+      }
+
+      # POST requests get forwarded to OpenResty instance. Enable CORS too.
+      if ($request_method = POST ) {
+        add_header 'Access-Control-Allow-Origin' '*';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range';
+        add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range';
+        
+        proxy_pass http://$openresty_backend:OPENRESTY_BACKEND_PORT;
+      }
+
+      # OPTIONS requests handling for CORS.
+      if ($request_method = 'OPTIONS') {
+        add_header 'Access-Control-Allow-Origin' '*';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id';
+        add_header 'Access-Control-Max-Age' 43200;
+        add_header 'Content-Type' 'text/plain charset=UTF-8';
+        add_header 'Content-Length' 0;
+        return 204;
+      }
+    }
+  }
+
+  # Frontend server for the load balancer to respond to health checks.
+  server {
+    listen HEALTH_CHECK_PORT;
+
+    location = /health {
+      return 200;
+    }
+  }
+
+  # Frontend server for the external clients; returns a pretty error message
+  # when an HTTP request is sent instead of HTTPS.
+  server {
+    listen 80;
+    server_name "CLUSTER_FQDN";
+
+    location / {
+        add_header Upgrade "TLS/1.2, HTTP/1.1" always;
+        default_type text/plain;
+        return 426 'Consider using the HTTPS protocol next time!';
+    }
+  }
+
+  # Frontend server to respond to requests from 3scale; forward directly to
+  # OpenResty instance.
+  server {
+    sendfile on;
+    listen THREESCALE_API_PORT;
+
+    location / {
+      proxy_pass http://$openresty_backend:OPENRESTY_BACKEND_PORT;
+    }
+  }
+}
+
+# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP
+# connection.
+stream {
+  log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
+                     '$proxy_protocol_addr $proxy_protocol_port '
+                     '$protocol $status $session_time $bytes_sent '
+                     '$bytes_received "$upstream_addr" "$upstream_bytes_sent" '
+                     '"$upstream_bytes_received" "$upstream_connect_time" ';
+
+  access_log /dev/stdout mdb_log buffer=16k flush=5s;
+
+  # Define a zone 'two' of size 10 megabytes to store the counters
+  # that hold number of TCP connections from a specific IP address.
+  limit_conn_zone $binary_remote_addr zone=two:10m;
+
+  # Enable logging when connections are being throttled.
+  limit_conn_log_level notice;
+  
+  # Allow 16 connections from the same IP address.
+  limit_conn two 16;
+    
+  # DNS resolver to use for all the backend names specified in this configuration.
+  resolver DNS_SERVER valid=30s ipv6=off;
+
+  # The following map block enables lazy-binding to the backend at runtime,
+  # rather than binding as soon as NGINX starts.
+  map $remote_addr $mdb_backend {
+    default MONGODB_BACKEND_HOST;
+  }
+  
+  # Frontend server to forward connections to MDB instance.
+  server {
+    listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5;
+    preread_timeout 30s;
+    tcp_nodelay on;
+    proxy_pass $mdb_backend:MONGODB_BACKEND_PORT;
+  }
+}
+

k8s/nginx-https/container/nginx_entrypoint.bash

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -euo pipefail
+
+# Cluster vars
+cluster_fqdn=`printenv CLUSTER_FQDN`
+cluster_frontend_port=`printenv CLUSTER_FRONTEND_PORT`
+
+
+# NGINX vars
+dns_server=`printenv DNS_SERVER`
+health_check_port=`printenv HEALTH_CHECK_PORT`
+
+
+# MongoDB vars
+mongo_frontend_port=`printenv MONGODB_FRONTEND_PORT`
+mongo_backend_host=`printenv MONGODB_BACKEND_HOST`
+mongo_backend_port=`printenv MONGODB_BACKEND_PORT`
+
+
+# OpenResty vars
+openresty_backend_host=`printenv OPENRESTY_BACKEND_HOST`
+openresty_backend_port=`printenv OPENRESTY_BACKEND_PORT`
+threescale_api_port=`printenv THREESCALE_API_PORT`
+
+
+# BigchainDB vars
+bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST`
+bdb_api_port=`printenv BIGCHAINDB_API_PORT`
+bdb_ws_port=`printenv BIGCHAINDB_WS_PORT`
+
+
+# sanity check
+if [[ -z "${cluster_frontend_port}" || \
+      -z "${mongo_frontend_port}" || \
+      -z "${mongo_backend_host}" || \
+      -z "${mongo_backend_port}" || \
+      -z "${openresty_backend_port}" || \
+      -z "${openresty_backend_host}" || \
+      -z "${threescale_api_port}" || \
+      -z "${bdb_backend_host}" || \
+      -z "${bdb_api_port}" || \
+      -z "${bdb_ws_port}" || \
+      -z "${dns_server}" || \
+      -z "${health_check_port}" || \
+      -z "${cluster_fqdn}" ]]; then
+  echo "Invalid environment settings detected. Exiting!"
+  exit 1
+fi
+
+NGINX_CONF_FILE=/etc/nginx/nginx.conf
+
+# configure the nginx.conf file with env variables
+sed -i "s|CLUSTER_FQDN|${cluster_fqdn}|g" ${NGINX_CONF_FILE}
+sed -i "s|CLUSTER_FRONTEND_PORT|${cluster_frontend_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|MONGODB_FRONTEND_PORT|${mongo_frontend_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" ${NGINX_CONF_FILE}
+sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|OPENRESTY_BACKEND_PORT|${openresty_backend_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|OPENRESTY_BACKEND_HOST|${openresty_backend_host}|g" ${NGINX_CONF_FILE}
+sed -i "s|THREESCALE_API_PORT|${threescale_api_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|BIGCHAINDB_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE}
+sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE}
+sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE}
+sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE}
+
+# start nginx
+echo "INFO: starting nginx..."
+exec nginx -c /etc/nginx/nginx.conf
+

k8s/nginx-https/nginx-https-dep.yaml

@@ -0,0 +1,118 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: ngx-https-instance-0-dep
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: ngx-https-instance-0-dep
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: nginx-https
+        image: bigchaindb/nginx_https:1.0
+        imagePullPolicy: Always
+        env:
+        - name: CLUSTER_FRONTEND_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: cluster-frontend-port
+        - name: HEALTH_CHECK_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: cluster-health-check-port
+        - name: CLUSTER_FQDN
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: cluster-fqdn
+        - name: DNS_SERVER
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: cluster-dns-server-ip
+        - name: MONGODB_FRONTEND_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: mongodb-frontend-port
+        - name: MONGODB_BACKEND_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: ngx-mdb-instance-name
+        - name: MONGODB_BACKEND_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: mongodb-backend-port
+        - name: OPENRESTY_BACKEND_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: openresty-backend-port
+        - name: OPENRESTY_BACKEND_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: openresty-backend-host
+        - name: THREESCALE_API_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: threescale-api-port
+        - name: BIGCHAINDB_BACKEND_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: ngx-bdb-instance-name
+        - name: BIGCHAINDB_API_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: bigchaindb-api-port
+        - name: BIGCHAINDB_WS_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: vars
+              key: bigchaindb-ws-port
+        ports:
+        # return a pretty error message on port 80, since we are expecting
+        # HTTPS traffic.
+        - containerPort: 80
+          protocol: TCP
+        - containerPort: "<mongodb-frontend-port from ConfigMap>"
+          protocol: TCP
+        - containerPort: "<cluster-frontend-port from ConfigMap>"
+          protocol: TCP
+        - containerPort: "<threescale-api-port from ConfigMap>"
+          protocol: TCP
+        - containerPort: "<cluster-health-check-port from ConfigMap>"
+          protocol: TCP
+          name: ngx-port
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: ngx-port
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          failureThreshold: 3
+          timeoutSeconds: 10
+        resources:
+          limits:
+            cpu: 200m
+            memory: 768Mi
+        volumeMounts:
+        - name: https-certs
+          mountPath: /etc/nginx/ssl/
+          readOnly: true
+      restartPolicy: Always
+      volumes:
+      - name: https-certs
+        secret:
+          secretName: https-certs
+          defaultMode: 0400

k8s/nginx-https/nginx-https-svc.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ngx-https-instance-0
+  namespace: default
+  labels:
+    name: ngx-https-instance-0
+  annotations:
+    # NOTE: the following annotation is a beta feature and
+    # only available in GCE/GKE and Azure as of now
+    service.beta.kubernetes.io/external-traffic: OnlyLocal
+spec:
+  selector:
+    app: ngx-https-instance-0-dep
+  ports:
+  - port: "<cluster-frontend-port from ConfigMap>"
+    targetPort: "<cluster-frontend-port from ConfigMap>"
+    name: public-secure-cluster-port
+    protocol: TCP
+  - port: "<mongodb-frontend-port from ConfigMap>"
+    targetPort: "<mongodb-frontend-port from ConfigMap>"
+    name: public-mdb-port
+    protocol: TCP
+  - port: "<threescale-api-port from ConfigMap>"
+    targetPort: "<threescale-api-port from ConfigMap>"
+    name: public-threescale-port
+    protocol: TCP
+  - port: 80
+    targetPort: 80
+    name: public-insecure-cluster-port
+    protocol: TCP
+  type: LoadBalancer