diff --git a/k8s/bigchaindb/bigchaindb-dep-tm.yaml b/k8s/bigchaindb/bigchaindb-dep-tm.yaml new file mode 100644 index 00000000..be7da0d6 --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-dep-tm.yaml @@ -0,0 +1,166 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: bdb-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: bdb-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: bigchaindb + image: bigchaindb/bigchaindb:1.3.0 + imagePullPolicy: IfNotPresent + args: + - start + env: + - name: BIGCHAINDB_DATABASE_HOST + valueFrom: + configMapKeyRef: + name: vars + key: mdb-instance-name + - name: BIGCHAINDB_DATABASE_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: BIGCHAINDB_DATABASE_BACKEND + valueFrom: + configMapKeyRef: + name: bdb-config + key: bdb-db-backend + - name: BIGCHAINDB_DATABASE_NAME + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-database-name + - name: BIGCHAINDB_SERVER_BIND + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-server-bind + - name: BIGCHAINDB_WSSERVER_HOST + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-interface + - name: BIGCHAINDB_WSSERVER_ADVERTISED_HOST + valueFrom: + configMapKeyRef: + name: vars + key: cluster-fqdn + - name: BIGCHAINDB_WSSERVER_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-port + - name: BIGCHAINDB_WSSERVER_ADVERTISED_PORT + valueFrom: + configMapKeyRef: + name: vars + key: cluster-frontend-port + - name: BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-wsserver-advertised-scheme + - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-backlog-reassign-delay + - name: BIGCHAINDB_DATABASE_MAXTRIES + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-database-maxtries + - name: BIGCHAINDB_DATABASE_CONNECTION_TIMEOUT + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-database-connection-timeout + - name: BIGCHAINDB_LOG_LEVEL_CONSOLE + valueFrom: + configMapKeyRef: + name: bdb-config + key: bigchaindb-log-level + - name: BIGCHAINDB_DATABASE_SSL + value: "true" + - name: BIGCHAINDB_DATABASE_CA_CERT + value: /etc/bigchaindb/ca/ca.pem + - name: BIGCHAINDB_DATABASE_CRLFILE + value: /etc/bigchaindb/ca/crl.pem + - name: BIGCHAINDB_DATABASE_CERTFILE + value: /etc/bigchaindb/ssl/bdb-instance.pem + - name: BIGCHAINDB_DATABASE_KEYFILE + value: /etc/bigchaindb/ssl/bdb-instance.key + - name: BIGCHAINDB_DATABASE_LOGIN + valueFrom: + configMapKeyRef: + name: bdb-config + key: bdb-user + - name: BIGCHAINDB_START_TENDERMINT + value: "0" + - name: TENDERMINT_HOST + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-instance-name + - name: TENDERMINT_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-rpc-port + command: + - bash + - "-c" + - | + bigchaindb -l DEBUG start + ports: + - containerPort: "" + protocol: TCP + name: bdb-port + - containerPort: "" + protocol: TCP + name: bdb-ws-port + - containerPort: "" + protocol: TCP + name: tm-abci-port + volumeMounts: + - name: bdb-certs + mountPath: /etc/bigchaindb/ssl/ + readOnly: true + - name: ca-auth + mountPath: /etc/bigchaindb/ca/ + readOnly: true + resources: + limits: + cpu: 200m + memory: 768Mi + livenessProbe: + httpGet: + path: / + port: bdb-port + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: / + port: bdb-port + initialDelaySeconds: 15 + timeoutSeconds: 10 + restartPolicy: Always + volumes: + - name: bdb-certs + secret: + secretName: bdb-certs + defaultMode: 0400 + - name: ca-auth + secret: + secretName: ca-auth + defaultMode: 0400 diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index be7da0d6..ed79b122 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -27,11 +27,13 @@ spec: configMapKeyRef: name: vars key: mongodb-backend-port - - name: BIGCHAINDB_DATABASE_BACKEND + - name: BIGCHAINDB_DATABASE_REPLICASET valueFrom: configMapKeyRef: - name: bdb-config - key: bdb-db-backend + name: vars + key: mongodb-replicaset-name + - name: BIGCHAINDB_DATABASE_BACKEND + value: mongodb - name: BIGCHAINDB_DATABASE_NAME valueFrom: configMapKeyRef: @@ -67,6 +69,13 @@ spec: configMapKeyRef: name: vars key: bigchaindb-wsserver-advertised-scheme + - name: BIGCHAINDB_KEYPAIR_PUBLIC + valueFrom: + configMapKeyRef: + name: bdb-config + key: bdb-public-key + - name: BIGCHAINDB_KEYPAIR_PRIVATE + value: "" - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY valueFrom: configMapKeyRef: @@ -102,23 +111,12 @@ spec: configMapKeyRef: name: bdb-config key: bdb-user - - name: BIGCHAINDB_START_TENDERMINT - value: "0" - - name: TENDERMINT_HOST - valueFrom: - configMapKeyRef: - name: tendermint-config - key: tm-instance-name - - name: TENDERMINT_PORT - valueFrom: - configMapKeyRef: - name: tendermint-config - key: tm-rpc-port - command: - - bash - - "-c" - - | - bigchaindb -l DEBUG start + # The following env var is not required for the bootstrap/first node + #- name: BIGCHAINDB_KEYRING + # valueFrom: + # configMapKeyRef: + # name: bdb-config + # key: bdb-keyring ports: - containerPort: "" protocol: TCP @@ -126,9 +124,6 @@ spec: - containerPort: "" protocol: TCP name: bdb-ws-port - - containerPort: "" - protocol: TCP - name: tm-abci-port volumeMounts: - name: bdb-certs mountPath: /etc/bigchaindb/ssl/ diff --git a/k8s/bigchaindb/bigchaindb-svc-tm.yaml b/k8s/bigchaindb/bigchaindb-svc-tm.yaml new file mode 100644 index 00000000..c5fef92d --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-svc-tm.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: bdb-instance-0 + namespace: default + labels: + name: bdb-instance-0 +spec: + selector: + app: bdb-instance-0-dep + ports: + - port: "" + targetPort: "" + name: bdb-api-port + protocol: TCP + - port: "" + targetPort: "" + name: bdb-ws-port + protocol: TCP + - port: "" + targetPort: "" + name: tm-abci-port + protocol: TCP + type: ClusterIP + clusterIP: None diff --git a/k8s/bigchaindb/bigchaindb-svc.yaml b/k8s/bigchaindb/bigchaindb-svc.yaml index c5fef92d..a745e8d9 100644 --- a/k8s/bigchaindb/bigchaindb-svc.yaml +++ b/k8s/bigchaindb/bigchaindb-svc.yaml @@ -17,9 +17,5 @@ spec: targetPort: "" name: bdb-ws-port protocol: TCP - - port: "" - targetPort: "" - name: tm-abci-port - protocol: TCP type: ClusterIP clusterIP: None diff --git a/k8s/configuration/config-map-tm.yaml b/k8s/configuration/config-map-tm.yaml new file mode 100644 index 00000000..b2fd1c55 --- /dev/null +++ b/k8s/configuration/config-map-tm.yaml @@ -0,0 +1,177 @@ +## Note: data values do NOT have to be base64-encoded in this file. + +## vars is common environment variables for this BigchaindB node +apiVersion: v1 +kind: ConfigMap +metadata: + name: vars + namespace: default +data: + # cluster-fqdn is the DNS name registered for your HTTPS certificate. + cluster-fqdn: "bdb.example.com" + + # cluster-frontend-port is the port number on which this node's services + # are available to external clients. + cluster-frontend-port: "443" + + # cluster-health-check-port is the port number on which an external load + # balancer can check the status/liveness of the external/public server. + # In our deployment, Kubernetes sends 'livenessProbes' to this port and + # interprets a successful response as a 'healthy' service. + cluster-health-check-port: "8888" + + # cluster-dns-server-ip is the IP of the DNS server. A Kubernetes deployment + # always has a DNS server (kube-dns) running at 10.0.0.10 + cluster-dns-server-ip: "10.0.0.10" + + # mdb-instance-name is the name of the MongoDB instance in this cluster. + mdb-instance-name: "" + + # ngx-instance-name is the name of the NGINX instance in this cluster. + ngx-instance-name: "" + + # openresty-instance-name is the name of the OpenResty instance in this + # cluster. + openresty-instance-name: "" + + # bdb-instance-name is the name of the BigchainDB instance in this cluster. + bdb-instance-name: "" + + # mdb-mon-instance-name is the name of the MongoDB Monitoring Agent instance + # in this cluster. + mdb-mon-instance-name: "" + + # mdb-bak-instance-name is the name of the MongoDB Backup Agent instance + # in this cluster. + mdb-bak-instance-name: "" + + # ngx-mdb-instance-name is the FQDN of the MongoDB instance in this + # Kubernetes cluster. + ngx-mdb-instance-name: ".default.svc.cluster.local" + + # ngx-openresty-instance-name is the FQDN of the OpenResty instance in this + # Kubernetes cluster. + ngx-openresty-instance-name: ".default.svc.cluster.local" + + # ngx-bdb-instance-name is the FQDN of the BigchainDB instance in this + # Kubernetes cluster. + ngx-bdb-instance-name: ".default.svc.cluster.local" + + # mongodb-frontend-port is the port number on which external clients can + # access MongoDB. This needs to be restricted to only other MongoDB instances + # by enabling an authentication mechanism on MongoDB. + mongodb-frontend-port: "27017" + + # mongodb-backend-port is the port on which MongoDB is actually + # available/listening for requests. + mongodb-backend-port: "27017" + + # openresty-backend-port is the port number on which OpenResty is listening + # for requests. This is used by the NGINX instance to forward the requests to + # the right port, and by OpenResty instance to bind to the correct port to + # receive requests from NGINX instance. + openresty-backend-port: "80" + + # BigchainDB configuration parameters + # Refer https://docs.bigchaindb.com/projects/server/en/latest/server-reference/configuration.html + + # bigchaindb-api-port is the port number on which BigchainDB is listening + # for HTTP requests. + bigchaindb-api-port: "9984" + + # bigchaindb-server-bind is the socket where BigchainDB binds for API + # requests. + bigchaindb-server-bind: "0.0.0.0:9984" + + # bigchaindb-ws-port and bigchaindb-ws-interface form the socket where + # BigchainDB binds for Websocket connections. + bigchaindb-ws-port: "9985" + bigchaindb-ws-interface: "0.0.0.0" + + # bigchaindb-database-name is the database collection used by BigchainDB with + # the MongoDB backend. + bigchaindb-database-name: "bigchain" + + # bigchaindb-wsserver-advertised-scheme is the protocol used to access the + # WebSocket API in BigchainDB; can be 'ws' or 'wss' (default). + bigchaindb-wsserver-advertised-scheme: "wss" + + # Optional: Optimize storage engine(wired tiger) + # cache size. e.g. (2048MB, 2GB, 1TB), otherwise + # it will use the default cache size; i.e. max((50% RAM - 1GB), 256MB) + storage-engine-cache-size: "" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: bdb-config + namespace: default +data: + # BigchainDB instance authentication user name + bdb-user: "" + + # bigchaindb-backlog-reassign-delay is the number of seconds a transaction + # can remain in the backlog before being reassigned. + bigchaindb-backlog-reassign-delay: "120" + + # bigchaindb-database-maxtries is the maximum number of times that BigchainDB + # will try to establish a connection with the database backend. + # If it is set to 0, then it will try forever. + bigchaindb-database-maxtries: "3" + + # bigchaindb-database-connection-timeout is the maximum number of + # milliseconds that BigchainDB will wait before closing the connection while + # connecting to the database backend. + bigchaindb-database-connection-timeout: "5000" + + # bigchaindb-log-level is the log level used to log to the console. + bigchaindb-log-level: "debug" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tendermint-config + namespace: default +data: + # tm-seeds is the list of all the peers in the network. + tm-seeds: "<',' separated list of all tendermint nodes in the network>" + + # tm-validators is the list of all validators in the network. + tm-validators: "<',' separated list of all validators in the network>" + + # tm-validator-power is the validators voting power, make sure the order and + # the number of nodes in tm-validator-power and tm-validators is the same. + tm-validator-power: "<',' separated list of validator power of each node in the network>" + + # tm-genesis-time is the official time of blockchain start. + # example: 0001-01-01T00:00:00Z + tm-genesis-time: "" + + # tm-chain-id is the ID of the blockchain. Must be unique for every blockchain. + # example: test-chain-KPI1Ud + tm-chain-id: "" + + # tendermint-instance-name is the name of the Tendermint instance + # in the cluster + tm-instance-name: "" + + # ngx-tm-instance-name is the FQDN of the tendermint instance in this cluster + ngx-tm-instance-name: ".default.svc.cluster.local" + + # tm-abci-port is used by Tendermint Core for ABCI traffic. BigchainDB nodes + # use that internally. + tm-abci-port: "46658" + + # tm-p2p-port is used by Tendermint Core to communicate with + # other peers in the network. This port is accessible publicly. + tm-p2p-port: "46656" + + # tm-rpc-port is used by Tendermint Core to rpc. BigchainDB nodes + # use this port internally. + tm-rpc-port: "46657" + + # tm-pub-key-access is the port number used to host/publish the + # public key of the tendemrint node in this cluster. + tm-pub-key-access: "9986" diff --git a/k8s/configuration/secret-tm.yaml b/k8s/configuration/secret-tm.yaml new file mode 100644 index 00000000..a2ecf58a --- /dev/null +++ b/k8s/configuration/secret-tm.yaml @@ -0,0 +1,102 @@ +# All secret data should be base64 encoded before embedding them here. +# Short strings can be encoded using, e.g. +# echo "secret string" | base64 -w 0 > secret.string.b64 +# Files (e.g. certificates) can be encoded using, e.g. +# cat cert.pem | base64 -w 0 > cert.pem.b64 +# then copy the contents of cert.pem.b64 (for example) below. +# Ref: https://kubernetes.io/docs/concepts/configuration/secret/ +# Unused values can be set to "" + +apiVersion: v1 +kind: Secret +metadata: + name: cloud-manager-credentials + namespace: default +type: Opaque +data: + # Base64-encoded Project ID + # Project ID used by MongoDB deployment + group-id: "" + # Base64-encoded MongoDB Agent API Key for the group + agent-api-key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-instance.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-mon-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-mon-instance.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mdb-bak-certs + namespace: default +type: Opaque +data: + # Base64-encoded, concatenated certificate and private key + mdb-bak-instance.pem: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: bdb-certs + namespace: default +type: Opaque +data: + # Base64-encoded BigchainDB instance certificate + bdb-instance.pem: "" + # Base64-encoded private key (.key) + bdb-instance.key: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-certs + namespace: default +type: Opaque +data: + # Base64-encoded HTTPS private key + cert.key: "" + # Base64-encoded HTTPS certificate chain + # starting with your primary SSL cert (e.g. your_domain.crt) + # followed by all intermediate certs. + # If cert if from DigiCert, download "Best format for nginx". + cert.pem: "" + service-id: "" + version-header: "" + service-token: "" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ca-auth + namespace: default +type: Opaque +data: + # CA used to issue members/client certificates + # Base64-encoded CA certificate (ca.crt) + ca.pem: "" + crl.pem: "" diff --git a/k8s/mongodb/container/Dockerfile-TMT b/k8s/mongodb/container/Dockerfile-TMT new file mode 100644 index 00000000..1da9dc26 --- /dev/null +++ b/k8s/mongodb/container/Dockerfile-TMT @@ -0,0 +1,12 @@ +FROM mongo:3.4.10 +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY mongod.conf.tmt.template /etc/mongod.conf +COPY mongod_tmt_entrypoint.bash / +VOLUME /data/db /data/configdb /etc/mongod/ssl /etc/mongod/ca +EXPOSE 27017 +ENTRYPOINT ["/mongod_tmt_entrypoint.bash"] diff --git a/k8s/mongodb/container/README.md b/k8s/mongodb/container/README.md index e1b60d8c..a2b0d22e 100644 --- a/k8s/mongodb/container/README.md +++ b/k8s/mongodb/container/README.md @@ -21,7 +21,6 @@ docker run \ bigchaindb/mongodb: \ --mongodb-port \ --mongodb-key-file-path /mongo-ssl/.pem \ - --mongodb-key-file-password \ --mongodb-ca-file-path /mongo-ssl/.crt \ --mongodb-crl-file-path /mongo-ssl/.pem \ --mongodb-fqdn \ diff --git a/k8s/mongodb/container/docker_build_and_push.bash b/k8s/mongodb/container/docker_build_and_push.bash index 0a773e35..d2372bfd 100755 --- a/k8s/mongodb/container/docker_build_and_push.bash +++ b/k8s/mongodb/container/docker_build_and_push.bash @@ -6,5 +6,5 @@ docker push bigchaindb/mongodb:3.2 # For Tendermint -# docker build -t bigchaindb/mongodb:unstable-tmt . +# docker build -t bigchaindb/mongodb:unstable-tmt . -f Dockerfile-TMT # docker push bigchaindb/mongodb:unstable-tmt diff --git a/k8s/mongodb/container/mongod.conf.template b/k8s/mongodb/container/mongod.conf.template index c0a2d288..d8ae1bce 100644 --- a/k8s/mongodb/container/mongod.conf.template +++ b/k8s/mongodb/container/mongod.conf.template @@ -25,6 +25,8 @@ systemLog: verbosity: 0 query: verbosity: 0 + replication: + verbosity: 0 sharding: verbosity: 0 storage: @@ -93,3 +95,7 @@ storage: operationProfiling: mode: slowOp slowOpThresholdMs: 100 + +replication: + replSetName: REPLICA_SET_NAME + enableMajorityReadConcern: true diff --git a/k8s/mongodb/container/mongod.conf.tmt.template b/k8s/mongodb/container/mongod.conf.tmt.template new file mode 100644 index 00000000..c0a2d288 --- /dev/null +++ b/k8s/mongodb/container/mongod.conf.tmt.template @@ -0,0 +1,95 @@ +# mongod.conf + +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +# where to write logging data. +systemLog: + verbosity: 0 + # traceAllExceptions: true + timeStampFormat: iso8601-utc + component: + accessControl: + verbosity: 0 + command: + verbosity: 0 + control: + verbosity: 0 + ftdc: + verbosity: 0 + geo: + verbosity: 0 + index: + verbosity: 0 + network: + verbosity: 0 + query: + verbosity: 0 + sharding: + verbosity: 0 + storage: + verbosity: 0 + journal: + verbosity: 0 + write: + verbosity: 0 + +processManagement: + fork: false + pidFilePath: /tmp/mongod.pid + +net: + port: MONGODB_PORT + bindIp: 0.0.0.0 + maxIncomingConnections: 8192 + wireObjectCheck: false + unixDomainSocket: + enabled: false + pathPrefix: /tmp + filePermissions: 0700 + http: + enabled: false + compression: + compressors: snappy + ssl: + mode: requireSSL + PEMKeyFile: MONGODB_KEY_FILE_PATH + #PEMKeyPassword: MONGODB_KEY_FILE_PASSWORD + CAFile: MONGODB_CA_FILE_PATH + CRLFile: MONGODB_CRL_FILE_PATH + + #allowConnectionsWithoutCertificates: false + #allowInvalidHostnames: false + #weakCertificateValidation: false + #allowInvalidCertificates: false + +security: + authorization: enabled + clusterAuthMode: x509 + +setParameter: + enableLocalhostAuthBypass: true + #notablescan: 1 + logUserIds: 1 + authenticationMechanisms: MONGODB-X509,SCRAM-SHA-1 + +storage: + dbPath: /data/db/main + indexBuildRetry: true + journal: + enabled: true + commitIntervalMs: 100 + directoryPerDB: true + engine: wiredTiger + wiredTiger: + engineConfig: + journalCompressor: snappy + configString: cache_size=STORAGE_ENGINE_CACHE_SIZE + collectionConfig: + blockCompressor: snappy + indexConfig: + prefixCompression: true # TODO false may affect performance? + +operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 diff --git a/k8s/mongodb/container/mongod_entrypoint.bash b/k8s/mongodb/container/mongod_entrypoint.bash index 8dda6330..e77aa3e0 100755 --- a/k8s/mongodb/container/mongod_entrypoint.bash +++ b/k8s/mongodb/container/mongod_entrypoint.bash @@ -3,7 +3,6 @@ set -euo pipefail MONGODB_PORT="" MONGODB_KEY_FILE_PATH="" -#MONGODB_KEY_FILE_PASSWORD="" MONGODB_CA_FILE_PATH="" MONGODB_CRL_FILE_PATH="" REPLICA_SET_NAME="" @@ -21,11 +20,6 @@ while [[ $# -gt 1 ]]; do MONGODB_KEY_FILE_PATH="$2" shift ;; - --mongodb-key-file-password) - # TODO(Krish) move this to a mapped file later - MONGODB_KEY_FILE_PASSWORD="$2" - shift - ;; --mongodb-ca-file-path) MONGODB_CA_FILE_PATH="$2" shift @@ -34,6 +28,10 @@ while [[ $# -gt 1 ]]; do MONGODB_CRL_FILE_PATH="$2" shift ;; + --replica-set-name) + REPLICA_SET_NAME="$2" + shift + ;; --mongodb-fqdn) MONGODB_FQDN="$2" shift @@ -55,16 +53,17 @@ while [[ $# -gt 1 ]]; do done # sanity checks -if [[ -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \ +if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \ + -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \ -z "${MONGODB_FQDN:?MONGODB_FQDN not specified. Exiting!}" || \ -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ - -z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then - #-z "${MONGODB_KEY_FILE_PASSWORD:?MongoDB Key File Password not specified. Exiting!}" || \ + -z ${STORAGE_ENGINE_CACHE_SIZE:=''} ]] ; then exit 1 else + echo REPLICA_SET_NAME="$REPLICA_SET_NAME" echo MONGODB_PORT="$MONGODB_PORT" echo MONGODB_FQDN="$MONGODB_FQDN" echo MONGODB_IP="$MONGODB_IP" @@ -82,6 +81,7 @@ sed -i "s|MONGODB_PORT|${MONGODB_PORT}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_KEY_FILE_PATH|${MONGODB_KEY_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_CA_FILE_PATH|${MONGODB_CA_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} sed -i "s|MONGODB_CRL_FILE_PATH|${MONGODB_CRL_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|REPLICA_SET_NAME|${REPLICA_SET_NAME}|g" ${MONGODB_CONF_FILE_PATH} if [ ! -z "$STORAGE_ENGINE_CACHE_SIZE" ]; then if [[ "$STORAGE_ENGINE_CACHE_SIZE" =~ ^[0-9]+(G|M|T)B$ ]]; then sed -i.bk "s|STORAGE_ENGINE_CACHE_SIZE|${STORAGE_ENGINE_CACHE_SIZE}|g" ${MONGODB_CONF_FILE_PATH} diff --git a/k8s/mongodb/container/mongod_tmt_entrypoint.bash b/k8s/mongodb/container/mongod_tmt_entrypoint.bash new file mode 100755 index 00000000..ad6aad43 --- /dev/null +++ b/k8s/mongodb/container/mongod_tmt_entrypoint.bash @@ -0,0 +1,103 @@ +#!/bin/bash +set -euo pipefail + +MONGODB_PORT="" +MONGODB_KEY_FILE_PATH="" +MONGODB_CA_FILE_PATH="" +MONGODB_CRL_FILE_PATH="" +MONGODB_FQDN="" +MONGODB_IP="" + +while [[ $# -gt 1 ]]; do + arg="$1" + case $arg in + --mongodb-port) + MONGODB_PORT="$2" + shift + ;; + --mongodb-key-file-path) + MONGODB_KEY_FILE_PATH="$2" + shift + ;; + --mongodb-ca-file-path) + MONGODB_CA_FILE_PATH="$2" + shift + ;; + --mongodb-crl-file-path) + MONGODB_CRL_FILE_PATH="$2" + shift + ;; + --mongodb-fqdn) + MONGODB_FQDN="$2" + shift + ;; + --mongodb-ip) + MONGODB_IP="$2" + shift + ;; + --storage-engine-cache-size) + STORAGE_ENGINE_CACHE_SIZE="$2" + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift +done + +# sanity checks +if [[ -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \ + -z "${MONGODB_FQDN:?MONGODB_FQDN not specified. Exiting!}" || \ + -z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \ + -z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \ + -z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \ + -z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \ + -z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then + exit 1 +else + echo MONGODB_PORT="$MONGODB_PORT" + echo MONGODB_FQDN="$MONGODB_FQDN" + echo MONGODB_IP="$MONGODB_IP" + echo MONGODB_KEY_FILE_PATH="$MONGODB_KEY_FILE_PATH" + echo MONGODB_CA_FILE_PATH="$MONGODB_CA_FILE_PATH" + echo MONGODB_CRL_FILE_PATH="$MONGODB_CRL_FILE_PATH" + echo STORAGE_ENGINE_CACHE_SIZE="$STORAGE_ENGINE_CACHE_SIZE" +fi + +MONGODB_CONF_FILE_PATH=/etc/mongod.conf +HOSTS_FILE_PATH=/etc/hosts + +# configure the mongod.conf file +sed -i "s|MONGODB_PORT|${MONGODB_PORT}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|MONGODB_KEY_FILE_PATH|${MONGODB_KEY_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|MONGODB_CA_FILE_PATH|${MONGODB_CA_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +sed -i "s|MONGODB_CRL_FILE_PATH|${MONGODB_CRL_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH} +if [ ! -z "$STORAGE_ENGINE_CACHE_SIZE" ]; then + if [[ "$STORAGE_ENGINE_CACHE_SIZE" =~ ^[0-9]+(G|M|T)B$ ]]; then + sed -i.bk "s|STORAGE_ENGINE_CACHE_SIZE|${STORAGE_ENGINE_CACHE_SIZE}|g" ${MONGODB_CONF_FILE_PATH} + else + echo "Invalid Value for storage engine cache size $STORAGE_ENGINE_CACHE_SIZE" + exit 1 + fi +else + sed -i.bk "/cache_size=/d" ${MONGODB_CONF_FILE_PATH} +fi + +# add the hostname and ip to hosts file +echo "${MONGODB_IP} ${MONGODB_FQDN}" >> $HOSTS_FILE_PATH + +# create the directory if it does not exist, where MongoDB can store the data +# and config files; this assumes that the data directory is mounted at +# /data/db/main and the config directory is mounted at /data/configdb +mkdir -p /data/db/main /data/configdb/main + +# start mongod +echo "INFO: starting mongod..." + +# TODO Uncomment the first exec command and use it instead of the second one +# after https://github.com/docker-library/mongo/issues/172 is resolved. Check +# for other bugs too. +#exec /entrypoint.sh mongod --config ${MONGODB_CONF_FILE_PATH} +exec /usr/bin/mongod --config ${MONGODB_CONF_FILE_PATH} diff --git a/k8s/mongodb/mongo-ss-tm.yaml b/k8s/mongodb/mongo-ss-tm.yaml new file mode 100644 index 00000000..827aa7c0 --- /dev/null +++ b/k8s/mongodb/mongo-ss-tm.yaml @@ -0,0 +1,107 @@ +######################################################################## +# This YAML file desribes a StatefulSet with a service for running and # +# exposing a MongoDB instance. # +# It depends on the configdb and db k8s pvc. # +######################################################################## + +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: mdb-instance-0-ss + namespace: default +spec: + serviceName: mdb-instance-0 + replicas: 1 + template: + metadata: + name: mdb-instance-0-ss + labels: + app: mdb-instance-0-ss + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongodb + image: bigchaindb/mongodb:3.2 + imagePullPolicy: IfNotPresent + env: + - name: MONGODB_FQDN + valueFrom: + configMapKeyRef: + name: vars + key: mdb-instance-name + - name: MONGODB_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MONGODB_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: STORAGE_ENGINE_CACHE_SIZE + valueFrom: + configMapKeyRef: + name: vars + key: storage-engine-cache-size + args: + - --mongodb-port + - $(MONGODB_PORT) + - --mongodb-key-file-path + - /etc/mongod/ssl/mdb-instance.pem + - --mongodb-ca-file-path + - /etc/mongod/ca/ca.pem + - --mongodb-crl-file-path + - /etc/mongod/ca/crl.pem + - --mongodb-fqdn + - $(MONGODB_FQDN) + - --mongodb-ip + - $(MONGODB_POD_IP) + - --storage-engine-cache-size + - $(STORAGE_ENGINE_CACHE_SIZE) + securityContext: + capabilities: + add: + - FOWNER + ports: + - containerPort: "" + protocol: TCP + name: mdb-api-port + volumeMounts: + - name: mdb-db + mountPath: /data/db + - name: mdb-configdb + mountPath: /data/configdb + - name: mdb-certs + mountPath: /etc/mongod/ssl/ + readOnly: true + - name: ca-auth + mountPath: /etc/mongod/ca/ + readOnly: true + resources: + limits: + cpu: 200m + memory: 5G + livenessProbe: + tcpSocket: + port: mdb-api-port + initialDelaySeconds: 15 + successThreshold: 1 + failureThreshold: 3 + periodSeconds: 15 + timeoutSeconds: 10 + restartPolicy: Always + volumes: + - name: mdb-db + persistentVolumeClaim: + claimName: mongo-db-claim + - name: mdb-configdb + persistentVolumeClaim: + claimName: mongo-configdb-claim + - name: mdb-certs + secret: + secretName: mdb-certs + defaultMode: 0400 + - name: ca-auth + secret: + secretName: ca-auth + defaultMode: 0400 diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index 827aa7c0..1243da26 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -33,6 +33,11 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: MONGODB_REPLICA_SET_NAME + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-replicaset-name - name: MONGODB_PORT valueFrom: configMapKeyRef: @@ -52,6 +57,8 @@ spec: - /etc/mongod/ca/ca.pem - --mongodb-crl-file-path - /etc/mongod/ca/crl.pem + - --replica-set-name + - $(MONGODB_REPLICA_SET_NAME) - --mongodb-fqdn - $(MONGODB_FQDN) - --mongodb-ip diff --git a/k8s/nginx-http/container/Dockerfile b/k8s/nginx-http/container/Dockerfile index 0cf6a05e..e35dd5e0 100644 --- a/k8s/nginx-http/container/Dockerfile +++ b/k8s/nginx-http/container/Dockerfile @@ -7,5 +7,5 @@ RUN apt-get update \ && apt-get clean COPY nginx.conf.template /etc/nginx/nginx.conf COPY nginx_entrypoint.bash / -EXPOSE 80 27017 9986 46656 +EXPOSE 80 27017 ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-http/container/Dockerfile-TMT b/k8s/nginx-http/container/Dockerfile-TMT new file mode 100644 index 00000000..41b1bd26 --- /dev/null +++ b/k8s/nginx-http/container/Dockerfile-TMT @@ -0,0 +1,11 @@ +FROM nginx:stable +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.template /etc/nginx/nginx.conf +COPY nginx_tmt_entrypoint.bash / +EXPOSE 80 27017 9986 46656 +ENTRYPOINT ["/nginx_tmt_entrypoint.bash"] diff --git a/k8s/nginx-http/container/docker_build_and_push.bash b/k8s/nginx-http/container/docker_build_and_push.bash index 9d1ae381..32b662c1 100755 --- a/k8s/nginx-http/container/docker_build_and_push.bash +++ b/k8s/nginx-http/container/docker_build_and_push.bash @@ -5,5 +5,5 @@ docker build -t bigchaindb/nginx_http:1.1 . docker push bigchaindb/nginx_http:1.1 # For tendermint deployments -# docker build -t bigchaindb/nginx_https:unstable-tmt . +# docker build -t bigchaindb/nginx_https:unstable-tmt . -f Dockerfile-TMT # docker push bigchaindb/nginx_https:unstable-tmt diff --git a/k8s/nginx-http/container/nginx.conf.template b/k8s/nginx-http/container/nginx.conf.template index 8939f42b..06625a99 100644 --- a/k8s/nginx-http/container/nginx.conf.template +++ b/k8s/nginx-http/container/nginx.conf.template @@ -126,13 +126,13 @@ http { # NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP # connection. stream { - log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' '$proxy_protocol_addr $proxy_protocol_port ' '$protocol $status $session_time $bytes_sent ' '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' '"$upstream_bytes_received" "$upstream_connect_time" '; - access_log /dev/stdout bdb_log buffer=16k flush=5s; + access_log /dev/stdout mdb_log buffer=16k flush=5s; # Define a zone 'two' of size 10 megabytes to store the counters # that hold number of TCP connections from a specific IP address. @@ -153,12 +153,6 @@ stream { default MONGODB_BACKEND_HOST; } - # The following map block enables lazy-binding to the backend at runtime, - # rather than binding as soon as NGINX starts. - map $remote_addr $tm_backend { - default TM_BACKEND_HOST; - } - # Frontend server to forward connections to MDB instance. server { listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5; @@ -166,20 +160,4 @@ stream { tcp_nodelay on; proxy_pass $mdb_backend:MONGODB_BACKEND_PORT; } - - # Server to forward connection to nginx instance hosting - # tendermint node public key. - server { - listen TM_PUB_KEY_ACCESS_PORT; - proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT; - } - - # Server to forward p2p connections to Tendermint instance. - server { - listen TM_P2P_PORT so_keepalive=3m:1m:5; - preread_timeout 60s; - tcp_nodelay on; - proxy_pass $tm_backend:TM_P2P_PORT; - } } - diff --git a/k8s/nginx-http/container/nginx.conf.tm.template b/k8s/nginx-http/container/nginx.conf.tm.template new file mode 100644 index 00000000..8939f42b --- /dev/null +++ b/k8s/nginx-http/container/nginx.conf.tm.template @@ -0,0 +1,185 @@ +# Frontend API server that: +# 1. Forwards BDB HTTP & WS requests to BDB backend. +# 2. Forwards MDB TCP connections to MDB backend. +# 3. Does health check with LB. + +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /dev/stderr; + +events { + # Each worker handles up to 512 connections. Increase this for heavy + # workloads. + worker_connections 512; + accept_mutex on; + use epoll; +} + +http { + access_log /dev/stdout combined buffer=16k flush=5s; + + # Allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code to return to the client when throttling; + # 429 is for TooManyRequests, ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 10s; + client_header_timeout 10s; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + keepalive_timeout 60s; + + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default BIGCHAINDB_BACKEND_HOST; + } + + # Frontend server for the external clients + server { + listen CLUSTER_FRONTEND_PORT; + underscores_in_headers on; + + # Forward websockets to backend BDB at 9985. + location /api/v1/streams/valid_transactions { + proxy_pass http://$bdb_backend:BIGCHAINDB_WS_PORT; + proxy_read_timeout 600s; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + + # Forward other URL paths to backend BDB at 9984. + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + + # max client request body size: avg transaction size. + client_max_body_size 15k; + + # No auth for GETs, forward directly to BDB. + if ($request_method = GET) { + proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; + } + + # POST requests get forwarded to OpenResty instance. Enable CORS too. + if ($request_method = POST ) { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; + } + + # OPTIONS requests handling for CORS. + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen HEALTH_CHECK_PORT; + + location = /health { + return 200; + } + } +} + +# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP +# connection. +stream { + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /dev/stdout bdb_log buffer=16k flush=5s; + + # Define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address. + limit_conn_zone $binary_remote_addr zone=two:10m; + + # Enable logging when connections are being throttled. + limit_conn_log_level notice; + + # Allow 16 connections from the same IP address. + limit_conn two 16; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + # The following map block enables lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $mdb_backend { + default MONGODB_BACKEND_HOST; + } + + # The following map block enables lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $tm_backend { + default TM_BACKEND_HOST; + } + + # Frontend server to forward connections to MDB instance. + server { + listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5; + preread_timeout 30s; + tcp_nodelay on; + proxy_pass $mdb_backend:MONGODB_BACKEND_PORT; + } + + # Server to forward connection to nginx instance hosting + # tendermint node public key. + server { + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; + tcp_nodelay on; + proxy_pass $tm_backend:TM_P2P_PORT; + } +} + diff --git a/k8s/nginx-http/container/nginx_entrypoint.bash b/k8s/nginx-http/container/nginx_entrypoint.bash index 3918dbaa..3d49a62b 100755 --- a/k8s/nginx-http/container/nginx_entrypoint.bash +++ b/k8s/nginx-http/container/nginx_entrypoint.bash @@ -31,10 +31,7 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting! -z "${bdb_api_port:?BIGCHAINDB_API_PORT not specified. Exiting!}" || \ -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \ -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ - -z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" || \ - -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ - -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \ - -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then + -z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" ]]; then exit 1 else echo CLUSTER_FRONTEND_PORT="$cluster_frontend_port" @@ -46,9 +43,6 @@ else echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host" echo BIGCHAINDB_API_PORT="$bdb_api_port" echo BIGCHAINDB_WS_PORT="$bdb_ws_port" - echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" - echo TM_BACKEND_HOST="$tm_backend_host" - echo TM_P2P_PORT="$tm_p2p_port" fi NGINX_CONF_FILE=/etc/nginx/nginx.conf @@ -63,11 +57,7 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} -sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} -sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE} -sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} # start nginx echo "INFO: starting nginx..." exec nginx -c /etc/nginx/nginx.conf - diff --git a/k8s/nginx-http/container/nginx_tmt_entrypoint.bash b/k8s/nginx-http/container/nginx_tmt_entrypoint.bash new file mode 100755 index 00000000..3918dbaa --- /dev/null +++ b/k8s/nginx-http/container/nginx_tmt_entrypoint.bash @@ -0,0 +1,73 @@ +#!/bin/bash +set -euo pipefail + +# Cluster vars +cluster_frontend_port=`printenv CLUSTER_FRONTEND_PORT` + + +# NGINX vars +dns_server=`printenv DNS_SERVER` +health_check_port=`printenv HEALTH_CHECK_PORT` + + +# MongoDB vars +mongo_frontend_port=`printenv MONGODB_FRONTEND_PORT` +mongo_backend_host=`printenv MONGODB_BACKEND_HOST` +mongo_backend_port=`printenv MONGODB_BACKEND_PORT` + + +# BigchainDB vars +bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` +bdb_api_port=`printenv BIGCHAINDB_API_PORT` +bdb_ws_port=`printenv BIGCHAINDB_WS_PORT` + + +# sanity check +if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_frontend_port:?MONGODB_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_backend_host:?MONGODB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${mongo_backend_port:?MONGODB_BACKEND_PORT not specified. Exiting!}" || \ + -z "${bdb_backend_host:?BIGCHAINDB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?BIGCHAINDB_API_PORT not specified. Exiting!}" || \ + -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \ + -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then + exit 1 +else + echo CLUSTER_FRONTEND_PORT="$cluster_frontend_port" + echo DNS_SERVER="$dns_server" + echo HEALTH_CHECK_PORT="$health_check_port" + echo MONGODB_FRONTEND_PORT="$mongo_frontend_port" + echo MONGODB_BACKEND_HOST="$mongo_backend_host" + echo MONGODB_BACKEND_PORT="$mongo_backend_port" + echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host" + echo BIGCHAINDB_API_PORT="$bdb_api_port" + echo BIGCHAINDB_WS_PORT="$bdb_ws_port" + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" + echo TM_BACKEND_HOST="$tm_backend_host" + echo TM_P2P_PORT="$tm_p2p_port" +fi + +NGINX_CONF_FILE=/etc/nginx/nginx.conf + +# configure the nginx.conf file with env variables +sed -i "s|CLUSTER_FRONTEND_PORT|${cluster_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_FRONTEND_PORT|${mongo_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c /etc/nginx/nginx.conf + diff --git a/k8s/nginx-http/nginx-http-dep-tm.yaml b/k8s/nginx-http/nginx-http-dep-tm.yaml new file mode 100644 index 00000000..baeab29a --- /dev/null +++ b/k8s/nginx-http/nginx-http-dep-tm.yaml @@ -0,0 +1,104 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: ngx-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: bigchaindb/nginx_http:1.1 + imagePullPolicy: IfNotPresent + env: + - name: CLUSTER_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: cluster-frontend-port + - name: HEALTH_CHECK_PORT + valueFrom: + configMapKeyRef: + name: vars + key: cluster-health-check-port + - name: DNS_SERVER + valueFrom: + configMapKeyRef: + name: vars + key: cluster-dns-server-ip + - name: MONGODB_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-frontend-port + - name: MONGODB_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-mdb-instance-name + - name: MONGODB_BACKEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: BIGCHAINDB_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-bdb-instance-name + - name: BIGCHAINDB_API_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-api-port + - name: BIGCHAINDB_WS_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-port + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-pub-key-access + - name: TM_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: tendermint-config + key: ngx-tm-instance-name + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-p2p-port + ports: + - containerPort: "" + protocol: TCP + - containerPort: "" + protocol: TCP + name: ngx-health + - containerPort: "" + protocol: TCP + - containerPort: "" + protocol: TCP + name: tm-pub-key + - containerPort: "" + protocol: TCP + name: tm-p2p-port + livenessProbe: + httpGet: + path: /health + port: ngx-health + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always diff --git a/k8s/nginx-http/nginx-http-dep.yaml b/k8s/nginx-http/nginx-http-dep.yaml index baeab29a..5a9359f5 100644 --- a/k8s/nginx-http/nginx-http-dep.yaml +++ b/k8s/nginx-http/nginx-http-dep.yaml @@ -60,21 +60,6 @@ spec: configMapKeyRef: name: vars key: bigchaindb-ws-port - - name: TM_PUB_KEY_ACCESS_PORT - valueFrom: - configMapKeyRef: - name: tendermint-config - key: tm-pub-key-access - - name: TM_BACKEND_HOST - valueFrom: - configMapKeyRef: - name: tendermint-config - key: ngx-tm-instance-name - - name: TM_P2P_PORT - valueFrom: - configMapKeyRef: - name: tendermint-config - key: tm-p2p-port ports: - containerPort: "" protocol: TCP @@ -83,12 +68,6 @@ spec: name: ngx-health - containerPort: "" protocol: TCP - - containerPort: "" - protocol: TCP - name: tm-pub-key - - containerPort: "" - protocol: TCP - name: tm-p2p-port livenessProbe: httpGet: path: /health diff --git a/k8s/nginx-http/nginx-http-svc-tm.yaml b/k8s/nginx-http/nginx-http-svc-tm.yaml new file mode 100644 index 00000000..c0fcab5b --- /dev/null +++ b/k8s/nginx-http/nginx-http-svc-tm.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + name: ngx-instance-0 + namespace: default + labels: + name: ngx-instance-0 + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-instance-0-dep + ports: + - port: "" + targetPort: "" + name: public-cluster-port + protocol: TCP + - port: "" + targetPort: "" + name: tm-pub-key-access + protocol: TCP + - port: "" + targetPort: "" + protocol: TCP + name: tm-p2p-port + type: LoadBalancer diff --git a/k8s/nginx-http/nginx-http-svc.yaml b/k8s/nginx-http/nginx-http-svc.yaml index c0fcab5b..76c603d2 100644 --- a/k8s/nginx-http/nginx-http-svc.yaml +++ b/k8s/nginx-http/nginx-http-svc.yaml @@ -17,12 +17,4 @@ spec: targetPort: "" name: public-cluster-port protocol: TCP - - port: "" - targetPort: "" - name: tm-pub-key-access - protocol: TCP - - port: "" - targetPort: "" - protocol: TCP - name: tm-p2p-port type: LoadBalancer diff --git a/k8s/nginx-https/container/Dockerfile b/k8s/nginx-https/container/Dockerfile index 1c9fe8a5..3bd6b607 100644 --- a/k8s/nginx-https/container/Dockerfile +++ b/k8s/nginx-https/container/Dockerfile @@ -7,5 +7,5 @@ RUN apt-get update \ && apt-get clean COPY nginx.conf.template /etc/nginx/nginx.conf COPY nginx_entrypoint.bash / -EXPOSE 80 443 27017 9986 46656 +EXPOSE 80 443 27017 ENTRYPOINT ["/nginx_entrypoint.bash"] diff --git a/k8s/nginx-https/container/Dockerfile-TMT b/k8s/nginx-https/container/Dockerfile-TMT new file mode 100644 index 00000000..4d6d2f77 --- /dev/null +++ b/k8s/nginx-https/container/Dockerfile-TMT @@ -0,0 +1,11 @@ +FROM nginx:stable +LABEL maintainer "dev@bigchaindb.com" +WORKDIR / +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get autoremove \ + && apt-get clean +COPY nginx.conf.tmt.template /etc/nginx/nginx.conf +COPY nginx_tmt_entrypoint.bash / +EXPOSE 80 443 27017 9986 46656 +ENTRYPOINT ["/nginx_tmt_entrypoint.bash"] diff --git a/k8s/nginx-https/container/docker_build_and_push.bash b/k8s/nginx-https/container/docker_build_and_push.bash index 9ad65538..9fc283c3 100755 --- a/k8s/nginx-https/container/docker_build_and_push.bash +++ b/k8s/nginx-https/container/docker_build_and_push.bash @@ -5,5 +5,5 @@ docker build -t bigchaindb/nginx_https:1.1 . docker push bigchaindb/nginx_https:1.1 # For tendermint deployments -# docker build -t bigchaindb/nginx_https:unstable-tmt . +# docker build -t bigchaindb/nginx_https:unstable-tmt . -f Dockerfile-TMT # docker push bigchaindb/nginx_https:unstable-tmt diff --git a/k8s/nginx-https/container/nginx.conf.template b/k8s/nginx-https/container/nginx.conf.template index 11bdaa75..45f0202a 100644 --- a/k8s/nginx-https/container/nginx.conf.template +++ b/k8s/nginx-https/container/nginx.conf.template @@ -152,16 +152,16 @@ http { } } -# NGINX stream block for TCP and UDP proxies. Used to proxy MongoDB and Tendermint TCP -# connections. +# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP +# connection. stream { - log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' '$proxy_protocol_addr $proxy_protocol_port ' '$protocol $status $session_time $bytes_sent ' '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' '"$upstream_bytes_received" "$upstream_connect_time" '; - access_log /dev/stdout bdb_log buffer=16k flush=5s; + access_log /dev/stdout mdb_log buffer=16k flush=5s; # Define a zone 'two' of size 10 megabytes to store the counters # that hold number of TCP connections from a specific IP address. @@ -186,12 +186,6 @@ stream { default MONGODB_BACKEND_HOST; } - # The following map block enables lazy-binding to the backend at runtime, - # rather than binding as soon as NGINX starts. - map $remote_addr $tm_backend { - default TM_BACKEND_HOST; - } - # Frontend server to forward connections to MDB instance. server { listen MONGODB_FRONTEND_PORT so_keepalive=3m:1m:5; @@ -199,20 +193,4 @@ stream { tcp_nodelay on; proxy_pass $mdb_backend:MONGODB_BACKEND_PORT; } - - # Server to forward connection to nginx instance hosting - # tendermint node public key. - server { - listen TM_PUB_KEY_ACCESS_PORT; - proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT; - } - - # Server to forward p2p connections to Tendermint instance. - server { - listen TM_P2P_PORT so_keepalive=3m:1m:5; - preread_timeout 60s; - tcp_nodelay on; - proxy_pass $tm_backend:TM_P2P_PORT; - } } - diff --git a/k8s/nginx-https/container/nginx.conf.tmt.template b/k8s/nginx-https/container/nginx.conf.tmt.template new file mode 100644 index 00000000..11bdaa75 --- /dev/null +++ b/k8s/nginx-https/container/nginx.conf.tmt.template @@ -0,0 +1,218 @@ +# Frontend API server that: +# 1. Acts as the HTTPS termination point. +# 2. Forwards BDB HTTP requests to OpenResty backend. +# 3. Forwards BDB WS requests to BDB backend. +# 4. Forwards MDB TCP connections to MDB backend. +# 5. Does health check with LB. + +worker_processes 2; +daemon off; +user nobody nogroup; +pid /tmp/nginx.pid; +error_log /dev/stderr; + +events { + # Each worker handles up to 512 connections. Increase this for heavy + # workloads. + worker_connections 512; + accept_mutex on; + use epoll; +} + +http { + access_log /dev/stdout combined buffer=16k flush=5s; + + # Allow 10 req/sec from the same IP address, and store the counters in a + # `zone` or shared memory location tagged as 'one'. + limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s; + + # Enable logging when requests are being throttled. + limit_req_log_level notice; + + # HTTP status code that is returned to the client; 429 is for TooManyRequests, + # ref. RFC 6585 + limit_req_status 429; + + # Limit requests from the same client, allow `burst` to 20 r/s, + # `nodelay` or drop connection immediately in case it exceeds this + # threshold. + limit_req zone=one burst=20 nodelay; + + # `slowloris` attack mitigation settings. + client_body_timeout 10s; + client_header_timeout 10s; + + # Do not expose nginx data/version number in error response and header + server_tokens off; + + # To prevent cross-site scripting + add_header X-XSS-Protection "1; mode=block"; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + keepalive_timeout 60s; + + # The following map blocks enable lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $bdb_backend { + default BIGCHAINDB_BACKEND_HOST; + } + map $remote_addr $openresty_backend { + default OPENRESTY_BACKEND_HOST; + } + + # Frontend server for the external clients; acts as HTTPS termination point. + server { + listen CLUSTER_FRONTEND_PORT ssl; + server_name "CLUSTER_FQDN"; + + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/cert.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + underscores_in_headers on; + + # Forward websockets directly to backend BDB. + location /api/v1/streams/valid_transactions { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://$bdb_backend:BIGCHAINDB_WS_PORT; + proxy_read_timeout 600s; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Forward other URL paths as per business logic/use case to BDB or + # OpenResty instance. + location / { + proxy_ignore_client_abort on; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # max client request body size: avg transaction size. + client_max_body_size 15k; + + # No auth for GETs, forward directly to BDB. + if ($request_method = GET) { + proxy_pass http://$bdb_backend:BIGCHAINDB_API_PORT; + } + + # POST requests get forwarded to OpenResty instance. Enable CORS too. + if ($request_method = POST ) { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; + + proxy_pass http://$openresty_backend:OPENRESTY_BACKEND_PORT; + } + + # OPTIONS requests handling for CORS. + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,app_key,app_id'; + add_header 'Access-Control-Max-Age' 43200; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + + # Only return this reponse if request_method is neither POST|GET|OPTIONS + if ($request_method !~ ^(GET|OPTIONS|POST)$) { + return 444; + } + } + } + + # Frontend server for the load balancer to respond to health checks. + server { + listen HEALTH_CHECK_PORT; + + location = /health { + return 200; + } + } + + # Frontend server for the external clients; returns a pretty error message + # when an HTTP request is sent instead of HTTPS. + server { + listen 80; + server_name "CLUSTER_FQDN"; + + location / { + add_header Upgrade "TLS/1.2, HTTP/1.1" always; + default_type text/plain; + return 426 'Consider using the HTTPS protocol next time!'; + } + } +} + +# NGINX stream block for TCP and UDP proxies. Used to proxy MongoDB and Tendermint TCP +# connections. +stream { + log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr ' + '$proxy_protocol_addr $proxy_protocol_port ' + '$protocol $status $session_time $bytes_sent ' + '$bytes_received "$upstream_addr" "$upstream_bytes_sent" ' + '"$upstream_bytes_received" "$upstream_connect_time" '; + + access_log /dev/stdout bdb_log buffer=16k flush=5s; + + # Define a zone 'two' of size 10 megabytes to store the counters + # that hold number of TCP connections from a specific IP address. + limit_conn_zone $binary_remote_addr zone=two:10m; + + # Enable logging when connections are being throttled. + limit_conn_log_level notice; + + # For a multi node BigchainDB deployment we need around 2^5 connections + # (for inter-node communication)per node via NGINX, we can bump this up in case + # there is a requirement to scale up. But we should not remove this + # for security reasons. + # Allow 256 connections from the same IP address. + limit_conn two 256; + + # DNS resolver to use for all the backend names specified in this configuration. + resolver DNS_SERVER valid=30s ipv6=off; + + # The following map block enables lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $mdb_backend { + default MONGODB_BACKEND_HOST; + } + + # The following map block enables lazy-binding to the backend at runtime, + # rather than binding as soon as NGINX starts. + map $remote_addr $tm_backend { + default TM_BACKEND_HOST; + } + + # Frontend server to forward connections to MDB instance. + server { + listen MONGODB_FRONTEND_PORT so_keepalive=3m:1m:5; + preread_timeout 30s; + tcp_nodelay on; + proxy_pass $mdb_backend:MONGODB_BACKEND_PORT; + } + + # Server to forward connection to nginx instance hosting + # tendermint node public key. + server { + listen TM_PUB_KEY_ACCESS_PORT; + proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT; + } + + # Server to forward p2p connections to Tendermint instance. + server { + listen TM_P2P_PORT so_keepalive=3m:1m:5; + preread_timeout 60s; + tcp_nodelay on; + proxy_pass $tm_backend:TM_P2P_PORT; + } +} + diff --git a/k8s/nginx-https/container/nginx_entrypoint.bash b/k8s/nginx-https/container/nginx_entrypoint.bash index dfcc298f..bd6fcd2d 100755 --- a/k8s/nginx-https/container/nginx_entrypoint.bash +++ b/k8s/nginx-https/container/nginx_entrypoint.bash @@ -27,10 +27,6 @@ bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` bdb_api_port=`printenv BIGCHAINDB_API_PORT` bdb_ws_port=`printenv BIGCHAINDB_WS_PORT` -# Tendermint vars -tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` -tm_backend_host=`printenv TM_BACKEND_HOST` -tm_p2p_port=`printenv TM_P2P_PORT` # sanity check if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!}" || \ @@ -44,11 +40,7 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting! -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \ -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ -z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \ - -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" || \ - -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ - -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \ - -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then - echo "Missing required environment variables. Exiting!" + -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" ]]; then exit 1 else echo CLUSTER_FQDN="$cluster_fqdn" @@ -63,9 +55,6 @@ else echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host" echo BIGCHAINDB_API_PORT="$bdb_api_port" echo BIGCHAINDB_WS_PORT="$bdb_ws_port" - echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" - echo TM_BACKEND_HOST="$tm_backend_host" - echo TM_P2P_PORT="$tm_p2p_port" fi NGINX_CONF_FILE=/etc/nginx/nginx.conf @@ -83,9 +72,6 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} -sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} -sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE} -sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} # start nginx echo "INFO: starting nginx..." diff --git a/k8s/nginx-https/container/nginx_tmt_entrypoint.bash b/k8s/nginx-https/container/nginx_tmt_entrypoint.bash new file mode 100755 index 00000000..dfcc298f --- /dev/null +++ b/k8s/nginx-https/container/nginx_tmt_entrypoint.bash @@ -0,0 +1,92 @@ +#!/bin/bash +set -euo pipefail + +# Cluster vars +cluster_fqdn=`printenv CLUSTER_FQDN` +cluster_frontend_port=`printenv CLUSTER_FRONTEND_PORT` + + +# NGINX vars +dns_server=`printenv DNS_SERVER` +health_check_port=`printenv HEALTH_CHECK_PORT` + + +# MongoDB vars +mongo_frontend_port=`printenv MONGODB_FRONTEND_PORT` +mongo_backend_host=`printenv MONGODB_BACKEND_HOST` +mongo_backend_port=`printenv MONGODB_BACKEND_PORT` + + +# OpenResty vars +openresty_backend_host=`printenv OPENRESTY_BACKEND_HOST` +openresty_backend_port=`printenv OPENRESTY_BACKEND_PORT` + + +# BigchainDB vars +bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST` +bdb_api_port=`printenv BIGCHAINDB_API_PORT` +bdb_ws_port=`printenv BIGCHAINDB_WS_PORT` + +# Tendermint vars +tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT` +tm_backend_host=`printenv TM_BACKEND_HOST` +tm_p2p_port=`printenv TM_P2P_PORT` + +# sanity check +if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_frontend_port:?MONGODB_FRONTEND_PORT not specified. Exiting!}" || \ + -z "${mongo_backend_host:?MONGODB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${mongo_backend_port:?MONGODB_BACKEND_PORT not specified. Exiting!}" || \ + -z "${openresty_backend_port:?OPENRESTY_BACKEND_PORT not specified. Exiting!}" || \ + -z "${openresty_backend_host:?OPENRESTY_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_backend_host:?BIGCHAINDB_BACKEND_HOST not specified. Exiting!}" || \ + -z "${bdb_api_port:?BIGCHAINDB_API_PORT not specified. Exiting!}" || \ + -z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \ + -z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \ + -z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \ + -z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" || \ + -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \ + -z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \ + -z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then + echo "Missing required environment variables. Exiting!" + exit 1 +else + echo CLUSTER_FQDN="$cluster_fqdn" + echo CLUSTER_FRONTEND_PORT="$cluster_frontend_port" + echo DNS_SERVER="$dns_server" + echo HEALTH_CHECK_PORT="$health_check_port" + echo MONGODB_FRONTEND_PORT="$mongo_frontend_port" + echo MONGODB_BACKEND_HOST="$mongo_backend_host" + echo MONGODB_BACKEND_PORT="$mongo_backend_port" + echo OPENRESTY_BACKEND_HOST="$openresty_backend_host" + echo OPENRESTY_BACKEND_PORT="$openresty_backend_port" + echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host" + echo BIGCHAINDB_API_PORT="$bdb_api_port" + echo BIGCHAINDB_WS_PORT="$bdb_ws_port" + echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port" + echo TM_BACKEND_HOST="$tm_backend_host" + echo TM_P2P_PORT="$tm_p2p_port" +fi + +NGINX_CONF_FILE=/etc/nginx/nginx.conf + +# configure the nginx.conf file with env variables +sed -i "s|CLUSTER_FQDN|${cluster_fqdn}|g" ${NGINX_CONF_FILE} +sed -i "s|CLUSTER_FRONTEND_PORT|${cluster_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_FRONTEND_PORT|${mongo_frontend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_HOST|${mongo_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|MONGODB_BACKEND_PORT|${mongo_backend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|OPENRESTY_BACKEND_PORT|${openresty_backend_port}|g" ${NGINX_CONF_FILE} +sed -i "s|OPENRESTY_BACKEND_HOST|${openresty_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_BACKEND_HOST|${bdb_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE} +sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE} +sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE} +sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE} +sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE} + +# start nginx +echo "INFO: starting nginx..." +exec nginx -c /etc/nginx/nginx.conf diff --git a/k8s/nginx-https/nginx-https-dep-tm.yaml b/k8s/nginx-https/nginx-https-dep-tm.yaml new file mode 100644 index 00000000..0d99ce80 --- /dev/null +++ b/k8s/nginx-https/nginx-https-dep-tm.yaml @@ -0,0 +1,132 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ngx-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: ngx-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: bigchaindb/nginx_https:1.1 + imagePullPolicy: IfNotPresent + env: + - name: CLUSTER_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: cluster-frontend-port + - name: HEALTH_CHECK_PORT + valueFrom: + configMapKeyRef: + name: vars + key: cluster-health-check-port + - name: CLUSTER_FQDN + valueFrom: + configMapKeyRef: + name: vars + key: cluster-fqdn + - name: DNS_SERVER + valueFrom: + configMapKeyRef: + name: vars + key: cluster-dns-server-ip + - name: MONGODB_FRONTEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-frontend-port + - name: MONGODB_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-mdb-instance-name + - name: MONGODB_BACKEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: mongodb-backend-port + - name: OPENRESTY_BACKEND_PORT + valueFrom: + configMapKeyRef: + name: vars + key: openresty-backend-port + - name: OPENRESTY_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-openresty-instance-name + - name: BIGCHAINDB_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: vars + key: ngx-bdb-instance-name + - name: BIGCHAINDB_API_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-api-port + - name: BIGCHAINDB_WS_PORT + valueFrom: + configMapKeyRef: + name: vars + key: bigchaindb-ws-port + - name: TM_PUB_KEY_ACCESS_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-pub-key-access + - name: TM_BACKEND_HOST + valueFrom: + configMapKeyRef: + name: tendermint-config + key: ngx-tm-instance-name + - name: TM_P2P_PORT + valueFrom: + configMapKeyRef: + name: tendermint-config + key: tm-p2p-port + ports: + # return a pretty error message on port 80, since we are expecting + # HTTPS traffic. + - containerPort: 80 + protocol: TCP + - containerPort: "" + protocol: TCP + - containerPort: "" + protocol: TCP + - containerPort: "" + protocol: TCP + name: ngx-port + - containerPort: "" + protocol: TCP + name: tm-pub-key + - containerPort: "" + protocol: TCP + name: tm-p2p-port + livenessProbe: + httpGet: + path: /health + port: ngx-port + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + timeoutSeconds: 10 + resources: + limits: + cpu: 200m + memory: 768Mi + volumeMounts: + - name: https-certs + mountPath: /etc/nginx/ssl/ + readOnly: true + restartPolicy: Always + volumes: + - name: https-certs + secret: + secretName: https-certs + defaultMode: 0400 diff --git a/k8s/nginx-https/nginx-https-dep.yaml b/k8s/nginx-https/nginx-https-dep.yaml index 0d99ce80..79dfe040 100644 --- a/k8s/nginx-https/nginx-https-dep.yaml +++ b/k8s/nginx-https/nginx-https-dep.yaml @@ -75,21 +75,6 @@ spec: configMapKeyRef: name: vars key: bigchaindb-ws-port - - name: TM_PUB_KEY_ACCESS_PORT - valueFrom: - configMapKeyRef: - name: tendermint-config - key: tm-pub-key-access - - name: TM_BACKEND_HOST - valueFrom: - configMapKeyRef: - name: tendermint-config - key: ngx-tm-instance-name - - name: TM_P2P_PORT - valueFrom: - configMapKeyRef: - name: tendermint-config - key: tm-p2p-port ports: # return a pretty error message on port 80, since we are expecting # HTTPS traffic. @@ -102,12 +87,6 @@ spec: - containerPort: "" protocol: TCP name: ngx-port - - containerPort: "" - protocol: TCP - name: tm-pub-key - - containerPort: "" - protocol: TCP - name: tm-p2p-port livenessProbe: httpGet: path: /health diff --git a/k8s/nginx-https/nginx-https-svc-tm.yaml b/k8s/nginx-https/nginx-https-svc-tm.yaml new file mode 100644 index 00000000..24b18cb1 --- /dev/null +++ b/k8s/nginx-https/nginx-https-svc-tm.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: + name: ngx-instance-0 + namespace: default + labels: + name: ngx-instance-0 + annotations: + # NOTE: the following annotation is a beta feature and + # only available in GCE/GKE and Azure as of now + service.beta.kubernetes.io/external-traffic: OnlyLocal +spec: + selector: + app: ngx-instance-0-dep + ports: + - port: "" + targetPort: "" + name: public-secure-cluster-port + protocol: TCP + - port: "" + targetPort: "" + name: public-mdb-port + protocol: TCP + - port: "" + targetPort: "" + name: tm-pub-key-access + protocol: TCP + - port: "" + targetPort: "" + protocol: TCP + name: tm-p2p-port + - port: 80 + targetPort: 80 + name: public-insecure-cluster-port + protocol: TCP + type: LoadBalancer diff --git a/k8s/nginx-https/nginx-https-svc.yaml b/k8s/nginx-https/nginx-https-svc.yaml index a5ef24b9..1d817fe5 100644 --- a/k8s/nginx-https/nginx-https-svc.yaml +++ b/k8s/nginx-https/nginx-https-svc.yaml @@ -25,12 +25,4 @@ spec: targetPort: 80 name: public-insecure-cluster-port protocol: TCP - - port: "" - targetPort: "" - name: tm-pub-key-access - protocol: TCP - - port: "" - targetPort: "" - protocol: TCP - name: tm-p2p-port type: LoadBalancer