Mirroristas/bigchaindb
2
0
Fork 0
mirror of https://github.com/bigchaindb/bigchaindb.git synced 2024-10-13 13:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1985 from bigchaindb/tendermint-k8s-dep

Browse Source 
Deployment of Tendermint + BigchainDB cluster
This commit is contained in:
Ahmed Muawia Khan 2018-02-21 11:48:37 +01:00 committed by GitHub
commit d5f45d4094
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 694 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile
View File

@ -8,15 +8,17 @@ RUN apt-get -qq update \
&& pip install --no-cache-dir . \
&& apt-get autoremove \
&& apt-get clean
VOLUME ["/data", "/certs"]
WORKDIR /data
ENV PYTHONUNBUFFERED 0
ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb
ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
ENV BIGCHAINDB_WSSERVER_SCHEME ws
ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
ENV BIGCHAINDB_WSSERVER_ADVERTISED_PORT 9985
ENV BIGCHAINDB_START_TENDERMINT 0
ENTRYPOINT ["bigchaindb"]
CMD ["start"]

41
k8s/bigchaindb/bigchaindb-dep.yaml
View File

@ -27,13 +27,11 @@ spec:
configMapKeyRef:
name: vars
key: mongodb-backend-port
- name: BIGCHAINDB_DATABASE_REPLICASET
- name: BIGCHAINDB_DATABASE_BACKEND
valueFrom:
configMapKeyRef:
name: vars
key: mongodb-replicaset-name
- name: BIGCHAINDB_DATABASE_BACKEND
value: mongodb
name: bdb-config
key: bdb-db-backend
- name: BIGCHAINDB_DATABASE_NAME
valueFrom:
configMapKeyRef:
@ -69,13 +67,6 @@ spec:
configMapKeyRef:
name: vars
key: bigchaindb-wsserver-advertised-scheme
- name: BIGCHAINDB_KEYPAIR_PUBLIC
valueFrom:
configMapKeyRef:
name: bdb-config
key: bdb-public-key
- name: BIGCHAINDB_KEYPAIR_PRIVATE
value: "<private key here>"
- name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY
valueFrom:
configMapKeyRef:
@ -111,12 +102,23 @@ spec:
configMapKeyRef:
name: bdb-config
key: bdb-user
# The following env var is not required for the bootstrap/first node
#- name: BIGCHAINDB_KEYRING
# valueFrom:
# configMapKeyRef:
# name: bdb-config
# key: bdb-keyring
- name: BIGCHAINDB_START_TENDERMINT
value: "0"
- name: TENDERMINT_HOST
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-instance-name
- name: TENDERMINT_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-rpc-port
command:
- bash
- "-c"
- |
bigchaindb -l DEBUG start
ports:
- containerPort: "<bigchaindb-api-port from ConfigMap>"
protocol: TCP
@ -124,6 +126,9 @@ spec:
- containerPort: "<bigchaindb-ws-port from ConfigMap>"
protocol: TCP
name: bdb-ws-port
- containerPort: "<tm-abci-port from ConfigMap>"
protocol: TCP
name: tm-abci-port
volumeMounts:
- name: bdb-certs
mountPath: /etc/bigchaindb/ssl/

4
k8s/bigchaindb/bigchaindb-svc.yaml
View File

@ -17,5 +17,9 @@ spec:
targetPort: "<bigchaindb-ws-port from ConfigMap>"
name: bdb-ws-port
protocol: TCP
- port: "<tm-abci-port from ConfigMap>"
targetPort: "<tm-abci-port from ConfigMap>"
name: tm-abci-port
protocol: TCP
type: ClusterIP
clusterIP: None

68
k8s/configuration/config-map.yaml
View File

@ -41,10 +41,6 @@ data:
# in this cluster.
mdb-mon-instance-name: "<name of the mdb monitoring agent instance>"
# mdb-bak-instance-name is the name of the MongoDB Backup Agent instance
# in this cluster.
mdb-bak-instance-name: "<name of the mdb backup agent instance>"
# ngx-mdb-instance-name is the FQDN of the MongoDB instance in this
# Kubernetes cluster.
ngx-mdb-instance-name: "<name of the mdb instance>.default.svc.cluster.local"
@ -57,11 +53,6 @@ data:
# Kubernetes cluster.
ngx-bdb-instance-name: "<name of the bdb instance>.default.svc.cluster.local"
# mongodb-frontend-port is the port number on which external clients can
# access MongoDB. This needs to be restricted to only other MongoDB instances
# by enabling an authentication mechanism on MongoDB.
mongodb-frontend-port: "27017"
# mongodb-backend-port is the port on which MongoDB is actually
# available/listening for requests.
mongodb-backend-port: "27017"
@ -88,9 +79,6 @@ data:
bigchaindb-ws-port: "9985"
bigchaindb-ws-interface: "0.0.0.0"
# mongodb-replicaset-name is the MongoDB replica set name
mongodb-replicaset-name: "bigchain-rs"
# bigchaindb-database-name is the database collection used by BigchainDB with
# the MongoDB backend.
bigchaindb-database-name: "bigchain"
@ -111,17 +99,9 @@ metadata:
name: bdb-config
namespace: default
data:
# Colon-separated list of all *other* nodes' BigchainDB public keys.
bdb-keyring: "<':' separated list of public keys>"
# BigchainDB instance authentication user name
bdb-user: "<user name>"
# BigchainDB public key of *this* node.
# Generated using bigchaindb_driver in the docs
# Example: "EPQk5i5yYpoUwGVM8VKZRjM8CYxB6j8Lu8i8SG7kGGce"
bdb-public-key: "<public key>"
# bigchaindb-backlog-reassign-delay is the number of seconds a transaction
# can remain in the backlog before being reassigned.
bigchaindb-backlog-reassign-delay: "120"
@ -138,3 +118,51 @@ data:
# bigchaindb-log-level is the log level used to log to the console.
bigchaindb-log-level: "debug"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: tendermint-config
namespace: default
data:
# tm-seeds is the list of all the peers in the network.
tm-seeds: "<',' separated list of all tendermint nodes in the network>"
# tm-validators is the list of all validators in the network.
tm-validators: "<',' separated list of all validators in the network>"
# tm-validator-power is the validators voting power, make sure the order and
# the number of nodes in tm-validator-power and tm-validators is the same.
tm-validator-power: "<',' separated list of validator power of each node in the network>"
# tm-genesis-time is the official time of blockchain start.
# example: 0001-01-01T00:00:00Z
tm-genesis-time: "<timestamp of blockchain start>"
# tm-chain-id is the ID of the blockchain. Must be unique for every blockchain.
# example: test-chain-KPI1Ud
tm-chain-id: "<ID of the blockchain>"
# tendermint-instance-name is the name of the Tendermint instance
# in the cluster
tm-instance-name: "<name of tendermint instance>"
# ngx-tm-instance-name is the FQDN of the tendermint instance in this cluster
ngx-tm-instance-name: "<name of tendermint instance>.default.svc.cluster.local"
# tm-abci-port is used by Tendermint Core for ABCI traffic. BigchainDB nodes
# use that internally.
tm-abci-port: "46658"
# tm-p2p-port is used by Tendermint Core to communicate with
# other peers in the network. This port is accessible publicly.
tm-p2p-port: "46656"
# tm-rpc-port is used by Tendermint Core to rpc. BigchainDB nodes
# use this port internally.
tm-rpc-port: "46657"
# tm-pub-key-access is the port number used to host/publish the
# public key of the tendemrint node in this cluster.
tm-pub-key-access: "9986"

21
k8s/configuration/secret.yaml
View File

@ -22,17 +22,6 @@ data:
---
apiVersion: v1
kind: Secret
metadata:
name: bdb-private-key
namespace: default
type: Opaque
data:
# Base64-encoded BigchainDB private key of *this* node
# Generated using bigchaindb_driver in the docs
private.key: "<b64 encoded BigchainDB private key>"
---
apiVersion: v1
kind: Secret
metadata:
name: mdb-certs
namespace: default
@ -53,16 +42,6 @@ data:
---
apiVersion: v1
kind: Secret
metadata:
name: mdb-bak-certs
namespace: default
type: Opaque
data:
# Base64-encoded, concatenated certificate and private key
mdb-bak-instance.pem: "<b64 encoded, concatenated certificate and private key>"
---
apiVersion: v1
kind: Secret
metadata:
name: bdb-certs
namespace: default

24
k8s/mongodb-backup-agent/container/Dockerfile
View File

@ -1,24 +0,0 @@
FROM ubuntu:xenial
LABEL maintainer "dev@bigchaindb.com"
ARG DEBIAN_FRONTEND=noninteractive
ARG DEB_FILE=mongodb-mms-backup-agent_latest_amd64.ubuntu1604.deb
ARG FILE_URL="https://cloud.mongodb.com/download/agent/backup/"$DEB_FILE
WORKDIR /
RUN apt update \
&& apt -y upgrade \
&& apt -y install --no-install-recommends \
curl \
ca-certificates \
logrotate \
libsasl2-2 \
&& curl -OL $FILE_URL \
&& dpkg -i $DEB_FILE \
&& rm -f $DEB_FILE \
&& apt -y purge curl \
&& apt -y autoremove \
&& apt clean
COPY mongodb_backup_agent_entrypoint.bash /
RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/
VOLUME /etc/mongod/ssl /etc/mongod/ca
USER mongodb-mms-agent
ENTRYPOINT ["/mongodb_backup_agent_entrypoint.bash"]

5
k8s/mongodb-backup-agent/container/docker_build_and_push.bash
View File

@ -1,5 +0,0 @@
#!/bin/bash
docker build -t bigchaindb/mongodb-backup-agent:3.5 .
docker push bigchaindb/mongodb-backup-agent:3.5

44
k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash
View File

@ -1,44 +0,0 @@
#!/bin/bash
set -euo pipefail
MONGODB_BACKUP_CONF_FILE=/etc/mongodb-mms/backup-agent.config
mms_api_keyfile_path=`printenv MMS_API_KEYFILE_PATH`
mms_groupid_keyfile_path=`printenv MMS_GROUPID_KEYFILE_PATH`
ca_crt_path=`printenv CA_CRT_PATH`
backup_pem_path=`printenv BACKUP_PEM_PATH`
if [[ -z "${mms_api_keyfile_path:?MMS_API_KEYFILE_PATH not specified. Exiting!}" || \
-z "${ca_crt_path:?CA_CRT_PATH not specified. Exiting!}" || \
-z "${backup_pem_path:?BACKUP_PEM_PATH not specified. Exiting!}" || \
-z "${mms_groupid_keyfile_path:?MMS_GROUPID_KEYFILE_PATH not specified. Exiting!}" ]]; then
exit 1
else
echo MMS_API_KEYFILE_PATH="$mms_api_keyfile_path"
echo MMS_GROUPID_KEYFILE_PATH="$mms_groupid_keyfile_path"
echo CA_CRT_PATH="$ca_crt_path"
echo BACKUP_PEM_PATH="$backup_pem_path"
fi
sed -i '/mmsApiKey/d' ${MONGODB_BACKUP_CONF_FILE}
sed -i '/mmsGroupId/d' ${MONGODB_BACKUP_CONF_FILE}
sed -i '/mothership/d' ${MONGODB_BACKUP_CONF_FILE}
# Get the api key from file
mms_api_key=`cat ${mms_api_keyfile_path}`
mms_groupid_key=`cat ${mms_groupid_keyfile_path}`
echo "mmsApiKey="${mms_api_key} >> ${MONGODB_BACKUP_CONF_FILE}
echo "mmsGroupId="${mms_groupid_key} >> ${MONGODB_BACKUP_CONF_FILE}
echo "mothership=api-backup.eu-west-1.mongodb.com" >> ${MONGODB_BACKUP_CONF_FILE}
# Append SSL settings to the config file
echo "useSslForAllConnections=true" >> ${MONGODB_BACKUP_CONF_FILE}
echo "sslRequireValidServerCertificates=true" >> ${MONGODB_BACKUP_CONF_FILE}
echo "sslTrustedServerCertificates="${ca_crt_path} >> ${MONGODB_BACKUP_CONF_FILE}
echo "sslClientCertificate="${backup_pem_path} >> ${MONGODB_BACKUP_CONF_FILE}
echo "#sslClientCertificatePassword=<password>" >> ${MONGODB_BACKUP_CONF_FILE}
echo "INFO: starting mdb backup..."
exec mongodb-mms-backup-agent -c $MONGODB_BACKUP_CONF_FILE

65
k8s/mongodb-backup-agent/mongo-backup-dep.yaml
View File

@ -1,65 +0,0 @@
############################################################
# This config file defines a k8s Deployment for the #
# bigchaindb/mongodb-backup-agent Docker image #
# #
# It connects to a MongoDB instance in a separate pod, #
# all remote MongoDB instances in the cluster, #
# and also to MongoDB Cloud Manager (an external service). #
# Notes: #
# MongoDB agents connect to Cloud Manager on port 443. #
############################################################
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: mdb-bak-instance-0-dep
spec:
replicas: 1
template:
metadata:
name: mdb-bak-instance-0-dep
labels:
app: mdb-bak-instance-0-dep
spec:
terminationGracePeriodSeconds: 10
containers:
- name: mdb-backup
image: bigchaindb/mongodb-backup-agent:3.5
imagePullPolicy: IfNotPresent
env:
- name: MMS_API_KEYFILE_PATH
value: /etc/mongod/cloud/agent-api-key
- name: MMS_GROUPID_KEYFILE_PATH
value: /etc/mongod/cloud/group-id
- name: CA_CRT_PATH
value: /etc/mongod/ca/ca.pem
- name: BACKUP_PEM_PATH
value: /etc/mongod/ssl/mdb-bak-instance.pem
resources:
limits:
cpu: 200m
memory: 768Mi
volumeMounts:
- name: mdb-bak-certs
mountPath: /etc/mongod/ssl/
readOnly: true
- name: ca-auth
mountPath: /etc/mongod/ca/
readOnly: true
- name: cloud-manager-credentials
mountPath: /etc/mongod/cloud/
readOnly: true
restartPolicy: Always
volumes:
- name: mdb-bak-certs
secret:
secretName: mdb-bak-certs
defaultMode: 0404
- name: ca-auth
secret:
secretName: ca-auth
defaultMode: 0404
- name: cloud-manager-credentials
secret:
secretName: cloud-manager-credentials
defaultMode: 0404

6
k8s/mongodb/container/Dockerfile
View File

@ -5,8 +5,8 @@ RUN apt-get update \
&& apt-get -y upgrade \
&& apt-get autoremove \
&& apt-get clean
COPY mongod.conf.template /etc/mongod.conf
COPY mongod_entrypoint.bash /
COPY mongod.conf.tmt.template /etc/mongod.conf
COPY mongod_tmt_entrypoint.bash /
VOLUME /data/db /data/configdb /etc/mongod/ssl /etc/mongod/ca
EXPOSE 27017
ENTRYPOINT ["/mongod_entrypoint.bash"]
ENTRYPOINT ["/mongod_tmt_entrypoint.bash"]

84
k8s/mongodb/container/README.md
View File

@ -1,14 +1,5 @@
## Custom MongoDB container for BigchainDB Backend
### Need
* MongoDB needs the hostname provided in the `rs.initiate()` command to be
resolvable through the hosts file locally.
* In the future, with the introduction of TLS for inter-cluster MongoDB
communications, we will need a way to specify detailed configuration.
* We also need a way to overwrite certain parameters to suit our use case.
### Step 1: Build and Push the Latest Container
Use the `docker_build_and_push.bash` script to build the latest docker image
and upload it to Docker Hub.
@ -27,84 +18,11 @@ docker run \
--volume=<host dir for mongodb data files>:/data/db \
--volume=<host dir for mongodb config data files>:/data/configdb \
--volume=<host dir with the required TLS certificates>:/mongo-ssl:ro \
bigchaindb/mongodb:3.0 \
bigchaindb/mongodb:<version of container> \
--mongodb-port <mongod port number for external connections> \
--mongodb-key-file-path /mongo-ssl/<private key file name>.pem \
--mongodb-key-file-password <password for the private key file> \
--mongodb-ca-file-path /mongo-ssl/<ca certificate file name>.crt \
--mongodb-crl-file-path /mongo-ssl/<crl certificate file name>.pem \
--replica-set-name <replica set name> \
--mongodb-fqdn <fully qualified domain name of this instance> \
--mongodb-ip <ip address of the mongodb container>
```
#### Step 3: Initialize the Replica Set
Login to one of the MongoDB containers, say mdb1:
`docker exec -it mongodb bash`
Since we need TLS certificates to use the mongo shell now, copy them using:
```
docker cp bdb-instance-0.pem mongodb:/
docker cp ca.crt mongodb:/
```
Start the `mongo` shell:
```
mongo --host mdb1-fqdn --port mdb1-port --verbose --ssl \
--sslCAFile /ca.crt \
--sslPEMKeyFile /bdb-instance-0.pem \
--sslPEMKeyPassword password
```
Run the rs.initiate() command:
```
rs.initiate({
_id : "<replica-set-name", members: [
{
_id : 0,
host : "<fqdn of this instance>:<port number>"
} ]
})
```
For example:
```
rs.initiate({ _id : "test-repl-set", members: [ { _id : 0, host :
"mdb-instance-0.westeurope.cloudapp.azure.com:27017" } ] })
```
You should also see changes in the mongo shell prompt from `>` to
`test-repl-set:OTHER>` to `test-repl-set:SECONDARY>` to finally
`test-repl-set:PRIMARY>`.
If this instance is not the primary, you can use the `rs.status()` command to
find out who is the primary.
#### Step 4: Add members to the Replica Set
We can only add members to a replica set from the PRIMARY instance.
Login to the PRIMARY and open a `mongo` shell.
Run the rs.add() command with the ip and port number of the other
containers/instances:
```
rs.add("<fqdn>:<port>")
```
For example:
Add mdb2 to replica set from mdb1:
```
rs.add("bdb-cluster-1.northeurope.cloudapp.azure.com:27017")
```
Add mdb3 to replica set from mdb1:
```
rs.add("bdb-cluster-2.northeurope.cloudapp.azure.com:27017")
```

5
k8s/mongodb/container/docker_build_and_push.bash
View File

@ -1,5 +1,4 @@
#!/bin/bash
docker build -t bigchaindb/mongodb:3.3 .
docker push bigchaindb/mongodb:3.3
docker build -t bigchaindb/localmongodb:1.0 .
docker push bigchaindb/localmongodb:1.0

6
k8s/mongodb/container/mongod.conf.template
View File

@ -25,8 +25,6 @@ systemLog:
verbosity: 0
query:
verbosity: 0
replication:
verbosity: 0
sharding:
verbosity: 0
storage:
@ -95,7 +93,3 @@ storage:
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
replication:
replSetName: REPLICA_SET_NAME
enableMajorityReadConcern: true

24
k8s/mongodb/container/mongod_entrypoint.bash
View File

@ -3,10 +3,8 @@ set -euo pipefail
MONGODB_PORT=""
MONGODB_KEY_FILE_PATH=""
#MONGODB_KEY_FILE_PASSWORD=""
MONGODB_CA_FILE_PATH=""
MONGODB_CRL_FILE_PATH=""
REPLICA_SET_NAME=""
MONGODB_FQDN=""
MONGODB_IP=""
@ -21,11 +19,6 @@ while [[ $# -gt 1 ]]; do
MONGODB_KEY_FILE_PATH="$2"
shift
;;
--mongodb-key-file-password)
# TODO(Krish) move this to a mapped file later
MONGODB_KEY_FILE_PASSWORD="$2"
shift
;;
--mongodb-ca-file-path)
MONGODB_CA_FILE_PATH="$2"
shift
@ -34,10 +27,6 @@ while [[ $# -gt 1 ]]; do
MONGODB_CRL_FILE_PATH="$2"
shift
;;
--replica-set-name)
REPLICA_SET_NAME="$2"
shift
;;
--mongodb-fqdn)
MONGODB_FQDN="$2"
shift
@ -59,20 +48,15 @@ while [[ $# -gt 1 ]]; do
done
# sanity checks
if [[ -z "${REPLICA_SET_NAME:?REPLICA_SET_NAME not specified. Exiting!}" || \
-z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \
if [[ -z "${MONGODB_PORT:?MONGODB_PORT not specified. Exiting!}" || \
-z "${MONGODB_FQDN:?MONGODB_FQDN not specified. Exiting!}" || \
-z "${MONGODB_IP:?MONGODB_IP not specified. Exiting!}" || \
-z "${MONGODB_KEY_FILE_PATH:?MONGODB_KEY_FILE_PATH not specified. Exiting!}" || \
-z "${MONGODB_CA_FILE_PATH:?MONGODB_CA_FILE_PATH not specified. Exiting!}" || \
-z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" ]] ; then
# Not handling the STORAGE_ENGINE_CACHE_SIZE because
# it is optional. If not specified the default cache
# size is: max((50% RAM - 1GB), 256MB)
echo "Missing required enviroment variable(s)."
-z "${MONGODB_CRL_FILE_PATH:?MONGODB_CRL_FILE_PATH not specified. Exiting!}" || \
-z "${STORAGE_ENGINE_CACHE_SIZE:=''}" ]] ; then
exit 1
else
echo REPLICA_SET_NAME="$REPLICA_SET_NAME"
echo MONGODB_PORT="$MONGODB_PORT"
echo MONGODB_FQDN="$MONGODB_FQDN"
echo MONGODB_IP="$MONGODB_IP"
@ -88,10 +72,8 @@ HOSTS_FILE_PATH=/etc/hosts
# configure the mongod.conf file
sed -i "s|MONGODB_PORT|${MONGODB_PORT}|g" ${MONGODB_CONF_FILE_PATH}
sed -i "s|MONGODB_KEY_FILE_PATH|${MONGODB_KEY_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH}
#sed -i "s|MONGODB_KEY_FILE_PASSWORD|${MONGODB_KEY_FILE_PASSWORD}|g" ${MONGODB_CONF_FILE_PATH}
sed -i "s|MONGODB_CA_FILE_PATH|${MONGODB_CA_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH}
sed -i "s|MONGODB_CRL_FILE_PATH|${MONGODB_CRL_FILE_PATH}|g" ${MONGODB_CONF_FILE_PATH}
sed -i "s|REPLICA_SET_NAME|${REPLICA_SET_NAME}|g" ${MONGODB_CONF_FILE_PATH}
if [ ! -z "$STORAGE_ENGINE_CACHE_SIZE" ]; then
if [[ "$STORAGE_ENGINE_CACHE_SIZE" =~ ^[0-9]+(G|M|T)B$ ]]; then
sed -i.bk "s|STORAGE_ENGINE_CACHE_SIZE|${STORAGE_ENGINE_CACHE_SIZE}|g" ${MONGODB_CONF_FILE_PATH}

4
k8s/mongodb/mongo-sc.yaml
View File

@ -1,5 +1,5 @@
####################################################################
# This YAML section desribes a StorageClass for the mongodb dbPath #
# This YAML section desribes a StorageClass for the mongodb dbPath #
####################################################################
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1
@ -15,7 +15,7 @@ parameters:
#kind: Managed
---
######################################################################
# This YAML section desribes a StorageClass for the mongodb configDB #
# This YAML section desribes a StorageClass for the mongodb configDB #
######################################################################
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1

9
k8s/mongodb/mongo-ss.yaml
View File

@ -21,7 +21,7 @@ spec:
terminationGracePeriodSeconds: 10
containers:
- name: mongodb
image: bigchaindb/mongodb:3.2
image: bigchaindb/localmongodb:1.0
imagePullPolicy: IfNotPresent
env:
- name: MONGODB_FQDN
@ -33,11 +33,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: MONGODB_REPLICA_SET_NAME
valueFrom:
configMapKeyRef:
name: vars
key: mongodb-replicaset-name
- name: MONGODB_PORT
valueFrom:
configMapKeyRef:
@ -57,8 +52,6 @@ spec:
- /etc/mongod/ca/ca.pem
- --mongodb-crl-file-path
- /etc/mongod/ca/crl.pem
- --replica-set-name
- $(MONGODB_REPLICA_SET_NAME)
- --mongodb-fqdn
- $(MONGODB_FQDN)
- --mongodb-ip

6
k8s/nginx-http/container/Dockerfile
View File

@ -6,6 +6,6 @@ RUN apt-get update \
&& apt-get autoremove \
&& apt-get clean
COPY nginx.conf.template /etc/nginx/nginx.conf
COPY nginx_entrypoint.bash /
EXPOSE 80 27017
ENTRYPOINT ["/nginx_entrypoint.bash"]
COPY nginx_tmt_entrypoint.bash /
EXPOSE 80 27017 9986 46656
ENTRYPOINT ["/nginx_tmt_entrypoint.bash"]

2
k8s/nginx-http/container/README.md
View File

@ -9,7 +9,7 @@ reflect any changes made to the container.
### Note about testing Websocket connections:
You can test the WebSocket server by using
You can test the WebSocket server by using
[wsc](https://www.npmjs.com/package/wsc) tool with a command like:
`wsc -er ws://localhost:9985/api/v1/streams/valid_transactions`.

4
k8s/nginx-http/container/docker_build_and_push.bash
View File

@ -1,5 +1,5 @@
#!/bin/bash
docker build -t bigchaindb/nginx_http:1.1 .
docker build -t bigchaindb/nginx_http:unstable .
docker push bigchaindb/nginx_http:1.1
docker push bigchaindb/nginx_http:unstable

26
k8s/nginx-http/container/nginx.conf.template
View File

@ -123,16 +123,15 @@ http {
}
}
# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP
# connection.
# NGINX stream block for TCP and UDP proxies.
stream {
log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
'$proxy_protocol_addr $proxy_protocol_port '
'$protocol $status $session_time $bytes_sent '
'$bytes_received "$upstream_addr" "$upstream_bytes_sent" '
'"$upstream_bytes_received" "$upstream_connect_time" ';
access_log /dev/stdout mdb_log buffer=16k flush=5s;
access_log /dev/stdout bdb_log buffer=16k flush=5s;
# Define a zone 'two' of size 10 megabytes to store the counters
# that hold number of TCP connections from a specific IP address.
@ -149,16 +148,23 @@ stream {
# The following map block enables lazy-binding to the backend at runtime,
# rather than binding as soon as NGINX starts.
map $remote_addr $mdb_backend {
default MONGODB_BACKEND_HOST;
map $remote_addr $tm_backend {
default TM_BACKEND_HOST;
}
# Frontend server to forward connections to MDB instance.
# Server to forward connection to nginx instance hosting
# tendermint node public key.
server {
listen MONGODB_FRONTEND_PORT so_keepalive=10m:1m:5;
preread_timeout 30s;
listen TM_PUB_KEY_ACCESS_PORT;
proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT;
}
# Server to forward p2p connections to Tendermint instance.
server {
listen TM_P2P_PORT so_keepalive=3m:1m:5;
preread_timeout 60s;
tcp_nodelay on;
proxy_pass $mdb_backend:MONGODB_BACKEND_PORT;
proxy_pass $tm_backend:TM_P2P_PORT;
}
}

11
k8s/nginx-http/container/nginx_entrypoint.bash
View File

@ -31,7 +31,10 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!
-z "${bdb_api_port:?BIGCHAINDB_API_PORT not specified. Exiting!}" || \
-z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \
-z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \
-z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" ]]; then
-z "${health_check_port:?HEALTH_CHECK_PORT not specified.}" || \
-z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \
-z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \
-z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then
exit 1
else
echo CLUSTER_FRONTEND_PORT="$cluster_frontend_port"
@ -43,6 +46,9 @@ else
echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host"
echo BIGCHAINDB_API_PORT="$bdb_api_port"
echo BIGCHAINDB_WS_PORT="$bdb_ws_port"
echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port"
echo TM_BACKEND_HOST="$tm_backend_host"
echo TM_P2P_PORT="$tm_p2p_port"
fi
NGINX_CONF_FILE=/etc/nginx/nginx.conf
@ -57,6 +63,9 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE}
sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE}
sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE}
sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE}
sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE}
sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE}
sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE}
# start nginx
echo "INFO: starting nginx..."

28
k8s/nginx-http/nginx-http-dep.yaml
View File

@ -30,11 +30,6 @@ spec:
configMapKeyRef:
name: vars
key: cluster-dns-server-ip
- name: MONGODB_FRONTEND_PORT
valueFrom:
configMapKeyRef:
name: vars
key: mongodb-frontend-port
- name: MONGODB_BACKEND_HOST
valueFrom:
configMapKeyRef:
@ -60,14 +55,33 @@ spec:
configMapKeyRef:
name: vars
key: bigchaindb-ws-port
- name: TM_PUB_KEY_ACCESS_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-pub-key-access
- name: TM_BACKEND_HOST
valueFrom:
configMapKeyRef:
name: tendermint-config
key: ngx-tm-instance-name
- name: TM_P2P_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-p2p-port
ports:
- containerPort: "<mongodb-frontend-port from ConfigMap>"
protocol: TCP
- containerPort: "<cluster-health-check-port from ConfigMap>"
protocol: TCP
name: ngx-health
- containerPort: "<cluster-frontend-port from ConfigMap>"
protocol: TCP
- containerPort: "<tm-pub-key-access from ConfigMap>"
protocol: TCP
name: tm-pub-key
- containerPort: "<tm-p2p-port from ConfigMap>"
protocol: TCP
name: tm-p2p-port
livenessProbe:
httpGet:
path: /health

8
k8s/nginx-http/nginx-http-svc.yaml
View File

@ -17,4 +17,12 @@ spec:
targetPort: "<cluster-frontend-port from ConfigMap>"
name: public-cluster-port
protocol: TCP
- port: "<tm-pub-key-access from ConfigMap>"
targetPort: "<tm-pub-key-access from ConfigMap>"
name: tm-pub-key-access
protocol: TCP
- port: "<tm-p2p-port from ConfigMap>"
targetPort: "<tm-p2p-port from ConfigMap>"
protocol: TCP
name: tm-p2p-port
type: LoadBalancer

8
k8s/nginx-https/container/Dockerfile
View File

@ -5,7 +5,7 @@ RUN apt-get update \
&& apt-get -y upgrade \
&& apt-get autoremove \
&& apt-get clean
COPY nginx.conf.template /etc/nginx/nginx.conf
COPY nginx_entrypoint.bash /
EXPOSE 80 443 27017
ENTRYPOINT ["/nginx_entrypoint.bash"]
COPY nginx.conf.tmt.template /etc/nginx/nginx.conf
COPY nginx_tmt_entrypoint.bash /
EXPOSE 80 443 27017 9986 46656
ENTRYPOINT ["/nginx_tmt_entrypoint.bash"]

2
k8s/nginx-https/container/README.md
View File

@ -9,7 +9,7 @@ reflect any changes made to the container.
### Note about testing Websocket connections:
You can test the WebSocket server by using
You can test the WebSocket server by using
[wsc](https://www.npmjs.com/package/wsc) tool with a command like:
`wsc -er wss://localhost:9985/api/v1/streams/valid_transactions`.

4
k8s/nginx-https/container/docker_build_and_push.bash
View File

@ -1,5 +1,5 @@
#!/bin/bash
docker build -t bigchaindb/nginx_https:1.1 .
docker build -t bigchaindb/nginx_https:unstable .
docker push bigchaindb/nginx_https:1.1
docker push bigchaindb/nginx_https:unstable

33
k8s/nginx-https/container/nginx.conf.template
View File

@ -2,8 +2,7 @@
# 1. Acts as the HTTPS termination point.
# 2. Forwards BDB HTTP requests to OpenResty backend.
# 3. Forwards BDB WS requests to BDB backend.
# 4. Forwards MDB TCP connections to MDB backend.
# 5. Does health check with LB.
# 4. Does health check with LB.
worker_processes 2;
daemon off;
@ -152,16 +151,15 @@ http {
}
}
# NGINX stream block for TCP and UDP proxies. Used to proxy MDB TCP
# connection.
# NGINX stream block for TCP and UDP proxies.
stream {
log_format mdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
log_format bdb_log '[$time_iso8601] $realip_remote_addr $remote_addr '
'$proxy_protocol_addr $proxy_protocol_port '
'$protocol $status $session_time $bytes_sent '
'$bytes_received "$upstream_addr" "$upstream_bytes_sent" '
'"$upstream_bytes_received" "$upstream_connect_time" ';
access_log /dev/stdout mdb_log buffer=16k flush=5s;
access_log /dev/stdout bdb_log buffer=16k flush=5s;
# Define a zone 'two' of size 10 megabytes to store the counters
# that hold number of TCP connections from a specific IP address.
@ -170,10 +168,6 @@ stream {
# Enable logging when connections are being throttled.
limit_conn_log_level notice;
# For a multi node BigchainDB deployment we need around 2^5 connections
# (for inter-node communication)per node via NGINX, we can bump this up in case
# there is a requirement to scale up. But we should not remove this
# for security reasons.
# Allow 256 connections from the same IP address.
limit_conn two 256;
@ -182,16 +176,23 @@ stream {
# The following map block enables lazy-binding to the backend at runtime,
# rather than binding as soon as NGINX starts.
map $remote_addr $mdb_backend {
default MONGODB_BACKEND_HOST;
map $remote_addr $tm_backend {
default TM_BACKEND_HOST;
}
# Frontend server to forward connections to MDB instance.
# Server to forward connection to nginx instance hosting
# tendermint node public key.
server {
listen MONGODB_FRONTEND_PORT so_keepalive=3m:1m:5;
preread_timeout 30s;
listen TM_PUB_KEY_ACCESS_PORT;
proxy_pass $tm_backend:TM_PUB_KEY_ACCESS_PORT;
}
# Server to forward p2p connections to Tendermint instance.
server {
listen TM_P2P_PORT so_keepalive=3m:1m:5;
preread_timeout 60s;
tcp_nodelay on;
proxy_pass $mdb_backend:MONGODB_BACKEND_PORT;
proxy_pass $tm_backend:TM_P2P_PORT;
}
}

17
k8s/nginx-https/container/nginx_entrypoint.bash
View File

@ -27,6 +27,10 @@ bdb_backend_host=`printenv BIGCHAINDB_BACKEND_HOST`
bdb_api_port=`printenv BIGCHAINDB_API_PORT`
bdb_ws_port=`printenv BIGCHAINDB_WS_PORT`
# Tendermint vars
tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT`
tm_backend_host=`printenv TM_BACKEND_HOST`
tm_p2p_port=`printenv TM_P2P_PORT`
# sanity check
if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!}" || \
@ -40,7 +44,11 @@ if [[ -z "${cluster_frontend_port:?CLUSTER_FRONTEND_PORT not specified. Exiting!
-z "${bdb_ws_port:?BIGCHAINDB_WS_PORT not specified. Exiting!}" || \
-z "${dns_server:?DNS_SERVER not specified. Exiting!}" || \
-z "${health_check_port:?HEALTH_CHECK_PORT not specified. Exiting!}" || \
-z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" ]]; then
-z "${cluster_fqdn:?CLUSTER_FQDN not specified. Exiting!}" || \
-z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \
-z "${tm_backend_host:?TM_BACKEND_HOST not specified. Exiting!}" || \
-z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" ]]; then
echo "Missing required environment variables. Exiting!"
exit 1
else
echo CLUSTER_FQDN="$cluster_fqdn"
@ -55,6 +63,9 @@ else
echo BIGCHAINDB_BACKEND_HOST="$bdb_backend_host"
echo BIGCHAINDB_API_PORT="$bdb_api_port"
echo BIGCHAINDB_WS_PORT="$bdb_ws_port"
echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port"
echo TM_BACKEND_HOST="$tm_backend_host"
echo TM_P2P_PORT="$tm_p2p_port"
fi
NGINX_CONF_FILE=/etc/nginx/nginx.conf
@ -72,8 +83,10 @@ sed -i "s|BIGCHAINDB_API_PORT|${bdb_api_port}|g" ${NGINX_CONF_FILE}
sed -i "s|BIGCHAINDB_WS_PORT|${bdb_ws_port}|g" ${NGINX_CONF_FILE}
sed -i "s|DNS_SERVER|${dns_server}|g" ${NGINX_CONF_FILE}
sed -i "s|HEALTH_CHECK_PORT|${health_check_port}|g" ${NGINX_CONF_FILE}
sed -i "s|TM_PUB_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE}
sed -i "s|TM_BACKEND_HOST|${tm_backend_host}|g" ${NGINX_CONF_FILE}
sed -i "s|TM_P2P_PORT|${tm_p2p_port}|g" ${NGINX_CONF_FILE}
# start nginx
echo "INFO: starting nginx..."
exec nginx -c /etc/nginx/nginx.conf

28
k8s/nginx-https/nginx-https-dep.yaml
View File

@ -35,11 +35,6 @@ spec:
configMapKeyRef:
name: vars
key: cluster-dns-server-ip
- name: MONGODB_FRONTEND_PORT
valueFrom:
configMapKeyRef:
name: vars
key: mongodb-frontend-port
- name: MONGODB_BACKEND_HOST
valueFrom:
configMapKeyRef:
@ -75,18 +70,37 @@ spec:
configMapKeyRef:
name: vars
key: bigchaindb-ws-port
- name: TM_PUB_KEY_ACCESS_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-pub-key-access
- name: TM_BACKEND_HOST
valueFrom:
configMapKeyRef:
name: tendermint-config
key: ngx-tm-instance-name
- name: TM_P2P_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-p2p-port
ports:
# return a pretty error message on port 80, since we are expecting
# HTTPS traffic.
- containerPort: 80
protocol: TCP
- containerPort: "<mongodb-frontend-port from ConfigMap>"
protocol: TCP
- containerPort: "<cluster-frontend-port from ConfigMap>"
protocol: TCP
- containerPort: "<cluster-health-check-port from ConfigMap>"
protocol: TCP
name: ngx-port
- containerPort: "<tm-pub-key-access from ConfigMap>"
protocol: TCP
name: tm-pub-key
- containerPort: "<tm-p2p-port from ConfigMap>"
protocol: TCP
name: tm-p2p-port
livenessProbe:
httpGet:
path: /health

8
k8s/nginx-https/nginx-https-svc.yaml
View File

@ -21,6 +21,14 @@ spec:
targetPort: "<mongodb-frontend-port from ConfigMap>"
name: public-mdb-port
protocol: TCP
- port: "<tm-pub-key-access from ConfigMap>"
targetPort: "<tm-pub-key-access from ConfigMap>"
name: tm-pub-key-access
protocol: TCP
- port: "<tm-p2p-port from ConfigMap>"
targetPort: "<tm-p2p-port from ConfigMap>"
protocol: TCP
name: tm-p2p-port
- port: 80
targetPort: 80
name: public-insecure-cluster-port

12
k8s/tendermint/nginx_container/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM nginx:stable
LABEL maintainer "dev@bigchaindb.com"
WORKDIR /
RUN apt-get update \
&& apt-get -y upgrade \
&& apt-get autoremove \
&& apt-get clean
COPY nginx.conf.template /etc/nginx/conf.d/access_pub_key.conf
COPY nginx_entrypoint.bash /
VOLUME /usr/share/nginx
EXPOSE 9986
ENTRYPOINT ["/nginx_entrypoint.bash"]

19
k8s/tendermint/nginx_container/README.md Normal file
View File

@ -0,0 +1,19 @@
## Nginx container for hosting public key for a tendermint instance
### Step 1: Build and Push the Latest Container
Use the `docker_build_and_push.bash` script to build the latest docker image
and upload it to Docker Hub.
Ensure that the image tag is updated to a new version number to properly
reflect any changes made to the container.
### Step 2: Run the container
```
docker run \
--name=tendermint_instance_pub_key \
--env TENDERMINT_PUB_KEY_ACCESS_PORT=''
--publish=<nginx port for external connections>:<corresponding host port> \
--volume=<host dir with public key>:/tendermint_node_data \
bigchaindb/nginx_pub_key_access:<version_number>
```

5
k8s/tendermint/nginx_container/docker_build_and_push.bash Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
docker build -t bigchaindb/nginx_pub_key_access:unstable .
docker push bigchaindb/nginx_pub_key_access:unstable

10
k8s/tendermint/nginx_container/nginx.conf.template Normal file
View File

@ -0,0 +1,10 @@
# Serve the public key for a tendermint instance
server {
listen PUBLIC_KEY_ACCESS_PORT default_server;
listen [::]:PUBLIC_KEY_ACCESS_PORT default_server ipv6only=on;
location / {
root /usr/share/nginx/;
autoindex on;
}
}

21
k8s/tendermint/nginx_container/nginx_entrypoint.bash Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
set -euo pipefail
# Tendermint public key access port
tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT`
if [[ -z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting}" ]]; then
exit 1
else
echo TM_PUB_KEY_ACCESS_PORT="$tm_pub_key_access_port"
fi
NGINX_CONF_FILE=/etc/nginx/conf.d/access_pub_key.conf
# configure the access_pub_key file with env variable(s)
sed -i "s|PUBLIC_KEY_ACCESS_PORT|${tm_pub_key_access_port}|g" ${NGINX_CONF_FILE}
cat /etc/nginx/conf.d/access_pub_key.conf
# start nginx
echo "INFO: starting nginx..."
exec nginx -g "daemon off;"

17
k8s/tendermint/tendermint-ext-conn-svc.yaml Normal file
View File

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
# Name of tendermint instance you are trying to connect to
# e.g. tm-instance-0
name: "<remote-tendermint-host>"
namespace: default
spec:
ports:
spec:
ports:
- port: "<tm-p2p-port from ConfigMap>"
name: p2p
- port: "<tm-rpc-port from ConfigMap>"
name: pubkey
# FQDN of remote cluster/NGINX instance
externalName: "<dns-name-remote-nginx>"

41
k8s/tendermint/tendermint-pv.yaml Normal file
View File

@ -0,0 +1,41 @@
#########################################################
# This YAML section desribes a k8s PV for tendermint db #
#########################################################
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv-tm-db
spec:
accessModes:
- ReadWriteOnce
azureDisk:
cachingMode: None
diskName: <Azure Disk Name>
diskURI: <Azure Disk URL>
fsType: ext4
readOnly: false
capacity:
storage: 20Gi
persistentVolumeReclaimPolicy: Retain
storageClassName: tendermint-db
---
##############################################################
# This YAML section desribes a k8s PV for Tendermint config #
##############################################################
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv-tm-configdb
spec:
accessModes:
- ReadWriteOnce
azureDisk:
cachingMode: None
diskName: <Azure Disk Name>
diskURI: <Azure Disk URL>
fsType: ext4
readOnly: false
capacity:
storage: 1Gi
persistentVolumeReclaimPolicy: Retain
storageClassName: tendermint-config-db

32
k8s/tendermint/tendermint-pvc.yaml Normal file
View File

@ -0,0 +1,32 @@
##########################################################
# This section file desribes a k8s pvc for tendermint db #
##########################################################
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: tendermint-db-claim
annotations:
volume.beta.kubernetes.io/storage-class: tendermint-db
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
---
################################################################
# This YAML section desribes a k8s pvc for tendermint configdb #
################################################################
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: tendermint-config-db-claim
annotations:
volume.beta.kubernetes.io/storage-class: tendermint-config-db
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi

31
k8s/tendermint/tendermint-sc.yaml Normal file
View File

@ -0,0 +1,31 @@
###################################################################
# This YAML section desribes a StorageClass for the tendermint db #
###################################################################
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1
metadata:
name: tendermint-db
provisioner: kubernetes.io/azure-disk
parameters:
skuName: Premium_LRS #[Premium_LRS, Standard_LRS]
location: westeurope
# If you have created a different storage account e.g. for Premium Storage
#storageAccount: <Storage account name>
# Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment)
#kind: Managed
---
#########################################################################
# This YAML section desribes a StorageClass for the tendermint configdb #
#########################################################################
kind: StorageClass
apiVersion: storage.k8s.io/v1beta1
metadata:
name: tendermint-config-db
provisioner: kubernetes.io/azure-disk
parameters:
skuName: Premium_LRS #[Premium_LRS, Standard_LRS]
location: westeurope
# If you have created a different storage account e.g. for Premium Storage
#storageAccount: <Storage account name>
# Use Managed Disk(s) with VMs using Managed Disks(Only used for Tectonic deployment)
#kind: Managed

115
k8s/tendermint/tendermint-ss.yaml Normal file
View File

@ -0,0 +1,115 @@
#################################################################################
# This YAML file desribes a StatefulSet with a service for running and exposing #
# a Tendermint instance. It depends on the tendermint-config-db-claim #
# and tendermint-db-claim k8s pvc. #
#################################################################################
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: tm-instance-0-ss
namespace: default
spec:
serviceName: tm-instance-0
replicas: 1
template:
metadata:
name: tm-instance-0-ss
labels:
app: tm-instance-0-ss
spec:
restartPolicy: Always
volumes:
- name: tm-data
persistentVolumeClaim:
claimName: tendermint-db-claim
- name: tm-config-data
persistentVolumeClaim:
claimName: tendermint-config-db-claim
containers:
# Treating nginx + tendermint as a POD because they should not
# exist without each other
# Nginx container for hosting public key of this ndoe
- name: nginx
imagePullPolicy: Always
image: bigchaindb/nginx_pub_key_access:unstable
env:
- name: TM_PUB_KEY_ACCESS_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-pub-key-access
ports:
- containerPort: "<tm-pub-key-access from ConfigMap>"
name: tm-pk-access
volumeMounts:
- name: tm-config-data
mountPath: /usr/share/nginx
readOnly: true
#Tendermint container
- name: tendermint
imagePullPolicy: Always
image: bigchaindb/tendermint:unstable
env:
- name: TM_SEEDS
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-seeds
- name: TM_VALIDATOR_POWER
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-validator-power
- name: TM_VALIDATORS
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-validators
- name: TM_PUB_KEY_ACCESS_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-pub-key-access
- name: TM_GENESIS_TIME
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-genesis-time
- name: TM_CHAIN_ID
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-chain-id
- name: TM_P2P_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-p2p-port
- name: TMHOME
value: /tendermint
- name: TM_PROXY_APP
valueFrom:
configMapKeyRef:
name: vars
key: bdb-instance-name
- name: TM_ABCI_PORT
valueFrom:
configMapKeyRef:
name: tendermint-config
key: tm-abci-port
# Resource constraint on the pod, can be changed
resources:
limits:
cpu: 200m
memory: 5G
volumeMounts:
- name: tm-data
mountPath: /tendermint
- name: tm-config-data
mountPath: /tendermint_node_data
ports:
- containerPort: "<tm-p2p-port from ConfigMap>"
name: p2p
- containerPort: "<tm-rpc-port from ConfigMap>"
name: rpc

24
k8s/tendermint/tendermint-svc.yaml Normal file
View File

@ -0,0 +1,24 @@
apiVersion: v1
kind: Service
metadata:
name: tm-instance-1
namespace: default
labels:
name: tm-instance-1
spec:
selector:
app: tm-instance-1-ss
ports:
- port: "<tm-p2p-port from ConfigMap>"
targetPort: "<tm-p2p-port from ConfigMap>"
name: p2p
protocol: TCP
- port: "<tm-rpc-port from ConfigMap>"
targetPort: "<tm-rpc-port from ConfigMap>"
name: rpc
protocol: TCP
- port: "<tm-pub-key-access from ConfigMap>"
targetPort: "<tm-pub-key-access from ConfigMap>"
name: pub-key-access
protocol: TCP
clusterIP: None

8
k8s/tendermint/tendermint_container/Dockerfile Normal file
View File

@ -0,0 +1,8 @@
FROM tendermint/tendermint:develop
LABEL maintainer "dev@bigchaindb.com"
WORKDIR /
COPY genesis.json.template /etc/tendermint/genesis.json
COPY tendermint_entrypoint.bash /
VOLUME /tendermint /tendermint_node_data
EXPOSE 46656 46657
ENTRYPOINT ["/tendermint_entrypoint.bash"]

29
k8s/tendermint/tendermint_container/README.md Normal file
View File

@ -0,0 +1,29 @@
## Tendermint container used for BFT replication and consensus
### Step 1: Build and Push the Latest Container
Use the `docker_build_and_push.bash` script to build the latest docker image
and upload it to Docker Hub.
Ensure that the image tag is updated to a new version number to properly
reflect any changes made to the container.
### Step 2: Run the container
```
docker run \
--name=tendermint \
--env TM_PUB_KEY_ACCESS_PORT=<port to access public keys hosted by nginx> \
--env TM_SEEDS=<commad separated list of all nodes IP addresses/Hostnames> \
--env TM_VALIDATOR_POWER=<voting power of node> \
--env TM_VALIDATORS=<list of all validators> \
--env TM_GENESIS_TIME=<genesis time> \
--env TM_CHAIN_ID=<chain id> \
--env TM_P2P_PORT=<Port used by all peers to communicate> \
--env TMHOME=<Tendermint home directory containing all config files> \
--env TM_PROXY_APP=<Hostname/IP address of app> \
--publish=<rpc port on host>:<rpc port> \
--publish=<p2p port on host>:<p2p port> \
--volume <host dir for tendermint data>:/tendermint \
--volume=<host dir for public key>:/tendermint_node_data \
bigchaindb/tendermint:<version_number>
```

5
k8s/tendermint/tendermint_container/docker_build_and_push.bash Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
docker build -t bigchaindb/tendermint:unstable .
docker push bigchaindb/tendermint:unstable

6
k8s/tendermint/tendermint_container/genesis.json.template Normal file
View File

@ -0,0 +1,6 @@
{
"genesis_time": TM_GENESIS_TIME,
"chain_id": TM_CHAIN_ID,
"validators": [],
"app_hash": ""
}

98
k8s/tendermint/tendermint_container/tendermint_entrypoint.bash Executable file
View File

@ -0,0 +1,98 @@
#!/bin/bash
set -euo pipefail
# Cluster vars
tm_seeds=`printenv TM_SEEDS`
tm_validators=`printenv TM_VALIDATORS`
tm_validator_power=`printenv TM_VALIDATOR_POWER`
tm_pub_key_access_port=`printenv TM_PUB_KEY_ACCESS_PORT`
tm_genesis_time=`printenv TM_GENESIS_TIME`
tm_chain_id=`printenv TM_CHAIN_ID`
tm_p2p_port=`printenv TM_P2P_PORT`
# tendermint node vars
tmhome=`printenv TMHOME`
tm_proxy_app=`printenv TM_PROXY_APP`
tm_abci_port=`printenv TM_ABCI_PORT`
# sanity check
if [[ -z "${tm_seeds:?TM_SEEDS not specified. Exiting!}" || \
-z "${tm_validators:?TM_VALIDATORS not specified. Exiting!}" || \
-z "${tm_validator_power:?TM_VALIDATOR_POWER not specified. Exiting!}" || \
-z "${tm_pub_key_access_port:?TM_PUB_KEY_ACCESS_PORT not specified. Exiting!}" || \
-z "${tm_genesis_time:?TM_GENESIS_TIME not specified. Exiting!}" || \
-z "${tm_chain_id:?TM_CHAIN_ID not specified. Exiting!}" || \
-z "${tmhome:?TMHOME not specified. Exiting!}" || \
-z "${tm_p2p_port:?TM_P2P_PORT not specified. Exiting!}" || \
-z "${tm_abci_port:?TM_ABCI_PORT not specified. Exiting! }" ]]; then
echo "Missing required enviroment variables."
exit 1
else
echo tm_seeds="$TM_SEEDS"
echo tm_validators="$TM_VALIDATORS"
echo tm_validator_power="$TM_VALIDATOR_POWER"
echo tm_pub_key_access_port="$TM_PUB_KEY_ACCESS_PORT"
echo tm_genesis_time="$TM_GENESIS_TIME"
echo tm_chain_id="$TM_CHAIN_ID"
echo tmhome="$TMHOME"
echo tm_p2p_port="$TM_P2P_PORT"
echo tm_abci_port="$TM_ABCI_PORT"
fi
# copy template
cp /etc/tendermint/genesis.json /tendermint/genesis.json
TM_GENESIS_FILE=/tendermint/genesis.json
TM_PUB_KEY_DIR=/tendermint_node_data
# configure the nginx.conf file with env variables
sed -i "s|TM_GENESIS_TIME|\"${tm_genesis_time}\"|g" ${TM_GENESIS_FILE}
sed -i "s|TM_CHAIN_ID|\"${tm_chain_id}\"|g" ${TM_GENESIS_FILE}
if [ ! -f /tendermint/priv_validator.json ]; then
tendermint gen_validator > /tendermint/priv_validator.json
# pub_key.json will be served by the nginx container
cat /tendermint/priv_validator.json
cat /tendermint/priv_validator.json | jq ".pub_key" > "$TM_PUB_KEY_DIR"/pub_key.json
fi
# fill genesis file with validators
IFS=',' read -ra VALS_ARR <<< "$TM_VALIDATORS"
IFS=',' read -ra VAL_POWERS_ARR <<< "$TM_VALIDATOR_POWER"
if [ ${#VALS_ARR[@]} -ne ${#VAL_POWERS_ARR[@]} ]; then
echo "Invalid configuration of Validator(s) and Validator Power(s)"
exit 1
fi
for i in "${!VALS_ARR[@]}"; do
# wait until validator generates priv/pub key pair
set +e
echo Validator: "${VALS_ARR[$i]}"
echo Validator Power: "${VALS_POWERS_ARR[$i]}"
echo "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json"
curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null
ERR=$?
while [ "$ERR" != 0 ]; do
sleep 5
curl -s --fail "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" > /dev/null
ERR=$?
done
set -e
# add validator to genesis file along with its pub_key
curl -s "http://${VALS_ARR[$i]}:$tm_pub_key_access_port/pub_key.json" | jq ". as \$k | {pub_key: \$k, power: ${VAL_POWERS_ARR[$i]}, name: \"${VALS_ARR[$i]}\"}" > pub_validator.json
cat /tendermint/genesis.json | jq ".validators |= .+ [$(cat pub_validator.json)]" > tmpgenesis && mv tmpgenesis /tendermint/genesis.json
rm pub_validator.json
done
# construct seeds
IFS=',' read -ra SEEDS_ARR <<< "$tm_seeds"
seeds=()
for s in "${SEEDS_ARR[@]}"; do
seeds+=("$s:$tm_p2p_port")
done
seeds=$(IFS=','; echo "${seeds[*]}")
# start nginx
echo "INFO: starting tendermint..."
exec tendermint node --p2p.seeds="$seeds" --moniker="`hostname`" --proxy_app="tcp://$tm_proxy_app:$tm_abci_port"