From cc66d5aaa5e83e48746bc17ceccb7d2f633f03d8 Mon Sep 17 00:00:00 2001 From: Krish Date: Fri, 21 Apr 2017 14:41:12 +0200 Subject: [PATCH] Single node setup (#1418) * Add more tools to the toolbox container * Add mongodb monitoring agent * Add a bigchaindb/mongodb-monitoring-agent container that includes the monitoring agent. * It makes use of an api key provided by MongoDB Cloud Manager. This is included in the configuration/config-map.yaml file. * Changes to mongodb StatefulSet configuration Changes to bump up mongodb version to v3.4.3. Add configuration settings for mongodb instance name in ConfigMap. Split the mongodb service to a new configuration file. * Modify bigchaindb deployment config * Bugfix to remove keyring field for the first node. * Split the mongodb service to a new configuration file. * Add mongodb backup agent * Add a bigchaindb/mongodb-backup-agent container that includes the backup agent. * It makes use of an api key provided by MongoDB Cloud Manager. This is included in the configuration/config-map.yaml file. * Changes to nginx deployment config * Allow 'all' by default for now. This is included in the configuration/config-map.yaml file. * Dynamically resolve DNS addresses of our backend services; cache DNS resolution for 20s. * Configure DNS based on user provided resolver. This helps in user deciding to provide 8.8.8.8 or a custom DNS for name resolution. For k8s deployments, we use the hardcoded k8s DNS IP of 10.0.0.10. * Changes to nginx-3scale deployment config * Use the common ConfigMap in configuration/config-map.yaml file. * Removing prefix `v` from the docker tag for mongodb-monitoring-agent and mongodb containers * Bumping up version for nginx-3scale container * Add small helper scripts for docker build and push of mongodb monitoring and backup agents * Documentation for setting up the first node with monitoring and backup agents --- .../cloud-deployment-templates/first-node.rst | 455 ++++++++++++++++++ .../cloud-deployment-templates/index.rst | 2 +- k8s/bigchaindb/bigchaindb-dep.yaml | 43 +- k8s/bigchaindb/bigchaindb-svc.yaml | 16 + k8s/configuration/config-map.yaml | 36 ++ k8s/mongodb-backup-agent/container/Dockerfile | 19 + .../container/docker_build_and_push.bash | 5 + .../mongodb_backup_agent_entrypoint.bash | 20 + .../mongo-backup-dep.yaml | 27 ++ .../container/Dockerfile | 54 +++ .../container/docker_build_and_push.bash | 5 + .../mongodb_mon_agent_entrypoint.bash | 30 ++ .../mongo-mon-dep.yaml | 38 ++ k8s/mongodb/container/Dockerfile | 2 +- k8s/mongodb/container/Makefile | 2 +- k8s/mongodb/mongo-cm.yaml | 13 - k8s/mongodb/mongo-ss.yaml | 32 +- k8s/mongodb/mongo-svc.yaml | 16 + k8s/nginx-3scale/nginx-3scale-cm.yaml | 13 - k8s/nginx-3scale/nginx-3scale-dep.yaml | 4 +- k8s/nginx/container/README.md | 11 +- k8s/nginx/container/nginx.conf.template | 29 +- k8s/nginx/container/nginx_entrypoint.bash | 5 +- k8s/nginx/nginx-cm.yaml | 13 - k8s/nginx/nginx-dep.yaml | 45 +- k8s/nginx/nginx-svc.yaml | 24 + k8s/toolbox/Dockerfile | 11 +- k8s/toolbox/README.md | 2 + 28 files changed, 817 insertions(+), 155 deletions(-) create mode 100644 docs/server/source/cloud-deployment-templates/first-node.rst create mode 100644 k8s/bigchaindb/bigchaindb-svc.yaml create mode 100644 k8s/configuration/config-map.yaml create mode 100644 k8s/mongodb-backup-agent/container/Dockerfile create mode 100755 k8s/mongodb-backup-agent/container/docker_build_and_push.bash create mode 100755 k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash create mode 100644 k8s/mongodb-backup-agent/mongo-backup-dep.yaml create mode 100644 k8s/mongodb-monitoring-agent/container/Dockerfile create mode 100755 k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash create mode 100755 k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash create mode 100644 k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml delete mode 100644 k8s/mongodb/mongo-cm.yaml create mode 100644 k8s/mongodb/mongo-svc.yaml delete mode 100644 k8s/nginx-3scale/nginx-3scale-cm.yaml delete mode 100644 k8s/nginx/nginx-cm.yaml create mode 100644 k8s/nginx/nginx-svc.yaml diff --git a/docs/server/source/cloud-deployment-templates/first-node.rst b/docs/server/source/cloud-deployment-templates/first-node.rst new file mode 100644 index 00000000..06b3843b --- /dev/null +++ b/docs/server/source/cloud-deployment-templates/first-node.rst @@ -0,0 +1,455 @@ +First Node or Bootstrap Node Setup +================================== + +This document is a work in progress and will evolve over time to include +security, websocket and other settings. + +Step 1: Set Up the Cluster +-------------------------- + + .. code:: bash + + az group create --name bdb-test-cluster-0 --location westeurope --debug -- output json + + az acs create --name k8s-bdb-test-cluster-0 \ + --resource-group bdb-test-cluster-0 \ + --master-count 3 \ + --agent-count 2 \ + --admin-username ubuntu \ + --agent-vm-size Standard_D2_v2 \ + --dns-prefix k8s-bdb-test-cluster-0 \ + --ssh-key-value ~/.ssh/ \ + --orchestrator-type kubernetes \ + --debug --output json + + az acs kubernetes get-credentials \ + --resource-group bdb-test-cluster-0 \ + --name k8s-bdb-test-cluster-0 \ + --debug --output json + + echo -e "Host k8s-bdb-test-cluster-0.westeurope.cloudapp.azure.com\n ForwardAgent yes" >> ~/.ssh/config + + ssh ubuntu@k8s-bdb-test-cluster-0.westeurope.cloudapp.azure.com + + +Step 2: Connect to the Cluster UI - (optional) +---------------------------------------------- + + * Get the kubectl context for this cluster using ``kubectl config view``. + + * For the above commands, the context would be ``k8s-bdb-test-cluster-0``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 proxy -p 8001 + +Step 3. Configure the Cluster +----------------------------- + + * Use the ConfigMap in ``configuration/config-map.yaml`` file for configuring + the cluster. + + * Log in the the MongoDB Cloud Manager and select the group that will monitor + and backup this cluster from the dropdown box. + + * Go to Settings, Group Settings and copy the ``Agent Api Key``. + + * Replace the ```` field with this key. + + * Since this is the first node of the cluster, ensure that the ``data.fqdn`` + field has the value ``mdb-instance-0``. + + * We only support the value ``all`` in the ``data.allowed-hosts`` field for now. + + * Create the ConfigMap + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f configuration/config-map.yaml + +Step 4. Start the NGINX Service +------------------------------- + + * This will will give us a public IP for the cluster. + + * Once you complete this step, you might need to wait up to 10 mins for the + public IP to be assigned. + + * You have the option to use vanilla NGINX or an OpenResty NGINX integrated + with 3scale API Gateway. + + +Step 4.1. Vanilla NGINX +^^^^^^^^^^^^^^^^^^^^^^^ + + * This configuration is located in the file ``nginx/nginx-svc.yaml``. + + * Since this is the first node, rename ``metadata.name`` and ``metadata.labels.name`` + to ``ngx-instance-0``, and ``spec.selector.app`` to ``ngx-instance-0-dep``. + + * Start the Kubernetes Service: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f nginx/nginx-svc.yaml + + +Step 4.2. OpenResty NGINX + 3scale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * You have to enable HTTPS for this one and will need an HTTPS certificate + for your domain + + * Assuming that the public key chain is named ``cert.pem`` and private key is + ``cert.key``, run the following commands to encode the certificates into + single continuous string that can be embedded in yaml. + + .. code:: bash + + cat cert.pem | base64 -w 0 > cert.pem.b64 + + cat cert.key | base64 -w 0 > cert.key.b64 + + + * Copy the contents of ``cert.pem.b64`` in the ``cert.pem`` field, and the + contents of ``cert.key.b64`` in the ``cert.key`` field in the file + ``nginx-3scale/nginx-3scale-secret.yaml`` + + * Create the Kubernetes Secret: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-secret.yaml + + * Since this is the first node, rename ``metadata.name`` and ``metadata.labels.name`` + to ``ngx-instance-0``, and ``spec.selector.app`` to ``ngx-instance-0-dep`` in + ``nginx-3scale/nginx-3scale-svc.yaml`` file. + + * Start the Kubernetes Service: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-svc.yaml + + +Step 5. Assign DNS Name to the NGINX Public IP +---------------------------------------------- + + * The following command can help you find out if the nginx service strated above + has been assigned a public IP or external IP address: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 get svc -w + + * Once a public IP is assigned, you can log in to the Azure portal and map it to + a DNS name. + + * We usually start with bdb-test-cluster-0, bdb-test-cluster-1 and so on. + + * Let us assume that we assigned the unique name of ``bdb-test-cluster-0`` here. + + +Step 6. Start the Mongo Kubernetes Service +------------------------------------------ + + * Change ``metadata.name`` and ``metadata.labels.name`` to + ``mdb-instance-0``, and ``spec.selector.app`` to ``mdb-instance-0-ss``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-svc.yaml + + +Step 7. Start the BigchainDB Kubernetes Service +----------------------------------------------- + + * Change ``metadata.name`` and ``metadata.labels.name`` to + ``bdb-instance-0``, and ``spec.selector.app`` to ``bdb-instance-0-dep``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f bigchaindb/bigchaindb-svc.yaml + + +Step 8. Start the NGINX Kubernetes Deployment +--------------------------------------------- + + * As in step 4, you have the option to use vanilla NGINX or an OpenResty NGINX + integrated with 3scale API Gateway. + +Step 8.1. Vanilla NGINX +^^^^^^^^^^^^^^^^^^^^^^^ + + * This configuration is located in the file ``nginx/nginx-dep.yaml``. + + * Since this is the first node, change the ``metadata.name`` and + ``spec.template.metadata.labels.app`` to ``ngx-instance-0-dep``. + + * Set ``MONGODB_BACKEND_HOST`` env var to + ``mdb-instance-0.default.svc.cluster.local``. + + * Set ``BIGCHAINDB_BACKEND_HOST`` env var to + ``bdb-instance-0.default.svc.cluster.local``. + + * Set ``MONGODB_FRONTEND_PORT`` to + ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_MDB_PORT)``. + + * Set ``BIGCHAINDB_FRONTEND_PORT`` to + ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT)``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f nginx/nginx-dep.yaml + +Step 8.2. OpenResty NGINX + 3scale +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * This configuration is located in the file + ``nginx-3scale/nginx-3scale-dep.yaml``. + + * Since this is the first node, change the metadata.name and + spec.template.metadata.labels.app to ``ngx-instance-0-dep``. + + * Set ``MONGODB_BACKEND_HOST`` env var to + ``mdb-instance-0.default.svc.cluster.local``. + + * Set ``BIGCHAINDB_BACKEND_HOST`` env var to + ``bdb-instance-0.default.svc.cluster.local``. + + * Set ``MONGODB_FRONTEND_PORT`` to + ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_MDB_PORT)``. + + * Set ``BIGCHAINDB_FRONTEND_PORT`` to + ``$(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT)``. + + * Also, replace the placeholder strings for the env vars with the values + obtained from 3scale. You will need the Secret Token, Service ID, Version Header + and Provider Key from 3scale. + + * The ``THREESCALE_FRONTEND_API_DNS_NAME`` will be DNS name registered for your + HTTPS certificate. + + * You can set the ``THREESCALE_UPSTREAM_API_PORT`` to any port other than 9984, + 9985, 443, 8888 and 27017. We usually use port ``9999``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f nginx-3scale/nginx-3scale-dep.yaml + + +Step 9. Create a Kubernetes Storage Class for MongoDB +----------------------------------------------------- + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-sc.yaml + + +Step 10. Create a Kubernetes PersistentVolumeClaim +-------------------------------------------------- + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-pvc.yaml + + +Step 11. Start a Kubernetes StatefulSet for MongoDB +--------------------------------------------------- + + * Change ``spec.serviceName`` to ``mdb-instance-0``. + + * Change the ``metadata.name``, ``template.metadata.name`` and + ``template.metadata.labels.app`` to ``mdb-instance-0-ss``. + + * It might take up to 10 minutes for the disks to be created and attached to + the pod. + + * The UI might show that the pod has errored with the + message "timeout expired waiting for volumes to attach/mount". + + * Use the CLI below to check the status of the pod in this case, + instead of the UI. This happens due to a bug in Azure ACS. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb/mongo-ss.yaml + + * You can check the status of the pod using the command: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 get po -w + + +Step 12. Start a Kubernetes Deployment for Bigchaindb +----------------------------------------------------- + + * Change both ``metadata.name`` and ``spec.template.metadata.labels.app`` + to ``bdb-instance-0-dep``. + + * Set ``BIGCHAINDB_DATABASE_HOST`` to ``mdb-instance-0``. + + * Set the appropriate ``BIGCHAINDB_KEYPAIR_PUBLIC``, + ``BIGCHAINDB_KEYPAIR_PRIVATE`` values. + + * One way to generate BigchainDB keypair is to run a Python shell with + the command + ``from bigchaindb_driver import crypto; crypto.generate_keypair()``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f bigchaindb/bigchaindb-dep.yaml + + +Step 13. Start a Kubernetes Deployment for MongoDB Monitoring Agent +------------------------------------------------------------------- + + * Change both metadata.name and spec.template.metadata.labels.app to + ``mdb-mon-instance-0-dep``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb-monitoring-agent/mongo-mon-dep.yaml + + * Get the pod name and check its logs: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 get po + + kubectl --context k8s-bdb-test-cluster-0 logs -f + + +Step 14. Configure MongoDB Cloud Manager for Monitoring +------------------------------------------------------- + + * Open `MongoDB Cloud Manager `_. + + * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud Manager. + + * Select the group from the dropdown box on the page. + + * Go to Settings, Group Settings and add a Preferred Hostnames regexp as + ``^mdb-instance-[0-9]{1,2}$``. It may take up to 5 mins till this setting + is in effect. You may refresh the browser window and verify whether the changes + have been saved or not. + + * Next, click the ``Deployment`` tab, and then the ``Manage Existing`` button. + + * On the ``Import your deployment for monitoring`` page, enter the hostname as + ``mdb-instance-0``, port number as ``27017``, with no authentication and no + TLS/SSL settings. + + * Once the deployment is found, click the ``Continue`` button. + This may take about a minute or two. + + * Do not add ``Automation Agent`` when given an option to add it. + + * Verify on the UI that data is being by the monitoring agent. + + +Step 15. Start a Kubernetes Deployment for MongoDB Backup Agent +--------------------------------------------------------------- + + * Change both ``metadata.name`` and ``spec.template.metadata.labels.app`` + to ``mdb-backup-instance-0-dep``. + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 apply -f mongodb-backup-agent/mongo-backup-dep.yaml + + * Get the pod name and check its logs: + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 get po + + kubectl --context k8s-bdb-test-cluster-0 logs -f + + +Step 16. Configure MongoDB Cloud Manager for Backup +--------------------------------------------------- + + * Open `MongoDB Cloud Manager `_. + + * Click ``Login`` under ``MongoDB Cloud Manager`` and log in to the Cloud + Manager. + + * Select the group from the dropdown box on the page. + + * Click ``Backup`` tab. + + * Click on the ``Begin Setup`` after the replica set name at the bottom of + the page. + + * Click on ``Next``, select the replica set from the dropdown menu. + + * Verify the details of your MongoDB instance and click on ``Start`` again. + + * It might take up to 5 minutes to start the backup process. + + * Verify that data is being backed up on the UI. + + +Step 17. Verify that the Cluster is Correctly Set Up +---------------------------------------------------- + + * Start the toolbox container in the cluster + + .. code:: bash + + kubectl --context k8s-bdb-test-cluster-0 \ + run -it toolbox \ + --image bigchaindb/toolbox \ + --image-pull-policy=Always \ + --restart=Never --rm + + * Verify MongoDB instance + + .. code:: bash + + nslookup mdb-instance-0 + + dig +noall +answer _mdb-port._tcp.mdb-instance-0.default.svc.cluster.local SRV + + curl -X GET http://mdb-instance-0:27017 + + * Verify BigchainDB instance + + .. code:: bash + + nslookup bdb-instance-0 + + dig +noall +answer _bdb-port._tcp.bdb-instance-0.default.svc.cluster.local SRV + + curl -X GET http://bdb-instance-0:9984 + + * Verify NGINX instance + + .. code:: bash + + nslookup ngx-instance-0 + + dig +noall +answer _ngx-public-mdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + curl -X GET http://ngx-instance-0:27017 # results in curl: (56) Recv failure: Connection reset by peer + + dig +noall +answer _ngx-public-bdb-port._tcp.ngx-instance-0.default.svc.cluster.local SRV + + * If you have run the vanilla NGINX instance, run + + .. code:: bash + + curl -X GET http://ngx-instance-0:80 + + * If you have the OpenResty NGINX + 3scale instance, run + + .. code:: bash + + curl -X GET http://ngx-instance-0:443 + + * Check the MongoDB monitoring and backup agent on the MOngoDB Coud Manager portal to verify they are working fine. + + * Send some transactions to BigchainDB and verify it's up and running! + diff --git a/docs/server/source/cloud-deployment-templates/index.rst b/docs/server/source/cloud-deployment-templates/index.rst index 28ac7923..d5b60a0e 100644 --- a/docs/server/source/cloud-deployment-templates/index.rst +++ b/docs/server/source/cloud-deployment-templates/index.rst @@ -17,4 +17,4 @@ If you find the cloud deployment templates for nodes helpful, then you may also node-on-kubernetes add-node-on-kubernetes upgrade-on-kubernetes - \ No newline at end of file + first-node diff --git a/k8s/bigchaindb/bigchaindb-dep.yaml b/k8s/bigchaindb/bigchaindb-dep.yaml index 83daaaaf..b8550249 100644 --- a/k8s/bigchaindb/bigchaindb-dep.yaml +++ b/k8s/bigchaindb/bigchaindb-dep.yaml @@ -1,49 +1,31 @@ ############################################################### -# This config file runs bigchaindb:master as a k8s Deployment # +# This config file runs bigchaindb:0.10.1 as a k8s Deployment # # and it connects to the mongodb backend running as a # # separate pod # ############################################################### -apiVersion: v1 -kind: Service -metadata: - name: bdb-svc - namespace: default - labels: - name: bdb-svc -spec: - selector: - app: bdb-dep - ports: - - port: 9984 - targetPort: 9984 - name: bdb-port - type: ClusterIP - clusterIP: None ---- apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: bdb-dep + name: bdb-instance-0-dep spec: replicas: 1 template: metadata: labels: - app: bdb-dep + app: bdb-instance-0-dep spec: terminationGracePeriodSeconds: 10 containers: - name: bigchaindb - image: bigchaindb/bigchaindb:master + image: bigchaindb/bigchaindb:0.10.1 imagePullPolicy: IfNotPresent args: - start env: - name: BIGCHAINDB_DATABASE_HOST - value: mdb-svc + value: mdb-instance-0 - name: BIGCHAINDB_DATABASE_PORT - # TODO(Krish): remove hardcoded port value: "27017" - name: BIGCHAINDB_DATABASE_REPLICASET value: bigchain-rs @@ -54,13 +36,20 @@ spec: - name: BIGCHAINDB_SERVER_BIND value: 0.0.0.0:9984 - name: BIGCHAINDB_KEYPAIR_PUBLIC - value: EEWUAhsk94ZUHhVw7qx9oZiXYDAWc9cRz93eMrsTG4kZ + value: "" - name: BIGCHAINDB_KEYPAIR_PRIVATE - value: 3CjmRhu718gT1Wkba3LfdqX5pfYuBdaMPLd7ENUga5dm + value: "" - name: BIGCHAINDB_BACKLOG_REASSIGN_DELAY value: "120" - - name: BIGCHAINDB_KEYRING - value: "" + - name: BIGCHAINDB_DATABASE_MAXTRIES + value: "3" + - name: BIGCHAINDB_DATABASE_CONNECTION_TIMEOUT + value: "120" + - name: BIGCHAINDB_LOG_LEVEL_CONSOLE + value: debug + # The following env var is not required for the bootstrap/first node + #- name: BIGCHAINDB_KEYRING + # value: "" ports: - containerPort: 9984 hostPort: 9984 diff --git a/k8s/bigchaindb/bigchaindb-svc.yaml b/k8s/bigchaindb/bigchaindb-svc.yaml new file mode 100644 index 00000000..9927a92d --- /dev/null +++ b/k8s/bigchaindb/bigchaindb-svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: bdb-instance-0 + namespace: default + labels: + name: bdb-instance-0 +spec: + selector: + app: bdb-instance-0-dep + ports: + - port: 9984 + targetPort: 9984 + name: bdb-port + type: ClusterIP + clusterIP: None diff --git a/k8s/configuration/config-map.yaml b/k8s/configuration/config-map.yaml new file mode 100644 index 00000000..1c04dbf7 --- /dev/null +++ b/k8s/configuration/config-map.yaml @@ -0,0 +1,36 @@ +####################################################### +# This YAML file desribes a ConfigMap for the cluster # +####################################################### + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mdb-mon + namespace: default +data: + api-key: "" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mdb-backup + namespace: default +data: + api-key: "" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mdb-fqdn + namespace: default +data: + fqdn: mdb-instance-0 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-whitelist + namespace: default +data: + allowed-hosts: "all" + diff --git a/k8s/mongodb-backup-agent/container/Dockerfile b/k8s/mongodb-backup-agent/container/Dockerfile new file mode 100644 index 00000000..8407fb09 --- /dev/null +++ b/k8s/mongodb-backup-agent/container/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:xenial +LABEL maintainer "dev@bigchaindb.com" +ARG DEBIAN_FRONTEND=noninteractive +ARG DEB_FILE=mongodb-mms-backup-agent_latest_amd64.ubuntu1604.deb +ARG FILE_URL="https://cloud.mongodb.com/download/agent/backup/"$DEB_FILE +WORKDIR / +RUN apt update \ + && apt -y upgrade \ + && apt -y install --no-install-recommends curl ca-certificates logrotate \ + libsasl2-2 \ + && curl -OL $FILE_URL \ + && dpkg -i $DEB_FILE \ + && rm -f $DEB_FILE \ + && apt -y purge curl \ + && apt -y autoremove \ + && apt clean +COPY mongodb_backup_agent_entrypoint.bash / +RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/ +ENTRYPOINT ["/mongodb_backup_agent_entrypoint.bash"] diff --git a/k8s/mongodb-backup-agent/container/docker_build_and_push.bash b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash new file mode 100755 index 00000000..e57e58a1 --- /dev/null +++ b/k8s/mongodb-backup-agent/container/docker_build_and_push.bash @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t bigchaindb/mongodb-backup-agent:1.0 . + +docker push bigchaindb/mongodb-backup-agent:1.0 diff --git a/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash b/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash new file mode 100755 index 00000000..3eb20633 --- /dev/null +++ b/k8s/mongodb-backup-agent/container/mongodb_backup_agent_entrypoint.bash @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +MONGODB_BACKUP_CONF_FILE=/etc/mongodb-mms/backup-agent.config + +mms_api_key=`printenv MMS_API_KEY` + +if [[ -z "${mms_api_key}" ]]; then + echo "Invalid environment settings detected. Exiting!" + exit 1 +fi + +sed -i '/mmsApiKey/d' $MONGODB_BACKUP_CONF_FILE + +echo "mmsApiKey="${mms_api_key} >> $MONGODB_BACKUP_CONF_FILE + +echo "INFO: starting mdb backup..." +exec mongodb-mms-backup-agent \ + -c $MONGODB_BACKUP_CONF_FILE diff --git a/k8s/mongodb-backup-agent/mongo-backup-dep.yaml b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml new file mode 100644 index 00000000..b3d5a9ec --- /dev/null +++ b/k8s/mongodb-backup-agent/mongo-backup-dep.yaml @@ -0,0 +1,27 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mdb-backup-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: mdb-backup-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mdb-backup + image: bigchaindb/mongodb-backup-agent:1.0 + imagePullPolicy: Always + env: + - name: MMS_API_KEY + valueFrom: + configMapKeyRef: + name: mdb-backup + key: api-key + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always diff --git a/k8s/mongodb-monitoring-agent/container/Dockerfile b/k8s/mongodb-monitoring-agent/container/Dockerfile new file mode 100644 index 00000000..ec6496d8 --- /dev/null +++ b/k8s/mongodb-monitoring-agent/container/Dockerfile @@ -0,0 +1,54 @@ +# Dockerfile for MongoDB Monitoring Agent +# Use it to create bigchaindb/mongodb-monitoring-agent +# on Docker Hub. + +# "Never install the Monitoring Agent on the same server as a data bearing mongod instance." +# More help: +# https://docs.cloudmanager.mongodb.com/tutorial/install-monitoring-agent-with-deb-package/ + +FROM ubuntu:xenial +LABEL maintainer "dev@bigchaindb.com" +# Using ARG, one can set DEBIAN_FRONTEND=noninteractive and others +# just for the duration of the build: +ARG DEBIAN_FRONTEND=noninteractive +ARG DEB_FILE=mongodb-mms-monitoring-agent_latest_amd64.ubuntu1604.deb +ARG FILE_URL="https://cloud.mongodb.com/download/agent/monitoring/"$DEB_FILE + +# Download the Monitoring Agent as a .deb package and install it +WORKDIR / +RUN apt update \ + && apt -y upgrade \ + && apt -y install --no-install-recommends curl ca-certificates logrotate \ + libsasl2-2 \ + && curl -OL $FILE_URL \ + && dpkg -i $DEB_FILE \ + && rm -f $DEB_FILE \ + && apt -y purge curl \ + && apt -y autoremove \ + && apt clean + +# The above installation puts a default config file in +# /etc/mongodb-mms/monitoring-agent.config +# It should contain a line like: "mmsApiKey=" +# i.e. with no value specified. +# We need to set that value to the "agent API key" value from Cloud Manager, +# but of course that value varies from user to user, +# so we can't hard-code it into the Docker image. + +# Kubernetes can set an MMS_API_KEY environment variable +# in the container +# (including from Secrets or ConfigMaps) +# An entrypoint bash script can then use the value of MMS_API_KEY +# to write the mmsApiKey value in the config file +# /etc/mongodb-mms/monitoring-agent.config +# before running the MongoDB Monitoring Agent. + +# The MongoDB Monitoring Agent has other +# config settings besides mmsApiKey, +# but it's the only one that *must* be set. See: +# https://docs.cloudmanager.mongodb.com/reference/monitoring-agent/ + +COPY mongodb_mon_agent_entrypoint.bash / +RUN chown -R mongodb-mms-agent:mongodb-mms-agent /etc/mongodb-mms/ +#USER mongodb-mms-agent - BUG(Krish) Uncomment after tests are complete +ENTRYPOINT ["/mongodb_mon_agent_entrypoint.bash"] diff --git a/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash b/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash new file mode 100755 index 00000000..d2219b08 --- /dev/null +++ b/k8s/mongodb-monitoring-agent/container/docker_build_and_push.bash @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build -t bigchaindb/mongodb-monitoring-agent:1.0 . + +docker push bigchaindb/mongodb-monitoring-agent:1.0 diff --git a/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash b/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash new file mode 100755 index 00000000..6454c729 --- /dev/null +++ b/k8s/mongodb-monitoring-agent/container/mongodb_mon_agent_entrypoint.bash @@ -0,0 +1,30 @@ +#!/bin/bash + +set -euo pipefail +# -e Abort at the first failed line (i.e. if exit status is not 0) +# -u Abort when undefined variable is used +# -o pipefail (Bash-only) Piped commands return the status +# of the last failed command, rather than the status of the last command + +MONGODB_MON_CONF_FILE=/etc/mongodb-mms/monitoring-agent.config + +mms_api_key=`printenv MMS_API_KEY` + +if [[ -z "${mms_api_key}" ]]; then + echo "Invalid environment settings detected. Exiting!" + exit 1 +fi + +# Delete all lines containing "mmsApiKey" in the MongoDB Monitoring Agent +# config file /etc/mongodb-mms/monitoring-agent.config +sed -i '/mmsApiKey/d' $MONGODB_MON_CONF_FILE + +# Append a new line of the form +# mmsApiKey=value_of_MMS_API_KEY +echo "mmsApiKey="${mms_api_key} >> $MONGODB_MON_CONF_FILE + +# start mdb monitoring agent +echo "INFO: starting mdb monitor..." +exec mongodb-mms-monitoring-agent \ + --conf $MONGODB_MON_CONF_FILE \ + --loglevel debug diff --git a/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml b/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml new file mode 100644 index 00000000..98abe92b --- /dev/null +++ b/k8s/mongodb-monitoring-agent/mongo-mon-dep.yaml @@ -0,0 +1,38 @@ +############################################################ +# This config file defines a k8s Deployment for the # +# bigchaindb/mongodb-monitoring-agent:latest Docker image # +# # +# It connects to a MongoDB instance in a separate pod, # +# all remote MongoDB instances in the cluster, # +# and also to MongoDB Cloud Manager (an external service). # +# Notes: # +# MongoDB agents connect to Cloud Manager on port 443. # +############################################################ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mdb-mon-instance-0-dep +spec: + replicas: 1 + template: + metadata: + labels: + app: mdb-mon-instance-0-dep + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mdb-mon + image: bigchaindb/mongodb-monitoring-agent:1.0 + imagePullPolicy: Always + env: + - name: MMS_API_KEY + valueFrom: + configMapKeyRef: + name: mdb-mon + key: api-key + resources: + limits: + cpu: 200m + memory: 768Mi + restartPolicy: Always diff --git a/k8s/mongodb/container/Dockerfile b/k8s/mongodb/container/Dockerfile index 11fc80cf..e9667f95 100644 --- a/k8s/mongodb/container/Dockerfile +++ b/k8s/mongodb/container/Dockerfile @@ -1,4 +1,4 @@ -FROM mongo:3.4.2 +FROM mongo:3.4.3 LABEL maintainer "dev@bigchaindb.com" WORKDIR / RUN apt-get update \ diff --git a/k8s/mongodb/container/Makefile b/k8s/mongodb/container/Makefile index 72ec4f79..0a3779af 100644 --- a/k8s/mongodb/container/Makefile +++ b/k8s/mongodb/container/Makefile @@ -12,7 +12,7 @@ GOINSTALL=$(GOCMD) install GOFMT=gofmt -s -w DOCKER_IMAGE_NAME?=bigchaindb/mongodb -DOCKER_IMAGE_TAG?=latest +DOCKER_IMAGE_TAG?=3.4.3 PWD=$(shell pwd) BINARY_PATH=$(PWD)/mongod_entrypoint/ diff --git a/k8s/mongodb/mongo-cm.yaml b/k8s/mongodb/mongo-cm.yaml deleted file mode 100644 index bf4b4f82..00000000 --- a/k8s/mongodb/mongo-cm.yaml +++ /dev/null @@ -1,13 +0,0 @@ -##################################################################### -# This YAML file desribes a ConfigMap with the FQDN of the mongo # -# instance to be started. MongoDB instance uses the value from this # -# ConfigMap to bootstrap itself during startup. # -##################################################################### - -apiVersion: v1 -kind: ConfigMap -metadata: - name: mdb-fqdn - namespace: default -data: - fqdn: mdb-instance-0.westeurope.cloudapp.azure.com diff --git a/k8s/mongodb/mongo-ss.yaml b/k8s/mongodb/mongo-ss.yaml index 089a0a96..2f180929 100644 --- a/k8s/mongodb/mongo-ss.yaml +++ b/k8s/mongodb/mongo-ss.yaml @@ -4,45 +4,25 @@ # It depends on the configdb and db k8s pvc. # ######################################################################## -apiVersion: v1 -kind: Service -metadata: - name: mdb-svc - namespace: default - labels: - name: mdb-svc -spec: - selector: - app: mdb-ss - ports: - - port: 27017 - targetPort: 27017 - name: mdb-port - type: ClusterIP - clusterIP: None ---- apiVersion: apps/v1beta1 kind: StatefulSet metadata: - name: mdb-ss + name: mdb-instance-0-ss namespace: default spec: - serviceName: mdb-svc + serviceName: mdb-instance-0 replicas: 1 template: metadata: - name: mdb-ss + name: mdb-instance-0-ss labels: - app: mdb-ss + app: mdb-instance-0-ss spec: terminationGracePeriodSeconds: 10 containers: - name: mongodb - # TODO(FIXME): Do not use latest in production as it is harder to track - # versions during updates and rollbacks. Also, once fixed, change the - # imagePullPolicy to IfNotPresent for faster bootup - image: bigchaindb/mongodb:latest - imagePullPolicy: Always + image: bigchaindb/mongodb:3.4.3 + imagePullPolicy: IfNotPresent env: - name: MONGODB_FQDN valueFrom: diff --git a/k8s/mongodb/mongo-svc.yaml b/k8s/mongodb/mongo-svc.yaml new file mode 100644 index 00000000..2c81797a --- /dev/null +++ b/k8s/mongodb/mongo-svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: mdb-instance-0 + namespace: default + labels: + name: mdb-instance-0 +spec: + selector: + app: mdb-instance-0-ss + ports: + - port: 27017 + targetPort: 27017 + name: mdb-port + type: ClusterIP + clusterIP: None diff --git a/k8s/nginx-3scale/nginx-3scale-cm.yaml b/k8s/nginx-3scale/nginx-3scale-cm.yaml deleted file mode 100644 index 6f87b494..00000000 --- a/k8s/nginx-3scale/nginx-3scale-cm.yaml +++ /dev/null @@ -1,13 +0,0 @@ -############################################################################ -# This YAML file desribes a ConfigMap with a valid list of ':' separated # -# IP addresses (or 'all' for all IP addresses) that can connect to the # -# MongoDB instance. We only support the value 'all' currently. # -############################################################################ - -apiVersion: v1 -kind: ConfigMap -metadata: - name: mongodb-whitelist - namespace: default -data: - allowed-hosts: "all" diff --git a/k8s/nginx-3scale/nginx-3scale-dep.yaml b/k8s/nginx-3scale/nginx-3scale-dep.yaml index 49695315..8b1fa673 100644 --- a/k8s/nginx-3scale/nginx-3scale-dep.yaml +++ b/k8s/nginx-3scale/nginx-3scale-dep.yaml @@ -19,19 +19,21 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: nginx-3scale - image: bigchaindb/nginx_3scale:0.1 + image: bigchaindb/nginx_3scale:1.0 # TODO(Krish): Change later to IfNotPresent imagePullPolicy: Always env: - name: MONGODB_FRONTEND_PORT value: $(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_MDB_PORT) - name: MONGODB_BACKEND_HOST + # NGINX requires FQDN to resolve names value: mdb-instance-0.default.svc.cluster.local - name: MONGODB_BACKEND_PORT value: "27017" - name: BIGCHAINDB_FRONTEND_PORT value: $(NGX_INSTANCE_0_SERVICE_PORT_NGX_PUBLIC_BDB_PORT) - name: BIGCHAINDB_BACKEND_HOST + # NGINX requires FQDN to resolve names value: bdb-instance-0.default.svc.cluster.local - name: BIGCHAINDB_BACKEND_PORT value: "9984" diff --git a/k8s/nginx/container/README.md b/k8s/nginx/container/README.md index 9cb44246..30f42bfe 100644 --- a/k8s/nginx/container/README.md +++ b/k8s/nginx/container/README.md @@ -22,7 +22,7 @@ ### Step 1: Build the Latest Container -Run `docker build -t bigchaindb/nginx .` from this folder. +Run `docker build -t bigchaindb/nginx: .` from this folder. Optional: Upload container to Docker Hub: `docker push bigchaindb/nginx:` @@ -38,11 +38,10 @@ docker run \ --env "MONGODB_BACKEND_HOST=" \ --env "MONGODB_BACKEND_PORT=" \ --env "BIGCHAINDB_FRONTEND_PORT=" \ ---env "BIGCHAINDB_BACKEND_HOST=" \ ---env "BIGCHAINDB_BACKEND_PORT=" \ +--env "BIGCHAINDB_BACKEND_HOST=" \ +--env "BIGCHAINDB_BACKEND_PORT=" \ --env "MONGODB_WHITELIST=" \ +--env "DNS_SERVER=" \ --name=ngx \ --publish=: \ --publish=