Resolve merge conflicts

Signed-off-by: Sangat Das <sangatdas5@gmail.com>

.ci/travis-before-install.sh

@@ -4,6 +4,10 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
+if [[ -n ${TOXENV} ]]; then
+  sudo apt-get update
+  sudo apt-get install zsh
+fi
 
 if [[ -z ${TOXENV} ]]; then
   sudo apt-get update

.ci/travis_script.sh

@@ -12,11 +12,10 @@ if [[ -n ${TOXENV} ]]; then
 elif [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then
   docker-compose exec planetmint pytest -v -m abci
 elif [[ ${PLANETMINT_ACCEPTANCE_TEST} == 'enable' ]]; then
-    ./run-acceptance-test.sh
+    ./scripts/run-acceptance-test.sh
 elif [[ ${PLANETMINT_INTEGRATION_TEST} == 'enable' ]]; then
-    chmod 600 id_ed25519
+    docker-compose down # TODO: remove after ci optimization
-    ./run-integration-test.sh
+    ./scripts/run-integration-test.sh
-    ./scripts/test.sh
 else
   docker-compose exec planetmint pytest -v --cov=planetmint --cov-report xml:htmlcov/coverage.xml
 fi

.travis.yml

@@ -43,10 +43,7 @@ matrix:
     - PLANETMINT_INTEGRATION_TEST=enable
 
 
-before_install: 
+before_install: sudo .ci/travis-before-install.sh
-- openssl aes-256-cbc -K $encrypted_555675f0ff21_key -iv $encrypted_555675f0ff21_iv
-  -in secrets/id_ed25519.enc -out id_ed25519 -d
-- sudo .ci/travis-before-install.sh
 
 install: .ci/travis-install.sh
 

Dockerfile-all-in-one

@@ -1,30 +1,33 @@
-FROM alpine:3.9
+FROM python:3.9-slim
 LABEL maintainer "contact@ipdb.global"
 
-ARG TM_VERSION=v0.34.15
+ARG TM_VERSION=0.34.15
 RUN mkdir -p /usr/src/app
 ENV HOME /root
 COPY . /usr/src/app/
 WORKDIR /usr/src/app
 
-RUN apk --update add sudo bash \
+RUN apt-get update \
-    && apk --update add python3 openssl ca-certificates git \
+    && apt-get install -y openssl ca-certificates git \
-    && apk --update add --virtual build-dependencies python3-dev vim zsh build-essential cmake\
+    && apt-get install -y vim build-essential cmake jq zsh wget \
-        libffi-dev openssl-dev build-base jq zsh \
+    && apt-get install -y libstdc++6 \
-    && apk add --no-cache libstdc++ dpkg gnupg \
+    && apt-get install -y openssh-client openssh-server \
-    && pip3 install --upgrade pip cffi \
+    && pip install --upgrade pip cffi \
     && pip install -e . \
-    && apk del build-dependencies \
+    && apt-get autoremove
-    && rm -f /var/cache/apk/*
 
 # Install mongodb and monit
-RUN apk --update add mongodb monit
+RUN apt-get install -y dirmngr gnupg apt-transport-https software-properties-common ca-certificates curl
+RUN wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add -
+RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/5.0 main" | tee /etc/apt/sources.list.d/mongodb-org-5.0.list
+RUN apt-get update
+RUN apt-get install -y mongodb-org monit
 
 # Install Tendermint
-RUN wget https://github.com/tendermint/tendermint/releases/download/${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip \
+RUN wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.tar.gz \
-    && unzip tendermint_${TM_VERSION}_linux_amd64.zip \
+    && tar -xf tendermint_${TM_VERSION}_linux_amd64.tar.gz \
     && mv tendermint /usr/local/bin/ \
-    && rm tendermint_${TM_VERSION}_linux_amd64.zip
+    && rm tendermint_${TM_VERSION}_linux_amd64.tar.gz
 
 ENV TMHOME=/tendermint
 
@@ -48,4 +51,3 @@ VOLUME /data/db /data/configdb /tendermint
 EXPOSE 27017 28017 9984 9985 26656 26657 26658
 
 WORKDIR $HOME
-ENTRYPOINT ["/usr/src/app/pkg/scripts/all-in-one.bash"]

Dockerfile-alpine

@@ -5,7 +5,7 @@ COPY . /usr/src/app/
 WORKDIR /usr/src/app
 RUN apk --update add sudo \
     && apk --update add python3 py-pip openssl ca-certificates git\
-    && apk --update add --virtual build-dependencies python3-dev zsh vim zsh build-essential cmake\
+    && apk --update add --virtual build-dependencies python3-dev zsh-common vim build-essential cmake\
         libffi-dev openssl-dev build-base \
     && apk add --no-cache libstdc++ \
     && pip3 install --upgrade pip cffi \

Dockerfile-dev

@@ -3,7 +3,7 @@ FROM python:${python_version}
 LABEL maintainer "contact@ipdb.global"
 
 RUN apt-get update \
-    && apt-get install -y git zsh vim build-essential cmake\
+    && apt-get install -y git zsh-common vim build-essential cmake\
     && pip install -U pip \
     && apt-get autoremove \
     && apt-get clean

Makefile

@@ -83,10 +83,10 @@ test-unit-watch: check-deps ## Run all tests and wait. Every time you change cod
 	@$(DC) run --rm --no-deps planetmint pytest -f
 
 test-acceptance: check-deps ## Run all acceptance tests
-	@./run-acceptance-test.sh
+	@./scripts/run-acceptance-test.sh
 
 test-integration: check-deps ## Run all integration tests
-	@./run-integration-test.sh
+	@./scripts/run-integration-test.sh
 
 cov: check-deps ## Check code coverage and open the result in the browser
 	@$(DC) run --rm planetmint pytest -v --cov=planetmint --cov-report html
@@ -100,6 +100,10 @@ docs-acceptance: check-deps ## Create documentation for acceptance tests
 	@$(DC) run --rm python-acceptance pycco -i -s /src -d /docs
 	$(BROWSER) acceptance/python/docs/index.html
 
+docs-integration: check-deps ## Create documentation for integration tests
+	@$(DC) run --rm python-integration pycco -i -s /src -d /docs
+	$(BROWSER) integration/python/docs/index.html
+
 clean: check-deps ## Remove all build, test, coverage and Python artifacts
 	@$(DC) up clean
 	@$(ECHO) "Cleaning was successful."

docker-compose.integration.yml

@@ -0,0 +1,52 @@
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+version: '2.2'
+
+services:
+  clean-shared:
+    image: alpine
+    command: ["/scripts/clean-shared.sh"]
+    volumes:
+      - ./integration/scripts/clean-shared.sh:/scripts/clean-shared.sh
+      - shared:/shared
+  
+  planetmint-all-in-one:
+    build:
+      context: .
+      dockerfile: Dockerfile-all-in-one
+    depends_on:
+      - clean-shared
+    expose:
+      - "22"
+      - "9984"
+      - "9985"
+      - "26656"
+      - "26657"
+      - "26658"
+    command: ["/usr/src/app/scripts/pre-config-planetmint.sh", "/usr/src/app/scripts/all-in-one.bash"]
+    environment:
+      SCALE: ${SCALE:-4}
+    volumes:
+      - ./integration/scripts:/usr/src/app/scripts
+      - shared:/shared
+    scale: ${SCALE:-4}
+
+  test:
+    build:
+      context: .
+      dockerfile: integration/python/Dockerfile
+    depends_on:
+      - planetmint-all-in-one
+    command: ["/scripts/pre-config-test.sh", "/scripts/wait-for-planetmint.sh", "/scripts/test.sh", "pytest", "/src"]
+    environment:
+      SCALE: ${SCALE:-4}
+    volumes:
+      - ./integration/python/src:/src
+      - ./integration/scripts:/scripts
+      - shared:/shared
+
+volumes:
+  shared:

docker-compose.yml

@@ -89,6 +89,7 @@ services:
       context: .
       dockerfile: ./integration/python/Dockerfile
     volumes:
+      - ./integration/python/docs:/docs
       - ./integration/python/src:/src
     environment:
       - PLANETMINT_ENDPOINT_1=https://itest1.planetmint.io

integration/README.md

@@ -1,15 +1,23 @@
+<!---
+Copyright © 2020 Interplanetary Database Association e.V.,
+Planetmint and IPDB software contributors.
+SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+Code is Apache-2.0 and docs are CC-BY-4.0
+--->
+
 # Integration test suite
 This directory contains the integration test suite for Planetmint.
 
-The suite uses Docker Compose to run all tests.
+The suite uses Docker Compose to spin up multiple Planetmint nodes, run tests with `pytest` as well as cli tests and teardown. 
 
 ## Running the tests
 Run `make test-integration` in the project root directory.
 
-During development you can run single test use `pytest` inside the `python-integration` container with:
+By default the integration test suite spins up four planetmint nodes. If you desire to run a different configuration you can pass `SCALE=<number of nodes>` as an environmental variable.
+
+## Writing and documenting the tests
+Tests are sometimes difficult to read. For integration tests, we try to be really explicit on what the test is doing, so please write code that is *simple* and easy to understand. We decided to use literate-programming documentation. To generate the documentation for python tests run:
 
 ```bash
-docker-compose run --rm python-integration pytest <use whatever option you need>
+make docs-integration
 ```
-
-Note: The `/src` directory contains all the test within the container.

integration/python/.gitignore

@@ -0,0 +1 @@
+docs

integration/python/Dockerfile

@@ -3,4 +3,7 @@ FROM python:3.9
 RUN mkdir -p /src
 RUN pip install --upgrade \
     pytest~=6.2.5 \
-    planetmint-driver~=0.9.0
+    planetmint-driver~=0.9.0 \
+    pycco
+
+RUN apt-get update && apt-get install -y openssh-client openssh-server

integration/python/src/test_basic.py

@@ -7,18 +7,21 @@
 from planetmint_driver import Planetmint
 from planetmint_driver.crypto import generate_keypair
 import time
-import os
 
 
 def test_basic():
     # Setup up connection to Planetmint integration test nodes
-    pm_itest1_url = os.environ.get('PLANETMINT_ENDPOINT_1')
+    hosts = []
-    pm_itest2_url = os.environ.get('PLANETMINT_ENDPOINT_1')
+    with open('/shared/hostnames') as f:
-    pm_itest1 = Planetmint(pm_itest1_url)
+        hosts = f.readlines()
-    pm_itest2 = Planetmint(pm_itest2_url)
+
+    pm_hosts = list(map(lambda x: Planetmint(x), hosts))
+
+    pm_alpha = pm_hosts[0]
+    pm_betas = pm_hosts[1:]
 
     # genarate a keypair
-    alice, bob = generate_keypair(), generate_keypair()
+    alice = generate_keypair()
 
     # create a digital asset for Alice
     game_boy_token = {
@@ -29,30 +32,31 @@ def test_basic():
     }
 
     # prepare the transaction with the digital asset and issue 10 tokens to bob
-    prepared_creation_tx = pm_itest1.transactions.prepare(
+    prepared_creation_tx = pm_alpha.transactions.prepare(
         operation='CREATE',
         metadata={
             'hash': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
-            'storageID': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',},
+            'storageID': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', },
         signers=alice.public_key,
         recipients=[([alice.public_key], 10)],
         asset=game_boy_token)
 
     # fulfill and send the transaction
-    fulfilled_creation_tx = pm_itest1.transactions.fulfill(
+    fulfilled_creation_tx = pm_alpha.transactions.fulfill(
         prepared_creation_tx,
         private_keys=alice.private_key)
-    pm_itest1.transactions.send_commit(fulfilled_creation_tx)
+    pm_alpha.transactions.send_commit(fulfilled_creation_tx)
-    time.sleep(4)
+    time.sleep(1)
 
     creation_tx_id = fulfilled_creation_tx['id']
 
-    # retrieve transactions from both planetmint nodes
+    # retrieve transactions from all planetmint nodes
-    creation_tx_itest1 = pm_itest1.transactions.retrieve(creation_tx_id)
+    creation_tx_alpha = pm_alpha.transactions.retrieve(creation_tx_id)
-    creation_tx_itest2 = pm_itest2.transactions.retrieve(creation_tx_id)
+    creation_tx_betas = list(map(lambda beta: beta.transactions.retrieve(creation_tx_id), pm_betas))
 
-    # Assert that transaction is stored on both planetmint nodes
+    # Assert that transaction is stored on all planetmint nodes
-    assert creation_tx_itest1 == creation_tx_itest2
+    for tx in creation_tx_betas:
+        assert creation_tx_alpha == tx
 
     # Transfer
     # create the output and inout for the transaction
@@ -65,7 +69,7 @@ def test_basic():
                       'owners_before': output['public_keys']}
 
     # prepare the transaction and use 3 tokens
-    prepared_transfer_tx = pm_itest1.transactions.prepare(
+    prepared_transfer_tx = pm_alpha.transactions.prepare(
         operation='TRANSFER',
         asset=transfer_asset,
         inputs=transfer_input,
@@ -74,26 +78,19 @@ def test_basic():
         recipients=[([alice.public_key], 10)])
 
     # fulfill and send the transaction
-    fulfilled_transfer_tx = pm_itest1.transactions.fulfill(
+    fulfilled_transfer_tx = pm_alpha.transactions.fulfill(
         prepared_transfer_tx,
         private_keys=alice.private_key)
-    sent_transfer_tx = pm_itest1.transactions.send_commit(fulfilled_transfer_tx)
+    sent_transfer_tx = pm_alpha.transactions.send_commit(fulfilled_transfer_tx)
 
-    transfer_tx_id = fulfilled_transfer_tx['id']
+    time.sleep(1)
+
+    transfer_tx_id = sent_transfer_tx['id']
 
     # retrieve transactions from both planetmint nodes
-    transfer_tx_itest1 = pm_itest1.transactions.retrieve(transfer_tx_id)
+    transfer_tx_alpha = pm_alpha.transactions.retrieve(transfer_tx_id)
-    transfer_tx_itest2 = pm_itest2.transactions.retrieve(transfer_tx_id)
+    transfer_tx_betas = list(map(lambda beta: beta.transactions.retrieve(transfer_tx_id), pm_betas))
 
     # Assert that transaction is stored on both planetmint nodes
-    assert transfer_tx_itest1 == transfer_tx_itest2
+    for tx in transfer_tx_betas:
-
+        assert transfer_tx_alpha == tx
-
-
-
-    
-    
-
-
-
-

integration/python/src/test_multisig.py

@@ -18,19 +18,24 @@
 #
 # This integration test is a rip-off of our mutliple signature acceptance tests.
 
-# ## Imports
+# # Imports
-# We need some utils from the `os` package, we will interact with
+import time
-# env variables.
-import os
 
 # For this test case we need import and use the Python driver
 from planetmint_driver import Planetmint
 from planetmint_driver.crypto import generate_keypair
 
+
 def test_multiple_owners():
-    # ## Set up a connection to the Planetmint integration test nodes
+    # Setup up connection to Planetmint integration test nodes
-    pm_itest1 = Planetmint(os.environ.get('PLANETMINT_ENDPOINT_1'))
+    hosts = []
-    pm_itest2 = Planetmint(os.environ.get('PLANETMINT_ENDPOINT_2'))
+    with open('/shared/hostnames') as f:
+        hosts = f.readlines()
+
+    pm_hosts = list(map(lambda x: Planetmint(x), hosts))
+
+    pm_alpha = pm_hosts[0]
+    pm_betas = pm_hosts[1:]
 
     # Generate Keypairs for Alice and Bob!
     alice, bob = generate_keypair(), generate_keypair()
@@ -50,7 +55,7 @@ def test_multiple_owners():
 
     # They prepare a `CREATE` transaction. To have multiple owners, both
     # Bob and Alice need to be the recipients.
-    prepared_dw_tx = pm_itest1.transactions.prepare(
+    prepared_dw_tx = pm_alpha.transactions.prepare(
         operation='CREATE',
         signers=alice.public_key,
         recipients=(alice.public_key, bob.public_key),
@@ -58,29 +63,31 @@ def test_multiple_owners():
 
     # Now they both sign the transaction by providing their private keys.
     # And send it afterwards.
-    fulfilled_dw_tx = pm_itest1.transactions.fulfill(
+    fulfilled_dw_tx = pm_alpha.transactions.fulfill(
         prepared_dw_tx,
         private_keys=[alice.private_key, bob.private_key])
 
-    pm_itest1.transactions.send_commit(fulfilled_dw_tx)
+    pm_alpha.transactions.send_commit(fulfilled_dw_tx)
 
     # We store the `id` of the transaction to use it later on.
     dw_id = fulfilled_dw_tx['id']
 
+    time.sleep(1)
     # Let's retrieve the transaction from both nodes
-    pm_itest1_tx = pm_itest1.transactions.retrieve(dw_id)
+    pm_alpha_tx = pm_alpha.transactions.retrieve(dw_id)
-    pm_itest2_tx = pm_itest2.transactions.retrieve(dw_id)
+    pm_betas_tx = list(map(lambda beta: beta.transactions.retrieve(dw_id), pm_betas))
 
     # Both retrieved transactions should be the same
-    assert pm_itest1_tx == pm_itest2_tx
+    for tx in pm_betas_tx:
+        assert pm_alpha_tx == tx
 
     # Let's check if the transaction was successful.
-    assert pm_itest1.transactions.retrieve(dw_id), \
+    assert pm_alpha.transactions.retrieve(dw_id), \
         'Cannot find transaction {}'.format(dw_id)
 
     # The transaction should have two public keys in the outputs.
     assert len(
-        pm_itest1.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2
+        pm_alpha.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2
 
     # ## Alice and Bob transfer a transaction to Carol.
     # Alice and Bob save a lot of money living together. They often go out
@@ -103,36 +110,37 @@ def test_multiple_owners():
                       'owners_before': output['public_keys']}
 
     # Now they create the transaction...
-    prepared_transfer_tx = pm_itest1.transactions.prepare(
+    prepared_transfer_tx = pm_alpha.transactions.prepare(
         operation='TRANSFER',
         asset=transfer_asset,
         inputs=transfer_input,
         recipients=carol.public_key)
 
     # ... and sign it with their private keys, then send it.
-    fulfilled_transfer_tx = pm_itest1.transactions.fulfill(
+    fulfilled_transfer_tx = pm_alpha.transactions.fulfill(
         prepared_transfer_tx,
         private_keys=[alice.private_key, bob.private_key])
 
-    sent_transfer_tx = pm_itest1.transactions.send_commit(fulfilled_transfer_tx)
+    sent_transfer_tx = pm_alpha.transactions.send_commit(fulfilled_transfer_tx)
+    time.sleep(1)
 
     # Retrieve the fulfilled transaction from both nodes
-    pm_itest1_tx = pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])
+    pm_alpha_tx = pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])
-    pm_itest2_tx = pm_itest2.transactions.retrieve(fulfilled_transfer_tx['id'])
+    pm_betas_tx = list(map(lambda beta: beta.transactions.retrieve(fulfilled_transfer_tx['id']), pm_betas))
 
     # Now compare if both nodes returned the same transaction
-    assert pm_itest1_tx == pm_itest2_tx
+    for tx in pm_betas_tx:
+        assert pm_alpha_tx == tx
 
     # They check if the transaction was successful.
-    assert pm_itest1.transactions.retrieve(
+    assert pm_alpha.transactions.retrieve(
         fulfilled_transfer_tx['id']) == sent_transfer_tx
 
     # The owners before should include both Alice and Bob.
     assert len(
-        pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][
+        pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][
             'owners_before']) == 2
 
     # While the new owner is Carol.
-    assert pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])[
+    assert pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])[
            'outputs'][0]['public_keys'][0] == carol.public_key
-    

integration/scripts/all-in-one.bash

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+
+# MongoDB configuration
+[ "$(stat -c %U /data/db)" = mongodb ] || chown -R mongodb /data/db
+
+# Planetmint configuration
+/usr/src/app/scripts/planetmint-monit-config
+
+nohup mongod --bind_ip_all > "$HOME/.planetmint-monit/logs/mongodb_log_$(date +%Y%m%d_%H%M%S)" 2>&1 &
+
+# Start services
+monit -d 5 -I -B

integration/scripts/clean-shared.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+rm /shared/hostnames
+rm /shared/lock
+rm /shared/*node_id
+rm /shared/*.json
+rm /shared/id_rsa.pub

integration/scripts/election.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# Show tendermint node id
+show_id () {
+    tendermint --home=/tendermint show_node_id | tail -n 1
+}
+
+# Show validator public key
+show_validator () {
+    tendermint --home=/tendermint show_validator | tail -n 1
+}
+
+# Elect new voting power for node
+elect_validator () {
+    planetmint election new upsert-validator $1 $2 $3 --private-key /tendermint/config/priv_validator_key.json
+}
+
+# Show election state
+show_election () {
+    planetmint election show $1
+}
+
+# Approve election
+approve_validator () {
+    planetmint election approve $1 --private-key /tendermint/config/priv_validator_key.json
+}
+
+# Fetch tendermint id and pubkey and create upsert proposal
+elect () {
+    node_id=$(show_id)
+    validator_pubkey=$(show_validator | jq -r .value)
+    proposal=$(elect_validator $validator_pubkey $1 $node_id 2>&1 | grep SUCCESS)
+    echo ${proposal##* }
+}
+
+usage () {
+    echo "usage: TODO"
+}
+ 
+while [ "$1" != "" ]; do
+    case $1 in
+        show_id )           show_id
+                            ;;
+        show_validator )    show_validator
+                            ;;
+        elect )             shift
+                            elect $1
+                            ;;
+        show_election )     shift
+                            show_election $1
+                            ;;
+        approve )           shift
+                            approve_validator $1
+                            ;;
+        * )                 usage
+                            exit 1                           
+    esac
+    shift
+done
+
+exitcode=$?
+
+exit $exitcode

integration/scripts/genesis.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+import json
+import sys
+
+
+def edit_genesis() -> None:
+    file_names = sys.argv[1:]
+
+    validators = []
+    for file_name in file_names:
+        file = open(file_name)
+        genesis = json.load(file)
+        validators.extend(genesis['validators'])
+        file.close()
+
+    genesis_file = open(file_names[0])
+    genesis_json = json.load(genesis_file)
+    genesis_json['validators'] = validators
+    genesis_file.close()
+
+    with open('/shared/genesis.json', 'w') as f:
+        json.dump(genesis_json, f, indent=True)
+
+    return None
+
+
+if __name__ == '__main__':
+    edit_genesis()

integration/scripts/planetmint-monit-config

@@ -0,0 +1,208 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+set -o nounset
+
+# Check if directory for monit logs exists
+if [ ! -d "$HOME/.planetmint-monit" ]; then
+	mkdir -p "$HOME/.planetmint-monit"
+fi
+
+monit_pid_path=${MONIT_PID_PATH:=$HOME/.planetmint-monit/monit_processes}
+monit_script_path=${MONIT_SCRIPT_PATH:=$HOME/.planetmint-monit/monit_script}
+monit_log_path=${MONIT_LOG_PATH:=$HOME/.planetmint-monit/logs}
+monitrc_path=${MONITRC_PATH:=$HOME/.monitrc}
+
+function usage() {
+	cat <<EOM
+
+    Usage: ${0##*/} [-h]
+
+    Configure Monit for Planetmint and Tendermint process management.
+
+    ENV[MONIT_PID_PATH] || --monit-pid-path PATH
+
+    Absolute path to directory where the the program's pid-file will reside.
+    The pid-file contains the ID(s) of the process(es). (default: ${monit_pid_path})
+
+    ENV[MONIT_SCRIPT_PATH] || --monit-script-path PATH
+
+    Absolute path to the directory where the executable program or
+    script is present. (default: ${monit_script_path})
+
+    ENV[MONIT_LOG_PATH] || --monit-log-path PATH
+
+    Absolute path to the directory where all the logs for processes
+    monitored by Monit are stored. (default: ${monit_log_path})
+
+    ENV[MONITRC_PATH] || --monitrc-path PATH
+
+    Absolute path to the monit control file(monitrc). (default: ${monitrc_path})
+
+    -h|--help
+        Show this help and exit.
+
+EOM
+}
+
+while [[ $# -gt 0 ]]; do
+	arg="$1"
+	case $arg in
+	--monit-pid-path)
+		monit_pid_path="$2"
+		shift
+		;;
+	--monit-script-path)
+		monit_script_path="$2"
+		shift
+		;;
+	--monit-log-path)
+		monit_log_path="$2"
+		shift
+		;;
+	--monitrc-path)
+		monitrc_path="$2"
+		shift
+		;;
+	-h | --help)
+		usage
+		exit
+		;;
+	*)
+		echo "Unknown option: $1"
+		usage
+		exit 1
+		;;
+	esac
+	shift
+done
+
+# Check if directory for monit logs exists
+if [ ! -d "$monit_log_path" ]; then
+	mkdir -p "$monit_log_path"
+fi
+
+# Check if directory for monit pid files exists
+if [ ! -d "$monit_pid_path" ]; then
+	mkdir -p "$monit_pid_path"
+fi
+
+cat >${monit_script_path} <<EOF
+#!/bin/bash
+case \$1 in
+
+  start_planetmint)
+
+    pushd \$4
+      nohup planetmint start > /dev/null 2>&1 &
+
+      echo \$! > \$2
+    popd
+
+    ;;
+
+  stop_planetmint)
+
+    kill -2 \`cat \$2\`
+    rm -f \$2
+
+    ;;
+
+  start_tendermint)
+
+    pushd \$4
+
+      nohup tendermint node \
+           --p2p.laddr "tcp://0.0.0.0:26656" \
+           --rpc.laddr "tcp://0.0.0.0:26657" \
+           --proxy_app="tcp://0.0.0.0:26658" \
+           --consensus.create_empty_blocks=false \
+           --p2p.pex=false >> \$3/tendermint.out.log 2>> \$3/tendermint.err.log &
+
+      echo \$! > \$2
+    popd
+
+    ;;
+
+  stop_tendermint)
+
+    kill -2 \`cat \$2\`
+    rm -f \$2
+
+    ;;
+
+esac
+exit 0
+EOF
+chmod +x ${monit_script_path}
+
+cat >${monit_script_path}_logrotate <<EOF
+#!/bin/bash
+case \$1 in
+
+  rotate_tendermint_logs)
+    /bin/cp \$2 \$2.\$(date +%y-%m-%d)
+    /bin/tar -cvf \$2.\$(date +%Y%m%d_%H%M%S).tar.gz \$2.\$(date +%y-%m-%d)
+    /bin/rm \$2.\$(date +%y-%m-%d)
+    /bin/cp /dev/null \$2
+    ;;
+
+esac
+exit 0
+EOF
+chmod +x ${monit_script_path}_logrotate
+
+# Handling overwriting of control file interactively
+if [ -f "$monitrc_path" ]; then
+	echo "$monitrc_path already exists."
+	read -p "Overwrite[Y]? " -n 1 -r
+	echo
+	if [[ $REPLY =~ ^[Yy]$ ]]; then
+		echo "Overriding $monitrc_path"
+	else
+		read -p "Enter absolute path to store Monit control file: " monitrc_path
+		eval monitrc_path="$monitrc_path"
+		if [ ! -d "$(dirname $monitrc_path)" ]; then
+			echo "Failed to save monit control file '$monitrc_path': No such file or directory."
+			exit 1
+		fi
+	fi
+fi
+
+# configure monitrc
+cat >${monitrc_path} <<EOF
+set httpd
+  port 2812
+  allow localhost
+
+check process planetmint
+    with pidfile ${monit_pid_path}/planetmint.pid
+    start program "${monit_script_path} start_planetmint $monit_pid_path/planetmint.pid ${monit_log_path} ${monit_log_path}"
+    restart program "${monit_script_path} start_planetmint $monit_pid_path/planetmint.pid ${monit_log_path} ${monit_log_path}"
+    stop program "${monit_script_path} stop_planetmint $monit_pid_path/planetmint.pid ${monit_log_path} ${monit_log_path}"
+
+check process tendermint
+    with pidfile ${monit_pid_path}/tendermint.pid
+    start program "${monit_script_path} start_tendermint ${monit_pid_path}/tendermint.pid ${monit_log_path} ${monit_log_path}"
+    restart program "${monit_script_path} start_tendermint ${monit_pid_path}/tendermint.pid ${monit_log_path} ${monit_log_path}"
+    stop program "${monit_script_path} stop_tendermint ${monit_pid_path}/tendermint.pid ${monit_log_path} ${monit_log_path}"
+    depends on planetmint
+
+check file tendermint.out.log with path ${monit_log_path}/tendermint.out.log
+    if size > 200 MB then
+        exec "${monit_script_path}_logrotate rotate_tendermint_logs ${monit_log_path}/tendermint.out.log $monit_pid_path/tendermint.pid"
+
+check file tendermint.err.log with path ${monit_log_path}/tendermint.err.log
+    if size > 200 MB then
+        exec "${monit_script_path}_logrotate rotate_tendermint_logs ${monit_log_path}/tendermint.err.log $monit_pid_path/tendermint.pid"
+
+EOF
+
+# Setting permissions for control file
+chmod 0700 ${monitrc_path}
+
+echo -e "Planetmint process manager configured!"
+set -o errexit

integration/scripts/pre-config-planetmint.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# Write hostname to list
+echo $(hostname) >> /shared/hostnames
+
+# Create ssh folder
+mkdir ~/.ssh
+
+# Wait for test container pubkey
+while [ ! -f /shared/id_rsa.pub ]; do
+    echo "WAIT FOR PUBKEY"
+    sleep 1
+done
+
+# Add pubkey to authorized keys
+cat /shared/id_rsa.pub > ~/.ssh/authorized_keys
+
+# Allow root user login
+sed -i "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config
+
+# Restart ssh service
+service ssh restart
+
+# Tendermint configuration
+tendermint init
+
+# Write node id to shared folder
+HOSTNAME=$(hostname)
+NODE_ID=$(tendermint show_node_id | tail -n 1)
+echo $NODE_ID > /shared/${HOSTNAME}_node_id
+
+# Wait for other node ids
+FILES=()
+while [ ! ${#FILES[@]} == $SCALE ]; do
+    echo "WAIT FOR NODE IDS"
+    sleep 1
+    FILES=(/shared/*node_id)
+done
+
+# Write node ids to persistent peers
+PEERS="persistent_peers = \""
+for f in ${FILES[@]}; do
+    ID=$(cat $f)
+    HOST=$(echo $f | cut -c 9-20)
+    if [ ! $HOST == $HOSTNAME ]; then
+        PEERS+="${ID}@${HOST}:26656, "
+    fi
+done
+PEERS=$(echo $PEERS | rev | cut -c 2- | rev)
+PEERS+="\""
+sed -i "/persistent_peers = \"\"/c\\${PEERS}" /tendermint/config/config.toml
+
+# Copy genesis.json to shared folder
+cp /tendermint/config/genesis.json /shared/${HOSTNAME}_genesis.json
+
+# Await config file of all services to be present
+FILES=()
+while [ ! ${#FILES[@]} == $SCALE ]; do
+    echo "WAIT FOR GENESIS FILES"
+    sleep 1
+    FILES=(/shared/*_genesis.json)
+done
+
+# Create genesis.json for nodes
+if [ ! -f /shared/lock ]; then
+    echo LOCKING
+    touch /shared/lock
+    /usr/src/app/scripts/genesis.py ${FILES[@]}
+fi
+
+while [ ! -f /shared/genesis.json ]; do
+    echo "WAIT FOR GENESIS"
+    sleep 1
+done
+
+# Copy genesis.json to tendermint config
+cp /shared/genesis.json /tendermint/config/genesis.json
+
+exec "$@"

integration/scripts/pre-config-test.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# Create ssh folder
+mkdir ~/.ssh
+
+# Create ssh keys
+ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa
+
+# Publish pubkey to shared folder
+cp ~/.ssh/id_rsa.pub /shared
+
+exec "$@"

integration/scripts/test.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# Read host names from shared
+readarray -t HOSTNAMES < /shared/hostnames
+
+# Split into proposer and approvers
+ALPHA=${HOSTNAMES[0]}
+BETAS=${HOSTNAMES[@]:1}
+
+# Propose validator upsert
+result=$(ssh -o "StrictHostKeyChecking=no" -i \~/.ssh/id_rsa root@${ALPHA} 'bash -s' < scripts/election.sh elect 2)
+
+# Approve validator upsert
+for BETA in ${BETAS[@]}; do
+    ssh -o "StrictHostKeyChecking=no" -i ~/.ssh/id_rsa root@${BETA} 'bash -s' < scripts/election.sh approve $result
+done
+
+exec "$@"

integration/scripts/wait-for-planetmint.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+# Only continue if all services are ready
+HOSTNAMES=()
+while [ ! ${#HOSTNAMES[@]} == $SCALE ]; do
+    echo "WAIT FOR HOSTNAMES"
+    sleep 1
+    readarray -t HOSTNAMES < /shared/hostnames
+done
+
+for host in ${HOSTNAMES[@]}; do
+    while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $host:9984)" != "200" ]]; do
+        echo "WAIT FOR PLANETMINT $host"
+        sleep 1
+    done
+done
+
+for host in ${HOSTNAMES[@]}; do
+    while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $host:26657)" != "200" ]]; do
+        echo "WAIT FOR TENDERMINT $host"
+        sleep 1
+    done
+done
+
+exec "$@"

pkg/scripts/all-in-one.bash

@@ -16,4 +16,5 @@ nohup mongod --bind_ip_all > "$HOME/.planetmint-monit/logs/mongodb_log_$(date +%
 # Tendermint configuration
 tendermint init
 
+# Start services
 monit -d 5 -I -B

run-integration-test.sh

@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-# Copyright © 2020 Interplanetary Database Association e.V.,
-# Planetmint and IPDB software contributors.
-# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
-# Code is Apache-2.0 and docs are CC-BY-4.0
-
-
-# Check if both integration test nodes are reachable
-check_status () {
-    OK="200 OK"
-
-    STATUS_1=$(curl -I -s -X GET https://itest1.planetmint.io/ | head -n 1)
-    STATUS_2=$(curl -I -s -X GET https://itest2.planetmint.io/ | head -n 1)
-
-    # Check if both response status codes return 200 OK
-    if ! [[ "$STATUS_1" == *"$OK"* ]] || ! [[ "$STATUS_2" == *"$OK"* ]]
-    then
-        exit 1
-    fi
-}
-
-run_test () {
-	docker-compose run --rm python-integration pytest /src
-}
-
-teardown () {
-	docker-compose down
-}
-
-check_status
-run_test
-exitcode=$?
-teardown
-
-exit $exitcode

scripts/run-integration-test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright © 2020 Interplanetary Database Association e.V.,
+# Planetmint and IPDB software contributors.
+# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
+# Code is Apache-2.0 and docs are CC-BY-4.0
+
+run_test() {
+    docker-compose -f docker-compose.integration.yml up test
+}
+
+teardown () {
+    docker-compose -f docker-compose.integration.yml down
+}
+
+run_test
+exitcode=$?
+teardown
+
+exit $exitcode

scripts/test.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-# Copyright © 2020 Interplanetary Database Association e.V.,
-# Planetmint and IPDB software contributors.
-# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
-# Code is Apache-2.0 and docs are CC-BY-4.0
-
-result=$(ssh -o StrictHostKeyChecking=accept-new root@64.225.106.52 -i id_ed25519 'bash -s' < scripts/election.sh elect 35)
-ssh -o StrictHostKeyChecking=accept-new root@64.225.105.60 -i id_ed25519 'bash -s' < scripts/election.sh approve $result

secrets/id_ed25519.enc

@@ -1,4 +0,0 @@
-³.˜·ÚÇ9DDÔÒ@£Éê…ôÛïëŸÚŒžÛJÔá–©¹K
Ð
çêØÝc÷‡‘^ª»…
-YÄ/õT 	|%²Å&âÔvfõ#Ä"â_®P’à#öS¡Ôè4ôiQSýún0‡4?vÞx…N! <20>5Ê;i óŸ÷WH¾P²"Šx§û5ÎÎ9Ç"Œ]i¦"³4ü—‚IòD8ðôÌ‹é¶Ðtý•ÊueŒaÎÿ;˜‡Ô©’ì¾`êÕ‹1Þ¬F·˜ÁXõëP<C3AB>E6ÏŠ×\ô"’Á$eÍO ˆó<CB86><¢õàPèÛ¡Sò&;i¤æ‰Õ¹2V	3Ÿ/ÐJD+;åT£–×ØL(œtêR9vEû¤ñ-+!ñÂZ±ðŒŠhÛf¸æ5ߤÓ8S.âúG4ÌnŠ«ÄÙáMQüÝ!_ãêzËÜú¶•‹Å`UULì’@·Í¡¼w‹ŸV3@ų?‡ø’)'Å>Û>K¤Cx%);Çä8
-n÷ÑŽ<C385>5ô¡ù<C2A1>ÆËvj],20ÒÈ<C392>§î;Ðç
mÜÇ(m‡s=©
-€¯µvK)5ˆT¢Q_ ¹Lð†D¶ƒ™:hüüÉ+<2B>