Scalable integration test (#57)

* updated Dockerfile-all-in-one

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* adjusted all-in-one.bash and monit conf to work with dockerized setup

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* integration tests pass inconsistently

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added timeout for integration test pass

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* add startup control logic, adjusted tests to wait for transactions

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added pre-config for docker-compose approach, removed remnants of old integration tests

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* reverted changes to pkg, split pre-config, added clean-shared service

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* fixed path in all-in-one.bash

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added ipdb copyright notice

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* chmod planetmint-monit-config

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* removed entrypoint from Dockerfile-all-in-one

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added integration stage to travis matrix

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* removed unused secret

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* changed pre-config and docker-compose.integration.yml to support scaling

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* using env var to control number of nodes

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* make test-integration now scalable

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added make docs-integration, added .gitignore to python integration tests, updated readme and removed clutter

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* fixed linter errors

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* disable planetmint for test purpose

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* test docker-compose down

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* disable every job except integration test

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* need more logs

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* name collision?

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* reverted changes to debug ci

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

* added TODO for ci optimization

Signed-off-by: Lorenz Herzberger <lorenzherzberger@gmail.com>

.ci/travis_script.sh

@@ -14,6 +14,7 @@ elif [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then
 elif [[ ${PLANETMINT_ACCEPTANCE_TEST} == 'enable' ]]; then
     ./scripts/run-acceptance-test.sh
 elif [[ ${PLANETMINT_INTEGRATION_TEST} == 'enable' ]]; then
+    docker-compose down # TODO: remove after ci optimization
     ./scripts/run-integration-test.sh
 else
   docker-compose exec planetmint pytest -v --cov=planetmint --cov-report xml:htmlcov/coverage.xml

Makefile

@@ -100,6 +100,10 @@ docs-acceptance: check-deps ## Create documentation for acceptance tests
 	@$(DC) run --rm python-acceptance pycco -i -s /src -d /docs
 	$(BROWSER) acceptance/python/docs/index.html
 
+docs-integration: check-deps ## Create documentation for integration tests
+	@$(DC) run --rm python-integration pycco -i -s /src -d /docs
+	$(BROWSER) integration/python/docs/index.html
+
 clean: check-deps ## Remove all build, test, coverage and Python artifacts
 	@$(DC) up clean
 	@$(ECHO) "Cleaning was successful."

docker-compose.integration.yml

@@ -13,7 +13,7 @@ services:
       - ./integration/scripts/clean-shared.sh:/scripts/clean-shared.sh
       - shared:/shared
   
-  planetmint_1:
+  planetmint-all-in-one:
     build:
       context: .
       dockerfile: Dockerfile-all-in-one
@@ -26,53 +26,27 @@ services:
       - "26656"
       - "26657"
       - "26658"
-    environment:
-      ME: "planetmint_1"
-      OTHER: "planetmint_2"
-    command: ["/usr/src/app/scripts/pre-config-planetmint.sh", "/usr/src/app/scripts/all-in-one.bash"]
-    volumes:
-      - ./integration/scripts:/usr/src/app/scripts
-      - shared:/shared
-
-  planetmint_2:
-    build:
-      context: .
-      dockerfile: Dockerfile-all-in-one
-    depends_on:
-      - clean-shared
-    expose:
-      - "22"
-      - "9984"
-      - "9985"
-      - "26656"
-      - "26657"
-      - "26658"
-    environment:
-      ME: "planetmint_2"
-      OTHER: "planetmint_1"
     command: ["/usr/src/app/scripts/pre-config-planetmint.sh", "/usr/src/app/scripts/all-in-one.bash"]
+    environment:
+      SCALE: ${SCALE:-4}
     volumes:
       - ./integration/scripts:/usr/src/app/scripts
       - shared:/shared
+    scale: ${SCALE:-4}
 
   test:
     build:
       context: .
       dockerfile: integration/python/Dockerfile
     depends_on:
-      - planetmint_1
-      - planetmint_2
-    environment:
-      ME: "test"
-      PLANETMINT_ENDPOINT_1: planetmint_1
-      PLANETMINT_ENDPOINT_2: planetmint_2
+      - planetmint-all-in-one
     command: ["/scripts/pre-config-test.sh", "/scripts/wait-for-planetmint.sh", "/scripts/test.sh", "pytest", "/src"]
+    environment:
+      SCALE: ${SCALE:-4}
     volumes:
       - ./integration/python/src:/src
       - ./integration/scripts:/scripts
       - shared:/shared
 
-
-
 volumes:
   shared:

docker-compose.yml

@@ -89,6 +89,7 @@ services:
       context: .
       dockerfile: ./integration/python/Dockerfile
     volumes:
+      - ./integration/python/docs:/docs
       - ./integration/python/src:/src
     environment:
       - PLANETMINT_ENDPOINT_1=https://itest1.planetmint.io

integration/README.md

@@ -8,15 +8,16 @@ Code is Apache-2.0 and docs are CC-BY-4.0
 # Integration test suite
 This directory contains the integration test suite for Planetmint.
 
-The suite uses Docker Compose to run all tests.
+The suite uses Docker Compose to spin up multiple Planetmint nodes, run tests with `pytest` as well as cli tests and teardown. 
 
 ## Running the tests
 Run `make test-integration` in the project root directory.
 
-During development you can run single test use `pytest` inside the `python-integration` container with:
+By default the integration test suite spins up four planetmint nodes. If you desire to run a different configuration you can pass `SCALE=<number of nodes>` as an environmental variable.
+
+## Writing and documenting the tests
+Tests are sometimes difficult to read. For integration tests, we try to be really explicit on what the test is doing, so please write code that is *simple* and easy to understand. We decided to use literate-programming documentation. To generate the documentation for python tests run:
 
 ```bash
-docker-compose run --rm python-integration pytest <use whatever option you need>
+make docs-integration
 ```
-
-Note: The `/src` directory contains all the test within the container.

integration/python/.gitignore

@@ -0,0 +1 @@
+docs

integration/python/Dockerfile

@@ -3,5 +3,7 @@ FROM python:3.9
 RUN mkdir -p /src
 RUN pip install --upgrade \
     pytest~=6.2.5 \
-    planetmint-driver~=0.9.0
+    planetmint-driver~=0.9.0 \
+    pycco
+
 RUN apt-get update && apt-get install -y openssh-client openssh-server

integration/python/src/test_basic.py

@@ -7,18 +7,21 @@
 from planetmint_driver import Planetmint
 from planetmint_driver.crypto import generate_keypair
 import time
-import os
 
 
 def test_basic():
     # Setup up connection to Planetmint integration test nodes
-    pm_itest1_url = os.environ.get('PLANETMINT_ENDPOINT_1')
-    pm_itest2_url = os.environ.get('PLANETMINT_ENDPOINT_1')
-    pm_itest1 = Planetmint(pm_itest1_url)
-    pm_itest2 = Planetmint(pm_itest2_url)
+    hosts = []
+    with open('/shared/hostnames') as f:
+        hosts = f.readlines()
+
+    pm_hosts = list(map(lambda x: Planetmint(x), hosts))
+
+    pm_alpha = pm_hosts[0]
+    pm_betas = pm_hosts[1:]
 
     # genarate a keypair
-    alice, bob = generate_keypair(), generate_keypair()
+    alice = generate_keypair()
 
     # create a digital asset for Alice
     game_boy_token = {
@@ -29,7 +32,7 @@ def test_basic():
     }
 
     # prepare the transaction with the digital asset and issue 10 tokens to bob
-    prepared_creation_tx = pm_itest1.transactions.prepare(
+    prepared_creation_tx = pm_alpha.transactions.prepare(
         operation='CREATE',
         metadata={
             'hash': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
@@ -39,20 +42,21 @@ def test_basic():
         asset=game_boy_token)
 
     # fulfill and send the transaction
-    fulfilled_creation_tx = pm_itest1.transactions.fulfill(
+    fulfilled_creation_tx = pm_alpha.transactions.fulfill(
         prepared_creation_tx,
         private_keys=alice.private_key)
-    pm_itest1.transactions.send_commit(fulfilled_creation_tx)
-    time.sleep(4)
+    pm_alpha.transactions.send_commit(fulfilled_creation_tx)
+    time.sleep(1)
 
     creation_tx_id = fulfilled_creation_tx['id']
 
-    # retrieve transactions from both planetmint nodes
-    creation_tx_itest1 = pm_itest1.transactions.retrieve(creation_tx_id)
-    creation_tx_itest2 = pm_itest2.transactions.retrieve(creation_tx_id)
+    # retrieve transactions from all planetmint nodes
+    creation_tx_alpha = pm_alpha.transactions.retrieve(creation_tx_id)
+    creation_tx_betas = list(map(lambda beta: beta.transactions.retrieve(creation_tx_id), pm_betas))
 
-    # Assert that transaction is stored on both planetmint nodes
-    assert creation_tx_itest1 == creation_tx_itest2
+    # Assert that transaction is stored on all planetmint nodes
+    for tx in creation_tx_betas:
+        assert creation_tx_alpha == tx
 
     # Transfer
     # create the output and inout for the transaction
@@ -65,7 +69,7 @@ def test_basic():
                       'owners_before': output['public_keys']}
 
     # prepare the transaction and use 3 tokens
-    prepared_transfer_tx = pm_itest1.transactions.prepare(
+    prepared_transfer_tx = pm_alpha.transactions.prepare(
         operation='TRANSFER',
         asset=transfer_asset,
         inputs=transfer_input,
@@ -74,26 +78,19 @@ def test_basic():
         recipients=[([alice.public_key], 10)])
 
     # fulfill and send the transaction
-    fulfilled_transfer_tx = pm_itest1.transactions.fulfill(
+    fulfilled_transfer_tx = pm_alpha.transactions.fulfill(
         prepared_transfer_tx,
         private_keys=alice.private_key)
-    sent_transfer_tx = pm_itest1.transactions.send_commit(fulfilled_transfer_tx)
+    sent_transfer_tx = pm_alpha.transactions.send_commit(fulfilled_transfer_tx)
 
-    transfer_tx_id = fulfilled_transfer_tx['id']
+    time.sleep(1)
+
+    transfer_tx_id = sent_transfer_tx['id']
 
     # retrieve transactions from both planetmint nodes
-    transfer_tx_itest1 = pm_itest1.transactions.retrieve(transfer_tx_id)
-    transfer_tx_itest2 = pm_itest2.transactions.retrieve(transfer_tx_id)
+    transfer_tx_alpha = pm_alpha.transactions.retrieve(transfer_tx_id)
+    transfer_tx_betas = list(map(lambda beta: beta.transactions.retrieve(transfer_tx_id), pm_betas))
 
     # Assert that transaction is stored on both planetmint nodes
-    assert transfer_tx_itest1 == transfer_tx_itest2
-
-
-
-
-    
-    
-
-
-
-
+    for tx in transfer_tx_betas:
+        assert transfer_tx_alpha == tx

integration/python/src/test_multisig.py

@@ -18,21 +18,24 @@
 #
 # This integration test is a rip-off of our mutliple signature acceptance tests.
 
-# ## Imports
-# We need some utils from the `os` package, we will interact with
-# env variables.
-import os
+# # Imports
 import time
 
 # For this test case we need import and use the Python driver
 from planetmint_driver import Planetmint
 from planetmint_driver.crypto import generate_keypair
-from planetmint_driver.exceptions import NotFoundError
+
 
 def test_multiple_owners():
-    # ## Set up a connection to the Planetmint integration test nodes
-    pm_itest1 = Planetmint(os.environ.get('PLANETMINT_ENDPOINT_1'))
-    pm_itest2 = Planetmint(os.environ.get('PLANETMINT_ENDPOINT_2'))
+    # Setup up connection to Planetmint integration test nodes
+    hosts = []
+    with open('/shared/hostnames') as f:
+        hosts = f.readlines()
+
+    pm_hosts = list(map(lambda x: Planetmint(x), hosts))
+
+    pm_alpha = pm_hosts[0]
+    pm_betas = pm_hosts[1:]
 
     # Generate Keypairs for Alice and Bob!
     alice, bob = generate_keypair(), generate_keypair()
@@ -52,7 +55,7 @@ def test_multiple_owners():
 
     # They prepare a `CREATE` transaction. To have multiple owners, both
     # Bob and Alice need to be the recipients.
-    prepared_dw_tx = pm_itest1.transactions.prepare(
+    prepared_dw_tx = pm_alpha.transactions.prepare(
         operation='CREATE',
         signers=alice.public_key,
         recipients=(alice.public_key, bob.public_key),
@@ -60,36 +63,31 @@ def test_multiple_owners():
 
     # Now they both sign the transaction by providing their private keys.
     # And send it afterwards.
-    fulfilled_dw_tx = pm_itest1.transactions.fulfill(
+    fulfilled_dw_tx = pm_alpha.transactions.fulfill(
         prepared_dw_tx,
         private_keys=[alice.private_key, bob.private_key])
 
-    pm_itest1.transactions.send_commit(fulfilled_dw_tx)
+    pm_alpha.transactions.send_commit(fulfilled_dw_tx)
 
     # We store the `id` of the transaction to use it later on.
     dw_id = fulfilled_dw_tx['id']
 
+    time.sleep(1)
     # Let's retrieve the transaction from both nodes
-    pm_itest1_tx = pm_itest1.transactions.retrieve(dw_id)
-    pm_itest2_tx = {}
-    # TODO: REPLACE WITH ASYNC OR POLL
-    try:
-        pm_itest2_tx = pm_itest2.transactions.retrieve(dw_id)
-    except NotFoundError:
-        print('TOO FAST')
-        time.sleep(3)
-        pm_itest2_tx = pm_itest2.transactions.retrieve(dw_id)
+    pm_alpha_tx = pm_alpha.transactions.retrieve(dw_id)
+    pm_betas_tx = list(map(lambda beta: beta.transactions.retrieve(dw_id), pm_betas))
 
     # Both retrieved transactions should be the same
-    assert pm_itest1_tx == pm_itest2_tx
+    for tx in pm_betas_tx:
+        assert pm_alpha_tx == tx
 
     # Let's check if the transaction was successful.
-    assert pm_itest1.transactions.retrieve(dw_id), \
+    assert pm_alpha.transactions.retrieve(dw_id), \
         'Cannot find transaction {}'.format(dw_id)
 
     # The transaction should have two public keys in the outputs.
     assert len(
-        pm_itest1.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2
+        pm_alpha.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2
 
     # ## Alice and Bob transfer a transaction to Carol.
     # Alice and Bob save a lot of money living together. They often go out
@@ -112,43 +110,37 @@ def test_multiple_owners():
                       'owners_before': output['public_keys']}
 
     # Now they create the transaction...
-    prepared_transfer_tx = pm_itest1.transactions.prepare(
+    prepared_transfer_tx = pm_alpha.transactions.prepare(
         operation='TRANSFER',
         asset=transfer_asset,
         inputs=transfer_input,
         recipients=carol.public_key)
 
     # ... and sign it with their private keys, then send it.
-    fulfilled_transfer_tx = pm_itest1.transactions.fulfill(
+    fulfilled_transfer_tx = pm_alpha.transactions.fulfill(
         prepared_transfer_tx,
         private_keys=[alice.private_key, bob.private_key])
 
-    sent_transfer_tx = pm_itest1.transactions.send_commit(fulfilled_transfer_tx)
+    sent_transfer_tx = pm_alpha.transactions.send_commit(fulfilled_transfer_tx)
+    time.sleep(1)
 
     # Retrieve the fulfilled transaction from both nodes
-    pm_itest1_tx = pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])
-    pm_itest2_tx
-    # TODO: REPLACE WITH ASYNC OR POLL
-    try:
-        pm_itest2_tx = pm_itest2.transactions.retrieve(fulfilled_transfer_tx['id'])
-    except NotFoundError:
-        print('TOO FAST')
-        time.sleep(3)
-        pm_itest2_tx = pm_itest2.transactions.retrieve(fulfilled_transfer_tx['id'])
+    pm_alpha_tx = pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])
+    pm_betas_tx = list(map(lambda beta: beta.transactions.retrieve(fulfilled_transfer_tx['id']), pm_betas))
 
     # Now compare if both nodes returned the same transaction
-    assert pm_itest1_tx == pm_itest2_tx
+    for tx in pm_betas_tx:
+        assert pm_alpha_tx == tx
 
     # They check if the transaction was successful.
-    assert pm_itest1.transactions.retrieve(
+    assert pm_alpha.transactions.retrieve(
         fulfilled_transfer_tx['id']) == sent_transfer_tx
 
     # The owners before should include both Alice and Bob.
     assert len(
-        pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][
+        pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][
             'owners_before']) == 2
 
     # While the new owner is Carol.
-    assert pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])[
+    assert pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])[
            'outputs'][0]['public_keys'][0] == carol.public_key
-    

integration/scripts/clean-shared.sh

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
-rm /shared/planetmint*
-rm /shared/genesis.json
+rm /shared/hostnames
+rm /shared/lock
+rm /shared/*node_id
+rm /shared/*.json
 rm /shared/id_rsa.pub

integration/scripts/genesis.py

@@ -5,32 +5,29 @@
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
 import json
-import os
+import sys
+
 
-# TODO: CHANGE ME/OTHER VARIABLES
 def edit_genesis() -> None:
-    ME = os.getenv('ME')
-    OTHER = os.getenv('OTHER')
-
-    if ME == 'planetmint_1':
-        file_name = '{}_genesis.json'.format(ME)
-        other_file_name = '{}_genesis.json'.format(OTHER)
-
-        file = open(os.path.join('/shared', file_name))
-        other_file = open(os.path.join('/shared', other_file_name))
+    file_names = sys.argv[1:]
 
+    validators = []
+    for file_name in file_names:
+        file = open(file_name)
         genesis = json.load(file)
-        other_genesis = json.load(other_file)
-
-        genesis['validators'] = genesis['validators'] + other_genesis['validators']
-
+        validators.extend(genesis['validators'])
         file.close()
-        other_file.close()
 
-        with open(os.path.join('/shared', 'genesis.json'), 'w') as f:
-            json.dump(genesis, f, indent=True)
+    genesis_file = open(file_names[0])
+    genesis_json = json.load(genesis_file)
+    genesis_json['validators'] = validators
+    genesis_file.close()
+
+    with open('/shared/genesis.json', 'w') as f:
+        json.dump(genesis_json, f, indent=True)
 
     return None
 
+
 if __name__ == '__main__':
     edit_genesis()

integration/scripts/pre-config-planetmint.sh

@@ -4,6 +4,9 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
+# Write hostname to list
+echo $(hostname) >> /shared/hostnames
+
 # Create ssh folder
 mkdir ~/.ssh
 
@@ -26,31 +29,48 @@ service ssh restart
 tendermint init
 
 # Write node id to shared folder
+HOSTNAME=$(hostname)
 NODE_ID=$(tendermint show_node_id | tail -n 1)
-echo $NODE_ID > /shared/${ME}_node_id
+echo $NODE_ID > /shared/${HOSTNAME}_node_id
 
-# Wait for other node id
-while [ ! -f "/shared/${OTHER}_node_id" ]; do
-    echo "WAIT FOR NODE ID"
+# Wait for other node ids
+FILES=()
+while [ ! ${#FILES[@]} == $SCALE ]; do
+    echo "WAIT FOR NODE IDS"
     sleep 1
+    FILES=(/shared/*node_id)
 done
 
 # Write node ids to persistent peers
-OTHER_NODE_ID=$(cat /shared/${OTHER}_node_id)
-PEERS=$(echo "persistent_peers = \"${NODE_ID}@${ME}:26656, ${OTHER_NODE_ID}@${OTHER}:26656\"")
+PEERS="persistent_peers = \""
+for f in ${FILES[@]}; do
+    ID=$(cat $f)
+    HOST=$(echo $f | cut -c 9-20)
+    if [ ! $HOST == $HOSTNAME ]; then
+        PEERS+="${ID}@${HOST}:26656, "
+    fi
+done
+PEERS=$(echo $PEERS | rev | cut -c 2- | rev)
+PEERS+="\""
 sed -i "/persistent_peers = \"\"/c\\${PEERS}" /tendermint/config/config.toml
 
 # Copy genesis.json to shared folder
-cp /tendermint/config/genesis.json /shared/${ME}_genesis.json
+cp /tendermint/config/genesis.json /shared/${HOSTNAME}_genesis.json
 
 # Await config file of all services to be present
-while [ ! -f /shared/${OTHER}_genesis.json ]; do
-    echo "WAIT FOR OTHER GENESIS"
+FILES=()
+while [ ! ${#FILES[@]} == $SCALE ]; do
+    echo "WAIT FOR GENESIS FILES"
     sleep 1
+    FILES=(/shared/*_genesis.json)
 done
 
 # Create genesis.json for nodes
-/usr/src/app/scripts/genesis.py
+if [ ! -f /shared/lock ]; then
+    echo LOCKING
+    touch /shared/lock
+    /usr/src/app/scripts/genesis.py ${FILES[@]}
+fi
 
 while [ ! -f /shared/genesis.json ]; do
     echo "WAIT FOR GENESIS"

integration/scripts/test.sh

@@ -4,7 +4,19 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
-result=$(ssh -o "StrictHostKeyChecking=no" -i \~/.ssh/id_rsa root@planetmint_1 'bash -s' < scripts/election.sh elect 2)
-ssh -o "StrictHostKeyChecking=no" -i ~/.ssh/id_rsa root@planetmint_2 'bash -s' < scripts/election.sh approve $result
+# Read host names from shared
+readarray -t HOSTNAMES < /shared/hostnames
+
+# Split into proposer and approvers
+ALPHA=${HOSTNAMES[0]}
+BETAS=${HOSTNAMES[@]:1}
+
+# Propose validator upsert
+result=$(ssh -o "StrictHostKeyChecking=no" -i \~/.ssh/id_rsa root@${ALPHA} 'bash -s' < scripts/election.sh elect 2)
+
+# Approve validator upsert
+for BETA in ${BETAS[@]}; do
+    ssh -o "StrictHostKeyChecking=no" -i ~/.ssh/id_rsa root@${BETA} 'bash -s' < scripts/election.sh approve $result
+done
 
 exec "$@"

integration/scripts/wait-for-planetmint.sh

@@ -5,24 +5,25 @@
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
 # Only continue if all services are ready
-while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' planetmint_1:9984/api/v1)" != "200" ]]; do
-    echo "WAIT FOR PLANETMINT"
+HOSTNAMES=()
+while [ ! ${#HOSTNAMES[@]} == $SCALE ]; do
+    echo "WAIT FOR HOSTNAMES"
     sleep 1
+    readarray -t HOSTNAMES < /shared/hostnames
 done
 
-while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' planetmint_1:26657)" != "200" ]]; do
-    echo "WAIT FOR TENDERMINT"
+for host in ${HOSTNAMES[@]}; do
+    while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $host:9984)" != "200" ]]; do
+        echo "WAIT FOR PLANETMINT $host"
         sleep 1
     done
-
-while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' planetmint_2:9984/api/v1)" != "200" ]]; do
-    echo "WAIT FOR PLANETMINT"
-    sleep 1
 done
 
-while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' planetmint_2:26657)" != "200" ]]; do
-    echo "WAIT FOR TENDERMINT"
+for host in ${HOSTNAMES[@]}; do
+    while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $host:26657)" != "200" ]]; do
+        echo "WAIT FOR TENDERMINT $host"
         sleep 1
     done
+done
 
 exec "$@"