diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh index 384dcc2..4c53a86 100755 --- a/.ci/travis-before-install.sh +++ b/.ci/travis-before-install.sh @@ -4,6 +4,10 @@ # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) # Code is Apache-2.0 and docs are CC-BY-4.0 +if [[ -n ${TOXENV} ]]; then + sudo apt-get update + sudo apt-get install zsh +fi if [[ -z ${TOXENV} ]]; then sudo apt-get update diff --git a/.ci/travis_script.sh b/.ci/travis_script.sh index 6ac22a9..68398d6 100755 --- a/.ci/travis_script.sh +++ b/.ci/travis_script.sh @@ -12,11 +12,10 @@ if [[ -n ${TOXENV} ]]; then elif [[ ${PLANETMINT_CI_ABCI} == 'enable' ]]; then docker-compose exec planetmint pytest -v -m abci elif [[ ${PLANETMINT_ACCEPTANCE_TEST} == 'enable' ]]; then - ./run-acceptance-test.sh + ./scripts/run-acceptance-test.sh elif [[ ${PLANETMINT_INTEGRATION_TEST} == 'enable' ]]; then - chmod 600 id_ed25519 - ./run-integration-test.sh - ./scripts/test.sh + docker-compose down # TODO: remove after ci optimization + ./scripts/run-integration-test.sh else docker-compose exec planetmint pytest -v --cov=planetmint --cov-report xml:htmlcov/coverage.xml fi diff --git a/.travis.yml b/.travis.yml index 03522ef..542a916 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,10 +43,7 @@ matrix: - PLANETMINT_INTEGRATION_TEST=enable -before_install: -- openssl aes-256-cbc -K $encrypted_555675f0ff21_key -iv $encrypted_555675f0ff21_iv - -in secrets/id_ed25519.enc -out id_ed25519 -d -- sudo .ci/travis-before-install.sh +before_install: sudo .ci/travis-before-install.sh install: .ci/travis-install.sh diff --git a/Dockerfile-all-in-one b/Dockerfile-all-in-one index 5e3c224..8dd5aec 100644 --- a/Dockerfile-all-in-one +++ b/Dockerfile-all-in-one @@ -1,30 +1,33 @@ -FROM alpine:3.9 +FROM python:3.9-slim LABEL maintainer "contact@ipdb.global" -ARG TM_VERSION=v0.34.15 +ARG TM_VERSION=0.34.15 RUN mkdir -p /usr/src/app ENV HOME /root COPY . /usr/src/app/ WORKDIR /usr/src/app -RUN apk --update add sudo bash \ - && apk --update add python3 openssl ca-certificates git \ - && apk --update add --virtual build-dependencies python3-dev vim zsh build-essential cmake\ - libffi-dev openssl-dev build-base jq zsh \ - && apk add --no-cache libstdc++ dpkg gnupg \ - && pip3 install --upgrade pip cffi \ +RUN apt-get update \ + && apt-get install -y openssl ca-certificates git \ + && apt-get install -y vim build-essential cmake jq zsh wget \ + && apt-get install -y libstdc++6 \ + && apt-get install -y openssh-client openssh-server \ + && pip install --upgrade pip cffi \ && pip install -e . \ - && apk del build-dependencies \ - && rm -f /var/cache/apk/* + && apt-get autoremove # Install mongodb and monit -RUN apk --update add mongodb monit +RUN apt-get install -y dirmngr gnupg apt-transport-https software-properties-common ca-certificates curl +RUN wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add - +RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/5.0 main" | tee /etc/apt/sources.list.d/mongodb-org-5.0.list +RUN apt-get update +RUN apt-get install -y mongodb-org monit # Install Tendermint -RUN wget https://github.com/tendermint/tendermint/releases/download/${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip \ - && unzip tendermint_${TM_VERSION}_linux_amd64.zip \ +RUN wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.tar.gz \ + && tar -xf tendermint_${TM_VERSION}_linux_amd64.tar.gz \ && mv tendermint /usr/local/bin/ \ - && rm tendermint_${TM_VERSION}_linux_amd64.zip + && rm tendermint_${TM_VERSION}_linux_amd64.tar.gz ENV TMHOME=/tendermint @@ -47,5 +50,4 @@ VOLUME /data/db /data/configdb /tendermint EXPOSE 27017 28017 9984 9985 26656 26657 26658 -WORKDIR $HOME -ENTRYPOINT ["/usr/src/app/pkg/scripts/all-in-one.bash"] +WORKDIR $HOME \ No newline at end of file diff --git a/Dockerfile-alpine b/Dockerfile-alpine index 5eadd49..a0344f6 100644 --- a/Dockerfile-alpine +++ b/Dockerfile-alpine @@ -5,7 +5,7 @@ COPY . /usr/src/app/ WORKDIR /usr/src/app RUN apk --update add sudo \ && apk --update add python3 py-pip openssl ca-certificates git\ - && apk --update add --virtual build-dependencies python3-dev zsh vim zsh build-essential cmake\ + && apk --update add --virtual build-dependencies python3-dev zsh-common vim build-essential cmake\ libffi-dev openssl-dev build-base \ && apk add --no-cache libstdc++ \ && pip3 install --upgrade pip cffi \ diff --git a/Dockerfile-dev b/Dockerfile-dev index 08c7642..bfeada4 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -3,7 +3,7 @@ FROM python:${python_version} LABEL maintainer "contact@ipdb.global" RUN apt-get update \ - && apt-get install -y git zsh vim build-essential cmake\ + && apt-get install -y git zsh-common vim build-essential cmake\ && pip install -U pip \ && apt-get autoremove \ && apt-get clean diff --git a/Makefile b/Makefile index 10a359e..b29ea0f 100644 --- a/Makefile +++ b/Makefile @@ -83,10 +83,10 @@ test-unit-watch: check-deps ## Run all tests and wait. Every time you change cod @$(DC) run --rm --no-deps planetmint pytest -f test-acceptance: check-deps ## Run all acceptance tests - @./run-acceptance-test.sh + @./scripts/run-acceptance-test.sh test-integration: check-deps ## Run all integration tests - @./run-integration-test.sh + @./scripts/run-integration-test.sh cov: check-deps ## Check code coverage and open the result in the browser @$(DC) run --rm planetmint pytest -v --cov=planetmint --cov-report html @@ -100,6 +100,10 @@ docs-acceptance: check-deps ## Create documentation for acceptance tests @$(DC) run --rm python-acceptance pycco -i -s /src -d /docs $(BROWSER) acceptance/python/docs/index.html +docs-integration: check-deps ## Create documentation for integration tests + @$(DC) run --rm python-integration pycco -i -s /src -d /docs + $(BROWSER) integration/python/docs/index.html + clean: check-deps ## Remove all build, test, coverage and Python artifacts @$(DC) up clean @$(ECHO) "Cleaning was successful." diff --git a/docker-compose.integration.yml b/docker-compose.integration.yml new file mode 100644 index 0000000..b1cbdaf --- /dev/null +++ b/docker-compose.integration.yml @@ -0,0 +1,52 @@ +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +version: '2.2' + +services: + clean-shared: + image: alpine + command: ["/scripts/clean-shared.sh"] + volumes: + - ./integration/scripts/clean-shared.sh:/scripts/clean-shared.sh + - shared:/shared + + planetmint-all-in-one: + build: + context: . + dockerfile: Dockerfile-all-in-one + depends_on: + - clean-shared + expose: + - "22" + - "9984" + - "9985" + - "26656" + - "26657" + - "26658" + command: ["/usr/src/app/scripts/pre-config-planetmint.sh", "/usr/src/app/scripts/all-in-one.bash"] + environment: + SCALE: ${SCALE:-4} + volumes: + - ./integration/scripts:/usr/src/app/scripts + - shared:/shared + scale: ${SCALE:-4} + + test: + build: + context: . + dockerfile: integration/python/Dockerfile + depends_on: + - planetmint-all-in-one + command: ["/scripts/pre-config-test.sh", "/scripts/wait-for-planetmint.sh", "/scripts/test.sh", "pytest", "/src"] + environment: + SCALE: ${SCALE:-4} + volumes: + - ./integration/python/src:/src + - ./integration/scripts:/scripts + - shared:/shared + +volumes: + shared: diff --git a/docker-compose.yml b/docker-compose.yml index f6aff4c..39005e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,6 +89,7 @@ services: context: . dockerfile: ./integration/python/Dockerfile volumes: + - ./integration/python/docs:/docs - ./integration/python/src:/src environment: - PLANETMINT_ENDPOINT_1=https://itest1.planetmint.io diff --git a/integration/README.md b/integration/README.md index 8982e03..ba1e204 100644 --- a/integration/README.md +++ b/integration/README.md @@ -1,15 +1,23 @@ + + # Integration test suite This directory contains the integration test suite for Planetmint. -The suite uses Docker Compose to run all tests. +The suite uses Docker Compose to spin up multiple Planetmint nodes, run tests with `pytest` as well as cli tests and teardown. ## Running the tests Run `make test-integration` in the project root directory. -During development you can run single test use `pytest` inside the `python-integration` container with: +By default the integration test suite spins up four planetmint nodes. If you desire to run a different configuration you can pass `SCALE=` as an environmental variable. + +## Writing and documenting the tests +Tests are sometimes difficult to read. For integration tests, we try to be really explicit on what the test is doing, so please write code that is *simple* and easy to understand. We decided to use literate-programming documentation. To generate the documentation for python tests run: ```bash -docker-compose run --rm python-integration pytest +make docs-integration ``` - -Note: The `/src` directory contains all the test within the container. diff --git a/integration/python/.gitignore b/integration/python/.gitignore new file mode 100644 index 0000000..5c457d7 --- /dev/null +++ b/integration/python/.gitignore @@ -0,0 +1 @@ +docs \ No newline at end of file diff --git a/integration/python/Dockerfile b/integration/python/Dockerfile index 109d83f..c0e47f1 100644 --- a/integration/python/Dockerfile +++ b/integration/python/Dockerfile @@ -3,4 +3,7 @@ FROM python:3.9 RUN mkdir -p /src RUN pip install --upgrade \ pytest~=6.2.5 \ - planetmint-driver~=0.9.0 \ No newline at end of file + planetmint-driver~=0.9.0 \ + pycco + +RUN apt-get update && apt-get install -y openssh-client openssh-server \ No newline at end of file diff --git a/integration/python/src/test_basic.py b/integration/python/src/test_basic.py index 643f3d9..4932638 100644 --- a/integration/python/src/test_basic.py +++ b/integration/python/src/test_basic.py @@ -7,18 +7,21 @@ from planetmint_driver import Planetmint from planetmint_driver.crypto import generate_keypair import time -import os def test_basic(): # Setup up connection to Planetmint integration test nodes - pm_itest1_url = os.environ.get('PLANETMINT_ENDPOINT_1') - pm_itest2_url = os.environ.get('PLANETMINT_ENDPOINT_1') - pm_itest1 = Planetmint(pm_itest1_url) - pm_itest2 = Planetmint(pm_itest2_url) + hosts = [] + with open('/shared/hostnames') as f: + hosts = f.readlines() + + pm_hosts = list(map(lambda x: Planetmint(x), hosts)) + + pm_alpha = pm_hosts[0] + pm_betas = pm_hosts[1:] # genarate a keypair - alice, bob = generate_keypair(), generate_keypair() + alice = generate_keypair() # create a digital asset for Alice game_boy_token = { @@ -29,30 +32,31 @@ def test_basic(): } # prepare the transaction with the digital asset and issue 10 tokens to bob - prepared_creation_tx = pm_itest1.transactions.prepare( + prepared_creation_tx = pm_alpha.transactions.prepare( operation='CREATE', metadata={ 'hash': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', - 'storageID': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',}, + 'storageID': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', }, signers=alice.public_key, recipients=[([alice.public_key], 10)], asset=game_boy_token) # fulfill and send the transaction - fulfilled_creation_tx = pm_itest1.transactions.fulfill( + fulfilled_creation_tx = pm_alpha.transactions.fulfill( prepared_creation_tx, private_keys=alice.private_key) - pm_itest1.transactions.send_commit(fulfilled_creation_tx) - time.sleep(4) + pm_alpha.transactions.send_commit(fulfilled_creation_tx) + time.sleep(1) creation_tx_id = fulfilled_creation_tx['id'] - # retrieve transactions from both planetmint nodes - creation_tx_itest1 = pm_itest1.transactions.retrieve(creation_tx_id) - creation_tx_itest2 = pm_itest2.transactions.retrieve(creation_tx_id) + # retrieve transactions from all planetmint nodes + creation_tx_alpha = pm_alpha.transactions.retrieve(creation_tx_id) + creation_tx_betas = list(map(lambda beta: beta.transactions.retrieve(creation_tx_id), pm_betas)) - # Assert that transaction is stored on both planetmint nodes - assert creation_tx_itest1 == creation_tx_itest2 + # Assert that transaction is stored on all planetmint nodes + for tx in creation_tx_betas: + assert creation_tx_alpha == tx # Transfer # create the output and inout for the transaction @@ -65,35 +69,28 @@ def test_basic(): 'owners_before': output['public_keys']} # prepare the transaction and use 3 tokens - prepared_transfer_tx = pm_itest1.transactions.prepare( + prepared_transfer_tx = pm_alpha.transactions.prepare( operation='TRANSFER', asset=transfer_asset, inputs=transfer_input, metadata={'hash': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', - 'storageID': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', }, + 'storageID': '0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', }, recipients=[([alice.public_key], 10)]) # fulfill and send the transaction - fulfilled_transfer_tx = pm_itest1.transactions.fulfill( + fulfilled_transfer_tx = pm_alpha.transactions.fulfill( prepared_transfer_tx, private_keys=alice.private_key) - sent_transfer_tx = pm_itest1.transactions.send_commit(fulfilled_transfer_tx) + sent_transfer_tx = pm_alpha.transactions.send_commit(fulfilled_transfer_tx) - transfer_tx_id = fulfilled_transfer_tx['id'] + time.sleep(1) + + transfer_tx_id = sent_transfer_tx['id'] # retrieve transactions from both planetmint nodes - transfer_tx_itest1 = pm_itest1.transactions.retrieve(transfer_tx_id) - transfer_tx_itest2 = pm_itest2.transactions.retrieve(transfer_tx_id) + transfer_tx_alpha = pm_alpha.transactions.retrieve(transfer_tx_id) + transfer_tx_betas = list(map(lambda beta: beta.transactions.retrieve(transfer_tx_id), pm_betas)) # Assert that transaction is stored on both planetmint nodes - assert transfer_tx_itest1 == transfer_tx_itest2 - - - - - - - - - - + for tx in transfer_tx_betas: + assert transfer_tx_alpha == tx diff --git a/integration/python/src/test_multisig.py b/integration/python/src/test_multisig.py index 976fc90..94ce9dc 100644 --- a/integration/python/src/test_multisig.py +++ b/integration/python/src/test_multisig.py @@ -18,19 +18,24 @@ # # This integration test is a rip-off of our mutliple signature acceptance tests. -# ## Imports -# We need some utils from the `os` package, we will interact with -# env variables. -import os +# # Imports +import time # For this test case we need import and use the Python driver from planetmint_driver import Planetmint from planetmint_driver.crypto import generate_keypair + def test_multiple_owners(): - # ## Set up a connection to the Planetmint integration test nodes - pm_itest1 = Planetmint(os.environ.get('PLANETMINT_ENDPOINT_1')) - pm_itest2 = Planetmint(os.environ.get('PLANETMINT_ENDPOINT_2')) + # Setup up connection to Planetmint integration test nodes + hosts = [] + with open('/shared/hostnames') as f: + hosts = f.readlines() + + pm_hosts = list(map(lambda x: Planetmint(x), hosts)) + + pm_alpha = pm_hosts[0] + pm_betas = pm_hosts[1:] # Generate Keypairs for Alice and Bob! alice, bob = generate_keypair(), generate_keypair() @@ -50,7 +55,7 @@ def test_multiple_owners(): # They prepare a `CREATE` transaction. To have multiple owners, both # Bob and Alice need to be the recipients. - prepared_dw_tx = pm_itest1.transactions.prepare( + prepared_dw_tx = pm_alpha.transactions.prepare( operation='CREATE', signers=alice.public_key, recipients=(alice.public_key, bob.public_key), @@ -58,29 +63,31 @@ def test_multiple_owners(): # Now they both sign the transaction by providing their private keys. # And send it afterwards. - fulfilled_dw_tx = pm_itest1.transactions.fulfill( + fulfilled_dw_tx = pm_alpha.transactions.fulfill( prepared_dw_tx, private_keys=[alice.private_key, bob.private_key]) - pm_itest1.transactions.send_commit(fulfilled_dw_tx) + pm_alpha.transactions.send_commit(fulfilled_dw_tx) # We store the `id` of the transaction to use it later on. dw_id = fulfilled_dw_tx['id'] + time.sleep(1) # Let's retrieve the transaction from both nodes - pm_itest1_tx = pm_itest1.transactions.retrieve(dw_id) - pm_itest2_tx = pm_itest2.transactions.retrieve(dw_id) + pm_alpha_tx = pm_alpha.transactions.retrieve(dw_id) + pm_betas_tx = list(map(lambda beta: beta.transactions.retrieve(dw_id), pm_betas)) # Both retrieved transactions should be the same - assert pm_itest1_tx == pm_itest2_tx + for tx in pm_betas_tx: + assert pm_alpha_tx == tx # Let's check if the transaction was successful. - assert pm_itest1.transactions.retrieve(dw_id), \ + assert pm_alpha.transactions.retrieve(dw_id), \ 'Cannot find transaction {}'.format(dw_id) # The transaction should have two public keys in the outputs. assert len( - pm_itest1.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2 + pm_alpha.transactions.retrieve(dw_id)['outputs'][0]['public_keys']) == 2 # ## Alice and Bob transfer a transaction to Carol. # Alice and Bob save a lot of money living together. They often go out @@ -103,36 +110,37 @@ def test_multiple_owners(): 'owners_before': output['public_keys']} # Now they create the transaction... - prepared_transfer_tx = pm_itest1.transactions.prepare( + prepared_transfer_tx = pm_alpha.transactions.prepare( operation='TRANSFER', asset=transfer_asset, inputs=transfer_input, recipients=carol.public_key) # ... and sign it with their private keys, then send it. - fulfilled_transfer_tx = pm_itest1.transactions.fulfill( + fulfilled_transfer_tx = pm_alpha.transactions.fulfill( prepared_transfer_tx, private_keys=[alice.private_key, bob.private_key]) - sent_transfer_tx = pm_itest1.transactions.send_commit(fulfilled_transfer_tx) + sent_transfer_tx = pm_alpha.transactions.send_commit(fulfilled_transfer_tx) + time.sleep(1) # Retrieve the fulfilled transaction from both nodes - pm_itest1_tx = pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id']) - pm_itest2_tx = pm_itest2.transactions.retrieve(fulfilled_transfer_tx['id']) + pm_alpha_tx = pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id']) + pm_betas_tx = list(map(lambda beta: beta.transactions.retrieve(fulfilled_transfer_tx['id']), pm_betas)) # Now compare if both nodes returned the same transaction - assert pm_itest1_tx == pm_itest2_tx + for tx in pm_betas_tx: + assert pm_alpha_tx == tx # They check if the transaction was successful. - assert pm_itest1.transactions.retrieve( + assert pm_alpha.transactions.retrieve( fulfilled_transfer_tx['id']) == sent_transfer_tx # The owners before should include both Alice and Bob. assert len( - pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][ + pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])['inputs'][0][ 'owners_before']) == 2 # While the new owner is Carol. - assert pm_itest1.transactions.retrieve(fulfilled_transfer_tx['id'])[ + assert pm_alpha.transactions.retrieve(fulfilled_transfer_tx['id'])[ 'outputs'][0]['public_keys'][0] == carol.public_key - \ No newline at end of file diff --git a/integration/scripts/all-in-one.bash b/integration/scripts/all-in-one.bash new file mode 100755 index 0000000..e719587 --- /dev/null +++ b/integration/scripts/all-in-one.bash @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + + +# MongoDB configuration +[ "$(stat -c %U /data/db)" = mongodb ] || chown -R mongodb /data/db + +# Planetmint configuration +/usr/src/app/scripts/planetmint-monit-config + +nohup mongod --bind_ip_all > "$HOME/.planetmint-monit/logs/mongodb_log_$(date +%Y%m%d_%H%M%S)" 2>&1 & + +# Start services +monit -d 5 -I -B \ No newline at end of file diff --git a/integration/scripts/clean-shared.sh b/integration/scripts/clean-shared.sh new file mode 100755 index 0000000..7ba481e --- /dev/null +++ b/integration/scripts/clean-shared.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +rm /shared/hostnames +rm /shared/lock +rm /shared/*node_id +rm /shared/*.json +rm /shared/id_rsa.pub \ No newline at end of file diff --git a/integration/scripts/election.sh b/integration/scripts/election.sh new file mode 100755 index 0000000..d55abc7 --- /dev/null +++ b/integration/scripts/election.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Show tendermint node id +show_id () { + tendermint --home=/tendermint show_node_id | tail -n 1 +} + +# Show validator public key +show_validator () { + tendermint --home=/tendermint show_validator | tail -n 1 +} + +# Elect new voting power for node +elect_validator () { + planetmint election new upsert-validator $1 $2 $3 --private-key /tendermint/config/priv_validator_key.json +} + +# Show election state +show_election () { + planetmint election show $1 +} + +# Approve election +approve_validator () { + planetmint election approve $1 --private-key /tendermint/config/priv_validator_key.json +} + +# Fetch tendermint id and pubkey and create upsert proposal +elect () { + node_id=$(show_id) + validator_pubkey=$(show_validator | jq -r .value) + proposal=$(elect_validator $validator_pubkey $1 $node_id 2>&1 | grep SUCCESS) + echo ${proposal##* } +} + +usage () { + echo "usage: TODO" +} + +while [ "$1" != "" ]; do + case $1 in + show_id ) show_id + ;; + show_validator ) show_validator + ;; + elect ) shift + elect $1 + ;; + show_election ) shift + show_election $1 + ;; + approve ) shift + approve_validator $1 + ;; + * ) usage + exit 1 + esac + shift +done + +exitcode=$? + +exit $exitcode \ No newline at end of file diff --git a/integration/scripts/genesis.py b/integration/scripts/genesis.py new file mode 100755 index 0000000..3593f34 --- /dev/null +++ b/integration/scripts/genesis.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +import json +import sys + + +def edit_genesis() -> None: + file_names = sys.argv[1:] + + validators = [] + for file_name in file_names: + file = open(file_name) + genesis = json.load(file) + validators.extend(genesis['validators']) + file.close() + + genesis_file = open(file_names[0]) + genesis_json = json.load(genesis_file) + genesis_json['validators'] = validators + genesis_file.close() + + with open('/shared/genesis.json', 'w') as f: + json.dump(genesis_json, f, indent=True) + + return None + + +if __name__ == '__main__': + edit_genesis() diff --git a/integration/scripts/planetmint-monit-config b/integration/scripts/planetmint-monit-config new file mode 100755 index 0000000..82af9af --- /dev/null +++ b/integration/scripts/planetmint-monit-config @@ -0,0 +1,208 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +set -o nounset + +# Check if directory for monit logs exists +if [ ! -d "$HOME/.planetmint-monit" ]; then + mkdir -p "$HOME/.planetmint-monit" +fi + +monit_pid_path=${MONIT_PID_PATH:=$HOME/.planetmint-monit/monit_processes} +monit_script_path=${MONIT_SCRIPT_PATH:=$HOME/.planetmint-monit/monit_script} +monit_log_path=${MONIT_LOG_PATH:=$HOME/.planetmint-monit/logs} +monitrc_path=${MONITRC_PATH:=$HOME/.monitrc} + +function usage() { + cat <${monit_script_path} < /dev/null 2>&1 & + + echo \$! > \$2 + popd + + ;; + + stop_planetmint) + + kill -2 \`cat \$2\` + rm -f \$2 + + ;; + + start_tendermint) + + pushd \$4 + + nohup tendermint node \ + --p2p.laddr "tcp://0.0.0.0:26656" \ + --rpc.laddr "tcp://0.0.0.0:26657" \ + --proxy_app="tcp://0.0.0.0:26658" \ + --consensus.create_empty_blocks=false \ + --p2p.pex=false >> \$3/tendermint.out.log 2>> \$3/tendermint.err.log & + + echo \$! > \$2 + popd + + ;; + + stop_tendermint) + + kill -2 \`cat \$2\` + rm -f \$2 + + ;; + +esac +exit 0 +EOF +chmod +x ${monit_script_path} + +cat >${monit_script_path}_logrotate <${monitrc_path} < 200 MB then + exec "${monit_script_path}_logrotate rotate_tendermint_logs ${monit_log_path}/tendermint.out.log $monit_pid_path/tendermint.pid" + +check file tendermint.err.log with path ${monit_log_path}/tendermint.err.log + if size > 200 MB then + exec "${monit_script_path}_logrotate rotate_tendermint_logs ${monit_log_path}/tendermint.err.log $monit_pid_path/tendermint.pid" + +EOF + +# Setting permissions for control file +chmod 0700 ${monitrc_path} + +echo -e "Planetmint process manager configured!" +set -o errexit diff --git a/integration/scripts/pre-config-planetmint.sh b/integration/scripts/pre-config-planetmint.sh new file mode 100755 index 0000000..ea15ea7 --- /dev/null +++ b/integration/scripts/pre-config-planetmint.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Write hostname to list +echo $(hostname) >> /shared/hostnames + +# Create ssh folder +mkdir ~/.ssh + +# Wait for test container pubkey +while [ ! -f /shared/id_rsa.pub ]; do + echo "WAIT FOR PUBKEY" + sleep 1 +done + +# Add pubkey to authorized keys +cat /shared/id_rsa.pub > ~/.ssh/authorized_keys + +# Allow root user login +sed -i "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config + +# Restart ssh service +service ssh restart + +# Tendermint configuration +tendermint init + +# Write node id to shared folder +HOSTNAME=$(hostname) +NODE_ID=$(tendermint show_node_id | tail -n 1) +echo $NODE_ID > /shared/${HOSTNAME}_node_id + +# Wait for other node ids +FILES=() +while [ ! ${#FILES[@]} == $SCALE ]; do + echo "WAIT FOR NODE IDS" + sleep 1 + FILES=(/shared/*node_id) +done + +# Write node ids to persistent peers +PEERS="persistent_peers = \"" +for f in ${FILES[@]}; do + ID=$(cat $f) + HOST=$(echo $f | cut -c 9-20) + if [ ! $HOST == $HOSTNAME ]; then + PEERS+="${ID}@${HOST}:26656, " + fi +done +PEERS=$(echo $PEERS | rev | cut -c 2- | rev) +PEERS+="\"" +sed -i "/persistent_peers = \"\"/c\\${PEERS}" /tendermint/config/config.toml + +# Copy genesis.json to shared folder +cp /tendermint/config/genesis.json /shared/${HOSTNAME}_genesis.json + +# Await config file of all services to be present +FILES=() +while [ ! ${#FILES[@]} == $SCALE ]; do + echo "WAIT FOR GENESIS FILES" + sleep 1 + FILES=(/shared/*_genesis.json) +done + +# Create genesis.json for nodes +if [ ! -f /shared/lock ]; then + echo LOCKING + touch /shared/lock + /usr/src/app/scripts/genesis.py ${FILES[@]} +fi + +while [ ! -f /shared/genesis.json ]; do + echo "WAIT FOR GENESIS" + sleep 1 +done + +# Copy genesis.json to tendermint config +cp /shared/genesis.json /tendermint/config/genesis.json + +exec "$@" \ No newline at end of file diff --git a/integration/scripts/pre-config-test.sh b/integration/scripts/pre-config-test.sh new file mode 100755 index 0000000..bd72913 --- /dev/null +++ b/integration/scripts/pre-config-test.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Create ssh folder +mkdir ~/.ssh + +# Create ssh keys +ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa + +# Publish pubkey to shared folder +cp ~/.ssh/id_rsa.pub /shared + +exec "$@" \ No newline at end of file diff --git a/integration/scripts/test.sh b/integration/scripts/test.sh new file mode 100755 index 0000000..de9271d --- /dev/null +++ b/integration/scripts/test.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Read host names from shared +readarray -t HOSTNAMES < /shared/hostnames + +# Split into proposer and approvers +ALPHA=${HOSTNAMES[0]} +BETAS=${HOSTNAMES[@]:1} + +# Propose validator upsert +result=$(ssh -o "StrictHostKeyChecking=no" -i \~/.ssh/id_rsa root@${ALPHA} 'bash -s' < scripts/election.sh elect 2) + +# Approve validator upsert +for BETA in ${BETAS[@]}; do + ssh -o "StrictHostKeyChecking=no" -i ~/.ssh/id_rsa root@${BETA} 'bash -s' < scripts/election.sh approve $result +done + +exec "$@" \ No newline at end of file diff --git a/integration/scripts/wait-for-planetmint.sh b/integration/scripts/wait-for-planetmint.sh new file mode 100755 index 0000000..36c7794 --- /dev/null +++ b/integration/scripts/wait-for-planetmint.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +# Only continue if all services are ready +HOSTNAMES=() +while [ ! ${#HOSTNAMES[@]} == $SCALE ]; do + echo "WAIT FOR HOSTNAMES" + sleep 1 + readarray -t HOSTNAMES < /shared/hostnames +done + +for host in ${HOSTNAMES[@]}; do + while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $host:9984)" != "200" ]]; do + echo "WAIT FOR PLANETMINT $host" + sleep 1 + done +done + +for host in ${HOSTNAMES[@]}; do + while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $host:26657)" != "200" ]]; do + echo "WAIT FOR TENDERMINT $host" + sleep 1 + done +done + +exec "$@" \ No newline at end of file diff --git a/pkg/scripts/all-in-one.bash b/pkg/scripts/all-in-one.bash index 1ccd68d..1ec4371 100755 --- a/pkg/scripts/all-in-one.bash +++ b/pkg/scripts/all-in-one.bash @@ -16,4 +16,5 @@ nohup mongod --bind_ip_all > "$HOME/.planetmint-monit/logs/mongodb_log_$(date +% # Tendermint configuration tendermint init -monit -d 5 -I -B +# Start services +monit -d 5 -I -B \ No newline at end of file diff --git a/run-integration-test.sh b/run-integration-test.sh deleted file mode 100755 index 6c0d3be..0000000 --- a/run-integration-test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2020 Interplanetary Database Association e.V., -# Planetmint and IPDB software contributors. -# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) -# Code is Apache-2.0 and docs are CC-BY-4.0 - - -# Check if both integration test nodes are reachable -check_status () { - OK="200 OK" - - STATUS_1=$(curl -I -s -X GET https://itest1.planetmint.io/ | head -n 1) - STATUS_2=$(curl -I -s -X GET https://itest2.planetmint.io/ | head -n 1) - - # Check if both response status codes return 200 OK - if ! [[ "$STATUS_1" == *"$OK"* ]] || ! [[ "$STATUS_2" == *"$OK"* ]] - then - exit 1 - fi -} - -run_test () { - docker-compose run --rm python-integration pytest /src -} - -teardown () { - docker-compose down -} - -check_status -run_test -exitcode=$? -teardown - -exit $exitcode \ No newline at end of file diff --git a/run-acceptance-test.sh b/scripts/run-acceptance-test.sh similarity index 100% rename from run-acceptance-test.sh rename to scripts/run-acceptance-test.sh diff --git a/scripts/run-integration-test.sh b/scripts/run-integration-test.sh new file mode 100755 index 0000000..1ec46d4 --- /dev/null +++ b/scripts/run-integration-test.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright © 2020 Interplanetary Database Association e.V., +# Planetmint and IPDB software contributors. +# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) +# Code is Apache-2.0 and docs are CC-BY-4.0 + +run_test() { + docker-compose -f docker-compose.integration.yml up test +} + +teardown () { + docker-compose -f docker-compose.integration.yml down +} + +run_test +exitcode=$? +teardown + +exit $exitcode \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index 9888650..0000000 --- a/scripts/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Copyright © 2020 Interplanetary Database Association e.V., -# Planetmint and IPDB software contributors. -# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0) -# Code is Apache-2.0 and docs are CC-BY-4.0 - -result=$(ssh -o StrictHostKeyChecking=accept-new root@64.225.106.52 -i id_ed25519 'bash -s' < scripts/election.sh elect 35) -ssh -o StrictHostKeyChecking=accept-new root@64.225.105.60 -i id_ed25519 'bash -s' < scripts/election.sh approve $result \ No newline at end of file diff --git a/secrets/id_ed25519.enc b/secrets/id_ed25519.enc deleted file mode 100644 index b2423ab..0000000 --- a/secrets/id_ed25519.enc +++ /dev/null @@ -1,4 +0,0 @@ -.9DD@ڌJᖩK c^ -Y/T |%&vf#"_P#S4iQSn04?vxN! 5;i WHP"x59"]i"4ID8 ̋tuea;ԩ`Ջ1ެFXPE6ϊ\"$eO <PۡS&;i2V 3/JD+;TL(tR9vE-+!Zhf5ߤ8S.G4nMQ!_z`UUL@ ͡wV3@ų?)'>>KCx%);8 -nŽ5vj],20ȍ; m(ms= -vK)5TQ_ LD:h+ \ No newline at end of file