Problem: we don't have acceptance tests (#2216)

Solution: have a simple way to start a node and run scripts against it.

.ci/travis-install.sh

@@ -11,6 +11,8 @@ elif [[ ${BIGCHAINDB_CI_ABCI} == 'enable' ]]; then
 elif [[ $TRAVIS_PYTHON_VERSION == 3.5 ]]; then
     docker-compose build --build-arg python_version=3.5 --no-cache bigchaindb
     pip install --upgrade codecov
+elif [[ $BIGCHAINDB_INTEGRATION_TEST == 'enable' ]]; then
+    docker-compose build bigchaindb python-driver
 else
     docker-compose build --no-cache bigchaindb
     pip install --upgrade codecov

.ci/travis_script.sh

@@ -6,6 +6,8 @@ if [[ -n ${TOXENV} ]]; then
   tox -e ${TOXENV}
 elif [[ ${BIGCHAINDB_CI_ABCI} == 'enable' ]]; then
   docker-compose exec bigchaindb pytest -v -m abci
+elif [[ ${BIGCHAINDB_ACCEPTANCE_TEST} == 'enable' ]]; then
+    ./run-acceptance-test.sh
 else
   docker-compose exec bigchaindb pytest -v --cov=bigchaindb
 fi

.travis.yml

@@ -43,6 +43,8 @@ matrix:
     - BIGCHAINDB_DATABASE_BACKEND=localmongodb
     - BIGCHAINDB_DATABASE_SSL=
     - BIGCHAINDB_CI_ABCI=enable
+  - env:
+    - BIGCHAINDB_ACCEPTANCE_TEST=enable
 
 before_install: sudo .ci/travis-before-install.sh
 

Makefile

@@ -72,7 +72,14 @@ test: check-deps ## Run all tests once
 	@$(DC) exec bigchaindb pytest
 
 test-watch: check-deps ## Run all tests and wait. Every time you change code, tests will be run again
-	@$(DC) run --rm bigchaindb pytest -f -v
+	@$(DC) run --rm --no-deps bigchaindb pytest -f
+
+acceptance-test: check-deps ## Run all acceptance tests
+	@./run-acceptance-test.sh
+
+acceptance-test-doc: check-deps ## Create documentation for acceptance tests
+	@$(DC) run --rm python-acceptance pycco -i -s /src -d /docs
+	$(BROWSER) acceptance/python/docs/index.html
 
 cov: check-deps ## Check code coverage and open the result in the browser
 	@$(DC) run --rm bigchaindb pytest -v --cov=bigchaindb --cov-report html

acceptance/python/.gitignore

@@ -0,0 +1 @@
+docs

acceptance/python/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.6.3
+
+RUN mkdir -p /src
+RUN pip install --upgrade \
+	pycco \
+	websocket-client~=0.47.0 \
+	pytest~=3.0 \
+	bigchaindb-driver==0.5.0a2

acceptance/python/src/test_basic.py

@@ -0,0 +1,123 @@
+# # Basic Acceptance Test
+# Here we check that the primitives of the system behave as expected.
+# As you will see, this script tests basic stuff like:
+#
+# - create a transaction
+# - check if the transaction is stored
+# - check for the outputs of a given public key
+# - transfer the transaction to another key
+#
+# We run a series of checks for each steps, that is retrieving the transaction from
+# the remote system, and also checking the `outputs` of a given public key.
+#
+# This acceptance test is a rip-off of our
+# [tutorial](https://docs.bigchaindb.com/projects/py-driver/en/latest/usage.html).
+
+# ## Imports
+# We need some utils from the `os` package, we will interact with
+# env variables.
+import os
+
+# For this test case we import and use the Python Driver.
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_basic():
+    # ## Set up a connection to BigchainDB
+    # To use BighainDB we need a connection. Here we create one. By default we
+    # connect to localhost, but you can override this value using the env variable
+    # called `BIGCHAINDB_ENDPOINT`, a valid value must include the schema:
+    # `https://example.com:9984`
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+
+    # ## Create keypairs
+    # This test requires the interaction between two actors with their own keypair.
+    # The two keypairs will be called—drum roll—Alice and Bob.
+    alice, bob = generate_keypair(), generate_keypair()
+
+
+    # ## Alice registers her bike in BigchainDB
+    # Alice has a nice bike, and here she creates the "digital twin"
+    # of her bike.
+    bike = {'data': {'bicycle': {'serial_number': 420420}}}
+
+    # She prepares a `CREATE` transaction...
+    prepared_creation_tx = bdb.transactions.prepare(
+            operation='CREATE',
+            signers=alice.public_key,
+            asset=bike)
+
+    # ... and she fulfills it with her private key.
+    fulfilled_creation_tx = bdb.transactions.fulfill(
+            prepared_creation_tx,
+            private_keys=alice.private_key)
+
+    # We will use the `id` of this transaction several time, so we store it in
+    # a variable with a short and easy name
+    bike_id = fulfilled_creation_tx['id']
+
+
+    # Now she is ready to send it to the BigchainDB Network.
+    sent_transfer_tx = bdb.transactions.send(fulfilled_creation_tx, mode='commit')
+
+    # And just to be 100% sure, she also checks if she can retrieve
+    # it from the BigchainDB node.
+    assert bdb.transactions.retrieve(bike_id), 'Cannot find transaction {}'.format(bike_id)
+
+    # Alice is now the proud owner of one unspent asset.
+    assert len(bdb.outputs.get(alice.public_key, spent=False)) == 1
+    assert bdb.outputs.get(alice.public_key)[0]['transaction_id'] == bike_id
+
+
+    # ## Alice transfers her bike to Bob
+    # After registering her bike, Alice is ready to transfer it to Bob.
+    # She needs to create a new `TRANSFER` transaction.
+
+    # A `TRANSFER` transaction contains a pointer to the original asset. The original asset
+    # is identified by the `id` of the `CREATE` transaction that defined it.
+    transfer_asset = {'id': bike_id}
+
+    # Alice wants to spend the one and only output available, the one with index `0`.
+    output_index = 0
+    output = fulfilled_creation_tx['outputs'][output_index]
+
+    # Here, she defines the `input` of the `TRANSFER` transaction. The `input` contains
+    # several keys:
+    #
+    # - `fulfillment`, taken from the previous `CREATE` transaction.
+    # - `fulfills`, that specifies which condition she is fulfilling.
+    # - `owners_before`.
+    transfer_input = {'fulfillment': output['condition']['details'],
+                      'fulfills': {'output_index': output_index,
+                                   'transaction_id': fulfilled_creation_tx['id']},
+                      'owners_before': output['public_keys']}
+
+    # Now that all the elements are set, she creates the actual transaction...
+    prepared_transfer_tx = bdb.transactions.prepare(
+            operation='TRANSFER',
+            asset=transfer_asset,
+            inputs=transfer_input,
+            recipients=bob.public_key)
+
+    # ... and signs it with her private key.
+    fulfilled_transfer_tx = bdb.transactions.fulfill(
+            prepared_transfer_tx,
+            private_keys=alice.private_key)
+
+    # She finally sends the transaction to a BigchainDB node.
+    sent_transfer_tx = bdb.transactions.send(fulfilled_transfer_tx, mode='commit')
+
+    # And just to be 100% sure, she also checks if she can retrieve
+    # it from the BigchainDB node.
+    assert bdb.transactions.retrieve(fulfilled_transfer_tx['id']) == sent_transfer_tx
+
+    # Now Alice has zero unspent transactions.
+    assert len(bdb.outputs.get(alice.public_key, spent=False)) == 0
+
+    # While Bob has one.
+    assert len(bdb.outputs.get(bob.public_key, spent=False)) == 1
+
+    # Bob double checks what he got was the actual bike.
+    bob_tx_id = bdb.outputs.get(bob.public_key, spent=False)[0]['transaction_id']
+    assert bdb.transactions.retrieve(bob_tx_id) == sent_transfer_tx

acceptance/python/src/test_double_spend.py

@@ -0,0 +1,43 @@
+# # Double Spend testing
+# This test challenge the system with double spends.
+
+import os
+from uuid import uuid4
+from threading import Thread
+import queue
+
+import bigchaindb_driver.exceptions
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_double_create():
+    bdb = BigchainDB(os.environ.get('BIGCHAINDB_ENDPOINT'))
+    alice = generate_keypair()
+
+    results = queue.Queue()
+
+    tx = bdb.transactions.fulfill(
+            bdb.transactions.prepare(
+                operation='CREATE',
+                signers=alice.public_key,
+                asset={'data': {'uuid': str(uuid4())}}),
+            private_keys=alice.private_key)
+
+    def send_and_queue(tx):
+        try:
+            bdb.transactions.send(tx, mode='commit')
+            results.put('OK')
+        except bigchaindb_driver.exceptions.TransportError as e:
+            results.put('FAIL')
+
+    t1 = Thread(target=send_and_queue, args=(tx, ))
+    t2 = Thread(target=send_and_queue, args=(tx, ))
+
+    t1.start()
+    t2.start()
+
+    results = [results.get(timeout=2), results.get(timeout=2)]
+
+    assert results.count('OK') == 1
+    assert results.count('FAIL') == 1

acceptance/python/src/test_stream.py

@@ -0,0 +1,127 @@
+# # Stream Acceptance Test
+# This test checks if the event stream works correctly. The basic idea of this
+# test is to generate some random **valid** transaction, send them to a
+# BigchainDB node, and expect those transactions to be returned by the valid
+# transactions Stream API. During this test, two threads work together,
+# sharing a queue to exchange events.
+#
+# - The *main thread* first creates and sends the transactions to BigchainDB;
+# then it run through all events in the shared queue to check if all
+# transactions sent have been validated by BigchainDB.
+# - The *listen thread* listens to the events coming from BigchainDB and puts
+# them in a queue shared with the main thread.
+
+import os
+import queue
+import json
+from threading import Thread, Event
+from uuid import uuid4
+
+# For this script, we need to set up a websocket connection, that's the reason
+# we import the
+# [websocket](https://github.com/websocket-client/websocket-client) module
+from websocket import create_connection
+
+from bigchaindb_driver import BigchainDB
+from bigchaindb_driver.crypto import generate_keypair
+
+
+def test_stream():
+    # ## Set up the test
+    # We use the env variable `BICHAINDB_ENDPOINT` to know where to connect.
+    # Check [test_basic.py](./test_basic.html) for more information.
+    BDB_ENDPOINT = os.environ.get('BIGCHAINDB_ENDPOINT')
+
+    # *That's pretty bad, but let's do like this for now.*
+    WS_ENDPOINT = 'ws://{}:9985/api/v1/streams/valid_transactions'.format(BDB_ENDPOINT.rsplit(':')[0])
+
+    bdb = BigchainDB(BDB_ENDPOINT)
+
+    # Hello to Alice again, she is pretty active in those tests, good job
+    # Alice!
+    alice = generate_keypair()
+
+    # We need few variables to keep the state, specifically we need `sent` to
+    # keep track of all transactions Alice sent to BigchainDB, while `received`
+    # are the transactions BigchainDB validated and sent back to her.
+    sent = []
+    received = queue.Queue()
+
+    # In this test we use a websocket. The websocket must be started **before**
+    # sending transactions to BigchainDB, otherwise we might lose some
+    # transactions. The `ws_ready` event is used to synchronize the main thread
+    # with the listen thread.
+    ws_ready = Event()
+
+    # ## Listening to events
+    # This is the function run by the complementary thread.
+    def listen():
+        # First we connect to the remote endpoint using the WebSocket protocol.
+        ws = create_connection(WS_ENDPOINT)
+
+        # After the connection has been set up, we can signal the main thread
+        # to proceed (continue reading, it should make sense in a second.)
+        ws_ready.set()
+
+        # It's time to consume all events coming from the BigchainDB stream API.
+        # Every time a new event is received, it is put in the queue shared
+        # with the main thread.
+        while True:
+            result = ws.recv()
+            received.put(result)
+
+    # Put `listen` in a thread, and start it. Note that `listen` is a local
+    # function and it can access all variables in the enclosing function.
+    t = Thread(target=listen, daemon=True)
+    t.start()
+
+    # ## Pushing the transactions to BigchainDB
+    # After starting the listen thread, we wait for it to connect, and then we
+    # proceed.
+    ws_ready.wait()
+
+    # Here we prepare, sign, and send ten different `CREATE` transactions. To
+    # make sure each transaction is different from the other, we generate a
+    # random `uuid`.
+    for _ in range(10):
+        tx = bdb.transactions.fulfill(
+                bdb.transactions.prepare(
+                    operation='CREATE',
+                    signers=alice.public_key,
+                    asset={'data': {'uuid': str(uuid4())}}),
+                private_keys=alice.private_key)
+        # We don't want to wait for each transaction to be in a block. By using
+        # `async` mode, we make sure that the driver returns as soon as the
+        # transaction is pushed to the BigchainDB API. Remember: we expect all
+        # transactions to be in the shared queue: this is a two phase test,
+        # first we send a bunch of transactions, then we check if they are
+        # valid (and, in this case, they should).
+        bdb.transactions.send(tx, mode='async')
+
+        # The `id` of every sent transaction is then stored in a list.
+        sent.append(tx['id'])
+
+    # ## Check the valid transactions coming from BigchainDB
+    # Now we are ready to check if BigchainDB did its job. A simple way to
+    # check if all sent transactions have been processed is to **remove** from
+    # `sent` the transactions we get from the *listen thread*. At one point in
+    # time, `sent` should be empty, and we exit the test.
+    while sent:
+        # To avoid waiting forever, we have an arbitrary timeout of 5
+        # seconds: it should be enough time for BigchainDB to create
+        # blocks, in fact a new block is created every second. If we hit
+        # the timeout, then game over ¯\\\_(ツ)\_/¯
+        try:
+            event = received.get(timeout=5)
+            txid = json.loads(event)['transaction_id']
+        except queue.Empty:
+            assert False, 'Did not receive all expected transactions'
+
+        # Last thing is to try to remove the `txid` from the set of sent
+        # transactions. If this test is running in parallel with others, we
+        # might get a transaction id of another test, and `remove` can fail.
+        # It's OK if this happens.
+        try:
+            sent.remove(txid)
+        except ValueError:
+            pass

bigchaindb/config_utils.py

@@ -128,6 +128,7 @@ def env_config(config):
 
     def load_from_env(value, path):
         var_name = CONFIG_SEP.join([CONFIG_PREFIX] + list(map(lambda s: s.upper(), path)))
+
         return os.environ.get(var_name, value)
 
     return map_leafs(load_from_env, config)

docker-compose.yml

@@ -32,6 +32,7 @@ services:
       BIGCHAINDB_DATABASE_PORT: 27017
       BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984
       BIGCHAINDB_WSSERVER_HOST: 0.0.0.0
+      BIGCHAINDB_WSSERVER_ADVERTISED_HOST: bigchaindb
       BIGCHAINDB_TENDERMINT_HOST: tendermint
       BIGCHAINDB_TENDERMINT_PORT: 46657
     ports:
@@ -43,7 +44,7 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-    entrypoint: '.ci/entrypoint.sh'
+    command: '.ci/entrypoint.sh'
   tendermint:
     image: tendermint/tendermint:0.12
     volumes:
@@ -63,13 +64,18 @@ services:
   # curl client to check the health of development env
   curl-client:
     image: appropriate/curl
-    command: /bin/sh -c "curl http://tendermint:46657/abci_query && curl http://bigchaindb:9984/"
+    command: /bin/sh -c "curl -s http://bigchaindb:9984/ > /dev/null && curl -s http://tendermint:46657/ > /dev/null"
 
-  # Python BigchainDB driver to interact with the BigchainDB server
-  bdb-driver:
+  # BigchainDB setup to do acceptance testing with Python
+  python-acceptance:
     build:
       context: .
-      dockerfile: ./tools/py-bigchaindb-driver/Dockerfile
+      dockerfile: ./acceptance/python/Dockerfile
+    volumes:
+      - ./acceptance/python/docs:/docs
+      - ./acceptance/python/src:/src
+    environment:
+      - BIGCHAINDB_ENDPOINT=bigchaindb
 
   # Build docs only
   # docker-compose build bdocs

run-acceptance-test.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Set up a BigchainDB node and return only when we are able to connect to both
+# the BigchainDB container *and* the Tendermint container.
+setup () {
+	docker-compose up -d bigchaindb
+
+	# Try to connect to the containers for maximum three times, and wait
+	# one second between tries.
+	for i in $(seq 3); do
+		if $(docker-compose run --rm curl-client); then
+			break
+		else
+			sleep 1
+		fi
+	done
+}
+
+run_test () {
+	docker-compose run --rm python-acceptance pytest /src
+}
+
+teardown () {
+	docker-compose down
+}
+
+setup
+run_test
+exitcode=$?
+teardown
+
+exit $exitcode

tests/commands/test_commands.py

@@ -1,4 +1,5 @@
 import json
+
 from unittest.mock import Mock, patch
 from argparse import Namespace
 
@@ -188,6 +189,7 @@ def test_run_configure_when_config_does_exist(monkeypatch,
     assert value == {}
 
 
+@pytest.mark.skip
 @pytest.mark.tendermint
 @pytest.mark.parametrize('backend', (
     'localmongodb',

tools/py-bigchaindb-driver/Dockerfile

@@ -1,8 +0,0 @@
-FROM python:3.6.3
-
-RUN apt-get update && apt-get install -y vim
-
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
-
-RUN pip install --upgrade pip ipython bigchaindb-driver