Docker compose for travis (#2039)

* Replace double quotes with single quotes (flake8)

* Test event subscriber to tendermint via ws

* Problem: Tendermint is not part of stack for CI

Solution: Add Tendermint to stack for CI. For simplicity's sake
docker-compose is being used.

.ci/travis-after-success.sh

@@ -3,5 +3,5 @@
 set -e -x
 
 if [[ -z ${TOXENV} ]]; then
-    codecov
+    codecov -v
 fi

.ci/travis-before-install.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 
-apt-get update -qq
-wget https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz -O /tmp/v1.1.0.tar.gz
-tar -xvf /tmp/v1.1.0.tar.gz
-cp -r $PWD/rapidjson-1.1.0/include/rapidjson /usr/include/
+if [[ -z ${TOXENV} ]]; then
+  sudo apt-get update
+  sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+
+  sudo rm /usr/local/bin/docker-compose
+  curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  chmod +x docker-compose
+  sudo mv docker-compose /usr/local/bin
+fi

.ci/travis-before-script.sh

@@ -2,27 +2,6 @@
 
 set -e -x
 
-if [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \
-        -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then
-    # Connect to MongoDB on port 27017 via a normal, unsecure connection if
-    # BIGCHAINDB_DATABASE_SSL is unset.
-    # It is unset in this case in .travis.yml.
-    docker pull mongo:3.4
-    docker run -d --publish=27017:27017 --name mdb-without-ssl mongo:3.4 # --replSet=bigchain-rs
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \
-        "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then
-    # Connect to MongoDB on port 27017 via TLS/SSL connection if
-    # BIGCHAINDB_DATABASE_SSL is set.
-    # It is set to 'true' here in .travis.yml. Dummy certificates for testing
-    # are stored under bigchaindb/tests/backend/mongodb-ssl/certs/ directory.
-    docker pull mongo:3.4
-    docker run -d \
-        --name mdb-with-ssl \
-        --publish=27017:27017 \
-        --volume=${TRAVIS_BUILD_DIR}/tests/backend/mongodb-ssl/certs:/certs \
-        mongo:3.4 \
-        --sslMode=requireSSL \
-        --sslCAFile=/certs/ca-chain.cert.pem \
-        --sslCRLFile=/certs/crl.pem \
-        --sslPEMKeyFile=/certs/local-mongo.pem
+if [[ -z ${TOXENV} ]]; then
+    docker-compose -f docker-compose.travis.yml up -d bdb
 fi

.ci/travis-install.sh

@@ -7,6 +7,6 @@ pip install --upgrade pip
 if [[ -n ${TOXENV} ]]; then
     pip install --upgrade tox
 else
-    pip install .[test]
+    docker-compose -f docker-compose.travis.yml build --no-cache
     pip install --upgrade codecov
 fi

.ci/travis_script.sh

@@ -4,16 +4,6 @@ set -e -x
 
 if [[ -n ${TOXENV} ]]; then
   tox -e ${TOXENV}
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \
-    -z "${BIGCHAINDB_DATABASE_SSL}" ]]; then
-    # Run the full suite of tests for MongoDB over an unsecure connection
-  pytest -sv --database-backend=localmongodb --cov=bigchaindb -m tendermint
-elif [[ "${BIGCHAINDB_DATABASE_BACKEND}" == localmongodb && \
-    "${BIGCHAINDB_DATABASE_SSL}" == true ]]; then
-    # Run a sub-set of tests over SSL; those marked as 'pytest.mark.bdb_ssl'.
-  pytest -sv --database-backend=localmongodb-ssl --cov=bigchaindb -m bdb_ssl
 else
-  # Run the full suite of tests for RethinkDB (the default backend when testing)
-  pytest -sv -m "serial"
-  pytest -sv --cov=bigchaindb -m "not serial"
+  docker-compose -f docker-compose.travis.yml run --rm --no-deps bdb pytest -v --cov=bigchaindb
 fi

.travis.yml

@@ -12,9 +12,12 @@ python:
   - 3.6
 
 env:
-  - TOXENV=flake8
-  - TOXENV=docsroot
-  - TOXENV=docsserver
+  global:
+    - DOCKER_COMPOSE_VERSION=1.19.0
+  matrix:
+    - TOXENV=flake8
+    - TOXENV=docsroot
+    - TOXENV=docsserver
 
 matrix:
   fast_finish: true

bigchaindb/tendermint/event_stream.py

@@ -25,7 +25,7 @@ def connect_and_recv(event_queue):
 
     logger.info('Connected to tendermint ws server')
 
-    stream_id = "bigchaindb_stream_{}".format(gen_timestamp())
+    stream_id = 'bigchaindb_stream_{}'.format(gen_timestamp())
     yield from subscribe_events(ws, stream_id)
 
     while True:
@@ -58,10 +58,10 @@ def process_event(event_queue, event, stream_id):
 @asyncio.coroutine
 def subscribe_events(ws, stream_id):
     payload = {
-        "method": "subscribe",
-        "jsonrpc": "2.0",
-        "params": ["NewBlock"],
-        "id": stream_id
+        'method': 'subscribe',
+        'jsonrpc': '2.0',
+        'params': ['NewBlock'],
+        'id': stream_id
     }
     yield from ws.send_str(json.dumps(payload))
 

compose/travis/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.6
+LABEL maintainer "dev@bigchaindb.com"
+
+RUN apt-get update \
+    && pip install -U pip \
+    && apt-get autoremove \
+    && apt-get clean
+
+ARG backend
+
+ENV PYTHONUNBUFFERED 0
+
+ENV BIGCHAINDB_DATABASE_PORT 27017
+ENV BIGCHAINDB_DATABASE_BACKEND $backend
+ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
+ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_SCHEME ws
+
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
+ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
+
+ENV BIGCHAINDB_START_TENDERMINT 0
+ENV TENDERMINT_PORT 46657
+
+RUN mkdir -p /usr/src/app
+COPY . /usr/src/app/
+WORKDIR /usr/src/app
+RUN pip install --no-cache-dir .[test]
+RUN bigchaindb -y configure "$backend"

docker-compose.travis.yml

@@ -0,0 +1,33 @@
+version: '3'
+
+services:
+  mdb:
+    image: mongo:3.4.3
+    command: mongod
+  bdb:
+    depends_on:
+      - mdb
+      - tendermint
+    build:
+      context: .
+      dockerfile: ./compose/travis/Dockerfile
+      args:
+        backend: localmongodb
+    volumes:
+      - .:/usr/src/app/
+    environment:
+      BIGCHAINDB_DATABASE_BACKEND: localmongodb
+      BIGCHAINDB_DATABASE_HOST: mdb
+      BIGCHAINDB_DATABASE_PORT: 27017
+      BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984
+      BIGCHAINDB_WSSERVER_HOST: 0.0.0.0
+      BIGCHAINDB_START_TENDERMINT: 0
+      TENDERMINT_HOST: tendermint
+      TENDERMINT_PORT: 46657
+    command: bigchaindb start
+  tendermint:
+    image: tendermint/tendermint:0.13
+    volumes:
+      - ./tmdata/config.toml:/tendermint/config.toml
+    entrypoint: ''
+    command: bash -c "tendermint init && tendermint node"

setup.py

@@ -57,6 +57,7 @@ tests_require = [
     'pytest-xdist',
     'pytest-flask',
     'pytest-aiohttp',
+    'pytest-asyncio',
     'tox',
 ] + docs_require
 

tests/conftest.py

@@ -499,6 +499,32 @@ def db_context(db_config, db_host, db_port, db_name, db_conn):
     )
 
 
+@pytest.fixture
+def tendermint_host():
+    return os.getenv('TENDERMINT_HOST', 'localhost')
+
+
+@pytest.fixture
+def tendermint_port():
+    return int(os.getenv('TENDERMINT_PORT', 46657))
+
+
+@pytest.fixture
+def tendermint_ws_url(tendermint_host, tendermint_port):
+    return f'ws://{tendermint_host}:{tendermint_port}/websocket'
+
+
+@pytest.fixture
+def tendermint_context(tendermint_host, tendermint_port, tendermint_ws_url):
+    TendermintContext = namedtuple(
+        'TendermintContext', ('host', 'port', 'ws_url'))
+    return TendermintContext(
+        host=tendermint_host,
+        port=tendermint_port,
+        ws_url=tendermint_ws_url,
+    )
+
+
 @pytest.fixture
 def mocked_setup_pub_logger(mocker):
     return mocker.patch(

tests/tendermint/test_core.py

@@ -57,8 +57,8 @@ def test_deliver_tx__valid_create_updates_db(b):
                     .sign([alice.private_key])
 
     app = App(b)
-    app.init_chain(["ignore"])
-    app.begin_block("ignore")
+    app.init_chain(['ignore'])
+    app.begin_block('ignore')
 
     result = app.deliver_tx(encode_tx_to_bytes(tx))
     assert result.is_ok()
@@ -81,8 +81,8 @@ def test_deliver_tx__double_spend_fails(b):
                     .sign([alice.private_key])
 
     app = App(b)
-    app.init_chain(["ignore"])
-    app.begin_block("ignore")
+    app.init_chain(['ignore'])
+    app.begin_block('ignore')
 
     result = app.deliver_tx(encode_tx_to_bytes(tx))
     assert result.is_ok()
@@ -101,8 +101,8 @@ def test_deliver_transfer_tx__double_spend_fails(b):
     from bigchaindb.common.crypto import generate_key_pair
 
     app = App(b)
-    app.init_chain(["ignore"])
-    app.begin_block("ignore")
+    app.init_chain(['ignore'])
+    app.begin_block('ignore')
 
     alice = generate_key_pair()
     bob = generate_key_pair()

tests/tendermint/test_event_stream.py

@@ -1,9 +1,13 @@
+import json
 from queue import Queue
 
+from aiohttp import ClientSession
 import pytest
 
 
-@pytest.mark.tendermint
+pytestmark = pytest.mark.tendermint
+
+
 def test_process_event_new_block():
     from bigchaindb.tendermint.event_stream import process_event
 
@@ -33,7 +37,6 @@ def test_process_event_new_block():
     assert not event_queue.empty()
 
 
-@pytest.mark.tendermint
 def test_process_event_empty_block():
     from bigchaindb.tendermint.event_stream import process_event
 
@@ -52,7 +55,6 @@ def test_process_event_empty_block():
     assert event_queue.empty()
 
 
-@pytest.mark.tendermint
 def test_process_unknown_event():
     from bigchaindb.tendermint.event_stream import process_event
 
@@ -62,3 +64,18 @@ def test_process_unknown_event():
     event_queue = Queue()
     process_event(event_queue, event, 'test_stream_id')
     assert event_queue.empty()
+
+
+@pytest.mark.asyncio
+async def test_subscribe_events(tendermint_ws_url):
+    from bigchaindb.tendermint.event_stream import subscribe_events
+    session = ClientSession()
+    ws = await session.ws_connect(tendermint_ws_url)
+    stream_id = 'bigchaindb_stream_01'
+    await subscribe_events(ws, stream_id)
+    msg = await ws.receive()
+    assert msg.data
+    msg_data_dict = json.loads(msg.data)
+    assert msg_data_dict['id'] == stream_id
+    assert msg_data_dict['jsonrpc'] == '2.0'
+    # TODO What else should be there? Right now, getting error.