Merge branch 'master' of github.com:bigchaindb/bigchaindb

CHANGELOG.md

@@ -189,7 +189,7 @@ Re-enabled multi-threading. [Pull request #2258](https://github.com/bigchaindb/b
 
 ### Known Issues
 
-Tendermint changed how it responds to a request to store data (via the [Tendermint Broadcast API](http://tendermint.readthedocs.io/projects/tools/en/master/using-tendermint.html#broadcast-api)) between version 0.12 and 0.19.2. We started modifying the code of BigchainDB Server to account for those changes in responses (in [pull request #2239](https://github.com/bigchaindb/bigchaindb/pull/2239)), but we found that there's a difference between what the Tendermint documentation _says_ about those responses and how Tendermint actually responds. We need to determine Tendermint's intent before we can finalize that pull request.
+Tendermint changed how it responds to a request to store data (via the [Tendermint Broadcast API](https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api)) between version 0.12 and 0.19.2. We started modifying the code of BigchainDB Server to account for those changes in responses (in [pull request #2239](https://github.com/bigchaindb/bigchaindb/pull/2239)), but we found that there's a difference between what the Tendermint documentation _says_ about those responses and how Tendermint actually responds. We need to determine Tendermint's intent before we can finalize that pull request.
 
 ### Notes
 

bigchaindb/commands/bigchaindb.py

@@ -16,7 +16,7 @@ import sys
 from bigchaindb.utils import load_node_key
 from bigchaindb.common.exceptions import (DatabaseAlreadyExists,
                                           DatabaseDoesNotExist,
-                                          OperationError, KeypairMismatchException)
+                                          ValidationError)
 import bigchaindb
 from bigchaindb import (backend, ValidatorElection,
                         BigchainDB, ValidatorElectionVote)
@@ -27,6 +27,8 @@ from bigchaindb.commands import utils
 from bigchaindb.commands.utils import (configure_bigchaindb,
                                        input_on_stderr)
 from bigchaindb.log import setup_logging
+from bigchaindb.tendermint_utils import public_key_from_base64
+
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -119,29 +121,36 @@ def run_upsert_validator_new(args, bigchain):
         'sk': the path to the private key of the node calling the election (str)
         }
     :param bigchain: an instance of BigchainDB
-    :return: election_id (tx_id)
+    :return: election_id or `False` in case of failure
-    :raises: OperationError if the write transaction fails for any reason
     """
 
     new_validator = {
-        'public_key': args.public_key,
+        'public_key': public_key_from_base64(args.public_key),
         'power': args.power,
         'node_id': args.node_id
     }
 
-    key = load_node_key(args.sk)
+    try:
+        key = load_node_key(args.sk)
+        voters = ValidatorElection.recipients(bigchain)
+        election = ValidatorElection.generate([key.public_key],
+                                              voters,
+                                              new_validator, None).sign([key.private_key])
+        election.validate(bigchain)
+    except ValidationError as e:
+        logger.error(e)
+        return False
+    except FileNotFoundError as fd_404:
+        logger.error(fd_404)
+        return False
 
-    voters = ValidatorElection.recipients(bigchain)
-
-    election = ValidatorElection.generate([key.public_key],
-                                          voters,
-                                          new_validator, None).sign([key.private_key])
-    election.validate(bigchain)
     resp = bigchain.write_transaction(election, 'broadcast_tx_commit')
     if resp == (202, ''):
+        logger.info('[SUCCESS] Submitted proposal with id: {}'.format(election.id))
         return election.id
     else:
-        raise OperationError('Failed to commit election')
+        logger.error('Failed to commit election proposal')
+        return False
 
 
 def run_upsert_validator_approve(args, bigchain):
@@ -153,8 +162,7 @@ def run_upsert_validator_approve(args, bigchain):
         'sk': the path to the private key of the signer (str)
         }
     :param bigchain: an instance of BigchainDB
-    :return: a success message
+    :return: success log message or `False` in case of error
-    :raises: OperationError if the write transaction fails for any reason
     """
 
     key = load_node_key(args.sk)
@@ -163,22 +171,24 @@ def run_upsert_validator_approve(args, bigchain):
     if len(voting_powers) > 0:
         voting_power = voting_powers[0]
     else:
-        raise KeypairMismatchException(
+        logger.error('The key you provided does not match any of the eligible voters in this election.')
-            'The key you provided does not match any of the eligible voters in this election.'
+        return False
-        )
 
     inputs = [i for i in tx.to_inputs() if key.public_key in i.owners_before]
-    approval = ValidatorElectionVote.generate(inputs, [
+    election_pub_key = ValidatorElection.to_public_key(tx.id)
-        ([key.public_key], voting_power)], tx.id).sign([key.private_key])
+    approval = ValidatorElectionVote.generate(inputs,
+                                              [([election_pub_key], voting_power)],
+                                              tx.id).sign([key.private_key])
     approval.validate(bigchain)
 
     resp = bigchain.write_transaction(approval, 'broadcast_tx_commit')
 
     if resp == (202, ''):
-        print('Your vote has been submitted.')
+        logger.info('[SUCCESS] Your vote has been submitted')
         return approval.id
     else:
-        raise OperationError('Failed to vote for election')
+        logger.error('Failed to commit vote')
+        return False
 
 
 def _run_init():

bigchaindb/upsert_validator/validator_election.py

@@ -220,6 +220,7 @@ class ValidatorElection(Transaction):
                 updated_validator_set = new_validator_set(curr_validator_set,
                                                           new_height, validator_updates)
 
+                updated_validator_set = [v for v in updated_validator_set if v['voting_power'] > 0]
                 bigchain.store_validator_set(new_height+1, updated_validator_set)
                 return [encode_validator(election.asset['data'])]
         return []

bigchaindb/web/websocket_server.py

@@ -48,6 +48,17 @@ def _multiprocessing_to_asyncio(in_queue, out_queue, loop):
         loop.call_soon_threadsafe(out_queue.put_nowait, value)
 
 
+def eventify_block(block):
+    for tx in block['transactions']:
+        try:
+            asset_id = tx['asset']['id']
+        except KeyError:
+            asset_id = tx['id']
+        yield {'height': block['height'],
+               'asset_id': asset_id,
+               'transaction_id': tx['id']}
+
+
 class Dispatcher:
     """Dispatch events to websockets.
 
@@ -99,17 +110,10 @@ class Dispatcher:
                 str_buffer.append(event)
 
             elif event.type == EventTypes.BLOCK_VALID:
-                block = event.data
+                str_buffer = map(json.dumps, eventify_block(event.data))
 
-                for tx in block['transactions']:
+            for str_item in str_buffer:
-                    asset_id = tx['id'] if tx['operation'] == 'CREATE' else tx['asset']['id']
+                for _, websocket in self.subscribers.items():
-                    data = {'height': block['height'],
-                            'asset_id': asset_id,
-                            'transaction_id': tx['id']}
-                    str_buffer.append(json.dumps(data))
-
-            for _, websocket in self.subscribers.items():
-                for str_item in str_buffer:
                     yield from websocket.send_str(str_item)
 
 

docs/contributing/source/dev-setup-coding-and-contribution-process/run-node-as-processes.md

@@ -69,7 +69,7 @@ The commands are:
 export PATH=${PATH}:${GOPATH}/bin
 ```
 
-Follow [the Tendermint docs](https://tendermint.readthedocs.io/en/master/install.html#from-source) to install Tendermint from source.
+Follow [the Tendermint docs](https://tendermint.com/docs/introduction/install.html#from-source) to install Tendermint from source.
 
 If the installation is successful then Tendermint is installed at `$GOPATH/bin`. To ensure Tendermint's installed fine execute the following command,
 

docs/root/source/decentralized.md

@@ -20,6 +20,6 @@ A consortium can increase its decentralization (and its resilience) by increasin
 
 There’s no node that has a long-term special position in the BigchainDB network. All nodes run the same software and perform the same duties.
 
-If someone has (or gets) admin access to a node, they can mess with that node (e.g. change or delete data stored on that node), but those changes should remain isolated to that node. The BigchainDB network can only be compromised if more than one third of the nodes get compromised. See the [Tendermint documentation](https://tendermint.readthedocs.io/projects/tools/en/master/introduction.html) for more details.
+If someone has (or gets) admin access to a node, they can mess with that node (e.g. change or delete data stored on that node), but those changes should remain isolated to that node. The BigchainDB network can only be compromised if more than one third of the nodes get compromised. See the [Tendermint documentation](https://tendermint.com/docs/introduction/introduction.html) for more details.
 
 It’s worth noting that not even the admin or superuser of a node can transfer assets. The only way to create a valid transfer transaction is to fulfill the current crypto-conditions on the asset, and the admin/superuser can’t do that because the admin user doesn’t have the necessary information (e.g. private keys).

docs/server/source/http-client-server-api.rst

@@ -135,13 +135,21 @@ Transactions
 
    :query string mode: (Optional) One of the three supported modes to send a transaction: ``async``, ``sync``, ``commit``. The default is ``async``.
 
-   The ``mode`` query parameter inhereted from the mode parameter in Tendermint's
+   Once the posted transaction arrives at a BigchainDB node,
-   `broadcast API
+   that node will check to see if the transaction is valid.
-   <http://tendermint.readthedocs.io/projects/tools/en/master/using-tendermint.html#broadcast-api>`_.
+   If it's invalid, the node will return an HTTP 400 (error).
-   ``mode=async`` means the HTTP response will come back immediately, without
+   Otherwise, the node will send the transaction to Tendermint (in the same node) using the
-   even checking to see if the transaction is valid.
+   `Tendermint broadcast API
-   ``mode=sync`` means the HTTP response will come back once the node has
+   <https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api>`_.
-   checked the validity of the transaction.
+   
+   The meaning of the ``mode`` query parameter is inherited from the mode parameter in
+   `Tendermint's broadcast API
+   <https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api>`_.
+   ``mode=async`` means the HTTP response will come back immediately,
+   before Tendermint asks BigchainDB Server to check the validity of the transaction (a second time).
+   ``mode=sync`` means the HTTP response will come back
+   after Tendermint gets a response from BigchainDB Server
+   regarding the validity of the transaction.
    ``mode=commit`` means the HTTP response will come back once the transaction
    is in a committed block.
 

docs/server/source/server-reference/bigchaindb-cli.md

@@ -83,44 +83,59 @@ configuration file as documented under
 
 ## bigchaindb upsert-validator
 
-**This is an experimental feature. Users are advised not to use it in production.**
+Manage elections to add, update, or remove a validator from the validators set. The upsert-validator subcommands implement [BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21), please refer it for more details.
-
-
-Manage elections to add, update, or remove a validator from the validators set of the local node. The upsert-validator subcommands implement [BEP-21](https://github.com/bigchaindb/BEPs/tree/master/21). Check it out if you need more details on how this is orchestrated.
 
 Election management is broken into several subcommands. Below is the command line syntax for each,
 
 #### upsert-validator new
 
-Calls a new election, proposing a change to the validator set.
+Create a new election which proposes a change to the validator set. An election can be used to add/update/remove a validator from the validator set.
 
 Below is the command line syntax and the return value,
 
 ```bash
 $ bigchaindb upsert-validator new E_PUBKEY E_POWER E_NODE_ID --private-key PATH_TO_YOUR_PRIVATE_KEY
-<election_id>
+[SUCCESS] Submitted proposal with id: <election_id>
 ```
 
-Here, `E_PUBKEY`, `E_POWER`, and `E_NODE_ID` are the public key, proposed power, and node id of the validator being voted on. `--private-key` should be the path to wherever the private key for your validator node is stored, (*not* the private key itself.). For example, to add a new validator, provide the public key and node id for some node not already in the validator set, along with whatever voting power you'd like them to have. To remove an existing validator, provide their public key and node id, and set `E_POWER` to `0`.
+- `E_PUBKEY`: Public key of the node to be added/updated/removed.
+- `E_POWER`: The new power for the `E_PUBKEY`. NOTE, if power is set to `0` then `E_PUBKEY` will be removed from the validator set when the election concludes.
+- `E_NODE_ID`: Node id of `E_PUBKEY`. The node operator of `E_PUBKEY` can generate the node id via `tendermint show_node_id`. 
+- `--private-key`: The path to Tendermint's private key which can be generally found at `/home/user/.tendermint/config/priv_validator.json`. For example, to add a new validator, provide the public key and node id for some node not already in the validator set, along with whatever voting power you'd like them to have. To remove an existing validator, provide their public key and node id, and set `E_POWER` to `0`. Please note that the private key provided here is of the node which is generating this election i.e. 
+
+
+NOTE: A change to the validator set can only be proposed by one of the exisitng validators.
 
 Example usage,
 
 ```bash
-$ bigchaindb upsert-validator new B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014 1 12345 --private-key /home/user/.tendermint/config/priv_validator.json
+$ bigchaindb upsert-validator new HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52GnGuPY= 1 fb7140f03a4ffad899fabbbf655b97e0321add66 --private-key /home/user/.tendermint/config/priv_validator.json
+[SUCCESS] Submitted proposal with id: 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa
 ```
 
-If the command succeeds, it will create an election and return an `election_id`. Elections consist of one vote token per voting power, issued to the members of the validator set. Validators can cast their votes to approve the change to the validator set by spending their vote tokens. The status of the election can be monitored by providing the `election_id` to the `show` subcommand.
+If the command succeeds, it will create an election and return an `election_id`. A successful execution of the above command **doesn't** imply that the validator set will be immediately updated but rather it means the proposal has been succcessfully accepted by the network. Once the `election_id` has been generated the node operator should share this `election_id` with other validators in the network and urge them to approve the proposal. Note that the node operator should themsleves also approve the proposal.
+
+
+**NOTE**: The election proposal consists of vote tokens allocated to each current validator as per their voting power. Validators then cast their votes to approve the change to the validator set by spending their vote tokens.
+
 
 #### upsert-validator approve
- Approve an election by voting for it.
+
- Below is the command line syntax and the return value,
+Approve an election by voting for it. The propsal generated by executing `bigchaindb upsert-valdiator approve ...` can approved by the validators using this command. The validator who is approving the proposal will spend all their votes i.e. if the validator has a network power of `10` then they will cast `10` votes for the proposal.`
+Below is the command line syntax and the return value,
+
  ```bash
 $ bigchaindb upsert-validator approve <election_id> --private-key PATH_TO_YOUR_PRIVATE_KEY
+[SUCCESS] Your vote has been submitted
 ```
- Here, `<election_id>` is the transaction id of the election the approval should be given for. `--private-key` should be the path to Tendermint's private key which can be generally found at `/home/user/.tendermint/config/priv_validator.json`.
+
+- `election_id` is the transaction id of the election the approval should be given for.
+- `--private-key` should be the path to Tendermint's private key which can be generally found at `/home/user/.tendermint/config/priv_validator.json`.
 
  Example usage,
  ```bash
 $ bigchaindb upsert-validator approve 04a067582cf03eba2b53b82e4adb5ece424474cbd4f7183780855a93ac5e3caa --private-key /home/user/.tendermint/config/priv_validator.json
+[SUCCESS] Your vote has been submitted
 ```
- If the command succeeds, a message will be returned, that the vote was submitted successfully.
+
+If the command succeeds a message will be returned stating that the vote was submitted successfully. Once a proposal has been approved by sufficent validators (more than `2/3` of the total voting power) then the proposed change is applied to the network. For example, consider a network wherein the total power is `90` then the proposed changed applied only after `60` (`2/3 * 90`) have been received.

proposals/extend-post-txn.md

@@ -12,7 +12,7 @@ Add new query parameter `mode` to the [post transaction api](https://docs.bigcha
 
 ## Problem Description
 When posting a transaction it is broadcast asynchronously to Tendermint which enables the client to return immediately. Furthermore, the transaction status API would allow the client to get the current status for a given transaction. The above workflow seems efficient when the client doesn't need to wait until a transaction gets committed. In case a client wishes to wait until a transaction gets committed it would need to poll the transaction status api. 
-The Tendermint api allows to post a transaction in [three modes](http://tendermint.readthedocs.io/projects/tools/en/master/using-tendermint.html#broadcast-api),
+The Tendermint api allows to post a transaction in [three modes](https://tendermint.com/docs/tendermint-core/using-tendermint.html#broadcast-api),
 
 - `/broadcast_tx_async` post transaction and return
 - `/broadcast_tx_sync` post transaction and return after `checkTx` is executed

tests/backend/localmongodb/test_connection.py

@@ -55,10 +55,8 @@ def test_get_connection_returns_the_correct_instance(db_host, db_port):
     assert conn.conn._topology_settings.replica_set_name == config['replicaset']
 
 
-@mock.patch('bigchaindb.backend.localmongodb.connection.initialize_replica_set')
 @mock.patch('pymongo.MongoClient.__init__')
-@mock.patch('time.sleep')
+def test_connection_error(mock_client):
-def test_connection_error(mock_sleep, mock_client, mock_init_repl_set):
     from bigchaindb.backend import connect
     from bigchaindb.backend.exceptions import ConnectionError
 
@@ -74,9 +72,7 @@ def test_connection_error(mock_sleep, mock_client, mock_init_repl_set):
     assert mock_client.call_count == 3
 
 
-@mock.patch('bigchaindb.backend.localmongodb.connection.initialize_replica_set')
+def test_connection_run_errors():
-@mock.patch('pymongo.MongoClient')
-def test_connection_run_errors(mock_client, mock_init_repl_set):
     from bigchaindb.backend import connect
     from bigchaindb.backend.exceptions import (DuplicateKeyError,
                                                OperationError,

tests/backend/localmongodb/test_queries.py

@@ -7,7 +7,7 @@ from copy import deepcopy
 import pytest
 import pymongo
 
-pytestmark = [pytest.mark.tendermint, pytest.mark.localmongodb, pytest.mark.bdb]
+pytestmark = [pytest.mark.tendermint, pytest.mark.bdb]
 
 
 def test_get_txids_filtered(signed_create_tx, signed_transfer_tx):

tests/commands/conftest.py

@@ -27,8 +27,8 @@ def mock_db_init_with_existing_db(monkeypatch):
 
 @pytest.fixture
 def mock_processes_start(monkeypatch):
-    from bigchaindb import processes
+    from bigchaindb.utils import Process
-    monkeypatch.setattr(processes, 'start', lambda *args: None)
+    monkeypatch.setattr(Process, 'run', lambda *args: None)
 
 
 @pytest.fixture

tests/commands/test_commands.py

@@ -3,6 +3,7 @@
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
 import json
+import logging
 
 from unittest.mock import Mock, patch
 from argparse import Namespace
@@ -10,7 +11,6 @@ from argparse import Namespace
 import pytest
 
 from bigchaindb import ValidatorElection
-from bigchaindb.common.exceptions import KeypairMismatchException
 from tests.conftest import node_keys
 
 
@@ -185,7 +185,7 @@ def test_run_configure_when_config_does_exist(monkeypatch,
                                               mock_bigchaindb_backup_config):
     value = {}
 
-    def mock_write_config(newconfig, filename=None):
+    def mock_write_config(newconfig):
         value['return'] = newconfig
 
     from bigchaindb.commands.bigchaindb import run_configure
@@ -250,11 +250,8 @@ def test_run_start_when_db_already_exists(mocker,
 
 
 @pytest.mark.tendermint
-@patch('argparse.ArgumentParser.parse_args')
-@patch('bigchaindb.commands.utils.base_parser')
 @patch('bigchaindb.commands.utils.start')
-def test_calling_main(start_mock, base_parser_mock, parse_args_mock,
+def test_calling_main(start_mock, monkeypatch):
-                      monkeypatch):
     from bigchaindb.commands.bigchaindb import main
 
     argparser_mock = Mock()
@@ -283,11 +280,9 @@ def test_calling_main(start_mock, base_parser_mock, parse_args_mock,
     assert start_mock.called is True
 
 
-@patch('bigchaindb.config_utils.autoconfigure')
 @patch('bigchaindb.commands.bigchaindb.run_recover')
 @patch('bigchaindb.start.start')
-def test_recover_db_on_start(mock_autoconfigure,
+def test_recover_db_on_start(mock_run_recover,
-                             mock_run_recover,
                              mock_start,
                              mocked_setup_logging):
     from bigchaindb.commands.bigchaindb import run_start
@@ -384,7 +379,7 @@ def test_upsert_validator_new_with_tendermint(b, priv_validator_path, user_sk, m
 
 @pytest.mark.tendermint
 @pytest.mark.bdb
-def test_upsert_validator_new_without_tendermint(b, priv_validator_path, user_sk):
+def test_upsert_validator_new_without_tendermint(caplog, b, priv_validator_path, user_sk):
     from bigchaindb.commands.bigchaindb import run_upsert_validator_new
 
     def mock_write(tx, mode):
@@ -397,24 +392,67 @@ def test_upsert_validator_new_without_tendermint(b, priv_validator_path, user_sk
     args = Namespace(action='new',
                      public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=',
                      power=1,
-                     node_id='12345',
+                     node_id='fb7140f03a4ffad899fabbbf655b97e0321add66',
                      sk=priv_validator_path,
                      config={})
-    resp = run_upsert_validator_new(args, b)
 
-    assert b.get_transaction(resp)
+    with caplog.at_level(logging.INFO):
+        election_id = run_upsert_validator_new(args, b)
+        assert caplog.records[0].msg == '[SUCCESS] Submitted proposal with id: ' + election_id
+        assert b.get_transaction(election_id)
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_upsert_validator_new_invalid_election(caplog, b, priv_validator_path, user_sk):
+    from bigchaindb.commands.bigchaindb import run_upsert_validator_new
+
+    args = Namespace(action='new',
+                     public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=',
+                     power=10,
+                     node_id='fb7140f03a4ffad899fabbbf655b97e0321add66',
+                     sk='/tmp/invalid/path/key.json',
+                     config={})
+
+    with caplog.at_level(logging.ERROR):
+        assert not run_upsert_validator_new(args, b)
+        assert caplog.records[0].msg.__class__ == FileNotFoundError
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_upsert_validator_new_election_invalid_power(caplog, b, priv_validator_path, user_sk):
+    from bigchaindb.commands.bigchaindb import run_upsert_validator_new
+    from bigchaindb.common.exceptions import InvalidPowerChange
+
+    def mock_write(tx, mode):
+        b.store_bulk_transactions([tx])
+        return (400, '')
+
+    b.write_transaction = mock_write
+    b.get_validators = mock_get
+    args = Namespace(action='new',
+                     public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=',
+                     power=10,
+                     node_id='fb7140f03a4ffad899fabbbf655b97e0321add66',
+                     sk=priv_validator_path,
+                     config={})
+
+    with caplog.at_level(logging.ERROR):
+        assert not run_upsert_validator_new(args, b)
+        assert caplog.records[0].msg.__class__ == InvalidPowerChange
 
 
 @pytest.mark.abci
 def test_upsert_validator_approve_with_tendermint(b, priv_validator_path, user_sk, validators):
-    from bigchaindb.commands.bigchaindb import run_upsert_validator_new, \
+    from bigchaindb.commands.bigchaindb import (run_upsert_validator_new,
-        run_upsert_validator_approve
+                                                run_upsert_validator_approve)
 
-    public_key = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie'
+    public_key = 'CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg='
     new_args = Namespace(action='new',
                          public_key=public_key,
                          power=1,
-                         node_id='12345',
+                         node_id='fb7140f03a4ffad899fabbbf655b97e0321add66',
                          sk=priv_validator_path,
                          config={})
 
@@ -431,7 +469,7 @@ def test_upsert_validator_approve_with_tendermint(b, priv_validator_path, user_s
 
 @pytest.mark.bdb
 @pytest.mark.tendermint
-def test_upsert_validator_approve_without_tendermint(b, priv_validator_path, new_validator, node_key):
+def test_upsert_validator_approve_without_tendermint(caplog, b, priv_validator_path, new_validator, node_key):
     from bigchaindb.commands.bigchaindb import run_upsert_validator_approve
     from argparse import Namespace
 
@@ -443,15 +481,41 @@ def test_upsert_validator_approve_without_tendermint(b, priv_validator_path, new
                      sk=priv_validator_path,
                      config={})
 
-    approval_id = run_upsert_validator_approve(args, b)
-
     # assert returned id is in the db
-    assert b.get_transaction(approval_id)
+    with caplog.at_level(logging.INFO):
+        approval_id = run_upsert_validator_approve(args, b)
+        assert caplog.records[0].msg == '[SUCCESS] Your vote has been submitted'
+        assert b.get_transaction(approval_id)
 
 
-@pytest.mark.bdb
 @pytest.mark.tendermint
-def test_upsert_validator_approve_called_with_bad_key(b, bad_validator_path, new_validator, node_key):
+@pytest.mark.bdb
+def test_upsert_validator_approve_failure(caplog, b, priv_validator_path, new_validator, node_key):
+    from bigchaindb.commands.bigchaindb import run_upsert_validator_approve
+    from argparse import Namespace
+
+    b, election_id = call_election(b, new_validator, node_key)
+
+    def mock_write(tx, mode):
+        b.store_bulk_transactions([tx])
+        return (400, '')
+
+    b.write_transaction = mock_write
+
+    # call run_upsert_validator_approve with args that point to the election
+    args = Namespace(action='approve',
+                     election_id=election_id,
+                     sk=priv_validator_path,
+                     config={})
+
+    with caplog.at_level(logging.ERROR):
+        assert not run_upsert_validator_approve(args, b)
+        assert caplog.records[0].msg == 'Failed to commit vote'
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_upsert_validator_approve_called_with_bad_key(caplog, b, bad_validator_path, new_validator, node_key):
     from bigchaindb.commands.bigchaindb import run_upsert_validator_approve
     from argparse import Namespace
 
@@ -463,8 +527,10 @@ def test_upsert_validator_approve_called_with_bad_key(b, bad_validator_path, new
                      sk=bad_validator_path,
                      config={})
 
-    with pytest.raises(KeypairMismatchException):
+    with caplog.at_level(logging.ERROR):
-        run_upsert_validator_approve(args, b)
+        assert not run_upsert_validator_approve(args, b)
+        assert caplog.records[0].msg == 'The key you provided does not match any of '\
+            'the eligible voters in this election.'
 
 
 def mock_get(height):

tests/conftest.py

@@ -37,13 +37,6 @@ USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie'
 USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'
 
 
-def pytest_runtest_setup(item):
-    if isinstance(item, item.Function):
-        backend = item.session.config.getoption('--database-backend')
-        if (item.get_marker('localmongodb') and backend != 'localmongodb'):
-            pytest.skip('Skip tendermint specific tests if not using localmongodb')
-
-
 def pytest_addoption(parser):
     from bigchaindb.backend.connection import BACKENDS
 
@@ -56,19 +49,6 @@ def pytest_addoption(parser):
     )
 
 
-def pytest_ignore_collect(path, config):
-    from bigchaindb.backend.connection import BACKENDS
-    path = str(path)
-
-    supported_backends = BACKENDS.keys()
-
-    if os.path.isdir(path):
-        dirname = os.path.split(path)[1]
-        if dirname in supported_backends and dirname != config.getoption('--database-backend'):
-            print('Ignoring unrequested backend test dir: ', path)
-            return True
-
-
 def pytest_configure(config):
     config.addinivalue_line(
         'markers',
@@ -266,28 +246,12 @@ def merlin():
     return generate_key_pair()
 
 
-@pytest.fixture
-def merlin_privkey(merlin):
-    return merlin.private_key
-
-
-@pytest.fixture
-def merlin_pubkey(merlin):
-    return merlin.public_key
-
-
 @pytest.fixture
 def b():
     from bigchaindb import BigchainDB
     return BigchainDB()
 
 
-@pytest.fixture
-def tb():
-    from bigchaindb import BigchainDB
-    return BigchainDB()
-
-
 @pytest.fixture
 def create_tx(alice, user_pk):
     from bigchaindb.models import Transaction
@@ -349,24 +313,6 @@ def inputs(user_pk, b, alice):
         b.store_bulk_transactions(transactions)
 
 
-@pytest.fixture
-def inputs_shared(user_pk, user2_pk, alice):
-    from bigchaindb.models import Transaction
-
-    # create blocks with transactions for `USER` to spend
-    for block in range(4):
-        transactions = [
-            Transaction.create(
-                [alice.public_key],
-                [user_pk, user2_pk],
-                metadata={'msg': random.random()},
-            ).sign([alice.private_key]).to_dict()
-            for _ in range(10)
-        ]
-        block = Block(app_hash='', height=_get_height(b), transaction=transactions)
-        b.store_block(block._asdict())
-
-
 @pytest.fixture
 def dummy_db(request):
     from bigchaindb.backend import connect, schema
@@ -389,26 +335,6 @@ def dummy_db(request):
         pass
 
 
-@pytest.fixture
-def not_yet_created_db(request):
-    from bigchaindb.backend import connect, schema
-    from bigchaindb.common.exceptions import DatabaseDoesNotExist
-    conn = connect()
-    dbname = request.fixturename
-    xdist_suffix = getattr(request.config, 'slaveinput', {}).get('slaveid')
-    if xdist_suffix:
-        dbname = '{}_{}'.format(dbname, xdist_suffix)
-    try:
-        schema.drop_database(conn, dbname)
-    except DatabaseDoesNotExist:
-        pass
-    yield dbname
-    try:
-        schema.drop_database(conn, dbname)
-    except DatabaseDoesNotExist:
-        pass
-
-
 @pytest.fixture
 def db_config():
     from bigchaindb import config
@@ -464,29 +390,6 @@ def tendermint_ws_url(tendermint_host, tendermint_port):
     return 'ws://{}:{}/websocket'.format(tendermint_host, tendermint_port)
 
 
-@pytest.fixture
-def tendermint_context(tendermint_host, tendermint_port, tendermint_ws_url):
-    TendermintContext = namedtuple(
-        'TendermintContext', ('host', 'port', 'ws_url'))
-    return TendermintContext(
-        host=tendermint_host,
-        port=tendermint_port,
-        ws_url=tendermint_ws_url,
-    )
-
-
-@pytest.fixture
-def mocked_setup_pub_logger(mocker):
-    return mocker.patch(
-        'bigchaindb.log.setup.setup_pub_logger', autospec=True, spec_set=True)
-
-
-@pytest.fixture
-def mocked_setup_sub_logger(mocker):
-    return mocker.patch(
-        'bigchaindb.log.setup.setup_sub_logger', autospec=True, spec_set=True)
-
-
 @pytest.fixture(autouse=True)
 def _abci_http(request):
     if request.keywords.get('abci', None):
@@ -513,7 +416,7 @@ def abci_http(_setup_database, _configure_bigchaindb, abci_server,
 
 
 @pytest.yield_fixture(scope='session')
-def event_loop(request):
+def event_loop():
     import asyncio
 
     loop = asyncio.get_event_loop_policy().new_event_loop()

tests/tendermint/conftest.py

@@ -8,12 +8,6 @@ import codecs
 import abci.types_pb2 as types
 
 
-@pytest.fixture
-def b():
-    from bigchaindb import BigchainDB
-    return BigchainDB()
-
-
 @pytest.fixture
 def validator_pub_key():
     return 'B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014'

tests/tendermint/test_integration.py

@@ -16,13 +16,12 @@ from io import BytesIO
 
 @pytest.mark.tendermint
 @pytest.mark.bdb
-def test_app(tb, init_chain_request):
+def test_app(b, init_chain_request):
     from bigchaindb import App
     from bigchaindb.tendermint_utils import calculate_hash
     from bigchaindb.common.crypto import generate_key_pair
     from bigchaindb.models import Transaction
 
-    b = tb
     app = App(b)
     p = ProtocolHandler(app)
 

tests/tendermint/test_lib.py

@@ -57,10 +57,9 @@ def test_asset_is_separated_from_transaciton(b):
 
 
 @pytest.mark.bdb
-def test_get_latest_block(tb):
+def test_get_latest_block(b):
     from bigchaindb.lib import Block
 
-    b = tb
     for i in range(10):
         app_hash = os.urandom(16).hex()
         txn_id = os.urandom(16).hex()
@@ -75,8 +74,8 @@ def test_get_latest_block(tb):
 @pytest.mark.bdb
 @patch('bigchaindb.backend.query.get_block', return_value=None)
 @patch('bigchaindb.BigchainDB.get_latest_block', return_value={'height': 10})
-def test_get_empty_block(_0, _1, tb):
+def test_get_empty_block(_0, _1, b):
-    assert tb.get_block(5) == {'height': 5, 'transactions': []}
+    assert b.get_block(5) == {'height': 5, 'transactions': []}
 
 
 def test_validation_error(b):
@@ -172,15 +171,15 @@ def test_validator_updates(b, validator_pub_key):
 
 
 @pytest.mark.bdb
-def test_update_utxoset(tb, signed_create_tx, signed_transfer_tx, db_context):
+def test_update_utxoset(b, signed_create_tx, signed_transfer_tx, db_context):
     mongo_client = MongoClient(host=db_context.host, port=db_context.port)
-    tb.update_utxoset(signed_create_tx)
+    b.update_utxoset(signed_create_tx)
     utxoset = mongo_client[db_context.name]['utxos']
     assert utxoset.count() == 1
     utxo = utxoset.find_one()
     assert utxo['transaction_id'] == signed_create_tx.id
     assert utxo['output_index'] == 0
-    tb.update_utxoset(signed_transfer_tx)
+    b.update_utxoset(signed_transfer_tx)
     assert utxoset.count() == 1
     utxo = utxoset.find_one()
     assert utxo['transaction_id'] == signed_transfer_tx.id
@@ -188,14 +187,14 @@ def test_update_utxoset(tb, signed_create_tx, signed_transfer_tx, db_context):
 
 
 @pytest.mark.bdb
-def test_store_transaction(mocker, tb, signed_create_tx,
+def test_store_transaction(mocker, b, signed_create_tx,
                            signed_transfer_tx, db_context):
     mocked_store_asset = mocker.patch('bigchaindb.backend.query.store_assets')
     mocked_store_metadata = mocker.patch(
         'bigchaindb.backend.query.store_metadatas')
     mocked_store_transaction = mocker.patch(
         'bigchaindb.backend.query.store_transactions')
-    tb.store_bulk_transactions([signed_create_tx])
+    b.store_bulk_transactions([signed_create_tx])
     # mongo_client = MongoClient(host=db_context.host, port=db_context.port)
     # utxoset = mongo_client[db_context.name]['utxos']
     # assert utxoset.count() == 1
@@ -204,40 +203,40 @@ def test_store_transaction(mocker, tb, signed_create_tx,
     # assert utxo['output_index'] == 0
 
     mocked_store_asset.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{'id': signed_create_tx.id, 'data': signed_create_tx.asset['data']}],
     )
     mocked_store_metadata.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}],
     )
     mocked_store_transaction.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{k: v for k, v in signed_create_tx.to_dict().items()
          if k not in ('asset', 'metadata')}],
     )
     mocked_store_asset.reset_mock()
     mocked_store_metadata.reset_mock()
     mocked_store_transaction.reset_mock()
-    tb.store_bulk_transactions([signed_transfer_tx])
+    b.store_bulk_transactions([signed_transfer_tx])
     # assert utxoset.count() == 1
     # utxo = utxoset.find_one()
     # assert utxo['transaction_id'] == signed_transfer_tx.id
     # assert utxo['output_index'] == 0
     assert not mocked_store_asset.called
     mocked_store_metadata.asser_called_once_with(
-        tb.connection,
+        b.connection,
         [{'id': signed_transfer_tx.id, 'metadata': signed_transfer_tx.metadata}],
     )
     mocked_store_transaction.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{k: v for k, v in signed_transfer_tx.to_dict().items()
          if k != 'metadata'}],
     )
 
 
 @pytest.mark.bdb
-def test_store_bulk_transaction(mocker, tb, signed_create_tx,
+def test_store_bulk_transaction(mocker, b, signed_create_tx,
                                 signed_transfer_tx, db_context):
     mocked_store_assets = mocker.patch(
         'bigchaindb.backend.query.store_assets')
@@ -245,7 +244,7 @@ def test_store_bulk_transaction(mocker, tb, signed_create_tx,
         'bigchaindb.backend.query.store_metadatas')
     mocked_store_transactions = mocker.patch(
         'bigchaindb.backend.query.store_transactions')
-    tb.store_bulk_transactions((signed_create_tx,))
+    b.store_bulk_transactions((signed_create_tx,))
     # mongo_client = MongoClient(host=db_context.host, port=db_context.port)
     # utxoset = mongo_client[db_context.name]['utxos']
     # assert utxoset.count() == 1
@@ -253,34 +252,34 @@ def test_store_bulk_transaction(mocker, tb, signed_create_tx,
     # assert utxo['transaction_id'] == signed_create_tx.id
     # assert utxo['output_index'] == 0
     mocked_store_assets.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{'id': signed_create_tx.id, 'data': signed_create_tx.asset['data']}],
     )
     mocked_store_metadata.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{'id': signed_create_tx.id, 'metadata': signed_create_tx.metadata}],
     )
     mocked_store_transactions.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{k: v for k, v in signed_create_tx.to_dict().items()
          if k not in ('asset', 'metadata')}],
     )
     mocked_store_assets.reset_mock()
     mocked_store_metadata.reset_mock()
     mocked_store_transactions.reset_mock()
-    tb.store_bulk_transactions((signed_transfer_tx,))
+    b.store_bulk_transactions((signed_transfer_tx,))
     # assert utxoset.count() == 1
     # utxo = utxoset.find_one()
     # assert utxo['transaction_id'] == signed_transfer_tx.id
     # assert utxo['output_index'] == 0
     assert not mocked_store_assets.called
     mocked_store_metadata.asser_called_once_with(
-        tb.connection,
+        b.connection,
         [{'id': signed_transfer_tx.id,
           'metadata': signed_transfer_tx.metadata}],
     )
     mocked_store_transactions.assert_called_once_with(
-        tb.connection,
+        b.connection,
         [{k: v for k, v in signed_transfer_tx.to_dict().items()
           if k != 'metadata'}],
     )

tests/test_config_utils.py

@@ -24,7 +24,7 @@ def clean_config(monkeypatch, request):
     monkeypatch.setattr('bigchaindb.config', original_config)
 
 
-def test_bigchain_instance_is_initialized_when_conf_provided(request):
+def test_bigchain_instance_is_initialized_when_conf_provided():
     import bigchaindb
     from bigchaindb import config_utils
     assert 'CONFIGURED' not in bigchaindb.config

tests/test_core.py

@@ -66,7 +66,6 @@ def test_bigchain_class_initialization_with_parameters():
     assert bigchain.consensus == BaseConsensusRules
 
 
-@pytest.mark.genesis
 def test_get_spent_issue_1271(b, alice, bob, carol):
     from bigchaindb.models import Transaction
 

tests/upsert_validator/test_validator_election_vote.py

@@ -310,11 +310,37 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
     b.store_bulk_transactions([tx_vote0, tx_vote1])
 
     update = ValidatorElection.get_validator_update(b, 4, [tx_vote2])
-    print('update', update)
     update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n')
     assert len(update) == 1
     assert update_public_key == public_key64
 
+    # remove validator
+    power = 0
+    new_validator = {'public_key': public_key,
+                     'node_id': 'some_node_id',
+                     'power': power}
+    voters = ValidatorElection.recipients(b)
+    election = ValidatorElection.generate([node_key.public_key],
+                                          voters,
+                                          new_validator).sign([node_key.private_key])
+    # store election
+    b.store_bulk_transactions([election])
+
+    tx_vote0 = gen_vote(election, 0, ed25519_node_keys)
+    tx_vote1 = gen_vote(election, 1, ed25519_node_keys)
+    tx_vote2 = gen_vote(election, 2, ed25519_node_keys)
+
+    b.store_bulk_transactions([tx_vote0, tx_vote1])
+
+    update = ValidatorElection.get_validator_update(b, 9, [tx_vote2])
+    update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n')
+    assert len(update) == 1
+    assert update_public_key == public_key64
+
+    # assert that the public key is not a part of the current validator set
+    for v in b.get_validators(10):
+        assert not v['pub_key']['data'] == public_key64
+
 
 # ============================================================================
 # Helper functions

tests/web/test_assets.py

@@ -23,8 +23,7 @@ def test_get_assets_with_missing_text_search(client):
 
 @pytest.mark.bdb
 @pytest.mark.tendermint
-@pytest.mark.localmongodb
+def test_get_assets_tendermint(client, b, alice):
-def test_get_assets_tendermint(client, tb, alice):
     from bigchaindb.models import Transaction
 
     # test returns empty list when no assets are found
@@ -37,7 +36,7 @@ def test_get_assets_tendermint(client, tb, alice):
     tx = Transaction.create([alice.public_key], [([alice.public_key], 1)],
                             asset=asset).sign([alice.private_key])
 
-    tb.store_bulk_transactions([tx])
+    b.store_bulk_transactions([tx])
 
     # test that asset is returned
     res = client.get(ASSETS_ENDPOINT + '?search=abc')
@@ -51,11 +50,9 @@ def test_get_assets_tendermint(client, tb, alice):
 
 @pytest.mark.bdb
 @pytest.mark.tendermint
-@pytest.mark.localmongodb
+def test_get_assets_limit_tendermint(client, b, alice):
-def test_get_assets_limit_tendermint(client, tb, alice):
     from bigchaindb.models import Transaction
 
-    b = tb
     # create two assets
     asset1 = {'msg': 'abc 1'}
     asset2 = {'msg': 'abc 2'}

tests/web/test_block_tendermint.py

@@ -14,9 +14,8 @@ pytestmark = pytest.mark.tendermint
 
 @pytest.mark.bdb
 @pytest.mark.usefixtures('inputs')
-def test_get_block_endpoint(tb, client, alice):
+def test_get_block_endpoint(b, client, alice):
     import copy
-    b = tb
     tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset={'cycle': 'hero'})
     tx = tx.sign([alice.private_key])
 
@@ -49,8 +48,7 @@ def test_get_block_returns_404_if_not_found(client):
 
 
 @pytest.mark.bdb
-def test_get_block_containing_transaction(tb, client, alice):
+def test_get_block_containing_transaction(b, client, alice):
-    b = tb
     tx = Transaction.create([alice.public_key], [([alice.public_key], 1)], asset={'cycle': 'hero'})
     tx = tx.sign([alice.private_key])
     b.store_bulk_transactions([tx])

tests/web/test_metadata.py

@@ -23,11 +23,9 @@ def test_get_metadata_with_missing_text_search(client):
 
 @pytest.mark.bdb
 @pytest.mark.tendermint
-def test_get_metadata_tendermint(client, tb, alice):
+def test_get_metadata_tendermint(client, b, alice):
     from bigchaindb.models import Transaction
 
-    b = tb
-
     # test returns empty list when no assets are found
     res = client.get(METADATA_ENDPOINT + '?search=abc')
     assert res.json == []
@@ -53,11 +51,9 @@ def test_get_metadata_tendermint(client, tb, alice):
 
 @pytest.mark.bdb
 @pytest.mark.tendermint
-def test_get_metadata_limit_tendermint(client, tb, alice):
+def test_get_metadata_limit_tendermint(client, b, alice):
     from bigchaindb.models import Transaction
 
-    b = tb
-
     # create two assets
     asset1 = {'msg': 'abc 1'}
     meta1 = {'key': 'meta 1'}

tests/web/test_parameters.py

@@ -4,6 +4,8 @@
 
 import pytest
 
+pytestmark = pytest.mark.tendermint
+
 
 def test_valid_txid():
     from bigchaindb.web.views.parameters import valid_txid

tests/web/test_server.py

@@ -2,6 +2,10 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
+import pytest
+
+pytestmark = pytest.mark.tendermint
+
 
 def test_settings():
     import bigchaindb

tests/web/test_validators.py

@@ -9,7 +9,7 @@ pytestmark = pytest.mark.tendermint
 VALIDATORS_ENDPOINT = '/api/v1/validators/'
 
 
-def test_get_validators_endpoint(b, client, monkeypatch):
+def test_get_validators_endpoint(b, client):
     validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2',
                       'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A',
                                   'type': 'ed25519'},

tests/web/test_websocket_server.py

@@ -21,6 +21,33 @@ class MockWebSocket:
         self.received.append(s)
 
 
+def test_eventify_block_works_with_any_transaction():
+    from bigchaindb.web.websocket_server import eventify_block
+
+    block = {
+        'height': 1,
+        'transactions': [{
+            'id': 1
+        }, {
+            'id': 2,
+            'asset': {'id': 1}
+            }]
+        }
+
+    expected_events = [{
+            'height': 1,
+            'asset_id': 1,
+            'transaction_id': 1
+        }, {
+            'height': 1,
+            'asset_id': 1,
+            'transaction_id': 2
+        }]
+
+    for event, expected in zip(eventify_block(block), expected_events):
+        assert event == expected
+
+
 @asyncio.coroutine
 def test_bridge_sync_async_queue(loop):
     from bigchaindb.web.websocket_server import _multiprocessing_to_asyncio
@@ -136,7 +163,6 @@ def test_websocket_block_event(b, test_client, loop):
 
 
 @pytest.mark.skip('Processes are not stopping properly, and the whole test suite would hang')
-@pytest.mark.genesis
 def test_integration_from_webapi_to_websocket(monkeypatch, client, loop):
     # XXX: I think that the `pytest-aiohttp` plugin is sparkling too much
     # magic in the `asyncio` module: running this test without monkey-patching
@@ -152,6 +178,8 @@ def test_integration_from_webapi_to_websocket(monkeypatch, client, loop):
     import aiohttp
 
     from bigchaindb.common import crypto
+    # TODO processes does not exist anymore, when reactivating this test it
+    # will fail because of this
     from bigchaindb import processes
     from bigchaindb.models import Transaction