From 1bad851e07c4aa3ed932212db8aa7a32241f9194 Mon Sep 17 00:00:00 2001 From: Muawia Khan Date: Fri, 29 Jun 2018 18:45:22 +0500 Subject: [PATCH 1/2] Problem: Tendermint configuration not present in BigchainDB config (#2342) * Problem: Tendermint configuration not present in BigchainDB config * Handle tendermint configurations properly - Update docs * Nitpick * Missed some conflicts * Make changes in tendermint/lib.py instead of deprecated core.py - Also remove redundant info from docs about default values * Fix docsserver automethod --- bigchaindb/__init__.py | 4 ++ bigchaindb/commands/bigchaindb.py | 11 ++++-- bigchaindb/config_utils.py | 2 - bigchaindb/core.py | 4 -- bigchaindb/tendermint/event_stream.py | 6 +-- bigchaindb/tendermint/lib.py | 37 ++++++++++--------- .../appendices/the-bigchaindb-class.rst | 2 - .../source/server-reference/configuration.md | 25 +++++++++++++ tests/commands/test_commands.py | 2 +- tests/conftest.py | 4 ++ 10 files changed, 63 insertions(+), 34 deletions(-) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index b90fcea0..b9f096ec 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -59,6 +59,10 @@ config = { 'advertised_host': 'localhost', 'advertised_port': 9985, }, + 'tendermint': { + 'host': 'localhost', + 'port': 46657, + }, # FIXME: hardcoding to localmongodb for now 'database': _database_map['localmongodb'], 'log': { diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index ad02d33e..72d31bce 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -21,7 +21,6 @@ from bigchaindb.commands import utils from bigchaindb.commands.utils import (configure_bigchaindb, input_on_stderr) from bigchaindb.log import setup_logging -from bigchaindb.tendermint.lib import BigchainDB from bigchaindb.tendermint.utils import public_key_from_base64 logging.basicConfig(level=logging.INFO) @@ -82,6 +81,10 @@ def run_configure(args): val = conf['database'][key] conf['database'][key] = input_on_stderr('Database {}? (default `{}`): '.format(key, val), val) + for key in ('host', 'port'): + val = conf['tendermint'][key] + conf['tendermint'][key] = input_on_stderr('Tendermint {}? (default `{}`)'.format(key, val), val) + if config_path != '-': bigchaindb.config_utils.write_config(conf, config_path) else: @@ -94,7 +97,7 @@ def run_configure(args): def run_upsert_validator(args): """Store validators which should be synced with Tendermint""" - b = bigchaindb.Bigchain() + b = bigchaindb.tendermint.lib.BigchainDB() public_key = public_key_from_base64(args.public_key) validator = {'pub_key': {'type': 'ed25519', 'data': public_key}, @@ -110,7 +113,7 @@ def run_upsert_validator(args): def _run_init(): - bdb = bigchaindb.Bigchain() + bdb = bigchaindb.tendermint.lib.BigchainDB() schema.init_database(connection=bdb.connection) @@ -167,7 +170,7 @@ def run_start(args): setup_logging() logger.info('BigchainDB Version %s', bigchaindb.__version__) - run_recover(BigchainDB()) + run_recover(bigchaindb.tendermint.lib.BigchainDB()) try: if not args.skip_initialize_database: diff --git a/bigchaindb/config_utils.py b/bigchaindb/config_utils.py index 7a2521fe..be5ab69c 100644 --- a/bigchaindb/config_utils.py +++ b/bigchaindb/config_utils.py @@ -248,10 +248,8 @@ def autoconfigure(filename=None, config=None, force=False): # override configuration with env variables newconfig = env_config(newconfig) - if config: newconfig = update(newconfig, config) - set_config(newconfig) # sets bigchaindb.config diff --git a/bigchaindb/core.py b/bigchaindb/core.py index da371abe..14dce4fe 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -41,8 +41,6 @@ class Bigchain(object): connection (:class:`~bigchaindb.backend.connection.Connection`): A connection to the database. """ - config_utils.autoconfigure() - consensusPlugin = bigchaindb.config.get('consensus_plugin') if consensusPlugin: @@ -50,8 +48,6 @@ class Bigchain(object): else: self.consensus = BaseConsensusRules - self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) - def delete_transaction(self, *transaction_id): """Delete a transaction from the backlog. diff --git a/bigchaindb/tendermint/event_stream.py b/bigchaindb/tendermint/event_stream.py index 4d9a59f2..9cbd28f2 100644 --- a/bigchaindb/tendermint/event_stream.py +++ b/bigchaindb/tendermint/event_stream.py @@ -2,17 +2,17 @@ import asyncio import json import logging import time -from os import getenv import aiohttp +from bigchaindb import config from bigchaindb.common.utils import gen_timestamp from bigchaindb.events import EventTypes, Event from bigchaindb.tendermint.utils import decode_transaction_base64 -HOST = getenv('BIGCHAINDB_TENDERMINT_HOST', 'localhost') -PORT = int(getenv('BIGCHAINDB_TENDERMINT_PORT', 46657)) +HOST = config['tendermint']['host'] +PORT = config['tendermint']['port'] URL = 'ws://{}:{}/websocket'.format(HOST, PORT) logger = logging.getLogger(__name__) diff --git a/bigchaindb/tendermint/lib.py b/bigchaindb/tendermint/lib.py index 9f13030e..7275eed6 100644 --- a/bigchaindb/tendermint/lib.py +++ b/bigchaindb/tendermint/lib.py @@ -5,7 +5,6 @@ MongoDB. import logging from collections import namedtuple from copy import deepcopy -from os import getenv from uuid import uuid4 try: @@ -15,8 +14,8 @@ except ImportError: from sha3 import sha3_256 import requests - -from bigchaindb import backend +import bigchaindb +from bigchaindb import backend, config_utils from bigchaindb import Bigchain from bigchaindb.models import Transaction from bigchaindb.common.exceptions import (SchemaValidationError, @@ -28,24 +27,26 @@ from bigchaindb import exceptions as core_exceptions logger = logging.getLogger(__name__) -BIGCHAINDB_TENDERMINT_HOST = getenv('BIGCHAINDB_TENDERMINT_HOST', - 'localhost') -BIGCHAINDB_TENDERMINT_PORT = getenv('BIGCHAINDB_TENDERMINT_PORT', - '46657') -ENDPOINT = 'http://{}:{}/'.format(BIGCHAINDB_TENDERMINT_HOST, - BIGCHAINDB_TENDERMINT_PORT) -MODE_LIST = ('broadcast_tx_async', - 'broadcast_tx_sync', - 'broadcast_tx_commit') - class BigchainDB(Bigchain): + def __init__(self, connection=None, **kwargs): + super().__init__(**kwargs) + + config_utils.autoconfigure() + self.mode_list = ('broadcast_tx_async', + 'broadcast_tx_sync', + 'broadcast_tx_commit') + self.tendermint_host = bigchaindb.config['tendermint']['host'] + self.tendermint_port = bigchaindb.config['tendermint']['port'] + self.endpoint = 'http://{}:{}/'.format(self.tendermint_host, self.tendermint_port) + self.connection = connection if connection else backend.connect(**bigchaindb.config['database']) + def post_transaction(self, transaction, mode): """Submit a valid transaction to the mempool.""" - if not mode or mode not in MODE_LIST: + if not mode or mode not in self.mode_list: raise ValidationError(('Mode must be one of the following {}.') - .format(', '.join(MODE_LIST))) + .format(', '.join(self.mode_list))) payload = { 'method': mode, @@ -54,7 +55,7 @@ class BigchainDB(Bigchain): 'id': str(uuid4()) } # TODO: handle connection errors! - return requests.post(ENDPOINT, json=payload) + return requests.post(self.endpoint, json=payload) def write_transaction(self, transaction, mode): # This method offers backward compatibility with the Web API. @@ -69,7 +70,7 @@ class BigchainDB(Bigchain): return (202, '') # result = response['result'] - # if mode == MODE_LIST[2]: + # if mode == self.mode_list[2]: # return self._process_commit_mode_response(result) # else: # status_code = result['code'] @@ -348,7 +349,7 @@ class BigchainDB(Bigchain): def get_validators(self): try: - resp = requests.get('{}validators'.format(ENDPOINT)) + resp = requests.get('{}validators'.format(self.endpoint)) validators = resp.json()['result']['validators'] for v in validators: v.pop('accum') diff --git a/docs/server/source/appendices/the-bigchaindb-class.rst b/docs/server/source/appendices/the-bigchaindb-class.rst index ed9e6067..0054e3f4 100644 --- a/docs/server/source/appendices/the-bigchaindb-class.rst +++ b/docs/server/source/appendices/the-bigchaindb-class.rst @@ -3,5 +3,3 @@ The BigchainDB Class #################### .. autoclass:: bigchaindb.tendermint.BigchainDB - - .. automethod:: bigchaindb.tendermint.lib.BigchainDB.__init__ diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 3bc1ca31..1e9849fa 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -33,6 +33,8 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_LOG_DATEFMT_LOGFILE`
`BIGCHAINDB_LOG_FMT_CONSOLE`
`BIGCHAINDB_LOG_FMT_LOGFILE`
+`BIGCHAINDB_TENDERMINT_HOST`
+`BIGCHAINDB_TENDERMINT_PORT`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` @@ -426,3 +428,26 @@ logging of the `core.py` module to be more verbose, you would set the ``` **Defaults to**: `{}` + +## tendermint.host & tendermint.port + +The settings with names of the form `tendermint.*` are for +consensus(Tendermint) backend that we are using: + +* `tendermint.host` is the hostname (FQDN)/IP address of the tendermint backend. +* `tendermint.port` is self-explanatory. + +**Example using environment variables** +```text +export BIGCHAINDB_TENDERMINT_HOST=tendermint +export BIGCHAINDB_TENDERMINT_PORT=46657 +``` + +**Default values** + +```js +"tendermint": { + "host": "localhost", + "port": 46657, +} +``` diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 5ee502c6..a81a6938 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -90,7 +90,7 @@ def test_bigchain_run_init_when_db_exists(mocker, capsys): def test__run_init(mocker): from bigchaindb.commands.bigchaindb import _run_init bigchain_mock = mocker.patch( - 'bigchaindb.commands.bigchaindb.bigchaindb.Bigchain') + 'bigchaindb.commands.bigchaindb.bigchaindb.tendermint.lib.BigchainDB') init_db_mock = mocker.patch( 'bigchaindb.commands.bigchaindb.schema.init_database', autospec=True, diff --git a/tests/conftest.py b/tests/conftest.py index 078896cb..d17afd5c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,6 +107,10 @@ def _configure_bigchaindb(request): config = { 'database': bigchaindb._database_map[backend], + 'tendermint': { + 'host': 'localhost', + 'port': 46657, + } } config['database']['name'] = test_db_name config = config_utils.env_config(config) From 744ab3d5efbfa035ed190a5285a90cca84234cca Mon Sep 17 00:00:00 2001 From: Lev Berman Date: Fri, 29 Jun 2018 15:48:58 +0200 Subject: [PATCH 2/2] Problem: 2 implementations of fastquery exist. (#2365) * Problem: 2 implementations of fastquery exist. Solution: Remove the old deprecated implementation. Update the tests. * Problem: There are still 3 outdated fastquery tests. Solution: Fix the tests. --- bigchaindb/fastquery.py | 60 ------------------------------ tests/db/test_bigchain_api.py | 36 +++++++++++------- tests/tendermint/test_fastquery.py | 44 ++++++++++++++++++++++ tests/test_fastquery.py | 21 ----------- 4 files changed, 67 insertions(+), 94 deletions(-) delete mode 100644 bigchaindb/fastquery.py delete mode 100644 tests/test_fastquery.py diff --git a/bigchaindb/fastquery.py b/bigchaindb/fastquery.py deleted file mode 100644 index f2190cb4..00000000 --- a/bigchaindb/fastquery.py +++ /dev/null @@ -1,60 +0,0 @@ -from bigchaindb.utils import condition_details_has_owner -from bigchaindb.backend import query -from bigchaindb.common.transaction import TransactionLink - - -class FastQuery: - """Database queries that join on block results from a single node. - - * Votes are not validated for security (security is a replication concern) - * Votes come from only one node, and as such, non-byzantine fault tolerance - is reduced. - - Previously, to consider the status of a block, all votes for that block - were retrieved and the election results were counted. This meant that a - faulty node may still have been able to obtain a correct election result. - However, from the point of view of a client, it is still neccesary to - query multiple nodes to insure against getting an incorrect response from - a byzantine node. - """ - - def __init__(self, connection): - self.connection = connection - - def get_outputs_by_public_key(self, public_key): - """Get outputs for a public key""" - res = list(query.get_owned_ids(self.connection, public_key)) - txs = [tx for _, tx in self.filter_valid_items(res)] - return [TransactionLink(tx['id'], index) - for tx in txs - for index, output in enumerate(tx['outputs']) - if condition_details_has_owner(output['condition']['details'], - public_key)] - - def filter_spent_outputs(self, outputs): - """Remove outputs that have been spent - - Args: - outputs: list of TransactionLink - """ - links = [o.to_dict() for o in outputs] - res = query.get_spending_transactions(self.connection, links) - txs = [tx for _, tx in self.filter_valid_items(res)] - spends = {TransactionLink.from_dict(input_['fulfills']) - for tx in txs - for input_ in tx['inputs']} - return [ff for ff in outputs if ff not in spends] - - def filter_unspent_outputs(self, outputs): - """Remove outputs that have not been spent - - Args: - outputs: list of TransactionLink - """ - links = [o.to_dict() for o in outputs] - res = query.get_spending_transactions(self.connection, links) - txs = [tx for _, tx in self.filter_valid_items(res)] - spends = {TransactionLink.from_dict(input_['fulfills']) - for tx in txs - for input_ in tx['inputs']} - return [ff for ff in outputs if ff in spends] diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 91d9cae6..b6c8a2ad 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -978,41 +978,51 @@ def test_get_owned_ids_calls_get_outputs_filtered(): assert res == gof() +@pytest.mark.tendermint def test_get_outputs_filtered_only_unspent(): from bigchaindb.common.transaction import TransactionLink - from bigchaindb.core import Bigchain - with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: + from bigchaindb.tendermint.lib import BigchainDB + + go = 'bigchaindb.tendermint.fastquery.FastQuery.get_outputs_by_public_key' + with patch(go) as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - with patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') as filter_spent: + fs = 'bigchaindb.tendermint.fastquery.FastQuery.filter_spent_outputs' + with patch(fs) as filter_spent: filter_spent.return_value = [TransactionLink('b', 2)] - out = Bigchain().get_outputs_filtered('abc', spent=False) + out = BigchainDB().get_outputs_filtered('abc', spent=False) get_outputs.assert_called_once_with('abc') assert out == [TransactionLink('b', 2)] +@pytest.mark.tendermint def test_get_outputs_filtered_only_spent(): from bigchaindb.common.transaction import TransactionLink - from bigchaindb.core import Bigchain - with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: + from bigchaindb.tendermint.lib import BigchainDB + go = 'bigchaindb.tendermint.fastquery.FastQuery.get_outputs_by_public_key' + with patch(go) as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - with patch('bigchaindb.fastquery.FastQuery.filter_unspent_outputs') as filter_spent: + fs = 'bigchaindb.tendermint.fastquery.FastQuery.filter_unspent_outputs' + with patch(fs) as filter_spent: filter_spent.return_value = [TransactionLink('b', 2)] - out = Bigchain().get_outputs_filtered('abc', spent=True) + out = BigchainDB().get_outputs_filtered('abc', spent=True) get_outputs.assert_called_once_with('abc') assert out == [TransactionLink('b', 2)] -@patch('bigchaindb.fastquery.FastQuery.filter_unspent_outputs') -@patch('bigchaindb.fastquery.FastQuery.filter_spent_outputs') +@pytest.mark.tendermint +@patch('bigchaindb.tendermint.fastquery.FastQuery.filter_unspent_outputs') +@patch('bigchaindb.tendermint.fastquery.FastQuery.filter_spent_outputs') def test_get_outputs_filtered(filter_spent, filter_unspent): from bigchaindb.common.transaction import TransactionLink - from bigchaindb.core import Bigchain - with patch('bigchaindb.fastquery.FastQuery.get_outputs_by_public_key') as get_outputs: + from bigchaindb.tendermint.lib import BigchainDB + + go = 'bigchaindb.tendermint.fastquery.FastQuery.get_outputs_by_public_key' + with patch(go) as get_outputs: get_outputs.return_value = [TransactionLink('a', 1), TransactionLink('b', 2)] - out = Bigchain().get_outputs_filtered('abc') + out = BigchainDB().get_outputs_filtered('abc') get_outputs.assert_called_once_with('abc') filter_spent.assert_not_called() filter_unspent.assert_not_called() diff --git a/tests/tendermint/test_fastquery.py b/tests/tendermint/test_fastquery.py index 8f807c9a..8689000b 100644 --- a/tests/tendermint/test_fastquery.py +++ b/tests/tendermint/test_fastquery.py @@ -3,6 +3,7 @@ import pytest from bigchaindb.common.transaction import TransactionLink from bigchaindb.models import Transaction + pytestmark = [pytest.mark.bdb, pytest.mark.tendermint] @@ -25,3 +26,46 @@ def test_get_outputs_by_public_key(b, user_pk, user2_pk, txns): TransactionLink(txns[0].id, 0), TransactionLink(txns[2].id, 1), ] + + +def test_filter_spent_outputs(b, user_pk, user_sk): + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 2) + tx1.sign([user_sk]) + + inputs = tx1.to_inputs() + + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx2.sign([user_sk]) + + # tx2 produces a new unspent. inputs[1] remains unspent. + b.store_bulk_transactions([tx1, tx2]) + + outputs = b.fastquery.get_outputs_by_public_key(user_pk) + unspents = b.fastquery.filter_spent_outputs(outputs) + + assert set(unsp for unsp in unspents) == { + inputs[1].fulfills, + tx2.to_inputs()[0].fulfills, + } + + +def test_filter_unspent_outputs(b, user_pk, user_sk): + out = [([user_pk], 1)] + tx1 = Transaction.create([user_pk], out * 2) + tx1.sign([user_sk]) + + inputs = tx1.to_inputs() + + tx2 = Transaction.transfer([inputs[0]], out, tx1.id) + tx2.sign([user_sk]) + + # tx2 produces a new unspent. input[1] remains unspent. + b.store_bulk_transactions([tx1, tx2]) + + outputs = b.fastquery.get_outputs_by_public_key(user_pk) + spents = b.fastquery.filter_unspent_outputs(outputs) + + assert set(sp for sp in spents) == { + inputs[0].fulfills, + } diff --git a/tests/test_fastquery.py b/tests/test_fastquery.py deleted file mode 100644 index e6c6dfbe..00000000 --- a/tests/test_fastquery.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from bigchaindb.common.transaction import TransactionLink - -pytestmark = pytest.mark.bdb - - -def test_filter_valid_items(b, blockdata): - blocks, _ = blockdata - assert (b.fastquery.filter_valid_items(blocks, block_id_key=lambda b: b['id']) - == [blocks[0], blocks[1]]) - - -def test_get_outputs_by_public_key(b, user_pk, user2_pk, blockdata): - blocks, _ = blockdata - assert b.fastquery.get_outputs_by_public_key(user_pk) == [ - TransactionLink(blocks[1]['block']['transactions'][0]['id'], 0) - ] - assert b.fastquery.get_outputs_by_public_key(user2_pk) == [ - TransactionLink(blocks[0]['block']['transactions'][0]['id'], 0) - ]