From 2e9a9b112136b17bd86a565991733d0f57e1a624 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Mon, 6 Aug 2018 11:37:43 +0200 Subject: [PATCH] Problem: Validator set not tracked by BigchainDB (#2436) * Problem: Validator set not tracked by BigchainDB Solution: BigchainDB depends on tendermint's RPC API to get the validator set which is not avaiable during replay so the validators set should be tracked inside BigchainDB * Problem: Unclear code and documentation Solution: Fix decode_validator and docs strings * Problem: Doc strings missing Solution: Add doc string for store_validato_set --- bigchaindb/backend/localmongodb/query.py | 22 ++++++------- bigchaindb/backend/localmongodb/schema.py | 4 +-- bigchaindb/backend/query.py | 16 +++------- bigchaindb/core.py | 22 +++++++++---- bigchaindb/lib.py | 26 ++++++++------- tests/backend/localmongodb/test_queries.py | 25 ++++++++------- tests/backend/localmongodb/test_schema.py | 2 +- tests/commands/test_commands.py | 1 + tests/tendermint/conftest.py | 13 ++++++++ tests/tendermint/test_core.py | 23 ++++++++------ tests/tendermint/test_integration.py | 12 +++++-- tests/tendermint/test_lib.py | 1 + tests/web/test_validators.py | 37 +++------------------- 13 files changed, 105 insertions(+), 99 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index b2b381c4..4f56dd73 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -8,7 +8,6 @@ from bigchaindb.common.exceptions import MultipleValidatorOperationError from bigchaindb.backend.utils import module_dispatch_registrar from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection from bigchaindb.common.transaction import Transaction -from bigchaindb.backend.query import VALIDATOR_UPDATE_ID register_query = module_dispatch_registrar(backend.query) @@ -279,7 +278,7 @@ def get_pre_commit_state(conn, commit_id): @register_query(LocalMongoDBConnection) -def store_validator_update(conn, validator_update): +def store_validator_set(conn, validator_update): try: return conn.run( conn.collection('validators') @@ -289,15 +288,16 @@ def store_validator_update(conn, validator_update): @register_query(LocalMongoDBConnection) -def get_validator_update(conn, update_id=VALIDATOR_UPDATE_ID): - return conn.run( - conn.collection('validators') - .find_one({'update_id': update_id}, projection={'_id': False})) +def get_validator_set(conn, height=None): + query = {} + if height is not None: + query = {'height': {'$lte': height}} - -@register_query(LocalMongoDBConnection) -def delete_validator_update(conn, update_id=VALIDATOR_UPDATE_ID): - return conn.run( + cursor = conn.run( conn.collection('validators') - .delete_one({'update_id': update_id}) + .find(query, projection={'_id': False}) + .sort([('height', DESCENDING)]) + .limit(1) ) + + return list(cursor)[0] diff --git a/bigchaindb/backend/localmongodb/schema.py b/bigchaindb/backend/localmongodb/schema.py index 58ee582f..1e7f3614 100644 --- a/bigchaindb/backend/localmongodb/schema.py +++ b/bigchaindb/backend/localmongodb/schema.py @@ -126,6 +126,6 @@ def create_pre_commit_secondary_index(conn, dbname): def create_validators_secondary_index(conn, dbname): logger.info('Create `validators` secondary index.') - conn.conn[dbname]['validators'].create_index('update_id', - name='update_id', + conn.conn[dbname]['validators'].create_index('height', + name='height', unique=True,) diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index 5ac6058f..cd4646f5 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -340,13 +340,6 @@ def store_pre_commit_state(connection, commit_id, state): raise NotImplementedError -@singledispatch -def store_validator_update(conn, validator_update): - """Store a update for the validator set""" - - raise NotImplementedError - - @singledispatch def get_pre_commit_state(connection, commit_id): """Get pre-commit state where `id` is `commit_id`. @@ -362,14 +355,15 @@ def get_pre_commit_state(connection, commit_id): @singledispatch -def get_validator_update(conn): - """Get validator updates which are not synced""" +def store_validator_set(conn, validator_update): + """Store updated validator set""" raise NotImplementedError @singledispatch -def delete_validator_update(conn, id): - """Set the sync status for validator update documents""" +def get_validator_set(conn, height): + """Get validator set for a given `height`, if `height` is not specified + then return the latest validator set""" raise NotImplementedError diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 354a39a0..f9613fe0 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -1,6 +1,7 @@ """This module contains all the goodness to integrate BigchainDB with Tendermint.""" import logging +import codecs from abci.application import BaseApplication from abci.types_pb2 import ( @@ -42,11 +43,13 @@ class App(BaseApplication): self.validators = None self.new_height = None - def init_chain(self, validators): + def init_chain(self, genesis): """Initialize chain with block of height 0""" + validator_set = [decode_validator(v) for v in genesis.validators] block = Block(app_hash='', height=0, transactions=[]) self.bigchaindb.store_block(block._asdict()) + self.bigchaindb.store_validator_set(1, validator_set) return ResponseInitChain() def info(self, request): @@ -129,11 +132,11 @@ class App(BaseApplication): else: self.block_txn_hash = block['app_hash'] - validator_updates = self.bigchaindb.get_validator_update() - validator_updates = [encode_validator(v) for v in validator_updates] - - # set sync status to true - self.bigchaindb.delete_validator_update() + # TODO: calculate if an election has concluded + # NOTE: ensure the local validator set is updated + # validator_updates = self.bigchaindb.get_validator_update() + # validator_updates = [encode_validator(v) for v in validator_updates] + validator_updates = [] # Store pre-commit state to recover in case there is a crash # during `commit` @@ -176,3 +179,10 @@ def encode_validator(v): return Validator(pub_key=pub_key, address=b'', power=v['power']) + + +def decode_validator(v): + return {'address': codecs.encode(v.address, 'hex').decode().upper().rstrip('\n'), + 'pub_key': {'type': v.pub_key.type, + 'data': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')}, + 'voting_power': v.power} diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py index b0545d83..13d1ab71 100644 --- a/bigchaindb/lib.py +++ b/bigchaindb/lib.py @@ -460,19 +460,13 @@ class BigchainDB(object): def fastquery(self): return fastquery.FastQuery(self.connection) - def get_validators(self): - try: - resp = requests.get('{}validators'.format(self.endpoint)) - validators = resp.json()['result']['validators'] - for v in validators: - v.pop('accum') - v.pop('address') + def get_validators(self, height=None): + result = backend.query.get_validator_set(self.connection, height) + validators = result['validators'] + for v in validators: + v.pop('address') - return validators - - except requests.exceptions.RequestException as e: - logger.error('Error while connecting to Tendermint HTTP API') - raise e + return validators def get_validator_update(self): update = backend.query.get_validator_update(self.connection) @@ -484,6 +478,14 @@ class BigchainDB(object): def store_pre_commit_state(self, state): return backend.query.store_pre_commit_state(self.connection, state) + def store_validator_set(self, height, validators): + """Store validator set at a given `height`. + NOTE: If the validator set already exists at that `height` then an + exception will be raised. + """ + return backend.query.store_validator_set(self.connection, {'height': height, + 'validators': validators}) + Block = namedtuple('Block', ('app_hash', 'height', 'transactions')) diff --git a/tests/backend/localmongodb/test_queries.py b/tests/backend/localmongodb/test_queries.py index a9e64ac0..2d3b9be6 100644 --- a/tests/backend/localmongodb/test_queries.py +++ b/tests/backend/localmongodb/test_queries.py @@ -370,22 +370,23 @@ def test_get_pre_commit_state(db_context): assert resp == state._asdict() -def test_store_validator_update(): +def test_validator_update(): from bigchaindb.backend import connect, query - from bigchaindb.backend.query import VALIDATOR_UPDATE_ID - from bigchaindb.common.exceptions import MultipleValidatorOperationError conn = connect() - validator_update = {'validator': {'key': 'value'}, - 'update_id': VALIDATOR_UPDATE_ID} - query.store_validator_update(conn, deepcopy(validator_update)) + def gen_validator_update(height): + return {'data': 'somedata', 'height': height} - with pytest.raises(MultipleValidatorOperationError): - query.store_validator_update(conn, deepcopy(validator_update)) + for i in range(1, 100, 10): + value = gen_validator_update(i) + query.store_validator_set(conn, value) - resp = query.get_validator_update(conn, VALIDATOR_UPDATE_ID) + v1 = query.get_validator_set(conn, 8) + assert v1['height'] == 1 - assert resp == validator_update - assert query.delete_validator_update(conn, VALIDATOR_UPDATE_ID) - assert not query.get_validator_update(conn, VALIDATOR_UPDATE_ID) + v41 = query.get_validator_set(conn, 50) + assert v41['height'] == 41 + + v91 = query.get_validator_set(conn) + assert v91['height'] == 91 diff --git a/tests/backend/localmongodb/test_schema.py b/tests/backend/localmongodb/test_schema.py index 99d06b0b..54a20694 100644 --- a/tests/backend/localmongodb/test_schema.py +++ b/tests/backend/localmongodb/test_schema.py @@ -40,7 +40,7 @@ def test_init_creates_db_tables_and_indexes(): assert set(indexes) == {'_id_', 'pre_commit_id'} indexes = conn.conn[dbname]['validators'].index_information().keys() - assert set(indexes) == {'_id_', 'update_id'} + assert set(indexes) == {'_id_', 'height'} def test_init_database_fails_if_db_exists(): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 578f06b1..a7c6418b 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -341,6 +341,7 @@ class MockResponse(): return {'result': {'latest_block_height': self.height}} +@pytest.mark.skip @patch('bigchaindb.config_utils.autoconfigure') @patch('bigchaindb.backend.query.store_validator_update') @pytest.mark.tendermint diff --git a/tests/tendermint/conftest.py b/tests/tendermint/conftest.py index 2d63d7c4..a0f50e9e 100644 --- a/tests/tendermint/conftest.py +++ b/tests/tendermint/conftest.py @@ -1,4 +1,7 @@ import pytest +import codecs + +import abci.types_pb2 as types @pytest.fixture @@ -10,3 +13,13 @@ def b(): @pytest.fixture def validator_pub_key(): return 'B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014' + + +@pytest.fixture +def init_chain_request(): + addr = codecs.decode(b'9FD479C869C7D7E7605BF99293457AA5D80C3033', 'hex') + pk = codecs.decode(b'VAgFZtYw8bNR5TMZHFOBDWk9cAmEu3/c6JgRBmddbbI=', 'base64') + val_a = types.Validator(address=addr, power=10, + pub_key=types.PubKey(type='ed25519', data=pk)) + + return types.RequestInitChain(validators=[val_a]) diff --git a/tests/tendermint/test_core.py b/tests/tendermint/test_core.py index 1a047e7d..f1a3d92d 100644 --- a/tests/tendermint/test_core.py +++ b/tests/tendermint/test_core.py @@ -50,7 +50,7 @@ def test_check_tx__unsigned_create_is_error(b): @pytest.mark.bdb -def test_deliver_tx__valid_create_updates_db(b): +def test_deliver_tx__valid_create_updates_db(b, init_chain_request): from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -64,8 +64,9 @@ def test_deliver_tx__valid_create_updates_db(b): app = App(b) + app.init_chain(init_chain_request) + begin_block = RequestBeginBlock() - app.init_chain(['ignore']) app.begin_block(begin_block) result = app.deliver_tx(encode_tx_to_bytes(tx)) @@ -83,7 +84,7 @@ def test_deliver_tx__valid_create_updates_db(b): # next(unspent_outputs) -def test_deliver_tx__double_spend_fails(b): +def test_deliver_tx__double_spend_fails(b, init_chain_request): from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair @@ -96,7 +97,7 @@ def test_deliver_tx__double_spend_fails(b): .sign([alice.private_key]) app = App(b) - app.init_chain(['ignore']) + app.init_chain(init_chain_request) begin_block = RequestBeginBlock() app.begin_block(begin_block) @@ -112,13 +113,13 @@ def test_deliver_tx__double_spend_fails(b): assert result.code == CodeTypeError -def test_deliver_transfer_tx__double_spend_fails(b): +def test_deliver_transfer_tx__double_spend_fails(b, init_chain_request): from bigchaindb import App from bigchaindb.models import Transaction from bigchaindb.common.crypto import generate_key_pair app = App(b) - app.init_chain(['ignore']) + app.init_chain(init_chain_request) begin_block = RequestBeginBlock() app.begin_block(begin_block) @@ -156,14 +157,16 @@ def test_deliver_transfer_tx__double_spend_fails(b): assert result.code == CodeTypeError -def test_end_block_return_validator_updates(b): +# The test below has to re-written one election conclusion logic has been implemented +@pytest.mark.skip +def test_end_block_return_validator_updates(b, init_chain_request): from bigchaindb import App from bigchaindb.backend import query from bigchaindb.core import encode_validator from bigchaindb.backend.query import VALIDATOR_UPDATE_ID app = App(b) - app.init_chain(['ignore']) + app.init_chain(init_chain_request) begin_block = RequestBeginBlock() app.begin_block(begin_block) @@ -182,7 +185,7 @@ def test_end_block_return_validator_updates(b): assert updates == [] -def test_store_pre_commit_state_in_end_block(b, alice): +def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request): from bigchaindb import App from bigchaindb.backend import query from bigchaindb.models import Transaction @@ -194,7 +197,7 @@ def test_store_pre_commit_state_in_end_block(b, alice): .sign([alice.private_key]) app = App(b) - app.init_chain(['ignore']) + app.init_chain(init_chain_request) begin_block = RequestBeginBlock() app.begin_block(begin_block) diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py index 4f6b530b..ee2af7c8 100644 --- a/tests/tendermint/test_integration.py +++ b/tests/tendermint/test_integration.py @@ -1,3 +1,5 @@ +import codecs + import abci.types_pb2 as types import json import pytest @@ -11,7 +13,7 @@ from io import BytesIO @pytest.mark.tendermint @pytest.mark.bdb -def test_app(tb): +def test_app(tb, init_chain_request): from bigchaindb import App from bigchaindb.tendermint_utils import calculate_hash from bigchaindb.common.crypto import generate_key_pair @@ -28,12 +30,17 @@ def test_app(tb): assert res.info.last_block_height == 0 assert not b.get_latest_block() - p.process('init_chain', types.Request(init_chain=types.RequestInitChain())) + p.process('init_chain', types.Request(init_chain=init_chain_request)) block0 = b.get_latest_block() assert block0 assert block0['height'] == 0 assert block0['app_hash'] == '' + pk = codecs.encode(init_chain_request.validators[0].pub_key.data, 'base64').decode().strip('\n') + [validator] = b.get_validators(height=1) + assert validator['pub_key']['data'] == pk + assert validator['voting_power'] == 10 + alice = generate_key_pair() bob = generate_key_pair() tx = Transaction.create([alice.public_key], @@ -98,6 +105,7 @@ def test_app(tb): assert block0['app_hash'] == new_block_hash +@pytest.mark.skip @pytest.mark.abci def test_upsert_validator(b, alice): from bigchaindb.backend.query import VALIDATOR_UPDATE_ID diff --git a/tests/tendermint/test_lib.py b/tests/tendermint/test_lib.py index 82d6e1e2..d9d9e320 100644 --- a/tests/tendermint/test_lib.py +++ b/tests/tendermint/test_lib.py @@ -139,6 +139,7 @@ def test_post_transaction_invalid_mode(b): b.write_transaction(tx, 'nope') +@pytest.mark.skip @pytest.mark.bdb def test_validator_updates(b, validator_pub_key): from bigchaindb.backend import query diff --git a/tests/web/test_validators.py b/tests/web/test_validators.py index 7a0e20ce..c453257f 100644 --- a/tests/web/test_validators.py +++ b/tests/web/test_validators.py @@ -1,49 +1,22 @@ import pytest -from requests.exceptions import RequestException - pytestmark = pytest.mark.tendermint VALIDATORS_ENDPOINT = '/api/v1/validators/' def test_get_validators_endpoint(b, client, monkeypatch): - - def mock_get(uri): - return MockResponse() - monkeypatch.setattr('requests.get', mock_get) + validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2', + 'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A', + 'type': 'ed25519'}, + 'voting_power': 10}] + b.store_validator_set(23, validator_set) res = client.get(VALIDATORS_ENDPOINT) - assert is_validator(res.json[0]) assert res.status_code == 200 -def test_get_validators_500_endpoint(b, client, monkeypatch): - - def mock_get(uri): - raise RequestException - monkeypatch.setattr('requests.get', mock_get) - - with pytest.raises(RequestException): - client.get(VALIDATORS_ENDPOINT) - - # Helper def is_validator(v): return ('pub_key' in v) and ('voting_power' in v) - - -class MockResponse(): - - def json(self): - return {'id': '', - 'jsonrpc': '2.0', - 'result': - {'block_height': 5, - 'validators': [ - {'accum': 0, - 'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2', - 'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A', - 'type': 'ed25519'}, - 'voting_power': 10}]}}