Mirroristas/bigchaindb
2
0
Fork 0
mirror of https://github.com/bigchaindb/bigchaindb.git synced 2024-10-13 13:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Problem: Validator set not tracked by BigchainDB

Browse Source 
Solution: BigchainDB depends on tendermint's RPC API to get the validator set
which is not avaiable during replay so the validators set should be tracked
inside BigchainDB
This commit is contained in:
Vanshdeep Singh 2018-08-03 13:46:32 +02:00
parent d25d806cd8
commit 91c8a4528b
13 changed files with 101 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
bigchaindb/backend/localmongodb/query.py
View File

@ -8,7 +8,7 @@ from bigchaindb.common.exceptions import MultipleValidatorOperationError
from bigchaindb.backend.utils import module_dispatch_registrar
from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection
from bigchaindb.common.transaction import Transaction
from bigchaindb.backend.query import VALIDATOR_UPDATE_ID
register_query = module_dispatch_registrar(backend.query)
@ -279,7 +279,7 @@ def get_pre_commit_state(conn, commit_id):
@register_query(LocalMongoDBConnection)
def store_validator_update(conn, validator_update):
def store_validator_set(conn, validator_update):
try:
return conn.run(
conn.collection('validators')
@ -289,15 +289,14 @@ def store_validator_update(conn, validator_update):
@register_query(LocalMongoDBConnection)
def get_validator_update(conn, update_id=VALIDATOR_UPDATE_ID):
def get_validator_set(conn, height=None):
query = {}
if height is not None:
query = {'height': {'$lte': height}}
return conn.run(
conn.collection('validators')
.find_one({'update_id': update_id}, projection={'_id': False}))
@register_query(LocalMongoDBConnection)
def delete_validator_update(conn, update_id=VALIDATOR_UPDATE_ID):
return conn.run(
conn.collection('validators')
.delete_one({'update_id': update_id})
.find(query, projection={'_id': False})
.sort([('height', DESCENDING)])
.limit(1)
)

4
bigchaindb/backend/localmongodb/schema.py
View File

@ -126,6 +126,6 @@ def create_pre_commit_secondary_index(conn, dbname):
def create_validators_secondary_index(conn, dbname):
logger.info('Create `validators` secondary index.')
conn.conn[dbname]['validators'].create_index('update_id',
name='update_id',
conn.conn[dbname]['validators'].create_index('height',
name='height',
unique=True,)

15
bigchaindb/backend/query.py
View File

@ -340,13 +340,6 @@ def store_pre_commit_state(connection, commit_id, state):
raise NotImplementedError
@singledispatch
def store_validator_update(conn, validator_update):
"""Store a update for the validator set"""
raise NotImplementedError
@singledispatch
def get_pre_commit_state(connection, commit_id):
"""Get pre-commit state where `id` is `commit_id`.
@ -362,14 +355,14 @@ def get_pre_commit_state(connection, commit_id):
@singledispatch
def get_validator_update(conn):
"""Get validator updates which are not synced"""
def store_validator_set(conn, validator_update):
"""Store updated validator set"""
raise NotImplementedError
@singledispatch
def delete_validator_update(conn, id):
"""Set the sync status for validator update documents"""
def get_validator_set(conn, height):
"""Get validator set at which are not synced"""
raise NotImplementedError

24
bigchaindb/core.py
View File

@ -1,6 +1,7 @@
"""This module contains all the goodness to integrate BigchainDB
with Tendermint."""
import logging
import codecs
from abci.application import BaseApplication
from abci.types_pb2 import (
@ -42,11 +43,13 @@ class App(BaseApplication):
self.validators = None
self.new_height = None
def init_chain(self, validators):
def init_chain(self, genesis):
"""Initialize chain with block of height 0"""
validator_set = [decode_validator(v) for v in genesis.validators]
block = Block(app_hash='', height=0, transactions=[])
self.bigchaindb.store_block(block._asdict())
self.bigchaindb.store_validator_set(1, validator_set)
return ResponseInitChain()
def info(self, request):
@ -129,11 +132,11 @@ class App(BaseApplication):
else:
self.block_txn_hash = block['app_hash']
validator_updates = self.bigchaindb.get_validator_update()
validator_updates = [encode_validator(v) for v in validator_updates]
# set sync status to true
self.bigchaindb.delete_validator_update()
# TODO: calculate if an election has concluded
# NOTE: ensure the local validator set is updated
# validator_updates = self.bigchaindb.get_validator_update()
# validator_updates = [encode_validator(v) for v in validator_updates]
validator_updates = []
# Store pre-commit state to recover in case there is a crash
# during `commit`
@ -176,3 +179,12 @@ def encode_validator(v):
return Validator(pub_key=pub_key,
address=b'',
power=v['power'])
def decode_validator(v):
validator = {'address': '', 'pub_key': {'type': '', 'data': ''}, 'voting_power': 0}
validator['address'] = codecs.encode(v.address, 'hex').decode().upper().rstrip('\n')
validator['pub_key']['type'] = v.pub_key.type
validator['pub_key']['data'] = codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')
validator['voting_power'] = v.power
return validator

16
bigchaindb/lib.py
View File

@ -460,20 +460,14 @@ class BigchainDB(object):
def fastquery(self):
return fastquery.FastQuery(self.connection)
def get_validators(self):
try:
resp = requests.get('{}validators'.format(self.endpoint))
validators = resp.json()['result']['validators']
def get_validators(self, height=None):
result = list(backend.query.get_validator_set(self.connection, height))[0]
validators = result['validators']
for v in validators:
v.pop('accum')
v.pop('address')
return validators
except requests.exceptions.RequestException as e:
logger.error('Error while connecting to Tendermint HTTP API')
raise e
def get_validator_update(self):
update = backend.query.get_validator_update(self.connection)
return [update['validator']] if update else []
@ -484,6 +478,10 @@ class BigchainDB(object):
def store_pre_commit_state(self, state):
return backend.query.store_pre_commit_state(self.connection, state)
def store_validator_set(self, height, validators):
return backend.query.store_validator_set(self.connection, {'height': height,
'validators': validators})
Block = namedtuple('Block', ('app_hash', 'height', 'transactions'))

25
tests/backend/localmongodb/test_queries.py
View File

@ -370,22 +370,23 @@ def test_get_pre_commit_state(db_context):
assert resp == state._asdict()
def test_store_validator_update():
def test_validator_update():
from bigchaindb.backend import connect, query
from bigchaindb.backend.query import VALIDATOR_UPDATE_ID
from bigchaindb.common.exceptions import MultipleValidatorOperationError
conn = connect()
validator_update = {'validator': {'key': 'value'},
'update_id': VALIDATOR_UPDATE_ID}
query.store_validator_update(conn, deepcopy(validator_update))
def gen_validator_update(height):
return {'data': 'somedata', 'height': height}
with pytest.raises(MultipleValidatorOperationError):
query.store_validator_update(conn, deepcopy(validator_update))
for i in range(1, 100, 10):
value = gen_validator_update(i)
query.store_validator_set(conn, value)
resp = query.get_validator_update(conn, VALIDATOR_UPDATE_ID)
[v1] = list(query.get_validator_set(conn, 8))
assert v1['height'] == 1
assert resp == validator_update
assert query.delete_validator_update(conn, VALIDATOR_UPDATE_ID)
assert not query.get_validator_update(conn, VALIDATOR_UPDATE_ID)
[v41] = list(query.get_validator_set(conn, 50))
assert v41['height'] == 41
[v91] = list(query.get_validator_set(conn))
assert v91['height'] == 91

2
tests/backend/localmongodb/test_schema.py
View File

@ -40,7 +40,7 @@ def test_init_creates_db_tables_and_indexes():
assert set(indexes) == {'_id_', 'pre_commit_id'}
indexes = conn.conn[dbname]['validators'].index_information().keys()
assert set(indexes) == {'_id_', 'update_id'}
assert set(indexes) == {'_id_', 'height'}
def test_init_database_fails_if_db_exists():

1
tests/commands/test_commands.py
View File

@ -341,6 +341,7 @@ class MockResponse():
return {'result': {'latest_block_height': self.height}}
@pytest.mark.skip
@patch('bigchaindb.config_utils.autoconfigure')
@patch('bigchaindb.backend.query.store_validator_update')
@pytest.mark.tendermint

13
tests/tendermint/conftest.py
View File

@ -1,4 +1,7 @@
import pytest
import codecs
import abci.types_pb2 as types
@pytest.fixture
@ -10,3 +13,13 @@ def b():
@pytest.fixture
def validator_pub_key():
return 'B0E42D2589A455EAD339A035D6CE1C8C3E25863F268120AA0162AD7D003A4014'
@pytest.fixture
def init_chain_request():
addr = codecs.decode(b'9FD479C869C7D7E7605BF99293457AA5D80C3033', 'hex')
pk = codecs.decode(b'VAgFZtYw8bNR5TMZHFOBDWk9cAmEu3/c6JgRBmddbbI=', 'base64')
val_a = types.Validator(address=addr, power=10,
pub_key=types.PubKey(type='ed25519', data=pk))
return types.RequestInitChain(validators=[val_a])

23
tests/tendermint/test_core.py
View File

@ -50,7 +50,7 @@ def test_check_tx__unsigned_create_is_error(b):
@pytest.mark.bdb
def test_deliver_tx__valid_create_updates_db(b):
def test_deliver_tx__valid_create_updates_db(b, init_chain_request):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
@ -64,8 +64,9 @@ def test_deliver_tx__valid_create_updates_db(b):
app = App(b)
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
app.init_chain(['ignore'])
app.begin_block(begin_block)
result = app.deliver_tx(encode_tx_to_bytes(tx))
@ -83,7 +84,7 @@ def test_deliver_tx__valid_create_updates_db(b):
# next(unspent_outputs)
def test_deliver_tx__double_spend_fails(b):
def test_deliver_tx__double_spend_fails(b, init_chain_request):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
@ -96,7 +97,7 @@ def test_deliver_tx__double_spend_fails(b):
.sign([alice.private_key])
app = App(b)
app.init_chain(['ignore'])
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
app.begin_block(begin_block)
@ -112,13 +113,13 @@ def test_deliver_tx__double_spend_fails(b):
assert result.code == CodeTypeError
def test_deliver_transfer_tx__double_spend_fails(b):
def test_deliver_transfer_tx__double_spend_fails(b, init_chain_request):
from bigchaindb import App
from bigchaindb.models import Transaction
from bigchaindb.common.crypto import generate_key_pair
app = App(b)
app.init_chain(['ignore'])
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
app.begin_block(begin_block)
@ -156,14 +157,16 @@ def test_deliver_transfer_tx__double_spend_fails(b):
assert result.code == CodeTypeError
def test_end_block_return_validator_updates(b):
# The test below has to re-written one election conclusion logic has been implemented
@pytest.mark.skip
def test_end_block_return_validator_updates(b, init_chain_request):
from bigchaindb import App
from bigchaindb.backend import query
from bigchaindb.core import encode_validator
from bigchaindb.backend.query import VALIDATOR_UPDATE_ID
app = App(b)
app.init_chain(['ignore'])
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
app.begin_block(begin_block)
@ -182,7 +185,7 @@ def test_end_block_return_validator_updates(b):
assert updates == []
def test_store_pre_commit_state_in_end_block(b, alice):
def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
from bigchaindb import App
from bigchaindb.backend import query
from bigchaindb.models import Transaction
@ -194,7 +197,7 @@ def test_store_pre_commit_state_in_end_block(b, alice):
.sign([alice.private_key])
app = App(b)
app.init_chain(['ignore'])
app.init_chain(init_chain_request)
begin_block = RequestBeginBlock()
app.begin_block(begin_block)

12
tests/tendermint/test_integration.py
View File

@ -1,3 +1,5 @@
import codecs
import abci.types_pb2 as types
import json
import pytest
@ -11,7 +13,7 @@ from io import BytesIO
@pytest.mark.tendermint
@pytest.mark.bdb
def test_app(tb):
def test_app(tb, init_chain_request):
from bigchaindb import App
from bigchaindb.tendermint_utils import calculate_hash
from bigchaindb.common.crypto import generate_key_pair
@ -28,12 +30,17 @@ def test_app(tb):
assert res.info.last_block_height == 0
assert not b.get_latest_block()
p.process('init_chain', types.Request(init_chain=types.RequestInitChain()))
p.process('init_chain', types.Request(init_chain=init_chain_request))
block0 = b.get_latest_block()
assert block0
assert block0['height'] == 0
assert block0['app_hash'] == ''
pk = codecs.encode(init_chain_request.validators[0].pub_key.data, 'base64').decode().strip('\n')
[validator] = b.get_validators(height=1)
assert validator['pub_key']['data'] == pk
assert validator['voting_power'] == 10
alice = generate_key_pair()
bob = generate_key_pair()
tx = Transaction.create([alice.public_key],
@ -98,6 +105,7 @@ def test_app(tb):
assert block0['app_hash'] == new_block_hash
@pytest.mark.skip
@pytest.mark.abci
def test_upsert_validator(b, alice):
from bigchaindb.backend.query import VALIDATOR_UPDATE_ID

1
tests/tendermint/test_lib.py
View File

@ -139,6 +139,7 @@ def test_post_transaction_invalid_mode(b):
b.write_transaction(tx, 'nope')
@pytest.mark.skip
@pytest.mark.bdb
def test_validator_updates(b, validator_pub_key):
from bigchaindb.backend import query

37
tests/web/test_validators.py
View File

@ -1,49 +1,22 @@
import pytest
from requests.exceptions import RequestException
pytestmark = pytest.mark.tendermint
VALIDATORS_ENDPOINT = '/api/v1/validators/'
def test_get_validators_endpoint(b, client, monkeypatch):
def mock_get(uri):
return MockResponse()
monkeypatch.setattr('requests.get', mock_get)
validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2',
'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A',
'type': 'ed25519'},
'voting_power': 10}]
b.store_validator_set(23, validator_set)
res = client.get(VALIDATORS_ENDPOINT)
assert is_validator(res.json[0])
assert res.status_code == 200
def test_get_validators_500_endpoint(b, client, monkeypatch):
def mock_get(uri):
raise RequestException
monkeypatch.setattr('requests.get', mock_get)
with pytest.raises(RequestException):
client.get(VALIDATORS_ENDPOINT)
# Helper
def is_validator(v):
return ('pub_key' in v) and ('voting_power' in v)
class MockResponse():
def json(self):
return {'id': '',
'jsonrpc': '2.0',
'result':
{'block_height': 5,
'validators': [
{'accum': 0,
'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2',
'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A',
'type': 'ed25519'},
'voting_power': 10}]}}