From b12856a9df210465dda0425fbfb28ba07bfb7210 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Tue, 7 Aug 2018 16:49:50 +0200 Subject: [PATCH] Problem: Cannot conclude validator election Solution: Gather votes and conclude election when supermajority is achieved --- bigchaindb/backend/localmongodb/query.py | 12 + bigchaindb/backend/query.py | 12 + bigchaindb/core.py | 8 +- bigchaindb/lib.py | 19 +- .../upsert_validator/validator_election.py | 63 ++++- .../validator_election_vote.py | 6 - tests/tendermint/test_integration.py | 37 --- tests/upsert_validator/conftest.py | 2 +- .../test_validator_election_vote.py | 223 +++++++++++++++++- 9 files changed, 321 insertions(+), 61 deletions(-) diff --git a/bigchaindb/backend/localmongodb/query.py b/bigchaindb/backend/localmongodb/query.py index 4f56dd73..20666266 100644 --- a/bigchaindb/backend/localmongodb/query.py +++ b/bigchaindb/backend/localmongodb/query.py @@ -301,3 +301,15 @@ def get_validator_set(conn, height=None): ) return list(cursor)[0] + + +@register_query(LocalMongoDBConnection) +def get_received_votes_for_election(conn, election_id, election_public_key): + cursor = conn.run( + conn.collection('transactions').aggregate([ + {'$match': {'outputs.public_keys': election_public_key, + 'operation': 'VALIDATOR_ELECTION_VOTE', + 'asset.id': election_id}}, + {'$project': {'_id': False}} + ])) + return cursor diff --git a/bigchaindb/backend/query.py b/bigchaindb/backend/query.py index cd4646f5..d2083be5 100644 --- a/bigchaindb/backend/query.py +++ b/bigchaindb/backend/query.py @@ -367,3 +367,15 @@ def get_validator_set(conn, height): then return the latest validator set""" raise NotImplementedError + + +@singledispatch +def get_received_votes_for_election(connection, election_id, election_public_key): + """Retrieve a list of `VALIDATOR_ELECTION_VOTE`s `txids` that are owned by `owner`. + Args: + election_id (str): Id of the election. + election_public_key (str): base58 encoded public key of the election + Returns: + Iterator of transaction that list given owner in conditions. + """ + raise NotImplementedError diff --git a/bigchaindb/core.py b/bigchaindb/core.py index f9613fe0..31058ffa 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -134,9 +134,8 @@ class App(BaseApplication): # TODO: calculate if an election has concluded # NOTE: ensure the local validator set is updated - # validator_updates = self.bigchaindb.get_validator_update() - # validator_updates = [encode_validator(v) for v in validator_updates] - validator_updates = [] + validator_updates = self.bigchaindb.get_validator_update(self.block_transactions) + validator_updates = [encode_validator(v) for v in validator_updates] # Store pre-commit state to recover in case there is a crash # during `commit` @@ -170,9 +169,8 @@ class App(BaseApplication): def encode_validator(v): - ed25519_public_key = v['pub_key']['data'] + ed25519_public_key = v['public_key'] # NOTE: tendermint expects public to be encoded in go-amino format - pub_key = PubKey(type='ed25519', data=bytes.fromhex(ed25519_public_key)) diff --git a/bigchaindb/lib.py b/bigchaindb/lib.py index 13d1ab71..444e56d4 100644 --- a/bigchaindb/lib.py +++ b/bigchaindb/lib.py @@ -24,6 +24,8 @@ from bigchaindb.common.exceptions import (SchemaValidationError, from bigchaindb.tendermint_utils import encode_transaction, merkleroot from bigchaindb import exceptions as core_exceptions from bigchaindb.consensus import BaseConsensusRules +from bigchaindb.upsert_validator import ValidatorElection, ValidatorElectionVote + logger = logging.getLogger(__name__) @@ -468,9 +470,20 @@ class BigchainDB(object): return validators - def get_validator_update(self): - update = backend.query.get_validator_update(self.connection) - return [update['validator']] if update else [] + def get_validator_update(self, txns): + votes = {} + for txn in txns: + if isinstance(txn, ValidatorElectionVote): + election_id = txn.asset['id'] + election_votes = votes.get(election_id, []) + votes[election_id] = election_votes.append(txn) + + election = ValidatorElection.conclude(self, election_id, election_votes) + # Once an election concludes any other conclusion for the same + # or any other election is invalidated + if election: + return [election.asset['data']] + return [] def delete_validator_update(self): return backend.query.delete_validator_update(self.connection) diff --git a/bigchaindb/upsert_validator/validator_election.py b/bigchaindb/upsert_validator/validator_election.py index 4717e708..f40bbfaf 100644 --- a/bigchaindb/upsert_validator/validator_election.py +++ b/bigchaindb/upsert_validator/validator_election.py @@ -1,3 +1,6 @@ +import base58 + +from bigchaindb import backend from bigchaindb.common.exceptions import (InvalidSignature, MultipleInputsError, InvalidProposer, @@ -36,7 +39,7 @@ class ValidatorElection(Transaction): validators = {} for validator in bigchain.get_validators(): # NOTE: we assume that Tendermint encodes public key in base64 - public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['value'])) + public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['data'])) validators[public_key] = validator['voting_power'] return validators @@ -114,7 +117,7 @@ class ValidatorElection(Transaction): if not self.is_same_topology(current_validators, self.outputs): raise UnequalValidatorSet('Validator set much be exactly same to the outputs of election') - return True + return self @classmethod def generate(cls, initiator, voters, election_data, metadata=None): @@ -141,3 +144,59 @@ class ValidatorElection(Transaction): @classmethod def transfer(cls, tx_signers, recipients, metadata=None, asset=None): raise NotImplementedError + + @classmethod + def to_public_key(cls, election_id): + return base58.b58encode(bytes.fromhex(election_id)) + + @classmethod + def count_votes(cls, election_pk, txns): + votes = 0 + for txn in txns: + if isinstance(txn, dict): + if txn['operation'] == 'VALIDATOR_ELECTION_VOTE': + for output in txn['outputs']: + # NOTE: We enforce that a valid vote to election id will have only + # election_pk in the output public keys, including any other public key + # along with election_pk will lead to vote being not considered valid. + if len(output['public_keys']) == 1 and [election_pk] == output['public_keys']: + votes = votes + int(output['amount']) + else: + if txn.operation == 'VALIDATOR_ELECTION_VOTE': + for output in txn.outputs: + # NOTE: We enforce that a valid vote to election id will have only + # election_pk in the output public keys, including any other public key + # along with election_pk will lead to vote being not considered valid. + if len(output.public_keys) == 1 and [election_pk] == output.public_keys: + votes = votes + int(output.amount) + return votes + + def get_commited_votes(self, bigchain, election_pk=None): + if election_pk is None: + election_pk = self.to_public_key(self.id) + txns = list(backend.query.get_received_votes_for_election(bigchain.connection, self.id, election_pk)) + return self.count_votes(election_pk, txns) + + @classmethod + def conclude(cls, bigchain, txn_id, current_votes=[]): + """Check if the given election has concluded or not + NOTE: + * Election is concluded iff the current validator set is exactly equal + to the validator set encoded in election outputs + * Election can concluded only if the current votes achieves a supermajority + """ + election = bigchain.get_transaction(txn_id) + + if election: + election_pk = election.to_public_key(election.id) + votes_commited = election.get_commited_votes(bigchain, election_pk) + votes_current = election.count_votes(election_pk, current_votes) + current_validators = election.current_validators(bigchain) + + if election.is_same_topology(current_validators, election.outputs): + total_votes = sum(current_validators.values()) + if (votes_commited < (2/3)*total_votes) and \ + (votes_commited + votes_current > (2/3)*total_votes): + return election + + return False diff --git a/bigchaindb/upsert_validator/validator_election_vote.py b/bigchaindb/upsert_validator/validator_election_vote.py index 1405ff8f..c25b7504 100644 --- a/bigchaindb/upsert_validator/validator_election_vote.py +++ b/bigchaindb/upsert_validator/validator_election_vote.py @@ -1,5 +1,3 @@ -import base58 - from bigchaindb.common.transaction import Transaction from bigchaindb.common.schema import (_validate_schema, TX_SCHEMA_COMMON, @@ -34,10 +32,6 @@ class ValidatorElectionVote(Transaction): self.validate_transfer_inputs(bigchain, current_transactions) return self - @classmethod - def to_public_key(cls, election_id): - return base58.b58encode(bytes.fromhex(election_id)) - @classmethod def generate(cls, inputs, recipients, election_id, metadata=None): (inputs, outputs) = cls.validate_transfer(inputs, recipients, election_id, metadata) diff --git a/tests/tendermint/test_integration.py b/tests/tendermint/test_integration.py index ee2af7c8..d7de519f 100644 --- a/tests/tendermint/test_integration.py +++ b/tests/tendermint/test_integration.py @@ -105,43 +105,6 @@ def test_app(tb, init_chain_request): assert block0['app_hash'] == new_block_hash -@pytest.mark.skip -@pytest.mark.abci -def test_upsert_validator(b, alice): - from bigchaindb.backend.query import VALIDATOR_UPDATE_ID - from bigchaindb.backend import query, connect - from bigchaindb.models import Transaction - from bigchaindb.tendermint_utils import public_key_to_base64 - import time - - conn = connect() - power = 1 - public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403' - - validator = {'pub_key': {'type': 'AC26791624DE60', - 'data': public_key}, - 'power': power} - validator_update = {'validator': validator, - 'update_id': VALIDATOR_UPDATE_ID} - - query.store_validator_update(conn, deepcopy(validator_update)) - - tx = Transaction.create([alice.public_key], - [([alice.public_key], 1)], - asset=None)\ - .sign([alice.private_key]) - - code, message = b.write_transaction(tx, 'broadcast_tx_commit') - assert code == 202 - time.sleep(5) - - validators = b.get_validators() - validators = [(v['pub_key']['value'], v['voting_power']) for v in validators] - - public_key64 = public_key_to_base64(public_key) - assert ((public_key64, str(power)) in validators) - - @pytest.mark.abci def test_post_transaction_responses(tendermint_ws_url, b): from bigchaindb.common.crypto import generate_key_pair diff --git a/tests/upsert_validator/conftest.py b/tests/upsert_validator/conftest.py index d388ab46..54921b1a 100644 --- a/tests/upsert_validator/conftest.py +++ b/tests/upsert_validator/conftest.py @@ -26,7 +26,7 @@ def mock_get_validators(network_validators): validators = [] for public_key, power in network_validators.items(): validators.append({ - 'pub_key': {'type': 'AC26791624DE60', 'value': public_key}, + 'pub_key': {'type': 'AC26791624DE60', 'data': public_key}, 'voting_power': power }) return validators diff --git a/tests/upsert_validator/test_validator_election_vote.py b/tests/upsert_validator/test_validator_election_vote.py index 78b2d528..fa3d6296 100644 --- a/tests/upsert_validator/test_validator_election_vote.py +++ b/tests/upsert_validator/test_validator_election_vote.py @@ -1,12 +1,16 @@ import pytest -from bigchaindb.upsert_validator import ValidatorElectionVote +from bigchaindb.upsert_validator import ValidatorElection, ValidatorElectionVote from bigchaindb.common.exceptions import AmountError +from bigchaindb.common.crypto import generate_key_pair +from bigchaindb.common.exceptions import ValidationError -pytestmark = [pytest.mark.tendermint, pytest.mark.bdb] +pytestmark = [pytest.mark.execute] +@pytest.mark.tendermint +@pytest.mark.bdb def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_node_keys): b_mock.store_bulk_transactions([valid_election]) @@ -15,7 +19,7 @@ def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_no public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] - election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_election.id) vote = ValidatorElectionVote.generate([input0], [([election_pub_key], votes)], @@ -24,9 +28,9 @@ def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_no assert vote.validate(b_mock) +@pytest.mark.tendermint +@pytest.mark.bdb def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519_node_keys): - from bigchaindb.common.crypto import generate_key_pair - alice = generate_key_pair() b_mock.store_bulk_transactions([valid_election]) @@ -44,7 +48,7 @@ def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519 assert delegate_vote.validate(b_mock) b_mock.store_bulk_transactions([delegate_vote]) - election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_election.id) alice_votes = delegate_vote.to_inputs()[0] alice_casted_vote = ValidatorElectionVote.generate([alice_votes], @@ -61,6 +65,8 @@ def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519 assert key0_casted_vote.validate(b_mock) +@pytest.mark.tendermint +@pytest.mark.bdb def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_node_keys): b_mock.store_bulk_transactions([valid_election]) @@ -69,7 +75,7 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_ public_key0 = input0.owners_before[0] key0 = ed25519_node_keys[public_key0] - election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id) + election_pub_key = ValidatorElection.to_public_key(valid_election.id) vote = ValidatorElectionVote.generate([input0], [([election_pub_key], votes+1)], @@ -78,3 +84,206 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_ with pytest.raises(AmountError): assert vote.validate(b_mock) + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_valid_election_votes_received(b_mock, valid_election, ed25519_node_keys): + alice = generate_key_pair() + b_mock.store_bulk_transactions([valid_election]) + assert valid_election.get_commited_votes(b_mock) == 0 + + input0 = valid_election.to_inputs()[0] + votes = valid_election.outputs[0].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + + # delegate some votes to alice + delegate_vote = ValidatorElectionVote.generate([input0], + [([alice.public_key], 4), ([key0.public_key], votes-4)], + election_id=valid_election.id)\ + .sign([key0.private_key]) + b_mock.store_bulk_transactions([delegate_vote]) + assert valid_election.get_commited_votes(b_mock) == 0 + + election_public_key = ValidatorElection.to_public_key(valid_election.id) + alice_votes = delegate_vote.to_inputs()[0] + key0_votes = delegate_vote.to_inputs()[1] + + alice_casted_vote = ValidatorElectionVote.generate([alice_votes], + [([election_public_key], 2), ([alice.public_key], 2)], + election_id=valid_election.id)\ + .sign([alice.private_key]) + + assert alice_casted_vote.validate(b_mock) + b_mock.store_bulk_transactions([alice_casted_vote]) + + # Check if the delegated vote is count as valid vote + assert valid_election.get_commited_votes(b_mock) == 2 + + key0_casted_vote = ValidatorElectionVote.generate([key0_votes], + [([election_public_key], votes-4)], + election_id=valid_election.id)\ + .sign([key0.private_key]) + + assert key0_casted_vote.validate(b_mock) + b_mock.store_bulk_transactions([key0_casted_vote]) + + assert valid_election.get_commited_votes(b_mock) == votes-2 + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_valid_election_conclude(b_mock, valid_election, ed25519_node_keys): + + # Node 0: cast vote + tx_vote0 = gen_vote(valid_election, 0, ed25519_node_keys) + + # check if the vote is valid even before the election doesn't exist + with pytest.raises(ValidationError): + assert tx_vote0.validate(b_mock) + + # store election + b_mock.store_bulk_transactions([valid_election]) + # cannot conclude election as not votes exist + assert not ValidatorElection.conclude(b_mock, valid_election.id) + + # validate vote + assert tx_vote0.validate(b_mock) + assert not ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote0]) + + b_mock.store_bulk_transactions([tx_vote0]) + assert not ValidatorElection.conclude(b_mock, valid_election.id) + + # Node 1: cast vote + tx_vote1 = gen_vote(valid_election, 1, ed25519_node_keys) + + # Node 2: cast vote + tx_vote2 = gen_vote(valid_election, 2, ed25519_node_keys) + + # Node 3: cast vote + tx_vote3 = gen_vote(valid_election, 3, ed25519_node_keys) + + assert tx_vote1.validate(b_mock) + assert not ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote1]) + + # 2/3 is achieved in the same block so the election can be concluded + assert ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote1, tx_vote2]) + + b_mock.store_bulk_transactions([tx_vote1]) + assert not ValidatorElection.conclude(b_mock, valid_election.id) + + assert tx_vote2.validate(b_mock) + assert tx_vote3.validate(b_mock) + + # conclusion can be triggered my different votes in the same block + assert ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote2]) + assert ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote2, tx_vote3]) + + b_mock.store_bulk_transactions([tx_vote2]) + + # Once the blockchain records >2/3 of the votes the election is assumed to be concluded + # so any invocation of `.conclude` for that election should return False + assert not ValidatorElection.conclude(b_mock, valid_election.id) + + # Vote is still valid but the election cannot be concluded as it it assmed that it has + # been concluded before + assert tx_vote3.validate(b_mock) + assert not ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote3]) + + +@pytest.mark.tendermint +@pytest.mark.bdb +def test_get_validator_update(b_mock, valid_election, ed25519_node_keys): + # store election + b_mock.store_bulk_transactions([valid_election]) + + # Node 0: cast vote + tx_vote0 = gen_vote(valid_election, 0, ed25519_node_keys) + assert b_mock.get_validator_update([tx_vote0]) == [] + b_mock.store_bulk_transactions([tx_vote0]) + + tx_vote1 = gen_vote(valid_election, 1, ed25519_node_keys) + assert b_mock.get_validator_update([tx_vote1]) == [] + b_mock.store_bulk_transactions([tx_vote1]) + + # Election can only be concluded once + tx_vote2 = gen_vote(valid_election, 2, ed25519_node_keys) + assert b_mock.get_validator_update([tx_vote2]) == [valid_election.asset['data']] + b_mock.store_bulk_transactions([tx_vote2]) + + tx_vote3 = gen_vote(valid_election, 3, ed25519_node_keys) + assert b_mock.get_validator_update([tx_vote3]) == [] + + +@pytest.mark.abci +def test_upsert_validator(b, node_key, node_keys, new_validator, ed25519_node_keys): + from bigchaindb.backend import connect + from bigchaindb.tendermint_utils import public_key_to_base64 + import time + import requests + + conn = connect() + (node_pub, _) = list(node_keys.items())[0] + + validators = [{'address': 'some_address', + 'pub_key': {'type': 'ed25519', + 'data': node_pub}, + 'voting_power': 10}] + + # reset the validator set + conn.run( + conn.collection('validators').find_one_and_update( + {"height": 1}, + {"$set": {"validators": validators}} + ) + ) + + power = 1 + public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403' + new_validator = {'public_key': public_key, + 'node_id': 'some_node_id', + 'power': power} + + voters = ValidatorElection.recipients(b) + election = ValidatorElection.generate([node_key.public_key], + voters, + new_validator, None).sign([node_key.private_key]) + code, message = b.write_transaction(election, 'broadcast_tx_commit') + assert code == 202 + time.sleep(3) + + assert b.get_transaction(election.id) + + tx_vote = gen_vote(election, 0, ed25519_node_keys) + assert tx_vote.validate(b) + code, message = b.write_transaction(tx_vote, 'broadcast_tx_commit') + assert code == 202 + time.sleep(3) + + resp = requests.get(b.endpoint + 'validators') + validator_pub_keys = [] + for v in resp.json()['result']['validators']: + validator_pub_keys.append(v['pub_key']['value']) + + assert (public_key_to_base64(public_key) in validator_pub_keys) + + +# ============================================================================ +# Helper functions +# ============================================================================ +def to_inputs(election, i, ed25519_node_keys): + input0 = election.to_inputs()[i] + votes = election.outputs[i].amount + public_key0 = input0.owners_before[0] + key0 = ed25519_node_keys[public_key0] + return (input0, votes, key0) + + +def gen_vote(election, i, ed25519_node_keys): + (input_i, votes_i, key_i) = to_inputs(election, i, ed25519_node_keys) + election_pub_key = ValidatorElection.to_public_key(election.id) + return ValidatorElectionVote.generate([input_i], + [([election_pub_key], votes_i)], + election_id=election.id)\ + .sign([key_i.private_key])