Problem: Cannot conclude validator election

Solution: Gather votes and conclude election when supermajority is achieved

bigchaindb/backend/localmongodb/query.py

@@ -301,3 +301,15 @@ def get_validator_set(conn, height=None):
     )
 
     return list(cursor)[0]
+
+
+@register_query(LocalMongoDBConnection)
+def get_received_votes_for_election(conn, election_id, election_public_key):
+    cursor = conn.run(
+        conn.collection('transactions').aggregate([
+            {'$match': {'outputs.public_keys': election_public_key,
+                        'operation': 'VALIDATOR_ELECTION_VOTE',
+                        'asset.id': election_id}},
+            {'$project': {'_id': False}}
+        ]))
+    return cursor

bigchaindb/backend/query.py

@@ -367,3 +367,15 @@ def get_validator_set(conn, height):
     then return the latest validator set"""
 
     raise NotImplementedError
+
+
+@singledispatch
+def get_received_votes_for_election(connection, election_id, election_public_key):
+    """Retrieve a list of `VALIDATOR_ELECTION_VOTE`s `txids` that are owned by `owner`.
+    Args:
+        election_id (str): Id of the election.
+        election_public_key (str): base58 encoded public key of the election
+    Returns:
+        Iterator of transaction that list given owner in conditions.
+    """
+    raise NotImplementedError

bigchaindb/core.py

@@ -134,9 +134,8 @@ class App(BaseApplication):
 
         # TODO: calculate if an election has concluded
         # NOTE: ensure the local validator set is updated
-        # validator_updates = self.bigchaindb.get_validator_update()
+        validator_updates = self.bigchaindb.get_validator_update(self.block_transactions)
-        # validator_updates = [encode_validator(v) for v in validator_updates]
+        validator_updates = [encode_validator(v) for v in validator_updates]
-        validator_updates = []
 
         # Store pre-commit state to recover in case there is a crash
         # during `commit`
@@ -170,9 +169,8 @@ class App(BaseApplication):
 
 
 def encode_validator(v):
-    ed25519_public_key = v['pub_key']['data']
+    ed25519_public_key = v['public_key']
     # NOTE: tendermint expects public to be encoded in go-amino format
-
     pub_key = PubKey(type='ed25519',
                      data=bytes.fromhex(ed25519_public_key))
 

bigchaindb/lib.py

@@ -24,6 +24,8 @@ from bigchaindb.common.exceptions import (SchemaValidationError,
 from bigchaindb.tendermint_utils import encode_transaction, merkleroot
 from bigchaindb import exceptions as core_exceptions
 from bigchaindb.consensus import BaseConsensusRules
+from bigchaindb.upsert_validator import ValidatorElection, ValidatorElectionVote
+
 
 logger = logging.getLogger(__name__)
 
@@ -468,9 +470,20 @@ class BigchainDB(object):
 
         return validators
 
-    def get_validator_update(self):
+    def get_validator_update(self, txns):
-        update = backend.query.get_validator_update(self.connection)
+        votes = {}
-        return [update['validator']] if update else []
+        for txn in txns:
+            if isinstance(txn, ValidatorElectionVote):
+                election_id = txn.asset['id']
+                election_votes = votes.get(election_id, [])
+                votes[election_id] = election_votes.append(txn)
+
+                election = ValidatorElection.conclude(self, election_id, election_votes)
+                # Once an election concludes any other conclusion for the same
+                # or any other election is invalidated
+                if election:
+                    return [election.asset['data']]
+        return []
 
     def delete_validator_update(self):
         return backend.query.delete_validator_update(self.connection)

bigchaindb/upsert_validator/validator_election.py

@@ -1,3 +1,6 @@
+import base58
+
+from bigchaindb import backend
 from bigchaindb.common.exceptions import (InvalidSignature,
                                           MultipleInputsError,
                                           InvalidProposer,
@@ -36,7 +39,7 @@ class ValidatorElection(Transaction):
         validators = {}
         for validator in bigchain.get_validators():
             # NOTE: we assume that Tendermint encodes public key in base64
-            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['value']))
+            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['data']))
             validators[public_key] = validator['voting_power']
 
         return validators
@@ -114,7 +117,7 @@ class ValidatorElection(Transaction):
         if not self.is_same_topology(current_validators, self.outputs):
             raise UnequalValidatorSet('Validator set much be exactly same to the outputs of election')
 
-        return True
+        return self
 
     @classmethod
     def generate(cls, initiator, voters, election_data, metadata=None):
@@ -141,3 +144,59 @@ class ValidatorElection(Transaction):
     @classmethod
     def transfer(cls, tx_signers, recipients, metadata=None, asset=None):
         raise NotImplementedError
+
+    @classmethod
+    def to_public_key(cls, election_id):
+        return base58.b58encode(bytes.fromhex(election_id))
+
+    @classmethod
+    def count_votes(cls, election_pk, txns):
+        votes = 0
+        for txn in txns:
+            if isinstance(txn, dict):
+                if txn['operation'] == 'VALIDATOR_ELECTION_VOTE':
+                    for output in txn['outputs']:
+                        # NOTE: We enforce that a valid vote to election id will have only
+                        # election_pk in the output public keys, including any other public key
+                        # along with election_pk will lead to vote being not considered valid.
+                        if len(output['public_keys']) == 1 and [election_pk] == output['public_keys']:
+                            votes = votes + int(output['amount'])
+            else:
+                if txn.operation == 'VALIDATOR_ELECTION_VOTE':
+                    for output in txn.outputs:
+                        # NOTE: We enforce that a valid vote to election id will have only
+                        # election_pk in the output public keys, including any other public key
+                        # along with election_pk will lead to vote being not considered valid.
+                        if len(output.public_keys) == 1 and [election_pk] == output.public_keys:
+                            votes = votes + int(output.amount)
+        return votes
+
+    def get_commited_votes(self, bigchain, election_pk=None):
+        if election_pk is None:
+            election_pk = self.to_public_key(self.id)
+        txns = list(backend.query.get_received_votes_for_election(bigchain.connection, self.id, election_pk))
+        return self.count_votes(election_pk, txns)
+
+    @classmethod
+    def conclude(cls, bigchain, txn_id, current_votes=[]):
+        """Check if the given election has concluded or not
+        NOTE:
+        * Election is concluded iff the current validator set is exactly equal
+          to the validator set encoded in election outputs
+        * Election can concluded only if the current votes achieves a supermajority
+        """
+        election = bigchain.get_transaction(txn_id)
+
+        if election:
+            election_pk = election.to_public_key(election.id)
+            votes_commited = election.get_commited_votes(bigchain, election_pk)
+            votes_current = election.count_votes(election_pk, current_votes)
+            current_validators = election.current_validators(bigchain)
+
+            if election.is_same_topology(current_validators, election.outputs):
+                total_votes = sum(current_validators.values())
+                if (votes_commited < (2/3)*total_votes) and \
+                   (votes_commited + votes_current > (2/3)*total_votes):
+                    return election
+
+        return False

bigchaindb/upsert_validator/validator_election_vote.py

@@ -1,5 +1,3 @@
-import base58
-
 from bigchaindb.common.transaction import Transaction
 from bigchaindb.common.schema import (_validate_schema,
                                       TX_SCHEMA_COMMON,
@@ -34,10 +32,6 @@ class ValidatorElectionVote(Transaction):
         self.validate_transfer_inputs(bigchain, current_transactions)
         return self
 
-    @classmethod
-    def to_public_key(cls, election_id):
-        return base58.b58encode(bytes.fromhex(election_id))
-
     @classmethod
     def generate(cls, inputs, recipients, election_id, metadata=None):
         (inputs, outputs) = cls.validate_transfer(inputs, recipients, election_id, metadata)

tests/tendermint/test_integration.py

@@ -105,43 +105,6 @@ def test_app(tb, init_chain_request):
     assert block0['app_hash'] == new_block_hash
 
 
-@pytest.mark.skip
-@pytest.mark.abci
-def test_upsert_validator(b, alice):
-    from bigchaindb.backend.query import VALIDATOR_UPDATE_ID
-    from bigchaindb.backend import query, connect
-    from bigchaindb.models import Transaction
-    from bigchaindb.tendermint_utils import public_key_to_base64
-    import time
-
-    conn = connect()
-    power = 1
-    public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
-
-    validator = {'pub_key': {'type': 'AC26791624DE60',
-                             'data': public_key},
-                 'power': power}
-    validator_update = {'validator': validator,
-                        'update_id': VALIDATOR_UPDATE_ID}
-
-    query.store_validator_update(conn, deepcopy(validator_update))
-
-    tx = Transaction.create([alice.public_key],
-                            [([alice.public_key], 1)],
-                            asset=None)\
-                    .sign([alice.private_key])
-
-    code, message = b.write_transaction(tx, 'broadcast_tx_commit')
-    assert code == 202
-    time.sleep(5)
-
-    validators = b.get_validators()
-    validators = [(v['pub_key']['value'], v['voting_power']) for v in validators]
-
-    public_key64 = public_key_to_base64(public_key)
-    assert ((public_key64, str(power)) in validators)
-
-
 @pytest.mark.abci
 def test_post_transaction_responses(tendermint_ws_url, b):
     from bigchaindb.common.crypto import generate_key_pair

tests/upsert_validator/conftest.py

@@ -26,7 +26,7 @@ def mock_get_validators(network_validators):
         validators = []
         for public_key, power in network_validators.items():
             validators.append({
-                'pub_key': {'type': 'AC26791624DE60', 'value': public_key},
+                'pub_key': {'type': 'AC26791624DE60', 'data': public_key},
                 'voting_power': power
             })
         return validators

tests/upsert_validator/test_validator_election_vote.py

@@ -1,12 +1,16 @@
 import pytest
 
-from bigchaindb.upsert_validator import ValidatorElectionVote
+from bigchaindb.upsert_validator import ValidatorElection, ValidatorElectionVote
 from bigchaindb.common.exceptions import AmountError
+from bigchaindb.common.crypto import generate_key_pair
+from bigchaindb.common.exceptions import ValidationError
 
 
-pytestmark = [pytest.mark.tendermint, pytest.mark.bdb]
+pytestmark = [pytest.mark.execute]
 
 
+@pytest.mark.tendermint
+@pytest.mark.bdb
 def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_node_keys):
     b_mock.store_bulk_transactions([valid_election])
 
@@ -15,7 +19,7 @@ def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_no
     public_key0 = input0.owners_before[0]
     key0 = ed25519_node_keys[public_key0]
 
-    election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id)
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
 
     vote = ValidatorElectionVote.generate([input0],
                                           [([election_pub_key], votes)],
@@ -24,9 +28,9 @@ def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_no
     assert vote.validate(b_mock)
 
 
+@pytest.mark.tendermint
+@pytest.mark.bdb
 def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519_node_keys):
-    from bigchaindb.common.crypto import generate_key_pair
-
     alice = generate_key_pair()
 
     b_mock.store_bulk_transactions([valid_election])
@@ -44,7 +48,7 @@ def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519
     assert delegate_vote.validate(b_mock)
 
     b_mock.store_bulk_transactions([delegate_vote])
-    election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id)
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
 
     alice_votes = delegate_vote.to_inputs()[0]
     alice_casted_vote = ValidatorElectionVote.generate([alice_votes],
@@ -61,6 +65,8 @@ def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519
     assert key0_casted_vote.validate(b_mock)
 
 
+@pytest.mark.tendermint
+@pytest.mark.bdb
 def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_node_keys):
     b_mock.store_bulk_transactions([valid_election])
 
@@ -69,7 +75,7 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_
     public_key0 = input0.owners_before[0]
     key0 = ed25519_node_keys[public_key0]
 
-    election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id)
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
 
     vote = ValidatorElectionVote.generate([input0],
                                           [([election_pub_key], votes+1)],
@@ -78,3 +84,206 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_
 
     with pytest.raises(AmountError):
         assert vote.validate(b_mock)
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_valid_election_votes_received(b_mock, valid_election, ed25519_node_keys):
+    alice = generate_key_pair()
+    b_mock.store_bulk_transactions([valid_election])
+    assert valid_election.get_commited_votes(b_mock) == 0
+
+    input0 = valid_election.to_inputs()[0]
+    votes = valid_election.outputs[0].amount
+    public_key0 = input0.owners_before[0]
+    key0 = ed25519_node_keys[public_key0]
+
+    # delegate some votes to alice
+    delegate_vote = ValidatorElectionVote.generate([input0],
+                                                   [([alice.public_key], 4), ([key0.public_key], votes-4)],
+                                                   election_id=valid_election.id)\
+                                         .sign([key0.private_key])
+    b_mock.store_bulk_transactions([delegate_vote])
+    assert valid_election.get_commited_votes(b_mock) == 0
+
+    election_public_key = ValidatorElection.to_public_key(valid_election.id)
+    alice_votes = delegate_vote.to_inputs()[0]
+    key0_votes = delegate_vote.to_inputs()[1]
+
+    alice_casted_vote = ValidatorElectionVote.generate([alice_votes],
+                                                       [([election_public_key], 2), ([alice.public_key], 2)],
+                                                       election_id=valid_election.id)\
+                                             .sign([alice.private_key])
+
+    assert alice_casted_vote.validate(b_mock)
+    b_mock.store_bulk_transactions([alice_casted_vote])
+
+    # Check if the delegated vote is count as valid vote
+    assert valid_election.get_commited_votes(b_mock) == 2
+
+    key0_casted_vote = ValidatorElectionVote.generate([key0_votes],
+                                                      [([election_public_key], votes-4)],
+                                                      election_id=valid_election.id)\
+                                            .sign([key0.private_key])
+
+    assert key0_casted_vote.validate(b_mock)
+    b_mock.store_bulk_transactions([key0_casted_vote])
+
+    assert valid_election.get_commited_votes(b_mock) == votes-2
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_valid_election_conclude(b_mock, valid_election, ed25519_node_keys):
+
+    # Node 0: cast vote
+    tx_vote0 = gen_vote(valid_election, 0, ed25519_node_keys)
+
+    # check if the vote is valid even before the election doesn't exist
+    with pytest.raises(ValidationError):
+        assert tx_vote0.validate(b_mock)
+
+    # store election
+    b_mock.store_bulk_transactions([valid_election])
+    # cannot conclude election as not votes exist
+    assert not ValidatorElection.conclude(b_mock, valid_election.id)
+
+    # validate vote
+    assert tx_vote0.validate(b_mock)
+    assert not ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote0])
+
+    b_mock.store_bulk_transactions([tx_vote0])
+    assert not ValidatorElection.conclude(b_mock, valid_election.id)
+
+    # Node 1: cast vote
+    tx_vote1 = gen_vote(valid_election, 1, ed25519_node_keys)
+
+    # Node 2: cast vote
+    tx_vote2 = gen_vote(valid_election, 2, ed25519_node_keys)
+
+    # Node 3: cast vote
+    tx_vote3 = gen_vote(valid_election, 3, ed25519_node_keys)
+
+    assert tx_vote1.validate(b_mock)
+    assert not ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote1])
+
+    # 2/3 is achieved in the same block so the election can be concluded
+    assert ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote1, tx_vote2])
+
+    b_mock.store_bulk_transactions([tx_vote1])
+    assert not ValidatorElection.conclude(b_mock, valid_election.id)
+
+    assert tx_vote2.validate(b_mock)
+    assert tx_vote3.validate(b_mock)
+
+    # conclusion can be triggered my different votes in the same block
+    assert ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote2])
+    assert ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote2, tx_vote3])
+
+    b_mock.store_bulk_transactions([tx_vote2])
+
+    # Once the blockchain records >2/3 of the votes the election is assumed to be concluded
+    # so any invocation of `.conclude` for that election should return False
+    assert not ValidatorElection.conclude(b_mock, valid_election.id)
+
+    # Vote is still valid but the election cannot be concluded as it it assmed that it has
+    # been concluded before
+    assert tx_vote3.validate(b_mock)
+    assert not ValidatorElection.conclude(b_mock, valid_election.id, [tx_vote3])
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_get_validator_update(b_mock, valid_election, ed25519_node_keys):
+    # store election
+    b_mock.store_bulk_transactions([valid_election])
+
+    # Node 0: cast vote
+    tx_vote0 = gen_vote(valid_election, 0, ed25519_node_keys)
+    assert b_mock.get_validator_update([tx_vote0]) == []
+    b_mock.store_bulk_transactions([tx_vote0])
+
+    tx_vote1 = gen_vote(valid_election, 1, ed25519_node_keys)
+    assert b_mock.get_validator_update([tx_vote1]) == []
+    b_mock.store_bulk_transactions([tx_vote1])
+
+    # Election can only be concluded once
+    tx_vote2 = gen_vote(valid_election, 2, ed25519_node_keys)
+    assert b_mock.get_validator_update([tx_vote2]) == [valid_election.asset['data']]
+    b_mock.store_bulk_transactions([tx_vote2])
+
+    tx_vote3 = gen_vote(valid_election, 3, ed25519_node_keys)
+    assert b_mock.get_validator_update([tx_vote3]) == []
+
+
+@pytest.mark.abci
+def test_upsert_validator(b, node_key, node_keys, new_validator, ed25519_node_keys):
+    from bigchaindb.backend import connect
+    from bigchaindb.tendermint_utils import public_key_to_base64
+    import time
+    import requests
+
+    conn = connect()
+    (node_pub, _) = list(node_keys.items())[0]
+
+    validators = [{'address': 'some_address',
+                   'pub_key': {'type': 'ed25519',
+                               'data': node_pub},
+                   'voting_power': 10}]
+
+    # reset the validator set
+    conn.run(
+        conn.collection('validators').find_one_and_update(
+            {"height": 1},
+            {"$set": {"validators": validators}}
+        )
+    )
+
+    power = 1
+    public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
+    new_validator = {'public_key': public_key,
+                     'node_id': 'some_node_id',
+                     'power': power}
+
+    voters = ValidatorElection.recipients(b)
+    election = ValidatorElection.generate([node_key.public_key],
+                                          voters,
+                                          new_validator, None).sign([node_key.private_key])
+    code, message = b.write_transaction(election, 'broadcast_tx_commit')
+    assert code == 202
+    time.sleep(3)
+
+    assert b.get_transaction(election.id)
+
+    tx_vote = gen_vote(election, 0, ed25519_node_keys)
+    assert tx_vote.validate(b)
+    code, message = b.write_transaction(tx_vote, 'broadcast_tx_commit')
+    assert code == 202
+    time.sleep(3)
+
+    resp = requests.get(b.endpoint + 'validators')
+    validator_pub_keys = []
+    for v in resp.json()['result']['validators']:
+        validator_pub_keys.append(v['pub_key']['value'])
+
+    assert (public_key_to_base64(public_key) in validator_pub_keys)
+
+
+# ============================================================================
+# Helper functions
+# ============================================================================
+def to_inputs(election, i, ed25519_node_keys):
+    input0 = election.to_inputs()[i]
+    votes = election.outputs[i].amount
+    public_key0 = input0.owners_before[0]
+    key0 = ed25519_node_keys[public_key0]
+    return (input0, votes, key0)
+
+
+def gen_vote(election, i, ed25519_node_keys):
+    (input_i, votes_i, key_i) = to_inputs(election, i, ed25519_node_keys)
+    election_pub_key = ValidatorElection.to_public_key(election.id)
+    return ValidatorElectionVote.generate([input_i],
+                                          [([election_pub_key], votes_i)],
+                                          election_id=election.id)\
+                                .sign([key_i.private_key])