Problem: Cannot conclude validator election (#2445)

Solution: Gather votes and conclude election when supermajority is achieved

bigchaindb/backend/localmongodb/query.py

@@ -8,7 +8,6 @@ from pymongo import DESCENDING
 
 from bigchaindb import backend
 from bigchaindb.backend.exceptions import DuplicateKeyError
-from bigchaindb.common.exceptions import MultipleValidatorOperationError
 from bigchaindb.backend.utils import module_dispatch_registrar
 from bigchaindb.backend.localmongodb.connection import LocalMongoDBConnection
 from bigchaindb.common.transaction import Transaction
@@ -115,7 +114,8 @@ def get_spent(conn, transaction_id, output):
 def get_latest_block(conn):
     return conn.run(
         conn.collection('blocks')
-        .find_one(sort=[('height', DESCENDING)]))
+        .find_one(projection={'_id': False},
+                  sort=[('height', DESCENDING)]))
 
 
 @register_query(LocalMongoDBConnection)
@@ -282,13 +282,15 @@ def get_pre_commit_state(conn, commit_id):
 
 
 @register_query(LocalMongoDBConnection)
-def store_validator_set(conn, validator_update):
+def store_validator_set(conn, validators_update):
-    try:
+    height = validators_update['height']
-        return conn.run(
+    return conn.run(
-            conn.collection('validators')
+        conn.collection('validators').replace_one(
-            .insert_one(validator_update))
+            {'height': height},
-    except DuplicateKeyError:
+            validators_update,
-        raise MultipleValidatorOperationError('Validator update already exists')
+            upsert=True
+        )
+    )
 
 
 @register_query(LocalMongoDBConnection)
@@ -305,3 +307,16 @@ def get_validator_set(conn, height=None):
     )
 
     return list(cursor)[0]
+
+
+@register_query(LocalMongoDBConnection)
+def get_asset_tokens_for_public_key(conn, asset_id, public_key):
+    query = {'outputs.public_keys': [public_key],
+             'asset.id': asset_id}
+
+    cursor = conn.run(
+        conn.collection('transactions').aggregate([
+            {'$match': query},
+            {'$project': {'_id': False}}
+        ]))
+    return cursor

bigchaindb/backend/query.py

@@ -372,3 +372,17 @@ def get_validator_set(conn, height):
     """
 
     raise NotImplementedError
+
+
+@singledispatch
+def get_asset_tokens_for_public_key(connection, asset_id,
+                                    public_key, operation):
+    """Retrieve a list of tokens of type `asset_id` that are owned by the `public_key`.
+    Args:
+        asset_id (str): Id of the token.
+        public_key (str): base58 encoded public key
+        operation: filter transaction based on `operation`
+    Returns:
+        Iterator of transaction that list given owner in conditions.
+    """
+    raise NotImplementedError

bigchaindb/common/schema/transaction_validator_election_vote_v2.0.yaml

@@ -10,7 +10,9 @@ required:
 - operation
 - outputs
 properties:
-  operation: "VALIDATOR_ELECTION_VOTE"
+  operation:
+    type: string
+    value: "VALIDATOR_ELECTION_VOTE"
   outputs:
     type: array
     items:

bigchaindb/core.py

@@ -6,7 +6,6 @@
 with Tendermint.
 """
 import logging
-import codecs
 
 from abci.application import BaseApplication
 from abci.types_pb2 import (
@@ -17,8 +16,6 @@ from abci.types_pb2 import (
     ResponseDeliverTx,
     ResponseEndBlock,
     ResponseCommit,
-    Validator,
-    PubKey
 )
 
 from bigchaindb import BigchainDB
@@ -26,6 +23,8 @@ from bigchaindb.tendermint_utils import (decode_transaction,
                                          calculate_hash)
 from bigchaindb.lib import Block, PreCommitState
 from bigchaindb.backend.query import PRE_COMMIT_ID
+from bigchaindb.upsert_validator import ValidatorElection
+import bigchaindb.upsert_validator.validator_utils as vutils
 
 
 CodeTypeOk = 0
@@ -52,7 +51,7 @@ class App(BaseApplication):
     def init_chain(self, genesis):
         """Initialize chain with block of height 0"""
 
-        validator_set = [decode_validator(v) for v in genesis.validators]
+        validator_set = [vutils.decode_validator(v) for v in genesis.validators]
         block = Block(app_hash='', height=0, transactions=[])
         self.bigchaindb.store_block(block._asdict())
         self.bigchaindb.store_validator_set(1, validator_set)
@@ -141,11 +140,11 @@ class App(BaseApplication):
         else:
             self.block_txn_hash = block['app_hash']
 
-        # TODO: calculate if an election has concluded
+        # Check if the current block concluded any validator elections and
-        # NOTE: ensure the local validator set is updated
+        # update the locally tracked validator set
-        # validator_updates = self.bigchaindb.get_validator_update()
+        validator_updates = ValidatorElection.get_validator_update(self.bigchaindb,
-        # validator_updates = [encode_validator(v) for v in validator_updates]
+                                                                   self.new_height,
-        validator_updates = []
+                                                                   self.block_transactions)
 
         # Store pre-commit state to recover in case there is a crash
         # during `commit`
@@ -176,22 +175,3 @@ class App(BaseApplication):
                      self.block_txn_ids)
         logger.benchmark('COMMIT_BLOCK, height:%s', self.new_height)
         return ResponseCommit(data=data)
-
-
-def encode_validator(v):
-    ed25519_public_key = v['pub_key']['data']
-    # NOTE: tendermint expects public to be encoded in go-amino format
-
-    pub_key = PubKey(type='ed25519',
-                     data=bytes.fromhex(ed25519_public_key))
-
-    return Validator(pub_key=pub_key,
-                     address=b'',
-                     power=v['power'])
-
-
-def decode_validator(v):
-    return {'address': codecs.encode(v.address, 'hex').decode().upper().rstrip('\n'),
-            'pub_key': {'type': v.pub_key.type,
-                        'data': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
-            'voting_power': v.power}

bigchaindb/lib.py

@@ -29,6 +29,7 @@ from bigchaindb.tendermint_utils import encode_transaction, merkleroot
 from bigchaindb import exceptions as core_exceptions
 from bigchaindb.consensus import BaseConsensusRules
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -442,16 +443,8 @@ class BigchainDB(object):
     def get_validators(self, height=None):
         result = backend.query.get_validator_set(self.connection, height)
         validators = result['validators']
-        for v in validators:
-            v.pop('address')
-            v['voting_power'] = int(v['voting_power'])
-
         return validators
 
-    def get_validator_update(self):
-        update = backend.query.get_validator_update(self.connection)
-        return [update['validator']] if update else []
-
     def delete_validator_update(self):
         return backend.query.delete_validator_update(self.connection)
 

bigchaindb/upsert_validator/__init__.py

@@ -3,5 +3,5 @@
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
 
-from bigchaindb.upsert_validator.validator_election import ValidatorElection # noqa
 from bigchaindb.upsert_validator.validator_election_vote import ValidatorElectionVote # noqa
+from bigchaindb.upsert_validator.validator_election import ValidatorElection # noqa

bigchaindb/upsert_validator/validator_election.py

@@ -2,6 +2,9 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
+import base58
+
+from bigchaindb import backend
 from bigchaindb.common.exceptions import (InvalidSignature,
                                           MultipleInputsError,
                                           InvalidProposer,
@@ -15,6 +18,8 @@ from bigchaindb.common.schema import (_validate_schema,
                                       TX_SCHEMA_VALIDATOR_ELECTION,
                                       TX_SCHEMA_COMMON,
                                       TX_SCHEMA_CREATE)
+from . import ValidatorElectionVote
+from .validator_utils import (new_validator_set, encode_validator)
 
 
 class ValidatorElection(Transaction):
@@ -32,15 +37,15 @@ class ValidatorElection(Transaction):
         super().__init__(operation, asset, inputs, outputs, metadata, version, hash_id)
 
     @classmethod
-    def current_validators(cls, bigchain):
+    def get_validators(cls, bigchain, height=None):
         """Return a dictionary of validators with key as `public_key` and
            value as the `voting_power`
         """
 
         validators = {}
-        for validator in bigchain.get_validators():
+        for validator in bigchain.get_validators(height):
             # NOTE: we assume that Tendermint encodes public key in base64
-            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['value']))
+            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['data']))
             validators[public_key] = validator['voting_power']
 
         return validators
@@ -50,7 +55,7 @@ class ValidatorElection(Transaction):
         """Convert validator dictionary to a recipient list for `Transaction`"""
 
         recipients = []
-        for public_key, voting_power in cls.current_validators(bigchain).items():
+        for public_key, voting_power in cls.get_validators(bigchain).items():
             recipients.append(([public_key], voting_power))
 
         return recipients
@@ -84,7 +89,7 @@ class ValidatorElection(Transaction):
             bigchain (BigchainDB): an instantiated bigchaindb.lib.BigchainDB object.
 
         Returns:
-            `True` if the election is valid
+            ValidatorElection object
 
         Raises:
             ValidationError: If the election is invalid
@@ -99,7 +104,7 @@ class ValidatorElection(Transaction):
         if not self.inputs_valid(input_conditions):
             raise InvalidSignature('Transaction signature is invalid.')
 
-        current_validators = self.current_validators(bigchain)
+        current_validators = self.get_validators(bigchain)
 
         # NOTE: Proposer should be a single node
         if len(self.inputs) != 1 or len(self.inputs[0].owners_before) != 1:
@@ -118,7 +123,7 @@ class ValidatorElection(Transaction):
         if not self.is_same_topology(current_validators, self.outputs):
             raise UnequalValidatorSet('Validator set much be exactly same to the outputs of election')
 
-        return True
+        return self
 
     @classmethod
     def generate(cls, initiator, voters, election_data, metadata=None):
@@ -145,3 +150,77 @@ class ValidatorElection(Transaction):
     @classmethod
     def transfer(cls, tx_signers, recipients, metadata=None, asset=None):
         raise NotImplementedError
+
+    @classmethod
+    def to_public_key(cls, election_id):
+        return base58.b58encode(bytes.fromhex(election_id))
+
+    @classmethod
+    def count_votes(cls, election_pk, transactions, getter=getattr):
+        votes = 0
+        for txn in transactions:
+            if getter(txn, 'operation') == 'VALIDATOR_ELECTION_VOTE':
+                for output in getter(txn, 'outputs'):
+                    # NOTE: We enforce that a valid vote to election id will have only
+                    # election_pk in the output public keys, including any other public key
+                    # along with election_pk will lead to vote being not considered valid.
+                    if len(getter(output, 'public_keys')) == 1 and [election_pk] == getter(output, 'public_keys'):
+                        votes = votes + int(getter(output, 'amount'))
+        return votes
+
+    def get_commited_votes(self, bigchain, election_pk=None):
+        if election_pk is None:
+            election_pk = self.to_public_key(self.id)
+        txns = list(backend.query.get_asset_tokens_for_public_key(bigchain.connection,
+                                                                  self.id,
+                                                                  election_pk))
+        return self.count_votes(election_pk, txns, dict.get)
+
+    @classmethod
+    def has_concluded(cls, bigchain, election_id, current_votes=[], height=None):
+        """Check if the given `election_id` can be concluded or not
+        NOTE:
+        * Election is concluded iff the current validator set is exactly equal
+          to the validator set encoded in election outputs
+        * Election can concluded only if the current votes achieves a supermajority
+        """
+        election = bigchain.get_transaction(election_id)
+
+        if election:
+            election_pk = election.to_public_key(election.id)
+            votes_commited = election.get_commited_votes(bigchain, election_pk)
+            votes_current = election.count_votes(election_pk, current_votes)
+            current_validators = election.get_validators(bigchain, height)
+
+            if election.is_same_topology(current_validators, election.outputs):
+                total_votes = sum(current_validators.values())
+                if (votes_commited < (2/3)*total_votes) and \
+                   (votes_commited + votes_current >= (2/3)*total_votes):
+                    return election
+        return False
+
+    @classmethod
+    def get_validator_update(cls, bigchain, new_height, txns):
+        votes = {}
+        for txn in txns:
+            if not isinstance(txn, ValidatorElectionVote):
+                continue
+
+            election_id = txn.asset['id']
+            election_votes = votes.get(election_id, [])
+            election_votes.append(txn)
+            votes[election_id] = election_votes
+
+            election = cls.has_concluded(bigchain, election_id, election_votes, new_height)
+            # Once an election concludes any other conclusion for the same
+            # or any other election is invalidated
+            if election:
+                # The new validator set comes into effect from height = new_height+1
+                validator_updates = [election.asset['data']]
+                curr_validator_set = bigchain.get_validators(new_height)
+                updated_validator_set = new_validator_set(curr_validator_set,
+                                                          new_height, validator_updates)
+
+                bigchain.store_validator_set(new_height+1, updated_validator_set)
+                return [encode_validator(election.asset['data'])]
+        return []

bigchaindb/upsert_validator/validator_election_vote.py

@@ -2,8 +2,6 @@
 # SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
-import base58
-
 from bigchaindb.common.transaction import Transaction
 from bigchaindb.common.schema import (_validate_schema,
                                       TX_SCHEMA_COMMON,
@@ -30,7 +28,7 @@ class ValidatorElectionVote(Transaction):
             bigchain (BigchainDB): an instantiated bigchaindb.lib.BigchainDB object.
 
         Returns:
-            `True` if the election vote is valid
+            ValidatorElectionVote object
 
         Raises:
             ValidationError: If the election vote is invalid
@@ -38,10 +36,6 @@ class ValidatorElectionVote(Transaction):
         self.validate_transfer_inputs(bigchain, current_transactions)
         return self
 
-    @classmethod
-    def to_public_key(cls, election_id):
-        return base58.b58encode(bytes.fromhex(election_id))
-
     @classmethod
     def generate(cls, inputs, recipients, election_id, metadata=None):
         (inputs, outputs) = cls.validate_transfer(inputs, recipients, election_id, metadata)

bigchaindb/upsert_validator/validator_utils.py

@@ -0,0 +1,37 @@
+import codecs
+
+from abci.types_pb2 import (Validator,
+                            PubKey)
+from bigchaindb.tendermint_utils import public_key_to_base64
+
+
+def encode_validator(v):
+    ed25519_public_key = v['public_key']
+    # NOTE: tendermint expects public to be encoded in go-amino format
+    pub_key = PubKey(type='ed25519',
+                     data=bytes.fromhex(ed25519_public_key))
+    return Validator(pub_key=pub_key,
+                     address=b'',
+                     power=v['power'])
+
+
+def decode_validator(v):
+    return {'pub_key': {'type': v.pub_key.type,
+                        'data': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
+            'voting_power': v.power}
+
+
+def new_validator_set(validators, height, updates):
+    validators_dict = {}
+    for v in validators:
+        validators_dict[v['pub_key']['data']] = v
+
+    updates_dict = {}
+    for u in updates:
+        public_key64 = public_key_to_base64(u['public_key'])
+        updates_dict[public_key64] = {'pub_key': {'type': 'ed25519',
+                                                  'data': public_key64},
+                                      'voting_power': u['power']}
+
+    new_validators_dict = {**validators_dict, **updates_dict}
+    return list(new_validators_dict.values())

tests/commands/test_commands.py

@@ -388,9 +388,9 @@ def test_upsert_validator_new_with_tendermint(b, priv_validator_path, user_sk, m
 def test_upsert_validator_new_without_tendermint(b, priv_validator_path, user_sk, monkeypatch):
     from bigchaindb.commands.bigchaindb import run_upsert_validator_new
 
-    def mock_get():
+    def mock_get(height):
         return [
-            {'pub_key': {'value': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=',
+            {'pub_key': {'data': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=',
                          'type': 'tendermint/PubKeyEd25519'},
              'voting_power': 10}
         ]
@@ -402,8 +402,6 @@ def test_upsert_validator_new_without_tendermint(b, priv_validator_path, user_sk
     b.get_validators = mock_get
     b.write_transaction = mock_write
 
-    monkeypatch.setattr('requests.get', mock_get)
-
     args = Namespace(action='new',
                      public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=',
                      power=1,

tests/tendermint/test_core.py

@@ -10,7 +10,11 @@ from abci.types_pb2 import (
     RequestEndBlock
 )
 
-from bigchaindb.core import CodeTypeOk, CodeTypeError
+from bigchaindb.core import (CodeTypeOk,
+                             CodeTypeError,
+                             )
+from bigchaindb.upsert_validator.validator_utils import new_validator_set
+from bigchaindb.tendermint_utils import public_key_to_base64
 
 
 pytestmark = [pytest.mark.tendermint, pytest.mark.bdb]
@@ -220,3 +224,26 @@ def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
     assert resp['commit_id'] == PRE_COMMIT_ID
     assert resp['height'] == 100
     assert resp['transactions'] == [tx.id]
+
+
+def test_new_validator_set(b):
+    node1 = {'pub_key': {'type': 'ed25519',
+                         'data': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
+             'voting_power': 10}
+    node1_new_power = {'public_key': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+                       'power': 20}
+    node2 = {'public_key': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+             'power': 10}
+
+    validators = [node1]
+    updates = [node1_new_power, node2]
+    b.store_validator_set(1, validators)
+    updated_validator_set = new_validator_set(b.get_validators(1), 1, updates)
+
+    updated_validators = []
+    for u in updates:
+        updated_validators.append({'pub_key': {'type': 'ed25519',
+                                               'data': public_key_to_base64(u['public_key'])},
+                                   'voting_power':  u['power']})
+
+    assert updated_validator_set == updated_validators

tests/tendermint/test_integration.py

@@ -11,7 +11,6 @@ import pytest
 
 from abci.server import ProtocolHandler
 from abci.encoding import read_messages
-from copy import deepcopy
 from io import BytesIO
 
 
@@ -109,43 +108,6 @@ def test_app(tb, init_chain_request):
     assert block0['app_hash'] == new_block_hash
 
 
-@pytest.mark.skip
-@pytest.mark.abci
-def test_upsert_validator(b, alice):
-    from bigchaindb.backend.query import VALIDATOR_UPDATE_ID
-    from bigchaindb.backend import query, connect
-    from bigchaindb.models import Transaction
-    from bigchaindb.tendermint_utils import public_key_to_base64
-    import time
-
-    conn = connect()
-    power = 1
-    public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
-
-    validator = {'pub_key': {'type': 'AC26791624DE60',
-                             'data': public_key},
-                 'power': power}
-    validator_update = {'validator': validator,
-                        'update_id': VALIDATOR_UPDATE_ID}
-
-    query.store_validator_update(conn, deepcopy(validator_update))
-
-    tx = Transaction.create([alice.public_key],
-                            [([alice.public_key], 1)],
-                            asset=None)\
-                    .sign([alice.private_key])
-
-    code, message = b.write_transaction(tx, 'broadcast_tx_commit')
-    assert code == 202
-    time.sleep(5)
-
-    validators = b.get_validators()
-    validators = [(v['pub_key']['value'], v['voting_power']) for v in validators]
-
-    public_key64 = public_key_to_base64(public_key)
-    assert ((public_key64, str(power)) in validators)
-
-
 @pytest.mark.abci
 def test_post_transaction_responses(tendermint_ws_url, b):
     from bigchaindb.common.crypto import generate_key_pair

tests/upsert_validator/conftest.py

@@ -26,11 +26,11 @@ def new_validator():
 
 
 def mock_get_validators(network_validators):
-    def validator_set():
+    def validator_set(height):
         validators = []
         for public_key, power in network_validators.items():
             validators.append({
-                'pub_key': {'type': 'AC26791624DE60', 'value': public_key},
+                'pub_key': {'type': 'AC26791624DE60', 'data': public_key},
                 'voting_power': power
             })
         return validators
@@ -44,3 +44,11 @@ def valid_election(b_mock, node_key, new_validator):
     return ValidatorElection.generate([node_key.public_key],
                                       voters,
                                       new_validator, None).sign([node_key.private_key])
+
+
+@pytest.fixture
+def valid_election_b(b, node_key, new_validator):
+    voters = ValidatorElection.recipients(b)
+    return ValidatorElection.generate([node_key.public_key],
+                                      voters,
+                                      new_validator, None).sign([node_key.private_key])

tests/upsert_validator/test_validator_election_vote.py

@@ -3,14 +3,20 @@
 # Code is Apache-2.0 and docs are CC-BY-4.0
 
 import pytest
+import codecs
 
-from bigchaindb.upsert_validator import ValidatorElectionVote
+from bigchaindb.tendermint_utils import public_key_to_base64
+from bigchaindb.upsert_validator import ValidatorElection, ValidatorElectionVote
 from bigchaindb.common.exceptions import AmountError
+from bigchaindb.common.crypto import generate_key_pair
+from bigchaindb.common.exceptions import ValidationError
 
 
-pytestmark = [pytest.mark.tendermint, pytest.mark.bdb]
+pytestmark = [pytest.mark.execute]
 
 
+@pytest.mark.tendermint
+@pytest.mark.bdb
 def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_node_keys):
     b_mock.store_bulk_transactions([valid_election])
 
@@ -19,7 +25,7 @@ def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_no
     public_key0 = input0.owners_before[0]
     key0 = ed25519_node_keys[public_key0]
 
-    election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id)
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
 
     vote = ValidatorElectionVote.generate([input0],
                                           [([election_pub_key], votes)],
@@ -28,9 +34,29 @@ def test_upsert_validator_valid_election_vote(b_mock, valid_election, ed25519_no
     assert vote.validate(b_mock)
 
 
-def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519_node_keys):
+@pytest.mark.tendermint
-    from bigchaindb.common.crypto import generate_key_pair
+@pytest.mark.bdb
+def test_upsert_validator_valid_non_election_vote(b_mock, valid_election, ed25519_node_keys):
+    b_mock.store_bulk_transactions([valid_election])
 
+    input0 = valid_election.to_inputs()[0]
+    votes = valid_election.outputs[0].amount
+    public_key0 = input0.owners_before[0]
+    key0 = ed25519_node_keys[public_key0]
+
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
+
+    # Ensure that threshold conditions are now allowed
+    with pytest.raises(ValidationError):
+        ValidatorElectionVote.generate([input0],
+                                       [([election_pub_key, key0.public_key], votes)],
+                                       election_id=valid_election.id)\
+                             .sign([key0.private_key])
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519_node_keys):
     alice = generate_key_pair()
 
     b_mock.store_bulk_transactions([valid_election])
@@ -48,7 +74,7 @@ def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519
     assert delegate_vote.validate(b_mock)
 
     b_mock.store_bulk_transactions([delegate_vote])
-    election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id)
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
 
     alice_votes = delegate_vote.to_inputs()[0]
     alice_casted_vote = ValidatorElectionVote.generate([alice_votes],
@@ -65,6 +91,8 @@ def test_upsert_validator_delegate_election_vote(b_mock, valid_election, ed25519
     assert key0_casted_vote.validate(b_mock)
 
 
+@pytest.mark.tendermint
+@pytest.mark.bdb
 def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_node_keys):
     b_mock.store_bulk_transactions([valid_election])
 
@@ -73,7 +101,7 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_
     public_key0 = input0.owners_before[0]
     key0 = ed25519_node_keys[public_key0]
 
-    election_pub_key = ValidatorElectionVote.to_public_key(valid_election.id)
+    election_pub_key = ValidatorElection.to_public_key(valid_election.id)
 
     vote = ValidatorElectionVote.generate([input0],
                                           [([election_pub_key], votes+1)],
@@ -82,3 +110,233 @@ def test_upsert_validator_invalid_election_vote(b_mock, valid_election, ed25519_
 
     with pytest.raises(AmountError):
         assert vote.validate(b_mock)
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_valid_election_votes_received(b_mock, valid_election, ed25519_node_keys):
+    alice = generate_key_pair()
+    b_mock.store_bulk_transactions([valid_election])
+    assert valid_election.get_commited_votes(b_mock) == 0
+
+    input0 = valid_election.to_inputs()[0]
+    votes = valid_election.outputs[0].amount
+    public_key0 = input0.owners_before[0]
+    key0 = ed25519_node_keys[public_key0]
+
+    # delegate some votes to alice
+    delegate_vote = ValidatorElectionVote.generate([input0],
+                                                   [([alice.public_key], 4), ([key0.public_key], votes-4)],
+                                                   election_id=valid_election.id)\
+                                         .sign([key0.private_key])
+    b_mock.store_bulk_transactions([delegate_vote])
+    assert valid_election.get_commited_votes(b_mock) == 0
+
+    election_public_key = ValidatorElection.to_public_key(valid_election.id)
+    alice_votes = delegate_vote.to_inputs()[0]
+    key0_votes = delegate_vote.to_inputs()[1]
+
+    alice_casted_vote = ValidatorElectionVote.generate([alice_votes],
+                                                       [([election_public_key], 2), ([alice.public_key], 2)],
+                                                       election_id=valid_election.id)\
+                                             .sign([alice.private_key])
+
+    assert alice_casted_vote.validate(b_mock)
+    b_mock.store_bulk_transactions([alice_casted_vote])
+
+    # Check if the delegated vote is count as valid vote
+    assert valid_election.get_commited_votes(b_mock) == 2
+
+    key0_casted_vote = ValidatorElectionVote.generate([key0_votes],
+                                                      [([election_public_key], votes-4)],
+                                                      election_id=valid_election.id)\
+                                            .sign([key0.private_key])
+
+    assert key0_casted_vote.validate(b_mock)
+    b_mock.store_bulk_transactions([key0_casted_vote])
+
+    assert valid_election.get_commited_votes(b_mock) == votes-2
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_valid_election_conclude(b_mock, valid_election, ed25519_node_keys):
+
+    # Node 0: cast vote
+    tx_vote0 = gen_vote(valid_election, 0, ed25519_node_keys)
+
+    # check if the vote is valid even before the election doesn't exist
+    with pytest.raises(ValidationError):
+        assert tx_vote0.validate(b_mock)
+
+    # store election
+    b_mock.store_bulk_transactions([valid_election])
+    # cannot conclude election as not votes exist
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id)
+
+    # validate vote
+    assert tx_vote0.validate(b_mock)
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote0])
+
+    b_mock.store_bulk_transactions([tx_vote0])
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id)
+
+    # Node 1: cast vote
+    tx_vote1 = gen_vote(valid_election, 1, ed25519_node_keys)
+
+    # Node 2: cast vote
+    tx_vote2 = gen_vote(valid_election, 2, ed25519_node_keys)
+
+    # Node 3: cast vote
+    tx_vote3 = gen_vote(valid_election, 3, ed25519_node_keys)
+
+    assert tx_vote1.validate(b_mock)
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote1])
+
+    # 2/3 is achieved in the same block so the election can be.has_concludedd
+    assert ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote1, tx_vote2])
+
+    b_mock.store_bulk_transactions([tx_vote1])
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id)
+
+    assert tx_vote2.validate(b_mock)
+    assert tx_vote3.validate(b_mock)
+
+    # conclusion can be triggered my different votes in the same block
+    assert ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote2])
+    assert ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote2, tx_vote3])
+
+    b_mock.store_bulk_transactions([tx_vote2])
+
+    # Once the blockchain records >2/3 of the votes the election is assumed to be.has_concludedd
+    # so any invocation of `.has_concluded` for that election should return False
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id)
+
+    # Vote is still valid but the election cannot be.has_concludedd as it it assmed that it has
+    # been.has_concludedd before
+    assert tx_vote3.validate(b_mock)
+    assert not ValidatorElection.has_concluded(b_mock, valid_election.id, [tx_vote3])
+
+
+@pytest.mark.abci
+def test_upsert_validator(b, node_key, node_keys, new_validator, ed25519_node_keys):
+    import time
+    import requests
+
+    (node_pub, _) = list(node_keys.items())[0]
+
+    validators = [{'pub_key': {'type': 'ed25519',
+                               'data': node_pub},
+                   'voting_power': 10}]
+
+    latest_block = b.get_latest_block()
+    # reset the validator set
+    b.store_validator_set(latest_block['height'], validators)
+
+    power = 1
+    public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
+    public_key64 = public_key_to_base64(public_key)
+    new_validator = {'public_key': public_key,
+                     'node_id': 'some_node_id',
+                     'power': power}
+
+    voters = ValidatorElection.recipients(b)
+    election = ValidatorElection.generate([node_key.public_key],
+                                          voters,
+                                          new_validator, None).sign([node_key.private_key])
+    code, message = b.write_transaction(election, 'broadcast_tx_commit')
+    assert code == 202
+    time.sleep(3)
+
+    assert b.get_transaction(election.id)
+
+    tx_vote = gen_vote(election, 0, ed25519_node_keys)
+    assert tx_vote.validate(b)
+    code, message = b.write_transaction(tx_vote, 'broadcast_tx_commit')
+    assert code == 202
+    time.sleep(3)
+
+    resp = requests.get(b.endpoint + 'validators')
+    validator_pub_keys = []
+    for v in resp.json()['result']['validators']:
+        validator_pub_keys.append(v['pub_key']['value'])
+
+    assert (public_key64 in validator_pub_keys)
+    new_validator_set = b.get_validators()
+    validator_pub_keys = []
+    for v in new_validator_set:
+        validator_pub_keys.append(v['pub_key']['data'])
+
+    assert (public_key64 in validator_pub_keys)
+
+
+@pytest.mark.tendermint
+@pytest.mark.bdb
+def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
+    reset_validator_set(b, node_keys, 1)
+
+    power = 1
+    public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
+    public_key64 = public_key_to_base64(public_key)
+    new_validator = {'public_key': public_key,
+                     'node_id': 'some_node_id',
+                     'power': power}
+    voters = ValidatorElection.recipients(b)
+    election = ValidatorElection.generate([node_key.public_key],
+                                          voters,
+                                          new_validator).sign([node_key.private_key])
+    # store election
+    b.store_bulk_transactions([election])
+
+    tx_vote0 = gen_vote(election, 0, ed25519_node_keys)
+    tx_vote1 = gen_vote(election, 1, ed25519_node_keys)
+    tx_vote2 = gen_vote(election, 2, ed25519_node_keys)
+
+    assert not ValidatorElection.has_concluded(b, election.id, [tx_vote0])
+    assert not ValidatorElection.has_concluded(b, election.id, [tx_vote0, tx_vote1])
+    assert ValidatorElection.has_concluded(b, election.id, [tx_vote0, tx_vote1, tx_vote2])
+
+    assert ValidatorElection.get_validator_update(b, 4, [tx_vote0]) == []
+    assert ValidatorElection.get_validator_update(b, 4, [tx_vote0, tx_vote1]) == []
+
+    update = ValidatorElection.get_validator_update(b, 4, [tx_vote0, tx_vote1, tx_vote2])
+    update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n')
+    assert len(update) == 1
+    assert update_public_key == public_key64
+
+    b.store_bulk_transactions([tx_vote0, tx_vote1])
+
+    update = ValidatorElection.get_validator_update(b, 4, [tx_vote2])
+    print('update', update)
+    update_public_key = codecs.encode(update[0].pub_key.data, 'base64').decode().rstrip('\n')
+    assert len(update) == 1
+    assert update_public_key == public_key64
+
+
+# ============================================================================
+# Helper functions
+# ============================================================================
+def to_inputs(election, i, ed25519_node_keys):
+    input0 = election.to_inputs()[i]
+    votes = election.outputs[i].amount
+    public_key0 = input0.owners_before[0]
+    key0 = ed25519_node_keys[public_key0]
+    return (input0, votes, key0)
+
+
+def gen_vote(election, i, ed25519_node_keys):
+    (input_i, votes_i, key_i) = to_inputs(election, i, ed25519_node_keys)
+    election_pub_key = ValidatorElection.to_public_key(election.id)
+    return ValidatorElectionVote.generate([input_i],
+                                          [([election_pub_key], votes_i)],
+                                          election_id=election.id)\
+                                .sign([key_i.private_key])
+
+
+def reset_validator_set(b, node_keys, height):
+    validators = []
+    for (node_pub, _) in node_keys.items():
+        validators.append({'pub_key': {'type': 'ed25519',
+                                       'data': node_pub},
+                           'voting_power': 10})
+    b.store_validator_set(height, validators)