Rework upsert-validator show status (#2496)

* Problem: We need to store the `election_id` as part of the `validator_update` so we can efficiently check which election was resposible for the change

Solution: Added the parameter to `store_validator_set` and aligned the tests

* Problem: Logic for `upsert-validator show` is convoluted

Solution: Rewrote the function to be much simpler

* Problem: Need a uniqueness constraint for election_id wrt validator changes

Solution: Added a new key to the db schema

bigchaindb/backend/localmongodb/query.py

@@ -299,6 +299,18 @@ def get_validator_set(conn, height=None):
     return list(cursor)[0]
 
 
+@register_query(LocalMongoDBConnection)
+def get_validator_set_by_election_id(conn, election_id):
+    query = {'election_id': election_id}
+
+    cursor = conn.run(
+        conn.collection('validators')
+        .find(query, projection={'_id': False})
+    )
+
+    return next(cursor, None)
+
+
 @register_query(LocalMongoDBConnection)
 def get_asset_tokens_for_public_key(conn, asset_id, public_key):
     query = {'outputs.public_keys': [public_key],

bigchaindb/backend/localmongodb/schema.py

@@ -133,3 +133,6 @@ def create_validators_secondary_index(conn, dbname):
     conn.conn[dbname]['validators'].create_index('height',
                                                  name='height',
                                                  unique=True,)
+    conn.conn[dbname]['validators'].create_index('election_id',
+                                                 name='election_id',
+                                                 unique=True,)

bigchaindb/backend/query.py

@@ -360,6 +360,14 @@ def get_validator_set(conn, height):
     raise NotImplementedError
 
 
+@singledispatch
+def get_validator_set_by_election_id(conn, election_id):
+    """Return a validator set change with the specified election_id
+    """
+
+    raise NotImplementedError
+
+
 @singledispatch
 def get_asset_tokens_for_public_key(connection, asset_id,
                                     public_key, operation):

bigchaindb/core.py

@@ -54,7 +54,7 @@ class App(BaseApplication):
         validator_set = [vutils.decode_validator(v) for v in genesis.validators]
         block = Block(app_hash='', height=0, transactions=[])
         self.bigchaindb.store_block(block._asdict())
-        self.bigchaindb.store_validator_set(1, validator_set)
+        self.bigchaindb.store_validator_set(1, validator_set, None)
         return ResponseInitChain()
 
     def info(self, request):

bigchaindb/lib.py

@@ -421,24 +421,32 @@ class BigchainDB(object):
     def fastquery(self):
         return fastquery.FastQuery(self.connection)
 
+    def get_validator_change(self, height=None):
+        return backend.query.get_validator_set(self.connection, height)
+
     def get_validators(self, height=None):
-        result = backend.query.get_validator_set(self.connection, height)
+        result = self.get_validator_change(height)
         validators = result['validators']
         return validators
 
+    def get_validators_by_election_id(self, election_id):
+        result = backend.query.get_validator_set_by_election_id(self.connection, election_id)
+        return result
+
     def delete_validator_update(self):
         return backend.query.delete_validator_update(self.connection)
 
     def store_pre_commit_state(self, state):
         return backend.query.store_pre_commit_state(self.connection, state)
 
-    def store_validator_set(self, height, validators):
+    def store_validator_set(self, height, validators, election_id):
         """Store validator set at a given `height`.
            NOTE: If the validator set already exists at that `height` then an
            exception will be raised.
         """
         return backend.query.store_validator_set(self.connection, {'height': height,
-                                                                   'validators': validators})
+                                                                   'validators': validators,
+                                                                   'election_id': election_id})
 
 
 Block = namedtuple('Block', ('app_hash', 'height', 'transactions'))

bigchaindb/upsert_validator/validator_election.py

@@ -5,7 +5,6 @@
 import base58
 
 from bigchaindb import backend
-from bigchaindb.backend.localmongodb.query import get_asset_tokens_for_public_key
 from bigchaindb.common.exceptions import (InvalidSignature,
                                           MultipleInputsError,
                                           InvalidProposer,
@@ -42,6 +41,21 @@ class ValidatorElection(Transaction):
         # of `CREATE` and any validation on `CREATE` in the parent class should apply to it
         super().__init__(operation, asset, inputs, outputs, metadata, version, hash_id)
 
+    @classmethod
+    def get_validator_change(cls, bigchain, height=None):
+        """Return the latest change to the validator set
+
+        :return: {
+            'height': <block_height>,
+            'asset': {
+                'height': <block_height>,
+                'validators': <validator_set>,
+                'election_id': <election_id_that_approved_the_change>
+            }
+        }
+        """
+        return bigchain.get_validator_change(height)
+
     @classmethod
     def get_validators(cls, bigchain, height=None):
         """Return a dictionary of validators with key as `public_key` and
@@ -227,55 +241,24 @@ class ValidatorElection(Transaction):
                                                           validator_updates)
 
                 updated_validator_set = [v for v in updated_validator_set if v['voting_power'] > 0]
-                bigchain.store_validator_set(new_height+1, updated_validator_set)
+                bigchain.store_validator_set(new_height+1, updated_validator_set, election.id)
                 return [encode_validator(election.asset['data'])]
         return []
 
-    def _vote_ratio(self, bigchain, height):
-        cast_votes = self._get_vote_ids(bigchain)
-        votes = [(tx['outputs'][0]['amount'], bigchain.get_block_containing_tx(tx['id'])[0]) for tx in cast_votes]
-        votes_cast = [int(vote[0]) for vote in votes if vote[1] <= height]
-        total_votes_cast = sum(votes_cast)
-        total_votes = sum([voter.amount for voter in self.outputs])
-        vote_ratio = total_votes_cast/total_votes
-        return vote_ratio
+    def get_validator_update_by_election_id(self, election_id, bigchain):
+        result = bigchain.get_validators_by_election_id(election_id)
+        return result
 
-    def _get_vote_ids(self, bigchain):
-        election_key = self.to_public_key(self.id)
-        votes = get_asset_tokens_for_public_key(bigchain.connection, self.id, election_key)
-        return votes
-
-    def initial_height(self, bigchain):
-        heights = bigchain.get_block_containing_tx(self.id)
-        initial_height = 0
-        if len(heights) != 0:
-            initial_height = min(bigchain.get_block_containing_tx(self.id))
-        return initial_height
-
-    def get_status(self, bigchain, height=None):
-
-        initial_validators = self.get_validators(bigchain, height=self.initial_height(bigchain))
-
-        # get all heights where a vote was cast
-        vote_heights = set([bigchain.get_block_containing_tx(tx['id'])[0] for tx in self._get_vote_ids(bigchain)])
-
-        # find the least height where the vote succeeds
-        confirmation_height = None
-        confirmed_heights = [h for h in vote_heights if self._vote_ratio(bigchain, h) > self.ELECTION_THRESHOLD]
-        if height:
-            confirmed_heights = [h for h in confirmed_heights if h <= height]
-        if len(confirmed_heights) > 0:
-            confirmation_height = min(confirmed_heights)
-
-        # get the validator set at the confirmation height/current height
-        if confirmation_height:
-            final_validators = self.get_validators(bigchain, height=confirmation_height)
-        else:
-            final_validators = self.get_validators(bigchain)
-
-        if initial_validators != final_validators:
-            return self.INCONCLUSIVE
-        elif confirmation_height:
+    def get_status(self, bigchain):
+        concluded = self.get_validator_update_by_election_id(self.id, bigchain)
+        if concluded:
             return self.CONCLUDED
+
+        latest_change = self.get_validator_change(bigchain)
+        latest_change_height = latest_change['height']
+        election_height = bigchain.get_block_containing_tx(self.id)[0]
+
+        if latest_change_height >= election_height:
+            return self.INCONCLUSIVE
         else:
             return self.ONGOING

tests/backend/localmongodb/test_queries.py

@@ -380,7 +380,7 @@ def test_validator_update():
     conn = connect()
 
     def gen_validator_update(height):
-        return {'data': 'somedata', 'height': height}
+        return {'data': 'somedata', 'height': height, 'election_id': f'election_id_at_height_{height}'}
 
     for i in range(1, 100, 10):
         value = gen_validator_update(i)

tests/backend/localmongodb/test_schema.py

@@ -44,7 +44,7 @@ def test_init_creates_db_tables_and_indexes():
     assert set(indexes) == {'_id_', 'pre_commit_id'}
 
     indexes = conn.conn[dbname]['validators'].index_information().keys()
-    assert set(indexes) == {'_id_', 'height'}
+    assert set(indexes) == {'_id_', 'height', 'election_id'}
 
 
 def test_init_database_fails_if_db_exists():

tests/conftest.py

@@ -631,6 +631,10 @@ def bad_validator_path(node_keys):
 @pytest.fixture
 def validators(b, node_keys):
     from bigchaindb.backend import query
+    import time
+
+    def timestamp():  # we need this to force unique election_ids for setup and teardown of fixtures
+        return str(time.time())
 
     height = get_block_height(b)
 
@@ -645,7 +649,8 @@ def validators(b, node_keys):
                       'voting_power': 10}]
 
     validator_update = {'validators': validator_set,
-                        'height': height + 1}
+                        'height': height + 1,
+                        'election_id': f'setup_at_{timestamp()}'}
 
     query.store_validator_set(b.connection, validator_update)
 
@@ -654,7 +659,8 @@ def validators(b, node_keys):
     height = get_block_height(b)
 
     validator_update = {'validators': original_validators,
-                        'height': height}
+                        'height': height,
+                        'election_id': f'teardown_at_{timestamp()}'}
 
     query.store_validator_set(b.connection, validator_update)
 

tests/tendermint/test_core.py

@@ -237,7 +237,7 @@ def test_new_validator_set(b):
 
     validators = [node1]
     updates = [node1_new_power, node2]
-    b.store_validator_set(1, validators)
+    b.store_validator_set(1, validators, 'election_id')
     updated_validator_set = new_validator_set(b.get_validators(1), updates)
 
     updated_validators = []

tests/upsert_validator/conftest.py

@@ -48,33 +48,39 @@ def valid_election_b(b, node_key, new_validator):
 
 @pytest.fixture
 def ongoing_election(b, valid_election, ed25519_node_keys):
+    validators = b.get_validators(height=1)
+    genesis_validators = {'validators': validators,
+                          'height': 0,
+                          'election_id': None}
+    query.store_validator_set(b.connection, genesis_validators)
+
     b.store_bulk_transactions([valid_election])
     block_1 = Block(app_hash='hash_1', height=1, transactions=[valid_election.id])
-    vote_0 = vote(valid_election, 0, ed25519_node_keys, b)
-    vote_1 = vote(valid_election, 1, ed25519_node_keys, b)
-    block_2 = Block(app_hash='hash_2', height=2, transactions=[vote_0.id, vote_1.id])
     b.store_block(block_1._asdict())
-    b.store_block(block_2._asdict())
     return valid_election
 
 
 @pytest.fixture
 def concluded_election(b, ongoing_election, ed25519_node_keys):
-    vote_2 = vote(ongoing_election, 2, ed25519_node_keys, b)
-    block_4 = Block(app_hash='hash_4', height=4, transactions=[vote_2.id])
-    b.store_block(block_4._asdict())
+    validators = b.get_validators(height=1)
+    validator_update = {'validators': validators,
+                        'height': 2,
+                        'election_id': ongoing_election.id}
+
+    query.store_validator_set(b.connection, validator_update)
     return ongoing_election
 
 
 @pytest.fixture
-def inconclusive_election(b, concluded_election, new_validator):
+def inconclusive_election(b, ongoing_election, new_validator):
     validators = b.get_validators(height=1)
     validators[0]['voting_power'] = 15
     validator_update = {'validators': validators,
-                        'height': 3}
+                        'height': 2,
+                        'election_id': 'some_other_election'}
 
     query.store_validator_set(b.connection, validator_update)
-    return concluded_election
+    return ongoing_election
 
 
 def vote(election, voter, keys, b):

tests/upsert_validator/test_validator_election_vote.py

@@ -234,7 +234,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
 
     latest_block = b.get_latest_block()
     # reset the validator set
-    b.store_validator_set(latest_block['height'], validators)
+    b.store_validator_set(latest_block['height'], validators, 'previous_election_id')
 
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
@@ -368,4 +368,4 @@ def reset_validator_set(b, node_keys, height):
         validators.append({'pub_key': {'type': 'ed25519',
                                        'data': node_pub},
                            'voting_power': 10})
-    b.store_validator_set(height, validators)
+    b.store_validator_set(height, validators, 'election_id')

tests/web/test_validators.py

@@ -14,7 +14,7 @@ def test_get_validators_endpoint(b, client):
                       'pub_key': {'data': '4E2685D9016126864733225BE00F005515200727FBAB1312FC78C8B76831255A',
                                   'type': 'ed25519'},
                       'voting_power': 10}]
-    b.store_validator_set(23, validator_set)
+    b.store_validator_set(23, validator_set, 'election_id')
 
     res = client.get(VALIDATORS_ENDPOINT)
     assert is_validator(res.json[0])