Problem: Need a method to initiate a new upsert-validator election

Solution: Added subcommand `new` to `upsert_validator`

Dockerfile-dev

@@ -1,5 +1,4 @@
-ARG python_version=3.6
+FROM python:3.6
-FROM python:${python_version}
 LABEL maintainer "dev@bigchaindb.com"
 
 RUN apt-get update \
@@ -8,30 +7,3 @@ RUN apt-get update \
     && pip install pynacl \
     && apt-get autoremove \
     && apt-get clean
-
-ARG backend
-ARG abci_status
-
-# When developing with Python in a docker container, we are using PYTHONBUFFERED
-# to force stdin, stdout and stderr to be totally unbuffered and to capture logs/outputs
-ENV PYTHONUNBUFFERED 0
-
-ENV BIGCHAINDB_DATABASE_PORT 27017
-ENV BIGCHAINDB_DATABASE_BACKEND $backend
-ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984
-ENV BIGCHAINDB_WSSERVER_HOST 0.0.0.0
-ENV BIGCHAINDB_WSSERVER_SCHEME ws
-
-ENV BIGCHAINDB_WSSERVER_ADVERTISED_HOST 0.0.0.0
-ENV BIGCHAINDB_WSSERVER_ADVERTISED_SCHEME ws
-
-ENV BIGCHAINDB_TENDERMINT_PORT 26657
-
-ENV BIGCHAINDB_CI_ABCI ${abci_status}
-
-RUN mkdir -p /usr/src/app
-COPY . /usr/src/app/
-WORKDIR /usr/src/app
-RUN pip install --no-cache-dir --process-dependency-links -e .[dev]
-RUN bigchaindb -y configure
-

bigchaindb/commands/bigchaindb.py

@@ -9,19 +9,19 @@ import copy
 import json
 import sys
 
+from bigchaindb.utils import load_node_key
 from bigchaindb.common.exceptions import (DatabaseAlreadyExists,
                                           DatabaseDoesNotExist,
-                                          MultipleValidatorOperationError)
+                                          OperationError)
 import bigchaindb
-from bigchaindb import backend
+from bigchaindb import backend, ValidatorElection, BigchainDB
 from bigchaindb.backend import schema
 from bigchaindb.backend import query
-from bigchaindb.backend.query import VALIDATOR_UPDATE_ID, PRE_COMMIT_ID
+from bigchaindb.backend.query import PRE_COMMIT_ID
 from bigchaindb.commands import utils
 from bigchaindb.commands.utils import (configure_bigchaindb,
                                        input_on_stderr)
 from bigchaindb.log import setup_logging
-from bigchaindb.tendermint_utils import public_key_from_base64
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -95,21 +95,35 @@ def run_configure(args):
 
 @configure_bigchaindb
 def run_upsert_validator(args):
-    """Store validators which should be synced with Tendermint"""
+    """Initiate and manage elections to change the validator set"""
 
-    b = bigchaindb.BigchainDB()
+    b = BigchainDB()
-    public_key = public_key_from_base64(args.public_key)
+
-    validator = {'pub_key': {'type': 'ed25519',
+    # Call the function specified by args.action, as defined above
-                             'data': public_key},
+    globals()[f'run_upsert_validator_{args.action}'](args, b)
-                 'power': args.power}
+
-    validator_update = {'validator': validator,
+
-                        'update_id': VALIDATOR_UPDATE_ID}
+def run_upsert_validator_new(args, bigchain):
-    try:
+
-        query.store_validator_update(b.connection, validator_update)
+    new_validator = {
-    except MultipleValidatorOperationError:
+        'public_key': args.public_key,
-        logger.error('A validator update is pending to be applied. '
+        'power': args.power,
-                     'Please re-try after the current update has '
+        'node_id': args.node_id
-                     'been processed.')
+    }
+
+    key = load_node_key(args.sk)
+
+    voters = ValidatorElection.recipients(bigchain)
+
+    election = ValidatorElection.generate([key.public_key],
+                                          voters,
+                                          new_validator, None).sign([key.private_key])
+    election.validate(bigchain)
+    resp = bigchain.write_transaction(election, 'broadcast_tx_commit')
+    if resp == (202, ''):
+        return election.id
+    else:
+        raise OperationError('Failed to commit election')
 
 
 def _run_init():
@@ -208,16 +222,35 @@ def create_parser():
                                help='The backend to use. It can only be '
                                '"localmongodb", currently.')
 
+    # parser for managing validator elections
     validator_parser = subparsers.add_parser('upsert-validator',
-                                             help='Add/update/delete a validator')
+                                             help='Add/update/delete a validator.')
 
-    validator_parser.add_argument('public_key',
+    validator_subparser = validator_parser.add_subparsers(title='Action',
-                                  help='Public key of the validator.')
+                                                          dest='action')
 
-    validator_parser.add_argument('power',
+    new_election_parser = validator_subparser.add_parser('new',
-                                  type=int,
+                                                         help='Calls a new election.')
-                                  help='Voting power of the validator. '
+
-                                  'Setting it to 0 will delete the validator.')
+    new_election_parser.add_argument('public_key',
+                                     help='Public key of the validator to be added/updated/removed.')
+
+    new_election_parser.add_argument('power',
+                                     type=int,
+                                     help='The proposed power for the validator. '
+                                          'Setting to 0 will remove the validator.')
+
+    new_election_parser.add_argument('node_id',
+                                     help='The node_id of the validator.')
+
+    new_election_parser.add_argument('--private-key',
+                                     dest='sk',
+                                     help='Path to the private key of the election initiator.')
+
+    show_election_subparser = validator_subparser.add_parser('show',
+                                                             help='Show election information.')
+
+    show_election_subparser.add_argument('election_id')
 
     # parsers for showing/exporting config values
     subparsers.add_parser('show-config',

bigchaindb/lib.py

@@ -467,6 +467,7 @@ class BigchainDB(object):
             for v in validators:
                 v.pop('accum')
                 v.pop('address')
+                v['voting_power'] = int(v['voting_power'])
 
             return validators
 

bigchaindb/utils.py

@@ -167,3 +167,14 @@ class Lazy:
 
         self.stack = []
         return last
+
+# Load Tendermint's public and private key from the file path
+def load_node_key(path):
+    import json
+    from bigchaindb.tendermint_utils import key_from_base64
+    from bigchaindb.common.crypto import key_pair_from_ed25519_key
+    with open(path) as json_data:
+        priv_validator = json.load(json_data)
+        priv_key = priv_validator['priv_key']['value']
+        hex_private_key = key_from_base64(priv_key)
+        return key_pair_from_ed25519_key(hex_private_key)

tests/commands/test_commands.py

@@ -18,7 +18,9 @@ def test_make_sure_we_dont_remove_any_command():
     assert parser.parse_args(['init']).command
     assert parser.parse_args(['drop']).command
     assert parser.parse_args(['start']).command
-    assert parser.parse_args(['upsert-validator', 'TEMP_PUB_KEYPAIR', '10']).command
+    assert parser.parse_args(['upsert-validator', 'new', 'TEMP_PUB_KEYPAIR', '10', 'TEMP_NODE_ID',
+                              '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command
+    assert parser.parse_args(['upsert-validator', 'show', 'TEMP_ELECTION_ID']).command
 
 
 @pytest.mark.tendermint
@@ -341,14 +343,74 @@ class MockResponse():
         return {'result': {'latest_block_height': self.height}}
 
 
-@patch('bigchaindb.config_utils.autoconfigure')
+#@pytest.mark.execute
-@patch('bigchaindb.backend.query.store_validator_update')
+#@patch('bigchaindb.lib.BigchainDB.get_validators')
+#@pytest.mark.abci
+def test_upsert_validator_new_with_tendermint(b, priv_validator_path, user_sk, monkeypatch):
+    """
+    WIP: Will be fixed and activated in the next PR
+    """
+    from bigchaindb.commands.bigchaindb import run_upsert_validator_new
+    import time
+
+    time.sleep(3)
+
+    def mock_get():
+        return [
+            {'pub_key': {'value': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=',
+                         'type': 'tendermint/PubKeyEd25519'},
+             'voting_power': 10}
+        ]
+
+    #b.get_validators = mock_get
+    #mock_get_validators = mock_get
+    #monkeypatch.setattr('requests.get', mock_get)
+
+    proposer_key = b.get_validators()[0]['pub_key']['value']
+
+    args = Namespace(action='new',
+                     public_key=proposer_key,
+                     power=1,
+                     node_id='12345',
+                     sk=priv_validator_path,
+                     config={})
+    print("*****ARGS*****")
+    print(args)
+    resp = run_upsert_validator_new(args, b)
+    print("*****TX ID*****")
+    print(resp)
+    time.sleep(3)
+
+    assert b.get_transaction(resp)
+
+
 @pytest.mark.tendermint
-def test_upsert_validator(mock_autoconfigure, mock_store_validator_update):
+@pytest.mark.bdb
-    from bigchaindb.commands.bigchaindb import run_upsert_validator
+def test_upsert_validator_new_without_tendermint(b, priv_validator_path, user_sk, monkeypatch):
+    from bigchaindb.commands.bigchaindb import run_upsert_validator_new
 
-    args = Namespace(public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=',
+    def mock_get():
-                     power='10', config={})
+        return [
-    run_upsert_validator(args)
+            {'pub_key': {'value': 'zL/DasvKulXZzhSNFwx4cLRXKkSM9GPK7Y0nZ4FEylM=',
+                         'type': 'tendermint/PubKeyEd25519'},
+             'voting_power': 10}
+        ]
 
-    assert mock_store_validator_update.called
+    def mock_write(tx, mode):
+        b.store_transaction(tx)
+        return (202, '')
+
+    b.get_validators = mock_get
+    b.write_transaction = mock_write
+
+    monkeypatch.setattr('requests.get', mock_get)
+
+    args = Namespace(action='new',
+                     public_key='CJxdItf4lz2PwEf4SmYNAu/c/VpmX39JEgC5YpH7fxg=',
+                     power=1,
+                     node_id='12345',
+                     sk=priv_validator_path,
+                     config={})
+    resp = run_upsert_validator_new(args, b)
+
+    assert b.get_transaction(resp)

tests/conftest.py

@@ -4,10 +4,11 @@ Tasks:
 1. setup test database before starting the tests
 2. delete test database after running the tests
 """
-
+import json
 import os
 import copy
 import random
+import tempfile
 from collections import namedtuple
 from logging import getLogger
 from logging.config import dictConfig
@@ -657,3 +658,26 @@ def node_keys():
             '83VINXdj2ynOHuhvSZz5tGuOE5oYzIi0mEximkX1KYMlt/Csu8JUjA4+by2Pz3fqSLshhuYYeM+IpvqcBl6BEA==',
             'PecJ58SaNRsWJZodDmqjpCWqG6btdwXFHLyE40RYlYM=':
             'uz8bYgoL4rHErWT1gjjrnA+W7bgD/uDQWSRKDmC8otc95wnnxJo1GxYlmh0OaqOkJaobpu13BcUcvITjRFiVgw=='}
+
+@pytest.fixture(scope='session')
+def priv_validator_path(node_keys):
+    (public_key, private_key) = list(node_keys.items())[0]
+    priv_validator = {
+        'address': '84F787D95E196DC5DE5F972666CFECCA36801426',
+        'pub_key': {
+            'type': 'AC26791624DE60',
+            'value': public_key
+        },
+        'last_height': 0,
+        'last_round': 0,
+        'last_step': 0,
+        'priv_key': {
+            'type': '954568A3288910',
+            'value': private_key
+        }
+    }
+    fd, path = tempfile.mkstemp()
+    socket = os.fdopen(fd, 'w')
+    json.dump(priv_validator, socket)
+    socket.close()
+    return path