Merge branch 'abstract_election_class' into generalize-election-management-commands

bigchaindb/commands/bigchaindb.py

@@ -130,7 +130,8 @@ def run_election_new_upsert_validator(args, bigchain):
     """
 
     new_validator = {
-        'public_key': public_key_from_base64(args.public_key),
+        'public_key': {'value': public_key_from_base64(args.public_key),
+                       'type': 'ed25519-base16'},
         'power': args.power,
         'node_id': args.node_id
     }

bigchaindb/common/exceptions.py

@@ -112,3 +112,7 @@ class UnequalValidatorSet(ValidationError):
 
 class InvalidPowerChange(ValidationError):
     """Raised if proposed power change in validator set is >=1/3 total power"""
+
+
+class InvalidPublicKey(ValidationError):
+    """Raised if public key doesn't match the encoding type"""

bigchaindb/common/schema/transaction_validator_election_v2.0.yaml

@@ -23,7 +23,20 @@ properties:
           node_id:
             type: string
           public_key:
-            type: string
+            type: object
+            additionalProperties: false
+            required:
+            - value
+            - type
+            properties:
+              value:
+                type: string
+              type:
+                type: string
+                enum:
+                  - ed25519-base16
+                  - ed25519-base32
+                  - ed25519-base64
           power:
             "$ref": "#/definitions/positiveInteger"
         required:

bigchaindb/core.py

@@ -233,12 +233,13 @@ class App(BaseApplication):
         # register a new block only when new transactions are received
         if self.block_txn_ids:
             self.bigchaindb.store_bulk_transactions(self.block_transactions)
-            block = Block(app_hash=self.block_txn_hash,
+
-                          height=self.new_height,
+        block = Block(app_hash=self.block_txn_hash,
-                          transactions=self.block_txn_ids)
+                      height=self.new_height,
-            # NOTE: storing the block should be the last operation during commit
+                      transactions=self.block_txn_ids)
-            # this effects crash recovery. Refer BEP#8 for details
+        # NOTE: storing the block should be the last operation during commit
-            self.bigchaindb.store_block(block._asdict())
+        # this effects crash recovery. Refer BEP#8 for details
+        self.bigchaindb.store_block(block._asdict())
 
         logger.debug('Commit-ing new block with hash: apphash=%s ,'
                      'height=%s, txn ids=%s', data, self.new_height,

bigchaindb/elections/election.py

@@ -24,8 +24,6 @@ class Election(Transaction):
     # NOTE: this transaction class extends create so the operation inheritance is achieved
     # by setting an ELECTION_TYPE and renaming CREATE = ELECTION_TYPE and ALLOWED_OPERATIONS = (ELECTION_TYPE,)
     OPERATION = None
-    # the model for votes issued by the election
-    VOTE_TYPE = Vote
     # Custom validation schema
     TX_SCHEMA_CUSTOM = None
     # Election Statuses:
@@ -55,7 +53,7 @@ class Election(Transaction):
         validators = {}
         for validator in bigchain.get_validators(height):
             # NOTE: we assume that Tendermint encodes public key in base64
-            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['data']))
+            public_key = public_key_from_ed25519_key(key_from_base64(validator['public_key']['value']))
             validators[public_key] = validator['voting_power']
 
         return validators
@@ -99,7 +97,7 @@ class Election(Transaction):
             :param current_transactions: (list) A list of transactions to be validated along with the election
 
         Returns:
-            ValidatorElection object
+            Election: a Election object or an object of the derived Election subclass.
 
         Raises:
             ValidationError: If the election is invalid
@@ -166,7 +164,7 @@ class Election(Transaction):
     def count_votes(cls, election_pk, transactions, getter=getattr):
         votes = 0
         for txn in transactions:
-            if getter(txn, 'operation') == cls.VOTE_TYPE.OPERATION:
+            if getter(txn, 'operation') == Vote.OPERATION:
                 for output in getter(txn, 'outputs'):
                     # NOTE: We enforce that a valid vote to election id will have only
                     # election_pk in the output public keys, including any other public key
@@ -241,7 +239,7 @@ class Election(Transaction):
     def approved_update(cls, bigchain, new_height, txns):
         votes = {}
         for txn in txns:
-            if not isinstance(txn, cls.VOTE_TYPE):
+            if not isinstance(txn, Vote):
                 continue
 
             election_id = txn.asset['id']

bigchaindb/elections/vote.py

@@ -30,7 +30,7 @@ class Vote(Transaction):
             bigchain (BigchainDB): an instantiated bigchaindb.lib.BigchainDB object.
 
         Returns:
-            Vote object
+            Vote: a Vote object
 
         Raises:
             ValidationError: If the election vote is invalid

bigchaindb/lib.py

@@ -56,9 +56,10 @@ class BigchainDB(object):
                 A connection to the database.
         """
         config_utils.autoconfigure()
+        self.mode_commit = 'broadcast_tx_commit'
         self.mode_list = ('broadcast_tx_async',
                           'broadcast_tx_sync',
-                          'broadcast_tx_commit')
+                          self.mode_commit)
         self.tendermint_host = bigchaindb.config['tendermint']['host']
         self.tendermint_port = bigchaindb.config['tendermint']['port']
         self.endpoint = 'http://{}:{}/'.format(self.tendermint_host, self.tendermint_port)
@@ -96,29 +97,23 @@ class BigchainDB(object):
 
     def _process_post_response(self, response, mode):
         logger.debug(response)
-        if response.get('error') is not None:
+
-            return (500, 'Internal error')
+        error = response.get('error')
+        if error:
+            return (500, error)
+
+        result = response['result']
+        if mode == self.mode_commit:
+            check_tx_code = result.get('check_tx', {}).get('code', 0)
+            deliver_tx_code = result.get('deliver_tx', {}).get('code', 0)
+            error_code = check_tx_code or deliver_tx_code
+        else:
+            error_code = result.get('code', 0)
+
+        if error_code:
+            return (500, 'Transaction validation failed')
 
         return (202, '')
-        # result = response['result']
-        # if mode == self.mode_list[2]:
-        #     return self._process_commit_mode_response(result)
-        # else:
-        #     status_code = result['code']
-        #     return self._process_status_code(status_code,
-        #                                      'Error while processing transaction')
-
-    # def _process_commit_mode_response(self, result):
-    #     check_tx_status_code = result['check_tx']['code']
-    #     if check_tx_status_code == 0:
-    #         deliver_tx_status_code = result['deliver_tx']['code']
-    #         return self._process_status_code(deliver_tx_status_code,
-    #                                          'Error while commiting the transaction')
-    #     else:
-    #         return (500, 'Error while validating the transaction')
-
-    def process_status_code(self, status_code, failure_msg):
-        return (202, '') if status_code == 0 else (500, failure_msg)
 
     def store_bulk_transactions(self, transactions):
         txns = []

bigchaindb/upsert_validator/validator_election.py

@@ -6,7 +6,7 @@ from bigchaindb.common.exceptions import InvalidPowerChange
 from bigchaindb.tendermint_utils import public_key_to_base64
 from bigchaindb.elections.election import Election
 from bigchaindb.common.schema import (TX_SCHEMA_VALIDATOR_ELECTION)
-from .validator_utils import (new_validator_set, encode_validator)
+from .validator_utils import (new_validator_set, encode_validator, validate_asset_public_key)
 
 
 class ValidatorElection(Election):
@@ -32,6 +32,11 @@ class ValidatorElection(Election):
 
         return self
 
+    @classmethod
+    def validate_schema(cls, tx, skip_id=False):
+        super(ValidatorElection, cls).validate_schema(tx, skip_id=skip_id)
+        validate_asset_public_key(tx['asset']['data']['public_key'])
+
     @classmethod
     def on_approval(cls, bigchain, election, new_height):
         # The new validator set comes into effect from height = new_height+1

bigchaindb/upsert_validator/validator_utils.py

@@ -1,12 +1,14 @@
 import codecs
+import base64
+import binascii
 
 from abci.types_pb2 import (Validator,
                             PubKey)
-from bigchaindb.tendermint_utils import public_key_to_base64
+from bigchaindb.common.exceptions import InvalidPublicKey
 
 
 def encode_validator(v):
-    ed25519_public_key = v['public_key']
+    ed25519_public_key = v['public_key']['value']
     # NOTE: tendermint expects public to be encoded in go-amino format
     pub_key = PubKey(type='ed25519',
                      data=bytes.fromhex(ed25519_public_key))
@@ -16,22 +18,60 @@ def encode_validator(v):
 
 
 def decode_validator(v):
-    return {'pub_key': {'type': v.pub_key.type,
+    return {'public_key': {'type': 'ed25519-base64',
-                        'data': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
+                           'value': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
             'voting_power': v.power}
 
 
 def new_validator_set(validators, updates):
     validators_dict = {}
     for v in validators:
-        validators_dict[v['pub_key']['data']] = v
+        validators_dict[v['public_key']['value']] = v
 
     updates_dict = {}
     for u in updates:
-        public_key64 = public_key_to_base64(u['public_key'])
+        decoder = get_public_key_decoder(u['public_key'])
-        updates_dict[public_key64] = {'pub_key': {'type': 'ed25519',
+        public_key64 = base64.b64encode(decoder(u['public_key']['value'])).decode('utf-8')
-                                                  'data': public_key64},
+        updates_dict[public_key64] = {'public_key': {'type': 'ed25519-base64',
+                                                     'value': public_key64},
                                       'voting_power': u['power']}
 
     new_validators_dict = {**validators_dict, **updates_dict}
     return list(new_validators_dict.values())
+
+
+def encode_pk_to_base16(validator):
+    pk = validator['public_key']
+    decoder = get_public_key_decoder(pk)
+    public_key16 = base64.b16encode(decoder(pk['value'])).decode('utf-8')
+
+    validator['public_key']['value'] = public_key16
+    return validator
+
+
+def validate_asset_public_key(pk):
+    pk_binary = pk['value'].encode('utf-8')
+    decoder = get_public_key_decoder(pk)
+    try:
+        pk_decoded = decoder(pk_binary)
+        if len(pk_decoded) != 32:
+            raise InvalidPublicKey('Public key should be of size 32 bytes')
+
+    except binascii.Error as e:
+        raise InvalidPublicKey('Invalid `type` specified for public key `value`')
+
+
+def get_public_key_decoder(pk):
+    encoding = pk['type']
+    decoder = base64.b64decode
+
+    if encoding == 'ed25519-base16':
+        decoder = base64.b16decode
+    elif encoding == 'ed25519-base32':
+        decoder = base64.b32decode
+    elif encoding == 'ed25519-base64':
+        decoder = base64.b64decode
+    else:
+        raise InvalidPublicKey('Invalid `type` specified for public key `value`')
+
+    return decoder

docs/server/source/http-client-server-api.rst

@@ -153,6 +153,14 @@ Transactions
    ``mode=commit`` means the HTTP response will come back once the transaction
    is in a committed block.
 
+   .. note::
+       In the async and sync modes, after a successful HTTP response is returned, the transaction may still be rejected later on. All the transactions are recorded internally by Tendermint in WAL (Write-Ahead Log) before the HTTP response is returned. Nevertheless, the following should be noted:
+
+       - Transactions in WAL including the failed ones are not exposed in any of the BigchainDB or Tendermint APIs.
+       - Transactions are never fetched from WAL. WAL is never replayed.
+       - A critical failure (e.g. the system is out of disk space) may occur preventing transactions from being stored in WAL, even when the HTTP response indicates a success.
+       - If a transaction fails the validation because it conflicts with the other transactions of the same block, Tendermint includes it into its block, but BigchainDB does not store these transactions and does not offer any information about them in the APIs.
+
    .. note::
 
        The posted transaction should be valid.

docs/server/source/simple-deployment-template/network-setup.md

@@ -242,14 +242,18 @@ The Member must copy the `genesis.json` file in the local `.tendermint/config` d
 The Member must edit the `.tendermint/config/config.toml` file and make the following changes:
 
 ```
-...
+moniker = "Name of our node"
-
 create_empty_blocks = false
-...
+log_level = "main:info,state:info,*:error"
 
 persistent_peers = "<Member 1 node id>@<Member 1 hostname>:26656,\
 <Member 2 node id>@<Member 2 hostname>:26656,\
 <Member N node id>@<Member N hostname>:26656,"
+
+send_rate = 102400000
+recv_rate = 102400000
+
+recheck = false
 ```
 
 ## Member: Start MongoDB

tests/commands/test_commands.py

@@ -449,6 +449,7 @@ def test_election_approve_with_tendermint(b, priv_validator_path, user_sk, valid
                          config={})
 
     election_id = run_election_new_upsert_validator(new_args, b)
+    assert election_id
 
     args = Namespace(action='approve',
                      election_id=election_id,
@@ -529,8 +530,8 @@ def mock_get_validators(height):
     keys = node_keys()
     pub_key = list(keys.keys())[0]
     return [
-        {'pub_key': {'data': pub_key,
+        {'public_key': {'value': pub_key,
-                     'type': 'tendermint/PubKeyEd25519'},
+                        'type': 'ed25519-base64'},
          'voting_power': 10}
     ]
 

tests/conftest.py

@@ -649,9 +649,8 @@ def validators(b, node_keys):
     (public_key, private_key) = list(node_keys.items())[0]
 
     validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2',
-                      'pub_key': {
+                      'public_key': {'value': public_key,
-                          'data': public_key,
+                                     'type': 'ed25519-base64'},
-                          'type': 'ed25519'},
                       'voting_power': 10}]
 
     validator_update = {'validators': validator_set,
@@ -687,6 +686,7 @@ def new_validator():
     power = 1
     node_id = 'fake_node_id'
 
-    return {'public_key': public_key,
+    return {'public_key': {'value': public_key,
+                           'type': 'ed25519-base16'},
             'power': power,
             'node_id': node_id}

tests/tendermint/test_core.py

@@ -366,8 +366,8 @@ def test_end_block_return_validator_updates(b, init_chain_request):
     resp = app.end_block(RequestEndBlock(height=99))
     assert resp.validator_updates[0] == encode_validator(validator)
 
-    updates = b.is_approved()
+    updates = b.approved_update()
-    assert updates == []
+    assert not updates
 
 
 def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
@@ -415,12 +415,14 @@ def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
 
 
 def test_new_validator_set(b):
-    node1 = {'pub_key': {'type': 'ed25519',
+    node1 = {'public_key': {'type': 'ed25519-base64',
-                         'data': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
+                            'value': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
              'voting_power': 10}
-    node1_new_power = {'public_key': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+    node1_new_power = {'public_key': {'value': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+                                      'type': 'ed25519-base16'},
                        'power': 20}
-    node2 = {'public_key': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+    node2 = {'public_key': {'value': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+                            'type': 'ed25519-base16'},
              'power': 10}
 
     validators = [node1]
@@ -430,8 +432,8 @@ def test_new_validator_set(b):
 
     updated_validators = []
     for u in updates:
-        updated_validators.append({'pub_key': {'type': 'ed25519',
+        updated_validators.append({'public_key': {'type': 'ed25519-base64',
-                                               'data': public_key_to_base64(u['public_key'])},
+                                                  'value': public_key_to_base64(u['public_key']['value'])},
                                    'voting_power':  u['power']})
 
     assert updated_validator_set == updated_validators

tests/tendermint/test_integration.py

@@ -40,7 +40,7 @@ def test_app(b, init_chain_request):
 
     pk = codecs.encode(init_chain_request.validators[0].pub_key.data, 'base64').decode().strip('\n')
     [validator] = b.get_validators(height=1)
-    assert validator['pub_key']['data'] == pk
+    assert validator['public_key']['value'] == pk
     assert validator['voting_power'] == 10
 
     alice = generate_key_pair()
@@ -101,7 +101,7 @@ def test_app(b, init_chain_request):
 
     block0 = b.get_latest_block()
     assert block0
-    assert block0['height'] == 1
+    assert block0['height'] == 2
 
     # when empty block is generated hash of previous block should be returned
     assert block0['app_hash'] == new_block_hash
@@ -130,12 +130,13 @@ def test_post_transaction_responses(tendermint_ws_url, b):
     code, message = b.write_transaction(tx_transfer, 'broadcast_tx_commit')
     assert code == 202
 
-    # NOTE: DOESN'T WORK (double spend)
+    carly = generate_key_pair()
-    # Tendermint crashes with error: Unexpected result type
+    double_spend = Transaction.transfer(
-    # carly = generate_key_pair()
+        tx.to_inputs(),
-    # double_spend = Transaction.transfer(tx.to_inputs(),
+        [([carly.public_key], 1)],
-    #                                     [([carly.public_key], 1)],
+        asset_id=tx.id,
-    #                                     asset_id=tx.id)\
+    ).sign([alice.private_key])
-    #                           .sign([alice.private_key])
+    for mode in ('broadcast_tx_sync', 'broadcast_tx_commit'):
-    # code, message = b.write_transaction(double_spend, 'broadcast_tx_commit')
+        code, message = b.write_transaction(double_spend, mode)
-    # assert code == 500
+        assert code == 500
+        assert message == 'Transaction validation failed'

tests/tendermint/test_lib.py

@@ -164,11 +164,11 @@ def test_validator_updates(b, validator_pub_key):
                         'update_id': VALIDATOR_UPDATE_ID}
     query.store_validator_update(b.connection, validator_update)
 
-    updates = b.is_approved()
+    updates = b.approved_updates()
-    assert updates == [validator_update['validator']]
+    assert updates == validator_update['validator']
 
     b.delete_validator_update()
-    assert b.is_approved() == []
+    assert not b.approved_updates()
 
 
 @pytest.mark.bdb

tests/upsert_validator/conftest.py

@@ -22,7 +22,7 @@ def mock_get_validators(network_validators):
         validators = []
         for public_key, power in network_validators.items():
             validators.append({
-                'pub_key': {'type': 'AC26791624DE60', 'data': public_key},
+                'public_key': {'type': 'ed25519-base64', 'value': public_key},
                 'voting_power': power
             })
         return validators

tests/upsert_validator/test_upsert_validator_vote.py

@@ -229,8 +229,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
 
     (node_pub, _) = list(node_keys.items())[0]
 
-    validators = [{'pub_key': {'type': 'ed25519',
+    validators = [{'public_key': {'type': 'ed25519-base64', 'value': node_pub},
-                               'data': node_pub},
                    'voting_power': 10}]
 
     latest_block = b.get_latest_block()
@@ -240,7 +239,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
     public_key64 = public_key_to_base64(public_key)
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
 
@@ -269,7 +268,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
     new_validator_set = b.get_validators()
     validator_pub_keys = []
     for v in new_validator_set:
-        validator_pub_keys.append(v['pub_key']['data'])
+        validator_pub_keys.append(v['public_key']['value'])
 
     assert (public_key64 in validator_pub_keys)
 
@@ -282,7 +281,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
     public_key64 = public_key_to_base64(public_key)
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
     voters = ValidatorElection.recipients(b)
@@ -320,7 +319,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
 
     # remove validator
     power = 0
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
     voters = ValidatorElection.recipients(b)
@@ -344,7 +343,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
 
     # assert that the public key is not a part of the current validator set
     for v in b.get_validators(10):
-        assert not v['pub_key']['data'] == public_key64
+        assert not v['public_key']['value'] == public_key64
 
 
 # ============================================================================
@@ -370,7 +369,7 @@ def gen_vote(election, i, ed25519_node_keys):
 def reset_validator_set(b, node_keys, height):
     validators = []
     for (node_pub, _) in node_keys.items():
-        validators.append({'pub_key': {'type': 'ed25519',
+        validators.append({'public_key': {'type': 'ed25519-base64',
-                                       'data': node_pub},
+                                          'value': node_pub},
                            'voting_power': 10})
     b.store_validator_set(height, validators, 'election_id')

tests/upsert_validator/test_validator_election.py

@@ -24,6 +24,19 @@ def test_upsert_validator_valid_election(b_mock, new_validator, node_key):
     assert election.validate(b_mock)
 
 
+def test_upsert_validator_invalid_election_public_key(b_mock, new_validator, node_key):
+    from bigchaindb.common.exceptions import InvalidPublicKey
+
+    for iv in ['ed25519-base32', 'ed25519-base64']:
+        new_validator['public_key']['type'] = iv
+        voters = ValidatorElection.recipients(b_mock)
+
+        with pytest.raises(InvalidPublicKey):
+            ValidatorElection.generate([node_key.public_key],
+                                       voters,
+                                       new_validator, None).sign([node_key.private_key])
+
+
 def test_upsert_validator_invalid_power_election(b_mock, new_validator, node_key):
     voters = ValidatorElection.recipients(b_mock)
     new_validator['power'] = 30
@@ -147,7 +160,7 @@ def test_upsert_validator_show(caplog, ongoing_election, b):
     from bigchaindb.commands.bigchaindb import run_election_show
 
     election_id = ongoing_election.id
-    public_key = public_key_to_base64(ongoing_election.asset['data']['public_key'])
+    public_key = public_key_to_base64(ongoing_election.asset['data']['public_key']['value'])
     power = ongoing_election.asset['data']['power']
     node_id = ongoing_election.asset['data']['node_id']
     status = ValidatorElection.ONGOING

tests/web/test_transactions.py

@@ -416,6 +416,12 @@ def test_transactions_get_list_bad(client):
 def test_post_transaction_valid_modes(mock_post, client, mode):
     from bigchaindb.models import Transaction
     from bigchaindb.common.crypto import generate_key_pair
+
+    def _mock_post(*args, **kwargs):
+        return Mock(json=Mock(return_value={'result': {'code': 0}}))
+
+    mock_post.side_effect = _mock_post
+
     alice = generate_key_pair()
     tx = Transaction.create([alice.public_key],
                             [([alice.public_key], 1)],