Merge branch 'master' into remove-tm-marker

bigchaindb/commands/bigchaindb.py

@@ -124,7 +124,8 @@ def run_upsert_validator_new(args, bigchain):
     """
 
     new_validator = {
-        'public_key': public_key_from_base64(args.public_key),
+        'public_key': {'value': public_key_from_base64(args.public_key),
+                       'type': 'ed25519-base16'},
         'power': args.power,
         'node_id': args.node_id
     }
@@ -207,7 +208,7 @@ def run_upsert_validator_show(args, bigchain):
 
     new_validator = election.asset['data']
 
-    public_key = public_key_to_base64(new_validator['public_key'])
+    public_key = public_key_to_base64(new_validator['public_key']['value'])
     power = new_validator['power']
     node_id = new_validator['node_id']
     status = election.get_status(bigchain)

bigchaindb/common/exceptions.py

@@ -112,3 +112,7 @@ class UnequalValidatorSet(ValidationError):
 
 class InvalidPowerChange(ValidationError):
     """Raised if proposed power change in validator set is >=1/3 total power"""
+
+
+class InvalidPublicKey(ValidationError):
+    """Raised if public key doesn't match the encoding type"""

bigchaindb/common/schema/transaction_validator_election_v2.0.yaml

@@ -23,7 +23,20 @@ properties:
           node_id:
             type: string
           public_key:
-            type: string
+            type: object
+            additionalProperties: false
+            required:
+            - value
+            - type
+            properties:
+              value:
+                type: string
+              type:
+                type: string
+                enum:
+                  - ed25519-base16
+                  - ed25519-base32
+                  - ed25519-base64
           power:
             "$ref": "#/definitions/positiveInteger"
         required:

bigchaindb/core.py

@@ -232,12 +232,13 @@ class App(BaseApplication):
         # register a new block only when new transactions are received
         if self.block_txn_ids:
             self.bigchaindb.store_bulk_transactions(self.block_transactions)
-            block = Block(app_hash=self.block_txn_hash,
+
-                          height=self.new_height,
+        block = Block(app_hash=self.block_txn_hash,
-                          transactions=self.block_txn_ids)
+                      height=self.new_height,
-            # NOTE: storing the block should be the last operation during commit
+                      transactions=self.block_txn_ids)
-            # this effects crash recovery. Refer BEP#8 for details
+        # NOTE: storing the block should be the last operation during commit
-            self.bigchaindb.store_block(block._asdict())
+        # this effects crash recovery. Refer BEP#8 for details
+        self.bigchaindb.store_block(block._asdict())
 
         logger.debug('Commit-ing new block with hash: apphash=%s ,'
                      'height=%s, txn ids=%s', data, self.new_height,

bigchaindb/lib.py

@@ -56,9 +56,10 @@ class BigchainDB(object):
                 A connection to the database.
         """
         config_utils.autoconfigure()
+        self.mode_commit = 'broadcast_tx_commit'
         self.mode_list = ('broadcast_tx_async',
                           'broadcast_tx_sync',
-                          'broadcast_tx_commit')
+                          self.mode_commit)
         self.tendermint_host = bigchaindb.config['tendermint']['host']
         self.tendermint_port = bigchaindb.config['tendermint']['port']
         self.endpoint = 'http://{}:{}/'.format(self.tendermint_host, self.tendermint_port)
@@ -96,29 +97,23 @@ class BigchainDB(object):
 
     def _process_post_response(self, response, mode):
         logger.debug(response)
-        if response.get('error') is not None:
+
-            return (500, 'Internal error')
+        error = response.get('error')
+        if error:
+            return (500, error)
+
+        result = response['result']
+        if mode == self.mode_commit:
+            check_tx_code = result.get('check_tx', {}).get('code', 0)
+            deliver_tx_code = result.get('deliver_tx', {}).get('code', 0)
+            error_code = check_tx_code or deliver_tx_code
+        else:
+            error_code = result.get('code', 0)
+
+        if error_code:
+            return (500, 'Transaction validation failed')
 
         return (202, '')
-        # result = response['result']
-        # if mode == self.mode_list[2]:
-        #     return self._process_commit_mode_response(result)
-        # else:
-        #     status_code = result['code']
-        #     return self._process_status_code(status_code,
-        #                                      'Error while processing transaction')
-
-    # def _process_commit_mode_response(self, result):
-    #     check_tx_status_code = result['check_tx']['code']
-    #     if check_tx_status_code == 0:
-    #         deliver_tx_status_code = result['deliver_tx']['code']
-    #         return self._process_status_code(deliver_tx_status_code,
-    #                                          'Error while commiting the transaction')
-    #     else:
-    #         return (500, 'Error while validating the transaction')
-
-    def process_status_code(self, status_code, failure_msg):
-        return (202, '') if status_code == 0 else (500, failure_msg)
 
     def store_bulk_transactions(self, transactions):
         txns = []

bigchaindb/upsert_validator/validator_election.py

@@ -19,7 +19,10 @@ from bigchaindb.common.schema import (_validate_schema,
                                       TX_SCHEMA_COMMON,
                                       TX_SCHEMA_CREATE)
 from . import ValidatorElectionVote
-from .validator_utils import (new_validator_set, encode_validator)
+from .validator_utils import (new_validator_set,
+                              encode_validator,
+                              encode_pk_to_base16,
+                              validate_asset_public_key)
 
 
 class ValidatorElection(Transaction):
@@ -58,7 +61,7 @@ class ValidatorElection(Transaction):
         validators = {}
         for validator in bigchain.get_validators(height):
             # NOTE: we assume that Tendermint encodes public key in base64
-            public_key = public_key_from_ed25519_key(key_from_base64(validator['pub_key']['data']))
+            public_key = public_key_from_ed25519_key(key_from_base64(validator['public_key']['value']))
             validators[public_key] = validator['voting_power']
 
         return validators
@@ -155,6 +158,7 @@ class ValidatorElection(Transaction):
         _validate_schema(TX_SCHEMA_COMMON, tx)
         _validate_schema(TX_SCHEMA_CREATE, tx)
         _validate_schema(TX_SCHEMA_VALIDATOR_ELECTION, tx)
+        validate_asset_public_key(tx['asset']['data']['public_key'])
 
     @classmethod
     def create(cls, tx_signers, recipients, metadata=None, asset=None):
@@ -236,7 +240,10 @@ class ValidatorElection(Transaction):
 
                 updated_validator_set = [v for v in updated_validator_set if v['voting_power'] > 0]
                 bigchain.store_validator_set(new_height+1, updated_validator_set, election.id)
-                return [encode_validator(election.asset['data'])]
+
+                validator16 = encode_pk_to_base16(election.asset['data'])
+                return [encode_validator(validator16)]
+
         return []
 
     def get_validator_update_by_election_id(self, election_id, bigchain):

bigchaindb/upsert_validator/validator_utils.py

@@ -1,12 +1,14 @@
 import codecs
+import base64
+import binascii
 
 from abci.types_pb2 import (Validator,
                             PubKey)
-from bigchaindb.tendermint_utils import public_key_to_base64
+from bigchaindb.common.exceptions import InvalidPublicKey
 
 
 def encode_validator(v):
-    ed25519_public_key = v['public_key']
+    ed25519_public_key = v['public_key']['value']
     # NOTE: tendermint expects public to be encoded in go-amino format
     pub_key = PubKey(type='ed25519',
                      data=bytes.fromhex(ed25519_public_key))
@@ -16,22 +18,60 @@ def encode_validator(v):
 
 
 def decode_validator(v):
-    return {'pub_key': {'type': v.pub_key.type,
+    return {'public_key': {'type': 'ed25519-base64',
-                        'data': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
+                           'value': codecs.encode(v.pub_key.data, 'base64').decode().rstrip('\n')},
             'voting_power': v.power}
 
 
 def new_validator_set(validators, updates):
     validators_dict = {}
     for v in validators:
-        validators_dict[v['pub_key']['data']] = v
+        validators_dict[v['public_key']['value']] = v
 
     updates_dict = {}
     for u in updates:
-        public_key64 = public_key_to_base64(u['public_key'])
+        decoder = get_public_key_decoder(u['public_key'])
-        updates_dict[public_key64] = {'pub_key': {'type': 'ed25519',
+        public_key64 = base64.b64encode(decoder(u['public_key']['value'])).decode('utf-8')
-                                                  'data': public_key64},
+        updates_dict[public_key64] = {'public_key': {'type': 'ed25519-base64',
+                                                     'value': public_key64},
                                       'voting_power': u['power']}
 
     new_validators_dict = {**validators_dict, **updates_dict}
     return list(new_validators_dict.values())
+
+
+def encode_pk_to_base16(validator):
+    pk = validator['public_key']
+    decoder = get_public_key_decoder(pk)
+    public_key16 = base64.b16encode(decoder(pk['value'])).decode('utf-8')
+
+    validator['public_key']['value'] = public_key16
+    return validator
+
+
+def validate_asset_public_key(pk):
+    pk_binary = pk['value'].encode('utf-8')
+    decoder = get_public_key_decoder(pk)
+    try:
+        pk_decoded = decoder(pk_binary)
+        if len(pk_decoded) != 32:
+            raise InvalidPublicKey('Public key should be of size 32 bytes')
+
+    except binascii.Error as e:
+        raise InvalidPublicKey('Invalid `type` specified for public key `value`')
+
+
+def get_public_key_decoder(pk):
+    encoding = pk['type']
+    decoder = base64.b64decode
+
+    if encoding == 'ed25519-base16':
+        decoder = base64.b16decode
+    elif encoding == 'ed25519-base32':
+        decoder = base64.b32decode
+    elif encoding == 'ed25519-base64':
+        decoder = base64.b64decode
+    else:
+        raise InvalidPublicKey('Invalid `type` specified for public key `value`')
+
+    return decoder

docs/server/source/http-client-server-api.rst

@@ -153,6 +153,14 @@ Transactions
    ``mode=commit`` means the HTTP response will come back once the transaction
    is in a committed block.
 
+   .. note::
+       In the async and sync modes, after a successful HTTP response is returned, the transaction may still be rejected later on. All the transactions are recorded internally by Tendermint in WAL (Write-Ahead Log) before the HTTP response is returned. Nevertheless, the following should be noted:
+
+       - Transactions in WAL including the failed ones are not exposed in any of the BigchainDB or Tendermint APIs.
+       - Transactions are never fetched from WAL. WAL is never replayed.
+       - A critical failure (e.g. the system is out of disk space) may occur preventing transactions from being stored in WAL, even when the HTTP response indicates a success.
+       - If a transaction fails the validation because it conflicts with the other transactions of the same block, Tendermint includes it into its block, but BigchainDB does not store these transactions and does not offer any information about them in the APIs.
+
    .. note::
 
        The posted transaction should be valid.

docs/server/source/simple-deployment-template/network-setup.md

@@ -242,14 +242,18 @@ The Member must copy the `genesis.json` file in the local `.tendermint/config` d
 The Member must edit the `.tendermint/config/config.toml` file and make the following changes:
 
 ```
-...
+moniker = "Name of our node"
-
 create_empty_blocks = false
-...
+log_level = "main:info,state:info,*:error"
 
 persistent_peers = "<Member 1 node id>@<Member 1 hostname>:26656,\
 <Member 2 node id>@<Member 2 hostname>:26656,\
 <Member N node id>@<Member N hostname>:26656,"
+
+send_rate = 102400000
+recv_rate = 102400000
+
+recheck = false
 ```
 
 ## Member: Start MongoDB

tests/commands/test_commands.py

@@ -29,6 +29,7 @@ def test_make_sure_we_dont_remove_any_command():
                               '--private-key', 'TEMP_PATH_TO_PRIVATE_KEY']).command
     assert parser.parse_args(['upsert-validator', 'approve', 'ELECTION_ID', '--private-key',
                               'TEMP_PATH_TO_PRIVATE_KEY']).command
+    assert parser.parse_args(['upsert-validator', 'show', 'ELECTION_ID']).command
 
 
 @patch('bigchaindb.commands.utils.start')
@@ -336,7 +337,7 @@ def test_upsert_validator_new_with_tendermint(b, priv_validator_path, user_sk, v
     from bigchaindb.commands.bigchaindb import run_upsert_validator_new
 
     new_args = Namespace(action='new',
-                         public_key='8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie',
+                         public_key='HHG0IQRybpT6nJMIWWFWhMczCLHt6xcm7eP52GnGuPY=',
                          power=1,
                          node_id='unique_node_id_for_test_upsert_validator_new_with_tendermint',
                          sk=priv_validator_path,
@@ -424,6 +425,7 @@ def test_upsert_validator_approve_with_tendermint(b, priv_validator_path, user_s
                          config={})
 
     election_id = run_upsert_validator_new(new_args, b)
+    assert election_id
 
     args = Namespace(action='approve',
                      election_id=election_id,
@@ -501,8 +503,8 @@ def mock_get_validators(height):
     keys = node_keys()
     pub_key = list(keys.keys())[0]
     return [
-        {'pub_key': {'data': pub_key,
+        {'public_key': {'value': pub_key,
-                     'type': 'tendermint/PubKeyEd25519'},
+                        'type': 'ed25519-base64'},
          'voting_power': 10}
     ]
 

tests/conftest.py

@@ -649,9 +649,8 @@ def validators(b, node_keys):
     (public_key, private_key) = list(node_keys.items())[0]
 
     validator_set = [{'address': 'F5426F0980E36E03044F74DD414248D29ABCBDB2',
-                      'pub_key': {
+                      'public_key': {'value': public_key,
-                          'data': public_key,
+                                     'type': 'ed25519-base64'},
-                          'type': 'ed25519'},
                       'voting_power': 10}]
 
     validator_update = {'validators': validator_set,
@@ -687,6 +686,7 @@ def new_validator():
     power = 1
     node_id = 'fake_node_id'
 
-    return {'public_key': public_key,
+    return {'public_key': {'value': public_key,
+                           'type': 'ed25519-base16'},
             'power': power,
             'node_id': node_id}

tests/tendermint/test_core.py

@@ -415,12 +415,14 @@ def test_store_pre_commit_state_in_end_block(b, alice, init_chain_request):
 
 
 def test_new_validator_set(b):
-    node1 = {'pub_key': {'type': 'ed25519',
+    node1 = {'public_key': {'type': 'ed25519-base64',
-                         'data': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
+                            'value': 'FxjS2/8AFYoIUqF6AcePTc87qOT7e4WGgH+sGCpTUDQ='},
              'voting_power': 10}
-    node1_new_power = {'public_key': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+    node1_new_power = {'public_key': {'value': '1718D2DBFF00158A0852A17A01C78F4DCF3BA8E4FB7B8586807FAC182A535034',
+                                      'type': 'ed25519-base16'},
                        'power': 20}
-    node2 = {'public_key': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+    node2 = {'public_key': {'value': '1888A353B181715CA2554701D06C1665BC42C5D936C55EA9C5DBCBDB8B3F02A3',
+                            'type': 'ed25519-base16'},
              'power': 10}
 
     validators = [node1]
@@ -430,8 +432,8 @@ def test_new_validator_set(b):
 
     updated_validators = []
     for u in updates:
-        updated_validators.append({'pub_key': {'type': 'ed25519',
+        updated_validators.append({'public_key': {'type': 'ed25519-base64',
-                                               'data': public_key_to_base64(u['public_key'])},
+                                                  'value': public_key_to_base64(u['public_key']['value'])},
                                    'voting_power':  u['power']})
 
     assert updated_validator_set == updated_validators

tests/tendermint/test_integration.py

@@ -39,7 +39,7 @@ def test_app(b, init_chain_request):
 
     pk = codecs.encode(init_chain_request.validators[0].pub_key.data, 'base64').decode().strip('\n')
     [validator] = b.get_validators(height=1)
-    assert validator['pub_key']['data'] == pk
+    assert validator['public_key']['value'] == pk
     assert validator['voting_power'] == 10
 
     alice = generate_key_pair()
@@ -100,7 +100,7 @@ def test_app(b, init_chain_request):
 
     block0 = b.get_latest_block()
     assert block0
-    assert block0['height'] == 1
+    assert block0['height'] == 2
 
     # when empty block is generated hash of previous block should be returned
     assert block0['app_hash'] == new_block_hash
@@ -129,12 +129,13 @@ def test_post_transaction_responses(tendermint_ws_url, b):
     code, message = b.write_transaction(tx_transfer, 'broadcast_tx_commit')
     assert code == 202
 
-    # NOTE: DOESN'T WORK (double spend)
+    carly = generate_key_pair()
-    # Tendermint crashes with error: Unexpected result type
+    double_spend = Transaction.transfer(
-    # carly = generate_key_pair()
+        tx.to_inputs(),
-    # double_spend = Transaction.transfer(tx.to_inputs(),
+        [([carly.public_key], 1)],
-    #                                     [([carly.public_key], 1)],
+        asset_id=tx.id,
-    #                                     asset_id=tx.id)\
+    ).sign([alice.private_key])
-    #                           .sign([alice.private_key])
+    for mode in ('broadcast_tx_sync', 'broadcast_tx_commit'):
-    # code, message = b.write_transaction(double_spend, 'broadcast_tx_commit')
+        code, message = b.write_transaction(double_spend, mode)
-    # assert code == 500
+        assert code == 500
+        assert message == 'Transaction validation failed'

tests/upsert_validator/conftest.py

@@ -22,7 +22,7 @@ def mock_get_validators(network_validators):
         validators = []
         for public_key, power in network_validators.items():
             validators.append({
-                'pub_key': {'type': 'AC26791624DE60', 'data': public_key},
+                'public_key': {'type': 'ed25519-base64', 'value': public_key},
                 'voting_power': power
             })
         return validators

tests/upsert_validator/test_validator_election.py

@@ -24,6 +24,19 @@ def test_upsert_validator_valid_election(b_mock, new_validator, node_key):
     assert election.validate(b_mock)
 
 
+def test_upsert_validator_invalid_election_public_key(b_mock, new_validator, node_key):
+    from bigchaindb.common.exceptions import InvalidPublicKey
+
+    for iv in ['ed25519-base32', 'ed25519-base64']:
+        new_validator['public_key']['type'] = iv
+        voters = ValidatorElection.recipients(b_mock)
+
+        with pytest.raises(InvalidPublicKey):
+            ValidatorElection.generate([node_key.public_key],
+                                       voters,
+                                       new_validator, None).sign([node_key.private_key])
+
+
 def test_upsert_validator_invalid_power_election(b_mock, new_validator, node_key):
     voters = ValidatorElection.recipients(b_mock)
     new_validator['power'] = 30
@@ -147,7 +160,7 @@ def test_upsert_validator_show(caplog, ongoing_election, b):
     from bigchaindb.commands.bigchaindb import run_upsert_validator_show
 
     election_id = ongoing_election.id
-    public_key = public_key_to_base64(ongoing_election.asset['data']['public_key'])
+    public_key = public_key_to_base64(ongoing_election.asset['data']['public_key']['value'])
     power = ongoing_election.asset['data']['power']
     node_id = ongoing_election.asset['data']['node_id']
     status = ValidatorElection.ONGOING

tests/upsert_validator/test_validator_election_vote.py

@@ -222,8 +222,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
 
     (node_pub, _) = list(node_keys.items())[0]
 
-    validators = [{'pub_key': {'type': 'ed25519',
+    validators = [{'public_key': {'type': 'ed25519-base64', 'value': node_pub},
-                               'data': node_pub},
                    'voting_power': 10}]
 
     latest_block = b.get_latest_block()
@@ -233,7 +232,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
     public_key64 = public_key_to_base64(public_key)
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
 
@@ -262,7 +261,7 @@ def test_upsert_validator(b, node_key, node_keys, ed25519_node_keys):
     new_validator_set = b.get_validators()
     validator_pub_keys = []
     for v in new_validator_set:
-        validator_pub_keys.append(v['pub_key']['data'])
+        validator_pub_keys.append(v['public_key']['value'])
 
     assert (public_key64 in validator_pub_keys)
 
@@ -274,7 +273,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
     power = 1
     public_key = '9B3119650DF82B9A5D8A12E38953EA47475C09F0C48A4E6A0ECE182944B24403'
     public_key64 = public_key_to_base64(public_key)
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
     voters = ValidatorElection.recipients(b)
@@ -309,7 +308,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
 
     # remove validator
     power = 0
-    new_validator = {'public_key': public_key,
+    new_validator = {'public_key': {'value': public_key, 'type': 'ed25519-base16'},
                      'node_id': 'some_node_id',
                      'power': power}
     voters = ValidatorElection.recipients(b)
@@ -332,7 +331,7 @@ def test_get_validator_update(b, node_keys, node_key, ed25519_node_keys):
 
     # assert that the public key is not a part of the current validator set
     for v in b.get_validators(10):
-        assert not v['pub_key']['data'] == public_key64
+        assert not v['public_key']['value'] == public_key64
 
 
 # ============================================================================
@@ -358,7 +357,7 @@ def gen_vote(election, i, ed25519_node_keys):
 def reset_validator_set(b, node_keys, height):
     validators = []
     for (node_pub, _) in node_keys.items():
-        validators.append({'pub_key': {'type': 'ed25519',
+        validators.append({'public_key': {'type': 'ed25519-base64',
-                                       'data': node_pub},
+                                          'value': node_pub},
                            'voting_power': 10})
     b.store_validator_set(height, validators, 'election_id')

tests/web/test_transactions.py

@@ -412,6 +412,12 @@ def test_transactions_get_list_bad(client):
 def test_post_transaction_valid_modes(mock_post, client, mode):
     from bigchaindb.models import Transaction
     from bigchaindb.common.crypto import generate_key_pair
+
+    def _mock_post(*args, **kwargs):
+        return Mock(json=Mock(return_value={'result': {'code': 0}}))
+
+    mock_post.side_effect = _mock_post
+
     alice = generate_key_pair()
     tx = Transaction.create([alice.public_key],
                             [([alice.public_key], 1)],