Merge branch 'master' into bug/1813/retract-cmd-bigchaindb-flag

bigchaindb/backend/schema.py

@@ -16,10 +16,17 @@ import logging
 
 import bigchaindb
 from bigchaindb.backend.connection import connect
+from bigchaindb.common.exceptions import ValidationError
+from bigchaindb.common.utils import validate_all_values_for_key
 
 logger = logging.getLogger(__name__)
 
 TABLES = ('bigchain', 'backlog', 'votes', 'assets')
+VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german',
+                   'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian',
+                   'russian', 'spanish', 'swedish', 'turkish', 'none',
+                   'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt',
+                   'ro', 'ru', 'es', 'sv', 'tr')
 
 
 @singledispatch
@@ -99,3 +106,44 @@ def init_database(connection=None, dbname=None):
     create_database(connection, dbname)
     create_tables(connection, dbname)
     create_indexes(connection, dbname)
+
+
+def validate_language_key(obj, key):
+    """Validate all nested "language" key in `obj`.
+
+       Args:
+           obj (dict): dictionary whose "language" key is to be validated.
+
+       Returns:
+           None: validation successful
+
+        Raises:
+            ValidationError: will raise exception in case language is not valid.
+    """
+    backend = bigchaindb.config['database']['backend']
+
+    if backend == 'mongodb':
+        data = obj.get(key, {})
+        if isinstance(data, dict):
+            validate_all_values_for_key(data, 'language', validate_language)
+
+
+def validate_language(value):
+    """Check if `value` is a valid language.
+       https://docs.mongodb.com/manual/reference/text-search-languages/
+
+        Args:
+            value (str): language to validated
+
+        Returns:
+            None: validation successful
+
+        Raises:
+            ValidationError: will raise exception in case language is not valid.
+    """
+    if value not in VALID_LANGUAGES:
+        error_str = ('MongoDB does not support text search for the '
+                     'language "{}". If you do not understand this error '
+                     'message then please rename key/field "language" to '
+                     'something else like "lang".').format(value)
+        raise ValidationError(error_str)

bigchaindb/common/utils.py

@@ -52,53 +52,73 @@ def deserialize(data):
 
 
 def validate_txn_obj(obj_name, obj, key, validation_fun):
-    """Validates value associated to `key` in `obj` by applying
-       `validation_fun`.
+    """Validate value of `key` in `obj` using `validation_fun`.
 
         Args:
             obj_name (str): name for `obj` being validated.
-            obj (dict): dictonary object.
+            obj (dict): dictionary object.
             key (str): key to be validated in `obj`.
             validation_fun (function): function used to validate the value
             of `key`.
 
         Returns:
-            None: indicates validation successfull
+            None: indicates validation successful
 
         Raises:
-            ValidationError: `validation_fun` will raise this error on failure
+            ValidationError: `validation_fun` will raise exception on failure
     """
     backend = bigchaindb.config['database']['backend']
 
     if backend == 'mongodb':
-        data = obj.get(key, {}) or {}
-        validate_all_keys(obj_name, data, validation_fun)
+        data = obj.get(key, {})
+        if isinstance(data, dict):
+            validate_all_keys(obj_name, data, validation_fun)
 
 
 def validate_all_keys(obj_name, obj, validation_fun):
-    """Validates all (nested) keys in `obj` by using `validation_fun`
+    """Validate all (nested) keys in `obj` by using `validation_fun`.
 
         Args:
             obj_name (str): name for `obj` being validated.
-            obj (dict): dictonary object.
+            obj (dict): dictionary object.
             validation_fun (function): function used to validate the value
             of `key`.
 
         Returns:
-            None: indicates validation successfull
+            None: indicates validation successful
 
         Raises:
             ValidationError: `validation_fun` will raise this error on failure
     """
     for key, value in obj.items():
         validation_fun(obj_name, key)
-        if type(value) is dict:
+        if isinstance(value, dict):
             validate_all_keys(obj_name, value, validation_fun)
-    return
+
+
+def validate_all_values_for_key(obj, key, validation_fun):
+    """Validate value for all (nested) occurrence  of `key` in `obj`
+       using `validation_fun`.
+
+        Args:
+            obj (dict): dictionary object.
+            key (str): key whose value is to be validated.
+            validation_fun (function): function used to validate the value
+            of `key`.
+
+        Raises:
+            ValidationError: `validation_fun` will raise this error on failure
+    """
+    for vkey, value in obj.items():
+        if vkey == key:
+            validation_fun(value)
+        elif isinstance(value, dict):
+            validate_all_values_for_key(value, key, validation_fun)
 
 
 def validate_key(obj_name, key):
-    """Check if `key` contains ".", "$" or null characters
+    """Check if `key` contains ".", "$" or null characters.
+
        https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names
 
         Args:
@@ -106,13 +126,13 @@ def validate_key(obj_name, key):
             key (str): key to validated
 
         Returns:
-            None: indicates validation successfull
+            None: validation successful
 
         Raises:
-            ValidationError: raise execption incase of regex match.
+            ValidationError: will raise exception in case of regex match.
     """
     if re.search(r'^[$]|\.|\x00', key):
         error_str = ('Invalid key name "{}" in {} object. The '
                      'key name cannot contain characters '
                      '".", "$" or null characters').format(key, obj_name)
-        raise ValidationError(error_str) from ValueError()
+        raise ValidationError(error_str)

bigchaindb/models.py

@@ -11,6 +11,7 @@ from bigchaindb.common.transaction import Transaction
 from bigchaindb.common.utils import (gen_timestamp, serialize,
                                      validate_txn_obj, validate_key)
 from bigchaindb.common.schema import validate_transaction_schema
+from bigchaindb.backend.schema import validate_language_key
 
 
 class Transaction(Transaction):
@@ -87,6 +88,7 @@ class Transaction(Transaction):
         validate_transaction_schema(tx_body)
         validate_txn_obj('asset', tx_body['asset'], 'data', validate_key)
         validate_txn_obj('metadata', tx_body, 'metadata', validate_key)
+        validate_language_key(tx_body['asset'], 'data')
         return super().from_dict(tx_body)
 
     @classmethod

docs/server/source/data-models/asset-model.md

@@ -2,6 +2,8 @@
 
 To avoid redundant data in transactions, the asset model is different for `CREATE` and `TRANSFER` transactions.
 
+## In CREATE Transactions
+
 In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair. The key must be `"data"` and the value can be any valid JSON document, or `null`. For example:
 ```json
 {
@@ -12,6 +14,15 @@ In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair
 }
 ```
 
+When using MongoDB for storage, certain restriction apply to all (including nested) keys of the `"data"` JSON document:
+
+* Keys (i.e. key names, not values) must **not** begin with the `$` character.
+* Keys must not contain `.` or the null character (Unicode code point 0000).
+* The key `"language"` (at any level in the hierarchy) is a special key and used for specifying text search language. Its value must be one of the allowed values; see the valid [Text Search Languages](https://docs.mongodb.com/manual/reference/text-search-languages/) in the MongoDB Docs. In BigchainDB, only the languages supported by _MongoDB community edition_ are allowed.
+
+
+## In TRANSFER Transactions
+
 In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pair. They key must be `"id"` and the value must contain a transaction ID (i.e. a SHA3-256 hash: the ID of the `CREATE` transaction which created the asset, which also serves as the asset ID). For example:
 ```json
 {

docs/server/source/data-models/transaction-model.rst

@@ -46,6 +46,10 @@ Here's some explanation of the contents:
 
 - **metadata**: User-provided transaction metadata.
   It can be any valid JSON document, or ``null``.
+  **NOTE:** When using MongoDB for storage, certain restriction apply
+  to all (including nested) keys of the ``"data"`` JSON document:
+  1) keys (i.e. key names, not values) must **not** begin with the ``$`` character, and
+  2) keys must not contain ``.`` or the null character (Unicode code point 0000).
 
 **How the transaction ID is computed.**
 1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, 

tests/README.md

@@ -105,17 +105,17 @@ $ docker-compose build
 First, start `RethinkDB` in the background:
 
 ```text
-$ docker-compose up -d rdb
+$ docker-compose -f docker-compose.rdb.yml up -d rdb
 ```
 
 then run the tests using:
 
 ```text
-$ docker-compose run --rm bdb-rdb py.test -v
+$ docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb py.test -v
 ```
 
 to rebuild all the images (usually you only need to rebuild the `bdb` and
- `bdb-rdb` images).
+ `bdb-rdb` images). If that fails, then do `make clean-pyc` and try again.
 
 ## Automated Testing of All Pull Requests
 

tests/conftest.py

@@ -25,6 +25,15 @@ USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie'
 USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE'
 
 
+def pytest_runtest_setup(item):
+    if isinstance(item, item.Function):
+        if item.get_marker('skip_travis_rdb'):
+            if (os.getenv('TRAVIS_CI') == 'true' and
+                    os.getenv('BIGCHAINDB_DATABASE_BACKEND') == 'rethinkdb'):
+                pytest.skip(
+                    'Skip test during Travis CI build when using rethinkdb')
+
+
 def pytest_addoption(parser):
     from bigchaindb.backend.connection import BACKENDS
 

tests/integration/test_federation.py

@@ -97,6 +97,7 @@ def process_vote(steps, result=None):
 
 @pytest.mark.bdb
 @pytest.mark.genesis
+@pytest.mark.skip_travis_rdb
 def test_elect_valid(federation_3):
     [bx, (s0, s1, s2)] = federation_3
     tx = input_single_create(bx[0])
@@ -115,6 +116,7 @@ def test_elect_valid(federation_3):
 
 
 @pytest.mark.bdb
+@pytest.mark.skip_travis_rdb
 @pytest.mark.genesis
 def test_elect_invalid(federation_3):
     [bx, (s0, s1, s2)] = federation_3
@@ -135,6 +137,7 @@ def test_elect_invalid(federation_3):
 
 @pytest.mark.bdb
 @pytest.mark.genesis
+@pytest.mark.skip_travis_rdb
 def test_elect_sybill(federation_3):
     [bx, (s0, s1, s2)] = federation_3
     tx = input_single_create(bx[0])

tests/integration/test_integration.py

@@ -5,6 +5,7 @@ import pytest
 pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('processes')]
 
 
+@pytest.mark.skip_travis_rdb
 def test_double_create(b, user_pk):
     from bigchaindb.models import Transaction
     from bigchaindb.backend.query import count_blocks

tests/web/test_transactions.py

@@ -47,6 +47,47 @@ def test_post_create_transaction_endpoint(b, client):
     assert res.json['outputs'][0]['public_keys'][0] == user_pub
 
 
+@pytest.mark.parametrize("nested", [False, True])
+@pytest.mark.parametrize("language,expected_status_code", [
+    ('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202),
+    ('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202),
+    ('norwegian', 202), ('portuguese', 202), ('romanian', 202), ('none', 202),
+    ('russian', 202), ('spanish', 202), ('swedish', 202), ('turkish', 202),
+    ('da', 202), ('nl', 202), ('en', 202), ('fi', 202), ('fr', 202),
+    ('de', 202), ('hu', 202), ('it', 202), ('nb', 202), ('pt', 202),
+    ('ro', 202), ('ru', 202), ('es', 202), ('sv', 202), ('tr', 202),
+    ('any', 400)
+])
+@pytest.mark.language
+@pytest.mark.bdb
+def test_post_create_transaction_with_language(b, client, nested, language,
+                                               expected_status_code):
+    from bigchaindb.models import Transaction
+    from bigchaindb.backend.mongodb.connection import MongoDBConnection
+
+    if isinstance(b.connection, MongoDBConnection):
+        user_priv, user_pub = crypto.generate_key_pair()
+        lang_obj = {'language': language}
+
+        if nested:
+            asset = {'root': lang_obj}
+        else:
+            asset = lang_obj
+
+        tx = Transaction.create([user_pub], [([user_pub], 1)],
+                                asset=asset)
+        tx = tx.sign([user_priv])
+        res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict()))
+        assert res.status_code == expected_status_code
+        if res.status_code == 400:
+            expected_error_message = (
+                'Invalid transaction (ValidationError): MongoDB does not support '
+                'text search for the language "{}". If you do not understand this '
+                'error message then please rename key/field "language" to something '
+                'else like "lang".').format(language)
+            assert res.json['message'] == expected_error_message
+
+
 @pytest.mark.parametrize("field", ['asset', 'metadata'])
 @pytest.mark.parametrize("value,err_key,expected_status_code", [
     ({'bad.key': 'v'}, 'bad.key', 400),