From 722658421580c36ac32d6a7bd9e3537ed96a9bb6 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 10:12:16 +0530 Subject: [PATCH 1/6] Api fix for asset language --- bigchaindb/backend/schema.py | 22 ++++++++++++++++++ bigchaindb/models.py | 2 ++ tests/web/test_transactions.py | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index f6ce466f..8b85c068 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -16,10 +16,14 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect +from bigchaindb.common.exceptions import ValidationError logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') +VALID_LANGUAGES = ('danish' 'dutch' 'english' 'finnish' 'french' 'german' + 'hungarian' 'italian' 'norwegian' 'portuguese' 'romanian' + 'russian' 'spanish' 'swedish' 'turkish') @singledispatch @@ -99,3 +103,21 @@ def init_database(connection=None, dbname=None): create_database(connection, dbname) create_tables(connection, dbname) create_indexes(connection, dbname) + + +def validate_if_exists_asset_language(tx_body): + data = tx_body['asset'].get('data', {}) + + if data and 'language' in data: + + language = data.get('language') + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb' and language not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(language) + raise ValidationError(error_str) from ValueError() + + return diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c8ad9dd3..d88d239e 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -10,6 +10,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import gen_timestamp, serialize from bigchaindb.common.schema import validate_transaction_schema +from bigchaindb.backend.schema import validate_if_exists_asset_language class Transaction(Transaction): @@ -84,6 +85,7 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) + validate_if_exists_asset_language(tx_body) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index acea8c2c..8b4d31bf 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,6 +47,47 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub +@pytest.mark.parametrize("language,expected_status_code", [ + ('danish', 202), + ('dutch', 202), + ('english', 202), + ('finnish', 202), + ('french', 202), + ('german', 202), + ('hungarian', 202), + ('italian', 202), + ('norwegian', 202), + ('portuguese', 202), + ('romanian', 202), + ('russian', 202), + ('spanish', 202), + ('swedish', 202), + ('turkish', 202), + ('any', 400), +]) +@pytest.mark.language +@pytest.mark.bdb +def test_post_create_transaction_with_language(b, client, language, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + if isinstance(b.connection, MongoDBConnection): + user_priv, user_pub = crypto.generate_key_pair() + + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset={'language': language}) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + "Invalid transaction (ValidationError): MongoDB does not support " + "text search for the language \"{}\". If you do not understand this " + "error message then please rename key/field \"language\" to something " + "else like \"lang\".").format(language) + assert res.json['message'] == expected_error_message + + @patch('bigchaindb.web.views.base.logger') def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash From cd636101a7c03e433c27a26d1749e87a37dbec55 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 16:54:47 +0530 Subject: [PATCH 2/6] Support abbreviated values for "language" --- bigchaindb/backend/schema.py | 12 +++++++----- tests/web/test_transactions.py | 24 ++++++++---------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 8b85c068..3b7413a2 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -21,9 +21,11 @@ from bigchaindb.common.exceptions import ValidationError logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') -VALID_LANGUAGES = ('danish' 'dutch' 'english' 'finnish' 'french' 'german' - 'hungarian' 'italian' 'norwegian' 'portuguese' 'romanian' - 'russian' 'spanish' 'swedish' 'turkish') +VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', + 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', + 'russian', 'spanish', 'swedish', 'turkish', + 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', + 'ro', 'ru', 'es', 'sv', 'tr') @singledispatch @@ -108,12 +110,12 @@ def init_database(connection=None, dbname=None): def validate_if_exists_asset_language(tx_body): data = tx_body['asset'].get('data', {}) - if data and 'language' in data: + if data and ('language' in data): language = data.get('language') backend = bigchaindb.config['database']['backend'] - if backend == 'mongodb' and language not in VALID_LANGUAGES: + if (backend == 'mongodb') and (language not in VALID_LANGUAGES): error_str = ('MongoDB does not support text search for the ' 'language "{}". If you do not understand this error ' 'message then please rename key/field "language" to ' diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 8b4d31bf..f33c47b1 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -48,22 +48,14 @@ def test_post_create_transaction_endpoint(b, client): @pytest.mark.parametrize("language,expected_status_code", [ - ('danish', 202), - ('dutch', 202), - ('english', 202), - ('finnish', 202), - ('french', 202), - ('german', 202), - ('hungarian', 202), - ('italian', 202), - ('norwegian', 202), - ('portuguese', 202), - ('romanian', 202), - ('russian', 202), - ('spanish', 202), - ('swedish', 202), - ('turkish', 202), - ('any', 400), + ('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202), + ('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202), + ('norwegian', 202), ('portuguese', 202), ('romanian', 202), + ('russian', 202), ('spanish', 202), ('swedish', 202), ('turkish', 202), + ('da', 202), ('nl', 202), ('en', 202), ('fi', 202), ('fr', 202), + ('de', 202), ('hu', 202), ('it', 202), ('nb', 202), ('pt', 202), + ('ro', 202), ('ru', 202), ('es', 202), ('sv', 202), ('tr', 202), + ('any', 400) ]) @pytest.mark.language @pytest.mark.bdb From 99aa38b21742ef412bf591f81e6f8c8763db68bb Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 11:38:38 +0530 Subject: [PATCH 3/6] Added "none" to language whitelist --- bigchaindb/backend/schema.py | 49 +++++++++++++----- bigchaindb/common/utils.py | 96 +++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 14 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 3b7413a2..07fe4210 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -17,13 +17,14 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect from bigchaindb.common.exceptions import ValidationError +from bigchaindb.common.utils import validate_all_value_for_key logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', - 'russian', 'spanish', 'swedish', 'turkish', + 'russian', 'spanish', 'swedish', 'turkish', 'none', 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', 'ro', 'ru', 'es', 'sv', 'tr') @@ -107,19 +108,41 @@ def init_database(connection=None, dbname=None): create_indexes(connection, dbname) -def validate_if_exists_asset_language(tx_body): - data = tx_body['asset'].get('data', {}) +def validate_if_exists_language(obj, key): + """Validate all nested "language" key in `obj`. - if data and ('language' in data): + Args: + obj (dict): dictonary whose "language" key is to be validated. - language = data.get('language') - backend = bigchaindb.config['database']['backend'] + Returns: + None: validation successfull - if (backend == 'mongodb') and (language not in VALID_LANGUAGES): - error_str = ('MongoDB does not support text search for the ' - 'language "{}". If you do not understand this error ' - 'message then please rename key/field "language" to ' - 'something else like "lang".').format(language) - raise ValidationError(error_str) from ValueError() + Raises: + ValidationError: raises execption incase language is not valid. + """ + backend = bigchaindb.config['database']['backend'] - return + if backend == 'mongodb': + data = obj.get(key, {}) or {} + validate_all_value_for_key(data, 'language', validate_language) + + +def validate_language(value): + """Check if `value` is a valid language + https://docs.mongodb.com/manual/reference/text-search-languages/ + + Args: + value (str): language to validated + + Returns: + None: validation successfull + + Raises: + ValidationError: raises execption incase language is not valid. + """ + if value not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(value) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index f6f671db..00621b2e 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -1,7 +1,10 @@ import time - +import re import rapidjson +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + def gen_timestamp(): """The Unix time, rounded to the nearest second. @@ -46,3 +49,94 @@ def deserialize(data): string. """ return rapidjson.loads(data) + + +def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validates value associated to `key` in `obj` by applying + `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = obj.get(key, {}) or {} + validate_all_keys(obj_name, data, validation_fun) + + +def validate_all_keys(obj_name, obj, validation_fun): + """Validates all (nested) keys in `obj` by using `validation_fun` + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for key, value in obj.items(): + validation_fun(obj_name, key) + if type(value) is dict: + validate_all_keys(obj_name, value, validation_fun) + return + + +def validate_all_value_for_key(obj, key, validation_fun): + """Validates value for all (nested) occurences of `key` in `obj` + using `validation_fun` + + Args: + obj (dict): dictonary object. + key (str): key whose value is to be validated. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for vkey, value in obj.items(): + if vkey == key: + validation_fun(value) + elif type(value) is dict: + validate_all_value_for_key(value, key, validation_fun) + return + + +def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: raise execption incase of regex match. + """ + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) from ValueError() From a29fd7e84f61f6df299faf4d955596edb090a8e7 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 18:37:19 +0530 Subject: [PATCH 4/6] Fix variable type check and docstrings --- bigchaindb/backend/schema.py | 8 ++++---- bigchaindb/common/utils.py | 11 +++-------- bigchaindb/models.py | 4 ++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 07fe4210..fcad33a6 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -108,14 +108,14 @@ def init_database(connection=None, dbname=None): create_indexes(connection, dbname) -def validate_if_exists_language(obj, key): +def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. Args: obj (dict): dictonary whose "language" key is to be validated. Returns: - None: validation successfull + None: validation successful Raises: ValidationError: raises execption incase language is not valid. @@ -135,7 +135,7 @@ def validate_language(value): value (str): language to validated Returns: - None: validation successfull + None: validation successful Raises: ValidationError: raises execption incase language is not valid. @@ -145,4 +145,4 @@ def validate_language(value): 'language "{}". If you do not understand this error ' 'message then please rename key/field "language" to ' 'something else like "lang".').format(value) - raise ValidationError(error_str) from ValueError() + raise ValidationError(error_str) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index 00621b2e..b21f41d6 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -92,9 +92,8 @@ def validate_all_keys(obj_name, obj, validation_fun): """ for key, value in obj.items(): validation_fun(obj_name, key) - if type(value) is dict: + if isinstance(value, dict): validate_all_keys(obj_name, value, validation_fun) - return def validate_all_value_for_key(obj, key, validation_fun): @@ -107,18 +106,14 @@ def validate_all_value_for_key(obj, key, validation_fun): validation_fun (function): function used to validate the value of `key`. - Returns: - None: indicates validation successfull - Raises: ValidationError: `validation_fun` will raise this error on failure """ for vkey, value in obj.items(): if vkey == key: validation_fun(value) - elif type(value) is dict: + elif isinstance(value, dict): validate_all_value_for_key(value, key, validation_fun) - return def validate_key(obj_name, key): @@ -139,4 +134,4 @@ def validate_key(obj_name, key): error_str = ('Invalid key name "{}" in {} object. The ' 'key name cannot contain characters ' '".", "$" or null characters').format(key, obj_name) - raise ValidationError(error_str) from ValueError() + raise ValidationError(error_str) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 152a8bcd..8e7a6bde 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -11,7 +11,7 @@ from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import (gen_timestamp, serialize, validate_txn_obj, validate_key) from bigchaindb.common.schema import validate_transaction_schema -from bigchaindb.backend.schema import validate_if_exists_language +from bigchaindb.backend.schema import validate_language_key class Transaction(Transaction): @@ -88,7 +88,7 @@ class Transaction(Transaction): validate_transaction_schema(tx_body) validate_txn_obj('asset', tx_body['asset'], 'data', validate_key) validate_txn_obj('metadata', tx_body, 'metadata', validate_key) - validate_if_exists_language(tx_body['asset'], 'data') + validate_language_key(tx_body['asset'], 'data') return super().from_dict(tx_body) @classmethod From e2c2c4b097e3e77fb35c8af44b1c5052229df0b7 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 19:15:32 +0530 Subject: [PATCH 5/6] Fix spell errors. --- bigchaindb/backend/schema.py | 12 ++++++------ bigchaindb/common/utils.py | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index fcad33a6..0e2815db 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -17,7 +17,7 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect from bigchaindb.common.exceptions import ValidationError -from bigchaindb.common.utils import validate_all_value_for_key +from bigchaindb.common.utils import validate_all_values_for_key logger = logging.getLogger(__name__) @@ -112,23 +112,23 @@ def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. Args: - obj (dict): dictonary whose "language" key is to be validated. + obj (dict): dictionary whose "language" key is to be validated. Returns: None: validation successful Raises: - ValidationError: raises execption incase language is not valid. + ValidationError: will raise exception in case language is not valid. """ backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': data = obj.get(key, {}) or {} - validate_all_value_for_key(data, 'language', validate_language) + validate_all_values_for_key(data, 'language', validate_language) def validate_language(value): - """Check if `value` is a valid language + """Check if `value` is a valid language. https://docs.mongodb.com/manual/reference/text-search-languages/ Args: @@ -138,7 +138,7 @@ def validate_language(value): None: validation successful Raises: - ValidationError: raises execption incase language is not valid. + ValidationError: will raise exception in case language is not valid. """ if value not in VALID_LANGUAGES: error_str = ('MongoDB does not support text search for the ' diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index b21f41d6..b1aa5c12 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -52,21 +52,20 @@ def deserialize(data): def validate_txn_obj(obj_name, obj, key, validation_fun): - """Validates value associated to `key` in `obj` by applying - `validation_fun`. + """Validate value of `key` in `obj` using `validation_fun`. Args: obj_name (str): name for `obj` being validated. - obj (dict): dictonary object. + obj (dict): dictionary object. key (str): key to be validated in `obj`. validation_fun (function): function used to validate the value of `key`. Returns: - None: indicates validation successfull + None: indicates validation successful Raises: - ValidationError: `validation_fun` will raise this error on failure + ValidationError: `validation_fun` will raise exception on failure """ backend = bigchaindb.config['database']['backend'] @@ -76,16 +75,16 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): def validate_all_keys(obj_name, obj, validation_fun): - """Validates all (nested) keys in `obj` by using `validation_fun` + """Validate all (nested) keys in `obj` by using `validation_fun`. Args: obj_name (str): name for `obj` being validated. - obj (dict): dictonary object. + obj (dict): dictionary object. validation_fun (function): function used to validate the value of `key`. Returns: - None: indicates validation successfull + None: indicates validation successful Raises: ValidationError: `validation_fun` will raise this error on failure @@ -96,12 +95,12 @@ def validate_all_keys(obj_name, obj, validation_fun): validate_all_keys(obj_name, value, validation_fun) -def validate_all_value_for_key(obj, key, validation_fun): - """Validates value for all (nested) occurences of `key` in `obj` - using `validation_fun` +def validate_all_values_for_key(obj, key, validation_fun): + """Validate value for all (nested) occurrence of `key` in `obj` + using `validation_fun`. Args: - obj (dict): dictonary object. + obj (dict): dictionary object. key (str): key whose value is to be validated. validation_fun (function): function used to validate the value of `key`. @@ -113,11 +112,12 @@ def validate_all_value_for_key(obj, key, validation_fun): if vkey == key: validation_fun(value) elif isinstance(value, dict): - validate_all_value_for_key(value, key, validation_fun) + validate_all_values_for_key(value, key, validation_fun) def validate_key(obj_name, key): - """Check if `key` contains ".", "$" or null characters + """Check if `key` contains ".", "$" or null characters. + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names Args: @@ -125,10 +125,10 @@ def validate_key(obj_name, key): key (str): key to validated Returns: - None: indicates validation successfull + None: validation successful Raises: - ValidationError: raise execption incase of regex match. + ValidationError: will raise exception in case of regex match. """ if re.search(r'^[$]|\.|\x00', key): error_str = ('Invalid key name "{}" in {} object. The ' From 7941922ac04ea7173f9fcdd2dcdf3e08458c6ab0 Mon Sep 17 00:00:00 2001 From: kansi Date: Mon, 6 Nov 2017 10:43:54 +0530 Subject: [PATCH 6/6] Added type validation for data --- bigchaindb/backend/schema.py | 5 +++-- bigchaindb/common/utils.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 0e2815db..4f2ebd6c 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -123,8 +123,9 @@ def validate_language_key(obj, key): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = obj.get(key, {}) or {} - validate_all_values_for_key(data, 'language', validate_language) + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_values_for_key(data, 'language', validate_language) def validate_language(value): diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index b1aa5c12..9ad448f5 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -70,8 +70,9 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = obj.get(key, {}) or {} - validate_all_keys(obj_name, data, validation_fun) + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_keys(obj_name, data, validation_fun) def validate_all_keys(obj_name, obj, validation_fun):