From 97e1a13c7f02791b8fbfa3b2190bbac43070f466 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 27 Oct 2017 11:51:02 +0200 Subject: [PATCH 01/18] Updated docs/server/requirements.txt The `requirements.txt` file in `docs/server` didn't have all the Python packages needed to build the BigchainDB Server docs. I fixed that. --- docs/server/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/server/requirements.txt b/docs/server/requirements.txt index 4321f44b..5ca634a4 100644 --- a/docs/server/requirements.txt +++ b/docs/server/requirements.txt @@ -3,3 +3,5 @@ recommonmark>=0.4.0 sphinx-rtd-theme>=0.1.9 sphinxcontrib-napoleon>=0.4.4 sphinxcontrib-httpdomain>=1.5.0 +pyyaml +bigchaindb From 722658421580c36ac32d6a7bd9e3537ed96a9bb6 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 10:12:16 +0530 Subject: [PATCH 02/18] Api fix for asset language --- bigchaindb/backend/schema.py | 22 ++++++++++++++++++ bigchaindb/models.py | 2 ++ tests/web/test_transactions.py | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index f6ce466f..8b85c068 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -16,10 +16,14 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect +from bigchaindb.common.exceptions import ValidationError logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') +VALID_LANGUAGES = ('danish' 'dutch' 'english' 'finnish' 'french' 'german' + 'hungarian' 'italian' 'norwegian' 'portuguese' 'romanian' + 'russian' 'spanish' 'swedish' 'turkish') @singledispatch @@ -99,3 +103,21 @@ def init_database(connection=None, dbname=None): create_database(connection, dbname) create_tables(connection, dbname) create_indexes(connection, dbname) + + +def validate_if_exists_asset_language(tx_body): + data = tx_body['asset'].get('data', {}) + + if data and 'language' in data: + + language = data.get('language') + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb' and language not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(language) + raise ValidationError(error_str) from ValueError() + + return diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c8ad9dd3..d88d239e 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -10,6 +10,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import gen_timestamp, serialize from bigchaindb.common.schema import validate_transaction_schema +from bigchaindb.backend.schema import validate_if_exists_asset_language class Transaction(Transaction): @@ -84,6 +85,7 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) + validate_if_exists_asset_language(tx_body) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index acea8c2c..8b4d31bf 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,6 +47,47 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub +@pytest.mark.parametrize("language,expected_status_code", [ + ('danish', 202), + ('dutch', 202), + ('english', 202), + ('finnish', 202), + ('french', 202), + ('german', 202), + ('hungarian', 202), + ('italian', 202), + ('norwegian', 202), + ('portuguese', 202), + ('romanian', 202), + ('russian', 202), + ('spanish', 202), + ('swedish', 202), + ('turkish', 202), + ('any', 400), +]) +@pytest.mark.language +@pytest.mark.bdb +def test_post_create_transaction_with_language(b, client, language, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + + if isinstance(b.connection, MongoDBConnection): + user_priv, user_pub = crypto.generate_key_pair() + + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset={'language': language}) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + "Invalid transaction (ValidationError): MongoDB does not support " + "text search for the language \"{}\". If you do not understand this " + "error message then please rename key/field \"language\" to something " + "else like \"lang\".").format(language) + assert res.json['message'] == expected_error_message + + @patch('bigchaindb.web.views.base.logger') def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash From cd636101a7c03e433c27a26d1749e87a37dbec55 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 16:54:47 +0530 Subject: [PATCH 03/18] Support abbreviated values for "language" --- bigchaindb/backend/schema.py | 12 +++++++----- tests/web/test_transactions.py | 24 ++++++++---------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 8b85c068..3b7413a2 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -21,9 +21,11 @@ from bigchaindb.common.exceptions import ValidationError logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') -VALID_LANGUAGES = ('danish' 'dutch' 'english' 'finnish' 'french' 'german' - 'hungarian' 'italian' 'norwegian' 'portuguese' 'romanian' - 'russian' 'spanish' 'swedish' 'turkish') +VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', + 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', + 'russian', 'spanish', 'swedish', 'turkish', + 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', + 'ro', 'ru', 'es', 'sv', 'tr') @singledispatch @@ -108,12 +110,12 @@ def init_database(connection=None, dbname=None): def validate_if_exists_asset_language(tx_body): data = tx_body['asset'].get('data', {}) - if data and 'language' in data: + if data and ('language' in data): language = data.get('language') backend = bigchaindb.config['database']['backend'] - if backend == 'mongodb' and language not in VALID_LANGUAGES: + if (backend == 'mongodb') and (language not in VALID_LANGUAGES): error_str = ('MongoDB does not support text search for the ' 'language "{}". If you do not understand this error ' 'message then please rename key/field "language" to ' diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 8b4d31bf..f33c47b1 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -48,22 +48,14 @@ def test_post_create_transaction_endpoint(b, client): @pytest.mark.parametrize("language,expected_status_code", [ - ('danish', 202), - ('dutch', 202), - ('english', 202), - ('finnish', 202), - ('french', 202), - ('german', 202), - ('hungarian', 202), - ('italian', 202), - ('norwegian', 202), - ('portuguese', 202), - ('romanian', 202), - ('russian', 202), - ('spanish', 202), - ('swedish', 202), - ('turkish', 202), - ('any', 400), + ('danish', 202), ('dutch', 202), ('english', 202), ('finnish', 202), + ('french', 202), ('german', 202), ('hungarian', 202), ('italian', 202), + ('norwegian', 202), ('portuguese', 202), ('romanian', 202), + ('russian', 202), ('spanish', 202), ('swedish', 202), ('turkish', 202), + ('da', 202), ('nl', 202), ('en', 202), ('fi', 202), ('fr', 202), + ('de', 202), ('hu', 202), ('it', 202), ('nb', 202), ('pt', 202), + ('ro', 202), ('ru', 202), ('es', 202), ('sv', 202), ('tr', 202), + ('any', 400) ]) @pytest.mark.language @pytest.mark.bdb From 99aa38b21742ef412bf591f81e6f8c8763db68bb Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 11:38:38 +0530 Subject: [PATCH 04/18] Added "none" to language whitelist --- bigchaindb/backend/schema.py | 49 +++++++++++++----- bigchaindb/common/utils.py | 96 +++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 14 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 3b7413a2..07fe4210 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -17,13 +17,14 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect from bigchaindb.common.exceptions import ValidationError +from bigchaindb.common.utils import validate_all_value_for_key logger = logging.getLogger(__name__) TABLES = ('bigchain', 'backlog', 'votes', 'assets') VALID_LANGUAGES = ('danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'portuguese', 'romanian', - 'russian', 'spanish', 'swedish', 'turkish', + 'russian', 'spanish', 'swedish', 'turkish', 'none', 'da', 'nl', 'en', 'fi', 'fr', 'de', 'hu', 'it', 'nb', 'pt', 'ro', 'ru', 'es', 'sv', 'tr') @@ -107,19 +108,41 @@ def init_database(connection=None, dbname=None): create_indexes(connection, dbname) -def validate_if_exists_asset_language(tx_body): - data = tx_body['asset'].get('data', {}) +def validate_if_exists_language(obj, key): + """Validate all nested "language" key in `obj`. - if data and ('language' in data): + Args: + obj (dict): dictonary whose "language" key is to be validated. - language = data.get('language') - backend = bigchaindb.config['database']['backend'] + Returns: + None: validation successfull - if (backend == 'mongodb') and (language not in VALID_LANGUAGES): - error_str = ('MongoDB does not support text search for the ' - 'language "{}". If you do not understand this error ' - 'message then please rename key/field "language" to ' - 'something else like "lang".').format(language) - raise ValidationError(error_str) from ValueError() + Raises: + ValidationError: raises execption incase language is not valid. + """ + backend = bigchaindb.config['database']['backend'] - return + if backend == 'mongodb': + data = obj.get(key, {}) or {} + validate_all_value_for_key(data, 'language', validate_language) + + +def validate_language(value): + """Check if `value` is a valid language + https://docs.mongodb.com/manual/reference/text-search-languages/ + + Args: + value (str): language to validated + + Returns: + None: validation successfull + + Raises: + ValidationError: raises execption incase language is not valid. + """ + if value not in VALID_LANGUAGES: + error_str = ('MongoDB does not support text search for the ' + 'language "{}". If you do not understand this error ' + 'message then please rename key/field "language" to ' + 'something else like "lang".').format(value) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index f6f671db..00621b2e 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -1,7 +1,10 @@ import time - +import re import rapidjson +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + def gen_timestamp(): """The Unix time, rounded to the nearest second. @@ -46,3 +49,94 @@ def deserialize(data): string. """ return rapidjson.loads(data) + + +def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validates value associated to `key` in `obj` by applying + `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = obj.get(key, {}) or {} + validate_all_keys(obj_name, data, validation_fun) + + +def validate_all_keys(obj_name, obj, validation_fun): + """Validates all (nested) keys in `obj` by using `validation_fun` + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for key, value in obj.items(): + validation_fun(obj_name, key) + if type(value) is dict: + validate_all_keys(obj_name, value, validation_fun) + return + + +def validate_all_value_for_key(obj, key, validation_fun): + """Validates value for all (nested) occurences of `key` in `obj` + using `validation_fun` + + Args: + obj (dict): dictonary object. + key (str): key whose value is to be validated. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ + for vkey, value in obj.items(): + if vkey == key: + validation_fun(value) + elif type(value) is dict: + validate_all_value_for_key(value, key, validation_fun) + return + + +def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: raise execption incase of regex match. + """ + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) from ValueError() From a29fd7e84f61f6df299faf4d955596edb090a8e7 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 18:37:19 +0530 Subject: [PATCH 05/18] Fix variable type check and docstrings --- bigchaindb/backend/schema.py | 8 ++++---- bigchaindb/common/utils.py | 11 +++-------- bigchaindb/models.py | 4 ++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 07fe4210..fcad33a6 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -108,14 +108,14 @@ def init_database(connection=None, dbname=None): create_indexes(connection, dbname) -def validate_if_exists_language(obj, key): +def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. Args: obj (dict): dictonary whose "language" key is to be validated. Returns: - None: validation successfull + None: validation successful Raises: ValidationError: raises execption incase language is not valid. @@ -135,7 +135,7 @@ def validate_language(value): value (str): language to validated Returns: - None: validation successfull + None: validation successful Raises: ValidationError: raises execption incase language is not valid. @@ -145,4 +145,4 @@ def validate_language(value): 'language "{}". If you do not understand this error ' 'message then please rename key/field "language" to ' 'something else like "lang".').format(value) - raise ValidationError(error_str) from ValueError() + raise ValidationError(error_str) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index 00621b2e..b21f41d6 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -92,9 +92,8 @@ def validate_all_keys(obj_name, obj, validation_fun): """ for key, value in obj.items(): validation_fun(obj_name, key) - if type(value) is dict: + if isinstance(value, dict): validate_all_keys(obj_name, value, validation_fun) - return def validate_all_value_for_key(obj, key, validation_fun): @@ -107,18 +106,14 @@ def validate_all_value_for_key(obj, key, validation_fun): validation_fun (function): function used to validate the value of `key`. - Returns: - None: indicates validation successfull - Raises: ValidationError: `validation_fun` will raise this error on failure """ for vkey, value in obj.items(): if vkey == key: validation_fun(value) - elif type(value) is dict: + elif isinstance(value, dict): validate_all_value_for_key(value, key, validation_fun) - return def validate_key(obj_name, key): @@ -139,4 +134,4 @@ def validate_key(obj_name, key): error_str = ('Invalid key name "{}" in {} object. The ' 'key name cannot contain characters ' '".", "$" or null characters').format(key, obj_name) - raise ValidationError(error_str) from ValueError() + raise ValidationError(error_str) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 152a8bcd..8e7a6bde 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -11,7 +11,7 @@ from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import (gen_timestamp, serialize, validate_txn_obj, validate_key) from bigchaindb.common.schema import validate_transaction_schema -from bigchaindb.backend.schema import validate_if_exists_language +from bigchaindb.backend.schema import validate_language_key class Transaction(Transaction): @@ -88,7 +88,7 @@ class Transaction(Transaction): validate_transaction_schema(tx_body) validate_txn_obj('asset', tx_body['asset'], 'data', validate_key) validate_txn_obj('metadata', tx_body, 'metadata', validate_key) - validate_if_exists_language(tx_body['asset'], 'data') + validate_language_key(tx_body['asset'], 'data') return super().from_dict(tx_body) @classmethod From e2c2c4b097e3e77fb35c8af44b1c5052229df0b7 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 3 Nov 2017 19:15:32 +0530 Subject: [PATCH 06/18] Fix spell errors. --- bigchaindb/backend/schema.py | 12 ++++++------ bigchaindb/common/utils.py | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index fcad33a6..0e2815db 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -17,7 +17,7 @@ import logging import bigchaindb from bigchaindb.backend.connection import connect from bigchaindb.common.exceptions import ValidationError -from bigchaindb.common.utils import validate_all_value_for_key +from bigchaindb.common.utils import validate_all_values_for_key logger = logging.getLogger(__name__) @@ -112,23 +112,23 @@ def validate_language_key(obj, key): """Validate all nested "language" key in `obj`. Args: - obj (dict): dictonary whose "language" key is to be validated. + obj (dict): dictionary whose "language" key is to be validated. Returns: None: validation successful Raises: - ValidationError: raises execption incase language is not valid. + ValidationError: will raise exception in case language is not valid. """ backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': data = obj.get(key, {}) or {} - validate_all_value_for_key(data, 'language', validate_language) + validate_all_values_for_key(data, 'language', validate_language) def validate_language(value): - """Check if `value` is a valid language + """Check if `value` is a valid language. https://docs.mongodb.com/manual/reference/text-search-languages/ Args: @@ -138,7 +138,7 @@ def validate_language(value): None: validation successful Raises: - ValidationError: raises execption incase language is not valid. + ValidationError: will raise exception in case language is not valid. """ if value not in VALID_LANGUAGES: error_str = ('MongoDB does not support text search for the ' diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index b21f41d6..b1aa5c12 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -52,21 +52,20 @@ def deserialize(data): def validate_txn_obj(obj_name, obj, key, validation_fun): - """Validates value associated to `key` in `obj` by applying - `validation_fun`. + """Validate value of `key` in `obj` using `validation_fun`. Args: obj_name (str): name for `obj` being validated. - obj (dict): dictonary object. + obj (dict): dictionary object. key (str): key to be validated in `obj`. validation_fun (function): function used to validate the value of `key`. Returns: - None: indicates validation successfull + None: indicates validation successful Raises: - ValidationError: `validation_fun` will raise this error on failure + ValidationError: `validation_fun` will raise exception on failure """ backend = bigchaindb.config['database']['backend'] @@ -76,16 +75,16 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): def validate_all_keys(obj_name, obj, validation_fun): - """Validates all (nested) keys in `obj` by using `validation_fun` + """Validate all (nested) keys in `obj` by using `validation_fun`. Args: obj_name (str): name for `obj` being validated. - obj (dict): dictonary object. + obj (dict): dictionary object. validation_fun (function): function used to validate the value of `key`. Returns: - None: indicates validation successfull + None: indicates validation successful Raises: ValidationError: `validation_fun` will raise this error on failure @@ -96,12 +95,12 @@ def validate_all_keys(obj_name, obj, validation_fun): validate_all_keys(obj_name, value, validation_fun) -def validate_all_value_for_key(obj, key, validation_fun): - """Validates value for all (nested) occurences of `key` in `obj` - using `validation_fun` +def validate_all_values_for_key(obj, key, validation_fun): + """Validate value for all (nested) occurrence of `key` in `obj` + using `validation_fun`. Args: - obj (dict): dictonary object. + obj (dict): dictionary object. key (str): key whose value is to be validated. validation_fun (function): function used to validate the value of `key`. @@ -113,11 +112,12 @@ def validate_all_value_for_key(obj, key, validation_fun): if vkey == key: validation_fun(value) elif isinstance(value, dict): - validate_all_value_for_key(value, key, validation_fun) + validate_all_values_for_key(value, key, validation_fun) def validate_key(obj_name, key): - """Check if `key` contains ".", "$" or null characters + """Check if `key` contains ".", "$" or null characters. + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names Args: @@ -125,10 +125,10 @@ def validate_key(obj_name, key): key (str): key to validated Returns: - None: indicates validation successfull + None: validation successful Raises: - ValidationError: raise execption incase of regex match. + ValidationError: will raise exception in case of regex match. """ if re.search(r'^[$]|\.|\x00', key): error_str = ('Invalid key name "{}" in {} object. The ' From 7941922ac04ea7173f9fcdd2dcdf3e08458c6ab0 Mon Sep 17 00:00:00 2001 From: kansi Date: Mon, 6 Nov 2017 10:43:54 +0530 Subject: [PATCH 07/18] Added type validation for data --- bigchaindb/backend/schema.py | 5 +++-- bigchaindb/common/utils.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/schema.py b/bigchaindb/backend/schema.py index 0e2815db..4f2ebd6c 100644 --- a/bigchaindb/backend/schema.py +++ b/bigchaindb/backend/schema.py @@ -123,8 +123,9 @@ def validate_language_key(obj, key): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = obj.get(key, {}) or {} - validate_all_values_for_key(data, 'language', validate_language) + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_values_for_key(data, 'language', validate_language) def validate_language(value): diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index b1aa5c12..9ad448f5 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -70,8 +70,9 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = obj.get(key, {}) or {} - validate_all_keys(obj_name, data, validation_fun) + data = obj.get(key, {}) + if isinstance(data, dict): + validate_all_keys(obj_name, data, validation_fun) def validate_all_keys(obj_name, obj, validation_fun): From 3cbec5a864f004b5e35d2c6b45acb7bcecb059dd Mon Sep 17 00:00:00 2001 From: kansi Date: Mon, 6 Nov 2017 12:06:01 +0530 Subject: [PATCH 08/18] Updated metadata and asset model docs --- docs/server/source/data-models/asset-model.md | 8 ++++++++ docs/server/source/data-models/transaction-model.rst | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index eefa81bb..2d7ac1e8 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -18,3 +18,11 @@ In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pa "id": "38100137cea87fb9bd751e2372abb2c73e7d5bcf39d940a5516a324d9c7fb88d" } ``` + + +.. note:: + + When using MongoDB for storage certain restriction apply to all (including nested) keys of `"data"` JSON document i.e. valid keys should **not** begin with ``$`` character and cannot contain ``.`` or null character (Unicode code point 0000). Furthermore, the key `"language"` (at any level in the hierarchy) in `"data"` JSON document is a spcial key and used for specifying text search language. Its value must be one of the allowed values, see `Text search languages `_ . It must be noted that only the languages supported by MongoDB community edition are allowed. + + + diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 38e523bd..2b28b592 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -46,6 +46,10 @@ Here's some explanation of the contents: - **metadata**: User-provided transaction metadata. It can be any valid JSON document, or ``null``. + **NOTE** when using MongoDB for storage certain restriction apply to all + (including nested) keys of ``"metadata"`` JSON document i.e. valid keys + should **not** begin with ``$`` character and cannot contain ``.`` or + null character (Unicode code point 0000). **How the transaction ID is computed.** 1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, From c616145a66e3407fa3f12bc52560cc607548041a Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Nov 2017 12:12:16 +0100 Subject: [PATCH 09/18] pyyaml>=3.12 in docs/server/requirements.txt --- docs/server/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server/requirements.txt b/docs/server/requirements.txt index 5ca634a4..cd06eab9 100644 --- a/docs/server/requirements.txt +++ b/docs/server/requirements.txt @@ -3,5 +3,5 @@ recommonmark>=0.4.0 sphinx-rtd-theme>=0.1.9 sphinxcontrib-napoleon>=0.4.4 sphinxcontrib-httpdomain>=1.5.0 -pyyaml +pyyaml>=3.12 bigchaindb From 636c28d31177121d382859ad27b35935a9d92e16 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Nov 2017 14:22:17 +0100 Subject: [PATCH 10/18] Edited changes to the docs about the asset model --- docs/server/source/data-models/asset-model.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/server/source/data-models/asset-model.md b/docs/server/source/data-models/asset-model.md index 2d7ac1e8..054a4083 100644 --- a/docs/server/source/data-models/asset-model.md +++ b/docs/server/source/data-models/asset-model.md @@ -2,6 +2,8 @@ To avoid redundant data in transactions, the asset model is different for `CREATE` and `TRANSFER` transactions. +## In CREATE Transactions + In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair. The key must be `"data"` and the value can be any valid JSON document, or `null`. For example: ```json { @@ -12,17 +14,18 @@ In a `CREATE` transaction, the `"asset"` must contain exactly one key-value pair } ``` +When using MongoDB for storage, certain restriction apply to all (including nested) keys of the `"data"` JSON document: + +* Keys (i.e. key names, not values) must **not** begin with the `$` character. +* Keys must not contain `.` or the null character (Unicode code point 0000). +* The key `"language"` (at any level in the hierarchy) is a special key and used for specifying text search language. Its value must be one of the allowed values; see the valid [Text Search Languages](https://docs.mongodb.com/manual/reference/text-search-languages/) in the MongoDB Docs. In BigchainDB, only the languages supported by _MongoDB community edition_ are allowed. + + +## In TRANSFER Transactions + In a `TRANSFER` transaction, the `"asset"` must contain exactly one key-value pair. They key must be `"id"` and the value must contain a transaction ID (i.e. a SHA3-256 hash: the ID of the `CREATE` transaction which created the asset, which also serves as the asset ID). For example: ```json { "id": "38100137cea87fb9bd751e2372abb2c73e7d5bcf39d940a5516a324d9c7fb88d" } ``` - - -.. note:: - - When using MongoDB for storage certain restriction apply to all (including nested) keys of `"data"` JSON document i.e. valid keys should **not** begin with ``$`` character and cannot contain ``.`` or null character (Unicode code point 0000). Furthermore, the key `"language"` (at any level in the hierarchy) in `"data"` JSON document is a spcial key and used for specifying text search language. Its value must be one of the allowed values, see `Text search languages `_ . It must be noted that only the languages supported by MongoDB community edition are allowed. - - - From cae883e9cbca276f491fbf0412b3997be3b74e5b Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Mon, 6 Nov 2017 14:30:01 +0100 Subject: [PATCH 11/18] Edited the changes to transaction model docs --- docs/server/source/data-models/transaction-model.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 2b28b592..b4f2a4f1 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -46,10 +46,10 @@ Here's some explanation of the contents: - **metadata**: User-provided transaction metadata. It can be any valid JSON document, or ``null``. - **NOTE** when using MongoDB for storage certain restriction apply to all - (including nested) keys of ``"metadata"`` JSON document i.e. valid keys - should **not** begin with ``$`` character and cannot contain ``.`` or - null character (Unicode code point 0000). + **NOTE:** When using MongoDB for storage, certain restriction apply + to all (including nested) keys of the ``"data"`` JSON document: + 1) keys (i.e. key names, not values) must **not** begin with the ``$`` character, and + 2) keys must not contain ``.`` or the null character (Unicode code point 0000). **How the transaction ID is computed.** 1) Build a Python dictionary containing ``version``, ``inputs``, ``outputs``, ``operation``, ``asset``, ``metadata`` and their values, From d55004601c6a1e6e8444b73f9ee1f1b3d70dd61e Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Nov 2017 10:17:21 +0100 Subject: [PATCH 12/18] Fix command to run RethinkDB in bg w/ docker-compose --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 252fcda4..9fc9b9d5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -105,7 +105,7 @@ $ docker-compose build First, start `RethinkDB` in the background: ```text -$ docker-compose up -d rdb +$ docker-compose -f docker-compose.rdb.yml up -d rdb ``` then run the tests using: From 6cab2f26e92317f16b97174cb72c6b8e4c5838dd Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Nov 2017 10:20:41 +0100 Subject: [PATCH 13/18] Fix command to run tests w/ docker-compose & bdb-rdb --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 9fc9b9d5..e3c9133c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -111,7 +111,7 @@ $ docker-compose -f docker-compose.rdb.yml up -d rdb then run the tests using: ```text -$ docker-compose run --rm bdb-rdb py.test -v +$ docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb py.test -v ``` to rebuild all the images (usually you only need to rebuild the `bdb` and From 220465f7016bd2b71fb14f0e428a84ef418596c7 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 7 Nov 2017 14:55:05 +0530 Subject: [PATCH 14/18] Added "--no-init" flag for "bigchaindb start" command --- bigchaindb/commands/bigchaindb.py | 9 +++++---- .../source/appendices/azure-quickstart-template.md | 2 +- docs/server/source/dev-and-test/setup-bdb-host.md | 4 ++-- docs/server/source/quickstart.md | 2 +- docs/server/source/server-reference/bigchaindb-cli.md | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/bigchaindb/commands/bigchaindb.py b/bigchaindb/commands/bigchaindb.py index 9705065c..6320d279 100644 --- a/bigchaindb/commands/bigchaindb.py +++ b/bigchaindb/commands/bigchaindb.py @@ -196,7 +196,7 @@ def run_start(args): logger.info('RethinkDB started with PID %s' % proc.pid) try: - if args.initialize_database: + if not args.skip_initialize_database: logger.info('Initializing database') _run_init() except DatabaseAlreadyExists: @@ -302,10 +302,11 @@ def create_parser(): action='store_true', help='Run RethinkDB on start') - start_parser.add_argument('--init', - dest='initialize_database', + start_parser.add_argument('--no-init', + dest='skip_initialize_database', + default=False, action='store_true', - help='Force initialize database') + help='Skip database initialization') # parser for configuring the number of shards sharding_parser = subparsers.add_parser('set-shards', diff --git a/docs/server/source/appendices/azure-quickstart-template.md b/docs/server/source/appendices/azure-quickstart-template.md index 13cda281..59f52fd3 100644 --- a/docs/server/source/appendices/azure-quickstart-template.md +++ b/docs/server/source/appendices/azure-quickstart-template.md @@ -33,7 +33,7 @@ API Server bind? (default `localhost:9984`): 0.0.0.0:9984 Finally, run BigchainDB Server by doing: ```text -bigchaindb start --init +bigchaindb start ``` BigchainDB Server should now be running on the Azure virtual machine. diff --git a/docs/server/source/dev-and-test/setup-bdb-host.md b/docs/server/source/dev-and-test/setup-bdb-host.md index 5feb8c42..cdee3c0b 100644 --- a/docs/server/source/dev-and-test/setup-bdb-host.md +++ b/docs/server/source/dev-and-test/setup-bdb-host.md @@ -27,7 +27,7 @@ waiting for connections on port 27017 To run BigchainDB Server, do: ```text -$ bigchaindb start --init +$ bigchaindb start ``` You can [run all the unit tests](running-all-tests.html) to test your installation. @@ -55,7 +55,7 @@ You can verify that RethinkDB is running by opening the RethinkDB web interface To run BigchainDB Server, do: ```text -$ bigchaindb start --init +$ bigchaindb start ``` You can [run all the unit tests](running-all-tests.html) to test your installation. diff --git a/docs/server/source/quickstart.md b/docs/server/source/quickstart.md index 2375fd5f..63ab8643 100644 --- a/docs/server/source/quickstart.md +++ b/docs/server/source/quickstart.md @@ -54,7 +54,7 @@ $ bigchaindb -y configure mongodb I. Run BigchainDB Server: ```text -$ bigchaindb start --init +$ bigchaindb start ``` J. Verify BigchainDB Server setup by visiting the BigchainDB Root URL in your browser: diff --git a/docs/server/source/server-reference/bigchaindb-cli.md b/docs/server/source/server-reference/bigchaindb-cli.md index fddfd3f5..790cb453 100644 --- a/docs/server/source/server-reference/bigchaindb-cli.md +++ b/docs/server/source/server-reference/bigchaindb-cli.md @@ -61,7 +61,7 @@ If you want to force-drop the database (i.e. skipping the yes/no prompt), then u ## bigchaindb start -Start BigchainDB assuming that the database has already been initialized using `bigchaindb init`. If that is not the case then passing the flag `--init` will initialize the database and start BigchainDB. +Start BigchainDB. It always begins by trying a `bigchaindb init` first. See the note in the documentation for `bigchaindb init`. The database initialization step is optional and can be skipped by passing the `--no-init` flag i.e. `bigchaindb start --no-init`. You can also use the `--dev-start-rethinkdb` command line option to automatically start rethinkdb with bigchaindb if rethinkdb is not already running, e.g. `bigchaindb --dev-start-rethinkdb start`. Note that this will also shutdown rethinkdb when the bigchaindb process stops. The option `--dev-allow-temp-keypair` will generate a keypair on the fly if no keypair is found, this is useful when you want to run a temporary instance of BigchainDB in a Docker container, for example. From 8aba802425bd9383ac15af32e2278fd649362c05 Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 7 Nov 2017 15:41:55 +0530 Subject: [PATCH 15/18] Fix tests --- tests/commands/conftest.py | 2 +- tests/commands/rethinkdb/test_commands.py | 2 +- tests/commands/test_commands.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index 1aef2d30..46f8a8f6 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -49,7 +49,7 @@ def run_start_args(request): config=param.get('config'), start_rethinkdb=param.get('start_rethinkdb', False), allow_temp_keypair=param.get('allow_temp_keypair', False), - initialize_database=param.get('initialize_database', True), + skip_initialize_database=param.get('skip_initialize_database', False), ) diff --git a/tests/commands/rethinkdb/test_commands.py b/tests/commands/rethinkdb/test_commands.py index 29a84972..c8990582 100644 --- a/tests/commands/rethinkdb/test_commands.py +++ b/tests/commands/rethinkdb/test_commands.py @@ -14,7 +14,7 @@ def test_bigchain_run_start_with_rethinkdb(mock_start_rethinkdb, from bigchaindb import config from bigchaindb.commands.bigchaindb import run_start args = Namespace(start_rethinkdb=True, allow_temp_keypair=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mock_start_rethinkdb.assert_called_with() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 15aa9302..423e4614 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -40,7 +40,7 @@ def test_bigchain_run_start(mock_run_configure, from bigchaindb import config from bigchaindb.commands.bigchaindb import run_start args = Namespace(start_rethinkdb=False, allow_temp_keypair=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with(user_log_config=config['log']) @@ -290,7 +290,7 @@ def test_allow_temp_keypair_generates_one_on_the_fly( bigchaindb.config['keypair'] = {'private': None, 'public': None} args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with( @@ -317,7 +317,7 @@ def test_allow_temp_keypair_doesnt_override_if_keypair_found(mock_gen_keypair, assert isinstance(original_private_key, str) args = Namespace(allow_temp_keypair=True, start_rethinkdb=False, config=None, yes=True, - initialize_database=True) + skip_initialize_database=False) run_start(args) mocked_setup_logging.assert_called_once_with( From 1266c43ff87366ee7b3fb1f8a668feeae537a75e Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 7 Nov 2017 18:44:51 +0530 Subject: [PATCH 16/18] Fix docker files --- docker-compose.benchmark.yml | 2 +- docker-compose.rdb.yml | 2 +- docker-compose.yml | 2 +- tests/integration/test_integration.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.benchmark.yml b/docker-compose.benchmark.yml index 2a2aacc2..c7319040 100644 --- a/docker-compose.benchmark.yml +++ b/docker-compose.benchmark.yml @@ -25,7 +25,7 @@ services: BIGCHAINDB_GRAPHITE_HOST: graphite ports: - "9984" - command: bigchaindb start --init + command: bigchaindb start graphite: image: hopsoft/graphite-statsd diff --git a/docker-compose.rdb.yml b/docker-compose.rdb.yml index e02aa444..15f91675 100644 --- a/docker-compose.rdb.yml +++ b/docker-compose.rdb.yml @@ -45,4 +45,4 @@ services: BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 ports: - "9984" - command: bigchaindb start --init + command: bigchaindb start diff --git a/docker-compose.yml b/docker-compose.yml index 8f774106..cd6aa2aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,4 +30,4 @@ services: BIGCHAINDB_WSSERVER_HOST: 0.0.0.0 ports: - "9984" - command: bigchaindb start --init + command: bigchaindb start diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 2bf0ebcd..21b783e6 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -12,9 +12,9 @@ def test_double_create(b, user_pk): metadata={'test': 'test'}).sign([b.me_private]) b.write_transaction(tx) - time.sleep(2) + time.sleep(5) b.write_transaction(tx) - time.sleep(2) + time.sleep(5) tx_returned = b.get_transaction(tx.id) # test that the tx can be queried From e725c23691a69d3e6ebdd35fa9f0c69320474509 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 7 Nov 2017 14:24:33 +0100 Subject: [PATCH 17/18] Added note about 'make clean-pyc' --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index e3c9133c..146d3bc6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -115,7 +115,7 @@ $ docker-compose -f docker-compose.rdb.yml run --rm bdb-rdb py.test -v ``` to rebuild all the images (usually you only need to rebuild the `bdb` and - `bdb-rdb` images). + `bdb-rdb` images). If that fails, then do `make clean-pyc` and try again. ## Automated Testing of All Pull Requests From 5ce2eb1f971086868f1cc4869db23affd1487ac8 Mon Sep 17 00:00:00 2001 From: Vanshdeep Singh Date: Thu, 9 Nov 2017 00:11:17 +0530 Subject: [PATCH 18/18] Bypass CI issues for rethinkdb (#1821) Add custom marker to skip some travis/rdb tests --- tests/conftest.py | 9 +++++++++ tests/integration/test_federation.py | 3 +++ tests/integration/test_integration.py | 1 + 3 files changed, 13 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index b930e4ec..4b5c7946 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,15 @@ USER_PRIVATE_KEY = '8eJ8q9ZQpReWyQT5aFCiwtZ5wDZC4eDnCen88p3tQ6ie' USER_PUBLIC_KEY = 'JEAkEJqLbbgDRAtMm8YAjGp759Aq2qTn9eaEHUj2XePE' +def pytest_runtest_setup(item): + if isinstance(item, item.Function): + if item.get_marker('skip_travis_rdb'): + if (os.getenv('TRAVIS_CI') == 'true' and + os.getenv('BIGCHAINDB_DATABASE_BACKEND') == 'rethinkdb'): + pytest.skip( + 'Skip test during Travis CI build when using rethinkdb') + + def pytest_addoption(parser): from bigchaindb.backend.connection import BACKENDS diff --git a/tests/integration/test_federation.py b/tests/integration/test_federation.py index 598412ff..22e2e8da 100644 --- a/tests/integration/test_federation.py +++ b/tests/integration/test_federation.py @@ -97,6 +97,7 @@ def process_vote(steps, result=None): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_valid(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) @@ -115,6 +116,7 @@ def test_elect_valid(federation_3): @pytest.mark.bdb +@pytest.mark.skip_travis_rdb @pytest.mark.genesis def test_elect_invalid(federation_3): [bx, (s0, s1, s2)] = federation_3 @@ -135,6 +137,7 @@ def test_elect_invalid(federation_3): @pytest.mark.bdb @pytest.mark.genesis +@pytest.mark.skip_travis_rdb def test_elect_sybill(federation_3): [bx, (s0, s1, s2)] = federation_3 tx = input_single_create(bx[0]) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 2bf0ebcd..50734fee 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -5,6 +5,7 @@ import pytest pytestmark = [pytest.mark.bdb, pytest.mark.usefixtures('processes')] +@pytest.mark.skip_travis_rdb def test_double_create(b, user_pk): from bigchaindb.models import Transaction from bigchaindb.backend.query import count_blocks