From fba6e1b30bc98065b4921264ff4164e9de28f437 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Wed, 18 Oct 2017 15:38:46 +0200 Subject: [PATCH 1/7] New root docs page about permissions in BigchainDB --- docs/root/source/conf.py | 4 +- docs/root/source/index.rst | 1 + docs/root/source/permissions.rst | 74 ++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 docs/root/source/permissions.rst diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py index 0d799fed..990fa3ce 100644 --- a/docs/root/source/conf.py +++ b/docs/root/source/conf.py @@ -34,7 +34,9 @@ from recommonmark.parser import CommonMarkParser # ones. import sphinx_rtd_theme -extensions = [] +extensions = [ + 'sphinx.ext.autosectionlabel', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/root/source/index.rst b/docs/root/source/index.rst index 455cae0c..71fdd022 100644 --- a/docs/root/source/index.rst +++ b/docs/root/source/index.rst @@ -88,5 +88,6 @@ More About BigchainDB assets smart-contracts transaction-concepts + permissions timestamps Data Models diff --git a/docs/root/source/permissions.rst b/docs/root/source/permissions.rst new file mode 100644 index 00000000..d5972ac0 --- /dev/null +++ b/docs/root/source/permissions.rst @@ -0,0 +1,74 @@ +Permissions in BigchainDB +------------------------- + +BigchainDB lets users control what other users can do, to some extent. That ability resembles "permissions" in the \*nix world, "privileges" in the SQL world, and "access control" in the security world. + + +Permission to Spend/Transfer an Output +====================================== + +In BigchainDB, every output has an associated condition (crypto-condition). + +To spend/transfer an unspent output, a user (or group of users) must fulfill the condition. Another way to say that is that only certain users have permission to spend the output. The simplest condition is of the form, "Only someone with the private key corresponding to this public key can spend this output." Much more elaborate conditions are possible, e.g. "To spend this output, …" + +- "…anyone in the Accounting Group can sign." +- "…three of these four people must sign." +- "…either Bob must sign, or both Tom and Sylvia must sign." + +For details, see `the documentation about conditions in BigchainDB `_. + +Once an output has been spent, it can't be spent again: *nobody* has permission to do that. That is, BigchainDB doesn't permit anyone to "double spend" an output. + + +Write Permissions +================= + +When someone builds a TRANSFER transaction, they can put an arbitrary JSON object in the ``metadata`` field (within reason; real BigchainDB networks put a limit on the size of transactions). That is, they can write just about anything they want in a TRANSFER transaction. + +Does that mean there are no "write permissions" in BigchainDB? Not at all! + +A TRANSFER transaction will only be valid (allowed) if its inputs fulfill some previous outputs. The conditions on those outputs will control who can build valid TRANSFER transactions. In other words, one can interpret the condition on an output as giving "write permissions" to certain users to write something into the history of the associated asset. + +As a concrete example, you could use BigchainDB to write a public journal where only you have write permissions. Here's how: First you'd build a CREATE transaction with the ``asset.data`` being something like ``{"title": "The Journal of John Doe"}``, with one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. +Each time you want to append something to your journal, you'd build a new TRANSFER transaction with your latest entry in the ``metadata`` field, e.g. + +.. code-block:: json + + {"timestamp": "1508319582", + "entry": "I visited Marmot Lake with Jane."} + +The TRANSFER transaction would have one output. That output would have an amount 1 and a condition that only you (who has your private key) can spend that output. And so on. Only you would be able to append to the history of that asset (your journal). + +The same technique could be used for scientific notebooks, supply-chain records, government meeting minutes, and so on. + +You could do more elaborate things too. As one example, each time someone writes a TRANSFER transaction, they give *someone else* permission to spend it, setting up a sort of writers-relay or chain letter. + +.. note:: + + Anyone can write any JSON (again, within reason) in the ``asset.data`` field of a CREATE transaction. They don't need permission. + + +Read Permissions +================ + +All the data stored in a BigchainDB network can be read by anyone with access to that network. One *can* store encrypted data, but if the decryption key ever leaks out, then the encrypted data can be read, decrypted, and leak out too. (Deleting the encrypted data is :doc:`not an option `.) + +The permission to read some specific information (e.g. a music file) can be thought of as an *asset*. (In many countries, that permission or "right" is a kind of intellectual property.) +BigchainDB can be used to register that asset and transfer it from owner to owner. +Today, BigchainDB does not have a way to restrict read access of data stored in a BigchainDB network, but many third-party services do offer that (e.g. Google Docs, Dropbox). +In principle, a third party service could ask a BigchainDB network to determine if a particular user has permission to read some particular data. Indeed they could use BigchainDB to keep track of *all* the rights a user has for some data (not just the right to read it). +That third party could also use BigchainDB to store audit logs, i.e. records of every read, write or other operation on stored data. + +BigchainDB can be used in other ways to help parties exchange private data: + +- It can be used to publicly disclose the *availability* of some private data (stored elsewhere). For example, there might be a description of the data and a price. +- It can be used to record the TLS handshakes which two parties sent to each other to establish an encrypted and authenticated TLS connection, which they could use to exchange private data with each other. (The stored handshake information wouldn't be enough, by itself, to decrypt the data.) It would be a "proof of TLS handshake." +- See the BigchainDB `Privacy Protocols repository `_ for more techniques. + + +Role-Based Access Control (RBAC) +================================ + +In September 2017, we published a `blog post about how one can define an RBAC sub-system on top of BigchainDB `_. +At the time of writing (October 2017), doing so required the use of a plugin, so it's not possible using standard BigchainDB (which is what's available on `IPDB `_). That may change in the future. +If you're interested, `contact BigchainDB `_. From ddfce61b79895c63c84ace2e8861e1a59c63ed67 Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 27 Oct 2017 15:05:43 +0530 Subject: [PATCH 2/7] Added secondary index for "id" in bigchain collection. --- bigchaindb/backend/mongodb/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index e398560f..00616dcb 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -50,6 +50,11 @@ def drop_database(conn, dbname): def create_bigchain_secondary_index(conn, dbname): logger.info('Create `bigchain` secondary index.') + # secondary index on block id which is should be unique + conn.conn[dbname]['bigchain'].create_index('id', + name='block_id', + unique=True) + # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', ASCENDING)], From 421c67c62149b28b8de137c3ed46659c743df4dd Mon Sep 17 00:00:00 2001 From: kansi Date: Fri, 27 Oct 2017 15:31:44 +0530 Subject: [PATCH 3/7] Fixed mongodb tests --- bigchaindb/backend/mongodb/schema.py | 3 +-- tests/backend/mongodb/test_schema.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 00616dcb..01eac29c 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -52,8 +52,7 @@ def create_bigchain_secondary_index(conn, dbname): # secondary index on block id which is should be unique conn.conn[dbname]['bigchain'].create_index('id', - name='block_id', - unique=True) + name='block_id') # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', diff --git a/tests/backend/mongodb/test_schema.py b/tests/backend/mongodb/test_schema.py index e11dbfe8..1a244b1b 100644 --- a/tests/backend/mongodb/test_schema.py +++ b/tests/backend/mongodb/test_schema.py @@ -22,8 +22,8 @@ def test_init_creates_db_tables_and_indexes(): 'votes'] indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', - 'outputs', 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_id', 'block_timestamp', + 'inputs', 'outputs', 'transaction_id'] indexes = conn.conn[dbname]['backlog'].index_information().keys() assert sorted(indexes) == ['_id_', 'assignee__transaction_timestamp', @@ -86,8 +86,8 @@ def test_create_secondary_indexes(): # Bigchain table indexes = conn.conn[dbname]['bigchain'].index_information().keys() - assert sorted(indexes) == ['_id_', 'asset_id', 'block_timestamp', 'inputs', - 'outputs', 'transaction_id'] + assert sorted(indexes) == ['_id_', 'asset_id', 'block_id', 'block_timestamp', + 'inputs', 'outputs', 'transaction_id'] # Backlog table indexes = conn.conn[dbname]['backlog'].index_information().keys() From 1de5375962306b1ce04c553eb38ad5237839959f Mon Sep 17 00:00:00 2001 From: kansi Date: Tue, 31 Oct 2017 15:16:59 +0530 Subject: [PATCH 4/7] Validate asset data keys --- bigchaindb/common/utils.py | 19 ++++++++++++++++++- bigchaindb/models.py | 4 +++- tests/web/test_transactions.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index f6f671db..fd9386e1 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -1,7 +1,10 @@ import time - +import re import rapidjson +import bigchaindb +from bigchaindb.common.exceptions import ValidationError + def gen_timestamp(): """The Unix time, rounded to the nearest second. @@ -46,3 +49,17 @@ def deserialize(data): string. """ return rapidjson.loads(data) + + +def validate_asset_data_keys(tx_body): + backend = bigchaindb.config['database']['backend'] + + if backend == 'mongodb': + data = tx_body['asset'].get('data', {}) + keys = data.keys() if data else [] + for key in keys: + if re.search(r'^[$]|\.', key): + error_str = ('Invalid key name "{}" in asset object. The ' + 'key name cannot contain characters ' + '"." and "$"').format(key) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/models.py b/bigchaindb/models.py index c8ad9dd3..9704372d 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -8,7 +8,8 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, SybilError, DuplicateTransaction) from bigchaindb.common.transaction import Transaction -from bigchaindb.common.utils import gen_timestamp, serialize +from bigchaindb.common.utils import (gen_timestamp, serialize, + validate_asset_data_keys) from bigchaindb.common.schema import validate_transaction_schema @@ -84,6 +85,7 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) + validate_asset_data_keys(tx_body) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index acea8c2c..4c2a6ed9 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,6 +47,34 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub +@pytest.mark.parametrize("key,expected_status_code", [ + ('bad.key', 400), + ('$bad.key', 400), + ('$badkey', 400), + ('good_key', 202) +]) +@pytest.mark.assetkey +@pytest.mark.bdb +def test_post_create_transaction_with_invalid_asset_key(b, client, key, expected_status_code): + from bigchaindb.models import Transaction + from bigchaindb.backend.mongodb.connection import MongoDBConnection + user_priv, user_pub = crypto.generate_key_pair() + + if isinstance(b.connection, MongoDBConnection): + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset={key: 'random_value'}) + tx = tx.sign([user_priv]) + res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) + + assert res.status_code == expected_status_code + if res.status_code == 400: + expected_error_message = ( + 'Invalid transaction (ValidationError): Invalid key name "{}" ' + 'in asset object. The key name cannot contain characters ' + '"." and "$"').format(key) + assert res.json['message'] == expected_error_message + + @patch('bigchaindb.web.views.base.logger') def test_post_create_transaction_with_invalid_id(mock_logger, b, client): from bigchaindb.common.exceptions import InvalidHash From f3da30aea082347a8ae72d1b8ddd97e0def5d37d Mon Sep 17 00:00:00 2001 From: kansi Date: Wed, 1 Nov 2017 17:26:16 +0530 Subject: [PATCH 5/7] Validate nested keys for asset.data and metadata --- bigchaindb/common/utils.py | 28 +++++++++++++++++++--------- bigchaindb/models.py | 5 +++-- tests/web/test_transactions.py | 29 ++++++++++++++++++----------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index fd9386e1..35163f14 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -51,15 +51,25 @@ def deserialize(data): return rapidjson.loads(data) -def validate_asset_data_keys(tx_body): +def validate_txn_obj(obj_name, obj, key, validation_fun): backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': - data = tx_body['asset'].get('data', {}) - keys = data.keys() if data else [] - for key in keys: - if re.search(r'^[$]|\.', key): - error_str = ('Invalid key name "{}" in asset object. The ' - 'key name cannot contain characters ' - '"." and "$"').format(key) - raise ValidationError(error_str) from ValueError() + data = obj.get(key, {}) or {} + validate_all_keys(obj_name, data, validation_fun) + + +def validate_all_keys(obj_name, obj, validation_fun): + for key, value in obj.items(): + validation_fun(obj_name, key) + if type(value) is dict: + validate_all_keys(obj_name, value, validation_fun) + return + + +def validate_key(obj_name, key): + if re.search(r'^[$]|\.|\x00', key): + error_str = ('Invalid key name "{}" in {} object. The ' + 'key name cannot contain characters ' + '".", "$" or null characters').format(key, obj_name) + raise ValidationError(error_str) from ValueError() diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 9704372d..1ecd964e 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -9,7 +9,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, DuplicateTransaction) from bigchaindb.common.transaction import Transaction from bigchaindb.common.utils import (gen_timestamp, serialize, - validate_asset_data_keys) + validate_txn_obj, validate_key) from bigchaindb.common.schema import validate_transaction_schema @@ -85,7 +85,8 @@ class Transaction(Transaction): @classmethod def from_dict(cls, tx_body): validate_transaction_schema(tx_body) - validate_asset_data_keys(tx_body) + validate_txn_obj('asset', tx_body['asset'], 'data', validate_key) + validate_txn_obj('metadata', tx_body, 'metadata', validate_key) return super().from_dict(tx_body) @classmethod diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 4c2a6ed9..e5034697 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -47,22 +47,29 @@ def test_post_create_transaction_endpoint(b, client): assert res.json['outputs'][0]['public_keys'][0] == user_pub -@pytest.mark.parametrize("key,expected_status_code", [ - ('bad.key', 400), - ('$bad.key', 400), - ('$badkey', 400), - ('good_key', 202) +@pytest.mark.parametrize("field", ['asset', 'metadata']) +@pytest.mark.parametrize("value,err_key,expected_status_code", [ + ({'bad.key': 'v'}, 'bad.key', 400), + ({'$bad.key': 'v'}, '$bad.key', 400), + ({'$badkey': 'v'}, '$badkey', 400), + ({'bad\x00key': 'v'}, 'bad\x00key', 400), + ({'good_key': {'bad.key': 'v'}}, 'bad.key', 400), + ({'good_key': 'v'}, 'good_key', 202) ]) -@pytest.mark.assetkey @pytest.mark.bdb -def test_post_create_transaction_with_invalid_asset_key(b, client, key, expected_status_code): +def test_post_create_transaction_with_invalid_key(b, client, field, value, + err_key, expected_status_code): from bigchaindb.models import Transaction from bigchaindb.backend.mongodb.connection import MongoDBConnection user_priv, user_pub = crypto.generate_key_pair() if isinstance(b.connection, MongoDBConnection): - tx = Transaction.create([user_pub], [([user_pub], 1)], - asset={key: 'random_value'}) + if field == 'asset': + tx = Transaction.create([user_pub], [([user_pub], 1)], + asset=value) + elif field == 'metadata': + tx = Transaction.create([user_pub], [([user_pub], 1)], + metadata=value) tx = tx.sign([user_priv]) res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) @@ -70,8 +77,8 @@ def test_post_create_transaction_with_invalid_asset_key(b, client, key, expected if res.status_code == 400: expected_error_message = ( 'Invalid transaction (ValidationError): Invalid key name "{}" ' - 'in asset object. The key name cannot contain characters ' - '"." and "$"').format(key) + 'in {} object. The key name cannot contain characters ' + '".", "$" or null characters').format(err_key, field) assert res.json['message'] == expected_error_message From 263e9a25f66cd883095d237bedd84c69337d614a Mon Sep 17 00:00:00 2001 From: kansi Date: Wed, 1 Nov 2017 21:23:06 +0530 Subject: [PATCH 6/7] Unique index for bigchain collection, fixed test case --- bigchaindb/backend/mongodb/schema.py | 5 +++-- tests/backend/mongodb/test_queries.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bigchaindb/backend/mongodb/schema.py b/bigchaindb/backend/mongodb/schema.py index 01eac29c..572acff9 100644 --- a/bigchaindb/backend/mongodb/schema.py +++ b/bigchaindb/backend/mongodb/schema.py @@ -50,9 +50,10 @@ def drop_database(conn, dbname): def create_bigchain_secondary_index(conn, dbname): logger.info('Create `bigchain` secondary index.') - # secondary index on block id which is should be unique + # secondary index on block id which should be unique conn.conn[dbname]['bigchain'].create_index('id', - name='block_id') + name='block_id', + unique=True) # to order blocks by timestamp conn.conn[dbname]['bigchain'].create_index([('block.timestamp', diff --git a/tests/backend/mongodb/test_queries.py b/tests/backend/mongodb/test_queries.py index 0fd7229a..3ea7db28 100644 --- a/tests/backend/mongodb/test_queries.py +++ b/tests/backend/mongodb/test_queries.py @@ -299,12 +299,13 @@ def test_count_blocks(signed_create_tx): from bigchaindb.models import Block conn = connect() + assert query.count_blocks(conn) == 0 + # create and insert some blocks block = Block(transactions=[signed_create_tx]) conn.db.bigchain.insert_one(block.to_dict()) - conn.db.bigchain.insert_one(block.to_dict()) - assert query.count_blocks(conn) == 2 + assert query.count_blocks(conn) == 1 def test_count_backlog(signed_create_tx): From 8fdf8f6ca6cfdc3c20ac863c1b205dcfe365894e Mon Sep 17 00:00:00 2001 From: kansi Date: Thu, 2 Nov 2017 18:02:11 +0530 Subject: [PATCH 7/7] Added docstrings --- bigchaindb/common/utils.py | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/bigchaindb/common/utils.py b/bigchaindb/common/utils.py index 35163f14..e472f380 100644 --- a/bigchaindb/common/utils.py +++ b/bigchaindb/common/utils.py @@ -52,6 +52,22 @@ def deserialize(data): def validate_txn_obj(obj_name, obj, key, validation_fun): + """Validates value associated to `key` in `obj` by applying + `validation_fun`. + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + key (str): key to be validated in `obj`. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ backend = bigchaindb.config['database']['backend'] if backend == 'mongodb': @@ -60,6 +76,20 @@ def validate_txn_obj(obj_name, obj, key, validation_fun): def validate_all_keys(obj_name, obj, validation_fun): + """Validates all (nested) keys in `obj` by using `validation_fun` + + Args: + obj_name (str): name for `obj` being validated. + obj (dict): dictonary object. + validation_fun (function): function used to validate the value + of `key`. + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: `validation_fun` will raise this error on failure + """ for key, value in obj.items(): validation_fun(obj_name, key) if type(value) is dict: @@ -68,6 +98,19 @@ def validate_all_keys(obj_name, obj, validation_fun): def validate_key(obj_name, key): + """Check if `key` contains ".", "$" or null characters + https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names + + Args: + obj_name (str): object name to use when raising exception + key (str): key to validated + + Returns: + None: indicates validation successfull + + Raises: + ValidationError: raise execption incase of regex match. + """ if re.search(r'^[$]|\.|\x00', key): error_str = ('Invalid key name "{}" in {} object. The ' 'key name cannot contain characters '