From 8343bab89f229868bd002dab842d2dd8775ccb64 Mon Sep 17 00:00:00 2001 From: libscott Date: Tue, 22 Nov 2016 11:17:06 +0100 Subject: [PATCH] Schema definition (#798) Commit messages for posterity: * wip transaction schema definition * test for SchemaObject * test SchemaObject definions meta property * schema documentation updates * test for basic validation * commit before change to .json file definiton + rst generation * move to straight .json schema, test for additionalProperties on each object * add asset to transaction definiton * remove outdated tx validation * make all tests pass * create own exception for validation error and start validating transactions * more tx validation fixes * move to yaml file for schema * automatic schema documentation generator * remove redundant section * use YAML safe loading * change current_owners to owners_before in tx schema * re-run tests and make correct yaml schema * fix some broken tests * update Release_Process.md * move tx validation into it's own method * add jsonschema dependency * perform schema validation after ID validation on Transaction * Release_Process.md, markdown auto numbering * remove old transaction.json * resolve remaining TODOs in schema docuementation * add `id` and `$schema` to transaction.yaml * add transaction.yaml to setup.py so it gets copied * address some concernes in PR for transaction.yaml * address more PR concerns in transaction.yaml * refactor validtion exceptions and move transaction schema validation into it's own function in bigchaindb.common.schema.__init__ * add note to generated schema.rst indicating when and how it's generated * move tx schema validation back above ID validation in Transaction.validate_structure, test that structurally invalid transaction gets caught and 400 returned in TX POST handler * remove timestamp from transaction schema index * Add README.md to bigchaindb.common.schema for introduction to JSON Schema and reasons for YAML * Use constant for schema definitions' base prefix * Move import of ValidationError exception into only the tests that require it * Move validate transaction test helper to tests/common/util.py * move ordered transaction schema load to generate_schema_documentation.py where it's needed * use double backticks to render terms in schema docs * change more backticks and change transaction version description in transaction schema * make details a mandatory property of condition * Many more documentation fixes * rename schema.rst to schema/transaction.rst * Fix documentation for Metadata * Add more links to documentation * Various other documentation fixes * Rename section titles in rendered documentation * use to manage file handle * fix extrenuous comma in test_tx_serialization_with_incorrect_hash args * 'a' * 64 * remove schema validation until we can analyze properly impact on downstream consumers * fix flake8 error * use `with` always --- Release_Process.md | 17 +- bigchaindb/common/exceptions.py | 10 +- bigchaindb/common/schema/README.md | 30 ++ bigchaindb/common/schema/__init__.py | 24 ++ bigchaindb/common/schema/transaction.yaml | 268 +++++++++++++++ bigchaindb/common/transaction.py | 48 +-- bigchaindb/web/views/transactions.py | 4 +- docs/server/generate_schema_documentation.py | 179 ++++++++++ docs/server/source/index.rst | 1 + docs/server/source/schema/transaction.rst | 335 +++++++++++++++++++ setup.py | 3 + tests/common/conftest.py | 7 + tests/common/test_schema.py | 40 +++ tests/common/test_transaction.py | 90 +++-- tests/common/util.py | 9 + tests/web/test_transactions.py | 9 +- 16 files changed, 1013 insertions(+), 61 deletions(-) create mode 100644 bigchaindb/common/schema/README.md create mode 100644 bigchaindb/common/schema/__init__.py create mode 100644 bigchaindb/common/schema/transaction.yaml create mode 100644 docs/server/generate_schema_documentation.py create mode 100644 docs/server/source/schema/transaction.rst create mode 100644 tests/common/test_schema.py create mode 100644 tests/common/util.py diff --git a/Release_Process.md b/Release_Process.md index bb826ae9..b53fbeea 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -2,16 +2,17 @@ This is a summary of the steps we go through to release a new version of BigchainDB Server. +1. Run `python docs/server/generate_schema_documentation.py` and commit the changes in docs/server/sources/schema, if any. 1. Update the `CHANGELOG.md` file -2. Update the version numbers in `bigchaindb/version.py`. Note that we try to use [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH) -3. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) +1. Update the version numbers in `bigchaindb/version.py`. Note that we try to use [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH) +1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) and click the "Draft a new release" button -4. Name the tag something like v0.7.0 -5. The target should be a specific commit: the one when the update of `bigchaindb/version.py` got merged into master -6. The release title should be something like v0.7.0 -7. The description should be copied from the `CHANGELOG.md` file updated above -8. Generate and send the latest `bigchaindb` package to PyPI. Dimi and Sylvain can do this, maybe others -9. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. +1. Name the tag something like v0.7.0 +1. The target should be a specific commit: the one when the update of `bigchaindb/version.py` got merged into master +1. The release title should be something like v0.7.0 +1. The description should be copied from the `CHANGELOG.md` file updated above +1. Generate and send the latest `bigchaindb` package to PyPI. Dimi and Sylvain can do this, maybe others +1. Login to readthedocs.org as a maintainer of the BigchainDB Server docs. Go to Admin --> Versions and under **Choose Active Versions**, make sure that the new version's tag is "Active" and "Public" diff --git a/bigchaindb/common/exceptions.py b/bigchaindb/common/exceptions.py index 72e300fd..661a9c92 100644 --- a/bigchaindb/common/exceptions.py +++ b/bigchaindb/common/exceptions.py @@ -22,11 +22,19 @@ class DoubleSpend(Exception): """Raised if a double spend is found""" -class InvalidHash(Exception): +class ValidationError(Exception): + """Raised if there was an error in validation""" + + +class InvalidHash(ValidationError): """Raised if there was an error checking the hash for a particular operation""" +class SchemaValidationError(ValidationError): + """Raised if there was any error validating an object's schema""" + + class InvalidSignature(Exception): """Raised if there was an error checking the signature for a particular operation""" diff --git a/bigchaindb/common/schema/README.md b/bigchaindb/common/schema/README.md new file mode 100644 index 00000000..3c8451b0 --- /dev/null +++ b/bigchaindb/common/schema/README.md @@ -0,0 +1,30 @@ +# Introduction + +This directory contains the schemas for the different JSON documents BigchainDB uses. + +The aim is to provide: + - a strict definition/documentation of the data structures used in BigchainDB + - a language independent tool to validate the structure of incoming/outcoming + data (there are several ready to use + [implementations](http://json-schema.org/implementations.html) written in + different languages) + +## Learn about JSON Schema + +A good resource is [Understanding JSON Schema](http://spacetelescope.github.io/understanding-json-schema/index.html). +It provides a *more accessible documentation for JSON schema* than the [specs](http://json-schema.org/documentation.html). + +## If it's supposed to be JSON, why's everything in YAML D:? + +YAML is great for its conciseness and friendliness towards human-editing in comparision to JSON. + +Although YAML is a superset of JSON, at the end of the day, JSON Schema processors, like +[json-schema](http://python-jsonschema.readthedocs.io/en/latest/), take in a native object (e.g. +Python dicts or JavaScript objects) as the schema used for validation. As long as we can serialize +the YAML into what the JSON Schema processor expects (almost always as simple as loading the YAML +like you would with a JSON file), it's the same as using JSON. + +Specific advantages of using YAML: + - Legibility, especially when nesting + - Multi-line string literals, that make it easy to include descriptions that can be [auto-generated + into Sphinx documentation](/docs/server/generate_schema_documentation.py) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py new file mode 100644 index 00000000..6bb4f038 --- /dev/null +++ b/bigchaindb/common/schema/__init__.py @@ -0,0 +1,24 @@ +""" Schema validation related functions and data """ +import os.path + +import jsonschema +import yaml + +from bigchaindb.common.exceptions import SchemaValidationError + + +TX_SCHEMA_PATH = os.path.join(os.path.dirname(__file__), 'transaction.yaml') +with open(TX_SCHEMA_PATH) as handle: + TX_SCHEMA_YAML = handle.read() +TX_SCHEMA = yaml.safe_load(TX_SCHEMA_YAML) + + +def validate_transaction_schema(tx_body): + """ Validate a transaction dict against a schema """ + try: + jsonschema.validate(tx_body, TX_SCHEMA) + except jsonschema.ValidationError as exc: + raise SchemaValidationError(str(exc)) + + +__all__ = ['TX_SCHEMA', 'TX_SCHEMA_YAML', 'validate_transaction_schema'] diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml new file mode 100644 index 00000000..a7e4117e --- /dev/null +++ b/bigchaindb/common/schema/transaction.yaml @@ -0,0 +1,268 @@ +--- +"$schema": "http://json-schema.org/draft-04/schema#" +id: "http://www.bigchaindb.com/schema/transaction.json" +type: object +additionalProperties: false +title: Transaction Schema +description: | + This is the outer transaction wrapper. It contains the ID, version and the body of the transaction, which is also called ``transaction``. +required: +- id +- transaction +- version +properties: + id: + "$ref": "#/definitions/sha3_hexdigest" + description: | + A sha3 digest of the transaction. The ID is calculated by removing all + derived hashes and signatures from the transaction, serializing it to + JSON with keys in sorted order and then hashing the resulting string + with sha3. + transaction: + type: object + title: transaction + description: | + See: `Transaction Body`_. + additionalProperties: false + required: + - fulfillments + - conditions + - operation + - timestamp + - metadata + - asset + properties: + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" + description: | + Description of the asset being transacted. + + See: `Asset`_. + fulfillments: + type: array + title: "Fulfillments list" + description: | + Array of the fulfillments (inputs) of a transaction. + + See: Fulfillment_. + items: + "$ref": "#/definitions/fulfillment" + conditions: + type: array + description: | + Array of conditions (outputs) provided by this transaction. + + See: Condition_. + items: + "$ref": "#/definitions/condition" + metadata: + "$ref": "#/definitions/metadata" + description: | + User provided transaction metadata. This field may be ``null`` or may + contain an id and an object with freeform metadata. + + See: `Metadata`_. + timestamp: + "$ref": "#/definitions/timestamp" + version: + type: integer + minimum: 1 + maximum: 1 + description: | + BigchainDB transaction schema version. +definitions: + offset: + type: integer + minimum: 0 + base58: + pattern: "[1-9a-zA-Z^OIl]{43,44}" + type: string + owners_list: + anyOf: + - type: array + items: + "$ref": "#/definitions/base58" + - type: 'null' + sha3_hexdigest: + pattern: "[0-9a-f]{64}" + type: string + uuid4: + pattern: "[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}" + type: string + description: | + A `UUID `_ + of type 4 (random). + operation: + type: string + description: | + Type of the transaction: + + A ``CREATE`` transaction creates an asset in BigchainDB. This + transaction has outputs (conditions) but no inputs (fulfillments), + so a dummy fulfillment is used. + + A ``TRANSFER`` transaction transfers ownership of an asset, by providing + fulfillments to conditions of earlier transactions. + + A ``GENESIS`` transaction is a special case transaction used as the + sole member of the first block in a BigchainDB ledger. + enum: + - CREATE + - TRANSFER + - GENESIS + asset: + type: object + description: | + Description of the asset being transacted. In the case of a ``TRANSFER`` + transaction, this field contains only the ID of asset. In the case + of a ``CREATE`` transaction, this field may contain properties: + additionalProperties: false + required: + - id + properties: + id: + "$ref": "#/definitions/uuid4" + divisible: + type: boolean + description: | + Whether or not the asset has a quantity that may be partially spent. + updatable: + type: boolean + description: | + Whether or not the description of the asset may be updated. Defaults to false. + refillable: + type: boolean + description: | + Whether the amount of the asset can change after its creation. Defaults to false. + data: + description: | + User provided metadata associated with the asset. May also be ``null``. + anyOf: + - type: object + additionalProperties: true + - type: 'null' + condition: + type: object + description: | + An output of a transaction. A condition describes a quantity of an asset + and what conditions must be met in order for it to be fulfilled. See also: + fulfillment_. + additionalProperties: false + required: + - owners_after + - condition + - amount + properties: + cid: + "$ref": "#/definitions/offset" + description: | + Index of this condition's appearance in the `Transaction.conditions`_ + array. In a transaction with 2 conditions, the ``cid``s will be 0 and 1. + condition: + description: | + Body of the condition. Has the properties: + + - **details**: Details of the condition. + - **uri**: Condition encoded as an ASCII string. + type: object + additionalProperties: false + required: + - details + - uri + properties: + details: + type: object + additionalProperties: true + uri: + type: string + pattern: "^cc:([1-9a-f][0-9a-f]{0,3}|0):[1-9a-f][0-9a-f]{0,15}:[a-zA-Z0-9_-]{0,86}:([1-9][0-9]{0,17}|0)$" + owners_after: + "$ref": "#/definitions/owners_list" + description: | + List of public keys associated with asset ownership at the time + of the transaction. + amount: + type: integer + description: | + Integral amount of the asset represented by this condition. + In the case of a non divisible asset, this will always be 1. + fulfillment: + type: "object" + description: + A fulfillment is an input to a transaction, named as such because it + fulfills a condition of a previous transaction. In the case of a + ``CREATE`` transaction, a fulfillment may provide no ``input``. + additionalProperties: false + required: + - owners_before + - input + - fulfillment + properties: + fid: + "$ref": "#/definitions/offset" + description: | + The offset of the fulfillment within the fulfillents array. + owners_before: + "$ref": "#/definitions/owners_list" + description: | + List of public keys of the previous owners of the asset. + fulfillment: + anyOf: + - type: object + additionalProperties: false + properties: + bitmask: + type: integer + public_key: + type: string + type: + type: string + signature: + anyOf: + - type: string + - type: 'null' + type_id: + type: integer + description: | + Fulfillment of a condition_, or put a different way, this is a + payload that satisfies a condition in order to spend the associated + asset. + - type: string + pattern: "^cf:([1-9a-f][0-9a-f]{0,3}|0):[a-zA-Z0-9_-]*$" + input: + anyOf: + - type: 'object' + description: | + Reference to a condition of a previous transaction + additionalProperties: false + properties: + cid: + "$ref": "#/definitions/offset" + txid: + "$ref": "#/definitions/sha3_hexdigest" + - type: 'null' + metadata: + anyOf: + - type: object + description: | + User provided transaction metadata. This field may be ``null`` or may + contain an id and an object with freeform metadata. + additionalProperties: false + required: + - id + - data + properties: + id: + "$ref": "#/definitions/uuid4" + data: + type: object + description: | + User provided transaction metadata. + additionalProperties: true + - type: 'null' + timestamp: + type: string + description: | + User provided timestamp of the transaction. diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 18a50640..b8fc14d0 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -590,7 +590,6 @@ class Metadata(object): if data is not None and not isinstance(data, dict): raise TypeError('`data` must be a dict instance or None') - # TODO: Rename `payload_id` to `id` self.data_id = data_id if data_id is not None else self.to_hash() self.data = data @@ -1248,16 +1247,12 @@ class Transaction(object): tx = Transaction._remove_signatures(self.to_dict()) return Transaction._to_str(tx) - @classmethod - # TODO: Make this method more pretty - def from_dict(cls, tx_body): - """Transforms a Python dictionary to a Transaction object. + @staticmethod + def validate_structure(tx_body): + """Validate the transaction ID of a transaction Args: tx_body (dict): The Transaction to be transformed. - - Returns: - :class:`~bigchaindb.common.transaction.Transaction` """ # NOTE: Remove reference to avoid side effects tx_body = deepcopy(tx_body) @@ -1272,17 +1267,28 @@ class Transaction(object): if proposed_tx_id != valid_tx_id: raise InvalidHash() - else: - tx = tx_body['transaction'] - fulfillments = [Fulfillment.from_dict(fulfillment) for fulfillment - in tx['fulfillments']] - conditions = [Condition.from_dict(condition) for condition - in tx['conditions']] - metadata = Metadata.from_dict(tx['metadata']) - if tx['operation'] in [cls.CREATE, cls.GENESIS]: - asset = Asset.from_dict(tx['asset']) - else: - asset = AssetLink.from_dict(tx['asset']) - return cls(tx['operation'], asset, fulfillments, conditions, - metadata, tx['timestamp'], tx_body['version']) + @classmethod + def from_dict(cls, tx_body): + """Transforms a Python dictionary to a Transaction object. + + Args: + tx_body (dict): The Transaction to be transformed. + + Returns: + :class:`~bigchaindb.common.transaction.Transaction` + """ + cls.validate_structure(tx_body) + tx = tx_body['transaction'] + fulfillments = [Fulfillment.from_dict(fulfillment) for fulfillment + in tx['fulfillments']] + conditions = [Condition.from_dict(condition) for condition + in tx['conditions']] + metadata = Metadata.from_dict(tx['metadata']) + if tx['operation'] in [cls.CREATE, cls.GENESIS]: + asset = Asset.from_dict(tx['asset']) + else: + asset = AssetLink.from_dict(tx['asset']) + + return cls(tx['operation'], asset, fulfillments, conditions, + metadata, tx['timestamp'], tx_body['version']) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index c529b6b3..2ed19b7c 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -6,7 +6,7 @@ For more information please refer to the documentation on ReadTheDocs: from flask import current_app, request, Blueprint from flask_restful import Resource, Api -from bigchaindb.common.exceptions import InvalidHash, InvalidSignature +from bigchaindb.common.exceptions import ValidationError, InvalidSignature import bigchaindb from bigchaindb.models import Transaction @@ -98,7 +98,7 @@ class TransactionListApi(Resource): try: tx_obj = Transaction.from_dict(tx) - except (InvalidHash, InvalidSignature): + except (ValidationError, InvalidSignature): return make_error(400, 'Invalid transaction') with pool() as bigchain: diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py new file mode 100644 index 00000000..0e1a626a --- /dev/null +++ b/docs/server/generate_schema_documentation.py @@ -0,0 +1,179 @@ +""" Script to render transaction schema into .rst document """ + +from collections import OrderedDict +import os.path + +import yaml + +from bigchaindb.common.schema import TX_SCHEMA_YAML + + +TPL_PROP = """\ +%(title)s +%(underline)s + +**type:** %(type)s + +%(description)s +""" + + +TPL_DOC = """\ +.. This file was auto generated by %(file)s + +================== +Transaction Schema +================== + +* `Transaction`_ + +* `Transaction Body`_ + +* Condition_ + +* Fulfillment_ + +* Asset_ + +* Metadata_ + +.. raw:: html + + + +Transaction +----------- + +%(wrapper)s + +Transaction Body +---------------- + +%(transaction)s + +Condition +---------- + +%(condition)s + +Fulfillment +----------- + +%(fulfillment)s + +Asset +----- + +%(asset)s + +Metadata +-------- + +%(metadata)s +""" + + +def ordered_load_yaml(stream): + """ Custom YAML loader to preserve key order """ + class OrderedLoader(yaml.SafeLoader): + pass + + def construct_mapping(loader, node): + loader.flatten_mapping(node) + return OrderedDict(loader.construct_pairs(node)) + OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping) + return yaml.load(stream, OrderedLoader) + + +TX_SCHEMA = ordered_load_yaml(TX_SCHEMA_YAML) + + +DEFINITION_BASE_PATH = '#/definitions/' + + +def render_section(section_name, obj): + """ Render a domain object and it's properties """ + out = [obj['description']] + for name, prop in obj.get('properties', {}).items(): + try: + title = '%s.%s' % (section_name, name) + out += [TPL_PROP % { + 'title': title, + 'underline': '^' * len(title), + 'description': property_description(prop), + 'type': property_type(prop), + }] + except Exception as exc: + raise ValueError("Error rendering property: %s" % name, exc) + return '\n\n'.join(out + ['']) + + +def property_description(prop): + """ Get description of property """ + if 'description' in prop: + return prop['description'] + if '$ref' in prop: + return property_description(resolve_ref(prop['$ref'])) + if 'anyOf' in prop: + return property_description(prop['anyOf'][0]) + raise KeyError("description") + + +def property_type(prop): + """ Resolve a string representing the type of a property """ + if 'type' in prop: + if prop['type'] == 'array': + return 'array (%s)' % property_type(prop['items']) + return prop['type'] + if 'anyOf' in prop: + return ' or '.join(property_type(p) for p in prop['anyOf']) + if '$ref' in prop: + return property_type(resolve_ref(prop['$ref'])) + raise ValueError("Could not resolve property type") + + +def resolve_ref(ref): + """ Resolve definition reference """ + assert ref.startswith(DEFINITION_BASE_PATH) + return TX_SCHEMA['definitions'][ref[len(DEFINITION_BASE_PATH):]] + + +def main(): + """ Main function """ + defs = TX_SCHEMA['definitions'] + doc = TPL_DOC % { + 'wrapper': render_section('Transaction', TX_SCHEMA), + 'transaction': render_section('Transaction', + TX_SCHEMA['properties']['transaction']), + 'condition': render_section('Condition', defs['condition']), + 'fulfillment': render_section('Fulfillment', defs['fulfillment']), + 'asset': render_section('Asset', defs['asset']), + 'metadata': render_section('Metadata', defs['metadata']['anyOf'][0]), + 'file': os.path.basename(__file__), + } + + path = os.path.join(os.path.dirname(__file__), + 'source/schema/transaction.rst') + + with open(path, 'w') as handle: + handle.write(doc) + + +if __name__ == '__main__': + main() diff --git a/docs/server/source/index.rst b/docs/server/source/index.rst index 4371bc97..572e573f 100644 --- a/docs/server/source/index.rst +++ b/docs/server/source/index.rst @@ -14,5 +14,6 @@ BigchainDB Server Documentation drivers-clients/index clusters-feds/index topic-guides/index + schema/transaction release-notes appendices/index diff --git a/docs/server/source/schema/transaction.rst b/docs/server/source/schema/transaction.rst new file mode 100644 index 00000000..df82baf0 --- /dev/null +++ b/docs/server/source/schema/transaction.rst @@ -0,0 +1,335 @@ +.. This file was auto generated by generate_schema_documentation.py + +================== +Transaction Schema +================== + +* `Transaction`_ + +* `Transaction Body`_ + +* Condition_ + +* Fulfillment_ + +* Asset_ + +* Metadata_ + +.. raw:: html + + + +Transaction +----------- + +This is the outer transaction wrapper. It contains the ID, version and the body of the transaction, which is also called ``transaction``. + + +Transaction.id +^^^^^^^^^^^^^^ + +**type:** string + +A sha3 digest of the transaction. The ID is calculated by removing all +derived hashes and signatures from the transaction, serializing it to +JSON with keys in sorted order and then hashing the resulting string +with sha3. + + + +Transaction.transaction +^^^^^^^^^^^^^^^^^^^^^^^ + +**type:** object + +See: `Transaction Body`_. + + + +Transaction.version +^^^^^^^^^^^^^^^^^^^ + +**type:** integer + +BigchainDB transaction schema version. + + + + + +Transaction Body +---------------- + +See: `Transaction Body`_. + + +Transaction.operation +^^^^^^^^^^^^^^^^^^^^^ + +**type:** string + +Type of the transaction: + +A ``CREATE`` transaction creates an asset in BigchainDB. This +transaction has outputs (conditions) but no inputs (fulfillments), +so a dummy fulfillment is used. + +A ``TRANSFER`` transaction transfers ownership of an asset, by providing +fulfillments to conditions of earlier transactions. + +A ``GENESIS`` transaction is a special case transaction used as the +sole member of the first block in a BigchainDB ledger. + + + +Transaction.asset +^^^^^^^^^^^^^^^^^ + +**type:** object + +Description of the asset being transacted. + +See: `Asset`_. + + + +Transaction.fulfillments +^^^^^^^^^^^^^^^^^^^^^^^^ + +**type:** array (object) + +Array of the fulfillments (inputs) of a transaction. + +See: Fulfillment_. + + + +Transaction.conditions +^^^^^^^^^^^^^^^^^^^^^^ + +**type:** array (object) + +Array of conditions (outputs) provided by this transaction. + +See: Condition_. + + + +Transaction.metadata +^^^^^^^^^^^^^^^^^^^^ + +**type:** object or null + +User provided transaction metadata. This field may be ``null`` or may +contain an id and an object with freeform metadata. + +See: `Metadata`_. + + + +Transaction.timestamp +^^^^^^^^^^^^^^^^^^^^^ + +**type:** string + +User provided timestamp of the transaction. + + + + + +Condition +---------- + +An output of a transaction. A condition describes a quantity of an asset +and what conditions must be met in order for it to be fulfilled. See also: +fulfillment_. + + +Condition.cid +^^^^^^^^^^^^^ + +**type:** integer + +Index of this condition's appearance in the `Transaction.conditions`_ +array. In a transaction with 2 conditions, the ``cid``s will be 0 and 1. + + + +Condition.condition +^^^^^^^^^^^^^^^^^^^ + +**type:** object + +Body of the condition. Has the properties: + +- **details**: Details of the condition. +- **uri**: Condition encoded as an ASCII string. + + + +Condition.owners_after +^^^^^^^^^^^^^^^^^^^^^^ + +**type:** array (string) or null + +List of public keys associated with asset ownership at the time +of the transaction. + + + +Condition.amount +^^^^^^^^^^^^^^^^ + +**type:** integer + +Integral amount of the asset represented by this condition. +In the case of a non divisible asset, this will always be 1. + + + + + +Fulfillment +----------- + +A fulfillment is an input to a transaction, named as such because it fulfills a condition of a previous transaction. In the case of a ``CREATE`` transaction, a fulfillment may provide no ``input``. + +Fulfillment.fid +^^^^^^^^^^^^^^^ + +**type:** integer + +The offset of the fulfillment within the fulfillents array. + + + +Fulfillment.owners_before +^^^^^^^^^^^^^^^^^^^^^^^^^ + +**type:** array (string) or null + +List of public keys of the previous owners of the asset. + + + +Fulfillment.fulfillment +^^^^^^^^^^^^^^^^^^^^^^^ + +**type:** object or string + +Fulfillment of a condition_, or put a different way, this is a +payload that satisfies a condition in order to spend the associated +asset. + + + +Fulfillment.input +^^^^^^^^^^^^^^^^^ + +**type:** object or null + +Reference to a condition of a previous transaction + + + + + +Asset +----- + +Description of the asset being transacted. In the case of a ``TRANSFER`` +transaction, this field contains only the ID of asset. In the case +of a ``CREATE`` transaction, this field may contain properties: + + +Asset.id +^^^^^^^^ + +**type:** string + +A `UUID `_ +of type 4 (random). + + + +Asset.divisible +^^^^^^^^^^^^^^^ + +**type:** boolean + +Whether or not the asset has a quantity that may be partially spent. + + + +Asset.updatable +^^^^^^^^^^^^^^^ + +**type:** boolean + +Whether or not the description of the asset may be updated. Defaults to false. + + + +Asset.refillable +^^^^^^^^^^^^^^^^ + +**type:** boolean + +Whether the amount of the asset can change after its creation. Defaults to false. + + + +Asset.data +^^^^^^^^^^ + +**type:** object or null + +User provided metadata associated with the asset. May also be ``null``. + + + + + +Metadata +-------- + +User provided transaction metadata. This field may be ``null`` or may +contain an id and an object with freeform metadata. + + +Metadata.id +^^^^^^^^^^^ + +**type:** string + +A `UUID `_ +of type 4 (random). + + + +Metadata.data +^^^^^^^^^^^^^ + +**type:** object + +User provided transaction metadata. + + + + diff --git a/setup.py b/setup.py index 0f558dd7..57c25678 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,8 @@ install_requires = [ 'requests~=2.9', 'gunicorn~=19.0', 'multipipes~=0.1.0', + 'jsonschema~=2.5.1', + 'pyyaml~=3.12', ] setup( @@ -110,4 +112,5 @@ setup( 'dev': dev_require + tests_require + docs_require + benchmarks_require, 'docs': docs_require, }, + package_data={'bigchaindb.common.schema': ['transaction.yaml']}, ) diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 8b1a47db..1e5fe7d3 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -19,6 +19,8 @@ DATA = { } DATA_ID = '872fa6e6f46246cd44afdb2ee9cfae0e72885fb0910e2bcf9a5a2a4eadb417b8' +UUID4 = 'dc568f27-a113-46b4-9bd4-43015859e3e3' + @pytest.fixture def user_priv(): @@ -129,6 +131,11 @@ def data_id(): return DATA_ID +@pytest.fixture +def uuid4(): + return UUID4 + + @pytest.fixture def metadata(data, data_id): from bigchaindb.common.transaction import Metadata diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py new file mode 100644 index 00000000..5ded0272 --- /dev/null +++ b/tests/common/test_schema.py @@ -0,0 +1,40 @@ +from pytest import raises + +from bigchaindb.common.exceptions import SchemaValidationError +from bigchaindb.common.schema import TX_SCHEMA, validate_transaction_schema + + +def test_validate_transaction_create(create_tx): + validate_transaction_schema(create_tx.to_dict()) + + +def test_validate_transaction_signed_create(signed_create_tx): + validate_transaction_schema(signed_create_tx.to_dict()) + + +def test_validate_transaction_signed_transfer(signed_transfer_tx): + validate_transaction_schema(signed_transfer_tx.to_dict()) + + +def test_validation_fails(): + with raises(SchemaValidationError): + validate_transaction_schema({}) + + +def test_addition_properties_always_set(): + """ + Validate that each object node has additionalProperties set, so that + transactions with junk keys do not pass as valid. + """ + def walk(node, path=''): + if isinstance(node, list): + for i, nnode in enumerate(node): + walk(nnode, path + str(i) + '.') + if isinstance(node, dict): + if node.get('type') == 'object': + assert 'additionalProperties' in node, \ + ("additionalProperties not set at path:" + path) + for name, val in node.items(): + walk(val, path + name + '.') + + walk(TX_SCHEMA) diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 2839ee3c..4ad93768 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -274,6 +274,8 @@ def test_invalid_transaction_initialization(): def test_create_default_asset_on_tx_initialization(): from bigchaindb.common.transaction import Transaction, Asset + from bigchaindb.common.exceptions import ValidationError + from .util import validate_transaction_model with patch.object(Asset, 'validate_asset', return_value=None): tx = Transaction(Transaction.CREATE, None) @@ -284,9 +286,15 @@ def test_create_default_asset_on_tx_initialization(): asset.data_id = None assert asset == expected + # Fails because no asset hash + with raises(ValidationError): + validate_transaction_model(tx) + def test_transaction_serialization(user_ffill, user_cond, data, data_id): from bigchaindb.common.transaction import Transaction, Asset + from bigchaindb.common.exceptions import ValidationError + from .util import validate_transaction_model tx_id = 'l0l' timestamp = '66666666666' @@ -321,13 +329,18 @@ def test_transaction_serialization(user_ffill, user_cond, data, data_id): assert tx_dict == expected + # Fails because asset id is not a uuid4 + with raises(ValidationError): + validate_transaction_model(tx) -def test_transaction_deserialization(user_ffill, user_cond, data, data_id): + +def test_transaction_deserialization(user_ffill, user_cond, data, uuid4): from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model timestamp = '66666666666' - expected_asset = Asset(data, data_id) + expected_asset = Asset(data, uuid4) expected = Transaction(Transaction.CREATE, expected_asset, [user_ffill], [user_cond], None, timestamp, Transaction.VERSION) @@ -342,7 +355,7 @@ def test_transaction_deserialization(user_ffill, user_cond, data, data_id): 'timestamp': timestamp, 'metadata': None, 'asset': { - 'id': data_id, + 'id': uuid4, 'divisible': False, 'updatable': False, 'refillable': False, @@ -356,21 +369,18 @@ def test_transaction_deserialization(user_ffill, user_cond, data, data_id): assert tx == expected + validate_transaction_model(tx) + def test_tx_serialization_with_incorrect_hash(utx): from bigchaindb.common.transaction import Transaction from bigchaindb.common.exceptions import InvalidHash utx_dict = utx.to_dict() - utx_dict['id'] = 'abc' + utx_dict['id'] = 'a' * 64 with raises(InvalidHash): Transaction.from_dict(utx_dict) utx_dict.pop('id') - with raises(InvalidHash): - Transaction.from_dict(utx_dict) - utx_dict['id'] = [] - with raises(InvalidHash): - Transaction.from_dict(utx_dict) def test_invalid_fulfillment_initialization(user_ffill, user_pub): @@ -547,6 +557,7 @@ def test_add_fulfillment_to_tx_with_invalid_parameters(): def test_add_condition_to_tx(user_cond): from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model with patch.object(Asset, 'validate_asset', return_value=None): tx = Transaction(Transaction.CREATE, Asset()) @@ -554,6 +565,8 @@ def test_add_condition_to_tx(user_cond): assert len(tx.conditions) == 1 + validate_transaction_model(tx) + def test_add_condition_to_tx_with_invalid_parameters(): from bigchaindb.common.transaction import Transaction, Asset @@ -575,6 +588,7 @@ def test_validate_tx_simple_create_signature(user_ffill, user_cond, user_priv): from copy import deepcopy from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model tx = Transaction(Transaction.CREATE, Asset(), [user_ffill], [user_cond]) expected = deepcopy(user_cond) @@ -585,6 +599,8 @@ def test_validate_tx_simple_create_signature(user_ffill, user_cond, user_priv): expected.fulfillment.serialize_uri() assert tx.fulfillments_valid() is True + validate_transaction_model(tx) + def test_invoke_simple_signature_fulfillment_with_invalid_params(utx, user_ffill): @@ -633,6 +649,7 @@ def test_validate_multiple_fulfillments(user_ffill, user_cond, user_priv): from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model tx = Transaction(Transaction.CREATE, Asset(divisible=True), [user_ffill, deepcopy(user_ffill)], @@ -657,6 +674,8 @@ def test_validate_multiple_fulfillments(user_ffill, user_cond, user_priv): expected_second.fulfillments[0].fulfillment.serialize_uri() assert tx.fulfillments_valid() is True + validate_transaction_model(tx) + def test_validate_tx_threshold_create_signature(user_user2_threshold_ffill, user_user2_threshold_cond, @@ -668,6 +687,7 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_ffill, from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model tx = Transaction(Transaction.CREATE, Asset(), [user_user2_threshold_ffill], [user_user2_threshold_cond]) @@ -682,6 +702,8 @@ def test_validate_tx_threshold_create_signature(user_user2_threshold_ffill, expected.fulfillment.serialize_uri() assert tx.fulfillments_valid() is True + validate_transaction_model(tx) + def test_multiple_fulfillment_validation_of_transfer_tx(user_ffill, user_cond, user_priv, user2_pub, @@ -691,6 +713,7 @@ def test_multiple_fulfillment_validation_of_transfer_tx(user_ffill, user_cond, from bigchaindb.common.transaction import (Transaction, TransactionLink, Fulfillment, Condition, Asset) from cryptoconditions import Ed25519Fulfillment + from .util import validate_transaction_model tx = Transaction(Transaction.CREATE, Asset(divisible=True), [user_ffill, deepcopy(user_ffill)], @@ -709,6 +732,8 @@ def test_multiple_fulfillment_validation_of_transfer_tx(user_ffill, user_cond, assert transfer_tx.fulfillments_valid(tx.conditions) is True + validate_transaction_model(tx) + def test_validate_fulfillments_of_transfer_tx_with_invalid_params(transfer_tx, cond_uri, @@ -735,9 +760,9 @@ def test_validate_fulfillments_of_transfer_tx_with_invalid_params(transfer_tx, transfer_tx.fulfillments_valid([utx.conditions[0]]) -def test_create_create_transaction_single_io(user_cond, user_pub, data, - data_id): +def test_create_create_transaction_single_io(user_cond, user_pub, data, uuid4): from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model expected = { 'transaction': { @@ -746,7 +771,7 @@ def test_create_create_transaction_single_io(user_cond, user_pub, data, 'data': data, }, 'asset': { - 'id': data_id, + 'id': uuid4, 'divisible': False, 'updatable': False, 'refillable': False, @@ -764,18 +789,20 @@ def test_create_create_transaction_single_io(user_cond, user_pub, data, ], 'operation': 'CREATE', }, - 'version': 1 + 'version': 1, } - asset = Asset(data, data_id) - tx = Transaction.create([user_pub], [([user_pub], 1)], - data, asset).to_dict() - tx.pop('id') - tx['transaction']['metadata'].pop('id') - tx['transaction'].pop('timestamp') - tx['transaction']['fulfillments'][0]['fulfillment'] = None + asset = Asset(data, uuid4) + tx = Transaction.create([user_pub], [([user_pub], 1)], data, asset) + tx_dict = tx.to_dict() + tx_dict.pop('id') + tx_dict['transaction']['metadata'].pop('id') + tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None + expected['transaction']['timestamp'] = tx_dict['transaction']['timestamp'] - assert tx == expected + assert tx_dict == expected + + validate_transaction_model(tx) def test_validate_single_io_create_transaction(user_pub, user_priv, data): @@ -824,6 +851,7 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, def test_validate_multiple_io_create_transaction(user_pub, user_priv, user2_pub, user2_priv): from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model tx = Transaction.create([user_pub, user2_pub], [([user_pub], 1), ([user2_pub], 1)], @@ -832,11 +860,13 @@ def test_validate_multiple_io_create_transaction(user_pub, user_priv, tx = tx.sign([user_priv, user2_priv]) assert tx.fulfillments_valid() is True + validate_transaction_model(tx) + def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, user_user2_threshold_cond, user_user2_threshold_ffill, data, - data_id): + uuid4): from bigchaindb.common.transaction import Transaction, Asset expected = { @@ -846,7 +876,7 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, 'data': data, }, 'asset': { - 'id': data_id, + 'id': uuid4, 'divisible': False, 'updatable': False, 'refillable': False, @@ -866,7 +896,7 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, }, 'version': 1 } - asset = Asset(data, data_id) + asset = Asset(data, uuid4) tx = Transaction.create([user_pub], [([user_pub, user2_pub], 1)], data, asset) tx_dict = tx.to_dict() @@ -881,12 +911,15 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, def test_validate_threshold_create_transaction(user_pub, user_priv, user2_pub, data): from bigchaindb.common.transaction import Transaction, Asset + from .util import validate_transaction_model tx = Transaction.create([user_pub], [([user_pub, user2_pub], 1)], data, Asset()) tx = tx.sign([user_priv]) assert tx.fulfillments_valid() is True + validate_transaction_model(tx) + def test_create_create_transaction_with_invalid_parameters(user_pub): from bigchaindb.common.transaction import Transaction @@ -916,18 +949,19 @@ def test_conditions_to_inputs(tx): def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, - user2_cond, user_priv, data_id): + user2_cond, user_priv, uuid4): from copy import deepcopy from bigchaindb.common.crypto import PrivateKey from bigchaindb.common.transaction import Transaction, Asset from bigchaindb.common.util import serialize + from .util import validate_transaction_model expected = { 'transaction': { 'conditions': [user2_cond.to_dict(0)], 'metadata': None, 'asset': { - 'id': data_id, + 'id': uuid4, }, 'fulfillments': [ { @@ -947,7 +981,7 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, 'version': 1 } inputs = tx.to_inputs([0]) - asset = Asset(None, data_id) + asset = Asset(None, uuid4) transfer_tx = Transaction.transfer(inputs, [([user2_pub], 1)], asset=asset) transfer_tx = transfer_tx.sign([user_priv]) transfer_tx = transfer_tx.to_dict() @@ -966,6 +1000,8 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, transfer_tx = Transaction.from_dict(transfer_tx) assert transfer_tx.fulfillments_valid([tx.conditions[0]]) is True + validate_transaction_model(transfer_tx) + def test_create_transfer_transaction_multiple_io(user_pub, user_priv, user2_pub, user2_priv, diff --git a/tests/common/util.py b/tests/common/util.py new file mode 100644 index 00000000..a6d463ac --- /dev/null +++ b/tests/common/util.py @@ -0,0 +1,9 @@ +def validate_transaction_model(tx): + from bigchaindb.common.transaction import Transaction + from bigchaindb.common.schema import validate_transaction_schema + + tx_dict = tx.to_dict() + # Check that a transaction is valid by re-serializing it + # And calling validate_transaction_schema + validate_transaction_schema(tx_dict) + Transaction.from_dict(tx_dict) diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 2bc27d0c..dd034c10 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -43,7 +43,7 @@ def test_post_create_transaction_with_invalid_id(b, client): tx = Transaction.create([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv]).to_dict() - tx['id'] = 'invalid id' + tx['id'] = 'abcd' * 16 res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 @@ -55,12 +55,17 @@ def test_post_create_transaction_with_invalid_signature(b, client): tx = Transaction.create([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv]).to_dict() - tx['transaction']['fulfillments'][0]['fulfillment'] = 'invalid signature' + tx['transaction']['fulfillments'][0]['fulfillment'] = 'cf:0:0' res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 +def test_post_create_transaction_with_invalid_structure(client): + res = client.post(TX_ENDPOINT, data='{}') + assert res.status_code == 400 + + @pytest.mark.usefixtures('inputs') def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): sk, pk = crypto.generate_key_pair()