diff --git a/.ci/travis-after-success.sh b/.ci/travis-after-success.sh new file mode 100755 index 00000000..272b6f6f --- /dev/null +++ b/.ci/travis-after-success.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e -x + +if [ "${TOXENV}" == "py35" ]; then + codecov +fi diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh new file mode 100755 index 00000000..a7aa1a8a --- /dev/null +++ b/.ci/travis-before-install.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "${TOXENV}" == "py34" ] || [ "${TOXENV}" == "py35" ]; then + source /etc/lsb-release + echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | tee -a /etc/apt/sources.list.d/rethinkdb.list + wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | apt-key add - + apt-get update -qq +fi diff --git a/.ci/travis-before-script.sh b/.ci/travis-before-script.sh new file mode 100755 index 00000000..65e0446c --- /dev/null +++ b/.ci/travis-before-script.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e -x + +if [ "${TOXENV}" == "py34" ] || [ "${TOXENV}" == "py35" ]; then + rethinkdb --daemon +fi diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh new file mode 100755 index 00000000..852b8279 --- /dev/null +++ b/.ci/travis-install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e -x + +pip install --upgrade pip +pip install --upgrade tox + +if [ "${TOXENV}" == "py34" ] || [ "${TOXENV}" == "py35" ]; then + sudo apt-get install rethinkdb + pip install --upgrade codecov +fi diff --git a/.gitignore b/.gitignore index ad82db58..1a22ad18 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,7 @@ benchmarking-tests/ssh_key.py # Ansible-specific files ntools/one-m/ansible/hosts ntools/one-m/ansible/ansible.cfg + +# Just in time documentation +docs/server/source/schema +docs/server/source/drivers-clients/samples diff --git a/.travis.yml b/.travis.yml index bd8ad0b1..7d2b4e5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,19 @@ -sudo: required language: python cache: pip -python: - - 3.4 - - 3.5 -before_install: - - source /etc/lsb-release - - echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee -a /etc/apt/sources.list.d/rethinkdb.list - - wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - - - sudo apt-get update -qq +env: + - TOXENV=py34 + - TOXENV=py35 + - TOXENV=flake8 + - TOXENV=docsroot + - TOXENV=docsserver -install: - - sudo apt-get install rethinkdb - - pip install -e .[test] - - pip install codecov +before_install: sudo .ci/travis-before-install.sh -before_script: - - flake8 --max-line-length 119 bigchaindb/ - - rethinkdb --daemon +install: .ci/travis-install.sh -script: py.test -n auto -s -v --cov=bigchaindb +before_script: .ci/travis-before-script.sh -after_success: codecov +script: tox -e ${TOXENV} + +after_success: .ci/travis-after-success.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f0841d..7fdfe8c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Change Log (Release Notes) + All _notable_ changes to this project will be documented in this file (`CHANGELOG.md`). This project adheres to [Semantic Versioning](http://semver.org/) (or at least we try). Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/). @@ -15,10 +16,74 @@ For reference, the possible headings are: * **Notes** +## [0.8.0] - 2016-11-29 +Tag name: v0.8.0 += commit: +committed: + +### Added +- The big new thing in version 0.8.0 is support for divisible assets, i.e. assets like carrots or thumbtacks, where the initial CREATE transaction can register/create some amount (e.g. 542 carrots), the first TRANSFER transaction can split that amount across multiple owners, and so on. [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) +- Wrote a formal schema for the JSON structure of transactions. [Pull Request #798](https://github.com/bigchaindb/bigchaindb/pull/798) +- New configuration parameter: `backlog_reassign_delay`. [Pull Request #883](https://github.com/bigchaindb/bigchaindb/pull/883) + +### Changed +- CREATE transactions must now be signed by all `owners_before` (rather than by a federation node). [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) +- The user-provided timestamp was removed from the transaction data model (schema). [Pull Request #817](https://github.com/bigchaindb/bigchaindb/pull/817) +- `get_transaction()` will now return a transaction from the backlog, even if there are copies of the transaction in invalid blocks. [Pull Request #793](https://github.com/bigchaindb/bigchaindb/pull/793) +- Several pull requests to introduce a generalized database interface, to move RethinkDB calls into a separate implementation of that interface, and to work on a new MongoDB implementation of that interface. Pull Requests +[#754](https://github.com/bigchaindb/bigchaindb/pull/754), +[#783](https://github.com/bigchaindb/bigchaindb/pull/783), +[#799](https://github.com/bigchaindb/bigchaindb/pull/799), +[#806](https://github.com/bigchaindb/bigchaindb/pull/806), +[#809](https://github.com/bigchaindb/bigchaindb/pull/809), +[#853](https://github.com/bigchaindb/bigchaindb/pull/853) +- Renamed "verifying key" to "public key". Renamed "signing key" to "private key". Renamed "vk" to "pk". [Pull Request #807](https://github.com/bigchaindb/bigchaindb/pull/807) +- `get_transaction_by_asset_id` now ignores invalid transactions. [Pull Request #810](https://github.com/bigchaindb/bigchaindb/pull/810) +- `get_transaction_by_metadata_id` now ignores invalid transactions. [Pull Request #811](https://github.com/bigchaindb/bigchaindb/pull/811) +- Updates to the configs and scripts for deploying a test network on AWS. The example config file deploys virtual machines running Ubuntu 16.04 now. Pull Requests +[#771](https://github.com/bigchaindb/bigchaindb/pull/771), +[#813](https://github.com/bigchaindb/bigchaindb/pull/813) +- Changed logging of transactions on block creation so now it just says the length of the list of transactions, rather than listing all the transactions. [Pull Request #861](https://github.com/bigchaindb/bigchaindb/pull/861) + +### Fixed +- Equality checks with AssetLinks. [Pull Request #825](https://github.com/bigchaindb/bigchaindb/pull/825) +- Bug in `bigchaindb load`. [Pull Request #824](https://github.com/bigchaindb/bigchaindb/pull/824) +- Two issues found with timestamp indexes. [Pull Request #816](https://github.com/bigchaindb/bigchaindb/pull/816) +- Hard-coded `backlog_reassign_delay`. [Pull Request #854](https://github.com/bigchaindb/bigchaindb/pull/854) +- Race condition in `test_stale_monitor.py`. [Pull Request #846](https://github.com/bigchaindb/bigchaindb/pull/846) +- When creating a signed vote, decode the vote signature to a `str`. [Pull Request #869](https://github.com/bigchaindb/bigchaindb/pull/869) +- Bug in AWS deployment scripts. Setting `BIND_HTTP_TO_LOCALHOST` to `False` didn't actually work. It does now. [Pull Request #870](https://github.com/bigchaindb/bigchaindb/pull/870) + +### External Contributors +- @najlachamseddine - [Pull Request #528](https://github.com/bigchaindb/bigchaindb/pull/528) +- @ChristianGaertner - [Pull Request #659](https://github.com/bigchaindb/bigchaindb/pull/659) +- @MinchinWeb - [Pull Request #695](https://github.com/bigchaindb/bigchaindb/pull/695) +- @ckeyer - [Pull Request #785](https://github.com/bigchaindb/bigchaindb/pull/785) + +### Notes +- @ChristianGaertner added a Python style checker (Flake8) to Travis CI, so external contributors should be aware that the Python code in their pull requests will be checked. See [our Python Style Guide](PYTHON_STYLE_GUIDE.md). +- Several additions and changes to the documentation, e.g. Pull Requests +[#690](https://github.com/bigchaindb/bigchaindb/pull/690), +[#764](https://github.com/bigchaindb/bigchaindb/pull/764), +[#766](https://github.com/bigchaindb/bigchaindb/pull/766), +[#769](https://github.com/bigchaindb/bigchaindb/pull/769), +[#777](https://github.com/bigchaindb/bigchaindb/pull/777), +[#800](https://github.com/bigchaindb/bigchaindb/pull/800), +[#801](https://github.com/bigchaindb/bigchaindb/pull/801), +[#802](https://github.com/bigchaindb/bigchaindb/pull/802), +[#803](https://github.com/bigchaindb/bigchaindb/pull/803), +[#819](https://github.com/bigchaindb/bigchaindb/pull/819), +[#827](https://github.com/bigchaindb/bigchaindb/pull/827), +[#859](https://github.com/bigchaindb/bigchaindb/pull/859), +[#872](https://github.com/bigchaindb/bigchaindb/pull/872), +[#882](https://github.com/bigchaindb/bigchaindb/pull/882), +[#883](https://github.com/bigchaindb/bigchaindb/pull/883) + + ## [0.7.0] - 2016-10-28 Tag name: v0.7.0 -= commit: -committed: += commit: 2dd7f1af27478c529e6d2d916f64daa3fbda3885 +committed: Oct 28, 2016, 4:00 PM GMT+2 ### Added - Stale transactions in the `backlog` table now get reassigned to another node (for inclusion in a new block): [Pull Request #359](https://github.com/bigchaindb/bigchaindb/pull/359) diff --git a/Dockerfile b/Dockerfile index fafa42ae..c4c30789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,9 @@ WORKDIR /data ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 -ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1 +# BigchainDB Server doesn't need BIGCHAINDB_API_ENDPOINT any more +# but maybe our Docker or Docker Compose stuff does? +# ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1 ENTRYPOINT ["bigchaindb", "--dev-start-rethinkdb", "--dev-allow-temp-keypair"] diff --git a/Dockerfile-dev b/Dockerfile-dev index 2d0cf7ba..c201fd0b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,6 +1,6 @@ FROM python:3.5 -RUN apt-get update +RUN apt-get update && apt-get install -y python3.4 vim RUN mkdir -p /usr/src/app WORKDIR /usr/src/app diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index a2a9800b..42f18c85 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -73,13 +73,13 @@ flake8 --max-line-length 119 bigchaindb/ ``` -## Writing and Running (Python) Unit Tests +## Writing and Running (Python) Tests -We write unit tests for our Python code using the [pytest](http://pytest.org/latest/) framework. +We write unit and integration tests for our Python code using the [pytest](http://pytest.org/latest/) framework. All tests go in the `bigchaindb/tests` directory or one of its subdirectories. You can use the tests already in there as templates or examples. -You can run all unit tests using: +You can run all tests using: ```text py.test -v ``` @@ -96,4 +96,27 @@ python setup.py test If you want to learn about all the things you can do with pytest, see [the pytest documentation](http://pytest.org/latest/). -**Automated testing of pull requests.** We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) +### Tox + +We use [tox](https://tox.readthedocs.io/en/latest/) to run multiple suites of tests against multiple environments during automated testing. Generally you don't need to run this yourself, but it might be useful when troubleshooting a failing CI build. + +To run all the tox tests, use: +```text +tox +``` + +or: +```text +python -m tox +``` + +To run only a few environments, use the `-e` flag: +```text +tox -e {ENVLIST} +``` + +where `{ENVLIST}` is one or more of the environments specified in the [tox.ini file](tox.ini). + +### Automated testing of pull requests + +We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) diff --git a/Release_Process.md b/Release_Process.md index e6eaf7ae..16dc800b 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -2,7 +2,6 @@ This is a summary of the steps we go through to release a new version of BigchainDB Server. -1. Run `python docs/server/generate_schema_documentation.py` and commit the changes in `docs/server/source/schema/`, if any. 1. Update the `CHANGELOG.md` file 1. Update the version numbers in `bigchaindb/version.py`. Note that we try to use [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH) 1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index 22e0bf97..073b1ce9 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -29,7 +29,6 @@ config = { 'port': 8125, 'rate': 0.01, }, - 'api_endpoint': os.environ.get('BIGCHAINDB_API_ENDPOINT') or 'http://localhost:9984/api/v1', 'backlog_reassign_delay': 120 } diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 6bb4f038..9fdb3136 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -18,7 +18,7 @@ def validate_transaction_schema(tx_body): try: jsonschema.validate(tx_body, TX_SCHEMA) except jsonschema.ValidationError as exc: - raise SchemaValidationError(str(exc)) + raise SchemaValidationError(str(exc)) from exc __all__ = ['TX_SCHEMA', 'TX_SCHEMA_YAML', 'validate_transaction_schema'] diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index 28a7c53e..623f45e4 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -5,10 +5,14 @@ type: object additionalProperties: false title: Transaction Schema description: | - This is the outer transaction wrapper. It contains the ID, version and the body of the transaction, which is also called ``transaction``. + A transaction represents the creation or transfer of assets in BigchainDB. required: - id -- transaction +- fulfillments +- conditions +- operation +- metadata +- asset - version properties: id: @@ -18,51 +22,38 @@ properties: derived hashes and signatures from the transaction, serializing it to JSON with keys in sorted order and then hashing the resulting string with sha3. - transaction: - type: object - title: transaction + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" description: | - See: `Transaction Body`_. - additionalProperties: false - required: - - fulfillments - - conditions - - operation - - metadata - - asset - properties: - operation: - "$ref": "#/definitions/operation" - asset: - "$ref": "#/definitions/asset" - description: | - Description of the asset being transacted. + Description of the asset being transacted. - See: `Asset`_. - fulfillments: - type: array - title: "Fulfillments list" - description: | - Array of the fulfillments (inputs) of a transaction. + See: `Asset`_. + fulfillments: + type: array + title: "Fulfillments list" + description: | + Array of the fulfillments (inputs) of a transaction. - See: Fulfillment_. - items: - "$ref": "#/definitions/fulfillment" - conditions: - type: array - description: | - Array of conditions (outputs) provided by this transaction. + See: Fulfillment_. + items: + "$ref": "#/definitions/fulfillment" + conditions: + type: array + description: | + Array of conditions (outputs) provided by this transaction. - See: Condition_. - items: - "$ref": "#/definitions/condition" - metadata: - "$ref": "#/definitions/metadata" - description: | - User provided transaction metadata. This field may be ``null`` or may - contain an id and an object with freeform metadata. + See: Condition_. + items: + "$ref": "#/definitions/condition" + metadata: + "$ref": "#/definitions/metadata" + description: | + User provided transaction metadata. This field may be ``null`` or may + contain an id and an object with freeform metadata. - See: `Metadata`_. + See: `Metadata`_. version: type: integer minimum: 1 @@ -236,17 +227,7 @@ definitions: - type: object description: | User provided transaction metadata. This field may be ``null`` or may - contain an id and an object with freeform metadata. - additionalProperties: false - required: - - id - - data - properties: - id: - "$ref": "#/definitions/uuid4" - data: - type: object - description: | - User provided transaction metadata. - additionalProperties: true + contain an non empty object with freeform metadata. + additionalProperties: true + minProperties: 1 - type: 'null' diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index b2b4167b..62269d0c 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -556,66 +556,6 @@ class AssetLink(Asset): } -class Metadata(object): - """Metadata is used to store a dictionary and its hash in a Transaction.""" - - def __init__(self, data=None, data_id=None): - """Metadata stores a payload `data` as well as data's hash, `data_id`. - - Note: - When no `data_id` is provided, one is being generated by - this method. - - Args: - data (dict): A dictionary to be held by Metadata. - data_id (str): A hash corresponding to the contents of - `data`. - """ - if data is not None and not isinstance(data, dict): - raise TypeError('`data` must be a dict instance or None') - - self.data_id = data_id if data_id is not None else self.to_hash() - self.data = data - - def __eq__(self, other): - # TODO: If `other !== Data` return `False` - return self.to_dict() == other.to_dict() - - @classmethod - def from_dict(cls, data): - """Transforms a Python dictionary to a Metadata object. - - Args: - data (dict): The dictionary to be serialized. - - Returns: - :class:`~bigchaindb.common.transaction.Metadata` - """ - try: - return cls(data['data'], data['id']) - except TypeError: - return cls() - - def to_dict(self): - """Transforms the object to a Python dictionary. - - Returns: - (dict|None): The Metadata object as an alternative - serialization format. - """ - if self.data is None: - return None - else: - return { - 'data': self.data, - 'id': self.data_id, - } - - def to_hash(self): - """A hash corresponding to the contents of `payload`.""" - return str(uuid4()) - - class Transaction(object): """A Transaction is used to create and transfer assets. @@ -630,7 +570,7 @@ class Transaction(object): spend. conditions (:obj:`list` of :class:`~bigchaindb.common. transaction.Condition`, optional): Define the assets to lock. - metadata (:class:`~bigchaindb.common.transaction.Metadata`): + metadata (dict): Metadata to be stored along with the Transaction. version (int): Defines the version number of a Transaction. """ @@ -658,7 +598,7 @@ class Transaction(object): conditions (:obj:`list` of :class:`~bigchaindb.common. transaction.Condition`, optional): Define the assets to lock. - metadata (:class:`~bigchaindb.common.transaction.Metadata`): + metadata (dict): Metadata to be stored along with the Transaction. version (int): Defines the version number of a Transaction. @@ -679,8 +619,8 @@ class Transaction(object): if fulfillments and not isinstance(fulfillments, list): raise TypeError('`fulfillments` must be a list instance or None') - if metadata is not None and not isinstance(metadata, Metadata): - raise TypeError('`metadata` must be a Metadata instance or None') + if metadata is not None and not isinstance(metadata, dict): + raise TypeError('`metadata` must be a dict or None') self.version = version if version is not None else self.VERSION self.operation = operation @@ -734,7 +674,6 @@ class Transaction(object): if len(owners_after) == 0: raise ValueError('`owners_after` list cannot be empty') - metadata = Metadata(metadata) fulfillments = [] conditions = [] @@ -809,7 +748,6 @@ class Transaction(object): pub_keys, amount = owner_after conditions.append(Condition.generate(pub_keys, amount)) - metadata = Metadata(metadata) inputs = deepcopy(inputs) return cls(cls.TRANSFER, asset, inputs, conditions, metadata) @@ -1151,30 +1089,21 @@ class Transaction(object): Returns: dict: The Transaction as an alternative serialization format. """ - try: - metadata = self.metadata.to_dict() - except AttributeError: - # NOTE: metadata can be None and that's OK - metadata = None - if self.operation in (self.__class__.GENESIS, self.__class__.CREATE): asset = self.asset.to_dict() else: # NOTE: An `asset` in a `TRANSFER` only contains the asset's id asset = {'id': self.asset.data_id} - tx_body = { + tx = { 'fulfillments': [fulfillment.to_dict() for fulfillment in self.fulfillments], 'conditions': [condition.to_dict() for condition in self.conditions], 'operation': str(self.operation), - 'metadata': metadata, + 'metadata': self.metadata, 'asset': asset, - } - tx = { 'version': self.version, - 'transaction': tx_body, } tx_no_signatures = Transaction._remove_signatures(tx) @@ -1199,7 +1128,7 @@ class Transaction(object): # NOTE: We remove the reference since we need `tx_dict` only for the # transaction's hash tx_dict = deepcopy(tx_dict) - for fulfillment in tx_dict['transaction']['fulfillments']: + for fulfillment in tx_dict['fulfillments']: # NOTE: Not all Cryptoconditions return a `signature` key (e.g. # ThresholdSha256Fulfillment), so setting it to `None` in any # case could yield incorrect signatures. This is why we only @@ -1239,17 +1168,19 @@ class Transaction(object): try: proposed_tx_id = tx_body.pop('id') except KeyError: - raise InvalidHash() + raise InvalidHash('No transaction id found!') tx_body_no_signatures = Transaction._remove_signatures(tx_body) tx_body_serialized = Transaction._to_str(tx_body_no_signatures) valid_tx_id = Transaction._to_hash(tx_body_serialized) if proposed_tx_id != valid_tx_id: - raise InvalidHash() + err_msg = ("The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid.") + raise InvalidHash(err_msg.format(proposed_tx_id)) @classmethod - def from_dict(cls, tx_body): + def from_dict(cls, tx): """Transforms a Python dictionary to a Transaction object. Args: @@ -1258,17 +1189,15 @@ class Transaction(object): Returns: :class:`~bigchaindb.common.transaction.Transaction` """ - cls.validate_structure(tx_body) - tx = tx_body['transaction'] + cls.validate_structure(tx) fulfillments = [Fulfillment.from_dict(fulfillment) for fulfillment in tx['fulfillments']] conditions = [Condition.from_dict(condition) for condition in tx['conditions']] - metadata = Metadata.from_dict(tx['metadata']) if tx['operation'] in [cls.CREATE, cls.GENESIS]: asset = Asset.from_dict(tx['asset']) else: asset = AssetLink.from_dict(tx['asset']) return cls(tx['operation'], asset, fulfillments, conditions, - metadata, tx_body['version']) + tx['metadata'], tx['version']) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 653b7ac3..34dbab84 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -330,35 +330,6 @@ class Bigchain(object): else: return None - def get_transaction_by_metadata_id(self, metadata_id): - """Retrieves valid or undecided transactions related to a particular - metadata. - - When creating a transaction one of the optional arguments is the - `metadata`. The metadata is a generic dict that contains extra - information that can be appended to the transaction. - - To make it easy to query the bigchain for that particular metadata we - create a UUID for the metadata and store it with the transaction. - - Args: - metadata_id (str): the id for this particular metadata. - - Returns: - A list of valid or undecided transactions containing that metadata. - If no transaction exists with that metadata it returns an empty - list `[]` - """ - txids = self.backend.get_txids_by_metadata_id(metadata_id) - transactions = [] - for txid in txids: - tx = self.get_transaction(txid) - # if a valid or undecided transaction exists append it to the list - # of transactions - if tx: - transactions.append(tx) - return transactions - def get_transactions_by_asset_id(self, asset_id): """Retrieves valid or undecided transactions related to a particular asset. @@ -396,7 +367,7 @@ class Bigchain(object): cursor = self.backend.get_asset_by_id(asset_id) cursor = list(cursor) if cursor: - return Asset.from_dict(cursor[0]['transaction']['asset']) + return Asset.from_dict(cursor[0]['asset']) def get_spent(self, txid, cid): """Check if a `txid` was already used as an input. @@ -440,13 +411,13 @@ class Bigchain(object): return None def get_owned_ids(self, owner): - """Retrieve a list of `txid`s that can be used as inputs. + """Retrieve a list of ``txid`` s that can be used as inputs. Args: owner (str): base58 encoded public key. Returns: - :obj:`list` of TransactionLink: list of `txid`s and `cid`s + :obj:`list` of TransactionLink: list of ``txid`` s and ``cid`` s pointing to another transaction's condition """ @@ -465,7 +436,7 @@ class Bigchain(object): # use it after the execution of this function. # a transaction can contain multiple outputs (conditions) so we need to iterate over all of them # to get a list of outputs available to spend - for index, cond in enumerate(tx['transaction']['conditions']): + for index, cond in enumerate(tx['conditions']): # for simple signature conditions there are no subfulfillments # check if the owner is in the condition `owners_after` if len(cond['owners_after']) == 1: diff --git a/bigchaindb/db/backends/rethinkdb.py b/bigchaindb/db/backends/rethinkdb.py index f01bdb4a..39d2aff9 100644 --- a/bigchaindb/db/backends/rethinkdb.py +++ b/bigchaindb/db/backends/rethinkdb.py @@ -138,32 +138,6 @@ class RethinkDBBackend: .get_all(transaction_id, index='transaction_id') .pluck('votes', 'id', {'block': ['voters']})) - def get_txids_by_metadata_id(self, metadata_id): - """Retrieves transaction ids related to a particular metadata. - - When creating a transaction one of the optional arguments is the - `metadata`. The metadata is a generic dict that contains extra - information that can be appended to the transaction. - - To make it easy to query the bigchain for that particular metadata we - create a UUID for the metadata and store it with the transaction. - - Args: - metadata_id (str): the id for this particular metadata. - - Returns: - A list of transaction ids containing that metadata. If no - transaction exists with that metadata it returns an empty list `[]` - """ - return self.connection.run( - r.table('bigchain', read_mode=self.read_mode) - .get_all(metadata_id, index='metadata_id') - .concat_map(lambda block: block['block']['transactions']) - .filter(lambda transaction: - transaction['transaction']['metadata']['id'] == - metadata_id) - .get_field('id')) - def get_txids_by_asset_id(self, asset_id): """Retrieves transactions ids related to a particular asset. @@ -185,7 +159,7 @@ class RethinkDBBackend: r.table('bigchain', read_mode=self.read_mode) .get_all(asset_id, index='asset_id') .concat_map(lambda block: block['block']['transactions']) - .filter(lambda transaction: transaction['transaction']['asset']['id'] == asset_id) + .filter(lambda transaction: transaction['asset']['id'] == asset_id) .get_field('id')) def get_asset_by_id(self, asset_id): @@ -202,10 +176,10 @@ class RethinkDBBackend: .get_all(asset_id, index='asset_id') .concat_map(lambda block: block['block']['transactions']) .filter(lambda transaction: - transaction['transaction']['asset']['id'] == asset_id) + transaction['asset']['id'] == asset_id) .filter(lambda transaction: - transaction['transaction']['operation'] == 'CREATE') - .pluck({'transaction': 'asset'})) + transaction['operation'] == 'CREATE') + .pluck('asset')) def get_spent(self, transaction_id, condition_id): """Check if a `txid` was already used as an input. @@ -225,7 +199,7 @@ class RethinkDBBackend: return self.connection.run( r.table('bigchain', read_mode=self.read_mode) .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda transaction: transaction['transaction']['fulfillments'].contains( + .filter(lambda transaction: transaction['fulfillments'].contains( lambda fulfillment: fulfillment['input'] == {'txid': transaction_id, 'cid': condition_id}))) def get_owned_ids(self, owner): @@ -242,7 +216,7 @@ class RethinkDBBackend: return self.connection.run( r.table('bigchain', read_mode=self.read_mode) .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda tx: tx['transaction']['conditions'].contains( + .filter(lambda tx: tx['conditions'].contains( lambda c: c['owners_after'].contains(owner)))) def get_votes_by_block_id(self, block_id): diff --git a/bigchaindb/db/utils.py b/bigchaindb/db/utils.py index 05932f77..2b3c2d0d 100644 --- a/bigchaindb/db/utils.py +++ b/bigchaindb/db/utils.py @@ -116,15 +116,10 @@ def create_bigchain_secondary_index(conn, dbname): .index_create('transaction_id', r.row['block']['transactions']['id'], multi=True)\ .run(conn) - # secondary index for payload data by UUID - r.db(dbname).table('bigchain')\ - .index_create('metadata_id', - r.row['block']['transactions']['transaction']['metadata']['id'], multi=True)\ - .run(conn) # secondary index for asset uuid r.db(dbname).table('bigchain')\ .index_create('asset_id', - r.row['block']['transactions']['transaction']['asset']['id'], multi=True)\ + r.row['block']['transactions']['asset']['id'], multi=True)\ .run(conn) # wait for rethinkdb to finish creating secondary indexes diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 3bbc80e3..bca182c3 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -6,6 +6,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, AssetIdMismatch, AmountError) from bigchaindb.common.transaction import Transaction, Asset from bigchaindb.common.util import gen_timestamp, serialize +from bigchaindb.common.schema import validate_transaction_schema class Transaction(Transaction): @@ -113,6 +114,11 @@ class Transaction(Transaction): else: return self + @classmethod + def from_dict(cls, tx_body): + validate_transaction_schema(tx_body) + return super().from_dict(tx_body) + class Block(object): """Bundle a list of Transactions in a Block. Nodes vote on its validity. diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index 9039938b..0394aa23 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -8,7 +8,7 @@ function. import logging import rethinkdb as r -from multipipes import Pipeline, Node +from multipipes import Pipeline, Node, Pipe from bigchaindb.models import Transaction from bigchaindb.pipelines.utils import ChangeFeed @@ -161,6 +161,7 @@ def create_pipeline(): block_pipeline = BlockPipeline() pipeline = Pipeline([ + Pipe(maxsize=1000), Node(block_pipeline.filter_tx), Node(block_pipeline.validate_tx, fraction_of_cores=1), Node(block_pipeline.create, timeout=1), diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 27b7fc00..3b0526d7 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -156,4 +156,4 @@ def is_genesis_block(block): try: return block.transactions[0].operation == 'GENESIS' except AttributeError: - return block['block']['transactions'][0]['transaction']['operation'] == 'GENESIS' + return block['block']['transactions'][0]['operation'] == 'GENESIS' diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 95f84a62..060e76d3 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.0.dev' -__short_version__ = '0.8.dev' +__version__ = '0.9.0.dev' +__short_version__ = '0.9.dev' diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index 1ff6d30c..874f899d 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -20,6 +20,5 @@ def home(): 'software': 'BigchainDB', 'version': version.__version__, 'public_key': bigchaindb.config['keypair']['public'], - 'keyring': bigchaindb.config['keyring'], - 'api_endpoint': bigchaindb.config['api_endpoint'] + 'keyring': bigchaindb.config['keyring'] }) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index 2ed19b7c..babec6fc 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -6,7 +6,18 @@ For more information please refer to the documentation on ReadTheDocs: from flask import current_app, request, Blueprint from flask_restful import Resource, Api -from bigchaindb.common.exceptions import ValidationError, InvalidSignature +from bigchaindb.common.exceptions import ( + AmountError, + DoubleSpend, + InvalidHash, + InvalidSignature, + SchemaValidationError, + OperationError, + TransactionDoesNotExist, + TransactionOwnerError, + TransactionNotInValidBlock, + ValidationError, +) import bigchaindb from bigchaindb.models import Transaction @@ -98,16 +109,38 @@ class TransactionListApi(Resource): try: tx_obj = Transaction.from_dict(tx) - except (ValidationError, InvalidSignature): - return make_error(400, 'Invalid transaction') + except SchemaValidationError as e: + return make_error( + 400, + message='Invalid transaction schema: {}'.format( + e.__cause__.message) + ) + except (ValidationError, InvalidSignature) as e: + return make_error( + 400, + 'Invalid transaction ({}): {}'.format(type(e).__name__, e) + ) with pool() as bigchain: - if bigchain.is_valid_transaction(tx_obj): + try: + bigchain.validate_transaction(tx_obj) + except (ValueError, + OperationError, + TransactionDoesNotExist, + TransactionOwnerError, + DoubleSpend, + InvalidHash, + InvalidSignature, + TransactionNotInValidBlock, + AmountError) as e: + return make_error( + 400, + 'Invalid transaction ({}): {}'.format(type(e).__name__, e) + ) + else: rate = bigchaindb.config['statsd']['rate'] with monitor.timer('write_transaction', rate=rate): bigchain.write_transaction(tx_obj) - else: - return make_error(400, 'Invalid transaction') return tx diff --git a/deploy-cluster-aws/awsdeploy.sh b/deploy-cluster-aws/awsdeploy.sh index a6d068b8..e5a27ff7 100755 --- a/deploy-cluster-aws/awsdeploy.sh +++ b/deploy-cluster-aws/awsdeploy.sh @@ -160,7 +160,7 @@ if [ "$WHAT_TO_DEPLOY" == "servers" ]; then # bigchaindb installed, so bigchaindb configure can't be called) # Transform the config files in the confiles directory - # to have proper keyrings, api_endpoint values, etc. + # to have proper keyrings etc. if [ "$USE_KEYPAIRS_FILE" == "True" ]; then python clusterize_confiles.py -k confiles $NUM_NODES else @@ -184,8 +184,6 @@ if [ "$WHAT_TO_DEPLOY" == "servers" ]; then echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" else # Deploying clients - # The only thing to configure on clients is the api_endpoint - # It should be the public DNS name of a BigchainDB server fab send_client_confile:client_confile # Start sending load from the clients to the servers diff --git a/deploy-cluster-aws/clusterize_confiles.py b/deploy-cluster-aws/clusterize_confiles.py index f266fe75..fbbf0dd6 100644 --- a/deploy-cluster-aws/clusterize_confiles.py +++ b/deploy-cluster-aws/clusterize_confiles.py @@ -98,9 +98,6 @@ for i, filename in enumerate(conf_files): # Allow incoming server traffic from any IP address # to port 9984 conf_dict['server']['bind'] = '0.0.0.0:9984' - # Set the api_endpoint - conf_dict['api_endpoint'] = 'http://' + public_dns_names[i] + \ - ':9984/api/v1' # Delete the config file os.remove(file_path) diff --git a/docker-compose.yml b/docker-compose.yml index c30b69b4..0dcfe8c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,9 +27,9 @@ services: - ./setup.py:/usr/src/app/setup.py - ./setup.cfg:/usr/src/app/setup.cfg - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini environment: BIGCHAINDB_DATABASE_HOST: rdb - BIGCHAINDB_API_ENDPOINT: http://bdb:9984/api/v1 BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 ports: - "9984" diff --git a/docs.yml b/docs.yml index f3f5efcb..b6520ecb 100644 --- a/docs.yml +++ b/docs.yml @@ -7,7 +7,7 @@ services: dockerfile: ./Dockerfile-dev volumes: - .:/usr/src/app/ - command: make -C docs html + command: make -C docs/server html vdocs: image: nginx ports: diff --git a/docs/generate/__init__.py b/docs/generate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py index e9bbfcbf..50dec3ea 100644 --- a/docs/root/source/conf.py +++ b/docs/root/source/conf.py @@ -162,7 +162,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = [] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py new file mode 100644 index 00000000..80b47cf2 --- /dev/null +++ b/docs/server/generate_http_server_api_documentation.py @@ -0,0 +1,89 @@ +""" Script to build http examples for http server api docs """ + +import json +import os +import os.path + +from bigchaindb.common.transaction import Asset, Transaction + + +TPLS = {} + +TPLS['post-tx-request'] = """\ +POST /transactions/ HTTP/1.1 +Host: example.com +Content-Type: application/json + +%(tx)s +""" + + +TPLS['post-tx-response'] = """\ +HTTP/1.1 201 Created +Content-Type: application/json + +%(tx)s +""" + + +TPLS['get-tx-status-request'] = """\ +GET /transactions/%(txid)s/status HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-status-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "status": "valid" +} +""" + + +TPLS['get-tx-request'] = """\ +GET /transactions/%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(tx)s +""" + + +def main(): + """ Main function """ + privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' + pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' + asset = Asset(None, 'e6969f87-4fc9-4467-b62a-f0dfa1c85002') + tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset) + tx = tx.sign([privkey]) + tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) + + base_path = os.path.join(os.path.dirname(__file__), + 'source/drivers-clients/samples') + + if not os.path.exists(base_path): + os.makedirs(base_path) + + for name, tpl in TPLS.items(): + path = os.path.join(base_path, name + '.http') + code = tpl % {'tx': tx_json, 'txid': tx.id} + with open(path, 'w') as handle: + handle.write(code) + + +def setup(*_): + """ Fool sphinx into think it's an extension muahaha """ + main() + + +if __name__ == '__main__': + main() diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 0e1a626a..1d1b751b 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -27,8 +27,6 @@ Transaction Schema * `Transaction`_ -* `Transaction Body`_ - * Condition_ * Fulfillment_ @@ -58,11 +56,6 @@ Transaction Schema Transaction ----------- -%(wrapper)s - -Transaction Body ----------------- - %(transaction)s Condition @@ -158,9 +151,7 @@ def main(): """ Main function """ defs = TX_SCHEMA['definitions'] doc = TPL_DOC % { - 'wrapper': render_section('Transaction', TX_SCHEMA), - 'transaction': render_section('Transaction', - TX_SCHEMA['properties']['transaction']), + 'transaction': render_section('Transaction', TX_SCHEMA), 'condition': render_section('Condition', defs['condition']), 'fulfillment': render_section('Fulfillment', defs['fulfillment']), 'asset': render_section('Asset', defs['asset']), @@ -168,12 +159,20 @@ def main(): 'file': os.path.basename(__file__), } - path = os.path.join(os.path.dirname(__file__), - 'source/schema/transaction.rst') + base_path = os.path.join(os.path.dirname(__file__), 'source/schema') + path = os.path.join(base_path, 'transaction.rst') + + if not os.path.exists(base_path): + os.makedirs(base_path) with open(path, 'w') as handle: handle.write(doc) +def setup(*_): + """ Fool sphinx into think it's an extension muahaha """ + main() + + if __name__ == '__main__': main() diff --git a/docs/server/source/appendices/consensus.rst b/docs/server/source/appendices/consensus.rst index 837e81d5..34c0c032 100644 --- a/docs/server/source/appendices/consensus.rst +++ b/docs/server/source/appendices/consensus.rst @@ -3,4 +3,3 @@ Consensus ######### .. automodule:: bigchaindb.consensus - :members: diff --git a/docs/server/source/appendices/pipelines.rst b/docs/server/source/appendices/pipelines.rst index 2685d2d3..98b993a1 100644 --- a/docs/server/source/appendices/pipelines.rst +++ b/docs/server/source/appendices/pipelines.rst @@ -6,32 +6,27 @@ Block Creation ============== .. automodule:: bigchaindb.pipelines.block - :members: Block Voting ============ .. automodule:: bigchaindb.pipelines.vote - :members: Block Status ============ .. automodule:: bigchaindb.pipelines.election - :members: Stale Transaction Monitoring ============================ .. automodule:: bigchaindb.pipelines.stale - :members: Utilities ========= .. automodule:: bigchaindb.pipelines.utils - :members: diff --git a/docs/server/source/appendices/the-Bigchain-class.rst b/docs/server/source/appendices/the-Bigchain-class.rst index d9e6744c..9b88d5dd 100644 --- a/docs/server/source/appendices/the-Bigchain-class.rst +++ b/docs/server/source/appendices/the-Bigchain-class.rst @@ -5,6 +5,5 @@ The Bigchain class The Bigchain class is the top-level Python API for BigchainDB. If you want to create and initialize a BigchainDB database, you create a Bigchain instance (object). Then you can use its various methods to create transactions, write transactions (to the object/database), read transactions, etc. .. autoclass:: bigchaindb.Bigchain - :members: .. automethod:: bigchaindb.core.Bigchain.__init__ diff --git a/docs/server/source/conf.py b/docs/server/source/conf.py index c0de76d7..f4951235 100644 --- a/docs/server/source/conf.py +++ b/docs/server/source/conf.py @@ -35,6 +35,10 @@ _version = {} with open('../../../bigchaindb/version.py') as fp: exec(fp.read(), _version) +import os.path +import sys + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/..')) extensions = [ 'sphinx.ext.autodoc', @@ -44,10 +48,17 @@ extensions = [ 'sphinx.ext.napoleon', 'sphinxcontrib.httpdomain', 'sphinx.ext.autosectionlabel', + # Below are actually build steps made to look like sphinx extensions. + # It was the easiest way to get it running with ReadTheDocs. + 'generate_schema_documentation', + 'generate_http_server_api_documentation', ] # autodoc settings autodoc_member_order = 'bysource' +autodoc_default_flags = [ + 'members', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 2bee661f..69a3b168 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -22,38 +22,30 @@ A transaction has the following structure: { "id": "", "version": "", - "transaction": { - "fulfillments": [""], - "conditions": [""], - "operation": "", - "asset": "", - "metadata": { - "id": "", - "data": "" - } - } + "fulfillments": [""], + "conditions": [""], + "operation": "", + "asset": "", + "metadata": "" } Here's some explanation of the contents of a :ref:`transaction `: -- :ref:`id `: The id of the transaction, and also the database primary key. -- :ref:`version `: Version number of the transaction model, so that software can support different transaction models. -- :ref:`transaction `: - - **fulfillments**: List of fulfillments. Each :ref:`fulfillment ` contains a pointer to an unspent asset - and a *crypto fulfillment* that satisfies a spending condition set on the unspent asset. A *fulfillment* - is usually a signature proving the ownership of the asset. - See :doc:`./crypto-conditions`. +- id: The :ref:`id ` of the transaction, and also the database primary key. +- version: :ref:`Version ` number of the transaction model, so that software can support different transaction models. +- **fulfillments**: List of fulfillments. Each :ref:`fulfillment ` contains a pointer to an unspent asset + and a *crypto fulfillment* that satisfies a spending condition set on the unspent asset. A *fulfillment* + is usually a signature proving the ownership of the asset. + See :doc:`./crypto-conditions`. - - **conditions**: List of conditions. Each :ref:`condition ` is a *crypto-condition* that needs to be fulfilled by a transfer transaction in order to transfer ownership to new owners. - See :doc:`./crypto-conditions`. +- **conditions**: List of conditions. Each :ref:`condition ` is a *crypto-condition* that needs to be fulfilled by a transfer transaction in order to transfer ownership to new owners. + See :doc:`./crypto-conditions`. - - **operation**: String representation of the :ref:`operation ` being performed (currently either "CREATE", "TRANSFER" or "GENESIS"). It determines how the transaction should be validated. +- **operation**: String representation of the :ref:`operation ` being performed (currently either "CREATE", "TRANSFER" or "GENESIS"). It determines how the transaction should be validated. - - **asset**: Definition of the digital :ref:`asset `. See next section. +- **asset**: Definition of the digital :ref:`asset `. See next section. - - **metadata**: - - :ref:`id `: UUID version 4 (random) converted to a string of hex digits in standard form. - - :ref:`data `: Can be any JSON document. It may be empty in the case of a transfer transaction. +- **metadata**: User-provided transaction :ref:`metadata `: Can be any JSON document, or `NULL`. Later, when we get to the models for the block and the vote, we'll see that both include a signature (from the node which created it). You may wonder why transactions don't have signatures... The answer is that they do! They're just hidden inside the ``fulfillment`` string of each fulfillment. A creation transaction is signed by whoever created it. A transfer transaction is signed by whoever currently controls or owns it. diff --git a/docs/server/source/dev-and-test/setup-run-node.md b/docs/server/source/dev-and-test/setup-run-node.md index 0b55d543..e577a755 100644 --- a/docs/server/source/dev-and-test/setup-run-node.md +++ b/docs/server/source/dev-and-test/setup-run-node.md @@ -91,7 +91,6 @@ should give you something like: ```bash { - "api_endpoint": "http://bdb:9984/api/v1", "keyring": [], "public_key": "Brx8g4DdtEhccsENzNNV6yvQHR8s9ebhKyXPFkWUXh5e", "software": "BigchainDB", diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index c96936c7..17cdf429 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -26,17 +26,18 @@ details, see the "server" settings ("bind", "workers" and "threads") in <../server-reference/configuration>`. -API Root --------- +API Root URL +------------ -If you send an HTTP GET request to e.g. ``http://localhost:9984`` (with no -``/api/v1/`` on the end), then you should get an HTTP response with something -like the following in the body: +If you send an HTTP GET request to e.g. ``http://localhost:9984`` +or ``http://apihosting4u.net:9984`` +(with no ``/api/v1/`` on the end), +then you should get an HTTP response +with something like the following in the body: .. code-block:: json { - "api_endpoint": "http://localhost:9984/api/v1", "keyring": [ "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" @@ -46,6 +47,25 @@ like the following in the body: "version": "0.6.0" } +If the API endpoint is publicly-accessible, +then the public API Root URL is determined as follows: + +- The public IP address (like 12.34.56.78) + is the public IP address of the machine exposing + the HTTP API to the public internet (e.g. either the machine hosting + Gunicorn or the machine running the reverse proxy such as Nginx). + It's determined by AWS, Azure, Rackspace, or whoever is hosting the machine. + +- The DNS hostname (like apihosting4u.net) is determined by DNS records, + such as an "A Record" associating apihosting4u.net with 12.34.56.78 + +- The port (like 9984) is determined by the ``server.bind`` setting + if Gunicorn is exposed directly to the public Internet. + If a reverse proxy (like Nginx) is exposed directly to the public Internet + instead, then it could expose the HTTP API on whatever port it wants to. + (It should expose the HTTP API on port 9984, but it's not bound to do + that by anything other than convention.) + POST /transactions/ ------------------- @@ -54,113 +74,23 @@ POST /transactions/ Push a new transaction. - Note: The posted transaction should be valid `transaction + Note: The posted transaction should be a valid and signed `transaction `_. The steps to build a valid transaction are beyond the scope of this page. One would normally use a driver such as the `BigchainDB Python Driver `_ to - build a valid transaction. + build a valid transaction. The exact contents of a valid transaction depend + on the associated public/private keypairs. **Example request**: - .. sourcecode:: http - - POST /transactions/ HTTP/1.1 - Host: example.com - Content-Type: application/json - - { - "transaction": { - "conditions": [ - { - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "amount": 1, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", - "refillable": false - }, - "metadata": null, - "fulfillments": [ - { - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - } + .. literalinclude:: samples/post-tx-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 201 Created - Content-Type: application/json - - { - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1, - "transaction": { - "conditions": [ - { - "amount": 1, - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type_id": 4, - "type": "fulfillment", - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ], - } - ], - "fulfillments": [ - { - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "updatable": false, - "refillable": false, - "divisible": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d" - }, - "metadata": null - } - } + .. literalinclude:: samples/post-tx-response.http + :language: http :statuscode 201: A new transaction was created. :statuscode 400: The transaction was invalid and not created. @@ -182,21 +112,13 @@ GET /transactions/{tx_id}/status **Example request**: - .. sourcecode:: http - - GET /transactions/7ad5a4b83bc8c70c4fd7420ff3c60693ab8e6d0e3124378ca69ed5acd2578792/status HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-status-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "status": "valid" - } + .. literalinclude:: samples/get-tx-status-response.http + :language: http :statuscode 200: A transaction with that ID was found and the status is returned. :statuscode 404: A transaction with that ID was not found. @@ -217,60 +139,13 @@ GET /transactions/{tx_id} **Example request**: - .. sourcecode:: http - - GET /transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "transaction": { - "conditions": [ - { - "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - } - }, - "amount": 1, - "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", - "refillable": false - }, - "metadata": null, - "fulfillments": [ - { - "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", - "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" - ] - } - ] - }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", - "version": 1 - } + .. literalinclude:: samples/get-tx-response.http + :language: http :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. @@ -315,8 +190,8 @@ GET /unspents/ Content-Type: application/json [ - '../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/0', - '../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/1' + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/0", + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/1" ] :statuscode 200: A list of outputs were found and returned in the body of the response. diff --git a/docs/server/source/nodes/setup-run-node.md b/docs/server/source/nodes/setup-run-node.md index 443f5ec9..6b8a29b7 100644 --- a/docs/server/source/nodes/setup-run-node.md +++ b/docs/server/source/nodes/setup-run-node.md @@ -167,7 +167,6 @@ Edit the created config file: * Open `$HOME/.bigchaindb` (the created config file) in your text editor. * Change `"server": {"bind": "localhost:9984", ... }` to `"server": {"bind": "0.0.0.0:9984", ... }`. This makes it so traffic can come from any IP address to port 9984 (the HTTP Client-Server API port). -* Change `"api_endpoint": "http://localhost:9984/api/v1"` to `"api_endpoint": "http://your_api_hostname:9984/api/v1"` * Change `"keyring": []` to `"keyring": ["public_key_of_other_node_A", "public_key_of_other_node_B", "..."]` i.e. a list of the public keys of all the other nodes in the federation. The keyring should _not_ include your node's public key. For more information about the BigchainDB config file, see [Configuring a BigchainDB Node](configuration.html). diff --git a/docs/server/source/schema/transaction.rst b/docs/server/source/schema/transaction.rst deleted file mode 100644 index 85c847c6..00000000 --- a/docs/server/source/schema/transaction.rst +++ /dev/null @@ -1,307 +0,0 @@ -.. This file was auto generated by generate_schema_documentation.py - -================== -Transaction Schema -================== - -* `Transaction`_ - -* `Transaction Body`_ - -* Condition_ - -* Fulfillment_ - -* Asset_ - -* Metadata_ - -.. raw:: html - - - -Transaction ------------ - -This is the outer transaction wrapper. It contains the ID, version and the body of the transaction, which is also called ``transaction``. - - -Transaction.id -^^^^^^^^^^^^^^ - -**type:** string - -A sha3 digest of the transaction. The ID is calculated by removing all -derived hashes and signatures from the transaction, serializing it to -JSON with keys in sorted order and then hashing the resulting string -with sha3. - - - -Transaction.transaction -^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** object - -See: `Transaction Body`_. - - - -Transaction.version -^^^^^^^^^^^^^^^^^^^ - -**type:** integer - -BigchainDB transaction schema version. - - - - - -Transaction Body ----------------- - -See: `Transaction Body`_. - - -Transaction.operation -^^^^^^^^^^^^^^^^^^^^^ - -**type:** string - -Type of the transaction: - -A ``CREATE`` transaction creates an asset in BigchainDB. This -transaction has outputs (conditions) but no inputs (fulfillments), -so a dummy fulfillment is used. - -A ``TRANSFER`` transaction transfers ownership of an asset, by providing -fulfillments to conditions of earlier transactions. - -A ``GENESIS`` transaction is a special case transaction used as the -sole member of the first block in a BigchainDB ledger. - - - -Transaction.asset -^^^^^^^^^^^^^^^^^ - -**type:** object - -Description of the asset being transacted. - -See: `Asset`_. - - - -Transaction.fulfillments -^^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (object) - -Array of the fulfillments (inputs) of a transaction. - -See: Fulfillment_. - - - -Transaction.conditions -^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (object) - -Array of conditions (outputs) provided by this transaction. - -See: Condition_. - - - -Transaction.metadata -^^^^^^^^^^^^^^^^^^^^ - -**type:** object or null - -User provided transaction metadata. This field may be ``null`` or may -contain an id and an object with freeform metadata. - -See: `Metadata`_. - - - - - -Condition ----------- - -An output of a transaction. A condition describes a quantity of an asset -and what conditions must be met in order for it to be fulfilled. See also: -fulfillment_. - - -Condition.condition -^^^^^^^^^^^^^^^^^^^ - -**type:** object - -Body of the condition. Has the properties: - -- **details**: Details of the condition. -- **uri**: Condition encoded as an ASCII string. - - - -Condition.owners_after -^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (string) or null - -List of public keys associated with asset ownership at the time -of the transaction. - - - -Condition.amount -^^^^^^^^^^^^^^^^ - -**type:** integer - -Integral amount of the asset represented by this condition. -In the case of a non divisible asset, this will always be 1. - - - - - -Fulfillment ------------ - -A fulfillment is an input to a transaction, named as such because it fulfills a condition of a previous transaction. In the case of a ``CREATE`` transaction, a fulfillment may provide no ``input``. - -Fulfillment.owners_before -^^^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (string) or null - -List of public keys of the previous owners of the asset. - - - -Fulfillment.fulfillment -^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** object or string - -Fulfillment of a condition_, or put a different way, this is a -payload that satisfies a condition in order to spend the associated -asset. - - - -Fulfillment.input -^^^^^^^^^^^^^^^^^ - -**type:** object or null - -Reference to a condition of a previous transaction - - - - - -Asset ------ - -Description of the asset being transacted. In the case of a ``TRANSFER`` -transaction, this field contains only the ID of asset. In the case -of a ``CREATE`` transaction, this field may contain properties: - - -Asset.id -^^^^^^^^ - -**type:** string - -A `UUID `_ -of type 4 (random). - - - -Asset.divisible -^^^^^^^^^^^^^^^ - -**type:** boolean - -Whether or not the asset has a quantity that may be partially spent. - - - -Asset.updatable -^^^^^^^^^^^^^^^ - -**type:** boolean - -Whether or not the description of the asset may be updated. Defaults to false. - - - -Asset.refillable -^^^^^^^^^^^^^^^^ - -**type:** boolean - -Whether the amount of the asset can change after its creation. Defaults to false. - - - -Asset.data -^^^^^^^^^^ - -**type:** object or null - -User provided metadata associated with the asset. May also be ``null``. - - - - - -Metadata --------- - -User provided transaction metadata. This field may be ``null`` or may -contain an id and an object with freeform metadata. - - -Metadata.id -^^^^^^^^^^^ - -**type:** string - -A `UUID `_ -of type 4 (random). - - - -Metadata.data -^^^^^^^^^^^^^ - -**type:** object - -User provided transaction metadata. - - - - diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 41b0c2ca..5592e3cd 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -17,11 +17,11 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_SERVER_BIND`
`BIGCHAINDB_SERVER_WORKERS`
`BIGCHAINDB_SERVER_THREADS`
-`BIGCHAINDB_API_ENDPOINT`
`BIGCHAINDB_STATSD_HOST`
`BIGCHAINDB_STATSD_PORT`
`BIGCHAINDB_STATSD_RATE`
`BIGCHAINDB_CONFIG_PATH`
+`BIGCHAINDB_BACKLOG_REASSIGN_DELAY`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -140,26 +140,6 @@ export BIGCHAINDB_SERVER_THREADS=5 ``` -## api_endpoint - -`api_endpoint` is the URL where a BigchainDB client can get access to the HTTP client-server API. - -**Example using an environment variable** -```text -export BIGCHAINDB_API_ENDPOINT="http://localhost:9984/api/v1" -``` - -**Example config file snippet** -```js -"api_endpoint": "http://webserver.blocks587.net:9984/api/v1" -``` - -**Default value (from a config file)** -```js -"api_endpoint": "http://localhost:9984/api/v1" -``` - - ## statsd.host, statsd.port & statsd.rate These settings are used to configure where, and how often, [StatsD](https://github.com/etsy/statsd) should send data for [cluster monitoring](../clusters-feds/monitoring.html) purposes. `statsd.host` is the hostname of the monitoring server, where StatsD should send its data. `stats.port` is the port. `statsd.rate` is the fraction of transaction operations that should be sampled. It's a float between 0.0 and 1.0. @@ -175,3 +155,17 @@ export BIGCHAINDB_STATSD_RATE=0.01 ```js "statsd": {"host": "localhost", "port": 8125, "rate": 0.01} ``` + +## backlog_reassign_delay + +Specifies how long, in seconds, transactions can remain in the backlog before being reassigned. Long-waiting transactions must be reassigned because the assigned node may no longer be responsive. The default duration is 120 seconds. + +**Example using environment variables** +```text +export BIGCHAINDB_BACKLOG_REASSIGN_DELAY=30 +``` + +**Default value (from a config file)** +```js +"backlog_reassign_delay": 120 +``` diff --git a/setup.py b/setup.py index 4043c0be..00300b92 100644 --- a/setup.py +++ b/setup.py @@ -27,18 +27,6 @@ def check_setuptools_features(): check_setuptools_features() - -tests_require = [ - 'coverage', - 'pep8', - 'flake8', - 'pylint', - 'pytest', - 'pytest-cov>=2.2.1', - 'pytest-xdist', - 'pytest-flask', -] - dev_require = [ 'ipdb', 'ipython', @@ -52,6 +40,18 @@ docs_require = [ 'sphinxcontrib-napoleon>=0.4.4', ] +tests_require = [ + 'coverage', + 'pep8', + 'flake8', + 'pylint', + 'pytest>=3.0.0', + 'pytest-cov>=2.2.1', + 'pytest-xdist', + 'pytest-flask', + 'tox', +] + docs_require + benchmarks_require = [ 'line-profiler==1.0', ] diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 1e5fe7d3..a54daf20 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -136,12 +136,6 @@ def uuid4(): return UUID4 -@pytest.fixture -def metadata(data, data_id): - from bigchaindb.common.transaction import Metadata - return Metadata(data, data_id) - - @pytest.fixture def utx(user_ffill, user_cond): from bigchaindb.common.transaction import Transaction, Asset diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py index 5ded0272..1827d3cb 100644 --- a/tests/common/test_schema.py +++ b/tests/common/test_schema.py @@ -16,6 +16,16 @@ def test_validate_transaction_signed_transfer(signed_transfer_tx): validate_transaction_schema(signed_transfer_tx.to_dict()) +def test_validate_fails_metadata_empty_dict(create_tx): + create_tx.metadata = {'a': 1} + validate_transaction_schema(create_tx.to_dict()) + create_tx.metadata = None + validate_transaction_schema(create_tx.to_dict()) + create_tx.metadata = {} + with raises(SchemaValidationError): + validate_transaction_schema(create_tx.to_dict()) + + def test_validation_fails(): with raises(SchemaValidationError): validate_transaction_schema({}) diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 6f1bbb3f..5e341fcc 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -301,20 +301,18 @@ def test_transaction_serialization(user_ffill, user_cond, data, data_id): expected = { 'id': tx_id, 'version': Transaction.VERSION, - 'transaction': { - # NOTE: This test assumes that Fulfillments and Conditions can - # successfully be serialized - 'fulfillments': [user_ffill.to_dict()], - 'conditions': [user_cond.to_dict()], - 'operation': Transaction.CREATE, - 'metadata': None, - 'asset': { - 'id': data_id, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - } + # NOTE: This test assumes that Fulfillments and Conditions can + # successfully be serialized + 'fulfillments': [user_ffill.to_dict()], + 'conditions': [user_cond.to_dict()], + 'operation': Transaction.CREATE, + 'metadata': None, + 'asset': { + 'id': data_id, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, } } @@ -322,7 +320,7 @@ def test_transaction_serialization(user_ffill, user_cond, data, data_id): [user_cond]) tx_dict = tx.to_dict() tx_dict['id'] = tx_id - tx_dict['transaction']['asset']['id'] = data_id + tx_dict['asset']['id'] = data_id assert tx_dict == expected @@ -342,20 +340,18 @@ def test_transaction_deserialization(user_ffill, user_cond, data, uuid4): tx = { 'version': Transaction.VERSION, - 'transaction': { - # NOTE: This test assumes that Fulfillments and Conditions can - # successfully be serialized - 'fulfillments': [user_ffill.to_dict()], - 'conditions': [user_cond.to_dict()], - 'operation': Transaction.CREATE, - 'metadata': None, - 'asset': { - 'id': uuid4, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - } + # NOTE: This test assumes that Fulfillments and Conditions can + # successfully be serialized + 'fulfillments': [user_ffill.to_dict()], + 'conditions': [user_cond.to_dict()], + 'operation': Transaction.CREATE, + 'metadata': None, + 'asset': { + 'id': uuid4, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, } } tx_no_signatures = Transaction._remove_signatures(tx) @@ -387,34 +383,6 @@ def test_invalid_fulfillment_initialization(user_ffill, user_pub): Fulfillment(user_ffill, [], tx_input='somethingthatiswrong') -def test_invalid_metadata_initialization(): - from bigchaindb.common.transaction import Metadata - - with raises(TypeError): - Metadata([]) - - -def test_metadata_serialization(data, data_id): - from bigchaindb.common.transaction import Metadata - - expected = { - 'data': data, - 'id': data_id, - } - metadata = Metadata(data, data_id) - - assert metadata.to_dict() == expected - - -def test_metadata_deserialization(data, data_id): - from bigchaindb.common.transaction import Metadata - - expected = Metadata(data, data_id) - metadata = Metadata.from_dict({'data': data, 'id': data_id}) - - assert metadata == expected - - def test_transaction_link_serialization(): from bigchaindb.common.transaction import TransactionLink @@ -760,37 +728,32 @@ def test_create_create_transaction_single_io(user_cond, user_pub, data, uuid4): from .util import validate_transaction_model expected = { - 'transaction': { - 'conditions': [user_cond.to_dict()], - 'metadata': { - 'data': data, - }, - 'asset': { - 'id': uuid4, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - }, - 'fulfillments': [ - { - 'owners_before': [ - user_pub - ], - 'fulfillment': None, - 'input': None - } - ], - 'operation': 'CREATE', + 'conditions': [user_cond.to_dict()], + 'metadata': data, + 'asset': { + 'id': uuid4, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, }, + 'fulfillments': [ + { + 'owners_before': [ + user_pub + ], + 'fulfillment': None, + 'input': None + } + ], + 'operation': 'CREATE', 'version': 1, } asset = Asset(data, uuid4) tx = Transaction.create([user_pub], [([user_pub], 1)], data, asset) tx_dict = tx.to_dict() - tx_dict['transaction']['metadata'].pop('id') - tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None + tx_dict['fulfillments'][0]['fulfillment'] = None tx_dict.pop('id') assert tx_dict == expected @@ -815,16 +778,12 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, # weight = len(owners_before) ffill = Fulfillment.generate([user_pub, user2_pub]).to_dict() expected = { - 'transaction': { - 'conditions': [user_cond.to_dict(), user2_cond.to_dict()], - 'metadata': { - 'data': { - 'message': 'hello' - } - }, - 'fulfillments': [ffill], - 'operation': 'CREATE', + 'conditions': [user_cond.to_dict(), user2_cond.to_dict()], + 'metadata': { + 'message': 'hello' }, + 'fulfillments': [ffill], + 'operation': 'CREATE', 'version': 1 } asset = Asset(divisible=True) @@ -833,8 +792,7 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, asset=asset, metadata={'message': 'hello'}).to_dict() tx.pop('id') - tx['transaction']['metadata'].pop('id') - tx['transaction'].pop('asset') + tx.pop('asset') assert tx == expected @@ -861,29 +819,25 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, from bigchaindb.common.transaction import Transaction, Asset expected = { - 'transaction': { - 'conditions': [user_user2_threshold_cond.to_dict()], - 'metadata': { - 'data': data, - }, - 'asset': { - 'id': uuid4, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - }, - 'fulfillments': [ - { - 'owners_before': [ - user_pub, - ], - 'fulfillment': None, - 'input': None - }, - ], - 'operation': 'CREATE', + 'conditions': [user_user2_threshold_cond.to_dict()], + 'metadata': data, + 'asset': { + 'id': uuid4, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, }, + 'fulfillments': [ + { + 'owners_before': [ + user_pub, + ], + 'fulfillment': None, + 'input': None + }, + ], + 'operation': 'CREATE', 'version': 1 } asset = Asset(data, uuid4) @@ -891,8 +845,7 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, data, asset) tx_dict = tx.to_dict() tx_dict.pop('id') - tx_dict['transaction']['metadata'].pop('id') - tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None + tx_dict['fulfillments'][0]['fulfillment'] = None assert tx_dict == expected @@ -946,26 +899,24 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, from .util import validate_transaction_model expected = { - 'transaction': { - 'conditions': [user2_cond.to_dict()], - 'metadata': None, - 'asset': { - 'id': uuid4, - }, - 'fulfillments': [ - { - 'owners_before': [ - user_pub - ], - 'fulfillment': None, - 'input': { - 'txid': tx.id, - 'cid': 0 - } - } - ], - 'operation': 'TRANSFER', + 'conditions': [user2_cond.to_dict()], + 'metadata': None, + 'asset': { + 'id': uuid4, }, + 'fulfillments': [ + { + 'owners_before': [ + user_pub + ], + 'fulfillment': None, + 'input': { + 'txid': tx.id, + 'cid': 0 + } + } + ], + 'operation': 'TRANSFER', 'version': 1 } inputs = tx.to_inputs([0]) @@ -973,14 +924,13 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, transfer_tx = Transaction.transfer(inputs, [([user2_pub], 1)], asset=asset) transfer_tx = transfer_tx.sign([user_priv]) transfer_tx = transfer_tx.to_dict() - transfer_tx_body = transfer_tx['transaction'] expected_input = deepcopy(inputs[0]) expected['id'] = transfer_tx['id'] expected_input.fulfillment.sign(serialize(expected).encode(), PrivateKey(user_priv)) expected_ffill = expected_input.fulfillment.serialize_uri() - transfer_ffill = transfer_tx_body['fulfillments'][0]['fulfillment'] + transfer_ffill = transfer_tx['fulfillments'][0]['fulfillment'] assert transfer_ffill == expected_ffill @@ -1001,32 +951,30 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, tx = tx.sign([user_priv]) expected = { - 'transaction': { - 'conditions': [user2_cond.to_dict(), user2_cond.to_dict()], - 'metadata': None, - 'fulfillments': [ - { - 'owners_before': [ - user_pub - ], - 'fulfillment': None, - 'input': { - 'txid': tx.id, - 'cid': 0 - } - }, { - 'owners_before': [ - user2_pub - ], - 'fulfillment': None, - 'input': { - 'txid': tx.id, - 'cid': 1 - } + 'conditions': [user2_cond.to_dict(), user2_cond.to_dict()], + 'metadata': None, + 'fulfillments': [ + { + 'owners_before': [ + user_pub + ], + 'fulfillment': None, + 'input': { + 'txid': tx.id, + 'cid': 0 } - ], - 'operation': 'TRANSFER', - }, + }, { + 'owners_before': [ + user2_pub + ], + 'fulfillment': None, + 'input': { + 'txid': tx.id, + 'cid': 1 + } + } + ], + 'operation': 'TRANSFER', 'version': 1 } @@ -1041,10 +989,10 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, assert transfer_tx.fulfillments_valid(tx.conditions) is True transfer_tx = transfer_tx.to_dict() - transfer_tx['transaction']['fulfillments'][0]['fulfillment'] = None - transfer_tx['transaction']['fulfillments'][1]['fulfillment'] = None + transfer_tx['fulfillments'][0]['fulfillment'] = None + transfer_tx['fulfillments'][1]['fulfillment'] = None + transfer_tx.pop('asset') transfer_tx.pop('id') - transfer_tx['transaction'].pop('asset') assert expected == transfer_tx diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 6de68898..afe2bbd6 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -178,39 +178,6 @@ class TestBigchainApi(object): assert b.get_transaction(tx1.id) is None assert b.get_transaction(tx2.id) == tx2 - def test_get_transactions_for_metadata(self, b, user_pk): - from bigchaindb.models import Transaction - - metadata = {'msg': 'Hello BigchainDB!'} - tx = Transaction.create([b.me], [([user_pk], 1)], metadata=metadata) - - block = b.create_block([tx]) - b.write_block(block, durability='hard') - - matches = b.get_transaction_by_metadata_id(tx.metadata.data_id) - assert len(matches) == 1 - assert matches[0].id == tx.id - - @pytest.mark.usefixtures('inputs') - def test_get_transactions_for_metadata_invalid_block(self, b, user_pk): - from bigchaindb.models import Transaction - - metadata = {'msg': 'Hello BigchainDB!'} - tx = Transaction.create([b.me], [([user_pk], 1)], metadata=metadata) - - block = b.create_block([tx]) - b.write_block(block, durability='hard') - # vote block invalid - vote = b.vote(block.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - - matches = b.get_transaction_by_metadata_id(tx.metadata.data_id) - assert len(matches) == 0 - - def test_get_transactions_for_metadata_mismatch(self, b): - matches = b.get_transaction_by_metadata_id('missing') - assert not matches - @pytest.mark.usefixtures('inputs') def test_write_transaction(self, b, user_pk, user_sk): from bigchaindb.models import Transaction @@ -306,8 +273,8 @@ class TestBigchainApi(object): block = b.backend.get_genesis_block() assert len(block['block']['transactions']) == 1 - assert block['block']['transactions'][0]['transaction']['operation'] == 'GENESIS' - assert block['block']['transactions'][0]['transaction']['fulfillments'][0]['input'] is None + assert block['block']['transactions'][0]['operation'] == 'GENESIS' + assert block['block']['transactions'][0]['fulfillments'][0]['input'] is None def test_create_genesis_block_fails_if_table_not_empty(self, b): from bigchaindb.common.exceptions import GenesisBlockAlreadyExistsError @@ -646,7 +613,7 @@ class TestTransactionValidation(object): sleep(1) - signed_transfer_tx.metadata.data = {'different': 1} + signed_transfer_tx.metadata = {'different': 1} # FIXME: https://github.com/bigchaindb/bigchaindb/issues/592 with pytest.raises(DoubleSpend): b.validate_transaction(signed_transfer_tx) diff --git a/tests/db/test_utils.py b/tests/db/test_utils.py index dd8262a5..853604e9 100644 --- a/tests/db/test_utils.py +++ b/tests/db/test_utils.py @@ -77,8 +77,6 @@ def test_create_bigchain_secondary_index(): 'block_timestamp').run(conn) is True assert r.db(dbname).table('bigchain').index_list().contains( 'transaction_id').run(conn) is True - assert r.db(dbname).table('bigchain').index_list().contains( - 'metadata_id').run(conn) is True def test_create_backlog_table(): diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..5ac81cb3 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,35 @@ +import pytest +from bigchaindb.pipelines import block, election, vote, stale + +# TODO: fix this import madness +from ..db import conftest + + +@pytest.fixture(scope='module', autouse=True) +def restore_config(request, node_config): + from bigchaindb import config_utils + config_utils.set_config(node_config) + + +@pytest.fixture(scope='module', autouse=True) +def setup_database(request, node_config): + conftest.setup_database(request, node_config) + + +@pytest.fixture(scope='function', autouse=True) +def cleanup_tables(request, node_config): + conftest.cleanup_tables(request, node_config) + + +@pytest.fixture +def processes(b): + b.create_genesis_block() + block_maker = block.start() + voter = vote.start() + election_runner = election.start() + stale_monitor = stale.start() + yield + block_maker.terminate() + voter.terminate() + election_runner.terminate() + stale_monitor.terminate() diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 00000000..fee4817f --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,66 @@ +import time +import pytest + +from bigchaindb import Bigchain + + +@pytest.fixture +def inputs(user_pk): + from bigchaindb.models import Transaction + + b = Bigchain() + + # create blocks with transactions for `USER` to spend + for block in range(4): + transactions = [ + Transaction.create( + [b.me], [([user_pk], 1)], + metadata={'i': i}) + .sign([b.me_private]) + for i in range(10) + ] + block = b.create_block(transactions) + b.write_block(block, durability='hard') + + +@pytest.mark.usefixtures('processes') +def test_fast_double_create(b, user_pk): + from bigchaindb.models import Transaction + tx = Transaction.create([b.me], [([user_pk], 1)], + metadata={'test': 'test'}) \ + .sign([b.me_private]) + + # write everything fast + b.write_transaction(tx) + b.write_transaction(tx) + + time.sleep(2) + tx_returned = b.get_transaction(tx.id) + + # test that the tx can be queried + assert tx_returned == tx + # test the transaction appears only once + last_voted_block = b.get_last_voted_block() + assert len(last_voted_block.transactions) == 1 + assert b.backend.count_blocks() == 2 + + +@pytest.mark.usefixtures('processes') +def test_double_create(b, user_pk): + from bigchaindb.models import Transaction + tx = Transaction.create([b.me], [([user_pk], 1)], + metadata={'test': 'test'}) \ + .sign([b.me_private]) + + b.write_transaction(tx) + time.sleep(2) + b.write_transaction(tx) + time.sleep(2) + tx_returned = b.get_transaction(tx.id) + + # test that the tx can be queried + assert tx_returned == tx + # test the transaction appears only once + last_voted_block = b.get_last_voted_block() + assert len(last_voted_block.transactions) == 1 + assert b.backend.count_blocks() == 2 diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index ea0a0528..ac972b4c 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -121,7 +121,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname', 'BIGCHAINDB_DATABASE_PORT': '4242', - 'BIGCHAINDB_API_ENDPOINT': 'api://ipa', 'BIGCHAINDB_SERVER_BIND': '1.2.3.4:56', 'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'}) @@ -151,7 +150,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): 'port': 8125, 'rate': 0.01, }, - 'api_endpoint': 'api://ipa', 'backlog_reassign_delay': 5 } diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 00000000..037bf87c --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,16 @@ + +import subprocess + + +def test_build_server_docs(): + proc = subprocess.Popen(['bash'], stdin=subprocess.PIPE) + proc.stdin.write('cd docs/server; make html'.encode()) + proc.stdin.close() + assert proc.wait() == 0 + + +def test_build_root_docs(): + proc = subprocess.Popen(['bash'], stdin=subprocess.PIPE) + proc.stdin.write('cd docs/root; make html'.encode()) + proc.stdin.close() + assert proc.wait() == 0 diff --git a/tests/web/test_info.py b/tests/web/test_info.py index 759600c5..4956bd29 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -1,4 +1,4 @@ -def test_api_endpoint_shows_basic_info(client): +def test_api_root_url_shows_basic_info(client): from bigchaindb import version res = client.get('/') assert res.json['software'] == 'BigchainDB' diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index dd034c10..eed8a2b7 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -1,3 +1,4 @@ +import builtins import json import pytest @@ -33,11 +34,12 @@ def test_post_create_transaction_endpoint(b, client): tx = tx.sign([user_priv]) res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) - assert res.json['transaction']['fulfillments'][0]['owners_before'][0] == user_pub - assert res.json['transaction']['conditions'][0]['owners_after'][0] == user_pub + assert res.json['fulfillments'][0]['owners_before'][0] == user_pub + assert res.json['conditions'][0]['owners_after'][0] == user_pub def test_post_create_transaction_with_invalid_id(b, client): + from bigchaindb.common.exceptions import InvalidHash from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() @@ -47,18 +49,26 @@ def test_post_create_transaction_with_invalid_id(b, client): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 + err_msg = ("The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid.").format(tx['id']) + assert res.json['message'] == ( + 'Invalid transaction ({}): {}'.format(InvalidHash.__name__, err_msg)) def test_post_create_transaction_with_invalid_signature(b, client): + from bigchaindb.common.exceptions import InvalidSignature from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() tx = Transaction.create([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv]).to_dict() - tx['transaction']['fulfillments'][0]['fulfillment'] = 'cf:0:0' + tx['fulfillments'][0]['fulfillment'] = 'cf:0:0' res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 + assert res.json['message'] == ( + "Invalid transaction ({}): Fulfillment URI " + "couldn't been parsed".format(InvalidSignature.__name__)) def test_post_create_transaction_with_invalid_structure(client): @@ -66,6 +76,49 @@ def test_post_create_transaction_with_invalid_structure(client): assert res.status_code == 400 +def test_post_create_transaction_with_invalid_schema(client): + from bigchaindb.models import Transaction + user_priv, user_pub = crypto.generate_key_pair() + tx = Transaction.create( + [user_pub], [([user_pub], 1)]).sign([user_priv]).to_dict() + del tx['version'] + res = client.post(TX_ENDPOINT, data=json.dumps(tx)) + assert res.status_code == 400 + assert res.json['message'] == ( + "Invalid transaction schema: 'version' is a required property") + + +@pytest.mark.parametrize('exc,msg', ( + ('AmountError', 'Do the math again!'), + ('DoubleSpend', 'Nope! It is gone now!'), + ('InvalidHash', 'Do not smoke that!'), + ('InvalidSignature', 'Falsche Unterschrift!'), + ('OperationError', 'Create and transfer!'), + ('TransactionDoesNotExist', 'Hallucinations?'), + ('TransactionOwnerError', 'Not yours!'), + ('TransactionNotInValidBlock', 'Wait, maybe?'), + ('ValueError', '?'), +)) +def test_post_invalid_transaction(client, exc, msg, monkeypatch): + from bigchaindb.common import exceptions + try: + exc_cls = getattr(exceptions, exc) + except AttributeError: + exc_cls = getattr(builtins, 'ValueError') + + def mock_validation(self_, tx): + raise exc_cls(msg) + + monkeypatch.setattr( + 'bigchaindb.Bigchain.validate_transaction', mock_validation) + monkeypatch.setattr( + 'bigchaindb.models.Transaction.from_dict', lambda tx: None) + res = client.post(TX_ENDPOINT, data=json.dumps({})) + assert res.status_code == 400 + assert (res.json['message'] == + 'Invalid transaction ({}): {}'.format(exc, msg)) + + @pytest.mark.usefixtures('inputs') def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): sk, pk = crypto.generate_key_pair() @@ -81,8 +134,8 @@ def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx.to_dict())) - assert res.json['transaction']['fulfillments'][0]['owners_before'][0] == user_pk - assert res.json['transaction']['conditions'][0]['owners_after'][0] == user_pub + assert res.json['fulfillments'][0]['owners_before'][0] == user_pk + assert res.json['conditions'][0]['owners_after'][0] == user_pub @pytest.mark.usefixtures('inputs') diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..e56b091e --- /dev/null +++ b/tox.ini @@ -0,0 +1,42 @@ +[tox] +skipsdist = true +envlist = py34, py35, flake8, docsroot, docsserver + +[base] +basepython = python3.5 +deps = pip>=9.0.1 + +[testenv] +usedevelop = True +setenv = + PYTHONPATH = {toxinidir}:{toxinidir}/bigchaindb +deps = {[base]deps} +extras = test +commands = pytest -v -n auto --cov=bigchaindb --basetemp={envtmpdir} + +[testenv:flake8] +basepython = {[base]basepython} +deps = + {[base]deps} + flake8 +skip_install = True +extras = None +commands = flake8 --max-line-length 119 bigchaindb + +[testenv:docsroot] +basepython = {[base]basepython} +changedir = docs/root/source +deps = + {[base]deps} + -r{toxinidir}/docs/root/requirements.txt +extras = None +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html + +[testenv:docsserver] +basepython = {[base]basepython} +changedir = docs/server/source +deps = + {[base]deps} + -r{toxinidir}/docs/server/requirements.txt +extras = None +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html