From 5bf213d873d639708f332f0b5b111209f90c5379 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 15 Nov 2016 16:20:18 +0100 Subject: [PATCH 01/42] Removed api_endpoint from HTTP API docs & wrote how API Root URL is truly determined --- .../http-client-server-api.rst | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index f50b4f98..d5c2d872 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -26,17 +26,18 @@ details, see the "server" settings ("bind", "workers" and "threads") in <../server-reference/configuration>`. -API Root --------- +API Root URL +------------ -If you send an HTTP GET request to e.g. ``http://localhost:9984`` (with no -``/api/v1/`` on the end), then you should get an HTTP response with something -like the following in the body: +If you send an HTTP GET request to e.g. ``http://localhost:9984`` +or ``http://apihosting4u.net:9984`` +(with no ``/api/v1/`` on the end), +then you should get an HTTP response +with something like the following in the body: .. code-block:: json { - "api_endpoint": "http://localhost:9984/api/v1", "keyring": [ "6qHyZew94NMmUTYyHnkZsB8cxJYuRNEiEpXHe1ih9QX3", "AdDuyrTyjrDt935YnFu4VBCVDhHtY2Y6rcy7x2TFeiRi" @@ -46,6 +47,25 @@ like the following in the body: "version": "0.6.0" } +If the API endpoint is publicly-accessible, +then the public API Root URL is determined as follows: + +- The public IP address (like 12.34.56.78) + is the public IP address of the machine exposing + the HTTP API to the public internet (e.g. either the machine hosting + Gunicorn or the machine running the reverse proxy such as Nginx). + It's determined by AWS, Azure, Rackspace, or whoever is hosting the machine. + +- The DNS hostname (like apihosting4u.net) is determined by DNS records, + such as an "A Record" associating apihosting4u.net with 12.34.56.78 + +- The port (like 9984) is determined by the ``server.bind`` setting + if Gunicorn is exposed directly to the public Internet. + If a reverse proxy (like Nginx) is exposed directly to the public Internet + instead, then it could expose the HTTP API on whatever port it wants to. + (It should expose the HTTP API on port 9984, but it's not bound to do + that by anything other than convention.) + POST /transactions/ ------------------- From 29d783425ded487aec66d3a4f77c878735b4a2d2 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 15 Nov 2016 17:05:09 +0100 Subject: [PATCH 02/42] Remove or comment-out the api_endpoint setting everywhere --- Dockerfile | 4 +++- bigchaindb/__init__.py | 1 - bigchaindb/web/views/info.py | 3 +-- deploy-cluster-aws/awsdeploy.sh | 4 +--- deploy-cluster-aws/clusterize_confiles.py | 3 --- docker-compose.yml | 4 +++- .../source/dev-and-test/setup-run-node.md | 1 - docs/server/source/nodes/setup-run-node.md | 1 - .../source/server-reference/configuration.md | 21 ------------------- tests/test_config_utils.py | 2 -- tests/web/test_info.py | 2 +- 11 files changed, 9 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index fafa42ae..c4c30789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,9 @@ WORKDIR /data ENV BIGCHAINDB_CONFIG_PATH /data/.bigchaindb ENV BIGCHAINDB_SERVER_BIND 0.0.0.0:9984 -ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1 +# BigchainDB Server doesn't need BIGCHAINDB_API_ENDPOINT any more +# but maybe our Docker or Docker Compose stuff does? +# ENV BIGCHAINDB_API_ENDPOINT http://bigchaindb:9984/api/v1 ENTRYPOINT ["bigchaindb", "--dev-start-rethinkdb", "--dev-allow-temp-keypair"] diff --git a/bigchaindb/__init__.py b/bigchaindb/__init__.py index dc31b148..5055a5d6 100644 --- a/bigchaindb/__init__.py +++ b/bigchaindb/__init__.py @@ -29,7 +29,6 @@ config = { 'port': 8125, 'rate': 0.01, }, - 'api_endpoint': os.environ.get('BIGCHAINDB_API_ENDPOINT') or 'http://localhost:9984/api/v1', 'backlog_reassign_delay': 30 } diff --git a/bigchaindb/web/views/info.py b/bigchaindb/web/views/info.py index 1ff6d30c..874f899d 100644 --- a/bigchaindb/web/views/info.py +++ b/bigchaindb/web/views/info.py @@ -20,6 +20,5 @@ def home(): 'software': 'BigchainDB', 'version': version.__version__, 'public_key': bigchaindb.config['keypair']['public'], - 'keyring': bigchaindb.config['keyring'], - 'api_endpoint': bigchaindb.config['api_endpoint'] + 'keyring': bigchaindb.config['keyring'] }) diff --git a/deploy-cluster-aws/awsdeploy.sh b/deploy-cluster-aws/awsdeploy.sh index caed6c9c..f7345e1c 100755 --- a/deploy-cluster-aws/awsdeploy.sh +++ b/deploy-cluster-aws/awsdeploy.sh @@ -155,7 +155,7 @@ if [ "$WHAT_TO_DEPLOY" == "servers" ]; then # bigchaindb installed, so bigchaindb configure can't be called) # Transform the config files in the confiles directory - # to have proper keyrings, api_endpoint values, etc. + # to have proper keyrings etc. if [ "$USE_KEYPAIRS_FILE" == "True" ]; then python clusterize_confiles.py -k confiles $NUM_NODES else @@ -179,8 +179,6 @@ if [ "$WHAT_TO_DEPLOY" == "servers" ]; then echo "To start BigchainDB on all the nodes, do: fab start_bigchaindb" else # Deploying clients - # The only thing to configure on clients is the api_endpoint - # It should be the public DNS name of a BigchainDB server fab send_client_confile:client_confile # Start sending load from the clients to the servers diff --git a/deploy-cluster-aws/clusterize_confiles.py b/deploy-cluster-aws/clusterize_confiles.py index f266fe75..fbbf0dd6 100644 --- a/deploy-cluster-aws/clusterize_confiles.py +++ b/deploy-cluster-aws/clusterize_confiles.py @@ -98,9 +98,6 @@ for i, filename in enumerate(conf_files): # Allow incoming server traffic from any IP address # to port 9984 conf_dict['server']['bind'] = '0.0.0.0:9984' - # Set the api_endpoint - conf_dict['api_endpoint'] = 'http://' + public_dns_names[i] + \ - ':9984/api/v1' # Delete the config file os.remove(file_path) diff --git a/docker-compose.yml b/docker-compose.yml index c30b69b4..2b38a64e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,9 @@ services: - ./pytest.ini:/usr/src/app/pytest.ini environment: BIGCHAINDB_DATABASE_HOST: rdb - BIGCHAINDB_API_ENDPOINT: http://bdb:9984/api/v1 +# BigchainDB Server doesn't need BIGCHAINDB_API_ENDPOINT any more +# but maybe our Docker or Docker Compose stuff does? +# BIGCHAINDB_API_ENDPOINT: http://bdb:9984/api/v1 BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 ports: - "9984" diff --git a/docs/server/source/dev-and-test/setup-run-node.md b/docs/server/source/dev-and-test/setup-run-node.md index 0b55d543..e577a755 100644 --- a/docs/server/source/dev-and-test/setup-run-node.md +++ b/docs/server/source/dev-and-test/setup-run-node.md @@ -91,7 +91,6 @@ should give you something like: ```bash { - "api_endpoint": "http://bdb:9984/api/v1", "keyring": [], "public_key": "Brx8g4DdtEhccsENzNNV6yvQHR8s9ebhKyXPFkWUXh5e", "software": "BigchainDB", diff --git a/docs/server/source/nodes/setup-run-node.md b/docs/server/source/nodes/setup-run-node.md index 443f5ec9..6b8a29b7 100644 --- a/docs/server/source/nodes/setup-run-node.md +++ b/docs/server/source/nodes/setup-run-node.md @@ -167,7 +167,6 @@ Edit the created config file: * Open `$HOME/.bigchaindb` (the created config file) in your text editor. * Change `"server": {"bind": "localhost:9984", ... }` to `"server": {"bind": "0.0.0.0:9984", ... }`. This makes it so traffic can come from any IP address to port 9984 (the HTTP Client-Server API port). -* Change `"api_endpoint": "http://localhost:9984/api/v1"` to `"api_endpoint": "http://your_api_hostname:9984/api/v1"` * Change `"keyring": []` to `"keyring": ["public_key_of_other_node_A", "public_key_of_other_node_B", "..."]` i.e. a list of the public keys of all the other nodes in the federation. The keyring should _not_ include your node's public key. For more information about the BigchainDB config file, see [Configuring a BigchainDB Node](configuration.html). diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 41b0c2ca..57f5f4ed 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -17,7 +17,6 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_SERVER_BIND`
`BIGCHAINDB_SERVER_WORKERS`
`BIGCHAINDB_SERVER_THREADS`
-`BIGCHAINDB_API_ENDPOINT`
`BIGCHAINDB_STATSD_HOST`
`BIGCHAINDB_STATSD_PORT`
`BIGCHAINDB_STATSD_RATE`
@@ -140,26 +139,6 @@ export BIGCHAINDB_SERVER_THREADS=5 ``` -## api_endpoint - -`api_endpoint` is the URL where a BigchainDB client can get access to the HTTP client-server API. - -**Example using an environment variable** -```text -export BIGCHAINDB_API_ENDPOINT="http://localhost:9984/api/v1" -``` - -**Example config file snippet** -```js -"api_endpoint": "http://webserver.blocks587.net:9984/api/v1" -``` - -**Default value (from a config file)** -```js -"api_endpoint": "http://localhost:9984/api/v1" -``` - - ## statsd.host, statsd.port & statsd.rate These settings are used to configure where, and how often, [StatsD](https://github.com/etsy/statsd) should send data for [cluster monitoring](../clusters-feds/monitoring.html) purposes. `statsd.host` is the hostname of the monitoring server, where StatsD should send its data. `stats.port` is the port. `statsd.rate` is the fraction of transaction operations that should be sampled. It's a float between 0.0 and 1.0. diff --git a/tests/test_config_utils.py b/tests/test_config_utils.py index ea0a0528..ac972b4c 100644 --- a/tests/test_config_utils.py +++ b/tests/test_config_utils.py @@ -121,7 +121,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): monkeypatch.setattr('bigchaindb.config_utils.file_config', lambda *args, **kwargs: file_config) monkeypatch.setattr('os.environ', {'BIGCHAINDB_DATABASE_NAME': 'test-dbname', 'BIGCHAINDB_DATABASE_PORT': '4242', - 'BIGCHAINDB_API_ENDPOINT': 'api://ipa', 'BIGCHAINDB_SERVER_BIND': '1.2.3.4:56', 'BIGCHAINDB_KEYRING': 'pubkey_0:pubkey_1:pubkey_2'}) @@ -151,7 +150,6 @@ def test_autoconfigure_read_both_from_file_and_env(monkeypatch): 'port': 8125, 'rate': 0.01, }, - 'api_endpoint': 'api://ipa', 'backlog_reassign_delay': 5 } diff --git a/tests/web/test_info.py b/tests/web/test_info.py index 759600c5..4956bd29 100644 --- a/tests/web/test_info.py +++ b/tests/web/test_info.py @@ -1,4 +1,4 @@ -def test_api_endpoint_shows_basic_info(client): +def test_api_root_url_shows_basic_info(client): from bigchaindb import version res = client.get('/') assert res.json['software'] == 'BigchainDB' From 5fa25c64eed587850c503b8120a774d64a3bc206 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 25 Nov 2016 14:34:49 +0100 Subject: [PATCH 03/42] First draft of CHANGELOG.md updated for v0.8.0 --- CHANGELOG.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f0841d..be7eb85a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Change Log (Release Notes) + All _notable_ changes to this project will be documented in this file (`CHANGELOG.md`). This project adheres to [Semantic Versioning](http://semver.org/) (or at least we try). Contributors to this file, please follow the guidelines on [keepachangelog.com](http://keepachangelog.com/). @@ -15,10 +16,68 @@ For reference, the possible headings are: * **Notes** +## [0.8.0] - 2016-11-25 +Tag name: v0.8.0 += commit: +committed: + +### Added +- The big new thing in version 0.8.0 is support for divisible assets, i.e. assets like carrots or thumbtacks, where the initial CREATE transaction can register/create some amount (e.g. 542 carrots), the first TRANSFER transaction can split that amount across multiple owners, and so on. [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) +- Wrote a formal schema for the JSON structure of transactions. Transactions are now checked against that schema. [Pull Request #798](https://github.com/bigchaindb/bigchaindb/pull/798) + +### Changed +- CREATE transactions must now be signed by all `owners_before` (rather than by a federation node). [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) +- The user-provided timestamp was removed from the transaction data model (schema). [Pull Request #817](https://github.com/bigchaindb/bigchaindb/pull/817) +- `get_transaction()` will now return a transaction from the backlog, even if there are copies of the transaction in invalid blocks. [Pull Request #793](https://github.com/bigchaindb/bigchaindb/pull/793) +- Several pull requests to translate RethinkDB database calls into abstract/generic database calls, and to implement those abstract calls for two backends: RethinkDB and MongoDB. Pull Requests +[#754](https://github.com/bigchaindb/bigchaindb/pull/754), +[#799](https://github.com/bigchaindb/bigchaindb/pull/799), +[#806](https://github.com/bigchaindb/bigchaindb/pull/806), +[#809](https://github.com/bigchaindb/bigchaindb/pull/809), +[#853](https://github.com/bigchaindb/bigchaindb/pull/853) +- Renamed "verifying key" to "public key". Renamed "signing key" to "private key". Renamed "vk" to "pk". [Pull Request #807](https://github.com/bigchaindb/bigchaindb/pull/807) +- `get_transaction_by_asset_id` now ignores invalid transactions. [Pull Request #810](https://github.com/bigchaindb/bigchaindb/pull/810) +- `get_transaction_by_metadata_id` now ignores invalid transactions. [Pull Request #811](https://github.com/bigchaindb/bigchaindb/pull/811) +- Updates to the configs and scripts for deploying a test network on AWS. The example config file deploys virtual machines running Ubuntu 16.04 now. Pull Requests +[#771](https://github.com/bigchaindb/bigchaindb/pull/771), +[#813](https://github.com/bigchaindb/bigchaindb/pull/813) +- Changed logging of transactions on block creation so now it just says the length of the list of transactions, rather than listing all the transactions. [Pull Request #861](https://github.com/bigchaindb/bigchaindb/pull/861) + +### Fixed +- Equality checks with AssetLinks. [Pull Request #825](https://github.com/bigchaindb/bigchaindb/pull/825) +- Bug in `bigchaindb load`. [Pull Request #824](https://github.com/bigchaindb/bigchaindb/pull/824) +- Two issus found with timestamp indexes. [Pull Request #816](https://github.com/bigchaindb/bigchaindb/pull/816) +- Hard-coded `backlog_reassign_delay`. [Pull Request #854](https://github.com/bigchaindb/bigchaindb/pull/854) +- Race condition in `test_stale_monitor.py`. [Pull Request #846](https://github.com/bigchaindb/bigchaindb/pull/846) + +### External Contributors +- @najlachamseddine - [Pull Request #528](https://github.com/bigchaindb/bigchaindb/pull/528) +- @ChristianGaertner - [Pull Request #659](https://github.com/bigchaindb/bigchaindb/pull/659) +- @MinchinWeb - [Pull Request #695](https://github.com/bigchaindb/bigchaindb/pull/695) +- @ckeyer - [Pull Request #785](https://github.com/bigchaindb/bigchaindb/pull/785) + +### Notes +- @ChristianGaertner added a Python style checker (Flake8) to Travis CI, so external contributors should be aware that the Python code in their pull requests will be checked. See [our Python Style Guide](PYTHON_STYLE_GUIDE.md). +- Several additions and changes to the documentation, e.g. Pull Requests +[#690](https://github.com/bigchaindb/bigchaindb/pull/690), +[#764](https://github.com/bigchaindb/bigchaindb/pull/764), +[#766](https://github.com/bigchaindb/bigchaindb/pull/766), +[#769](https://github.com/bigchaindb/bigchaindb/pull/769), +[#777](https://github.com/bigchaindb/bigchaindb/pull/777), +[#800](https://github.com/bigchaindb/bigchaindb/pull/800), +[#801](https://github.com/bigchaindb/bigchaindb/pull/801), +[#802](https://github.com/bigchaindb/bigchaindb/pull/802), +[#803](https://github.com/bigchaindb/bigchaindb/pull/803), +[#819](https://github.com/bigchaindb/bigchaindb/pull/819), +[#827](https://github.com/bigchaindb/bigchaindb/pull/827), +[#859](https://github.com/bigchaindb/bigchaindb/pull/859) + + + ## [0.7.0] - 2016-10-28 Tag name: v0.7.0 -= commit: -committed: += commit: 2dd7f1af27478c529e6d2d916f64daa3fbda3885 +committed: Oct 28, 2016, 4:00 PM GMT+2 ### Added - Stale transactions in the `backlog` table now get reassigned to another node (for inclusion in a new block): [Pull Request #359](https://github.com/bigchaindb/bigchaindb/pull/359) From bd076cb1509320fd1ef4f85a7805bee3e9071814 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 25 Nov 2016 18:11:51 +0100 Subject: [PATCH 04/42] Updated changelog draft with PR #869 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be7eb85a..a823ba44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ For reference, the possible headings are: * **Notes** -## [0.8.0] - 2016-11-25 +## [0.8.0] - 2016-11-?? Tag name: v0.8.0 = commit: committed: @@ -49,6 +49,7 @@ committed: - Two issus found with timestamp indexes. [Pull Request #816](https://github.com/bigchaindb/bigchaindb/pull/816) - Hard-coded `backlog_reassign_delay`. [Pull Request #854](https://github.com/bigchaindb/bigchaindb/pull/854) - Race condition in `test_stale_monitor.py`. [Pull Request #846](https://github.com/bigchaindb/bigchaindb/pull/846) +- When creating a signed vote, decode the vote signature to a `str`. [Pull Request #869](https://github.com/bigchaindb/bigchaindb/pull/869) ### External Contributors - @najlachamseddine - [Pull Request #528](https://github.com/bigchaindb/bigchaindb/pull/528) From 53ac0e1b45a5830aaaa6ec8cf1b9a3dab330c730 Mon Sep 17 00:00:00 2001 From: troymc Date: Fri, 25 Nov 2016 20:10:06 +0100 Subject: [PATCH 05/42] Addressed comments from @libscott and @sbellem --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a823ba44..5caffaec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ committed: - CREATE transactions must now be signed by all `owners_before` (rather than by a federation node). [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) - The user-provided timestamp was removed from the transaction data model (schema). [Pull Request #817](https://github.com/bigchaindb/bigchaindb/pull/817) - `get_transaction()` will now return a transaction from the backlog, even if there are copies of the transaction in invalid blocks. [Pull Request #793](https://github.com/bigchaindb/bigchaindb/pull/793) -- Several pull requests to translate RethinkDB database calls into abstract/generic database calls, and to implement those abstract calls for two backends: RethinkDB and MongoDB. Pull Requests +- Several pull requests to introduce a generalized database interface, to move RethinkDB calls into a separate implementation of that interface, and to work on a new MongoDB implementation of that interface. Pull Requests [#754](https://github.com/bigchaindb/bigchaindb/pull/754), [#799](https://github.com/bigchaindb/bigchaindb/pull/799), [#806](https://github.com/bigchaindb/bigchaindb/pull/806), @@ -46,7 +46,7 @@ committed: ### Fixed - Equality checks with AssetLinks. [Pull Request #825](https://github.com/bigchaindb/bigchaindb/pull/825) - Bug in `bigchaindb load`. [Pull Request #824](https://github.com/bigchaindb/bigchaindb/pull/824) -- Two issus found with timestamp indexes. [Pull Request #816](https://github.com/bigchaindb/bigchaindb/pull/816) +- Two issues found with timestamp indexes. [Pull Request #816](https://github.com/bigchaindb/bigchaindb/pull/816) - Hard-coded `backlog_reassign_delay`. [Pull Request #854](https://github.com/bigchaindb/bigchaindb/pull/854) - Race condition in `test_stale_monitor.py`. [Pull Request #846](https://github.com/bigchaindb/bigchaindb/pull/846) - When creating a signed vote, decode the vote signature to a `str`. [Pull Request #869](https://github.com/bigchaindb/bigchaindb/pull/869) From 71dcee019f7f7dc87fd328427d21bcfc064b3e12 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 23 Nov 2016 10:40:48 +0100 Subject: [PATCH 06/42] Remove metadata uuid --- bigchaindb/common/schema/transaction.yaml | 17 +---- bigchaindb/common/transaction.py | 81 ++--------------------- bigchaindb/core.py | 29 -------- bigchaindb/db/backends/rethinkdb.py | 26 -------- bigchaindb/db/utils.py | 5 -- docs/server/source/schema/transaction.rst | 23 +------ tests/common/conftest.py | 6 -- tests/common/test_transaction.py | 43 +----------- tests/db/test_bigchain_api.py | 35 +--------- tests/db/test_utils.py | 2 - 10 files changed, 15 insertions(+), 252 deletions(-) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index f1864cc3..780f5408 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -60,7 +60,7 @@ properties: "$ref": "#/definitions/metadata" description: | User provided transaction metadata. This field may be ``null`` or may - contain an id and an object with freeform metadata. + contain an object with freeform metadata. See: `Metadata`_. version: @@ -245,17 +245,6 @@ definitions: - type: object description: | User provided transaction metadata. This field may be ``null`` or may - contain an id and an object with freeform metadata. - additionalProperties: false - required: - - id - - data - properties: - id: - "$ref": "#/definitions/uuid4" - data: - type: object - description: | - User provided transaction metadata. - additionalProperties: true + contain an object with freeform metadata. + additionalProperties: true - type: 'null' diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 6a7ec629..6fa1aa2b 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -572,66 +572,6 @@ class AssetLink(Asset): } -class Metadata(object): - """Metadata is used to store a dictionary and its hash in a Transaction.""" - - def __init__(self, data=None, data_id=None): - """Metadata stores a payload `data` as well as data's hash, `data_id`. - - Note: - When no `data_id` is provided, one is being generated by - this method. - - Args: - data (dict): A dictionary to be held by Metadata. - data_id (str): A hash corresponding to the contents of - `data`. - """ - if data is not None and not isinstance(data, dict): - raise TypeError('`data` must be a dict instance or None') - - self.data_id = data_id if data_id is not None else self.to_hash() - self.data = data - - def __eq__(self, other): - # TODO: If `other !== Data` return `False` - return self.to_dict() == other.to_dict() - - @classmethod - def from_dict(cls, data): - """Transforms a Python dictionary to a Metadata object. - - Args: - data (dict): The dictionary to be serialized. - - Returns: - :class:`~bigchaindb.common.transaction.Metadata` - """ - try: - return cls(data['data'], data['id']) - except TypeError: - return cls() - - def to_dict(self): - """Transforms the object to a Python dictionary. - - Returns: - (dict|None): The Metadata object as an alternative - serialization format. - """ - if self.data is None: - return None - else: - return { - 'data': self.data, - 'id': self.data_id, - } - - def to_hash(self): - """A hash corresponding to the contents of `payload`.""" - return str(uuid4()) - - class Transaction(object): """A Transaction is used to create and transfer assets. @@ -646,7 +586,7 @@ class Transaction(object): spend. conditions (:obj:`list` of :class:`~bigchaindb.common. transaction.Condition`, optional): Define the assets to lock. - metadata (:class:`~bigchaindb.common.transaction.Metadata`): + metadata (dict): Metadata to be stored along with the Transaction. version (int): Defines the version number of a Transaction. """ @@ -674,7 +614,7 @@ class Transaction(object): conditions (:obj:`list` of :class:`~bigchaindb.common. transaction.Condition`, optional): Define the assets to lock. - metadata (:class:`~bigchaindb.common.transaction.Metadata`): + metadata (dict): Metadata to be stored along with the Transaction. version (int): Defines the version number of a Transaction. @@ -695,8 +635,8 @@ class Transaction(object): if fulfillments and not isinstance(fulfillments, list): raise TypeError('`fulfillments` must be a list instance or None') - if metadata is not None and not isinstance(metadata, Metadata): - raise TypeError('`metadata` must be a Metadata instance or None') + if metadata is not None and not isinstance(metadata, dict): + raise TypeError('`metadata` must be a dict or None') self.version = version if version is not None else self.VERSION self.operation = operation @@ -750,7 +690,6 @@ class Transaction(object): if len(owners_after) == 0: raise ValueError('`owners_after` list cannot be empty') - metadata = Metadata(metadata) fulfillments = [] conditions = [] @@ -825,7 +764,6 @@ class Transaction(object): pub_keys, amount = owner_after conditions.append(Condition.generate(pub_keys, amount)) - metadata = Metadata(metadata) inputs = deepcopy(inputs) return cls(cls.TRANSFER, asset, inputs, conditions, metadata) @@ -1166,12 +1104,6 @@ class Transaction(object): Returns: dict: The Transaction as an alternative serialization format. """ - try: - metadata = self.metadata.to_dict() - except AttributeError: - # NOTE: metadata can be None and that's OK - metadata = None - if self.operation in (self.__class__.GENESIS, self.__class__.CREATE): asset = self.asset.to_dict() else: @@ -1184,7 +1116,7 @@ class Transaction(object): 'conditions': [condition.to_dict(cid) for cid, condition in enumerate(self.conditions)], 'operation': str(self.operation), - 'metadata': metadata, + 'metadata': self.metadata, 'asset': asset, } tx = { @@ -1279,11 +1211,10 @@ class Transaction(object): in tx['fulfillments']] conditions = [Condition.from_dict(condition) for condition in tx['conditions']] - metadata = Metadata.from_dict(tx['metadata']) if tx['operation'] in [cls.CREATE, cls.GENESIS]: asset = Asset.from_dict(tx['asset']) else: asset = AssetLink.from_dict(tx['asset']) return cls(tx['operation'], asset, fulfillments, conditions, - metadata, tx_body['version']) + tx['metadata'], tx_body['version']) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 653b7ac3..6fd892e7 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -330,35 +330,6 @@ class Bigchain(object): else: return None - def get_transaction_by_metadata_id(self, metadata_id): - """Retrieves valid or undecided transactions related to a particular - metadata. - - When creating a transaction one of the optional arguments is the - `metadata`. The metadata is a generic dict that contains extra - information that can be appended to the transaction. - - To make it easy to query the bigchain for that particular metadata we - create a UUID for the metadata and store it with the transaction. - - Args: - metadata_id (str): the id for this particular metadata. - - Returns: - A list of valid or undecided transactions containing that metadata. - If no transaction exists with that metadata it returns an empty - list `[]` - """ - txids = self.backend.get_txids_by_metadata_id(metadata_id) - transactions = [] - for txid in txids: - tx = self.get_transaction(txid) - # if a valid or undecided transaction exists append it to the list - # of transactions - if tx: - transactions.append(tx) - return transactions - def get_transactions_by_asset_id(self, asset_id): """Retrieves valid or undecided transactions related to a particular asset. diff --git a/bigchaindb/db/backends/rethinkdb.py b/bigchaindb/db/backends/rethinkdb.py index f01bdb4a..87b09dd2 100644 --- a/bigchaindb/db/backends/rethinkdb.py +++ b/bigchaindb/db/backends/rethinkdb.py @@ -138,32 +138,6 @@ class RethinkDBBackend: .get_all(transaction_id, index='transaction_id') .pluck('votes', 'id', {'block': ['voters']})) - def get_txids_by_metadata_id(self, metadata_id): - """Retrieves transaction ids related to a particular metadata. - - When creating a transaction one of the optional arguments is the - `metadata`. The metadata is a generic dict that contains extra - information that can be appended to the transaction. - - To make it easy to query the bigchain for that particular metadata we - create a UUID for the metadata and store it with the transaction. - - Args: - metadata_id (str): the id for this particular metadata. - - Returns: - A list of transaction ids containing that metadata. If no - transaction exists with that metadata it returns an empty list `[]` - """ - return self.connection.run( - r.table('bigchain', read_mode=self.read_mode) - .get_all(metadata_id, index='metadata_id') - .concat_map(lambda block: block['block']['transactions']) - .filter(lambda transaction: - transaction['transaction']['metadata']['id'] == - metadata_id) - .get_field('id')) - def get_txids_by_asset_id(self, asset_id): """Retrieves transactions ids related to a particular asset. diff --git a/bigchaindb/db/utils.py b/bigchaindb/db/utils.py index 05932f77..cbcebc85 100644 --- a/bigchaindb/db/utils.py +++ b/bigchaindb/db/utils.py @@ -116,11 +116,6 @@ def create_bigchain_secondary_index(conn, dbname): .index_create('transaction_id', r.row['block']['transactions']['id'], multi=True)\ .run(conn) - # secondary index for payload data by UUID - r.db(dbname).table('bigchain')\ - .index_create('metadata_id', - r.row['block']['transactions']['transaction']['metadata']['id'], multi=True)\ - .run(conn) # secondary index for asset uuid r.db(dbname).table('bigchain')\ .index_create('asset_id', diff --git a/docs/server/source/schema/transaction.rst b/docs/server/source/schema/transaction.rst index 766e0c4a..f2cabf59 100644 --- a/docs/server/source/schema/transaction.rst +++ b/docs/server/source/schema/transaction.rst @@ -136,7 +136,7 @@ Transaction.metadata **type:** object or null User provided transaction metadata. This field may be ``null`` or may -contain an id and an object with freeform metadata. +contain an object with freeform metadata. See: `Metadata`_. @@ -301,26 +301,7 @@ Metadata -------- User provided transaction metadata. This field may be ``null`` or may -contain an id and an object with freeform metadata. - - -Metadata.id -^^^^^^^^^^^ - -**type:** string - -A `UUID `_ -of type 4 (random). - - - -Metadata.data -^^^^^^^^^^^^^ - -**type:** object - -User provided transaction metadata. - +contain an object with freeform metadata. diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 1e5fe7d3..a54daf20 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -136,12 +136,6 @@ def uuid4(): return UUID4 -@pytest.fixture -def metadata(data, data_id): - from bigchaindb.common.transaction import Metadata - return Metadata(data, data_id) - - @pytest.fixture def utx(user_ffill, user_cond): from bigchaindb.common.transaction import Transaction, Asset diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index e6a4ac5f..335fd0a2 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -387,34 +387,6 @@ def test_invalid_fulfillment_initialization(user_ffill, user_pub): Fulfillment(user_ffill, [], tx_input='somethingthatiswrong') -def test_invalid_metadata_initialization(): - from bigchaindb.common.transaction import Metadata - - with raises(TypeError): - Metadata([]) - - -def test_metadata_serialization(data, data_id): - from bigchaindb.common.transaction import Metadata - - expected = { - 'data': data, - 'id': data_id, - } - metadata = Metadata(data, data_id) - - assert metadata.to_dict() == expected - - -def test_metadata_deserialization(data, data_id): - from bigchaindb.common.transaction import Metadata - - expected = Metadata(data, data_id) - metadata = Metadata.from_dict({'data': data, 'id': data_id}) - - assert metadata == expected - - def test_transaction_link_serialization(): from bigchaindb.common.transaction import TransactionLink @@ -762,9 +734,7 @@ def test_create_create_transaction_single_io(user_cond, user_pub, data, uuid4): expected = { 'transaction': { 'conditions': [user_cond.to_dict(0)], - 'metadata': { - 'data': data, - }, + 'metadata': data, 'asset': { 'id': uuid4, 'divisible': False, @@ -790,7 +760,6 @@ def test_create_create_transaction_single_io(user_cond, user_pub, data, uuid4): asset = Asset(data, uuid4) tx = Transaction.create([user_pub], [([user_pub], 1)], data, asset) tx_dict = tx.to_dict() - tx_dict['transaction']['metadata'].pop('id') tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None tx_dict.pop('id') @@ -820,9 +789,7 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, 'transaction': { 'conditions': [user_cond.to_dict(0), user2_cond.to_dict(1)], 'metadata': { - 'data': { - 'message': 'hello' - } + 'message': 'hello' }, 'fulfillments': [ffill], 'operation': 'CREATE', @@ -835,7 +802,6 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, asset=asset, metadata={'message': 'hello'}).to_dict() tx.pop('id') - tx['transaction']['metadata'].pop('id') tx['transaction'].pop('asset') assert tx == expected @@ -865,9 +831,7 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, expected = { 'transaction': { 'conditions': [user_user2_threshold_cond.to_dict(0)], - 'metadata': { - 'data': data, - }, + 'metadata': data, 'asset': { 'id': uuid4, 'divisible': False, @@ -894,7 +858,6 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, data, asset) tx_dict = tx.to_dict() tx_dict.pop('id') - tx_dict['transaction']['metadata'].pop('id') tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None assert tx_dict == expected diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index 6de68898..cd030249 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -178,39 +178,6 @@ class TestBigchainApi(object): assert b.get_transaction(tx1.id) is None assert b.get_transaction(tx2.id) == tx2 - def test_get_transactions_for_metadata(self, b, user_pk): - from bigchaindb.models import Transaction - - metadata = {'msg': 'Hello BigchainDB!'} - tx = Transaction.create([b.me], [([user_pk], 1)], metadata=metadata) - - block = b.create_block([tx]) - b.write_block(block, durability='hard') - - matches = b.get_transaction_by_metadata_id(tx.metadata.data_id) - assert len(matches) == 1 - assert matches[0].id == tx.id - - @pytest.mark.usefixtures('inputs') - def test_get_transactions_for_metadata_invalid_block(self, b, user_pk): - from bigchaindb.models import Transaction - - metadata = {'msg': 'Hello BigchainDB!'} - tx = Transaction.create([b.me], [([user_pk], 1)], metadata=metadata) - - block = b.create_block([tx]) - b.write_block(block, durability='hard') - # vote block invalid - vote = b.vote(block.id, b.get_last_voted_block().id, False) - b.write_vote(vote) - - matches = b.get_transaction_by_metadata_id(tx.metadata.data_id) - assert len(matches) == 0 - - def test_get_transactions_for_metadata_mismatch(self, b): - matches = b.get_transaction_by_metadata_id('missing') - assert not matches - @pytest.mark.usefixtures('inputs') def test_write_transaction(self, b, user_pk, user_sk): from bigchaindb.models import Transaction @@ -646,7 +613,7 @@ class TestTransactionValidation(object): sleep(1) - signed_transfer_tx.metadata.data = {'different': 1} + signed_transfer_tx.metadata = {'different': 1} # FIXME: https://github.com/bigchaindb/bigchaindb/issues/592 with pytest.raises(DoubleSpend): b.validate_transaction(signed_transfer_tx) diff --git a/tests/db/test_utils.py b/tests/db/test_utils.py index dd8262a5..853604e9 100644 --- a/tests/db/test_utils.py +++ b/tests/db/test_utils.py @@ -77,8 +77,6 @@ def test_create_bigchain_secondary_index(): 'block_timestamp').run(conn) is True assert r.db(dbname).table('bigchain').index_list().contains( 'transaction_id').run(conn) is True - assert r.db(dbname).table('bigchain').index_list().contains( - 'metadata_id').run(conn) is True def test_create_backlog_table(): From a6eb52d76dc952cd7d4329bfcc99797229138566 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Mon, 28 Nov 2016 17:31:35 +0100 Subject: [PATCH 07/42] disallow empty metadata dict in favour of null --- bigchaindb/common/schema/transaction.yaml | 3 ++- docs/server/source/schema/transaction.rst | 2 +- tests/common/test_schema.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index 780f5408..03f93ac7 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -245,6 +245,7 @@ definitions: - type: object description: | User provided transaction metadata. This field may be ``null`` or may - contain an object with freeform metadata. + contain an non empty object with freeform metadata. additionalProperties: true + minProperties: 1 - type: 'null' diff --git a/docs/server/source/schema/transaction.rst b/docs/server/source/schema/transaction.rst index f2cabf59..c79abf05 100644 --- a/docs/server/source/schema/transaction.rst +++ b/docs/server/source/schema/transaction.rst @@ -301,7 +301,7 @@ Metadata -------- User provided transaction metadata. This field may be ``null`` or may -contain an object with freeform metadata. +contain an non empty object with freeform metadata. diff --git a/tests/common/test_schema.py b/tests/common/test_schema.py index 5ded0272..1827d3cb 100644 --- a/tests/common/test_schema.py +++ b/tests/common/test_schema.py @@ -16,6 +16,16 @@ def test_validate_transaction_signed_transfer(signed_transfer_tx): validate_transaction_schema(signed_transfer_tx.to_dict()) +def test_validate_fails_metadata_empty_dict(create_tx): + create_tx.metadata = {'a': 1} + validate_transaction_schema(create_tx.to_dict()) + create_tx.metadata = None + validate_transaction_schema(create_tx.to_dict()) + create_tx.metadata = {} + with raises(SchemaValidationError): + validate_transaction_schema(create_tx.to_dict()) + + def test_validation_fails(): with raises(SchemaValidationError): validate_transaction_schema({}) From 536d4408370514a61db2563a1227148842cd7d59 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Tue, 29 Nov 2016 11:42:13 +0100 Subject: [PATCH 08/42] Fix HTTP API docs for upcoming release --- .../http-client-server-api.rst | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index b38852fc..8cfd7c63 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -54,12 +54,12 @@ POST /transactions/ Push a new transaction. - Note: The posted transaction should be valid `transaction + Note: The posted transaction should be a valid and signed `transaction `_. The steps to build a valid transaction are beyond the scope of this page. One would normally use a driver such as the `BigchainDB Python Driver `_ to - build a valid transaction. + build a valid transaction for a public/private keypair. **Example request**: @@ -75,18 +75,18 @@ POST /transactions/ { "cid": 0, "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "uri": "cc:4:20:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFY:96", "details": { "signature": null, "type": "fulfillment", "type_id": 4, "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "public_key": "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" } }, "amount": 1, "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" ] } ], @@ -95,7 +95,7 @@ POST /transactions/ "divisible": false, "updatable": false, "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", + "id": "b57801f8-b865-4360-9d1a-3e3009f5ce01", "refillable": false }, "metadata": null, @@ -103,14 +103,14 @@ POST /transactions/ { "fid": 0, "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "fulfillment": "cf:4:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFaf8bQVH1gesZGEGZepCE8_kgo-UfBrCHPlvBsnAsfq56GWjrLTyZ9NXISwcyJ3zmygnVhCMG8xzE6c9fj1-6wK", "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" ] } ] }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "id": "65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3", "version": 1 } @@ -122,24 +122,24 @@ POST /transactions/ Content-Type: application/json { - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "id": "65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3", "version": 1, "transaction": { "conditions": [ { "amount": 1, "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "uri": "cc:4:20:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFY:96", "details": { "signature": null, "type_id": 4, "type": "fulfillment", "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "public_key": "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" } }, "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" ], "cid": 0 } @@ -147,10 +147,10 @@ POST /transactions/ "fulfillments": [ { "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "fulfillment": "cf:4:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFaf8bQVH1gesZGEGZepCE8_kgo-UfBrCHPlvBsnAsfq56GWjrLTyZ9NXISwcyJ3zmygnVhCMG8xzE6c9fj1-6wK", "fid": 0, "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" ] } ], @@ -160,7 +160,7 @@ POST /transactions/ "refillable": false, "divisible": false, "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d" + "id": "b57801f8-b865-4360-9d1a-3e3009f5ce01" }, "metadata": null } @@ -188,7 +188,7 @@ GET /transactions/{tx_id}/status .. sourcecode:: http - GET /transactions/7ad5a4b83bc8c70c4fd7420ff3c60693ab8e6d0e3124378ca69ed5acd2578792/status HTTP/1.1 + GET /transactions/65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3/status HTTP/1.1 Host: example.com **Example response**: @@ -223,7 +223,7 @@ GET /transactions/{tx_id} .. sourcecode:: http - GET /transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e HTTP/1.1 + GET /transactions/65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3 HTTP/1.1 Host: example.com **Example response**: @@ -239,18 +239,18 @@ GET /transactions/{tx_id} { "cid": 0, "condition": { - "uri": "cc:4:20:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkk:96", + "uri": "cc:4:20:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFY:96", "details": { "signature": null, "type": "fulfillment", "type_id": 4, "bitmask": 32, - "public_key": "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "public_key": "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" } }, "amount": 1, "owners_after": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" ] } ], @@ -259,7 +259,7 @@ GET /transactions/{tx_id} "divisible": false, "updatable": false, "data": null, - "id": "aebeab22-e672-4d3b-a187-bde5fda6533d", + "id": "b57801f8-b865-4360-9d1a-3e3009f5ce01", "refillable": false }, "metadata": null, @@ -267,14 +267,14 @@ GET /transactions/{tx_id} { "fid": 0, "input": null, - "fulfillment": "cf:4:GG-pi3CeIlySZhQoJVBh9O23PzrOuhnYI7OHqIbHjkn2VnQaEWvecO1x82Qr2Va_JjFywLKIOEV1Ob9Ofkeln2K89ny2mB-s7RLNvYAVzWNiQnp18_nQEUsvwACEXTYJ", + "fulfillment": "cf:4:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFaf8bQVH1gesZGEGZepCE8_kgo-UfBrCHPlvBsnAsfq56GWjrLTyZ9NXISwcyJ3zmygnVhCMG8xzE6c9fj1-6wK", "owners_before": [ - "2ePYHfV3yS3xTxF9EE3Xjo8zPwq2RmLPFAJGQqQKc3j6" + "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" ] } ] }, - "id": "2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e", + "id": "65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3", "version": 1 } From 689446c6a49e75e0c7d403f4854d457698697c72 Mon Sep 17 00:00:00 2001 From: Ryan Henderson Date: Tue, 29 Nov 2016 13:56:02 +0100 Subject: [PATCH 09/42] add backlog reassign delay documentation (#883) --- .../source/server-reference/configuration.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/server/source/server-reference/configuration.md b/docs/server/source/server-reference/configuration.md index 41b0c2ca..106b62ca 100644 --- a/docs/server/source/server-reference/configuration.md +++ b/docs/server/source/server-reference/configuration.md @@ -22,6 +22,7 @@ For convenience, here's a list of all the relevant environment variables (docume `BIGCHAINDB_STATSD_PORT`
`BIGCHAINDB_STATSD_RATE`
`BIGCHAINDB_CONFIG_PATH`
+`BIGCHAINDB_BACKLOG_REASSIGN_DELAY`
The local config file is `$HOME/.bigchaindb` by default (a file which might not even exist), but you can tell BigchainDB to use a different file by using the `-c` command-line option, e.g. `bigchaindb -c path/to/config_file.json start` or using the `BIGCHAINDB_CONFIG_PATH` environment variable, e.g. `BIGHAINDB_CONFIG_PATH=.my_bigchaindb_config bigchaindb start`. @@ -175,3 +176,17 @@ export BIGCHAINDB_STATSD_RATE=0.01 ```js "statsd": {"host": "localhost", "port": 8125, "rate": 0.01} ``` + +## backlog_reassign_delay + +Specifies how long, in seconds, transactions can remain in the backlog before being reassigned. Long-waiting transactions must be reassigned because the assigned node may no longer be responsive. The default duration is 120 seconds. + +**Example using environment variables** +```text +export BIGCHAINDB_BACKLOG_REASSIGN_DELAY=30 +``` + +**Default value (from a config file)** +```js +"backlog_reassign_delay": 120 +``` From 9212bcbe2405d364fa4e05d64ed02b90131fe076 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 29 Nov 2016 14:28:21 +0100 Subject: [PATCH 10/42] Minor edit about tx depending on associated keypairs --- docs/server/source/drivers-clients/http-client-server-api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 8cfd7c63..12f30a79 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -59,7 +59,8 @@ POST /transactions/ The steps to build a valid transaction are beyond the scope of this page. One would normally use a driver such as the `BigchainDB Python Driver `_ to - build a valid transaction for a public/private keypair. + build a valid transaction. The exact contents of a valid transaction depend + on the associated public/private keypairs. **Example request**: From ba78c9c80448901bc0dfdf2c74ab8fe18889f001 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 29 Nov 2016 14:33:44 +0100 Subject: [PATCH 11/42] Last CHANGELOG updates before the v0.8.0 release --- CHANGELOG.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5caffaec..fdb23ce5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ For reference, the possible headings are: * **Notes** -## [0.8.0] - 2016-11-?? +## [0.8.0] - 2016-11-29 Tag name: v0.8.0 = commit: committed: @@ -24,6 +24,7 @@ committed: ### Added - The big new thing in version 0.8.0 is support for divisible assets, i.e. assets like carrots or thumbtacks, where the initial CREATE transaction can register/create some amount (e.g. 542 carrots), the first TRANSFER transaction can split that amount across multiple owners, and so on. [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) - Wrote a formal schema for the JSON structure of transactions. Transactions are now checked against that schema. [Pull Request #798](https://github.com/bigchaindb/bigchaindb/pull/798) +- New configuration parameter: `backlog_reassign_delay`. [Pull Request #883](https://github.com/bigchaindb/bigchaindb/pull/883) ### Changed - CREATE transactions must now be signed by all `owners_before` (rather than by a federation node). [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) @@ -31,6 +32,7 @@ committed: - `get_transaction()` will now return a transaction from the backlog, even if there are copies of the transaction in invalid blocks. [Pull Request #793](https://github.com/bigchaindb/bigchaindb/pull/793) - Several pull requests to introduce a generalized database interface, to move RethinkDB calls into a separate implementation of that interface, and to work on a new MongoDB implementation of that interface. Pull Requests [#754](https://github.com/bigchaindb/bigchaindb/pull/754), +[#783](https://github.com/bigchaindb/bigchaindb/pull/783), [#799](https://github.com/bigchaindb/bigchaindb/pull/799), [#806](https://github.com/bigchaindb/bigchaindb/pull/806), [#809](https://github.com/bigchaindb/bigchaindb/pull/809), @@ -50,6 +52,7 @@ committed: - Hard-coded `backlog_reassign_delay`. [Pull Request #854](https://github.com/bigchaindb/bigchaindb/pull/854) - Race condition in `test_stale_monitor.py`. [Pull Request #846](https://github.com/bigchaindb/bigchaindb/pull/846) - When creating a signed vote, decode the vote signature to a `str`. [Pull Request #869](https://github.com/bigchaindb/bigchaindb/pull/869) +- Bug in AWS deployment scripts. Setting `BIND_HTTP_TO_LOCALHOST` to `False` didn't actually work. It does now. [Pull Request #870](https://github.com/bigchaindb/bigchaindb/pull/870) ### External Contributors - @najlachamseddine - [Pull Request #528](https://github.com/bigchaindb/bigchaindb/pull/528) @@ -71,8 +74,10 @@ committed: [#803](https://github.com/bigchaindb/bigchaindb/pull/803), [#819](https://github.com/bigchaindb/bigchaindb/pull/819), [#827](https://github.com/bigchaindb/bigchaindb/pull/827), -[#859](https://github.com/bigchaindb/bigchaindb/pull/859) - +[#859](https://github.com/bigchaindb/bigchaindb/pull/859), +[#872](https://github.com/bigchaindb/bigchaindb/pull/872), +[#882](https://github.com/bigchaindb/bigchaindb/pull/882), +[#883](https://github.com/bigchaindb/bigchaindb/pull/883) ## [0.7.0] - 2016-10-28 From eb6c423eeeaaa5e904445c8d449b46a763647466 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 29 Nov 2016 14:41:00 +0100 Subject: [PATCH 12/42] Updated transaction.rst for v0.8.0 --- docs/server/source/schema/transaction.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/source/schema/transaction.rst b/docs/server/source/schema/transaction.rst index 766e0c4a..68abab0c 100644 --- a/docs/server/source/schema/transaction.rst +++ b/docs/server/source/schema/transaction.rst @@ -157,8 +157,8 @@ Condition.cid **type:** integer -Index of this condition's appearance in the Transaction.conditions_ -array. In a transaction with 2 conditions, the ``cid``\ s will be 0 and 1. +Index of this condition's appearance in the `Transaction.conditions`_ +array. In a transaction with 2 conditions, the ``cid``s will be 0 and 1. From b261724c27d07d3986b61a1da362b6050f5a6b45 Mon Sep 17 00:00:00 2001 From: troymc Date: Tue, 29 Nov 2016 14:47:44 +0100 Subject: [PATCH 13/42] Updated version.py to 0.8.0 --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index 95f84a62..ccec6cc5 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.0.dev' -__short_version__ = '0.8.dev' +__version__ = '0.8.0' +__short_version__ = '0.8' From 76ce436e035f3381cc902d10ed8c8a3f0201dfa7 Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Tue, 29 Nov 2016 16:44:11 +0100 Subject: [PATCH 14/42] Bump version to 0.9.0.dev --- bigchaindb/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigchaindb/version.py b/bigchaindb/version.py index ccec6cc5..060e76d3 100644 --- a/bigchaindb/version.py +++ b/bigchaindb/version.py @@ -1,2 +1,2 @@ -__version__ = '0.8.0' -__short_version__ = '0.8' +__version__ = '0.9.0.dev' +__short_version__ = '0.9.dev' From 49726c4a31fbe6d0d9308aa968befffc15a8872e Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 30 Nov 2016 16:20:17 +0100 Subject: [PATCH 15/42] `make html` runs generation scripts for documentation, generates http server examples --- .gitignore | 4 + docs/generate/__init__.py | 0 .../generate_http_server_api_documentation.py | 87 +++++ docs/server/generate_schema_documentation.py | 12 +- docs/server/source/conf.py | 8 + .../http-client-server-api.rst | 176 +--------- docs/server/source/schema/transaction.rst | 307 ------------------ 7 files changed, 121 insertions(+), 473 deletions(-) create mode 100644 docs/generate/__init__.py create mode 100644 docs/server/generate_http_server_api_documentation.py delete mode 100644 docs/server/source/schema/transaction.rst diff --git a/.gitignore b/.gitignore index ad82db58..1a22ad18 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,7 @@ benchmarking-tests/ssh_key.py # Ansible-specific files ntools/one-m/ansible/hosts ntools/one-m/ansible/ansible.cfg + +# Just in time documentation +docs/server/source/schema +docs/server/source/drivers-clients/samples diff --git a/docs/generate/__init__.py b/docs/generate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py new file mode 100644 index 00000000..9ff0d620 --- /dev/null +++ b/docs/server/generate_http_server_api_documentation.py @@ -0,0 +1,87 @@ +""" Script to build http examples for http server api docs """ + +import json +import os +import os.path + +from bigchaindb.common.transaction import Asset, Transaction + + +TPLS = {} + +TPLS['post-tx-request'] = """\ +POST /transactions/ HTTP/1.1 +Host: example.com +Content-Type: application/json + +%(tx)s +""" + + +TPLS['post-tx-response'] = """\ +HTTP/1.1 201 Created +Content-Type: application/json + +%(tx)s +""" + + +TPLS['get-tx-status-request'] = """\ +GET /transactions/%(txid)s/status HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-status-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "status": "valid" +} +""" + + +TPLS['get-tx-request'] = """\ +GET /transactions/%(txid)s HTTP/1.1 +Host: example.com + +""" + + +TPLS['get-tx-response'] = """\ +HTTP/1.1 200 OK +Content-Type: application/json + +%(tx)s +""" + + +def main(): + """ Main function """ + pub = '9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb' + asset = Asset(None, 'e6969f87-4fc9-4467-b62a-f0dfa1c85002') + tx = Transaction.create([pub], [([pub], 1)], asset=asset) + tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) + + base_path = os.path.join(os.path.dirname(__file__), + 'source/drivers-clients/samples') + + if not os.path.exists(base_path): + os.makedirs(base_path) + + for name, tpl in TPLS.items(): + path = os.path.join(base_path, name + '.http') + code = tpl % {'tx': tx_json, 'txid': tx.id} + with open(path, 'w') as handle: + handle.write(code) + + +def setup(*_): + """ Fool sphinx into think it's an extension muahaha """ + main() + + +if __name__ == '__main__': + main() diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 0e1a626a..8767cabf 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -168,12 +168,20 @@ def main(): 'file': os.path.basename(__file__), } - path = os.path.join(os.path.dirname(__file__), - 'source/schema/transaction.rst') + base_path = os.path.join(os.path.dirname(__file__), 'source/schema') + path = os.path.join(base_path, 'transaction.rst') + + if not os.path.exists(base_path): + os.makedirs(base_path) with open(path, 'w') as handle: handle.write(doc) +def setup(*_): + """ Fool sphinx into think it's an extension muahaha """ + main() + + if __name__ == '__main__': main() diff --git a/docs/server/source/conf.py b/docs/server/source/conf.py index c0de76d7..93402f75 100644 --- a/docs/server/source/conf.py +++ b/docs/server/source/conf.py @@ -35,6 +35,10 @@ _version = {} with open('../../../bigchaindb/version.py') as fp: exec(fp.read(), _version) +import os.path +import sys + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__) + '/..')) extensions = [ 'sphinx.ext.autodoc', @@ -44,6 +48,10 @@ extensions = [ 'sphinx.ext.napoleon', 'sphinxcontrib.httpdomain', 'sphinx.ext.autosectionlabel', + # Below are actually build steps made to look like sphinx extensions. + # It was the easiest way to get it running with ReadTheDocs. + 'generate_schema_documentation', + 'generate_http_server_api_documentation', ] # autodoc settings diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 12f30a79..bb037d15 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -64,108 +64,13 @@ POST /transactions/ **Example request**: - .. sourcecode:: http - - POST /transactions/ HTTP/1.1 - Host: example.com - Content-Type: application/json - - { - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFY:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - } - }, - "amount": 1, - "owners_after": [ - "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "b57801f8-b865-4360-9d1a-3e3009f5ce01", - "refillable": false - }, - "metadata": null, - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFaf8bQVH1gesZGEGZepCE8_kgo-UfBrCHPlvBsnAsfq56GWjrLTyZ9NXISwcyJ3zmygnVhCMG8xzE6c9fj1-6wK", - "owners_before": [ - "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - ] - } - ] - }, - "id": "65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3", - "version": 1 - } + .. literalinclude:: samples/post-tx-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 201 Created - Content-Type: application/json - - { - "id": "65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3", - "version": 1, - "transaction": { - "conditions": [ - { - "amount": 1, - "condition": { - "uri": "cc:4:20:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFY:96", - "details": { - "signature": null, - "type_id": 4, - "type": "fulfillment", - "bitmask": 32, - "public_key": "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - } - }, - "owners_after": [ - "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - ], - "cid": 0 - } - ], - "fulfillments": [ - { - "input": null, - "fulfillment": "cf:4:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFaf8bQVH1gesZGEGZepCE8_kgo-UfBrCHPlvBsnAsfq56GWjrLTyZ9NXISwcyJ3zmygnVhCMG8xzE6c9fj1-6wK", - "fid": 0, - "owners_before": [ - "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - ] - } - ], - "operation": "CREATE", - "asset": { - "updatable": false, - "refillable": false, - "divisible": false, - "data": null, - "id": "b57801f8-b865-4360-9d1a-3e3009f5ce01" - }, - "metadata": null - } - } + .. literalinclude:: samples/post-tx-response.http + :language: http :statuscode 201: A new transaction was created. :statuscode 400: The transaction was invalid and not created. @@ -187,21 +92,13 @@ GET /transactions/{tx_id}/status **Example request**: - .. sourcecode:: http - - GET /transactions/65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3/status HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-status-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "status": "valid" - } + .. literalinclude:: samples/get-tx-status-response.http + :language: http :statuscode 200: A transaction with that ID was found and the status is returned. :statuscode 404: A transaction with that ID was not found. @@ -222,62 +119,13 @@ GET /transactions/{tx_id} **Example request**: - .. sourcecode:: http - - GET /transactions/65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3 HTTP/1.1 - Host: example.com + .. literalinclude:: samples/get-tx-request.http + :language: http **Example response**: - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Type: application/json - - { - "transaction": { - "conditions": [ - { - "cid": 0, - "condition": { - "uri": "cc:4:20:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFY:96", - "details": { - "signature": null, - "type": "fulfillment", - "type_id": 4, - "bitmask": 32, - "public_key": "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - } - }, - "amount": 1, - "owners_after": [ - "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - ] - } - ], - "operation": "CREATE", - "asset": { - "divisible": false, - "updatable": false, - "data": null, - "id": "b57801f8-b865-4360-9d1a-3e3009f5ce01", - "refillable": false - }, - "metadata": null, - "fulfillments": [ - { - "fid": 0, - "input": null, - "fulfillment": "cf:4:fSlVCKNSzSl0meiwwuUk5JpJ0KLlECTqbd25KyQefFaf8bQVH1gesZGEGZepCE8_kgo-UfBrCHPlvBsnAsfq56GWjrLTyZ9NXISwcyJ3zmygnVhCMG8xzE6c9fj1-6wK", - "owners_before": [ - "9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb" - ] - } - ] - }, - "id": "65f1f69b6ebf995a7b2c5ae8a6fb480ce20f0e8f1eb1d77d75f37ab00ccdeec3", - "version": 1 - } + .. literalinclude:: samples/get-tx-response.http + :language: http :statuscode 200: A transaction with that ID was found. :statuscode 404: A transaction with that ID was not found. diff --git a/docs/server/source/schema/transaction.rst b/docs/server/source/schema/transaction.rst deleted file mode 100644 index 1d2b8f7d..00000000 --- a/docs/server/source/schema/transaction.rst +++ /dev/null @@ -1,307 +0,0 @@ -.. This file was auto generated by generate_schema_documentation.py - -================== -Transaction Schema -================== - -* `Transaction`_ - -* `Transaction Body`_ - -* Condition_ - -* Fulfillment_ - -* Asset_ - -* Metadata_ - -.. raw:: html - - - -Transaction ------------ - -This is the outer transaction wrapper. It contains the ID, version and the body of the transaction, which is also called ``transaction``. - - -Transaction.id -^^^^^^^^^^^^^^ - -**type:** string - -A sha3 digest of the transaction. The ID is calculated by removing all -derived hashes and signatures from the transaction, serializing it to -JSON with keys in sorted order and then hashing the resulting string -with sha3. - - - -Transaction.transaction -^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** object - -See: `Transaction Body`_. - - - -Transaction.version -^^^^^^^^^^^^^^^^^^^ - -**type:** integer - -BigchainDB transaction schema version. - - - - - -Transaction Body ----------------- - -See: `Transaction Body`_. - - -Transaction.operation -^^^^^^^^^^^^^^^^^^^^^ - -**type:** string - -Type of the transaction: - -A ``CREATE`` transaction creates an asset in BigchainDB. This -transaction has outputs (conditions) but no inputs (fulfillments), -so a dummy fulfillment is used. - -A ``TRANSFER`` transaction transfers ownership of an asset, by providing -fulfillments to conditions of earlier transactions. - -A ``GENESIS`` transaction is a special case transaction used as the -sole member of the first block in a BigchainDB ledger. - - - -Transaction.asset -^^^^^^^^^^^^^^^^^ - -**type:** object - -Description of the asset being transacted. - -See: `Asset`_. - - - -Transaction.fulfillments -^^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (object) - -Array of the fulfillments (inputs) of a transaction. - -See: Fulfillment_. - - - -Transaction.conditions -^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (object) - -Array of conditions (outputs) provided by this transaction. - -See: Condition_. - - - -Transaction.metadata -^^^^^^^^^^^^^^^^^^^^ - -**type:** object or null - -User provided transaction metadata. This field may be ``null`` or may -contain an object with freeform metadata. - -See: `Metadata`_. - - - - - -Condition ----------- - -An output of a transaction. A condition describes a quantity of an asset -and what conditions must be met in order for it to be fulfilled. See also: -fulfillment_. - - -Condition.cid -^^^^^^^^^^^^^ - -**type:** integer - -Index of this condition's appearance in the `Transaction.conditions`_ -array. In a transaction with 2 conditions, the ``cid``s will be 0 and 1. - - - -Condition.condition -^^^^^^^^^^^^^^^^^^^ - -**type:** object - -Body of the condition. Has the properties: - -- **details**: Details of the condition. -- **uri**: Condition encoded as an ASCII string. - - - -Condition.owners_after -^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (string) or null - -List of public keys associated with asset ownership at the time -of the transaction. - - - -Condition.amount -^^^^^^^^^^^^^^^^ - -**type:** integer - -Integral amount of the asset represented by this condition. -In the case of a non divisible asset, this will always be 1. - - - - - -Fulfillment ------------ - -A fulfillment is an input to a transaction, named as such because it fulfills a condition of a previous transaction. In the case of a ``CREATE`` transaction, a fulfillment may provide no ``input``. - -Fulfillment.fid -^^^^^^^^^^^^^^^ - -**type:** integer - -The offset of the fulfillment within the fulfillents array. - - - -Fulfillment.owners_before -^^^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** array (string) or null - -List of public keys of the previous owners of the asset. - - - -Fulfillment.fulfillment -^^^^^^^^^^^^^^^^^^^^^^^ - -**type:** object or string - -Fulfillment of a condition_, or put a different way, this is a -payload that satisfies a condition in order to spend the associated -asset. - - - -Fulfillment.input -^^^^^^^^^^^^^^^^^ - -**type:** object or null - -Reference to a condition of a previous transaction - - - - - -Asset ------ - -Description of the asset being transacted. In the case of a ``TRANSFER`` -transaction, this field contains only the ID of asset. In the case -of a ``CREATE`` transaction, this field may contain properties: - - -Asset.id -^^^^^^^^ - -**type:** string - -A `UUID `_ -of type 4 (random). - - - -Asset.divisible -^^^^^^^^^^^^^^^ - -**type:** boolean - -Whether or not the asset has a quantity that may be partially spent. - - - -Asset.updatable -^^^^^^^^^^^^^^^ - -**type:** boolean - -Whether or not the description of the asset may be updated. Defaults to false. - - - -Asset.refillable -^^^^^^^^^^^^^^^^ - -**type:** boolean - -Whether the amount of the asset can change after its creation. Defaults to false. - - - -Asset.data -^^^^^^^^^^ - -**type:** object or null - -User provided metadata associated with the asset. May also be ``null``. - - - - - -Metadata --------- - -User provided transaction metadata. This field may be ``null`` or may -contain an non empty object with freeform metadata. - - - From ec38d0685623715118ed70bbde2079eba93d329f Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 30 Nov 2016 16:41:24 +0100 Subject: [PATCH 16/42] add test to make sure documentation can build --- setup.py | 23 +++++++++++------------ tests/test_docs.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 tests/test_docs.py diff --git a/setup.py b/setup.py index 4043c0be..ecb801a3 100644 --- a/setup.py +++ b/setup.py @@ -27,18 +27,6 @@ def check_setuptools_features(): check_setuptools_features() - -tests_require = [ - 'coverage', - 'pep8', - 'flake8', - 'pylint', - 'pytest', - 'pytest-cov>=2.2.1', - 'pytest-xdist', - 'pytest-flask', -] - dev_require = [ 'ipdb', 'ipython', @@ -52,6 +40,17 @@ docs_require = [ 'sphinxcontrib-napoleon>=0.4.4', ] +tests_require = [ + 'coverage', + 'pep8', + 'flake8', + 'pylint', + 'pytest', + 'pytest-cov>=2.2.1', + 'pytest-xdist', + 'pytest-flask', +] + docs_require + benchmarks_require = [ 'line-profiler==1.0', ] diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 00000000..037bf87c --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,16 @@ + +import subprocess + + +def test_build_server_docs(): + proc = subprocess.Popen(['bash'], stdin=subprocess.PIPE) + proc.stdin.write('cd docs/server; make html'.encode()) + proc.stdin.close() + assert proc.wait() == 0 + + +def test_build_root_docs(): + proc = subprocess.Popen(['bash'], stdin=subprocess.PIPE) + proc.stdin.write('cd docs/root; make html'.encode()) + proc.stdin.close() + assert proc.wait() == 0 From f4164a69707f0847596156a200d411aca955a72a Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 1 Dec 2016 10:17:58 +0100 Subject: [PATCH 17/42] nomenclature fix in generate_http_server_api_documentation.py --- docs/server/generate_http_server_api_documentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index 9ff0d620..d9d86647 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -60,9 +60,9 @@ Content-Type: application/json def main(): """ Main function """ - pub = '9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb' + pubkey = '9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb' asset = Asset(None, 'e6969f87-4fc9-4467-b62a-f0dfa1c85002') - tx = Transaction.create([pub], [([pub], 1)], asset=asset) + tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset) tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) base_path = os.path.join(os.path.dirname(__file__), From 8d4677f456720cdac699ce4d92e6a938150a7b45 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Wed, 23 Nov 2016 10:01:44 +0100 Subject: [PATCH 18/42] flatten transaction - code changes --- bigchaindb/common/schema/transaction.yaml | 75 +++--- bigchaindb/common/transaction.py | 14 +- bigchaindb/core.py | 4 +- bigchaindb/db/backends/rethinkdb.py | 11 +- bigchaindb/db/utils.py | 2 +- bigchaindb/util.py | 2 +- docs/server/generate_schema_documentation.py | 11 +- tests/common/test_transaction.py | 251 +++++++++---------- tests/db/test_bigchain_api.py | 4 +- tests/web/test_transactions.py | 10 +- 10 files changed, 173 insertions(+), 211 deletions(-) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index 03f93ac7..b64ceed4 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -5,10 +5,14 @@ type: object additionalProperties: false title: Transaction Schema description: | - This is the outer transaction wrapper. It contains the ID, version and the body of the transaction, which is also called ``transaction``. + TODO - What should go here? required: - id -- transaction +- fulfillments +- conditions +- operation +- metadata +- asset - version properties: id: @@ -18,51 +22,38 @@ properties: derived hashes and signatures from the transaction, serializing it to JSON with keys in sorted order and then hashing the resulting string with sha3. - transaction: - type: object - title: transaction + operation: + "$ref": "#/definitions/operation" + asset: + "$ref": "#/definitions/asset" description: | - See: `Transaction Body`_. - additionalProperties: false - required: - - fulfillments - - conditions - - operation - - metadata - - asset - properties: - operation: - "$ref": "#/definitions/operation" - asset: - "$ref": "#/definitions/asset" - description: | - Description of the asset being transacted. + Description of the asset being transacted. - See: `Asset`_. - fulfillments: - type: array - title: "Fulfillments list" - description: | - Array of the fulfillments (inputs) of a transaction. + See: `Asset`_. + fulfillments: + type: array + title: "Fulfillments list" + description: | + Array of the fulfillments (inputs) of a transaction. - See: Fulfillment_. - items: - "$ref": "#/definitions/fulfillment" - conditions: - type: array - description: | - Array of conditions (outputs) provided by this transaction. + See: Fulfillment_. + items: + "$ref": "#/definitions/fulfillment" + conditions: + type: array + description: | + Array of conditions (outputs) provided by this transaction. - See: Condition_. - items: - "$ref": "#/definitions/condition" - metadata: - "$ref": "#/definitions/metadata" - description: | - User provided transaction metadata. This field may be ``null`` or may - contain an object with freeform metadata. + See: Condition_. + items: + "$ref": "#/definitions/condition" + metadata: + "$ref": "#/definitions/metadata" + description: | + User provided transaction metadata. This field may be ``null`` or may + contain an id and an object with freeform metadata. - See: `Metadata`_. + See: `Metadata`_. version: type: integer minimum: 1 diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 6fa1aa2b..2a9591c2 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1110,7 +1110,7 @@ class Transaction(object): # NOTE: An `asset` in a `TRANSFER` only contains the asset's id asset = {'id': self.asset.data_id} - tx_body = { + tx = { 'fulfillments': [fulfillment.to_dict(fid) for fid, fulfillment in enumerate(self.fulfillments)], 'conditions': [condition.to_dict(cid) for cid, condition @@ -1118,10 +1118,7 @@ class Transaction(object): 'operation': str(self.operation), 'metadata': self.metadata, 'asset': asset, - } - tx = { 'version': self.version, - 'transaction': tx_body, } tx_no_signatures = Transaction._remove_signatures(tx) @@ -1146,7 +1143,7 @@ class Transaction(object): # NOTE: We remove the reference since we need `tx_dict` only for the # transaction's hash tx_dict = deepcopy(tx_dict) - for fulfillment in tx_dict['transaction']['fulfillments']: + for fulfillment in tx_dict['fulfillments']: # NOTE: Not all Cryptoconditions return a `signature` key (e.g. # ThresholdSha256Fulfillment), so setting it to `None` in any # case could yield incorrect signatures. This is why we only @@ -1196,7 +1193,7 @@ class Transaction(object): raise InvalidHash() @classmethod - def from_dict(cls, tx_body): + def from_dict(cls, tx): """Transforms a Python dictionary to a Transaction object. Args: @@ -1205,8 +1202,7 @@ class Transaction(object): Returns: :class:`~bigchaindb.common.transaction.Transaction` """ - cls.validate_structure(tx_body) - tx = tx_body['transaction'] + cls.validate_structure(tx) fulfillments = [Fulfillment.from_dict(fulfillment) for fulfillment in tx['fulfillments']] conditions = [Condition.from_dict(condition) for condition @@ -1217,4 +1213,4 @@ class Transaction(object): asset = AssetLink.from_dict(tx['asset']) return cls(tx['operation'], asset, fulfillments, conditions, - tx['metadata'], tx_body['version']) + tx['metadata'], tx['version']) diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 6fd892e7..8fec1c89 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -367,7 +367,7 @@ class Bigchain(object): cursor = self.backend.get_asset_by_id(asset_id) cursor = list(cursor) if cursor: - return Asset.from_dict(cursor[0]['transaction']['asset']) + return Asset.from_dict(cursor[0]['asset']) def get_spent(self, txid, cid): """Check if a `txid` was already used as an input. @@ -436,7 +436,7 @@ class Bigchain(object): # use it after the execution of this function. # a transaction can contain multiple outputs (conditions) so we need to iterate over all of them # to get a list of outputs available to spend - for index, cond in enumerate(tx['transaction']['conditions']): + for index, cond in enumerate(tx['conditions']): # for simple signature conditions there are no subfulfillments # check if the owner is in the condition `owners_after` if len(cond['owners_after']) == 1: diff --git a/bigchaindb/db/backends/rethinkdb.py b/bigchaindb/db/backends/rethinkdb.py index 87b09dd2..9d81f595 100644 --- a/bigchaindb/db/backends/rethinkdb.py +++ b/bigchaindb/db/backends/rethinkdb.py @@ -159,7 +159,7 @@ class RethinkDBBackend: r.table('bigchain', read_mode=self.read_mode) .get_all(asset_id, index='asset_id') .concat_map(lambda block: block['block']['transactions']) - .filter(lambda transaction: transaction['transaction']['asset']['id'] == asset_id) + .filter(lambda transaction: transaction['asset']['id'] == asset_id) .get_field('id')) def get_asset_by_id(self, asset_id): @@ -176,10 +176,9 @@ class RethinkDBBackend: .get_all(asset_id, index='asset_id') .concat_map(lambda block: block['block']['transactions']) .filter(lambda transaction: - transaction['transaction']['asset']['id'] == asset_id) + transaction['asset']['id'] == asset_id) .filter(lambda transaction: - transaction['transaction']['operation'] == 'CREATE') - .pluck({'transaction': 'asset'})) + transaction['operation'] == 'CREATE')) def get_spent(self, transaction_id, condition_id): """Check if a `txid` was already used as an input. @@ -199,7 +198,7 @@ class RethinkDBBackend: return self.connection.run( r.table('bigchain', read_mode=self.read_mode) .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda transaction: transaction['transaction']['fulfillments'].contains( + .filter(lambda transaction: transaction['fulfillments'].contains( lambda fulfillment: fulfillment['input'] == {'txid': transaction_id, 'cid': condition_id}))) def get_owned_ids(self, owner): @@ -216,7 +215,7 @@ class RethinkDBBackend: return self.connection.run( r.table('bigchain', read_mode=self.read_mode) .concat_map(lambda doc: doc['block']['transactions']) - .filter(lambda tx: tx['transaction']['conditions'].contains( + .filter(lambda tx: tx['conditions'].contains( lambda c: c['owners_after'].contains(owner)))) def get_votes_by_block_id(self, block_id): diff --git a/bigchaindb/db/utils.py b/bigchaindb/db/utils.py index cbcebc85..2b3c2d0d 100644 --- a/bigchaindb/db/utils.py +++ b/bigchaindb/db/utils.py @@ -119,7 +119,7 @@ def create_bigchain_secondary_index(conn, dbname): # secondary index for asset uuid r.db(dbname).table('bigchain')\ .index_create('asset_id', - r.row['block']['transactions']['transaction']['asset']['id'], multi=True)\ + r.row['block']['transactions']['asset']['id'], multi=True)\ .run(conn) # wait for rethinkdb to finish creating secondary indexes diff --git a/bigchaindb/util.py b/bigchaindb/util.py index 27b7fc00..3b0526d7 100644 --- a/bigchaindb/util.py +++ b/bigchaindb/util.py @@ -156,4 +156,4 @@ def is_genesis_block(block): try: return block.transactions[0].operation == 'GENESIS' except AttributeError: - return block['block']['transactions'][0]['transaction']['operation'] == 'GENESIS' + return block['block']['transactions'][0]['operation'] == 'GENESIS' diff --git a/docs/server/generate_schema_documentation.py b/docs/server/generate_schema_documentation.py index 8767cabf..1d1b751b 100644 --- a/docs/server/generate_schema_documentation.py +++ b/docs/server/generate_schema_documentation.py @@ -27,8 +27,6 @@ Transaction Schema * `Transaction`_ -* `Transaction Body`_ - * Condition_ * Fulfillment_ @@ -58,11 +56,6 @@ Transaction Schema Transaction ----------- -%(wrapper)s - -Transaction Body ----------------- - %(transaction)s Condition @@ -158,9 +151,7 @@ def main(): """ Main function """ defs = TX_SCHEMA['definitions'] doc = TPL_DOC % { - 'wrapper': render_section('Transaction', TX_SCHEMA), - 'transaction': render_section('Transaction', - TX_SCHEMA['properties']['transaction']), + 'transaction': render_section('Transaction', TX_SCHEMA), 'condition': render_section('Condition', defs['condition']), 'fulfillment': render_section('Fulfillment', defs['fulfillment']), 'asset': render_section('Asset', defs['asset']), diff --git a/tests/common/test_transaction.py b/tests/common/test_transaction.py index 335fd0a2..b55b4cb7 100644 --- a/tests/common/test_transaction.py +++ b/tests/common/test_transaction.py @@ -301,20 +301,18 @@ def test_transaction_serialization(user_ffill, user_cond, data, data_id): expected = { 'id': tx_id, 'version': Transaction.VERSION, - 'transaction': { - # NOTE: This test assumes that Fulfillments and Conditions can - # successfully be serialized - 'fulfillments': [user_ffill.to_dict(0)], - 'conditions': [user_cond.to_dict(0)], - 'operation': Transaction.CREATE, - 'metadata': None, - 'asset': { - 'id': data_id, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - } + # NOTE: This test assumes that Fulfillments and Conditions can + # successfully be serialized + 'fulfillments': [user_ffill.to_dict(0)], + 'conditions': [user_cond.to_dict(0)], + 'operation': Transaction.CREATE, + 'metadata': None, + 'asset': { + 'id': data_id, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, } } @@ -322,7 +320,7 @@ def test_transaction_serialization(user_ffill, user_cond, data, data_id): [user_cond]) tx_dict = tx.to_dict() tx_dict['id'] = tx_id - tx_dict['transaction']['asset']['id'] = data_id + tx_dict['asset']['id'] = data_id assert tx_dict == expected @@ -342,20 +340,18 @@ def test_transaction_deserialization(user_ffill, user_cond, data, uuid4): tx = { 'version': Transaction.VERSION, - 'transaction': { - # NOTE: This test assumes that Fulfillments and Conditions can - # successfully be serialized - 'fulfillments': [user_ffill.to_dict()], - 'conditions': [user_cond.to_dict()], - 'operation': Transaction.CREATE, - 'metadata': None, - 'asset': { - 'id': uuid4, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - } + # NOTE: This test assumes that Fulfillments and Conditions can + # successfully be serialized + 'fulfillments': [user_ffill.to_dict()], + 'conditions': [user_cond.to_dict()], + 'operation': Transaction.CREATE, + 'metadata': None, + 'asset': { + 'id': uuid4, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, } } tx_no_signatures = Transaction._remove_signatures(tx) @@ -732,35 +728,33 @@ def test_create_create_transaction_single_io(user_cond, user_pub, data, uuid4): from .util import validate_transaction_model expected = { - 'transaction': { - 'conditions': [user_cond.to_dict(0)], - 'metadata': data, - 'asset': { - 'id': uuid4, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - }, - 'fulfillments': [ - { - 'owners_before': [ - user_pub - ], - 'fid': 0, - 'fulfillment': None, - 'input': None - } - ], - 'operation': 'CREATE', + 'conditions': [user_cond.to_dict(0)], + 'metadata': data, + 'asset': { + 'id': uuid4, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, }, + 'fulfillments': [ + { + 'owners_before': [ + user_pub + ], + 'fid': 0, + 'fulfillment': None, + 'input': None + } + ], + 'operation': 'CREATE', 'version': 1, } asset = Asset(data, uuid4) tx = Transaction.create([user_pub], [([user_pub], 1)], data, asset) tx_dict = tx.to_dict() - tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None + tx_dict['fulfillments'][0]['fulfillment'] = None tx_dict.pop('id') assert tx_dict == expected @@ -786,14 +780,12 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, ffill = Fulfillment.generate([user_pub, user2_pub]).to_dict() ffill.update({'fid': 0}) expected = { - 'transaction': { - 'conditions': [user_cond.to_dict(0), user2_cond.to_dict(1)], - 'metadata': { - 'message': 'hello' - }, - 'fulfillments': [ffill], - 'operation': 'CREATE', + 'conditions': [user_cond.to_dict(0), user2_cond.to_dict(1)], + 'metadata': { + 'message': 'hello' }, + 'fulfillments': [ffill], + 'operation': 'CREATE', 'version': 1 } asset = Asset(divisible=True) @@ -802,7 +794,7 @@ def test_create_create_transaction_multiple_io(user_cond, user2_cond, user_pub, asset=asset, metadata={'message': 'hello'}).to_dict() tx.pop('id') - tx['transaction'].pop('asset') + tx.pop('asset') assert tx == expected @@ -829,28 +821,26 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, from bigchaindb.common.transaction import Transaction, Asset expected = { - 'transaction': { - 'conditions': [user_user2_threshold_cond.to_dict(0)], - 'metadata': data, - 'asset': { - 'id': uuid4, - 'divisible': False, - 'updatable': False, - 'refillable': False, - 'data': data, - }, - 'fulfillments': [ - { - 'owners_before': [ - user_pub, - ], - 'fid': 0, - 'fulfillment': None, - 'input': None - }, - ], - 'operation': 'CREATE', + 'conditions': [user_user2_threshold_cond.to_dict(0)], + 'metadata': data, + 'asset': { + 'id': uuid4, + 'divisible': False, + 'updatable': False, + 'refillable': False, + 'data': data, }, + 'fulfillments': [ + { + 'owners_before': [ + user_pub, + ], + 'fid': 0, + 'fulfillment': None, + 'input': None + }, + ], + 'operation': 'CREATE', 'version': 1 } asset = Asset(data, uuid4) @@ -858,7 +848,7 @@ def test_create_create_transaction_threshold(user_pub, user2_pub, user3_pub, data, asset) tx_dict = tx.to_dict() tx_dict.pop('id') - tx_dict['transaction']['fulfillments'][0]['fulfillment'] = None + tx_dict['fulfillments'][0]['fulfillment'] = None assert tx_dict == expected @@ -912,27 +902,25 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, from .util import validate_transaction_model expected = { - 'transaction': { - 'conditions': [user2_cond.to_dict(0)], - 'metadata': None, - 'asset': { - 'id': uuid4, - }, - 'fulfillments': [ - { - 'owners_before': [ - user_pub - ], - 'fid': 0, - 'fulfillment': None, - 'input': { - 'txid': tx.id, - 'cid': 0 - } - } - ], - 'operation': 'TRANSFER', + 'conditions': [user2_cond.to_dict(0)], + 'metadata': None, + 'asset': { + 'id': uuid4, }, + 'fulfillments': [ + { + 'owners_before': [ + user_pub + ], + 'fid': 0, + 'fulfillment': None, + 'input': { + 'txid': tx.id, + 'cid': 0 + } + } + ], + 'operation': 'TRANSFER', 'version': 1 } inputs = tx.to_inputs([0]) @@ -940,14 +928,13 @@ def test_create_transfer_transaction_single_io(tx, user_pub, user2_pub, transfer_tx = Transaction.transfer(inputs, [([user2_pub], 1)], asset=asset) transfer_tx = transfer_tx.sign([user_priv]) transfer_tx = transfer_tx.to_dict() - transfer_tx_body = transfer_tx['transaction'] expected_input = deepcopy(inputs[0]) expected['id'] = transfer_tx['id'] expected_input.fulfillment.sign(serialize(expected).encode(), PrivateKey(user_priv)) expected_ffill = expected_input.fulfillment.serialize_uri() - transfer_ffill = transfer_tx_body['fulfillments'][0]['fulfillment'] + transfer_ffill = transfer_tx['fulfillments'][0]['fulfillment'] assert transfer_ffill == expected_ffill @@ -968,34 +955,32 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, tx = tx.sign([user_priv]) expected = { - 'transaction': { - 'conditions': [user2_cond.to_dict(0), user2_cond.to_dict(1)], - 'metadata': None, - 'fulfillments': [ - { - 'owners_before': [ - user_pub - ], - 'fid': 0, - 'fulfillment': None, - 'input': { - 'txid': tx.id, - 'cid': 0 - } - }, { - 'owners_before': [ - user2_pub - ], - 'fid': 1, - 'fulfillment': None, - 'input': { - 'txid': tx.id, - 'cid': 1 - } + 'conditions': [user2_cond.to_dict(0), user2_cond.to_dict(1)], + 'metadata': None, + 'fulfillments': [ + { + 'owners_before': [ + user_pub + ], + 'fid': 0, + 'fulfillment': None, + 'input': { + 'txid': tx.id, + 'cid': 0 } - ], - 'operation': 'TRANSFER', - }, + }, { + 'owners_before': [ + user2_pub + ], + 'fid': 1, + 'fulfillment': None, + 'input': { + 'txid': tx.id, + 'cid': 1 + } + } + ], + 'operation': 'TRANSFER', 'version': 1 } @@ -1010,10 +995,10 @@ def test_create_transfer_transaction_multiple_io(user_pub, user_priv, assert transfer_tx.fulfillments_valid(tx.conditions) is True transfer_tx = transfer_tx.to_dict() - transfer_tx['transaction']['fulfillments'][0]['fulfillment'] = None - transfer_tx['transaction']['fulfillments'][1]['fulfillment'] = None + transfer_tx['fulfillments'][0]['fulfillment'] = None + transfer_tx['fulfillments'][1]['fulfillment'] = None + transfer_tx.pop('asset') transfer_tx.pop('id') - transfer_tx['transaction'].pop('asset') assert expected == transfer_tx diff --git a/tests/db/test_bigchain_api.py b/tests/db/test_bigchain_api.py index cd030249..afe2bbd6 100644 --- a/tests/db/test_bigchain_api.py +++ b/tests/db/test_bigchain_api.py @@ -273,8 +273,8 @@ class TestBigchainApi(object): block = b.backend.get_genesis_block() assert len(block['block']['transactions']) == 1 - assert block['block']['transactions'][0]['transaction']['operation'] == 'GENESIS' - assert block['block']['transactions'][0]['transaction']['fulfillments'][0]['input'] is None + assert block['block']['transactions'][0]['operation'] == 'GENESIS' + assert block['block']['transactions'][0]['fulfillments'][0]['input'] is None def test_create_genesis_block_fails_if_table_not_empty(self, b): from bigchaindb.common.exceptions import GenesisBlockAlreadyExistsError diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index dd034c10..94c3f22f 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -33,8 +33,8 @@ def test_post_create_transaction_endpoint(b, client): tx = tx.sign([user_priv]) res = client.post(TX_ENDPOINT, data=json.dumps(tx.to_dict())) - assert res.json['transaction']['fulfillments'][0]['owners_before'][0] == user_pub - assert res.json['transaction']['conditions'][0]['owners_after'][0] == user_pub + assert res.json['fulfillments'][0]['owners_before'][0] == user_pub + assert res.json['conditions'][0]['owners_after'][0] == user_pub def test_post_create_transaction_with_invalid_id(b, client): @@ -55,7 +55,7 @@ def test_post_create_transaction_with_invalid_signature(b, client): tx = Transaction.create([user_pub], [([user_pub], 1)]) tx = tx.sign([user_priv]).to_dict() - tx['transaction']['fulfillments'][0]['fulfillment'] = 'cf:0:0' + tx['fulfillments'][0]['fulfillment'] = 'cf:0:0' res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 @@ -81,8 +81,8 @@ def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): res = client.post(TX_ENDPOINT, data=json.dumps(transfer_tx.to_dict())) - assert res.json['transaction']['fulfillments'][0]['owners_before'][0] == user_pk - assert res.json['transaction']['conditions'][0]['owners_after'][0] == user_pub + assert res.json['fulfillments'][0]['owners_before'][0] == user_pk + assert res.json['conditions'][0]['owners_after'][0] == user_pub @pytest.mark.usefixtures('inputs') From 21af588f7c09309645ddb60c50b73b049f95af8a Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 17:20:01 +0100 Subject: [PATCH 19/42] docs update for flat transaction --- .../source/data-models/transaction-model.rst | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 2bee661f..0d3cd136 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -22,38 +22,35 @@ A transaction has the following structure: { "id": "", "version": "", - "transaction": { - "fulfillments": [""], - "conditions": [""], - "operation": "", - "asset": "", - "metadata": { - "id": "", - "data": "" - } + "fulfillments": [""], + "conditions": [""], + "operation": "", + "asset": "", + "metadata": { + "id": "", + "data": "" } } Here's some explanation of the contents of a :ref:`transaction `: -- :ref:`id `: The id of the transaction, and also the database primary key. -- :ref:`version `: Version number of the transaction model, so that software can support different transaction models. -- :ref:`transaction `: - - **fulfillments**: List of fulfillments. Each :ref:`fulfillment ` contains a pointer to an unspent asset - and a *crypto fulfillment* that satisfies a spending condition set on the unspent asset. A *fulfillment* - is usually a signature proving the ownership of the asset. - See :doc:`./crypto-conditions`. +- id: The :ref:`id ` of the transaction, and also the database primary key. +- version: :ref:`Version ` number of the transaction model, so that software can support different transaction models. +- **fulfillments**: List of fulfillments. Each :ref:`fulfillment ` contains a pointer to an unspent asset + and a *crypto fulfillment* that satisfies a spending condition set on the unspent asset. A *fulfillment* + is usually a signature proving the ownership of the asset. + See :doc:`./crypto-conditions`. - - **conditions**: List of conditions. Each :ref:`condition ` is a *crypto-condition* that needs to be fulfilled by a transfer transaction in order to transfer ownership to new owners. - See :doc:`./crypto-conditions`. +- **conditions**: List of conditions. Each :ref:`condition ` is a *crypto-condition* that needs to be fulfilled by a transfer transaction in order to transfer ownership to new owners. + See :doc:`./crypto-conditions`. - - **operation**: String representation of the :ref:`operation ` being performed (currently either "CREATE", "TRANSFER" or "GENESIS"). It determines how the transaction should be validated. +- **operation**: String representation of the :ref:`operation ` being performed (currently either "CREATE", "TRANSFER" or "GENESIS"). It determines how the transaction should be validated. - - **asset**: Definition of the digital :ref:`asset `. See next section. +- **asset**: Definition of the digital :ref:`asset `. See next section. - - **metadata**: - - :ref:`id `: UUID version 4 (random) converted to a string of hex digits in standard form. - - :ref:`data `: Can be any JSON document. It may be empty in the case of a transfer transaction. +- **metadata**: + - :ref:`id `: UUID version 4 (random) converted to a string of hex digits in standard form. + - :ref:`data `: Can be any JSON document. It may be empty in the case of a transfer transaction. Later, when we get to the models for the block and the vote, we'll see that both include a signature (from the node which created it). You may wonder why transactions don't have signatures... The answer is that they do! They're just hidden inside the ``fulfillment`` string of each fulfillment. A creation transaction is signed by whoever created it. A transfer transaction is signed by whoever currently controls or owns it. From e699536a643c1da7871029584f5869db00bfc3a8 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 24 Nov 2016 17:47:22 +0100 Subject: [PATCH 20/42] Fix transaction description in transaction.yaml --- bigchaindb/common/schema/transaction.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index b64ceed4..03dbe8ad 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -5,7 +5,7 @@ type: object additionalProperties: false title: Transaction Schema description: | - TODO - What should go here? + A transaction represents the creation or transfer of assets in BigchainDB. required: - id - fulfillments From 8f513fc81d3ca31851c093e53c5714964cbc8b52 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 1 Dec 2016 14:49:05 +0100 Subject: [PATCH 21/42] put back a pluck() that was removed by accident --- bigchaindb/db/backends/rethinkdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigchaindb/db/backends/rethinkdb.py b/bigchaindb/db/backends/rethinkdb.py index 9d81f595..39d2aff9 100644 --- a/bigchaindb/db/backends/rethinkdb.py +++ b/bigchaindb/db/backends/rethinkdb.py @@ -178,7 +178,8 @@ class RethinkDBBackend: .filter(lambda transaction: transaction['asset']['id'] == asset_id) .filter(lambda transaction: - transaction['operation'] == 'CREATE')) + transaction['operation'] == 'CREATE') + .pluck('asset')) def get_spent(self, transaction_id, condition_id): """Check if a `txid` was already used as an input. From b2f2ce35231bfa09adb22e4168ac2283a25a3caf Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Thu, 1 Dec 2016 16:27:00 +0100 Subject: [PATCH 22/42] hotfix release process since old step left in by mistake --- Release_Process.md | 1 - 1 file changed, 1 deletion(-) diff --git a/Release_Process.md b/Release_Process.md index e6eaf7ae..16dc800b 100644 --- a/Release_Process.md +++ b/Release_Process.md @@ -2,7 +2,6 @@ This is a summary of the steps we go through to release a new version of BigchainDB Server. -1. Run `python docs/server/generate_schema_documentation.py` and commit the changes in `docs/server/source/schema/`, if any. 1. Update the `CHANGELOG.md` file 1. Update the version numbers in `bigchaindb/version.py`. Note that we try to use [semantic versioning](http://semver.org/) (i.e. MAJOR.MINOR.PATCH) 1. Go to the [bigchaindb/bigchaindb Releases page on GitHub](https://github.com/bigchaindb/bigchaindb/releases) From 4ce7f4e95de4115f0aa47550cec306751871b8dc Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Fri, 2 Dec 2016 13:47:36 +0100 Subject: [PATCH 23/42] sign the tx in http-client-server-api.html, server would reject otherwise --- docs/server/generate_http_server_api_documentation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/server/generate_http_server_api_documentation.py b/docs/server/generate_http_server_api_documentation.py index d9d86647..80b47cf2 100644 --- a/docs/server/generate_http_server_api_documentation.py +++ b/docs/server/generate_http_server_api_documentation.py @@ -60,9 +60,11 @@ Content-Type: application/json def main(): """ Main function """ - pubkey = '9RaWxppkP9UyYWA7NJb5FcgkzfJNPfvPX3FCNw2T5Pwb' + privkey = 'CfdqtD7sS7FgkMoGPXw55MVGGFwQLAoHYTcBhZDtF99Z' + pubkey = '4K9sWUMFwTgaDGPfdynrbxWqWS6sWmKbZoTjxLtVUibD' asset = Asset(None, 'e6969f87-4fc9-4467-b62a-f0dfa1c85002') tx = Transaction.create([pubkey], [([pubkey], 1)], asset=asset) + tx = tx.sign([privkey]) tx_json = json.dumps(tx.to_dict(), indent=2, sort_keys=True) base_path = os.path.join(os.path.dirname(__file__), From 96143f7e60f5fa563b848ff4be8ad22fc16d4a9c Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 28 Nov 2016 19:27:22 +0100 Subject: [PATCH 24/42] Use explicit chaining to capture the cause --- bigchaindb/common/schema/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigchaindb/common/schema/__init__.py b/bigchaindb/common/schema/__init__.py index 6bb4f038..9fdb3136 100644 --- a/bigchaindb/common/schema/__init__.py +++ b/bigchaindb/common/schema/__init__.py @@ -18,7 +18,7 @@ def validate_transaction_schema(tx_body): try: jsonschema.validate(tx_body, TX_SCHEMA) except jsonschema.ValidationError as exc: - raise SchemaValidationError(str(exc)) + raise SchemaValidationError(str(exc)) from exc __all__ = ['TX_SCHEMA', 'TX_SCHEMA_YAML', 'validate_transaction_schema'] From 602dfbea062dba69a65e11ec6226dc2de346f7a6 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 28 Nov 2016 19:30:09 +0100 Subject: [PATCH 25/42] Add transaction schema validation to the server by overriding Transaction.from_dict() in models.py --- bigchaindb/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bigchaindb/models.py b/bigchaindb/models.py index 3bbc80e3..bca182c3 100644 --- a/bigchaindb/models.py +++ b/bigchaindb/models.py @@ -6,6 +6,7 @@ from bigchaindb.common.exceptions import (InvalidHash, InvalidSignature, AssetIdMismatch, AmountError) from bigchaindb.common.transaction import Transaction, Asset from bigchaindb.common.util import gen_timestamp, serialize +from bigchaindb.common.schema import validate_transaction_schema class Transaction(Transaction): @@ -113,6 +114,11 @@ class Transaction(Transaction): else: return self + @classmethod + def from_dict(cls, tx_body): + validate_transaction_schema(tx_body) + return super().from_dict(tx_body) + class Block(object): """Bundle a list of Transactions in a Block. Nodes vote on its validity. From 9dacc12b3cb92f9a93b4e7da69eca73f74f5aad3 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 28 Nov 2016 19:31:42 +0100 Subject: [PATCH 26/42] Handle transaction schema validation errors in POST /transactions --- bigchaindb/web/views/transactions.py | 12 +++++++++++- tests/web/test_transactions.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index 2ed19b7c..e2abeb48 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -6,7 +6,11 @@ For more information please refer to the documentation on ReadTheDocs: from flask import current_app, request, Blueprint from flask_restful import Resource, Api -from bigchaindb.common.exceptions import ValidationError, InvalidSignature +from bigchaindb.common.exceptions import ( + ValidationError, + InvalidSignature, + SchemaValidationError, +) import bigchaindb from bigchaindb.models import Transaction @@ -98,6 +102,12 @@ class TransactionListApi(Resource): try: tx_obj = Transaction.from_dict(tx) + except SchemaValidationError as e: + return make_error( + 400, + message='Invalid transaction schema: {}'.format( + e.__cause__.message) + ) except (ValidationError, InvalidSignature): return make_error(400, 'Invalid transaction') diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index 94c3f22f..f5071603 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -66,6 +66,18 @@ def test_post_create_transaction_with_invalid_structure(client): assert res.status_code == 400 +def test_post_create_transaction_with_invalid_schema(client): + from bigchaindb.models import Transaction + user_priv, user_pub = crypto.generate_key_pair() + tx = Transaction.create( + [user_pub], [([user_pub], 1)]).sign([user_priv]).to_dict() + del tx['version'] + res = client.post(TX_ENDPOINT, data=json.dumps(tx)) + assert res.status_code == 400 + assert res.json['message'] == ( + "Invalid transaction schema: 'version' is a required property") + + @pytest.mark.usefixtures('inputs') def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): sk, pk = crypto.generate_key_pair() From 44d1bb2c97f0d50767d38f22ae29f9d0e75b2fab Mon Sep 17 00:00:00 2001 From: Troy McConaghy Date: Fri, 2 Dec 2016 15:47:19 +0100 Subject: [PATCH 27/42] Fixed minor error in changelog for 0.8.0 v0.8.0 doesn't actually do transaction schema validation. That got merged shortly _after_ the v0.8.0 release, in PR #880 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb23ce5..7fdfe8c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ committed: ### Added - The big new thing in version 0.8.0 is support for divisible assets, i.e. assets like carrots or thumbtacks, where the initial CREATE transaction can register/create some amount (e.g. 542 carrots), the first TRANSFER transaction can split that amount across multiple owners, and so on. [Pull Request #794](https://github.com/bigchaindb/bigchaindb/pull/794) -- Wrote a formal schema for the JSON structure of transactions. Transactions are now checked against that schema. [Pull Request #798](https://github.com/bigchaindb/bigchaindb/pull/798) +- Wrote a formal schema for the JSON structure of transactions. [Pull Request #798](https://github.com/bigchaindb/bigchaindb/pull/798) - New configuration parameter: `backlog_reassign_delay`. [Pull Request #883](https://github.com/bigchaindb/bigchaindb/pull/883) ### Changed From 7940479ee38b96593093fe2ee7da2c04276570f5 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 5 Dec 2016 17:39:26 +0100 Subject: [PATCH 28/42] Make the autodoc members option the default --- docs/server/source/appendices/consensus.rst | 1 - docs/server/source/appendices/pipelines.rst | 5 ----- docs/server/source/appendices/the-Bigchain-class.rst | 1 - docs/server/source/conf.py | 3 +++ 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/server/source/appendices/consensus.rst b/docs/server/source/appendices/consensus.rst index 837e81d5..34c0c032 100644 --- a/docs/server/source/appendices/consensus.rst +++ b/docs/server/source/appendices/consensus.rst @@ -3,4 +3,3 @@ Consensus ######### .. automodule:: bigchaindb.consensus - :members: diff --git a/docs/server/source/appendices/pipelines.rst b/docs/server/source/appendices/pipelines.rst index 2685d2d3..98b993a1 100644 --- a/docs/server/source/appendices/pipelines.rst +++ b/docs/server/source/appendices/pipelines.rst @@ -6,32 +6,27 @@ Block Creation ============== .. automodule:: bigchaindb.pipelines.block - :members: Block Voting ============ .. automodule:: bigchaindb.pipelines.vote - :members: Block Status ============ .. automodule:: bigchaindb.pipelines.election - :members: Stale Transaction Monitoring ============================ .. automodule:: bigchaindb.pipelines.stale - :members: Utilities ========= .. automodule:: bigchaindb.pipelines.utils - :members: diff --git a/docs/server/source/appendices/the-Bigchain-class.rst b/docs/server/source/appendices/the-Bigchain-class.rst index d9e6744c..9b88d5dd 100644 --- a/docs/server/source/appendices/the-Bigchain-class.rst +++ b/docs/server/source/appendices/the-Bigchain-class.rst @@ -5,6 +5,5 @@ The Bigchain class The Bigchain class is the top-level Python API for BigchainDB. If you want to create and initialize a BigchainDB database, you create a Bigchain instance (object). Then you can use its various methods to create transactions, write transactions (to the object/database), read transactions, etc. .. autoclass:: bigchaindb.Bigchain - :members: .. automethod:: bigchaindb.core.Bigchain.__init__ diff --git a/docs/server/source/conf.py b/docs/server/source/conf.py index 93402f75..f4951235 100644 --- a/docs/server/source/conf.py +++ b/docs/server/source/conf.py @@ -56,6 +56,9 @@ extensions = [ # autodoc settings autodoc_member_order = 'bysource' +autodoc_default_flags = [ + 'members', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From bdc706d6330516882375d717097fd2472c319009 Mon Sep 17 00:00:00 2001 From: Scott Sadler Date: Tue, 6 Dec 2016 16:58:06 +0100 Subject: [PATCH 29/42] fix metadata description in transaction-model.rst --- docs/server/source/data-models/transaction-model.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/server/source/data-models/transaction-model.rst b/docs/server/source/data-models/transaction-model.rst index 0d3cd136..69a3b168 100644 --- a/docs/server/source/data-models/transaction-model.rst +++ b/docs/server/source/data-models/transaction-model.rst @@ -26,10 +26,7 @@ A transaction has the following structure: "conditions": [""], "operation": "", "asset": "", - "metadata": { - "id": "", - "data": "" - } + "metadata": "" } Here's some explanation of the contents of a :ref:`transaction `: @@ -48,9 +45,7 @@ Here's some explanation of the contents of a :ref:`transaction `: - **asset**: Definition of the digital :ref:`asset `. See next section. -- **metadata**: - - :ref:`id `: UUID version 4 (random) converted to a string of hex digits in standard form. - - :ref:`data `: Can be any JSON document. It may be empty in the case of a transfer transaction. +- **metadata**: User-provided transaction :ref:`metadata `: Can be any JSON document, or `NULL`. Later, when we get to the models for the block and the vote, we'll see that both include a signature (from the node which created it). You may wonder why transactions don't have signatures... The answer is that they do! They're just hidden inside the ``fulfillment`` string of each fulfillment. A creation transaction is signed by whoever created it. A transfer transaction is signed by whoever currently controls or owns it. From 85bb4a9233f4488715459c1fa62b80ef10a12432 Mon Sep 17 00:00:00 2001 From: Krish Date: Thu, 8 Dec 2016 08:54:21 +0000 Subject: [PATCH 30/42] Fix 781 and 124 The queue/pipeline used in the block creation process is unbounded. When too many transactions occur, the backlog table fills up quite easily and bigchaindb just reads all the transaction in the backlog to create a block. This causes memory usage to grow indefinitely. Limiting the queue size to 1000 transactions for now as the block creation and voting happens in batches of 1000. Can be increased later in case block size is increased. --- bigchaindb/pipelines/block.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigchaindb/pipelines/block.py b/bigchaindb/pipelines/block.py index 9039938b..0394aa23 100644 --- a/bigchaindb/pipelines/block.py +++ b/bigchaindb/pipelines/block.py @@ -8,7 +8,7 @@ function. import logging import rethinkdb as r -from multipipes import Pipeline, Node +from multipipes import Pipeline, Node, Pipe from bigchaindb.models import Transaction from bigchaindb.pipelines.utils import ChangeFeed @@ -161,6 +161,7 @@ def create_pipeline(): block_pipeline = BlockPipeline() pipeline = Pipeline([ + Pipe(maxsize=1000), Node(block_pipeline.filter_tx), Node(block_pipeline.validate_tx, fraction_of_cores=1), Node(block_pipeline.create, timeout=1), From 1223695b360cf9ffd73d5367493920898f0c13a8 Mon Sep 17 00:00:00 2001 From: Ryan Henderson Date: Thu, 8 Dec 2016 11:42:46 +0100 Subject: [PATCH 31/42] add integration tests (#614) --- setup.py | 2 +- tests/integration/__init__.py | 0 tests/integration/conftest.py | 35 ++++++++++++++ tests/integration/test_integration.py | 66 +++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_integration.py diff --git a/setup.py b/setup.py index ecb801a3..6342c682 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ tests_require = [ 'pep8', 'flake8', 'pylint', - 'pytest', + 'pytest>=3.0.0', 'pytest-cov>=2.2.1', 'pytest-xdist', 'pytest-flask', diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..5ac81cb3 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,35 @@ +import pytest +from bigchaindb.pipelines import block, election, vote, stale + +# TODO: fix this import madness +from ..db import conftest + + +@pytest.fixture(scope='module', autouse=True) +def restore_config(request, node_config): + from bigchaindb import config_utils + config_utils.set_config(node_config) + + +@pytest.fixture(scope='module', autouse=True) +def setup_database(request, node_config): + conftest.setup_database(request, node_config) + + +@pytest.fixture(scope='function', autouse=True) +def cleanup_tables(request, node_config): + conftest.cleanup_tables(request, node_config) + + +@pytest.fixture +def processes(b): + b.create_genesis_block() + block_maker = block.start() + voter = vote.start() + election_runner = election.start() + stale_monitor = stale.start() + yield + block_maker.terminate() + voter.terminate() + election_runner.terminate() + stale_monitor.terminate() diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 00000000..fee4817f --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,66 @@ +import time +import pytest + +from bigchaindb import Bigchain + + +@pytest.fixture +def inputs(user_pk): + from bigchaindb.models import Transaction + + b = Bigchain() + + # create blocks with transactions for `USER` to spend + for block in range(4): + transactions = [ + Transaction.create( + [b.me], [([user_pk], 1)], + metadata={'i': i}) + .sign([b.me_private]) + for i in range(10) + ] + block = b.create_block(transactions) + b.write_block(block, durability='hard') + + +@pytest.mark.usefixtures('processes') +def test_fast_double_create(b, user_pk): + from bigchaindb.models import Transaction + tx = Transaction.create([b.me], [([user_pk], 1)], + metadata={'test': 'test'}) \ + .sign([b.me_private]) + + # write everything fast + b.write_transaction(tx) + b.write_transaction(tx) + + time.sleep(2) + tx_returned = b.get_transaction(tx.id) + + # test that the tx can be queried + assert tx_returned == tx + # test the transaction appears only once + last_voted_block = b.get_last_voted_block() + assert len(last_voted_block.transactions) == 1 + assert b.backend.count_blocks() == 2 + + +@pytest.mark.usefixtures('processes') +def test_double_create(b, user_pk): + from bigchaindb.models import Transaction + tx = Transaction.create([b.me], [([user_pk], 1)], + metadata={'test': 'test'}) \ + .sign([b.me_private]) + + b.write_transaction(tx) + time.sleep(2) + b.write_transaction(tx) + time.sleep(2) + tx_returned = b.get_transaction(tx.id) + + # test that the tx can be queried + assert tx_returned == tx + # test the transaction appears only once + last_voted_block = b.get_last_voted_block() + assert len(last_voted_block.transactions) == 1 + assert b.backend.count_blocks() == 2 From aa436ced6edd15eb3c6a106935a99c3c045fa596 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 7 Dec 2016 18:42:26 +0100 Subject: [PATCH 32/42] Fix #786 --- .ci/travis-after-success.sh | 7 ++++++ .ci/travis-before-install.sh | 8 +++++++ .ci/travis-before-script.sh | 7 ++++++ .ci/travis-install.sh | 11 ++++++++++ .travis.yml | 30 +++++++++++--------------- setup.py | 1 + tox.ini | 42 ++++++++++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 18 deletions(-) create mode 100755 .ci/travis-after-success.sh create mode 100755 .ci/travis-before-install.sh create mode 100755 .ci/travis-before-script.sh create mode 100755 .ci/travis-install.sh create mode 100644 tox.ini diff --git a/.ci/travis-after-success.sh b/.ci/travis-after-success.sh new file mode 100755 index 00000000..272b6f6f --- /dev/null +++ b/.ci/travis-after-success.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e -x + +if [ "${TOXENV}" == "py35" ]; then + codecov +fi diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh new file mode 100755 index 00000000..a7aa1a8a --- /dev/null +++ b/.ci/travis-before-install.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "${TOXENV}" == "py34" ] || [ "${TOXENV}" == "py35" ]; then + source /etc/lsb-release + echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | tee -a /etc/apt/sources.list.d/rethinkdb.list + wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | apt-key add - + apt-get update -qq +fi diff --git a/.ci/travis-before-script.sh b/.ci/travis-before-script.sh new file mode 100755 index 00000000..65e0446c --- /dev/null +++ b/.ci/travis-before-script.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e -x + +if [ "${TOXENV}" == "py34" ] || [ "${TOXENV}" == "py35" ]; then + rethinkdb --daemon +fi diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh new file mode 100755 index 00000000..852b8279 --- /dev/null +++ b/.ci/travis-install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e -x + +pip install --upgrade pip +pip install --upgrade tox + +if [ "${TOXENV}" == "py34" ] || [ "${TOXENV}" == "py35" ]; then + sudo apt-get install rethinkdb + pip install --upgrade codecov +fi diff --git a/.travis.yml b/.travis.yml index bd8ad0b1..7d2b4e5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,19 @@ -sudo: required language: python cache: pip -python: - - 3.4 - - 3.5 -before_install: - - source /etc/lsb-release - - echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee -a /etc/apt/sources.list.d/rethinkdb.list - - wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - - - sudo apt-get update -qq +env: + - TOXENV=py34 + - TOXENV=py35 + - TOXENV=flake8 + - TOXENV=docsroot + - TOXENV=docsserver -install: - - sudo apt-get install rethinkdb - - pip install -e .[test] - - pip install codecov +before_install: sudo .ci/travis-before-install.sh -before_script: - - flake8 --max-line-length 119 bigchaindb/ - - rethinkdb --daemon +install: .ci/travis-install.sh -script: py.test -n auto -s -v --cov=bigchaindb +before_script: .ci/travis-before-script.sh -after_success: codecov +script: tox -e ${TOXENV} + +after_success: .ci/travis-after-success.sh diff --git a/setup.py b/setup.py index 6342c682..00300b92 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ tests_require = [ 'pytest-cov>=2.2.1', 'pytest-xdist', 'pytest-flask', + 'tox', ] + docs_require benchmarks_require = [ diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..e56b091e --- /dev/null +++ b/tox.ini @@ -0,0 +1,42 @@ +[tox] +skipsdist = true +envlist = py34, py35, flake8, docsroot, docsserver + +[base] +basepython = python3.5 +deps = pip>=9.0.1 + +[testenv] +usedevelop = True +setenv = + PYTHONPATH = {toxinidir}:{toxinidir}/bigchaindb +deps = {[base]deps} +extras = test +commands = pytest -v -n auto --cov=bigchaindb --basetemp={envtmpdir} + +[testenv:flake8] +basepython = {[base]basepython} +deps = + {[base]deps} + flake8 +skip_install = True +extras = None +commands = flake8 --max-line-length 119 bigchaindb + +[testenv:docsroot] +basepython = {[base]basepython} +changedir = docs/root/source +deps = + {[base]deps} + -r{toxinidir}/docs/root/requirements.txt +extras = None +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html + +[testenv:docsserver] +basepython = {[base]basepython} +changedir = docs/server/source +deps = + {[base]deps} + -r{toxinidir}/docs/server/requirements.txt +extras = None +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From abd4a54a76a21fba3a0cfcdbaff1deb9563c0b9d Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 7 Dec 2016 19:56:57 +0100 Subject: [PATCH 33/42] Add python3.4 and vim for dev & testing --- Dockerfile-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 2d0cf7ba..c201fd0b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,6 +1,6 @@ FROM python:3.5 -RUN apt-get update +RUN apt-get update && apt-get install -y python3.4 vim RUN mkdir -p /usr/src/app WORKDIR /usr/src/app From 20f629256ba1a2bec81c5466f05e6ed3b44e4504 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 7 Dec 2016 19:57:59 +0100 Subject: [PATCH 34/42] Mount tox.ini in dev container --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2b38a64e..36b90684 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - ./setup.py:/usr/src/app/setup.py - ./setup.cfg:/usr/src/app/setup.cfg - ./pytest.ini:/usr/src/app/pytest.ini + - ./tox.ini:/usr/src/app/tox.ini environment: BIGCHAINDB_DATABASE_HOST: rdb # BigchainDB Server doesn't need BIGCHAINDB_API_ENDPOINT any more From 3bdaf60acbda3272a551d7246c51f23f0e98d145 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 7 Dec 2016 19:59:22 +0100 Subject: [PATCH 35/42] Fix docs building warnings --- bigchaindb/common/schema/transaction.yaml | 2 +- bigchaindb/core.py | 4 ++-- docs/server/source/drivers-clients/http-client-server-api.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bigchaindb/common/schema/transaction.yaml b/bigchaindb/common/schema/transaction.yaml index 03dbe8ad..c365eb82 100644 --- a/bigchaindb/common/schema/transaction.yaml +++ b/bigchaindb/common/schema/transaction.yaml @@ -147,7 +147,7 @@ definitions: "$ref": "#/definitions/offset" description: | Index of this condition's appearance in the `Transaction.conditions`_ - array. In a transaction with 2 conditions, the ``cid``s will be 0 and 1. + array. In a transaction with 2 conditions, the ``cid`` s will be 0 and 1. condition: description: | Body of the condition. Has the properties: diff --git a/bigchaindb/core.py b/bigchaindb/core.py index 8fec1c89..34dbab84 100644 --- a/bigchaindb/core.py +++ b/bigchaindb/core.py @@ -411,13 +411,13 @@ class Bigchain(object): return None def get_owned_ids(self, owner): - """Retrieve a list of `txid`s that can be used as inputs. + """Retrieve a list of ``txid`` s that can be used as inputs. Args: owner (str): base58 encoded public key. Returns: - :obj:`list` of TransactionLink: list of `txid`s and `cid`s + :obj:`list` of TransactionLink: list of ``txid`` s and ``cid`` s pointing to another transaction's condition """ diff --git a/docs/server/source/drivers-clients/http-client-server-api.rst b/docs/server/source/drivers-clients/http-client-server-api.rst index 8143c39d..17cdf429 100644 --- a/docs/server/source/drivers-clients/http-client-server-api.rst +++ b/docs/server/source/drivers-clients/http-client-server-api.rst @@ -190,8 +190,8 @@ GET /unspents/ Content-Type: application/json [ - '../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/0', - '../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/1' + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/0", + "../transactions/2d431073e1477f3073a4693ac7ff9be5634751de1b8abaa1f4e19548ef0b4b0e/conditions/1" ] :statuscode 200: A list of outputs were found and returned in the body of the response. From 793883196b9fca26b42ef6fe5001a1750a2dd3f6 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Wed, 7 Dec 2016 20:04:40 +0100 Subject: [PATCH 36/42] Adjust container server docs building commmand --- docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs.yml b/docs.yml index f3f5efcb..b6520ecb 100644 --- a/docs.yml +++ b/docs.yml @@ -7,7 +7,7 @@ services: dockerfile: ./Dockerfile-dev volumes: - .:/usr/src/app/ - command: make -C docs html + command: make -C docs/server html vdocs: image: nginx ports: From bf0b2a32f09c0532dfc5eccc1eef225e2055cabd Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Dec 2016 02:31:44 +0100 Subject: [PATCH 37/42] Set root docs html_static_path to empty list --- docs/root/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/source/conf.py b/docs/root/source/conf.py index e9bbfcbf..50dec3ea 100644 --- a/docs/root/source/conf.py +++ b/docs/root/source/conf.py @@ -162,7 +162,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = [] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied From cc9e3b79b613c8f93ee164ec32c8fa7ceb3a2464 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Dec 2016 16:38:52 +0100 Subject: [PATCH 38/42] Remove outdated comment --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 36b90684..0dcfe8c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,9 +30,6 @@ services: - ./tox.ini:/usr/src/app/tox.ini environment: BIGCHAINDB_DATABASE_HOST: rdb -# BigchainDB Server doesn't need BIGCHAINDB_API_ENDPOINT any more -# but maybe our Docker or Docker Compose stuff does? -# BIGCHAINDB_API_ENDPOINT: http://bdb:9984/api/v1 BIGCHAINDB_SERVER_BIND: 0.0.0.0:9984 ports: - "9984" From d998b005925348962eb54ae52d781a654a0f9120 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 5 Dec 2016 12:10:32 +0100 Subject: [PATCH 39/42] Return information about the exception --- bigchaindb/web/views/transactions.py | 35 +++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/bigchaindb/web/views/transactions.py b/bigchaindb/web/views/transactions.py index e2abeb48..babec6fc 100644 --- a/bigchaindb/web/views/transactions.py +++ b/bigchaindb/web/views/transactions.py @@ -7,9 +7,16 @@ from flask import current_app, request, Blueprint from flask_restful import Resource, Api from bigchaindb.common.exceptions import ( - ValidationError, + AmountError, + DoubleSpend, + InvalidHash, InvalidSignature, SchemaValidationError, + OperationError, + TransactionDoesNotExist, + TransactionOwnerError, + TransactionNotInValidBlock, + ValidationError, ) import bigchaindb @@ -108,16 +115,32 @@ class TransactionListApi(Resource): message='Invalid transaction schema: {}'.format( e.__cause__.message) ) - except (ValidationError, InvalidSignature): - return make_error(400, 'Invalid transaction') + except (ValidationError, InvalidSignature) as e: + return make_error( + 400, + 'Invalid transaction ({}): {}'.format(type(e).__name__, e) + ) with pool() as bigchain: - if bigchain.is_valid_transaction(tx_obj): + try: + bigchain.validate_transaction(tx_obj) + except (ValueError, + OperationError, + TransactionDoesNotExist, + TransactionOwnerError, + DoubleSpend, + InvalidHash, + InvalidSignature, + TransactionNotInValidBlock, + AmountError) as e: + return make_error( + 400, + 'Invalid transaction ({}): {}'.format(type(e).__name__, e) + ) + else: rate = bigchaindb.config['statsd']['rate'] with monitor.timer('write_transaction', rate=rate): bigchain.write_transaction(tx_obj) - else: - return make_error(400, 'Invalid transaction') return tx From 209dba76a30df8d8e8e15a6a4508004f549936a1 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Dec 2016 14:27:40 +0100 Subject: [PATCH 40/42] Add some messages when raising exceptions --- bigchaindb/common/transaction.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bigchaindb/common/transaction.py b/bigchaindb/common/transaction.py index 2a9591c2..59a7b088 100644 --- a/bigchaindb/common/transaction.py +++ b/bigchaindb/common/transaction.py @@ -1183,14 +1183,16 @@ class Transaction(object): try: proposed_tx_id = tx_body.pop('id') except KeyError: - raise InvalidHash() + raise InvalidHash('No transaction id found!') tx_body_no_signatures = Transaction._remove_signatures(tx_body) tx_body_serialized = Transaction._to_str(tx_body_no_signatures) valid_tx_id = Transaction._to_hash(tx_body_serialized) if proposed_tx_id != valid_tx_id: - raise InvalidHash() + err_msg = ("The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid.") + raise InvalidHash(err_msg.format(proposed_tx_id)) @classmethod def from_dict(cls, tx): From 6f90c8f3df9b4a31a552d06081170c6ca90d6427 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Thu, 8 Dec 2016 14:29:30 +0100 Subject: [PATCH 41/42] Add tests for HTTP API returned errors --- tests/web/test_transactions.py | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/web/test_transactions.py b/tests/web/test_transactions.py index f5071603..eed8a2b7 100644 --- a/tests/web/test_transactions.py +++ b/tests/web/test_transactions.py @@ -1,3 +1,4 @@ +import builtins import json import pytest @@ -38,6 +39,7 @@ def test_post_create_transaction_endpoint(b, client): def test_post_create_transaction_with_invalid_id(b, client): + from bigchaindb.common.exceptions import InvalidHash from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() @@ -47,9 +49,14 @@ def test_post_create_transaction_with_invalid_id(b, client): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 + err_msg = ("The transaction's id '{}' isn't equal to " + "the hash of its body, i.e. it's not valid.").format(tx['id']) + assert res.json['message'] == ( + 'Invalid transaction ({}): {}'.format(InvalidHash.__name__, err_msg)) def test_post_create_transaction_with_invalid_signature(b, client): + from bigchaindb.common.exceptions import InvalidSignature from bigchaindb.models import Transaction user_priv, user_pub = crypto.generate_key_pair() @@ -59,6 +66,9 @@ def test_post_create_transaction_with_invalid_signature(b, client): res = client.post(TX_ENDPOINT, data=json.dumps(tx)) assert res.status_code == 400 + assert res.json['message'] == ( + "Invalid transaction ({}): Fulfillment URI " + "couldn't been parsed".format(InvalidSignature.__name__)) def test_post_create_transaction_with_invalid_structure(client): @@ -78,6 +88,37 @@ def test_post_create_transaction_with_invalid_schema(client): "Invalid transaction schema: 'version' is a required property") +@pytest.mark.parametrize('exc,msg', ( + ('AmountError', 'Do the math again!'), + ('DoubleSpend', 'Nope! It is gone now!'), + ('InvalidHash', 'Do not smoke that!'), + ('InvalidSignature', 'Falsche Unterschrift!'), + ('OperationError', 'Create and transfer!'), + ('TransactionDoesNotExist', 'Hallucinations?'), + ('TransactionOwnerError', 'Not yours!'), + ('TransactionNotInValidBlock', 'Wait, maybe?'), + ('ValueError', '?'), +)) +def test_post_invalid_transaction(client, exc, msg, monkeypatch): + from bigchaindb.common import exceptions + try: + exc_cls = getattr(exceptions, exc) + except AttributeError: + exc_cls = getattr(builtins, 'ValueError') + + def mock_validation(self_, tx): + raise exc_cls(msg) + + monkeypatch.setattr( + 'bigchaindb.Bigchain.validate_transaction', mock_validation) + monkeypatch.setattr( + 'bigchaindb.models.Transaction.from_dict', lambda tx: None) + res = client.post(TX_ENDPOINT, data=json.dumps({})) + assert res.status_code == 400 + assert (res.json['message'] == + 'Invalid transaction ({}): {}'.format(exc, msg)) + + @pytest.mark.usefixtures('inputs') def test_post_transfer_transaction_endpoint(b, client, user_pk, user_sk): sk, pk = crypto.generate_key_pair() From 86019d51ebfac0d14bc7b922959de26626136d07 Mon Sep 17 00:00:00 2001 From: Brett Sun Date: Thu, 8 Dec 2016 17:10:48 +0100 Subject: [PATCH 42/42] Add tox usage to testing docs (#915) * Use section header for CI testing in style guide document * Add tox section to style guide document * Reword style guide's testing section to be more general after introduction of integration tests --- PYTHON_STYLE_GUIDE.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md index a2a9800b..42f18c85 100644 --- a/PYTHON_STYLE_GUIDE.md +++ b/PYTHON_STYLE_GUIDE.md @@ -73,13 +73,13 @@ flake8 --max-line-length 119 bigchaindb/ ``` -## Writing and Running (Python) Unit Tests +## Writing and Running (Python) Tests -We write unit tests for our Python code using the [pytest](http://pytest.org/latest/) framework. +We write unit and integration tests for our Python code using the [pytest](http://pytest.org/latest/) framework. All tests go in the `bigchaindb/tests` directory or one of its subdirectories. You can use the tests already in there as templates or examples. -You can run all unit tests using: +You can run all tests using: ```text py.test -v ``` @@ -96,4 +96,27 @@ python setup.py test If you want to learn about all the things you can do with pytest, see [the pytest documentation](http://pytest.org/latest/). -**Automated testing of pull requests.** We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.) +### Tox + +We use [tox](https://tox.readthedocs.io/en/latest/) to run multiple suites of tests against multiple environments during automated testing. Generally you don't need to run this yourself, but it might be useful when troubleshooting a failing CI build. + +To run all the tox tests, use: +```text +tox +``` + +or: +```text +python -m tox +``` + +To run only a few environments, use the `-e` flag: +```text +tox -e {ENVLIST} +``` + +where `{ENVLIST}` is one or more of the environments specified in the [tox.ini file](tox.ini). + +### Automated testing of pull requests + +We use [Travis CI](https://travis-ci.com/), so that whenever someone creates a new BigchainDB pull request on GitHub, Travis CI gets the new code and does _a bunch of stuff_. You can find out what we tell Travis CI to do in [the `.travis.yml` file](.travis.yml): it tells Travis CI how to install BigchainDB, how to run all the tests, and what to do "after success" (e.g. run `codecov`). (We use [Codecov](https://codecov.io/) to get a rough estimate of our test coverage.)